@filefreak.io/sdk 0.4.0 → 0.6.0

package/README.md

@@ -5,7 +5,8 @@ later and in browsers. No dependencies.
 
 Every endpoint is a function, typed from the same table the API is built
 from. Around them: uploads sent straight to storage in parts and carried on
-after an interruption, downloads fetched in ranges, listings and changes read
+after an interruption, a new version of a large file sending only what
+changed, downloads fetched in ranges, listings and changes read
 to the end, files under Shard Protection sealed as they are sent and read
 through their maps, what a share link holds saved with no key, and requests sent again only when that
 cannot make anything twice. The reference is at
@@ -33,7 +34,9 @@ const bytes = await ff.bytes(file.id); // for a file small enough to hold in mem
 than memory is fine. `toPath` writes beside the file and gives it its name
 only once every byte is there: a download that fails leaves nothing behind.
 A file under Shard Protection has no address, and `download` and `bytes`
-read it through its map instead, each fragment checked as it arrives.
+read it through its map instead, each fragment checked as it arrives. One
+written in version 2 of the shard format, in blocks cut by its content, is
+read the same way, its outline checked before any block is fetched.
 Into a folder under Shard Protection, `upload` seals each part with the
 file's own key before it is sent, so none of the file is ever plain in
 storage. The key is asked of the API for the upload and kept nowhere, and
@@ -41,6 +44,28 @@ an upload carried on asks which fragments storage has before it sends any.
 
 Keep keys on a server. A key sent from a web page is refused.
 
+## A new version of a large file
+
+A file of 16 MiB or more is read through before it is sent and cut where
+its content says, and each part is one of its blocks. A new version of
+it, sent with `onConflict: "replace"`, sends only the blocks the file does
+not have already: those it has are put together with the new ones inside
+storage. The SDK and the CLI cut a file alike, so the next version of a file
+either one sent keeps its blocks whichever sends it.
+
+```ts
+await ff.upload(await fromPath("disk.img"), { parentId: folder.id, onConflict: "replace" });
+```
+
+Into a folder under Shard Protection the same holds, with every block
+encrypted before it leaves. A protected file of more than 16 MiB is cut by
+its own key, so that nobody without the key can cut a copy of it the same
+way and compare, and its next version keeps the blocks the last one has
+when the key may read files; a key that may only write starts a new file
+key and sends every block. Each block is read again as it is sent and
+checked against what the upload began with, so a file changed while it is
+sent stops the upload (`source_changed`) rather than storing a mix of two.
+
 ## In a web page
 
 A page is given a token by your server instead. Your server makes it with its
@@ -188,7 +213,9 @@ await ff.files.delete(file.id, { ifRevision: file.revision });
 
 What reached storage is kept until the upload expires. Keep the upload's id
 from `onBegun`, and carry on with the same file: only the parts storage does
-not have are sent.
+not have are sent. A file cut by its content is carried on block by block,
+each checked against what the upload began with, so carrying on with another
+file is refused at its first block.
 
 ```ts
 import { UploadStopped } from "@filefreak.io/sdk";

package/dist/client.js

@@ -6,6 +6,7 @@ import { bind, OPERATIONS, } from "./generated.js";
 import { sendFrame } from "./frames.js";
 import { fetchRanges } from "./ranges.js";
 import { asItIsKept, readProtected } from "./reader.js";
+import { READS_FORMATS } from "./shards.js";
 import { FileFreakError, sleep, Transport } from "./transport.js";
 import { upload } from "./uploads.js";
 /** How long a follow of the changes waits for one before asking again, unless told. */
@@ -139,7 +140,7 @@ export class FileFreak {
             const { url } = await this.files.downloadUrl(item.id, {}, { signal });
             await fetchRanges({ url, sizeBytes: item.sizeBytes, sink, signal, progress, ranges: options.ranges, fetch: this.transport.fetch, sleep: (ms) => sleep(ms, signal) });
         }, () => readProtected({
-            pages: (from, pageSignal) => this.files.shards(item.id, { from }, { signal: pageSignal }),
+            pages: (from, pageSignal) => this.files.shards(item.id, { from, formats: READS_FORMATS }, { signal: pageSignal }),
             sink,
             sizeBytes: item.sizeBytes,
             signal,

package/dist/cut.d.ts

@@ -0,0 +1,96 @@
+import { type FileKeys, type OutlineBlock } from "./shards.js";
+/** How a file is cut. One rule for every file, so that a file cut once is cut the same way next time, which is what lets its blocks be found again. */
+export interface Rule {
+    /** A smaller plain file is not cut: it goes in one or two parts, and there is nothing to gain. */
+    cutFrom: number;
+    /** FastCDC's sizes. No piece but a file's last is shorter than least, and none is longer than most. */
+    least: number;
+    average: number;
+    most: number;
+    /** FastCDC's patterns at level 1: the stricter until a piece reaches the average, the looser after. Within the low 48 bits. */
+    strict: bigint;
+    loose: bigint;
+    /** Past this size pieces are put together in runs, about four to a run, so that a file of 150 GB is still within the API's 10,000 blocks. */
+    groupFrom: number;
+    /** A run ends once it is this long, so that one more piece, 16 MiB at most, leaves it within the API's 64 MiB. */
+    groupMost: number;
+    /** The most blocks the API takes. */
+    mostBlocks: number;
+    /** How much of a Blob is read at a time. */
+    readBytes: number;
+}
+export declare const RULE: Rule;
+/** FastCDC's 256 numbers: whole, for the hash at a cut, and their low 48 bits in halves of 24, for the hash as the bytes go by. */
+export declare class CutTable {
+    readonly values: BigUint64Array;
+    readonly low: Int32Array<ArrayBuffer>;
+    readonly high: Int32Array<ArrayBuffer>;
+    constructor(values: BigUint64Array);
+}
+/**
+ * The fastcdc crate's own table, which a plain file is cut with: for each
+ * value, the first eight bytes, big-endian, of the MD5 of 64 bytes of it
+ * (the crate's examples/table64.rs, after the C implementation it follows).
+ * Made here rather than copied in.
+ */
+export declare function publicTable(): CutTable;
+/** A protected file's own table: 2,048 bytes of HKDF-SHA256 from its key, with no salt and the info "filefreak-shard-v2 cut", as 256 big-endian numbers. */
+export declare function keyedTable(raw: Uint8Array): Promise<CutTable>;
+/** A piece of a file: how many bytes, and the hash it was cut at. */
+export interface Piece {
+    length: number;
+    hash: bigint;
+}
+/**
+ * The piece `data` begins with, where `data` runs to the end of the largest
+ * piece or of the file, whichever comes first. As the crate cuts: the hash
+ * starts `least` bytes in; places are tested two at a time, so the last byte
+ * of a piece cut short at an odd length is hashed and never tested; and a
+ * piece cut at a match ends before the byte that matched.
+ */
+export declare function piece(data: Uint8Array, table: CutTable, rule?: Rule): Piece;
+/**
+ * Whether a piece cut at `hash` ends a run, in a file past groupFrom: one in
+ * four, decided by the bytes where it was cut, so that a run ends at the
+ * same content wherever it moves to. Mixed first, because a hash at a cut
+ * matches a pattern and some of its bits are always zero.
+ */
+export declare function endsRun(hash: bigint): boolean;
+/** What is cut: its bytes, or a way to read them a window at a time. */
+export type Bytes = Uint8Array | {
+    size: number;
+    read(from: number, to: number): Promise<Uint8Array>;
+};
+/** A block of a file: how many bytes, and what names it. */
+export interface Named<T> {
+    length: number;
+    name: T;
+}
+/**
+ * A file cut into blocks with the table given: a piece to a block, or in a
+ * file past groupFrom, runs of pieces. `name` is given each block's bytes in
+ * order. Null as soon as there are more blocks than the API takes, without
+ * reading the rest. What is read is kept only while a piece or an open run
+ * still needs it.
+ */
+export declare function cut<T>(bytes: Bytes, table: CutTable, name: (block: Uint8Array) => Promise<T>, rule?: Rule, signal?: AbortSignal): Promise<Array<Named<T>> | null>;
+/** A plain block as beginning an upload takes it. */
+export interface PlainBlock {
+    size: number;
+    /** In lower-case hex. */
+    sha256: string;
+}
+/**
+ * A plain file cut for beginning its upload, each block its size and
+ * SHA-256. Null for a file too small to be worth cutting, or cut into more
+ * blocks than the API takes: either is sent in parts of one size.
+ */
+export declare function cutPlain(bytes: Bytes, rule?: Rule, signal?: AbortSignal): Promise<PlainBlock[] | null>;
+/**
+ * A protected file cut for version 2 of the shard format, with its own key's
+ * table, each block its id and length, as uploads.blocks takes them and the
+ * outline holds them. Null for a file not cut into two to ten thousand
+ * blocks, which version 2 does not take: it is written in version 1.
+ */
+export declare function cutSealed(bytes: Bytes, raw: Uint8Array, keys: FileKeys, rule?: Rule, signal?: AbortSignal): Promise<OutlineBlock[] | null>;
+export declare function hex(bytes: Uint8Array): string;

package/dist/cut.js

@@ -0,0 +1,220 @@
+// Cutting a file by its content, so that its next version sends only what
+// changed. The rule is the CLI's (cli/crates/transfer/src/blocks.rs), so
+// that of a file either one sent, the other's next version keeps the blocks.
+//
+// Where a piece ends is decided by the bytes there: FastCDC, as the fastcdc
+// crate runs it (version 5, its 2020 form, normalised at level 1), hashes the
+// bytes as it reads and cuts where the hash matches a pattern. A byte put in
+// near the start of a file moves the end of the piece it falls in and leaves
+// every later piece as it was, where parts of one size would all move. A
+// plain file is cut with the crate's own table of numbers and each block
+// named by its SHA-256, as beginning an upload takes them. A file under Shard
+// Protection, written in version 2 of the shard format, is cut with a table
+// made from the file's own key and each block named by its id
+// (docs/reference/shard-format.md, "Cutting a file"): with the public table,
+// whoever has the database could cut a file they hold and compare the
+// lengths. docs/reference/shard-format-v2.json holds both to the crate.
+//
+// The hash runs to 64 bits, more than a JavaScript number holds exactly. The
+// patterns look only at its low 48 bits, which nothing above them changes, so
+// only those are kept as the bytes go by, as two numbers of 24 bits. The
+// whole hash is wanted only at a cut, where it says whether a run ends in a
+// file past 32 GiB, and it depends only on the last 64 bytes hashed: it is
+// worked out again from those, as a BigInt.
+import { Md5 } from "./md5.js";
+import { blockId } from "./shards.js";
+const MIB = 1024 * 1024;
+const HALF = 0xff_ffff;
+export const RULE = {
+    cutFrom: 16 * MIB,
+    least: MIB,
+    average: 4 * MIB,
+    most: 16 * MIB,
+    strict: 0x0000d93777537000n,
+    loose: 0x0000d91767537000n,
+    groupFrom: 32 * 1024 * MIB,
+    groupMost: 48 * MIB,
+    mostBlocks: 10_000,
+    readBytes: 32 * MIB,
+};
+/** FastCDC's 256 numbers: whole, for the hash at a cut, and their low 48 bits in halves of 24, for the hash as the bytes go by. */
+export class CutTable {
+    values;
+    low = new Int32Array(256);
+    high = new Int32Array(256);
+    constructor(values) {
+        this.values = values;
+        if (values.length !== 256)
+            throw new RangeError(`A cutting table has 256 numbers, not ${values.length}.`);
+        for (let i = 0; i < 256; i++) {
+            this.low[i] = Number(values[i] & BigInt(HALF));
+            this.high[i] = Number((values[i] >> 24n) & BigInt(HALF));
+        }
+    }
+}
+let crateTable = null;
+/**
+ * The fastcdc crate's own table, which a plain file is cut with: for each
+ * value, the first eight bytes, big-endian, of the MD5 of 64 bytes of it
+ * (the crate's examples/table64.rs, after the C implementation it follows).
+ * Made here rather than copied in.
+ */
+export function publicTable() {
+    if (!crateTable) {
+        const values = new BigUint64Array(256);
+        for (let i = 0; i < 256; i++) {
+            const digest = new Md5().update(new Uint8Array(64).fill(i)).digest();
+            values[i] = new DataView(digest.buffer, digest.byteOffset, 8).getBigUint64(0, false);
+        }
+        crateTable = new CutTable(values);
+    }
+    return crateTable;
+}
+/** A protected file's own table: 2,048 bytes of HKDF-SHA256 from its key, with no salt and the info "filefreak-shard-v2 cut", as 256 big-endian numbers. */
+export async function keyedTable(raw) {
+    if (raw.length !== 32)
+        throw new RangeError(`A shard key is 32 bytes, not ${raw.length}.`);
+    const base = await crypto.subtle.importKey("raw", new Uint8Array(raw), "HKDF", false, ["deriveBits"]);
+    const bits = await crypto.subtle.deriveBits({ name: "HKDF", hash: "SHA-256", salt: new Uint8Array(0), info: new TextEncoder().encode("filefreak-shard-v2 cut") }, base, 2048 * 8);
+    const view = new DataView(bits);
+    const values = new BigUint64Array(256);
+    for (let i = 0; i < 256; i++)
+        values[i] = view.getBigUint64(i * 8, false);
+    return new CutTable(values);
+}
+/** A pattern as two halves of 24 bits, refused if it looks above the 48 bits the hash is kept to. */
+function halves(pattern) {
+    if (pattern >> 48n !== 0n)
+        throw new RangeError(`A cutting pattern looks only at the low 48 bits of the hash, and ${pattern.toString(16)} does not.`);
+    return [Number(pattern & BigInt(HALF)), Number(pattern >> 24n)];
+}
+/**
+ * The piece `data` begins with, where `data` runs to the end of the largest
+ * piece or of the file, whichever comes first. As the crate cuts: the hash
+ * starts `least` bytes in; places are tested two at a time, so the last byte
+ * of a piece cut short at an odd length is hashed and never tested; and a
+ * piece cut at a match ends before the byte that matched.
+ */
+export function piece(data, table, rule = RULE) {
+    const left = data.length;
+    if (left <= rule.least)
+        return { length: left, hash: 0n };
+    const end = Math.min(left, rule.most);
+    const strictTo = 2 * Math.floor(Math.min(rule.average, end) / 2);
+    const testedTo = 2 * Math.floor(end / 2);
+    const low = table.low;
+    const high = table.high;
+    let [maskLow, maskHigh] = halves(rule.strict);
+    const [looseLow, looseHigh] = halves(rule.loose);
+    let lo = 0;
+    let hi = 0;
+    for (let i = rule.least; i < testedTo; i++) {
+        if (i === strictTo) {
+            maskLow = looseLow;
+            maskHigh = looseHigh;
+        }
+        const b = data[i];
+        const sum = 2 * lo + low[b];
+        hi = (2 * hi + high[b] + (sum >>> 24)) & HALF;
+        lo = sum & HALF;
+        if ((lo & maskLow) === 0 && (hi & maskHigh) === 0)
+            return { length: i, hash: hashAt(data, i, table, rule, i % 2 === 0) };
+    }
+    return { length: end, hash: hashAt(data, end - 1, table, rule, false) };
+}
+/**
+ * The whole hash once byte `i` is hashed, from the bytes that still count in
+ * it: those from `least` on, and of them only the last 64, since each byte
+ * doubles it and 64 doublings leave nothing of an earlier one. Doubled again
+ * for a match at an even place, as the crate gives it.
+ */
+function hashAt(data, i, table, rule, doubled) {
+    let h = 0n;
+    for (let j = Math.max(rule.least, i - 63); j <= i; j++)
+        h = BigInt.asUintN(64, (h << 1n) + table.values[data[j]]);
+    return doubled ? BigInt.asUintN(64, h << 1n) : h;
+}
+/**
+ * Whether a piece cut at `hash` ends a run, in a file past groupFrom: one in
+ * four, decided by the bytes where it was cut, so that a run ends at the
+ * same content wherever it moves to. Mixed first, because a hash at a cut
+ * matches a pattern and some of its bits are always zero.
+ */
+export function endsRun(hash) {
+    return BigInt.asUintN(64, hash * 0x9e3779b97f4a7c15n) >> 62n === 0n;
+}
+/**
+ * A file cut into blocks with the table given: a piece to a block, or in a
+ * file past groupFrom, runs of pieces. `name` is given each block's bytes in
+ * order. Null as soon as there are more blocks than the API takes, without
+ * reading the rest. What is read is kept only while a piece or an open run
+ * still needs it.
+ */
+export async function cut(bytes, table, name, rule = RULE, signal) {
+    const reader = bytes instanceof Uint8Array ? null : bytes;
+    const size = sizeOf(bytes);
+    let buffer = bytes instanceof Uint8Array ? bytes : new Uint8Array(0);
+    let bufferAt = 0;
+    const grouping = size > rule.groupFrom;
+    const blocks = [];
+    let runAt = null;
+    for (let at = 0; at < size;) {
+        signal?.throwIfAborted();
+        const want = Math.min(size, at + rule.most);
+        if (reader && bufferAt + buffer.length < want) {
+            const keep = runAt ?? at;
+            const had = bufferAt + buffer.length;
+            const upTo = Math.min(size, Math.max(want, had + rule.readBytes));
+            const more = await reader.read(had, upTo);
+            if (more.length !== upTo - had)
+                throw new RangeError(`Reading bytes ${had} to ${upTo} gave ${more.length}: the file has changed since it was opened.`);
+            const joined = new Uint8Array(had - keep + more.length);
+            joined.set(buffer.subarray(keep - bufferAt));
+            joined.set(more, had - keep);
+            buffer = joined;
+            bufferAt = keep;
+        }
+        const { length, hash } = piece(buffer.subarray(at - bufferAt, want - bufferAt), table, rule);
+        const end = at + length;
+        runAt ??= at;
+        if (!grouping || end - runAt >= rule.groupMost || endsRun(hash) || end === size) {
+            blocks.push({ length: end - runAt, name: await name(buffer.subarray(runAt - bufferAt, end - bufferAt)) });
+            if (blocks.length > rule.mostBlocks)
+                return null;
+            runAt = null;
+        }
+        at = end;
+    }
+    return blocks;
+}
+/**
+ * A plain file cut for beginning its upload, each block its size and
+ * SHA-256. Null for a file too small to be worth cutting, or cut into more
+ * blocks than the API takes: either is sent in parts of one size.
+ */
+export async function cutPlain(bytes, rule = RULE, signal) {
+    if (sizeOf(bytes) < rule.cutFrom)
+        return null;
+    const blocks = await cut(bytes, publicTable(), async (block) => hex(new Uint8Array(await crypto.subtle.digest("SHA-256", own(block)))), rule, signal);
+    return blocks && blocks.map((b) => ({ size: b.length, sha256: b.name }));
+}
+/**
+ * A protected file cut for version 2 of the shard format, with its own key's
+ * table, each block its id and length, as uploads.blocks takes them and the
+ * outline holds them. Null for a file not cut into two to ten thousand
+ * blocks, which version 2 does not take: it is written in version 1.
+ */
+export async function cutSealed(bytes, raw, keys, rule = RULE, signal) {
+    const blocks = await cut(bytes, await keyedTable(raw), (block) => blockId(keys, block), rule, signal);
+    return blocks && blocks.length >= 2 ? blocks.map((b) => ({ id: b.name, length: b.length })) : null;
+}
+function sizeOf(bytes) {
+    return bytes instanceof Uint8Array ? bytes.length : bytes.size;
+}
+/** The bytes as WebCrypto's types take them, copied only when they are not in an ArrayBuffer of their own (a SharedArrayBuffer's). */
+function own(bytes) {
+    return bytes.buffer instanceof ArrayBuffer ? bytes : new Uint8Array(bytes);
+}
+export function hex(bytes) {
+    return Array.from(bytes, (b) => b.toString(16).padStart(2, "0")).join("");
+}

package/dist/generated.d.ts

@@ -157,11 +157,13 @@ export interface SendTo {
 }
 /** A page of a protected file's map: its key, and where each fragment is. */
 export interface ShardMap {
-    /** The content's own id, which each fragment's check is bound to with its place and the count. */
+    /** The content's own id, which each fragment's check is bound to with its place and the count, and in version 2 the outline. */
     contentId: string;
+    /** The version of the shard format the file is written in: 1, or 2, in which the file is cut into blocks by its content and its next version keeps the blocks that did not change. Only a version named in formats is answered. */
+    format: number;
     /** The file's size. */
     sizeBytes: number;
-    /** How many bytes of the file each fragment holds, except the last, which holds what remains. */
+    /** How many bytes of the file each fragment holds, except the last, which holds what remains. In version 2, the largest block's: each block's own length is in the outline. */
     fragmentBytes: number;
     /** How many fragments the file is. */
     count: number;
@@ -171,11 +173,23 @@ export interface ShardMap {
     contentType: string | null;
     /** The file's revision whose content this is. Its fragments never change, however late they are read. */
     revision: number;
+    /** Version 2, on the first page: every block of the file and the MAC over them. Null on a later page, and in version 1. */
+    outline: {
+        /** The MAC over the content id, the size and every block's id and length, in base64. Check it with the file's key, as the SDK's checkOutline does, before using a byte of the file. */
+        mac: string;
+        /** Every block of the file, in order of place. */
+        blocks: Array<{
+            /** The block's id in base64: 32 bytes, the HMAC of its bytes under the file's key. Each block is checked against it once opened. */
+            id: string;
+            /** How many bytes of the file the block holds. */
+            length: number;
+        }>;
+    } | null;
     /** The first place on this page. */
     from: number;
     /** Up to 1000, in order of place. */
     fragments: Array<{
-        /** Counted from zero. Its bytes of the file begin at place times fragmentBytes. */
+        /** Counted from zero. In version 1 its bytes of the file begin at place times fragmentBytes; in version 2 after the blocks before it in the outline. */
         place: number;
         /** Where to fetch the stored fragment, with a plain GET and no key: the address is the permission. */
         url: string;
@@ -314,6 +328,8 @@ export type UploadsBeginBody = {
     }>;
     /** true when this program can seal fragments as it sends them, as the SDK and the CLI do. Into a protected place the answer's shards is then set, and each part is a fragment. */
     protectable?: boolean;
+    /** With protectable: the versions of the shard format this program can write. Left out, 1 alone. With 2, a sealed file of more than 16 MiB is written in version 2. */
+    shardFormats?: Array<number>;
 };
 /** What POST /v1/uploads answers with. */
 export type UploadsBeginAnswer = {
@@ -324,11 +340,13 @@ export type UploadsBeginAnswer = {
     reused: Array<number>;
     /** Set when the upload is sealed as it is sent: how the file is cut into fragments. The key to seal them with is given by uploads.get. Null for an upload sent as it is. */
     shards: {
-        /** The content's own id, which each fragment's seal is bound to with its place and the count. */
+        /** The version of the shard format to write it in: 1, or 2 when shardFormats names it and the file is large enough. */
+        format: number;
+        /** The content's own id, which each fragment's seal is bound to with its place and the count, and in version 2 the outline. */
         contentId: string;
-        /** How many bytes of the file each fragment holds, except the last, which holds what remains. */
+        /** How many bytes of the file each fragment holds, except the last, which holds what remains. In version 2, 0: the blocks are declared with uploads.blocks. */
         fragmentBytes: number;
-        /** How many fragments, one to each part. */
+        /** How many fragments, one to each part. In version 2, 0 until the blocks are declared. */
         count: number;
     } | null;
 };
@@ -346,16 +364,42 @@ export type UploadsGetAnswer = {
     reused: Array<number>;
     /** For an upload sealed as it is sent: how the file is cut, and the key to seal each fragment with. Null for an upload sent as it is, and for one that has ended. */
     shards: {
-        /** The content's own id, which each fragment's seal is bound to with its place and the count. */
+        /** The version of the shard format to write it in. */
+        format: number;
+        /** The content's own id, which each fragment's seal is bound to with its place and the count, and in version 2 the outline. */
         contentId: string;
         /** The file's own key, in base64: 32 bytes, which open its fragments. In this answer and nowhere else. */
         key: string;
-        /** How many bytes of the file each fragment holds, except the last, which holds what remains. */
+        /** How many bytes of the file each fragment holds, except the last, which holds what remains. In version 2, the largest block's once they are declared, and 0 before. */
         fragmentBytes: number;
-        /** How many fragments, one to each part. */
+        /** How many fragments, one to each part. In version 2, how many blocks once they are declared, and 0 before. */
         count: number;
+        /** Version 2: the blocks as declared, in order, or null until they are. Null in version 1. */
+        blocks: Array<{
+            size: number;
+            /** Its id, in base64. */
+            id: string;
+        }> | null;
     } | null;
 };
+/** What POST /v1/uploads/:id/blocks takes as its body. */
+export type UploadsBlocksBody = {
+    /** Every block of the file, in order. */
+    blocks: Array<{
+        /** How many bytes of the file the block holds. */
+        size: number;
+        /** The block's id, in base64: 32 bytes. */
+        id: string;
+    }>;
+};
+/** What POST /v1/uploads/:id/blocks answers with. */
+export type UploadsBlocksAnswer = {
+    upload: Upload;
+    /** The places whose block the file has already, under its key. Never sent, and counted as arrived. */
+    reused: Array<number>;
+    /** Addresses for the first blocks to send, each at its place. Places in reused are not among them. */
+    parts: Array<Part>;
+};
 /** What GET /v1/uploads/:id/parts takes in its query. */
 export type UploadsPartsQuery = {
     /** The first part wanted, counted from zero. */
@@ -384,6 +428,8 @@ export type UploadsFragmentsAnswer = {
 export type UploadsCompleteBody = {
     /** The file's SHA-256 in hex, as you worked it out. Kept with the file as your word, since this service never sees the bytes, and given back as clientSha256, so a copy can be checked later without reading the file again. */
     sha256?: string;
+    /** For an upload written in version 2 of the shard format, and required for one: the outline's MAC in base64, as the SDK's outlineMac makes it from the content id, the size and the declared blocks. Checked with the file's key. */
+    outline?: string;
 };
 /** What POST /v1/uploads/:id/complete answers with. */
 export type UploadsCompleteAnswer = {
@@ -417,6 +463,8 @@ export type FilesDownloadUrlAnswer = {
 export type FilesShardsQuery = {
     /** The first fragment wanted, counted from zero: 0, or the `next` of the page before. */
     from?: number;
+    /** The versions of the shard format this program reads, as 1,2. Left out, 1 alone. */
+    formats?: string;
 };
 /** What GET /v1/files/:id/shards answers with. */
 export type FilesShardsAnswer = ShardMap;
@@ -689,6 +737,12 @@ export declare const OPERATIONS: {
         readonly idempotent: false;
         readonly repeatable: false;
     };
+    readonly "uploads.blocks": {
+        readonly method: "POST";
+        readonly path: "/v1/uploads/:id/blocks";
+        readonly idempotent: false;
+        readonly repeatable: true;
+    };
     readonly "uploads.parts": {
         readonly method: "GET";
         readonly path: "/v1/uploads/:id/parts";
@@ -910,6 +964,8 @@ export declare function bind(call: Call): {
          *
          * A file under Shard Protection is kept as fragments, each sealed with the file's own key and stored across several storage accounts. Its map says where each fragment is, up to 1,000 at a time, with the key. Fetch each fragment from its address (no key: the address is the permission), check and open it as the SDK's openFragment does, and write it at its place. The first page counts the file's size toward the month's downloads; later pages count nothing. Addresses work for four hours: before then, ask for the page again from the first fragment not yet fetched. The key is in this answer and nowhere else. This API never keeps it, and neither should a program, beyond the read.
          *
+         * A file in version 2 of the shard format (format 2) is kept as blocks of their own lengths, cut by its content. The first page carries its outline: check it with the key before using a byte, open each block as the SDK's openBlock does, and write it after the blocks before it. A program says which versions it reads with formats, and a file in another is answered with 409 format_unsupported, so a program made before version 2 is told to update rather than failing on the file.
+         *
          * GET /v1/files/:id/shards. Needs the scope files:read.
          */
         shards: (id: string, query?: FilesShardsQuery, options?: CallOptions) => Promise<FilesShardsAnswer>;
@@ -966,6 +1022,8 @@ export declare function bind(call: Call): {
          *
          * Into a place under Shard Protection, an upload begun with protectable true is sealed as it is sent. A place is protected by the setting nearest it (files.update's protection), or with none by the drive's default: an app's, which its owner turns on from the app's page, or for a device the account's. The answer's shards then says how the file is cut, and uploads.get gives its key. Each part is a fragment: the file's bytes from its number times fragmentBytes, sealed with that key as the SDK's sealFragment does, under a nonce of its own, and sent to the part's address. uploads.fragments confirms them, and completing makes the file, protected from its first byte, with a null clientSha256. A sealed upload sets blocks aside: none is kept, and none reused. Begun without protectable, a file lands as an ordinary file and the service protects it within a few minutes; from then on it is read through its map (files.shards).
          *
+         * A program that writes version 2 of the shard format says so with shardFormats. A sealed file of more than 16 MiB is then written in it: shards says format 2, with no fragments and no parts yet. Get the key from uploads.get, cut the file into blocks by its content with it, and declare them with uploads.blocks, which answers with the parts to send. Replacing a file whose content is in version 2, with a key that may read the file, the key is the file's own, and the blocks the file has already are not sent again. Complete it with outline, the outline's MAC.
+         *
          * POST /v1/uploads. Needs the scope files:write. Honours an Idempotency-Key.
          */
         begin: (body: UploadsBeginBody, options?: CallOptions) => Promise<UploadsBeginAnswer>;
@@ -977,6 +1035,14 @@ export declare function bind(call: Call): {
          * GET /v1/uploads/:id. Needs the scope files:write.
          */
         get: (id: string, options?: CallOptions) => Promise<UploadsGetAnswer>;
+        /**
+         * Declare a sealed upload's blocks.
+         *
+         * For an upload written in version 2 of the shard format (its shards say format 2): the blocks the file is cut into by its content with the file's key, in order, each its size and its id, the HMAC of its bytes under the file's key. From 2 to 10,000 blocks of 1 byte to 64 MiB, adding up to sizeBytes. Answers with the places not to send, whose block the file has already under its key, and addresses for the first of the rest: ask for more with uploads.parts, and confirm them with uploads.fragments. Each part is its block, sealed as the SDK's sealBlock does. Declared once: the same blocks again are answered the same, and other blocks are refused with 409 blocks_declared.
+         *
+         * POST /v1/uploads/:id/blocks. Needs the scope files:write. Safe to send again.
+         */
+        blocks: (id: string, body: UploadsBlocksBody, options?: CallOptions) => Promise<UploadsBlocksAnswer>;
         /**
          * Addresses for more parts.
          *

package/dist/generated.js

@@ -16,6 +16,7 @@ export const OPERATIONS = {
     "changes.list": { method: "GET", path: "/v1/changes", idempotent: false, repeatable: false },
     "uploads.begin": { method: "POST", path: "/v1/uploads", idempotent: true, repeatable: false },
     "uploads.get": { method: "GET", path: "/v1/uploads/:id", idempotent: false, repeatable: false },
+    "uploads.blocks": { method: "POST", path: "/v1/uploads/:id/blocks", idempotent: false, repeatable: true },
     "uploads.parts": { method: "GET", path: "/v1/uploads/:id/parts", idempotent: false, repeatable: false },
     "uploads.fragments": { method: "POST", path: "/v1/uploads/:id/fragments", idempotent: false, repeatable: true },
     "uploads.complete": { method: "POST", path: "/v1/uploads/:id/complete", idempotent: false, repeatable: true },
@@ -115,6 +116,8 @@ export function bind(call) {
              *
              * A file under Shard Protection is kept as fragments, each sealed with the file's own key and stored across several storage accounts. Its map says where each fragment is, up to 1,000 at a time, with the key. Fetch each fragment from its address (no key: the address is the permission), check and open it as the SDK's openFragment does, and write it at its place. The first page counts the file's size toward the month's downloads; later pages count nothing. Addresses work for four hours: before then, ask for the page again from the first fragment not yet fetched. The key is in this answer and nowhere else. This API never keeps it, and neither should a program, beyond the read.
              *
+             * A file in version 2 of the shard format (format 2) is kept as blocks of their own lengths, cut by its content. The first page carries its outline: check it with the key before using a byte, open each block as the SDK's openBlock does, and write it after the blocks before it. A program says which versions it reads with formats, and a file in another is answered with 409 format_unsupported, so a program made before version 2 is told to update rather than failing on the file.
+             *
              * GET /v1/files/:id/shards. Needs the scope files:read.
              */
             shards: (id, query, options) => call("files.shards", { params: { id }, query }, options),
@@ -171,6 +174,8 @@ export function bind(call) {
              *
              * Into a place under Shard Protection, an upload begun with protectable true is sealed as it is sent. A place is protected by the setting nearest it (files.update's protection), or with none by the drive's default: an app's, which its owner turns on from the app's page, or for a device the account's. The answer's shards then says how the file is cut, and uploads.get gives its key. Each part is a fragment: the file's bytes from its number times fragmentBytes, sealed with that key as the SDK's sealFragment does, under a nonce of its own, and sent to the part's address. uploads.fragments confirms them, and completing makes the file, protected from its first byte, with a null clientSha256. A sealed upload sets blocks aside: none is kept, and none reused. Begun without protectable, a file lands as an ordinary file and the service protects it within a few minutes; from then on it is read through its map (files.shards).
              *
+             * A program that writes version 2 of the shard format says so with shardFormats. A sealed file of more than 16 MiB is then written in it: shards says format 2, with no fragments and no parts yet. Get the key from uploads.get, cut the file into blocks by its content with it, and declare them with uploads.blocks, which answers with the parts to send. Replacing a file whose content is in version 2, with a key that may read the file, the key is the file's own, and the blocks the file has already are not sent again. Complete it with outline, the outline's MAC.
+             *
              * POST /v1/uploads. Needs the scope files:write. Honours an Idempotency-Key.
              */
             begin: (body, options) => call("uploads.begin", { body }, options),
@@ -182,6 +187,14 @@ export function bind(call) {
              * GET /v1/uploads/:id. Needs the scope files:write.
              */
             get: (id, options) => call("uploads.get", { params: { id } }, options),
+            /**
+             * Declare a sealed upload's blocks.
+             *
+             * For an upload written in version 2 of the shard format (its shards say format 2): the blocks the file is cut into by its content with the file's key, in order, each its size and its id, the HMAC of its bytes under the file's key. From 2 to 10,000 blocks of 1 byte to 64 MiB, adding up to sizeBytes. Answers with the places not to send, whose block the file has already under its key, and addresses for the first of the rest: ask for more with uploads.parts, and confirm them with uploads.fragments. Each part is its block, sealed as the SDK's sealBlock does. Declared once: the same blocks again are answered the same, and other blocks are refused with 409 blocks_declared.
+             *
+             * POST /v1/uploads/:id/blocks. Needs the scope files:write. Safe to send again.
+             */
+            blocks: (id, body, options) => call("uploads.blocks", { params: { id }, body }, options),
             /**
              * Addresses for more parts.
              *

package/dist/index.d.ts

@@ -3,6 +3,6 @@ export * from "./generated.js";
 export { FRAME_MAX_BYTES, takesFrame } from "./frames.js";
 export { get, type GetOptions, type Got, type Into, type LinkedFile } from "./links.js";
 export { type Sink } from "./ranges.js";
-export { checkOutline, fileKeys, MOST_BLOCK_BYTES, openBlock, openFragment, outlineMac, sealBlock, sealFragment, SHARD_OVERHEAD, SHARD_VERSION, SHARD_VERSION_2, ShardError, shardKey, ShardLayout, type FileKeys, type FragmentAt, type Outline, type OutlineBlock, type ShardErrorCode, type ShardKey, } from "./shards.js";
+export { BlockLayout, checkOutline, fileKeys, MOST_BLOCK_BYTES, openBlock, openFragment, outlineMac, READS_FORMATS, sealBlock, sealFragment, SHARD_OVERHEAD, SHARD_VERSION, SHARD_VERSION_2, ShardError, shardKey, ShardLayout, type FileKeys, type FragmentAt, type Layout, type Outline, type OutlineBlock, type ShardErrorCode, type ShardKey, } from "./shards.js";
 export { FileFreakError, type CallOptions, type ClientOptions, type TokenSource } from "./transport.js";
 export { UploadStopped, type Source, type UploadOptions } from "./uploads.js";

package/dist/index.js

@@ -13,6 +13,6 @@ export { FileFreak, Walk } from "./client.js";
 export * from "./generated.js";
 export { FRAME_MAX_BYTES, takesFrame } from "./frames.js";
 export { get } from "./links.js";
-export { checkOutline, fileKeys, MOST_BLOCK_BYTES, openBlock, openFragment, outlineMac, sealBlock, sealFragment, SHARD_OVERHEAD, SHARD_VERSION, SHARD_VERSION_2, ShardError, shardKey, ShardLayout, } from "./shards.js";
+export { BlockLayout, checkOutline, fileKeys, MOST_BLOCK_BYTES, openBlock, openFragment, outlineMac, READS_FORMATS, sealBlock, sealFragment, SHARD_OVERHEAD, SHARD_VERSION, SHARD_VERSION_2, ShardError, shardKey, ShardLayout, } from "./shards.js";
 export { FileFreakError } from "./transport.js";
 export { UploadStopped } from "./uploads.js";