@filefreak.io/sdk 0.2.0 → 0.3.1

package/README.md

@@ -6,8 +6,8 @@ later and in browsers. No dependencies.
 Every endpoint is a function, typed from the same table the API is built
 from. Around them: uploads sent straight to storage in parts and carried on
 after an interruption, downloads fetched in ranges, listings and changes read
-to the end, files under Shard Protection read through their maps, what a
-share link holds saved with no key, and requests sent again only when that
+to the end, files under Shard Protection sealed as they are sent and read
+through their maps, what a share link holds saved with no key, and requests sent again only when that
 cannot make anything twice. The reference is at
 https://filefreak.io/developers/reference.
 
@@ -34,6 +34,10 @@ than memory is fine. `toPath` writes beside the file and gives it its name
 only once every byte is there: a download that fails leaves nothing behind.
 A file under Shard Protection has no address, and `download` and `bytes`
 read it through its map instead, each fragment checked as it arrives.
+Into a folder under Shard Protection, `upload` seals each part with the
+file's own key before it is sent, so none of the file is ever plain in
+storage. The key is asked of the API for the upload and kept nowhere, and
+an upload carried on asks which fragments storage has before it sends any.
 
 Keep keys on a server. A key sent from a web page is refused.
 

package/dist/generated.d.ts

@@ -25,6 +25,8 @@ export interface Item {
     clientSha256: string | null;
     /** Whether the file is under Shard Protection: kept as sealed fragments, and read through its map (files.shards) rather than from an address. */
     protected: boolean;
+    /** Its own Shard Protection setting. inherit follows the nearest folder above it that has one, and with none the drive's default: the app's, which its owner turns on from the app's page, or for a device the account's. The nearest setting wins. A file is converted within a few minutes of a change, and protected says when it has been. */
+    protection: "on" | "off" | "inherit";
     createdAt: string;
     updatedAt: string;
 }
@@ -214,6 +216,8 @@ export type FilesUpdateBody = {
     parentId?: string | null;
     /** public, private, or inherit to follow the folder it is in. */
     visibility?: "private" | "public" | "inherit";
+    /** on, off, or inherit to follow the folder it is in. */
+    protection?: "on" | "off" | "inherit";
 };
 /** What PATCH /v1/files/:id answers with. */
 export type FilesUpdateAnswer = {
@@ -304,6 +308,8 @@ export type UploadsBeginBody = {
         /** The block's SHA-256 in hex. */
         sha256: string;
     }>;
+    /** true when this program can seal fragments as it sends them, as the SDK and the CLI do. Into a protected place the answer's shards is then set, and each part is a fragment. */
+    protectable?: boolean;
 };
 /** What POST /v1/uploads answers with. */
 export type UploadsBeginAnswer = {
@@ -312,6 +318,15 @@ export type UploadsBeginAnswer = {
     parts: Array<Part>;
     /** For an upload begun with blocks: the parts not to send, because storage has them already, from the file this replaces, or because they repeat a block an earlier part sends. Empty otherwise. Nothing is taken from the file this replaces when the key may not read it, when the file has not been opened or changed in 90 days, or when copying it would take the account past 1000 GB copied this way in an hour: every block is sent then. */
     reused: Array<number>;
+    /** Set when the upload is sealed as it is sent: how the file is cut into fragments. The key to seal them with is given by uploads.get. Null for an upload sent as it is. */
+    shards: {
+        /** The content's own id, which each fragment's seal is bound to with its place and the count. */
+        contentId: string;
+        /** How many bytes of the file each fragment holds, except the last, which holds what remains. */
+        fragmentBytes: number;
+        /** How many fragments, one to each part. */
+        count: number;
+    } | null;
 };
 /** What GET /v1/uploads/:id answers with. */
 export type UploadsGetAnswer = {
@@ -325,6 +340,17 @@ export type UploadsGetAnswer = {
     }> | null;
     /** As in the answer that began the upload: the parts not to send. */
     reused: Array<number>;
+    /** For an upload sealed as it is sent: how the file is cut, and the key to seal each fragment with. Null for an upload sent as it is, and for one that has ended. */
+    shards: {
+        /** The content's own id, which each fragment's seal is bound to with its place and the count. */
+        contentId: string;
+        /** The file's own key, in base64: 32 bytes, which open its fragments. In this answer and nowhere else. */
+        key: string;
+        /** How many bytes of the file each fragment holds, except the last, which holds what remains. */
+        fragmentBytes: number;
+        /** How many fragments, one to each part. */
+        count: number;
+    } | null;
 };
 /** What GET /v1/uploads/:id/parts takes in its query. */
 export type UploadsPartsQuery = {
@@ -336,6 +362,20 @@ export type UploadsPartsQuery = {
 export type UploadsPartsAnswer = {
     parts: Array<Part>;
 };
+/** What POST /v1/uploads/:id/fragments takes as its body. */
+export type UploadsFragmentsBody = {
+    /** The places of fragments that have been sent: up to 100 at a time. */
+    places: Array<number>;
+};
+/** What POST /v1/uploads/:id/fragments answers with. */
+export type UploadsFragmentsAnswer = {
+    /** At their own names, the size their places hold, and recorded. Never send them again. */
+    recorded: Array<number>;
+    /** Still being copied, or being confirmed by another request. Ask again shortly. */
+    pending: Array<number>;
+    /** Not in storage, or not their size and removed. Seal each again, under a new nonce, and send and confirm it. */
+    resend: Array<number>;
+};
 /** What POST /v1/uploads/:id/complete takes as its body. */
 export type UploadsCompleteBody = {
     /** The file's SHA-256 in hex, as you worked it out. Kept with the file as your word, since this service never sees the bytes, and given back as clientSha256, so a copy can be checked later without reading the file again. */
@@ -396,8 +436,8 @@ export type UsageGetAnswer = {
         plan: string;
         storageUsedBytes: number;
         storageIncludedBytes: number;
-        /** Where uploads are refused: what the plan includes, plus what the account's spending limit allows beyond it. */
-        storageLimitBytes: number;
+        /** Where uploads are refused: what the plan includes, plus what the account's spending limit allows beyond it. Null when nothing is: Pro that FileFreak has given the account has no storage limit. */
+        storageLimitBytes: number | null;
         downloadedBytesThisMonth: number;
         downloadsIncludedBytes: number;
         apiCallsThisMonth: number;
@@ -642,6 +682,12 @@ export declare const OPERATIONS: {
         readonly idempotent: false;
         readonly repeatable: false;
     };
+    readonly "uploads.fragments": {
+        readonly method: "POST";
+        readonly path: "/v1/uploads/:id/fragments";
+        readonly idempotent: false;
+        readonly repeatable: true;
+    };
     readonly "uploads.complete": {
         readonly method: "POST";
         readonly path: "/v1/uploads/:id/complete";
@@ -803,6 +849,8 @@ export declare function bind(call: Call): {
          *
          * Any of the three, in one request. They are applied in that order, each as its own change: if the second is refused the first has still happened, and the answer says which was refused. Moving a folder moves everything in it and is one change however much that is. Making a folder public makes everything in it public, except what has a setting of its own.
          *
+         * protection is a fourth, applied last: on puts the file or folder under Shard Protection, off takes it out, and inherit has it follow the folder it is in. A folder's setting reaches everything in it that has none of its own. Turning it on is refused while the account's plan does not include it (402 shard_protection_needs_pro); turning it off never is. Nothing is converted in this request: the service converts each file within a few minutes, and its protected says when. A key confined to a folder decides about what is in it, and not about the folder.
+         *
          * PATCH /v1/files/:id. Needs the scope files:write.
          */
         update: (id: string, body: FilesUpdateBody, options?: CallOptions) => Promise<FilesUpdateAnswer>;
@@ -889,7 +937,7 @@ export declare function bind(call: Call): {
          *
          * The room is set aside now, so an upload that would not fit is refused before a byte is sent. The parts can be sent by anything that can make a PUT request, including a browser that has never seen the API key: hand it the addresses.
          *
-         * Into a folder under Shard Protection, which its owner turns on from the website, a file lands as an ordinary file, as it would anywhere else, and the service protects it within a few minutes: its content, and every kept version, are spread over the storage lanes as fragments. Until then it can be read like any other file. From then on it is read through its map (files.shards), and its clientSha256 is null. An upload from a program is not sealed as it is sent yet.
+         * Into a place under Shard Protection, an upload begun with protectable true is sealed as it is sent. A place is protected by the setting nearest it (files.update's protection), or with none by the drive's default: an app's, which its owner turns on from the app's page, or for a device the account's. The answer's shards then says how the file is cut, and uploads.get gives its key. Each part is a fragment: the file's bytes from its number times fragmentBytes, sealed with that key as the SDK's sealFragment does, under a nonce of its own, and sent to the part's address. uploads.fragments confirms them, and completing makes the file, protected from its first byte, with a null clientSha256. A sealed upload sets blocks aside: none is kept, and none reused. Begun without protectable, a file lands as an ordinary file and the service protects it within a few minutes; from then on it is read through its map (files.shards).
          *
          * POST /v1/uploads. Needs the scope files:write. Honours an Idempotency-Key.
          */
@@ -897,7 +945,7 @@ export declare function bind(call: Call): {
         /**
          * An upload, and which parts have arrived.
          *
-         * For carrying on after an interruption: storage itself says which parts it has, so nothing the client remembered is needed.
+         * For carrying on after an interruption: storage itself says which parts it has, so nothing the client remembered is needed. For an upload sealed as it is sent, uploadedParts are the fragments confirmed and recorded, and shards gives the key, which this API keeps nowhere else. Confirm any fragment that may have been sent before sending it again. This answer is never kept, by this API or anything between it and the caller.
          *
          * GET /v1/uploads/:id. Needs the scope files:write.
          */
@@ -905,11 +953,19 @@ export declare function bind(call: Call): {
         /**
          * Addresses for more parts.
          *
-         * Addresses are given out a window at a time and each lasts half an hour. Ask again for any that have run out.
+         * Addresses are given out a window at a time and each lasts half an hour. Ask again for any that have run out. For an upload sealed as it is sent, each number is a fragment's place, and its address takes the sealed fragment, with its Content-MD5. Confirm fragments with uploads.fragments once they are sent.
          *
          * GET /v1/uploads/:id/parts. Needs the scope files:write.
          */
         parts: (id: string, query?: UploadsPartsQuery, options?: CallOptions) => Promise<UploadsPartsAnswer>;
+        /**
+         * Confirm sealed fragments.
+         *
+         * Confirms fragments of a sealed upload that have been sent. Each is copied inside storage to its own name, measured, and recorded once it is the size its place holds. recorded: done, never send it again. pending: storage is still copying it, or another confirmation of it is under way, so ask again shortly. resend: not there, or not its size and removed, so seal it again, under a new nonce, send it and confirm it. Safe to send again.
+         *
+         * POST /v1/uploads/:id/fragments. Needs the scope files:write. Safe to send again.
+         */
+        fragments: (id: string, body: UploadsFragmentsBody, options?: CallOptions) => Promise<UploadsFragmentsAnswer>;
         /**
          * Finish an upload.
          *

package/dist/generated.js

@@ -17,6 +17,7 @@ export const OPERATIONS = {
     "uploads.begin": { method: "POST", path: "/v1/uploads", idempotent: true, repeatable: false },
     "uploads.get": { method: "GET", path: "/v1/uploads/:id", idempotent: false, repeatable: false },
     "uploads.parts": { method: "GET", path: "/v1/uploads/:id/parts", idempotent: false, repeatable: false },
+    "uploads.fragments": { method: "POST", path: "/v1/uploads/:id/fragments", idempotent: false, repeatable: true },
     "uploads.complete": { method: "POST", path: "/v1/uploads/:id/complete", idempotent: false, repeatable: true },
     "uploads.cancel": { method: "DELETE", path: "/v1/uploads/:id", idempotent: false, repeatable: false },
     "files.downloadUrl": { method: "GET", path: "/v1/files/:id/download", idempotent: false, repeatable: false },
@@ -71,6 +72,8 @@ export function bind(call) {
              *
              * Any of the three, in one request. They are applied in that order, each as its own change: if the second is refused the first has still happened, and the answer says which was refused. Moving a folder moves everything in it and is one change however much that is. Making a folder public makes everything in it public, except what has a setting of its own.
              *
+             * protection is a fourth, applied last: on puts the file or folder under Shard Protection, off takes it out, and inherit has it follow the folder it is in. A folder's setting reaches everything in it that has none of its own. Turning it on is refused while the account's plan does not include it (402 shard_protection_needs_pro); turning it off never is. Nothing is converted in this request: the service converts each file within a few minutes, and its protected says when. A key confined to a folder decides about what is in it, and not about the folder.
+             *
              * PATCH /v1/files/:id. Needs the scope files:write.
              */
             update: (id, body, options) => call("files.update", { params: { id }, body }, options),
@@ -157,7 +160,7 @@ export function bind(call) {
              *
              * The room is set aside now, so an upload that would not fit is refused before a byte is sent. The parts can be sent by anything that can make a PUT request, including a browser that has never seen the API key: hand it the addresses.
              *
-             * Into a folder under Shard Protection, which its owner turns on from the website, a file lands as an ordinary file, as it would anywhere else, and the service protects it within a few minutes: its content, and every kept version, are spread over the storage lanes as fragments. Until then it can be read like any other file. From then on it is read through its map (files.shards), and its clientSha256 is null. An upload from a program is not sealed as it is sent yet.
+             * Into a place under Shard Protection, an upload begun with protectable true is sealed as it is sent. A place is protected by the setting nearest it (files.update's protection), or with none by the drive's default: an app's, which its owner turns on from the app's page, or for a device the account's. The answer's shards then says how the file is cut, and uploads.get gives its key. Each part is a fragment: the file's bytes from its number times fragmentBytes, sealed with that key as the SDK's sealFragment does, under a nonce of its own, and sent to the part's address. uploads.fragments confirms them, and completing makes the file, protected from its first byte, with a null clientSha256. A sealed upload sets blocks aside: none is kept, and none reused. Begun without protectable, a file lands as an ordinary file and the service protects it within a few minutes; from then on it is read through its map (files.shards).
              *
              * POST /v1/uploads. Needs the scope files:write. Honours an Idempotency-Key.
              */
@@ -165,7 +168,7 @@ export function bind(call) {
             /**
              * An upload, and which parts have arrived.
              *
-             * For carrying on after an interruption: storage itself says which parts it has, so nothing the client remembered is needed.
+             * For carrying on after an interruption: storage itself says which parts it has, so nothing the client remembered is needed. For an upload sealed as it is sent, uploadedParts are the fragments confirmed and recorded, and shards gives the key, which this API keeps nowhere else. Confirm any fragment that may have been sent before sending it again. This answer is never kept, by this API or anything between it and the caller.
              *
              * GET /v1/uploads/:id. Needs the scope files:write.
              */
@@ -173,11 +176,19 @@ export function bind(call) {
             /**
              * Addresses for more parts.
              *
-             * Addresses are given out a window at a time and each lasts half an hour. Ask again for any that have run out.
+             * Addresses are given out a window at a time and each lasts half an hour. Ask again for any that have run out. For an upload sealed as it is sent, each number is a fragment's place, and its address takes the sealed fragment, with its Content-MD5. Confirm fragments with uploads.fragments once they are sent.
              *
              * GET /v1/uploads/:id/parts. Needs the scope files:write.
              */
             parts: (id, query, options) => call("uploads.parts", { params: { id }, query }, options),
+            /**
+             * Confirm sealed fragments.
+             *
+             * Confirms fragments of a sealed upload that have been sent. Each is copied inside storage to its own name, measured, and recorded once it is the size its place holds. recorded: done, never send it again. pending: storage is still copying it, or another confirmation of it is under way, so ask again shortly. resend: not there, or not its size and removed, so seal it again, under a new nonce, send it and confirm it. Safe to send again.
+             *
+             * POST /v1/uploads/:id/fragments. Needs the scope files:write. Safe to send again.
+             */
+            fragments: (id, body, options) => call("uploads.fragments", { params: { id }, body }, options),
             /**
              * Finish an upload.
              *

package/dist/index.d.ts

@@ -2,6 +2,6 @@ export { FileFreak, Walk, type DownloadOptions, type FollowOptions } from "./cli
 export * from "./generated.js";
 export { get, type GetOptions, type Got, type Into, type LinkedFile } from "./links.js";
 export { type Sink } from "./ranges.js";
-export { openFragment, sealFragment, SHARD_OVERHEAD, SHARD_VERSION, ShardError, shardKey, ShardLayout, type FragmentAt, type ShardErrorCode, type ShardKey, } from "./shards.js";
+export { checkOutline, fileKeys, MOST_BLOCK_BYTES, openBlock, openFragment, outlineMac, sealBlock, sealFragment, SHARD_OVERHEAD, SHARD_VERSION, SHARD_VERSION_2, ShardError, shardKey, ShardLayout, type FileKeys, type FragmentAt, type Outline, type OutlineBlock, type ShardErrorCode, type ShardKey, } from "./shards.js";
 export { FileFreakError, type CallOptions, type ClientOptions, type TokenSource } from "./transport.js";
 export { UploadStopped, type Source, type UploadOptions } from "./uploads.js";

package/dist/index.js

@@ -12,6 +12,6 @@
 export { FileFreak, Walk } from "./client.js";
 export * from "./generated.js";
 export { get } from "./links.js";
-export { openFragment, sealFragment, SHARD_OVERHEAD, SHARD_VERSION, ShardError, shardKey, ShardLayout, } from "./shards.js";
+export { checkOutline, fileKeys, MOST_BLOCK_BYTES, openBlock, openFragment, outlineMac, sealBlock, sealFragment, SHARD_OVERHEAD, SHARD_VERSION, SHARD_VERSION_2, ShardError, shardKey, ShardLayout, } from "./shards.js";
 export { FileFreakError } from "./transport.js";
 export { UploadStopped } from "./uploads.js";

package/dist/shards.d.ts

@@ -4,9 +4,10 @@ export declare const SHARD_OVERHEAD = 29;
 /**
  * Why a fragment or a description was refused. A reader checks in this
  * order, and says the first that fails: short, version, length,
- * authentication. layout is a description of a file no fragment can match.
+ * authentication, and in version 2 id. layout is a description of a file no
+ * fragment can match; outline is a version-2 outline whose MAC does not hold.
  */
-export type ShardErrorCode = "short" | "version" | "length" | "authentication" | "layout";
+export type ShardErrorCode = "short" | "version" | "length" | "authentication" | "id" | "layout" | "outline";
 export declare class ShardError extends Error {
     readonly code: ShardErrorCode;
     constructor(code: ShardErrorCode, message: string);
@@ -43,17 +44,75 @@ export declare function shardKey(raw: Uint8Array): Promise<ShardKey>;
 /** The bytes a fragment's tag also covers: the label, the content id, the place and the count. */
 export declare function associatedData(at: FragmentAt): Uint8Array<ArrayBuffer>;
 /** One fragment of a file, encrypted and ready to store, under a nonce of its own. */
-export declare function sealFragment(key: ShardKey, at: FragmentAt, plaintext: Uint8Array): Promise<Uint8Array>;
+export declare function sealFragment(key: ShardKey, at: FragmentAt, plaintext: Uint8Array): Promise<Uint8Array<ArrayBuffer>>;
 /**
  * sealFragment with the nonce chosen by the caller, which only the test
  * files may do: two fragments sealed under one key with one nonce give away
  * the difference between them, and the key's authentication with it. It is
  * not exported from the package.
  */
-export declare function sealWithNonce(key: ShardKey, at: FragmentAt, plaintext: Uint8Array, nonce: Uint8Array): Promise<Uint8Array>;
+export declare function sealWithNonce(key: ShardKey, at: FragmentAt, plaintext: Uint8Array, nonce: Uint8Array): Promise<Uint8Array<ArrayBuffer>>;
 /**
  * A stored fragment's bytes of the file, once it is known to be this file's
  * fragment at this place, unchanged. Nothing of a fragment that fails is
  * returned: WebCrypto gives no plaintext until the tag has been checked.
  */
 export declare function openFragment(key: ShardKey, at: FragmentAt, blob: Uint8Array): Promise<Uint8Array<ArrayBuffer>>;
+export declare const SHARD_VERSION_2 = 2;
+/** The most bytes of a file one block holds, as the API takes blocks. */
+export declare const MOST_BLOCK_BYTES: number;
+/** A block id: the HMAC-SHA256 of its bytes under the file's block key. */
+export declare const BLOCK_ID_BYTES = 32;
+/**
+ * A file's key in version 2, as the three keys derived from it: never the
+ * key itself, so that sealing, naming blocks and vouching for the outline
+ * cannot be played against each other.
+ */
+export interface FileKeys {
+    seal: ShardKey;
+    block: ShardKey;
+    outline: ShardKey;
+}
+/** One block as an outline gives it: its id and how many bytes of the file it holds. */
+export interface OutlineBlock {
+    id: Uint8Array;
+    length: number;
+}
+/** What a content in version 2 is, as its outline gives it, before any block is fetched. */
+export interface Outline {
+    contentId: string;
+    size: number;
+    blocks: OutlineBlock[];
+}
+/** The three keys of a file's key: HKDF-SHA256 with no salt, each named by its use. */
+export declare function fileKeys(raw: Uint8Array): Promise<FileKeys>;
+/** A block's id: what the map holds in place of a hash anybody could make of the same bytes. */
+export declare function blockId(keys: FileKeys, plaintext: Uint8Array): Promise<Uint8Array<ArrayBuffer>>;
+/** The bytes a block's tag also covers: the label and its id, and nothing about where it is, so that a later version can keep it as it is. */
+export declare function blockAssociatedData(id: Uint8Array): Uint8Array<ArrayBuffer>;
+/** One block, encrypted and ready to store under a nonce of its own, with its id. */
+export declare function sealBlock(keys: FileKeys, plaintext: Uint8Array): Promise<{
+    id: Uint8Array<ArrayBuffer>;
+    blob: Uint8Array<ArrayBuffer>;
+}>;
+/**
+ * sealBlock with the id and the nonce chosen by the caller, which only the
+ * test files may do, as with sealWithNonce. Not exported from the package.
+ */
+export declare function sealBlockWithNonce(keys: FileKeys, id: Uint8Array, plaintext: Uint8Array, nonce: Uint8Array): Promise<Uint8Array<ArrayBuffer>>;
+/**
+ * A stored block's bytes of the file, once it is known to be the block the
+ * outline names, unchanged: in this order, short, version, length,
+ * authentication (the tag, which binds it to its id), and id (that its
+ * bytes are the ones the id was made of). Nothing of a block that fails is
+ * returned.
+ */
+export declare function openBlock(keys: FileKeys, block: OutlineBlock, blob: Uint8Array): Promise<Uint8Array<ArrayBuffer>>;
+/** The outline's MAC, which the writer sends with the upload's completion and every reader checks before it releases a byte. */
+export declare function outlineMac(keys: FileKeys, outline: Outline): Promise<Uint8Array<ArrayBuffer>>;
+/**
+ * Refuse an outline unless its MAC holds under the file's key: whoever has
+ * the map and not the key cannot reorder, drop, repeat or swap a content's
+ * blocks, or give one content's blocks as another's.
+ */
+export declare function checkOutline(keys: FileKeys, outline: Outline, mac: Uint8Array): Promise<void>;

package/dist/shards.js

@@ -1,9 +1,13 @@
-// The shard format, version 1: how a file under Shard Protection is kept, as
-// fragments each encrypted on its own. docs/reference/shard-format.md is the
-// definition, and docs/reference/shard-format-v1.json holds the test files
-// this is checked against (test/shards.test.ts), as the CLI's copy is: a
-// writer here whose bytes differ by one from the CLI's makes files the CLI
-// cannot read, and the reverse.
+// The shard format: how a file under Shard Protection is kept, as fragments
+// each encrypted on its own. Version 1 cuts a content into fragments of one
+// size, each sealed to its content's own key, id, place and count. Version 2
+// cuts it into blocks by content, sealed under a key the file's versions
+// share, so that a block a new version still has can be kept as it is.
+// docs/reference/shard-format.md is the definition, and
+// docs/reference/shard-format-v1.json and -v2.json hold the test files this
+// is checked against (test/shards.test.ts), as the CLI's copy is: a writer
+// here whose bytes differ by one from the CLI's makes files the CLI cannot
+// read, and the reverse.
 //
 // WebCrypto, so the same code runs in a page, in a worker and in Node.
 export const SHARD_VERSION = 1;
@@ -128,6 +132,158 @@ export async function openFragment(key, at, blob) {
     }
     return new Uint8Array(plain);
 }
+// ---------------------------------------------------------------------------
+// Version 2: blocks cut by content, under a file's key that versions share
+// ---------------------------------------------------------------------------
+export const SHARD_VERSION_2 = 2;
+/** The most bytes of a file one block holds, as the API takes blocks. */
+export const MOST_BLOCK_BYTES = 64 * 1024 * 1024;
+/** A block id: the HMAC-SHA256 of its bytes under the file's block key. */
+export const BLOCK_ID_BYTES = 32;
+const LABEL_2 = new TextEncoder().encode("filefreak-shard-v2");
+const OUTLINE_LABEL = new TextEncoder().encode("filefreak-outline-v2");
+/** The three keys of a file's key: HKDF-SHA256 with no salt, each named by its use. */
+export async function fileKeys(raw) {
+    if (raw.length !== 32)
+        throw new RangeError(`A shard key is 32 bytes, not ${raw.length}.`);
+    const base = await crypto.subtle.importKey("raw", new Uint8Array(raw), "HKDF", false, ["deriveBits"]);
+    const derive = (use) => crypto.subtle.deriveBits({ name: "HKDF", hash: "SHA-256", salt: new Uint8Array(0), info: new TextEncoder().encode(`filefreak-shard-v2 ${use}`) }, base, 256);
+    const [seal, block, outline] = await Promise.all([derive("seal"), derive("block"), derive("outline")]);
+    const hmac = (raw) => crypto.subtle.importKey("raw", raw, { name: "HMAC", hash: "SHA-256" }, false, ["sign", "verify"]);
+    return {
+        seal: await crypto.subtle.importKey("raw", seal, { name: "AES-GCM" }, false, ["encrypt", "decrypt"]),
+        block: await hmac(block),
+        outline: await hmac(outline),
+    };
+}
+/** A block's id: what the map holds in place of a hash anybody could make of the same bytes. */
+export async function blockId(keys, plaintext) {
+    return new Uint8Array(await crypto.subtle.sign("HMAC", keys.block, new Uint8Array(plaintext)));
+}
+/** The bytes a block's tag also covers: the label and its id, and nothing about where it is, so that a later version can keep it as it is. */
+export function blockAssociatedData(id) {
+    if (id.length !== BLOCK_ID_BYTES)
+        throw new RangeError(`A block id is ${BLOCK_ID_BYTES} bytes, not ${id.length}.`);
+    const out = new Uint8Array(LABEL_2.length + BLOCK_ID_BYTES);
+    out.set(LABEL_2, 0);
+    out.set(id, LABEL_2.length);
+    return out;
+}
+/** One block, encrypted and ready to store under a nonce of its own, with its id. */
+export async function sealBlock(keys, plaintext) {
+    const id = await blockId(keys, plaintext);
+    return { id, blob: await sealBlockWithNonce(keys, id, plaintext, crypto.getRandomValues(new Uint8Array(NONCE_BYTES))) };
+}
+/**
+ * sealBlock with the id and the nonce chosen by the caller, which only the
+ * test files may do, as with sealWithNonce. Not exported from the package.
+ */
+export async function sealBlockWithNonce(keys, id, plaintext, nonce) {
+    if (plaintext.length > MOST_BLOCK_BYTES)
+        throw new ShardError("length", `A block holds at most ${MOST_BLOCK_BYTES} bytes of the file, not ${plaintext.length}.`);
+    if (nonce.length !== NONCE_BYTES)
+        throw new RangeError(`A nonce is ${NONCE_BYTES} bytes, not ${nonce.length}.`);
+    const iv = new Uint8Array(nonce);
+    const sealed = await crypto.subtle.encrypt({ name: "AES-GCM", iv, additionalData: blockAssociatedData(id), tagLength: 128 }, keys.seal, new Uint8Array(plaintext));
+    const out = new Uint8Array(HEADER_BYTES + sealed.byteLength);
+    out[0] = SHARD_VERSION_2;
+    out.set(iv, 1);
+    out.set(new Uint8Array(sealed), HEADER_BYTES);
+    return out;
+}
+/**
+ * A stored block's bytes of the file, once it is known to be the block the
+ * outline names, unchanged: in this order, short, version, length,
+ * authentication (the tag, which binds it to its id), and id (that its
+ * bytes are the ones the id was made of). Nothing of a block that fails is
+ * returned.
+ */
+export async function openBlock(keys, block, blob) {
+    const named = hexOf(block.id).slice(0, 12);
+    if (blob.length < SHARD_OVERHEAD) {
+        throw new ShardError("short", `Block ${named} is ${blob.length} bytes, shorter than any block can be (${SHARD_OVERHEAD}).`);
+    }
+    if (blob[0] !== SHARD_VERSION_2) {
+        throw new ShardError("version", `Block ${named} is in version ${blob[0]} of the format, where version ${SHARD_VERSION_2} was expected.`);
+    }
+    if (blob.length !== block.length + SHARD_OVERHEAD) {
+        throw new ShardError("length", `Block ${named} is ${blob.length} bytes, and holding ${block.length} bytes of the file it would be ${block.length + SHARD_OVERHEAD}.`);
+    }
+    let plain;
+    try {
+        plain = await crypto.subtle.decrypt({ name: "AES-GCM", iv: new Uint8Array(blob.subarray(1, HEADER_BYTES)), additionalData: blockAssociatedData(block.id), tagLength: 128 }, keys.seal, new Uint8Array(blob.subarray(HEADER_BYTES)));
+    }
+    catch (e) {
+        if (e instanceof DOMException && e.name === "OperationError") {
+            throw new ShardError("authentication", `Block ${named} is not what was written as this file's block ${named}: it has been changed, or it belongs to another file or block.`);
+        }
+        throw e;
+    }
+    const bytes = new Uint8Array(plain);
+    // The tag says somebody with the key sealed these bytes under this id; this says the id is theirs.
+    if (!(await crypto.subtle.verify("HMAC", keys.block, new Uint8Array(block.id), bytes))) {
+        throw new ShardError("id", `Block ${named} opens, but its bytes are not the ones its id was made of.`);
+    }
+    return bytes;
+}
+/** Refuse an outline that describes no file: no blocks, a block too long, an empty block in a file that is not empty, or lengths that do not add up. */
+function checkShape(outline) {
+    const { size, blocks } = outline;
+    if (!Number.isSafeInteger(size) || size < 0)
+        throw new ShardError("layout", `A file's size is a whole number of bytes, not ${size}.`);
+    if (blocks.length === 0)
+        throw new ShardError("layout", "An outline names at least one block: an empty file is one empty block.");
+    if (blocks.length > MOST_FRAGMENTS)
+        throw new ShardError("layout", `An outline of ${blocks.length} blocks is more than the format can number.`);
+    let total = 0;
+    for (const [place, b] of blocks.entries()) {
+        if (b.id.length !== BLOCK_ID_BYTES)
+            throw new ShardError("layout", `Block ${place}'s id is ${b.id.length} bytes, not ${BLOCK_ID_BYTES}.`);
+        if (!Number.isSafeInteger(b.length) || b.length < 0 || b.length > MOST_BLOCK_BYTES) {
+            throw new ShardError("layout", `Block ${place} holds ${b.length} bytes, outside 0 to ${MOST_BLOCK_BYTES}.`);
+        }
+        if (b.length === 0 && size > 0)
+            throw new ShardError("layout", `Block ${place} is empty, and only an empty file has an empty block.`);
+        total += b.length;
+    }
+    if (total !== size)
+        throw new ShardError("layout", `The blocks hold ${total} bytes, and the file is ${size}.`);
+}
+/** The bytes the outline's MAC is over: the label, the content id, the size, the count, and each block's id and length in order. */
+function outlineBytes(outline) {
+    const head = OUTLINE_LABEL.length + 16 + 8 + 4;
+    const out = new Uint8Array(head + outline.blocks.length * (BLOCK_ID_BYTES + 4));
+    const view = new DataView(out.buffer);
+    out.set(OUTLINE_LABEL, 0);
+    out.set(uuidBytes(outline.contentId), OUTLINE_LABEL.length);
+    view.setBigUint64(OUTLINE_LABEL.length + 16, BigInt(outline.size), false);
+    view.setUint32(OUTLINE_LABEL.length + 24, outline.blocks.length, false);
+    for (const [i, b] of outline.blocks.entries()) {
+        const at = head + i * (BLOCK_ID_BYTES + 4);
+        out.set(b.id, at);
+        view.setUint32(at + BLOCK_ID_BYTES, b.length, false);
+    }
+    return out;
+}
+/** The outline's MAC, which the writer sends with the upload's completion and every reader checks before it releases a byte. */
+export async function outlineMac(keys, outline) {
+    checkShape(outline);
+    return new Uint8Array(await crypto.subtle.sign("HMAC", keys.outline, outlineBytes(outline)));
+}
+/**
+ * Refuse an outline unless its MAC holds under the file's key: whoever has
+ * the map and not the key cannot reorder, drop, repeat or swap a content's
+ * blocks, or give one content's blocks as another's.
+ */
+export async function checkOutline(keys, outline, mac) {
+    checkShape(outline);
+    if (!(await crypto.subtle.verify("HMAC", keys.outline, new Uint8Array(mac), outlineBytes(outline)))) {
+        throw new ShardError("outline", "The file's outline is not what was written for it: its blocks have been reordered, dropped or swapped, or it is another file's.");
+    }
+}
+function hexOf(b) {
+    return Array.from(b, (x) => x.toString(16).padStart(2, "0")).join("");
+}
 function uuidBytes(id) {
     if (!/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(id)) {
         throw new RangeError(`A content id is a UUID, and ${JSON.stringify(id)} is not one.`);

package/dist/uploads.d.ts

@@ -1,4 +1,4 @@
-import type { FilesGetAnswer, Item, Upload, UploadsBeginAnswer, UploadsBeginBody, UploadsCompleteAnswer, UploadsCompleteBody, UploadsGetAnswer, UploadsPartsAnswer, UploadsPartsQuery } from "./generated.js";
+import type { FilesGetAnswer, Item, Upload, UploadsBeginAnswer, UploadsBeginBody, UploadsCompleteAnswer, UploadsCompleteBody, UploadsFragmentsAnswer, UploadsFragmentsBody, UploadsGetAnswer, UploadsPartsAnswer, UploadsPartsQuery } from "./generated.js";
 import { type CallOptions } from "./transport.js";
 /** What can be uploaded. A File's name and type are used when none are given. */
 export type Source = Blob | ArrayBuffer | Uint8Array;
@@ -45,6 +45,7 @@ export interface UploadCalls {
     begin(body: UploadsBeginBody, options?: CallOptions): Promise<UploadsBeginAnswer>;
     get(id: string, options?: CallOptions): Promise<UploadsGetAnswer>;
     parts(id: string, query?: UploadsPartsQuery, options?: CallOptions): Promise<UploadsPartsAnswer>;
+    fragments(id: string, body: UploadsFragmentsBody, options?: CallOptions): Promise<UploadsFragmentsAnswer>;
     complete(id: string, body?: UploadsCompleteBody, options?: CallOptions): Promise<UploadsCompleteAnswer>;
     file(id: string, options?: CallOptions): Promise<FilesGetAnswer>;
 }

package/dist/uploads.js

@@ -17,7 +17,21 @@
 //
 // A part is read into memory whole to be hashed and sent, so the memory an
 // upload needs is about partBytes times the parts sent at once.
+//
+// SEALED AS IT IS SENT (plan 04, F6 step 4, answer 46). Every begin says the
+// SDK can seal (protectable), as the website and the CLI do. Into a place
+// under Shard Protection the API answers with shards, and each part is then
+// a fragment: the file's bytes at that place, sealed with the file's own key
+// (sealFragment, a nonce of its own each time) before it is hashed and sent.
+// The key comes from uploads.get, whose answer nothing keeps, and is held
+// only while the upload runs. Fragments sent are confirmed (uploads.fragments)
+// as they go, at most one confirmation at a time: each is copied inside
+// storage to its own name and recorded, or is to be sealed again and sent.
+// Carrying on, every fragment not recorded is confirmed before it is sent
+// again, since storage may hold it already. A fragment is up to 64 MiB and
+// sealed in memory, so the memory a sealed upload needs is the same.
 import { base64, Md5 } from "./md5.js";
+import { sealFragment, ShardError, shardKey, ShardLayout } from "./shards.js";
 import { FileFreakError, sleep } from "./transport.js";
 const MIB = 1024 * 1024;
 /** The API's bounds on a part. */
@@ -31,6 +45,14 @@ const PART_ATTEMPTS = 6;
 const ADDRESS_MARGIN_MS = 60_000;
 /** How long to go on asking while storage puts a very large file together. */
 const ASSEMBLY_PATIENCE_MS = 30 * 60_000;
+/** Fragments sent are confirmed this often, or as soon as this many are waiting, and at most this many at once (the API's window). */
+const CONFIRM_MS = 2000;
+const CONFIRM_AT = 32;
+const CONFIRM_MOST = 100;
+/** How long confirming may answer only "still copying" before the upload stops and says so. */
+const CONFIRM_PATIENCE_MS = 5 * 60_000;
+/** How many times fragments storage did not keep are sealed again and sent before the upload stops. */
+const SEALED_ROUNDS = 4;
 /** An upload that stopped part way. What arrived stays in storage until the upload expires: carry on with `resume: uploadId`, or cancel it. */
 export class UploadStopped extends Error {
     uploadId;
@@ -41,6 +63,34 @@ export class UploadStopped extends Error {
         this.name = "UploadStopped";
     }
 }
+/**
+ * The key and the cut of a sealed upload, from what uploads.get says, with
+ * every fragment it has not recorded waiting to be confirmed: one sent
+ * before, by this process or another, may be in storage already.
+ */
+async function sealingOf(state, size) {
+    const shards = state.shards;
+    if (!shards)
+        throw new FileFreakError(0, "unreadable", "The API began a sealed upload and then gave no key for it.");
+    let layout;
+    let raw;
+    try {
+        layout = new ShardLayout(size, shards.fragmentBytes);
+        raw = Uint8Array.from(atob(shards.key), (c) => c.charCodeAt(0));
+    }
+    catch (e) {
+        throw Object.assign(new FileFreakError(0, "unreadable", "The API answered with a sealed upload whose cut or key could not be read."), { cause: e });
+    }
+    if (layout.count !== shards.count || layout.count !== state.upload.partCount || raw.length !== 32) {
+        throw new FileFreakError(0, "layout", `The API cut this file into ${shards.count} fragments, where ${size} bytes in fragments of ${shards.fragmentBytes} make ${layout.count}.`);
+    }
+    const recorded = new Set(state.uploadedParts);
+    const waiting = new Set();
+    for (let n = 0; n < layout.count; n++)
+        if (!recorded.has(n))
+            waiting.add(n);
+    return { key: await shardKey(raw), contentId: shards.contentId, layout, recorded, waiting };
+}
 /** The part size to ask for: 32 MiB, halved while that would leave fewer parts than are sent at once. */
 export function partSizeFor(fileBytes, atOnce) {
     let size = 32 * MIB;
@@ -81,7 +131,8 @@ export async function upload(calls, fetcher, source, options = {}) {
         if (state.blocks) {
             throw new FileFreakError(400, "cut_by_content", "That upload was begun with its file cut by content, which this SDK does not do. Carry it on with the program that began it, or cancel it and begin again.");
         }
-        job = { source, size, upload: state.upload, addresses: new Map(), fetching: null, done: new Set(state.uploadedParts), sentBytes: 0 };
+        const sealing = state.shards ? await sealingOf(state, size) : null;
+        job = { source, size, upload: state.upload, addresses: new Map(), fetching: null, done: new Set(state.uploadedParts), sentBytes: 0, sealing };
     }
     else {
         const name = options.name ?? (typeof File !== "undefined" && source instanceof File ? source.name : undefined);
@@ -93,6 +144,8 @@ export async function upload(calls, fetcher, source, options = {}) {
             sizeBytes: size,
             parentId: options.parentId ?? null,
             partBytes: Math.max(LEAST_PART, Math.min(MOST_PART, options.partBytes ?? partSizeFor(size, atOnce))),
+            // Into a protected place, the parts are fragments this sends sealed.
+            protectable: true,
         };
         if (options.onConflict)
             body.onConflict = options.onConflict;
@@ -103,12 +156,23 @@ export async function upload(calls, fetcher, source, options = {}) {
         if (options.imageSizes)
             body.imageSizes = options.imageSizes;
         const begun = await calls.begin(body, { signal, ...(options.idempotencyKey ? { idempotencyKey: options.idempotencyKey } : {}) });
-        job = { source, size, upload: begun.upload, addresses: new Map(begun.parts.map((p) => [p.number, p])), fetching: null, done: new Set(), sentBytes: 0 };
+        job = { source, size, upload: begun.upload, addresses: new Map(begun.parts.map((p) => [p.number, p])), fetching: null, done: new Set(), sentBytes: 0, sealing: null };
+        if (begun.shards) {
+            // Begin's answer is kept by the API under its Idempotency-Key, so the key is asked for here.
+            try {
+                job.sealing = await sealingOf(await calls.get(begun.upload.id, call), size);
+            }
+            catch (e) {
+                throw new UploadStopped(begun.upload.id, e);
+            }
+            // Nothing of this upload can have been sent yet.
+            job.sealing.waiting.clear();
+        }
     }
     options.onBegun?.(job.upload);
     try {
         tally(job, options);
-        await sendParts(calls, fetcher, job, atOnce, options);
+        await deliver(calls, fetcher, job, atOnce, options);
         return await finish(calls, fetcher, job, atOnce, options);
     }
     catch (e) {
@@ -124,6 +188,71 @@ function tally(job, options) {
     job.sentBytes = sent;
     options.onProgress?.(sent, job.size);
 }
+/** Every part storage does not have, sent: as they are for a plain upload, and confirmed until each is recorded for a sealed one. */
+async function deliver(calls, fetcher, job, atOnce, options) {
+    const sealing = job.sealing;
+    if (!sealing)
+        return await sendParts(calls, fetcher, job, atOnce, options);
+    for (let round = 0;; round++) {
+        // Whatever may be in storage already is asked about before it is sent again.
+        await confirmAll(calls, job, sealing, options);
+        if (sealing.recorded.size === job.upload.partCount)
+            return;
+        // Round 0 sends each fragment once; each later round sends again what storage did not keep.
+        if (round > SEALED_ROUNDS) {
+            const left = job.upload.partCount - sealing.recorded.size;
+            throw new FileFreakError(0, "fragments_refused", `Storage did not keep ${left} of this file's fragments after ${SEALED_ROUNDS + 1} tries.`);
+        }
+        let sending = true;
+        const sent = sendParts(calls, fetcher, job, atOnce, options).finally(() => (sending = false));
+        await Promise.all([sent, confirmWhile(calls, job, sealing, () => sending, options)]);
+    }
+}
+/** While fragments are being sent: those sent are confirmed every CONFIRM_MS, or as soon as CONFIRM_AT are waiting. */
+async function confirmWhile(calls, job, sealing, sending, options) {
+    let last = Date.now();
+    while (sending()) {
+        await sleep(100, options.signal);
+        if (!sending())
+            return;
+        if (sealing.waiting.size >= CONFIRM_AT || (sealing.waiting.size > 0 && Date.now() - last >= CONFIRM_MS)) {
+            await confirmSome(calls, job, sealing, options);
+            last = Date.now();
+        }
+    }
+}
+/** Every fragment waiting asked about until none is: each recorded, or to be sealed again and sent. */
+async function confirmAll(calls, job, sealing, options) {
+    let heard = Date.now();
+    while (sealing.waiting.size > 0) {
+        if (await confirmSome(calls, job, sealing, options)) {
+            heard = Date.now();
+            continue;
+        }
+        if (Date.now() - heard > CONFIRM_PATIENCE_MS) {
+            throw new FileFreakError(0, "still_copying", "Storage has not finished taking part of this file. Nothing is lost: carry it on with resume in a few minutes.");
+        }
+        // Only fragments storage is still copying, or another confirmation holds, are left.
+        await sleep(CONFIRM_MS, options.signal);
+    }
+}
+/** Ask about the first CONFIRM_MOST fragments waiting. True when one was recorded or is to be sent again: either way it stops waiting. */
+async function confirmSome(calls, job, sealing, options) {
+    const places = [...sealing.waiting].sort((a, b) => a - b).slice(0, CONFIRM_MOST);
+    const answer = await calls.fragments(job.upload.id, { places }, { signal: options.signal });
+    for (const n of answer.recorded) {
+        sealing.recorded.add(n);
+        sealing.waiting.delete(n);
+        job.done.add(n);
+    }
+    for (const n of answer.resend) {
+        sealing.waiting.delete(n);
+        // Sent again, sealed again: never the same nonce over other bytes.
+        job.done.delete(n);
+    }
+    tally(job, options);
+    return answer.recorded.length > 0 || answer.resend.length > 0;
+}
 async function sendParts(calls, fetcher, job, atOnce, options) {
     const todo = [];
     for (let n = 0; n < job.upload.partCount; n++)
@@ -146,6 +275,7 @@ async function sendParts(calls, fetcher, job, atOnce, options) {
                 return;
             await sendPart(calls, fetcher, job, n, stop.signal);
             job.done.add(n);
+            job.sealing?.waiting.add(n);
             tally(job, options);
         }
     };
@@ -160,16 +290,27 @@ async function sendParts(calls, fetcher, job, atOnce, options) {
     }
 }
 async function sendPart(calls, fetcher, job, n, signal) {
-    const from = n * job.upload.partBytes;
+    const sealing = job.sealing;
+    const { start: from, end: to } = sealing ? sealing.layout.range(n) : { start: n * job.upload.partBytes, end: Math.min(job.size, (n + 1) * job.upload.partBytes) };
     let bytes;
     try {
-        bytes = await bytesOf(job.source, from, Math.min(job.size, from + job.upload.partBytes));
+        bytes = await bytesOf(job.source, from, to);
     }
     catch (e) {
         // Browsers and Node both refuse to read a file that changed after it was
         // opened, which is what keeps one from going up half old and half new.
         throw Object.assign(new FileFreakError(0, "unreadable", "The file could not be read. It may have been changed, moved or deleted since it was opened: open it again, unchanged, to carry on."), { cause: e });
     }
+    if (sealing) {
+        try {
+            bytes = await sealFragment(sealing.key, { contentId: sealing.contentId, layout: sealing.layout, place: n }, bytes);
+        }
+        catch (e) {
+            if (e instanceof ShardError)
+                throw Object.assign(new FileFreakError(0, "layout", e.message), { cause: e });
+            throw e;
+        }
+    }
     const md5 = base64(new Md5().update(bytes).digest());
     for (let attempt = 1;; attempt++) {
         const part = await address(calls, job, n, signal);
@@ -249,8 +390,13 @@ async function finish(calls, fetcher, job, atOnce, options) {
             if (e instanceof FileFreakError && e.code === "upload_incomplete" && round++ < 2) {
                 const have = Array.isArray(e.body?.uploadedParts) ? e.body.uploadedParts.filter((n) => typeof n === "number") : [];
                 job.done = new Set(have);
+                if (job.sealing) {
+                    // The fragments recorded: any other may be in storage, and is confirmed first.
+                    job.sealing.recorded = new Set(have);
+                    job.sealing.waiting = new Set(Array.from({ length: job.upload.partCount }, (_, n) => n).filter((n) => !job.sealing.recorded.has(n)));
+                }
                 tally(job, options);
-                await sendParts(calls, fetcher, job, atOnce, options);
+                await deliver(calls, fetcher, job, atOnce, options);
                 continue;
             }
             throw e;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@filefreak.io/sdk",
-  "version": "0.2.0",
+  "version": "0.3.1",
   "description": "FileFreak's developer API for Node and the browser: every endpoint, uploads straight to storage in parts, downloads in ranges, and the changes to follow.",
   "license": "Apache-2.0",
   "homepage": "https://filefreak.io/developers/reference",