@filefreak.io/sdk 0.4.0 → 0.5.0

package/README.md

@@ -33,7 +33,9 @@ const bytes = await ff.bytes(file.id); // for a file small enough to hold in mem
 than memory is fine. `toPath` writes beside the file and gives it its name
 only once every byte is there: a download that fails leaves nothing behind.
 A file under Shard Protection has no address, and `download` and `bytes`
-read it through its map instead, each fragment checked as it arrives.
+read it through its map instead, each fragment checked as it arrives. One
+written in version 2 of the shard format, in blocks cut by its content, is
+read the same way, its outline checked before any block is fetched.
 Into a folder under Shard Protection, `upload` seals each part with the
 file's own key before it is sent, so none of the file is ever plain in
 storage. The key is asked of the API for the upload and kept nowhere, and

package/dist/client.js

@@ -6,6 +6,7 @@ import { bind, OPERATIONS, } from "./generated.js";
 import { sendFrame } from "./frames.js";
 import { fetchRanges } from "./ranges.js";
 import { asItIsKept, readProtected } from "./reader.js";
+import { READS_FORMATS } from "./shards.js";
 import { FileFreakError, sleep, Transport } from "./transport.js";
 import { upload } from "./uploads.js";
 /** How long a follow of the changes waits for one before asking again, unless told. */
@@ -139,7 +140,7 @@ export class FileFreak {
             const { url } = await this.files.downloadUrl(item.id, {}, { signal });
             await fetchRanges({ url, sizeBytes: item.sizeBytes, sink, signal, progress, ranges: options.ranges, fetch: this.transport.fetch, sleep: (ms) => sleep(ms, signal) });
         }, () => readProtected({
-            pages: (from, pageSignal) => this.files.shards(item.id, { from }, { signal: pageSignal }),
+            pages: (from, pageSignal) => this.files.shards(item.id, { from, formats: READS_FORMATS }, { signal: pageSignal }),
             sink,
             sizeBytes: item.sizeBytes,
             signal,

package/dist/generated.d.ts

@@ -157,11 +157,13 @@ export interface SendTo {
 }
 /** A page of a protected file's map: its key, and where each fragment is. */
 export interface ShardMap {
-    /** The content's own id, which each fragment's check is bound to with its place and the count. */
+    /** The content's own id, which each fragment's check is bound to with its place and the count, and in version 2 the outline. */
     contentId: string;
+    /** The version of the shard format the file is written in: 1, or 2, in which the file is cut into blocks by its content and its next version keeps the blocks that did not change. Only a version named in formats is answered. */
+    format: number;
     /** The file's size. */
     sizeBytes: number;
-    /** How many bytes of the file each fragment holds, except the last, which holds what remains. */
+    /** How many bytes of the file each fragment holds, except the last, which holds what remains. In version 2, the largest block's: each block's own length is in the outline. */
     fragmentBytes: number;
     /** How many fragments the file is. */
     count: number;
@@ -171,11 +173,23 @@ export interface ShardMap {
     contentType: string | null;
     /** The file's revision whose content this is. Its fragments never change, however late they are read. */
     revision: number;
+    /** Version 2, on the first page: every block of the file and the MAC over them. Null on a later page, and in version 1. */
+    outline: {
+        /** The MAC over the content id, the size and every block's id and length, in base64. Check it with the file's key, as the SDK's checkOutline does, before using a byte of the file. */
+        mac: string;
+        /** Every block of the file, in order of place. */
+        blocks: Array<{
+            /** The block's id in base64: 32 bytes, the HMAC of its bytes under the file's key. Each block is checked against it once opened. */
+            id: string;
+            /** How many bytes of the file the block holds. */
+            length: number;
+        }>;
+    } | null;
     /** The first place on this page. */
     from: number;
     /** Up to 1000, in order of place. */
     fragments: Array<{
-        /** Counted from zero. Its bytes of the file begin at place times fragmentBytes. */
+        /** Counted from zero. In version 1 its bytes of the file begin at place times fragmentBytes; in version 2 after the blocks before it in the outline. */
         place: number;
         /** Where to fetch the stored fragment, with a plain GET and no key: the address is the permission. */
         url: string;
@@ -314,6 +328,8 @@ export type UploadsBeginBody = {
     }>;
     /** true when this program can seal fragments as it sends them, as the SDK and the CLI do. Into a protected place the answer's shards is then set, and each part is a fragment. */
     protectable?: boolean;
+    /** With protectable: the versions of the shard format this program can write. Left out, 1 alone. With 2, a sealed file of more than 16 MiB is written in version 2. */
+    shardFormats?: Array<number>;
 };
 /** What POST /v1/uploads answers with. */
 export type UploadsBeginAnswer = {
@@ -324,11 +340,13 @@ export type UploadsBeginAnswer = {
     reused: Array<number>;
     /** Set when the upload is sealed as it is sent: how the file is cut into fragments. The key to seal them with is given by uploads.get. Null for an upload sent as it is. */
     shards: {
-        /** The content's own id, which each fragment's seal is bound to with its place and the count. */
+        /** The version of the shard format to write it in: 1, or 2 when shardFormats names it and the file is large enough. */
+        format: number;
+        /** The content's own id, which each fragment's seal is bound to with its place and the count, and in version 2 the outline. */
         contentId: string;
-        /** How many bytes of the file each fragment holds, except the last, which holds what remains. */
+        /** How many bytes of the file each fragment holds, except the last, which holds what remains. In version 2, 0: the blocks are declared with uploads.blocks. */
         fragmentBytes: number;
-        /** How many fragments, one to each part. */
+        /** How many fragments, one to each part. In version 2, 0 until the blocks are declared. */
         count: number;
     } | null;
 };
@@ -346,16 +364,42 @@ export type UploadsGetAnswer = {
     reused: Array<number>;
     /** For an upload sealed as it is sent: how the file is cut, and the key to seal each fragment with. Null for an upload sent as it is, and for one that has ended. */
     shards: {
-        /** The content's own id, which each fragment's seal is bound to with its place and the count. */
+        /** The version of the shard format to write it in. */
+        format: number;
+        /** The content's own id, which each fragment's seal is bound to with its place and the count, and in version 2 the outline. */
         contentId: string;
         /** The file's own key, in base64: 32 bytes, which open its fragments. In this answer and nowhere else. */
         key: string;
-        /** How many bytes of the file each fragment holds, except the last, which holds what remains. */
+        /** How many bytes of the file each fragment holds, except the last, which holds what remains. In version 2, the largest block's once they are declared, and 0 before. */
         fragmentBytes: number;
-        /** How many fragments, one to each part. */
+        /** How many fragments, one to each part. In version 2, how many blocks once they are declared, and 0 before. */
         count: number;
+        /** Version 2: the blocks as declared, in order, or null until they are. Null in version 1. */
+        blocks: Array<{
+            size: number;
+            /** Its id, in base64. */
+            id: string;
+        }> | null;
     } | null;
 };
+/** What POST /v1/uploads/:id/blocks takes as its body. */
+export type UploadsBlocksBody = {
+    /** Every block of the file, in order. */
+    blocks: Array<{
+        /** How many bytes of the file the block holds. */
+        size: number;
+        /** The block's id, in base64: 32 bytes. */
+        id: string;
+    }>;
+};
+/** What POST /v1/uploads/:id/blocks answers with. */
+export type UploadsBlocksAnswer = {
+    upload: Upload;
+    /** The places whose block the file has already, under its key. Never sent, and counted as arrived. */
+    reused: Array<number>;
+    /** Addresses for the first blocks to send, each at its place. Places in reused are not among them. */
+    parts: Array<Part>;
+};
 /** What GET /v1/uploads/:id/parts takes in its query. */
 export type UploadsPartsQuery = {
     /** The first part wanted, counted from zero. */
@@ -384,6 +428,8 @@ export type UploadsFragmentsAnswer = {
 export type UploadsCompleteBody = {
     /** The file's SHA-256 in hex, as you worked it out. Kept with the file as your word, since this service never sees the bytes, and given back as clientSha256, so a copy can be checked later without reading the file again. */
     sha256?: string;
+    /** For an upload written in version 2 of the shard format, and required for one: the outline's MAC in base64, as the SDK's outlineMac makes it from the content id, the size and the declared blocks. Checked with the file's key. */
+    outline?: string;
 };
 /** What POST /v1/uploads/:id/complete answers with. */
 export type UploadsCompleteAnswer = {
@@ -417,6 +463,8 @@ export type FilesDownloadUrlAnswer = {
 export type FilesShardsQuery = {
     /** The first fragment wanted, counted from zero: 0, or the `next` of the page before. */
     from?: number;
+    /** The versions of the shard format this program reads, as 1,2. Left out, 1 alone. */
+    formats?: string;
 };
 /** What GET /v1/files/:id/shards answers with. */
 export type FilesShardsAnswer = ShardMap;
@@ -689,6 +737,12 @@ export declare const OPERATIONS: {
         readonly idempotent: false;
         readonly repeatable: false;
     };
+    readonly "uploads.blocks": {
+        readonly method: "POST";
+        readonly path: "/v1/uploads/:id/blocks";
+        readonly idempotent: false;
+        readonly repeatable: true;
+    };
     readonly "uploads.parts": {
         readonly method: "GET";
         readonly path: "/v1/uploads/:id/parts";
@@ -910,6 +964,8 @@ export declare function bind(call: Call): {
          *
          * A file under Shard Protection is kept as fragments, each sealed with the file's own key and stored across several storage accounts. Its map says where each fragment is, up to 1,000 at a time, with the key. Fetch each fragment from its address (no key: the address is the permission), check and open it as the SDK's openFragment does, and write it at its place. The first page counts the file's size toward the month's downloads; later pages count nothing. Addresses work for four hours: before then, ask for the page again from the first fragment not yet fetched. The key is in this answer and nowhere else. This API never keeps it, and neither should a program, beyond the read.
          *
+         * A file in version 2 of the shard format (format 2) is kept as blocks of their own lengths, cut by its content. The first page carries its outline: check it with the key before using a byte, open each block as the SDK's openBlock does, and write it after the blocks before it. A program says which versions it reads with formats, and a file in another is answered with 409 format_unsupported, so a program made before version 2 is told to update rather than failing on the file.
+         *
          * GET /v1/files/:id/shards. Needs the scope files:read.
          */
         shards: (id: string, query?: FilesShardsQuery, options?: CallOptions) => Promise<FilesShardsAnswer>;
@@ -966,6 +1022,8 @@ export declare function bind(call: Call): {
          *
          * Into a place under Shard Protection, an upload begun with protectable true is sealed as it is sent. A place is protected by the setting nearest it (files.update's protection), or with none by the drive's default: an app's, which its owner turns on from the app's page, or for a device the account's. The answer's shards then says how the file is cut, and uploads.get gives its key. Each part is a fragment: the file's bytes from its number times fragmentBytes, sealed with that key as the SDK's sealFragment does, under a nonce of its own, and sent to the part's address. uploads.fragments confirms them, and completing makes the file, protected from its first byte, with a null clientSha256. A sealed upload sets blocks aside: none is kept, and none reused. Begun without protectable, a file lands as an ordinary file and the service protects it within a few minutes; from then on it is read through its map (files.shards).
          *
+         * A program that writes version 2 of the shard format says so with shardFormats. A sealed file of more than 16 MiB is then written in it: shards says format 2, with no fragments and no parts yet. Get the key from uploads.get, cut the file into blocks by its content with it, and declare them with uploads.blocks, which answers with the parts to send. Replacing a file whose content is in version 2, with a key that may read the file, the key is the file's own, and the blocks the file has already are not sent again. Complete it with outline, the outline's MAC.
+         *
          * POST /v1/uploads. Needs the scope files:write. Honours an Idempotency-Key.
          */
         begin: (body: UploadsBeginBody, options?: CallOptions) => Promise<UploadsBeginAnswer>;
@@ -977,6 +1035,14 @@ export declare function bind(call: Call): {
          * GET /v1/uploads/:id. Needs the scope files:write.
          */
         get: (id: string, options?: CallOptions) => Promise<UploadsGetAnswer>;
+        /**
+         * Declare a sealed upload's blocks.
+         *
+         * For an upload written in version 2 of the shard format (its shards say format 2): the blocks the file is cut into by its content with the file's key, in order, each its size and its id, the HMAC of its bytes under the file's key. From 2 to 10,000 blocks of 1 byte to 64 MiB, adding up to sizeBytes. Answers with the places not to send, whose block the file has already under its key, and addresses for the first of the rest: ask for more with uploads.parts, and confirm them with uploads.fragments. Each part is its block, sealed as the SDK's sealBlock does. Declared once: the same blocks again are answered the same, and other blocks are refused with 409 blocks_declared.
+         *
+         * POST /v1/uploads/:id/blocks. Needs the scope files:write. Safe to send again.
+         */
+        blocks: (id: string, body: UploadsBlocksBody, options?: CallOptions) => Promise<UploadsBlocksAnswer>;
         /**
          * Addresses for more parts.
          *

package/dist/generated.js

@@ -16,6 +16,7 @@ export const OPERATIONS = {
     "changes.list": { method: "GET", path: "/v1/changes", idempotent: false, repeatable: false },
     "uploads.begin": { method: "POST", path: "/v1/uploads", idempotent: true, repeatable: false },
     "uploads.get": { method: "GET", path: "/v1/uploads/:id", idempotent: false, repeatable: false },
+    "uploads.blocks": { method: "POST", path: "/v1/uploads/:id/blocks", idempotent: false, repeatable: true },
     "uploads.parts": { method: "GET", path: "/v1/uploads/:id/parts", idempotent: false, repeatable: false },
     "uploads.fragments": { method: "POST", path: "/v1/uploads/:id/fragments", idempotent: false, repeatable: true },
     "uploads.complete": { method: "POST", path: "/v1/uploads/:id/complete", idempotent: false, repeatable: true },
@@ -115,6 +116,8 @@ export function bind(call) {
              *
              * A file under Shard Protection is kept as fragments, each sealed with the file's own key and stored across several storage accounts. Its map says where each fragment is, up to 1,000 at a time, with the key. Fetch each fragment from its address (no key: the address is the permission), check and open it as the SDK's openFragment does, and write it at its place. The first page counts the file's size toward the month's downloads; later pages count nothing. Addresses work for four hours: before then, ask for the page again from the first fragment not yet fetched. The key is in this answer and nowhere else. This API never keeps it, and neither should a program, beyond the read.
              *
+             * A file in version 2 of the shard format (format 2) is kept as blocks of their own lengths, cut by its content. The first page carries its outline: check it with the key before using a byte, open each block as the SDK's openBlock does, and write it after the blocks before it. A program says which versions it reads with formats, and a file in another is answered with 409 format_unsupported, so a program made before version 2 is told to update rather than failing on the file.
+             *
              * GET /v1/files/:id/shards. Needs the scope files:read.
              */
             shards: (id, query, options) => call("files.shards", { params: { id }, query }, options),
@@ -171,6 +174,8 @@ export function bind(call) {
              *
              * Into a place under Shard Protection, an upload begun with protectable true is sealed as it is sent. A place is protected by the setting nearest it (files.update's protection), or with none by the drive's default: an app's, which its owner turns on from the app's page, or for a device the account's. The answer's shards then says how the file is cut, and uploads.get gives its key. Each part is a fragment: the file's bytes from its number times fragmentBytes, sealed with that key as the SDK's sealFragment does, under a nonce of its own, and sent to the part's address. uploads.fragments confirms them, and completing makes the file, protected from its first byte, with a null clientSha256. A sealed upload sets blocks aside: none is kept, and none reused. Begun without protectable, a file lands as an ordinary file and the service protects it within a few minutes; from then on it is read through its map (files.shards).
              *
+             * A program that writes version 2 of the shard format says so with shardFormats. A sealed file of more than 16 MiB is then written in it: shards says format 2, with no fragments and no parts yet. Get the key from uploads.get, cut the file into blocks by its content with it, and declare them with uploads.blocks, which answers with the parts to send. Replacing a file whose content is in version 2, with a key that may read the file, the key is the file's own, and the blocks the file has already are not sent again. Complete it with outline, the outline's MAC.
+             *
              * POST /v1/uploads. Needs the scope files:write. Honours an Idempotency-Key.
              */
             begin: (body, options) => call("uploads.begin", { body }, options),
@@ -182,6 +187,14 @@ export function bind(call) {
              * GET /v1/uploads/:id. Needs the scope files:write.
              */
             get: (id, options) => call("uploads.get", { params: { id } }, options),
+            /**
+             * Declare a sealed upload's blocks.
+             *
+             * For an upload written in version 2 of the shard format (its shards say format 2): the blocks the file is cut into by its content with the file's key, in order, each its size and its id, the HMAC of its bytes under the file's key. From 2 to 10,000 blocks of 1 byte to 64 MiB, adding up to sizeBytes. Answers with the places not to send, whose block the file has already under its key, and addresses for the first of the rest: ask for more with uploads.parts, and confirm them with uploads.fragments. Each part is its block, sealed as the SDK's sealBlock does. Declared once: the same blocks again are answered the same, and other blocks are refused with 409 blocks_declared.
+             *
+             * POST /v1/uploads/:id/blocks. Needs the scope files:write. Safe to send again.
+             */
+            blocks: (id, body, options) => call("uploads.blocks", { params: { id }, body }, options),
             /**
              * Addresses for more parts.
              *

package/dist/index.d.ts

@@ -3,6 +3,6 @@ export * from "./generated.js";
 export { FRAME_MAX_BYTES, takesFrame } from "./frames.js";
 export { get, type GetOptions, type Got, type Into, type LinkedFile } from "./links.js";
 export { type Sink } from "./ranges.js";
-export { checkOutline, fileKeys, MOST_BLOCK_BYTES, openBlock, openFragment, outlineMac, sealBlock, sealFragment, SHARD_OVERHEAD, SHARD_VERSION, SHARD_VERSION_2, ShardError, shardKey, ShardLayout, type FileKeys, type FragmentAt, type Outline, type OutlineBlock, type ShardErrorCode, type ShardKey, } from "./shards.js";
+export { BlockLayout, checkOutline, fileKeys, MOST_BLOCK_BYTES, openBlock, openFragment, outlineMac, READS_FORMATS, sealBlock, sealFragment, SHARD_OVERHEAD, SHARD_VERSION, SHARD_VERSION_2, ShardError, shardKey, ShardLayout, type FileKeys, type FragmentAt, type Layout, type Outline, type OutlineBlock, type ShardErrorCode, type ShardKey, } from "./shards.js";
 export { FileFreakError, type CallOptions, type ClientOptions, type TokenSource } from "./transport.js";
 export { UploadStopped, type Source, type UploadOptions } from "./uploads.js";

package/dist/index.js

@@ -13,6 +13,6 @@ export { FileFreak, Walk } from "./client.js";
 export * from "./generated.js";
 export { FRAME_MAX_BYTES, takesFrame } from "./frames.js";
 export { get } from "./links.js";
-export { checkOutline, fileKeys, MOST_BLOCK_BYTES, openBlock, openFragment, outlineMac, sealBlock, sealFragment, SHARD_OVERHEAD, SHARD_VERSION, SHARD_VERSION_2, ShardError, shardKey, ShardLayout, } from "./shards.js";
+export { BlockLayout, checkOutline, fileKeys, MOST_BLOCK_BYTES, openBlock, openFragment, outlineMac, READS_FORMATS, sealBlock, sealFragment, SHARD_OVERHEAD, SHARD_VERSION, SHARD_VERSION_2, ShardError, shardKey, ShardLayout, } from "./shards.js";
 export { FileFreakError } from "./transport.js";
 export { UploadStopped } from "./uploads.js";

package/dist/links.js

@@ -17,6 +17,7 @@
 // API call is counted, since no key is used.
 import { fetchRanges } from "./ranges.js";
 import { asItIsKept, readProtected } from "./reader.js";
+import { READS_FORMATS, SHARD_VERSION } from "./shards.js";
 import { backoff, FileFreakError, sleep } from "./transport.js";
 const UUID = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
 /** As the API's route takes a token (api/src/routes/public.ts). */
@@ -106,8 +107,30 @@ class Reader {
             this.fail();
         if (!isCount(v.sizeBytes) || !isCount(v.fragmentBytes) || !isCount(v.count) || !Array.isArray(v.fragments))
             this.fail();
+        // A site from before version 2 of the format says no version, and every file it has is in version 1.
+        const format = v.format === undefined ? SHARD_VERSION : v.format;
+        if (!isCount(format))
+            this.fail();
+        const outline = v.outline === undefined || v.outline === null ? null : this.outline(v.outline);
         const fragments = v.fragments.map((f) => (isObject(f) && isCount(f.place) && isWebAddress(f.url) ? { place: f.place, url: f.url } : this.fail()));
-        return { contentId: v.contentId, sizeBytes: v.sizeBytes, fragmentBytes: v.fragmentBytes, count: v.count, key: v.key, fragments, expiresAt: v.expiresAt, grant: v.grant };
+        return {
+            contentId: v.contentId,
+            format,
+            sizeBytes: v.sizeBytes,
+            fragmentBytes: v.fragmentBytes,
+            count: v.count,
+            key: v.key,
+            outline,
+            fragments,
+            expiresAt: v.expiresAt,
+            grant: v.grant,
+        };
+    }
+    outline(v) {
+        if (!isObject(v) || typeof v.mac !== "string" || !Array.isArray(v.blocks))
+            this.fail();
+        const blocks = v.blocks.map((b) => (isObject(b) && typeof b.id === "string" && isCount(b.length) ? { id: b.id, length: b.length } : this.fail()));
+        return { mac: v.mac, blocks };
     }
 }
 /** The link's own site, asked with nothing that says who is asking. */
@@ -216,7 +239,7 @@ async function openShare(site, token, password) {
         downloadsLeft: view.downloadsLeft,
         list: async (folder, offset) => site.read.listing(await site.ask(`${base}/items?parent=${folder}&offset=${offset}&limit=${PAGE}`, { share })),
         address: async (id) => site.read.address(await site.ask(`${base}/files/${id}/url`, { share })),
-        page: async (id, from, read, signal) => site.read.map(await site.ask(`${base}/files/${id}/shards?from=${from}`, { share, read }, undefined, signal)),
+        page: async (id, from, read, signal) => site.read.map(await site.ask(`${base}/files/${id}/shards?from=${from}&formats=${READS_FORMATS}`, { share, read }, undefined, signal)),
     };
 }
 async function openPublic(site, id, username) {
@@ -232,7 +255,7 @@ async function openPublic(site, id, username) {
         downloadsLeft: null,
         list: async (folder, offset) => site.read.listing(await site.ask(`/api/public/files/${folder}/items?offset=${offset}&limit=${PAGE}`)),
         address: async (file) => site.read.address(await site.ask(`/api/public/files/${file}/url`)),
-        page: async (file, from, read, signal) => site.read.map(await site.ask(`/api/public/files/${file}/shards?from=${from}`, { read }, undefined, signal)),
+        page: async (file, from, read, signal) => site.read.map(await site.ask(`/api/public/files/${file}/shards?from=${from}&formats=${READS_FORMATS}`, { read }, undefined, signal)),
     };
 }
 /** Every file in a shared folder, with the folders it is in, read folder by folder. */

package/dist/reader.d.ts

@@ -10,11 +10,22 @@ export declare const NOT_FETCHED = "Part of this file could not be fetched from
 /** A page of a protected file's map, as the reader uses it: files.shards's answer, or a link's. */
 export interface MapPage {
     contentId: string;
+    /** The version of the shard format the file is in, 1 or 2. */
+    format: number;
     sizeBytes: number;
+    /** In version 2, the largest block. */
     fragmentBytes: number;
     count: number;
     /** The file's key, in base64. */
     key: string;
+    /** In version 2, on the first page only: the outline's MAC, and each block's id and length, in base64. */
+    outline: {
+        mac: string;
+        blocks: ReadonlyArray<{
+            id: string;
+            length: number;
+        }>;
+    } | null;
     fragments: ReadonlyArray<{
         place: number;
         url: string;

package/dist/reader.js

@@ -28,7 +28,13 @@
 //
 // The key is a WebCrypto key that cannot be exported, held inside one read
 // and let go of with it. No error here carries it.
-import { openFragment, ShardError, shardKey, ShardLayout, SHARD_OVERHEAD } from "./shards.js";
+//
+// A file in version 2 of the format is read the same way, a block at each
+// place, once its outline is checked: the map's first page carries it, and
+// nothing is fetched until its MAC holds under the file's key, since a map
+// whose blocks were reordered or swapped would otherwise be written out in
+// the wrong order with every block passing its own check.
+import { BlockLayout, checkOutline, fileKeys, openBlock, openFragment, ShardError, shardKey, ShardLayout, SHARD_OVERHEAD, SHARD_VERSION, SHARD_VERSION_2, } from "./shards.js";
 import { backoff, FileFreakError, sleep } from "./transport.js";
 const MIB = 1024 * 1024;
 /** The most bytes of fragments being fetched, or fetched and not yet written, at once in this program: five of the largest. */
@@ -131,11 +137,17 @@ function unreadable(what) {
 const damaged = (cause) => stopped("fragment_damaged", NOT_WHAT_WAS_STORED, cause);
 const notFetched = (cause) => stopped("fragment_unavailable", NOT_FETCHED, cause);
 /**
- * A page whose count, or one of whose places, the file's size and fragment
- * size do not allow: a map that does not add up is not read from.
+ * A page whose count, or one of whose places, the file's layout does not
+ * allow: a map that does not add up is not read from. In version 2 its
+ * count and fragment size are the outline's, its block count and largest.
  */
 function agrees(page, layout) {
-    if (page.count !== layout.count) {
+    if (layout instanceof BlockLayout) {
+        if (page.count !== layout.count || page.fragmentBytes !== layout.fragmentSize) {
+            throw stopped("layout", `This file's map does not add up: it says ${page.count} blocks of at most ${page.fragmentBytes} bytes, and its outline names ${layout.count} of at most ${layout.fragmentSize}. Nothing was saved.`);
+        }
+    }
+    else if (page.count !== layout.count) {
         throw stopped("layout", `This file's map does not add up: it says ${page.count} fragments, where ${layout.size} bytes in fragments of ${layout.fragmentSize} make ${layout.count}. Nothing was saved.`);
     }
     const wrong = page.fragments.find((f) => !Number.isInteger(f.place) || f.place < 0 || f.place >= layout.count);
@@ -143,17 +155,50 @@ function agrees(page, layout) {
         throw stopped("layout", `This file's map names fragment ${wrong.place}, and the file has ${layout.count}, numbered from 0. Nothing was saved.`);
     }
 }
-async function keyOf(text) {
-    let raw;
+/** Base64 from the map, as bytes, or the map is unreadable. */
+function bytesOf(text, what) {
     try {
-        raw = Uint8Array.from(atob(text), (c) => c.charCodeAt(0));
+        return Uint8Array.from(atob(text), (c) => c.charCodeAt(0));
     }
     catch {
-        throw unreadable("key");
+        throw unreadable(what);
     }
+}
+/**
+ * The file's pieces as its first page describes them. In version 2 the
+ * outline is checked under the key here, before a block is fetched, and one
+ * that fails is the same failure as a fragment that fails its check.
+ */
+async function piecesOf(first) {
+    const raw = bytesOf(first.key, "key");
     if (raw.length !== 32)
         throw unreadable("key");
-    return await shardKey(raw);
+    try {
+        if (first.format === SHARD_VERSION) {
+            const layout = new ShardLayout(first.sizeBytes, first.fragmentBytes);
+            const key = await shardKey(raw);
+            return { layout, open: (place, blob) => openFragment(key, { contentId: first.contentId, layout, place }, blob) };
+        }
+        if (first.format === SHARD_VERSION_2) {
+            if (!first.outline)
+                throw unreadable("outline");
+            const outline = {
+                contentId: first.contentId,
+                size: first.sizeBytes,
+                blocks: first.outline.blocks.map((b) => ({ id: bytesOf(b.id, "outline"), length: b.length })),
+            };
+            const layout = new BlockLayout(outline);
+            const keys = await fileKeys(raw);
+            await checkOutline(keys, outline, bytesOf(first.outline.mac, "outline"));
+            return { layout, open: (place, blob) => openBlock(keys, outline.blocks[place], blob) };
+        }
+    }
+    catch (e) {
+        if (e instanceof ShardError)
+            throw e.code === "layout" ? stopped("layout", `${e.message} Nothing was saved.`) : damaged(e);
+        throw e;
+    }
+    throw stopped("format_unsupported", `This file is in version ${first.format} of the shard format, which this version of the SDK does not read. Update it to read the file.`);
 }
 /**
  * Read the protected file whose map `pages` gives into `sink`. Resolves once
@@ -172,19 +217,11 @@ export async function readProtected(options) {
         const first = await options.pages(0, stop.signal);
         if (options.sizeBytes !== undefined && first.sizeBytes !== options.sizeBytes)
             throw stopped("content_changed", CHANGED);
-        let layout;
-        try {
-            layout = new ShardLayout(first.sizeBytes, first.fragmentBytes);
-        }
-        catch (e) {
-            if (e instanceof ShardError)
-                throw stopped("layout", `${e.message} Nothing was saved.`);
-            throw e;
-        }
-        agrees(first, layout);
         if (!UUID.test(first.contentId))
             throw unreadable("content id");
-        const reading = new Reading(options, stop, layout, first.contentId, await keyOf(first.key));
+        const pieces = await piecesOf(first);
+        agrees(first, pieces.layout);
+        const reading = new Reading(options, stop, pieces, first.contentId, first.format);
         reading.hold(first);
         await reading.fill();
     }
@@ -216,9 +253,9 @@ export async function asItIsKept(isProtected, plain, mapped) {
 class Reading {
     options;
     stop;
-    layout;
+    pieces;
     contentId;
-    key;
+    format;
     addresses = new Map();
     /** Held while a page is asked for, so that fragments waiting on the same page ask once. */
     asking = new Gate(1);
@@ -230,13 +267,15 @@ class Reading {
     wait;
     now;
     quietMs;
+    layout;
     written = 0;
-    constructor(options, stop, layout, contentId, key) {
+    constructor(options, stop, pieces, contentId, format) {
         this.options = options;
         this.stop = stop;
-        this.layout = layout;
+        this.pieces = pieces;
         this.contentId = contentId;
-        this.key = key;
+        this.format = format;
+        this.layout = pieces.layout;
         this.budget = options.budget ?? shared();
         this.fetcher = options.fetch ?? fetch.bind(globalThis);
         this.wait = options.sleep ?? sleep;
@@ -248,7 +287,10 @@ class Reading {
     }
     /** Keep a page's addresses, once it is known to be a page of this file's map. */
     hold(page) {
-        if (page.contentId !== this.contentId || page.sizeBytes !== this.layout.size || page.fragmentBytes !== this.layout.fragmentSize) {
+        if (page.contentId !== this.contentId ||
+            page.format !== this.format ||
+            page.sizeBytes !== this.layout.size ||
+            page.fragmentBytes !== this.layout.fragmentSize) {
             throw stopped("content_changed", CHANGED);
         }
         agrees(page, this.layout);
@@ -424,7 +466,7 @@ class Reading {
         }
         let plain;
         try {
-            plain = await openFragment(this.key, { contentId: this.contentId, layout: this.layout, place }, blob);
+            plain = await this.pieces.open(place, blob);
         }
         catch (e) {
             throw damaged(e);

package/dist/shards.d.ts

@@ -25,6 +25,8 @@ export declare class ShardLayout {
     };
     /** How many bytes of the file fragment `place` holds. */
     length(place: number): number;
+    /** The place of the fragment that holds byte `offset` of the file. */
+    placeAt(offset: number): number;
 }
 /**
  * A content's key as WebCrypto holds it. Named through crypto.subtle and not
@@ -59,6 +61,8 @@ export declare function sealWithNonce(key: ShardKey, at: FragmentAt, plaintext:
  */
 export declare function openFragment(key: ShardKey, at: FragmentAt, blob: Uint8Array): Promise<Uint8Array<ArrayBuffer>>;
 export declare const SHARD_VERSION_2 = 2;
+/** The versions of the format this reads, as a reader asks for a map in them (formats). */
+export declare const READS_FORMATS = "1,2";
 /** The most bytes of a file one block holds, as the API takes blocks. */
 export declare const MOST_BLOCK_BYTES: number;
 /** A block id: the HMAC-SHA256 of its bytes under the file's block key. */
@@ -108,6 +112,33 @@ export declare function sealBlockWithNonce(keys: FileKeys, id: Uint8Array, plain
  * returned.
  */
 export declare function openBlock(keys: FileKeys, block: OutlineBlock, blob: Uint8Array): Promise<Uint8Array<ArrayBuffer>>;
+/**
+ * Where each block of a file in version 2 is, from its outline: what
+ * ShardLayout is for version 1, to a reader that writes each block at its
+ * place in the file, or reads a range from the blocks that hold it. It
+ * refuses an outline that describes no file, before anything is fetched by
+ * it; that the outline is the file's is checkOutline's to say.
+ */
+export declare class BlockLayout {
+    readonly size: number;
+    readonly count: number;
+    /** The largest block, which is what a map of a file in version 2 gives as its fragment size. */
+    readonly fragmentSize: number;
+    /** Where each block begins, then the file's size: count + 1 offsets, rising. */
+    private readonly starts;
+    constructor(outline: Outline);
+    /** Where in the file block `place` begins, and where it ends (exclusive). */
+    range(place: number): {
+        start: number;
+        end: number;
+    };
+    /** How many bytes of the file block `place` holds. */
+    length(place: number): number;
+    /** The place of the block that holds byte `offset` of the file: the last that begins at or before it. */
+    placeAt(offset: number): number;
+}
+/** Where each piece of a file is, in either version: the reader's view of it, whichever the map says. */
+export type Layout = ShardLayout | BlockLayout;
 /** The outline's MAC, which the writer sends with the upload's completion and every reader checks before it releases a byte. */
 export declare function outlineMac(keys: FileKeys, outline: Outline): Promise<Uint8Array<ArrayBuffer>>;
 /**

package/dist/shards.js

@@ -60,6 +60,13 @@ export class ShardLayout {
         const { start, end } = this.range(place);
         return end - start;
     }
+    /** The place of the fragment that holds byte `offset` of the file. */
+    placeAt(offset) {
+        if (!Number.isSafeInteger(offset) || offset < 0 || offset >= this.size) {
+            throw new RangeError(`There is no byte ${offset}: this file has ${this.size}, numbered from 0.`);
+        }
+        return Math.floor(offset / this.fragmentSize);
+    }
 }
 /** A content's key, 32 bytes, made usable for its fragments and nothing else. */
 export function shardKey(raw) {
@@ -136,6 +143,8 @@ export async function openFragment(key, at, blob) {
 // Version 2: blocks cut by content, under a file's key that versions share
 // ---------------------------------------------------------------------------
 export const SHARD_VERSION_2 = 2;
+/** The versions of the format this reads, as a reader asks for a map in them (formats). */
+export const READS_FORMATS = `${SHARD_VERSION},${SHARD_VERSION_2}`;
 /** The most bytes of a file one block holds, as the API takes blocks. */
 export const MOST_BLOCK_BYTES = 64 * 1024 * 1024;
 /** A block id: the HMAC-SHA256 of its bytes under the file's block key. */
@@ -249,6 +258,61 @@ function checkShape(outline) {
     if (total !== size)
         throw new ShardError("layout", `The blocks hold ${total} bytes, and the file is ${size}.`);
 }
+/**
+ * Where each block of a file in version 2 is, from its outline: what
+ * ShardLayout is for version 1, to a reader that writes each block at its
+ * place in the file, or reads a range from the blocks that hold it. It
+ * refuses an outline that describes no file, before anything is fetched by
+ * it; that the outline is the file's is checkOutline's to say.
+ */
+export class BlockLayout {
+    size;
+    count;
+    /** The largest block, which is what a map of a file in version 2 gives as its fragment size. */
+    fragmentSize;
+    /** Where each block begins, then the file's size: count + 1 offsets, rising. */
+    starts;
+    constructor(outline) {
+        checkShape(outline);
+        this.size = outline.size;
+        this.count = outline.blocks.length;
+        this.starts = [0];
+        let largest = 0;
+        for (const b of outline.blocks) {
+            this.starts.push(this.starts[this.starts.length - 1] + b.length);
+            largest = Math.max(largest, b.length);
+        }
+        this.fragmentSize = largest;
+    }
+    /** Where in the file block `place` begins, and where it ends (exclusive). */
+    range(place) {
+        if (!Number.isInteger(place) || place < 0 || place >= this.count) {
+            throw new RangeError(`There is no block ${place}: this file has ${this.count}, numbered from 0.`);
+        }
+        return { start: this.starts[place], end: this.starts[place + 1] };
+    }
+    /** How many bytes of the file block `place` holds. */
+    length(place) {
+        const { start, end } = this.range(place);
+        return end - start;
+    }
+    /** The place of the block that holds byte `offset` of the file: the last that begins at or before it. */
+    placeAt(offset) {
+        if (!Number.isSafeInteger(offset) || offset < 0 || offset >= this.size) {
+            throw new RangeError(`There is no byte ${offset}: this file has ${this.size}, numbered from 0.`);
+        }
+        let low = 0;
+        let high = this.count - 1;
+        while (low < high) {
+            const middle = Math.ceil((low + high) / 2);
+            if (this.starts[middle] <= offset)
+                low = middle;
+            else
+                high = middle - 1;
+        }
+        return low;
+    }
+}
 /** The bytes the outline's MAC is over: the label, the content id, the size, the count, and each block's id and length in order. */
 function outlineBytes(outline) {
     const head = OUTLINE_LABEL.length + 16 + 8 + 4;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@filefreak.io/sdk",
-  "version": "0.4.0",
+  "version": "0.5.0",
   "description": "FileFreak's developer API for Node and the browser: every endpoint, uploads straight to storage in parts, downloads in ranges, and the changes to follow.",
   "license": "Apache-2.0",
   "homepage": "https://filefreak.io/developers/reference",