@filefreak.io/sdk 0.4.0 → 0.6.0

package/dist/links.js

@@ -17,6 +17,7 @@
 // API call is counted, since no key is used.
 import { fetchRanges } from "./ranges.js";
 import { asItIsKept, readProtected } from "./reader.js";
+import { READS_FORMATS, SHARD_VERSION } from "./shards.js";
 import { backoff, FileFreakError, sleep } from "./transport.js";
 const UUID = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
 /** As the API's route takes a token (api/src/routes/public.ts). */
@@ -106,8 +107,30 @@ class Reader {
             this.fail();
         if (!isCount(v.sizeBytes) || !isCount(v.fragmentBytes) || !isCount(v.count) || !Array.isArray(v.fragments))
             this.fail();
+        // A site from before version 2 of the format says no version, and every file it has is in version 1.
+        const format = v.format === undefined ? SHARD_VERSION : v.format;
+        if (!isCount(format))
+            this.fail();
+        const outline = v.outline === undefined || v.outline === null ? null : this.outline(v.outline);
         const fragments = v.fragments.map((f) => (isObject(f) && isCount(f.place) && isWebAddress(f.url) ? { place: f.place, url: f.url } : this.fail()));
-        return { contentId: v.contentId, sizeBytes: v.sizeBytes, fragmentBytes: v.fragmentBytes, count: v.count, key: v.key, fragments, expiresAt: v.expiresAt, grant: v.grant };
+        return {
+            contentId: v.contentId,
+            format,
+            sizeBytes: v.sizeBytes,
+            fragmentBytes: v.fragmentBytes,
+            count: v.count,
+            key: v.key,
+            outline,
+            fragments,
+            expiresAt: v.expiresAt,
+            grant: v.grant,
+        };
+    }
+    outline(v) {
+        if (!isObject(v) || typeof v.mac !== "string" || !Array.isArray(v.blocks))
+            this.fail();
+        const blocks = v.blocks.map((b) => (isObject(b) && typeof b.id === "string" && isCount(b.length) ? { id: b.id, length: b.length } : this.fail()));
+        return { mac: v.mac, blocks };
     }
 }
 /** The link's own site, asked with nothing that says who is asking. */
@@ -216,7 +239,7 @@ async function openShare(site, token, password) {
         downloadsLeft: view.downloadsLeft,
         list: async (folder, offset) => site.read.listing(await site.ask(`${base}/items?parent=${folder}&offset=${offset}&limit=${PAGE}`, { share })),
         address: async (id) => site.read.address(await site.ask(`${base}/files/${id}/url`, { share })),
-        page: async (id, from, read, signal) => site.read.map(await site.ask(`${base}/files/${id}/shards?from=${from}`, { share, read }, undefined, signal)),
+        page: async (id, from, read, signal) => site.read.map(await site.ask(`${base}/files/${id}/shards?from=${from}&formats=${READS_FORMATS}`, { share, read }, undefined, signal)),
     };
 }
 async function openPublic(site, id, username) {
@@ -232,7 +255,7 @@ async function openPublic(site, id, username) {
         downloadsLeft: null,
         list: async (folder, offset) => site.read.listing(await site.ask(`/api/public/files/${folder}/items?offset=${offset}&limit=${PAGE}`)),
         address: async (file) => site.read.address(await site.ask(`/api/public/files/${file}/url`)),
-        page: async (file, from, read, signal) => site.read.map(await site.ask(`/api/public/files/${file}/shards?from=${from}`, { read }, undefined, signal)),
+        page: async (file, from, read, signal) => site.read.map(await site.ask(`/api/public/files/${file}/shards?from=${from}&formats=${READS_FORMATS}`, { read }, undefined, signal)),
     };
 }
 /** Every file in a shared folder, with the folders it is in, read folder by folder. */

package/dist/reader.d.ts

@@ -10,11 +10,22 @@ export declare const NOT_FETCHED = "Part of this file could not be fetched from
 /** A page of a protected file's map, as the reader uses it: files.shards's answer, or a link's. */
 export interface MapPage {
     contentId: string;
+    /** The version of the shard format the file is in, 1 or 2. */
+    format: number;
     sizeBytes: number;
+    /** In version 2, the largest block. */
     fragmentBytes: number;
     count: number;
     /** The file's key, in base64. */
     key: string;
+    /** In version 2, on the first page only: the outline's MAC, and each block's id and length, in base64. */
+    outline: {
+        mac: string;
+        blocks: ReadonlyArray<{
+            id: string;
+            length: number;
+        }>;
+    } | null;
     fragments: ReadonlyArray<{
         place: number;
         url: string;

package/dist/reader.js

@@ -28,7 +28,13 @@
 //
 // The key is a WebCrypto key that cannot be exported, held inside one read
 // and let go of with it. No error here carries it.
-import { openFragment, ShardError, shardKey, ShardLayout, SHARD_OVERHEAD } from "./shards.js";
+//
+// A file in version 2 of the format is read the same way, a block at each
+// place, once its outline is checked: the map's first page carries it, and
+// nothing is fetched until its MAC holds under the file's key, since a map
+// whose blocks were reordered or swapped would otherwise be written out in
+// the wrong order with every block passing its own check.
+import { BlockLayout, checkOutline, fileKeys, openBlock, openFragment, ShardError, shardKey, ShardLayout, SHARD_OVERHEAD, SHARD_VERSION, SHARD_VERSION_2, } from "./shards.js";
 import { backoff, FileFreakError, sleep } from "./transport.js";
 const MIB = 1024 * 1024;
 /** The most bytes of fragments being fetched, or fetched and not yet written, at once in this program: five of the largest. */
@@ -131,11 +137,17 @@ function unreadable(what) {
 const damaged = (cause) => stopped("fragment_damaged", NOT_WHAT_WAS_STORED, cause);
 const notFetched = (cause) => stopped("fragment_unavailable", NOT_FETCHED, cause);
 /**
- * A page whose count, or one of whose places, the file's size and fragment
- * size do not allow: a map that does not add up is not read from.
+ * A page whose count, or one of whose places, the file's layout does not
+ * allow: a map that does not add up is not read from. In version 2 its
+ * count and fragment size are the outline's, its block count and largest.
  */
 function agrees(page, layout) {
-    if (page.count !== layout.count) {
+    if (layout instanceof BlockLayout) {
+        if (page.count !== layout.count || page.fragmentBytes !== layout.fragmentSize) {
+            throw stopped("layout", `This file's map does not add up: it says ${page.count} blocks of at most ${page.fragmentBytes} bytes, and its outline names ${layout.count} of at most ${layout.fragmentSize}. Nothing was saved.`);
+        }
+    }
+    else if (page.count !== layout.count) {
         throw stopped("layout", `This file's map does not add up: it says ${page.count} fragments, where ${layout.size} bytes in fragments of ${layout.fragmentSize} make ${layout.count}. Nothing was saved.`);
     }
     const wrong = page.fragments.find((f) => !Number.isInteger(f.place) || f.place < 0 || f.place >= layout.count);
@@ -143,17 +155,50 @@ function agrees(page, layout) {
         throw stopped("layout", `This file's map names fragment ${wrong.place}, and the file has ${layout.count}, numbered from 0. Nothing was saved.`);
     }
 }
-async function keyOf(text) {
-    let raw;
+/** Base64 from the map, as bytes, or the map is unreadable. */
+function bytesOf(text, what) {
     try {
-        raw = Uint8Array.from(atob(text), (c) => c.charCodeAt(0));
+        return Uint8Array.from(atob(text), (c) => c.charCodeAt(0));
     }
     catch {
-        throw unreadable("key");
+        throw unreadable(what);
     }
+}
+/**
+ * The file's pieces as its first page describes them. In version 2 the
+ * outline is checked under the key here, before a block is fetched, and one
+ * that fails is the same failure as a fragment that fails its check.
+ */
+async function piecesOf(first) {
+    const raw = bytesOf(first.key, "key");
     if (raw.length !== 32)
         throw unreadable("key");
-    return await shardKey(raw);
+    try {
+        if (first.format === SHARD_VERSION) {
+            const layout = new ShardLayout(first.sizeBytes, first.fragmentBytes);
+            const key = await shardKey(raw);
+            return { layout, open: (place, blob) => openFragment(key, { contentId: first.contentId, layout, place }, blob) };
+        }
+        if (first.format === SHARD_VERSION_2) {
+            if (!first.outline)
+                throw unreadable("outline");
+            const outline = {
+                contentId: first.contentId,
+                size: first.sizeBytes,
+                blocks: first.outline.blocks.map((b) => ({ id: bytesOf(b.id, "outline"), length: b.length })),
+            };
+            const layout = new BlockLayout(outline);
+            const keys = await fileKeys(raw);
+            await checkOutline(keys, outline, bytesOf(first.outline.mac, "outline"));
+            return { layout, open: (place, blob) => openBlock(keys, outline.blocks[place], blob) };
+        }
+    }
+    catch (e) {
+        if (e instanceof ShardError)
+            throw e.code === "layout" ? stopped("layout", `${e.message} Nothing was saved.`) : damaged(e);
+        throw e;
+    }
+    throw stopped("format_unsupported", `This file is in version ${first.format} of the shard format, which this version of the SDK does not read. Update it to read the file.`);
 }
 /**
  * Read the protected file whose map `pages` gives into `sink`. Resolves once
@@ -172,19 +217,11 @@ export async function readProtected(options) {
         const first = await options.pages(0, stop.signal);
         if (options.sizeBytes !== undefined && first.sizeBytes !== options.sizeBytes)
             throw stopped("content_changed", CHANGED);
-        let layout;
-        try {
-            layout = new ShardLayout(first.sizeBytes, first.fragmentBytes);
-        }
-        catch (e) {
-            if (e instanceof ShardError)
-                throw stopped("layout", `${e.message} Nothing was saved.`);
-            throw e;
-        }
-        agrees(first, layout);
         if (!UUID.test(first.contentId))
             throw unreadable("content id");
-        const reading = new Reading(options, stop, layout, first.contentId, await keyOf(first.key));
+        const pieces = await piecesOf(first);
+        agrees(first, pieces.layout);
+        const reading = new Reading(options, stop, pieces, first.contentId, first.format);
         reading.hold(first);
         await reading.fill();
     }
@@ -216,9 +253,9 @@ export async function asItIsKept(isProtected, plain, mapped) {
 class Reading {
     options;
     stop;
-    layout;
+    pieces;
     contentId;
-    key;
+    format;
     addresses = new Map();
     /** Held while a page is asked for, so that fragments waiting on the same page ask once. */
     asking = new Gate(1);
@@ -230,13 +267,15 @@ class Reading {
     wait;
     now;
     quietMs;
+    layout;
     written = 0;
-    constructor(options, stop, layout, contentId, key) {
+    constructor(options, stop, pieces, contentId, format) {
         this.options = options;
         this.stop = stop;
-        this.layout = layout;
+        this.pieces = pieces;
         this.contentId = contentId;
-        this.key = key;
+        this.format = format;
+        this.layout = pieces.layout;
         this.budget = options.budget ?? shared();
         this.fetcher = options.fetch ?? fetch.bind(globalThis);
         this.wait = options.sleep ?? sleep;
@@ -248,7 +287,10 @@ class Reading {
     }
     /** Keep a page's addresses, once it is known to be a page of this file's map. */
     hold(page) {
-        if (page.contentId !== this.contentId || page.sizeBytes !== this.layout.size || page.fragmentBytes !== this.layout.fragmentSize) {
+        if (page.contentId !== this.contentId ||
+            page.format !== this.format ||
+            page.sizeBytes !== this.layout.size ||
+            page.fragmentBytes !== this.layout.fragmentSize) {
             throw stopped("content_changed", CHANGED);
         }
         agrees(page, this.layout);
@@ -424,7 +466,7 @@ class Reading {
         }
         let plain;
         try {
-            plain = await openFragment(this.key, { contentId: this.contentId, layout: this.layout, place }, blob);
+            plain = await this.pieces.open(place, blob);
         }
         catch (e) {
             throw damaged(e);

package/dist/shards.d.ts

@@ -25,6 +25,8 @@ export declare class ShardLayout {
     };
     /** How many bytes of the file fragment `place` holds. */
     length(place: number): number;
+    /** The place of the fragment that holds byte `offset` of the file. */
+    placeAt(offset: number): number;
 }
 /**
  * A content's key as WebCrypto holds it. Named through crypto.subtle and not
@@ -59,6 +61,8 @@ export declare function sealWithNonce(key: ShardKey, at: FragmentAt, plaintext:
  */
 export declare function openFragment(key: ShardKey, at: FragmentAt, blob: Uint8Array): Promise<Uint8Array<ArrayBuffer>>;
 export declare const SHARD_VERSION_2 = 2;
+/** The versions of the format this reads, as a reader asks for a map in them (formats). */
+export declare const READS_FORMATS = "1,2";
 /** The most bytes of a file one block holds, as the API takes blocks. */
 export declare const MOST_BLOCK_BYTES: number;
 /** A block id: the HMAC-SHA256 of its bytes under the file's block key. */
@@ -108,6 +112,33 @@ export declare function sealBlockWithNonce(keys: FileKeys, id: Uint8Array, plain
  * returned.
  */
 export declare function openBlock(keys: FileKeys, block: OutlineBlock, blob: Uint8Array): Promise<Uint8Array<ArrayBuffer>>;
+/**
+ * Where each block of a file in version 2 is, from its outline: what
+ * ShardLayout is for version 1, to a reader that writes each block at its
+ * place in the file, or reads a range from the blocks that hold it. It
+ * refuses an outline that describes no file, before anything is fetched by
+ * it; that the outline is the file's is checkOutline's to say.
+ */
+export declare class BlockLayout {
+    readonly size: number;
+    readonly count: number;
+    /** The largest block, which is what a map of a file in version 2 gives as its fragment size. */
+    readonly fragmentSize: number;
+    /** Where each block begins, then the file's size: count + 1 offsets, rising. */
+    private readonly starts;
+    constructor(outline: Outline);
+    /** Where in the file block `place` begins, and where it ends (exclusive). */
+    range(place: number): {
+        start: number;
+        end: number;
+    };
+    /** How many bytes of the file block `place` holds. */
+    length(place: number): number;
+    /** The place of the block that holds byte `offset` of the file: the last that begins at or before it. */
+    placeAt(offset: number): number;
+}
+/** Where each piece of a file is, in either version: the reader's view of it, whichever the map says. */
+export type Layout = ShardLayout | BlockLayout;
 /** The outline's MAC, which the writer sends with the upload's completion and every reader checks before it releases a byte. */
 export declare function outlineMac(keys: FileKeys, outline: Outline): Promise<Uint8Array<ArrayBuffer>>;
 /**

package/dist/shards.js

@@ -60,6 +60,13 @@ export class ShardLayout {
         const { start, end } = this.range(place);
         return end - start;
     }
+    /** The place of the fragment that holds byte `offset` of the file. */
+    placeAt(offset) {
+        if (!Number.isSafeInteger(offset) || offset < 0 || offset >= this.size) {
+            throw new RangeError(`There is no byte ${offset}: this file has ${this.size}, numbered from 0.`);
+        }
+        return Math.floor(offset / this.fragmentSize);
+    }
 }
 /** A content's key, 32 bytes, made usable for its fragments and nothing else. */
 export function shardKey(raw) {
@@ -136,6 +143,8 @@ export async function openFragment(key, at, blob) {
 // Version 2: blocks cut by content, under a file's key that versions share
 // ---------------------------------------------------------------------------
 export const SHARD_VERSION_2 = 2;
+/** The versions of the format this reads, as a reader asks for a map in them (formats). */
+export const READS_FORMATS = `${SHARD_VERSION},${SHARD_VERSION_2}`;
 /** The most bytes of a file one block holds, as the API takes blocks. */
 export const MOST_BLOCK_BYTES = 64 * 1024 * 1024;
 /** A block id: the HMAC-SHA256 of its bytes under the file's block key. */
@@ -249,6 +258,61 @@ function checkShape(outline) {
     if (total !== size)
         throw new ShardError("layout", `The blocks hold ${total} bytes, and the file is ${size}.`);
 }
+/**
+ * Where each block of a file in version 2 is, from its outline: what
+ * ShardLayout is for version 1, to a reader that writes each block at its
+ * place in the file, or reads a range from the blocks that hold it. It
+ * refuses an outline that describes no file, before anything is fetched by
+ * it; that the outline is the file's is checkOutline's to say.
+ */
+export class BlockLayout {
+    size;
+    count;
+    /** The largest block, which is what a map of a file in version 2 gives as its fragment size. */
+    fragmentSize;
+    /** Where each block begins, then the file's size: count + 1 offsets, rising. */
+    starts;
+    constructor(outline) {
+        checkShape(outline);
+        this.size = outline.size;
+        this.count = outline.blocks.length;
+        this.starts = [0];
+        let largest = 0;
+        for (const b of outline.blocks) {
+            this.starts.push(this.starts[this.starts.length - 1] + b.length);
+            largest = Math.max(largest, b.length);
+        }
+        this.fragmentSize = largest;
+    }
+    /** Where in the file block `place` begins, and where it ends (exclusive). */
+    range(place) {
+        if (!Number.isInteger(place) || place < 0 || place >= this.count) {
+            throw new RangeError(`There is no block ${place}: this file has ${this.count}, numbered from 0.`);
+        }
+        return { start: this.starts[place], end: this.starts[place + 1] };
+    }
+    /** How many bytes of the file block `place` holds. */
+    length(place) {
+        const { start, end } = this.range(place);
+        return end - start;
+    }
+    /** The place of the block that holds byte `offset` of the file: the last that begins at or before it. */
+    placeAt(offset) {
+        if (!Number.isSafeInteger(offset) || offset < 0 || offset >= this.size) {
+            throw new RangeError(`There is no byte ${offset}: this file has ${this.size}, numbered from 0.`);
+        }
+        let low = 0;
+        let high = this.count - 1;
+        while (low < high) {
+            const middle = Math.ceil((low + high) / 2);
+            if (this.starts[middle] <= offset)
+                low = middle;
+            else
+                high = middle - 1;
+        }
+        return low;
+    }
+}
 /** The bytes the outline's MAC is over: the label, the content id, the size, the count, and each block's id and length in order. */
 function outlineBytes(outline) {
     const head = OUTLINE_LABEL.length + 16 + 8 + 4;

package/dist/uploads.d.ts

@@ -1,4 +1,5 @@
-import type { FilesFrameAnswer, FilesGetAnswer, Item, Upload, UploadsBeginAnswer, UploadsBeginBody, UploadsCompleteAnswer, UploadsCompleteBody, UploadsFragmentsAnswer, UploadsFragmentsBody, UploadsGetAnswer, UploadsPartsAnswer, UploadsPartsQuery } from "./generated.js";
+import { type Rule } from "./cut.js";
+import type { FilesFrameAnswer, FilesGetAnswer, Item, Upload, UploadsBeginAnswer, UploadsBeginBody, UploadsBlocksAnswer, UploadsBlocksBody, UploadsCancelAnswer, UploadsCompleteAnswer, UploadsCompleteBody, UploadsFragmentsAnswer, UploadsFragmentsBody, UploadsGetAnswer, UploadsPartsAnswer, UploadsPartsQuery } from "./generated.js";
 import { type CallOptions } from "./transport.js";
 /** What can be uploaded. A File's name and type are used when none are given. */
 export type Source = Blob | ArrayBuffer | Uint8Array;
@@ -18,7 +19,7 @@ export interface UploadOptions {
     contentType?: string;
     /** Smaller copies to make of an image as soon as it is stored. */
     imageSizes?: number[];
-    /** Bytes in each part, from 8 MiB to 64 MiB. Left out, 32 MiB, or less for a small file. */
+    /** Bytes in each part, from 8 MiB to 64 MiB, for a file not cut by its content: one under 16 MiB, or one of more blocks than the API takes. Left out, 32 MiB, or less for a small file. */
     partBytes?: number;
     /** Parts sent at once. */
     concurrency?: number;
@@ -52,11 +53,13 @@ export interface UploadCalls {
     begin(body: UploadsBeginBody, options?: CallOptions): Promise<UploadsBeginAnswer>;
     get(id: string, options?: CallOptions): Promise<UploadsGetAnswer>;
     parts(id: string, query?: UploadsPartsQuery, options?: CallOptions): Promise<UploadsPartsAnswer>;
+    blocks(id: string, body: UploadsBlocksBody, options?: CallOptions): Promise<UploadsBlocksAnswer>;
     fragments(id: string, body: UploadsFragmentsBody, options?: CallOptions): Promise<UploadsFragmentsAnswer>;
     complete(id: string, body?: UploadsCompleteBody, options?: CallOptions): Promise<UploadsCompleteAnswer>;
+    cancel(id: string, options?: CallOptions): Promise<UploadsCancelAnswer>;
     file(id: string, options?: CallOptions): Promise<FilesGetAnswer>;
     frame(id: string, options?: CallOptions): Promise<FilesFrameAnswer>;
 }
 /** The part size to ask for: 32 MiB, halved while that would leave fewer parts than are sent at once. */
 export declare function partSizeFor(fileBytes: number, atOnce: number): number;
-export declare function upload(calls: UploadCalls, fetcher: typeof fetch, source: Source, options?: UploadOptions): Promise<Item>;
+export declare function upload(calls: UploadCalls, fetcher: typeof fetch, source: Source, options?: UploadOptions, rule?: Rule): Promise<Item>;