@filefreak.io/sdk 0.2.0 → 0.3.0

package/README.md

@@ -6,8 +6,8 @@ later and in browsers. No dependencies.
 Every endpoint is a function, typed from the same table the API is built
 from. Around them: uploads sent straight to storage in parts and carried on
 after an interruption, downloads fetched in ranges, listings and changes read
-to the end, files under Shard Protection read through their maps, what a
-share link holds saved with no key, and requests sent again only when that
+to the end, files under Shard Protection sealed as they are sent and read
+through their maps, what a share link holds saved with no key, and requests sent again only when that
 cannot make anything twice. The reference is at
 https://filefreak.io/developers/reference.
 
@@ -34,6 +34,10 @@ than memory is fine. `toPath` writes beside the file and gives it its name
 only once every byte is there: a download that fails leaves nothing behind.
 A file under Shard Protection has no address, and `download` and `bytes`
 read it through its map instead, each fragment checked as it arrives.
+Into a folder under Shard Protection, `upload` seals each part with the
+file's own key before it is sent, so none of the file is ever plain in
+storage. The key is asked of the API for the upload and kept nowhere, and
+an upload carried on asks which fragments storage has before it sends any.
 
 Keep keys on a server. A key sent from a web page is refused.
 

package/dist/generated.d.ts

@@ -25,6 +25,8 @@ export interface Item {
     clientSha256: string | null;
     /** Whether the file is under Shard Protection: kept as sealed fragments, and read through its map (files.shards) rather than from an address. */
     protected: boolean;
+    /** Its own Shard Protection setting. inherit follows the nearest folder above it that has one, and with none the drive's default: the app's, which its owner turns on from the app's page, or for a device the account's. The nearest setting wins. A file is converted within a few minutes of a change, and protected says when it has been. */
+    protection: "on" | "off" | "inherit";
     createdAt: string;
     updatedAt: string;
 }
@@ -214,6 +216,8 @@ export type FilesUpdateBody = {
     parentId?: string | null;
     /** public, private, or inherit to follow the folder it is in. */
     visibility?: "private" | "public" | "inherit";
+    /** on, off, or inherit to follow the folder it is in. */
+    protection?: "on" | "off" | "inherit";
 };
 /** What PATCH /v1/files/:id answers with. */
 export type FilesUpdateAnswer = {
@@ -304,6 +308,8 @@ export type UploadsBeginBody = {
         /** The block's SHA-256 in hex. */
         sha256: string;
     }>;
+    /** true when this program can seal fragments as it sends them, as the SDK and the CLI do. Into a protected place the answer's shards is then set, and each part is a fragment. */
+    protectable?: boolean;
 };
 /** What POST /v1/uploads answers with. */
 export type UploadsBeginAnswer = {
@@ -312,6 +318,15 @@ export type UploadsBeginAnswer = {
     parts: Array<Part>;
     /** For an upload begun with blocks: the parts not to send, because storage has them already, from the file this replaces, or because they repeat a block an earlier part sends. Empty otherwise. Nothing is taken from the file this replaces when the key may not read it, when the file has not been opened or changed in 90 days, or when copying it would take the account past 1000 GB copied this way in an hour: every block is sent then. */
     reused: Array<number>;
+    /** Set when the upload is sealed as it is sent: how the file is cut into fragments. The key to seal them with is given by uploads.get. Null for an upload sent as it is. */
+    shards: {
+        /** The content's own id, which each fragment's seal is bound to with its place and the count. */
+        contentId: string;
+        /** How many bytes of the file each fragment holds, except the last, which holds what remains. */
+        fragmentBytes: number;
+        /** How many fragments, one to each part. */
+        count: number;
+    } | null;
 };
 /** What GET /v1/uploads/:id answers with. */
 export type UploadsGetAnswer = {
@@ -325,6 +340,17 @@ export type UploadsGetAnswer = {
     }> | null;
     /** As in the answer that began the upload: the parts not to send. */
     reused: Array<number>;
+    /** For an upload sealed as it is sent: how the file is cut, and the key to seal each fragment with. Null for an upload sent as it is, and for one that has ended. */
+    shards: {
+        /** The content's own id, which each fragment's seal is bound to with its place and the count. */
+        contentId: string;
+        /** The file's own key, in base64: 32 bytes, which open its fragments. In this answer and nowhere else. */
+        key: string;
+        /** How many bytes of the file each fragment holds, except the last, which holds what remains. */
+        fragmentBytes: number;
+        /** How many fragments, one to each part. */
+        count: number;
+    } | null;
 };
 /** What GET /v1/uploads/:id/parts takes in its query. */
 export type UploadsPartsQuery = {
@@ -336,6 +362,20 @@ export type UploadsPartsQuery = {
 export type UploadsPartsAnswer = {
     parts: Array<Part>;
 };
+/** What POST /v1/uploads/:id/fragments takes as its body. */
+export type UploadsFragmentsBody = {
+    /** The places of fragments that have been sent: up to 100 at a time. */
+    places: Array<number>;
+};
+/** What POST /v1/uploads/:id/fragments answers with. */
+export type UploadsFragmentsAnswer = {
+    /** At their own names, the size their places hold, and recorded. Never send them again. */
+    recorded: Array<number>;
+    /** Still being copied, or being confirmed by another request. Ask again shortly. */
+    pending: Array<number>;
+    /** Not in storage, or not their size and removed. Seal each again, under a new nonce, and send and confirm it. */
+    resend: Array<number>;
+};
 /** What POST /v1/uploads/:id/complete takes as its body. */
 export type UploadsCompleteBody = {
     /** The file's SHA-256 in hex, as you worked it out. Kept with the file as your word, since this service never sees the bytes, and given back as clientSha256, so a copy can be checked later without reading the file again. */
@@ -642,6 +682,12 @@ export declare const OPERATIONS: {
         readonly idempotent: false;
         readonly repeatable: false;
     };
+    readonly "uploads.fragments": {
+        readonly method: "POST";
+        readonly path: "/v1/uploads/:id/fragments";
+        readonly idempotent: false;
+        readonly repeatable: true;
+    };
     readonly "uploads.complete": {
         readonly method: "POST";
         readonly path: "/v1/uploads/:id/complete";
@@ -803,6 +849,8 @@ export declare function bind(call: Call): {
          *
          * Any of the three, in one request. They are applied in that order, each as its own change: if the second is refused the first has still happened, and the answer says which was refused. Moving a folder moves everything in it and is one change however much that is. Making a folder public makes everything in it public, except what has a setting of its own.
          *
+         * protection is a fourth, applied last: on puts the file or folder under Shard Protection, off takes it out, and inherit has it follow the folder it is in. A folder's setting reaches everything in it that has none of its own. Turning it on is refused while the account's plan does not include it (402 shard_protection_needs_pro); turning it off never is. Nothing is converted in this request: the service converts each file within a few minutes, and its protected says when. A key confined to a folder decides about what is in it, and not about the folder.
+         *
          * PATCH /v1/files/:id. Needs the scope files:write.
          */
         update: (id: string, body: FilesUpdateBody, options?: CallOptions) => Promise<FilesUpdateAnswer>;
@@ -889,7 +937,7 @@ export declare function bind(call: Call): {
          *
          * The room is set aside now, so an upload that would not fit is refused before a byte is sent. The parts can be sent by anything that can make a PUT request, including a browser that has never seen the API key: hand it the addresses.
          *
-         * Into a folder under Shard Protection, which its owner turns on from the website, a file lands as an ordinary file, as it would anywhere else, and the service protects it within a few minutes: its content, and every kept version, are spread over the storage lanes as fragments. Until then it can be read like any other file. From then on it is read through its map (files.shards), and its clientSha256 is null. An upload from a program is not sealed as it is sent yet.
+         * Into a place under Shard Protection, an upload begun with protectable true is sealed as it is sent. A place is protected by the setting nearest it (files.update's protection), or with none by the drive's default: an app's, which its owner turns on from the app's page, or for a device the account's. The answer's shards then says how the file is cut, and uploads.get gives its key. Each part is a fragment: the file's bytes from its number times fragmentBytes, sealed with that key as the SDK's sealFragment does, under a nonce of its own, and sent to the part's address. uploads.fragments confirms them, and completing makes the file, protected from its first byte, with a null clientSha256. A sealed upload sets blocks aside: none is kept, and none reused. Begun without protectable, a file lands as an ordinary file and the service protects it within a few minutes; from then on it is read through its map (files.shards).
          *
          * POST /v1/uploads. Needs the scope files:write. Honours an Idempotency-Key.
          */
@@ -897,7 +945,7 @@ export declare function bind(call: Call): {
         /**
          * An upload, and which parts have arrived.
          *
-         * For carrying on after an interruption: storage itself says which parts it has, so nothing the client remembered is needed.
+         * For carrying on after an interruption: storage itself says which parts it has, so nothing the client remembered is needed. For an upload sealed as it is sent, uploadedParts are the fragments confirmed and recorded, and shards gives the key, which this API keeps nowhere else. Confirm any fragment that may have been sent before sending it again. This answer is never kept, by this API or anything between it and the caller.
          *
          * GET /v1/uploads/:id. Needs the scope files:write.
          */
@@ -905,11 +953,19 @@ export declare function bind(call: Call): {
         /**
          * Addresses for more parts.
          *
-         * Addresses are given out a window at a time and each lasts half an hour. Ask again for any that have run out.
+         * Addresses are given out a window at a time and each lasts half an hour. Ask again for any that have run out. For an upload sealed as it is sent, each number is a fragment's place, and its address takes the sealed fragment, with its Content-MD5. Confirm fragments with uploads.fragments once they are sent.
          *
          * GET /v1/uploads/:id/parts. Needs the scope files:write.
          */
         parts: (id: string, query?: UploadsPartsQuery, options?: CallOptions) => Promise<UploadsPartsAnswer>;
+        /**
+         * Confirm sealed fragments.
+         *
+         * Confirms fragments of a sealed upload that have been sent. Each is copied inside storage to its own name, measured, and recorded once it is the size its place holds. recorded: done, never send it again. pending: storage is still copying it, or another confirmation of it is under way, so ask again shortly. resend: not there, or not its size and removed, so seal it again, under a new nonce, send it and confirm it. Safe to send again.
+         *
+         * POST /v1/uploads/:id/fragments. Needs the scope files:write. Safe to send again.
+         */
+        fragments: (id: string, body: UploadsFragmentsBody, options?: CallOptions) => Promise<UploadsFragmentsAnswer>;
         /**
          * Finish an upload.
          *

package/dist/generated.js

@@ -17,6 +17,7 @@ export const OPERATIONS = {
     "uploads.begin": { method: "POST", path: "/v1/uploads", idempotent: true, repeatable: false },
     "uploads.get": { method: "GET", path: "/v1/uploads/:id", idempotent: false, repeatable: false },
     "uploads.parts": { method: "GET", path: "/v1/uploads/:id/parts", idempotent: false, repeatable: false },
+    "uploads.fragments": { method: "POST", path: "/v1/uploads/:id/fragments", idempotent: false, repeatable: true },
     "uploads.complete": { method: "POST", path: "/v1/uploads/:id/complete", idempotent: false, repeatable: true },
     "uploads.cancel": { method: "DELETE", path: "/v1/uploads/:id", idempotent: false, repeatable: false },
     "files.downloadUrl": { method: "GET", path: "/v1/files/:id/download", idempotent: false, repeatable: false },
@@ -71,6 +72,8 @@ export function bind(call) {
              *
              * Any of the three, in one request. They are applied in that order, each as its own change: if the second is refused the first has still happened, and the answer says which was refused. Moving a folder moves everything in it and is one change however much that is. Making a folder public makes everything in it public, except what has a setting of its own.
              *
+             * protection is a fourth, applied last: on puts the file or folder under Shard Protection, off takes it out, and inherit has it follow the folder it is in. A folder's setting reaches everything in it that has none of its own. Turning it on is refused while the account's plan does not include it (402 shard_protection_needs_pro); turning it off never is. Nothing is converted in this request: the service converts each file within a few minutes, and its protected says when. A key confined to a folder decides about what is in it, and not about the folder.
+             *
              * PATCH /v1/files/:id. Needs the scope files:write.
              */
             update: (id, body, options) => call("files.update", { params: { id }, body }, options),
@@ -157,7 +160,7 @@ export function bind(call) {
              *
              * The room is set aside now, so an upload that would not fit is refused before a byte is sent. The parts can be sent by anything that can make a PUT request, including a browser that has never seen the API key: hand it the addresses.
              *
-             * Into a folder under Shard Protection, which its owner turns on from the website, a file lands as an ordinary file, as it would anywhere else, and the service protects it within a few minutes: its content, and every kept version, are spread over the storage lanes as fragments. Until then it can be read like any other file. From then on it is read through its map (files.shards), and its clientSha256 is null. An upload from a program is not sealed as it is sent yet.
+             * Into a place under Shard Protection, an upload begun with protectable true is sealed as it is sent. A place is protected by the setting nearest it (files.update's protection), or with none by the drive's default: an app's, which its owner turns on from the app's page, or for a device the account's. The answer's shards then says how the file is cut, and uploads.get gives its key. Each part is a fragment: the file's bytes from its number times fragmentBytes, sealed with that key as the SDK's sealFragment does, under a nonce of its own, and sent to the part's address. uploads.fragments confirms them, and completing makes the file, protected from its first byte, with a null clientSha256. A sealed upload sets blocks aside: none is kept, and none reused. Begun without protectable, a file lands as an ordinary file and the service protects it within a few minutes; from then on it is read through its map (files.shards).
              *
              * POST /v1/uploads. Needs the scope files:write. Honours an Idempotency-Key.
              */
@@ -165,7 +168,7 @@ export function bind(call) {
             /**
              * An upload, and which parts have arrived.
              *
-             * For carrying on after an interruption: storage itself says which parts it has, so nothing the client remembered is needed.
+             * For carrying on after an interruption: storage itself says which parts it has, so nothing the client remembered is needed. For an upload sealed as it is sent, uploadedParts are the fragments confirmed and recorded, and shards gives the key, which this API keeps nowhere else. Confirm any fragment that may have been sent before sending it again. This answer is never kept, by this API or anything between it and the caller.
              *
              * GET /v1/uploads/:id. Needs the scope files:write.
              */
@@ -173,11 +176,19 @@ export function bind(call) {
             /**
              * Addresses for more parts.
              *
-             * Addresses are given out a window at a time and each lasts half an hour. Ask again for any that have run out.
+             * Addresses are given out a window at a time and each lasts half an hour. Ask again for any that have run out. For an upload sealed as it is sent, each number is a fragment's place, and its address takes the sealed fragment, with its Content-MD5. Confirm fragments with uploads.fragments once they are sent.
              *
              * GET /v1/uploads/:id/parts. Needs the scope files:write.
              */
             parts: (id, query, options) => call("uploads.parts", { params: { id }, query }, options),
+            /**
+             * Confirm sealed fragments.
+             *
+             * Confirms fragments of a sealed upload that have been sent. Each is copied inside storage to its own name, measured, and recorded once it is the size its place holds. recorded: done, never send it again. pending: storage is still copying it, or another confirmation of it is under way, so ask again shortly. resend: not there, or not its size and removed, so seal it again, under a new nonce, send it and confirm it. Safe to send again.
+             *
+             * POST /v1/uploads/:id/fragments. Needs the scope files:write. Safe to send again.
+             */
+            fragments: (id, body, options) => call("uploads.fragments", { params: { id }, body }, options),
             /**
              * Finish an upload.
              *

package/dist/shards.d.ts

@@ -43,14 +43,14 @@ export declare function shardKey(raw: Uint8Array): Promise<ShardKey>;
 /** The bytes a fragment's tag also covers: the label, the content id, the place and the count. */
 export declare function associatedData(at: FragmentAt): Uint8Array<ArrayBuffer>;
 /** One fragment of a file, encrypted and ready to store, under a nonce of its own. */
-export declare function sealFragment(key: ShardKey, at: FragmentAt, plaintext: Uint8Array): Promise<Uint8Array>;
+export declare function sealFragment(key: ShardKey, at: FragmentAt, plaintext: Uint8Array): Promise<Uint8Array<ArrayBuffer>>;
 /**
  * sealFragment with the nonce chosen by the caller, which only the test
  * files may do: two fragments sealed under one key with one nonce give away
  * the difference between them, and the key's authentication with it. It is
  * not exported from the package.
  */
-export declare function sealWithNonce(key: ShardKey, at: FragmentAt, plaintext: Uint8Array, nonce: Uint8Array): Promise<Uint8Array>;
+export declare function sealWithNonce(key: ShardKey, at: FragmentAt, plaintext: Uint8Array, nonce: Uint8Array): Promise<Uint8Array<ArrayBuffer>>;
 /**
  * A stored fragment's bytes of the file, once it is known to be this file's
  * fragment at this place, unchanged. Nothing of a fragment that fails is

package/dist/uploads.d.ts

@@ -1,4 +1,4 @@
-import type { FilesGetAnswer, Item, Upload, UploadsBeginAnswer, UploadsBeginBody, UploadsCompleteAnswer, UploadsCompleteBody, UploadsGetAnswer, UploadsPartsAnswer, UploadsPartsQuery } from "./generated.js";
+import type { FilesGetAnswer, Item, Upload, UploadsBeginAnswer, UploadsBeginBody, UploadsCompleteAnswer, UploadsCompleteBody, UploadsFragmentsAnswer, UploadsFragmentsBody, UploadsGetAnswer, UploadsPartsAnswer, UploadsPartsQuery } from "./generated.js";
 import { type CallOptions } from "./transport.js";
 /** What can be uploaded. A File's name and type are used when none are given. */
 export type Source = Blob | ArrayBuffer | Uint8Array;
@@ -45,6 +45,7 @@ export interface UploadCalls {
     begin(body: UploadsBeginBody, options?: CallOptions): Promise<UploadsBeginAnswer>;
     get(id: string, options?: CallOptions): Promise<UploadsGetAnswer>;
     parts(id: string, query?: UploadsPartsQuery, options?: CallOptions): Promise<UploadsPartsAnswer>;
+    fragments(id: string, body: UploadsFragmentsBody, options?: CallOptions): Promise<UploadsFragmentsAnswer>;
     complete(id: string, body?: UploadsCompleteBody, options?: CallOptions): Promise<UploadsCompleteAnswer>;
     file(id: string, options?: CallOptions): Promise<FilesGetAnswer>;
 }

package/dist/uploads.js

@@ -17,7 +17,21 @@
 //
 // A part is read into memory whole to be hashed and sent, so the memory an
 // upload needs is about partBytes times the parts sent at once.
+//
+// SEALED AS IT IS SENT (plan 04, F6 step 4, answer 46). Every begin says the
+// SDK can seal (protectable), as the website and the CLI do. Into a place
+// under Shard Protection the API answers with shards, and each part is then
+// a fragment: the file's bytes at that place, sealed with the file's own key
+// (sealFragment, a nonce of its own each time) before it is hashed and sent.
+// The key comes from uploads.get, whose answer nothing keeps, and is held
+// only while the upload runs. Fragments sent are confirmed (uploads.fragments)
+// as they go, at most one confirmation at a time: each is copied inside
+// storage to its own name and recorded, or is to be sealed again and sent.
+// Carrying on, every fragment not recorded is confirmed before it is sent
+// again, since storage may hold it already. A fragment is up to 64 MiB and
+// sealed in memory, so the memory a sealed upload needs is the same.
 import { base64, Md5 } from "./md5.js";
+import { sealFragment, ShardError, shardKey, ShardLayout } from "./shards.js";
 import { FileFreakError, sleep } from "./transport.js";
 const MIB = 1024 * 1024;
 /** The API's bounds on a part. */
@@ -31,6 +45,14 @@ const PART_ATTEMPTS = 6;
 const ADDRESS_MARGIN_MS = 60_000;
 /** How long to go on asking while storage puts a very large file together. */
 const ASSEMBLY_PATIENCE_MS = 30 * 60_000;
+/** Fragments sent are confirmed this often, or as soon as this many are waiting, and at most this many at once (the API's window). */
+const CONFIRM_MS = 2000;
+const CONFIRM_AT = 32;
+const CONFIRM_MOST = 100;
+/** How long confirming may answer only "still copying" before the upload stops and says so. */
+const CONFIRM_PATIENCE_MS = 5 * 60_000;
+/** How many times fragments storage did not keep are sealed again and sent before the upload stops. */
+const SEALED_ROUNDS = 4;
 /** An upload that stopped part way. What arrived stays in storage until the upload expires: carry on with `resume: uploadId`, or cancel it. */
 export class UploadStopped extends Error {
     uploadId;
@@ -41,6 +63,34 @@ export class UploadStopped extends Error {
         this.name = "UploadStopped";
     }
 }
+/**
+ * The key and the cut of a sealed upload, from what uploads.get says, with
+ * every fragment it has not recorded waiting to be confirmed: one sent
+ * before, by this process or another, may be in storage already.
+ */
+async function sealingOf(state, size) {
+    const shards = state.shards;
+    if (!shards)
+        throw new FileFreakError(0, "unreadable", "The API began a sealed upload and then gave no key for it.");
+    let layout;
+    let raw;
+    try {
+        layout = new ShardLayout(size, shards.fragmentBytes);
+        raw = Uint8Array.from(atob(shards.key), (c) => c.charCodeAt(0));
+    }
+    catch (e) {
+        throw Object.assign(new FileFreakError(0, "unreadable", "The API answered with a sealed upload whose cut or key could not be read."), { cause: e });
+    }
+    if (layout.count !== shards.count || layout.count !== state.upload.partCount || raw.length !== 32) {
+        throw new FileFreakError(0, "layout", `The API cut this file into ${shards.count} fragments, where ${size} bytes in fragments of ${shards.fragmentBytes} make ${layout.count}.`);
+    }
+    const recorded = new Set(state.uploadedParts);
+    const waiting = new Set();
+    for (let n = 0; n < layout.count; n++)
+        if (!recorded.has(n))
+            waiting.add(n);
+    return { key: await shardKey(raw), contentId: shards.contentId, layout, recorded, waiting };
+}
 /** The part size to ask for: 32 MiB, halved while that would leave fewer parts than are sent at once. */
 export function partSizeFor(fileBytes, atOnce) {
     let size = 32 * MIB;
@@ -81,7 +131,8 @@ export async function upload(calls, fetcher, source, options = {}) {
         if (state.blocks) {
             throw new FileFreakError(400, "cut_by_content", "That upload was begun with its file cut by content, which this SDK does not do. Carry it on with the program that began it, or cancel it and begin again.");
         }
-        job = { source, size, upload: state.upload, addresses: new Map(), fetching: null, done: new Set(state.uploadedParts), sentBytes: 0 };
+        const sealing = state.shards ? await sealingOf(state, size) : null;
+        job = { source, size, upload: state.upload, addresses: new Map(), fetching: null, done: new Set(state.uploadedParts), sentBytes: 0, sealing };
     }
     else {
         const name = options.name ?? (typeof File !== "undefined" && source instanceof File ? source.name : undefined);
@@ -93,6 +144,8 @@ export async function upload(calls, fetcher, source, options = {}) {
             sizeBytes: size,
             parentId: options.parentId ?? null,
             partBytes: Math.max(LEAST_PART, Math.min(MOST_PART, options.partBytes ?? partSizeFor(size, atOnce))),
+            // Into a protected place, the parts are fragments this sends sealed.
+            protectable: true,
         };
         if (options.onConflict)
             body.onConflict = options.onConflict;
@@ -103,12 +156,23 @@ export async function upload(calls, fetcher, source, options = {}) {
         if (options.imageSizes)
             body.imageSizes = options.imageSizes;
         const begun = await calls.begin(body, { signal, ...(options.idempotencyKey ? { idempotencyKey: options.idempotencyKey } : {}) });
-        job = { source, size, upload: begun.upload, addresses: new Map(begun.parts.map((p) => [p.number, p])), fetching: null, done: new Set(), sentBytes: 0 };
+        job = { source, size, upload: begun.upload, addresses: new Map(begun.parts.map((p) => [p.number, p])), fetching: null, done: new Set(), sentBytes: 0, sealing: null };
+        if (begun.shards) {
+            // Begin's answer is kept by the API under its Idempotency-Key, so the key is asked for here.
+            try {
+                job.sealing = await sealingOf(await calls.get(begun.upload.id, call), size);
+            }
+            catch (e) {
+                throw new UploadStopped(begun.upload.id, e);
+            }
+            // Nothing of this upload can have been sent yet.
+            job.sealing.waiting.clear();
+        }
     }
     options.onBegun?.(job.upload);
     try {
         tally(job, options);
-        await sendParts(calls, fetcher, job, atOnce, options);
+        await deliver(calls, fetcher, job, atOnce, options);
         return await finish(calls, fetcher, job, atOnce, options);
     }
     catch (e) {
@@ -124,6 +188,71 @@ function tally(job, options) {
     job.sentBytes = sent;
     options.onProgress?.(sent, job.size);
 }
+/** Every part storage does not have, sent: as they are for a plain upload, and confirmed until each is recorded for a sealed one. */
+async function deliver(calls, fetcher, job, atOnce, options) {
+    const sealing = job.sealing;
+    if (!sealing)
+        return await sendParts(calls, fetcher, job, atOnce, options);
+    for (let round = 0;; round++) {
+        // Whatever may be in storage already is asked about before it is sent again.
+        await confirmAll(calls, job, sealing, options);
+        if (sealing.recorded.size === job.upload.partCount)
+            return;
+        // Round 0 sends each fragment once; each later round sends again what storage did not keep.
+        if (round > SEALED_ROUNDS) {
+            const left = job.upload.partCount - sealing.recorded.size;
+            throw new FileFreakError(0, "fragments_refused", `Storage did not keep ${left} of this file's fragments after ${SEALED_ROUNDS + 1} tries.`);
+        }
+        let sending = true;
+        const sent = sendParts(calls, fetcher, job, atOnce, options).finally(() => (sending = false));
+        await Promise.all([sent, confirmWhile(calls, job, sealing, () => sending, options)]);
+    }
+}
+/** While fragments are being sent: those sent are confirmed every CONFIRM_MS, or as soon as CONFIRM_AT are waiting. */
+async function confirmWhile(calls, job, sealing, sending, options) {
+    let last = Date.now();
+    while (sending()) {
+        await sleep(100, options.signal);
+        if (!sending())
+            return;
+        if (sealing.waiting.size >= CONFIRM_AT || (sealing.waiting.size > 0 && Date.now() - last >= CONFIRM_MS)) {
+            await confirmSome(calls, job, sealing, options);
+            last = Date.now();
+        }
+    }
+}
+/** Every fragment waiting asked about until none is: each recorded, or to be sealed again and sent. */
+async function confirmAll(calls, job, sealing, options) {
+    let heard = Date.now();
+    while (sealing.waiting.size > 0) {
+        if (await confirmSome(calls, job, sealing, options)) {
+            heard = Date.now();
+            continue;
+        }
+        if (Date.now() - heard > CONFIRM_PATIENCE_MS) {
+            throw new FileFreakError(0, "still_copying", "Storage has not finished taking part of this file. Nothing is lost: carry it on with resume in a few minutes.");
+        }
+        // Only fragments storage is still copying, or another confirmation holds, are left.
+        await sleep(CONFIRM_MS, options.signal);
+    }
+}
+/** Ask about the first CONFIRM_MOST fragments waiting. True when one was recorded or is to be sent again: either way it stops waiting. */
+async function confirmSome(calls, job, sealing, options) {
+    const places = [...sealing.waiting].sort((a, b) => a - b).slice(0, CONFIRM_MOST);
+    const answer = await calls.fragments(job.upload.id, { places }, { signal: options.signal });
+    for (const n of answer.recorded) {
+        sealing.recorded.add(n);
+        sealing.waiting.delete(n);
+        job.done.add(n);
+    }
+    for (const n of answer.resend) {
+        sealing.waiting.delete(n);
+        // Sent again, sealed again: never the same nonce over other bytes.
+        job.done.delete(n);
+    }
+    tally(job, options);
+    return answer.recorded.length > 0 || answer.resend.length > 0;
+}
 async function sendParts(calls, fetcher, job, atOnce, options) {
     const todo = [];
     for (let n = 0; n < job.upload.partCount; n++)
@@ -146,6 +275,7 @@ async function sendParts(calls, fetcher, job, atOnce, options) {
                 return;
             await sendPart(calls, fetcher, job, n, stop.signal);
             job.done.add(n);
+            job.sealing?.waiting.add(n);
             tally(job, options);
         }
     };
@@ -160,16 +290,27 @@ async function sendParts(calls, fetcher, job, atOnce, options) {
     }
 }
 async function sendPart(calls, fetcher, job, n, signal) {
-    const from = n * job.upload.partBytes;
+    const sealing = job.sealing;
+    const { start: from, end: to } = sealing ? sealing.layout.range(n) : { start: n * job.upload.partBytes, end: Math.min(job.size, (n + 1) * job.upload.partBytes) };
     let bytes;
     try {
-        bytes = await bytesOf(job.source, from, Math.min(job.size, from + job.upload.partBytes));
+        bytes = await bytesOf(job.source, from, to);
     }
     catch (e) {
         // Browsers and Node both refuse to read a file that changed after it was
         // opened, which is what keeps one from going up half old and half new.
         throw Object.assign(new FileFreakError(0, "unreadable", "The file could not be read. It may have been changed, moved or deleted since it was opened: open it again, unchanged, to carry on."), { cause: e });
     }
+    if (sealing) {
+        try {
+            bytes = await sealFragment(sealing.key, { contentId: sealing.contentId, layout: sealing.layout, place: n }, bytes);
+        }
+        catch (e) {
+            if (e instanceof ShardError)
+                throw Object.assign(new FileFreakError(0, "layout", e.message), { cause: e });
+            throw e;
+        }
+    }
     const md5 = base64(new Md5().update(bytes).digest());
     for (let attempt = 1;; attempt++) {
         const part = await address(calls, job, n, signal);
@@ -249,8 +390,13 @@ async function finish(calls, fetcher, job, atOnce, options) {
             if (e instanceof FileFreakError && e.code === "upload_incomplete" && round++ < 2) {
                 const have = Array.isArray(e.body?.uploadedParts) ? e.body.uploadedParts.filter((n) => typeof n === "number") : [];
                 job.done = new Set(have);
+                if (job.sealing) {
+                    // The fragments recorded: any other may be in storage, and is confirmed first.
+                    job.sealing.recorded = new Set(have);
+                    job.sealing.waiting = new Set(Array.from({ length: job.upload.partCount }, (_, n) => n).filter((n) => !job.sealing.recorded.has(n)));
+                }
                 tally(job, options);
-                await sendParts(calls, fetcher, job, atOnce, options);
+                await deliver(calls, fetcher, job, atOnce, options);
                 continue;
             }
             throw e;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@filefreak.io/sdk",
-  "version": "0.2.0",
+  "version": "0.3.0",
   "description": "FileFreak's developer API for Node and the browser: every endpoint, uploads straight to storage in parts, downloads in ranges, and the changes to follow.",
   "license": "Apache-2.0",
   "homepage": "https://filefreak.io/developers/reference",