@droponair/sdk-js 0.35.0 → 0.37.0

package/CHANGELOG.md

@@ -1,5 +1,23 @@
 # Changelog
 
+## 0.37.0
+
+### Added
+
+- **Uploads carry on where they stopped.** A file over 8 MB goes up in parts, and
+  an upload broken by a dead connection sends only the parts the storage does not
+  have yet. Nothing to change in an app: it happens inside
+  `prepareAttachmentAndUpload`, and `onUploadProgress` counts the whole file as
+  before.
+
+## 0.36.0
+
+### Added
+
+- **Downloads carry on where they stopped.** A download that breaks keeps what it
+  already had and asks the storage for the rest by range, retrying six times with a
+  growing wait. Nothing to change in an app: it happens inside `downloadAttachment`.
+
 ## 0.35.0
 
 ### Added

package/dist/attachment/attachment-client.d.ts

@@ -69,6 +69,18 @@ export declare class AttachmentClient {
     toWire(ref: AttachmentRef): WireAttachmentRef;
     /** Convert a wire AttachmentRef (from a received message) into the public type. */
     static fromWire(wire: WireAttachmentRef): AttachmentRef;
+    /**
+     * Sends a file a part at a time, skipping the parts the storage already has.
+     *
+     * Each part is asked for a link of its own, sent, and its tag kept; when they are
+     * all there the storage puts them together. An upload that broke leaves its parts
+     * behind, so trying again sends only what is missing.
+     */
+    private uploadInParts;
+    /** What the storage already holds for this upload. */
+    private partsAlreadyStored;
+    private partUrl;
+    private finishParts;
     private uploadBytes;
     private wrapKeyForRecipients;
     private unwrapKey;

package/dist/attachment/attachment-client.js

@@ -40,6 +40,11 @@ async function readWithProgress(resp, expectedBytes, onProgress) {
     }
     return out;
 }
+/** How many times a broken download is tried again, and how long between tries. */
+/** Above this a file goes up in parts rather than in one request. */
+const IN_PARTS_ABOVE_BYTES = 8 * 1024 * 1024;
+const DOWNLOAD_ATTEMPTS = 6;
+const DOWNLOAD_BACKOFF_MS = [2000, 4000, 8000, 15000, 30000];
 async function sha256Hex(bytes) {
     const digest = await crypto.subtle.digest('SHA-256', asBufferSource(bytes));
     const hex = [];
@@ -80,6 +85,8 @@ class AttachmentClient {
             body.retentionSeconds = opts.retentionSeconds;
         if (opts.deleteWhenEveryoneHasIt)
             body.deleteWhenEveryoneHasIt = true;
+        if (opts.inParts)
+            body.inParts = true;
         const resp = await this.deps.fetchFn(`${this.deps.httpUrl}/v1/attachments/upload-session`, {
             method: 'POST',
             headers: { Authorization: `Bearer ${jwt}`, 'Content-Type': 'application/json' },
@@ -164,8 +171,16 @@ class AttachmentClient {
             recipientUserIds,
             retentionSeconds: opts.retentionSeconds,
             deleteWhenEveryoneHasIt: opts.deleteWhenEveryoneHasIt,
+            // A file big enough that losing the connection near the end would mean
+            // sending all of it again goes up in parts instead.
+            inParts: storedBytes.length > IN_PARTS_ABOVE_BYTES,
         });
-        await this.uploadBytes(session, storedBytes, opts.onUploadProgress);
+        if (session.uploadId) {
+            await this.uploadInParts(session, storedBytes, opts.onUploadProgress);
+        }
+        else {
+            await this.uploadBytes(session, storedBytes, opts.onUploadProgress);
+        }
         await this.finalize(session.attachmentId, sha256);
         let wrappedKeys = [];
         if (encryptionType === 'E2EE' && fileKey) {
@@ -225,18 +240,49 @@ class AttachmentClient {
      * current device, unwraps it, then decrypts the stored bytes.
      */
     async downloadAttachment(ref, opts = {}) {
-        const info = await this.getDownloadUrl(ref.attachmentId);
-        const resp = await this.deps.fetchFn(info.url, { method: info.method, headers: info.headers });
-        if (!resp.ok) {
-            const text = await resp.text().catch(() => '');
-            const message = `Failed to download attachment bytes (HTTP ${resp.status}): ${text}`;
-            // The link was good and the bytes are not there: deleted from storage
-            // underneath a record that still existed.
-            if (resp.status === 404)
-                throw new attachment_types_1.AttachmentUnavailableError(ref.attachmentId, 404, message);
-            throw new Error(message);
+        // What has already arrived is kept between attempts, and the rest asked for by
+        // range, so a download that breaks does not start the file again. A fresh link
+        // is asked for each attempt, since a link lasts minutes.
+        let had = new Uint8Array(0);
+        let info;
+        for (let attempt = 1;; attempt++) {
+            try {
+                info = await this.getDownloadUrl(ref.attachmentId);
+                const headers = { ...info.headers };
+                if (had.length > 0)
+                    headers.Range = `bytes=${had.length}-`;
+                const resp = await this.deps.fetchFn(info.url, { method: info.method, headers });
+                if (resp.status === 416 && had.length > 0)
+                    break;
+                if (!resp.ok) {
+                    const text = await resp.text().catch(() => '');
+                    const message = `Failed to download attachment bytes (HTTP ${resp.status}): ${text}`;
+                    // The link was good and the bytes are not there: deleted from storage
+                    // underneath a record that still existed.
+                    if (resp.status === 404)
+                        throw new attachment_types_1.AttachmentUnavailableError(ref.attachmentId, 404, message);
+                    throw new Error(message);
+                }
+                const resuming = resp.status === 206 && had.length > 0;
+                const base = resuming ? had : new Uint8Array(0);
+                const rest = await readWithProgress(resp, Math.max(0, info.sizeBytes - base.length), opts.onProgress
+                    ? (received, total) => opts.onProgress(base.length + received, base.length + total)
+                    : undefined);
+                const joined = new Uint8Array(base.length + rest.length);
+                joined.set(base, 0);
+                joined.set(rest, base.length);
+                had = joined;
+                break;
+            }
+            catch (error) {
+                if (error instanceof attachment_types_1.AttachmentUnavailableError)
+                    throw error;
+                if (attempt >= DOWNLOAD_ATTEMPTS)
+                    throw error;
+                await new Promise((resolve) => setTimeout(resolve, DOWNLOAD_BACKOFF_MS[Math.min(attempt - 1, DOWNLOAD_BACKOFF_MS.length - 1)]));
+            }
         }
-        let stored = await readWithProgress(resp, info.sizeBytes, opts.onProgress);
+        let stored = had;
         if (ref.encryptionType === 'CLEARTEXT') {
             await this.confirmReceived(ref.attachmentId);
             return { attachmentId: ref.attachmentId, mimeType: info.mimeType, sizeBytes: info.sizeBytes, sha256: info.sha256, bytes: stored };
@@ -318,6 +364,83 @@ class AttachmentClient {
         };
     }
     // ─── internals ───
+    /**
+     * Sends a file a part at a time, skipping the parts the storage already has.
+     *
+     * Each part is asked for a link of its own, sent, and its tag kept; when they are
+     * all there the storage puts them together. An upload that broke leaves its parts
+     * behind, so trying again sends only what is missing.
+     */
+    async uploadInParts(session, bytes, onProgress) {
+        const partSize = session.partSizeBytes ?? 8 * 1024 * 1024;
+        const count = Math.max(1, Math.ceil(bytes.length / partSize));
+        const already = new Map();
+        for (const part of await this.partsAlreadyStored(session.attachmentId)) {
+            already.set(part.partNumber, { etag: part.etag, sizeBytes: part.sizeBytes });
+        }
+        const tags = new Map();
+        let sent = 0;
+        onProgress?.(0, bytes.length);
+        for (let number = 1; number <= count; number++) {
+            const from = (number - 1) * partSize;
+            const size = Math.min(partSize, bytes.length - from);
+            const had = already.get(number);
+            if (had && had.sizeBytes === size) {
+                tags.set(number, had.etag);
+                sent += size;
+                onProgress?.(sent, bytes.length);
+                continue;
+            }
+            const link = await this.partUrl(session.attachmentId, number);
+            const resp = await this.deps.fetchFn(link.uploadUrl, {
+                method: 'PUT',
+                headers: link.uploadHeaders ?? {},
+                body: asBufferSource(bytes.subarray(from, from + size)),
+            });
+            if (!resp.ok) {
+                const text = await resp.text().catch(() => '');
+                throw new Error(`Part ${number} failed (HTTP ${resp.status}): ${text.slice(0, 400)}`);
+            }
+            tags.set(number, resp.headers?.get?.('etag') ?? '');
+            sent += size;
+            onProgress?.(sent, bytes.length);
+        }
+        await this.finishParts(session.attachmentId, tags, partSize, bytes.length);
+    }
+    /** What the storage already holds for this upload. */
+    async partsAlreadyStored(attachmentId) {
+        const jwt = await this.deps.getValidDropOnAirJwt();
+        const resp = await this.deps.fetchFn(`${this.deps.httpUrl}/v1/attachments/${encodeURIComponent(attachmentId)}/parts`, { method: 'GET', headers: { Authorization: `Bearer ${jwt}` } });
+        if (!resp.ok)
+            return [];
+        return (await resp.json());
+    }
+    async partUrl(attachmentId, partNumber) {
+        const jwt = await this.deps.getValidDropOnAirJwt();
+        const resp = await this.deps.fetchFn(`${this.deps.httpUrl}/v1/attachments/${encodeURIComponent(attachmentId)}/parts/${partNumber}/url`, { method: 'POST', headers: { Authorization: `Bearer ${jwt}` } });
+        if (!resp.ok) {
+            const text = await resp.text().catch(() => '');
+            throw new Error(`No link for part ${partNumber} (HTTP ${resp.status}): ${text.slice(0, 400)}`);
+        }
+        return (await resp.json());
+    }
+    async finishParts(attachmentId, tags, partSize, totalBytes) {
+        const parts = [...tags.keys()].sort((a, b) => a - b).map((number) => ({
+            partNumber: number,
+            etag: tags.get(number) ?? '',
+            sizeBytes: Math.min(partSize, totalBytes - (number - 1) * partSize),
+        }));
+        const jwt = await this.deps.getValidDropOnAirJwt();
+        const resp = await this.deps.fetchFn(`${this.deps.httpUrl}/v1/attachments/${encodeURIComponent(attachmentId)}/parts/finish`, {
+            method: 'POST',
+            headers: { Authorization: `Bearer ${jwt}`, 'Content-Type': 'application/json' },
+            body: JSON.stringify({ parts }),
+        });
+        if (!resp.ok) {
+            const text = await resp.text().catch(() => '');
+            throw new Error(`The parts could not be put together (HTTP ${resp.status}): ${text.slice(0, 400)}`);
+        }
+    }
     async uploadBytes(session, bytes, onProgress) {
         // In a browser an XMLHttpRequest can say how far the body has got; fetch cannot.
         if (onProgress && typeof XMLHttpRequest !== 'undefined') {

package/dist/attachment/attachment-types.d.ts

@@ -36,6 +36,11 @@ export interface AttachmentRef {
 }
 /** Optional metadata for {@link DropOnAirClient.createUploadSession}. */
 export interface CreateUploadSessionOptions {
+    /**
+     * Send the file in parts, so an upload broken by a dead connection carries on
+     * with the parts that are missing. The SDK asks for this itself for big files.
+     */
+    inParts?: boolean;
     mimeType?: string;
     sizeBytes: number;
     encryptionType?: AttachmentEncryptionType;
@@ -58,6 +63,10 @@ export interface CreateUploadSessionOptions {
 }
 /** Result of reserving an upload session. */
 export interface UploadSession {
+    /** Set when the file goes up in parts: what the storage calls this upload. */
+    uploadId?: string;
+    /** How big each part is, the last one apart. */
+    partSizeBytes?: number;
     attachmentId: string;
     storageHint: string;
     uploadUrl: string;

package/dist/version.d.ts

@@ -7,7 +7,7 @@
  *   MINOR, additive feature (e.g. multi-device payloads, new call event type)
  *   PATCH, bug-fix / perf improvement with no wire or API change
  */
-export declare const SDK_VERSION = "0.35.0";
+export declare const SDK_VERSION = "0.37.0";
 /**
  * Binary encrypted-payload format version.
  * Included as the first byte of every encrypted payload so receivers can

package/dist/version.js

@@ -10,7 +10,7 @@ exports.PROTOCOL_VERSION = exports.PAYLOAD_FORMAT_VERSION = exports.SDK_VERSION
  *   MINOR, additive feature (e.g. multi-device payloads, new call event type)
  *   PATCH, bug-fix / perf improvement with no wire or API change
  */
-exports.SDK_VERSION = '0.35.0';
+exports.SDK_VERSION = '0.37.0';
 /**
  * Binary encrypted-payload format version.
  * Included as the first byte of every encrypted payload so receivers can

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@droponair/sdk-js",
-  "version": "0.35.0",
+  "version": "0.37.0",
   "description": "End-to-end encrypted messaging, voice and video calling SDK. The relay never sees your keys or message content.",
   "main": "dist/index.js",
   "types": "dist/index.d.ts",