@woodleighschool/bloby-client 1.2.0 → 1.3.0

package/README.md

@@ -1,20 +1,28 @@
 # @woodleighschool/bloby-client
 
-Headless browser transfers for [Bloby](../../bloby). The server chooses direct or multipart ingestion. `upload` resolves after the complete transfer is accepted by storage; applications own publication, attachment and UI.
+Headless browser transfers for [Bloby](../../bloby). `digest` produces what an application declares when it creates an upload. The server chooses direct or multipart ingestion. `upload` resolves once storage has accepted every byte; applications own creating and finalizing the upload, attachment and UI.
 
 ```ts
+const content = await digest(file, { signal, onProgress });
+// { size_bytes, sha256, crc64nvme }, declared when creating the upload.
+
 await upload({
   strategy: "multipart",
   blob: file,
   signal,
   onProgress,
   multipart: {
-    signPart: (partNumber, signal) => signPartRequest(partNumber, signal),
-    complete: (parts, signal) => completeMultipartRequest(parts, signal),
+    signPart: (partNumber, crc64nvme, signal) => signPartRequest(partNumber, crc64nvme, signal),
   },
 });
 ```
 
-The callbacks adapt application endpoints. Multipart completion must be idempotent: the client retries a failed completion once without retransmitting the parts. Part transfers also retry once with a fresh signed target. Cancellation preserves the supplied `AbortSignal` reason and stops further signing, transfers and completion retries. Finalization or attachment of the resulting object is a separate application operation.
+`digest` reads the blob once, in bounded slices, without holding it in memory. It returns the size with the SHA-256 and CRC-64/NVME of the whole blob as lowercase hex, and reports progress at each whole percent. It runs on the main thread and returns to the event loop between slices.
+
+For a direct upload, pass the server's `{ strategy: "direct-put", target }` action together with the blob and options. The client sends `target.headers` as given; storage accepts only the declared bytes.
+
+A multipart upload sends 64 MiB parts, four at a time; parts grow when a blob would otherwise need more than 10,000. For each part the client computes its CRC-64/NVME, asks `signPart` for a target signed for that checksum, and sends the part with the returned headers. A failed part is attempted three times in total, each with a fresh target, after a short backoff. The first part to exhaust its attempts stops the others and rejects `upload` with its error. Progress is the sum of finished parts and bytes in flight, so it steps back when a part is retried.
+
+`signPart` adapts an application endpoint. Cancellation preserves the supplied `AbortSignal` reason and stops further signing and transfers. Completing the multipart upload and finalizing or attaching the object are server and application operations.
 
-For a direct upload, pass the server's `{ strategy: "direct-put", target }` action together with the blob and options. The core package has no React, router or API-client dependency.
+The package has no React, router or API-client dependency, and uses no WebAssembly, workers or `eval`, so it runs under `script-src 'self'`.

package/dist/index.d.ts

@@ -8,9 +8,13 @@ export interface UploadTarget {
     method: "PUT";
     headers?: Record<string, string>;
 }
+export interface Content {
+    size_bytes: number;
+    sha256: string;
+    crc64nvme: string;
+}
 export interface MultipartUploadRequest {
-    signPart: (partNumber: number, signal?: AbortSignal) => Promise<UploadTarget>;
-    complete: (parts: CompletedPart[], signal?: AbortSignal) => Promise<void>;
+    signPart: (partNumber: number, crc64nvme: string, signal?: AbortSignal) => Promise<UploadTarget>;
 }
 export type UploadAction = {
     strategy: "direct-put";
@@ -24,13 +28,13 @@ export type UploadRequest = Extract<UploadAction, {
     strategy: "multipart";
     multipart: MultipartUploadRequest;
 };
-export interface CompletedPart {
-    part_number: number;
-    etag: string;
-}
 export interface UploadOptions {
     blob: Blob;
     signal?: AbortSignal;
     onProgress?: (progress: UploadProgress) => void;
 }
+export declare function digest(blob: Blob, options?: {
+    signal?: AbortSignal;
+    onProgress?: (progress: UploadProgress) => void;
+}): Promise<Content>;
 export declare function upload(request: UploadRequest & UploadOptions): Promise<void>;

package/dist/index.js

@@ -1,67 +1,104 @@
+import { Crc64Nvme } from "@aws-sdk/crc64-nvme";
+import { SHA256 } from "@stablelib/sha256";
+import pMap from "p-map";
+import pRetry from "p-retry";
+import { uint8ArrayToHex } from "uint8array-extras";
+// A blob is hashed one slice at a time. The slice bounds the memory held and
+// how long a hashing step occupies the main thread; awaiting the next slice
+// returns to the event loop. Smaller slices spend a growing share of the pass
+// on reads.
+const readSize = 512 * 1024;
 // Bloby chooses 64 MiB as its browser base part size. S3 separately limits a
 // multipart upload to 10,000 parts.
 const multipartPartSize = 64 * 1024 * 1024;
 const maximumMultipartParts = 10_000;
-// Resolves after storage accepts the complete upload. Publishing and attaching
-// the object remain application operations.
+const partConcurrency = 4;
+const partAttempts = 3;
+// Reads the blob once, without holding it in memory. Progress is reported at
+// each whole percent, not at each slice, which would be hundreds of times a
+// second.
+export async function digest(blob, options = {}) {
+    const sha = new SHA256();
+    const crc = new Crc64Nvme();
+    let loaded = 0;
+    let reported = 0;
+    for await (const bytes of read(blob, options.signal)) {
+        sha.update(bytes);
+        crc.update(bytes);
+        loaded += bytes.length;
+        const progress = uploadProgress(loaded, blob.size);
+        if (progress.percent !== reported || loaded === blob.size) {
+            reported = progress.percent;
+            options.onProgress?.(progress);
+        }
+    }
+    return {
+        size_bytes: blob.size,
+        sha256: uint8ArrayToHex(sha.digest()),
+        crc64nvme: uint8ArrayToHex(await crc.digest()),
+    };
+}
+// Resolves once storage has accepted every byte. Completing a multipart upload
+// and finalizing the object remain server and application operations.
 export async function upload(request) {
+    const { blob, signal, onProgress } = request;
+    const report = (loaded) => onProgress?.(uploadProgress(loaded, blob.size));
     if (request.strategy === "direct-put") {
-        await uploadTarget(request.target, request.blob, request.blob.size, 0, request.onProgress, request.signal);
+        await put(request.target, blob, signal, report);
         return;
     }
-    return uploadWithMultipartProgress(request);
+    await uploadParts(request.multipart, blob, signal, report);
 }
-async function uploadWithMultipartProgress(request) {
-    const { blob, signal, onProgress } = request;
-    signal?.throwIfAborted();
+async function uploadParts(multipart, blob, external, report) {
     if (blob.size === 0)
         throw new Error("Multipart uploads require a nonempty blob.");
     const partSize = Math.max(multipartPartSize, Math.ceil(blob.size / maximumMultipartParts));
-    const parts = [];
-    let completedBytes = 0;
-    for (let offset = 0, partNumber = 1; offset < blob.size; offset += partSize, partNumber++) {
-        const chunk = blob.slice(offset, Math.min(offset + partSize, blob.size));
-        const etag = await uploadPart(request.multipart, partNumber, chunk, blob.size, completedBytes, onProgress, signal);
-        parts.push({ part_number: partNumber, etag });
-        completedBytes += chunk.size;
-        onProgress?.(uploadProgress(completedBytes, blob.size));
-    }
-    // Completion is idempotent: a lost response must not require transferring
-    // the parts again. Only the storage completion callback is retried.
-    for (let attempt = 0;; attempt++) {
-        signal?.throwIfAborted();
+    const partCount = Math.ceil(blob.size / partSize);
+    // The first part to fail aborts the others with its error.
+    const failure = new AbortController();
+    const signal = external ? AbortSignal.any([external, failure.signal]) : failure.signal;
+    let loaded = 0;
+    await pMap(Array.from({ length: partCount }, (_, index) => index), async (index) => {
+        const part = blob.slice(index * partSize, (index + 1) * partSize);
+        let partLoaded = 0;
         try {
-            await request.multipart.complete(parts, signal);
-            return;
+            await uploadPart(multipart, index + 1, part, signal, (bytes) => {
+                loaded += bytes - partLoaded;
+                partLoaded = bytes;
+                report(loaded);
+            });
         }
         catch (error) {
-            signal?.throwIfAborted();
-            if (attempt === 1)
-                throw error;
+            failure.abort(error);
+            throw error;
         }
-    }
+    }, { concurrency: partConcurrency, signal });
+}
+async function uploadPart(multipart, partNumber, part, signal, onLoaded) {
+    const crc = new Crc64Nvme();
+    for await (const bytes of read(part, signal))
+        crc.update(bytes);
+    const checksum = uint8ArrayToHex(await crc.digest());
+    // Every attempt asks for a fresh target; an earlier signature may have expired.
+    await pRetry(async () => put(await multipart.signPart(partNumber, checksum, signal), part, signal, onLoaded), {
+        retries: partAttempts - 1,
+        signal,
+        onFailedAttempt: () => {
+            if (!signal.aborted)
+                onLoaded(0);
+        },
+    });
+    onLoaded(part.size);
 }
-async function uploadPart(multipart, partNumber, chunk, totalBytes, completedBytes, onProgress, signal) {
-    let lastError;
-    for (let attempt = 0; attempt < 2; attempt++) {
+async function* read(blob, signal) {
+    signal?.throwIfAborted();
+    for (let offset = 0; offset < blob.size; offset += readSize) {
+        const bytes = new Uint8Array(await blob.slice(offset, offset + readSize).arrayBuffer());
         signal?.throwIfAborted();
-        try {
-            const target = await multipart.signPart(partNumber, signal);
-            const etag = await uploadTarget(target, chunk, totalBytes, completedBytes, onProgress, signal);
-            if (!etag) {
-                throw new Error(`Multipart part ${partNumber} did not return an ETag.`);
-            }
-            return etag;
-        }
-        catch (error) {
-            signal?.throwIfAborted();
-            lastError = error;
-            onProgress?.(uploadProgress(completedBytes, totalBytes));
-        }
+        yield bytes;
     }
-    throw lastError;
 }
-function uploadTarget(target, body, totalBytes, completedBytes, onProgress, signal) {
+function put(target, body, signal, onLoaded) {
     return new Promise((resolve, reject) => {
         if (signal?.aborted) {
             reject(signal.reason);
@@ -70,13 +107,11 @@ function uploadTarget(target, body, totalBytes, completedBytes, onProgress, sign
         const xhr = new XMLHttpRequest();
         const finish = () => signal?.removeEventListener("abort", abort);
         const abort = () => xhr.abort();
-        xhr.upload.addEventListener("progress", (event) => {
-            onProgress?.(uploadProgress(completedBytes + event.loaded, totalBytes));
-        });
+        xhr.upload.addEventListener("progress", (event) => onLoaded(event.loaded));
         xhr.addEventListener("load", () => {
             finish();
             if (xhr.status >= 200 && xhr.status < 300) {
-                resolve(xhr.getResponseHeader("ETag"));
+                resolve();
                 return;
             }
             reject(new Error(`Upload failed with HTTP ${xhr.status}.`));

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@woodleighschool/bloby-client",
-  "version": "1.2.0",
+  "version": "1.3.0",
   "description": "Headless browser transfers for Bloby direct and multipart uploads.",
   "homepage": "https://github.com/woodleighschool/goodies/tree/main/packages/bloby-client",
   "bugs": "https://github.com/woodleighschool/goodies/issues",
@@ -33,9 +33,16 @@
     "prepublishOnly": "pnpm run typecheck && pnpm run test",
     "prepare": "pnpm run build"
   },
+  "dependencies": {
+    "@aws-sdk/crc64-nvme": "^3.972.41",
+    "@stablelib/sha256": "^2.0.1",
+    "p-map": "^7.0.8",
+    "p-retry": "^8.0.1",
+    "uint8array-extras": "^1.6.0"
+  },
   "devDependencies": {
-    "@types/node": "^26.6.1",
-    "tsx": "^4.23.13",
+    "@types/node": "^26.6.4",
+    "tsx": "^4.23.15",
     "typescript": "~7.0.2"
   }
 }