@sequenceholdings/storage 0.1.0

package/dist/transfer.js

@@ -0,0 +1,257 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.StorageTransferError = void 0;
+exports.transferStorageUpload = transferStorageUpload;
+/** GCS sequential resumable transfer. No Atlas endpoint receives file bytes. */
+const crc32c_js_1 = require("./crc32c.js");
+class StorageTransferError extends Error {
+    status;
+    constructor(message, status) {
+        super(message);
+        this.status = status;
+        this.name = 'StorageTransferError';
+    }
+}
+exports.StorageTransferError = StorageTransferError;
+function checkedUploadUrl(value) {
+    const url = new URL(value);
+    if (url.protocol !== 'https:' ||
+        url.hostname !== 'storage.googleapis.com' ||
+        !url.pathname.startsWith('/upload/') ||
+        !url.searchParams.has('upload_id') ||
+        url.port ||
+        url.username ||
+        url.password) {
+        throw new StorageTransferError('Invalid storage upload destination');
+    }
+    return url.href;
+}
+async function* bytes(source) {
+    if (source instanceof Blob || source instanceof ReadableStream) {
+        const reader = (source instanceof Blob ? source.stream() : source).getReader();
+        try {
+            for (;;) {
+                const part = await reader.read();
+                if (part.done)
+                    return;
+                yield part.value;
+            }
+        }
+        finally {
+            await reader.cancel().catch(() => { });
+            reader.releaseLock();
+        }
+    }
+    else {
+        yield* source;
+    }
+}
+/**
+ * Retain at most two fixed-size chunks, independent of total file size. Every
+ * source byte is hashed as it is read, including the acknowledged prefix a
+ * resume skips: the digest describes the object the source says it is, so a
+ * source that yields different bytes on re-read cannot match what GCS stored.
+ */
+async function* chunks(source, size, skip, hash, signal) {
+    let buffer = new Uint8Array(size);
+    let used = 0;
+    for await (const input of bytes(source)) {
+        signal?.throwIfAborted();
+        hash.update(input);
+        let position = Math.min(skip, input.byteLength);
+        skip -= position;
+        while (position < input.byteLength) {
+            const length = Math.min(size - used, input.byteLength - position);
+            buffer.set(input.subarray(position, position + length), used);
+            used += length;
+            position += length;
+            if (used === size) {
+                yield buffer;
+                buffer = new Uint8Array(size);
+                used = 0;
+            }
+        }
+    }
+    if (skip)
+        throw new StorageTransferError('Upload source is shorter than the saved offset');
+    if (used)
+        yield buffer.subarray(0, used);
+}
+function receivedOffset(response) {
+    const range = response.headers.get('Range');
+    if (!range)
+        return 0;
+    const match = /^bytes=0-(\d+)$/.exec(range);
+    const offset = match ? Number(match[1]) + 1 : NaN;
+    if (!Number.isSafeInteger(offset) || offset < 1)
+        throw new StorageTransferError('Storage returned an invalid upload offset');
+    return offset;
+}
+// GCS uses 308 for upload progress. Fetch treats it as a redirect; Google's
+// no-308 protocol preserves the acknowledgement without following a Location.
+function incomplete(response) {
+    return (response.status === 308 ||
+        response.headers.get('X-Http-Status-Code-Override') === '308');
+}
+function retryable(status) {
+    return status === 408 || status === 429 || status >= 500;
+}
+async function delay(attempt, signal) {
+    signal?.throwIfAborted();
+    await new Promise((resolve, reject) => {
+        const timer = setTimeout(() => {
+            signal?.removeEventListener('abort', abort);
+            resolve();
+        }, Math.min(250 * 2 ** attempt, 4000));
+        const abort = () => {
+            clearTimeout(timer);
+            reject(signal?.reason ?? new Error('Upload cancelled'));
+        };
+        signal?.addEventListener('abort', abort, { once: true });
+    });
+}
+async function digest(source, signal) {
+    const hash = new crc32c_js_1.Crc32c();
+    for await (const input of bytes(source)) {
+        signal?.throwIfAborted();
+        hash.update(input);
+    }
+    return hash.digest();
+}
+/**
+ * Resume a session with the original source, starting at byte zero. The
+ * acknowledged prefix is read again (and discarded with bounded memory) so the
+ * returned CRC32C covers the whole object, not just the bytes this call sent.
+ * Session URLs are bearer credentials: never persist them in logs or telemetry.
+ */
+async function transferStorageUpload({ uploadUrl, source, size, maxBytes, ...options }) {
+    const url = checkedUploadUrl(uploadUrl);
+    const fetcher = options.fetch ?? globalThis.fetch;
+    const total = source instanceof Blob ? source.size : size;
+    if (total !== undefined &&
+        (!Number.isSafeInteger(total) || total < 0 || total > maxBytes))
+        throw new StorageTransferError('Upload exceeds its reserved size');
+    let chunkBytes = options.chunkBytes ?? 8 * 1024 * 1024;
+    if (chunkBytes < 262144 ||
+        chunkBytes > 32 * 1024 * 1024 ||
+        chunkBytes % 262144)
+        throw new StorageTransferError('Invalid upload chunk size');
+    if (options.method === 'simple' &&
+        source instanceof Blob &&
+        source.size <= 32 * 1024 * 1024)
+        chunkBytes = Math.max(262144, Math.ceil(source.size / 262144) * 262144);
+    async function probe() {
+        for (let attempt = 0;; attempt++) {
+            let response;
+            try {
+                response = await fetcher(url, {
+                    method: 'PUT',
+                    headers: {
+                        'X-GUploader-No-308': 'yes',
+                        'Content-Range': `bytes */${total ?? '*'}`,
+                    },
+                    body: new Uint8Array(),
+                    signal: options.signal,
+                    redirect: 'error',
+                });
+            }
+            catch {
+                options.signal?.throwIfAborted();
+                if (attempt >= 4)
+                    throw new StorageTransferError('Upload status is unavailable; resume this session later');
+                await delay(attempt, options.signal);
+                continue;
+            }
+            if (response.ok && !incomplete(response))
+                return { offset: total ?? 0, complete: true };
+            if (incomplete(response))
+                return { offset: receivedOffset(response), complete: false };
+            if (!retryable(response.status) || attempt >= 4)
+                throw new StorageTransferError('Upload session is unavailable or expired', response.status);
+            await delay(attempt, options.signal);
+        }
+    }
+    const initial = await probe();
+    if (initial.complete) {
+        if (total !== undefined)
+            options.onProgress?.(total);
+        // An earlier call finalized the object; nothing is sent, but the checksum
+        // must still describe the source so completion can compare it.
+        return { crc32c: await digest(source, options.signal) };
+    }
+    let offset = initial.offset;
+    if (offset > maxBytes || (total !== undefined && offset > total))
+        throw new StorageTransferError('Upload offset exceeds the source size');
+    options.onProgress?.(offset);
+    const hash = new crc32c_js_1.Crc32c();
+    const iterable = chunks(source, chunkBytes, offset, hash, options.signal);
+    const iterator = iterable[Symbol.asyncIterator]();
+    try {
+        let current = await iterator.next();
+        for (;;) {
+            const next = current.done ? current : await iterator.next();
+            const chunk = current.done ? new Uint8Array() : current.value;
+            const end = offset + chunk.byteLength;
+            const last = Boolean(next.done);
+            if (total === undefined && end > maxBytes)
+                throw new StorageTransferError(`Upload stream exceeds its ${maxBytes}-byte reservation; declare \`size\` when starting the upload to send a larger file`);
+            if (end > maxBytes ||
+                (total !== undefined && (end > total || (last && end !== total))))
+                throw new StorageTransferError('Upload source does not match its reserved size');
+            const length = total ?? (last ? end : '*');
+            let acknowledged = offset;
+            for (let attempt = 0;; attempt++) {
+                options.signal?.throwIfAborted();
+                let response;
+                try {
+                    response = await fetcher(url, {
+                        method: 'PUT',
+                        signal: options.signal,
+                        redirect: 'error',
+                        headers: {
+                            'X-GUploader-No-308': 'yes',
+                            'Content-Range': chunk.byteLength
+                                ? `bytes ${acknowledged}-${end - 1}/${length}`
+                                : `bytes */${length}`,
+                        },
+                        body: new Uint8Array(chunk.subarray(acknowledged - offset)),
+                    });
+                }
+                catch {
+                    options.signal?.throwIfAborted();
+                }
+                if (response?.ok && !incomplete(response)) {
+                    if (!last)
+                        throw new StorageTransferError('Storage completed before the source ended');
+                    options.onProgress?.(end);
+                    return { crc32c: hash.digest() };
+                }
+                if (response && !incomplete(response) && !retryable(response.status))
+                    throw new StorageTransferError('Storage rejected the upload chunk', response.status);
+                const status = response && incomplete(response)
+                    ? { offset: receivedOffset(response), complete: false }
+                    : await probe();
+                if (status.complete) {
+                    if (!last)
+                        throw new StorageTransferError('Storage completed before the source ended');
+                    options.onProgress?.(end);
+                    return { crc32c: hash.digest() };
+                }
+                if (status.offset < acknowledged || status.offset > end)
+                    throw new StorageTransferError('Storage returned an inconsistent upload offset');
+                acknowledged = status.offset;
+                options.onProgress?.(acknowledged);
+                if (acknowledged === end && !last)
+                    break;
+                if (attempt >= 4)
+                    throw new StorageTransferError('Upload interrupted; resume this session to continue');
+                await delay(attempt, options.signal);
+            }
+            offset = end;
+            current = next;
+        }
+    }
+    finally {
+        await iterator.return(undefined);
+    }
+}

package/package.json

@@ -0,0 +1,38 @@
+{
+  "name": "@sequenceholdings/storage",
+  "version": "0.1.0",
+  "description": "Atlas workspace storage contracts and streaming client",
+  "license": "UNLICENSED",
+  "type": "commonjs",
+  "main": "./dist/index.js",
+  "types": "./dist/index.d.ts",
+  "sideEffects": false,
+  "exports": {
+    ".": {
+      "types": "./dist/index.d.ts",
+      "default": "./dist/index.js"
+    }
+  },
+  "files": [
+    "dist",
+    "README.md"
+  ],
+  "dependencies": {
+    "zod": "^4.1.13"
+  },
+  "devDependencies": {
+    "typescript": "^5.3.3",
+    "vitest": "^4.1.11"
+  },
+  "publishConfig": {
+    "access": "public"
+  },
+  "engines": {
+    "node": ">=20"
+  },
+  "scripts": {
+    "build": "tsc",
+    "type-check": "tsc --noEmit",
+    "test": "vitest run"
+  }
+}