@boxrouter/sdk 0.0.0-stage → 0.12.190

package/dist/transfer.js

@@ -0,0 +1,389 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.TRANSFER_TIMEOUTS = exports.TransferError = void 0;
+exports.openSource = openSource;
+exports.sendParts = sendParts;
+exports.retryCompletion = retryCompletion;
+exports.downloadStream = downloadStream;
+exports.downloadToFile = downloadToFile;
+/**
+ * The one transfer engine behind volume and sandbox uploads and downloads.
+ *
+ * It does exactly what the API's part and link objects say: the `method` (PUT by default), the
+ * `headers`, and either the raw bytes or a multipart form with one file field of the given name.
+ * It never inspects a URL or looks at which service is behind it. Every request carries an exact Content-Length
+ * and is never chunked: bodies are lazy `Blob`s (a file on disk is read as the request is
+ * written), which `fetch` sends with their length. Nothing here logs a URL or a header value,
+ * and no error message contains one.
+ *
+ * Retries are a caller decision, `replayParts`. A volume part is a presigned PUT that replaces
+ * the earlier copy of the part, so volume callers pass `true` and a part is sent again after a
+ * network error or a 5xx answer. A sandbox upload writes a live file, so sandbox callers pass
+ * `false`: a part is retried only when its connection never opened, and once any byte may have
+ * been sent the engine raises and the caller aborts the upload.
+ *
+ * Requests are never redirected (a redirect is an error). Timeouts apply per socket operation
+ * (`TRANSFER_TIMEOUTS`), not to the whole transfer, so a large part succeeds while a stalled
+ * connection fails.
+ */
+const api_error_1 = require("./api-error");
+class TransferError extends Error {
+    status;
+    constructor(message, status) {
+        super(message);
+        this.name = 'TransferError';
+        this.status = status;
+    }
+}
+exports.TransferError = TransferError;
+// Transfer links last one to six hours; a 401 or 403 is most often a link that has ended.
+const EXPIRED_LINK = ': the transfer link expired or was refused; start the transfer again';
+function refusalHint(status) {
+    if (status === 401 || status === 403)
+        return EXPIRED_LINK;
+    return status === 507 ? ': the sandbox has no room for the file; use a path on its disk, not a memory-backed one' : '';
+}
+const PART_RETRY_LIMIT = 3;
+const COMPLETE_RETRY_DELAYS_MS = [2_000, 4_000, 8_000, 16_000, 32_000];
+const CONNECT_ERROR_CODES = new Set(['ECONNREFUSED', 'ENOTFOUND', 'EAI_AGAIN', 'ENETUNREACH', 'EHOSTUNREACH',
+    'UND_ERR_CONNECT_TIMEOUT']);
+// Set by the engine from the body it builds; a caller-supplied value could contradict it.
+const ENGINE_HEADERS = new Set(['content-length', 'transfer-encoding', 'host']);
+const undiciSpecifier = 'undici';
+/** Per socket operation: a 5 GiB part is fine while it moves, a connection that stalls for 300 s fails. */
+exports.TRANSFER_TIMEOUTS = { connectMs: 10_000, headersMs: 300_000, bodyMs: 300_000 };
+let dispatcher;
+// Node's own fetch would use its defaults; the engine states its timeouts, in Node, on an Agent of its own.
+function transferDispatcher() {
+    const runtime = globalThis.process;
+    if (!runtime?.versions?.node)
+        return Promise.resolve(undefined);
+    dispatcher ??= import(/* @vite-ignore */ undiciSpecifier).then(module => new module.Agent({
+        headersTimeout: exports.TRANSFER_TIMEOUTS.headersMs, bodyTimeout: exports.TRANSFER_TIMEOUTS.bodyMs, connect: { timeout: exports.TRANSFER_TIMEOUTS.connectMs },
+    }), () => undefined);
+    return dispatcher;
+}
+async function transferFetch(url, init) {
+    return globalThis.fetch(url, { ...init, redirect: 'manual', dispatcher: await transferDispatcher() });
+}
+function errorCode(error) {
+    const cause = error instanceof Error ? error.cause : undefined;
+    const code = cause?.code ?? error?.code;
+    return typeof code === 'string' ? code : undefined;
+}
+function connectFailed(error) {
+    const code = errorCode(error);
+    return code !== undefined && CONNECT_ERROR_CODES.has(code);
+}
+function describeFailure(label, error) {
+    return `${label} request failed${errorCode(error) ? ` (${errorCode(error)})` : ''}`;
+}
+function pause(ms, signal) {
+    signal?.throwIfAborted();
+    return new Promise((resolve, reject) => {
+        const abort = () => {
+            clearTimeout(timeout);
+            reject(signal.reason);
+        };
+        const timeout = setTimeout(() => {
+            signal?.removeEventListener('abort', abort);
+            resolve();
+        }, ms);
+        signal?.addEventListener('abort', abort, { once: true });
+    });
+}
+function requestHeaders(headers, dropContentType = false) {
+    const result = {};
+    for (const [name, value] of Object.entries(headers ?? {})) {
+        const lower = name.toLowerCase();
+        if (ENGINE_HEADERS.has(lower) || (dropContentType && lower === 'content-type'))
+            continue;
+        result[name] = value;
+    }
+    return result;
+}
+/** Open an upload source and learn its size without reading it into memory. A stream is copied to disk
+ * while `signal` is live: aborting it cancels the pending read and removes the temporary data. */
+async function openSource({ source, size }, signal) {
+    const sized = (actual) => {
+        if (size !== undefined && size !== actual)
+            throw new TypeError(`size ${size} does not match the source (${actual} bytes)`);
+        return actual;
+    };
+    if (typeof source === 'string') {
+        const fs = await import('node:fs');
+        const info = await fs.promises.stat(source);
+        if (!info.isFile())
+            throw new TypeError('source must be a regular file');
+        const blob = await fs.openAsBlob(source);
+        return { size: sized(blob.size), slice: (start, end) => blob.slice(start, end), close: async () => undefined };
+    }
+    if (source instanceof Blob) {
+        return { size: sized(source.size), slice: (start, end) => source.slice(start, end), close: async () => undefined };
+    }
+    if (source instanceof Uint8Array) {
+        return { size: sized(source.byteLength), slice: (start, end) => source.subarray(start, end), close: async () => undefined };
+    }
+    if (size === undefined)
+        throw new TypeError('size is required when uploading a stream');
+    return spoolStream(source, size, signal);
+}
+// Writes all of `bytes`: a write may take fewer bytes than it was given.
+async function writeFully(handle, bytes) {
+    for (let offset = 0; offset < bytes.byteLength;) {
+        const { bytesWritten } = await handle.write(bytes, offset, bytes.byteLength - offset);
+        if (bytesWritten <= 0)
+            throw new Error('writing the upload to a temporary file made no progress');
+        offset += bytesWritten;
+    }
+}
+// fetch can only send a known length from a Blob or bytes, so a stream is written to a temporary
+// file (never held in memory) and sent from there. Its length is checked here, before any request.
+async function spoolStream(stream, size, signal) {
+    signal?.throwIfAborted();
+    const [fs, os, paths] = await Promise.all([import('node:fs'), import('node:os'), import('node:path')]);
+    const directory = await fs.promises.mkdtemp(paths.join(os.tmpdir(), 'boxrouter-upload-'));
+    const close = () => fs.promises.rm(directory, { recursive: true, force: true });
+    try {
+        const file = paths.join(directory, 'body');
+        const handle = await fs.promises.open(file, 'wx', 0o600);
+        let written = 0;
+        try {
+            const reader = stream.getReader();
+            // A producer that stalls never answers read(); cancelling the reader ends it so the abort is seen.
+            const stop = () => { reader.cancel(signal.reason).catch(() => undefined); };
+            signal?.addEventListener('abort', stop, { once: true });
+            try {
+                for (let chunk = await reader.read(); !chunk.done; chunk = await reader.read()) {
+                    signal?.throwIfAborted();
+                    written += chunk.value.byteLength;
+                    if (written > size)
+                        throw new TypeError(`stream is longer than the declared size of ${size} bytes`);
+                    await writeFully(handle, chunk.value);
+                }
+                signal?.throwIfAborted();
+            }
+            finally {
+                signal?.removeEventListener('abort', stop);
+                reader.releaseLock();
+            }
+        }
+        finally {
+            await handle.close();
+        }
+        if (written !== size)
+            throw new TypeError(`stream ended after ${written} bytes; the declared size is ${size}`);
+        const blob = await fs.openAsBlob(file);
+        return { size, slice: (start, end) => blob.slice(start, end), close };
+    }
+    catch (error) {
+        await close();
+        throw error;
+    }
+}
+function partOffsets(parts, total) {
+    const ordered = [...parts].sort((left, right) => Number(left.number) - Number(right.number));
+    let offset = 0;
+    const ranges = ordered.map(part => {
+        const range = { part, start: offset, end: offset + Number(part.size) };
+        offset = range.end;
+        return range;
+    });
+    if (offset !== total)
+        throw new TransferError(`Upload parts cover ${offset} bytes but the source is ${total} bytes`);
+    return ranges;
+}
+async function sendPart(part, body, options, signal) {
+    const label = options.label ?? 'Transfer';
+    const form = part.form_field !== undefined && part.form_field !== null;
+    for (let attempt = 0;; attempt++) {
+        signal?.throwIfAborted();
+        let payload;
+        const headers = requestHeaders(part.headers, form);
+        if (form) {
+            const data = new FormData();
+            data.append(part.form_field, body instanceof Blob ? body : new Blob([body]), 'file');
+            payload = data;
+        }
+        else {
+            payload = body;
+            // Fetch derives the exact length from Blob/bytes. A manual header is rejected by Node 22's
+            // bundled fetch when it dispatches through the SDK's separately versioned undici Agent.
+        }
+        let response;
+        try {
+            response = await transferFetch(part.url, {
+                method: part.method ?? 'PUT', headers, body: payload, signal,
+            });
+        }
+        catch (error) {
+            if (signal?.aborted)
+                throw signal.reason ?? error;
+            if (attempt < PART_RETRY_LIMIT && (options.replayParts || connectFailed(error))) {
+                await pause(100 * (attempt + 1), signal);
+                continue;
+            }
+            throw new TransferError(describeFailure(`${label} upload part`, error));
+        }
+        if (response.status >= 500 && response.status <= 599 && options.replayParts && attempt < PART_RETRY_LIMIT) {
+            await response.body?.cancel().catch(() => undefined);
+            await pause(100 * (attempt + 1), signal);
+            continue;
+        }
+        await response.body?.cancel().catch(() => undefined);
+        if (!response.ok)
+            throw new TransferError(`${label} upload part returned HTTP ${response.status}${refusalHint(response.status)}`, response.status);
+        return response.headers.get('etag') ?? undefined;
+    }
+}
+/**
+ * Send every part with bounded parallelism and return each part's ETag when the response had one.
+ * The first failure stops the other parts and is raised; aborting the upload session is the caller's.
+ */
+async function sendParts(source, parts, options) {
+    if (!Number.isInteger(options.concurrency) || options.concurrency < 1)
+        throw new TypeError('concurrency must be a positive integer');
+    const ranges = partOffsets(parts, source.size);
+    const inner = new AbortController();
+    const signal = options.signal ? AbortSignal.any([options.signal, inner.signal]) : inner.signal;
+    const results = new Map();
+    let next = 0;
+    let done = 0;
+    let failure;
+    const workers = Array.from({ length: Math.min(options.concurrency, ranges.length) }, async () => {
+        while (next < ranges.length && failure === undefined) {
+            const { part, start, end } = ranges[next++];
+            try {
+                results.set(Number(part.number), await sendPart(part, source.slice(start, end), options, signal));
+                done += end - start;
+                options.progress?.(done, source.size);
+            }
+            catch (error) {
+                failure ??= { error };
+                inner.abort(error);
+            }
+        }
+    });
+    await Promise.all(workers);
+    if (failure !== undefined)
+        throw failure.error;
+    return ranges.map(({ part }) => {
+        const etag = results.get(Number(part.number));
+        return etag === undefined ? { number: Number(part.number) } : { number: Number(part.number), etag };
+    });
+}
+/**
+ * Run a completion call, retrying while the answer is unknown (503 or no response). Every retry
+ * carries the idempotency key the first send used (`call` gets it, undefined on the first attempt), so a
+ * completion whose answer was lost is recovered rather than repeated under a new key.
+ */
+async function retryCompletion(call, signal) {
+    let key;
+    for (let retry = 0;; retry++) {
+        try {
+            return await call(key);
+        }
+        catch (error) {
+            const delay = COMPLETE_RETRY_DELAYS_MS[retry];
+            const unknown = (error instanceof api_error_1.APIError && error.status === 503) ||
+                (error instanceof api_error_1.RequestError && error.response === undefined);
+            if (delay === undefined || !unknown || signal?.aborted)
+                throw error;
+            key ??= error.idempotencyKey;
+            await pause(delay, signal);
+        }
+    }
+}
+async function openLink(link, options) {
+    const label = options.label ?? 'Transfer';
+    let response;
+    try {
+        // identity: the bytes on the wire are the file's bytes, so the length checks hold.
+        response = await transferFetch(link.url, {
+            method: link.method ?? 'GET', headers: { ...requestHeaders(link.headers), 'Accept-Encoding': 'identity' },
+            signal: options.signal,
+        });
+    }
+    catch (error) {
+        if (options.signal?.aborted)
+            throw options.signal.reason ?? error;
+        throw new TransferError(describeFailure(`${label} download`, error));
+    }
+    if (!response.ok) {
+        await response.body?.cancel().catch(() => undefined);
+        throw new TransferError(`${label} download returned HTTP ${response.status}${refusalHint(response.status)}`, response.status);
+    }
+    return response;
+}
+// Counts the bytes, reports progress and fails the stream if it ends at a length other than the expected one.
+function checkedBody(response, options) {
+    const label = options.label ?? 'Transfer';
+    const expected = options.size ?? declaredLength(response);
+    let received = 0;
+    const empty = new ReadableStream({ start(controller) { controller.close(); } });
+    return (response.body ?? empty).pipeThrough(new TransformStream({
+        transform(chunk, controller) {
+            received += chunk.byteLength;
+            if (expected !== undefined && received > expected) {
+                throw new TransferError(`${label} download is longer than the expected ${expected} bytes`);
+            }
+            controller.enqueue(chunk);
+            options.progress?.(received, expected);
+        },
+        flush() {
+            if (expected !== undefined && received !== expected) {
+                throw new TransferError(`${label} download ended after ${received} of ${expected} bytes`);
+            }
+        },
+    }));
+}
+// Content-Length counts encoded bytes, so it is the expected length only for an unencoded body.
+function declaredLength(response) {
+    const length = response.headers.get('content-length');
+    if (length === null || !/^[0-9]+$/.test(length) || response.headers.has('content-encoding'))
+        return undefined;
+    return Number(length);
+}
+/** Stream a download. The stream errors if the body does not match `size`. */
+async function downloadStream(link, options = {}) {
+    return checkedBody(await openLink(link, options), options);
+}
+const stopBody = (response) => response.body?.cancel().catch(() => undefined);
+/** Download to `destination` through `<destination>.part-<random>`, renaming only after the full length is on disk. */
+async function downloadToFile(link, destination, options = {}) {
+    const [fs, paths, crypto, stream] = await Promise.all([
+        import('node:fs/promises'), import('node:path'), import('node:crypto'), import('node:stream'),
+    ]);
+    const response = await openLink(link, options);
+    const temporary = `${destination}.part-${crypto.randomBytes(6).toString('hex')}`;
+    let handle;
+    try {
+        await fs.mkdir(paths.dirname(destination), { recursive: true });
+        // The file keeps the mode of the one it replaces, set before any byte is written so a private
+        // file is never readable through its replacement; a new file has the default mode of a plain write.
+        const existing = await fs.stat(destination).catch(() => undefined);
+        handle = await fs.open(temporary, 'wx', existing ? existing.mode & 0o7777 : 0o666);
+        if (existing)
+            await handle.chmod(existing.mode & 0o7777);
+        const body = checkedBody(response, options);
+        // The stream owns the handle from here and closes it when it ends or fails.
+        const output = handle.createWriteStream();
+        handle = undefined;
+        await stream.promises.pipeline(stream.Readable.fromWeb(body), output, { signal: options.signal });
+        const synced = await fs.open(temporary, 'r');
+        try {
+            await synced.sync();
+        }
+        finally {
+            await synced.close();
+        }
+        await fs.rename(temporary, destination);
+    }
+    catch (error) {
+        // Nothing reads the body when setup fails; a body already piped is cancelled by the pipeline.
+        await stopBody(response);
+        await handle?.close().catch(() => undefined);
+        await fs.rm(temporary, { force: true });
+        throw error;
+    }
+}