@backblaze-labs/b2-sdk 0.1.0 → 0.2.0

package/dist/upload/large.js

@@ -1,222 +1,305 @@
-import { IncrementalSha1 } from "../streams/hash.js";
-import { ProgressTracker } from "../streams/progress.js";
-import { DEFAULT_TRANSFER_CONCURRENCY, DEFAULT_CONTENT_TYPE } from "../util/defaults.js";
-import { planRanges } from "../util/plan-ranges.js";
-import { cancelLargeFileBestEffort } from "./cancel.js";
+import { createAbortScope, raceWithAbort, throwRejectedOrAbortReason } from "./abort-scope.js";
+import { ResumeFileIdMismatchError } from "../errors/index.js";
+import { cancelLargeFileBestEffort, cleanupAfterLargeFileError } from "./cancel.js";
 import { Semaphore } from "./concurrency.js";
-import { collectPartSha1s, findResumeCandidate } from "./resume.js";
+import { finishLargeFileWithAbortReconciliation } from "./finish.js";
+import "../util/defaults.js";
+import { planRanges } from "../util/plan-ranges.js";
+import { ProgressTracker } from "../streams/progress.js";
+import { IncrementalSha1 } from "../streams/hash.js";
+import { readNextNonEmptyStreamChunk } from "../streams/source.js";
+import { findResumeCandidate } from "./resume.js";
+import { resolveRetryResponseBodyFailures, uploadPartWithFreshUrl } from "./retry.js";
+//#region src/upload/large.ts
+var MAX_CONSECUTIVE_EMPTY_STREAM_CHUNKS = 1024;
+function createResumeCandidateCriteria(options, request, totalSize, partSize, parts) {
+	return {
+		contentType: request.contentType,
+		fileInfo: request.fileInfo,
+		sourceSize: totalSize,
+		partSize,
+		parts,
+		...options.signal !== void 0 ? { signal: options.signal } : {},
+		...request.serverSideEncryption !== void 0 ? { serverSideEncryption: request.serverSideEncryption } : options.bucketDefaultServerSideEncryption !== void 0 ? { serverSideEncryption: options.bucketDefaultServerSideEncryption } : {},
+		...request.fileRetention !== void 0 ? { fileRetention: request.fileRetention } : options.bucketDefaultRetention !== void 0 ? { defaultFileRetention: options.bucketDefaultRetention } : options.bucketDefaultRetentionUnreadable === true ? { defaultFileRetentionUnreadable: true } : {},
+		...request.legalHold !== void 0 ? { legalHold: request.legalHold } : {},
+		...options.resumeDiscoveryTimeoutMs !== void 0 ? { discoveryTimeoutMs: options.resumeDiscoveryTimeoutMs } : {},
+		...options.onResumeCandidateRejected !== void 0 ? { onCandidateRejected: options.onResumeCandidateRejected } : {},
+		...options.resumeMaxListPages !== void 0 ? { maxListPages: options.resumeMaxListPages } : {},
+		...options.resumeMaxPartCandidates !== void 0 ? { maxPartCandidates: options.resumeMaxPartCandidates } : {},
+		...options.resumeMaxPartPages !== void 0 ? { maxPartPages: options.resumeMaxPartPages } : {}
+	};
+}
+/**
+* Uploads a file using the B2 multipart (large file) protocol.
+*
+* The source is sliced into parts and uploaded concurrently via
+* `b2_upload_part`. This is appropriate for files larger than the recommended
+* part size. For smaller files, use {@link uploadSmallFile} which sends the
+* entire payload in a single request.
+*
+* On failure, the in-progress large file is cancelled on a best-effort basis.
+*
+* @param raw - Low-level B2 API client.
+* @param accountInfo - Authorized account state (tokens, URLs, upload URL pool).
+* @param options - Upload parameters including part size and concurrency.
+*
+* @returns The resulting {@link FileVersion} metadata.
+*/
 async function uploadLargeFile(raw, accountInfo, options) {
-  const recommendedPartSize = accountInfo.getRecommendedPartSize();
-  const minPartSize = accountInfo.getAbsoluteMinimumPartSize();
-  const partSize = Math.max(options.partSize ?? recommendedPartSize, minPartSize);
-  const concurrency = options.concurrency ?? DEFAULT_TRANSFER_CONCURRENCY;
-  const totalSize = options.source.size;
-  const parts = planRanges(totalSize, partSize);
-  const fileInfo = { ...options.fileInfo };
-  const startLargeFileRequest = {
-    bucketId: options.bucketId,
-    fileName: options.fileName,
-    contentType: options.contentType ?? DEFAULT_CONTENT_TYPE,
-    fileInfo,
-    ...options.serverSideEncryption !== void 0 ? { serverSideEncryption: options.serverSideEncryption } : {},
-    ...options.fileRetention !== void 0 ? { fileRetention: options.fileRetention } : {},
-    ...options.legalHold !== void 0 ? { legalHold: options.legalHold } : {}
-  };
-  let largeFileId;
-  let preUploaded;
-  if (options.resumeFileId !== void 0) {
-    largeFileId = options.resumeFileId;
-    preUploaded = await collectPartSha1s(raw, accountInfo, largeFileId);
-  } else if (options.resume === true) {
-    const candidate = await findResumeCandidate(
-      raw,
-      accountInfo,
-      options.bucketId,
-      options.fileName
-    );
-    if (candidate) {
-      largeFileId = candidate.fileId;
-      preUploaded = candidate.uploadedPartSha1s;
-    } else {
-      const startResp = await raw.startLargeFile(
-        accountInfo.getApiUrl(),
-        accountInfo.getAuthToken(),
-        startLargeFileRequest
-      );
-      largeFileId = startResp.fileId;
-      preUploaded = /* @__PURE__ */ new Map();
-    }
-  } else {
-    const startResp = await raw.startLargeFile(
-      accountInfo.getApiUrl(),
-      accountInfo.getAuthToken(),
-      startLargeFileRequest
-    );
-    largeFileId = startResp.fileId;
-    preUploaded = /* @__PURE__ */ new Map();
-  }
-  const partSha1s = new Array(parts.length);
-  const tracker = new ProgressTracker(options.onProgress, totalSize, parts.length);
-  const sem = new Semaphore(concurrency);
-  if (!options.source.canSlice) {
-    if (options.resume === true || options.resumeFileId !== void 0) {
-      await cancelLargeFileBestEffort(raw, accountInfo, largeFileId);
-      throw new Error(
-        "uploadLargeFile: resume is not supported on non-sliceable sources (e.g. StreamSource)."
-      );
-    }
-    try {
-      await uploadPartsSequentially(
-        raw,
-        accountInfo,
-        options,
-        largeFileId,
-        parts,
-        partSha1s,
-        tracker
-      );
-      return await raw.finishLargeFile(accountInfo.getApiUrl(), accountInfo.getAuthToken(), {
-        fileId: largeFileId,
-        partSha1Array: partSha1s
-      });
-    } catch (err) {
-      await cancelLargeFileBestEffort(raw, accountInfo, largeFileId);
-      throw err;
-    }
-  }
-  try {
-    const tasks = parts.map(async (part) => {
-      await sem.acquire();
-      try {
-        options.signal?.throwIfAborted();
-        const partSource = options.source.slice(part.offset, part.offset + part.length);
-        const data = new Uint8Array(await partSource.toArrayBuffer());
-        const partSha1 = new IncrementalSha1();
-        await partSha1.update(data);
-        const sha1Hex = await partSha1.digest();
-        const serverSha1 = preUploaded.get(part.partNumber);
-        if (serverSha1 !== void 0 && serverSha1 === sha1Hex) {
-          partSha1s[part.partNumber - 1] = serverSha1;
-          tracker.addBytes(data.byteLength);
-          tracker.completePart();
-          return;
-        }
-        let uploadEntry = accountInfo.checkoutPartUploadUrl(largeFileId);
-        if (!uploadEntry) {
-          const resp = await raw.getUploadPartUrl(
-            accountInfo.getApiUrl(),
-            accountInfo.getAuthToken(),
-            { fileId: largeFileId }
-          );
-          uploadEntry = { uploadUrl: resp.uploadUrl, authorizationToken: resp.authorizationToken };
-        }
-        try {
-          const result2 = await raw.uploadPart(
-            uploadEntry.uploadUrl,
-            {
-              authorization: uploadEntry.authorizationToken,
-              partNumber: part.partNumber,
-              contentLength: data.byteLength,
-              contentSha1: sha1Hex,
-              ...options.serverSideEncryption !== void 0 ? { serverSideEncryption: options.serverSideEncryption } : {}
-            },
-            data,
-            options.signal
-          );
-          accountInfo.returnPartUploadUrl(largeFileId, uploadEntry);
-          partSha1s[part.partNumber - 1] = result2.contentSha1;
-          tracker.addBytes(data.byteLength);
-          tracker.completePart();
-        } catch (err) {
-          accountInfo.evictPartUploadUrl(largeFileId, uploadEntry);
-          throw err;
-        }
-      } finally {
-        sem.release();
-      }
-    });
-    await Promise.all(tasks);
-    const result = await raw.finishLargeFile(accountInfo.getApiUrl(), accountInfo.getAuthToken(), {
-      fileId: largeFileId,
-      partSha1Array: partSha1s
-    });
-    return result;
-  } catch (err) {
-    await cancelLargeFileBestEffort(raw, accountInfo, largeFileId);
-    throw err;
-  }
+	const recommendedPartSize = accountInfo.getRecommendedPartSize();
+	const minPartSize = accountInfo.getAbsoluteMinimumPartSize();
+	const partSize = Math.max(options.partSize ?? recommendedPartSize, minPartSize);
+	const concurrency = options.concurrency ?? 4;
+	const totalSize = options.source.size;
+	const parts = planRanges(totalSize, partSize);
+	const fileInfo = Object.create(null);
+	if (options.fileInfo !== void 0) for (const [key, value] of Object.entries(options.fileInfo)) fileInfo[key] = value;
+	const startLargeFileRequest = {
+		bucketId: options.bucketId,
+		fileName: options.fileName,
+		contentType: options.contentType ?? "b2/x-auto",
+		fileInfo,
+		...options.serverSideEncryption !== void 0 ? { serverSideEncryption: options.serverSideEncryption } : {},
+		...options.fileRetention !== void 0 ? { fileRetention: options.fileRetention } : {},
+		...options.legalHold !== void 0 ? { legalHold: options.legalHold } : {}
+	};
+	const resumeCandidateCriteria = createResumeCandidateCriteria(options, startLargeFileRequest, totalSize, partSize, parts);
+	if (!options.source.canSlice && options.resumeFileId !== void 0) throw new Error("uploadLargeFile: resume is not supported on non-sliceable sources.");
+	let largeFileId;
+	let preUploaded = /* @__PURE__ */ new Map();
+	let createdLargeFile = false;
+	const abortScope = createAbortScope(options.signal);
+	const startFreshLargeFile = async () => {
+		if (abortScope.signal.aborted && !options.source.canSlice) await cancelForwardOnlySource(options.source, abortScope.signal.reason);
+		abortScope.signal.throwIfAborted();
+		const startPromise = raw.startLargeFile(accountInfo.getApiUrl(), accountInfo.getAuthToken(), startLargeFileRequest, {
+			signal: abortScope.signal,
+			...options.retry !== void 0 ? { retry: options.retry } : {}
+		});
+		try {
+			largeFileId = (await raceWithAbort(startPromise, abortScope.signal)).fileId;
+		} catch (err) {
+			if (abortScope.signal.aborted) {
+				if (!options.source.canSlice) await cancelForwardOnlySource(options.source, abortScope.signal.reason).catch(() => {});
+				cancelLargeFileAfterStart(startPromise, raw, accountInfo, options.onCleanupFailure);
+			}
+			throw err;
+		}
+		preUploaded = /* @__PURE__ */ new Map();
+		createdLargeFile = true;
+	};
+	try {
+		if (abortScope.signal.aborted && !options.source.canSlice) await cancelForwardOnlySource(options.source, abortScope.signal.reason);
+		abortScope.signal.throwIfAborted();
+		if (options.resumeFileId !== void 0) {
+			const candidate = await findResumeCandidate(raw, accountInfo, options.bucketId, options.fileName, {
+				...resumeCandidateCriteria,
+				resumeFileId: options.resumeFileId
+			});
+			if (candidate === null) throw new ResumeFileIdMismatchError(options.resumeFileId, options.fileName);
+			largeFileId = candidate.fileId;
+			preUploaded = candidate.uploadedPartSha1s;
+		} else if (options.resume === true && options.source.canSlice) {
+			const candidate = await findResumeCandidate(raw, accountInfo, options.bucketId, options.fileName, resumeCandidateCriteria);
+			if (candidate) {
+				largeFileId = candidate.fileId;
+				preUploaded = /* @__PURE__ */ new Map();
+			} else await startFreshLargeFile();
+		} else await startFreshLargeFile();
+		const activeLargeFileId = largeFileId;
+		if (activeLargeFileId === void 0) throw new Error("uploadLargeFile: start did not return a large file ID.");
+		const partSha1s = new Array(parts.length);
+		const tracker = new ProgressTracker(options.onProgress, totalSize, parts.length);
+		const sem = new Semaphore(concurrency);
+		if (!options.source.canSlice) {
+			await uploadPartsSequentially(raw, accountInfo, options, activeLargeFileId, parts, partSha1s, tracker, abortScope.signal);
+			return await finishLargeFileWithAbortReconciliation(raw, accountInfo, {
+				fileId: activeLargeFileId,
+				bucketId: options.bucketId,
+				fileName: options.fileName,
+				partSha1s,
+				signal: abortScope.signal,
+				...options.retry !== void 0 ? { retry: options.retry } : {}
+			});
+		}
+		const tasks = parts.map(async (part) => {
+			await sem.acquire();
+			try {
+				abortScope.signal.throwIfAborted();
+				const partSource = options.source.slice(part.offset, part.offset + part.length);
+				const data = new Uint8Array(await partSource.toArrayBuffer({ signal: abortScope.signal }));
+				abortScope.signal.throwIfAborted();
+				const partSha1 = new IncrementalSha1();
+				await partSha1.update(data);
+				const sha1Hex = await partSha1.digest();
+				abortScope.signal.throwIfAborted();
+				const serverSha1 = preUploaded.get(part.partNumber);
+				if (serverSha1 !== void 0 && serverSha1 === sha1Hex) {
+					notifyResumePartReused(options.onResumePartReused, {
+						fileName: options.fileName,
+						fileId: activeLargeFileId,
+						partNumber: part.partNumber,
+						contentLength: data.byteLength,
+						contentSha1: serverSha1
+					});
+					partSha1s[part.partNumber - 1] = serverSha1;
+					tracker.addBytes(data.byteLength);
+					tracker.completePart();
+					return;
+				}
+				const result = await uploadPartWithFreshUrl(raw, accountInfo, activeLargeFileId, {
+					fileName: options.fileName,
+					partNumber: part.partNumber,
+					data,
+					contentLength: data.byteLength,
+					contentSha1: sha1Hex,
+					retry: options.retry,
+					signal: abortScope.signal,
+					onUploadRetry: options.onUploadRetry,
+					retryResponseBodyFailures: resolveRetryResponseBodyFailures(options.retryResponseBodyFailures),
+					...options.serverSideEncryption !== void 0 ? { serverSideEncryption: options.serverSideEncryption } : {}
+				});
+				partSha1s[part.partNumber - 1] = result.contentSha1;
+				tracker.addBytes(data.byteLength);
+				tracker.completePart();
+			} catch (err) {
+				abortScope.abort(err);
+				throw err;
+			} finally {
+				sem.release();
+			}
+		});
+		throwRejectedOrAbortReason(await Promise.allSettled(tasks), abortScope);
+		return await finishLargeFileWithAbortReconciliation(raw, accountInfo, {
+			fileId: activeLargeFileId,
+			bucketId: options.bucketId,
+			fileName: options.fileName,
+			partSha1s,
+			signal: abortScope.signal,
+			...options.retry !== void 0 ? { retry: options.retry } : {}
+		});
+	} catch (err) {
+		abortScope.abort(err);
+		if (largeFileId === void 0) throw err;
+		return await cleanupAfterUploadLargeFileError(err, raw, accountInfo, options, largeFileId, createdLargeFile);
+	} finally {
+		abortScope.dispose();
+	}
+}
+function cancelLargeFileAfterStart(started, raw, accountInfo, onCleanupFailure) {
+	started.then((resp) => cancelLargeFileBestEffort(raw, accountInfo, resp.fileId, onCleanupFailure === void 0 ? void 0 : { onCleanupFailure })).catch(() => {});
+}
+async function cleanupAfterUploadLargeFileError(err, raw, accountInfo, options, largeFileId, createdLargeFile) {
+	return await cleanupAfterLargeFileError(err, raw, accountInfo, {
+		fileId: largeFileId,
+		bucketId: options.bucketId,
+		fileName: options.fileName,
+		signal: options.signal,
+		onCleanupFailure: options.onCleanupFailure
+	}, { cancelOnError: createdLargeFile });
+}
+/**
+* Sequential upload path for non-sliceable sources.
+*
+* Reads the source's `stream()` once and accumulates exactly `partSize`
+* bytes into an in-memory buffer per iteration. Each filled buffer is
+* hashed, dispatched to `b2_upload_part`, then released before the next
+* part starts — so peak memory is ~partSize regardless of file size.
+*
+* Concurrency is forced to 1 here because the stream is a single
+* forward-only cursor; the engine can't read part N+1 until part N is
+* fully consumed.
+*
+* @param raw - Low-level B2 API client.
+* @param accountInfo - Authorized account state.
+* @param options - The original `uploadLargeFile` options.
+* @param largeFileId - ID of the in-progress large file (already started).
+* @param parts - Pre-planned part layout (used for part numbers + count).
+* @param partSha1s - Output array, written in-place at index `partNumber - 1`.
+* @param tracker - Progress tracker; bytes added per chunk, part completed
+*   each time a part finishes.
+* @param signal - Linked abort signal for source reads and part uploads.
+*/
+async function uploadPartsSequentially(raw, accountInfo, options, largeFileId, parts, partSha1s, tracker, signal) {
+	const reader = options.source.stream().getReader();
+	let bytesRead = 0;
+	let carry = null;
+	try {
+		for (const planned of parts) {
+			signal.throwIfAborted();
+			const buf = new Uint8Array(planned.length);
+			let filled = 0;
+			if (carry !== null) {
+				const take = Math.min(carry.byteLength, buf.byteLength - filled);
+				buf.set(carry.subarray(0, take), filled);
+				filled += take;
+				carry = take < carry.byteLength ? carry.subarray(take) : null;
+			}
+			while (filled < buf.byteLength) {
+				const { done, value } = await readNextNonEmptyStreamChunk(reader, emptyChunkError(), signal);
+				if (done) throw new Error(`uploadLargeFile: source stream ended after ${bytesRead} bytes, expected ${options.source.size}.`);
+				bytesRead += value.byteLength;
+				const take = Math.min(value.byteLength, buf.byteLength - filled);
+				buf.set(value.subarray(0, take), filled);
+				filled += take;
+				if (take < value.byteLength) carry = value.subarray(take);
+			}
+			signal.throwIfAborted();
+			const data = buf;
+			const partSha1 = new IncrementalSha1();
+			await partSha1.update(data);
+			const sha1Hex = await partSha1.digest();
+			signal.throwIfAborted();
+			const result = await uploadPartWithFreshUrl(raw, accountInfo, largeFileId, {
+				fileName: options.fileName,
+				partNumber: planned.partNumber,
+				data,
+				contentLength: data.byteLength,
+				contentSha1: sha1Hex,
+				retry: options.retry,
+				signal,
+				onUploadRetry: options.onUploadRetry,
+				retryResponseBodyFailures: resolveRetryResponseBodyFailures(options.retryResponseBodyFailures),
+				...options.serverSideEncryption !== void 0 ? { serverSideEncryption: options.serverSideEncryption } : {}
+			});
+			partSha1s[planned.partNumber - 1] = result.contentSha1;
+			tracker.addBytes(data.byteLength);
+			tracker.completePart();
+		}
+		if (carry !== null && carry.byteLength > 0) throw new Error(tooManyBytesError(options.source.size));
+		const extra = await readNextNonEmptyStreamChunk(reader, emptyChunkError(), signal);
+		if (!extra.done) {
+			bytesRead += extra.value.byteLength;
+			throw new Error(tooManyBytesError(options.source.size));
+		}
+	} catch (err) {
+		await reader.cancel(err).catch(() => {});
+		throw err;
+	} finally {
+		reader.releaseLock();
+	}
+}
+function emptyChunkError() {
+	return `uploadLargeFile: source stream emitted more than ${MAX_CONSECUTIVE_EMPTY_STREAM_CHUNKS} consecutive empty chunks. too many empty chunks.`;
+}
+function tooManyBytesError(advertisedSize) {
+	return `uploadLargeFile: source stream emitted more than advertised ${advertisedSize} bytes. source stream emitted more bytes than advertised size.`;
+}
+async function cancelForwardOnlySource(source, reason) {
+	const reader = source.stream().getReader();
+	try {
+		await reader.cancel(reason);
+	} finally {
+		reader.releaseLock();
+	}
 }
-async function uploadPartsSequentially(raw, accountInfo, options, largeFileId, parts, partSha1s, tracker) {
-  const reader = options.source.stream().getReader();
-  let partNumber = 1;
-  let carry = null;
-  try {
-    for (const planned of parts) {
-      options.signal?.throwIfAborted();
-      const buf = new Uint8Array(planned.length);
-      let filled = 0;
-      if (carry !== null) {
-        const take = Math.min(carry.byteLength, buf.byteLength - filled);
-        buf.set(carry.subarray(0, take), filled);
-        filled += take;
-        carry = take < carry.byteLength ? carry.subarray(take) : null;
-      }
-      while (filled < buf.byteLength) {
-        const { done, value } = await reader.read();
-        if (done) break;
-        const take = Math.min(value.byteLength, buf.byteLength - filled);
-        buf.set(value.subarray(0, take), filled);
-        filled += take;
-        if (take < value.byteLength) {
-          carry = value.subarray(take);
-        }
-      }
-      const data = filled === buf.byteLength ? buf : buf.subarray(0, filled);
-      if (data.byteLength === 0) {
-        throw new Error(
-          `uploadLargeFile: source stream ended before part ${partNumber}; advertised size does not match emitted bytes.`
-        );
-      }
-      const partSha1 = new IncrementalSha1();
-      await partSha1.update(data);
-      const sha1Hex = await partSha1.digest();
-      let uploadEntry = accountInfo.checkoutPartUploadUrl(largeFileId);
-      if (!uploadEntry) {
-        const resp = await raw.getUploadPartUrl(
-          accountInfo.getApiUrl(),
-          accountInfo.getAuthToken(),
-          { fileId: largeFileId }
-        );
-        uploadEntry = { uploadUrl: resp.uploadUrl, authorizationToken: resp.authorizationToken };
-      }
-      try {
-        const result = await raw.uploadPart(
-          uploadEntry.uploadUrl,
-          {
-            authorization: uploadEntry.authorizationToken,
-            partNumber: planned.partNumber,
-            contentLength: data.byteLength,
-            contentSha1: sha1Hex,
-            ...options.serverSideEncryption !== void 0 ? { serverSideEncryption: options.serverSideEncryption } : {}
-          },
-          data,
-          options.signal
-        );
-        accountInfo.returnPartUploadUrl(largeFileId, uploadEntry);
-        partSha1s[planned.partNumber - 1] = result.contentSha1;
-        tracker.addBytes(data.byteLength);
-        tracker.completePart();
-      } catch (err) {
-        accountInfo.evictPartUploadUrl(largeFileId, uploadEntry);
-        throw err;
-      }
-      partNumber++;
-    }
-  } finally {
-    reader.releaseLock();
-  }
+function notifyResumePartReused(listener, event) {
+	try {
+		listener?.(event);
+	} catch {}
 }
-export {
-  uploadLargeFile
-};
-//# sourceMappingURL=large.js.map
+//#endregion
+export { uploadLargeFile };
+
+//# sourceMappingURL=large.js.map