@lossless.org/client 2.1.0 → 2.3.0

package/ts/objectstorage/internal.multipart.stream.ts

@@ -60,6 +60,7 @@ const failureMessages: Record<TMultipartStreamFailureKind, string> = {
     'The multipart source emitted too many consecutive empty chunks.',
   'source-short': 'The multipart source ended before the declared content length.',
   'source-extra': 'The multipart source exceeded the declared content length.',
+  'source-over-maximum': 'The multipart source exceeded the maximum content length.',
   'part-callback': 'A multipart part callback failed.',
 };
 const failureSettlementMaximumTimeoutMs = 30_000;
@@ -96,13 +97,6 @@ export class MultipartStreamConsumptionError extends Error {
 }
 
 const validatePlan = (planArg: TMultipartUploadPlan): void => {
-  if (
-    !Number.isSafeInteger(planArg.objectSize) ||
-    planArg.objectSize < 0 ||
-    planArg.objectSize > MULTIPART_MAX_OBJECT_SIZE
-  ) {
-    throw new TypeError('plan.objectSize is invalid.');
-  }
   if (
     !Number.isSafeInteger(planArg.concurrency) ||
     planArg.concurrency < 1 ||
@@ -110,6 +104,32 @@ const validatePlan = (planArg: TMultipartUploadPlan): void => {
   ) {
     throw new TypeError('plan.concurrency is invalid.');
   }
+  if (planArg.classification === 'bounded') {
+    if (
+      !Number.isSafeInteger(planArg.maxObjectSize) ||
+      planArg.maxObjectSize < 1 ||
+      planArg.maxObjectSize > MULTIPART_MAX_OBJECT_SIZE ||
+      !Number.isSafeInteger(planArg.partSize) ||
+      planArg.partSize < MULTIPART_MIN_PART_SIZE ||
+      planArg.partSize > MULTIPART_MAX_PART_SIZE ||
+      planArg.maxPartCount !== Math.ceil(planArg.maxObjectSize / planArg.partSize) ||
+      planArg.maxPartCount > MULTIPART_MAX_PART_COUNT ||
+      planArg.concurrency > planArg.maxPartCount ||
+      planArg.maximumBufferedBytes !==
+        Math.min(planArg.maxObjectSize, planArg.partSize * planArg.concurrency) +
+          Math.min(planArg.maxObjectSize, planArg.partSize)
+    ) {
+      throw new TypeError('The bounded multipart plan is inconsistent.');
+    }
+    return;
+  }
+  if (
+    !Number.isSafeInteger(planArg.objectSize) ||
+    planArg.objectSize < 0 ||
+    planArg.objectSize > MULTIPART_MAX_OBJECT_SIZE
+  ) {
+    throw new TypeError('plan.objectSize is invalid.');
+  }
   if (planArg.classification === 'zero') {
     if (
       planArg.objectSize !== 0 ||
@@ -165,6 +185,14 @@ export const consumeExactMultipartStream = async (
     );
   }
 
+  const plan = optionsArg.plan;
+  // A bounded plan reads parts until the source ends and refuses more than
+  // its maximum; an exact plan reads exactly the declared length.
+  const bounded = plan.classification === 'bounded';
+  const sizeLimit = bounded ? plan.maxObjectSize : plan.objectSize;
+  const overLimitKind: TMultipartStreamFailureKind = bounded
+    ? 'source-over-maximum'
+    : 'source-extra';
   const reader = optionsArg.readableStream.getReader();
   const callbackAbortController = new AbortController();
   const inFlightParts = new Map<symbol, IInFlightPart>();
@@ -289,7 +317,11 @@ export const consumeExactMultipartStream = async (
     }
   };
 
-  const dispatchPart = (dataArg: Uint8Array, partNumberArg: number): void => {
+  const dispatchPart = (
+    dataArg: Uint8Array,
+    partNumberArg: number,
+    isLastPartArg: boolean,
+  ): void => {
     const token = Symbol(`multipart-part-${partNumberArg}`);
     partCallbacksStarted++;
     currentPartBytes = 0;
@@ -299,8 +331,8 @@ export const consumeExactMultipartStream = async (
       .then(() =>
         optionsArg.onPart({
           partNumber: partNumberArg,
-          byteOffset: (partNumberArg - 1) * optionsArg.plan.partSize,
-          isLastPart: partNumberArg === optionsArg.plan.partCount,
+          byteOffset: (partNumberArg - 1) * plan.partSize,
+          isLastPart: isLastPartArg,
           data: dataArg,
           signal: callbackAbortController.signal,
         }),
@@ -347,15 +379,19 @@ export const consumeExactMultipartStream = async (
 
     for (
       let partNumber = 1;
-      partNumber <= optionsArg.plan.partCount;
+      bounded ? !sourceClosed : partNumber <= plan.partCount;
       partNumber++
     ) {
       await waitForCapacity();
       const partByteLength = Math.min(
-        optionsArg.plan.partSize,
-        optionsArg.plan.objectSize -
-          (partNumber - 1) * optionsArg.plan.partSize,
+        plan.partSize,
+        sizeLimit - (partNumber - 1) * plan.partSize,
       );
+      if (partByteLength <= 0) {
+        // A bounded source filled its maximum, which ends at or inside the
+        // previous part; the trailing read below refuses any further byte.
+        break;
+      }
       if (
         inFlightBytes + partByteLength >
         optionsArg.plan.maximumBufferedBytes
@@ -396,6 +432,9 @@ export const consumeExactMultipartStream = async (
         const readResult = await readWhileObservingCallbacks();
         if (readResult.done) {
           sourceClosed = true;
+          if (bounded) {
+            break;
+          }
           throw {
             kind: 'source-short',
           } satisfies IPrimaryFailure;
@@ -410,16 +449,16 @@ export const consumeExactMultipartStream = async (
           continue;
         }
         observedBytes += readResult.value.byteLength;
-        if (observedBytes > optionsArg.plan.objectSize) {
+        if (observedBytes > sizeLimit) {
           throw {
-            kind: 'source-extra',
+            kind: overLimitKind,
           } satisfies IPrimaryFailure;
         }
-        if (readResult.value.byteLength > optionsArg.plan.partSize) {
+        if (readResult.value.byteLength > plan.partSize) {
           throw {
             kind: 'source-buffer-limit',
             cause: new RangeError(
-              `Source chunks must not exceed ${optionsArg.plan.partSize} bytes.`,
+              `Source chunks must not exceed ${plan.partSize} bytes.`,
             ),
           } satisfies IPrimaryFailure;
         }
@@ -428,7 +467,24 @@ export const consumeExactMultipartStream = async (
         observeBufferedBytes();
       }
 
-      dispatchPart(partData, partNumber);
+      if (partOffset === 0) {
+        // A bounded source ended at a part boundary.
+        currentPartBytes = 0;
+        break;
+      }
+      if (partOffset < partByteLength) {
+        // Only a bounded source ends inside a part; its last part keeps the
+        // bytes read, as an exact copy.
+        dispatchPart(partData.slice(0, partOffset), partNumber, true);
+      } else {
+        dispatchPart(
+          partData,
+          partNumber,
+          bounded
+            ? sourceClosed || observedBytes === sizeLimit
+            : partNumber === plan.partCount,
+        );
+      }
     }
 
     while (!sourceClosed) {
@@ -448,7 +504,7 @@ export const consumeExactMultipartStream = async (
       }
       observedBytes += readResult.value.byteLength;
       throw {
-        kind: 'source-extra',
+        kind: overLimitKind,
       } satisfies IPrimaryFailure;
     }
 
@@ -571,7 +627,9 @@ export const consumeExactMultipartStream = async (
         completionAuthorized: false,
         kind: failure.kind,
         cause: failure.cause,
-        expectedBytes: optionsArg.plan.objectSize,
+        ...(bounded
+          ? { maxBytes: plan.maxObjectSize }
+          : { expectedBytes: plan.objectSize }),
         observedBytes,
         partCallbacksStarted,
         completedPartNumbers: [...completedPartNumbers].sort(

package/ts/objectstorage/internal.multipart.types.ts

@@ -4,6 +4,12 @@ export interface IMultipartPlanningOptions {
   requestedConcurrency?: number;
 }
 
+export interface IMultipartBoundedPlanningOptions {
+  maxObjectSize: number;
+  requestedPartSize?: number;
+  requestedConcurrency?: number;
+}
+
 interface IMultipartUploadPlanBase {
   objectSize: number;
   concurrency: number;
@@ -24,11 +30,34 @@ export interface IMultipartDataUploadPlan extends IMultipartUploadPlanBase {
   partCount: number;
 }
 
-export type TMultipartUploadPlan = IMultipartZeroUploadPlan | IMultipartDataUploadPlan;
+/**
+ * A plan for a source of undeclared length: full parts are read until the
+ * source ends, and a source longer than `maxObjectSize` fails. The part size
+ * is chosen so that `maxObjectSize` fits into the multipart part limit.
+ */
+export interface IMultipartBoundedUploadPlan {
+  classification: 'bounded';
+  maxObjectSize: number;
+  partSize: number;
+  /** Parts that `maxObjectSize` needs; the source may need fewer. */
+  maxPartCount: number;
+  concurrency: number;
+  maximumBufferedBytes: number;
+}
+
+/** A plan for a source of declared length. */
+export type TMultipartDeclaredUploadPlan = IMultipartZeroUploadPlan | IMultipartDataUploadPlan;
+
+export type TMultipartUploadPlan = TMultipartDeclaredUploadPlan | IMultipartBoundedUploadPlan;
 
 export interface IMultipartStreamPart {
   partNumber: number;
   byteOffset: number;
+  /**
+   * Under a bounded plan a part is only known to be the last when the source
+   * ended inside it or it reached the maximum; a source that ends exactly at
+   * a part boundary leaves its last part marked false.
+   */
   isLastPart: boolean;
   /**
    * An exact, exclusively owned copy. The consumer never mutates or reuses it.
@@ -61,6 +90,7 @@ export type TMultipartStreamFailureKind =
   | 'source-empty-chunk-limit'
   | 'source-short'
   | 'source-extra'
+  | 'source-over-maximum'
   | 'part-callback';
 
 export type TMultipartOwnedReaderCleanup =
@@ -97,7 +127,10 @@ export interface IMultipartStreamFailureState {
   completionAuthorized: false;
   kind: TMultipartStreamFailureKind;
   cause?: unknown;
-  expectedBytes: number;
+  /** The declared length; absent under a bounded plan. */
+  expectedBytes?: number;
+  /** The maximum length of a bounded plan. */
+  maxBytes?: number;
   observedBytes: number;
   partCallbacksStarted: number;
   completedPartNumbers: number[];

package/ts/objectstorage/internal.multipartcopy.ts

@@ -0,0 +1,169 @@
+import * as plugins from './plugins.js';
+import type { IBasicBucketOperation } from './internal.exactupload.capability.js';
+import { abortMultipartUpload } from './internal.exactupload.operations.js';
+
+const MiB = 1024 * 1024;
+const GiB = 1024 * MiB;
+
+/**
+ * Sources larger than this are copied with UploadPartCopy. It is the largest
+ * object one CopyObject may copy, so every copy that one CopyObject can do
+ * stays one. A capped MinIO (2 CPUs, 2 GiB) answered a single CopyObject after
+ * about 3 s per GiB, inside the default 60 s request timeout at this size.
+ */
+export const multipartCopyThresholdBytes = 5 * GiB;
+/** One 256 MiB UploadPartCopy answered within 0.9 s on the same MinIO. */
+export const multipartCopyPartSizeBytes = 256 * MiB;
+const maximumCopyPartSizeBytes = 5 * GiB;
+const maximumCopyPartCount = 10_000;
+const maximumCopyObjectSizeBytes = 5 * 1024 * GiB;
+
+export interface IMultipartCopyPlan {
+  partSize: number;
+  partCount: number;
+}
+
+/**
+ * Grows the part size so the object fits in 10,000 parts and refuses objects
+ * that S3 cannot hold.
+ */
+export const planMultipartCopy = (
+  objectSizeArg: number,
+  partSizeArg = multipartCopyPartSizeBytes,
+): IMultipartCopyPlan => {
+  if (
+    !Number.isSafeInteger(objectSizeArg) ||
+    objectSizeArg <= 0 ||
+    objectSizeArg > maximumCopyObjectSizeBytes
+  ) {
+    throw new RangeError(
+      `A multipart copy needs a source of 1 byte to ${maximumCopyObjectSizeBytes} bytes, not ${objectSizeArg}.`,
+    );
+  }
+  const partSize = Math.max(
+    partSizeArg,
+    Math.ceil(objectSizeArg / maximumCopyPartCount),
+  );
+  if (partSize > maximumCopyPartSizeBytes) {
+    throw new RangeError(
+      `A multipart copy part cannot exceed ${maximumCopyPartSizeBytes} bytes.`,
+    );
+  }
+  return {
+    partSize,
+    partCount: Math.ceil(objectSizeArg / partSize),
+  };
+};
+
+export interface IMultipartCopyOptions {
+  copySource: string;
+  sourceSize: number;
+  /** Every part copy requires the source to still carry this ETag. */
+  sourceEtag: string;
+  targetBucketName: string;
+  targetKey: string;
+  /** Object properties of the new object; a multipart upload copies none itself. */
+  objectProperties: Pick<
+    plugins.s3.CreateMultipartUploadCommandInput,
+    | 'Metadata'
+    | 'ContentType'
+    | 'CacheControl'
+    | 'ContentDisposition'
+    | 'ContentEncoding'
+    | 'ContentLanguage'
+    | 'Expires'
+  >;
+  partSize?: number;
+  /** Receives cleanup that outlived its deadline; the operation must not finish before it settles. */
+  onDeferredCleanup: (settlementArg: Promise<void>) => void;
+}
+
+/**
+ * Copies one object part by part. Any failure or abort aborts the multipart
+ * upload instead of completing it.
+ */
+export const copyObjectMultipart = async (
+  operationArg: IBasicBucketOperation,
+  optionsArg: IMultipartCopyOptions,
+): Promise<void> => {
+  const plan = planMultipartCopy(optionsArg.sourceSize, optionsArg.partSize);
+  const abortUpload = (uploadIdArg: string) =>
+    abortMultipartUpload(
+      operationArg,
+      optionsArg.targetKey,
+      uploadIdArg,
+      optionsArg.targetBucketName,
+    );
+  const createOutput = await operationArg.runProvider(
+    () => operationArg.client.send(
+      new plugins.s3.CreateMultipartUploadCommand({
+        Bucket: optionsArg.targetBucketName,
+        Key: optionsArg.targetKey,
+        ...optionsArg.objectProperties,
+      }),
+      { abortSignal: operationArg.signal },
+    ),
+    {
+      // A create that ignored cancellation still owns an upload.
+      disposeLateResult: async (lateOutputArg) => {
+        if (lateOutputArg.UploadId) {
+          const abortResult = await abortUpload(lateOutputArg.UploadId);
+          await abortResult.settlement;
+        }
+      },
+    },
+  );
+  const uploadId = createOutput.UploadId;
+  if (typeof uploadId !== 'string' || uploadId.length === 0) {
+    throw new Error('CreateMultipartUpload returned no UploadId.');
+  }
+  try {
+    const parts: plugins.s3.CompletedPart[] = [];
+    for (let partNumber = 1; partNumber <= plan.partCount; partNumber++) {
+      const start = (partNumber - 1) * plan.partSize;
+      const end = Math.min(optionsArg.sourceSize, start + plan.partSize) - 1;
+      const partOutput = await operationArg.runProvider(
+        () => operationArg.client.send(
+          new plugins.s3.UploadPartCopyCommand({
+            Bucket: optionsArg.targetBucketName,
+            Key: optionsArg.targetKey,
+            UploadId: uploadId,
+            PartNumber: partNumber,
+            CopySource: optionsArg.copySource,
+            CopySourceRange: `bytes=${start}-${end}`,
+            CopySourceIfMatch: optionsArg.sourceEtag,
+          }),
+          { abortSignal: operationArg.signal },
+        ),
+      );
+      const etag = partOutput.CopyPartResult?.ETag;
+      if (typeof etag !== 'string' || etag.length === 0) {
+        throw new Error('UploadPartCopy returned no ETag.');
+      }
+      parts.push({ PartNumber: partNumber, ETag: etag });
+    }
+    await operationArg.runProvider(
+      () => operationArg.client.send(
+        new plugins.s3.CompleteMultipartUploadCommand({
+          Bucket: optionsArg.targetBucketName,
+          Key: optionsArg.targetKey,
+          UploadId: uploadId,
+          MultipartUpload: { Parts: parts },
+        }),
+        { abortSignal: operationArg.signal },
+      ),
+    );
+  } catch (error) {
+    const abortResult = await abortUpload(uploadId);
+    if (abortResult.cleanup.status === 'abort-failed') {
+      throw new AggregateError(
+        [error, abortResult.cleanup.cause],
+        `The multipart copy failed and its upload '${uploadId}' could not be aborted.`,
+      );
+    }
+    if (abortResult.timedOut) {
+      optionsArg.onDeferredCleanup(abortResult.settlement);
+    }
+    throw error;
+  }
+};