@lossless.org/client 2.1.0 → 2.3.0

package/ts/objectstorage/classes.bucket.ts

@@ -18,6 +18,10 @@ import {
   purgeExactPath as purgeExactPathOperation,
 } from './internal.exactpathpurge.operations.js';
 import { beginBasicBucketOperation } from './internal.exactupload.capability.js';
+import {
+  copyObjectMultipart,
+  multipartCopyThresholdBytes,
+} from './internal.multipartcopy.js';
 import {
   getProviderStatus,
   headBucketExists,
@@ -930,8 +934,18 @@ export class Bucket {
     return purgeExactPathOperation(this, optionsArg);
   }
 
+  /**
+   * Copies an object within the storage account. Sources up to 5 GiB are
+   * copied with one CopyObject; larger sources, which CopyObject refuses, are
+   * copied part by part with UploadPartCopy, each part conditional on the
+   * source ETag read first.
+   * A failed or aborted part copy aborts the multipart upload. Object tags,
+   * storage class, server-side encryption settings and a website redirect are
+   * not carried by a part-by-part copy.
+   */
   public async fastCopy(optionsArg: interfaces.IFastCopyOptions): Promise<void> {
     const operation = beginBasicBucketOperation(this, optionsArg.signal);
+    const deferredCleanups: Promise<void>[] = [];
     try {
       const targetBucketName = optionsArg.targetBucket ? optionsArg.targetBucket.name : this.name;
 
@@ -951,10 +965,41 @@ export class Bucket {
 
       // Define the copy operation
       const copySource = `${operation.bucketName}/${optionsArg.sourcePath}`;
+      const targetKey = optionsArg.destinationPath || optionsArg.sourcePath;
+      const sourceSize = currentObjInfo.ContentLength;
+      if (typeof sourceSize === 'number' && sourceSize > multipartCopyThresholdBytes) {
+        if (typeof currentObjInfo.ETag !== 'string' || currentObjInfo.ETag.length === 0) {
+          throw new Error('HeadObject returned no ETag for the copy source.');
+        }
+        await copyObjectMultipart(operation, {
+          copySource,
+          sourceSize,
+          sourceEtag: currentObjInfo.ETag,
+          targetBucketName,
+          targetKey,
+          // Mirrors CopyObject: COPY keeps the source metadata and content
+          // headers, REPLACE sets only the given metadata.
+          objectProperties: optionsArg.deleteExistingNativeMetadata
+            ? { Metadata: newNativeMetadata }
+            : {
+                Metadata: currentObjInfo.Metadata,
+                ContentType: currentObjInfo.ContentType,
+                CacheControl: currentObjInfo.CacheControl,
+                ContentDisposition: currentObjInfo.ContentDisposition,
+                ContentEncoding: currentObjInfo.ContentEncoding,
+                ContentLanguage: currentObjInfo.ContentLanguage,
+                Expires: currentObjInfo.Expires,
+              },
+          onDeferredCleanup: (settlementArg) => {
+            deferredCleanups.push(settlementArg);
+          },
+        });
+        return;
+      }
       const command = new plugins.s3.CopyObjectCommand({
         Bucket: targetBucketName,
         CopySource: copySource,
-        Key: optionsArg.destinationPath || optionsArg.sourcePath,
+        Key: targetKey,
         Metadata: newNativeMetadata,
         MetadataDirective: optionsArg.deleteExistingNativeMetadata ? 'REPLACE' : 'COPY',
       });
@@ -962,7 +1007,11 @@ export class Bucket {
         () => operation.client.send(command, { abortSignal: operation.signal }),
       );
     } finally {
-      operation.finish();
+      if (deferredCleanups.length > 0) {
+        void Promise.allSettled(deferredCleanups).then(() => operation.finish());
+      } else {
+        operation.finish();
+      }
     }
   }
 

package/ts/objectstorage/interfaces.ts

@@ -264,7 +264,18 @@ export interface IFastPutStreamExactOptions {
    * size (5 MiB by default).
    */
   readableStream: Readable | ReadableStream<Uint8Array>;
-  contentLength: number;
+  /**
+   * The exact source length. Exactly one of `contentLength` and
+   * `maxContentLength` must be given.
+   */
+  contentLength?: number;
+  /**
+   * The upper bound for a source of undeclared length. The part size grows so
+   * that the maximum fits in 10,000 parts, and buffering stays bounded to the
+   * planned parts in flight. A source that exceeds the maximum fails the
+   * upload, and the multipart upload is aborted instead of completed.
+   */
+  maxContentLength?: number;
   nativeMetadata?: Record<string, string>;
   /**
    * Native S3 object properties that must retain their semantic header
@@ -316,7 +327,10 @@ export type TExactUploadMultipartCleanup =
 export interface IExactUploadFailureState {
   phase: TExactUploadFailurePhase;
   destinationPath: string;
-  expectedContentLength: number;
+  /** The declared source length, when the upload declared one. */
+  expectedContentLength?: number;
+  /** The source maximum, when the upload had an undeclared length. */
+  maxContentLength?: number;
   /**
    * Whether this operation can prove that it did not publish the destination.
    * A conditional PutObject or CompleteMultipartUpload response can be lost

package/ts/objectstorage/internal.exactupload.operations.ts

@@ -15,7 +15,10 @@ import {
   mintExactUploadCapability,
   type IExactUploadOperation,
 } from './internal.exactupload.capability.js';
-import { planMultipartUpload } from './internal.multipart.planner.js';
+import {
+  planBoundedMultipartUpload,
+  planMultipartUpload,
+} from './internal.multipart.planner.js';
 import {
   consumeExactMultipartStream,
   MultipartStreamConsumptionError,
@@ -44,7 +47,7 @@ interface IIndependentCleanupResult {
   settlement: Promise<void>;
 }
 
-interface IMultipartAbortResult {
+export interface IMultipartAbortResult {
   cleanup: TExactUploadMultipartCleanup;
   timedOut: boolean;
   settlement: Promise<void>;
@@ -1263,10 +1266,11 @@ export const probeExactUploadCapability = async (
   }
 };
 
-const abortMultipartUpload = async (
+export const abortMultipartUpload = async (
   operationArg: IExactUploadOperation,
   keyArg: string,
   uploadIdArg: string,
+  bucketNameArg = operationArg.bucketName,
 ): Promise<IMultipartAbortResult> => {
   let reconciliationResult: 'aborted' | 'absent' | undefined;
   const cleanupResult = await runIndependentCleanup(
@@ -1274,7 +1278,7 @@ const abortMultipartUpload = async (
       async (signalArg) => {
         reconciliationResult = await reconcileMultipartUploadAbort(
           operationArg.client,
-          operationArg.bucketName,
+          bucketNameArg,
           keyArg,
           uploadIdArg,
           signalArg,
@@ -1306,6 +1310,69 @@ const abortMultipartUpload = async (
   };
 };
 
+/**
+ * Plans the upload from its declared length, or from the maximum of a source
+ * of undeclared length, and names that length for failure states.
+ */
+const planExactUpload = (optionsArg: IFastPutStreamExactOptions) => {
+  const declaresLength = optionsArg.contentLength !== undefined;
+  if (declaresLength === (optionsArg.maxContentLength !== undefined)) {
+    throw new TypeError(
+      'Exactly one of contentLength and maxContentLength must be given.',
+    );
+  }
+  if (declaresLength) {
+    return {
+      plan: planMultipartUpload({
+        objectSize: optionsArg.contentLength!,
+        requestedPartSize: optionsArg.requestedPartSize,
+        requestedConcurrency: optionsArg.requestedConcurrency,
+      }),
+      lengthState: {
+        expectedContentLength: optionsArg.contentLength,
+      },
+    };
+  }
+  return {
+    plan: planBoundedMultipartUpload({
+      maxObjectSize: optionsArg.maxContentLength!,
+      requestedPartSize: optionsArg.requestedPartSize,
+      requestedConcurrency: optionsArg.requestedConcurrency,
+    }),
+    lengthState: {
+      maxContentLength: optionsArg.maxContentLength,
+    },
+  };
+};
+
+const sendConditionalZeroPut = (
+  operationArg: IExactUploadOperation,
+  optionsArg: IFastPutStreamExactOptions,
+  contentTypeArg: string | undefined,
+  onAttemptArg: () => void,
+) => {
+  return sendOperationCommand(
+    operationArg,
+    (signalArg) => {
+      onAttemptArg();
+      return operationArg.client.send(
+        new plugins.s3.PutObjectCommand({
+          Bucket: operationArg.bucketName,
+          Key: optionsArg.path,
+          Body: new Uint8Array(0),
+          ContentLength: 0,
+          Metadata: optionsArg.nativeMetadata,
+          ContentType: contentTypeArg,
+          IfNoneMatch: '*',
+        }),
+        {
+          abortSignal: signalArg,
+        },
+      );
+    },
+  );
+};
+
 export const fastPutStreamExact = async (
   bucketRefArg: IExactUploadBucketRef,
   optionsArg: IFastPutStreamExactOptions,
@@ -1316,11 +1383,7 @@ export const fastPutStreamExact = async (
   const nativeObjectProperties = snapshotNativeObjectProperties(
     optionsArg.nativeObjectProperties,
   );
-  const plan = planMultipartUpload({
-    objectSize: optionsArg.contentLength,
-    requestedPartSize: optionsArg.requestedPartSize,
-    requestedConcurrency: optionsArg.requestedConcurrency,
-  });
+  const { plan, lengthState } = planExactUpload(optionsArg);
 
   let operation: IExactUploadOperation;
   try {
@@ -1334,7 +1397,7 @@ export const fastPutStreamExact = async (
       {
         phase: 'capability',
         destinationPath: optionsArg.path,
-        expectedContentLength: optionsArg.contentLength,
+        ...lengthState,
         destinationPublication: 'not-published',
         multipartCleanup: {
           status: 'not-required',
@@ -1353,7 +1416,7 @@ export const fastPutStreamExact = async (
       {
         phase: 'source',
         destinationPath: optionsArg.path,
-        expectedContentLength: optionsArg.contentLength,
+        ...lengthState,
         destinationPublication: 'not-published',
         multipartCleanup: {
           status: 'not-required',
@@ -1393,7 +1456,7 @@ export const fastPutStreamExact = async (
           {
             phase: 'source',
             destinationPath: optionsArg.path,
-            expectedContentLength: optionsArg.contentLength,
+            ...lengthState,
             destinationPublication: 'not-published',
             multipartCleanup: {
               status: 'not-required',
@@ -1404,24 +1467,12 @@ export const fastPutStreamExact = async (
       }
 
       try {
-        const putOutput = await sendOperationCommand(
+        const putOutput = await sendConditionalZeroPut(
           operation,
-          (signalArg) => {
+          optionsArg,
+          nativeObjectProperties?.contentType,
+          () => {
             putRequestAttempted = true;
-            return operation.client.send(
-              new plugins.s3.PutObjectCommand({
-                Bucket: operation.bucketName,
-                Key: optionsArg.path,
-                Body: new Uint8Array(0),
-                ContentLength: 0,
-                Metadata: optionsArg.nativeMetadata,
-                ContentType: nativeObjectProperties?.contentType,
-                IfNoneMatch: '*',
-              }),
-              {
-                abortSignal: signalArg,
-              },
-            );
           },
         );
         return {
@@ -1441,7 +1492,7 @@ export const fastPutStreamExact = async (
           {
             phase: 'put-zero',
             destinationPath: optionsArg.path,
-            expectedContentLength: optionsArg.contentLength,
+            ...lengthState,
             destinationPublication:
               putRequestAttempted &&
               !isDefinitiveConditionalWriteRejection(error)
@@ -1465,36 +1516,64 @@ export const fastPutStreamExact = async (
     }
   }
 
-  let phase: TExactUploadFailurePhase = 'create-multipart';
+  const bounded = plan.classification === 'bounded';
+  let phase: TExactUploadFailurePhase = bounded ? 'source' : 'create-multipart';
   let uploadId: string | undefined;
   let createRequestAttempted = false;
+  let createFailed = false;
+  let createSettled = false;
+  let zeroPutAttempted = false;
   let completeRequestAttempted = false;
   let operationFinishSettlement: Promise<void> | undefined;
   let detachedStreamError: MultipartStreamConsumptionError | undefined;
-  try {
-    const createOutput = await sendOperationCommand(
+  let multipartUploadCreation: Promise<string> | undefined;
+  // A source of undeclared length creates the multipart upload with its first
+  // part, so an empty source never leaves one behind.
+  const ensureMultipartUpload = (): Promise<string> => {
+    multipartUploadCreation ??= sendOperationCommand(
       operation,
       (signalArg) => {
         createRequestAttempted = true;
-        return operation.client.send(
-          new plugins.s3.CreateMultipartUploadCommand({
-            Bucket: operation.bucketName,
-            Key: optionsArg.path,
-            Metadata: optionsArg.nativeMetadata,
-            ContentType: nativeObjectProperties?.contentType,
-          }),
-          {
-            abortSignal: signalArg,
-          },
+        return operation.client
+          .send(
+            new plugins.s3.CreateMultipartUploadCommand({
+              Bucket: operation.bucketName,
+              Key: optionsArg.path,
+              Metadata: optionsArg.nativeMetadata,
+              ContentType: nativeObjectProperties?.contentType,
+            }),
+            {
+              abortSignal: signalArg,
+            },
+          )
+          .then((createOutputArg) => {
+            // A create that ignored cancellation still owns its upload, so
+            // deferred cleanup aborts it by this id.
+            uploadId = getOptionalNonEmptyString(createOutputArg.UploadId);
+            return createOutputArg;
+          });
+      },
+    ).then(
+      (createOutputArg) => {
+        createSettled = true;
+        return requireNonEmptyString(
+          createOutputArg.UploadId,
+          'CreateMultipartUpload UploadId',
         );
       },
+      (error: unknown) => {
+        createSettled = true;
+        createFailed = true;
+        throw error;
+      },
     );
-    uploadId = requireNonEmptyString(
-      createOutput.UploadId,
-      'CreateMultipartUpload UploadId',
-    );
-
-    phase = 'source';
+    return multipartUploadCreation;
+  };
+  try {
+    if (!bounded) {
+      await ensureMultipartUpload();
+      phase = 'source';
+    }
     const completedParts: plugins.s3.CompletedPart[] = [];
     let consumptionResult;
     try {
@@ -1505,12 +1584,14 @@ export const fastPutStreamExact = async (
         failureSettlementTimeoutMs:
           optionsArg.failureSettlementTimeoutMs,
         async onPart(partArg) {
+          operation.assertCurrent();
+          const partUploadId = await ensureMultipartUpload();
           operation.assertCurrent();
           const uploadPartOutput = await operation.client.send(
             new plugins.s3.UploadPartCommand({
               Bucket: operation.bucketName,
               Key: optionsArg.path,
-              UploadId: uploadId!,
+              UploadId: partUploadId,
               PartNumber: partArg.partNumber,
               Body: partArg.data,
               ContentLength: partArg.data.byteLength,
@@ -1532,9 +1613,15 @@ export const fastPutStreamExact = async (
     } catch (error) {
       if (
         error instanceof MultipartStreamConsumptionError &&
-        error.state.kind === 'part-callback'
+        (error.state.kind === 'part-callback' || error.state.kind === 'aborted')
       ) {
-        phase = 'upload-part';
+        // A source of undeclared length creates its upload with the first
+        // part, so a failed or still outstanding create owns the failure.
+        if (createFailed || (multipartUploadCreation && !createSettled)) {
+          phase = 'create-multipart';
+        } else if (error.state.kind === 'part-callback') {
+          phase = 'upload-part';
+        }
       }
       if (
         error instanceof MultipartStreamConsumptionError &&
@@ -1548,6 +1635,28 @@ export const fastPutStreamExact = async (
       throw error;
     }
 
+    if (!multipartUploadCreation) {
+      // Only an empty source of undeclared length reaches this point.
+      phase = 'put-zero';
+      const putOutput = await sendConditionalZeroPut(
+        operation,
+        optionsArg,
+        nativeObjectProperties?.contentType,
+        () => {
+          zeroPutAttempted = true;
+        },
+      );
+      return {
+        destinationPath: optionsArg.path,
+        contentLength: 0,
+        etag: requireNonEmptyString(putOutput.ETag, 'PutObject ETag'),
+        versionId: getOptionalNonEmptyString(putOutput.VersionId),
+        partCount: 0,
+        peakBufferedBytes: consumptionResult.peakBufferedBytes,
+        maximumBufferedBytes: consumptionResult.maximumBufferedBytes,
+      };
+    }
+    const completeUploadId = await multipartUploadCreation;
     phase = 'complete-multipart';
     const completeOutput = await sendOperationCommand(
       operation,
@@ -1557,12 +1666,12 @@ export const fastPutStreamExact = async (
           new plugins.s3.CompleteMultipartUploadCommand({
             Bucket: operation.bucketName,
             Key: optionsArg.path,
-            UploadId: uploadId,
+            UploadId: completeUploadId,
             MultipartUpload: {
               Parts: completedParts,
             },
             IfNoneMatch: '*',
-            MpuObjectSize: optionsArg.contentLength,
+            MpuObjectSize: consumptionResult.bytesConsumed,
           }),
           {
             abortSignal: signalArg,
@@ -1572,7 +1681,7 @@ export const fastPutStreamExact = async (
     );
     return {
       destinationPath: optionsArg.path,
-      contentLength: optionsArg.contentLength,
+      contentLength: consumptionResult.bytesConsumed,
       etag: requireNonEmptyString(
         completeOutput.ETag,
         'CompleteMultipartUpload ETag',
@@ -1586,21 +1695,21 @@ export const fastPutStreamExact = async (
     let multipartCleanup: TExactUploadMultipartCleanup;
     const primaryProviderSettlement =
       getDetachedOperationSendSettlement(operation.signal);
-    if (
-      uploadId &&
-      detachedStreamError?.state.cleanup.callbacks === 'detached'
-    ) {
-      multipartCleanup = {
-        status: 'deferred',
-      };
-      const callbackGatedCleanup = detachedStreamError.callbackSettlement
+    // Aborts the multipart upload once the given work settles. A create that
+    // was still in flight may only report its UploadId by then.
+    const abortAfterSettlement = (settlementArg: Promise<unknown>) =>
+      settlementArg
         .then(async () => {
+          await getDetachedOperationSendSettlement(operation.signal);
+          if (!uploadId) {
+            return;
+          }
           try {
             operation.assertLifecycleCurrent();
             const abortResult = await abortMultipartUpload(
               operation,
               optionsArg.path,
-              uploadId!,
+              uploadId,
             );
             await abortResult.settlement;
           } catch {
@@ -1611,35 +1720,27 @@ export const fastPutStreamExact = async (
           () => undefined,
           () => undefined,
         );
+    if (
+      (uploadId || createRequestAttempted) &&
+      detachedStreamError?.state.cleanup.callbacks === 'detached'
+    ) {
+      multipartCleanup = {
+        status: 'deferred',
+      };
       operationFinishSettlement = Promise.allSettled([
-        callbackGatedCleanup,
+        abortAfterSettlement(detachedStreamError.callbackSettlement),
         detachedStreamError.ownedSettlement,
       ]).then(() => undefined);
-    } else if (uploadId && primaryProviderSettlement) {
+    } else if (
+      (uploadId || createRequestAttempted) &&
+      primaryProviderSettlement
+    ) {
       multipartCleanup = {
         status: 'deferred',
       };
-      const providerGatedCleanup = primaryProviderSettlement
-        .then(async () => {
-          try {
-            operation.assertLifecycleCurrent();
-            const abortResult = await abortMultipartUpload(
-              operation,
-              optionsArg.path,
-              uploadId!,
-            );
-            await abortResult.settlement;
-          } catch {
-            // Lifecycle close makes deferred cleanup indeterminate.
-          }
-        })
-        .then(
-          () => undefined,
-          () => undefined,
-        );
       operationFinishSettlement = Promise.allSettled([
         primaryProviderSettlement,
-        providerGatedCleanup,
+        abortAfterSettlement(primaryProviderSettlement),
       ]).then(() => undefined);
     } else if (uploadId) {
       const abortResult = await abortMultipartUpload(
@@ -1653,20 +1754,11 @@ export const fastPutStreamExact = async (
       }
     } else if (createRequestAttempted) {
       multipartCleanup = {
-        ...(primaryProviderSettlement
-          ? {
-              status: 'deferred' as const,
-            }
-          : {
-              status: 'abort-failed' as const,
-              cause: new Error(
-                'The multipart upload could not be aborted because no UploadId was returned.',
-              ),
-            }),
+        status: 'abort-failed',
+        cause: new Error(
+          'The multipart upload could not be aborted because no UploadId was returned.',
+        ),
       };
-      if (primaryProviderSettlement) {
-        operationFinishSettlement = primaryProviderSettlement;
-      }
     } else {
       multipartCleanup = {
         status: 'not-required',
@@ -1691,10 +1783,12 @@ export const fastPutStreamExact = async (
       {
         phase,
         destinationPath: optionsArg.path,
-        expectedContentLength: optionsArg.contentLength,
+        ...lengthState,
         destinationPublication:
-          phase === 'complete-multipart' &&
-          completeRequestAttempted &&
+          (
+            (phase === 'complete-multipart' && completeRequestAttempted) ||
+            (phase === 'put-zero' && zeroPutAttempted)
+          ) &&
           !isDefinitiveConditionalWriteRejection(error)
             ? 'ambiguous'
             : 'not-published',

package/ts/objectstorage/internal.multipart.planner.ts

@@ -1,6 +1,8 @@
 import type {
+  IMultipartBoundedPlanningOptions,
+  IMultipartBoundedUploadPlan,
   IMultipartPlanningOptions,
-  TMultipartUploadPlan,
+  TMultipartDeclaredUploadPlan,
 } from './internal.multipart.types.js';
 
 export const MULTIPART_MIN_PART_SIZE = 5 * 1024 * 1024;
@@ -43,7 +45,7 @@ const normalizeRequestedConcurrency = (
 
 export const planMultipartUpload = (
   optionsArg: IMultipartPlanningOptions,
-): TMultipartUploadPlan => {
+): TMultipartDeclaredUploadPlan => {
   requireSafeNonNegativeInteger(optionsArg.objectSize, 'objectSize');
   if (optionsArg.objectSize > MULTIPART_MAX_OBJECT_SIZE) {
     throw new RangeError(
@@ -88,3 +90,45 @@ export const planMultipartUpload = (
     ) + Math.min(optionsArg.objectSize, partSize),
   };
 };
+
+/**
+ * Plans an upload of undeclared length that must not exceed
+ * `maxObjectSize`. The part size grows with the maximum so that the largest
+ * allowed source still fits into the multipart part limit.
+ */
+export const planBoundedMultipartUpload = (
+  optionsArg: IMultipartBoundedPlanningOptions,
+): IMultipartBoundedUploadPlan => {
+  if (!Number.isSafeInteger(optionsArg.maxObjectSize) || optionsArg.maxObjectSize < 1) {
+    throw new TypeError('maxObjectSize must be a positive safe integer.');
+  }
+  if (optionsArg.maxObjectSize > MULTIPART_MAX_OBJECT_SIZE) {
+    throw new RangeError(
+      `maxObjectSize exceeds the ${MULTIPART_MAX_OBJECT_SIZE} byte multipart limit.`,
+    );
+  }
+  const requestedConcurrency = normalizeRequestedConcurrency(
+    optionsArg.requestedConcurrency,
+  );
+  const preferredPartSize = normalizeRequestedPartSize(optionsArg.requestedPartSize);
+  const partSize = Math.max(
+    preferredPartSize,
+    Math.ceil(optionsArg.maxObjectSize / MULTIPART_MAX_PART_COUNT),
+  );
+  if (partSize > MULTIPART_MAX_PART_SIZE) {
+    throw new RangeError('The maximum cannot be represented by legal multipart parts.');
+  }
+  const maxPartCount = Math.ceil(optionsArg.maxObjectSize / partSize);
+  const concurrency = Math.min(requestedConcurrency, maxPartCount);
+  return {
+    classification: 'bounded',
+    maxObjectSize: optionsArg.maxObjectSize,
+    partSize,
+    maxPartCount,
+    concurrency,
+    maximumBufferedBytes: Math.min(
+      optionsArg.maxObjectSize,
+      partSize * concurrency,
+    ) + Math.min(optionsArg.maxObjectSize, partSize),
+  };
+};