@lossless.org/client 2.1.0 → 2.2.0

package/ts/objectstorage/internal.exactupload.operations.ts

@@ -15,7 +15,10 @@ import {
   mintExactUploadCapability,
   type IExactUploadOperation,
 } from './internal.exactupload.capability.js';
-import { planMultipartUpload } from './internal.multipart.planner.js';
+import {
+  planBoundedMultipartUpload,
+  planMultipartUpload,
+} from './internal.multipart.planner.js';
 import {
   consumeExactMultipartStream,
   MultipartStreamConsumptionError,
@@ -1306,6 +1309,69 @@ const abortMultipartUpload = async (
   };
 };
 
+/**
+ * Plans the upload from its declared length, or from the maximum of a source
+ * of undeclared length, and names that length for failure states.
+ */
+const planExactUpload = (optionsArg: IFastPutStreamExactOptions) => {
+  const declaresLength = optionsArg.contentLength !== undefined;
+  if (declaresLength === (optionsArg.maxContentLength !== undefined)) {
+    throw new TypeError(
+      'Exactly one of contentLength and maxContentLength must be given.',
+    );
+  }
+  if (declaresLength) {
+    return {
+      plan: planMultipartUpload({
+        objectSize: optionsArg.contentLength!,
+        requestedPartSize: optionsArg.requestedPartSize,
+        requestedConcurrency: optionsArg.requestedConcurrency,
+      }),
+      lengthState: {
+        expectedContentLength: optionsArg.contentLength,
+      },
+    };
+  }
+  return {
+    plan: planBoundedMultipartUpload({
+      maxObjectSize: optionsArg.maxContentLength!,
+      requestedPartSize: optionsArg.requestedPartSize,
+      requestedConcurrency: optionsArg.requestedConcurrency,
+    }),
+    lengthState: {
+      maxContentLength: optionsArg.maxContentLength,
+    },
+  };
+};
+
+const sendConditionalZeroPut = (
+  operationArg: IExactUploadOperation,
+  optionsArg: IFastPutStreamExactOptions,
+  contentTypeArg: string | undefined,
+  onAttemptArg: () => void,
+) => {
+  return sendOperationCommand(
+    operationArg,
+    (signalArg) => {
+      onAttemptArg();
+      return operationArg.client.send(
+        new plugins.s3.PutObjectCommand({
+          Bucket: operationArg.bucketName,
+          Key: optionsArg.path,
+          Body: new Uint8Array(0),
+          ContentLength: 0,
+          Metadata: optionsArg.nativeMetadata,
+          ContentType: contentTypeArg,
+          IfNoneMatch: '*',
+        }),
+        {
+          abortSignal: signalArg,
+        },
+      );
+    },
+  );
+};
+
 export const fastPutStreamExact = async (
   bucketRefArg: IExactUploadBucketRef,
   optionsArg: IFastPutStreamExactOptions,
@@ -1316,11 +1382,7 @@ export const fastPutStreamExact = async (
   const nativeObjectProperties = snapshotNativeObjectProperties(
     optionsArg.nativeObjectProperties,
   );
-  const plan = planMultipartUpload({
-    objectSize: optionsArg.contentLength,
-    requestedPartSize: optionsArg.requestedPartSize,
-    requestedConcurrency: optionsArg.requestedConcurrency,
-  });
+  const { plan, lengthState } = planExactUpload(optionsArg);
 
   let operation: IExactUploadOperation;
   try {
@@ -1334,7 +1396,7 @@ export const fastPutStreamExact = async (
       {
         phase: 'capability',
         destinationPath: optionsArg.path,
-        expectedContentLength: optionsArg.contentLength,
+        ...lengthState,
         destinationPublication: 'not-published',
         multipartCleanup: {
           status: 'not-required',
@@ -1353,7 +1415,7 @@ export const fastPutStreamExact = async (
       {
         phase: 'source',
         destinationPath: optionsArg.path,
-        expectedContentLength: optionsArg.contentLength,
+        ...lengthState,
         destinationPublication: 'not-published',
         multipartCleanup: {
           status: 'not-required',
@@ -1393,7 +1455,7 @@ export const fastPutStreamExact = async (
           {
             phase: 'source',
             destinationPath: optionsArg.path,
-            expectedContentLength: optionsArg.contentLength,
+            ...lengthState,
             destinationPublication: 'not-published',
             multipartCleanup: {
               status: 'not-required',
@@ -1404,24 +1466,12 @@ export const fastPutStreamExact = async (
       }
 
       try {
-        const putOutput = await sendOperationCommand(
+        const putOutput = await sendConditionalZeroPut(
           operation,
-          (signalArg) => {
+          optionsArg,
+          nativeObjectProperties?.contentType,
+          () => {
             putRequestAttempted = true;
-            return operation.client.send(
-              new plugins.s3.PutObjectCommand({
-                Bucket: operation.bucketName,
-                Key: optionsArg.path,
-                Body: new Uint8Array(0),
-                ContentLength: 0,
-                Metadata: optionsArg.nativeMetadata,
-                ContentType: nativeObjectProperties?.contentType,
-                IfNoneMatch: '*',
-              }),
-              {
-                abortSignal: signalArg,
-              },
-            );
           },
         );
         return {
@@ -1441,7 +1491,7 @@ export const fastPutStreamExact = async (
           {
             phase: 'put-zero',
             destinationPath: optionsArg.path,
-            expectedContentLength: optionsArg.contentLength,
+            ...lengthState,
             destinationPublication:
               putRequestAttempted &&
               !isDefinitiveConditionalWriteRejection(error)
@@ -1465,36 +1515,64 @@ export const fastPutStreamExact = async (
     }
   }
 
-  let phase: TExactUploadFailurePhase = 'create-multipart';
+  const bounded = plan.classification === 'bounded';
+  let phase: TExactUploadFailurePhase = bounded ? 'source' : 'create-multipart';
   let uploadId: string | undefined;
   let createRequestAttempted = false;
+  let createFailed = false;
+  let createSettled = false;
+  let zeroPutAttempted = false;
   let completeRequestAttempted = false;
   let operationFinishSettlement: Promise<void> | undefined;
   let detachedStreamError: MultipartStreamConsumptionError | undefined;
-  try {
-    const createOutput = await sendOperationCommand(
+  let multipartUploadCreation: Promise<string> | undefined;
+  // A source of undeclared length creates the multipart upload with its first
+  // part, so an empty source never leaves one behind.
+  const ensureMultipartUpload = (): Promise<string> => {
+    multipartUploadCreation ??= sendOperationCommand(
       operation,
       (signalArg) => {
         createRequestAttempted = true;
-        return operation.client.send(
-          new plugins.s3.CreateMultipartUploadCommand({
-            Bucket: operation.bucketName,
-            Key: optionsArg.path,
-            Metadata: optionsArg.nativeMetadata,
-            ContentType: nativeObjectProperties?.contentType,
-          }),
-          {
-            abortSignal: signalArg,
-          },
+        return operation.client
+          .send(
+            new plugins.s3.CreateMultipartUploadCommand({
+              Bucket: operation.bucketName,
+              Key: optionsArg.path,
+              Metadata: optionsArg.nativeMetadata,
+              ContentType: nativeObjectProperties?.contentType,
+            }),
+            {
+              abortSignal: signalArg,
+            },
+          )
+          .then((createOutputArg) => {
+            // A create that ignored cancellation still owns its upload, so
+            // deferred cleanup aborts it by this id.
+            uploadId = getOptionalNonEmptyString(createOutputArg.UploadId);
+            return createOutputArg;
+          });
+      },
+    ).then(
+      (createOutputArg) => {
+        createSettled = true;
+        return requireNonEmptyString(
+          createOutputArg.UploadId,
+          'CreateMultipartUpload UploadId',
         );
       },
+      (error: unknown) => {
+        createSettled = true;
+        createFailed = true;
+        throw error;
+      },
     );
-    uploadId = requireNonEmptyString(
-      createOutput.UploadId,
-      'CreateMultipartUpload UploadId',
-    );
-
-    phase = 'source';
+    return multipartUploadCreation;
+  };
+  try {
+    if (!bounded) {
+      await ensureMultipartUpload();
+      phase = 'source';
+    }
     const completedParts: plugins.s3.CompletedPart[] = [];
     let consumptionResult;
     try {
@@ -1505,12 +1583,14 @@ export const fastPutStreamExact = async (
         failureSettlementTimeoutMs:
           optionsArg.failureSettlementTimeoutMs,
         async onPart(partArg) {
+          operation.assertCurrent();
+          const partUploadId = await ensureMultipartUpload();
           operation.assertCurrent();
           const uploadPartOutput = await operation.client.send(
             new plugins.s3.UploadPartCommand({
               Bucket: operation.bucketName,
               Key: optionsArg.path,
-              UploadId: uploadId!,
+              UploadId: partUploadId,
               PartNumber: partArg.partNumber,
               Body: partArg.data,
               ContentLength: partArg.data.byteLength,
@@ -1532,9 +1612,15 @@ export const fastPutStreamExact = async (
     } catch (error) {
       if (
         error instanceof MultipartStreamConsumptionError &&
-        error.state.kind === 'part-callback'
+        (error.state.kind === 'part-callback' || error.state.kind === 'aborted')
       ) {
-        phase = 'upload-part';
+        // A source of undeclared length creates its upload with the first
+        // part, so a failed or still outstanding create owns the failure.
+        if (createFailed || (multipartUploadCreation && !createSettled)) {
+          phase = 'create-multipart';
+        } else if (error.state.kind === 'part-callback') {
+          phase = 'upload-part';
+        }
       }
       if (
         error instanceof MultipartStreamConsumptionError &&
@@ -1548,6 +1634,28 @@ export const fastPutStreamExact = async (
       throw error;
     }
 
+    if (!multipartUploadCreation) {
+      // Only an empty source of undeclared length reaches this point.
+      phase = 'put-zero';
+      const putOutput = await sendConditionalZeroPut(
+        operation,
+        optionsArg,
+        nativeObjectProperties?.contentType,
+        () => {
+          zeroPutAttempted = true;
+        },
+      );
+      return {
+        destinationPath: optionsArg.path,
+        contentLength: 0,
+        etag: requireNonEmptyString(putOutput.ETag, 'PutObject ETag'),
+        versionId: getOptionalNonEmptyString(putOutput.VersionId),
+        partCount: 0,
+        peakBufferedBytes: consumptionResult.peakBufferedBytes,
+        maximumBufferedBytes: consumptionResult.maximumBufferedBytes,
+      };
+    }
+    const completeUploadId = await multipartUploadCreation;
     phase = 'complete-multipart';
     const completeOutput = await sendOperationCommand(
       operation,
@@ -1557,12 +1665,12 @@ export const fastPutStreamExact = async (
           new plugins.s3.CompleteMultipartUploadCommand({
             Bucket: operation.bucketName,
             Key: optionsArg.path,
-            UploadId: uploadId,
+            UploadId: completeUploadId,
             MultipartUpload: {
               Parts: completedParts,
             },
             IfNoneMatch: '*',
-            MpuObjectSize: optionsArg.contentLength,
+            MpuObjectSize: consumptionResult.bytesConsumed,
           }),
           {
             abortSignal: signalArg,
@@ -1572,7 +1680,7 @@ export const fastPutStreamExact = async (
     );
     return {
       destinationPath: optionsArg.path,
-      contentLength: optionsArg.contentLength,
+      contentLength: consumptionResult.bytesConsumed,
       etag: requireNonEmptyString(
         completeOutput.ETag,
         'CompleteMultipartUpload ETag',
@@ -1586,21 +1694,21 @@ export const fastPutStreamExact = async (
     let multipartCleanup: TExactUploadMultipartCleanup;
     const primaryProviderSettlement =
       getDetachedOperationSendSettlement(operation.signal);
-    if (
-      uploadId &&
-      detachedStreamError?.state.cleanup.callbacks === 'detached'
-    ) {
-      multipartCleanup = {
-        status: 'deferred',
-      };
-      const callbackGatedCleanup = detachedStreamError.callbackSettlement
+    // Aborts the multipart upload once the given work settles. A create that
+    // was still in flight may only report its UploadId by then.
+    const abortAfterSettlement = (settlementArg: Promise<unknown>) =>
+      settlementArg
         .then(async () => {
+          await getDetachedOperationSendSettlement(operation.signal);
+          if (!uploadId) {
+            return;
+          }
           try {
             operation.assertLifecycleCurrent();
             const abortResult = await abortMultipartUpload(
               operation,
               optionsArg.path,
-              uploadId!,
+              uploadId,
             );
             await abortResult.settlement;
           } catch {
@@ -1611,35 +1719,27 @@ export const fastPutStreamExact = async (
           () => undefined,
           () => undefined,
         );
+    if (
+      (uploadId || createRequestAttempted) &&
+      detachedStreamError?.state.cleanup.callbacks === 'detached'
+    ) {
+      multipartCleanup = {
+        status: 'deferred',
+      };
       operationFinishSettlement = Promise.allSettled([
-        callbackGatedCleanup,
+        abortAfterSettlement(detachedStreamError.callbackSettlement),
         detachedStreamError.ownedSettlement,
       ]).then(() => undefined);
-    } else if (uploadId && primaryProviderSettlement) {
+    } else if (
+      (uploadId || createRequestAttempted) &&
+      primaryProviderSettlement
+    ) {
       multipartCleanup = {
         status: 'deferred',
       };
-      const providerGatedCleanup = primaryProviderSettlement
-        .then(async () => {
-          try {
-            operation.assertLifecycleCurrent();
-            const abortResult = await abortMultipartUpload(
-              operation,
-              optionsArg.path,
-              uploadId!,
-            );
-            await abortResult.settlement;
-          } catch {
-            // Lifecycle close makes deferred cleanup indeterminate.
-          }
-        })
-        .then(
-          () => undefined,
-          () => undefined,
-        );
       operationFinishSettlement = Promise.allSettled([
         primaryProviderSettlement,
-        providerGatedCleanup,
+        abortAfterSettlement(primaryProviderSettlement),
       ]).then(() => undefined);
     } else if (uploadId) {
       const abortResult = await abortMultipartUpload(
@@ -1653,20 +1753,11 @@ export const fastPutStreamExact = async (
       }
     } else if (createRequestAttempted) {
       multipartCleanup = {
-        ...(primaryProviderSettlement
-          ? {
-              status: 'deferred' as const,
-            }
-          : {
-              status: 'abort-failed' as const,
-              cause: new Error(
-                'The multipart upload could not be aborted because no UploadId was returned.',
-              ),
-            }),
+        status: 'abort-failed',
+        cause: new Error(
+          'The multipart upload could not be aborted because no UploadId was returned.',
+        ),
       };
-      if (primaryProviderSettlement) {
-        operationFinishSettlement = primaryProviderSettlement;
-      }
     } else {
       multipartCleanup = {
         status: 'not-required',
@@ -1691,10 +1782,12 @@ export const fastPutStreamExact = async (
       {
         phase,
         destinationPath: optionsArg.path,
-        expectedContentLength: optionsArg.contentLength,
+        ...lengthState,
         destinationPublication:
-          phase === 'complete-multipart' &&
-          completeRequestAttempted &&
+          (
+            (phase === 'complete-multipart' && completeRequestAttempted) ||
+            (phase === 'put-zero' && zeroPutAttempted)
+          ) &&
           !isDefinitiveConditionalWriteRejection(error)
             ? 'ambiguous'
             : 'not-published',

package/ts/objectstorage/internal.multipart.planner.ts

@@ -1,6 +1,8 @@
 import type {
+  IMultipartBoundedPlanningOptions,
+  IMultipartBoundedUploadPlan,
   IMultipartPlanningOptions,
-  TMultipartUploadPlan,
+  TMultipartDeclaredUploadPlan,
 } from './internal.multipart.types.js';
 
 export const MULTIPART_MIN_PART_SIZE = 5 * 1024 * 1024;
@@ -43,7 +45,7 @@ const normalizeRequestedConcurrency = (
 
 export const planMultipartUpload = (
   optionsArg: IMultipartPlanningOptions,
-): TMultipartUploadPlan => {
+): TMultipartDeclaredUploadPlan => {
   requireSafeNonNegativeInteger(optionsArg.objectSize, 'objectSize');
   if (optionsArg.objectSize > MULTIPART_MAX_OBJECT_SIZE) {
     throw new RangeError(
@@ -88,3 +90,45 @@ export const planMultipartUpload = (
     ) + Math.min(optionsArg.objectSize, partSize),
   };
 };
+
+/**
+ * Plans an upload of undeclared length that must not exceed
+ * `maxObjectSize`. The part size grows with the maximum so that the largest
+ * allowed source still fits into the multipart part limit.
+ */
+export const planBoundedMultipartUpload = (
+  optionsArg: IMultipartBoundedPlanningOptions,
+): IMultipartBoundedUploadPlan => {
+  if (!Number.isSafeInteger(optionsArg.maxObjectSize) || optionsArg.maxObjectSize < 1) {
+    throw new TypeError('maxObjectSize must be a positive safe integer.');
+  }
+  if (optionsArg.maxObjectSize > MULTIPART_MAX_OBJECT_SIZE) {
+    throw new RangeError(
+      `maxObjectSize exceeds the ${MULTIPART_MAX_OBJECT_SIZE} byte multipart limit.`,
+    );
+  }
+  const requestedConcurrency = normalizeRequestedConcurrency(
+    optionsArg.requestedConcurrency,
+  );
+  const preferredPartSize = normalizeRequestedPartSize(optionsArg.requestedPartSize);
+  const partSize = Math.max(
+    preferredPartSize,
+    Math.ceil(optionsArg.maxObjectSize / MULTIPART_MAX_PART_COUNT),
+  );
+  if (partSize > MULTIPART_MAX_PART_SIZE) {
+    throw new RangeError('The maximum cannot be represented by legal multipart parts.');
+  }
+  const maxPartCount = Math.ceil(optionsArg.maxObjectSize / partSize);
+  const concurrency = Math.min(requestedConcurrency, maxPartCount);
+  return {
+    classification: 'bounded',
+    maxObjectSize: optionsArg.maxObjectSize,
+    partSize,
+    maxPartCount,
+    concurrency,
+    maximumBufferedBytes: Math.min(
+      optionsArg.maxObjectSize,
+      partSize * concurrency,
+    ) + Math.min(optionsArg.maxObjectSize, partSize),
+  };
+};