@lossless.org/client 2.0.0 → 2.2.0

package/ts/objectstorage/internal.multipart.stream.ts

@@ -60,6 +60,7 @@ const failureMessages: Record<TMultipartStreamFailureKind, string> = {
     'The multipart source emitted too many consecutive empty chunks.',
   'source-short': 'The multipart source ended before the declared content length.',
   'source-extra': 'The multipart source exceeded the declared content length.',
+  'source-over-maximum': 'The multipart source exceeded the maximum content length.',
   'part-callback': 'A multipart part callback failed.',
 };
 const failureSettlementMaximumTimeoutMs = 30_000;
@@ -96,13 +97,6 @@ export class MultipartStreamConsumptionError extends Error {
 }
 
 const validatePlan = (planArg: TMultipartUploadPlan): void => {
-  if (
-    !Number.isSafeInteger(planArg.objectSize) ||
-    planArg.objectSize < 0 ||
-    planArg.objectSize > MULTIPART_MAX_OBJECT_SIZE
-  ) {
-    throw new TypeError('plan.objectSize is invalid.');
-  }
   if (
     !Number.isSafeInteger(planArg.concurrency) ||
     planArg.concurrency < 1 ||
@@ -110,6 +104,32 @@ const validatePlan = (planArg: TMultipartUploadPlan): void => {
   ) {
     throw new TypeError('plan.concurrency is invalid.');
   }
+  if (planArg.classification === 'bounded') {
+    if (
+      !Number.isSafeInteger(planArg.maxObjectSize) ||
+      planArg.maxObjectSize < 1 ||
+      planArg.maxObjectSize > MULTIPART_MAX_OBJECT_SIZE ||
+      !Number.isSafeInteger(planArg.partSize) ||
+      planArg.partSize < MULTIPART_MIN_PART_SIZE ||
+      planArg.partSize > MULTIPART_MAX_PART_SIZE ||
+      planArg.maxPartCount !== Math.ceil(planArg.maxObjectSize / planArg.partSize) ||
+      planArg.maxPartCount > MULTIPART_MAX_PART_COUNT ||
+      planArg.concurrency > planArg.maxPartCount ||
+      planArg.maximumBufferedBytes !==
+        Math.min(planArg.maxObjectSize, planArg.partSize * planArg.concurrency) +
+          Math.min(planArg.maxObjectSize, planArg.partSize)
+    ) {
+      throw new TypeError('The bounded multipart plan is inconsistent.');
+    }
+    return;
+  }
+  if (
+    !Number.isSafeInteger(planArg.objectSize) ||
+    planArg.objectSize < 0 ||
+    planArg.objectSize > MULTIPART_MAX_OBJECT_SIZE
+  ) {
+    throw new TypeError('plan.objectSize is invalid.');
+  }
   if (planArg.classification === 'zero') {
     if (
       planArg.objectSize !== 0 ||
@@ -165,6 +185,14 @@ export const consumeExactMultipartStream = async (
     );
   }
 
+  const plan = optionsArg.plan;
+  // A bounded plan reads parts until the source ends and refuses more than
+  // its maximum; an exact plan reads exactly the declared length.
+  const bounded = plan.classification === 'bounded';
+  const sizeLimit = bounded ? plan.maxObjectSize : plan.objectSize;
+  const overLimitKind: TMultipartStreamFailureKind = bounded
+    ? 'source-over-maximum'
+    : 'source-extra';
   const reader = optionsArg.readableStream.getReader();
   const callbackAbortController = new AbortController();
   const inFlightParts = new Map<symbol, IInFlightPart>();
@@ -289,7 +317,11 @@ export const consumeExactMultipartStream = async (
     }
   };
 
-  const dispatchPart = (dataArg: Uint8Array, partNumberArg: number): void => {
+  const dispatchPart = (
+    dataArg: Uint8Array,
+    partNumberArg: number,
+    isLastPartArg: boolean,
+  ): void => {
     const token = Symbol(`multipart-part-${partNumberArg}`);
     partCallbacksStarted++;
     currentPartBytes = 0;
@@ -299,8 +331,8 @@ export const consumeExactMultipartStream = async (
       .then(() =>
         optionsArg.onPart({
           partNumber: partNumberArg,
-          byteOffset: (partNumberArg - 1) * optionsArg.plan.partSize,
-          isLastPart: partNumberArg === optionsArg.plan.partCount,
+          byteOffset: (partNumberArg - 1) * plan.partSize,
+          isLastPart: isLastPartArg,
           data: dataArg,
           signal: callbackAbortController.signal,
         }),
@@ -347,15 +379,19 @@ export const consumeExactMultipartStream = async (
 
     for (
       let partNumber = 1;
-      partNumber <= optionsArg.plan.partCount;
+      bounded ? !sourceClosed : partNumber <= plan.partCount;
       partNumber++
     ) {
       await waitForCapacity();
       const partByteLength = Math.min(
-        optionsArg.plan.partSize,
-        optionsArg.plan.objectSize -
-          (partNumber - 1) * optionsArg.plan.partSize,
+        plan.partSize,
+        sizeLimit - (partNumber - 1) * plan.partSize,
       );
+      if (partByteLength <= 0) {
+        // A bounded source filled its maximum, which ends at or inside the
+        // previous part; the trailing read below refuses any further byte.
+        break;
+      }
       if (
         inFlightBytes + partByteLength >
         optionsArg.plan.maximumBufferedBytes
@@ -396,6 +432,9 @@ export const consumeExactMultipartStream = async (
         const readResult = await readWhileObservingCallbacks();
         if (readResult.done) {
           sourceClosed = true;
+          if (bounded) {
+            break;
+          }
           throw {
             kind: 'source-short',
           } satisfies IPrimaryFailure;
@@ -410,16 +449,16 @@ export const consumeExactMultipartStream = async (
           continue;
         }
         observedBytes += readResult.value.byteLength;
-        if (observedBytes > optionsArg.plan.objectSize) {
+        if (observedBytes > sizeLimit) {
           throw {
-            kind: 'source-extra',
+            kind: overLimitKind,
           } satisfies IPrimaryFailure;
         }
-        if (readResult.value.byteLength > optionsArg.plan.partSize) {
+        if (readResult.value.byteLength > plan.partSize) {
           throw {
             kind: 'source-buffer-limit',
             cause: new RangeError(
-              `Source chunks must not exceed ${optionsArg.plan.partSize} bytes.`,
+              `Source chunks must not exceed ${plan.partSize} bytes.`,
             ),
           } satisfies IPrimaryFailure;
         }
@@ -428,7 +467,24 @@ export const consumeExactMultipartStream = async (
         observeBufferedBytes();
       }
 
-      dispatchPart(partData, partNumber);
+      if (partOffset === 0) {
+        // A bounded source ended at a part boundary.
+        currentPartBytes = 0;
+        break;
+      }
+      if (partOffset < partByteLength) {
+        // Only a bounded source ends inside a part; its last part keeps the
+        // bytes read, as an exact copy.
+        dispatchPart(partData.slice(0, partOffset), partNumber, true);
+      } else {
+        dispatchPart(
+          partData,
+          partNumber,
+          bounded
+            ? sourceClosed || observedBytes === sizeLimit
+            : partNumber === plan.partCount,
+        );
+      }
     }
 
     while (!sourceClosed) {
@@ -448,7 +504,7 @@ export const consumeExactMultipartStream = async (
       }
       observedBytes += readResult.value.byteLength;
       throw {
-        kind: 'source-extra',
+        kind: overLimitKind,
       } satisfies IPrimaryFailure;
     }
 
@@ -571,7 +627,9 @@ export const consumeExactMultipartStream = async (
         completionAuthorized: false,
         kind: failure.kind,
         cause: failure.cause,
-        expectedBytes: optionsArg.plan.objectSize,
+        ...(bounded
+          ? { maxBytes: plan.maxObjectSize }
+          : { expectedBytes: plan.objectSize }),
         observedBytes,
         partCallbacksStarted,
         completedPartNumbers: [...completedPartNumbers].sort(

package/ts/objectstorage/internal.multipart.types.ts

@@ -4,6 +4,12 @@ export interface IMultipartPlanningOptions {
   requestedConcurrency?: number;
 }
 
+export interface IMultipartBoundedPlanningOptions {
+  maxObjectSize: number;
+  requestedPartSize?: number;
+  requestedConcurrency?: number;
+}
+
 interface IMultipartUploadPlanBase {
   objectSize: number;
   concurrency: number;
@@ -24,11 +30,34 @@ export interface IMultipartDataUploadPlan extends IMultipartUploadPlanBase {
   partCount: number;
 }
 
-export type TMultipartUploadPlan = IMultipartZeroUploadPlan | IMultipartDataUploadPlan;
+/**
+ * A plan for a source of undeclared length: full parts are read until the
+ * source ends, and a source longer than `maxObjectSize` fails. The part size
+ * is chosen so that `maxObjectSize` fits into the multipart part limit.
+ */
+export interface IMultipartBoundedUploadPlan {
+  classification: 'bounded';
+  maxObjectSize: number;
+  partSize: number;
+  /** Parts that `maxObjectSize` needs; the source may need fewer. */
+  maxPartCount: number;
+  concurrency: number;
+  maximumBufferedBytes: number;
+}
+
+/** A plan for a source of declared length. */
+export type TMultipartDeclaredUploadPlan = IMultipartZeroUploadPlan | IMultipartDataUploadPlan;
+
+export type TMultipartUploadPlan = TMultipartDeclaredUploadPlan | IMultipartBoundedUploadPlan;
 
 export interface IMultipartStreamPart {
   partNumber: number;
   byteOffset: number;
+  /**
+   * Under a bounded plan a part is only known to be the last when the source
+   * ended inside it or it reached the maximum; a source that ends exactly at
+   * a part boundary leaves its last part marked false.
+   */
   isLastPart: boolean;
   /**
    * An exact, exclusively owned copy. The consumer never mutates or reuses it.
@@ -61,6 +90,7 @@ export type TMultipartStreamFailureKind =
   | 'source-empty-chunk-limit'
   | 'source-short'
   | 'source-extra'
+  | 'source-over-maximum'
   | 'part-callback';
 
 export type TMultipartOwnedReaderCleanup =
@@ -97,7 +127,10 @@ export interface IMultipartStreamFailureState {
   completionAuthorized: false;
   kind: TMultipartStreamFailureKind;
   cause?: unknown;
-  expectedBytes: number;
+  /** The declared length; absent under a bounded plan. */
+  expectedBytes?: number;
+  /** The maximum length of a bounded plan. */
+  maxBytes?: number;
   observedBytes: number;
   partCallbacksStarted: number;
   completedPartNumbers: number[];