@lossless.org/client 0.1.0

package/ts/objectstorage/internal.multipart.planner.ts

@@ -0,0 +1,90 @@
+import type {
+  IMultipartPlanningOptions,
+  TMultipartUploadPlan,
+} from './internal.multipart.types.js';
+
+export const MULTIPART_MIN_PART_SIZE = 5 * 1024 * 1024;
+export const MULTIPART_MAX_PART_SIZE = 5 * 1024 * 1024 * 1024;
+export const MULTIPART_MAX_PART_COUNT = 10_000;
+export const MULTIPART_MAX_OBJECT_SIZE = 5 * 1024 * 1024 * 1024 * 1024;
+export const MULTIPART_DEFAULT_CONCURRENCY = 1;
+export const MULTIPART_MAX_CONCURRENCY = 32;
+
+const requireSafeNonNegativeInteger = (valueArg: number, nameArg: string): void => {
+  if (!Number.isSafeInteger(valueArg) || valueArg < 0) {
+    throw new TypeError(`${nameArg} must be a non-negative safe integer.`);
+  }
+};
+
+const normalizeRequestedPartSize = (requestedPartSizeArg: number | undefined): number => {
+  if (requestedPartSizeArg === undefined) {
+    return MULTIPART_MIN_PART_SIZE;
+  }
+  if (!Number.isSafeInteger(requestedPartSizeArg) || requestedPartSizeArg <= 0) {
+    throw new TypeError('requestedPartSize must be a positive safe integer.');
+  }
+  return Math.min(
+    MULTIPART_MAX_PART_SIZE,
+    Math.max(MULTIPART_MIN_PART_SIZE, requestedPartSizeArg),
+  );
+};
+
+const normalizeRequestedConcurrency = (
+  requestedConcurrencyArg: number | undefined,
+): number => {
+  if (requestedConcurrencyArg === undefined) {
+    return MULTIPART_DEFAULT_CONCURRENCY;
+  }
+  if (!Number.isSafeInteger(requestedConcurrencyArg) || requestedConcurrencyArg <= 0) {
+    throw new TypeError('requestedConcurrency must be a positive safe integer.');
+  }
+  return Math.min(MULTIPART_MAX_CONCURRENCY, requestedConcurrencyArg);
+};
+
+export const planMultipartUpload = (
+  optionsArg: IMultipartPlanningOptions,
+): TMultipartUploadPlan => {
+  requireSafeNonNegativeInteger(optionsArg.objectSize, 'objectSize');
+  if (optionsArg.objectSize > MULTIPART_MAX_OBJECT_SIZE) {
+    throw new RangeError(
+      `objectSize exceeds the ${MULTIPART_MAX_OBJECT_SIZE} byte multipart limit.`,
+    );
+  }
+
+  const requestedConcurrency = normalizeRequestedConcurrency(
+    optionsArg.requestedConcurrency,
+  );
+  if (optionsArg.objectSize === 0) {
+    return {
+      classification: 'zero',
+      objectSize: 0,
+      partSize: 0,
+      partCount: 0,
+      concurrency: requestedConcurrency,
+      maximumBufferedBytes: 0,
+    };
+  }
+
+  const preferredPartSize = normalizeRequestedPartSize(optionsArg.requestedPartSize);
+  const minimumPartSizeForCount = Math.ceil(
+    optionsArg.objectSize / MULTIPART_MAX_PART_COUNT,
+  );
+  const partSize = Math.max(preferredPartSize, minimumPartSizeForCount);
+  if (partSize > MULTIPART_MAX_PART_SIZE) {
+    throw new RangeError('The object cannot be represented by legal multipart parts.');
+  }
+
+  const partCount = Math.ceil(optionsArg.objectSize / partSize);
+  const concurrency = Math.min(requestedConcurrency, partCount);
+  return {
+    classification: 'multipart',
+    objectSize: optionsArg.objectSize,
+    partSize,
+    partCount,
+    concurrency,
+    maximumBufferedBytes: Math.min(
+      optionsArg.objectSize,
+      partSize * concurrency,
+    ) + Math.min(optionsArg.objectSize, partSize),
+  };
+};

package/ts/objectstorage/internal.multipart.stream.ts

@@ -0,0 +1,606 @@
+import {
+  MULTIPART_MAX_CONCURRENCY,
+  MULTIPART_MAX_OBJECT_SIZE,
+  MULTIPART_MAX_PART_COUNT,
+  MULTIPART_MAX_PART_SIZE,
+  MULTIPART_MIN_PART_SIZE,
+} from './internal.multipart.planner.js';
+import type {
+  IConsumeExactMultipartStreamOptions,
+  IMultipartStreamConsumptionResult,
+  IMultipartStreamFailureState,
+  TMultipartOwnedReaderCleanup,
+  TMultipartStreamFailureKind,
+  TMultipartUploadPlan,
+} from './internal.multipart.types.js';
+
+interface IPrimaryFailure {
+  kind: TMultipartStreamFailureKind;
+  cause?: unknown;
+}
+
+type TPartOutcome =
+  | {
+      token: symbol;
+      partNumber: number;
+      succeeded: true;
+    }
+  | {
+      token: symbol;
+      partNumber: number;
+      succeeded: false;
+      cause: unknown;
+    };
+
+interface IInFlightPart {
+  byteLength: number;
+  outcome: Promise<TPartOutcome>;
+}
+
+type TReadOutcome =
+  | {
+      type: 'read';
+      result: ReadableStreamReadResult<Uint8Array>;
+    }
+  | {
+      type: 'read-error';
+      cause: unknown;
+    };
+
+type TAbortOutcome = {
+  type: 'abort';
+  cause: unknown;
+};
+
+const failureMessages: Record<TMultipartStreamFailureKind, string> = {
+  aborted: 'Multipart stream consumption was aborted.',
+  'source-read': 'The multipart source stream failed while being read.',
+  'source-buffer-limit': 'The multipart source emitted a chunk larger than the buffer limit.',
+  'source-empty-chunk-limit':
+    'The multipart source emitted too many consecutive empty chunks.',
+  'source-short': 'The multipart source ended before the declared content length.',
+  'source-extra': 'The multipart source exceeded the declared content length.',
+  'part-callback': 'A multipart part callback failed.',
+};
+const failureSettlementMaximumTimeoutMs = 30_000;
+const maximumConsecutiveEmptyChunks = 1_024;
+
+export class MultipartStreamConsumptionError extends Error {
+  public readonly state: IMultipartStreamFailureState;
+  /**
+   * Non-rejecting settlement of every started part callback.
+   */
+  public readonly callbackSettlement: Promise<void>;
+  /**
+   * Non-rejecting settlement of callbacks and owned reader cancellation.
+   */
+  public readonly ownedSettlement: Promise<void>;
+
+  constructor(
+    stateArg: IMultipartStreamFailureState,
+    settlementsArg?: {
+      callbackSettlement: Promise<void>;
+      ownedSettlement: Promise<void>;
+    },
+  ) {
+    super(failureMessages[stateArg.kind], {
+      cause: stateArg.cause,
+    });
+    this.name = 'MultipartStreamConsumptionError';
+    this.state = stateArg;
+    this.callbackSettlement =
+      settlementsArg?.callbackSettlement ?? Promise.resolve();
+    this.ownedSettlement =
+      settlementsArg?.ownedSettlement ?? Promise.resolve();
+  }
+}
+
+const validatePlan = (planArg: TMultipartUploadPlan): void => {
+  if (
+    !Number.isSafeInteger(planArg.objectSize) ||
+    planArg.objectSize < 0 ||
+    planArg.objectSize > MULTIPART_MAX_OBJECT_SIZE
+  ) {
+    throw new TypeError('plan.objectSize is invalid.');
+  }
+  if (
+    !Number.isSafeInteger(planArg.concurrency) ||
+    planArg.concurrency < 1 ||
+    planArg.concurrency > MULTIPART_MAX_CONCURRENCY
+  ) {
+    throw new TypeError('plan.concurrency is invalid.');
+  }
+  if (planArg.classification === 'zero') {
+    if (
+      planArg.objectSize !== 0 ||
+      planArg.partSize !== 0 ||
+      planArg.partCount !== 0 ||
+      planArg.maximumBufferedBytes !== 0
+    ) {
+      throw new TypeError('The zero-byte multipart plan is inconsistent.');
+    }
+    return;
+  }
+
+  if (
+    planArg.objectSize === 0 ||
+    !Number.isSafeInteger(planArg.partSize) ||
+    planArg.partSize < MULTIPART_MIN_PART_SIZE ||
+    planArg.partSize > MULTIPART_MAX_PART_SIZE ||
+    planArg.partCount !== Math.ceil(planArg.objectSize / planArg.partSize) ||
+    planArg.partCount < 1 ||
+    planArg.partCount > MULTIPART_MAX_PART_COUNT ||
+    planArg.concurrency > planArg.partCount ||
+    planArg.maximumBufferedBytes !==
+      Math.min(planArg.objectSize, planArg.partSize * planArg.concurrency) +
+        Math.min(planArg.objectSize, planArg.partSize)
+  ) {
+    throw new TypeError('The multipart plan is inconsistent.');
+  }
+};
+
+export const consumeExactMultipartStream = async (
+  optionsArg: IConsumeExactMultipartStreamOptions,
+): Promise<IMultipartStreamConsumptionResult> => {
+  validatePlan(optionsArg.plan);
+  if (
+    !optionsArg.readableStream ||
+    typeof optionsArg.readableStream.getReader !== 'function'
+  ) {
+    throw new TypeError('readableStream must be a web ReadableStream.');
+  }
+  if (typeof optionsArg.onPart !== 'function') {
+    throw new TypeError('onPart must be a function.');
+  }
+  const failureSettlementTimeoutMs =
+    optionsArg.failureSettlementTimeoutMs ??
+    failureSettlementMaximumTimeoutMs;
+  if (
+    !Number.isSafeInteger(failureSettlementTimeoutMs) ||
+    failureSettlementTimeoutMs <= 0 ||
+    failureSettlementTimeoutMs > failureSettlementMaximumTimeoutMs
+  ) {
+    throw new TypeError(
+      `failureSettlementTimeoutMs must be a positive safe integer no greater than ${failureSettlementMaximumTimeoutMs}.`,
+    );
+  }
+
+  const reader = optionsArg.readableStream.getReader();
+  const callbackAbortController = new AbortController();
+  const inFlightParts = new Map<symbol, IInFlightPart>();
+  const completedPartNumbers = new Set<number>();
+  let inFlightBytes = 0;
+  let currentPartBytes = 0;
+  let pendingChunkBytes = 0;
+  let peakBufferedBytes = 0;
+  let observedBytes = 0;
+  let partCallbacksStarted = 0;
+  let sourceClosed = false;
+  let consecutiveEmptyChunks = 0;
+
+  let resolveAbort!: (outcomeArg: TAbortOutcome) => void;
+  const abortOutcome = new Promise<TAbortOutcome>((resolveArg) => {
+    resolveAbort = resolveArg;
+  });
+  const handleAbort = (): void => {
+    const cause = optionsArg.signal?.reason ?? new Error('Multipart stream aborted.');
+    callbackAbortController.abort(cause);
+    resolveAbort({
+      type: 'abort',
+      cause,
+    });
+  };
+  optionsArg.signal?.addEventListener('abort', handleAbort, { once: true });
+  if (optionsArg.signal?.aborted) {
+    handleAbort();
+  }
+
+  const processPartOutcome = (outcomeArg: TPartOutcome): void => {
+    const inFlightPart = inFlightParts.get(outcomeArg.token);
+    if (!inFlightPart) {
+      return;
+    }
+    inFlightParts.delete(outcomeArg.token);
+    inFlightBytes -= inFlightPart.byteLength;
+    if (outcomeArg.succeeded) {
+      completedPartNumbers.add(outcomeArg.partNumber);
+      return;
+    }
+    throw {
+      kind: 'part-callback',
+      cause: outcomeArg.cause,
+    } satisfies IPrimaryFailure;
+  };
+
+  const observeBufferedBytes = (): void => {
+    const bufferedBytes = inFlightBytes + currentPartBytes + pendingChunkBytes;
+    if (bufferedBytes > optionsArg.plan.maximumBufferedBytes) {
+      throw {
+        kind: 'source-buffer-limit',
+        cause: new RangeError('Multipart buffering exceeded its planned bound.'),
+      } satisfies IPrimaryFailure;
+    }
+    peakBufferedBytes = Math.max(peakBufferedBytes, bufferedBytes);
+  };
+
+  const waitForOnePart = async (): Promise<void> => {
+    const outcome = await Promise.race(
+      [
+        abortOutcome,
+        ...[...inFlightParts.values()].map(
+          (inFlightPartArg) => inFlightPartArg.outcome,
+        ),
+      ],
+    );
+    if (!('token' in outcome)) {
+      throw {
+        kind: 'aborted',
+        cause: outcome.cause,
+      } satisfies IPrimaryFailure;
+    }
+    processPartOutcome(outcome);
+  };
+
+  const waitForCapacity = async (): Promise<void> => {
+    while (inFlightParts.size >= optionsArg.plan.concurrency) {
+      await waitForOnePart();
+    }
+  };
+
+  const readWhileObservingCallbacks = async (): Promise<
+    ReadableStreamReadResult<Uint8Array>
+  > => {
+    const readOutcome: Promise<TReadOutcome> = reader.read().then(
+      (resultArg) => ({
+        type: 'read',
+        result: resultArg,
+      }),
+      (causeArg: unknown) => ({
+        type: 'read-error',
+        cause: causeArg,
+      }),
+    );
+
+    while (true) {
+      const outcome = await Promise.race([
+        readOutcome,
+        abortOutcome,
+        ...[...inFlightParts.values()].map(
+          (inFlightPartArg) => inFlightPartArg.outcome,
+        ),
+      ]);
+      if ('token' in outcome) {
+        processPartOutcome(outcome);
+        continue;
+      }
+      if (outcome.type === 'abort') {
+        throw {
+          kind: 'aborted',
+          cause: outcome.cause,
+        } satisfies IPrimaryFailure;
+      }
+      if (outcome.type === 'read-error') {
+        throw {
+          kind: 'source-read',
+          cause: outcome.cause,
+        } satisfies IPrimaryFailure;
+      }
+      return outcome.result;
+    }
+  };
+
+  const dispatchPart = (dataArg: Uint8Array, partNumberArg: number): void => {
+    const token = Symbol(`multipart-part-${partNumberArg}`);
+    partCallbacksStarted++;
+    currentPartBytes = 0;
+    inFlightBytes += dataArg.byteLength;
+    observeBufferedBytes();
+    const outcome = Promise.resolve()
+      .then(() =>
+        optionsArg.onPart({
+          partNumber: partNumberArg,
+          byteOffset: (partNumberArg - 1) * optionsArg.plan.partSize,
+          isLastPart: partNumberArg === optionsArg.plan.partCount,
+          data: dataArg,
+          signal: callbackAbortController.signal,
+        }),
+      )
+      .then<TPartOutcome, TPartOutcome>(
+        () => ({
+          token,
+          partNumber: partNumberArg,
+          succeeded: true,
+        }),
+        (causeArg: unknown) => ({
+          token,
+          partNumber: partNumberArg,
+          succeeded: false,
+          cause: causeArg,
+        }),
+      );
+    inFlightParts.set(token, {
+      byteLength: dataArg.byteLength,
+      outcome,
+    });
+  };
+
+  const acceptSourceChunk = (chunkArg: Uint8Array): boolean => {
+    if (chunkArg.byteLength > 0) {
+      consecutiveEmptyChunks = 0;
+      return true;
+    }
+    consecutiveEmptyChunks++;
+    if (consecutiveEmptyChunks > maximumConsecutiveEmptyChunks) {
+      throw {
+        kind: 'source-empty-chunk-limit',
+        cause: new RangeError(
+          `The multipart source emitted more than ${maximumConsecutiveEmptyChunks} consecutive empty chunks.`,
+        ),
+      } satisfies IPrimaryFailure;
+    }
+    return false;
+  };
+
+  try {
+    let pendingChunk: Uint8Array | undefined;
+    let pendingChunkOffset = 0;
+
+    for (
+      let partNumber = 1;
+      partNumber <= optionsArg.plan.partCount;
+      partNumber++
+    ) {
+      await waitForCapacity();
+      const partByteLength = Math.min(
+        optionsArg.plan.partSize,
+        optionsArg.plan.objectSize -
+          (partNumber - 1) * optionsArg.plan.partSize,
+      );
+      if (
+        inFlightBytes + partByteLength >
+        optionsArg.plan.maximumBufferedBytes
+      ) {
+        throw new Error('The multipart plan cannot honor its buffering bound.');
+      }
+      const partData = new Uint8Array(partByteLength);
+      currentPartBytes = partByteLength;
+      observeBufferedBytes();
+      let partOffset = 0;
+
+      while (partOffset < partByteLength) {
+        if (
+          pendingChunk &&
+          pendingChunkOffset < pendingChunk.byteLength
+        ) {
+          const copyByteLength = Math.min(
+            partByteLength - partOffset,
+            pendingChunk.byteLength - pendingChunkOffset,
+          );
+          partData.set(
+            pendingChunk.subarray(
+              pendingChunkOffset,
+              pendingChunkOffset + copyByteLength,
+            ),
+            partOffset,
+          );
+          pendingChunkOffset += copyByteLength;
+          partOffset += copyByteLength;
+          if (pendingChunkOffset === pendingChunk.byteLength) {
+            pendingChunk = undefined;
+            pendingChunkOffset = 0;
+            pendingChunkBytes = 0;
+          }
+          continue;
+        }
+
+        const readResult = await readWhileObservingCallbacks();
+        if (readResult.done) {
+          sourceClosed = true;
+          throw {
+            kind: 'source-short',
+          } satisfies IPrimaryFailure;
+        }
+        if (!(readResult.value instanceof Uint8Array)) {
+          throw {
+            kind: 'source-read',
+            cause: new TypeError('The multipart source emitted a non-Uint8Array chunk.'),
+          } satisfies IPrimaryFailure;
+        }
+        if (!acceptSourceChunk(readResult.value)) {
+          continue;
+        }
+        observedBytes += readResult.value.byteLength;
+        if (observedBytes > optionsArg.plan.objectSize) {
+          throw {
+            kind: 'source-extra',
+          } satisfies IPrimaryFailure;
+        }
+        if (readResult.value.byteLength > optionsArg.plan.partSize) {
+          throw {
+            kind: 'source-buffer-limit',
+            cause: new RangeError(
+              `Source chunks must not exceed ${optionsArg.plan.partSize} bytes.`,
+            ),
+          } satisfies IPrimaryFailure;
+        }
+        pendingChunk = readResult.value;
+        pendingChunkBytes = readResult.value.byteLength;
+        observeBufferedBytes();
+      }
+
+      dispatchPart(partData, partNumber);
+    }
+
+    while (!sourceClosed) {
+      const readResult = await readWhileObservingCallbacks();
+      if (readResult.done) {
+        sourceClosed = true;
+        break;
+      }
+      if (!(readResult.value instanceof Uint8Array)) {
+        throw {
+          kind: 'source-read',
+          cause: new TypeError('The multipart source emitted a non-Uint8Array chunk.'),
+        } satisfies IPrimaryFailure;
+      }
+      if (!acceptSourceChunk(readResult.value)) {
+        continue;
+      }
+      observedBytes += readResult.value.byteLength;
+      throw {
+        kind: 'source-extra',
+      } satisfies IPrimaryFailure;
+    }
+
+    while (inFlightParts.size > 0) {
+      await waitForOnePart();
+    }
+    if (optionsArg.signal?.aborted) {
+      throw {
+        kind: 'aborted',
+        cause: optionsArg.signal.reason,
+      } satisfies IPrimaryFailure;
+    }
+
+    return {
+      completionAuthorized: true,
+      bytesConsumed: observedBytes,
+      partsConsumed: completedPartNumbers.size,
+      peakBufferedBytes,
+      maximumBufferedBytes: optionsArg.plan.maximumBufferedBytes,
+    };
+  } catch (error) {
+    const failure =
+      typeof error === 'object' &&
+      error !== null &&
+      'kind' in error &&
+      typeof error.kind === 'string'
+        ? (error as IPrimaryFailure)
+        : ({
+            kind: 'source-read',
+            cause: error,
+          } satisfies IPrimaryFailure);
+    callbackAbortController.abort(failure.cause ?? new Error(failureMessages[failure.kind]));
+
+    const readerCleanupPromise: Promise<TMultipartOwnedReaderCleanup> = sourceClosed
+      ? Promise.resolve({
+          status: 'closed',
+        })
+      : Promise.resolve()
+          .then(() => reader.cancel(failure.cause))
+          .then<TMultipartOwnedReaderCleanup, TMultipartOwnedReaderCleanup>(
+            () => ({
+              status: 'cancelled',
+            }),
+            (causeArg: unknown) => ({
+              status: 'cancel-failed',
+              cause: causeArg,
+            }),
+          );
+
+    let callbacksSettled = false;
+    const callbackSettlement = Promise.all(
+      [...inFlightParts.values()].map(
+        (inFlightPartArg) => inFlightPartArg.outcome,
+      ),
+    )
+      .then((remainingOutcomesArg) => {
+        for (const outcome of remainingOutcomesArg) {
+          if (outcome.succeeded) {
+            completedPartNumbers.add(outcome.partNumber);
+          }
+        }
+        inFlightParts.clear();
+        inFlightBytes = 0;
+      })
+      .then(
+        () => undefined,
+        () => undefined,
+      );
+    const observedCallbackSettlement = callbackSettlement.then(() => {
+      callbacksSettled = true;
+    });
+    let readerCleanup: TMultipartOwnedReaderCleanup | undefined;
+    let readerCleanupSettled = false;
+    const observedReaderCleanup = readerCleanupPromise.then(
+      (readerCleanupArg) => {
+        readerCleanup = readerCleanupArg;
+        readerCleanupSettled = true;
+      },
+      () => {
+        readerCleanup = {
+          status: 'cancel-failed',
+          cause: new Error('Reader cancellation rejected unexpectedly.'),
+        };
+        readerCleanupSettled = true;
+      },
+    );
+    const ownedSettlement = Promise.allSettled([
+      observedCallbackSettlement,
+      observedReaderCleanup,
+    ]).then(() => undefined);
+    const settlementDeadlineToken = Symbol(
+      'multipart-failure-settlement-deadline',
+    );
+    let resolveSettlementDeadline!: (
+      valueArg: typeof settlementDeadlineToken,
+    ) => void;
+    const settlementDeadlineOutcome = new Promise<
+      typeof settlementDeadlineToken
+    >((resolveArg) => {
+      resolveSettlementDeadline = resolveArg;
+    });
+    const settlementDeadline = setTimeout(() => {
+      resolveSettlementDeadline(settlementDeadlineToken);
+    }, failureSettlementTimeoutMs);
+    await Promise.race([ownedSettlement, settlementDeadlineOutcome]);
+    clearTimeout(settlementDeadline);
+    const retry =
+      partCallbacksStarted === 0
+        ? ({
+            disposition: 'safe-to-retry',
+            requiresMultipartAbort: false,
+          } as const)
+        : ({
+            disposition: 'abort-multipart-first',
+            requiresMultipartAbort: true,
+          } as const);
+
+    throw new MultipartStreamConsumptionError(
+      {
+        completionAuthorized: false,
+        kind: failure.kind,
+        cause: failure.cause,
+        expectedBytes: optionsArg.plan.objectSize,
+        observedBytes,
+        partCallbacksStarted,
+        completedPartNumbers: [...completedPartNumbers].sort(
+          (leftArg, rightArg) => {
+            return leftArg - rightArg;
+          },
+        ),
+        cleanup: {
+          callbacks: callbacksSettled ? 'settled' : 'detached',
+          reader:
+            readerCleanupSettled && readerCleanup
+              ? readerCleanup
+              : {
+                  status: 'cancel-detached',
+                },
+        },
+        retry,
+      },
+      {
+        callbackSettlement,
+        ownedSettlement,
+      },
+    );
+  } finally {
+    optionsArg.signal?.removeEventListener('abort', handleAbort);
+    try {
+      reader.releaseLock();
+    } catch {
+      // Detached reader cancellation retains ownership until it settles.
+    }
+  }
+};