@od-oneapp/storage 2026.1.1301

package/src/multipart.ts

@@ -0,0 +1,497 @@
+/**
+ * @fileoverview Unified Multipart Upload Manager
+ *
+ * Provides a consistent interface for multipart uploads across all storage providers.
+ * Handles provider-specific differences and provides retry logic, progress tracking,
+ * and error recovery.
+ *
+ * Features:
+ * - Automatic part size calculation
+ * - Concurrent part uploads
+ * - Progress tracking
+ * - Resume support
+ * - Error recovery
+ *
+ * @module @repo/storage/multipart
+ */
+
+import { logWarn } from '@repo/shared/logs';
+
+import {
+  type MultipartUploadResult as BaseMultipartUploadResult,
+  type MultipartUploadOptions,
+  type StorageProvider,
+  type UploadOptions,
+  type UploadProgress,
+} from '../types';
+
+import { MULTIPART_THRESHOLDS, PART_SIZES, STORAGE_CONSTANTS } from './constants';
+
+export interface MultipartUploadState {
+  uploadId: string;
+  key: string;
+  provider: string;
+  parts: Array<{
+    partNumber: number;
+    etag?: string;
+    size: number;
+    uploaded: boolean;
+  }>;
+  totalSize: number;
+  uploadedSize: number;
+  completed: boolean;
+  aborted: boolean;
+  error?: string;
+}
+
+/**
+ * Extended multipart upload result with detailed part information
+ * Extends the base MultipartUploadResult from types.ts
+ */
+export interface MultipartUploadResult extends BaseMultipartUploadResult {
+  parts: Array<{ partNumber: number; etag: string; size: number }>;
+  totalParts: number;
+}
+
+export class MultipartUploadManager {
+  private provider: StorageProvider;
+  private state: MultipartUploadState | null = null;
+  private abortController: AbortController | null = null;
+
+  constructor(provider: StorageProvider) {
+    this.provider = provider;
+  }
+
+  /**
+   * Check if provider supports multipart uploads
+   */
+  supportsMultipart(): boolean {
+    const capabilities = this.provider.getCapabilities?.();
+    return capabilities?.multipart ?? false;
+  }
+
+  /**
+   * Create a new multipart upload
+   *
+   * @param key - Storage key
+   * @param totalSize - Total file size in bytes
+   * @param options - Upload options
+   * @returns Upload state
+   */
+  async createUpload(
+    key: string,
+    totalSize: number,
+    options: MultipartUploadOptions = {},
+  ): Promise<MultipartUploadState> {
+    if (!this.supportsMultipart()) {
+      throw new Error('Provider does not support multipart uploads');
+    }
+
+    if (this.state && !this.state.completed && !this.state.aborted) {
+      throw new Error('Upload already in progress. Abort current upload first.');
+    }
+
+    // Calculate optimal part size
+    const partSize = this.calculatePartSize(totalSize, options.partSize);
+    const totalParts = Math.ceil(totalSize / partSize);
+
+    // Create multipart upload
+    const uploadOptions: UploadOptions = {
+      onProgress: options.onProgress
+        ? (progress: UploadProgress) => {
+            options.onProgress?.({
+              key: progress.key,
+              loaded: progress.loaded,
+              total: progress.total,
+              part: progress.part ?? 0,
+              percentage: progress.percentage ?? 0,
+            });
+          }
+        : undefined,
+    };
+    const result = await this.provider.createMultipartUpload?.(key, uploadOptions);
+
+    if (!result) {
+      throw new Error('Failed to create multipart upload');
+    }
+
+    // Initialize state
+    this.state = {
+      uploadId: result.uploadId,
+      key: result.key,
+      provider: this.getProviderName(),
+      parts: Array.from({ length: totalParts }, (_, i) => ({
+        partNumber: i + 1,
+        size: i === totalParts - 1 ? totalSize - i * partSize : partSize,
+        uploaded: false,
+      })),
+      totalSize,
+      uploadedSize: 0,
+      completed: false,
+      aborted: false,
+    };
+
+    this.abortController = new AbortController();
+
+    return this.state;
+  }
+
+  /**
+   * Upload a part
+   *
+   * @param partNumber - Part number (1-based)
+   * @param data - Part data
+   * @param options - Upload options
+   * @returns Upload result
+   */
+  async uploadPart(
+    partNumber: number,
+    data: ArrayBuffer | Blob | Buffer,
+    options: UploadOptions = {},
+  ): Promise<{ etag: string; partNumber: number; size: number }> {
+    if (!this.state) {
+      throw new Error('No active upload. Call createUpload first.');
+    }
+
+    if (this.state.completed || this.state.aborted) {
+      throw new Error('Upload is completed or aborted');
+    }
+
+    const part = this.state.parts.find(p => p.partNumber === partNumber);
+    if (!part) {
+      throw new Error(`Part ${partNumber} not found`);
+    }
+
+    if (part.uploaded) {
+      throw new Error(`Part ${partNumber} already uploaded`);
+    }
+
+    // Check abort signal
+    if (this.abortController?.signal.aborted) {
+      throw new Error('Upload aborted');
+    }
+
+    try {
+      // Upload part with retry logic
+      const result = await this.uploadPartWithRetry(partNumber, data, options);
+
+      // Update state
+      part.etag = result.etag;
+      part.uploaded = true;
+      this.state.uploadedSize += part.size;
+
+      // Report progress
+      if (options.onProgress) {
+        options.onProgress({
+          key: this.state.key,
+          loaded: this.state.uploadedSize,
+          total: this.state.totalSize,
+          part: partNumber,
+          percentage: (this.state.uploadedSize / this.state.totalSize) * 100,
+        });
+      }
+
+      return result;
+    } catch (error) {
+      this.state.error = error instanceof Error ? error.message : String(error);
+      throw error;
+    }
+  }
+
+  /**
+   * Complete the multipart upload
+   *
+   * @returns Final upload result
+   */
+  async completeUpload(): Promise<MultipartUploadResult> {
+    if (!this.state) {
+      throw new Error('No active upload. Call createUpload first.');
+    }
+
+    if (this.state.completed) {
+      throw new Error('Upload already completed');
+    }
+
+    if (this.state.aborted) {
+      throw new Error('Upload was aborted');
+    }
+
+    // Check if all parts are uploaded
+    const unuploadedParts = this.state.parts.filter(p => !p.uploaded);
+    if (unuploadedParts.length > 0) {
+      throw new Error(`Upload incomplete: ${unuploadedParts.length} parts not uploaded`);
+    }
+
+    try {
+      // Complete multipart upload
+      const parts = this.state.parts
+        .filter(p => p.etag)
+        .map(p => ({ partNumber: p.partNumber, etag: p.etag ?? '' }))
+        .filter(p => p.etag !== '');
+
+      const result = await this.provider.completeMultipartUpload?.(this.state.uploadId, parts);
+
+      if (!result) {
+        throw new Error('Failed to complete multipart upload');
+      }
+
+      // Update state
+      this.state.completed = true;
+
+      return {
+        ...result,
+        key: result.key ?? this.state.key,
+        uploadId: this.state.uploadId,
+        parts: this.state.parts
+          .filter(p => p.etag)
+          .map(p => ({
+            partNumber: p.partNumber,
+            etag: p.etag ?? '',
+            size: p.size,
+          })),
+        totalParts: this.state.parts.length,
+      };
+    } catch (error) {
+      this.state.error = error instanceof Error ? error.message : String(error);
+      throw error;
+    }
+  }
+
+  /**
+   * Abort the multipart upload
+   */
+  async abortUpload(): Promise<void> {
+    if (!this.state) {
+      return;
+    }
+
+    if (this.state.completed) {
+      return;
+    }
+
+    // Signal abort
+    this.abortController?.abort();
+    this.state.aborted = true;
+
+    try {
+      // Abort on provider
+      await this.provider.abortMultipartUpload?.(this.state.uploadId);
+    } catch (error) {
+      // Log error but don't throw - we want to clean up state
+
+      // Use structured logging instead of console.warn
+      logWarn('Failed to abort upload on provider', {
+        error: error instanceof Error ? error.message : String(error),
+        uploadId: this.state.uploadId,
+        key: this.state.key,
+      });
+    }
+
+    // Reset state
+    this.state = null;
+    this.abortController = null;
+  }
+
+  /**
+   * Get current upload state
+   */
+  getState(): MultipartUploadState | null {
+    return this.state;
+  }
+
+  /**
+   * Resume upload from state (for recovery)
+   *
+   * @param state - Previous upload state
+   */
+  async resumeUpload(state: MultipartUploadState): Promise<void> {
+    if (this.state && !this.state.completed && !this.state.aborted) {
+      throw new Error('Upload already in progress');
+    }
+
+    this.state = { ...state };
+    this.abortController = new AbortController();
+  }
+
+  /**
+   * Upload file in chunks automatically
+   *
+   * @param key - Storage key
+   * @param data - File data
+   * @param options - Upload options
+   * @returns Upload result
+   */
+  async uploadFile(
+    key: string,
+    data: ArrayBuffer | Blob | Buffer,
+    options: MultipartUploadOptions = {},
+  ): Promise<MultipartUploadResult> {
+    const totalSize =
+      data instanceof ArrayBuffer
+        ? data.byteLength
+        : data instanceof Buffer
+          ? data.length
+          : (data as Blob).size;
+
+    // Create upload
+    await this.createUpload(key, totalSize, options);
+
+    const partSize = this.calculatePartSize(totalSize, options.partSize);
+    const totalParts = Math.ceil(totalSize / partSize);
+
+    // Upload parts
+    for (let i = 0; i < totalParts; i++) {
+      const start = i * partSize;
+      const end = Math.min(start + partSize, totalSize);
+      const partData = data.slice(start, end);
+
+      await this.uploadPart(i + 1, partData, {
+        onProgress: options.onProgress
+          ? (progress: UploadProgress) => {
+              options.onProgress?.({
+                key: progress.key,
+                loaded: progress.loaded,
+                total: progress.total,
+                part: progress.part ?? 0,
+                percentage: progress.percentage ?? 0,
+              });
+            }
+          : undefined,
+      });
+    }
+
+    // Complete upload
+    return await this.completeUpload();
+  }
+
+  private async uploadPartWithRetry(
+    partNumber: number,
+    data: ArrayBuffer | Blob | Buffer,
+    options: UploadOptions,
+    maxRetries: number = 3,
+  ): Promise<{ etag: string; partNumber: number; size: number }> {
+    let lastError: Error | null = null;
+
+    for (let attempt = 1; attempt <= maxRetries; attempt++) {
+      try {
+        if (!this.state?.uploadId) {
+          throw new Error('Upload ID not initialized');
+        }
+
+        const result = await this.provider.uploadPart?.(this.state.uploadId, partNumber, data, {
+          ...options,
+          abortSignal: this.abortController?.signal,
+        });
+
+        if (!result) {
+          throw new Error('Failed to upload part');
+        }
+
+        return {
+          etag: result.etag ?? '',
+          partNumber: result.partNumber ?? partNumber,
+          size:
+            data instanceof ArrayBuffer
+              ? data.byteLength
+              : data instanceof Buffer
+                ? data.length
+                : (data as Blob).size,
+        };
+      } catch (error) {
+        lastError = error instanceof Error ? error : new Error(String(error));
+
+        // Don't retry on abort
+        if (this.abortController?.signal.aborted) {
+          throw lastError;
+        }
+
+        // Don't retry on validation errors
+        if (lastError.message.includes('validation') || lastError.message.includes('invalid')) {
+          throw lastError;
+        }
+
+        // Wait before retry (exponential backoff)
+        if (attempt < maxRetries) {
+          await new Promise(resolve =>
+            setTimeout(resolve, Math.pow(2, attempt) * STORAGE_CONSTANTS.RETRY_BASE_DELAY_MS),
+          );
+        }
+      }
+    }
+
+    throw lastError ?? new Error('Upload failed after retries');
+  }
+
+  private calculatePartSize(totalSize: number, userPartSize?: number): number {
+    if (userPartSize) {
+      return Math.min(userPartSize, totalSize);
+    }
+
+    // Use constants for part size thresholds
+    if (totalSize < MULTIPART_THRESHOLDS.SMALL_FILE) {
+      return PART_SIZES.SMALL;
+    } else if (totalSize < MULTIPART_THRESHOLDS.MEDIUM_FILE) {
+      return PART_SIZES.MEDIUM;
+    } else {
+      // Scale part size to ensure we never exceed 10,000 parts (S3/R2 limit)
+      const maxParts = 9999; // Stay safely under 10,000 limit
+      const minPartSize = Math.ceil(totalSize / maxParts);
+      return Math.max(PART_SIZES.LARGE, minPartSize);
+    }
+  }
+
+  private getProviderName(): string {
+    return this.provider.constructor.name;
+  }
+}
+
+/**
+ * Create a multipart upload manager for a provider
+ *
+ * @param provider - Storage provider
+ * @returns Multipart upload manager
+ */
+export function createMultipartUploadManager(provider: StorageProvider): MultipartUploadManager {
+  if (!hasMultipartSupport(provider)) {
+    throw new Error('The provided storage provider does not support multipart uploads');
+  }
+  return new MultipartUploadManager(provider);
+}
+
+/**
+ * Determines whether the given storage provider supports multipart uploads.
+ *
+ * @param provider - The storage provider to check
+ * @returns `true` if the provider supports multipart uploads, `false` otherwise.
+ */
+export function hasMultipartSupport(provider: StorageProvider): boolean {
+  const capabilities = provider.getCapabilities?.();
+  return capabilities?.multipart ?? false;
+}
+
+/**
+ * Selects an appropriate multipart upload part size for a file.
+ *
+ * If `userPartSize` is provided, it will be capped at the file size; otherwise the part size
+ * is chosen from configured thresholds for small, medium, or large files.
+ *
+ * @param fileSize - File size in bytes
+ * @param userPartSize - Optional user-specified part size in bytes; capped to `fileSize` when provided
+ * @returns The selected part size in bytes
+ */
+export function getOptimalPartSize(fileSize: number, userPartSize?: number): number {
+  if (userPartSize) {
+    return Math.min(userPartSize, fileSize);
+  }
+
+  if (fileSize < MULTIPART_THRESHOLDS.SMALL_FILE) {
+    return PART_SIZES.SMALL;
+  } else if (fileSize < MULTIPART_THRESHOLDS.MEDIUM_FILE) {
+    return PART_SIZES.MEDIUM;
+  } else {
+    // Scale part size to ensure we never exceed 10,000 parts (S3/R2 limit)
+    const maxParts = 9999; // Stay safely under 10,000 limit
+    const minPartSize = Math.ceil(fileSize / maxParts);
+    return Math.max(PART_SIZES.LARGE, minPartSize);
+  }
+}

package/src/retry-utils.test.ts

@@ -0,0 +1,118 @@
+/**
+ * @fileoverview retry-utils.test.ts
+ */
+
+import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';
+
+import { STORAGE_CONSTANTS } from './constants';
+import { withRetry } from './retry-utils';
+import { StorageError, StorageErrorCode } from './validation';
+
+describe('withRetry', () => {
+  beforeEach(() => {
+    vi.useFakeTimers();
+  });
+
+  afterEach(() => {
+    vi.useRealTimers();
+    vi.restoreAllMocks();
+  });
+
+  it('returns immediately when the operation succeeds on the first attempt', async () => {
+    const operation = vi.fn().mockResolvedValue('success');
+
+    await expect(withRetry(operation)).resolves.toBe('success');
+    expect(operation).toHaveBeenCalledTimes(1);
+  });
+
+  it('retries when the operation throws a retryable error and eventually succeeds', async () => {
+    const retryableError = new StorageError(
+      'temporary timeout',
+      StorageErrorCode.TIMEOUT,
+      undefined,
+      true,
+    );
+    const operation = vi
+      .fn<() => Promise<string>>()
+      .mockRejectedValueOnce(retryableError)
+      .mockResolvedValueOnce('eventual-success');
+
+    const resultPromise = withRetry(operation, 3);
+
+    await vi.runAllTimersAsync();
+
+    await expect(resultPromise).resolves.toBe('eventual-success');
+    expect(operation).toHaveBeenCalledTimes(2);
+  });
+
+  it('waits using exponential backoff between retry attempts', async () => {
+    const retryableError = new StorageError(
+      'temporary failure',
+      StorageErrorCode.NETWORK_ERROR,
+      undefined,
+      true,
+    );
+    const operation = vi
+      .fn<() => Promise<string>>()
+      .mockRejectedValueOnce(retryableError)
+      .mockRejectedValueOnce(retryableError)
+      .mockResolvedValueOnce('done');
+
+    const setTimeoutSpy = vi.spyOn(global, 'setTimeout');
+
+    const resultPromise = withRetry(operation, 3);
+
+    // Execute pending timers sequentially to inspect delay values
+    await vi.advanceTimersByTimeAsync(STORAGE_CONSTANTS.RETRY_BASE_DELAY_MS);
+    await vi.advanceTimersByTimeAsync(STORAGE_CONSTANTS.RETRY_BASE_DELAY_MS * 2);
+
+    await expect(resultPromise).resolves.toBe('done');
+    expect(operation).toHaveBeenCalledTimes(3);
+    expect(setTimeoutSpy).toHaveBeenNthCalledWith(
+      1,
+      expect.any(Function),
+      STORAGE_CONSTANTS.RETRY_BASE_DELAY_MS,
+    );
+    expect(setTimeoutSpy).toHaveBeenNthCalledWith(
+      2,
+      expect.any(Function),
+      STORAGE_CONSTANTS.RETRY_BASE_DELAY_MS * 2,
+    );
+  });
+
+  it('throws immediately for non-retryable errors', async () => {
+    const fatalError = new Error('fatal error');
+    const operation = vi.fn().mockRejectedValue(fatalError);
+
+    await expect(withRetry(operation, 5)).rejects.toBe(fatalError);
+    expect(operation).toHaveBeenCalledTimes(1);
+  });
+
+  it('throws when all retry attempts fail', async () => {
+    const retryableError = new StorageError(
+      'still failing',
+      StorageErrorCode.UPLOAD_FAILED,
+      undefined,
+      true,
+    );
+    const operation = vi.fn().mockRejectedValue(retryableError);
+
+    // Start the retry operation and attach the catch handler immediately
+    const promise = withRetry(operation, 2).catch(e => e);
+
+    // Run timers to complete all retry attempts
+    await vi.runAllTimersAsync();
+
+    // The promise should resolve to the error (since we caught it above)
+    const error = await promise;
+    expect(error).toStrictEqual(retryableError);
+    expect(operation).toHaveBeenCalledTimes(2);
+  });
+
+  it('validates the maxRetries parameter', async () => {
+    const operation = vi.fn().mockResolvedValue('anything');
+
+    await expect(withRetry(operation, 0)).rejects.toThrowError('maxRetries must be at least 1');
+    expect(operation).not.toHaveBeenCalled();
+  });
+});

package/src/retry-utils.ts

@@ -0,0 +1,59 @@
+/**
+ * @fileoverview Retry utility for AWS SDK operations
+ *
+ * Wraps AWS SDK calls with retry logic for transient failures.
+ * Provides exponential backoff and retryable error detection.
+ *
+ * @module @repo/storage/retry-utils
+ */
+
+import { STORAGE_CONSTANTS } from './constants';
+import { isRetryableError } from './validation';
+
+/**
+ * Run an operation with retry and exponential backoff for retryable errors.
+ *
+ * Retries the provided operation up to `maxRetries` attempts. Between attempts it waits using
+ * exponential backoff: delay = 2^attempt * RETRY_BASE_DELAY_MS. If a non-retryable error is
+ * encountered or all attempts are exhausted, the last error is thrown.
+ *
+ * @param operation - Function that performs the operation and returns a promise of the result
+ * @param maxRetries - Maximum number of attempts (initial attempt counts as one)
+ * @returns The resolved value from a successful operation
+ * @throws The last encountered error when a non-retryable error occurs or all retries fail
+ */
+export async function withRetry<T>(
+  operation: () => Promise<T>,
+  maxRetries: number = STORAGE_CONSTANTS.DEFAULT_MAX_RETRIES,
+): Promise<T> {
+  // Validate maxRetries up-front
+  if (maxRetries < 1) {
+    throw new Error('maxRetries must be at least 1');
+  }
+
+  let lastError: Error | null = null;
+
+  for (let attempt = 1; attempt <= maxRetries; attempt++) {
+    try {
+      return await operation();
+    } catch (error) {
+      lastError = error instanceof Error ? error : new Error(String(error));
+
+      // Don't retry if error is not retryable
+      if (!isRetryableError(error)) {
+        throw lastError;
+      }
+
+      // Don't retry on last attempt
+      if (attempt === maxRetries) {
+        throw lastError;
+      }
+
+      // Exponential backoff: first retry uses base delay (2^0 * base)
+      const delay = Math.pow(2, attempt - 1) * STORAGE_CONSTANTS.RETRY_BASE_DELAY_MS;
+      await new Promise(resolve => setTimeout(resolve, delay));
+    }
+  }
+
+  throw lastError ?? new Error('Operation failed after retries');
+}