@uploadista/data-store-s3 0.0.3

package/src/services/s3-client.service.ts

@@ -0,0 +1,359 @@
+import type AWS from "@aws-sdk/client-s3";
+import type { S3ClientConfig } from "@aws-sdk/client-s3";
+import { NoSuchKey, NotFound, S3 } from "@aws-sdk/client-s3";
+import type { UploadistaError } from "@uploadista/core/errors";
+import { withS3ApiMetrics } from "@uploadista/observability";
+import { Context, Effect, Layer } from "effect";
+import type { MultipartUploadInfo, S3OperationContext } from "../types";
+import {
+  handleS3Error,
+  handleS3NotFoundError,
+  partKey,
+  toReadableStream,
+} from "../utils";
+
+export class S3ClientService extends Context.Tag("S3ClientService")<
+  S3ClientService,
+  {
+    readonly bucket: string;
+
+    // Basic S3 operations
+    readonly getObject: (
+      key: string,
+    ) => Effect.Effect<ReadableStream, UploadistaError>;
+    readonly headObject: (
+      key: string,
+    ) => Effect.Effect<number | undefined, UploadistaError>;
+    readonly putObject: (
+      key: string,
+      body: Uint8Array,
+    ) => Effect.Effect<string, UploadistaError>;
+    readonly deleteObject: (
+      key: string,
+    ) => Effect.Effect<void, UploadistaError>;
+    readonly deleteObjects: (
+      keys: string[],
+    ) => Effect.Effect<AWS.DeleteObjectsCommandOutput, UploadistaError>;
+
+    // Multipart upload operations
+    readonly createMultipartUpload: (
+      context: S3OperationContext,
+    ) => Effect.Effect<MultipartUploadInfo, UploadistaError>;
+    readonly uploadPart: (
+      context: S3OperationContext & { partNumber: number; data: Uint8Array },
+    ) => Effect.Effect<string, UploadistaError>;
+    readonly completeMultipartUpload: (
+      context: S3OperationContext,
+      parts: Array<AWS.Part>,
+    ) => Effect.Effect<string | undefined, UploadistaError>;
+    readonly abortMultipartUpload: (
+      context: S3OperationContext,
+    ) => Effect.Effect<void, UploadistaError>;
+    readonly listParts: (
+      context: S3OperationContext & { partNumberMarker?: string },
+    ) => Effect.Effect<
+      {
+        parts: AWS.Part[];
+        isTruncated: boolean;
+        nextPartNumberMarker?: string;
+      },
+      UploadistaError
+    >;
+    readonly listMultipartUploads: (
+      keyMarker?: string,
+      uploadIdMarker?: string,
+    ) => Effect.Effect<AWS.ListMultipartUploadsCommandOutput, UploadistaError>;
+
+    // Incomplete part operations
+    readonly getIncompletePart: (
+      id: string,
+    ) => Effect.Effect<ReadableStream | undefined, UploadistaError>;
+    readonly getIncompletePartSize: (
+      id: string,
+    ) => Effect.Effect<number | undefined, UploadistaError>;
+    readonly putIncompletePart: (
+      id: string,
+      data: Uint8Array,
+    ) => Effect.Effect<string, UploadistaError>;
+    readonly deleteIncompletePart: (
+      id: string,
+    ) => Effect.Effect<void, UploadistaError>;
+  }
+>() {}
+
+export const makeS3ClientService = (
+  s3ClientConfig: S3ClientConfig,
+  bucket: string,
+) => {
+  const s3Client = new S3(s3ClientConfig);
+  const getObject = (key: string) =>
+    Effect.tryPromise({
+      try: async () => {
+        const data = await s3Client.getObject({
+          Bucket: bucket,
+          Key: key,
+        });
+        return toReadableStream(data.Body);
+      },
+      catch: (error) => handleS3Error("getObject", error, { key, bucket }),
+    });
+
+  const headObject = (key: string) =>
+    Effect.tryPromise({
+      try: async () => {
+        try {
+          const data = await s3Client.headObject({
+            Bucket: bucket,
+            Key: key,
+          });
+          return data.ContentLength;
+        } catch (error) {
+          if (error instanceof NotFound) {
+            return undefined;
+          }
+          throw error;
+        }
+      },
+      catch: (error) => handleS3Error("headObject", error, { key, bucket }),
+    });
+
+  const putObject = (key: string, body: Uint8Array) =>
+    Effect.tryPromise({
+      try: async () => {
+        const response = await s3Client.putObject({
+          Bucket: bucket,
+          Key: key,
+          Body: body,
+        });
+        return response.ETag || "";
+      },
+      catch: (error) =>
+        handleS3Error("putObject", error, { key, bucket, size: body.length }),
+    });
+
+  const deleteObject = (key: string) =>
+    Effect.tryPromise({
+      try: async () => {
+        await s3Client.deleteObject({
+          Bucket: bucket,
+          Key: key,
+        });
+      },
+      catch: (error) => handleS3Error("deleteObject", error, { key, bucket }),
+    });
+
+  const deleteObjects = (keys: string[]) =>
+    Effect.tryPromise({
+      try: () =>
+        s3Client.deleteObjects({
+          Bucket: bucket,
+          Delete: {
+            Objects: keys.map((key) => ({ Key: key })),
+          },
+        }),
+      catch: (error) =>
+        handleS3Error("deleteObjects", error, { keys: keys.length, bucket }),
+    });
+
+  const createMultipartUpload = (context: S3OperationContext) =>
+    withS3ApiMetrics(
+      "createMultipartUpload",
+      Effect.tryPromise({
+        try: async () => {
+          const request: AWS.CreateMultipartUploadCommandInput = {
+            Bucket: context.bucket,
+            Key: context.key,
+          };
+
+          if (context.contentType) {
+            request.ContentType = context.contentType;
+          }
+
+          if (context.cacheControl) {
+            request.CacheControl = context.cacheControl;
+          }
+
+          const res = await s3Client.createMultipartUpload(request);
+
+          if (!res.UploadId) {
+            throw new Error("Upload ID is undefined");
+          }
+          if (!res.Key) {
+            throw new Error("Key is undefined");
+          }
+
+          return {
+            uploadId: res.UploadId,
+            bucket: context.bucket,
+            key: res.Key,
+          };
+        },
+        catch: (error) =>
+          handleS3Error("createMultipartUpload", error, context),
+      }),
+    );
+
+  const uploadPart = (
+    context: S3OperationContext & { partNumber: number; data: Uint8Array },
+  ) =>
+    withS3ApiMetrics(
+      "uploadPart",
+      Effect.tryPromise({
+        try: () =>
+          s3Client.uploadPart({
+            Bucket: context.bucket,
+            Key: context.key,
+            UploadId: context.uploadId,
+            PartNumber: context.partNumber,
+            Body: context.data,
+          }),
+        catch: (error) =>
+          handleS3Error("uploadPart", error, {
+            upload_id: context.key,
+            part_number: context.partNumber,
+            part_size: context.data.length,
+            s3_bucket: context.bucket,
+          }),
+      }).pipe(Effect.map((response) => response.ETag as string)),
+    );
+
+  const completeMultipartUpload = (
+    context: S3OperationContext,
+    parts: Array<AWS.Part>,
+  ) =>
+    withS3ApiMetrics(
+      "completeMultipartUpload",
+      Effect.tryPromise({
+        try: () =>
+          s3Client
+            .completeMultipartUpload({
+              Bucket: context.bucket,
+              Key: context.key,
+              UploadId: context.uploadId,
+              MultipartUpload: {
+                Parts: parts.map((part) => ({
+                  ETag: part.ETag,
+                  PartNumber: part.PartNumber,
+                })),
+              },
+            })
+            .then((response) => response.Location),
+        catch: (error) =>
+          handleS3Error("completeMultipartUpload", error, {
+            upload_id: context.key,
+            parts_count: parts.length,
+            s3_bucket: context.bucket,
+          }),
+      }),
+    );
+
+  const abortMultipartUpload = (context: S3OperationContext) =>
+    Effect.tryPromise({
+      try: async () => {
+        await s3Client.abortMultipartUpload({
+          Bucket: context.bucket,
+          Key: context.key,
+          UploadId: context.uploadId,
+        });
+      },
+      catch: (error) =>
+        handleS3NotFoundError("abortMultipartUpload", error, {
+          upload_id: context.key,
+          s3_bucket: context.bucket,
+        }),
+    });
+
+  const listParts = (
+    context: S3OperationContext & { partNumberMarker?: string },
+  ) =>
+    Effect.tryPromise({
+      try: async () => {
+        const params: AWS.ListPartsCommandInput = {
+          Bucket: context.bucket,
+          Key: context.key,
+          UploadId: context.uploadId,
+          PartNumberMarker: context.partNumberMarker,
+        };
+
+        const data = await s3Client.listParts(params);
+
+        return {
+          parts: data.Parts ?? [],
+          isTruncated: data.IsTruncated ?? false,
+          nextPartNumberMarker: data.NextPartNumberMarker,
+        };
+      },
+      catch: (error) =>
+        handleS3Error("listParts", error, {
+          upload_id: context.key,
+          s3_bucket: context.bucket,
+        }),
+    });
+
+  const listMultipartUploads = (keyMarker?: string, uploadIdMarker?: string) =>
+    Effect.tryPromise({
+      try: () =>
+        s3Client.listMultipartUploads({
+          Bucket: bucket,
+          KeyMarker: keyMarker,
+          UploadIdMarker: uploadIdMarker,
+        }),
+      catch: (error) =>
+        handleS3Error("listMultipartUploads", error, { bucket }),
+    });
+
+  const getIncompletePart = (id: string) =>
+    Effect.tryPromise({
+      try: async () => {
+        try {
+          const data = await s3Client.getObject({
+            Bucket: bucket,
+            Key: partKey(id),
+          });
+          return toReadableStream(data.Body);
+        } catch (error) {
+          if (error instanceof NoSuchKey) {
+            return undefined;
+          }
+          throw error;
+        }
+      },
+      catch: (error) =>
+        handleS3Error("getIncompletePart", error, { upload_id: id, bucket }),
+    });
+
+  const getIncompletePartSize = (id: string) => headObject(partKey(id));
+
+  const putIncompletePart = (id: string, data: Uint8Array) =>
+    putObject(partKey(id), data).pipe(
+      Effect.tap(() =>
+        Effect.logInfo("Incomplete part uploaded").pipe(
+          Effect.annotateLogs({ upload_id: id }),
+        ),
+      ),
+    );
+
+  const deleteIncompletePart = (id: string) => deleteObject(partKey(id));
+
+  return {
+    bucket,
+    getObject,
+    headObject,
+    putObject,
+    deleteObject,
+    deleteObjects,
+    createMultipartUpload,
+    uploadPart,
+    completeMultipartUpload,
+    abortMultipartUpload,
+    listParts,
+    listMultipartUploads,
+    getIncompletePart,
+    getIncompletePartSize,
+    putIncompletePart,
+    deleteIncompletePart,
+  };
+};
+
+export const S3ClientLayer = (s3ClientConfig: S3ClientConfig, bucket: string) =>
+  Layer.succeed(S3ClientService, makeS3ClientService(s3ClientConfig, bucket));

package/src/types.ts

@@ -0,0 +1,96 @@
+import type { S3ClientConfig } from "@aws-sdk/client-s3";
+import type { UploadistaError } from "@uploadista/core/errors";
+import type {
+  DataStoreCapabilities,
+  DataStoreWriteOptions,
+  KvStore,
+  UploadFile,
+  UploadStrategy,
+} from "@uploadista/core/types";
+import type { Effect } from "effect";
+
+export type S3StoreOptions = {
+  deliveryUrl: string;
+  /**
+   * The preferred part size for parts send to S3. Can not be lower than 5MiB or more than 5GiB.
+   * The server calculates the optimal part size, which takes this size into account,
+   * but may increase it to not exceed the S3 10K parts limit.
+   */
+  partSize?: number;
+  /**
+   * The minimal part size for parts.
+   * Can be used to ensure that all non-trailing parts are exactly the same size.
+   * Can not be lower than 5MiB or more than 5GiB.
+   */
+  minPartSize?: number;
+  /**
+   * The maximum number of parts allowed in a multipart upload. Defaults to 10,000.
+   */
+  maxMultipartParts?: number;
+  useTags?: boolean;
+  maxConcurrentPartUploads?: number;
+  expirationPeriodInMilliseconds?: number;
+  // Options to pass to the AWS S3 SDK.
+  s3ClientConfig: S3ClientConfig & { bucket: string };
+};
+
+export type ChunkInfo = {
+  partNumber: number;
+  data: Uint8Array;
+  size: number;
+  isFinalPart?: boolean;
+};
+
+export type S3OperationContext = {
+  uploadId: string;
+  bucket: string;
+  key: string;
+  partNumber?: number;
+  partSize?: number;
+  contentType?: string;
+  cacheControl?: string;
+};
+
+export type PartUploadResult = {
+  etag: string;
+  partNumber: number;
+};
+
+export type MultipartUploadInfo = {
+  uploadId: string;
+  bucket: string;
+  key: string;
+};
+
+export type UploadProgress = {
+  bytesUploaded: number;
+  totalBytes: number;
+  currentOffset: number;
+};
+
+export type S3Store = {
+  bucket: string;
+  create: (upload: UploadFile) => Effect.Effect<UploadFile, UploadistaError>;
+  remove: (id: string) => Effect.Effect<void, UploadistaError>;
+  write: (
+    options: DataStoreWriteOptions,
+    dependencies: { onProgress?: (chunkSize: number) => void },
+  ) => Effect.Effect<number, UploadistaError>;
+  getUpload: (id: string) => Effect.Effect<UploadFile, UploadistaError>;
+  read: (id: string) => Effect.Effect<ReadableStream, UploadistaError>;
+  deleteExpired: Effect.Effect<number, UploadistaError>;
+  getCapabilities: () => DataStoreCapabilities;
+  getChunkerConstraints: () => {
+    minChunkSize: number;
+    maxChunkSize: number;
+    optimalChunkSize: number;
+    requiresOrderedChunks: boolean;
+  };
+  validateUploadStrategy: (
+    strategy: UploadStrategy,
+  ) => Effect.Effect<boolean, never>;
+};
+
+export type S3StoreConfig = S3StoreOptions & {
+  kvStore: KvStore<UploadFile>;
+};

package/src/utils/calculations.ts

@@ -0,0 +1,61 @@
+import type AWS from "@aws-sdk/client-s3";
+
+export const calcOffsetFromParts = (parts?: Array<AWS.Part>): number => {
+  return parts && parts.length > 0
+    ? parts.reduce((a, b) => a + (b?.Size ?? 0), 0)
+    : 0;
+};
+
+export const calcOptimalPartSize = (
+  initSize: number | undefined,
+  preferredPartSize: number,
+  minPartSize: number,
+  maxMultipartParts: number,
+  maxUploadSize = 5_497_558_138_880, // 5TiB
+): number => {
+  const size = initSize ?? maxUploadSize;
+  let optimalPartSize: number;
+
+  if (size <= preferredPartSize) {
+    // For files smaller than preferred part size, use the file size
+    // but ensure it meets S3's minimum requirements for multipart uploads
+    optimalPartSize = size;
+  } else if (size <= preferredPartSize * maxMultipartParts) {
+    // File fits within max parts limit using preferred part size
+    optimalPartSize = preferredPartSize;
+  } else {
+    // File is too large for preferred part size, calculate minimum needed
+    optimalPartSize = Math.ceil(size / maxMultipartParts);
+  }
+
+  // Ensure we respect minimum part size for multipart uploads
+  // Exception: if the file is smaller than minPartSize, use the file size directly
+  const finalPartSize =
+    initSize && initSize < minPartSize
+      ? optimalPartSize // Single part upload for small files
+      : Math.max(optimalPartSize, minPartSize); // Enforce minimum for multipart
+
+  // Round up to ensure consistent part sizes and align to reasonable boundaries
+  // This helps ensure all parts except the last one will have exactly the same size
+  const alignment = 1024; // 1KB alignment for better consistency
+  return Math.ceil(finalPartSize / alignment) * alignment;
+};
+
+export const partKey = (id: string): string => {
+  return `${id}.part`;
+};
+
+export const shouldUseExpirationTags = (
+  expirationPeriodInMilliseconds: number,
+  useTags: boolean,
+): boolean => {
+  return expirationPeriodInMilliseconds !== 0 && useTags;
+};
+
+export const getExpirationDate = (
+  createdAt: string,
+  expirationPeriodInMilliseconds: number,
+): Date => {
+  const date = new Date(createdAt);
+  return new Date(date.getTime() + expirationPeriodInMilliseconds);
+};

package/src/utils/error-handling.ts

@@ -0,0 +1,52 @@
+import { UploadistaError } from "@uploadista/core/errors";
+import { trackS3Error as logS3Error } from "@uploadista/observability";
+import { Effect } from "effect";
+
+export const handleS3Error = (
+  operation: string,
+  error: unknown,
+  context: Record<string, unknown> = {},
+): UploadistaError => {
+  // Log the error with context
+  Effect.runSync(logS3Error(operation, error, context));
+
+  return UploadistaError.fromCode("FILE_WRITE_ERROR", error as Error);
+};
+
+export const handleS3NotFoundError = (
+  operation: string,
+  error: unknown,
+  context: Record<string, unknown> = {},
+): UploadistaError => {
+  if (
+    typeof error === "object" &&
+    error !== null &&
+    "code" in error &&
+    typeof error.code === "string" &&
+    ["NotFound", "NoSuchKey", "NoSuchUpload"].includes(error.code)
+  ) {
+    Effect.runSync(
+      Effect.logWarning(`File not found during ${operation} operation`).pipe(
+        Effect.annotateLogs({
+          error_code: error.code,
+          ...context,
+        }),
+      ),
+    );
+    return UploadistaError.fromCode("FILE_NOT_FOUND");
+  }
+
+  return handleS3Error(operation, error, context);
+};
+
+export const isUploadNotFoundError = (
+  error: unknown,
+): error is { code: "NoSuchUpload" | "NoSuchKey" } => {
+  return (
+    typeof error === "object" &&
+    error !== null &&
+    "code" in error &&
+    typeof error.code === "string" &&
+    (error.code === "NoSuchUpload" || error.code === "NoSuchKey")
+  );
+};

package/src/utils/index.ts

@@ -0,0 +1,3 @@
+export * from "./calculations";
+export * from "./error-handling";
+export * from "./stream-adapter";

package/src/utils/stream-adapter.ts

@@ -0,0 +1,50 @@
+/**
+ * Stream adapter utility to handle AWS SDK Body responses across different environments.
+ *
+ * In Node.js environments, AWS SDK returns Node.js Readable streams.
+ * In Cloudflare Workers, it returns Web Streams API ReadableStreams.
+ * This utility normalizes both to Web Streams API ReadableStreams.
+ */
+
+/**
+ * Converts various stream types to a Web Streams API ReadableStream
+ * @param body The body from AWS SDK response (could be Node.js Readable or Web ReadableStream)
+ * @returns A Web Streams API ReadableStream
+ */
+export function toReadableStream(body: unknown): ReadableStream {
+  // If it's already a Web ReadableStream, return as-is
+  if (body instanceof ReadableStream) {
+    return body;
+  }
+
+  // If it has a getReader method, it's likely already a ReadableStream
+  if (body && typeof body === "object" && "getReader" in body) {
+    return body as ReadableStream;
+  }
+
+  // Check if it's a Node.js Readable stream
+  if (body && typeof body === "object" && "pipe" in body && "on" in body) {
+    const nodeStream = body as NodeJS.ReadableStream;
+
+    return new ReadableStream({
+      start(controller) {
+        nodeStream.on("data", (chunk) => {
+          controller.enqueue(new Uint8Array(chunk));
+        });
+
+        nodeStream.on("end", () => {
+          controller.close();
+        });
+
+        nodeStream.on("error", (error) => {
+          controller.error(error);
+        });
+      },
+    });
+  }
+
+  // If it's some other type, try to handle it gracefully
+  throw new Error(
+    `Unsupported body type: ${typeof body}. Expected ReadableStream or Node.js Readable.`,
+  );
+}

package/tsconfig.json

@@ -0,0 +1,19 @@
+{
+  "extends": "@uploadista/typescript-config/server.json",
+  "compilerOptions": {
+    "baseUrl": "./",
+    "paths": {
+      "@/*": ["./src/*"]
+    },
+    "outDir": "./dist",
+    "rootDir": "./src"
+  },
+  "include": ["src"],
+  "exclude": [
+    "src/**/*.test.ts",
+    "src/**/*.test.tsx",
+    "src/**/*.spec.ts",
+    "src/**/*.spec.tsx",
+    "src/**/__tests__/**"
+  ]
+}

package/tsconfig.tsbuildinfo

@@ -0,0 +1 @@
+{"root":["./src/index.ts","./src/s3-store.ts","./src/types.ts","./src/services/index.ts","./src/services/s3-client.service.ts","./src/services/__mocks__/s3-client-mock.service.ts","./src/utils/calculations.ts","./src/utils/error-handling.ts","./src/utils/index.ts","./src/utils/stream-adapter.ts"],"version":"5.9.3"}

package/vitest.config.ts

@@ -0,0 +1,15 @@
+import { defineConfig } from 'vitest/config';
+
+export default defineConfig({
+  test: {
+    globals: true,
+    environment: 'node',
+    include: ['src/**/*.{test,spec}.{js,mjs,cjs,ts,mts,cts,jsx,tsx}'],
+    exclude: ['node_modules', 'dist'],
+    coverage: {
+      provider: 'v8',
+      reporter: ['text', 'json', 'html'],
+      exclude: ['node_modules/', 'dist/', '**/*.d.ts', '**/*.test.ts', '**/*.spec.ts']
+    }
+  }
+});