@uploadista/data-store-r2 0.0.9

package/src/services/r2-client.service.ts

@@ -0,0 +1,288 @@
+import type {
+  R2Bucket,
+  ReadableStream,
+} from "@cloudflare/workers-types";
+import type { UploadistaError } from "@uploadista/core/errors";
+import { withS3ApiMetrics } from "@uploadista/observability";
+import { Context, Effect, Layer } from "effect";
+import type { MultipartUploadInfo, R2OperationContext, R2UploadedPart } from "../types";
+import { handleR2Error, handleR2NotFoundError, partKey } from "../utils";
+
+export class R2ClientService extends Context.Tag("R2ClientService")<
+  R2ClientService,
+  {
+    readonly bucket: string;
+
+    // Basic S3 operations
+    readonly getObject: (
+      key: string,
+    ) => Effect.Effect<ReadableStream, UploadistaError>;
+    readonly headObject: (
+      key: string,
+    ) => Effect.Effect<number | undefined, UploadistaError>;
+    readonly putObject: (
+      key: string,
+      body: Uint8Array,
+    ) => Effect.Effect<string, UploadistaError>;
+    readonly deleteObject: (
+      key: string,
+    ) => Effect.Effect<void, UploadistaError>;
+    readonly deleteObjects: (
+      keys: string[],
+    ) => Effect.Effect<void, UploadistaError>;
+
+    // Multipart upload operations
+    readonly createMultipartUpload: (
+      context: R2OperationContext,
+    ) => Effect.Effect<MultipartUploadInfo, UploadistaError>;
+    readonly uploadPart: (
+      context: R2OperationContext & { partNumber: number; data: Uint8Array },
+    ) => Effect.Effect<string, UploadistaError>;
+    readonly completeMultipartUpload: (
+      context: R2OperationContext,
+      parts: Array<R2UploadedPart>,
+    ) => Effect.Effect<string | undefined, UploadistaError>;
+    readonly abortMultipartUpload: (
+      context: R2OperationContext,
+    ) => Effect.Effect<void, UploadistaError>;
+
+    // Incomplete part operations
+    readonly getIncompletePart: (
+      id: string,
+    ) => Effect.Effect<ReadableStream | undefined, UploadistaError>;
+    readonly getIncompletePartSize: (
+      id: string,
+    ) => Effect.Effect<number | undefined, UploadistaError>;
+    readonly putIncompletePart: (
+      id: string,
+      data: Uint8Array,
+    ) => Effect.Effect<string, UploadistaError>;
+    readonly deleteIncompletePart: (
+      id: string,
+    ) => Effect.Effect<void, UploadistaError>;
+  }
+>() {}
+
+export const makeR2ClientService = (
+  r2Bucket: R2Bucket,
+  r2BucketName: string,
+) => {
+  const getObject = (key: string) =>
+    Effect.gen(function* () {
+      const data = yield* Effect.tryPromise({
+        try: async () => {
+          const result = await r2Bucket.get(key);
+          if (!result) {
+            throw new Error(`Object not found: ${key}`);
+          }
+          return result.body;
+        },
+        catch: (error) =>
+          handleR2Error("getObject", error, { key, bucket: r2BucketName }),
+      });
+      return data;
+    });
+
+  const headObject = (key: string) =>
+    Effect.tryPromise({
+      try: async () => {
+        const data = await r2Bucket.head(key);
+        if (!data) {
+          return undefined;
+        }
+        return data.size;
+      },
+      catch: (error) =>
+        handleR2Error("headObject", error, { key, bucket: r2BucketName }),
+    });
+
+  const putObject = (key: string, body: Uint8Array) =>
+    Effect.tryPromise({
+      try: async () => {
+        const response = await r2Bucket.put(key, body);
+        if (!response) {
+          throw new Error("Failed to put object");
+        }
+        return response.etag;
+      },
+      catch: (error) =>
+        handleR2Error("putObject", error, {
+          key,
+          bucket: r2BucketName,
+          size: body.length,
+        }),
+    });
+
+  const deleteObject = (key: string) =>
+    Effect.tryPromise({
+      try: async () => {
+        await r2Bucket.delete(key);
+      },
+      catch: (error) =>
+        handleR2Error("deleteObject", error, { key, bucket: r2BucketName }),
+    });
+
+  const deleteObjects = (keys: string[]) =>
+    Effect.tryPromise({
+      try: () => r2Bucket.delete(keys),
+      catch: (error) =>
+        handleR2Error("deleteObjects", error, {
+          keys: keys.length,
+          bucket: r2BucketName,
+        }),
+    });
+
+  const createMultipartUpload = (context: R2OperationContext) =>
+    withS3ApiMetrics(
+      "createMultipartUpload",
+      Effect.tryPromise({
+        try: async () => {
+          const multipartUpload = await r2Bucket.createMultipartUpload(
+            context.key,
+          );
+
+          if (!multipartUpload.uploadId) {
+            throw new Error("Upload ID is undefined");
+          }
+          if (!multipartUpload.key) {
+            throw new Error("Key is undefined");
+          }
+
+          return {
+            uploadId: multipartUpload.uploadId,
+            bucket: context.bucket,
+            key: multipartUpload.key,
+          };
+        },
+        catch: (error) =>
+          handleR2Error("createMultipartUpload", error, context),
+      }),
+    );
+
+  const uploadPart = (
+    context: R2OperationContext & { partNumber: number; data: Uint8Array },
+  ) =>
+    withS3ApiMetrics(
+      "uploadPart",
+      Effect.tryPromise({
+        try: async () => {
+          const multipartUpload = await r2Bucket.resumeMultipartUpload(
+            context.key,
+            context.uploadId,
+          );
+          const part = await multipartUpload.uploadPart(
+            context.partNumber,
+            context.data,
+          );
+          if (!part) {
+            throw new Error("Part is undefined");
+          }
+          return part.etag;
+        },
+        catch: (error) =>
+          handleR2Error("uploadPart", error, {
+            upload_id: context.key,
+            part_number: context.partNumber,
+            part_size: context.data.length,
+            s3_bucket: context.bucket,
+          }),
+      }).pipe(Effect.map((response) => response)),
+    );
+
+  const completeMultipartUpload = (
+    context: R2OperationContext,
+    parts: Array<R2UploadedPart>,
+  ) =>
+    withS3ApiMetrics(
+      "completeMultipartUpload",
+      Effect.tryPromise({
+        try: async () => {
+          const multipartUpload = await r2Bucket.resumeMultipartUpload(
+            context.key,
+            context.uploadId,
+          );
+          const complete = await multipartUpload.complete(parts);
+          if (!complete) {
+            throw new Error("Complete is undefined");
+          }
+          return complete.key;
+        },
+        catch: (error) =>
+          handleR2Error("completeMultipartUpload", error, {
+            upload_id: context.key,
+            parts_count: parts.length,
+            s3_bucket: context.bucket,
+          }),
+      }),
+    );
+
+  const abortMultipartUpload = (context: R2OperationContext) =>
+    Effect.tryPromise({
+      try: async () => {
+        const multipartUpload = await r2Bucket.resumeMultipartUpload(
+          context.key,
+          context.uploadId,
+        );
+        await multipartUpload.abort();
+      },
+      catch: (error) =>
+        handleR2NotFoundError("abortMultipartUpload", error, {
+          upload_id: context.key,
+          s3_bucket: context.bucket,
+        }),
+    });
+
+  // Note: R2 does not provide a listParts API like S3
+  // Parts are tracked in the KV store instead (see r2-store.ts)
+  // Note: R2 also does not provide listMultipartUploads API
+  // For cleanup, use R2's native expiration policies instead
+
+  const getIncompletePart = (id: string) =>
+    Effect.tryPromise({
+      try: async () => {
+        const data = await r2Bucket.get(partKey(id));
+        if (!data || !data.body) {
+          return undefined;
+        }
+        return data.body;
+      },
+      catch: (error) =>
+        handleR2Error("getIncompletePart", error, {
+          upload_id: id,
+          bucket: r2BucketName,
+        }),
+    });
+
+  const getIncompletePartSize = (id: string) => headObject(partKey(id));
+
+  const putIncompletePart = (id: string, data: Uint8Array) =>
+    putObject(partKey(id), data).pipe(
+      Effect.tap(() =>
+        Effect.logInfo("Incomplete part uploaded").pipe(
+          Effect.annotateLogs({ upload_id: id }),
+        ),
+      ),
+    );
+
+  const deleteIncompletePart = (id: string) => deleteObject(partKey(id));
+
+  return {
+    bucket: r2BucketName,
+    getObject,
+    headObject,
+    putObject,
+    deleteObject,
+    deleteObjects,
+    createMultipartUpload,
+    uploadPart,
+    completeMultipartUpload,
+    abortMultipartUpload,
+    getIncompletePart,
+    getIncompletePartSize,
+    putIncompletePart,
+    deleteIncompletePart,
+  };
+};
+
+export const R2ClientLayer = (r2Bucket: R2Bucket, r2BucketName: string) =>
+  Layer.succeed(R2ClientService, makeR2ClientService(r2Bucket, r2BucketName));

package/src/types.ts

@@ -0,0 +1,106 @@
+import type {
+  R2Bucket,
+  R2UploadedPart as CloudflareR2UploadedPart,
+  ReadableStream,
+} from "@cloudflare/workers-types";
+import type { UploadistaError } from "@uploadista/core/errors";
+import type {
+  DataStoreCapabilities,
+  DataStoreWriteOptions,
+  UploadFile,
+  UploadStrategy,
+} from "@uploadista/core/types";
+import type { Effect } from "effect";
+
+/**
+ * Extended R2UploadedPart type that includes size property
+ * Cloudflare's R2UploadedPart doesn't include size, but we need it for tracking
+ */
+export type R2UploadedPart = CloudflareR2UploadedPart & {
+  size: number;
+};
+
+export type R2StoreOptions = {
+  deliveryUrl: string;
+  /**
+   * The preferred part size for parts send to S3. Can not be lower than 5MiB or more than 5GiB.
+   * The server calculates the optimal part size, which takes this size into account,
+   * but may increase it to not exceed the S3 10K parts limit.
+   */
+  partSize?: number;
+  /**
+   * The minimal part size for parts.
+   * Can be used to ensure that all non-trailing parts are exactly the same size.
+   * Can not be lower than 5MiB or more than 5GiB.
+   */
+  minPartSize?: number;
+  /**
+   * The maximum number of parts allowed in a multipart upload. Defaults to 10,000.
+   */
+  maxMultipartParts?: number;
+  useTags?: boolean;
+  maxConcurrentPartUploads?: number;
+  expirationPeriodInMilliseconds?: number;
+  // Options to pass to the Cloudflare R2 SDK.
+  bucket: string;
+  r2Bucket: R2Bucket;
+};
+
+export type ChunkInfo = {
+  partNumber: number;
+  data: Uint8Array;
+  size: number;
+  isFinalPart?: boolean;
+};
+
+export type R2OperationContext = {
+  uploadId: string;
+  bucket: string;
+  key: string;
+  partNumber?: number;
+  partSize?: number;
+  contentType?: string;
+  cacheControl?: string;
+};
+
+export type PartUploadResult = {
+  etag: string;
+  partNumber: number;
+};
+
+export type MultipartUploadInfo = {
+  uploadId: string;
+  bucket: string;
+  key: string;
+};
+
+export type UploadProgress = {
+  bytesUploaded: number;
+  totalBytes: number;
+  currentOffset: number;
+};
+
+export type R2Store = {
+  bucket: string;
+  create: (upload: UploadFile) => Effect.Effect<UploadFile, UploadistaError>;
+  remove: (id: string) => Effect.Effect<void, UploadistaError>;
+  write: (
+    options: DataStoreWriteOptions,
+    dependencies: { onProgress?: (chunkSize: number) => void },
+  ) => Effect.Effect<number, UploadistaError>;
+  getUpload: (id: string) => Effect.Effect<UploadFile, UploadistaError>;
+  read: (id: string) => Effect.Effect<ReadableStream, UploadistaError>;
+  deleteExpired: Effect.Effect<number, UploadistaError>;
+  getCapabilities: () => DataStoreCapabilities;
+  getChunkerConstraints: () => {
+    minChunkSize: number;
+    maxChunkSize: number;
+    optimalChunkSize: number;
+    requiresOrderedChunks: boolean;
+  };
+  validateUploadStrategy: (
+    strategy: UploadStrategy,
+  ) => Effect.Effect<boolean, never>;
+};
+
+export type R2StoreConfig = R2StoreOptions;

package/src/utils/calculations.ts

@@ -0,0 +1,61 @@
+import type { R2UploadedPart } from "../types";
+
+export const calcOffsetFromParts = (parts?: Array<R2UploadedPart>): number => {
+  return parts && parts.length > 0
+    ? parts.reduce((a, b) => a + (b?.size ?? 0), 0)
+    : 0;
+};
+
+export const calcOptimalPartSize = (
+  initSize: number | undefined,
+  preferredPartSize: number,
+  minPartSize: number,
+  maxMultipartParts: number,
+  maxUploadSize = 5_497_558_138_880, // 5TiB
+): number => {
+  const size = initSize ?? maxUploadSize;
+  let optimalPartSize: number;
+
+  if (size <= preferredPartSize) {
+    // For files smaller than preferred part size, use the file size
+    // but ensure it meets S3's minimum requirements for multipart uploads
+    optimalPartSize = size;
+  } else if (size <= preferredPartSize * maxMultipartParts) {
+    // File fits within max parts limit using preferred part size
+    optimalPartSize = preferredPartSize;
+  } else {
+    // File is too large for preferred part size, calculate minimum needed
+    optimalPartSize = Math.ceil(size / maxMultipartParts);
+  }
+
+  // Ensure we respect minimum part size for multipart uploads
+  // Exception: if the file is smaller than minPartSize, use the file size directly
+  const finalPartSize =
+    initSize && initSize < minPartSize
+      ? optimalPartSize // Single part upload for small files
+      : Math.max(optimalPartSize, minPartSize); // Enforce minimum for multipart
+
+  // Round up to ensure consistent part sizes and align to reasonable boundaries
+  // This helps ensure all parts except the last one will have exactly the same size
+  const alignment = 1024; // 1KB alignment for better consistency
+  return Math.ceil(finalPartSize / alignment) * alignment;
+};
+
+export const partKey = (id: string): string => {
+  return `${id}.part`;
+};
+
+export const shouldUseExpirationTags = (
+  expirationPeriodInMilliseconds: number,
+  useTags: boolean,
+): boolean => {
+  return expirationPeriodInMilliseconds !== 0 && useTags;
+};
+
+export const getExpirationDate = (
+  createdAt: string,
+  expirationPeriodInMilliseconds: number,
+): Date => {
+  const date = new Date(createdAt);
+  return new Date(date.getTime() + expirationPeriodInMilliseconds);
+};

package/src/utils/error-handling.ts

@@ -0,0 +1,52 @@
+import { UploadistaError } from "@uploadista/core/errors";
+import { trackS3Error as logR2Error } from "@uploadista/observability";
+import { Effect } from "effect";
+
+export const handleR2Error = (
+  operation: string,
+  error: unknown,
+  context: Record<string, unknown> = {},
+): UploadistaError => {
+  // Log the error with context
+  Effect.runSync(logR2Error(operation, error, context));
+
+  return UploadistaError.fromCode("FILE_WRITE_ERROR", error as Error);
+};
+
+export const handleR2NotFoundError = (
+  operation: string,
+  error: unknown,
+  context: Record<string, unknown> = {},
+): UploadistaError => {
+  if (
+    typeof error === "object" &&
+    error !== null &&
+    "code" in error &&
+    typeof error.code === "string" &&
+    ["NotFound", "NoSuchKey", "NoSuchUpload"].includes(error.code)
+  ) {
+    Effect.runSync(
+      Effect.logWarning(`File not found during ${operation} operation`).pipe(
+        Effect.annotateLogs({
+          error_code: error.code,
+          ...context,
+        }),
+      ),
+    );
+    return UploadistaError.fromCode("FILE_NOT_FOUND");
+  }
+
+  return handleR2Error(operation, error, context);
+};
+
+export const isUploadNotFoundError = (
+  error: unknown,
+): error is { code: "NoSuchUpload" | "NoSuchKey" } => {
+  return (
+    typeof error === "object" &&
+    error !== null &&
+    "code" in error &&
+    typeof error.code === "string" &&
+    (error.code === "NoSuchUpload" || error.code === "NoSuchKey")
+  );
+};

package/src/utils/index.ts

@@ -0,0 +1,2 @@
+export * from "./calculations";
+export * from "./error-handling";

package/tsconfig.json

@@ -0,0 +1,19 @@
+{
+  "extends": "@uploadista/typescript-config/server.json",
+  "compilerOptions": {
+    "baseUrl": "./",
+    "paths": {
+      "@/*": ["./src/*"]
+    },
+    "outDir": "./dist",
+    "rootDir": "./src"
+  },
+  "include": ["src"],
+  "exclude": [
+    "src/**/*.test.ts",
+    "src/**/*.test.tsx",
+    "src/**/*.spec.ts",
+    "src/**/*.spec.tsx",
+    "src/**/__tests__/**"
+  ]
+}

package/tsdown.config.ts

@@ -0,0 +1,11 @@
+import { defineConfig } from "tsdown";
+
+export default defineConfig({
+  entry: {
+    index: "src/index.ts",
+  },
+  minify: true,
+  format: ["esm", "cjs"],
+  dts: true,
+  outDir: "dist",
+});

package/vitest.config.ts

@@ -0,0 +1,15 @@
+import { defineConfig } from 'vitest/config';
+
+export default defineConfig({
+  test: {
+    globals: true,
+    environment: 'node',
+    include: ['src/**/*.{test,spec}.{js,mjs,cjs,ts,mts,cts,jsx,tsx}'],
+    exclude: ['node_modules', 'dist'],
+    coverage: {
+      provider: 'v8',
+      reporter: ['text', 'json', 'html'],
+      exclude: ['node_modules/', 'dist/', '**/*.d.ts', '**/*.test.ts', '**/*.spec.ts']
+    }
+  }
+});