@uploadista/data-store-s3 0.0.3

package/src/__tests__/utils/test-data-generator.ts

@@ -0,0 +1,331 @@
+import type { UploadistaError } from "@uploadista/core/errors";
+import { Option, Stream } from "effect";
+
+export interface TestFileSize {
+  name: string;
+  size: number;
+  description: string;
+}
+
+export const TEST_FILE_SIZES: Record<string, TestFileSize> = {
+  TINY: {
+    name: "tiny",
+    size: 1024, // 1KB
+    description: "Tiny file for edge cases",
+  },
+  SMALL_BASIC: {
+    name: "small-basic",
+    size: 1024 * 1024, // 1MB
+    description: "Basic small file",
+  },
+  SMALL_LARGE: {
+    name: "small-large",
+    size: Math.floor(4.9 * 1024 * 1024), // 4.9MB (just under S3 multipart threshold)
+    description: "Large small file (single part)",
+  },
+  MEDIUM_MIN: {
+    name: "medium-min",
+    size: 5 * 1024 * 1024, // 5MB (S3 minimum multipart size)
+    description: "Minimum medium file (multipart threshold)",
+  },
+  MEDIUM: {
+    name: "medium",
+    size: 10 * 1024 * 1024, // 10MB
+    description: "Standard medium file",
+  },
+  MEDIUM_LARGE: {
+    name: "medium-large",
+    size: 49 * 1024 * 1024, // 49MB
+    description: "Large medium file",
+  },
+  LARGE: {
+    name: "large",
+    size: 50 * 1024 * 1024, // 50MB
+    description: "Standard large file",
+  },
+  LARGE_XL: {
+    name: "large-xl",
+    size: 100 * 1024 * 1024, // 100MB
+    description: "Extra large file",
+  },
+  STRESS_TEST: {
+    name: "stress-test",
+    size: 200 * 1024 * 1024, // 200MB
+    description: "Stress test file",
+  },
+} as const;
+
+export interface TestFilePattern {
+  type: "random" | "zeros" | "ones" | "pattern" | "text";
+  pattern?: Uint8Array;
+  seed?: number;
+}
+
+/**
+ * Generate test data with different patterns for comprehensive testing
+ */
+
+/**
+ * Generate random data with optional seed for reproducibility
+ */
+export function generateRandomData(size: number, seed?: number): Uint8Array {
+  const data = new Uint8Array(size);
+
+  if (seed !== undefined) {
+    // Simple LCG for reproducible randomness
+    let rng = seed;
+    for (let i = 0; i < size; i++) {
+      rng = (rng * 1664525 + 1013904223) >>> 0;
+      data[i] = (rng >>> 24) & 0xff;
+    }
+  } else {
+    // crypto.getRandomValues has a 65,536 byte limit, so we need to generate in chunks
+    const maxChunkSize = 65536;
+    let offset = 0;
+
+    while (offset < size) {
+      const chunkSize = Math.min(maxChunkSize, size - offset);
+      const chunk = data.subarray(offset, offset + chunkSize);
+      crypto.getRandomValues(chunk);
+      offset += chunkSize;
+    }
+  }
+
+  return data;
+}
+
+/**
+ * Generate data filled with zeros (good for compression testing)
+ */
+export function generateZeroData(size: number): Uint8Array {
+  return new Uint8Array(size);
+}
+
+/**
+ * Generate data filled with ones
+ */
+export function generateOnesData(size: number): Uint8Array {
+  const data = new Uint8Array(size);
+  data.fill(255);
+  return data;
+}
+
+/**
+ * Generate data with repeating pattern
+ */
+export function generatePatternData(
+  size: number,
+  pattern: Uint8Array,
+): Uint8Array {
+  const data = new Uint8Array(size);
+  for (let i = 0; i < size; i++) {
+    data[i] = pattern[i % pattern.length];
+  }
+  return data;
+}
+
+/**
+ * Generate text-like data
+ */
+export function generateTextData(size: number): Uint8Array {
+  const text = "Lorem ipsum dolor sit amet, consectetur adipiscing elit. ";
+  const encoder = new TextEncoder();
+  const baseData = encoder.encode(text);
+  const data = new Uint8Array(size);
+
+  for (let i = 0; i < size; i++) {
+    data[i] = baseData[i % baseData.length];
+  }
+
+  return data;
+}
+
+/**
+ * Generate data based on pattern configuration
+ */
+export function generateData(
+  size: number,
+  pattern: TestFilePattern,
+): Uint8Array {
+  switch (pattern.type) {
+    case "random":
+      return generateRandomData(size, pattern.seed);
+    case "zeros":
+      return generateZeroData(size);
+    case "ones":
+      return generateOnesData(size);
+    case "pattern":
+      if (!pattern.pattern) {
+        throw new Error("Pattern must be provided for pattern type");
+      }
+      return generatePatternData(size, pattern.pattern);
+    case "text":
+      return generateTextData(size);
+    default:
+      throw new Error(`Unknown pattern type: ${pattern.type}`);
+  }
+}
+
+/**
+ * Create a stream from test data
+ */
+export const createTestDataStream = (
+  size: number,
+  pattern: TestFilePattern = { type: "random" },
+  chunkSize: number = 64 * 1024, // 64KB chunks
+): Stream.Stream<Uint8Array, UploadistaError> => {
+  const data = generateData(size, pattern);
+
+  return Stream.unfold(0, (offset) => {
+    if (offset >= data.length) {
+      return Option.none();
+    }
+
+    const end = Math.min(offset + chunkSize, data.length);
+    const chunk = data.slice(offset, end);
+
+    return Option.some([chunk, end] as const);
+  });
+};
+
+/**
+ * Create multiple streams for concurrent testing
+ */
+export const createMultipleTestStreams = (
+  count: number,
+  size: number,
+  pattern: TestFilePattern = { type: "random" },
+): Stream.Stream<Uint8Array, UploadistaError>[] => {
+  return Array.from({ length: count }, (_, i) =>
+    createTestDataStream(size, {
+      ...pattern,
+      seed: pattern.seed ? pattern.seed + i : i,
+    }),
+  );
+};
+
+/**
+ * Utility to compare two Uint8Arrays for testing
+ */
+export const compareArrays = (a: Uint8Array, b: Uint8Array): boolean => {
+  if (a.length !== b.length) {
+    return false;
+  }
+
+  for (let i = 0; i < a.length; i++) {
+    if (a[i] !== b[i]) {
+      return false;
+    }
+  }
+
+  return true;
+};
+
+/**
+ * Read all data from a stream for comparison
+ */
+export const streamToArray = async (
+  stream: ReadableStream<Uint8Array>,
+): Promise<Uint8Array> => {
+  const reader = stream.getReader();
+  const chunks: Uint8Array[] = [];
+  let totalLength = 0;
+
+  try {
+    while (true) {
+      const { done, value } = await reader.read();
+      if (done) break;
+      chunks.push(value);
+      totalLength += value.length;
+    }
+  } finally {
+    reader.releaseLock();
+  }
+
+  const result = new Uint8Array(totalLength);
+  let offset = 0;
+  for (const chunk of chunks) {
+    result.set(chunk, offset);
+    offset += chunk.length;
+  }
+
+  return result;
+};
+
+/**
+ * Create test files with metadata for comprehensive testing
+ */
+export interface TestFile {
+  id: string;
+  name: string;
+  size: number;
+  data: Uint8Array;
+  stream: Stream.Stream<Uint8Array, UploadistaError>;
+  pattern: TestFilePattern;
+  metadata?: {
+    contentType?: string;
+    cacheControl?: string;
+  };
+}
+
+export const createTestFile = (
+  id: string,
+  testSize: TestFileSize,
+  pattern: TestFilePattern = { type: "random" },
+  metadata?: TestFile["metadata"],
+): TestFile => {
+  const data = generateData(testSize.size, pattern);
+
+  return {
+    id,
+    name: testSize.name,
+    size: testSize.size,
+    data,
+    stream: createTestDataStream(testSize.size, pattern),
+    pattern,
+    metadata,
+  };
+};
+
+/**
+ * Create a set of standard test files for comprehensive testing
+ */
+export const createStandardTestFiles = (): TestFile[] => {
+  return [
+    createTestFile("tiny-random", TEST_FILE_SIZES.TINY, {
+      type: "random",
+      seed: 1,
+    }),
+    createTestFile("tiny-zeros", TEST_FILE_SIZES.TINY, { type: "zeros" }),
+    createTestFile("small-basic", TEST_FILE_SIZES.SMALL_BASIC, {
+      type: "random",
+      seed: 2,
+    }),
+    createTestFile("small-large", TEST_FILE_SIZES.SMALL_LARGE, {
+      type: "random",
+      seed: 3,
+    }),
+    createTestFile("medium-min", TEST_FILE_SIZES.MEDIUM_MIN, {
+      type: "random",
+      seed: 4,
+    }),
+    createTestFile("medium", TEST_FILE_SIZES.MEDIUM, {
+      type: "random",
+      seed: 5,
+    }),
+    createTestFile("large", TEST_FILE_SIZES.LARGE, { type: "random", seed: 6 }),
+    createTestFile(
+      "text-file",
+      TEST_FILE_SIZES.MEDIUM,
+      { type: "text" },
+      {
+        contentType: "text/plain",
+        cacheControl: "no-cache",
+      },
+    ),
+    createTestFile("pattern-file", TEST_FILE_SIZES.MEDIUM, {
+      type: "pattern",
+      pattern: new Uint8Array([0xde, 0xad, 0xbe, 0xef]),
+    }),
+  ];
+};

package/src/__tests__/utils/test-setup.ts

@@ -0,0 +1,256 @@
+import type { UploadFile } from "@uploadista/core/types";
+import { UploadFileKVStore } from "@uploadista/core/types";
+import { memoryKvStore } from "@uploadista/kv-store-memory";
+import { Effect, Layer } from "effect";
+import {
+  MockS3ClientLayer,
+  type MockS3Config,
+  type MockS3TestMethods,
+} from "../../services/__mocks__/s3-client-mock.service";
+import { S3ClientService } from "../../services/s3-client.service";
+
+// Re-export types for tests
+export type { MockS3TestMethods };
+
+import type { S3StoreConfig } from "../../types";
+
+// Default test configuration
+export const DEFAULT_TEST_CONFIG: MockS3Config = {
+  simulateLatency: 0, // No latency by default for faster tests
+  errorRate: 0,
+  uploadFailureRate: 0,
+  enableErrorInjection: true,
+};
+
+// Test bucket configuration
+export const TEST_BUCKET = "test-uploadista-bucket";
+export const TEST_DELIVERY_URL = "https://test-cdn.example.com";
+
+// Common S3 store configuration for tests
+export const createTestS3StoreConfig = (
+  overrides: Partial<S3StoreConfig> = {},
+): Omit<S3StoreConfig, "kvStore"> => ({
+  deliveryUrl: TEST_DELIVERY_URL,
+  partSize: 8 * 1024 * 1024, // 8MB default part size
+  minPartSize: 5 * 1024 * 1024, // 5MB S3 minimum
+  maxMultipartParts: 10_000,
+  useTags: true,
+  maxConcurrentPartUploads: 10,
+  expirationPeriodInMilliseconds: 7 * 24 * 60 * 60 * 1000, // 1 week
+  s3ClientConfig: {
+    region: "us-east-1",
+    bucket: TEST_BUCKET,
+  },
+  ...overrides,
+});
+
+// Layer that provides both KV store and S3 client for testing
+// Creates a fresh isolated KV store for each test
+export const TestLayersWithMockS3 = (
+  mockConfig: MockS3Config = DEFAULT_TEST_CONFIG,
+) =>
+  Layer.mergeAll(
+    Layer.effect(
+      UploadFileKVStore,
+      Effect.sync(() => memoryKvStore<UploadFile>()),
+    ),
+    MockS3ClientLayer(TEST_BUCKET, mockConfig),
+  );
+
+// Not implemented : Layer with real S3 client (for integration tests with LocalStack/Minio)
+export const TestLayersWithRealS3 = () =>
+  Layer.mergeAll(
+    Layer.effect(
+      UploadFileKVStore,
+      Effect.sync(() => memoryKvStore<UploadFile>()),
+    ),
+    Layer.succeed(S3ClientService, {
+      bucket: TEST_BUCKET,
+      // Add dummy implementations for the interface
+      getObject: () => Effect.die("Not implemented in test setup"),
+      headObject: () => Effect.die("Not implemented in test setup"),
+      putObject: () => Effect.die("Not implemented in test setup"),
+      deleteObject: () => Effect.die("Not implemented in test setup"),
+      deleteObjects: () => Effect.die("Not implemented in test setup"),
+      createMultipartUpload: () => Effect.die("Not implemented in test setup"),
+      uploadPart: () => Effect.die("Not implemented in test setup"),
+      completeMultipartUpload: () =>
+        Effect.die("Not implemented in test setup"),
+      abortMultipartUpload: () => Effect.die("Not implemented in test setup"),
+      listParts: () => Effect.die("Not implemented in test setup"),
+      listMultipartUploads: () => Effect.die("Not implemented in test setup"),
+      getIncompletePart: () => Effect.die("Not implemented in test setup"),
+      getIncompletePartSize: () => Effect.die("Not implemented in test setup"),
+      putIncompletePart: () => Effect.die("Not implemented in test setup"),
+      deleteIncompletePart: () => Effect.die("Not implemented in test setup"),
+    }),
+  );
+
+// Helper to get the mock service with testing methods
+export const getMockS3Service = (): Effect.Effect<
+  S3ClientService["Type"] & MockS3TestMethods,
+  never,
+  S3ClientService
+> =>
+  Effect.gen(function* () {
+    const service = yield* S3ClientService;
+    // Type assertion since we know this is our mock when using TestLayersWithMockS3
+    return service as S3ClientService["Type"] & MockS3TestMethods;
+  });
+
+// Helper to create a test upload file
+export const createTestUploadFile = (
+  id: string,
+  size: number,
+  overrides: Partial<UploadFile> = {},
+): UploadFile => ({
+  id,
+  offset: 0,
+  size,
+  metadata: {
+    contentType: "application/octet-stream",
+    ...overrides.metadata,
+  },
+  storage: {
+    id: id,
+    type: "s3",
+    path: id,
+    ...overrides.storage,
+  },
+  url: `${TEST_DELIVERY_URL}/${id}`,
+  ...overrides,
+});
+
+// Test environment setup that can be used in beforeEach
+export const setupTestEnvironment = (
+  mockConfig: MockS3Config = DEFAULT_TEST_CONFIG,
+) =>
+  Effect.gen(function* () {
+    const mockService = yield* getMockS3Service();
+
+    // Clear storage and reset configuration
+    yield* mockService.clearStorage();
+    yield* mockService.setConfig({ ...DEFAULT_TEST_CONFIG, ...mockConfig });
+
+    return mockService;
+  });
+
+// Helper to run tests with proper error handling and logging
+export const runTestWithTimeout = <A, E>(
+  effect: Effect.Effect<A, E>,
+  timeoutMs: number = 10000,
+): Promise<A> =>
+  Effect.runPromise(
+    effect.pipe(
+      Effect.timeout(`${timeoutMs} millis`),
+      Effect.tapError((error) =>
+        Effect.logError("Test failed").pipe(
+          Effect.annotateLogs({ error: String(error) }),
+        ),
+      ),
+    ),
+  );
+
+// Assertion helpers for common test patterns
+export const assertFileUploaded = (
+  mockService: S3ClientService["Type"] & MockS3TestMethods,
+  fileId: string,
+  expectedSize: number,
+) =>
+  Effect.gen(function* () {
+    const storage = yield* mockService.getStorage();
+    const uploadedFile = storage.objects.get(fileId);
+
+    if (!uploadedFile) {
+      return yield* Effect.fail(new Error(`File ${fileId} was not uploaded`));
+    }
+
+    if (uploadedFile.length !== expectedSize) {
+      yield* Effect.fail(
+        new Error(
+          `File ${fileId} size ${uploadedFile.length} does not match expected ${expectedSize}`,
+        ),
+      );
+    }
+
+    return uploadedFile;
+  });
+
+export const assertMultipartUploadExists = (
+  mockService: S3ClientService["Type"] & MockS3TestMethods,
+  uploadId: string,
+) =>
+  Effect.gen(function* () {
+    const storage = yield* mockService.getStorage();
+    const upload = storage.multipartUploads.get(uploadId);
+
+    if (!upload) {
+      yield* Effect.fail(new Error(`Multipart upload ${uploadId} not found`));
+    }
+
+    return upload;
+  });
+
+export const assertMetricsRecorded = (
+  mockService: S3ClientService["Type"] & MockS3TestMethods,
+  operation: string,
+  expectedCount: number = 1,
+) =>
+  Effect.gen(function* () {
+    const metrics = yield* mockService.getMetrics();
+    const actualCount = metrics.operationCounts.get(operation) || 0;
+
+    if (actualCount !== expectedCount) {
+      yield* Effect.fail(
+        new Error(
+          `Operation ${operation} was called ${actualCount} times, expected ${expectedCount}`,
+        ),
+      );
+    }
+  });
+
+// Helper to create test scenarios with different configurations
+export interface TestScenario {
+  name: string;
+  config: MockS3Config;
+  description: string;
+}
+
+export const createTestScenarios = (): TestScenario[] => [
+  {
+    name: "normal",
+    config: { ...DEFAULT_TEST_CONFIG },
+    description: "Normal operation without errors or latency",
+  },
+  {
+    name: "with-latency",
+    config: { ...DEFAULT_TEST_CONFIG, simulateLatency: 10 },
+    description: "Operation with 10ms simulated latency",
+  },
+  {
+    name: "with-errors",
+    config: { ...DEFAULT_TEST_CONFIG, errorRate: 0.1 },
+    description: "Operation with 10% random error rate",
+  },
+  {
+    name: "upload-failures",
+    config: { ...DEFAULT_TEST_CONFIG, uploadFailureRate: 0.05 },
+    description: "Operation with 5% upload failure rate",
+  },
+  {
+    name: "high-latency",
+    config: { ...DEFAULT_TEST_CONFIG, simulateLatency: 100 },
+    description: "Operation with 100ms simulated latency",
+  },
+];
+
+// Helper for parameterized tests
+export const runParameterizedTest = <T>(
+  scenarios: T[],
+  testFn: (scenario: T) => Effect.Effect<void, unknown>,
+) =>
+  Effect.gen(function* () {
+    for (const scenario of scenarios) {
+      yield* testFn(scenario);
+    }
+  });

package/src/index.ts

@@ -0,0 +1 @@
+export * from "./s3-store";