@uploadista/core 0.0.2

package/src/types/kv-store.ts

@@ -0,0 +1,375 @@
+import { Context, Effect, Layer } from "effect";
+import { UploadistaError } from "../errors";
+import type { FlowJob } from "../flow";
+import type { UploadFile } from "./upload-file";
+
+/**
+ * Base key-value store interface for raw string storage.
+ *
+ * This is the low-level interface that storage adapters implement.
+ * It stores raw string values without type safety or serialization.
+ *
+ * @property get - Retrieves a value by key, returns null if not found
+ * @property set - Stores a value with the given key
+ * @property delete - Removes a value by key
+ * @property list - Optional operation to list all keys with a given prefix
+ *
+ * @example
+ * ```typescript
+ * // Implement a BaseKvStore with Redis
+ * const redisKvStore: BaseKvStore = {
+ *   get: (key) => Effect.tryPromise({
+ *     try: () => redis.get(key),
+ *     catch: (error) => UploadistaError.fromCode("UNKNOWN_ERROR", { cause: error })
+ *   }),
+ *
+ *   set: (key, value) => Effect.tryPromise({
+ *     try: () => redis.set(key, value),
+ *     catch: (error) => UploadistaError.fromCode("UNKNOWN_ERROR", { cause: error })
+ *   }),
+ *
+ *   delete: (key) => Effect.tryPromise({
+ *     try: () => redis.del(key),
+ *     catch: (error) => UploadistaError.fromCode("UNKNOWN_ERROR", { cause: error })
+ *   }),
+ *
+ *   list: (prefix) => Effect.tryPromise({
+ *     try: () => redis.keys(`${prefix}*`),
+ *     catch: (error) => UploadistaError.fromCode("UNKNOWN_ERROR", { cause: error })
+ *   })
+ * };
+ * ```
+ */
+export interface BaseKvStore {
+  readonly get: (key: string) => Effect.Effect<string | null, UploadistaError>;
+  readonly set: (
+    key: string,
+    value: string,
+  ) => Effect.Effect<void, UploadistaError>;
+  readonly delete: (key: string) => Effect.Effect<void, UploadistaError>;
+  readonly list?: (
+    keyPrefix: string,
+  ) => Effect.Effect<Array<string>, UploadistaError>;
+}
+
+/**
+ * Type-safe key-value store interface with automatic serialization.
+ *
+ * This wraps a BaseKvStore and handles JSON serialization/deserialization
+ * for a specific data type, providing type safety and eliminating the need
+ * for manual JSON.stringify/parse calls.
+ *
+ * @template TData - The type of data stored in this KV store
+ *
+ * @property get - Retrieves and deserializes a value, fails if not found
+ * @property set - Serializes and stores a value
+ * @property delete - Removes a value by key
+ * @property list - Optional operation to list all keys (without prefix)
+ *
+ * @example
+ * ```typescript
+ * // Use a typed KV store
+ * const uploadStore: KvStore<UploadFile> = new TypedKvStore(
+ *   baseStore,
+ *   "uploads:",
+ *   jsonSerializer.serialize,
+ *   jsonSerializer.deserialize
+ * );
+ *
+ * // Store and retrieve typed data
+ * const program = Effect.gen(function* () {
+ *   const file: UploadFile = {
+ *     id: "file123",
+ *     offset: 0,
+ *     storage: { id: "s3", type: "s3" }
+ *   };
+ *
+ *   // Automatic serialization
+ *   yield* uploadStore.set("file123", file);
+ *
+ *   // Automatic deserialization with type safety
+ *   const retrieved = yield* uploadStore.get("file123");
+ *   console.log(retrieved.offset); // TypeScript knows this is a number
+ * });
+ * ```
+ */
+export type KvStore<TData> = {
+  readonly get: (key: string) => Effect.Effect<TData, UploadistaError>;
+  readonly set: (
+    key: string,
+    value: TData,
+  ) => Effect.Effect<void, UploadistaError>;
+  readonly delete: (key: string) => Effect.Effect<void, UploadistaError>;
+  readonly list?: () => Effect.Effect<Array<string>, UploadistaError>;
+};
+
+/**
+ * Typed wrapper class that adds serialization to a BaseKvStore.
+ *
+ * This class implements the KvStore interface by wrapping a BaseKvStore
+ * and handling serialization/deserialization for a specific type. It also
+ * adds a key prefix to isolate different data types in the same store.
+ *
+ * @template TData - The type of data to store
+ *
+ * @example
+ * ```typescript
+ * // Create a typed store for UploadFile
+ * const uploadFileStore = new TypedKvStore<UploadFile>(
+ *   baseKvStore,
+ *   "uploadista:upload-file:", // All keys will be prefixed
+ *   (data) => JSON.stringify(data),
+ *   (str) => JSON.parse(str) as UploadFile
+ * );
+ *
+ * // Use the store
+ * const effect = Effect.gen(function* () {
+ *   const file: UploadFile = { ... };
+ *   yield* uploadFileStore.set("abc123", file);
+ *   // Internally stores at key "uploadista:upload-file:abc123"
+ *
+ *   const retrieved = yield* uploadFileStore.get("abc123");
+ *   return retrieved;
+ * });
+ *
+ * // Custom serialization for binary data
+ * const binaryStore = new TypedKvStore<Uint8Array>(
+ *   baseKvStore,
+ *   "binary:",
+ *   (data) => btoa(String.fromCharCode(...data)), // Base64 encode
+ *   (str) => Uint8Array.from(atob(str), c => c.charCodeAt(0)) // Base64 decode
+ * );
+ * ```
+ */
+export class TypedKvStore<TData> implements KvStore<TData> {
+  constructor(
+    private baseStore: BaseKvStore,
+    private keyPrefix: string,
+    private serialize: (data: TData) => string,
+    private deserialize: (str: string) => TData,
+  ) {}
+
+  get = (key: string): Effect.Effect<TData, UploadistaError> =>
+    this.baseStore.get(this.keyPrefix + key).pipe(
+      Effect.flatMap((value) => {
+        if (value === null) {
+          return Effect.fail(
+            UploadistaError.fromCode("FILE_NOT_FOUND", {
+              cause: `Key "${key}" not found`,
+            }),
+          );
+        }
+        try {
+          return Effect.succeed(this.deserialize(value));
+        } catch (error) {
+          return Effect.fail(
+            new UploadistaError({
+              code: "VALIDATION_ERROR",
+              status: 400,
+              body: `Failed to deserialize value for key "${key}"`,
+              cause: error,
+            }),
+          );
+        }
+      }),
+    );
+
+  set = (key: string, value: TData): Effect.Effect<void, UploadistaError> => {
+    try {
+      const serialized = this.serialize(value);
+      return this.baseStore.set(this.keyPrefix + key, serialized);
+    } catch (error) {
+      return Effect.fail(
+        new UploadistaError({
+          code: "VALIDATION_ERROR",
+          status: 400,
+          body: `Failed to serialize value for key "${key}"`,
+          cause: error,
+        }),
+      );
+    }
+  };
+
+  delete = (key: string): Effect.Effect<void, UploadistaError> =>
+    this.baseStore.delete(this.keyPrefix + key);
+
+  list = (): Effect.Effect<Array<string>, UploadistaError> => {
+    if (this.baseStore.list) {
+      return this.baseStore.list(this.keyPrefix);
+    }
+    return Effect.fail(
+      new UploadistaError({
+        code: "UNKNOWN_ERROR",
+        status: 501,
+        body: "List operation not supported by this store",
+      }),
+    );
+  };
+}
+
+/**
+ * Default JSON serialization helpers.
+ *
+ * These functions provide standard JSON serialization for use with TypedKvStore.
+ * They work with any JSON-serializable type.
+ *
+ * @example
+ * ```typescript
+ * const store = new TypedKvStore<MyType>(
+ *   baseStore,
+ *   "mydata:",
+ *   jsonSerializer.serialize,
+ *   jsonSerializer.deserialize
+ * );
+ * ```
+ */
+export const jsonSerializer = {
+  serialize: <T>(data: T): string => JSON.stringify(data),
+  deserialize: <T>(str: string): T => JSON.parse(str),
+};
+
+/**
+ * Effect-TS context tag for the base untyped KV store.
+ *
+ * This is the low-level store that storage adapter implementations provide.
+ * Most application code should use typed stores like UploadFileKVStore instead.
+ *
+ * @example
+ * ```typescript
+ * // Provide a base store implementation
+ * const baseStoreLayer = Layer.succeed(BaseKvStoreService, redisKvStore);
+ *
+ * // Use in an Effect
+ * const effect = Effect.gen(function* () {
+ *   const baseStore = yield* BaseKvStoreService;
+ *   yield* baseStore.set("raw-key", "raw-value");
+ * });
+ * ```
+ */
+export class BaseKvStoreService extends Context.Tag("BaseKvStore")<
+  BaseKvStoreService,
+  BaseKvStore
+>() {}
+
+/**
+ * Effect-TS context tag for the UploadFile typed KV store.
+ *
+ * This provides type-safe storage for UploadFile metadata. It's the primary
+ * way to store and retrieve upload metadata in the system.
+ *
+ * @example
+ * ```typescript
+ * const uploadEffect = Effect.gen(function* () {
+ *   const kvStore = yield* UploadFileKVStore;
+ *
+ *   // Store upload metadata
+ *   const file: UploadFile = {
+ *     id: "upload123",
+ *     offset: 0,
+ *     storage: { id: "s3", type: "s3" }
+ *   };
+ *   yield* kvStore.set("upload123", file);
+ *
+ *   // Retrieve with type safety
+ *   const retrieved = yield* kvStore.get("upload123");
+ *   return retrieved;
+ * });
+ * ```
+ */
+export class UploadFileKVStore extends Context.Tag("UploadFileKVStore")<
+  UploadFileKVStore,
+  KvStore<UploadFile>
+>() {}
+
+/**
+ * Effect Layer that creates the UploadFileKVStore from a BaseKvStore.
+ *
+ * This layer automatically wires up JSON serialization for UploadFile objects
+ * with the "uploadista:upload-file:" key prefix.
+ *
+ * @example
+ * ```typescript
+ * const program = Effect.gen(function* () {
+ *   const kvStore = yield* UploadFileKVStore;
+ *   // Use the store...
+ * }).pipe(
+ *   Effect.provide(uploadFileKvStore),
+ *   Effect.provide(baseStoreLayer)
+ * );
+ * ```
+ */
+export const uploadFileKvStore = Layer.effect(
+  UploadFileKVStore,
+  Effect.gen(function* () {
+    const baseStore = yield* BaseKvStoreService;
+    return new TypedKvStore<UploadFile>(
+      baseStore,
+      "uploadista:upload-file:",
+      jsonSerializer.serialize,
+      jsonSerializer.deserialize,
+    );
+  }),
+);
+
+/**
+ * Effect-TS context tag for the FlowJob typed KV store.
+ *
+ * This provides type-safe storage for FlowJob metadata, tracking the
+ * execution state of flow processing jobs.
+ *
+ * @example
+ * ```typescript
+ * const flowEffect = Effect.gen(function* () {
+ *   const jobStore = yield* FlowJobKVStore;
+ *
+ *   // Store job state
+ *   const job: FlowJob = {
+ *     id: "job123",
+ *     flowId: "flow_resize",
+ *     status: "running",
+ *     tasks: [],
+ *     createdAt: new Date(),
+ *     updatedAt: new Date()
+ *   };
+ *   yield* jobStore.set("job123", job);
+ *
+ *   // Retrieve and check status
+ *   const retrieved = yield* jobStore.get("job123");
+ *   return retrieved.status;
+ * });
+ * ```
+ */
+export class FlowJobKVStore extends Context.Tag("FlowJobKVStore")<
+  FlowJobKVStore,
+  KvStore<FlowJob>
+>() {}
+
+/**
+ * Effect Layer that creates the FlowJobKVStore from a BaseKvStore.
+ *
+ * This layer automatically wires up JSON serialization for FlowJob objects
+ * with the "uploadista:flow-job:" key prefix.
+ *
+ * @example
+ * ```typescript
+ * const program = Effect.gen(function* () {
+ *   const jobStore = yield* FlowJobKVStore;
+ *   // Use the store...
+ * }).pipe(
+ *   Effect.provide(flowJobKvStore),
+ *   Effect.provide(baseStoreLayer)
+ * );
+ * ```
+ */
+export const flowJobKvStore = Layer.effect(
+  FlowJobKVStore,
+  Effect.gen(function* () {
+    const baseStore = yield* BaseKvStoreService;
+    return new TypedKvStore<FlowJob>(
+      baseStore,
+      "uploadista:flow-job:",
+      jsonSerializer.serialize,
+      jsonSerializer.deserialize,
+    );
+  }),
+);

package/src/types/middleware.ts

@@ -0,0 +1,54 @@
+import { Context, Effect, Layer } from "effect";
+import type { UploadistaError } from "../errors";
+
+export type MiddlewareContext = {
+  request: Request;
+  uploadId?: string;
+  metadata?: Record<string, string>;
+};
+
+export type MiddlewareNext = () => Promise<Response>;
+
+export type Middleware = (
+  context: MiddlewareContext,
+  next: MiddlewareNext,
+) => Promise<Response>;
+
+// Effect-based Middleware service
+export class MiddlewareService extends Context.Tag("MiddlewareService")<
+  MiddlewareService,
+  {
+    readonly execute: (
+      middlewares: Middleware[],
+      context: MiddlewareContext,
+      handler: MiddlewareNext,
+    ) => Effect.Effect<Response, UploadistaError>;
+  }
+>() {}
+
+export const MiddlewareServiceLive = Layer.succeed(
+  MiddlewareService,
+  MiddlewareService.of({
+    execute: (middlewares, context, handler) =>
+      Effect.gen(function* () {
+        if (middlewares.length === 0) {
+          return yield* Effect.tryPromise({
+            try: () => handler(),
+            catch: (error) => error as UploadistaError,
+          });
+        }
+
+        const chain = middlewares.reduceRight(
+          (next: MiddlewareNext, middleware: Middleware) => {
+            return () => middleware(context, next);
+          },
+          handler,
+        );
+
+        return yield* Effect.tryPromise({
+          try: () => chain(),
+          catch: (error) => error as UploadistaError,
+        });
+      }),
+  }),
+);

package/src/types/upload-event.ts

@@ -0,0 +1,75 @@
+import { z } from "zod";
+import { uploadFileSchema } from "./upload-file";
+
+export enum UploadEventType {
+  UPLOAD_STARTED = "upload-started",
+  UPLOAD_PROGRESS = "upload-progress",
+  UPLOAD_COMPLETE = "upload-complete",
+  UPLOAD_FAILED = "upload-failed",
+  UPLOAD_VALIDATION_SUCCESS = "upload-validation-success",
+  UPLOAD_VALIDATION_FAILED = "upload-validation-failed",
+  UPLOAD_VALIDATION_WARNING = "upload-validation-warning",
+}
+
+const flowContextSchema = z.object({
+  flowId: z.string(),
+  nodeId: z.string(),
+  jobId: z.string(),
+}).optional();
+
+export const uploadEventSchema = z.union([
+  z.object({
+    type: z.union([
+      z.literal(UploadEventType.UPLOAD_STARTED),
+      z.literal(UploadEventType.UPLOAD_COMPLETE),
+    ]),
+    data: uploadFileSchema,
+    flow: flowContextSchema,
+  }),
+  z.object({
+    type: z.literal(UploadEventType.UPLOAD_PROGRESS),
+    data: z.object({
+      id: z.string(),
+      progress: z.number(),
+      total: z.number(),
+    }),
+    flow: flowContextSchema,
+  }),
+  z.object({
+    type: z.literal(UploadEventType.UPLOAD_FAILED),
+    data: z.object({
+      id: z.string(),
+      error: z.string(),
+    }),
+    flow: flowContextSchema,
+  }),
+  z.object({
+    type: z.literal(UploadEventType.UPLOAD_VALIDATION_SUCCESS),
+    data: z.object({
+      id: z.string(),
+      validationType: z.enum(["checksum", "mimetype"]),
+      algorithm: z.string().optional(),
+    }),
+    flow: flowContextSchema,
+  }),
+  z.object({
+    type: z.literal(UploadEventType.UPLOAD_VALIDATION_FAILED),
+    data: z.object({
+      id: z.string(),
+      reason: z.string(),
+      expected: z.string(),
+      actual: z.string(),
+    }),
+    flow: flowContextSchema,
+  }),
+  z.object({
+    type: z.literal(UploadEventType.UPLOAD_VALIDATION_WARNING),
+    data: z.object({
+      id: z.string(),
+      message: z.string(),
+    }),
+    flow: flowContextSchema,
+  }),
+]);
+
+export type UploadEvent = z.infer<typeof uploadEventSchema>;

package/src/types/upload-file.ts

@@ -0,0 +1,139 @@
+import { z } from "zod";
+
+/**
+ * Zod schema for validating UploadFile objects.
+ *
+ * This schema defines the structure and validation rules for upload file metadata.
+ * Use this schema to parse and validate UploadFile data from external sources.
+ *
+ * @see {@link UploadFile} for the TypeScript type
+ */
+export const uploadFileSchema = z.object({
+  id: z.string(),
+  size: z.number().optional(),
+  offset: z.number(),
+  metadata: z.record(z.string(), z.union([z.string(), z.number(), z.boolean()])).optional(),
+  creationDate: z.string().optional(),
+  url: z.string().optional(),
+  sizeIsDeferred: z.boolean().optional(),
+  checksum: z.string().optional(),
+  checksumAlgorithm: z.string().optional(),
+  storage: z.object({
+    id: z.string(),
+    type: z.string(),
+    path: z.string().optional(),
+    uploadId: z.string().optional(),
+    bucket: z.string().optional(),
+  }),
+  flow: z
+    .object({
+      flowId: z.string(),
+      nodeId: z.string(),
+      jobId: z.string(),
+    })
+    .optional(),
+});
+
+/**
+ * Represents an uploaded file with its metadata and storage information.
+ *
+ * This is the core data structure that tracks file uploads throughout their lifecycle.
+ * It contains all metadata needed to resume uploads, track progress, and locate files
+ * in storage backends.
+ *
+ * @property id - Unique identifier for this upload
+ * @property offset - Current byte offset (how many bytes have been uploaded)
+ * @property storage - Storage backend information
+ * @property storage.id - Storage backend identifier (e.g., "s3-production")
+ * @property storage.type - Storage backend type (e.g., "s3", "azure", "gcs")
+ * @property storage.path - Optional path prefix within the storage backend
+ * @property storage.uploadId - Optional backend-specific upload ID (e.g., S3 multipart upload ID)
+ * @property storage.bucket - Optional bucket or container name
+ * @property flow - Optional flow processing information (when file is part of a flow)
+ * @property flow.flowId - ID of the flow processing this file
+ * @property flow.nodeId - ID of the flow node that created this file
+ * @property flow.jobId - ID of the flow job execution
+ * @property size - Total file size in bytes (undefined if deferred)
+ * @property metadata - Custom key-value metadata attached to the file
+ * @property creationDate - ISO 8601 timestamp when upload was created
+ * @property url - Optional public URL to access the file
+ * @property sizeIsDeferred - True if file size is not known at upload start
+ * @property checksum - Optional file checksum/hash value
+ * @property checksumAlgorithm - Algorithm used for checksum (e.g., "md5", "sha256")
+ *
+ * @example
+ * ```typescript
+ * // Create an UploadFile for a new upload
+ * const uploadFile: UploadFile = {
+ *   id: "upload_abc123",
+ *   offset: 0,
+ *   size: 1024000,
+ *   storage: {
+ *     id: "s3-production",
+ *     type: "s3",
+ *     bucket: "my-uploads",
+ *     path: "files/"
+ *   },
+ *   metadata: {
+ *     fileName: "image.jpg",
+ *     contentType: "image/jpeg",
+ *     userId: "user_123"
+ *   },
+ *   creationDate: new Date().toISOString(),
+ *   checksum: "5d41402abc4b2a76b9719d911017c592",
+ *   checksumAlgorithm: "md5"
+ * };
+ *
+ * // UploadFile with flow processing
+ * const flowFile: UploadFile = {
+ *   id: "upload_xyz789",
+ *   offset: 0,
+ *   size: 2048000,
+ *   storage: {
+ *     id: "s3-temp",
+ *     type: "s3",
+ *     bucket: "temp-processing"
+ *   },
+ *   flow: {
+ *     flowId: "flow_resize_optimize",
+ *     nodeId: "input_1",
+ *     jobId: "job_456"
+ *   }
+ * };
+ *
+ * // Resume an interrupted upload
+ * const resumingFile: UploadFile = {
+ *   id: "upload_resume",
+ *   offset: 524288, // Already uploaded 512KB
+ *   size: 1024000,
+ *   storage: {
+ *     id: "s3-production",
+ *     type: "s3",
+ *     uploadId: "multipart_xyz" // S3 multipart upload ID
+ *   }
+ * };
+ * ```
+ */
+export type UploadFile = {
+  id: string;
+  offset: number;
+  storage: {
+    id: string;
+    type: string;
+    path?: string | undefined;
+    uploadId?: string | undefined;
+    bucket?: string | undefined;
+  };
+  flow?: {
+    flowId: string;
+    nodeId: string;
+    jobId: string;
+  };
+  size?: number | undefined;
+  metadata?: Record<string, string | number | boolean> | undefined;
+  creationDate?: string | undefined;
+  url?: string | undefined;
+  sizeIsDeferred?: boolean | undefined;
+  checksum?: string | undefined;
+  checksumAlgorithm?: string | undefined;
+};

package/src/types/websocket.ts

@@ -0,0 +1,65 @@
+import z from "zod";
+import { uploadEventSchema } from "./upload-event";
+
+/**
+ * Platform-agnostic WebSocket connection interface
+ */
+export interface WebSocketConnection {
+  send(data: string): void;
+  close(code?: number, reason?: string): void;
+  readonly readyState: number;
+  readonly id: string;
+}
+
+/**
+ * WebSocket message that can be sent/received
+ */
+
+export const webSocketMessageSchema = z.union([
+  z.object({
+    type: z.literal("upload_event"),
+    payload: uploadEventSchema,
+    timestamp: z.string().optional(),
+  }),
+  z.object({
+    type: z.literal("flow_event"),
+    payload: z.any(), // FlowEvent doesn't have a zod schema, using z.any() for now
+    timestamp: z.string().optional(),
+  }),
+  z.object({
+    type: z.literal("subscribed"),
+    payload: z.object({ eventKey: z.string() }),
+    timestamp: z.string().optional(),
+  }),
+  z.object({
+    type: z.literal("error"),
+    message: z.string().optional(),
+  }),
+  z.object({
+    type: z.literal("pong"),
+    timestamp: z.string().optional(),
+  }),
+  z.object({
+    type: z.literal("ping"),
+    timestamp: z.string().optional(),
+  }),
+  z.object({
+    type: z.literal("connection"),
+    message: z.string().optional(),
+    uploadId: z.string().optional(),
+    timestamp: z.string().optional(),
+  }),
+]);
+
+export type WebSocketMessage<TEvent = unknown> =
+  | z.infer<typeof webSocketMessageSchema>
+  | {
+      type: "upload_event";
+      payload: TEvent;
+      timestamp?: string;
+    }
+  | {
+      type: "flow_event";
+      payload: TEvent;
+      timestamp?: string;
+    };