@uploadista/server 0.0.18-beta.4 → 0.0.18-beta.6

package/src/core/http-handlers/dlq-http-handlers.ts

@@ -0,0 +1,177 @@
+import { DeadLetterQueueService } from "@uploadista/core/flow";
+import { Effect } from "effect";
+import type {
+  DlqCleanupRequest,
+  DlqCleanupResponse,
+  DlqDeleteRequest,
+  DlqDeleteResponse,
+  DlqGetRequest,
+  DlqGetResponse,
+  DlqListRequest,
+  DlqListResponse,
+  DlqResolveRequest,
+  DlqResolveResponse,
+  DlqRetryAllRequest,
+  DlqRetryAllResponse,
+  DlqRetryRequest,
+  DlqRetryResponse,
+  DlqStatsRequest,
+  DlqStatsResponse,
+} from "../routes";
+
+/**
+ * Handle GET /api/dlq - List DLQ items
+ */
+export const handleDlqList = (req: DlqListRequest) =>
+  Effect.gen(function* () {
+    const dlq = yield* DeadLetterQueueService;
+    const result = yield* dlq.list(req.options);
+
+    return {
+      type: "dlq-list",
+      status: 200,
+      headers: { "Content-Type": "application/json" },
+      body: result,
+    } satisfies DlqListResponse;
+  });
+
+/**
+ * Handle GET /api/dlq/:itemId - Get a specific DLQ item
+ */
+export const handleDlqGet = (req: DlqGetRequest) =>
+  Effect.gen(function* () {
+    const dlq = yield* DeadLetterQueueService;
+    const item = yield* dlq.get(req.itemId);
+
+    return {
+      type: "dlq-get",
+      status: 200,
+      headers: { "Content-Type": "application/json" },
+      body: item,
+    } satisfies DlqGetResponse;
+  });
+
+/**
+ * Handle POST /api/dlq/:itemId/retry - Retry a specific DLQ item
+ */
+export const handleDlqRetry = (req: DlqRetryRequest) =>
+  Effect.gen(function* () {
+    const dlq = yield* DeadLetterQueueService;
+
+    // Mark item as retrying
+    yield* dlq.markRetrying(req.itemId);
+
+    // TODO: Implement actual retry logic by re-executing the flow
+    // This would require access to FlowServer and the original job context
+    // For now, we just mark it as retrying and return success
+    // The actual retry would be handled by a background scheduler
+
+    return {
+      type: "dlq-retry",
+      status: 200,
+      headers: { "Content-Type": "application/json" },
+      body: { success: true },
+    } satisfies DlqRetryResponse;
+  });
+
+/**
+ * Handle POST /api/dlq/retry-all - Retry all matching DLQ items
+ */
+export const handleDlqRetryAll = (req: DlqRetryAllRequest) =>
+  Effect.gen(function* () {
+    const dlq = yield* DeadLetterQueueService;
+
+    // List items matching the filter
+    const { items } = yield* dlq.list({
+      status: req.options?.status,
+      flowId: req.options?.flowId,
+    });
+
+    let succeeded = 0;
+    let failed = 0;
+
+    // Mark each item for retry
+    for (const item of items) {
+      const result = yield* Effect.either(dlq.markRetrying(item.id));
+      if (result._tag === "Right") {
+        succeeded++;
+      } else {
+        failed++;
+      }
+    }
+
+    return {
+      type: "dlq-retry-all",
+      status: 200,
+      headers: { "Content-Type": "application/json" },
+      body: {
+        retried: items.length,
+        succeeded,
+        failed,
+      },
+    } satisfies DlqRetryAllResponse;
+  });
+
+/**
+ * Handle DELETE /api/dlq/:itemId - Delete a DLQ item
+ */
+export const handleDlqDelete = (req: DlqDeleteRequest) =>
+  Effect.gen(function* () {
+    const dlq = yield* DeadLetterQueueService;
+    yield* dlq.delete(req.itemId);
+
+    return {
+      type: "dlq-delete",
+      status: 200,
+      headers: { "Content-Type": "application/json" },
+      body: { success: true },
+    } satisfies DlqDeleteResponse;
+  });
+
+/**
+ * Handle POST /api/dlq/:itemId/resolve - Manually resolve a DLQ item
+ */
+export const handleDlqResolve = (req: DlqResolveRequest) =>
+  Effect.gen(function* () {
+    const dlq = yield* DeadLetterQueueService;
+    const item = yield* dlq.markResolved(req.itemId);
+
+    return {
+      type: "dlq-resolve",
+      status: 200,
+      headers: { "Content-Type": "application/json" },
+      body: item,
+    } satisfies DlqResolveResponse;
+  });
+
+/**
+ * Handle POST /api/dlq/cleanup - Cleanup old DLQ items
+ */
+export const handleDlqCleanup = (req: DlqCleanupRequest) =>
+  Effect.gen(function* () {
+    const dlq = yield* DeadLetterQueueService;
+    const result = yield* dlq.cleanup(req.options);
+
+    return {
+      type: "dlq-cleanup",
+      status: 200,
+      headers: { "Content-Type": "application/json" },
+      body: result,
+    } satisfies DlqCleanupResponse;
+  });
+
+/**
+ * Handle GET /api/dlq/stats - Get DLQ statistics
+ */
+export const handleDlqStats = (_req: DlqStatsRequest) =>
+  Effect.gen(function* () {
+    const dlq = yield* DeadLetterQueueService;
+    const stats = yield* dlq.getStats();
+
+    return {
+      type: "dlq-stats",
+      status: 200,
+      headers: { "Content-Type": "application/json" },
+      body: stats,
+    } satisfies DlqStatsResponse;
+  });

package/src/core/http-handlers/http-handlers.ts

@@ -1,5 +1,15 @@
 import { Effect } from "effect";
 import type { UploadistaRequest, UploadistaResponse } from "../routes";
+import {
+  handleDlqCleanup,
+  handleDlqDelete,
+  handleDlqGet,
+  handleDlqList,
+  handleDlqResolve,
+  handleDlqRetry,
+  handleDlqRetryAll,
+  handleDlqStats,
+} from "./dlq-http-handlers";
 import {
   handleCancelFlow,
   handleGetFlow,
@@ -44,6 +54,23 @@ export const handleUploadistaRequest = <TRequirements>(
         return (yield* handlePauseFlow(req)) as UploadistaResponse;
       case "cancel-flow":
         return (yield* handleCancelFlow(req)) as UploadistaResponse;
+      // DLQ Admin routes
+      case "dlq-list":
+        return (yield* handleDlqList(req)) as UploadistaResponse;
+      case "dlq-get":
+        return (yield* handleDlqGet(req)) as UploadistaResponse;
+      case "dlq-retry":
+        return (yield* handleDlqRetry(req)) as UploadistaResponse;
+      case "dlq-retry-all":
+        return (yield* handleDlqRetryAll(req)) as UploadistaResponse;
+      case "dlq-delete":
+        return (yield* handleDlqDelete(req)) as UploadistaResponse;
+      case "dlq-resolve":
+        return (yield* handleDlqResolve(req)) as UploadistaResponse;
+      case "dlq-cleanup":
+        return (yield* handleDlqCleanup(req)) as UploadistaResponse;
+      case "dlq-stats":
+        return (yield* handleDlqStats(req)) as UploadistaResponse;
       case "not-found":
         return {
           status: 404,

package/src/core/routes.ts

@@ -1,5 +1,11 @@
 import type {
   DataStoreCapabilities,
+  DeadLetterCleanupOptions,
+  DeadLetterCleanupResult,
+  DeadLetterItem,
+  DeadLetterItemStatus,
+  DeadLetterListOptions,
+  DeadLetterQueueStats,
   FlowData,
   FlowJob,
   UploadFile,
@@ -17,6 +23,16 @@ export type UploadistaRouteType =
   | "resume-flow"
   | "pause-flow"
   | "cancel-flow"
+  // DLQ Admin routes
+  | "dlq-list"
+  | "dlq-get"
+  | "dlq-retry"
+  | "dlq-retry-all"
+  | "dlq-delete"
+  | "dlq-resolve"
+  | "dlq-cleanup"
+  | "dlq-stats"
+  // Error routes
   | "not-found"
   | "bad-request"
   | "method-not-allowed"
@@ -154,6 +170,69 @@ export type CancelFlowResponse = UploadistaStandardResponse<
   FlowJob
 >;
 
+// ============================================================================
+// Dead Letter Queue Admin Routes
+// ============================================================================
+
+export type DlqListRequest = UploadistaRoute<"dlq-list"> & {
+  options?: DeadLetterListOptions;
+};
+export type DlqListResponse = UploadistaStandardResponse<
+  "dlq-list",
+  { items: DeadLetterItem[]; total: number }
+>;
+
+export type DlqGetRequest = UploadistaRoute<"dlq-get"> & {
+  itemId: string;
+};
+export type DlqGetResponse = UploadistaStandardResponse<"dlq-get", DeadLetterItem>;
+
+export type DlqRetryRequest = UploadistaRoute<"dlq-retry"> & {
+  itemId: string;
+};
+export type DlqRetryResponse = UploadistaStandardResponse<
+  "dlq-retry",
+  { success: boolean; newJobId?: string }
+>;
+
+export type DlqRetryAllRequest = UploadistaRoute<"dlq-retry-all"> & {
+  options?: { status?: DeadLetterItemStatus; flowId?: string };
+};
+export type DlqRetryAllResponse = UploadistaStandardResponse<
+  "dlq-retry-all",
+  { retried: number; succeeded: number; failed: number }
+>;
+
+export type DlqDeleteRequest = UploadistaRoute<"dlq-delete"> & {
+  itemId: string;
+};
+export type DlqDeleteResponse = UploadistaStandardResponse<
+  "dlq-delete",
+  { success: boolean }
+>;
+
+export type DlqResolveRequest = UploadistaRoute<"dlq-resolve"> & {
+  itemId: string;
+};
+export type DlqResolveResponse = UploadistaStandardResponse<
+  "dlq-resolve",
+  DeadLetterItem
+>;
+
+export type DlqCleanupRequest = UploadistaRoute<"dlq-cleanup"> & {
+  options?: DeadLetterCleanupOptions;
+};
+export type DlqCleanupResponse = UploadistaStandardResponse<
+  "dlq-cleanup",
+  DeadLetterCleanupResult
+>;
+
+export type DlqStatsRequest = UploadistaRoute<"dlq-stats">;
+export type DlqStatsResponse = UploadistaStandardResponse<
+  "dlq-stats",
+  DeadLetterQueueStats
+>;
+
 export type UploadistaRequest =
   | CreateUploadRequest
   | GetCapabilitiesRequest
@@ -165,6 +244,16 @@ export type UploadistaRequest =
   | ResumeFlowRequest
   | PauseFlowRequest
   | CancelFlowRequest
+  // DLQ Admin requests
+  | DlqListRequest
+  | DlqGetRequest
+  | DlqRetryRequest
+  | DlqRetryAllRequest
+  | DlqDeleteRequest
+  | DlqResolveRequest
+  | DlqCleanupRequest
+  | DlqStatsRequest
+  // Error requests
   | NotFoundRequest
   | BadRequestRequest
   | MethodNotAllowedRequest
@@ -181,6 +270,16 @@ export type UploadistaResponse =
   | ResumeFlowResponse
   | PauseFlowResponse
   | CancelFlowResponse
+  // DLQ Admin responses
+  | DlqListResponse
+  | DlqGetResponse
+  | DlqRetryResponse
+  | DlqRetryAllResponse
+  | DlqDeleteResponse
+  | DlqResolveResponse
+  | DlqCleanupResponse
+  | DlqStatsResponse
+  // Error responses
   | NotFoundResponse
   | BadRequestResponse
   | MethodNotAllowedResponse

package/src/core/server.ts

@@ -1,5 +1,6 @@
 import type { PluginLayer, UploadistaError } from "@uploadista/core";
 import {
+  deadLetterQueueService,
   type Flow,
   FlowProvider,
   FlowWaitUntil,
@@ -7,6 +8,7 @@ import {
 } from "@uploadista/core/flow";
 import {
   createDataStoreLayer,
+  deadLetterQueueKvStore,
   type UploadFileDataStores,
   type UploadFileKVStore,
 } from "@uploadista/core/types";
@@ -197,6 +199,7 @@ export const createUploadistaServer = async <
   adapter,
   authCacheConfig,
   circuitBreaker = true,
+  deadLetterQueue = false,
 }: UploadistaServerConfig<
   TContext,
   TResponse,
@@ -272,22 +275,30 @@ export const createUploadistaServer = async <
     ? kvCircuitBreakerStoreLayer.pipe(Layer.provide(kvStore))
     : null;
 
+  // Create dead letter queue layer if enabled (uses the provided kvStore)
+  // The DLQ layer provides both the KV store wrapper and the service
+  const dlqLayer = deadLetterQueue
+    ? deadLetterQueueService.pipe(
+        Layer.provide(deadLetterQueueKvStore),
+        Layer.provide(kvStore),
+      )
+    : null;
+
   /**
    * Merge all server layers including plugins.
    *
    * This combines the core server infrastructure (upload server, flow server,
-   * metrics, auth cache, circuit breaker) with user-provided plugin layers.
+   * metrics, auth cache, circuit breaker, dead letter queue) with user-provided plugin layers.
    */
-  const baseServerLayer = Layer.mergeAll(
+  const serverLayerRaw = Layer.mergeAll(
     uploadServerLayer,
     flowServerLayer,
     effectiveMetricsLayer,
     authCacheLayer,
     ...plugins,
+    ...(circuitBreakerStoreLayer ? [circuitBreakerStoreLayer] : []),
+    ...(dlqLayer ? [dlqLayer] : []),
   );
-  const serverLayerRaw = circuitBreakerStoreLayer
-    ? Layer.merge(baseServerLayer, circuitBreakerStoreLayer)
-    : baseServerLayer;
 
   /**
    * Type Casting Rationale for Plugin System
@@ -485,7 +496,7 @@ export const createUploadistaServer = async <
         }
       }
 
-      // Combine auth context, auth cache, metrics layers, plugins, circuit breaker, and waitUntil
+      // Combine auth context, auth cache, metrics layers, plugins, circuit breaker, DLQ, and waitUntil
       // This ensures that flow nodes have access to all required services
       const baseRequestContextLayer = Layer.mergeAll(
         authContextLayer,
@@ -494,9 +505,12 @@ export const createUploadistaServer = async <
         ...plugins,
         ...waitUntilLayers,
       );
-      const requestContextLayer = circuitBreakerStoreLayer
+      const withCircuitBreakerContext = circuitBreakerStoreLayer
         ? Layer.merge(baseRequestContextLayer, circuitBreakerStoreLayer)
         : baseRequestContextLayer;
+      const requestContextLayer = dlqLayer
+        ? Layer.merge(withCircuitBreakerContext, dlqLayer)
+        : withCircuitBreakerContext;
 
       // Check for baseUrl/api/ prefix
       if (uploadistaRequest.type === "not-found") {

package/src/core/types.ts

@@ -323,6 +323,34 @@ export interface UploadistaServerConfig<
    * ```
    */
   circuitBreaker?: boolean;
+
+  /**
+   * Optional: Enable dead letter queue for failed flow jobs.
+   *
+   * When enabled, failed flow jobs are captured in a DLQ with full context
+   * for debugging and retry. The DLQ state is stored in the KV store,
+   * allowing it to be shared across multiple server instances.
+   *
+   * Set to `false` to disable the DLQ entirely.
+   *
+   * @default false
+   *
+   * @example
+   * ```typescript
+   * // Enable DLQ (uses the provided kvStore)
+   * const server = await createUploadistaServer({
+   *   kvStore: redisKvStore,
+   *   deadLetterQueue: true
+   * });
+   *
+   * // DLQ is disabled by default
+   * const server = await createUploadistaServer({
+   *   kvStore: redisKvStore,
+   *   // deadLetterQueue: false (default)
+   * });
+   * ```
+   */
+  deadLetterQueue?: boolean;
 }
 
 /**