@uploadista/core 0.2.0 → 1.0.0-beta.3

package/src/flow/types/flow-queue-item.ts

@@ -0,0 +1,154 @@
+/**
+ * Flow Queue item types and configuration.
+ *
+ * A FlowQueueItem represents a queued flow execution request, tracking its
+ * lifecycle from pending → running → completed | failed.
+ *
+ * @module flow/types/flow-queue-item
+ * @see {@link FlowQueueService} for queue operations
+ */
+
+/**
+ * Status of a flow queue item.
+ *
+ * Item lifecycle: pending → running → completed | failed
+ *
+ * - `pending`: Waiting for a concurrency slot to become available
+ * - `running`: Currently being executed by the flow engine
+ * - `completed`: Flow execution finished successfully
+ * - `failed`: Flow execution ended with an error
+ */
+export type FlowQueueItemStatus = "pending" | "running" | "completed" | "failed";
+
+/**
+ * Represents a single queued flow execution request.
+ *
+ * FlowQueueItems are created when a caller enqueues a flow for execution.
+ * The worker loop picks up pending items, transitions them to running, and
+ * dispatches them to FlowEngine. On completion or failure the item is updated.
+ *
+ * @property id - Unique queue item identifier (UUID)
+ * @property flowId - The flow definition to execute
+ * @property storageId - Target storage for flow outputs
+ * @property input - Original input payload passed to the flow
+ * @property clientId - Client who initiated the request (null for anonymous)
+ * @property status - Current lifecycle status of the queue item
+ * @property dlqItemId - Set when this item is a DLQ retry; links back to the DLQ item
+ * @property enqueuedAt - When the item was added to the queue
+ * @property startedAt - When the worker started executing this item
+ * @property completedAt - When execution finished (success or failure)
+ * @property error - Error message recorded if status is "failed"
+ *
+ * @example
+ * ```typescript
+ * const item: FlowQueueItem = {
+ *   id: "q_abc123",
+ *   flowId: "image-resize-pipeline",
+ *   storageId: "s3-production",
+ *   input: { input: { uploadId: "upload_xyz" } },
+ *   clientId: "client_456",
+ *   status: "pending",
+ *   enqueuedAt: new Date(),
+ * };
+ * ```
+ */
+export interface FlowQueueItem {
+  /** Unique queue item identifier (UUID) */
+  id: string;
+  /** The flow definition to execute */
+  flowId: string;
+  /** Target storage for flow outputs */
+  storageId: string;
+  /** Original input payload passed to the flow */
+  input: unknown;
+  /** Client who initiated the request (null for anonymous) */
+  clientId: string | null;
+  /** Current lifecycle status */
+  status: FlowQueueItemStatus;
+  /** Set when this is a DLQ retry; references the DLQ item for result correlation */
+  dlqItemId?: string;
+  /** When the item was added to the queue */
+  enqueuedAt: Date;
+  /** When the worker began executing this item */
+  startedAt?: Date;
+  /** When execution finished (success or failure) */
+  completedAt?: Date;
+  /** Error message if status is "failed" */
+  error?: string;
+}
+
+/**
+ * Aggregate statistics about the flow queue.
+ *
+ * Provides counts by status and concurrency information for monitoring.
+ *
+ * @property pending - Number of items waiting for a concurrency slot
+ * @property running - Number of items currently being executed
+ * @property completed - Number of items that finished successfully
+ * @property failed - Number of items that ended with an error
+ * @property maxConcurrency - Configured maximum simultaneous executions
+ * @property currentConcurrency - Number of items currently running
+ */
+export interface FlowQueueStats {
+  /** Number of items waiting for a concurrency slot */
+  pending: number;
+  /** Number of items currently being executed */
+  running: number;
+  /** Number of items that finished successfully */
+  completed: number;
+  /** Number of items that ended with an error */
+  failed: number;
+  /** Configured maximum simultaneous executions */
+  maxConcurrency: number;
+  /** Number of items currently running */
+  currentConcurrency: number;
+}
+
+/**
+ * Configuration options for the FlowQueueService.
+ *
+ * All fields are optional; defaults are applied via DEFAULT_QUEUE_CONFIG.
+ *
+ * @property maxConcurrency - Maximum number of simultaneously running flows (default: 4)
+ * @property dlqRetryIntervalMs - How often the DLQ retry loop fires in milliseconds (default: 30_000)
+ * @property dlqRetryBatchSize - Maximum DLQ items processed per retry loop tick (default: 10)
+ *
+ * @example
+ * ```typescript
+ * const config: FlowQueueConfig = {
+ *   maxConcurrency: 8,
+ *   dlqRetryIntervalMs: 60_000, // check every 60 seconds
+ *   dlqRetryBatchSize: 5,
+ * };
+ * ```
+ */
+export interface FlowQueueConfig {
+  /**
+   * Maximum number of simultaneously running flows.
+   * Flows beyond this limit remain pending until a slot opens.
+   * @default 4
+   */
+  maxConcurrency?: number;
+  /**
+   * Interval in milliseconds between DLQ retry loop ticks.
+   * Only relevant when DeadLetterQueueService is present.
+   * @default 30_000
+   */
+  dlqRetryIntervalMs?: number;
+  /**
+   * Maximum number of DLQ items to re-enqueue per retry loop tick.
+   * @default 10
+   */
+  dlqRetryBatchSize?: number;
+}
+
+/**
+ * Default configuration values for FlowQueueService.
+ *
+ * Applied when specific config fields are omitted.
+ */
+export const DEFAULT_QUEUE_CONFIG: Required<FlowQueueConfig> = {
+  maxConcurrency: 4,
+  dlqRetryIntervalMs: 30_000,
+  dlqRetryBatchSize: 10,
+};

package/src/types/data-store.ts

@@ -213,10 +213,10 @@ export type DataStoreCapabilities = {
  *   write: ({ file_id, stream, offset }, { onProgress }) => Effect.gen(function* () {
  *     // Write chunks to storage
  *     let bytesWritten = offset;
- *     yield* Stream.runForEach(stream, (chunk) => Effect.sync(() => {
+ *     yield* Stream.runForEach(stream, (chunk) => Effect.gen(function* () {
  *       writeChunk(file_id, chunk, bytesWritten);
  *       bytesWritten += chunk.byteLength;
- *       onProgress?.(chunk.byteLength);
+ *       yield* (onProgress?.(bytesWritten) ?? Effect.void);
  *     }));
  *     return bytesWritten;
  *   }),
@@ -286,7 +286,7 @@ export type DataStore<TData = unknown> = {
   readonly write: (
     options: DataStoreWriteOptions,
     dependencies: {
-      onProgress?: (chunkSize: number) => void;
+      onProgress?: (offset: number) => Effect.Effect<void>;
     },
   ) => Effect.Effect<number, UploadistaError>;
   /**

package/src/types/kv-store.ts

@@ -1,6 +1,8 @@
 import { Context, Effect, Layer } from "effect";
 import { UploadistaError } from "../errors";
-import type { DeadLetterItem, FlowJob } from "../flow";
+import type { DeadLetterItem } from "../flow/types/dead-letter-item";
+import type { FlowJob } from "../flow/types/flow-job";
+import type { FlowQueueItem } from "../flow/types/flow-queue-item";
 import type { UploadFile } from "./upload-file";
 
 /**
@@ -454,3 +456,31 @@ export const deadLetterQueueKvStore = Layer.effect(
     );
   }),
 );
+
+/**
+ * Effect-TS context tag for the FlowQueueItem typed KV store.
+ */
+export class FlowQueueKVStore extends Context.Tag("FlowQueueKVStore")<
+  FlowQueueKVStore,
+  KvStore<FlowQueueItem>
+>() {}
+
+/**
+ * Effect Layer that creates the FlowQueueKVStore from a BaseKvStore.
+ *
+ * Stores queue items as JSON under the "uploadista:queue-item:" prefix.
+ * Used by FlowQueueService.fromKvStore() so the queue can be backed by
+ * any BaseKvStoreService (Redis, filesystem, Cloudflare KV, etc.).
+ */
+export const flowQueueKvStore = Layer.effect(
+  FlowQueueKVStore,
+  Effect.gen(function* () {
+    const baseStore = yield* BaseKvStoreService;
+    return new TypedKvStore<FlowQueueItem>(
+      baseStore,
+      "uploadista:queue-item:",
+      jsonSerializer.serialize,
+      jsonSerializer.deserialize,
+    );
+  }),
+);

package/src/upload/write-to-store.ts

@@ -1,4 +1,4 @@
-import { Effect, Ref } from "effect";
+import { Effect } from "effect";
 import { UploadistaError } from "../errors";
 import { StreamLimiterEffect } from "../streams/stream-limiter";
 import type { DataStore, UploadEvent, UploadFile } from "../types";
@@ -108,8 +108,13 @@ export function writeToStore({
       Effect.sync(() => ({ signal, onAbort })),
       ({ signal: _signal }) =>
         Effect.gen(function* () {
-          // Create a ref to track the last progress emission time for throttling
-          const lastEmitTime = yield* Ref.make(0);
+          // Plain mutable variable for throttle timing.
+          // Must NOT use a Ref here: the onProgress callback is synchronous and
+          // called from within the data-store's stream fiber. Using Effect.runPromise
+          // inside onProgress causes a race condition where all calls capture the same
+          // `now` timestamp before any update runs, so only 1 event fires per PATCH.
+          // A plain let variable updates synchronously, preventing the race.
+          let lastEmitTimeMs = 0;
 
           // Create the stream limiter
           const limiter = StreamLimiterEffect.limit({
@@ -127,33 +132,23 @@ export function writeToStore({
               offset: upload.offset,
             },
             {
-              onProgress: (newOffset: number) => {
-                // Simple throttling using timestamp check
+              onProgress: (newOffset: number): Effect.Effect<void> => {
                 const now = Date.now();
-                Ref.get(lastEmitTime)
-                  .pipe(
-                    Effect.flatMap((lastTime) => {
-                      if (now - lastTime >= uploadProgressInterval) {
-                        return Effect.gen(function* () {
-                          yield* Ref.set(lastEmitTime, now);
-                          yield* eventEmitter.emit(upload.id, {
-                            type: UploadEventType.UPLOAD_PROGRESS,
-                            data: {
-                              id: upload.id,
-                              progress: newOffset,
-                              total: upload.size ?? 0,
-                            },
-                            flow: upload.flow,
-                          });
-                        });
-                      }
-                      return Effect.void;
-                    }),
-                    Effect.runPromise,
-                  )
-                  .catch(() => {
-                    // Ignore errors during progress emission
-                  });
+                if (now - lastEmitTimeMs >= uploadProgressInterval) {
+                  lastEmitTimeMs = now;
+                  return eventEmitter
+                    .emit(upload.id, {
+                      type: UploadEventType.UPLOAD_PROGRESS,
+                      data: {
+                        id: upload.id,
+                        progress: newOffset,
+                        total: upload.size ?? 0,
+                      },
+                      flow: upload.flow,
+                    })
+                    .pipe(Effect.catchAll(() => Effect.void));
+                }
+                return Effect.void;
               },
             },
           );

package/tests/flow-queue-store.test.ts

@@ -0,0 +1,150 @@
+import { Effect } from "effect";
+import { describe, expect, it, beforeEach } from "vitest";
+import { MemoryFlowQueueStore } from "../src/flow/flow-queue-store";
+import type { FlowQueueItem } from "../src/flow/types/flow-queue-item";
+
+function makeItem(overrides: Partial<FlowQueueItem> = {}): FlowQueueItem {
+  return {
+    id: `q_${Math.random().toString(36).slice(2)}`,
+    flowId: "test-flow",
+    storageId: "s3-test",
+    input: { files: ["a.png"] },
+    clientId: "client-1",
+    status: "pending",
+    enqueuedAt: new Date("2024-01-01T10:00:00Z"),
+    ...overrides,
+  };
+}
+
+describe("MemoryFlowQueueStore", () => {
+  let store: MemoryFlowQueueStore;
+
+  beforeEach(() => {
+    store = new MemoryFlowQueueStore();
+  });
+
+  describe("createItem", () => {
+    it("stores and returns the item", async () => {
+      const item = makeItem();
+      const result = await Effect.runPromise(store.createItem(item));
+      expect(result).toMatchObject({ id: item.id, status: "pending" });
+    });
+
+    it("allows creating multiple items", async () => {
+      const a = makeItem({ id: "q_a" });
+      const b = makeItem({ id: "q_b" });
+      await Effect.runPromise(
+        Effect.all([store.createItem(a), store.createItem(b)]),
+      );
+      const pending = await Effect.runPromise(store.listByStatus("pending"));
+      expect(pending).toHaveLength(2);
+    });
+  });
+
+  describe("getItem", () => {
+    it("returns the item when it exists", async () => {
+      const item = makeItem({ id: "q_known" });
+      await Effect.runPromise(store.createItem(item));
+      const result = await Effect.runPromise(store.getItem("q_known"));
+      expect(result).not.toBeNull();
+      expect(result!.id).toBe("q_known");
+    });
+
+    it("returns null for unknown IDs", async () => {
+      const result = await Effect.runPromise(store.getItem("q_unknown"));
+      expect(result).toBeNull();
+    });
+  });
+
+  describe("updateItem", () => {
+    it("applies partial updates", async () => {
+      const item = makeItem({ id: "q_upd" });
+      await Effect.runPromise(store.createItem(item));
+
+      const updated = await Effect.runPromise(
+        store.updateItem("q_upd", { status: "running", startedAt: new Date() }),
+      );
+
+      expect(updated.status).toBe("running");
+      expect(updated.startedAt).toBeInstanceOf(Date);
+      expect(updated.flowId).toBe("test-flow"); // unchanged
+    });
+
+    it("fails with FLOW_JOB_NOT_FOUND for unknown ID", async () => {
+      const result = await Effect.runPromise(
+        Effect.either(store.updateItem("q_missing", { status: "running" })),
+      );
+      expect(result._tag).toBe("Left");
+      if (result._tag === "Left") {
+        expect(result.left.code).toBe("FLOW_JOB_NOT_FOUND");
+      }
+    });
+  });
+
+  describe("listByStatus", () => {
+    it("filters by status correctly", async () => {
+      const pending = makeItem({ id: "q_p1", status: "pending" });
+      const running = makeItem({ id: "q_r1", status: "running" });
+      const completed = makeItem({ id: "q_c1", status: "completed" });
+      await Effect.runPromise(
+        Effect.all([
+          store.createItem(pending),
+          store.createItem(running),
+          store.createItem(completed),
+        ]),
+      );
+
+      const pendingList = await Effect.runPromise(
+        store.listByStatus("pending"),
+      );
+      expect(pendingList).toHaveLength(1);
+      expect(pendingList[0].id).toBe("q_p1");
+
+      const runningList = await Effect.runPromise(
+        store.listByStatus("running"),
+      );
+      expect(runningList).toHaveLength(1);
+      expect(runningList[0].id).toBe("q_r1");
+    });
+
+    it("returns pending items in FIFO order (oldest first)", async () => {
+      const early = makeItem({
+        id: "q_early",
+        enqueuedAt: new Date("2024-01-01T09:00:00Z"),
+      });
+      const late = makeItem({
+        id: "q_late",
+        enqueuedAt: new Date("2024-01-01T11:00:00Z"),
+      });
+      // Insert late first to verify sorting
+      await Effect.runPromise(
+        Effect.all([store.createItem(late), store.createItem(early)]),
+      );
+
+      const pending = await Effect.runPromise(store.listByStatus("pending"));
+      expect(pending[0].id).toBe("q_early");
+      expect(pending[1].id).toBe("q_late");
+    });
+
+    it("returns empty array when no items match status", async () => {
+      const pending = await Effect.runPromise(store.listByStatus("completed"));
+      expect(pending).toHaveLength(0);
+    });
+  });
+
+  describe("deleteItem", () => {
+    it("removes the item", async () => {
+      const item = makeItem({ id: "q_del" });
+      await Effect.runPromise(store.createItem(item));
+      await Effect.runPromise(store.deleteItem("q_del"));
+      const result = await Effect.runPromise(store.getItem("q_del"));
+      expect(result).toBeNull();
+    });
+
+    it("is idempotent for unknown IDs", async () => {
+      await expect(
+        Effect.runPromise(store.deleteItem("q_nonexistent")),
+      ).resolves.toBeUndefined();
+    });
+  });
+});