@vercel/queue 0.0.0-alpha.4 → 0.0.0-alpha.6

package/README.md

@@ -89,21 +89,11 @@ try {
 }
 ```
 
-Run the script
-
-```bash
-# Install dotenv-cli and ts-node if you need it
-npm i -g dotenv-cli ts-node typescript
-
-# Run the script with the OIDC token
-dotenv -e .env.local ts-node index.ts
-```
-
 ## Usage with Vercel
 
 When deploying on Vercel, rather than having a persistent server subscribed to a
-queue, Vercel can trigger a callback route when a message is ready for
-consumption.
+queue, Vercel can automatically trigger your API routes when messages are ready for
+consumption based on your vercel.json configuration.
 
 To demonstrate using queues on Vercel, let's use a Next.js app. You can use an
 existing app or create one using
@@ -138,11 +128,6 @@ export async function publishTestMessage(message: string) {
   const { messageId } = await send(
     "my-topic",
     { message, timestamp: Date.now() },
-    {
-      // Provide a callback URL to invoke a consumer when the message is ready to be processed
-      callback: {
-        url: getCallbackUrl() // implementation below
-      },
     },
   );
 
@@ -159,44 +144,11 @@ export async function publishTestMessage(message: string) {
   // Publish the message
   const { messageId } = await topic.publish(
     { message, timestamp: Date.now() },
-    {
-      // Provide multiple callback URLs to invoke multiple consumer groups
-      callback: {
-        {
-          "consumer-group-1": {
-            url: getCallbackUrl()
-          },
-          "consumer-group-2": {
-            url: getCallbackUrl()
-            delay: 5 // Delay callback by 5 seconds
-          },
-        }
-      },
-    },
   );
 
   console.log(`Published message ${messageId}`);
 }
 
-// Helper function to generate a local callback URL
-function getCallbackUrl() {
-  const callbackUrl = new URL(
-    process.env.VERCEL_URL
-      ? `https://${process.env.VERCEL_URL}/api/queue/handle`
-      : "http://localhost:3000/api/queue/handle"
-  );
-
-  // Add Vercel automation bypass secret if available (for preview deployments)
-  if (process.env.VERCEL_AUTOMATION_BYPASS_SECRET) {
-    callbackUrl.searchParams.set(
-      "x-vercel-protection-bypass",
-      process.env.VERCEL_AUTOMATION_BYPASS_SECRET
-    );
-  }
-
-  return callbackUrl.toString();
-}
-
 ```
 
 Now wire up the server function to your app
@@ -218,72 +170,160 @@ export default function Page() {
 
 ### Consuming the queue
 
-Instead of running a persistent server that subscribes to the queue, we use the
-callback functionality of Vercel queues to consume messages on the fly, when a
-message is ready to be processed.
+Messages are consumed using consumer groups, which provide load balancing and parallel processing capabilities.
+
+## Usage with Vercel
+
+To consume queue messages in a Vercel deployment, you need to create (Next.js) API routes and configure them in your `vercel.json` file.
+
+### 1. Create API Routes
 
-The `handleCallback` helper function simplifies queue callback handling in NextJS:
+Create API routes to handle incoming queue messages using the `handleCallback` helper:
 
 ```typescript
 // app/api/queue/handle/route.ts
 import { handleCallback } from "@vercel/queue";
 
-// Option 1: Specify a single handler for the topic
+// Option 1: Single topic with multiple consumer groups
 export const POST = handleCallback({
-  "my-topic": (message, metadata) => {
-    console.log(`Received message:`, message, metadata);
-    // metadata: { messageId, deliveryCount, timestamp }
+  "my-topic": {
+    "consumer-group-1": async (message, metadata) => {
+      console.log(`Consumer group 1 processing:`, message, metadata);
+      // Handle consumer group 1 logic
+      await processGroup1(message);
+    },
+    "consumer-group-2": async (message, metadata) => {
+      console.log(`Consumer group 2 processing:`, message, metadata);
+      // Handle consumer group 2 logic
+      await processGroup2(message);
+    },
   },
-
-  // .. more topic handlers can be provided here
 });
 
-// This consumes messages on the "default" consumer group, which is used when no consumer groups
-// were specified in the publish `callback` earlierA
+async function processGroup1(message: any) {
+  // Consumer group 1 specific logic
+}
 
-// Option 2: Multiple consumer groups
+async function processGroup2(message: any) {
+  // Consumer group 2 specific logic
+}
+```
+
+```typescript
+// Alternative: Multiple topics in one handler
 export const POST = handleCallback({
-  // topic: "my-topic"
-  "my-topic": {
-    // consumer group: "compress"
-    "consumer-group-1": (message, metadata) => {
-      console.log("Message:", message);
+  "user-events": {
+    welcome: async (message, metadata) => {
+      console.log(`New user event:`, message, metadata);
+      await sendWelcomeEmail(message.email);
+    },
+  },
+  "order-events": {
+    fulfillment: async (order, metadata) => {
+      console.log(`Processing order:`, order, metadata);
+      await fulfillOrder(order);
     },
-    // consumer group: "resize"
-    "consume-group-2": (message, metadata) => {
-      console.log("Message", message);
+    analytics: async (order, metadata) => {
+      console.log(`Tracking order:`, order, metadata);
+      await trackOrder(order);
     },
   },
 });
 ```
 
-## Key Features
+### 2. Configure vercel.json
 
-### Streaming Support
+Create a `vercel.json` file in your project root to declare which topics and consumer groups each API route handles:
 
-Handle large files and data streams without loading them into memory:
+```json
+{
+  "functions": {
+    "app/api/queue/handle/route.ts": {
+      "experimentalTriggers": [
+        {
+          "type": "queue/v1beta",
+          "topic": "my-topic",
+          "consumer": "consumer-group-1"
+        },
+        {
+          "type": "queue/v1beta",
+          "topic": "my-topic",
+          "consumer": "consumer-group-2"
+        }
+      ]
+    }
+  }
+}
+```
+
+### 3. Multiple API Routes
+
+You can also create separate API routes for different topics:
 
 ```typescript
-import { createTopic, StreamTransport } from "@vercel/queue";
+// app/api/queue/users/route.ts - Handle user events
+import { handleCallback } from "@vercel/queue";
 
-const videoTopic = createTopic<ReadableStream<Uint8Array>>(
-  "video-processing",
-  new StreamTransport(),
-);
+export const POST = handleCallback({
+  "user-events": {
+    processors: async (user, metadata) => {
+      console.log(`Processing user event:`, user, metadata);
+      await sendWelcomeEmail(user.email);
+    },
+  },
+});
+```
 
-// Process large video files efficiently
-const processor = videoTopic.consumerGroup("processors");
-await processor.consume(async (videoStream) => {
-  // Process stream chunk by chunk
-  const reader = videoStream.getReader();
-  while (true) {
-    const { done, value } = await reader.read();
-    if (done) break;
-    await processChunk(value);
-  }
+```typescript
+// app/api/queue/orders/route.ts - Handle order events
+import { handleCallback } from "@vercel/queue";
+
+export const POST = handleCallback({
+  "order-events": {
+    fulfillment: async (order, metadata) => {
+      console.log(`Processing order:`, order, metadata);
+      await fulfillOrder(order);
+    },
+  },
 });
 ```
 
+With corresponding `vercel.json`:
+
+```json
+{
+  "functions": {
+    "app/api/queue/users/route.ts": {
+      "experimentalTriggers": [
+        {
+          "type": "queue/v1beta",
+          "topic": "user-events",
+          "consumer": "processors"
+        }
+      ]
+    },
+    "app/api/queue/orders/route.ts": {
+      "experimentalTriggers": [
+        {
+          "type": "queue/v1beta",
+          "topic": "order-events",
+          "consumer": "fulfillment"
+        }
+      ]
+    }
+  }
+}
+```
+
+### Key Points
+
+- **Automatic Triggering**: Vercel automatically triggers your API routes when messages are available for the configured topic/consumer combinations
+- **Message Processing**: Your API routes receive the message ID and other metadata via headers, then use the queue client to process the specific message
+- **Configuration Required**: The `vercel.json` file is essential - it tells Vercel which topics and consumers each route should handle
+- **No Polling**: Unlike traditional queue consumers, you don't need to poll for messages - Vercel handles the triggering automatically
+
+## Key Features
+
 ### Consumer Groups
 
 Multiple consumers can process messages from the same topic in parallel:
@@ -351,55 +391,15 @@ const topic = createTopic<{ data: any }>("json-topic");
 Buffers the entire payload into memory as a Buffer - suitable for binary data
 that fits in memory.
 
-```typescript
-import { BufferTransport, createTopic } from "@vercel/queue";
-
-const topic = createTopic<Buffer>("binary-topic", new BufferTransport());
-const binaryData = Buffer.from("Binary data", "utf8");
-await topic.publish(binaryData);
-```
-
 #### StreamTransport
 
 **True streaming support** - passes ReadableStream directly without buffering.
 Ideal for large files and memory-efficient processing.
 
-```typescript
-import { createTopic, StreamTransport } from "@vercel/queue";
-
-const topic = createTopic<ReadableStream<Uint8Array>>(
-  "streaming-topic",
-  new StreamTransport(),
-);
-
-// Send large file as stream without loading into memory
-const fileStream = new ReadableStream<Uint8Array>({
-  start(controller) {
-    // Read file in chunks
-    for (const chunk of readFileInChunks("large-file.bin")) {
-      controller.enqueue(chunk);
-    }
-    controller.close();
-  },
-});
-
-await topic.publish(fileStream);
-```
-
 ### Custom Transport
 
 You can create your own serialization format by implementing the `Transport`
-interface:
-
-```typescript
-import { Transport } from "@vercel/queue";
-
-interface Transport<T = unknown> {
-  serialize(value: T): Buffer | ReadableStream<Uint8Array>;
-  deserialize(stream: ReadableStream<Uint8Array>): Promise<T>;
-  contentType: string;
-}
-```
+interface.
 
 ### Choosing the Right Transport
 
@@ -439,21 +439,6 @@ const topic = new Topic<T>(customClient, topicName, transport?);
 // Publish a message (uses topic's transport)
 await topic.publish(payload, options?);
 
-// Trigger a callback URL when the message is
-// ready for consumption
-await topic.publish(payload, {
-  callback: { url: "https://example.com/webhook" }
-});
-
-// Or provide multiple callbacks (each URL is called
-// with a separate consumer group)
-await topic.publish(payload, {
-  callback: {
-    group1: { url: "https://example.com/webhook1" },
-    group2: { url: "https://example.com/webhook2", delay: 30 }
-  }
-});
-
 // Create a consumer group (can override transport)
 const consumer = topic.consumerGroup<U>(groupName, options?);
 ```
@@ -469,7 +454,6 @@ await send<T>(topicName, payload, {
   transport?: Transport<T>;
   idempotencyKey?: string;
   retentionSeconds?: number;
-  callback?: Record<string, CallbackConfig> | CallbackConfig;
 });
 
 // Examples:
@@ -477,16 +461,11 @@ await send("notifications", { userId: "123", message: "Welcome!" });
 
 await send("images", imageBuffer, {
   transport: new BufferTransport(),
-  callback: { url: "https://example.com/process-image" }
 });
 
 await send("events", eventData, {
   idempotencyKey: "unique-key-123",
   retentionSeconds: 3600,
-  callback: {
-    analytics: { url: "https://analytics.example.com/webhook" },
-    notifications: { url: "https://notifications.example.com/webhook", delay: 30 }
-  }
 });
 ```
 
@@ -547,44 +526,33 @@ interface Transport<T = unknown> {
 }
 ```
 
-### Callback Utilities
+### Callback Handler
 
 ```typescript
-// Parse queue callback request headers
-function parseCallbackRequest(request: Request): CallbackMessageOptions;
+// Create a callback handler for Next.js route handlers
+function handleCallback(
+  handlers: CallbackHandlers,
+): (request: Request) => Promise<Response>;
 
-// Callback options type
-interface CallbackMessageOptions {
-  queueName: string;
-  consumerGroup: string;
-  messageId: string;
-}
-// Create a callback handler for NextJS route handlers
-function handleCallback(handlers: CallbackHandlers): (request: Request) => Promise<Response>;
-
-// Configuration object with handlers for different topics
+// Configuration object with handlers for different topics and consumer groups
 type CallbackHandlers = {
-  [topicName: string]:
-    | MessageHandler // Single handler (uses 'default' consumer group)
-    | { [consumerGroup: string]: MessageHandler }; // Multiple consumer group handlers
+  [topicName: string]: { [consumerGroup: string]: MessageHandler };
 };
 
 // Example usage:
 export const POST = handleCallback({
-  // Topic handler (uses 'default' consumer group)
-  "new-users": (message, metadata) => {
-    console.log(`New user event:`, message, metadata);
+  "user-events": {
+    welcome: (message, metadata) => {
+      console.log(`New user event:`, message, metadata);
+    },
   },
 
-  // Consumer group specific handlers
+  // Multiple consumer groups per topic
   "image-processing": {
-    "compress": (message, metadata) => console.log("Compressing image", message),
-    "resize": (message, metadata) => console.log("Resizing image", message),
-  }
+    compress: (message, metadata) => console.log("Compressing image", message),
+    resize: (message, metadata) => console.log("Resizing image", message),
+  },
 });
-
-// Error thrown for invalid callback requests
-class InvalidCallbackError extends Error;
 ```
 
 ## Examples
@@ -649,8 +617,6 @@ try {
 } catch (error) {
   if (error.message.includes("not found or not available")) {
     console.log("Message was already processed or does not exist");
-  } else if (error.message.includes("FIFO ordering violation")) {
-    console.log("FIFO queue requires processing messages in order");
   } else {
     console.error("Error processing message:", error);
   }
@@ -674,7 +640,7 @@ try {
   if (error instanceof QueueEmptyError) {
     console.log("No messages available");
   } else if (error instanceof MessageLockedError) {
-    console.log("Next message is locked (FIFO queue)");
+    console.log("Next message is locked");
     if (error.retryAfter) {
       console.log(`Retry after ${error.retryAfter} seconds`);
     }
@@ -763,117 +729,6 @@ try {
 }
 ```
 
-### Complete Example: Video Processing Pipeline
-
-Here's a comprehensive example showing a video processing pipeline that
-processes videos with FFmpeg and stores the results in Vercel Blob:
-
-```typescript
-import { createTopic, StreamTransport } from "@vercel/queue";
-import { spawn } from "child_process";
-import ffmpeg from "ffmpeg-static";
-import { put } from "@vercel/blob";
-
-// Input topic with unoptimized videos
-const unoptimizedVideosTopic = createTopic<ReadableStream<Uint8Array>>(
-  "unoptimized-videos",
-  new StreamTransport(),
-);
-
-// Output topic for optimized videos
-const optimizedVideosTopic = createTopic<ReadableStream<Uint8Array>>(
-  "optimized-videos",
-  new StreamTransport(),
-);
-
-// Step 1: Process videos with FFmpeg
-const videoProcessor = unoptimizedVideosTopic.consumerGroup("processors");
-
-try {
-  await videoProcessor.consume(async (inputVideoStream) => {
-    console.log("Processing video...");
-
-    if (!ffmpeg) {
-      throw new Error("FFmpeg not available");
-    }
-
-    // Create optimized video stream using FFmpeg
-    const optimizedStream = new ReadableStream<Uint8Array>({
-      start(controller) {
-        const ffmpegProcess = spawn(
-          ffmpeg,
-          [
-            "-i",
-            "pipe:0", // Input from stdin
-            "-c:v",
-            "libvpx-vp9", // Video codec
-            "-c:a",
-            "libopus", // Audio codec
-            "-crf",
-            "23", // Quality
-            "-f",
-            "webm", // Output format
-            "pipe:1", // Output to stdout
-          ],
-          { stdio: ["pipe", "pipe", "pipe"] },
-        );
-
-        // Pipe input stream to FFmpeg
-        const reader = inputVideoStream.getReader();
-        const pipeInput = async () => {
-          while (true) {
-            const { done, value } = await reader.read();
-            if (done) {
-              ffmpegProcess.stdin?.end();
-              break;
-            }
-            ffmpegProcess.stdin?.write(value);
-          }
-        };
-        pipeInput();
-
-        // Stream FFmpeg output
-        ffmpegProcess.stdout?.on("data", (chunk) => {
-          controller.enqueue(new Uint8Array(chunk));
-        });
-
-        ffmpegProcess.on("close", (code) => {
-          if (code === 0) {
-            controller.close();
-          } else {
-            controller.error(new Error(`FFmpeg failed with code ${code}`));
-          }
-        });
-      },
-    });
-
-    // Publish optimized video to next topic
-    await optimizedVideosTopic.publish(optimizedStream);
-    console.log("Video optimized and published");
-  });
-} catch (error) {
-  console.error("Video processing error:", error);
-}
-
-// Step 2: Store optimized videos in Vercel Blob
-const blobUploader = optimizedVideosTopic.consumerGroup("blob-uploaders");
-
-try {
-  await blobUploader.consume(async (optimizedVideo) => {
-    // Upload to Vercel Blob storage
-    const filename = `optimized-${Date.now()}.webm`;
-    const blob = await put(filename, optimizedVideo, {
-      access: "public",
-      contentType: "video/webm",
-    });
-
-    console.log(`Video uploaded to blob: ${blob.url} (${blob.size} bytes)`);
-  });
-} catch (error) {
-  console.error("Blob upload error:", error);
-}
-```
-
 ## Error Handling
 
 The queue client provides specific error types for different failure scenarios:
@@ -890,7 +745,7 @@ The queue client provides specific error types for different failure scenarios:
   status)
 
   - Contains optional `retryAfter` property with seconds to wait before retry
-  - For `consume()` without options: the next message in FIFO sequence is locked
+  - For `consume()` without options: the next message is locked
   - For `consume()` with messageId: the requested message is locked
 
 - **`MessageNotFoundError`**: Message doesn't exist (404 status)
@@ -898,24 +753,11 @@ The queue client provides specific error types for different failure scenarios:
 - **`MessageNotAvailableError`**: Message exists but isn't available for
   processing (409 status)
 
-- **`FifoOrderingViolationError`**: FIFO queue ordering violation (409 status
-  with nextMessageId)
-
-  - Contains `nextMessageId` property indicating which message to process first
-
-- **`FailedDependencyError`**: FIFO ordering violation when receiving by ID (424
-  status)
-
-  - Contains `nextMessageId` property indicating which message must be processed
-    first
-  - Similar to `FifoOrderingViolationError` but specifically for receive-by-ID
-    operations
-
 - **`MessageCorruptedError`**: Message data is corrupted or can't be parsed
 
 - **`BadRequestError`**: Invalid request parameters (400 status)
 
-  - Invalid queue names, FIFO limit violations, missing required parameters
+  - Invalid queue names, missing required parameters
 
 - **`UnauthorizedError`**: Authentication failure (401 status)
 
@@ -933,8 +775,7 @@ The queue client provides specific error types for different failure scenarios:
 ```typescript
 import {
   BadRequestError,
-  FailedDependencyError,
-  FifoOrderingViolationError,
+
   ForbiddenError,
   InternalServerError,
   MessageLockedError,
@@ -951,7 +792,7 @@ try {
   if (error instanceof QueueEmptyError) {
     console.log("Queue is empty, retry later");
   } else if (error instanceof MessageLockedError) {
-    console.log("Next message in FIFO queue is locked");
+    console.log("Next message is locked");
     if (error.retryAfter) {
       console.log(`Retry after ${error.retryAfter} seconds`);
     }
@@ -968,10 +809,7 @@ try {
       console.log(`Retry after ${error.retryAfter} seconds`);
       setTimeout(() => retry(), error.retryAfter * 1000);
     }
-  } else if (error instanceof FailedDependencyError) {
-    // FIFO ordering violation for receive by ID
-    console.log(`Must process ${error.nextMessageId} first`);
-  }
+
 }
 
 // Handle authentication and authorization errors