@vercel/queue 0.0.0-alpha.3 → 0.0.0-alpha.5

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -39,16 +39,33 @@ Publishing and consuming messages on a queue
39
39
 
40
40
  ```typescript
41
41
  // index.ts
42
- import { createTopic, JsonTransport, QueueClient } from "@vercel/queue";
42
+ import { send, receive } from "@vercel/queue";
43
43
 
44
- // Create a client - automatically authenticated using the OIDC token
45
- const client = await QueueClient.fromVercelFunction();
44
+ type Message = {
45
+ message: string;
46
+ timestamp: number;
47
+ };
48
+
49
+ // Option 1: Using the send and receive helpers (simplest)
50
+ // Automatically uses default client and JSON transport
51
+ await send<Message>("my-topic", {
52
+ message: "Hello, World!",
53
+ timestamp: Date.now(),
54
+ });
55
+
56
+ // Consume a single message off the queue
57
+ // (Often wrapped in a loop to keep polling messages off the queue)
58
+ await receive<Message>("my-topic", "my-consumer-group", (m) => {
59
+ console.log(m.message);
60
+ });
61
+
62
+ // Option 2: Using createTopic for more control
63
+
64
+ import { createTopic } from "@vercel/queue";
46
65
 
47
66
  // Create a topic with JSON serialization (default)
48
- const topic = createTopic<{ message: string; timestamp: number }>(
49
- client,
50
- "my-topic",
51
- );
67
+ // Uses default QueueClient automatically authenticated from Vercel environment
68
+ const topic = createTopic<Message>("my-topic");
52
69
 
53
70
  // Publish a message
54
71
  await topic.publish({
@@ -57,30 +74,29 @@ await topic.publish({
57
74
  });
58
75
 
59
76
  // Create a consumer group
60
- const consumer = topic.consumerGroup("my-processors");
61
-
62
- // Process messages continuously with cancellation support
63
- const controller = new AbortController();
77
+ const consumer = topic.consumerGroup("my-consumer-group");
64
78
 
65
- // Start processing (blocks until aborted or error)
79
+ // Process next available message (one-shot processing)
66
80
  try {
67
- await consumer.subscribe(controller.signal, async (message) => {
68
- console.log("Received:", message.payload.message);
69
- console.log("Timestamp:", new Date(message.payload.timestamp));
81
+ await consumer.consume(async (message, metadata) => {
82
+ console.log("Received:", message.message);
83
+ console.log("Timestamp:", new Date(message.timestamp));
84
+ console.log("Message Metadata", metadata);
85
+ // => { messageId, deliveryCount, timestamp }
70
86
  });
71
87
  } catch (error) {
72
- console.error("Processing stopped due to error:", error);
88
+ console.error("Processing error:", error);
73
89
  }
74
-
75
- // Stop processing from elsewhere in your code
76
- // controller.abort();
77
90
  ```
78
91
 
79
92
  Run the script
80
93
 
81
94
  ```bash
82
- # Using dotenv to load the OIDC token
83
- dotenv -e .env.local node index.ts
95
+ # Install dotenv-cli and ts-node if you need it
96
+ npm i -g dotenv-cli ts-node typescript
97
+
98
+ # Run the script with the OIDC token
99
+ dotenv -e .env.local ts-node index.ts
84
100
  ```
85
101
 
86
102
  ## Usage with Vercel
@@ -112,36 +128,75 @@ package export resolution:
112
128
  Create a new server function to publish messages
113
129
 
114
130
  ```typescript
115
- // app/action.ts
131
+ // app/actions.ts
116
132
  "use server";
117
133
 
118
- import { createTopic, QueueClient } from "@vercel/queue";
134
+ import { send } from "@vercel/queue";
119
135
 
120
136
  export async function publishTestMessage(message: string) {
121
- // Initialize a queue client
122
- const client = await QueueClient.fromVercelFunction();
123
-
124
- // Create a topic with JSON serialization (default)
125
- const topic = createTopic<{ message: string; timestamp: number }>(
126
- client,
137
+ // Option 1: Using simple send shorthand
138
+ const { messageId } = await send(
127
139
  "my-topic",
140
+ { message, timestamp: Date.now() },
141
+ {
142
+ // Provide a callback URL to invoke a consumer when the message is ready to be processed
143
+ callback: {
144
+ url: getCallbackUrl() // implementation below
145
+ },
146
+ },
128
147
  );
129
148
 
149
+ console.log(`Published message ${messageId}`);
150
+ }
151
+
152
+ // Option 2: Customize the topic, transport, consumer groups, etc.
153
+ import { createTopic } from "@vercel/queue";
154
+
155
+ export async function publishTestMessage(message: string) {
156
+ // Create a topic with JSON serialization (default)
157
+ const topic = createTopic<{ message: string; timestamp: number }>("my-topic");
158
+
130
159
  // Publish the message
131
160
  const { messageId } = await topic.publish(
132
161
  { message, timestamp: Date.now() },
133
162
  {
134
- // Provide a callback URL to invoke a consumer when the message is ready to be processed
163
+ // Provide multiple callback URLs to invoke multiple consumer groups
135
164
  callback: {
136
- url: process.env.VERCEL_PROJECT_PRODUCTION_URL
137
- ? `https://${process.env.VERCEL_PROJECT_PRODUCTION_URL}/api/queue/handle`
138
- : "http://localhost:3000/api/queue/handle",
165
+ {
166
+ "consumer-group-1": {
167
+ url: getCallbackUrl()
168
+ },
169
+ "consumer-group-2": {
170
+ url: getCallbackUrl()
171
+ delay: 5 // Delay callback by 5 seconds
172
+ },
173
+ }
139
174
  },
140
175
  },
141
176
  );
142
177
 
143
178
  console.log(`Published message ${messageId}`);
144
179
  }
180
+
181
+ // Helper function to generate a local callback URL
182
+ function getCallbackUrl() {
183
+ const callbackUrl = new URL(
184
+ process.env.VERCEL_URL
185
+ ? `https://${process.env.VERCEL_URL}/api/queue/handle`
186
+ : "http://localhost:3000/api/queue/handle"
187
+ );
188
+
189
+ // Add Vercel automation bypass secret if available (for preview deployments)
190
+ if (process.env.VERCEL_AUTOMATION_BYPASS_SECRET) {
191
+ callbackUrl.searchParams.set(
192
+ "x-vercel-protection-bypass",
193
+ process.env.VERCEL_AUTOMATION_BYPASS_SECRET
194
+ );
195
+ }
196
+
197
+ return callbackUrl.toString();
198
+ }
199
+
145
200
  ```
146
201
 
147
202
  Now wire up the server function to your app
@@ -151,12 +206,12 @@ Now wire up the server function to your app
151
206
  "use client";
152
207
  import { publishTestMessage } from "./actions";
153
208
 
154
- export default function Button() {
209
+ export default function Page() {
155
210
  return (
156
211
  // ...
157
- <Button onClick={() => publishTestMessage("Hello world")} >
212
+ <button onClick={() => publishTestMessage("Hello world")}>
158
213
  Publish Test Message
159
- </a>
214
+ </button>
160
215
  );
161
216
  }
162
217
  ```
@@ -173,30 +228,30 @@ The `handleCallback` helper function simplifies queue callback handling in NextJ
173
228
  // app/api/queue/handle/route.ts
174
229
  import { handleCallback } from "@vercel/queue";
175
230
 
231
+ // Option 1: Specify a single handler for the topic
176
232
  export const POST = handleCallback({
177
- // Handle messages sent on the "new-users" topic (the consumer
178
- // group "default" will be used)
179
233
  "my-topic": (message, metadata) => {
180
234
  console.log(`Received message:`, message, metadata);
181
235
  // metadata: { messageId, deliveryCount, timestamp }
182
236
  },
237
+
238
+ // .. more topic handlers can be provided here
183
239
  });
184
240
 
185
- // Or, specify separate handlers for separate consumer groups
241
+ // This consumes messages on the "default" consumer group, which is used when no consumer groups
242
+ // were specified in the publish `callback` earlierA
243
+
244
+ // Option 2: Multiple consumer groups
186
245
  export const POST = handleCallback({
187
246
  // topic: "my-topic"
188
247
  "my-topic": {
189
248
  // consumer group: "compress"
190
- compress: (message, metadata) => {
191
- console.log("Compressing image:", message);
249
+ "consumer-group-1": (message, metadata) => {
250
+ console.log("Message:", message);
192
251
  },
193
252
  // consumer group: "resize"
194
- resize: (message, metadata) => {
195
- console.log("Resizing image:", message);
196
- },
197
- // consumer group: "watermark"
198
- watermark: (message, metadata) => {
199
- console.log("Adding watermark:", message);
253
+ "consume-group-2": (message, metadata) => {
254
+ console.log("Message", message);
200
255
  },
201
256
  },
202
257
  });
@@ -209,18 +264,16 @@ export const POST = handleCallback({
209
264
  Handle large files and data streams without loading them into memory:
210
265
 
211
266
  ```typescript
212
- import { StreamTransport } from "@vercel/queue";
267
+ import { createTopic, StreamTransport } from "@vercel/queue";
213
268
 
214
269
  const videoTopic = createTopic<ReadableStream<Uint8Array>>(
215
- client,
216
270
  "video-processing",
217
271
  new StreamTransport(),
218
272
  );
219
273
 
220
274
  // Process large video files efficiently
221
275
  const processor = videoTopic.consumerGroup("processors");
222
- await processor.subscribe(signal, async (message) => {
223
- const videoStream = message.payload;
276
+ await processor.consume(async (videoStream) => {
224
277
  // Process stream chunk by chunk
225
278
  const reader = videoStream.getReader();
226
279
  while (true) {
@@ -252,10 +305,10 @@ const webhooks = topic.consumerGroup("webhooks");
252
305
  - **Topics**: Named message channels with configurable serialization
253
306
  - **Consumer Groups**: Named groups of consumers that process messages in
254
307
  parallel
255
- - `subscribe()`: Continuously process messages with automatic polling
256
- - `receiveMessage()`: Process a specific message by ID
257
- - `receiveNextMessage()`: Process the next available message (one-shot)
258
- - `handleMessage()`: Process message metadata only (without payload)
308
+ - `consume()`: Process messages with flexible consumption patterns
309
+ - No options: Process next available message
310
+ - With `messageId`: Process specific message by ID
311
+ - With `skipPayload: true`: Process message metadata only (without payload)
259
312
  - **Transports**: Pluggable serialization/deserialization for different data
260
313
  types
261
314
  - **Streaming**: Memory-efficient processing of large payloads
@@ -288,13 +341,9 @@ memory.
288
341
  ```typescript
289
342
  import { createTopic, JsonTransport } from "@vercel/queue";
290
343
 
291
- const topic = createTopic<{ data: any }>(
292
- client,
293
- "json-topic",
294
- new JsonTransport(),
295
- );
344
+ const topic = createTopic<{ data: any }>("json-topic", new JsonTransport());
296
345
  // or simply (JsonTransport is the default):
297
- const topic = createTopic<{ data: any }>(client, "json-topic");
346
+ const topic = createTopic<{ data: any }>("json-topic");
298
347
  ```
299
348
 
300
349
  #### BufferTransport
@@ -305,11 +354,7 @@ that fits in memory.
305
354
  ```typescript
306
355
  import { BufferTransport, createTopic } from "@vercel/queue";
307
356
 
308
- const topic = createTopic<Buffer>(
309
- client,
310
- "binary-topic",
311
- new BufferTransport(),
312
- );
357
+ const topic = createTopic<Buffer>("binary-topic", new BufferTransport());
313
358
  const binaryData = Buffer.from("Binary data", "utf8");
314
359
  await topic.publish(binaryData);
315
360
  ```
@@ -323,7 +368,6 @@ Ideal for large files and memory-efficient processing.
323
368
  import { createTopic, StreamTransport } from "@vercel/queue";
324
369
 
325
370
  const topic = createTopic<ReadableStream<Uint8Array>>(
326
- client,
327
371
  "streaming-topic",
328
372
  new StreamTransport(),
329
373
  );
@@ -372,8 +416,12 @@ interface Transport<T = unknown> {
372
416
  ### QueueClient
373
417
 
374
418
  ```typescript
419
+ // Simple usage - automatically gets OIDC token from Vercel environment
420
+ const client = new QueueClient();
421
+
422
+ // Or with options
375
423
  const client = new QueueClient({
376
- token: string;
424
+ token?: string; // Optional - will auto-detect if not provided
377
425
  baseUrl?: string; // defaults to 'https://vqs.vercel.sh'
378
426
  });
379
427
  ```
@@ -381,7 +429,12 @@ const client = new QueueClient({
381
429
  ### Topic
382
430
 
383
431
  ```typescript
384
- const topic = createTopic<T>(client, topicName, transport?);
432
+ // Simple usage with default client
433
+ const topic = createTopic<T>(topicName, transport?);
434
+
435
+ // For custom client configuration, use Topic constructor directly
436
+ const customClient = new QueueClient({ baseUrl: "https://custom.vqs.vercel.sh" });
437
+ const topic = new Topic<T>(customClient, topicName, transport?);
385
438
 
386
439
  // Publish a message (uses topic's transport)
387
440
  await topic.publish(payload, options?);
@@ -405,28 +458,59 @@ await topic.publish(payload, {
405
458
  const consumer = topic.consumerGroup<U>(groupName, options?);
406
459
  ```
407
460
 
408
- ### ConsumerGroup
461
+ ### Send (Shorthand)
409
462
 
410
463
  ```typescript
411
- // Start continuous processing (blocks until signal is aborted or error occurs)
412
- await consumer.subscribe(signal, handler, options?);
464
+ // Simple send - automatically uses default client and JSON transport
465
+ await send<T>(topicName, payload);
466
+
467
+ // Send with options including custom transport
468
+ await send<T>(topicName, payload, {
469
+ transport?: Transport<T>;
470
+ idempotencyKey?: string;
471
+ retentionSeconds?: number;
472
+ callback?: Record<string, CallbackConfig> | CallbackConfig;
473
+ });
413
474
 
414
- // Process a specific message by ID
415
- await consumer.receiveMessage(messageId, handler);
475
+ // Examples:
476
+ await send("notifications", { userId: "123", message: "Welcome!" });
416
477
 
417
- // Process the next available message
418
- await consumer.receiveNextMessage(handler);
478
+ await send("images", imageBuffer, {
479
+ transport: new BufferTransport(),
480
+ callback: { url: "https://example.com/process-image" }
481
+ });
419
482
 
420
- // Handle a specific message by ID without payload
421
- await consumer.handleMessage(messageId, handler);
483
+ await send("events", eventData, {
484
+ idempotencyKey: "unique-key-123",
485
+ retentionSeconds: 3600,
486
+ callback: {
487
+ analytics: { url: "https://analytics.example.com/webhook" },
488
+ notifications: { url: "https://notifications.example.com/webhook", delay: 30 }
489
+ }
490
+ });
491
+ ```
492
+
493
+ ### ConsumerGroup
494
+
495
+ ```typescript
496
+ // Process next available message (simplest form)
497
+ await consumer.consume(handler);
498
+
499
+ // Process specific message by ID with payload
500
+ await consumer.consume(handler, { messageId: "message-id" });
501
+
502
+ // Process specific message by ID without payload (metadata only)
503
+ // handler will be called with `undefined` as the payload
504
+ await consumer.consume(handler, { messageId: "message-id", skipPayload: true });
422
505
  ```
423
506
 
424
507
  ### Message Handler
425
508
 
426
509
  ```typescript
427
510
  // Handler function signature
428
- type MessageHandler<T> = (
429
- message: Message<T>,
511
+ type MessageHandler<T = unknown> = (
512
+ message: T,
513
+ metadata: MessageMetadata,
430
514
  ) => Promise<MessageHandlerResult> | MessageHandlerResult;
431
515
 
432
516
  // Handler result types
@@ -435,6 +519,22 @@ type MessageHandlerResult = void | MessageTimeoutResult;
435
519
  interface MessageTimeoutResult {
436
520
  timeoutSeconds: number; // seconds before message becomes available again
437
521
  }
522
+
523
+ // Message Metadata
524
+ interface MessageMetadata {
525
+ messageId: string;
526
+ deliveryCount: number;
527
+ timestamp: string;
528
+ }
529
+ ```
530
+
531
+ ### ConsumeOptions Interface
532
+
533
+ ```typescript
534
+ interface ConsumeOptions {
535
+ messageId?: string; // Process specific message by ID
536
+ skipPayload?: boolean; // Skip payload download (requires messageId)
537
+ }
438
538
  ```
439
539
 
440
540
  ### Transport Interface
@@ -462,24 +562,11 @@ interface CallbackMessageOptions {
462
562
  // Create a callback handler for NextJS route handlers
463
563
  function handleCallback(handlers: CallbackHandlers): (request: Request) => Promise<Response>;
464
564
 
465
- // Handler function signature for callbacks
466
- type Handler<T = unknown> = (
467
- payload: T,
468
- metadata: MessageMetadata
469
- ) => Promise<MessageHandlerResult> | MessageHandlerResult;
470
-
471
- // Message metadata provided to handlers
472
- interface MessageMetadata {
473
- messageId: string;
474
- deliveryCount: number;
475
- timestamp: string;
476
- }
477
-
478
565
  // Configuration object with handlers for different topics
479
566
  type CallbackHandlers = {
480
567
  [topicName: string]:
481
- | Handler // Single handler (uses 'default' consumer group)
482
- | { [consumerGroup: string]: Handler }; // Multiple consumer group handlers
568
+ | MessageHandler // Single handler (uses 'default' consumer group)
569
+ | { [consumerGroup: string]: MessageHandler }; // Multiple consumer group handlers
483
570
  };
484
571
 
485
572
  // Example usage:
@@ -511,7 +598,15 @@ interface UserEvent {
511
598
  timestamp: number;
512
599
  }
513
600
 
514
- const userTopic = createTopic<UserEvent>(client, "user-events");
601
+ // Option 1: Using send shorthand
602
+ await send<UserEvent>("user-events", {
603
+ userId: "123",
604
+ action: "login",
605
+ timestamp: Date.now(),
606
+ });
607
+
608
+ // Option 2: Using createTopic for consumers
609
+ const userTopic = createTopic<UserEvent>("user-events");
515
610
 
516
611
  await userTopic.publish({
517
612
  userId: "123",
@@ -520,27 +615,21 @@ await userTopic.publish({
520
615
  });
521
616
 
522
617
  const consumer = userTopic.consumerGroup("processors");
523
- const controller = new AbortController();
524
618
 
619
+ // Process next available message
525
620
  try {
526
- await consumer.subscribe(controller.signal, async (message) => {
527
- console.log(
528
- `User ${message.payload.userId} performed ${message.payload.action}`,
529
- );
621
+ await consumer.consume(async (message) => {
622
+ console.log(`User ${message.userId} performed ${message.action}`);
530
623
  });
531
624
  } catch (error) {
532
625
  console.error("Processing error:", error);
533
626
  }
534
-
535
- // Stop processing when needed
536
- // controller.abort();
537
627
  ```
538
628
 
539
629
  ### Processing Specific Messages by ID
540
630
 
541
631
  ```typescript
542
632
  const userTopic = createTopic<{ userId: string; action: string }>(
543
- client,
544
633
  "user-events",
545
634
  );
546
635
  const consumer = userTopic.consumerGroup("processors");
@@ -549,12 +638,13 @@ const consumer = userTopic.consumerGroup("processors");
549
638
  const messageId = "01234567-89ab-cdef-0123-456789abcdef";
550
639
 
551
640
  try {
552
- await consumer.receiveMessage(messageId, async (message) => {
553
- console.log(`Processing specific message: ${message.messageId}`);
554
- console.log(
555
- `User ${message.payload.userId} performed ${message.payload.action}`,
556
- );
557
- });
641
+ await consumer.consume(
642
+ async (message, { messageId }) => {
643
+ console.log(`Processing specific message: ${messageId}`);
644
+ console.log(`User ${message.userId} performed ${message.action}`);
645
+ },
646
+ { messageId },
647
+ );
558
648
  console.log("Message processed successfully");
559
649
  } catch (error) {
560
650
  if (error.message.includes("not found or not available")) {
@@ -570,17 +660,14 @@ try {
570
660
  ### Processing Next Available Message
571
661
 
572
662
  ```typescript
573
- const workTopic = createTopic<{ taskType: string; data: any }>(
574
- client,
575
- "work-queue",
576
- );
663
+ const workTopic = createTopic<{ taskType: string; data: any }>("work-queue");
577
664
  const worker = workTopic.consumerGroup("workers");
578
665
 
579
666
  // Process the next available message (one-shot processing)
580
667
  try {
581
- await worker.receiveNextMessage(async (message) => {
582
- console.log(`Processing task: ${message.payload.taskType}`);
583
- await processTask(message.payload.taskType, message.payload.data);
668
+ await worker.consume(async (message) => {
669
+ console.log(`Processing task: ${message.taskType}`);
670
+ await processTask(message.taskType, message.data);
584
671
  });
585
672
  console.log("Message processed successfully");
586
673
  } catch (error) {
@@ -596,31 +683,37 @@ try {
596
683
  }
597
684
  }
598
685
 
599
- // You can also use it with timeout results
600
- await worker.receiveNextMessage(async (message) => {
601
- if (!canProcessTaskType(message.payload.taskType)) {
686
+ // Handle conditional timeouts
687
+ await worker.consume(async (message) => {
688
+ if (!canProcessTaskType(message.taskType)) {
602
689
  // Return timeout to retry later
603
690
  return { timeoutSeconds: 60 };
604
691
  }
605
692
 
606
- await processTask(message.payload.taskType, message.payload.data);
693
+ await processTask(message.taskType, message.data);
607
694
  });
695
+
696
+ // Process specific message metadata only (no payload download)
697
+ await worker.consume(
698
+ async (_, metadata) => {
699
+ console.log(`Message ID: ${metadata.messageId}`);
700
+ console.log(`Delivery count: ${metadata.deliveryCount}`);
701
+ console.log(`Timestamp: ${metadata.timestamp}`);
702
+ // _ is undefined - no payload was downloaded
703
+ },
704
+ { messageId: "specific-message-id", skipPayload: true },
705
+ );
608
706
  ```
609
707
 
610
708
  ### Timing Out Messages
611
709
 
612
710
  ```typescript
613
- const workTopic = createTopic<{ taskType: string; data: any }>(
614
- client,
615
- "work-queue",
616
- );
711
+ const workTopic = createTopic<{ taskType: string; data: any }>("work-queue");
617
712
  const worker = workTopic.consumerGroup("workers");
618
- const controller = new AbortController();
619
713
 
714
+ // Process a message with conditional timeout
620
715
  try {
621
- await worker.subscribe(controller.signal, async (message) => {
622
- const { taskType, data } = message.payload;
623
-
716
+ await worker.consume(async ({ taskType, data }) => {
624
717
  // Check if we can process this task type right now
625
718
  if (taskType === "heavy-computation" && isSystemOverloaded()) {
626
719
  // Return timeout to retry later (5 minutes)
@@ -643,15 +736,12 @@ try {
643
736
  }
644
737
 
645
738
  // Example with exponential backoff
646
- const backoffController = new AbortController();
647
-
648
739
  try {
649
- await worker.subscribe(backoffController.signal, async (message) => {
740
+ await worker.consume(async (message, { deliveryCount }) => {
650
741
  const maxRetries = 3;
651
- const deliveryCount = message.deliveryCount;
652
742
 
653
743
  try {
654
- await processMessage(message.payload);
744
+ await processMessage(message);
655
745
  // Successful processing - message will be deleted
656
746
  } catch (error) {
657
747
  if (deliveryCount < maxRetries) {
@@ -679,111 +769,97 @@ Here's a comprehensive example showing a video processing pipeline that
679
769
  processes videos with FFmpeg and stores the results in Vercel Blob:
680
770
 
681
771
  ```typescript
682
- import { createTopic, QueueClient, StreamTransport } from "@vercel/queue";
772
+ import { createTopic, StreamTransport } from "@vercel/queue";
683
773
  import { spawn } from "child_process";
684
774
  import ffmpeg from "ffmpeg-static";
685
775
  import { put } from "@vercel/blob";
686
776
 
687
- const client = new QueueClient({
688
- token: "your-vercel-oidc-token",
689
- });
690
-
691
777
  // Input topic with unoptimized videos
692
778
  const unoptimizedVideosTopic = createTopic<ReadableStream<Uint8Array>>(
693
- client,
694
779
  "unoptimized-videos",
695
780
  new StreamTransport(),
696
781
  );
697
782
 
698
783
  // Output topic for optimized videos
699
784
  const optimizedVideosTopic = createTopic<ReadableStream<Uint8Array>>(
700
- client,
701
785
  "optimized-videos",
702
786
  new StreamTransport(),
703
787
  );
704
788
 
705
789
  // Step 1: Process videos with FFmpeg
706
790
  const videoProcessor = unoptimizedVideosTopic.consumerGroup("processors");
707
- const processingController = new AbortController();
708
791
 
709
792
  try {
710
- await videoProcessor.subscribe(
711
- processingController.signal,
712
- async (message) => {
713
- const inputVideoStream = message.payload;
714
- console.log("Processing video...");
715
-
716
- if (!ffmpeg) {
717
- throw new Error("FFmpeg not available");
718
- }
793
+ await videoProcessor.consume(async (inputVideoStream) => {
794
+ console.log("Processing video...");
719
795
 
720
- // Create optimized video stream using FFmpeg
721
- const optimizedStream = new ReadableStream<Uint8Array>({
722
- start(controller) {
723
- const ffmpegProcess = spawn(
724
- ffmpeg,
725
- [
726
- "-i",
727
- "pipe:0", // Input from stdin
728
- "-c:v",
729
- "libvpx-vp9", // Video codec
730
- "-c:a",
731
- "libopus", // Audio codec
732
- "-crf",
733
- "23", // Quality
734
- "-f",
735
- "webm", // Output format
736
- "pipe:1", // Output to stdout
737
- ],
738
- { stdio: ["pipe", "pipe", "pipe"] },
739
- );
740
-
741
- // Pipe input stream to FFmpeg
742
- const reader = inputVideoStream.getReader();
743
- const pipeInput = async () => {
744
- while (true) {
745
- const { done, value } = await reader.read();
746
- if (done) {
747
- ffmpegProcess.stdin?.end();
748
- break;
749
- }
750
- ffmpegProcess.stdin?.write(value);
751
- }
752
- };
753
- pipeInput();
754
-
755
- // Stream FFmpeg output
756
- ffmpegProcess.stdout?.on("data", (chunk) => {
757
- controller.enqueue(new Uint8Array(chunk));
758
- });
759
-
760
- ffmpegProcess.on("close", (code) => {
761
- if (code === 0) {
762
- controller.close();
763
- } else {
764
- controller.error(new Error(`FFmpeg failed with code ${code}`));
796
+ if (!ffmpeg) {
797
+ throw new Error("FFmpeg not available");
798
+ }
799
+
800
+ // Create optimized video stream using FFmpeg
801
+ const optimizedStream = new ReadableStream<Uint8Array>({
802
+ start(controller) {
803
+ const ffmpegProcess = spawn(
804
+ ffmpeg,
805
+ [
806
+ "-i",
807
+ "pipe:0", // Input from stdin
808
+ "-c:v",
809
+ "libvpx-vp9", // Video codec
810
+ "-c:a",
811
+ "libopus", // Audio codec
812
+ "-crf",
813
+ "23", // Quality
814
+ "-f",
815
+ "webm", // Output format
816
+ "pipe:1", // Output to stdout
817
+ ],
818
+ { stdio: ["pipe", "pipe", "pipe"] },
819
+ );
820
+
821
+ // Pipe input stream to FFmpeg
822
+ const reader = inputVideoStream.getReader();
823
+ const pipeInput = async () => {
824
+ while (true) {
825
+ const { done, value } = await reader.read();
826
+ if (done) {
827
+ ffmpegProcess.stdin?.end();
828
+ break;
765
829
  }
766
- });
767
- },
768
- });
830
+ ffmpegProcess.stdin?.write(value);
831
+ }
832
+ };
833
+ pipeInput();
834
+
835
+ // Stream FFmpeg output
836
+ ffmpegProcess.stdout?.on("data", (chunk) => {
837
+ controller.enqueue(new Uint8Array(chunk));
838
+ });
839
+
840
+ ffmpegProcess.on("close", (code) => {
841
+ if (code === 0) {
842
+ controller.close();
843
+ } else {
844
+ controller.error(new Error(`FFmpeg failed with code ${code}`));
845
+ }
846
+ });
847
+ },
848
+ });
769
849
 
770
- // Publish optimized video to next topic
771
- await optimizedVideosTopic.publish(optimizedStream);
772
- console.log("Video optimized and published");
773
- },
774
- );
850
+ // Publish optimized video to next topic
851
+ await optimizedVideosTopic.publish(optimizedStream);
852
+ console.log("Video optimized and published");
853
+ });
775
854
  } catch (error) {
776
855
  console.error("Video processing error:", error);
777
856
  }
778
857
 
779
858
  // Step 2: Store optimized videos in Vercel Blob
780
859
  const blobUploader = optimizedVideosTopic.consumerGroup("blob-uploaders");
781
- const uploadController = new AbortController();
782
860
 
783
861
  try {
784
- await blobUploader.subscribe(uploadController.signal, async (message) => {
785
- const optimizedVideo = message.payload;
786
-
862
+ await blobUploader.consume(async (optimizedVideo) => {
787
863
  // Upload to Vercel Blob storage
788
864
  const filename = `optimized-${Date.now()}.webm`;
789
865
  const blob = await put(filename, optimizedVideo, {
@@ -796,12 +872,6 @@ try {
796
872
  } catch (error) {
797
873
  console.error("Blob upload error:", error);
798
874
  }
799
-
800
- // Graceful shutdown
801
- process.on("SIGINT", () => {
802
- processingController.abort();
803
- uploadController.abort();
804
- });
805
875
  ```
806
876
 
807
877
  ## Error Handling
@@ -813,18 +883,15 @@ The queue client provides specific error types for different failure scenarios:
813
883
  - **`QueueEmptyError`**: Thrown when attempting to receive messages from an
814
884
  empty queue (204 status)
815
885
 
816
- - Only thrown when directly using `client.receiveMessages()`
817
- - `ConsumerGroup.subscribe()` handles this error internally and continues
818
- polling
886
+ - Thrown by `consume()` when no messages are available
887
+ - Also thrown when directly using `client.receiveMessages()`
819
888
 
820
889
  - **`MessageLockedError`**: Thrown when a message is temporarily locked (423
821
890
  status)
822
891
 
823
892
  - Contains optional `retryAfter` property with seconds to wait before retry
824
- - For `receiveMessages()` on FIFO queues: the next message in sequence is
825
- locked
826
- - For `receiveMessageById()`: the requested message is locked
827
- - `ConsumerGroup.subscribe()` handles this error internally when polling
893
+ - For `consume()` without options: the next message in FIFO sequence is locked
894
+ - For `consume()` with messageId: the requested message is locked
828
895
 
829
896
  - **`MessageNotFoundError`**: Message doesn't exist (404 status)
830
897
 
@@ -893,7 +960,7 @@ try {
893
960
 
894
961
  // Handle locked message with retry
895
962
  try {
896
- await consumer.receiveMessage(messageId, handler);
963
+ await consumer.consume(handler, { messageId });
897
964
  } catch (error) {
898
965
  if (error instanceof MessageLockedError) {
899
966
  console.log("Message is locked by another consumer");