@vercel/queue 0.0.0-alpha.3 → 0.0.0-alpha.5
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +290 -223
- package/dist/index.d.mts +146 -123
- package/dist/index.d.ts +146 -123
- package/dist/index.js +138 -289
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +135 -287
- package/dist/index.mjs.map +1 -1
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -39,16 +39,33 @@ Publishing and consuming messages on a queue
|
|
|
39
39
|
|
|
40
40
|
```typescript
|
|
41
41
|
// index.ts
|
|
42
|
-
import {
|
|
42
|
+
import { send, receive } from "@vercel/queue";
|
|
43
43
|
|
|
44
|
-
|
|
45
|
-
|
|
44
|
+
type Message = {
|
|
45
|
+
message: string;
|
|
46
|
+
timestamp: number;
|
|
47
|
+
};
|
|
48
|
+
|
|
49
|
+
// Option 1: Using the send and receive helpers (simplest)
|
|
50
|
+
// Automatically uses default client and JSON transport
|
|
51
|
+
await send<Message>("my-topic", {
|
|
52
|
+
message: "Hello, World!",
|
|
53
|
+
timestamp: Date.now(),
|
|
54
|
+
});
|
|
55
|
+
|
|
56
|
+
// Consume a single message off the queue
|
|
57
|
+
// (Often wrapped in a loop to keep polling messages off the queue)
|
|
58
|
+
await receive<Message>("my-topic", "my-consumer-group", (m) => {
|
|
59
|
+
console.log(m.message);
|
|
60
|
+
});
|
|
61
|
+
|
|
62
|
+
// Option 2: Using createTopic for more control
|
|
63
|
+
|
|
64
|
+
import { createTopic } from "@vercel/queue";
|
|
46
65
|
|
|
47
66
|
// Create a topic with JSON serialization (default)
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
"my-topic",
|
|
51
|
-
);
|
|
67
|
+
// Uses default QueueClient automatically authenticated from Vercel environment
|
|
68
|
+
const topic = createTopic<Message>("my-topic");
|
|
52
69
|
|
|
53
70
|
// Publish a message
|
|
54
71
|
await topic.publish({
|
|
@@ -57,30 +74,29 @@ await topic.publish({
|
|
|
57
74
|
});
|
|
58
75
|
|
|
59
76
|
// Create a consumer group
|
|
60
|
-
const consumer = topic.consumerGroup("my-
|
|
61
|
-
|
|
62
|
-
// Process messages continuously with cancellation support
|
|
63
|
-
const controller = new AbortController();
|
|
77
|
+
const consumer = topic.consumerGroup("my-consumer-group");
|
|
64
78
|
|
|
65
|
-
//
|
|
79
|
+
// Process next available message (one-shot processing)
|
|
66
80
|
try {
|
|
67
|
-
await consumer.
|
|
68
|
-
console.log("Received:", message.
|
|
69
|
-
console.log("Timestamp:", new Date(message.
|
|
81
|
+
await consumer.consume(async (message, metadata) => {
|
|
82
|
+
console.log("Received:", message.message);
|
|
83
|
+
console.log("Timestamp:", new Date(message.timestamp));
|
|
84
|
+
console.log("Message Metadata", metadata);
|
|
85
|
+
// => { messageId, deliveryCount, timestamp }
|
|
70
86
|
});
|
|
71
87
|
} catch (error) {
|
|
72
|
-
console.error("Processing
|
|
88
|
+
console.error("Processing error:", error);
|
|
73
89
|
}
|
|
74
|
-
|
|
75
|
-
// Stop processing from elsewhere in your code
|
|
76
|
-
// controller.abort();
|
|
77
90
|
```
|
|
78
91
|
|
|
79
92
|
Run the script
|
|
80
93
|
|
|
81
94
|
```bash
|
|
82
|
-
#
|
|
83
|
-
|
|
95
|
+
# Install dotenv-cli and ts-node if you need it
|
|
96
|
+
npm i -g dotenv-cli ts-node typescript
|
|
97
|
+
|
|
98
|
+
# Run the script with the OIDC token
|
|
99
|
+
dotenv -e .env.local ts-node index.ts
|
|
84
100
|
```
|
|
85
101
|
|
|
86
102
|
## Usage with Vercel
|
|
@@ -112,36 +128,75 @@ package export resolution:
|
|
|
112
128
|
Create a new server function to publish messages
|
|
113
129
|
|
|
114
130
|
```typescript
|
|
115
|
-
// app/
|
|
131
|
+
// app/actions.ts
|
|
116
132
|
"use server";
|
|
117
133
|
|
|
118
|
-
import {
|
|
134
|
+
import { send } from "@vercel/queue";
|
|
119
135
|
|
|
120
136
|
export async function publishTestMessage(message: string) {
|
|
121
|
-
//
|
|
122
|
-
const
|
|
123
|
-
|
|
124
|
-
// Create a topic with JSON serialization (default)
|
|
125
|
-
const topic = createTopic<{ message: string; timestamp: number }>(
|
|
126
|
-
client,
|
|
137
|
+
// Option 1: Using simple send shorthand
|
|
138
|
+
const { messageId } = await send(
|
|
127
139
|
"my-topic",
|
|
140
|
+
{ message, timestamp: Date.now() },
|
|
141
|
+
{
|
|
142
|
+
// Provide a callback URL to invoke a consumer when the message is ready to be processed
|
|
143
|
+
callback: {
|
|
144
|
+
url: getCallbackUrl() // implementation below
|
|
145
|
+
},
|
|
146
|
+
},
|
|
128
147
|
);
|
|
129
148
|
|
|
149
|
+
console.log(`Published message ${messageId}`);
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
// Option 2: Customize the topic, transport, consumer groups, etc.
|
|
153
|
+
import { createTopic } from "@vercel/queue";
|
|
154
|
+
|
|
155
|
+
export async function publishTestMessage(message: string) {
|
|
156
|
+
// Create a topic with JSON serialization (default)
|
|
157
|
+
const topic = createTopic<{ message: string; timestamp: number }>("my-topic");
|
|
158
|
+
|
|
130
159
|
// Publish the message
|
|
131
160
|
const { messageId } = await topic.publish(
|
|
132
161
|
{ message, timestamp: Date.now() },
|
|
133
162
|
{
|
|
134
|
-
// Provide
|
|
163
|
+
// Provide multiple callback URLs to invoke multiple consumer groups
|
|
135
164
|
callback: {
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
165
|
+
{
|
|
166
|
+
"consumer-group-1": {
|
|
167
|
+
url: getCallbackUrl()
|
|
168
|
+
},
|
|
169
|
+
"consumer-group-2": {
|
|
170
|
+
url: getCallbackUrl()
|
|
171
|
+
delay: 5 // Delay callback by 5 seconds
|
|
172
|
+
},
|
|
173
|
+
}
|
|
139
174
|
},
|
|
140
175
|
},
|
|
141
176
|
);
|
|
142
177
|
|
|
143
178
|
console.log(`Published message ${messageId}`);
|
|
144
179
|
}
|
|
180
|
+
|
|
181
|
+
// Helper function to generate a local callback URL
|
|
182
|
+
function getCallbackUrl() {
|
|
183
|
+
const callbackUrl = new URL(
|
|
184
|
+
process.env.VERCEL_URL
|
|
185
|
+
? `https://${process.env.VERCEL_URL}/api/queue/handle`
|
|
186
|
+
: "http://localhost:3000/api/queue/handle"
|
|
187
|
+
);
|
|
188
|
+
|
|
189
|
+
// Add Vercel automation bypass secret if available (for preview deployments)
|
|
190
|
+
if (process.env.VERCEL_AUTOMATION_BYPASS_SECRET) {
|
|
191
|
+
callbackUrl.searchParams.set(
|
|
192
|
+
"x-vercel-protection-bypass",
|
|
193
|
+
process.env.VERCEL_AUTOMATION_BYPASS_SECRET
|
|
194
|
+
);
|
|
195
|
+
}
|
|
196
|
+
|
|
197
|
+
return callbackUrl.toString();
|
|
198
|
+
}
|
|
199
|
+
|
|
145
200
|
```
|
|
146
201
|
|
|
147
202
|
Now wire up the server function to your app
|
|
@@ -151,12 +206,12 @@ Now wire up the server function to your app
|
|
|
151
206
|
"use client";
|
|
152
207
|
import { publishTestMessage } from "./actions";
|
|
153
208
|
|
|
154
|
-
export default function
|
|
209
|
+
export default function Page() {
|
|
155
210
|
return (
|
|
156
211
|
// ...
|
|
157
|
-
<
|
|
212
|
+
<button onClick={() => publishTestMessage("Hello world")}>
|
|
158
213
|
Publish Test Message
|
|
159
|
-
</
|
|
214
|
+
</button>
|
|
160
215
|
);
|
|
161
216
|
}
|
|
162
217
|
```
|
|
@@ -173,30 +228,30 @@ The `handleCallback` helper function simplifies queue callback handling in NextJ
|
|
|
173
228
|
// app/api/queue/handle/route.ts
|
|
174
229
|
import { handleCallback } from "@vercel/queue";
|
|
175
230
|
|
|
231
|
+
// Option 1: Specify a single handler for the topic
|
|
176
232
|
export const POST = handleCallback({
|
|
177
|
-
// Handle messages sent on the "new-users" topic (the consumer
|
|
178
|
-
// group "default" will be used)
|
|
179
233
|
"my-topic": (message, metadata) => {
|
|
180
234
|
console.log(`Received message:`, message, metadata);
|
|
181
235
|
// metadata: { messageId, deliveryCount, timestamp }
|
|
182
236
|
},
|
|
237
|
+
|
|
238
|
+
// .. more topic handlers can be provided here
|
|
183
239
|
});
|
|
184
240
|
|
|
185
|
-
//
|
|
241
|
+
// This consumes messages on the "default" consumer group, which is used when no consumer groups
|
|
242
|
+
// were specified in the publish `callback` earlierA
|
|
243
|
+
|
|
244
|
+
// Option 2: Multiple consumer groups
|
|
186
245
|
export const POST = handleCallback({
|
|
187
246
|
// topic: "my-topic"
|
|
188
247
|
"my-topic": {
|
|
189
248
|
// consumer group: "compress"
|
|
190
|
-
|
|
191
|
-
console.log("
|
|
249
|
+
"consumer-group-1": (message, metadata) => {
|
|
250
|
+
console.log("Message:", message);
|
|
192
251
|
},
|
|
193
252
|
// consumer group: "resize"
|
|
194
|
-
|
|
195
|
-
console.log("
|
|
196
|
-
},
|
|
197
|
-
// consumer group: "watermark"
|
|
198
|
-
watermark: (message, metadata) => {
|
|
199
|
-
console.log("Adding watermark:", message);
|
|
253
|
+
"consume-group-2": (message, metadata) => {
|
|
254
|
+
console.log("Message", message);
|
|
200
255
|
},
|
|
201
256
|
},
|
|
202
257
|
});
|
|
@@ -209,18 +264,16 @@ export const POST = handleCallback({
|
|
|
209
264
|
Handle large files and data streams without loading them into memory:
|
|
210
265
|
|
|
211
266
|
```typescript
|
|
212
|
-
import { StreamTransport } from "@vercel/queue";
|
|
267
|
+
import { createTopic, StreamTransport } from "@vercel/queue";
|
|
213
268
|
|
|
214
269
|
const videoTopic = createTopic<ReadableStream<Uint8Array>>(
|
|
215
|
-
client,
|
|
216
270
|
"video-processing",
|
|
217
271
|
new StreamTransport(),
|
|
218
272
|
);
|
|
219
273
|
|
|
220
274
|
// Process large video files efficiently
|
|
221
275
|
const processor = videoTopic.consumerGroup("processors");
|
|
222
|
-
await processor.
|
|
223
|
-
const videoStream = message.payload;
|
|
276
|
+
await processor.consume(async (videoStream) => {
|
|
224
277
|
// Process stream chunk by chunk
|
|
225
278
|
const reader = videoStream.getReader();
|
|
226
279
|
while (true) {
|
|
@@ -252,10 +305,10 @@ const webhooks = topic.consumerGroup("webhooks");
|
|
|
252
305
|
- **Topics**: Named message channels with configurable serialization
|
|
253
306
|
- **Consumer Groups**: Named groups of consumers that process messages in
|
|
254
307
|
parallel
|
|
255
|
-
- `
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
|
|
308
|
+
- `consume()`: Process messages with flexible consumption patterns
|
|
309
|
+
- No options: Process next available message
|
|
310
|
+
- With `messageId`: Process specific message by ID
|
|
311
|
+
- With `skipPayload: true`: Process message metadata only (without payload)
|
|
259
312
|
- **Transports**: Pluggable serialization/deserialization for different data
|
|
260
313
|
types
|
|
261
314
|
- **Streaming**: Memory-efficient processing of large payloads
|
|
@@ -288,13 +341,9 @@ memory.
|
|
|
288
341
|
```typescript
|
|
289
342
|
import { createTopic, JsonTransport } from "@vercel/queue";
|
|
290
343
|
|
|
291
|
-
const topic = createTopic<{ data: any }>(
|
|
292
|
-
client,
|
|
293
|
-
"json-topic",
|
|
294
|
-
new JsonTransport(),
|
|
295
|
-
);
|
|
344
|
+
const topic = createTopic<{ data: any }>("json-topic", new JsonTransport());
|
|
296
345
|
// or simply (JsonTransport is the default):
|
|
297
|
-
const topic = createTopic<{ data: any }>(
|
|
346
|
+
const topic = createTopic<{ data: any }>("json-topic");
|
|
298
347
|
```
|
|
299
348
|
|
|
300
349
|
#### BufferTransport
|
|
@@ -305,11 +354,7 @@ that fits in memory.
|
|
|
305
354
|
```typescript
|
|
306
355
|
import { BufferTransport, createTopic } from "@vercel/queue";
|
|
307
356
|
|
|
308
|
-
const topic = createTopic<Buffer>(
|
|
309
|
-
client,
|
|
310
|
-
"binary-topic",
|
|
311
|
-
new BufferTransport(),
|
|
312
|
-
);
|
|
357
|
+
const topic = createTopic<Buffer>("binary-topic", new BufferTransport());
|
|
313
358
|
const binaryData = Buffer.from("Binary data", "utf8");
|
|
314
359
|
await topic.publish(binaryData);
|
|
315
360
|
```
|
|
@@ -323,7 +368,6 @@ Ideal for large files and memory-efficient processing.
|
|
|
323
368
|
import { createTopic, StreamTransport } from "@vercel/queue";
|
|
324
369
|
|
|
325
370
|
const topic = createTopic<ReadableStream<Uint8Array>>(
|
|
326
|
-
client,
|
|
327
371
|
"streaming-topic",
|
|
328
372
|
new StreamTransport(),
|
|
329
373
|
);
|
|
@@ -372,8 +416,12 @@ interface Transport<T = unknown> {
|
|
|
372
416
|
### QueueClient
|
|
373
417
|
|
|
374
418
|
```typescript
|
|
419
|
+
// Simple usage - automatically gets OIDC token from Vercel environment
|
|
420
|
+
const client = new QueueClient();
|
|
421
|
+
|
|
422
|
+
// Or with options
|
|
375
423
|
const client = new QueueClient({
|
|
376
|
-
token
|
|
424
|
+
token?: string; // Optional - will auto-detect if not provided
|
|
377
425
|
baseUrl?: string; // defaults to 'https://vqs.vercel.sh'
|
|
378
426
|
});
|
|
379
427
|
```
|
|
@@ -381,7 +429,12 @@ const client = new QueueClient({
|
|
|
381
429
|
### Topic
|
|
382
430
|
|
|
383
431
|
```typescript
|
|
384
|
-
|
|
432
|
+
// Simple usage with default client
|
|
433
|
+
const topic = createTopic<T>(topicName, transport?);
|
|
434
|
+
|
|
435
|
+
// For custom client configuration, use Topic constructor directly
|
|
436
|
+
const customClient = new QueueClient({ baseUrl: "https://custom.vqs.vercel.sh" });
|
|
437
|
+
const topic = new Topic<T>(customClient, topicName, transport?);
|
|
385
438
|
|
|
386
439
|
// Publish a message (uses topic's transport)
|
|
387
440
|
await topic.publish(payload, options?);
|
|
@@ -405,28 +458,59 @@ await topic.publish(payload, {
|
|
|
405
458
|
const consumer = topic.consumerGroup<U>(groupName, options?);
|
|
406
459
|
```
|
|
407
460
|
|
|
408
|
-
###
|
|
461
|
+
### Send (Shorthand)
|
|
409
462
|
|
|
410
463
|
```typescript
|
|
411
|
-
//
|
|
412
|
-
await
|
|
464
|
+
// Simple send - automatically uses default client and JSON transport
|
|
465
|
+
await send<T>(topicName, payload);
|
|
466
|
+
|
|
467
|
+
// Send with options including custom transport
|
|
468
|
+
await send<T>(topicName, payload, {
|
|
469
|
+
transport?: Transport<T>;
|
|
470
|
+
idempotencyKey?: string;
|
|
471
|
+
retentionSeconds?: number;
|
|
472
|
+
callback?: Record<string, CallbackConfig> | CallbackConfig;
|
|
473
|
+
});
|
|
413
474
|
|
|
414
|
-
//
|
|
415
|
-
await
|
|
475
|
+
// Examples:
|
|
476
|
+
await send("notifications", { userId: "123", message: "Welcome!" });
|
|
416
477
|
|
|
417
|
-
|
|
418
|
-
|
|
478
|
+
await send("images", imageBuffer, {
|
|
479
|
+
transport: new BufferTransport(),
|
|
480
|
+
callback: { url: "https://example.com/process-image" }
|
|
481
|
+
});
|
|
419
482
|
|
|
420
|
-
|
|
421
|
-
|
|
483
|
+
await send("events", eventData, {
|
|
484
|
+
idempotencyKey: "unique-key-123",
|
|
485
|
+
retentionSeconds: 3600,
|
|
486
|
+
callback: {
|
|
487
|
+
analytics: { url: "https://analytics.example.com/webhook" },
|
|
488
|
+
notifications: { url: "https://notifications.example.com/webhook", delay: 30 }
|
|
489
|
+
}
|
|
490
|
+
});
|
|
491
|
+
```
|
|
492
|
+
|
|
493
|
+
### ConsumerGroup
|
|
494
|
+
|
|
495
|
+
```typescript
|
|
496
|
+
// Process next available message (simplest form)
|
|
497
|
+
await consumer.consume(handler);
|
|
498
|
+
|
|
499
|
+
// Process specific message by ID with payload
|
|
500
|
+
await consumer.consume(handler, { messageId: "message-id" });
|
|
501
|
+
|
|
502
|
+
// Process specific message by ID without payload (metadata only)
|
|
503
|
+
// handler will be called with `undefined` as the payload
|
|
504
|
+
await consumer.consume(handler, { messageId: "message-id", skipPayload: true });
|
|
422
505
|
```
|
|
423
506
|
|
|
424
507
|
### Message Handler
|
|
425
508
|
|
|
426
509
|
```typescript
|
|
427
510
|
// Handler function signature
|
|
428
|
-
type MessageHandler<T> = (
|
|
429
|
-
message:
|
|
511
|
+
type MessageHandler<T = unknown> = (
|
|
512
|
+
message: T,
|
|
513
|
+
metadata: MessageMetadata,
|
|
430
514
|
) => Promise<MessageHandlerResult> | MessageHandlerResult;
|
|
431
515
|
|
|
432
516
|
// Handler result types
|
|
@@ -435,6 +519,22 @@ type MessageHandlerResult = void | MessageTimeoutResult;
|
|
|
435
519
|
interface MessageTimeoutResult {
|
|
436
520
|
timeoutSeconds: number; // seconds before message becomes available again
|
|
437
521
|
}
|
|
522
|
+
|
|
523
|
+
// Message Metadata
|
|
524
|
+
interface MessageMetadata {
|
|
525
|
+
messageId: string;
|
|
526
|
+
deliveryCount: number;
|
|
527
|
+
timestamp: string;
|
|
528
|
+
}
|
|
529
|
+
```
|
|
530
|
+
|
|
531
|
+
### ConsumeOptions Interface
|
|
532
|
+
|
|
533
|
+
```typescript
|
|
534
|
+
interface ConsumeOptions {
|
|
535
|
+
messageId?: string; // Process specific message by ID
|
|
536
|
+
skipPayload?: boolean; // Skip payload download (requires messageId)
|
|
537
|
+
}
|
|
438
538
|
```
|
|
439
539
|
|
|
440
540
|
### Transport Interface
|
|
@@ -462,24 +562,11 @@ interface CallbackMessageOptions {
|
|
|
462
562
|
// Create a callback handler for NextJS route handlers
|
|
463
563
|
function handleCallback(handlers: CallbackHandlers): (request: Request) => Promise<Response>;
|
|
464
564
|
|
|
465
|
-
// Handler function signature for callbacks
|
|
466
|
-
type Handler<T = unknown> = (
|
|
467
|
-
payload: T,
|
|
468
|
-
metadata: MessageMetadata
|
|
469
|
-
) => Promise<MessageHandlerResult> | MessageHandlerResult;
|
|
470
|
-
|
|
471
|
-
// Message metadata provided to handlers
|
|
472
|
-
interface MessageMetadata {
|
|
473
|
-
messageId: string;
|
|
474
|
-
deliveryCount: number;
|
|
475
|
-
timestamp: string;
|
|
476
|
-
}
|
|
477
|
-
|
|
478
565
|
// Configuration object with handlers for different topics
|
|
479
566
|
type CallbackHandlers = {
|
|
480
567
|
[topicName: string]:
|
|
481
|
-
|
|
|
482
|
-
| { [consumerGroup: string]:
|
|
568
|
+
| MessageHandler // Single handler (uses 'default' consumer group)
|
|
569
|
+
| { [consumerGroup: string]: MessageHandler }; // Multiple consumer group handlers
|
|
483
570
|
};
|
|
484
571
|
|
|
485
572
|
// Example usage:
|
|
@@ -511,7 +598,15 @@ interface UserEvent {
|
|
|
511
598
|
timestamp: number;
|
|
512
599
|
}
|
|
513
600
|
|
|
514
|
-
|
|
601
|
+
// Option 1: Using send shorthand
|
|
602
|
+
await send<UserEvent>("user-events", {
|
|
603
|
+
userId: "123",
|
|
604
|
+
action: "login",
|
|
605
|
+
timestamp: Date.now(),
|
|
606
|
+
});
|
|
607
|
+
|
|
608
|
+
// Option 2: Using createTopic for consumers
|
|
609
|
+
const userTopic = createTopic<UserEvent>("user-events");
|
|
515
610
|
|
|
516
611
|
await userTopic.publish({
|
|
517
612
|
userId: "123",
|
|
@@ -520,27 +615,21 @@ await userTopic.publish({
|
|
|
520
615
|
});
|
|
521
616
|
|
|
522
617
|
const consumer = userTopic.consumerGroup("processors");
|
|
523
|
-
const controller = new AbortController();
|
|
524
618
|
|
|
619
|
+
// Process next available message
|
|
525
620
|
try {
|
|
526
|
-
await consumer.
|
|
527
|
-
console.log(
|
|
528
|
-
`User ${message.payload.userId} performed ${message.payload.action}`,
|
|
529
|
-
);
|
|
621
|
+
await consumer.consume(async (message) => {
|
|
622
|
+
console.log(`User ${message.userId} performed ${message.action}`);
|
|
530
623
|
});
|
|
531
624
|
} catch (error) {
|
|
532
625
|
console.error("Processing error:", error);
|
|
533
626
|
}
|
|
534
|
-
|
|
535
|
-
// Stop processing when needed
|
|
536
|
-
// controller.abort();
|
|
537
627
|
```
|
|
538
628
|
|
|
539
629
|
### Processing Specific Messages by ID
|
|
540
630
|
|
|
541
631
|
```typescript
|
|
542
632
|
const userTopic = createTopic<{ userId: string; action: string }>(
|
|
543
|
-
client,
|
|
544
633
|
"user-events",
|
|
545
634
|
);
|
|
546
635
|
const consumer = userTopic.consumerGroup("processors");
|
|
@@ -549,12 +638,13 @@ const consumer = userTopic.consumerGroup("processors");
|
|
|
549
638
|
const messageId = "01234567-89ab-cdef-0123-456789abcdef";
|
|
550
639
|
|
|
551
640
|
try {
|
|
552
|
-
await consumer.
|
|
553
|
-
|
|
554
|
-
|
|
555
|
-
`User ${message.
|
|
556
|
-
|
|
557
|
-
|
|
641
|
+
await consumer.consume(
|
|
642
|
+
async (message, { messageId }) => {
|
|
643
|
+
console.log(`Processing specific message: ${messageId}`);
|
|
644
|
+
console.log(`User ${message.userId} performed ${message.action}`);
|
|
645
|
+
},
|
|
646
|
+
{ messageId },
|
|
647
|
+
);
|
|
558
648
|
console.log("Message processed successfully");
|
|
559
649
|
} catch (error) {
|
|
560
650
|
if (error.message.includes("not found or not available")) {
|
|
@@ -570,17 +660,14 @@ try {
|
|
|
570
660
|
### Processing Next Available Message
|
|
571
661
|
|
|
572
662
|
```typescript
|
|
573
|
-
const workTopic = createTopic<{ taskType: string; data: any }>(
|
|
574
|
-
client,
|
|
575
|
-
"work-queue",
|
|
576
|
-
);
|
|
663
|
+
const workTopic = createTopic<{ taskType: string; data: any }>("work-queue");
|
|
577
664
|
const worker = workTopic.consumerGroup("workers");
|
|
578
665
|
|
|
579
666
|
// Process the next available message (one-shot processing)
|
|
580
667
|
try {
|
|
581
|
-
await worker.
|
|
582
|
-
console.log(`Processing task: ${message.
|
|
583
|
-
await processTask(message.
|
|
668
|
+
await worker.consume(async (message) => {
|
|
669
|
+
console.log(`Processing task: ${message.taskType}`);
|
|
670
|
+
await processTask(message.taskType, message.data);
|
|
584
671
|
});
|
|
585
672
|
console.log("Message processed successfully");
|
|
586
673
|
} catch (error) {
|
|
@@ -596,31 +683,37 @@ try {
|
|
|
596
683
|
}
|
|
597
684
|
}
|
|
598
685
|
|
|
599
|
-
//
|
|
600
|
-
await worker.
|
|
601
|
-
if (!canProcessTaskType(message.
|
|
686
|
+
// Handle conditional timeouts
|
|
687
|
+
await worker.consume(async (message) => {
|
|
688
|
+
if (!canProcessTaskType(message.taskType)) {
|
|
602
689
|
// Return timeout to retry later
|
|
603
690
|
return { timeoutSeconds: 60 };
|
|
604
691
|
}
|
|
605
692
|
|
|
606
|
-
await processTask(message.
|
|
693
|
+
await processTask(message.taskType, message.data);
|
|
607
694
|
});
|
|
695
|
+
|
|
696
|
+
// Process specific message metadata only (no payload download)
|
|
697
|
+
await worker.consume(
|
|
698
|
+
async (_, metadata) => {
|
|
699
|
+
console.log(`Message ID: ${metadata.messageId}`);
|
|
700
|
+
console.log(`Delivery count: ${metadata.deliveryCount}`);
|
|
701
|
+
console.log(`Timestamp: ${metadata.timestamp}`);
|
|
702
|
+
// _ is undefined - no payload was downloaded
|
|
703
|
+
},
|
|
704
|
+
{ messageId: "specific-message-id", skipPayload: true },
|
|
705
|
+
);
|
|
608
706
|
```
|
|
609
707
|
|
|
610
708
|
### Timing Out Messages
|
|
611
709
|
|
|
612
710
|
```typescript
|
|
613
|
-
const workTopic = createTopic<{ taskType: string; data: any }>(
|
|
614
|
-
client,
|
|
615
|
-
"work-queue",
|
|
616
|
-
);
|
|
711
|
+
const workTopic = createTopic<{ taskType: string; data: any }>("work-queue");
|
|
617
712
|
const worker = workTopic.consumerGroup("workers");
|
|
618
|
-
const controller = new AbortController();
|
|
619
713
|
|
|
714
|
+
// Process a message with conditional timeout
|
|
620
715
|
try {
|
|
621
|
-
await worker.
|
|
622
|
-
const { taskType, data } = message.payload;
|
|
623
|
-
|
|
716
|
+
await worker.consume(async ({ taskType, data }) => {
|
|
624
717
|
// Check if we can process this task type right now
|
|
625
718
|
if (taskType === "heavy-computation" && isSystemOverloaded()) {
|
|
626
719
|
// Return timeout to retry later (5 minutes)
|
|
@@ -643,15 +736,12 @@ try {
|
|
|
643
736
|
}
|
|
644
737
|
|
|
645
738
|
// Example with exponential backoff
|
|
646
|
-
const backoffController = new AbortController();
|
|
647
|
-
|
|
648
739
|
try {
|
|
649
|
-
await worker.
|
|
740
|
+
await worker.consume(async (message, { deliveryCount }) => {
|
|
650
741
|
const maxRetries = 3;
|
|
651
|
-
const deliveryCount = message.deliveryCount;
|
|
652
742
|
|
|
653
743
|
try {
|
|
654
|
-
await processMessage(message
|
|
744
|
+
await processMessage(message);
|
|
655
745
|
// Successful processing - message will be deleted
|
|
656
746
|
} catch (error) {
|
|
657
747
|
if (deliveryCount < maxRetries) {
|
|
@@ -679,111 +769,97 @@ Here's a comprehensive example showing a video processing pipeline that
|
|
|
679
769
|
processes videos with FFmpeg and stores the results in Vercel Blob:
|
|
680
770
|
|
|
681
771
|
```typescript
|
|
682
|
-
import { createTopic,
|
|
772
|
+
import { createTopic, StreamTransport } from "@vercel/queue";
|
|
683
773
|
import { spawn } from "child_process";
|
|
684
774
|
import ffmpeg from "ffmpeg-static";
|
|
685
775
|
import { put } from "@vercel/blob";
|
|
686
776
|
|
|
687
|
-
const client = new QueueClient({
|
|
688
|
-
token: "your-vercel-oidc-token",
|
|
689
|
-
});
|
|
690
|
-
|
|
691
777
|
// Input topic with unoptimized videos
|
|
692
778
|
const unoptimizedVideosTopic = createTopic<ReadableStream<Uint8Array>>(
|
|
693
|
-
client,
|
|
694
779
|
"unoptimized-videos",
|
|
695
780
|
new StreamTransport(),
|
|
696
781
|
);
|
|
697
782
|
|
|
698
783
|
// Output topic for optimized videos
|
|
699
784
|
const optimizedVideosTopic = createTopic<ReadableStream<Uint8Array>>(
|
|
700
|
-
client,
|
|
701
785
|
"optimized-videos",
|
|
702
786
|
new StreamTransport(),
|
|
703
787
|
);
|
|
704
788
|
|
|
705
789
|
// Step 1: Process videos with FFmpeg
|
|
706
790
|
const videoProcessor = unoptimizedVideosTopic.consumerGroup("processors");
|
|
707
|
-
const processingController = new AbortController();
|
|
708
791
|
|
|
709
792
|
try {
|
|
710
|
-
await videoProcessor.
|
|
711
|
-
|
|
712
|
-
async (message) => {
|
|
713
|
-
const inputVideoStream = message.payload;
|
|
714
|
-
console.log("Processing video...");
|
|
715
|
-
|
|
716
|
-
if (!ffmpeg) {
|
|
717
|
-
throw new Error("FFmpeg not available");
|
|
718
|
-
}
|
|
793
|
+
await videoProcessor.consume(async (inputVideoStream) => {
|
|
794
|
+
console.log("Processing video...");
|
|
719
795
|
|
|
720
|
-
|
|
721
|
-
|
|
722
|
-
|
|
723
|
-
|
|
724
|
-
|
|
725
|
-
|
|
726
|
-
|
|
727
|
-
|
|
728
|
-
|
|
729
|
-
|
|
730
|
-
|
|
731
|
-
|
|
732
|
-
|
|
733
|
-
|
|
734
|
-
|
|
735
|
-
|
|
736
|
-
|
|
737
|
-
|
|
738
|
-
|
|
739
|
-
|
|
740
|
-
|
|
741
|
-
|
|
742
|
-
|
|
743
|
-
|
|
744
|
-
|
|
745
|
-
|
|
746
|
-
|
|
747
|
-
|
|
748
|
-
|
|
749
|
-
|
|
750
|
-
|
|
751
|
-
|
|
752
|
-
|
|
753
|
-
pipeInput();
|
|
754
|
-
|
|
755
|
-
// Stream FFmpeg output
|
|
756
|
-
ffmpegProcess.stdout?.on("data", (chunk) => {
|
|
757
|
-
controller.enqueue(new Uint8Array(chunk));
|
|
758
|
-
});
|
|
759
|
-
|
|
760
|
-
ffmpegProcess.on("close", (code) => {
|
|
761
|
-
if (code === 0) {
|
|
762
|
-
controller.close();
|
|
763
|
-
} else {
|
|
764
|
-
controller.error(new Error(`FFmpeg failed with code ${code}`));
|
|
796
|
+
if (!ffmpeg) {
|
|
797
|
+
throw new Error("FFmpeg not available");
|
|
798
|
+
}
|
|
799
|
+
|
|
800
|
+
// Create optimized video stream using FFmpeg
|
|
801
|
+
const optimizedStream = new ReadableStream<Uint8Array>({
|
|
802
|
+
start(controller) {
|
|
803
|
+
const ffmpegProcess = spawn(
|
|
804
|
+
ffmpeg,
|
|
805
|
+
[
|
|
806
|
+
"-i",
|
|
807
|
+
"pipe:0", // Input from stdin
|
|
808
|
+
"-c:v",
|
|
809
|
+
"libvpx-vp9", // Video codec
|
|
810
|
+
"-c:a",
|
|
811
|
+
"libopus", // Audio codec
|
|
812
|
+
"-crf",
|
|
813
|
+
"23", // Quality
|
|
814
|
+
"-f",
|
|
815
|
+
"webm", // Output format
|
|
816
|
+
"pipe:1", // Output to stdout
|
|
817
|
+
],
|
|
818
|
+
{ stdio: ["pipe", "pipe", "pipe"] },
|
|
819
|
+
);
|
|
820
|
+
|
|
821
|
+
// Pipe input stream to FFmpeg
|
|
822
|
+
const reader = inputVideoStream.getReader();
|
|
823
|
+
const pipeInput = async () => {
|
|
824
|
+
while (true) {
|
|
825
|
+
const { done, value } = await reader.read();
|
|
826
|
+
if (done) {
|
|
827
|
+
ffmpegProcess.stdin?.end();
|
|
828
|
+
break;
|
|
765
829
|
}
|
|
766
|
-
|
|
767
|
-
|
|
768
|
-
|
|
830
|
+
ffmpegProcess.stdin?.write(value);
|
|
831
|
+
}
|
|
832
|
+
};
|
|
833
|
+
pipeInput();
|
|
834
|
+
|
|
835
|
+
// Stream FFmpeg output
|
|
836
|
+
ffmpegProcess.stdout?.on("data", (chunk) => {
|
|
837
|
+
controller.enqueue(new Uint8Array(chunk));
|
|
838
|
+
});
|
|
839
|
+
|
|
840
|
+
ffmpegProcess.on("close", (code) => {
|
|
841
|
+
if (code === 0) {
|
|
842
|
+
controller.close();
|
|
843
|
+
} else {
|
|
844
|
+
controller.error(new Error(`FFmpeg failed with code ${code}`));
|
|
845
|
+
}
|
|
846
|
+
});
|
|
847
|
+
},
|
|
848
|
+
});
|
|
769
849
|
|
|
770
|
-
|
|
771
|
-
|
|
772
|
-
|
|
773
|
-
|
|
774
|
-
);
|
|
850
|
+
// Publish optimized video to next topic
|
|
851
|
+
await optimizedVideosTopic.publish(optimizedStream);
|
|
852
|
+
console.log("Video optimized and published");
|
|
853
|
+
});
|
|
775
854
|
} catch (error) {
|
|
776
855
|
console.error("Video processing error:", error);
|
|
777
856
|
}
|
|
778
857
|
|
|
779
858
|
// Step 2: Store optimized videos in Vercel Blob
|
|
780
859
|
const blobUploader = optimizedVideosTopic.consumerGroup("blob-uploaders");
|
|
781
|
-
const uploadController = new AbortController();
|
|
782
860
|
|
|
783
861
|
try {
|
|
784
|
-
await blobUploader.
|
|
785
|
-
const optimizedVideo = message.payload;
|
|
786
|
-
|
|
862
|
+
await blobUploader.consume(async (optimizedVideo) => {
|
|
787
863
|
// Upload to Vercel Blob storage
|
|
788
864
|
const filename = `optimized-${Date.now()}.webm`;
|
|
789
865
|
const blob = await put(filename, optimizedVideo, {
|
|
@@ -796,12 +872,6 @@ try {
|
|
|
796
872
|
} catch (error) {
|
|
797
873
|
console.error("Blob upload error:", error);
|
|
798
874
|
}
|
|
799
|
-
|
|
800
|
-
// Graceful shutdown
|
|
801
|
-
process.on("SIGINT", () => {
|
|
802
|
-
processingController.abort();
|
|
803
|
-
uploadController.abort();
|
|
804
|
-
});
|
|
805
875
|
```
|
|
806
876
|
|
|
807
877
|
## Error Handling
|
|
@@ -813,18 +883,15 @@ The queue client provides specific error types for different failure scenarios:
|
|
|
813
883
|
- **`QueueEmptyError`**: Thrown when attempting to receive messages from an
|
|
814
884
|
empty queue (204 status)
|
|
815
885
|
|
|
816
|
-
-
|
|
817
|
-
- `
|
|
818
|
-
polling
|
|
886
|
+
- Thrown by `consume()` when no messages are available
|
|
887
|
+
- Also thrown when directly using `client.receiveMessages()`
|
|
819
888
|
|
|
820
889
|
- **`MessageLockedError`**: Thrown when a message is temporarily locked (423
|
|
821
890
|
status)
|
|
822
891
|
|
|
823
892
|
- Contains optional `retryAfter` property with seconds to wait before retry
|
|
824
|
-
- For `
|
|
825
|
-
|
|
826
|
-
- For `receiveMessageById()`: the requested message is locked
|
|
827
|
-
- `ConsumerGroup.subscribe()` handles this error internally when polling
|
|
893
|
+
- For `consume()` without options: the next message in FIFO sequence is locked
|
|
894
|
+
- For `consume()` with messageId: the requested message is locked
|
|
828
895
|
|
|
829
896
|
- **`MessageNotFoundError`**: Message doesn't exist (404 status)
|
|
830
897
|
|
|
@@ -893,7 +960,7 @@ try {
|
|
|
893
960
|
|
|
894
961
|
// Handle locked message with retry
|
|
895
962
|
try {
|
|
896
|
-
await consumer.
|
|
963
|
+
await consumer.consume(handler, { messageId });
|
|
897
964
|
} catch (error) {
|
|
898
965
|
if (error instanceof MessageLockedError) {
|
|
899
966
|
console.log("Message is locked by another consumer");
|