@vercel/queue 0.0.0-alpha.4 → 0.0.0-alpha.40
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +467 -827
- package/dist/index.d.mts +436 -584
- package/dist/index.d.ts +436 -584
- package/dist/index.js +1452 -986
- package/dist/index.js.map +1 -1
- package/dist/index.mjs +1434 -976
- package/dist/index.mjs.map +1 -1
- package/package.json +8 -5
package/README.md
CHANGED
|
@@ -1,20 +1,15 @@
|
|
|
1
1
|
# Vercel Queues
|
|
2
2
|
|
|
3
|
-
A TypeScript client library for interacting with the Vercel Queue Service API
|
|
4
|
-
with customizable serialization/deserialization (transport) support, including
|
|
5
|
-
**streaming support** for memory-efficient processing of large payloads.
|
|
3
|
+
A TypeScript client library for interacting with the Vercel Queue Service API, designed for seamless integration with Vercel deployments.
|
|
6
4
|
|
|
7
5
|
## Features
|
|
8
6
|
|
|
9
|
-
- **
|
|
10
|
-
|
|
11
|
-
- **
|
|
12
|
-
|
|
13
|
-
- **
|
|
14
|
-
|
|
15
|
-
- **Pub/Sub Pattern**: Topic-based messaging with consumer groups
|
|
16
|
-
- **Type Safety**: Full TypeScript support with generic types
|
|
17
|
-
- **Automatic Retries**: Built-in visibility timeout management
|
|
7
|
+
- **Simple API**: `send` and `receive` are all you need
|
|
8
|
+
- **Automatic Triggering on Vercel**: Vercel invokes your route handlers when messages are ready
|
|
9
|
+
- **Works Anywhere**: `send` and `receive` work in any Node.js environment
|
|
10
|
+
- **Type Safety**: Full TypeScript generics support
|
|
11
|
+
- **Customizable Serialization**: Built-in JSON, Buffer, and Stream transports
|
|
12
|
+
- **Local Dev Mode**: Messages sent locally trigger your handlers automatically
|
|
18
13
|
|
|
19
14
|
## Installation
|
|
20
15
|
|
|
@@ -24,992 +19,637 @@ npm install @vercel/queue
|
|
|
24
19
|
|
|
25
20
|
## Quick Start
|
|
26
21
|
|
|
27
|
-
|
|
28
|
-
(including the OIDC token):
|
|
22
|
+
Set up your region via environment variables. If your framework supports `.env` files (Next.js, Vite, Nuxt, etc.):
|
|
29
23
|
|
|
30
24
|
```bash
|
|
31
|
-
#
|
|
32
|
-
|
|
25
|
+
# .env.production (on Vercel, inherits the platform's region)
|
|
26
|
+
QUEUE_REGION=${VERCEL_REGION}
|
|
33
27
|
|
|
34
|
-
#
|
|
35
|
-
|
|
28
|
+
# .env.development (fixed region for local dev — iad1 is recommended)
|
|
29
|
+
QUEUE_REGION=iad1
|
|
36
30
|
```
|
|
37
31
|
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
```typescript
|
|
41
|
-
// index.ts
|
|
42
|
-
import { send, receive } from "@vercel/queue";
|
|
43
|
-
|
|
44
|
-
type Message = {
|
|
45
|
-
message: string;
|
|
46
|
-
timestamp: number;
|
|
47
|
-
};
|
|
48
|
-
|
|
49
|
-
// Option 1: Using the send and receive helpers (simplest)
|
|
50
|
-
// Automatically uses default client and JSON transport
|
|
51
|
-
await send<Message>("my-topic", {
|
|
52
|
-
message: "Hello, World!",
|
|
53
|
-
timestamp: Date.now(),
|
|
54
|
-
});
|
|
55
|
-
|
|
56
|
-
// Consume a single message off the queue
|
|
57
|
-
// (Often wrapped in a loop to keep polling messages off the queue)
|
|
58
|
-
await receive<Message>("my-topic", "my-consumer-group", (m) => {
|
|
59
|
-
console.log(m.message);
|
|
60
|
-
});
|
|
61
|
-
|
|
62
|
-
// Option 2: Using createTopic for more control
|
|
32
|
+
Otherwise, set `QUEUE_REGION` in your environment directly (e.g. via your hosting provider's dashboard or a `dotenv` setup).
|
|
63
33
|
|
|
64
|
-
|
|
34
|
+
Create a shared queue client:
|
|
65
35
|
|
|
66
|
-
|
|
67
|
-
//
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
// Publish a message
|
|
71
|
-
await topic.publish({
|
|
72
|
-
message: "Hello, World!",
|
|
73
|
-
timestamp: Date.now(),
|
|
74
|
-
});
|
|
75
|
-
|
|
76
|
-
// Create a consumer group
|
|
77
|
-
const consumer = topic.consumerGroup("my-consumer-group");
|
|
36
|
+
```typescript
|
|
37
|
+
// lib/queue.ts
|
|
38
|
+
import { QueueClient } from "@vercel/queue";
|
|
78
39
|
|
|
79
|
-
|
|
80
|
-
|
|
81
|
-
await consumer.consume(async (message, metadata) => {
|
|
82
|
-
console.log("Received:", message.message);
|
|
83
|
-
console.log("Timestamp:", new Date(message.timestamp));
|
|
84
|
-
console.log("Message Metadata", metadata);
|
|
85
|
-
// => { messageId, deliveryCount, timestamp }
|
|
86
|
-
});
|
|
87
|
-
} catch (error) {
|
|
88
|
-
console.error("Processing error:", error);
|
|
89
|
-
}
|
|
40
|
+
const queue = new QueueClient({ region: process.env.QUEUE_REGION! });
|
|
41
|
+
export const { send, receive, handleCallback, handleNodeCallback } = queue;
|
|
90
42
|
```
|
|
91
43
|
|
|
92
|
-
|
|
44
|
+
Send a message anywhere in your app:
|
|
93
45
|
|
|
94
|
-
```
|
|
95
|
-
|
|
96
|
-
npm i -g dotenv-cli ts-node typescript
|
|
46
|
+
```typescript
|
|
47
|
+
import { send } from "@/lib/queue";
|
|
97
48
|
|
|
98
|
-
|
|
99
|
-
dotenv -e .env.local ts-node index.ts
|
|
49
|
+
await send("my-topic", { message: "Hello world" });
|
|
100
50
|
```
|
|
101
51
|
|
|
102
|
-
|
|
52
|
+
Handle incoming messages with a route handler:
|
|
103
53
|
|
|
104
|
-
|
|
105
|
-
queue
|
|
106
|
-
|
|
107
|
-
|
|
108
|
-
To demonstrate using queues on Vercel, let's use a Next.js app. You can use an
|
|
109
|
-
existing app or create one using
|
|
110
|
-
[create-next-app](https://nextjs.org/docs/app/api-reference/cli/create-next-app).
|
|
54
|
+
```typescript
|
|
55
|
+
// app/api/queue/my-topic/route.ts
|
|
56
|
+
import { handleCallback } from "@/lib/queue";
|
|
111
57
|
|
|
112
|
-
|
|
58
|
+
export const POST = handleCallback(async (message, metadata) => {
|
|
59
|
+
console.log("Processing:", message);
|
|
60
|
+
});
|
|
61
|
+
```
|
|
113
62
|
|
|
114
|
-
|
|
115
|
-
package export resolution:
|
|
63
|
+
Configure your `vercel.json`:
|
|
116
64
|
|
|
117
65
|
```json
|
|
118
66
|
{
|
|
119
|
-
"
|
|
120
|
-
"
|
|
121
|
-
|
|
67
|
+
"functions": {
|
|
68
|
+
"app/api/queue/my-topic/route.ts": {
|
|
69
|
+
"experimentalTriggers": [{ "type": "queue/v2beta", "topic": "my-topic" }]
|
|
70
|
+
}
|
|
122
71
|
}
|
|
123
72
|
}
|
|
124
73
|
```
|
|
125
74
|
|
|
126
|
-
###
|
|
75
|
+
### Project Setup
|
|
127
76
|
|
|
128
|
-
|
|
77
|
+
For local development, link your Vercel project:
|
|
129
78
|
|
|
130
|
-
```
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
export async function publishTestMessage(message: string) {
|
|
137
|
-
// Option 1: Using simple send shorthand
|
|
138
|
-
const { messageId } = await send(
|
|
139
|
-
"my-topic",
|
|
140
|
-
{ message, timestamp: Date.now() },
|
|
141
|
-
{
|
|
142
|
-
// Provide a callback URL to invoke a consumer when the message is ready to be processed
|
|
143
|
-
callback: {
|
|
144
|
-
url: getCallbackUrl() // implementation below
|
|
145
|
-
},
|
|
146
|
-
},
|
|
147
|
-
);
|
|
79
|
+
```bash
|
|
80
|
+
npm i -g vercel
|
|
81
|
+
vc link
|
|
82
|
+
vc env pull
|
|
83
|
+
```
|
|
148
84
|
|
|
149
|
-
|
|
150
|
-
}
|
|
85
|
+
## Local Development
|
|
151
86
|
|
|
152
|
-
|
|
153
|
-
import { createTopic } from "@vercel/queue";
|
|
87
|
+
**Queues just work locally.** When you `send()` messages in development mode, the library sends them to the real Vercel Queue Service, reads your `vercel.json` configuration, discovers your queue handlers, and triggers them automatically via local HTTP requests. This means your local dev environment behaves identically to production — no surprising behavior differences.
|
|
154
88
|
|
|
155
|
-
|
|
156
|
-
// Create a topic with JSON serialization (default)
|
|
157
|
-
const topic = createTopic<{ message: string; timestamp: number }>("my-topic");
|
|
89
|
+
> **Note:** Local dev mode is enabled when `NODE_ENV=development`. Most frameworks (Next.js, etc.) set this automatically during `npm run dev`.
|
|
158
90
|
|
|
159
|
-
|
|
160
|
-
const { messageId } = await topic.publish(
|
|
161
|
-
{ message, timestamp: Date.now() },
|
|
162
|
-
{
|
|
163
|
-
// Provide multiple callback URLs to invoke multiple consumer groups
|
|
164
|
-
callback: {
|
|
165
|
-
{
|
|
166
|
-
"consumer-group-1": {
|
|
167
|
-
url: getCallbackUrl()
|
|
168
|
-
},
|
|
169
|
-
"consumer-group-2": {
|
|
170
|
-
url: getCallbackUrl()
|
|
171
|
-
delay: 5 // Delay callback by 5 seconds
|
|
172
|
-
},
|
|
173
|
-
}
|
|
174
|
-
},
|
|
175
|
-
},
|
|
176
|
-
);
|
|
91
|
+
## Publishing Messages
|
|
177
92
|
|
|
178
|
-
|
|
179
|
-
}
|
|
93
|
+
```typescript
|
|
94
|
+
import { QueueClient } from "@vercel/queue";
|
|
180
95
|
|
|
181
|
-
|
|
182
|
-
function getCallbackUrl() {
|
|
183
|
-
const callbackUrl = new URL(
|
|
184
|
-
process.env.VERCEL_URL
|
|
185
|
-
? `https://${process.env.VERCEL_URL}/api/queue/handle`
|
|
186
|
-
: "http://localhost:3000/api/queue/handle"
|
|
187
|
-
);
|
|
188
|
-
|
|
189
|
-
// Add Vercel automation bypass secret if available (for preview deployments)
|
|
190
|
-
if (process.env.VERCEL_AUTOMATION_BYPASS_SECRET) {
|
|
191
|
-
callbackUrl.searchParams.set(
|
|
192
|
-
"x-vercel-protection-bypass",
|
|
193
|
-
process.env.VERCEL_AUTOMATION_BYPASS_SECRET
|
|
194
|
-
);
|
|
195
|
-
}
|
|
96
|
+
const { send } = new QueueClient({ region: process.env.QUEUE_REGION! });
|
|
196
97
|
|
|
197
|
-
|
|
198
|
-
}
|
|
98
|
+
// Simple send
|
|
99
|
+
await send("my-topic", { message: "Hello world" });
|
|
199
100
|
|
|
101
|
+
// With options
|
|
102
|
+
await send(
|
|
103
|
+
"my-topic",
|
|
104
|
+
{ message: "Hello world" },
|
|
105
|
+
{
|
|
106
|
+
idempotencyKey: "unique-key", // Prevent duplicate messages
|
|
107
|
+
retentionSeconds: 3600, // 1 hour TTL (default: 24h)
|
|
108
|
+
delaySeconds: 60, // Delay delivery by 1 minute
|
|
109
|
+
},
|
|
110
|
+
);
|
|
200
111
|
```
|
|
201
112
|
|
|
202
|
-
|
|
113
|
+
Example usage in an API route:
|
|
203
114
|
|
|
204
|
-
```
|
|
205
|
-
// app/
|
|
206
|
-
|
|
207
|
-
import { publishTestMessage } from "./actions";
|
|
115
|
+
```typescript
|
|
116
|
+
// app/api/send-message/route.ts
|
|
117
|
+
import { send } from "@/lib/queue";
|
|
208
118
|
|
|
209
|
-
export
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
Publish Test Message
|
|
214
|
-
</button>
|
|
215
|
-
);
|
|
119
|
+
export async function POST(request: Request) {
|
|
120
|
+
const body = await request.json();
|
|
121
|
+
const { messageId } = await send("my-topic", { message: body.message });
|
|
122
|
+
return Response.json({ messageId });
|
|
216
123
|
}
|
|
217
124
|
```
|
|
218
125
|
|
|
219
|
-
|
|
126
|
+
> **Note:** `messageId` is `null` when the server accepts the message for deferred processing (e.g. during a server-side outage). The message will still be delivered.
|
|
220
127
|
|
|
221
|
-
|
|
222
|
-
callback functionality of Vercel queues to consume messages on the fly, when a
|
|
223
|
-
message is ready to be processed.
|
|
128
|
+
## Consuming Messages
|
|
224
129
|
|
|
225
|
-
|
|
130
|
+
### On Vercel
|
|
226
131
|
|
|
227
|
-
|
|
228
|
-
// app/api/queue/handle/route.ts
|
|
229
|
-
import { handleCallback } from "@vercel/queue";
|
|
230
|
-
|
|
231
|
-
// Option 1: Specify a single handler for the topic
|
|
232
|
-
export const POST = handleCallback({
|
|
233
|
-
"my-topic": (message, metadata) => {
|
|
234
|
-
console.log(`Received message:`, message, metadata);
|
|
235
|
-
// metadata: { messageId, deliveryCount, timestamp }
|
|
236
|
-
},
|
|
132
|
+
On Vercel, messages are consumed using API route handlers that Vercel automatically invokes when messages are available. Use `handleCallback` or `handleNodeCallback` to create these route handlers.
|
|
237
133
|
|
|
238
|
-
|
|
239
|
-
});
|
|
134
|
+
#### Web API — `handleCallback`
|
|
240
135
|
|
|
241
|
-
|
|
242
|
-
// were specified in the publish `callback` earlierA
|
|
136
|
+
Returns `(Request) => Promise<Response>`. For frameworks that export Web API route handlers (Next.js App Router, Hono, etc.).
|
|
243
137
|
|
|
244
|
-
|
|
245
|
-
export const POST = handleCallback({
|
|
246
|
-
// topic: "my-topic"
|
|
247
|
-
"my-topic": {
|
|
248
|
-
// consumer group: "compress"
|
|
249
|
-
"consumer-group-1": (message, metadata) => {
|
|
250
|
-
console.log("Message:", message);
|
|
251
|
-
},
|
|
252
|
-
// consumer group: "resize"
|
|
253
|
-
"consume-group-2": (message, metadata) => {
|
|
254
|
-
console.log("Message", message);
|
|
255
|
-
},
|
|
256
|
-
},
|
|
257
|
-
});
|
|
258
|
-
```
|
|
138
|
+
**Next.js App Router:**
|
|
259
139
|
|
|
260
|
-
|
|
140
|
+
```typescript
|
|
141
|
+
// app/api/queue/my-topic/route.ts
|
|
142
|
+
import { handleCallback } from "@/lib/queue";
|
|
261
143
|
|
|
262
|
-
|
|
144
|
+
export const POST = handleCallback(async (message, metadata) => {
|
|
145
|
+
// metadata: { messageId, deliveryCount, createdAt, expiresAt?, topicName, consumerGroup, region }
|
|
146
|
+
await processMessage(message);
|
|
147
|
+
// Throwing an error will automatically retry the message
|
|
148
|
+
});
|
|
149
|
+
```
|
|
263
150
|
|
|
264
|
-
|
|
151
|
+
**Hono:**
|
|
265
152
|
|
|
266
153
|
```typescript
|
|
267
|
-
import {
|
|
268
|
-
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
154
|
+
import { Hono } from "hono";
|
|
155
|
+
import { handleCallback } from "@/lib/queue";
|
|
156
|
+
|
|
157
|
+
const app = new Hono();
|
|
158
|
+
app.post(
|
|
159
|
+
"/api/queue",
|
|
160
|
+
handleCallback(async (message, metadata) => {
|
|
161
|
+
await processMessage(message);
|
|
162
|
+
}),
|
|
272
163
|
);
|
|
273
|
-
|
|
274
|
-
// Process large video files efficiently
|
|
275
|
-
const processor = videoTopic.consumerGroup("processors");
|
|
276
|
-
await processor.consume(async (videoStream) => {
|
|
277
|
-
// Process stream chunk by chunk
|
|
278
|
-
const reader = videoStream.getReader();
|
|
279
|
-
while (true) {
|
|
280
|
-
const { done, value } = await reader.read();
|
|
281
|
-
if (done) break;
|
|
282
|
-
await processChunk(value);
|
|
283
|
-
}
|
|
284
|
-
});
|
|
164
|
+
export default app;
|
|
285
165
|
```
|
|
286
166
|
|
|
287
|
-
|
|
167
|
+
#### Connect-style — `handleNodeCallback`
|
|
168
|
+
|
|
169
|
+
Returns `(req, res) => Promise<void>`. For frameworks that export Connect-style handlers (Express, Next.js Pages Router, etc.).
|
|
288
170
|
|
|
289
|
-
|
|
171
|
+
**Next.js Pages Router:**
|
|
290
172
|
|
|
291
173
|
```typescript
|
|
292
|
-
//
|
|
293
|
-
|
|
294
|
-
const worker2 = topic.consumerGroup("workers"); // Same group name
|
|
295
|
-
// worker1 and worker2 will receive different messages (load balancing)
|
|
174
|
+
// pages/api/queue/my-topic.ts
|
|
175
|
+
import { handleNodeCallback } from "@/lib/queue";
|
|
296
176
|
|
|
297
|
-
|
|
298
|
-
|
|
299
|
-
|
|
300
|
-
// analytics and webhooks will both receive every message
|
|
177
|
+
export default handleNodeCallback(async (message, metadata) => {
|
|
178
|
+
await processMessage(message);
|
|
179
|
+
});
|
|
301
180
|
```
|
|
302
181
|
|
|
303
|
-
|
|
304
|
-
|
|
305
|
-
- **Topics**: Named message channels with configurable serialization
|
|
306
|
-
- **Consumer Groups**: Named groups of consumers that process messages in
|
|
307
|
-
parallel
|
|
308
|
-
- `consume()`: Process messages with flexible consumption patterns
|
|
309
|
-
- No options: Process next available message
|
|
310
|
-
- With `messageId`: Process specific message by ID
|
|
311
|
-
- With `skipPayload: true`: Process message metadata only (without payload)
|
|
312
|
-
- **Transports**: Pluggable serialization/deserialization for different data
|
|
313
|
-
types
|
|
314
|
-
- **Streaming**: Memory-efficient processing of large payloads
|
|
315
|
-
- **Visibility Timeouts**: Automatic message lifecycle management
|
|
182
|
+
**Express:**
|
|
316
183
|
|
|
317
|
-
|
|
184
|
+
```typescript
|
|
185
|
+
import express from "express";
|
|
186
|
+
import { handleNodeCallback } from "@/lib/queue";
|
|
187
|
+
|
|
188
|
+
const app = express();
|
|
189
|
+
app.use(express.json());
|
|
190
|
+
app.post(
|
|
191
|
+
"/api/queue/my-topic",
|
|
192
|
+
handleNodeCallback(async (message, metadata) => {
|
|
193
|
+
await processMessage(message);
|
|
194
|
+
}),
|
|
195
|
+
);
|
|
196
|
+
export default app;
|
|
197
|
+
```
|
|
318
198
|
|
|
319
|
-
|
|
199
|
+
### 2. Configure vercel.json
|
|
320
200
|
|
|
321
|
-
|
|
322
|
-
- **Memory Efficient**: No buffering of complete payloads
|
|
323
|
-
- **Fast Parsing**: Native Buffer operations for ~50% performance improvement
|
|
324
|
-
- **Scalable**: Can handle arbitrarily large responses without memory
|
|
325
|
-
constraints
|
|
201
|
+
Tell Vercel which routes handle which topics:
|
|
326
202
|
|
|
327
|
-
|
|
203
|
+
```json
|
|
204
|
+
{
|
|
205
|
+
"functions": {
|
|
206
|
+
"app/api/queue/my-topic/route.ts": {
|
|
207
|
+
"experimentalTriggers": [
|
|
208
|
+
{
|
|
209
|
+
"type": "queue/v2beta",
|
|
210
|
+
"topic": "my-topic",
|
|
211
|
+
"retryAfterSeconds": 60,
|
|
212
|
+
"initialDelaySeconds": 0
|
|
213
|
+
}
|
|
214
|
+
]
|
|
215
|
+
},
|
|
216
|
+
"app/api/queue/orders/fulfillment/route.ts": {
|
|
217
|
+
"experimentalTriggers": [
|
|
218
|
+
{ "type": "queue/v2beta", "topic": "order-events" }
|
|
219
|
+
]
|
|
220
|
+
},
|
|
221
|
+
"app/api/queue/orders/analytics/route.ts": {
|
|
222
|
+
"experimentalTriggers": [
|
|
223
|
+
{
|
|
224
|
+
"type": "queue/v2beta",
|
|
225
|
+
"topic": "order-events",
|
|
226
|
+
"retryAfterSeconds": 300
|
|
227
|
+
}
|
|
228
|
+
]
|
|
229
|
+
}
|
|
230
|
+
}
|
|
231
|
+
}
|
|
232
|
+
```
|
|
328
233
|
|
|
329
|
-
|
|
330
|
-
interface with **streaming support** for memory-efficient processing. Transport
|
|
331
|
-
can be configured at the **topic level** when creating a topic, or at the
|
|
332
|
-
**consumer group level** when creating a consumer group.
|
|
234
|
+
Multiple route files for the same topic create separate consumer groups — each receives a copy of every message.
|
|
333
235
|
|
|
334
|
-
###
|
|
236
|
+
### 3. Retry and Backoff
|
|
335
237
|
|
|
336
|
-
|
|
238
|
+
When a handler throws, the message is not acknowledged and becomes available for redelivery after the `retryAfterSeconds` interval configured in `vercel.json`. Retries continue until the handler succeeds or the message expires (default: 24 hours).
|
|
337
239
|
|
|
338
|
-
|
|
339
|
-
memory.
|
|
240
|
+
For finer control over retry timing, pass a `retry` option:
|
|
340
241
|
|
|
341
242
|
```typescript
|
|
342
|
-
|
|
343
|
-
|
|
344
|
-
|
|
345
|
-
|
|
346
|
-
|
|
243
|
+
export const POST = handleCallback(
|
|
244
|
+
async (message, metadata) => {
|
|
245
|
+
await processMessage(message);
|
|
246
|
+
},
|
|
247
|
+
{
|
|
248
|
+
retry: (error, metadata) => {
|
|
249
|
+
if (error instanceof RateLimitError) return { afterSeconds: 60 };
|
|
250
|
+
// Return undefined to let the error propagate normally
|
|
251
|
+
},
|
|
252
|
+
},
|
|
253
|
+
);
|
|
347
254
|
```
|
|
348
255
|
|
|
349
|
-
|
|
256
|
+
When `retry` returns `{ afterSeconds: N }`, the message is rescheduled for redelivery after N seconds. Return `{ acknowledge: true }` to acknowledge the message so it is never retried. When it returns `undefined`, the error propagates normally and the message is retried at the default interval.
|
|
350
257
|
|
|
351
|
-
|
|
352
|
-
that fits in memory.
|
|
258
|
+
**Exponential backoff** uses `metadata.deliveryCount` (starts at 1, increments each delivery):
|
|
353
259
|
|
|
354
260
|
```typescript
|
|
355
|
-
|
|
356
|
-
|
|
357
|
-
|
|
358
|
-
|
|
359
|
-
|
|
261
|
+
export const POST = handleCallback(
|
|
262
|
+
async (message, metadata) => {
|
|
263
|
+
await processMessage(message);
|
|
264
|
+
},
|
|
265
|
+
{
|
|
266
|
+
retry: (error, metadata) => {
|
|
267
|
+
// 5s → 10s → 20s → 40s → ... capped at 5 min
|
|
268
|
+
const delay = Math.min(300, 2 ** metadata.deliveryCount * 5);
|
|
269
|
+
return { afterSeconds: delay };
|
|
270
|
+
},
|
|
271
|
+
},
|
|
272
|
+
);
|
|
360
273
|
```
|
|
361
274
|
|
|
362
|
-
|
|
363
|
-
|
|
364
|
-
**True streaming support** - passes ReadableStream directly without buffering.
|
|
365
|
-
Ideal for large files and memory-efficient processing.
|
|
275
|
+
**Conditional retry** — only retry transient errors:
|
|
366
276
|
|
|
367
277
|
```typescript
|
|
368
|
-
|
|
369
|
-
|
|
370
|
-
|
|
371
|
-
"streaming-topic",
|
|
372
|
-
new StreamTransport(),
|
|
373
|
-
);
|
|
374
|
-
|
|
375
|
-
// Send large file as stream without loading into memory
|
|
376
|
-
const fileStream = new ReadableStream<Uint8Array>({
|
|
377
|
-
start(controller) {
|
|
378
|
-
// Read file in chunks
|
|
379
|
-
for (const chunk of readFileInChunks("large-file.bin")) {
|
|
380
|
-
controller.enqueue(chunk);
|
|
381
|
-
}
|
|
382
|
-
controller.close();
|
|
278
|
+
export const POST = handleCallback(
|
|
279
|
+
async (message, metadata) => {
|
|
280
|
+
await processMessage(message);
|
|
383
281
|
},
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
282
|
+
{
|
|
283
|
+
retry: (error, metadata) => {
|
|
284
|
+
if (error instanceof RateLimitError) return { afterSeconds: 60 };
|
|
285
|
+
if (error instanceof TemporaryError) return { afterSeconds: 30 };
|
|
286
|
+
// Permanent errors: return undefined → retried at the default interval
|
|
287
|
+
},
|
|
288
|
+
},
|
|
289
|
+
);
|
|
387
290
|
```
|
|
388
291
|
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
You can create your own serialization format by implementing the `Transport`
|
|
392
|
-
interface:
|
|
292
|
+
**Acknowledging poison messages** — stop retrying messages that can never succeed:
|
|
393
293
|
|
|
394
294
|
```typescript
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
}
|
|
295
|
+
export const POST = handleCallback(
|
|
296
|
+
async (message, metadata) => {
|
|
297
|
+
await processMessage(message);
|
|
298
|
+
},
|
|
299
|
+
{
|
|
300
|
+
retry: (error, metadata) => {
|
|
301
|
+
if (error instanceof ValidationError) return { acknowledge: true };
|
|
302
|
+
if (metadata.deliveryCount > 5) return { acknowledge: true };
|
|
303
|
+
return { afterSeconds: Math.min(300, 2 ** metadata.deliveryCount * 5) };
|
|
304
|
+
},
|
|
305
|
+
},
|
|
306
|
+
);
|
|
402
307
|
```
|
|
403
308
|
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
| Use Case | Recommended Transport | Memory Usage | Performance |
|
|
407
|
-
| ---------------------- | --------------------- | ------------ | ----------- |
|
|
408
|
-
| Small JSON objects | `JsonTransport` | Low | High |
|
|
409
|
-
| Binary files < 100MB | `BufferTransport` | Medium | High |
|
|
410
|
-
| Large files > 100MB | `StreamTransport` | Very Low | Medium |
|
|
411
|
-
| Real-time data streams | `StreamTransport` | Very Low | High |
|
|
412
|
-
| Custom protocols | Custom implementation | Varies | Varies |
|
|
309
|
+
The `retry` option is available on `handleCallback`, `handleNodeCallback`, and `receive`.
|
|
413
310
|
|
|
414
|
-
##
|
|
311
|
+
## Custom Client Configuration
|
|
415
312
|
|
|
416
|
-
|
|
313
|
+
All configuration lives on the `QueueClient`:
|
|
417
314
|
|
|
418
315
|
```typescript
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
//
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
|
|
316
|
+
import { QueueClient, BufferTransport } from "@vercel/queue";
|
|
317
|
+
|
|
318
|
+
const queue = new QueueClient({
|
|
319
|
+
region: process.env.QUEUE_REGION!, // Required — see Quick Start for env setup
|
|
320
|
+
token: "my-token", // Auth token (default: OIDC auto-detection)
|
|
321
|
+
transport: new BufferTransport(), // Serialization (default: JsonTransport)
|
|
322
|
+
headers: { "X-Custom": "header" }, // Custom headers on all requests
|
|
323
|
+
deploymentId: null, // null = unpinned, omit = auto from env, or explicit string
|
|
426
324
|
});
|
|
427
|
-
```
|
|
428
325
|
|
|
429
|
-
|
|
326
|
+
// Use directly
|
|
327
|
+
await queue.send("my-topic", myBuffer);
|
|
430
328
|
|
|
431
|
-
|
|
432
|
-
|
|
433
|
-
|
|
329
|
+
// Or destructure
|
|
330
|
+
export const { send, receive, handleCallback, handleNodeCallback } = queue;
|
|
331
|
+
```
|
|
434
332
|
|
|
435
|
-
|
|
436
|
-
const customClient = new QueueClient({ baseUrl: "https://custom.vqs.vercel.sh" });
|
|
437
|
-
const topic = new Topic<T>(customClient, topicName, transport?);
|
|
333
|
+
The client sends requests to `https://${region}.vercel-queue.com`. When `handleCallback` receives a message, it reads the `ce-vqsregion` header and routes follow-up API calls to the correct regional endpoint.
|
|
438
334
|
|
|
439
|
-
|
|
440
|
-
await topic.publish(payload, options?);
|
|
335
|
+
To customize the URL scheme, provide a `resolveBaseUrl`:
|
|
441
336
|
|
|
442
|
-
|
|
443
|
-
|
|
444
|
-
|
|
445
|
-
|
|
337
|
+
```typescript
|
|
338
|
+
const queue = new QueueClient({
|
|
339
|
+
region: process.env.QUEUE_REGION!,
|
|
340
|
+
resolveBaseUrl: (region) => `https://${region}.my-proxy.example`,
|
|
446
341
|
});
|
|
342
|
+
```
|
|
447
343
|
|
|
448
|
-
|
|
449
|
-
// with a separate consumer group)
|
|
450
|
-
await topic.publish(payload, {
|
|
451
|
-
callback: {
|
|
452
|
-
group1: { url: "https://example.com/webhook1" },
|
|
453
|
-
group2: { url: "https://example.com/webhook2", delay: 30 }
|
|
454
|
-
}
|
|
455
|
-
});
|
|
344
|
+
## Transports
|
|
456
345
|
|
|
457
|
-
|
|
458
|
-
const consumer = topic.consumerGroup<U>(groupName, options?);
|
|
459
|
-
```
|
|
346
|
+
The transport controls how message payloads are serialized and deserialized.
|
|
460
347
|
|
|
461
|
-
|
|
348
|
+
| Use Case | Transport | Memory Usage | Notes |
|
|
349
|
+
| --------------- | ----------------- | ------------ | ----------------------- |
|
|
350
|
+
| Structured data | `JsonTransport` | Low | Default, JSON encoding |
|
|
351
|
+
| Binary data | `BufferTransport` | Medium | Raw bytes |
|
|
352
|
+
| Large payloads | `StreamTransport` | Very Low | No buffering, streaming |
|
|
462
353
|
|
|
463
354
|
```typescript
|
|
464
|
-
|
|
465
|
-
|
|
466
|
-
|
|
467
|
-
|
|
468
|
-
|
|
469
|
-
|
|
470
|
-
idempotencyKey?: string;
|
|
471
|
-
retentionSeconds?: number;
|
|
472
|
-
callback?: Record<string, CallbackConfig> | CallbackConfig;
|
|
473
|
-
});
|
|
355
|
+
import {
|
|
356
|
+
QueueClient,
|
|
357
|
+
JsonTransport,
|
|
358
|
+
BufferTransport,
|
|
359
|
+
StreamTransport,
|
|
360
|
+
} from "@vercel/queue";
|
|
474
361
|
|
|
475
|
-
//
|
|
476
|
-
|
|
362
|
+
// JSON with custom serialization
|
|
363
|
+
const queue = new QueueClient({
|
|
364
|
+
region: process.env.QUEUE_REGION!,
|
|
365
|
+
transport: new JsonTransport({
|
|
366
|
+
replacer: (key, value) => (key === "password" ? undefined : value),
|
|
367
|
+
reviver: (key, value) => (key === "date" ? new Date(value) : value),
|
|
368
|
+
}),
|
|
369
|
+
});
|
|
477
370
|
|
|
478
|
-
|
|
371
|
+
// Binary data
|
|
372
|
+
const binQueue = new QueueClient({
|
|
373
|
+
region: process.env.QUEUE_REGION!,
|
|
479
374
|
transport: new BufferTransport(),
|
|
480
|
-
callback: { url: "https://example.com/process-image" }
|
|
481
375
|
});
|
|
376
|
+
await binQueue.send("binary-topic", myBuffer);
|
|
482
377
|
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
|
|
486
|
-
|
|
487
|
-
analytics: { url: "https://analytics.example.com/webhook" },
|
|
488
|
-
notifications: { url: "https://notifications.example.com/webhook", delay: 30 }
|
|
489
|
-
}
|
|
378
|
+
// Streaming for large payloads
|
|
379
|
+
const streamQueue = new QueueClient({
|
|
380
|
+
region: process.env.QUEUE_REGION!,
|
|
381
|
+
transport: new StreamTransport(),
|
|
490
382
|
});
|
|
383
|
+
await streamQueue.send("large-file", myReadableStream);
|
|
491
384
|
```
|
|
492
385
|
|
|
493
|
-
|
|
494
|
-
|
|
495
|
-
```typescript
|
|
496
|
-
// Process next available message (simplest form)
|
|
497
|
-
await consumer.consume(handler);
|
|
498
|
-
|
|
499
|
-
// Process specific message by ID with payload
|
|
500
|
-
await consumer.consume(handler, { messageId: "message-id" });
|
|
386
|
+
## Manual Receive
|
|
501
387
|
|
|
502
|
-
|
|
503
|
-
// handler will be called with `undefined` as the payload
|
|
504
|
-
await consumer.consume(handler, { messageId: "message-id", skipPayload: true });
|
|
505
|
-
```
|
|
506
|
-
|
|
507
|
-
### Message Handler
|
|
388
|
+
Use `receive` to pull and process messages directly. This is an advanced alternative to `handleCallback` that works in any Node.js environment, both on and off Vercel.
|
|
508
389
|
|
|
509
|
-
|
|
510
|
-
// Handler function signature
|
|
511
|
-
type MessageHandler<T = unknown> = (
|
|
512
|
-
message: T,
|
|
513
|
-
metadata: MessageMetadata,
|
|
514
|
-
) => Promise<MessageHandlerResult> | MessageHandlerResult;
|
|
390
|
+
### Region considerations
|
|
515
391
|
|
|
516
|
-
|
|
517
|
-
type MessageHandlerResult = void | MessageTimeoutResult;
|
|
392
|
+
Messages can only be received from the region they were sent to. When using `receive`, use a **fixed region** (e.g. `"iad1"`) for both sending and receiving — do not use `VERCEL_REGION` (or `QUEUE_REGION=${VERCEL_REGION}`), because Vercel may route requests to different regions due to failover or load balancing, distributing your messages across regions unpredictably.
|
|
518
393
|
|
|
519
|
-
|
|
520
|
-
|
|
521
|
-
|
|
394
|
+
```bash
|
|
395
|
+
# .env.production — fixed region for manual receive workflows
|
|
396
|
+
QUEUE_REGION=iad1
|
|
522
397
|
|
|
523
|
-
|
|
524
|
-
|
|
525
|
-
messageId: string;
|
|
526
|
-
deliveryCount: number;
|
|
527
|
-
timestamp: string;
|
|
528
|
-
}
|
|
398
|
+
# .env.development
|
|
399
|
+
QUEUE_REGION=iad1
|
|
529
400
|
```
|
|
530
401
|
|
|
531
|
-
|
|
402
|
+
A single region is still highly available — Vercel deploys across 3+ availability zones within each region. If you need multi-region availability, you are responsible for designing your own HA strategy (e.g. sending to multiple regions and receiving from each).
|
|
532
403
|
|
|
533
|
-
|
|
534
|
-
interface ConsumeOptions {
|
|
535
|
-
messageId?: string; // Process specific message by ID
|
|
536
|
-
skipPayload?: boolean; // Skip payload download (requires messageId)
|
|
537
|
-
}
|
|
538
|
-
```
|
|
404
|
+
For most use cases on Vercel, `handleCallback` is the recommended approach — the platform handles region routing automatically and the SDK routes follow-up calls to the correct region via the `ce-vqsregion` header.
|
|
539
405
|
|
|
540
|
-
###
|
|
406
|
+
### Usage
|
|
541
407
|
|
|
542
408
|
```typescript
|
|
543
|
-
|
|
544
|
-
serialize(value: T): Buffer | ReadableStream<Uint8Array>;
|
|
545
|
-
deserialize(stream: ReadableStream<Uint8Array>): Promise<T>;
|
|
546
|
-
contentType: string;
|
|
547
|
-
}
|
|
548
|
-
```
|
|
409
|
+
import { QueueClient } from "@vercel/queue";
|
|
549
410
|
|
|
550
|
-
|
|
551
|
-
|
|
552
|
-
```typescript
|
|
553
|
-
// Parse queue callback request headers
|
|
554
|
-
function parseCallbackRequest(request: Request): CallbackMessageOptions;
|
|
411
|
+
const { receive } = new QueueClient({ region: "iad1" });
|
|
555
412
|
|
|
556
|
-
//
|
|
557
|
-
|
|
558
|
-
|
|
559
|
-
|
|
560
|
-
|
|
561
|
-
|
|
562
|
-
// Create a callback handler for NextJS route handlers
|
|
563
|
-
function handleCallback(handlers: CallbackHandlers): (request: Request) => Promise<Response>;
|
|
564
|
-
|
|
565
|
-
// Configuration object with handlers for different topics
|
|
566
|
-
type CallbackHandlers = {
|
|
567
|
-
[topicName: string]:
|
|
568
|
-
| MessageHandler // Single handler (uses 'default' consumer group)
|
|
569
|
-
| { [consumerGroup: string]: MessageHandler }; // Multiple consumer group handlers
|
|
570
|
-
};
|
|
571
|
-
|
|
572
|
-
// Example usage:
|
|
573
|
-
export const POST = handleCallback({
|
|
574
|
-
// Topic handler (uses 'default' consumer group)
|
|
575
|
-
"new-users": (message, metadata) => {
|
|
576
|
-
console.log(`New user event:`, message, metadata);
|
|
413
|
+
// Process next available message
|
|
414
|
+
const result = await receive(
|
|
415
|
+
"my-topic",
|
|
416
|
+
"my-group",
|
|
417
|
+
async (message, metadata) => {
|
|
418
|
+
console.log("Processing:", message);
|
|
577
419
|
},
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
"
|
|
581
|
-
"compress": (message, metadata) => console.log("Compressing image", message),
|
|
582
|
-
"resize": (message, metadata) => console.log("Resizing image", message),
|
|
583
|
-
}
|
|
584
|
-
});
|
|
585
|
-
|
|
586
|
-
// Error thrown for invalid callback requests
|
|
587
|
-
class InvalidCallbackError extends Error;
|
|
588
|
-
```
|
|
589
|
-
|
|
590
|
-
## Examples
|
|
591
|
-
|
|
592
|
-
### Basic JSON Processing
|
|
593
|
-
|
|
594
|
-
```typescript
|
|
595
|
-
interface UserEvent {
|
|
596
|
-
userId: string;
|
|
597
|
-
action: string;
|
|
598
|
-
timestamp: number;
|
|
420
|
+
);
|
|
421
|
+
if (!result.ok) {
|
|
422
|
+
console.log("Queue was empty:", result.reason);
|
|
599
423
|
}
|
|
600
424
|
|
|
601
|
-
//
|
|
602
|
-
await
|
|
603
|
-
userId: "123",
|
|
604
|
-
action: "login",
|
|
605
|
-
timestamp: Date.now(),
|
|
606
|
-
});
|
|
607
|
-
|
|
608
|
-
// Option 2: Using createTopic for consumers
|
|
609
|
-
const userTopic = createTopic<UserEvent>("user-events");
|
|
610
|
-
|
|
611
|
-
await userTopic.publish({
|
|
612
|
-
userId: "123",
|
|
613
|
-
action: "login",
|
|
614
|
-
timestamp: Date.now(),
|
|
615
|
-
});
|
|
616
|
-
|
|
617
|
-
const consumer = userTopic.consumerGroup("processors");
|
|
425
|
+
// Batch processing: up to 10 messages in one request
|
|
426
|
+
await receive("my-topic", "my-group", handler, { limit: 10 });
|
|
618
427
|
|
|
619
|
-
// Process
|
|
620
|
-
|
|
621
|
-
await consumer.consume(async (message) => {
|
|
622
|
-
console.log(`User ${message.userId} performed ${message.action}`);
|
|
623
|
-
});
|
|
624
|
-
} catch (error) {
|
|
625
|
-
console.error("Processing error:", error);
|
|
626
|
-
}
|
|
428
|
+
// Process a specific message by ID
|
|
429
|
+
await receive("my-topic", "my-group", handler, { messageId: "msg-123" });
|
|
627
430
|
```
|
|
628
431
|
|
|
629
|
-
|
|
432
|
+
> **Note:** `limit` and `messageId` are mutually exclusive options. The handler is never called when the queue is empty — check `result.ok` instead.
|
|
630
433
|
|
|
631
|
-
|
|
632
|
-
const userTopic = createTopic<{ userId: string; action: string }>(
|
|
633
|
-
"user-events",
|
|
634
|
-
);
|
|
635
|
-
const consumer = userTopic.consumerGroup("processors");
|
|
434
|
+
## Error Handling
|
|
636
435
|
|
|
637
|
-
|
|
638
|
-
|
|
436
|
+
```typescript
|
|
437
|
+
import {
|
|
438
|
+
BadRequestError,
|
|
439
|
+
DuplicateMessageError,
|
|
440
|
+
ForbiddenError,
|
|
441
|
+
InternalServerError,
|
|
442
|
+
UnauthorizedError,
|
|
443
|
+
} from "@vercel/queue";
|
|
444
|
+
import { send } from "@/lib/queue";
|
|
639
445
|
|
|
640
446
|
try {
|
|
641
|
-
await
|
|
642
|
-
async (message, { messageId }) => {
|
|
643
|
-
console.log(`Processing specific message: ${messageId}`);
|
|
644
|
-
console.log(`User ${message.userId} performed ${message.action}`);
|
|
645
|
-
},
|
|
646
|
-
{ messageId },
|
|
647
|
-
);
|
|
648
|
-
console.log("Message processed successfully");
|
|
447
|
+
await send("my-topic", payload);
|
|
649
448
|
} catch (error) {
|
|
650
|
-
if (error
|
|
651
|
-
console.log("
|
|
652
|
-
} else if (error
|
|
653
|
-
console.log("
|
|
654
|
-
} else {
|
|
655
|
-
console.
|
|
449
|
+
if (error instanceof UnauthorizedError) {
|
|
450
|
+
console.log("Invalid token - refresh authentication");
|
|
451
|
+
} else if (error instanceof ForbiddenError) {
|
|
452
|
+
console.log("Environment mismatch - check configuration");
|
|
453
|
+
} else if (error instanceof BadRequestError) {
|
|
454
|
+
console.log("Invalid parameters:", error.message);
|
|
455
|
+
} else if (error instanceof DuplicateMessageError) {
|
|
456
|
+
console.log("Duplicate message:", error.idempotencyKey);
|
|
457
|
+
} else if (error instanceof InternalServerError) {
|
|
458
|
+
console.log("Server error - retry with backoff");
|
|
656
459
|
}
|
|
657
460
|
}
|
|
658
461
|
```
|
|
659
462
|
|
|
660
|
-
|
|
463
|
+
All error types:
|
|
464
|
+
|
|
465
|
+
| Error | Description |
|
|
466
|
+
| ------------------------------------ | --------------------------------------------- |
|
|
467
|
+
| `BadRequestError` | Invalid request parameters |
|
|
468
|
+
| `UnauthorizedError` | Authentication failed (invalid/missing token) |
|
|
469
|
+
| `ForbiddenError` | Access denied (wrong environment/project) |
|
|
470
|
+
| `DuplicateMessageError` | Idempotency key already used |
|
|
471
|
+
| `ConsumerDiscoveryError` | Could not reach consumer deployment |
|
|
472
|
+
| `ConsumerRegistryNotConfiguredError` | Project not configured for queues |
|
|
473
|
+
| `InternalServerError` | Unexpected server error |
|
|
474
|
+
| `InvalidLimitError` | Batch limit outside valid range (1-10) |
|
|
475
|
+
| `MessageNotFoundError` | Message doesn't exist or expired |
|
|
476
|
+
| `MessageNotAvailableError` | Message exists but cannot be claimed |
|
|
477
|
+
| `MessageAlreadyProcessedError` | Message already successfully processed |
|
|
478
|
+
| `MessageLockedError` | Message being processed by another consumer |
|
|
479
|
+
| `MessageCorruptedError` | Message data could not be parsed |
|
|
480
|
+
| `QueueEmptyError` | No messages available in queue |
|
|
481
|
+
|
|
482
|
+
## Environment Variables
|
|
483
|
+
|
|
484
|
+
| Variable | Description | Default |
|
|
485
|
+
| ---------------------- | ----------------------------------------------- | ------- |
|
|
486
|
+
| `QUEUE_REGION` | Region code for the queue client (user-defined) | - |
|
|
487
|
+
| `VERCEL_REGION` | Current region (auto-set by Vercel) | - |
|
|
488
|
+
| `VERCEL_QUEUE_DEBUG` | Enable debug logging (`1` or `true`) | - |
|
|
489
|
+
| `VERCEL_DEPLOYMENT_ID` | Deployment ID (auto-set by Vercel) | - |
|
|
490
|
+
|
|
491
|
+
## Service Limits & Constraints
|
|
492
|
+
|
|
493
|
+
### Throughput & Storage
|
|
494
|
+
|
|
495
|
+
| Limit | Value | Notes |
|
|
496
|
+
| --------------------------- | --------------------- | ----------------------------------- |
|
|
497
|
+
| Message throughput | 10,000+ msg/sec/topic | Scales horizontally |
|
|
498
|
+
| Payload size | 1 GB | Smaller messages have lower latency |
|
|
499
|
+
| Number of topics | Unlimited | No hard limit |
|
|
500
|
+
| Consumer groups per message | ~4,000 | Per-message limit |
|
|
501
|
+
| Messages per queue | Unlimited | No hard limit |
|
|
502
|
+
|
|
503
|
+
### Parameter Constraints
|
|
504
|
+
|
|
505
|
+
#### Publishing Messages
|
|
506
|
+
|
|
507
|
+
| Parameter | Default | Min | Max | Notes |
|
|
508
|
+
| ------------------ | ------------ | --- | ----------- | ----------------------------------- |
|
|
509
|
+
| `retentionSeconds` | 86,400 (24h) | 60 | 86,400 | Message TTL |
|
|
510
|
+
| `delaySeconds` | 0 | 0 | ≤ retention | Cannot exceed retention |
|
|
511
|
+
| `idempotencyKey` | — | — | — | Dedup window: `min(retention, 24h)` |
|
|
512
|
+
|
|
513
|
+
#### Receiving Messages
|
|
514
|
+
|
|
515
|
+
| Parameter | Default | Min | Max | Notes |
|
|
516
|
+
| -------------------------- | ------- | --- | ----- | ------------------------------- |
|
|
517
|
+
| `visibilityTimeoutSeconds` | 300 | 30 | 3,600 | Lock duration during processing |
|
|
518
|
+
| `limit` | 1 | 1 | 10 | Messages per request |
|
|
519
|
+
|
|
520
|
+
### Identifier Formats
|
|
521
|
+
|
|
522
|
+
| Identifier | Pattern | Example |
|
|
523
|
+
| -------------- | ---------------- | ----------------------------------- |
|
|
524
|
+
| Topic name | `[A-Za-z0-9_-]+` | `my-queue`, `task_queue_v2` |
|
|
525
|
+
| Consumer group | `[A-Za-z0-9_-]+` | `worker-1`, `analytics_consumer` |
|
|
526
|
+
| Message ID | Opaque string | `0-1`, `3-7K9mNpQrS` |
|
|
527
|
+
| Receipt handle | Opaque string | Used for acknowledge/visibility ops |
|
|
528
|
+
|
|
529
|
+
### Wildcard Topics
|
|
661
530
|
|
|
662
|
-
```
|
|
663
|
-
|
|
664
|
-
|
|
665
|
-
|
|
666
|
-
|
|
667
|
-
try {
|
|
668
|
-
await worker.consume(async (message) => {
|
|
669
|
-
console.log(`Processing task: ${message.taskType}`);
|
|
670
|
-
await processTask(message.taskType, message.data);
|
|
671
|
-
});
|
|
672
|
-
console.log("Message processed successfully");
|
|
673
|
-
} catch (error) {
|
|
674
|
-
if (error instanceof QueueEmptyError) {
|
|
675
|
-
console.log("No messages available");
|
|
676
|
-
} else if (error instanceof MessageLockedError) {
|
|
677
|
-
console.log("Next message is locked (FIFO queue)");
|
|
678
|
-
if (error.retryAfter) {
|
|
679
|
-
console.log(`Retry after ${error.retryAfter} seconds`);
|
|
531
|
+
```json
|
|
532
|
+
{
|
|
533
|
+
"functions": {
|
|
534
|
+
"app/api/queue/route.ts": {
|
|
535
|
+
"experimentalTriggers": [{ "type": "queue/v2beta", "topic": "user-*" }]
|
|
680
536
|
}
|
|
681
|
-
} else {
|
|
682
|
-
console.error("Error processing message:", error);
|
|
683
537
|
}
|
|
684
538
|
}
|
|
685
|
-
|
|
686
|
-
// Handle conditional timeouts
|
|
687
|
-
await worker.consume(async (message) => {
|
|
688
|
-
if (!canProcessTaskType(message.taskType)) {
|
|
689
|
-
// Return timeout to retry later
|
|
690
|
-
return { timeoutSeconds: 60 };
|
|
691
|
-
}
|
|
692
|
-
|
|
693
|
-
await processTask(message.taskType, message.data);
|
|
694
|
-
});
|
|
695
|
-
|
|
696
|
-
// Process specific message metadata only (no payload download)
|
|
697
|
-
await worker.consume(
|
|
698
|
-
async (_, metadata) => {
|
|
699
|
-
console.log(`Message ID: ${metadata.messageId}`);
|
|
700
|
-
console.log(`Delivery count: ${metadata.deliveryCount}`);
|
|
701
|
-
console.log(`Timestamp: ${metadata.timestamp}`);
|
|
702
|
-
// _ is undefined - no payload was downloaded
|
|
703
|
-
},
|
|
704
|
-
{ messageId: "specific-message-id", skipPayload: true },
|
|
705
|
-
);
|
|
706
539
|
```
|
|
707
540
|
|
|
708
|
-
|
|
541
|
+
- `*` may only appear **once** in the pattern
|
|
542
|
+
- `*` must be at the **end** of the topic name
|
|
543
|
+
- Valid: `user-*`, `orders-*`
|
|
544
|
+
- Invalid: `*-events`, `user-*-data`
|
|
709
545
|
|
|
710
|
-
|
|
711
|
-
const workTopic = createTopic<{ taskType: string; data: any }>("work-queue");
|
|
712
|
-
const worker = workTopic.consumerGroup("workers");
|
|
546
|
+
## API Reference
|
|
713
547
|
|
|
714
|
-
|
|
715
|
-
try {
|
|
716
|
-
await worker.consume(async ({ taskType, data }) => {
|
|
717
|
-
// Check if we can process this task type right now
|
|
718
|
-
if (taskType === "heavy-computation" && isSystemOverloaded()) {
|
|
719
|
-
// Return timeout to retry later (5 minutes)
|
|
720
|
-
return { timeoutSeconds: 300 };
|
|
721
|
-
}
|
|
548
|
+
### `QueueClient`
|
|
722
549
|
|
|
723
|
-
|
|
724
|
-
|
|
725
|
-
|
|
726
|
-
|
|
727
|
-
|
|
728
|
-
|
|
729
|
-
|
|
730
|
-
|
|
731
|
-
|
|
732
|
-
|
|
733
|
-
|
|
734
|
-
} catch (error) {
|
|
735
|
-
console.error("Worker processing error:", error);
|
|
736
|
-
}
|
|
550
|
+
```typescript
|
|
551
|
+
import { QueueClient } from "@vercel/queue";
|
|
552
|
+
|
|
553
|
+
const queue = new QueueClient({
|
|
554
|
+
region: process.env.QUEUE_REGION!, // Required — see Quick Start for env setup
|
|
555
|
+
resolveBaseUrl: (r) => `https://${r}.vercel-queue.com`, // Default resolver
|
|
556
|
+
token: "my-token", // Auto-fetched via OIDC if omitted
|
|
557
|
+
headers: { "X-Custom": "value" },
|
|
558
|
+
transport: new JsonTransport(), // Default: JsonTransport
|
|
559
|
+
deploymentId: undefined, // omit = auto from env (pinned), null = unpinned, or explicit string
|
|
560
|
+
});
|
|
737
561
|
|
|
738
|
-
//
|
|
739
|
-
|
|
740
|
-
await worker.consume(async (message, { deliveryCount }) => {
|
|
741
|
-
const maxRetries = 3;
|
|
742
|
-
|
|
743
|
-
try {
|
|
744
|
-
await processMessage(message);
|
|
745
|
-
// Successful processing - message will be deleted
|
|
746
|
-
} catch (error) {
|
|
747
|
-
if (deliveryCount < maxRetries) {
|
|
748
|
-
// Exponential backoff: 2^deliveryCount minutes
|
|
749
|
-
const timeoutSeconds = Math.pow(2, deliveryCount) * 60;
|
|
750
|
-
console.log(
|
|
751
|
-
`Retrying message in ${timeoutSeconds} seconds (attempt ${deliveryCount})`,
|
|
752
|
-
);
|
|
753
|
-
return { timeoutSeconds: timeoutSeconds };
|
|
754
|
-
} else {
|
|
755
|
-
// Max retries reached, let the message fail and be deleted
|
|
756
|
-
console.error("Max retries reached, message will be discarded:", error);
|
|
757
|
-
throw error;
|
|
758
|
-
}
|
|
759
|
-
}
|
|
760
|
-
});
|
|
761
|
-
} catch (error) {
|
|
762
|
-
console.error("Backoff processing error:", error);
|
|
763
|
-
}
|
|
562
|
+
// Methods (arrow functions — safe to destructure)
|
|
563
|
+
const { send, receive, handleCallback, handleNodeCallback } = queue;
|
|
764
564
|
```
|
|
765
565
|
|
|
766
|
-
###
|
|
566
|
+
### `send(topicName, payload, options?)`
|
|
767
567
|
|
|
768
|
-
|
|
769
|
-
processes videos with FFmpeg and stores the results in Vercel Blob:
|
|
568
|
+
Returns `{ messageId: string | null }`. `messageId` is `null` when the server accepted the message for deferred processing (e.g. during a server-side outage).
|
|
770
569
|
|
|
771
570
|
```typescript
|
|
772
|
-
|
|
773
|
-
|
|
774
|
-
|
|
775
|
-
|
|
776
|
-
|
|
777
|
-
|
|
778
|
-
|
|
779
|
-
"unoptimized-videos",
|
|
780
|
-
new StreamTransport(),
|
|
781
|
-
);
|
|
782
|
-
|
|
783
|
-
// Output topic for optimized videos
|
|
784
|
-
const optimizedVideosTopic = createTopic<ReadableStream<Uint8Array>>(
|
|
785
|
-
"optimized-videos",
|
|
786
|
-
new StreamTransport(),
|
|
787
|
-
);
|
|
788
|
-
|
|
789
|
-
// Step 1: Process videos with FFmpeg
|
|
790
|
-
const videoProcessor = unoptimizedVideosTopic.consumerGroup("processors");
|
|
571
|
+
const { messageId } = await send("my-topic", payload, {
|
|
572
|
+
idempotencyKey: "unique-key", // Dedup window: min(retention, 24h)
|
|
573
|
+
retentionSeconds: 3600, // Message TTL (default: 86400)
|
|
574
|
+
delaySeconds: 60, // Delay before visible (default: 0)
|
|
575
|
+
headers: { "X-Custom": "val" }, // Custom headers
|
|
576
|
+
});
|
|
577
|
+
```
|
|
791
578
|
|
|
792
|
-
|
|
793
|
-
await videoProcessor.consume(async (inputVideoStream) => {
|
|
794
|
-
console.log("Processing video...");
|
|
579
|
+
### `receive(topicName, consumerGroup, handler, options?)`
|
|
795
580
|
|
|
796
|
-
|
|
797
|
-
throw new Error("FFmpeg not available");
|
|
798
|
-
}
|
|
581
|
+
Returns a discriminated result: `{ ok: true }` on success, or `{ ok: false, reason }` when no message was processed. The handler is never called when the queue is empty.
|
|
799
582
|
|
|
800
|
-
|
|
801
|
-
const optimizedStream = new ReadableStream<Uint8Array>({
|
|
802
|
-
start(controller) {
|
|
803
|
-
const ffmpegProcess = spawn(
|
|
804
|
-
ffmpeg,
|
|
805
|
-
[
|
|
806
|
-
"-i",
|
|
807
|
-
"pipe:0", // Input from stdin
|
|
808
|
-
"-c:v",
|
|
809
|
-
"libvpx-vp9", // Video codec
|
|
810
|
-
"-c:a",
|
|
811
|
-
"libopus", // Audio codec
|
|
812
|
-
"-crf",
|
|
813
|
-
"23", // Quality
|
|
814
|
-
"-f",
|
|
815
|
-
"webm", // Output format
|
|
816
|
-
"pipe:1", // Output to stdout
|
|
817
|
-
],
|
|
818
|
-
{ stdio: ["pipe", "pipe", "pipe"] },
|
|
819
|
-
);
|
|
820
|
-
|
|
821
|
-
// Pipe input stream to FFmpeg
|
|
822
|
-
const reader = inputVideoStream.getReader();
|
|
823
|
-
const pipeInput = async () => {
|
|
824
|
-
while (true) {
|
|
825
|
-
const { done, value } = await reader.read();
|
|
826
|
-
if (done) {
|
|
827
|
-
ffmpegProcess.stdin?.end();
|
|
828
|
-
break;
|
|
829
|
-
}
|
|
830
|
-
ffmpegProcess.stdin?.write(value);
|
|
831
|
-
}
|
|
832
|
-
};
|
|
833
|
-
pipeInput();
|
|
834
|
-
|
|
835
|
-
// Stream FFmpeg output
|
|
836
|
-
ffmpegProcess.stdout?.on("data", (chunk) => {
|
|
837
|
-
controller.enqueue(new Uint8Array(chunk));
|
|
838
|
-
});
|
|
839
|
-
|
|
840
|
-
ffmpegProcess.on("close", (code) => {
|
|
841
|
-
if (code === 0) {
|
|
842
|
-
controller.close();
|
|
843
|
-
} else {
|
|
844
|
-
controller.error(new Error(`FFmpeg failed with code ${code}`));
|
|
845
|
-
}
|
|
846
|
-
});
|
|
847
|
-
},
|
|
848
|
-
});
|
|
849
|
-
|
|
850
|
-
// Publish optimized video to next topic
|
|
851
|
-
await optimizedVideosTopic.publish(optimizedStream);
|
|
852
|
-
console.log("Video optimized and published");
|
|
853
|
-
});
|
|
854
|
-
} catch (error) {
|
|
855
|
-
console.error("Video processing error:", error);
|
|
856
|
-
}
|
|
857
|
-
|
|
858
|
-
// Step 2: Store optimized videos in Vercel Blob
|
|
859
|
-
const blobUploader = optimizedVideosTopic.consumerGroup("blob-uploaders");
|
|
583
|
+
For receive-by-id, operational errors are returned instead of thrown:
|
|
860
584
|
|
|
861
|
-
|
|
862
|
-
|
|
863
|
-
|
|
864
|
-
|
|
865
|
-
|
|
866
|
-
|
|
867
|
-
|
|
868
|
-
});
|
|
869
|
-
|
|
870
|
-
console.log(`Video uploaded to blob: ${blob.url} (${blob.size} bytes)`);
|
|
871
|
-
});
|
|
872
|
-
} catch (error) {
|
|
873
|
-
console.error("Blob upload error:", error);
|
|
585
|
+
```typescript
|
|
586
|
+
const result = await receive("my-topic", "my-group", handler, {
|
|
587
|
+
messageId: "msg-123",
|
|
588
|
+
});
|
|
589
|
+
if (!result.ok) {
|
|
590
|
+
// result.reason is "not_found" | "not_available" | "already_processed"
|
|
591
|
+
console.log(result.reason, result.messageId);
|
|
874
592
|
}
|
|
875
593
|
```
|
|
876
594
|
|
|
877
|
-
|
|
878
|
-
|
|
879
|
-
|
|
880
|
-
|
|
881
|
-
|
|
882
|
-
|
|
883
|
-
|
|
884
|
-
empty queue (204 status)
|
|
885
|
-
|
|
886
|
-
- Thrown by `consume()` when no messages are available
|
|
887
|
-
- Also thrown when directly using `client.receiveMessages()`
|
|
888
|
-
|
|
889
|
-
- **`MessageLockedError`**: Thrown when a message is temporarily locked (423
|
|
890
|
-
status)
|
|
891
|
-
|
|
892
|
-
- Contains optional `retryAfter` property with seconds to wait before retry
|
|
893
|
-
- For `consume()` without options: the next message in FIFO sequence is locked
|
|
894
|
-
- For `consume()` with messageId: the requested message is locked
|
|
895
|
-
|
|
896
|
-
- **`MessageNotFoundError`**: Message doesn't exist (404 status)
|
|
897
|
-
|
|
898
|
-
- **`MessageNotAvailableError`**: Message exists but isn't available for
|
|
899
|
-
processing (409 status)
|
|
900
|
-
|
|
901
|
-
- **`FifoOrderingViolationError`**: FIFO queue ordering violation (409 status
|
|
902
|
-
with nextMessageId)
|
|
903
|
-
|
|
904
|
-
- Contains `nextMessageId` property indicating which message to process first
|
|
905
|
-
|
|
906
|
-
- **`FailedDependencyError`**: FIFO ordering violation when receiving by ID (424
|
|
907
|
-
status)
|
|
908
|
-
|
|
909
|
-
- Contains `nextMessageId` property indicating which message must be processed
|
|
910
|
-
first
|
|
911
|
-
- Similar to `FifoOrderingViolationError` but specifically for receive-by-ID
|
|
912
|
-
operations
|
|
913
|
-
|
|
914
|
-
- **`MessageCorruptedError`**: Message data is corrupted or can't be parsed
|
|
915
|
-
|
|
916
|
-
- **`BadRequestError`**: Invalid request parameters (400 status)
|
|
917
|
-
|
|
918
|
-
- Invalid queue names, FIFO limit violations, missing required parameters
|
|
919
|
-
|
|
920
|
-
- **`UnauthorizedError`**: Authentication failure (401 status)
|
|
595
|
+
```typescript
|
|
596
|
+
// Batch mode
|
|
597
|
+
const result = await receive("my-topic", "my-group", handler, {
|
|
598
|
+
limit: 10, // Max messages (default: 1, max: 10)
|
|
599
|
+
visibilityTimeoutSeconds: 60, // Lock duration (default: 300)
|
|
600
|
+
});
|
|
601
|
+
```
|
|
921
602
|
|
|
922
|
-
|
|
603
|
+
### `handleCallback(handler, options?)`
|
|
923
604
|
|
|
924
|
-
|
|
605
|
+
Vercel only. Returns `(request: Request) => Promise<Response>` — for frameworks that export Web API route handlers.
|
|
925
606
|
|
|
926
|
-
|
|
607
|
+
```typescript
|
|
608
|
+
export const POST = handleCallback(
|
|
609
|
+
async (message, metadata) => {
|
|
610
|
+
await processMessage(message);
|
|
611
|
+
},
|
|
612
|
+
{
|
|
613
|
+
visibilityTimeoutSeconds: 300, // Lock duration (default: 300)
|
|
614
|
+
retry: (error, metadata) => {
|
|
615
|
+
// Optional: return { afterSeconds: N } to reschedule, { acknowledge: true } to ack, or undefined to propagate
|
|
616
|
+
},
|
|
617
|
+
},
|
|
618
|
+
);
|
|
619
|
+
```
|
|
927
620
|
|
|
928
|
-
|
|
929
|
-
- Unexpected server errors, service unavailable, etc.
|
|
621
|
+
### `handleNodeCallback(handler, options?)`
|
|
930
622
|
|
|
931
|
-
|
|
623
|
+
Vercel only. Returns `(req, res) => Promise<void>` — for frameworks that export Connect-style handlers.
|
|
932
624
|
|
|
933
625
|
```typescript
|
|
934
|
-
|
|
935
|
-
|
|
936
|
-
|
|
937
|
-
|
|
938
|
-
|
|
939
|
-
|
|
940
|
-
|
|
941
|
-
|
|
942
|
-
|
|
943
|
-
|
|
944
|
-
|
|
945
|
-
// Handle empty queue or locked messages
|
|
946
|
-
try {
|
|
947
|
-
for await (const message of client.receiveMessages(options, transport)) {
|
|
948
|
-
// Process messages
|
|
949
|
-
}
|
|
950
|
-
} catch (error) {
|
|
951
|
-
if (error instanceof QueueEmptyError) {
|
|
952
|
-
console.log("Queue is empty, retry later");
|
|
953
|
-
} else if (error instanceof MessageLockedError) {
|
|
954
|
-
console.log("Next message in FIFO queue is locked");
|
|
955
|
-
if (error.retryAfter) {
|
|
956
|
-
console.log(`Retry after ${error.retryAfter} seconds`);
|
|
957
|
-
}
|
|
958
|
-
}
|
|
959
|
-
}
|
|
626
|
+
// pages/api/queue/my-topic.ts
|
|
627
|
+
export default handleNodeCallback(
|
|
628
|
+
async (message, metadata) => {
|
|
629
|
+
await processMessage(message);
|
|
630
|
+
},
|
|
631
|
+
{
|
|
632
|
+
retry: (error, metadata) => ({ afterSeconds: 60 }),
|
|
633
|
+
},
|
|
634
|
+
);
|
|
635
|
+
```
|
|
960
636
|
|
|
961
|
-
|
|
962
|
-
try {
|
|
963
|
-
await consumer.consume(handler, { messageId });
|
|
964
|
-
} catch (error) {
|
|
965
|
-
if (error instanceof MessageLockedError) {
|
|
966
|
-
console.log("Message is locked by another consumer");
|
|
967
|
-
if (error.retryAfter) {
|
|
968
|
-
console.log(`Retry after ${error.retryAfter} seconds`);
|
|
969
|
-
setTimeout(() => retry(), error.retryAfter * 1000);
|
|
970
|
-
}
|
|
971
|
-
} else if (error instanceof FailedDependencyError) {
|
|
972
|
-
// FIFO ordering violation for receive by ID
|
|
973
|
-
console.log(`Must process ${error.nextMessageId} first`);
|
|
974
|
-
}
|
|
975
|
-
}
|
|
637
|
+
### Handler Signature
|
|
976
638
|
|
|
977
|
-
|
|
978
|
-
|
|
979
|
-
|
|
980
|
-
|
|
981
|
-
|
|
982
|
-
console.log("Invalid token - refresh authentication");
|
|
983
|
-
} else if (error instanceof ForbiddenError) {
|
|
984
|
-
console.log("Environment mismatch - check token/queue configuration");
|
|
985
|
-
} else if (error instanceof BadRequestError) {
|
|
986
|
-
console.log("Invalid parameters:", error.message);
|
|
987
|
-
} else if (error instanceof InternalServerError) {
|
|
988
|
-
console.log("Server error - retry with backoff");
|
|
989
|
-
}
|
|
990
|
-
}
|
|
639
|
+
```typescript
|
|
640
|
+
type MessageHandler<T> = (
|
|
641
|
+
message: T,
|
|
642
|
+
metadata: MessageMetadata,
|
|
643
|
+
) => Promise<void> | void;
|
|
991
644
|
|
|
992
|
-
|
|
993
|
-
|
|
994
|
-
|
|
995
|
-
|
|
996
|
-
|
|
997
|
-
|
|
998
|
-
|
|
999
|
-
|
|
1000
|
-
) {
|
|
1001
|
-
// Authentication/authorization errors - need to fix configuration
|
|
1002
|
-
console.log("Auth error - check credentials");
|
|
1003
|
-
} else if (error instanceof BadRequestError) {
|
|
1004
|
-
// Client error - fix the request
|
|
1005
|
-
console.log("Invalid request:", error.message);
|
|
1006
|
-
} else if (error instanceof InternalServerError) {
|
|
1007
|
-
// Server error - implement exponential backoff
|
|
1008
|
-
console.log("Server error - retry with backoff");
|
|
1009
|
-
} else {
|
|
1010
|
-
// Unknown error
|
|
1011
|
-
console.error("Unexpected error:", error);
|
|
1012
|
-
}
|
|
645
|
+
interface MessageMetadata {
|
|
646
|
+
messageId: string;
|
|
647
|
+
deliveryCount: number;
|
|
648
|
+
createdAt: Date;
|
|
649
|
+
expiresAt?: Date;
|
|
650
|
+
topicName: string;
|
|
651
|
+
consumerGroup: string;
|
|
652
|
+
region: string;
|
|
1013
653
|
}
|
|
1014
654
|
```
|
|
1015
655
|
|