@zintrust/queue-cloudflare 2.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +158 -0
- package/dist/CloudflareJobStore.d.ts +50 -0
- package/dist/CloudflareJobStore.js +435 -0
- package/dist/CloudflareQueueConsumer.d.ts +39 -0
- package/dist/CloudflareQueueConsumer.js +184 -0
- package/dist/CloudflareQueueCoordinator.d.ts +21 -0
- package/dist/CloudflareQueueCoordinator.js +57 -0
- package/dist/CloudflareQueueMigrator.d.ts +14 -0
- package/dist/CloudflareQueueMigrator.js +22 -0
- package/dist/CloudflareQueueScheduler.d.ts +43 -0
- package/dist/CloudflareQueueScheduler.js +99 -0
- package/dist/cli/MigrateCloudflareQueueCommand.d.ts +10 -0
- package/dist/cli/MigrateCloudflareQueueCommand.js +51 -0
- package/dist/cli-register.d.ts +9 -0
- package/dist/cli-register.js +21 -0
- package/dist/index.d.ts +50 -0
- package/dist/index.js +499 -0
- package/dist/migrationSql.d.ts +2 -0
- package/dist/migrationSql.js +104 -0
- package/dist/migrations/20260608000001_create_cloudflare_queue_jobs_table.d.ts +6 -0
- package/dist/migrations/20260608000001_create_cloudflare_queue_jobs_table.js +37 -0
- package/dist/migrations/20260608000002_create_cloudflare_queue_job_events_table.d.ts +6 -0
- package/dist/migrations/20260608000002_create_cloudflare_queue_job_events_table.js +21 -0
- package/dist/migrations/20260608000003_create_cloudflare_queue_job_logs_table.d.ts +6 -0
- package/dist/migrations/20260608000003_create_cloudflare_queue_job_logs_table.js +22 -0
- package/dist/migrations/20260608000004_create_cloudflare_queue_repeatables_table.d.ts +6 -0
- package/dist/migrations/20260608000004_create_cloudflare_queue_repeatables_table.js +28 -0
- package/dist/migrations/20260608000005_create_cloudflare_queue_flow_dependencies_table.d.ts +6 -0
- package/dist/migrations/20260608000005_create_cloudflare_queue_flow_dependencies_table.js +22 -0
- package/dist/migrations/index.d.ts +2 -0
- package/dist/migrations/index.js +12 -0
- package/dist/register.d.ts +5 -0
- package/dist/register.js +20 -0
- package/dist/types.d.ts +240 -0
- package/dist/types.js +1 -0
- package/package.json +52 -0
package/dist/index.js
ADDED
|
@@ -0,0 +1,499 @@
|
|
|
1
|
+
import { Cloudflare } from '@zintrust/core/cloudflare';
|
|
2
|
+
import { ErrorFactory } from '@zintrust/core/errors';
|
|
3
|
+
import { generateUuid } from '@zintrust/core/utils';
|
|
4
|
+
import { CloudflareJobStore } from './CloudflareJobStore.js';
|
|
5
|
+
import { CloudflareQueueConsumer } from './CloudflareQueueConsumer.js';
|
|
6
|
+
import { CloudflareQueueMigrator } from './CloudflareQueueMigrator.js';
|
|
7
|
+
import { CloudflareQueueScheduler } from './CloudflareQueueScheduler.js';
|
|
8
|
+
export { CloudflareJobStore } from './CloudflareJobStore.js';
|
|
9
|
+
export { CloudflareQueueConsumer } from './CloudflareQueueConsumer.js';
|
|
10
|
+
export { CloudflareQueueCoordinator } from './CloudflareQueueCoordinator.js';
|
|
11
|
+
export { CloudflareQueueMigrator } from './CloudflareQueueMigrator.js';
|
|
12
|
+
export { CloudflareQueueScheduler } from './CloudflareQueueScheduler.js';
|
|
13
|
+
const LEASES_MAX_ENTRIES = 10000;
|
|
14
|
+
const LEASES_TTL_MS = 15 * 60 * 1000;
|
|
15
|
+
const envValue = (key) => {
|
|
16
|
+
const workersValue = Cloudflare.getWorkersVar(key);
|
|
17
|
+
if (workersValue !== null)
|
|
18
|
+
return workersValue;
|
|
19
|
+
const nodeEnv = typeof process !== 'undefined' && process.env !== undefined ? process.env[key] : undefined;
|
|
20
|
+
return typeof nodeEnv === 'string' ? nodeEnv : '';
|
|
21
|
+
};
|
|
22
|
+
const nonEmpty = (value) => {
|
|
23
|
+
if (typeof value !== 'string')
|
|
24
|
+
return undefined;
|
|
25
|
+
const trimmed = value.trim();
|
|
26
|
+
return trimmed === '' ? undefined : trimmed;
|
|
27
|
+
};
|
|
28
|
+
const resolveBindingName = (queue, config) => {
|
|
29
|
+
return (nonEmpty(config?.bindingName) ??
|
|
30
|
+
nonEmpty(envValue('CLOUDFLARE_QUEUE_BINDING')) ??
|
|
31
|
+
nonEmpty(envValue('QUEUE_BINDING')) ??
|
|
32
|
+
queue);
|
|
33
|
+
};
|
|
34
|
+
const resolveBinding = (queue, config) => {
|
|
35
|
+
const bindingName = resolveBindingName(queue, config);
|
|
36
|
+
const configured = config?.bindings?.[bindingName];
|
|
37
|
+
if (configured !== undefined)
|
|
38
|
+
return configured;
|
|
39
|
+
const env = Cloudflare.getWorkersEnv();
|
|
40
|
+
const candidate = env?.[bindingName];
|
|
41
|
+
if (candidate !== undefined && typeof candidate.send === 'function')
|
|
42
|
+
return candidate;
|
|
43
|
+
return null;
|
|
44
|
+
};
|
|
45
|
+
const resolveApiBaseUrl = (config) => {
|
|
46
|
+
return (nonEmpty(config?.apiBaseUrl) ??
|
|
47
|
+
nonEmpty(envValue('CLOUDFLARE_API_BASE_URL')) ??
|
|
48
|
+
'https://api.cloudflare.com/client/v4').replace(/\/$/, '');
|
|
49
|
+
};
|
|
50
|
+
const resolveAccountId = (config) => {
|
|
51
|
+
const accountId = nonEmpty(config?.accountId) ??
|
|
52
|
+
nonEmpty(envValue('CLOUDFLARE_ACCOUNT_ID')) ??
|
|
53
|
+
nonEmpty(envValue('CF_ACCOUNT_ID'));
|
|
54
|
+
if (accountId === undefined) {
|
|
55
|
+
throw ErrorFactory.createConfigError('Cloudflare Queues requires CLOUDFLARE_ACCOUNT_ID');
|
|
56
|
+
}
|
|
57
|
+
return accountId;
|
|
58
|
+
};
|
|
59
|
+
const resolveQueueId = (queue, config) => {
|
|
60
|
+
const queueId = nonEmpty(config?.queueId) ??
|
|
61
|
+
nonEmpty(envValue('CLOUDFLARE_QUEUE_ID')) ??
|
|
62
|
+
nonEmpty(envValue('CF_QUEUE_ID')) ??
|
|
63
|
+
nonEmpty(queue);
|
|
64
|
+
if (queueId === undefined) {
|
|
65
|
+
throw ErrorFactory.createConfigError('Cloudflare Queues requires a queue id or queue name');
|
|
66
|
+
}
|
|
67
|
+
return queueId;
|
|
68
|
+
};
|
|
69
|
+
const resolveApiToken = (config) => {
|
|
70
|
+
const apiToken = nonEmpty(config?.apiToken) ??
|
|
71
|
+
nonEmpty(envValue('CLOUDFLARE_API_TOKEN')) ??
|
|
72
|
+
nonEmpty(envValue('CF_API_TOKEN'));
|
|
73
|
+
if (apiToken === undefined) {
|
|
74
|
+
throw ErrorFactory.createConfigError('Cloudflare Queues requires CLOUDFLARE_API_TOKEN');
|
|
75
|
+
}
|
|
76
|
+
return apiToken;
|
|
77
|
+
};
|
|
78
|
+
const resolveSendOptions = (config) => {
|
|
79
|
+
const options = {};
|
|
80
|
+
if (config?.contentType !== undefined)
|
|
81
|
+
options.contentType = config.contentType;
|
|
82
|
+
if (config?.delaySeconds !== undefined)
|
|
83
|
+
options.delaySeconds = config.delaySeconds;
|
|
84
|
+
return options;
|
|
85
|
+
};
|
|
86
|
+
const toApiContentType = (contentType) => {
|
|
87
|
+
return contentType === 'text' ? 'text' : 'json';
|
|
88
|
+
};
|
|
89
|
+
const createEnvelope = (id, payload) => ({
|
|
90
|
+
id,
|
|
91
|
+
payload,
|
|
92
|
+
attempts: 0,
|
|
93
|
+
});
|
|
94
|
+
const pruneLeases = (state) => {
|
|
95
|
+
const now = Date.now();
|
|
96
|
+
for (const [id, value] of state.leases.entries()) {
|
|
97
|
+
if (now - value.seenAt > LEASES_TTL_MS)
|
|
98
|
+
state.leases.delete(id);
|
|
99
|
+
}
|
|
100
|
+
if (state.leases.size <= LEASES_MAX_ENTRIES)
|
|
101
|
+
return;
|
|
102
|
+
const overflow = state.leases.size - LEASES_MAX_ENTRIES;
|
|
103
|
+
let removed = 0;
|
|
104
|
+
for (const id of state.leases.keys()) {
|
|
105
|
+
state.leases.delete(id);
|
|
106
|
+
removed += 1;
|
|
107
|
+
if (removed >= overflow)
|
|
108
|
+
break;
|
|
109
|
+
}
|
|
110
|
+
};
|
|
111
|
+
const apiUrl = (queue, config, path) => {
|
|
112
|
+
const accountId = encodeURIComponent(resolveAccountId(config));
|
|
113
|
+
const queueId = encodeURIComponent(resolveQueueId(queue, config));
|
|
114
|
+
return `${resolveApiBaseUrl(config)}/accounts/${accountId}/queues/${queueId}${path}`;
|
|
115
|
+
};
|
|
116
|
+
const apiFetch = async (queue, config, path, init) => {
|
|
117
|
+
const response = await fetch(apiUrl(queue, config, path), {
|
|
118
|
+
...init,
|
|
119
|
+
headers: {
|
|
120
|
+
authorization: `Bearer ${resolveApiToken(config)}`,
|
|
121
|
+
'content-type': 'application/json',
|
|
122
|
+
...(init?.headers ?? {}),
|
|
123
|
+
},
|
|
124
|
+
});
|
|
125
|
+
const text = await response.text();
|
|
126
|
+
const parsed = text === '' ? {} : JSON.parse(text);
|
|
127
|
+
if (!response.ok || parsed.success === false) {
|
|
128
|
+
const message = parsed.errors?.map((error) => error.message).filter(Boolean).join('; ') ||
|
|
129
|
+
`Cloudflare Queues API error (${response.status})`;
|
|
130
|
+
throw ErrorFactory.createConnectionError(message, { status: response.status, body: parsed });
|
|
131
|
+
}
|
|
132
|
+
return (parsed.result ?? parsed);
|
|
133
|
+
};
|
|
134
|
+
const decodePulledBody = (message) => {
|
|
135
|
+
const body = message.body;
|
|
136
|
+
if (body !== null &&
|
|
137
|
+
typeof body === 'object' &&
|
|
138
|
+
typeof body.id === 'string' &&
|
|
139
|
+
'payload' in body) {
|
|
140
|
+
const envelope = body;
|
|
141
|
+
return {
|
|
142
|
+
id: envelope.id,
|
|
143
|
+
payload: envelope.payload,
|
|
144
|
+
attempts: typeof envelope.attempts === 'number' ? envelope.attempts : (message.attempts ?? 0),
|
|
145
|
+
};
|
|
146
|
+
}
|
|
147
|
+
return {
|
|
148
|
+
id: message.id ?? generateUuid(),
|
|
149
|
+
payload: body,
|
|
150
|
+
attempts: message.attempts ?? 0,
|
|
151
|
+
};
|
|
152
|
+
};
|
|
153
|
+
const createEnqueue = (config) => async (queue, payload) => {
|
|
154
|
+
const id = generateUuid();
|
|
155
|
+
const envelope = createEnvelope(id, payload);
|
|
156
|
+
const options = resolveSendOptions(config);
|
|
157
|
+
const binding = resolveBinding(queue, config);
|
|
158
|
+
if (binding !== null) {
|
|
159
|
+
await binding.send(envelope, options);
|
|
160
|
+
return id;
|
|
161
|
+
}
|
|
162
|
+
const contentType = toApiContentType(config?.contentType);
|
|
163
|
+
await apiFetch(queue, config, '/messages', {
|
|
164
|
+
method: 'POST',
|
|
165
|
+
body: JSON.stringify({
|
|
166
|
+
body: contentType === 'text' ? JSON.stringify(envelope) : envelope,
|
|
167
|
+
content_type: contentType,
|
|
168
|
+
delay_seconds: config?.delaySeconds,
|
|
169
|
+
}),
|
|
170
|
+
});
|
|
171
|
+
return id;
|
|
172
|
+
};
|
|
173
|
+
const sendRaw = (config) => async (queue, payload, optionsOverride) => {
|
|
174
|
+
const options = { ...resolveSendOptions(config), ...(optionsOverride ?? {}) };
|
|
175
|
+
const binding = resolveBinding(queue, config);
|
|
176
|
+
if (binding !== null) {
|
|
177
|
+
await binding.send(payload, options);
|
|
178
|
+
return;
|
|
179
|
+
}
|
|
180
|
+
const contentType = toApiContentType(options.contentType ?? config?.contentType);
|
|
181
|
+
await apiFetch(queue, config, '/messages', {
|
|
182
|
+
method: 'POST',
|
|
183
|
+
body: JSON.stringify({
|
|
184
|
+
body: contentType === 'text' ? JSON.stringify(payload) : payload,
|
|
185
|
+
content_type: contentType,
|
|
186
|
+
delay_seconds: options.delaySeconds,
|
|
187
|
+
}),
|
|
188
|
+
});
|
|
189
|
+
};
|
|
190
|
+
const sendRawBatch = (config) => async (queue, messages) => {
|
|
191
|
+
if (messages.length === 0)
|
|
192
|
+
return;
|
|
193
|
+
const binding = resolveBinding(queue, config);
|
|
194
|
+
if (binding?.sendBatch !== undefined) {
|
|
195
|
+
for (let index = 0; index < messages.length; index += 100) {
|
|
196
|
+
await binding.sendBatch(messages.slice(index, index + 100));
|
|
197
|
+
}
|
|
198
|
+
return;
|
|
199
|
+
}
|
|
200
|
+
const rawSender = sendRaw(config);
|
|
201
|
+
for (const message of messages) {
|
|
202
|
+
await rawSender(queue, message.body, message.options);
|
|
203
|
+
}
|
|
204
|
+
};
|
|
205
|
+
const createDequeue = (state, config) => async (queue) => {
|
|
206
|
+
pruneLeases(state);
|
|
207
|
+
const body = {};
|
|
208
|
+
if (config?.batchSize !== undefined)
|
|
209
|
+
body['batch_size'] = config.batchSize;
|
|
210
|
+
if (config?.visibilityTimeoutMs !== undefined) {
|
|
211
|
+
body['visibility_timeout_ms'] = config.visibilityTimeoutMs;
|
|
212
|
+
}
|
|
213
|
+
const pulled = await apiFetch(queue, config, '/messages/pull', {
|
|
214
|
+
method: 'POST',
|
|
215
|
+
body: JSON.stringify(body),
|
|
216
|
+
});
|
|
217
|
+
state.lastBacklogCount =
|
|
218
|
+
pulled.message_backlog_count ?? pulled.metadata?.metrics?.backlog_count ?? state.lastBacklogCount;
|
|
219
|
+
const message = pulled.messages?.[0];
|
|
220
|
+
if (message === undefined)
|
|
221
|
+
return undefined;
|
|
222
|
+
const decoded = decodePulledBody(message);
|
|
223
|
+
if (message.lease_id !== undefined) {
|
|
224
|
+
state.leases.set(decoded.id, { leaseId: message.lease_id, seenAt: Date.now() });
|
|
225
|
+
}
|
|
226
|
+
return decoded;
|
|
227
|
+
};
|
|
228
|
+
const createAck = (state, config) => async (queue, id) => {
|
|
229
|
+
pruneLeases(state);
|
|
230
|
+
const lease = state.leases.get(id);
|
|
231
|
+
if (lease === undefined)
|
|
232
|
+
return;
|
|
233
|
+
state.leases.delete(id);
|
|
234
|
+
await apiFetch(queue, config, '/messages/ack', {
|
|
235
|
+
method: 'POST',
|
|
236
|
+
body: JSON.stringify({
|
|
237
|
+
acks: [{ lease_id: lease.leaseId }],
|
|
238
|
+
retries: [],
|
|
239
|
+
}),
|
|
240
|
+
});
|
|
241
|
+
};
|
|
242
|
+
const createLength = (state, config) => async (queue) => {
|
|
243
|
+
const metrics = await apiFetch(queue, config, '/metrics', {
|
|
244
|
+
method: 'GET',
|
|
245
|
+
});
|
|
246
|
+
const count = metrics.metadata?.metrics?.backlog_count ?? state.lastBacklogCount ?? 0;
|
|
247
|
+
return Number.isFinite(count) ? count : 0;
|
|
248
|
+
};
|
|
249
|
+
const createDrain = (config) => async (queue) => {
|
|
250
|
+
await apiFetch(queue, config, '/purge', {
|
|
251
|
+
method: 'POST',
|
|
252
|
+
body: JSON.stringify({ delete_messages_permanently: true }),
|
|
253
|
+
});
|
|
254
|
+
};
|
|
255
|
+
const createStore = (config) => {
|
|
256
|
+
return CloudflareJobStore.create(config?.state);
|
|
257
|
+
};
|
|
258
|
+
const toQueueEnvelope = (job) => ({
|
|
259
|
+
protocol: 'zintrust.cf.queue.v1',
|
|
260
|
+
jobId: job.id,
|
|
261
|
+
queueName: job.queueName,
|
|
262
|
+
name: job.name,
|
|
263
|
+
attempt: job.attemptsMade,
|
|
264
|
+
availableAt: job.availableAt,
|
|
265
|
+
});
|
|
266
|
+
const resolveCoordinator = (config, queueName) => {
|
|
267
|
+
const direct = config?.state?.coordinator;
|
|
268
|
+
if (direct?.getByName !== undefined)
|
|
269
|
+
return direct.getByName(queueName);
|
|
270
|
+
if (direct?.idFromName !== undefined && direct.get !== undefined) {
|
|
271
|
+
return direct.get(direct.idFromName(queueName));
|
|
272
|
+
}
|
|
273
|
+
const env = Cloudflare.getWorkersEnv();
|
|
274
|
+
const bindingName = config?.state?.coordinatorBindingName ?? 'QUEUE_COORDINATOR';
|
|
275
|
+
const binding = env?.[bindingName];
|
|
276
|
+
if (binding?.getByName !== undefined)
|
|
277
|
+
return binding.getByName(queueName);
|
|
278
|
+
if (binding?.idFromName !== undefined && binding.get !== undefined) {
|
|
279
|
+
return binding.get(binding.idFromName(queueName));
|
|
280
|
+
}
|
|
281
|
+
return null;
|
|
282
|
+
};
|
|
283
|
+
const shouldSendImmediately = (options) => {
|
|
284
|
+
const delay = options?.delay ?? 0;
|
|
285
|
+
if (options?.priority !== undefined || options?.lifo === true)
|
|
286
|
+
return false;
|
|
287
|
+
return delay <= 43200 * 1000;
|
|
288
|
+
};
|
|
289
|
+
const delayOptions = (options) => {
|
|
290
|
+
const delaySeconds = Math.max(0, Math.floor((options?.delay ?? 0) / 1000));
|
|
291
|
+
return delaySeconds > 0 ? { delaySeconds } : undefined;
|
|
292
|
+
};
|
|
293
|
+
function createCloudflareQueueDriver(config) {
|
|
294
|
+
const state = { leases: new Map() };
|
|
295
|
+
const rawSender = sendRaw(config);
|
|
296
|
+
const rawBatchSender = sendRawBatch(config);
|
|
297
|
+
let storeRef;
|
|
298
|
+
const getStore = () => {
|
|
299
|
+
storeRef ??= createStore(config);
|
|
300
|
+
return storeRef;
|
|
301
|
+
};
|
|
302
|
+
return {
|
|
303
|
+
enqueue: createEnqueue(config),
|
|
304
|
+
dequeue: createDequeue(state, config),
|
|
305
|
+
ack: createAck(state, config),
|
|
306
|
+
length: createLength(state, config),
|
|
307
|
+
drain: createDrain(config),
|
|
308
|
+
async add(queue, name, data, options) {
|
|
309
|
+
const job = await getStore().createJob({ queueName: queue, name, data, options });
|
|
310
|
+
const coordinator = resolveCoordinator(config, queue);
|
|
311
|
+
if ((await coordinator?.isPaused?.()) === true)
|
|
312
|
+
return job;
|
|
313
|
+
const rate = config?.state?.rateLimit;
|
|
314
|
+
if (rate !== undefined) {
|
|
315
|
+
const result = await coordinator?.rateLimit?.({
|
|
316
|
+
key: rate.key ?? queue,
|
|
317
|
+
max: rate.max,
|
|
318
|
+
durationMs: rate.durationMs,
|
|
319
|
+
});
|
|
320
|
+
if (result !== undefined && !result.allowed) {
|
|
321
|
+
await getStore().updateState({
|
|
322
|
+
queueName: queue,
|
|
323
|
+
jobId: job.id,
|
|
324
|
+
state: job.state,
|
|
325
|
+
availableAt: new Date(result.resetAt).toISOString(),
|
|
326
|
+
});
|
|
327
|
+
return job;
|
|
328
|
+
}
|
|
329
|
+
}
|
|
330
|
+
if (shouldSendImmediately(options)) {
|
|
331
|
+
await rawSender(queue, toQueueEnvelope(job), delayOptions(options));
|
|
332
|
+
await getStore().markDispatched(queue, job.id);
|
|
333
|
+
}
|
|
334
|
+
return job;
|
|
335
|
+
},
|
|
336
|
+
async addBulk(queue, jobs) {
|
|
337
|
+
const created = [];
|
|
338
|
+
const batch = [];
|
|
339
|
+
const coordinator = resolveCoordinator(config, queue);
|
|
340
|
+
const rate = config?.state?.rateLimit;
|
|
341
|
+
const rateResult = rate === undefined
|
|
342
|
+
? undefined
|
|
343
|
+
: await coordinator?.rateLimit?.({
|
|
344
|
+
key: rate.key ?? queue,
|
|
345
|
+
max: rate.max,
|
|
346
|
+
durationMs: rate.durationMs,
|
|
347
|
+
});
|
|
348
|
+
for (const jobInput of jobs) {
|
|
349
|
+
const job = await getStore().createJob({
|
|
350
|
+
queueName: queue,
|
|
351
|
+
name: jobInput.name,
|
|
352
|
+
data: jobInput.data,
|
|
353
|
+
options: jobInput.options,
|
|
354
|
+
});
|
|
355
|
+
created.push(job);
|
|
356
|
+
if (rateResult !== undefined && !rateResult.allowed) {
|
|
357
|
+
await getStore().updateState({
|
|
358
|
+
queueName: queue,
|
|
359
|
+
jobId: job.id,
|
|
360
|
+
state: job.state,
|
|
361
|
+
availableAt: new Date(rateResult.resetAt).toISOString(),
|
|
362
|
+
});
|
|
363
|
+
continue;
|
|
364
|
+
}
|
|
365
|
+
if (shouldSendImmediately(jobInput.options)) {
|
|
366
|
+
batch.push({
|
|
367
|
+
body: toQueueEnvelope(job),
|
|
368
|
+
options: delayOptions(jobInput.options),
|
|
369
|
+
id: job.id,
|
|
370
|
+
});
|
|
371
|
+
}
|
|
372
|
+
}
|
|
373
|
+
if ((await resolveCoordinator(config, queue)?.isPaused?.()) !== true) {
|
|
374
|
+
await rawBatchSender(queue, batch.map((message) => ({ body: message.body, options: message.options })));
|
|
375
|
+
for (const message of batch) {
|
|
376
|
+
await getStore().markDispatched(queue, message.id);
|
|
377
|
+
}
|
|
378
|
+
}
|
|
379
|
+
return created;
|
|
380
|
+
},
|
|
381
|
+
async getJob(queue, id) {
|
|
382
|
+
return await getStore().getJob(queue, id);
|
|
383
|
+
},
|
|
384
|
+
async getJobs(queue, states, limit) {
|
|
385
|
+
return await getStore().getJobs(queue, states, limit);
|
|
386
|
+
},
|
|
387
|
+
async getJobCounts(queue, ...states) {
|
|
388
|
+
return await getStore().getJobCounts(queue, ...states);
|
|
389
|
+
},
|
|
390
|
+
async getMetrics(queue) {
|
|
391
|
+
return await getStore().getMetrics(queue);
|
|
392
|
+
},
|
|
393
|
+
async updateProgress(queue, id, progress) {
|
|
394
|
+
await getStore().updateProgress(queue, id, progress);
|
|
395
|
+
},
|
|
396
|
+
async log(queue, id, message, data) {
|
|
397
|
+
await getStore().recordLog(queue, id, 'info', message, data);
|
|
398
|
+
},
|
|
399
|
+
async remove(queue, id) {
|
|
400
|
+
await getStore().updateState({ queueName: queue, jobId: id, state: 'canceled' });
|
|
401
|
+
},
|
|
402
|
+
async retry(queue, id) {
|
|
403
|
+
const job = await getStore().getJob(queue, id);
|
|
404
|
+
if (job === null)
|
|
405
|
+
return;
|
|
406
|
+
await getStore().updateState({ queueName: queue, jobId: id, state: 'waiting' });
|
|
407
|
+
await rawSender(queue, toQueueEnvelope(job));
|
|
408
|
+
await getStore().markDispatched(queue, id);
|
|
409
|
+
},
|
|
410
|
+
async promote(queue, id) {
|
|
411
|
+
const job = await getStore().getJob(queue, id);
|
|
412
|
+
if (job === null)
|
|
413
|
+
return;
|
|
414
|
+
await getStore().updateState({
|
|
415
|
+
queueName: queue,
|
|
416
|
+
jobId: id,
|
|
417
|
+
state: 'waiting',
|
|
418
|
+
availableAt: new Date().toISOString(),
|
|
419
|
+
});
|
|
420
|
+
await rawSender(queue, toQueueEnvelope(job));
|
|
421
|
+
await getStore().markDispatched(queue, id);
|
|
422
|
+
},
|
|
423
|
+
async clean(queue, states, olderThanMs) {
|
|
424
|
+
await getStore().clean(queue, states, olderThanMs);
|
|
425
|
+
},
|
|
426
|
+
async upsertJobScheduler(queue, name, data, repeat, id) {
|
|
427
|
+
return await getStore().upsertRepeatable({ id, queueName: queue, name, data, options: repeat });
|
|
428
|
+
},
|
|
429
|
+
async removeJobScheduler(id) {
|
|
430
|
+
await getStore().removeRepeatable(id);
|
|
431
|
+
},
|
|
432
|
+
async runScheduler(queue, limit) {
|
|
433
|
+
return await CloudflareQueueScheduler.create({
|
|
434
|
+
queueName: queue,
|
|
435
|
+
store: getStore(),
|
|
436
|
+
queue: { enqueue: async (queueName, payload) => {
|
|
437
|
+
await rawSender(queueName, payload);
|
|
438
|
+
return generateUuid();
|
|
439
|
+
} },
|
|
440
|
+
batchSize: limit,
|
|
441
|
+
}).run();
|
|
442
|
+
},
|
|
443
|
+
async reconcileStalled(queue, olderThanMs, limit) {
|
|
444
|
+
return await CloudflareQueueScheduler.create({
|
|
445
|
+
queueName: queue,
|
|
446
|
+
store: getStore(),
|
|
447
|
+
queue: { enqueue: async (queueName, payload) => {
|
|
448
|
+
await rawSender(queueName, payload);
|
|
449
|
+
return generateUuid();
|
|
450
|
+
} },
|
|
451
|
+
batchSize: limit,
|
|
452
|
+
stalledAfterMs: olderThanMs,
|
|
453
|
+
}).reconcileStalled();
|
|
454
|
+
},
|
|
455
|
+
async createFlow(input) {
|
|
456
|
+
return await getStore().createFlow(input);
|
|
457
|
+
},
|
|
458
|
+
async getFlowChildren(queue, parentJobId) {
|
|
459
|
+
return await getStore().getFlowChildren(queue, parentJobId);
|
|
460
|
+
},
|
|
461
|
+
async markFlowChildCompleted(queue, childJobId) {
|
|
462
|
+
const released = await getStore().markFlowChildCompleted(queue, childJobId);
|
|
463
|
+
for (const parent of released) {
|
|
464
|
+
await rawSender(parent.queueName, toQueueEnvelope(parent));
|
|
465
|
+
await getStore().markDispatched(parent.queueName, parent.id);
|
|
466
|
+
}
|
|
467
|
+
return released;
|
|
468
|
+
},
|
|
469
|
+
createConsumer(processor, queueName) {
|
|
470
|
+
return CloudflareQueueConsumer.create({
|
|
471
|
+
queueName,
|
|
472
|
+
store: getStore(),
|
|
473
|
+
processor,
|
|
474
|
+
state: config?.state,
|
|
475
|
+
queue: { enqueue: async (queueName, payload) => {
|
|
476
|
+
await rawSender(queueName, payload);
|
|
477
|
+
} },
|
|
478
|
+
});
|
|
479
|
+
},
|
|
480
|
+
async migrateState() {
|
|
481
|
+
if (config?.state?.d1 !== undefined) {
|
|
482
|
+
await CloudflareQueueMigrator.up({ d1: config.state.d1 });
|
|
483
|
+
return;
|
|
484
|
+
}
|
|
485
|
+
if (config?.state?.db !== undefined) {
|
|
486
|
+
await CloudflareQueueMigrator.up({ db: config.state.db });
|
|
487
|
+
return;
|
|
488
|
+
}
|
|
489
|
+
throw ErrorFactory.createConfigError('Cloudflare queue state migration requires config.state.d1 or config.state.db');
|
|
490
|
+
},
|
|
491
|
+
};
|
|
492
|
+
}
|
|
493
|
+
export const CloudflareQueues = Object.freeze({
|
|
494
|
+
create(config) {
|
|
495
|
+
return createCloudflareQueueDriver(config);
|
|
496
|
+
},
|
|
497
|
+
});
|
|
498
|
+
export const CloudflareQueue = CloudflareQueues;
|
|
499
|
+
export default CloudflareQueues;
|
|
@@ -0,0 +1,104 @@
|
|
|
1
|
+
export const cloudflareQueueMigrationStatements = Object.freeze([
|
|
2
|
+
`CREATE TABLE IF NOT EXISTS cf_queue_jobs (
|
|
3
|
+
id TEXT PRIMARY KEY,
|
|
4
|
+
queue_name TEXT NOT NULL,
|
|
5
|
+
name TEXT NOT NULL,
|
|
6
|
+
payload TEXT NOT NULL,
|
|
7
|
+
opts TEXT,
|
|
8
|
+
state TEXT NOT NULL,
|
|
9
|
+
priority INTEGER NOT NULL DEFAULT 0,
|
|
10
|
+
attempts INTEGER NOT NULL DEFAULT 0,
|
|
11
|
+
max_attempts INTEGER NOT NULL DEFAULT 1,
|
|
12
|
+
progress TEXT,
|
|
13
|
+
result TEXT,
|
|
14
|
+
error TEXT,
|
|
15
|
+
dedupe_key TEXT,
|
|
16
|
+
idempotency_key TEXT,
|
|
17
|
+
available_at TEXT NOT NULL,
|
|
18
|
+
started_at TEXT,
|
|
19
|
+
completed_at TEXT,
|
|
20
|
+
failed_at TEXT,
|
|
21
|
+
stalled_at TEXT,
|
|
22
|
+
created_at TEXT NOT NULL,
|
|
23
|
+
updated_at TEXT NOT NULL
|
|
24
|
+
)`,
|
|
25
|
+
`CREATE INDEX IF NOT EXISTS idx_cf_queue_jobs_queue_state_priority_available_at
|
|
26
|
+
ON cf_queue_jobs(queue_name, state, priority, available_at)`,
|
|
27
|
+
`CREATE INDEX IF NOT EXISTS idx_cf_queue_jobs_queue_dedupe_key
|
|
28
|
+
ON cf_queue_jobs(queue_name, dedupe_key)`,
|
|
29
|
+
`CREATE INDEX IF NOT EXISTS idx_cf_queue_jobs_queue_idempotency_key
|
|
30
|
+
ON cf_queue_jobs(queue_name, idempotency_key)`,
|
|
31
|
+
`CREATE INDEX IF NOT EXISTS idx_cf_queue_jobs_queue_updated_at
|
|
32
|
+
ON cf_queue_jobs(queue_name, updated_at)`,
|
|
33
|
+
`CREATE TABLE IF NOT EXISTS cf_queue_job_events (
|
|
34
|
+
id TEXT PRIMARY KEY,
|
|
35
|
+
job_id TEXT NOT NULL,
|
|
36
|
+
queue_name TEXT NOT NULL,
|
|
37
|
+
event TEXT NOT NULL,
|
|
38
|
+
data TEXT,
|
|
39
|
+
created_at TEXT NOT NULL
|
|
40
|
+
)`,
|
|
41
|
+
`CREATE INDEX IF NOT EXISTS idx_cf_queue_job_events_job_queue
|
|
42
|
+
ON cf_queue_job_events(job_id, queue_name)`,
|
|
43
|
+
`CREATE INDEX IF NOT EXISTS idx_cf_queue_job_events_queue_event_created_at
|
|
44
|
+
ON cf_queue_job_events(queue_name, event, created_at)`,
|
|
45
|
+
`CREATE INDEX IF NOT EXISTS idx_cf_queue_job_events_created_at
|
|
46
|
+
ON cf_queue_job_events(created_at)`,
|
|
47
|
+
`CREATE TABLE IF NOT EXISTS cf_queue_job_logs (
|
|
48
|
+
id TEXT PRIMARY KEY,
|
|
49
|
+
job_id TEXT NOT NULL,
|
|
50
|
+
queue_name TEXT NOT NULL,
|
|
51
|
+
level TEXT NOT NULL,
|
|
52
|
+
message TEXT NOT NULL,
|
|
53
|
+
data TEXT,
|
|
54
|
+
created_at TEXT NOT NULL
|
|
55
|
+
)`,
|
|
56
|
+
`CREATE INDEX IF NOT EXISTS idx_cf_queue_job_logs_job_queue
|
|
57
|
+
ON cf_queue_job_logs(job_id, queue_name)`,
|
|
58
|
+
`CREATE INDEX IF NOT EXISTS idx_cf_queue_job_logs_queue_level_created_at
|
|
59
|
+
ON cf_queue_job_logs(queue_name, level, created_at)`,
|
|
60
|
+
`CREATE INDEX IF NOT EXISTS idx_cf_queue_job_logs_created_at
|
|
61
|
+
ON cf_queue_job_logs(created_at)`,
|
|
62
|
+
`CREATE TABLE IF NOT EXISTS cf_queue_repeatables (
|
|
63
|
+
id TEXT PRIMARY KEY,
|
|
64
|
+
queue_name TEXT NOT NULL,
|
|
65
|
+
name TEXT NOT NULL,
|
|
66
|
+
payload TEXT NOT NULL,
|
|
67
|
+
cron TEXT,
|
|
68
|
+
every_ms INTEGER,
|
|
69
|
+
start_at TEXT,
|
|
70
|
+
end_at TEXT,
|
|
71
|
+
limit_count INTEGER,
|
|
72
|
+
run_count INTEGER NOT NULL DEFAULT 0,
|
|
73
|
+
next_run_at TEXT NOT NULL,
|
|
74
|
+
active INTEGER NOT NULL DEFAULT 1,
|
|
75
|
+
created_at TEXT NOT NULL,
|
|
76
|
+
updated_at TEXT NOT NULL
|
|
77
|
+
)`,
|
|
78
|
+
`CREATE INDEX IF NOT EXISTS idx_cf_queue_repeatables_active_next_run_at
|
|
79
|
+
ON cf_queue_repeatables(active, next_run_at)`,
|
|
80
|
+
`CREATE INDEX IF NOT EXISTS idx_cf_queue_repeatables_queue_active
|
|
81
|
+
ON cf_queue_repeatables(queue_name, active)`,
|
|
82
|
+
`CREATE TABLE IF NOT EXISTS cf_queue_flow_dependencies (
|
|
83
|
+
id TEXT PRIMARY KEY,
|
|
84
|
+
queue_name TEXT NOT NULL,
|
|
85
|
+
parent_job_id TEXT NOT NULL,
|
|
86
|
+
child_job_id TEXT NOT NULL,
|
|
87
|
+
state TEXT NOT NULL,
|
|
88
|
+
created_at TEXT NOT NULL,
|
|
89
|
+
updated_at TEXT NOT NULL
|
|
90
|
+
)`,
|
|
91
|
+
`CREATE INDEX IF NOT EXISTS idx_cf_queue_flow_dependencies_queue_parent
|
|
92
|
+
ON cf_queue_flow_dependencies(queue_name, parent_job_id)`,
|
|
93
|
+
`CREATE INDEX IF NOT EXISTS idx_cf_queue_flow_dependencies_queue_child
|
|
94
|
+
ON cf_queue_flow_dependencies(queue_name, child_job_id)`,
|
|
95
|
+
`CREATE INDEX IF NOT EXISTS idx_cf_queue_flow_dependencies_queue_state
|
|
96
|
+
ON cf_queue_flow_dependencies(queue_name, state)`,
|
|
97
|
+
]);
|
|
98
|
+
export const cloudflareQueueRollbackStatements = Object.freeze([
|
|
99
|
+
'DROP TABLE IF EXISTS cf_queue_flow_dependencies',
|
|
100
|
+
'DROP TABLE IF EXISTS cf_queue_repeatables',
|
|
101
|
+
'DROP TABLE IF EXISTS cf_queue_job_logs',
|
|
102
|
+
'DROP TABLE IF EXISTS cf_queue_job_events',
|
|
103
|
+
'DROP TABLE IF EXISTS cf_queue_jobs',
|
|
104
|
+
]);
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
import { MigrationSchema } from '@zintrust/core/database';
|
|
2
|
+
export const migration = {
|
|
3
|
+
async up(db) {
|
|
4
|
+
const schema = MigrationSchema.create(db);
|
|
5
|
+
await schema.create('cf_queue_jobs', (table) => {
|
|
6
|
+
table.string('id').primary();
|
|
7
|
+
table.string('queue_name');
|
|
8
|
+
table.string('name');
|
|
9
|
+
table.text('payload');
|
|
10
|
+
table.text('opts').nullable();
|
|
11
|
+
table.string('state');
|
|
12
|
+
table.integer('priority').default(0);
|
|
13
|
+
table.integer('attempts').default(0);
|
|
14
|
+
table.integer('max_attempts').default(1);
|
|
15
|
+
table.text('progress').nullable();
|
|
16
|
+
table.text('result').nullable();
|
|
17
|
+
table.text('error').nullable();
|
|
18
|
+
table.string('dedupe_key').nullable();
|
|
19
|
+
table.string('idempotency_key').nullable();
|
|
20
|
+
table.timestamp('available_at');
|
|
21
|
+
table.timestamp('started_at').nullable();
|
|
22
|
+
table.timestamp('completed_at').nullable();
|
|
23
|
+
table.timestamp('failed_at').nullable();
|
|
24
|
+
table.timestamp('stalled_at').nullable();
|
|
25
|
+
table.timestamp('created_at');
|
|
26
|
+
table.timestamp('updated_at');
|
|
27
|
+
table.index(['queue_name', 'state', 'priority', 'available_at']);
|
|
28
|
+
table.index(['queue_name', 'dedupe_key']);
|
|
29
|
+
table.index(['queue_name', 'idempotency_key']);
|
|
30
|
+
table.index(['queue_name', 'updated_at']);
|
|
31
|
+
});
|
|
32
|
+
},
|
|
33
|
+
async down(db) {
|
|
34
|
+
const schema = MigrationSchema.create(db);
|
|
35
|
+
await schema.dropIfExists('cf_queue_jobs');
|
|
36
|
+
},
|
|
37
|
+
};
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
import { MigrationSchema } from '@zintrust/core/database';
|
|
2
|
+
export const migration = {
|
|
3
|
+
async up(db) {
|
|
4
|
+
const schema = MigrationSchema.create(db);
|
|
5
|
+
await schema.create('cf_queue_job_events', (table) => {
|
|
6
|
+
table.string('id').primary();
|
|
7
|
+
table.string('job_id');
|
|
8
|
+
table.string('queue_name');
|
|
9
|
+
table.string('event');
|
|
10
|
+
table.text('data').nullable();
|
|
11
|
+
table.timestamp('created_at');
|
|
12
|
+
table.index(['job_id', 'queue_name']);
|
|
13
|
+
table.index(['queue_name', 'event', 'created_at']);
|
|
14
|
+
table.index('created_at');
|
|
15
|
+
});
|
|
16
|
+
},
|
|
17
|
+
async down(db) {
|
|
18
|
+
const schema = MigrationSchema.create(db);
|
|
19
|
+
await schema.dropIfExists('cf_queue_job_events');
|
|
20
|
+
},
|
|
21
|
+
};
|