@zintrust/queue-cloudflare 2.4.1 → 2.4.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/CloudflareJobStore.d.ts +1 -1
- package/dist/CloudflareJobStore.js +22 -9
- package/dist/CloudflareQueueConsumer.d.ts +2 -2
- package/dist/CloudflareQueueConsumer.js +123 -97
- package/dist/CloudflareQueueCoordinator.js +1 -0
- package/dist/CloudflareQueueMigrator.js +2 -6
- package/dist/CloudflareQueueScheduler.d.ts +8 -8
- package/dist/CloudflareQueueScheduler.js +51 -42
- package/dist/build-manifest.json +161 -0
- package/dist/index.d.ts +2 -2
- package/dist/index.js +31 -12
- package/package.json +1 -1
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import type {
|
|
1
|
+
import type { CloudflareFlowInput, CloudflareFlowResult, CloudflareJobOptions, CloudflareQueueJob, CloudflareQueueMetrics, CloudflareQueueState, CloudflareQueueStateConfig, CloudflareRepeatableRow } from './types.js';
|
|
2
2
|
type CreateJobInput<T = unknown> = {
|
|
3
3
|
queueName: string;
|
|
4
4
|
name: string;
|
|
@@ -1,3 +1,10 @@
|
|
|
1
|
+
/* eslint-disable no-await-in-loop, complexity, max-lines-per-function */
|
|
2
|
+
/* eslint-disable no-negated-condition, no-nested-ternary, no-restricted-syntax */
|
|
3
|
+
/* eslint-disable @typescript-eslint/explicit-function-return-type */
|
|
4
|
+
/* sonarqube-disable cognitive-complexity, no-nested-conditional, no-nested-ternary */
|
|
5
|
+
/* sonarqube-disable no-negated-condition, S3776 */
|
|
6
|
+
/* Sequential processing required, complex functions by design */
|
|
7
|
+
import { ErrorFactory } from '@zintrust/core/errors';
|
|
1
8
|
import { generateUuid } from '@zintrust/core/utils';
|
|
2
9
|
const nowIso = () => new Date().toISOString();
|
|
3
10
|
const serialize = (value) => JSON.stringify(value ?? null);
|
|
@@ -35,7 +42,7 @@ const resolveTarget = (config) => {
|
|
|
35
42
|
const d1 = env?.[bindingName];
|
|
36
43
|
if (d1 !== undefined && typeof d1.prepare === 'function')
|
|
37
44
|
return { kind: 'd1', db: d1 };
|
|
38
|
-
throw
|
|
45
|
+
throw ErrorFactory.createConfigError(`Cloudflare queue state store requires a D1 binding (${bindingName}) or ZinTrust database`);
|
|
39
46
|
};
|
|
40
47
|
const resolveKv = (config) => {
|
|
41
48
|
if (config?.kv !== undefined)
|
|
@@ -47,20 +54,29 @@ const resolveKv = (config) => {
|
|
|
47
54
|
};
|
|
48
55
|
const queryRows = async (target, sql, params = []) => {
|
|
49
56
|
if (target.kind === 'd1') {
|
|
50
|
-
const result = await target.db
|
|
57
|
+
const result = await target.db
|
|
58
|
+
.prepare(sql)
|
|
59
|
+
.bind(...params)
|
|
60
|
+
.all();
|
|
51
61
|
return result.results ?? [];
|
|
52
62
|
}
|
|
53
63
|
return (await target.db.query(sql, params));
|
|
54
64
|
};
|
|
55
65
|
const queryOne = async (target, sql, params = []) => {
|
|
56
66
|
if (target.kind === 'd1') {
|
|
57
|
-
return await target.db
|
|
67
|
+
return await target.db
|
|
68
|
+
.prepare(sql)
|
|
69
|
+
.bind(...params)
|
|
70
|
+
.first();
|
|
58
71
|
}
|
|
59
72
|
return (await target.db.queryOne(sql, params)) ?? null;
|
|
60
73
|
};
|
|
61
74
|
const execute = async (target, sql, params = []) => {
|
|
62
75
|
if (target.kind === 'd1') {
|
|
63
|
-
await target.db
|
|
76
|
+
await target.db
|
|
77
|
+
.prepare(sql)
|
|
78
|
+
.bind(...params)
|
|
79
|
+
.run();
|
|
64
80
|
return;
|
|
65
81
|
}
|
|
66
82
|
await target.db.execute(sql, params);
|
|
@@ -78,7 +94,7 @@ const toJob = (row, store) => {
|
|
|
78
94
|
progress: deserialize(row.progress),
|
|
79
95
|
result: deserialize(row.result),
|
|
80
96
|
error: deserialize(row.error),
|
|
81
|
-
opts:
|
|
97
|
+
opts: deserialize(row.opts) ?? {},
|
|
82
98
|
availableAt: row.available_at,
|
|
83
99
|
createdAt: row.created_at,
|
|
84
100
|
updatedAt: row.updated_at,
|
|
@@ -366,10 +382,7 @@ function createCloudflareJobStore(config) {
|
|
|
366
382
|
await execute(target, 'UPDATE cf_queue_repeatables SET run_count = ?, next_run_at = ?, active = ?, updated_at = ? WHERE id = ?', [runCount, nextRunAt, shouldDeactivate ? 0 : 1, nowIso(), row.id]);
|
|
367
383
|
},
|
|
368
384
|
async removeRepeatable(id) {
|
|
369
|
-
await execute(target, 'UPDATE cf_queue_repeatables SET active = 0, updated_at = ? WHERE id = ?', [
|
|
370
|
-
nowIso(),
|
|
371
|
-
id,
|
|
372
|
-
]);
|
|
385
|
+
await execute(target, 'UPDATE cf_queue_repeatables SET active = 0, updated_at = ? WHERE id = ?', [nowIso(), id]);
|
|
373
386
|
},
|
|
374
387
|
async createFlow(input) {
|
|
375
388
|
const parent = await store.createJob({
|
|
@@ -25,8 +25,8 @@ export type CloudflareQueueConsumerConfig<T = unknown, TResult = unknown> = {
|
|
|
25
25
|
};
|
|
26
26
|
export declare const CloudflareQueueConsumer: Readonly<{
|
|
27
27
|
create: <T = unknown, TResult = unknown>(config: CloudflareQueueConsumerConfig<T, TResult>) => {
|
|
28
|
-
processMessage(message: CloudflareMessageLike, batchQueueName?: string)
|
|
29
|
-
processBatch(batch: CloudflareBatchLike)
|
|
28
|
+
processMessage: (message: CloudflareMessageLike, batchQueueName?: string) => Promise<void>;
|
|
29
|
+
processBatch: (batch: CloudflareBatchLike) => Promise<void>;
|
|
30
30
|
};
|
|
31
31
|
createDeadLetter(input: {
|
|
32
32
|
queueName?: string;
|
|
@@ -4,22 +4,23 @@ const isEnvelope = (value) => {
|
|
|
4
4
|
value.protocol === 'zintrust.cf.queue.v1' &&
|
|
5
5
|
typeof value.jobId === 'string');
|
|
6
6
|
};
|
|
7
|
-
const
|
|
8
|
-
|
|
9
|
-
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
return direct.get(direct.idFromName(queueName ?? 'default'));
|
|
7
|
+
const getCoordinatorFromStub = (stub, queueName) => {
|
|
8
|
+
if (stub?.getByName !== undefined)
|
|
9
|
+
return stub.getByName(queueName);
|
|
10
|
+
if (stub?.idFromName !== undefined && stub.get !== undefined) {
|
|
11
|
+
return stub.get(stub.idFromName(queueName));
|
|
13
12
|
}
|
|
13
|
+
return null;
|
|
14
|
+
};
|
|
15
|
+
const resolveCoordinator = (config, queueName = 'default') => {
|
|
16
|
+
const direct = config?.coordinator;
|
|
17
|
+
const fromDirect = getCoordinatorFromStub(direct, queueName);
|
|
18
|
+
if (fromDirect !== null)
|
|
19
|
+
return fromDirect;
|
|
14
20
|
const env = globalThis.env;
|
|
15
21
|
const bindingName = config?.coordinatorBindingName ?? 'QUEUE_COORDINATOR';
|
|
16
22
|
const binding = env?.[bindingName];
|
|
17
|
-
|
|
18
|
-
return binding.getByName(queueName ?? 'default');
|
|
19
|
-
if (binding?.idFromName !== undefined && binding.get !== undefined) {
|
|
20
|
-
return binding.get(binding.idFromName(queueName ?? 'default'));
|
|
21
|
-
}
|
|
22
|
-
return null;
|
|
23
|
+
return getCoordinatorFromStub(binding, queueName);
|
|
23
24
|
};
|
|
24
25
|
const resolveWorkerId = (config) => {
|
|
25
26
|
return config?.workerId ?? 'queue-cloudflare-worker';
|
|
@@ -43,40 +44,124 @@ const serializeError = (error) => {
|
|
|
43
44
|
return { name: error.name, message: error.message, stack: error.stack };
|
|
44
45
|
return error;
|
|
45
46
|
};
|
|
47
|
+
const createProcessorContext = (config, queueName, job, coordinator, message) => ({
|
|
48
|
+
job,
|
|
49
|
+
attempt: message.attempts,
|
|
50
|
+
updateProgress: async (progress) => {
|
|
51
|
+
await config.store.updateProgress(queueName, job.id, progress);
|
|
52
|
+
},
|
|
53
|
+
log: async (logMessage, data) => {
|
|
54
|
+
await config.store.recordLog(queueName, job.id, 'info', logMessage, data);
|
|
55
|
+
},
|
|
56
|
+
heartbeat: async () => {
|
|
57
|
+
await coordinator?.heartbeat?.({
|
|
58
|
+
queueName,
|
|
59
|
+
jobId: job.id,
|
|
60
|
+
workerId: resolveWorkerId(config.state),
|
|
61
|
+
ttlMs: config.state?.heartbeatTtlMs ?? 60_000,
|
|
62
|
+
});
|
|
63
|
+
await config.store.updateState({
|
|
64
|
+
queueName,
|
|
65
|
+
jobId: job.id,
|
|
66
|
+
state: 'active',
|
|
67
|
+
});
|
|
68
|
+
},
|
|
69
|
+
});
|
|
70
|
+
const handleJobSuccess = async (config, queueName, job, result, coordinator, message) => {
|
|
71
|
+
await config.store.updateState({
|
|
72
|
+
queueName,
|
|
73
|
+
jobId: job.id,
|
|
74
|
+
state: 'completed',
|
|
75
|
+
result,
|
|
76
|
+
});
|
|
77
|
+
const releasedParents = await config.store.markFlowChildCompleted(queueName, job.id);
|
|
78
|
+
for (const parent of releasedParents) {
|
|
79
|
+
await config.queue?.enqueue(parent.queueName, toEnvelope(parent));
|
|
80
|
+
await config.store.markDispatched(parent.queueName, parent.id);
|
|
81
|
+
}
|
|
82
|
+
await config.store.applyRetention(queueName, job.id, 'completed');
|
|
83
|
+
await coordinator?.releaseLease?.({
|
|
84
|
+
queueName,
|
|
85
|
+
jobId: job.id,
|
|
86
|
+
workerId: resolveWorkerId(config.state),
|
|
87
|
+
ttlMs: config.state?.heartbeatTtlMs ?? 60_000,
|
|
88
|
+
});
|
|
89
|
+
message.ack();
|
|
90
|
+
};
|
|
91
|
+
const handleJobFailure = async (config, queueName, job, error, coordinator, message) => {
|
|
92
|
+
const exhausted = job.attemptsMade + 1 >= job.maxAttempts;
|
|
93
|
+
await config.store.updateState({
|
|
94
|
+
queueName,
|
|
95
|
+
jobId: job.id,
|
|
96
|
+
state: exhausted ? 'failed' : 'retrying',
|
|
97
|
+
error: serializeError(error),
|
|
98
|
+
});
|
|
99
|
+
if (exhausted) {
|
|
100
|
+
await config.store.applyRetention(queueName, job.id, 'failed');
|
|
101
|
+
await coordinator?.releaseLease?.({
|
|
102
|
+
queueName,
|
|
103
|
+
jobId: job.id,
|
|
104
|
+
workerId: resolveWorkerId(config.state),
|
|
105
|
+
ttlMs: config.state?.heartbeatTtlMs ?? 60_000,
|
|
106
|
+
});
|
|
107
|
+
message.ack();
|
|
108
|
+
return;
|
|
109
|
+
}
|
|
110
|
+
message.retry({
|
|
111
|
+
delaySeconds: resolveRetryDelay(job, message.attempts, config.retryDelaySeconds ?? 30),
|
|
112
|
+
});
|
|
113
|
+
};
|
|
114
|
+
const resolveEnvelope = (message, configQueueName, batchQueueName) => {
|
|
115
|
+
if (isEnvelope(message.body))
|
|
116
|
+
return message.body;
|
|
117
|
+
return {
|
|
118
|
+
protocol: 'zintrust.cf.queue.v1',
|
|
119
|
+
jobId: message.id,
|
|
120
|
+
queueName: configQueueName ?? batchQueueName ?? 'default',
|
|
121
|
+
name: 'default',
|
|
122
|
+
attempt: message.attempts,
|
|
123
|
+
availableAt: new Date().toISOString(),
|
|
124
|
+
};
|
|
125
|
+
};
|
|
126
|
+
const shouldSkipProcessing = async (coordinator, retryDelaySeconds, message) => {
|
|
127
|
+
if ((await coordinator?.isPaused?.()) === true) {
|
|
128
|
+
message.retry({ delaySeconds: retryDelaySeconds });
|
|
129
|
+
return true;
|
|
130
|
+
}
|
|
131
|
+
return false;
|
|
132
|
+
};
|
|
133
|
+
const acquireJobLease = async (coordinator, queueName, jobId, workerId, ttlMs, retryDelaySeconds, message) => {
|
|
134
|
+
const lease = await coordinator?.acquireLease?.({
|
|
135
|
+
queueName,
|
|
136
|
+
jobId,
|
|
137
|
+
workerId,
|
|
138
|
+
ttlMs,
|
|
139
|
+
});
|
|
140
|
+
if (lease !== undefined && !lease.acquired) {
|
|
141
|
+
message.retry({ delaySeconds: retryDelaySeconds });
|
|
142
|
+
return false;
|
|
143
|
+
}
|
|
144
|
+
return true;
|
|
145
|
+
};
|
|
46
146
|
const createConsumer = (config) => {
|
|
47
147
|
return {
|
|
48
148
|
async processMessage(message, batchQueueName) {
|
|
49
|
-
const envelope =
|
|
50
|
-
? message.body
|
|
51
|
-
: {
|
|
52
|
-
protocol: 'zintrust.cf.queue.v1',
|
|
53
|
-
jobId: message.id,
|
|
54
|
-
queueName: config.queueName ?? batchQueueName ?? 'default',
|
|
55
|
-
name: 'default',
|
|
56
|
-
attempt: message.attempts,
|
|
57
|
-
availableAt: new Date().toISOString(),
|
|
58
|
-
};
|
|
149
|
+
const envelope = resolveEnvelope(message, config.queueName, batchQueueName);
|
|
59
150
|
const queueName = config.queueName ?? envelope.queueName;
|
|
60
151
|
const coordinator = resolveCoordinator(config.state, queueName);
|
|
61
|
-
|
|
62
|
-
|
|
152
|
+
const retryDelaySeconds = config.retryDelaySeconds ?? 30;
|
|
153
|
+
if (await shouldSkipProcessing(coordinator, retryDelaySeconds, message))
|
|
63
154
|
return;
|
|
64
|
-
}
|
|
65
155
|
const job = await config.store.getJob(queueName, envelope.jobId);
|
|
66
156
|
if (job === null || job.state === 'canceled') {
|
|
67
157
|
message.ack();
|
|
68
158
|
return;
|
|
69
159
|
}
|
|
70
|
-
const
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
ttlMs: config.state?.heartbeatTtlMs ?? 60_000,
|
|
75
|
-
});
|
|
76
|
-
if (lease !== undefined && !lease.acquired) {
|
|
77
|
-
message.retry({ delaySeconds: config.retryDelaySeconds ?? 30 });
|
|
160
|
+
const workerId = resolveWorkerId(config.state);
|
|
161
|
+
const ttlMs = config.state?.heartbeatTtlMs ?? 60_000;
|
|
162
|
+
const leaseAcquired = await acquireJobLease(coordinator, queueName, job.id, workerId, ttlMs, retryDelaySeconds, message);
|
|
163
|
+
if (!leaseAcquired)
|
|
78
164
|
return;
|
|
79
|
-
}
|
|
80
165
|
await config.store.updateState({
|
|
81
166
|
queueName,
|
|
82
167
|
jobId: job.id,
|
|
@@ -84,71 +169,12 @@ const createConsumer = (config) => {
|
|
|
84
169
|
incrementAttempts: true,
|
|
85
170
|
});
|
|
86
171
|
try {
|
|
87
|
-
const
|
|
88
|
-
|
|
89
|
-
|
|
90
|
-
updateProgress: async (progress) => {
|
|
91
|
-
await config.store.updateProgress(queueName, job.id, progress);
|
|
92
|
-
},
|
|
93
|
-
log: async (logMessage, data) => {
|
|
94
|
-
await config.store.recordLog(queueName, job.id, 'info', logMessage, data);
|
|
95
|
-
},
|
|
96
|
-
heartbeat: async () => {
|
|
97
|
-
await coordinator?.heartbeat?.({
|
|
98
|
-
queueName,
|
|
99
|
-
jobId: job.id,
|
|
100
|
-
workerId: resolveWorkerId(config.state),
|
|
101
|
-
ttlMs: config.state?.heartbeatTtlMs ?? 60_000,
|
|
102
|
-
});
|
|
103
|
-
await config.store.updateState({
|
|
104
|
-
queueName,
|
|
105
|
-
jobId: job.id,
|
|
106
|
-
state: 'active',
|
|
107
|
-
});
|
|
108
|
-
},
|
|
109
|
-
});
|
|
110
|
-
await config.store.updateState({
|
|
111
|
-
queueName,
|
|
112
|
-
jobId: job.id,
|
|
113
|
-
state: 'completed',
|
|
114
|
-
result,
|
|
115
|
-
});
|
|
116
|
-
const releasedParents = await config.store.markFlowChildCompleted(queueName, job.id);
|
|
117
|
-
for (const parent of releasedParents) {
|
|
118
|
-
await config.queue?.enqueue(parent.queueName, toEnvelope(parent));
|
|
119
|
-
await config.store.markDispatched(parent.queueName, parent.id);
|
|
120
|
-
}
|
|
121
|
-
await config.store.applyRetention(queueName, job.id, 'completed');
|
|
122
|
-
await coordinator?.releaseLease?.({
|
|
123
|
-
queueName,
|
|
124
|
-
jobId: job.id,
|
|
125
|
-
workerId: resolveWorkerId(config.state),
|
|
126
|
-
ttlMs: config.state?.heartbeatTtlMs ?? 60_000,
|
|
127
|
-
});
|
|
128
|
-
message.ack();
|
|
172
|
+
const context = createProcessorContext(config, queueName, job, coordinator, message);
|
|
173
|
+
const result = await config.processor(job.data, context);
|
|
174
|
+
await handleJobSuccess(config, queueName, job, result, coordinator, message);
|
|
129
175
|
}
|
|
130
176
|
catch (error) {
|
|
131
|
-
|
|
132
|
-
await config.store.updateState({
|
|
133
|
-
queueName,
|
|
134
|
-
jobId: job.id,
|
|
135
|
-
state: exhausted ? 'failed' : 'retrying',
|
|
136
|
-
error: serializeError(error),
|
|
137
|
-
});
|
|
138
|
-
if (exhausted) {
|
|
139
|
-
await config.store.applyRetention(queueName, job.id, 'failed');
|
|
140
|
-
await coordinator?.releaseLease?.({
|
|
141
|
-
queueName,
|
|
142
|
-
jobId: job.id,
|
|
143
|
-
workerId: resolveWorkerId(config.state),
|
|
144
|
-
ttlMs: config.state?.heartbeatTtlMs ?? 60_000,
|
|
145
|
-
});
|
|
146
|
-
message.ack();
|
|
147
|
-
return;
|
|
148
|
-
}
|
|
149
|
-
message.retry({
|
|
150
|
-
delaySeconds: resolveRetryDelay(job, message.attempts, config.retryDelaySeconds ?? 30),
|
|
151
|
-
});
|
|
177
|
+
await handleJobFailure(config, queueName, job, error, coordinator, message);
|
|
152
178
|
}
|
|
153
179
|
},
|
|
154
180
|
async processBatch(batch) {
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
const leaseKey = (queueName, jobId) => `lease:${queueName}:${jobId}`;
|
|
2
2
|
const pausedKey = (queueName) => `paused:${queueName}`;
|
|
3
3
|
const rateLimitKey = (key) => `rate:${key}`;
|
|
4
|
+
// eslint-disable-next-line no-restricted-syntax
|
|
4
5
|
export class CloudflareQueueCoordinator {
|
|
5
6
|
state;
|
|
6
7
|
constructor(state) {
|
|
@@ -10,13 +10,9 @@ export const CloudflareQueueMigrator = Object.freeze({
|
|
|
10
10
|
statements: cloudflareQueueMigrationStatements,
|
|
11
11
|
rollbackStatements: cloudflareQueueRollbackStatements,
|
|
12
12
|
async up(target) {
|
|
13
|
-
|
|
14
|
-
await runStatement(target, statement);
|
|
15
|
-
}
|
|
13
|
+
await Promise.all(cloudflareQueueMigrationStatements.map((statement) => runStatement(target, statement)));
|
|
16
14
|
},
|
|
17
15
|
async down(target) {
|
|
18
|
-
|
|
19
|
-
await runStatement(target, statement);
|
|
20
|
-
}
|
|
16
|
+
await Promise.all(cloudflareQueueRollbackStatements.map((statement) => runStatement(target, statement)));
|
|
21
17
|
},
|
|
22
18
|
});
|
|
@@ -12,10 +12,10 @@ export type CloudflareQueueSchedulerConfig = {
|
|
|
12
12
|
};
|
|
13
13
|
export declare const CloudflareQueueScheduler: Readonly<{
|
|
14
14
|
create: (config: CloudflareQueueSchedulerConfig) => {
|
|
15
|
-
dispatchDueJobs()
|
|
16
|
-
dispatchRepeatables()
|
|
17
|
-
reconcileStalled()
|
|
18
|
-
run()
|
|
15
|
+
dispatchDueJobs: () => Promise<number>;
|
|
16
|
+
dispatchRepeatables: () => Promise<number>;
|
|
17
|
+
reconcileStalled: () => Promise<number>;
|
|
18
|
+
run: () => Promise<{
|
|
19
19
|
jobs: number;
|
|
20
20
|
repeatables: number;
|
|
21
21
|
}>;
|
|
@@ -27,10 +27,10 @@ export declare const CloudflareQueueScheduler: Readonly<{
|
|
|
27
27
|
queue: SchedulerQueueApi;
|
|
28
28
|
batchSize?: number;
|
|
29
29
|
}): {
|
|
30
|
-
dispatchDueJobs()
|
|
31
|
-
dispatchRepeatables()
|
|
32
|
-
reconcileStalled()
|
|
33
|
-
run()
|
|
30
|
+
dispatchDueJobs: () => Promise<number>;
|
|
31
|
+
dispatchRepeatables: () => Promise<number>;
|
|
32
|
+
reconcileStalled: () => Promise<number>;
|
|
33
|
+
run: () => Promise<{
|
|
34
34
|
jobs: number;
|
|
35
35
|
repeatables: number;
|
|
36
36
|
}>;
|
|
@@ -6,20 +6,61 @@ const toEnvelope = (input) => ({
|
|
|
6
6
|
attempt: input.attempt,
|
|
7
7
|
availableAt: input.availableAt,
|
|
8
8
|
});
|
|
9
|
+
const dispatchJob = async (config, job) => {
|
|
10
|
+
await config.queue.enqueue(config.queueName, toEnvelope({
|
|
11
|
+
queueName: job.queueName,
|
|
12
|
+
jobId: job.id,
|
|
13
|
+
name: job.name,
|
|
14
|
+
attempt: job.attemptsMade,
|
|
15
|
+
availableAt: job.availableAt,
|
|
16
|
+
}));
|
|
17
|
+
await config.store.markDispatched(job.queueName, job.id);
|
|
18
|
+
};
|
|
19
|
+
const dispatchRepeatable = async (config, row) => {
|
|
20
|
+
const payload = JSON.parse(row.payload);
|
|
21
|
+
const job = await config.store.createJob({
|
|
22
|
+
queueName: row.queue_name,
|
|
23
|
+
name: row.name,
|
|
24
|
+
data: payload,
|
|
25
|
+
});
|
|
26
|
+
await config.queue.enqueue(row.queue_name, toEnvelope({
|
|
27
|
+
queueName: row.queue_name,
|
|
28
|
+
jobId: job.id,
|
|
29
|
+
name: row.name,
|
|
30
|
+
attempt: 0,
|
|
31
|
+
availableAt: job.availableAt,
|
|
32
|
+
}));
|
|
33
|
+
await config.store.markDispatched(job.queueName, job.id);
|
|
34
|
+
await config.store.updateRepeatableAfterRun(row);
|
|
35
|
+
};
|
|
36
|
+
const retryStalledJob = async (config, job) => {
|
|
37
|
+
await config.store.updateState({
|
|
38
|
+
queueName: job.queueName,
|
|
39
|
+
jobId: job.id,
|
|
40
|
+
state: 'stalled',
|
|
41
|
+
});
|
|
42
|
+
await config.store.updateState({
|
|
43
|
+
queueName: job.queueName,
|
|
44
|
+
jobId: job.id,
|
|
45
|
+
state: 'retrying',
|
|
46
|
+
availableAt: new Date().toISOString(),
|
|
47
|
+
});
|
|
48
|
+
await config.queue.enqueue(job.queueName, toEnvelope({
|
|
49
|
+
queueName: job.queueName,
|
|
50
|
+
jobId: job.id,
|
|
51
|
+
name: job.name,
|
|
52
|
+
attempt: job.attemptsMade,
|
|
53
|
+
availableAt: job.availableAt,
|
|
54
|
+
}));
|
|
55
|
+
await config.store.markDispatched(job.queueName, job.id);
|
|
56
|
+
};
|
|
9
57
|
const createScheduler = (config) => {
|
|
10
58
|
return {
|
|
11
59
|
async dispatchDueJobs() {
|
|
12
60
|
const jobs = await config.store.getDueJobs(config.queueName, config.batchSize);
|
|
13
61
|
let dispatched = 0;
|
|
14
62
|
for (const job of jobs) {
|
|
15
|
-
await
|
|
16
|
-
queueName: job.queueName,
|
|
17
|
-
jobId: job.id,
|
|
18
|
-
name: job.name,
|
|
19
|
-
attempt: job.attemptsMade,
|
|
20
|
-
availableAt: job.availableAt,
|
|
21
|
-
}));
|
|
22
|
-
await config.store.markDispatched(job.queueName, job.id);
|
|
63
|
+
await dispatchJob(config, job);
|
|
23
64
|
dispatched += 1;
|
|
24
65
|
}
|
|
25
66
|
return dispatched;
|
|
@@ -28,21 +69,7 @@ const createScheduler = (config) => {
|
|
|
28
69
|
const rows = await config.store.getDueRepeatables(config.batchSize);
|
|
29
70
|
let dispatched = 0;
|
|
30
71
|
for (const row of rows) {
|
|
31
|
-
|
|
32
|
-
const job = await config.store.createJob({
|
|
33
|
-
queueName: row.queue_name,
|
|
34
|
-
name: row.name,
|
|
35
|
-
data: payload,
|
|
36
|
-
});
|
|
37
|
-
await config.queue.enqueue(row.queue_name, toEnvelope({
|
|
38
|
-
queueName: row.queue_name,
|
|
39
|
-
jobId: job.id,
|
|
40
|
-
name: row.name,
|
|
41
|
-
attempt: 0,
|
|
42
|
-
availableAt: job.availableAt,
|
|
43
|
-
}));
|
|
44
|
-
await config.store.markDispatched(job.queueName, job.id);
|
|
45
|
-
await config.store.updateRepeatableAfterRun(row);
|
|
72
|
+
await dispatchRepeatable(config, row);
|
|
46
73
|
dispatched += 1;
|
|
47
74
|
}
|
|
48
75
|
return dispatched;
|
|
@@ -51,25 +78,7 @@ const createScheduler = (config) => {
|
|
|
51
78
|
const stalled = await config.store.getStalledJobs(config.queueName, config.stalledAfterMs ?? 60_000, config.batchSize);
|
|
52
79
|
let retried = 0;
|
|
53
80
|
for (const job of stalled) {
|
|
54
|
-
await config
|
|
55
|
-
queueName: job.queueName,
|
|
56
|
-
jobId: job.id,
|
|
57
|
-
state: 'stalled',
|
|
58
|
-
});
|
|
59
|
-
await config.store.updateState({
|
|
60
|
-
queueName: job.queueName,
|
|
61
|
-
jobId: job.id,
|
|
62
|
-
state: 'retrying',
|
|
63
|
-
availableAt: new Date().toISOString(),
|
|
64
|
-
});
|
|
65
|
-
await config.queue.enqueue(job.queueName, toEnvelope({
|
|
66
|
-
queueName: job.queueName,
|
|
67
|
-
jobId: job.id,
|
|
68
|
-
name: job.name,
|
|
69
|
-
attempt: job.attemptsMade,
|
|
70
|
-
availableAt: job.availableAt,
|
|
71
|
-
}));
|
|
72
|
-
await config.store.markDispatched(job.queueName, job.id);
|
|
81
|
+
await retryStalledJob(config, job);
|
|
73
82
|
retried += 1;
|
|
74
83
|
}
|
|
75
84
|
return retried;
|
|
@@ -0,0 +1,161 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "@zintrust/queue-cloudflare",
|
|
3
|
+
"version": "2.4.1",
|
|
4
|
+
"buildDate": "2026-06-11T20:48:26.547Z",
|
|
5
|
+
"buildEnvironment": {
|
|
6
|
+
"node": "v22.22.1",
|
|
7
|
+
"platform": "darwin",
|
|
8
|
+
"arch": "arm64"
|
|
9
|
+
},
|
|
10
|
+
"git": {
|
|
11
|
+
"commit": "5a1a7000",
|
|
12
|
+
"branch": "release"
|
|
13
|
+
},
|
|
14
|
+
"package": {
|
|
15
|
+
"engines": {
|
|
16
|
+
"node": ">=20.0.0"
|
|
17
|
+
},
|
|
18
|
+
"dependencies": [],
|
|
19
|
+
"peerDependencies": [
|
|
20
|
+
"@zintrust/core"
|
|
21
|
+
]
|
|
22
|
+
},
|
|
23
|
+
"files": {
|
|
24
|
+
"CloudflareJobStore.d.ts": {
|
|
25
|
+
"size": 2901,
|
|
26
|
+
"sha256": "906d0a4ede39894eec99e2da4215b701947a3717696ad5735947ad6a672c078f"
|
|
27
|
+
},
|
|
28
|
+
"CloudflareJobStore.js": {
|
|
29
|
+
"size": 20661,
|
|
30
|
+
"sha256": "6226a96500b6fc01117624cea84d2211fc7ec916bdd9221adaa42c08974f8893"
|
|
31
|
+
},
|
|
32
|
+
"CloudflareQueueConsumer.d.ts": {
|
|
33
|
+
"size": 1411,
|
|
34
|
+
"sha256": "9a1a5d9b19f46811403c9b23e9b2ce7c15fee03505de0e0fd54950f9518e93af"
|
|
35
|
+
},
|
|
36
|
+
"CloudflareQueueConsumer.js": {
|
|
37
|
+
"size": 7714,
|
|
38
|
+
"sha256": "da6b078b05d8e3a57b5e19fc4c325a01ef21c6350e6bc4687c42aa6752ffb2fc"
|
|
39
|
+
},
|
|
40
|
+
"CloudflareQueueCoordinator.d.ts": {
|
|
41
|
+
"size": 898,
|
|
42
|
+
"sha256": "93e028ce222f7dbf9c17640e2a23620dbb24d1f545bfd7dcd739f97132f2aea1"
|
|
43
|
+
},
|
|
44
|
+
"CloudflareQueueCoordinator.js": {
|
|
45
|
+
"size": 2541,
|
|
46
|
+
"sha256": "2025c203e84adac03a9db753326bc4d97b4540eef3ee29ab1f1fd729959ff178"
|
|
47
|
+
},
|
|
48
|
+
"CloudflareQueueMigrator.d.ts": {
|
|
49
|
+
"size": 481,
|
|
50
|
+
"sha256": "462f69b84e99a67cdc4867effd9aafd4153562683889e4e54b4249d0ff89fd0c"
|
|
51
|
+
},
|
|
52
|
+
"CloudflareQueueMigrator.js": {
|
|
53
|
+
"size": 821,
|
|
54
|
+
"sha256": "6a5b9d17795f17b1b6d3a944cf665352f315ba5cbb29939ac5f4bc746676fa81"
|
|
55
|
+
},
|
|
56
|
+
"CloudflareQueueScheduler.d.ts": {
|
|
57
|
+
"size": 1373,
|
|
58
|
+
"sha256": "561bfd82597e6171f6a02accec7b7eaaf05a415d03c36806622326cbb3319338"
|
|
59
|
+
},
|
|
60
|
+
"CloudflareQueueScheduler.js": {
|
|
61
|
+
"size": 3751,
|
|
62
|
+
"sha256": "2574729f296c5470d9d2e2a8f2a750f0e2a504bc5c2fa72551c03aec58250475"
|
|
63
|
+
},
|
|
64
|
+
"cli/MigrateCloudflareQueueCommand.d.ts": {
|
|
65
|
+
"size": 279,
|
|
66
|
+
"sha256": "7053823798a95d919634e449a1a61bc872ea78e2a72d438fd60b9775457f30e8"
|
|
67
|
+
},
|
|
68
|
+
"cli/MigrateCloudflareQueueCommand.js": {
|
|
69
|
+
"size": 2214,
|
|
70
|
+
"sha256": "b0283c36ab377249b28fde4118fdc07dfcecfe56e0e60c5efa293fcdfc90cd69"
|
|
71
|
+
},
|
|
72
|
+
"cli-register.d.ts": {
|
|
73
|
+
"size": 265,
|
|
74
|
+
"sha256": "6816103b5a0b5a76bcf10b9e33f8405f42911a9d0f46c9b28b9af8cd2fac1671"
|
|
75
|
+
},
|
|
76
|
+
"cli-register.js": {
|
|
77
|
+
"size": 799,
|
|
78
|
+
"sha256": "1b2d4af1ba820e9d4ade4981dd6ad66cd7497b4d57847755c6272422871bd8aa"
|
|
79
|
+
},
|
|
80
|
+
"index.d.ts": {
|
|
81
|
+
"size": 3797,
|
|
82
|
+
"sha256": "716eceb48e844c1bbc80b63c73a40495dd88bf95ab3a8b332791efcd87fb0aa5"
|
|
83
|
+
},
|
|
84
|
+
"index.js": {
|
|
85
|
+
"size": 19749,
|
|
86
|
+
"sha256": "4f36fc2b0caa6eb99a4199a23493f14ead410e76f6b57b909919e6c0e3155f70"
|
|
87
|
+
},
|
|
88
|
+
"migrationSql.d.ts": {
|
|
89
|
+
"size": 151,
|
|
90
|
+
"sha256": "fe04e08b27af1d72c0a7a5f53c1a7344a74b76bca6828dbcaab4aad9bf11121c"
|
|
91
|
+
},
|
|
92
|
+
"migrationSql.js": {
|
|
93
|
+
"size": 4000,
|
|
94
|
+
"sha256": "5771630205e8a1f6ff8a40de13ed410f78ebc3948aa95635f043560c7027056d"
|
|
95
|
+
},
|
|
96
|
+
"migrations/20260608000001_create_cloudflare_queue_jobs_table.d.ts": {
|
|
97
|
+
"size": 210,
|
|
98
|
+
"sha256": "37a4d000493f61c93bcccd7d6253beda0112f656c4c57ba1415d69fcd86f12f8"
|
|
99
|
+
},
|
|
100
|
+
"migrations/20260608000001_create_cloudflare_queue_jobs_table.js": {
|
|
101
|
+
"size": 1589,
|
|
102
|
+
"sha256": "3da5123da3fbf5d3a297d916b030f62022e854a81451e1421c582c18eac92f97"
|
|
103
|
+
},
|
|
104
|
+
"migrations/20260608000002_create_cloudflare_queue_job_events_table.d.ts": {
|
|
105
|
+
"size": 210,
|
|
106
|
+
"sha256": "37a4d000493f61c93bcccd7d6253beda0112f656c4c57ba1415d69fcd86f12f8"
|
|
107
|
+
},
|
|
108
|
+
"migrations/20260608000002_create_cloudflare_queue_job_events_table.js": {
|
|
109
|
+
"size": 772,
|
|
110
|
+
"sha256": "824e2c98ae5d58255d7a5b489e89000464c362b650f25634513720980748ee73"
|
|
111
|
+
},
|
|
112
|
+
"migrations/20260608000003_create_cloudflare_queue_job_logs_table.d.ts": {
|
|
113
|
+
"size": 210,
|
|
114
|
+
"sha256": "37a4d000493f61c93bcccd7d6253beda0112f656c4c57ba1415d69fcd86f12f8"
|
|
115
|
+
},
|
|
116
|
+
"migrations/20260608000003_create_cloudflare_queue_job_logs_table.js": {
|
|
117
|
+
"size": 803,
|
|
118
|
+
"sha256": "0731df82ab7d7368bafb8a7c1bf13eb50bf45569302a387b977e333c49d225f6"
|
|
119
|
+
},
|
|
120
|
+
"migrations/20260608000004_create_cloudflare_queue_repeatables_table.d.ts": {
|
|
121
|
+
"size": 210,
|
|
122
|
+
"sha256": "37a4d000493f61c93bcccd7d6253beda0112f656c4c57ba1415d69fcd86f12f8"
|
|
123
|
+
},
|
|
124
|
+
"migrations/20260608000004_create_cloudflare_queue_repeatables_table.js": {
|
|
125
|
+
"size": 1117,
|
|
126
|
+
"sha256": "0b43fb6196bb0725fcd304cd13a91be17da78711ee39731b882b64be21227030"
|
|
127
|
+
},
|
|
128
|
+
"migrations/20260608000005_create_cloudflare_queue_flow_dependencies_table.d.ts": {
|
|
129
|
+
"size": 210,
|
|
130
|
+
"sha256": "37a4d000493f61c93bcccd7d6253beda0112f656c4c57ba1415d69fcd86f12f8"
|
|
131
|
+
},
|
|
132
|
+
"migrations/20260608000005_create_cloudflare_queue_flow_dependencies_table.js": {
|
|
133
|
+
"size": 846,
|
|
134
|
+
"sha256": "cbc981e809d3d7e1c14ef02b198bae30751f58c689c80cada42795c200b200dd"
|
|
135
|
+
},
|
|
136
|
+
"migrations/index.d.ts": {
|
|
137
|
+
"size": 194,
|
|
138
|
+
"sha256": "4865154f5157c6af231010ffabfb6939df98804fee48de0725b54dd55214eafb"
|
|
139
|
+
},
|
|
140
|
+
"migrations/index.js": {
|
|
141
|
+
"size": 673,
|
|
142
|
+
"sha256": "fbfc44a57c6f53e5e7778a4e4e4cc8476a7f85e7369b62ad48d14839cd0c60e2"
|
|
143
|
+
},
|
|
144
|
+
"register.d.ts": {
|
|
145
|
+
"size": 175,
|
|
146
|
+
"sha256": "b6f999615d49ceb0acc6b07221e6a77a45d00348b702ab0650ebdb79c19f7b20"
|
|
147
|
+
},
|
|
148
|
+
"register.js": {
|
|
149
|
+
"size": 601,
|
|
150
|
+
"sha256": "e880096f4f293f3f4586e6ec8c4162773617574dfb8dc0f081b1b11fee2a793c"
|
|
151
|
+
},
|
|
152
|
+
"types.d.ts": {
|
|
153
|
+
"size": 7286,
|
|
154
|
+
"sha256": "229f59486e848824998d4a58bfac55cebaf05b069fcb97d5bd23e7577e0bbe39"
|
|
155
|
+
},
|
|
156
|
+
"types.js": {
|
|
157
|
+
"size": 11,
|
|
158
|
+
"sha256": "8e609bb71c20b858c77f0e9f90bb1319db8477b13f9f965f1a1e18524bf50881"
|
|
159
|
+
}
|
|
160
|
+
}
|
|
161
|
+
}
|
package/dist/index.d.ts
CHANGED
|
@@ -1,11 +1,11 @@
|
|
|
1
1
|
import { CloudflareQueueConsumer } from './CloudflareQueueConsumer.js';
|
|
2
|
-
import type {
|
|
3
|
-
export type { CloudflareJobOptions, CloudflareFlowInput, CloudflareFlowResult, CloudflareQueueBackoff, CloudflareQueueBinding, CloudflareQueueConfig, CloudflareQueueContentType, CloudflareQueueEnvelope, CloudflareQueueJob, CloudflareQueueMetrics, CloudflareQueueProcessor, CloudflareQueueProcessorContext, CloudflareQueueState, CloudflareQueueStateConfig, CloudflareQueueRetention, CloudflareRepeatOptions, QueueMessage, } from './types.js';
|
|
2
|
+
import type { CloudflareFlowInput, CloudflareFlowResult, CloudflareJobOptions, CloudflareQueueConfig, CloudflareQueueJob, CloudflareQueueMetrics, CloudflareQueueProcessor, CloudflareQueueState, QueueMessage } from './types.js';
|
|
4
3
|
export { CloudflareJobStore } from './CloudflareJobStore.js';
|
|
5
4
|
export { CloudflareQueueConsumer } from './CloudflareQueueConsumer.js';
|
|
6
5
|
export { CloudflareQueueCoordinator } from './CloudflareQueueCoordinator.js';
|
|
7
6
|
export { CloudflareQueueMigrator } from './CloudflareQueueMigrator.js';
|
|
8
7
|
export { CloudflareQueueScheduler } from './CloudflareQueueScheduler.js';
|
|
8
|
+
export type { CloudflareFlowInput, CloudflareFlowResult, CloudflareJobOptions, CloudflareQueueBackoff, CloudflareQueueBinding, CloudflareQueueConfig, CloudflareQueueContentType, CloudflareQueueEnvelope, CloudflareQueueJob, CloudflareQueueMetrics, CloudflareQueueProcessor, CloudflareQueueProcessorContext, CloudflareQueueRetention, CloudflareQueueState, CloudflareQueueStateConfig, CloudflareRepeatOptions, QueueMessage, } from './types.js';
|
|
9
9
|
type CloudflareQueueDriver = {
|
|
10
10
|
enqueue<T = unknown>(queue: string, payload: T): Promise<string>;
|
|
11
11
|
dequeue<T = unknown>(queue: string): Promise<QueueMessage<T> | undefined>;
|
package/dist/index.js
CHANGED
|
@@ -1,7 +1,9 @@
|
|
|
1
|
+
/* eslint-disable no-await-in-loop, complexity */
|
|
2
|
+
/* Sequential processing required for queue operations, some functions are complex by design */
|
|
1
3
|
import { Cloudflare } from '@zintrust/core/cloudflare';
|
|
2
4
|
import { ErrorFactory } from '@zintrust/core/errors';
|
|
3
5
|
import { generateUuid } from '@zintrust/core/utils';
|
|
4
|
-
import { CloudflareJobStore } from './CloudflareJobStore.js';
|
|
6
|
+
import { CloudflareJobStore, } from './CloudflareJobStore.js';
|
|
5
7
|
import { CloudflareQueueConsumer } from './CloudflareQueueConsumer.js';
|
|
6
8
|
import { CloudflareQueueMigrator } from './CloudflareQueueMigrator.js';
|
|
7
9
|
import { CloudflareQueueScheduler } from './CloudflareQueueScheduler.js';
|
|
@@ -125,8 +127,10 @@ const apiFetch = async (queue, config, path, init) => {
|
|
|
125
127
|
const text = await response.text();
|
|
126
128
|
const parsed = text === '' ? {} : JSON.parse(text);
|
|
127
129
|
if (!response.ok || parsed.success === false) {
|
|
128
|
-
const message = parsed.errors
|
|
129
|
-
|
|
130
|
+
const message = parsed.errors
|
|
131
|
+
?.map((error) => error.message)
|
|
132
|
+
.filter(Boolean)
|
|
133
|
+
.join('; ') || `Cloudflare Queues API error (${response.status})`;
|
|
130
134
|
throw ErrorFactory.createConnectionError(message, { status: response.status, body: parsed });
|
|
131
135
|
}
|
|
132
136
|
return (parsed.result ?? parsed);
|
|
@@ -215,7 +219,9 @@ const createDequeue = (state, config) => async (queue) => {
|
|
|
215
219
|
body: JSON.stringify(body),
|
|
216
220
|
});
|
|
217
221
|
state.lastBacklogCount =
|
|
218
|
-
pulled.message_backlog_count ??
|
|
222
|
+
pulled.message_backlog_count ??
|
|
223
|
+
pulled.metadata?.metrics?.backlog_count ??
|
|
224
|
+
state.lastBacklogCount;
|
|
219
225
|
const message = pulled.messages?.[0];
|
|
220
226
|
if (message === undefined)
|
|
221
227
|
return undefined;
|
|
@@ -290,6 +296,7 @@ const delayOptions = (options) => {
|
|
|
290
296
|
const delaySeconds = Math.max(0, Math.floor((options?.delay ?? 0) / 1000));
|
|
291
297
|
return delaySeconds > 0 ? { delaySeconds } : undefined;
|
|
292
298
|
};
|
|
299
|
+
// eslint-disable-next-line max-lines-per-function -- Complex driver with many methods
|
|
293
300
|
function createCloudflareQueueDriver(config) {
|
|
294
301
|
const state = { leases: new Map() };
|
|
295
302
|
const rawSender = sendRaw(config);
|
|
@@ -424,7 +431,13 @@ function createCloudflareQueueDriver(config) {
|
|
|
424
431
|
await getStore().clean(queue, states, olderThanMs);
|
|
425
432
|
},
|
|
426
433
|
async upsertJobScheduler(queue, name, data, repeat, id) {
|
|
427
|
-
return await getStore().upsertRepeatable({
|
|
434
|
+
return await getStore().upsertRepeatable({
|
|
435
|
+
id,
|
|
436
|
+
queueName: queue,
|
|
437
|
+
name,
|
|
438
|
+
data,
|
|
439
|
+
options: repeat,
|
|
440
|
+
});
|
|
428
441
|
},
|
|
429
442
|
async removeJobScheduler(id) {
|
|
430
443
|
await getStore().removeRepeatable(id);
|
|
@@ -433,10 +446,12 @@ function createCloudflareQueueDriver(config) {
|
|
|
433
446
|
return await CloudflareQueueScheduler.create({
|
|
434
447
|
queueName: queue,
|
|
435
448
|
store: getStore(),
|
|
436
|
-
queue: {
|
|
449
|
+
queue: {
|
|
450
|
+
enqueue: async (queueName, payload) => {
|
|
437
451
|
await rawSender(queueName, payload);
|
|
438
452
|
return generateUuid();
|
|
439
|
-
}
|
|
453
|
+
},
|
|
454
|
+
},
|
|
440
455
|
batchSize: limit,
|
|
441
456
|
}).run();
|
|
442
457
|
},
|
|
@@ -444,10 +459,12 @@ function createCloudflareQueueDriver(config) {
|
|
|
444
459
|
return await CloudflareQueueScheduler.create({
|
|
445
460
|
queueName: queue,
|
|
446
461
|
store: getStore(),
|
|
447
|
-
queue: {
|
|
462
|
+
queue: {
|
|
463
|
+
enqueue: async (queueName, payload) => {
|
|
448
464
|
await rawSender(queueName, payload);
|
|
449
465
|
return generateUuid();
|
|
450
|
-
}
|
|
466
|
+
},
|
|
467
|
+
},
|
|
451
468
|
batchSize: limit,
|
|
452
469
|
stalledAfterMs: olderThanMs,
|
|
453
470
|
}).reconcileStalled();
|
|
@@ -472,9 +489,11 @@ function createCloudflareQueueDriver(config) {
|
|
|
472
489
|
store: getStore(),
|
|
473
490
|
processor,
|
|
474
491
|
state: config?.state,
|
|
475
|
-
queue: {
|
|
476
|
-
|
|
477
|
-
|
|
492
|
+
queue: {
|
|
493
|
+
enqueue: async (targetQueue, payload) => {
|
|
494
|
+
await rawSender(targetQueue, payload);
|
|
495
|
+
},
|
|
496
|
+
},
|
|
478
497
|
});
|
|
479
498
|
},
|
|
480
499
|
async migrateState() {
|