enqiu 0.1.2 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +298 -0
- package/README.md +132 -235
- package/dist/api.d.ts +16 -236
- package/dist/api.js +383 -478
- package/dist/backend.d.ts +25 -0
- package/dist/backend.js +18 -0
- package/dist/definition.d.ts +13 -0
- package/dist/definition.js +51 -0
- package/dist/errors.d.ts +57 -0
- package/dist/errors.js +83 -0
- package/dist/events.d.ts +30 -0
- package/dist/events.js +53 -0
- package/dist/index.d.ts +4 -5
- package/dist/index.js +3 -3
- package/dist/mapping.d.ts +99 -0
- package/dist/mapping.js +167 -0
- package/dist/markers.d.ts +25 -0
- package/dist/markers.js +51 -0
- package/dist/runner.d.ts +20 -0
- package/dist/runner.js +101 -0
- package/dist/serialize.d.ts +15 -0
- package/dist/serialize.js +90 -0
- package/dist/types.d.ts +326 -0
- package/dist/types.js +9 -0
- package/package.json +29 -13
- package/dist/codec.d.ts +0 -8
- package/dist/codec.js +0 -74
- package/dist/cron.d.ts +0 -19
- package/dist/cron.js +0 -217
- package/dist/memory-scheduler.d.ts +0 -24
- package/dist/memory-scheduler.js +0 -163
- package/dist/memory.d.ts +0 -344
- package/dist/memory.js +0 -1201
- package/dist/redis.d.ts +0 -202
- package/dist/redis.js +0 -2180
package/dist/api.js
CHANGED
|
@@ -1,519 +1,424 @@
|
|
|
1
|
-
|
|
2
|
-
|
|
3
|
-
|
|
4
|
-
|
|
5
|
-
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
|
|
1
|
+
/**
|
|
2
|
+
* `enqiu()` — a typed layer over BullMQ.
|
|
3
|
+
*
|
|
4
|
+
* Enqiu owns the developer experience: inferred job names, schema-validated
|
|
5
|
+
* input, and one object per job that you call like a function. BullMQ owns
|
|
6
|
+
* storage, scheduling and execution. Anything BullMQ's open-source tier cannot
|
|
7
|
+
* express is absent rather than faked, with two exceptions Enqiu enforces
|
|
8
|
+
* itself around the handler because they cost nothing to add: `timeout` and
|
|
9
|
+
* `expiresIn`.
|
|
10
|
+
*
|
|
11
|
+
* This file composes; the parts it composes live next to it.
|
|
12
|
+
*/
|
|
13
|
+
import { Queue, Worker } from "bullmq";
|
|
14
|
+
import { normalizeDefinition, validateInput } from "./definition.js";
|
|
15
|
+
import { QueueEventStream } from "./events.js";
|
|
16
|
+
import { CancellationMarkers } from "./markers.js";
|
|
17
|
+
import { JobRunner } from "./runner.js";
|
|
18
|
+
import { assertJobValue } from "./serialize.js";
|
|
19
|
+
import { JobCancelledError, JobFailedError, QueueClosedError, decodeFailure, failureToError, toError, } from "./errors.js";
|
|
20
|
+
import { bullStates, decodeCursor, encodeCursor, everyState, isFinished, mergePage, queueEventMap, toJobsOptions, toSnapshot, toStatus, } from "./mapping.js";
|
|
21
|
+
export { job } from "./definition.js";
|
|
22
|
+
/**
|
|
23
|
+
* What a handle needs to answer questions about its job.
|
|
24
|
+
*
|
|
25
|
+
* Three methods, and everything awkward about them is inside: that a cancelled
|
|
26
|
+
* job may no longer exist, that BullMQ hands a failure back as a bare string,
|
|
27
|
+
* and that only the worker holding a running job can abort it.
|
|
28
|
+
*/
|
|
29
|
+
class JobStore {
|
|
30
|
+
queue;
|
|
31
|
+
events;
|
|
32
|
+
markers;
|
|
33
|
+
worker;
|
|
34
|
+
constructor(queue, events, markers, worker) {
|
|
35
|
+
this.queue = queue;
|
|
36
|
+
this.events = events;
|
|
37
|
+
this.markers = markers;
|
|
38
|
+
this.worker = worker;
|
|
10
39
|
}
|
|
11
|
-
|
|
12
|
-
|
|
40
|
+
/**
|
|
41
|
+
* `state` says what an event already proved, so the state need not be re-read.
|
|
42
|
+
*
|
|
43
|
+
* The job and its state are fetched together: `getJobState` needs only the id,
|
|
44
|
+
* so waiting for the job first was a round trip spent for nothing.
|
|
45
|
+
*/
|
|
46
|
+
async snapshot(id, state) {
|
|
47
|
+
const [bull, settled] = await Promise.all([
|
|
48
|
+
this.queue.getJob(id),
|
|
49
|
+
state ?? this.queue.getJobState(id),
|
|
50
|
+
]);
|
|
51
|
+
// Gone means the marker is all there is — and the marker is a snapshot.
|
|
52
|
+
if (!bull)
|
|
53
|
+
return this.markers.read(id);
|
|
54
|
+
// An aborted job settles as failed; the marker is what distinguishes a
|
|
55
|
+
// deliberate cancellation from one that simply threw.
|
|
56
|
+
if (settled === "failed") {
|
|
57
|
+
const cancelled = await this.markers.read(id);
|
|
58
|
+
if (cancelled)
|
|
59
|
+
return cancelled;
|
|
60
|
+
}
|
|
61
|
+
return toSnapshot(bull, toStatus(settled));
|
|
13
62
|
}
|
|
14
|
-
|
|
15
|
-
|
|
63
|
+
async result(bull) {
|
|
64
|
+
const id = String(bull.id);
|
|
65
|
+
try {
|
|
66
|
+
return await bull.waitUntilFinished(await this.events.open());
|
|
67
|
+
}
|
|
68
|
+
catch (cause) {
|
|
69
|
+
const error = toError(cause);
|
|
70
|
+
// The free check first: a stored envelope needs no round trip to read.
|
|
71
|
+
const failure = decodeFailure(error.message);
|
|
72
|
+
if (failure)
|
|
73
|
+
throw failureToError(failure);
|
|
74
|
+
const cancelled = await this.markers.read(id);
|
|
75
|
+
if (cancelled)
|
|
76
|
+
throw new JobCancelledError(id, cancelled.error?.message);
|
|
77
|
+
throw new JobFailedError(id, error.message, { cause: error });
|
|
78
|
+
}
|
|
16
79
|
}
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
24
|
-
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
|
|
80
|
+
async cancel(bull, reason) {
|
|
81
|
+
const id = String(bull.id);
|
|
82
|
+
const state = await bull.getState();
|
|
83
|
+
if (state === "unknown" || isFinished(state))
|
|
84
|
+
return false;
|
|
85
|
+
// Taken before anything is destroyed: once the job is removed this is the
|
|
86
|
+
// only record that it ever existed, let alone what it carried. Finished
|
|
87
|
+
// here, so a reader has nothing left to reassemble.
|
|
88
|
+
const snapshot = toSnapshot(bull, "cancelled");
|
|
89
|
+
snapshot.finishedAt = Date.now();
|
|
90
|
+
snapshot.error = { name: "JobCancelledError", message: reason };
|
|
91
|
+
if (state === "active") {
|
|
92
|
+
// A running job cannot be removed, but this process's worker can abort
|
|
93
|
+
// it if it is the one holding it. Another worker's job is not ours to
|
|
94
|
+
// cancel, and BullMQ offers no cross-process signal for that.
|
|
95
|
+
if (!this.worker?.cancelJob(id, reason))
|
|
96
|
+
return false;
|
|
97
|
+
}
|
|
98
|
+
else {
|
|
99
|
+
try {
|
|
100
|
+
await bull.remove();
|
|
101
|
+
}
|
|
102
|
+
catch {
|
|
103
|
+
return false;
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
await this.markers.write(id, snapshot);
|
|
107
|
+
return true;
|
|
28
108
|
}
|
|
29
109
|
}
|
|
30
|
-
class
|
|
31
|
-
|
|
110
|
+
class Handle {
|
|
111
|
+
bull;
|
|
112
|
+
store;
|
|
113
|
+
deduplicated;
|
|
32
114
|
resultPromise;
|
|
33
|
-
constructor(
|
|
34
|
-
this.
|
|
115
|
+
constructor(bull, store, deduplicated) {
|
|
116
|
+
this.bull = bull;
|
|
117
|
+
this.store = store;
|
|
118
|
+
this.deduplicated = deduplicated;
|
|
35
119
|
}
|
|
36
120
|
get id() {
|
|
37
|
-
return this.
|
|
121
|
+
return String(this.bull.id);
|
|
38
122
|
}
|
|
39
123
|
get name() {
|
|
40
|
-
return this.
|
|
124
|
+
return this.bull.name;
|
|
41
125
|
}
|
|
42
126
|
get input() {
|
|
43
|
-
return this.
|
|
44
|
-
}
|
|
45
|
-
get status() {
|
|
46
|
-
return this.legacy.status;
|
|
47
|
-
}
|
|
48
|
-
get deduplicated() {
|
|
49
|
-
return this.legacy.deduplicated;
|
|
127
|
+
return this.bull.data;
|
|
50
128
|
}
|
|
51
129
|
get result() {
|
|
52
|
-
this.resultPromise ??= this.
|
|
130
|
+
this.resultPromise ??= this.store.result(this.bull);
|
|
53
131
|
return this.resultPromise;
|
|
54
132
|
}
|
|
55
|
-
|
|
56
|
-
return this.
|
|
133
|
+
cancel(reason) {
|
|
134
|
+
return this.store.cancel(this.bull, reason ?? "Job was cancelled");
|
|
57
135
|
}
|
|
58
136
|
async refresh() {
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
return
|
|
137
|
+
const snapshot = await this.store.snapshot(this.id);
|
|
138
|
+
if (!snapshot)
|
|
139
|
+
throw new Error(`Job "${this.id}" no longer exists`);
|
|
140
|
+
return snapshot;
|
|
63
141
|
}
|
|
64
142
|
}
|
|
65
|
-
|
|
66
|
-
options;
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
if (reservedNames.has(name)) {
|
|
78
|
-
throw new TypeError(`"${name}" is reserved by enqiu`);
|
|
79
|
-
}
|
|
80
|
-
const normalized = normalizeDefinition(definition);
|
|
81
|
-
this.definitions.set(name, normalized);
|
|
82
|
-
handlers[name] = async (input, context) => {
|
|
83
|
-
const output = await normalized.run(input, createContext(context, options.name ?? "default", options.telemetry));
|
|
84
|
-
return cloneJobValue(output);
|
|
85
|
-
};
|
|
86
|
-
}
|
|
87
|
-
if (this.definitions.size === 0) {
|
|
88
|
-
throw new TypeError("At least one job definition is required");
|
|
143
|
+
function createJobCallable(name, definition, runtime) {
|
|
144
|
+
const options = (submit) => toJobsOptions(submit, definition.policy, runtime.defaults);
|
|
145
|
+
const submit = async (input, submitOptions = {}) => {
|
|
146
|
+
await runtime.ready();
|
|
147
|
+
const value = assertJobValue(await validateInput(name, definition.schema, input));
|
|
148
|
+
// Ask before adding: on a hit BullMQ returns the *existing* job, so
|
|
149
|
+
// afterwards its id is the deduplication owner and the two are
|
|
150
|
+
// indistinguishable.
|
|
151
|
+
let deduplicated = false;
|
|
152
|
+
if (submitOptions.idempotencyKey !== undefined) {
|
|
153
|
+
const owner = await runtime.queue.getDeduplicationJobId(submitOptions.idempotencyKey);
|
|
154
|
+
deduplicated = owner !== undefined && owner !== null;
|
|
89
155
|
}
|
|
90
|
-
const
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
this.redis = new RedisQueue(handlers, compact({
|
|
97
|
-
driver: options.driver,
|
|
98
|
-
name: options.name,
|
|
99
|
-
worker: options.worker !== false,
|
|
100
|
-
concurrency,
|
|
101
|
-
autoStart,
|
|
102
|
-
retry: retry,
|
|
103
|
-
timeout: options.timeout,
|
|
104
|
-
historyLimit: options.historyLimit,
|
|
105
|
-
logLimit: options.logLimit,
|
|
106
|
-
}));
|
|
107
|
-
this.workerRunning = autoStart && options.worker !== false;
|
|
108
|
-
}
|
|
109
|
-
else {
|
|
110
|
-
this.memory = new MemoryQueue(handlers, compact({
|
|
111
|
-
name: options.name,
|
|
112
|
-
concurrency,
|
|
113
|
-
autoStart,
|
|
114
|
-
retry,
|
|
115
|
-
timeout: options.timeout,
|
|
116
|
-
historyLimit: options.historyLimit,
|
|
117
|
-
logLimit: options.logLimit,
|
|
118
|
-
}));
|
|
119
|
-
this.memoryScheduler = new MemoryScheduler();
|
|
120
|
-
this.workerRunning = autoStart;
|
|
121
|
-
}
|
|
122
|
-
const target = {};
|
|
123
|
-
for (const name of this.definitions.keys()) {
|
|
124
|
-
target[name] = this.createCallable(name);
|
|
125
|
-
}
|
|
126
|
-
target.queue = this.createQueueApi();
|
|
127
|
-
target.worker = this.createWorkerApi();
|
|
128
|
-
this.api = Object.freeze(target);
|
|
129
|
-
this.connectTelemetry();
|
|
130
|
-
}
|
|
131
|
-
get queue() {
|
|
132
|
-
return this.redis ?? this.memory;
|
|
133
|
-
}
|
|
134
|
-
createCallable(name) {
|
|
135
|
-
const definition = this.definitions.get(name);
|
|
136
|
-
if (!definition) {
|
|
137
|
-
throw new TypeError(`Unknown job "${name}"`);
|
|
156
|
+
const bull = await runtime.queue.add(name, value, options(submitOptions));
|
|
157
|
+
return new Handle(bull, runtime.store, deduplicated);
|
|
158
|
+
};
|
|
159
|
+
const bulk = async (inputs, bulkOptions = {}) => {
|
|
160
|
+
if (bulkOptions.ids && bulkOptions.ids.length !== inputs.length) {
|
|
161
|
+
throw new RangeError("bulk ids must match the number of inputs");
|
|
138
162
|
}
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
163
|
+
await runtime.ready();
|
|
164
|
+
const values = await Promise.all(inputs.map(async (input) => assertJobValue(await validateInput(name, definition.schema, input))));
|
|
165
|
+
// Only the id varies across the batch, so the rest is resolved once.
|
|
166
|
+
const shared = options(bulkOptions);
|
|
167
|
+
const created = await runtime.queue.addBulk(values.map((data, index) => {
|
|
168
|
+
const id = bulkOptions.ids?.[index];
|
|
169
|
+
return {
|
|
170
|
+
name,
|
|
171
|
+
data,
|
|
172
|
+
opts: id === undefined ? shared : { ...shared, jobId: id },
|
|
173
|
+
};
|
|
174
|
+
}));
|
|
175
|
+
return created.map((bull) => new Handle(bull, runtime.store, false));
|
|
176
|
+
};
|
|
177
|
+
const schedule = async (scheduleOptions) => {
|
|
178
|
+
runtime.assertOpen();
|
|
179
|
+
const value = assertJobValue(await validateInput(name, definition.schema, scheduleOptions.input));
|
|
180
|
+
const id = scheduleOptions.id?.trim() || name;
|
|
181
|
+
await runtime.queue.upsertJobScheduler(id, {
|
|
182
|
+
pattern: scheduleOptions.cron,
|
|
183
|
+
...(scheduleOptions.timezone === undefined
|
|
184
|
+
? {}
|
|
185
|
+
: { tz: scheduleOptions.timezone }),
|
|
186
|
+
}, { name, data: value });
|
|
187
|
+
return scheduleHandle(id, name, runtime.queue);
|
|
188
|
+
};
|
|
189
|
+
return Object.assign(submit, { bulk, schedule, input: definition.schema });
|
|
190
|
+
}
|
|
191
|
+
function scheduleHandle(id, jobName, queue) {
|
|
192
|
+
return {
|
|
193
|
+
id,
|
|
194
|
+
remove: async () => {
|
|
195
|
+
await queue.removeJobScheduler(id);
|
|
196
|
+
},
|
|
197
|
+
refresh: async () => {
|
|
198
|
+
const scheduler = await queue.getJobScheduler(id);
|
|
199
|
+
if (!scheduler)
|
|
200
|
+
throw new Error(`Schedule "${id}" does not exist`);
|
|
201
|
+
return {
|
|
202
|
+
id,
|
|
203
|
+
jobName,
|
|
204
|
+
cron: String(scheduler.pattern ?? ""),
|
|
205
|
+
timezone: String(scheduler.tz ?? "UTC"),
|
|
206
|
+
nextRunAt: Number(scheduler.next ?? 0),
|
|
207
|
+
input: scheduler.template?.data,
|
|
208
|
+
};
|
|
209
|
+
},
|
|
210
|
+
};
|
|
211
|
+
}
|
|
212
|
+
function createQueueApi(runtime) {
|
|
213
|
+
const { queue, markers, store, events } = runtime;
|
|
214
|
+
return Object.freeze({
|
|
215
|
+
get: async (id) => (await store.snapshot(id)),
|
|
216
|
+
list: async (query) => {
|
|
217
|
+
const limit = query.limit ?? 100;
|
|
218
|
+
if (!Number.isInteger(limit) || limit < 1 || limit > 1000) {
|
|
219
|
+
throw new RangeError("list.limit must be an integer between 1 and 1000");
|
|
144
220
|
}
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
221
|
+
const states = bullStates[query.status];
|
|
222
|
+
if (states.length === 0)
|
|
223
|
+
return { jobs: [] };
|
|
224
|
+
// One range per state, each with its own offset, because BullMQ applies
|
|
225
|
+
// a range to every state separately.
|
|
226
|
+
const offsets = decodeCursor(query.cursor, states.length);
|
|
227
|
+
const pages = await Promise.all(states.map(async (state, index) => {
|
|
228
|
+
const offset = offsets[index] ?? 0;
|
|
229
|
+
return {
|
|
230
|
+
offset,
|
|
231
|
+
items: await queue.getJobs([state], offset, offset + limit - 1),
|
|
232
|
+
};
|
|
233
|
+
}));
|
|
234
|
+
const { items, next } = mergePage(pages, limit);
|
|
235
|
+
const jobs = items.map((bull) => toSnapshot(bull, query.status));
|
|
236
|
+
return items.length === limit
|
|
237
|
+
? { jobs, cursor: encodeCursor(next) }
|
|
238
|
+
: { jobs };
|
|
239
|
+
},
|
|
240
|
+
stats: async () => {
|
|
241
|
+
const counts = await queue.getJobCounts(...everyState);
|
|
242
|
+
const sum = (status) => bullStates[status].reduce((n, state) => n + (counts[state] ?? 0), 0);
|
|
243
|
+
const stats = {
|
|
244
|
+
queued: sum("queued"),
|
|
245
|
+
scheduled: sum("scheduled"),
|
|
246
|
+
running: sum("running"),
|
|
247
|
+
succeeded: sum("succeeded"),
|
|
248
|
+
failed: sum("failed"),
|
|
249
|
+
};
|
|
250
|
+
return {
|
|
251
|
+
...stats,
|
|
252
|
+
total: Object.values(stats).reduce((a, b) => a + b, 0),
|
|
253
|
+
};
|
|
254
|
+
},
|
|
255
|
+
redrive: async (id) => {
|
|
256
|
+
const bull = await queue.getJob(id);
|
|
257
|
+
if (!bull)
|
|
258
|
+
throw new Error(`Job "${id}" cannot be redriven`);
|
|
259
|
+
const state = await bull.getState();
|
|
260
|
+
if (!isFinished(state))
|
|
261
|
+
throw new Error(`Job "${id}" cannot be redriven`);
|
|
262
|
+
await bull.retry(state);
|
|
263
|
+
return new Handle(bull, store, false);
|
|
264
|
+
},
|
|
265
|
+
cleanup: async (query = {}) => {
|
|
266
|
+
const olderThan = query.olderThan ?? 0;
|
|
267
|
+
if (!Number.isFinite(olderThan) || olderThan < 0) {
|
|
268
|
+
throw new RangeError("olderThan must be a non-negative finite number");
|
|
150
269
|
}
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
return this.redis.upsertSchedule({
|
|
168
|
-
...options,
|
|
169
|
-
input: value,
|
|
170
|
-
jobName: name,
|
|
171
|
-
submit: toLegacyOptions({}, definition.policy, name, value),
|
|
172
|
-
});
|
|
270
|
+
// Every state the status is made of: cleaning only the first left
|
|
271
|
+
// prioritized jobs behind and reported success.
|
|
272
|
+
const limit = query.limit ?? 1000;
|
|
273
|
+
const removed = await Promise.all(bullStates[query.status ?? "succeeded"].map((state) => queue.clean(olderThan, limit, state)));
|
|
274
|
+
await markers.prune(Date.now() - olderThan);
|
|
275
|
+
return removed.flat();
|
|
276
|
+
},
|
|
277
|
+
onIdle: async () => {
|
|
278
|
+
// BullMQ has no idle signal, so poll the counts it already maintains
|
|
279
|
+
// rather than tracking the same state in parallel. The wait widens so
|
|
280
|
+
// that draining a long queue does not sit at 50 polls a second.
|
|
281
|
+
for (let wait = 20;; wait = Math.min(wait * 2, 250)) {
|
|
282
|
+
const outstanding = await queue.getJobCountByTypes("waiting", "active", "delayed", "prioritized");
|
|
283
|
+
if (outstanding === 0)
|
|
284
|
+
return;
|
|
285
|
+
await new Promise((resolve) => setTimeout(resolve, wait));
|
|
173
286
|
}
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
287
|
+
},
|
|
288
|
+
on: (event, listener) => {
|
|
289
|
+
const { name, state } = queueEventMap[event];
|
|
290
|
+
const deliver = listener;
|
|
291
|
+
// `error` carries an Error rather than a job, and which one this is was
|
|
292
|
+
// settled when on() was called — not on every delivery.
|
|
293
|
+
const handler = event === "error"
|
|
294
|
+
? deliver
|
|
295
|
+
: ({ jobId }) => {
|
|
296
|
+
// The event already says what state the job is in, so only the
|
|
297
|
+
// job itself has to be fetched.
|
|
298
|
+
void store.snapshot(jobId, state).then((snapshot) => {
|
|
299
|
+
if (snapshot)
|
|
300
|
+
deliver(snapshot);
|
|
182
301
|
});
|
|
183
|
-
void handle;
|
|
184
|
-
},
|
|
185
|
-
});
|
|
186
|
-
};
|
|
187
|
-
Object.defineProperties(callable, {
|
|
188
|
-
bulk: { value: bulk, enumerable: true },
|
|
189
|
-
schedule: { value: schedule, enumerable: true },
|
|
190
|
-
input: { value: definition.schema, enumerable: true },
|
|
191
|
-
});
|
|
192
|
-
return callable;
|
|
193
|
-
}
|
|
194
|
-
createQueueApi() {
|
|
195
|
-
return Object.freeze({
|
|
196
|
-
get: async (id) => (await this.queue.get(id)),
|
|
197
|
-
list: async (query = {}) => {
|
|
198
|
-
if (this.redis) {
|
|
199
|
-
return this.redis.list(query);
|
|
200
|
-
}
|
|
201
|
-
let jobs = this.memory?.list(query.status) ?? [];
|
|
202
|
-
if (query.name) {
|
|
203
|
-
jobs = jobs.filter((item) => item.name === query.name);
|
|
204
|
-
}
|
|
205
|
-
if (query.after !== undefined) {
|
|
206
|
-
jobs = jobs.filter((item) => item.createdAt > query.after);
|
|
207
|
-
}
|
|
208
|
-
if (query.before !== undefined) {
|
|
209
|
-
jobs = jobs.filter((item) => item.createdAt < query.before);
|
|
210
|
-
}
|
|
211
|
-
const limit = query.limit ?? 100;
|
|
212
|
-
return {
|
|
213
|
-
jobs: jobs.slice(0, limit),
|
|
214
302
|
};
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
else {
|
|
222
|
-
this.memory?.pause();
|
|
223
|
-
}
|
|
224
|
-
},
|
|
225
|
-
resume: async () => {
|
|
226
|
-
if (this.redis) {
|
|
227
|
-
await this.redis.resumeQueue();
|
|
228
|
-
}
|
|
229
|
-
else {
|
|
230
|
-
this.memory?.start();
|
|
231
|
-
}
|
|
232
|
-
},
|
|
233
|
-
setConcurrency: async (limit) => {
|
|
234
|
-
if (this.redis) {
|
|
235
|
-
await this.redis.setGlobalConcurrency(limit);
|
|
303
|
+
// Attaching has to wait for the stream to open, so unsubscribing before
|
|
304
|
+
// that has to be remembered rather than applied.
|
|
305
|
+
let detach;
|
|
306
|
+
let unsubscribed = false;
|
|
307
|
+
void events.open().then((stream) => {
|
|
308
|
+
if (unsubscribed)
|
|
236
309
|
return;
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
},
|
|
240
|
-
redrive: async (id) => {
|
|
241
|
-
if (this.redis) {
|
|
242
|
-
return new PublicJobHandle((await this.redis.redrive(id)));
|
|
243
|
-
}
|
|
244
|
-
const handle = this.memory?.retry(id);
|
|
245
|
-
if (!handle) {
|
|
246
|
-
throw new Error(`Job "${id}" cannot be redriven`);
|
|
247
|
-
}
|
|
248
|
-
return new PublicJobHandle(handle);
|
|
249
|
-
},
|
|
250
|
-
cleanup: async (query = {}) => {
|
|
251
|
-
if (this.redis) {
|
|
252
|
-
return this.redis.cleanup(query);
|
|
253
|
-
}
|
|
254
|
-
return (this.memory?.cleanup(compact({
|
|
255
|
-
olderThan: query.olderThan,
|
|
256
|
-
limit: query.limit,
|
|
257
|
-
})) ?? []);
|
|
258
|
-
},
|
|
259
|
-
on: (event, listener) => {
|
|
260
|
-
const events = this.queue;
|
|
261
|
-
return events.on(event, listener);
|
|
262
|
-
},
|
|
263
|
-
});
|
|
264
|
-
}
|
|
265
|
-
createWorkerApi() {
|
|
266
|
-
const facade = this;
|
|
267
|
-
return Object.freeze({
|
|
268
|
-
get running() {
|
|
269
|
-
return facade.workerRunning;
|
|
270
|
-
},
|
|
271
|
-
start: async (options = {}) => {
|
|
272
|
-
if (options.concurrency !== undefined) {
|
|
273
|
-
if (facade.redis) {
|
|
274
|
-
facade.redis.setWorkerConcurrency(options.concurrency);
|
|
275
|
-
}
|
|
276
|
-
else {
|
|
277
|
-
facade.memory.concurrency =
|
|
278
|
-
options.concurrency;
|
|
279
|
-
}
|
|
280
|
-
}
|
|
281
|
-
facade.queue.start();
|
|
282
|
-
facade.workerRunning = true;
|
|
283
|
-
},
|
|
284
|
-
pause: async () => {
|
|
285
|
-
facade.queue.pause();
|
|
286
|
-
facade.workerRunning = false;
|
|
287
|
-
},
|
|
288
|
-
resume: async () => {
|
|
289
|
-
facade.queue.start();
|
|
290
|
-
facade.workerRunning = true;
|
|
291
|
-
},
|
|
292
|
-
onIdle: async () => facade.queue.onIdle(),
|
|
293
|
-
close: async (options) => {
|
|
294
|
-
await facade.queue.close(options);
|
|
295
|
-
facade.memoryScheduler?.close();
|
|
296
|
-
facade.workerRunning = false;
|
|
297
|
-
},
|
|
298
|
-
});
|
|
299
|
-
}
|
|
300
|
-
connectTelemetry() {
|
|
301
|
-
const telemetry = this.options.telemetry;
|
|
302
|
-
if (!telemetry) {
|
|
303
|
-
return;
|
|
304
|
-
}
|
|
305
|
-
const events = [
|
|
306
|
-
"added",
|
|
307
|
-
"started",
|
|
308
|
-
"retry",
|
|
309
|
-
"succeeded",
|
|
310
|
-
"failed",
|
|
311
|
-
"cancelled",
|
|
312
|
-
"expired",
|
|
313
|
-
];
|
|
314
|
-
const source = this.queue;
|
|
315
|
-
for (const event of events) {
|
|
316
|
-
source.on(event, (payload) => {
|
|
317
|
-
const snapshot = "job" in Object(payload)
|
|
318
|
-
? payload.job
|
|
319
|
-
: payload;
|
|
320
|
-
telemetry.emit({
|
|
321
|
-
type: `job.${event}`,
|
|
322
|
-
queue: this.options.name ?? "default",
|
|
323
|
-
timestamp: Date.now(),
|
|
324
|
-
job: snapshot,
|
|
325
|
-
});
|
|
310
|
+
stream.on(name, handler);
|
|
311
|
+
detach = () => stream.off(name, handler);
|
|
326
312
|
});
|
|
327
|
-
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
331
|
-
|
|
332
|
-
}
|
|
333
|
-
return this.memory.add(name, value, options);
|
|
334
|
-
}
|
|
335
|
-
}
|
|
336
|
-
export function enqiu(definitions, options = {}) {
|
|
337
|
-
return new EnqiuFacade(definitions, options).api;
|
|
338
|
-
}
|
|
339
|
-
function normalizeDefinition(definition) {
|
|
340
|
-
if (typeof definition === "function") {
|
|
341
|
-
return {
|
|
342
|
-
schema: undefined,
|
|
343
|
-
run: definition,
|
|
344
|
-
policy: {},
|
|
345
|
-
};
|
|
346
|
-
}
|
|
347
|
-
if (!definition ||
|
|
348
|
-
typeof definition !== "object" ||
|
|
349
|
-
definition[definitionMarker] !== true) {
|
|
350
|
-
throw new TypeError("Every job must be a handler or a definition created with job()");
|
|
351
|
-
}
|
|
352
|
-
const { input, run, retry, timeout, expiresIn, concurrency, throttle, debounce, } = definition;
|
|
353
|
-
return {
|
|
354
|
-
schema: input,
|
|
355
|
-
run: run,
|
|
356
|
-
policy: compact({
|
|
357
|
-
retry,
|
|
358
|
-
timeout,
|
|
359
|
-
expiresIn,
|
|
360
|
-
concurrency,
|
|
361
|
-
throttle,
|
|
362
|
-
debounce,
|
|
363
|
-
}),
|
|
364
|
-
};
|
|
365
|
-
}
|
|
366
|
-
async function validateInput(name, schema, input) {
|
|
367
|
-
if (!schema) {
|
|
368
|
-
return input;
|
|
369
|
-
}
|
|
370
|
-
const result = await schema["~standard"].validate(input);
|
|
371
|
-
if (result.issues) {
|
|
372
|
-
throw new JobValidationError(name, result.issues);
|
|
373
|
-
}
|
|
374
|
-
return result.value;
|
|
375
|
-
}
|
|
376
|
-
function isStandardSchema(value) {
|
|
377
|
-
if (!value || typeof value !== "object") {
|
|
378
|
-
return false;
|
|
379
|
-
}
|
|
380
|
-
const standard = value["~standard"];
|
|
381
|
-
return (standard?.version === 1 &&
|
|
382
|
-
typeof standard.vendor === "string" &&
|
|
383
|
-
typeof standard.validate === "function");
|
|
384
|
-
}
|
|
385
|
-
function normalizeLegacyRetry(value) {
|
|
386
|
-
if (typeof value === "number" || value === undefined) {
|
|
387
|
-
return value;
|
|
388
|
-
}
|
|
389
|
-
if (!Number.isInteger(value.attempts) || value.attempts < 1) {
|
|
390
|
-
throw new RangeError("retry.attempts must be a positive integer");
|
|
391
|
-
}
|
|
392
|
-
return compact({
|
|
393
|
-
retries: value.attempts - 1,
|
|
394
|
-
backoff: value.backoff,
|
|
395
|
-
when: value.when,
|
|
313
|
+
return () => {
|
|
314
|
+
unsubscribed = true;
|
|
315
|
+
detach?.();
|
|
316
|
+
};
|
|
317
|
+
},
|
|
396
318
|
});
|
|
397
319
|
}
|
|
398
|
-
function
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
320
|
+
function createWorkerApi(worker) {
|
|
321
|
+
return Object.freeze({
|
|
322
|
+
/**
|
|
323
|
+
* Read from BullMQ rather than mirrored here. A caller holding
|
|
324
|
+
* `bull.worker` can pause it, and a flag maintained alongside would go on
|
|
325
|
+
* claiming otherwise.
|
|
326
|
+
*/
|
|
327
|
+
get running() {
|
|
328
|
+
return worker !== undefined && worker.isRunning() && !worker.isPaused();
|
|
329
|
+
},
|
|
330
|
+
start: async (options = {}) => {
|
|
331
|
+
if (!worker) {
|
|
332
|
+
throw new TypeError("This queue was created with worker: false and cannot run jobs");
|
|
408
333
|
}
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
const debounce = policy.debounce
|
|
422
|
-
? {
|
|
423
|
-
wait: policy.debounce.wait,
|
|
424
|
-
mode: policy.debounce.mode,
|
|
425
|
-
key: resolvePolicyKey("debounce.by", policy.debounce.by(input)),
|
|
426
|
-
}
|
|
427
|
-
: undefined;
|
|
428
|
-
return compact({
|
|
429
|
-
id: options.id,
|
|
430
|
-
key: options.idempotencyKey,
|
|
431
|
-
keyRetention: options.idempotencyKey
|
|
432
|
-
? options.idempotencyTtl ?? 24 * 60 * 60 * 1000
|
|
433
|
-
: undefined,
|
|
434
|
-
delay: options.delay,
|
|
435
|
-
priority,
|
|
436
|
-
retry: normalizeLegacyRetry(options.retry ?? policy.retry),
|
|
437
|
-
timeout: options.timeout ?? policy.timeout,
|
|
438
|
-
expiresIn: options.expiresIn ?? policy.expiresIn,
|
|
439
|
-
concurrency,
|
|
440
|
-
throttle,
|
|
441
|
-
debounce,
|
|
442
|
-
signal: options.signal,
|
|
334
|
+
if (options.concurrency !== undefined) {
|
|
335
|
+
worker.concurrency = options.concurrency;
|
|
336
|
+
}
|
|
337
|
+
// Awaited on purpose. BullMQ's resume() restarts the main loop only if it
|
|
338
|
+
// has *already* exited, and right after a pause the loop is still
|
|
339
|
+
// unwinding — so checking isRunning() before that settles saw a live loop
|
|
340
|
+
// about to die, and left the worker resumed but not running.
|
|
341
|
+
if (worker.isPaused())
|
|
342
|
+
await worker.resume();
|
|
343
|
+
if (!worker.isRunning())
|
|
344
|
+
void worker.run();
|
|
345
|
+
},
|
|
443
346
|
});
|
|
444
347
|
}
|
|
445
|
-
|
|
446
|
-
|
|
447
|
-
|
|
348
|
+
/** Build a typed job API backed by a BullMQ queue. */
|
|
349
|
+
export function enqiu(definitions, options) {
|
|
350
|
+
const runtimeDefinitions = new Map();
|
|
351
|
+
for (const [name, definition] of Object.entries(definitions)) {
|
|
352
|
+
runtimeDefinitions.set(name, normalizeDefinition(definition));
|
|
448
353
|
}
|
|
449
|
-
|
|
450
|
-
|
|
451
|
-
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
validateProgress(progress);
|
|
460
|
-
const safeProgress = cloneJobValue(progress);
|
|
461
|
-
legacy.progress(safeProgress);
|
|
462
|
-
telemetry?.emit({
|
|
463
|
-
type: "job.progress",
|
|
464
|
-
queue,
|
|
465
|
-
timestamp: Date.now(),
|
|
466
|
-
fields: {
|
|
467
|
-
jobId: legacy.id,
|
|
468
|
-
jobName: legacy.name,
|
|
469
|
-
progress: safeProgress,
|
|
470
|
-
},
|
|
471
|
-
});
|
|
472
|
-
},
|
|
473
|
-
log,
|
|
354
|
+
if (runtimeDefinitions.size === 0) {
|
|
355
|
+
throw new TypeError("At least one job definition is required");
|
|
356
|
+
}
|
|
357
|
+
if (!options.connection) {
|
|
358
|
+
throw new TypeError("enqiu() requires a BullMQ connection");
|
|
359
|
+
}
|
|
360
|
+
const queueName = options.name ?? "default";
|
|
361
|
+
const base = {
|
|
362
|
+
connection: options.connection,
|
|
363
|
+
...(options.prefix === undefined ? {} : { prefix: options.prefix }),
|
|
474
364
|
};
|
|
475
|
-
|
|
476
|
-
|
|
477
|
-
|
|
478
|
-
|
|
479
|
-
|
|
480
|
-
|
|
481
|
-
|
|
482
|
-
|
|
483
|
-
|
|
484
|
-
|
|
485
|
-
...
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
telemetry?.emit({
|
|
489
|
-
type: `job.log.${level}`,
|
|
490
|
-
queue,
|
|
491
|
-
timestamp: Date.now(),
|
|
492
|
-
fields: {
|
|
493
|
-
jobId: context.id,
|
|
494
|
-
jobName: context.name,
|
|
495
|
-
message: entry.message,
|
|
496
|
-
...(entry.fields ?? {}),
|
|
497
|
-
},
|
|
365
|
+
const queue = new Queue(queueName, base);
|
|
366
|
+
let worker;
|
|
367
|
+
if (options.worker !== false) {
|
|
368
|
+
const { concurrency, autoStart = true } = options.worker ?? {};
|
|
369
|
+
const runner = new JobRunner(runtimeDefinitions, { timeout: options.timeout });
|
|
370
|
+
worker = new Worker(queueName,
|
|
371
|
+
// Three parameters on purpose: BullMQ decides whether to create an
|
|
372
|
+
// AbortController by reading `processor.length >= 3`. Declaring fewer
|
|
373
|
+
// means worker.cancelJob() can never abort anything.
|
|
374
|
+
async (bull, _token, signal) => runner.run(bull, signal), {
|
|
375
|
+
...base,
|
|
376
|
+
...(concurrency === undefined ? {} : { concurrency }),
|
|
377
|
+
autorun: false,
|
|
498
378
|
});
|
|
379
|
+
if (autoStart)
|
|
380
|
+
void worker.run();
|
|
381
|
+
}
|
|
382
|
+
const events = new QueueEventStream(queue, base);
|
|
383
|
+
const markers = new CancellationMarkers(queue);
|
|
384
|
+
const store = new JobStore(queue, events, markers, worker);
|
|
385
|
+
const runtime = {
|
|
386
|
+
queue,
|
|
387
|
+
events,
|
|
388
|
+
markers,
|
|
389
|
+
store,
|
|
390
|
+
defaults: {
|
|
391
|
+
logLimit: options.logLimit ?? 100,
|
|
392
|
+
retry: options.retry,
|
|
393
|
+
},
|
|
394
|
+
// Read from BullMQ, so closing `bull.queue` underneath is seen here too.
|
|
395
|
+
assertOpen: () => {
|
|
396
|
+
if (queue.closing !== undefined)
|
|
397
|
+
throw new QueueClosedError(queueName);
|
|
398
|
+
},
|
|
399
|
+
ready: async () => {
|
|
400
|
+
runtime.assertOpen();
|
|
401
|
+
await events.settle();
|
|
402
|
+
},
|
|
499
403
|
};
|
|
404
|
+
const jobs = {};
|
|
405
|
+
for (const [name, definition] of runtimeDefinitions) {
|
|
406
|
+
jobs[name] = createJobCallable(name, definition, runtime);
|
|
407
|
+
}
|
|
408
|
+
const queueApi = createQueueApi(runtime);
|
|
409
|
+
const workerApi = createWorkerApi(worker);
|
|
500
410
|
return Object.freeze({
|
|
501
|
-
|
|
502
|
-
|
|
503
|
-
|
|
504
|
-
|
|
411
|
+
jobs: Object.freeze(jobs),
|
|
412
|
+
queue: queueApi,
|
|
413
|
+
worker: workerApi,
|
|
414
|
+
bull: Object.freeze({ queue, worker }),
|
|
415
|
+
close: async (closeOptions) => {
|
|
416
|
+
if ((closeOptions?.drain ?? true) && workerApi.running) {
|
|
417
|
+
await queueApi.onIdle();
|
|
418
|
+
}
|
|
419
|
+
// Three independent connections: closing the worker waits out the jobs
|
|
420
|
+
// it holds, which the other two have no reason to wait for.
|
|
421
|
+
await Promise.all([worker?.close(), events.close(), queue.close()]);
|
|
422
|
+
},
|
|
505
423
|
});
|
|
506
424
|
}
|
|
507
|
-
function validateProgress(progress) {
|
|
508
|
-
if (!Number.isFinite(progress.completed) ||
|
|
509
|
-
!Number.isFinite(progress.total) ||
|
|
510
|
-
progress.completed < 0 ||
|
|
511
|
-
progress.total <= 0 ||
|
|
512
|
-
progress.completed > progress.total) {
|
|
513
|
-
throw new RangeError("Progress requires 0 <= completed <= total and total > 0");
|
|
514
|
-
}
|
|
515
|
-
}
|
|
516
|
-
function compact(value) {
|
|
517
|
-
return Object.fromEntries(Object.entries(value).filter(([, entry]) => entry !== undefined));
|
|
518
|
-
}
|
|
519
|
-
export { JobCancelledError, JobExpiredError, JobFailedError, JobSerializationError, JobTimeoutError, QueueClosedError, };
|