@cinnabun/queue 0.0.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/adapters/memory.adapter.d.ts +22 -0
- package/dist/adapters/memory.adapter.js +132 -0
- package/dist/adapters/redis.adapter.d.ts +27 -0
- package/dist/adapters/redis.adapter.js +140 -0
- package/dist/cli/register-queue-commands.d.ts +2 -0
- package/dist/cli/register-queue-commands.js +44 -0
- package/dist/decorators/job.d.ts +1 -0
- package/dist/decorators/job.js +6 -0
- package/dist/decorators/processor.d.ts +1 -0
- package/dist/decorators/processor.js +6 -0
- package/dist/index.d.ts +15 -0
- package/dist/index.js +11 -0
- package/dist/interfaces/job-options.d.ts +6 -0
- package/dist/interfaces/job-options.js +1 -0
- package/dist/interfaces/job.d.ts +14 -0
- package/dist/interfaces/job.js +1 -0
- package/dist/interfaces/queue-adapter.d.ts +12 -0
- package/dist/interfaces/queue-adapter.js +1 -0
- package/dist/interfaces/queue-options.d.ts +11 -0
- package/dist/interfaces/queue-options.js +1 -0
- package/dist/metadata/queue-storage.d.ts +21 -0
- package/dist/metadata/queue-storage.js +24 -0
- package/dist/queue-service-holder.d.ts +3 -0
- package/dist/queue-service-holder.js +12 -0
- package/dist/queue.module.d.ts +7 -0
- package/dist/queue.module.js +43 -0
- package/dist/queue.plugin.d.ts +9 -0
- package/dist/queue.plugin.js +41 -0
- package/dist/services/queue.service.d.ts +12 -0
- package/dist/services/queue.service.js +31 -0
- package/dist/services/worker.service.d.ts +13 -0
- package/dist/services/worker.service.js +52 -0
- package/dist/worker-bootstrap.d.ts +6 -0
- package/dist/worker-bootstrap.js +79 -0
- package/package.json +37 -0
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
import type { Job } from "../interfaces/job.js";
|
|
2
|
+
import type { JobOptions } from "../interfaces/job-options.js";
|
|
3
|
+
import type { QueueAdapter } from "../interfaces/queue-adapter.js";
|
|
4
|
+
export declare class MemoryAdapter implements QueueAdapter {
|
|
5
|
+
private connected;
|
|
6
|
+
private processing;
|
|
7
|
+
private readonly queues;
|
|
8
|
+
private readonly handlers;
|
|
9
|
+
private readonly defaultOptions;
|
|
10
|
+
private processInterval;
|
|
11
|
+
constructor(defaultOptions?: JobOptions);
|
|
12
|
+
connect(): Promise<void>;
|
|
13
|
+
disconnect(): Promise<void>;
|
|
14
|
+
enqueue<T>(queueName: string, jobName: string, data: T, options?: JobOptions): Promise<Job<T>>;
|
|
15
|
+
process(queueName: string, jobName: string, handler: (job: Job) => Promise<void>): void;
|
|
16
|
+
getJob(queueName: string, id: string): Promise<Job | null>;
|
|
17
|
+
removeJob(queueName: string, id: string): Promise<void>;
|
|
18
|
+
startProcessing(): Promise<void>;
|
|
19
|
+
stopProcessing(): Promise<void>;
|
|
20
|
+
private sortQueue;
|
|
21
|
+
private processNext;
|
|
22
|
+
}
|
|
@@ -0,0 +1,132 @@
|
|
|
1
|
+
export class MemoryAdapter {
|
|
2
|
+
connected = false;
|
|
3
|
+
processing = false;
|
|
4
|
+
queues = new Map();
|
|
5
|
+
handlers = new Map();
|
|
6
|
+
defaultOptions;
|
|
7
|
+
processInterval = null;
|
|
8
|
+
constructor(defaultOptions = {}) {
|
|
9
|
+
this.defaultOptions = {
|
|
10
|
+
attempts: 3,
|
|
11
|
+
backoff: 1000,
|
|
12
|
+
...defaultOptions,
|
|
13
|
+
};
|
|
14
|
+
}
|
|
15
|
+
async connect() {
|
|
16
|
+
this.connected = true;
|
|
17
|
+
}
|
|
18
|
+
async disconnect() {
|
|
19
|
+
await this.stopProcessing();
|
|
20
|
+
this.connected = false;
|
|
21
|
+
}
|
|
22
|
+
async enqueue(queueName, jobName, data, options) {
|
|
23
|
+
if (!this.connected) {
|
|
24
|
+
throw new Error("Adapter not connected");
|
|
25
|
+
}
|
|
26
|
+
const opts = { ...this.defaultOptions, ...options };
|
|
27
|
+
const maxAttempts = opts.attempts ?? 3;
|
|
28
|
+
const delay = opts.delay ?? 0;
|
|
29
|
+
const priority = opts.priority ?? 0;
|
|
30
|
+
const job = {
|
|
31
|
+
id: `mem-${Date.now()}-${Math.random().toString(36).slice(2, 11)}`,
|
|
32
|
+
name: jobName,
|
|
33
|
+
data,
|
|
34
|
+
attempts: 0,
|
|
35
|
+
maxAttempts,
|
|
36
|
+
priority,
|
|
37
|
+
delay,
|
|
38
|
+
createdAt: new Date(),
|
|
39
|
+
};
|
|
40
|
+
const runAt = Date.now() + delay;
|
|
41
|
+
const queued = { job, runAt };
|
|
42
|
+
if (!this.queues.has(queueName)) {
|
|
43
|
+
this.queues.set(queueName, []);
|
|
44
|
+
}
|
|
45
|
+
this.queues.get(queueName).push(queued);
|
|
46
|
+
this.sortQueue(queueName);
|
|
47
|
+
return job;
|
|
48
|
+
}
|
|
49
|
+
process(queueName, jobName, handler) {
|
|
50
|
+
const key = `${queueName}:${jobName}`;
|
|
51
|
+
this.handlers.set(key, handler);
|
|
52
|
+
}
|
|
53
|
+
async getJob(queueName, id) {
|
|
54
|
+
const queue = this.queues.get(queueName);
|
|
55
|
+
if (!queue)
|
|
56
|
+
return null;
|
|
57
|
+
const queued = queue.find((q) => q.job.id === id);
|
|
58
|
+
return queued ? queued.job : null;
|
|
59
|
+
}
|
|
60
|
+
async removeJob(queueName, id) {
|
|
61
|
+
const queue = this.queues.get(queueName);
|
|
62
|
+
if (!queue)
|
|
63
|
+
return;
|
|
64
|
+
const idx = queue.findIndex((q) => q.job.id === id);
|
|
65
|
+
if (idx >= 0) {
|
|
66
|
+
queue.splice(idx, 1);
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
async startProcessing() {
|
|
70
|
+
if (this.processing)
|
|
71
|
+
return;
|
|
72
|
+
this.processing = true;
|
|
73
|
+
this.processInterval = setInterval(() => {
|
|
74
|
+
this.processNext().catch(() => { });
|
|
75
|
+
}, 50);
|
|
76
|
+
}
|
|
77
|
+
async stopProcessing() {
|
|
78
|
+
this.processing = false;
|
|
79
|
+
if (this.processInterval) {
|
|
80
|
+
clearInterval(this.processInterval);
|
|
81
|
+
this.processInterval = null;
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
sortQueue(queueName) {
|
|
85
|
+
const queue = this.queues.get(queueName);
|
|
86
|
+
if (!queue)
|
|
87
|
+
return;
|
|
88
|
+
queue.sort((a, b) => {
|
|
89
|
+
if (a.runAt !== b.runAt)
|
|
90
|
+
return a.runAt - b.runAt;
|
|
91
|
+
return (b.job.priority ?? 0) - (a.job.priority ?? 0);
|
|
92
|
+
});
|
|
93
|
+
}
|
|
94
|
+
async processNext() {
|
|
95
|
+
const now = Date.now();
|
|
96
|
+
for (const [queueName, queue] of this.queues) {
|
|
97
|
+
if (queue.length === 0)
|
|
98
|
+
continue;
|
|
99
|
+
const next = queue[0];
|
|
100
|
+
if (next.runAt > now)
|
|
101
|
+
continue;
|
|
102
|
+
queue.shift();
|
|
103
|
+
const job = next.job;
|
|
104
|
+
const handlerKey = `${queueName}:${job.name}`;
|
|
105
|
+
const handler = this.handlers.get(handlerKey);
|
|
106
|
+
if (!handler) {
|
|
107
|
+
continue;
|
|
108
|
+
}
|
|
109
|
+
try {
|
|
110
|
+
job.processedAt = new Date();
|
|
111
|
+
job.attempts++;
|
|
112
|
+
await handler(job);
|
|
113
|
+
job.completedAt = new Date();
|
|
114
|
+
}
|
|
115
|
+
catch (err) {
|
|
116
|
+
const error = err instanceof Error ? err.message : String(err);
|
|
117
|
+
job.error = error;
|
|
118
|
+
job.failedAt = new Date();
|
|
119
|
+
const maxAttempts = job.maxAttempts ?? 3;
|
|
120
|
+
if (job.attempts < maxAttempts) {
|
|
121
|
+
const backoff = this.defaultOptions.backoff ?? 1000;
|
|
122
|
+
const retry = {
|
|
123
|
+
job,
|
|
124
|
+
runAt: now + backoff,
|
|
125
|
+
};
|
|
126
|
+
queue.push(retry);
|
|
127
|
+
this.sortQueue(queueName);
|
|
128
|
+
}
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
}
|
|
132
|
+
}
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
import type { Job } from "../interfaces/job.js";
|
|
2
|
+
import type { JobOptions } from "../interfaces/job-options.js";
|
|
3
|
+
import type { QueueAdapter } from "../interfaces/queue-adapter.js";
|
|
4
|
+
export declare class RedisAdapter implements QueueAdapter {
|
|
5
|
+
private readonly redisConfig;
|
|
6
|
+
private readonly defaultOptions;
|
|
7
|
+
private readonly bullQueues;
|
|
8
|
+
private readonly bullWorkers;
|
|
9
|
+
private connected;
|
|
10
|
+
constructor(redisConfig: {
|
|
11
|
+
host: string;
|
|
12
|
+
port: number;
|
|
13
|
+
password?: string;
|
|
14
|
+
db?: number;
|
|
15
|
+
}, defaultOptions?: JobOptions);
|
|
16
|
+
connect(): Promise<void>;
|
|
17
|
+
disconnect(): Promise<void>;
|
|
18
|
+
private getConnection;
|
|
19
|
+
private getQueue;
|
|
20
|
+
enqueue<T>(queueName: string, jobName: string, data: T, options?: JobOptions): Promise<Job<T>>;
|
|
21
|
+
process(queueName: string, jobName: string, handler: (job: Job) => Promise<void>): void;
|
|
22
|
+
getJob(queueName: string, id: string): Promise<Job | null>;
|
|
23
|
+
removeJob(queueName: string, id: string): Promise<void>;
|
|
24
|
+
startProcessing(): Promise<void>;
|
|
25
|
+
stopProcessing(): Promise<void>;
|
|
26
|
+
private bullJobToJob;
|
|
27
|
+
}
|
|
@@ -0,0 +1,140 @@
|
|
|
1
|
+
export class RedisAdapter {
|
|
2
|
+
redisConfig;
|
|
3
|
+
defaultOptions;
|
|
4
|
+
bullQueues = new Map();
|
|
5
|
+
bullWorkers = new Map();
|
|
6
|
+
connected = false;
|
|
7
|
+
constructor(redisConfig, defaultOptions = {}) {
|
|
8
|
+
this.redisConfig = redisConfig;
|
|
9
|
+
this.defaultOptions = {
|
|
10
|
+
attempts: 3,
|
|
11
|
+
backoff: 1000,
|
|
12
|
+
...defaultOptions,
|
|
13
|
+
};
|
|
14
|
+
}
|
|
15
|
+
async connect() {
|
|
16
|
+
this.connected = true;
|
|
17
|
+
}
|
|
18
|
+
async disconnect() {
|
|
19
|
+
for (const worker of this.bullWorkers.values()) {
|
|
20
|
+
await worker.close();
|
|
21
|
+
}
|
|
22
|
+
this.bullWorkers.clear();
|
|
23
|
+
for (const queue of this.bullQueues.values()) {
|
|
24
|
+
await queue.close();
|
|
25
|
+
}
|
|
26
|
+
this.bullQueues.clear();
|
|
27
|
+
this.connected = false;
|
|
28
|
+
}
|
|
29
|
+
getConnection() {
|
|
30
|
+
return {
|
|
31
|
+
host: this.redisConfig.host,
|
|
32
|
+
port: this.redisConfig.port,
|
|
33
|
+
password: this.redisConfig.password,
|
|
34
|
+
db: this.redisConfig.db,
|
|
35
|
+
};
|
|
36
|
+
}
|
|
37
|
+
getQueue(queueName) {
|
|
38
|
+
let queue = this.bullQueues.get(queueName);
|
|
39
|
+
if (!queue) {
|
|
40
|
+
const { Queue } = require("bullmq");
|
|
41
|
+
queue = new Queue(queueName, {
|
|
42
|
+
connection: this.getConnection(),
|
|
43
|
+
defaultJobOptions: {
|
|
44
|
+
attempts: this.defaultOptions.attempts ?? 3,
|
|
45
|
+
backoff: {
|
|
46
|
+
type: "fixed",
|
|
47
|
+
delay: this.defaultOptions.backoff ?? 1000,
|
|
48
|
+
},
|
|
49
|
+
removeOnComplete: false,
|
|
50
|
+
removeOnFail: false,
|
|
51
|
+
},
|
|
52
|
+
});
|
|
53
|
+
this.bullQueues.set(queueName, queue);
|
|
54
|
+
}
|
|
55
|
+
return queue;
|
|
56
|
+
}
|
|
57
|
+
async enqueue(queueName, jobName, data, options) {
|
|
58
|
+
if (!this.connected) {
|
|
59
|
+
throw new Error("Adapter not connected");
|
|
60
|
+
}
|
|
61
|
+
const queue = this.getQueue(queueName);
|
|
62
|
+
const opts = {};
|
|
63
|
+
if (options?.priority != null)
|
|
64
|
+
opts.priority = options.priority;
|
|
65
|
+
if (options?.delay != null)
|
|
66
|
+
opts.delay = options.delay;
|
|
67
|
+
if (options?.attempts != null)
|
|
68
|
+
opts.attempts = options.attempts;
|
|
69
|
+
if (options?.backoff != null) {
|
|
70
|
+
opts.backoff = { type: "fixed", delay: options.backoff };
|
|
71
|
+
}
|
|
72
|
+
const bullJob = await queue.add(jobName, data, opts);
|
|
73
|
+
return this.bullJobToJob(bullJob, jobName, data);
|
|
74
|
+
}
|
|
75
|
+
process(queueName, jobName, handler) {
|
|
76
|
+
const workerKey = queueName;
|
|
77
|
+
let worker = this.bullWorkers.get(workerKey);
|
|
78
|
+
if (!worker) {
|
|
79
|
+
const { Worker } = require("bullmq");
|
|
80
|
+
worker = new Worker(queueName, async (bullJob) => {
|
|
81
|
+
const handlerKey = `${queueName}:${bullJob.name}`;
|
|
82
|
+
const registeredHandler = worker.__handlers?.get(handlerKey);
|
|
83
|
+
if (registeredHandler) {
|
|
84
|
+
const job = this.bullJobToJob(bullJob, bullJob.name, bullJob.data);
|
|
85
|
+
await registeredHandler(job);
|
|
86
|
+
}
|
|
87
|
+
}, {
|
|
88
|
+
connection: this.getConnection(),
|
|
89
|
+
concurrency: 1,
|
|
90
|
+
});
|
|
91
|
+
worker.__handlers = new Map();
|
|
92
|
+
this.bullWorkers.set(workerKey, worker);
|
|
93
|
+
}
|
|
94
|
+
const handlerKey = `${queueName}:${jobName}`;
|
|
95
|
+
worker.__handlers.set(handlerKey, handler);
|
|
96
|
+
}
|
|
97
|
+
async getJob(queueName, id) {
|
|
98
|
+
const queue = this.getQueue(queueName);
|
|
99
|
+
const bullJob = await queue.getJob(id);
|
|
100
|
+
if (!bullJob)
|
|
101
|
+
return null;
|
|
102
|
+
return this.bullJobToJob(bullJob, bullJob.name, bullJob.data);
|
|
103
|
+
}
|
|
104
|
+
async removeJob(queueName, id) {
|
|
105
|
+
const queue = this.getQueue(queueName);
|
|
106
|
+
const bullJob = await queue.getJob(id);
|
|
107
|
+
if (bullJob) {
|
|
108
|
+
await bullJob.remove();
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
async startProcessing() {
|
|
112
|
+
// BullMQ workers start processing when created
|
|
113
|
+
// We register handlers via process(), workers are created on first process() call
|
|
114
|
+
}
|
|
115
|
+
async stopProcessing() {
|
|
116
|
+
for (const worker of this.bullWorkers.values()) {
|
|
117
|
+
await worker.pause();
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
bullJobToJob(bullJob, name, data) {
|
|
121
|
+
return {
|
|
122
|
+
id: bullJob.id ?? String(bullJob.id),
|
|
123
|
+
name,
|
|
124
|
+
data,
|
|
125
|
+
attempts: bullJob.attemptsMade ?? 0,
|
|
126
|
+
maxAttempts: bullJob.opts?.attempts,
|
|
127
|
+
priority: bullJob.opts?.priority,
|
|
128
|
+
delay: bullJob.opts?.delay,
|
|
129
|
+
createdAt: new Date(bullJob.timestamp ?? Date.now()),
|
|
130
|
+
processedAt: bullJob.processedOn
|
|
131
|
+
? new Date(bullJob.processedOn)
|
|
132
|
+
: undefined,
|
|
133
|
+
completedAt: bullJob.finishedOn
|
|
134
|
+
? new Date(bullJob.finishedOn)
|
|
135
|
+
: undefined,
|
|
136
|
+
failedAt: bullJob.failedReason ? new Date() : undefined,
|
|
137
|
+
error: bullJob.failedReason,
|
|
138
|
+
};
|
|
139
|
+
}
|
|
140
|
+
}
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
import { pathToFileURL } from "url";
|
|
2
|
+
export function registerQueueCommands(program) {
|
|
3
|
+
program
|
|
4
|
+
.command("worker")
|
|
5
|
+
.description("Start the queue worker process")
|
|
6
|
+
.option("-e, --entry <path>", "Entry file (default: src/main.ts or src/worker.ts)", "")
|
|
7
|
+
.action(async (options) => {
|
|
8
|
+
try {
|
|
9
|
+
const entry = options.entry || (await findWorkerEntry());
|
|
10
|
+
const mod = await import(pathToFileURL(entry).href);
|
|
11
|
+
const appClass = mod.App ?? mod.default;
|
|
12
|
+
if (!appClass) {
|
|
13
|
+
console.error("Entry file must export App or default. Example:\n" +
|
|
14
|
+
" export const App = MyApp;\n" +
|
|
15
|
+
" // or: export default MyApp;");
|
|
16
|
+
process.exit(1);
|
|
17
|
+
}
|
|
18
|
+
const { runWorker } = await import("../worker-bootstrap.js");
|
|
19
|
+
await runWorker(appClass);
|
|
20
|
+
}
|
|
21
|
+
catch (error) {
|
|
22
|
+
console.error("Worker failed:", error instanceof Error ? error.message : error);
|
|
23
|
+
process.exit(1);
|
|
24
|
+
}
|
|
25
|
+
});
|
|
26
|
+
}
|
|
27
|
+
async function findWorkerEntry() {
|
|
28
|
+
const { resolve } = await import("path");
|
|
29
|
+
const { existsSync } = await import("fs");
|
|
30
|
+
const cwd = process.cwd();
|
|
31
|
+
const candidates = [
|
|
32
|
+
resolve(cwd, "src/worker.ts"),
|
|
33
|
+
resolve(cwd, "src/worker.js"),
|
|
34
|
+
resolve(cwd, "src/main.ts"),
|
|
35
|
+
resolve(cwd, "src/main.js"),
|
|
36
|
+
];
|
|
37
|
+
for (const p of candidates) {
|
|
38
|
+
if (existsSync(p)) {
|
|
39
|
+
return p;
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
throw new Error("No entry file found. Create src/worker.ts or src/main.ts, or use -e <path>.\n" +
|
|
43
|
+
"Worker entry must export: export const App = YourAppClass;");
|
|
44
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export declare function Job(name: string): MethodDecorator;
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export declare function Processor(queueName: string): ClassDecorator;
|
package/dist/index.d.ts
ADDED
|
@@ -0,0 +1,15 @@
|
|
|
1
|
+
export type { Job as JobData } from "./interfaces/job.js";
|
|
2
|
+
export type { JobOptions } from "./interfaces/job-options.js";
|
|
3
|
+
export type { QueueAdapter } from "./interfaces/queue-adapter.js";
|
|
4
|
+
export type { QueueModuleOptions } from "./interfaces/queue-options.js";
|
|
5
|
+
export { MemoryAdapter } from "./adapters/memory.adapter.js";
|
|
6
|
+
export { RedisAdapter } from "./adapters/redis.adapter.js";
|
|
7
|
+
export { Job } from "./decorators/job.js";
|
|
8
|
+
export { Processor } from "./decorators/processor.js";
|
|
9
|
+
export { QueueService } from "./services/queue.service.js";
|
|
10
|
+
export { WorkerService } from "./services/worker.service.js";
|
|
11
|
+
export { QueueModule } from "./queue.module.js";
|
|
12
|
+
export { QueuePlugin } from "./queue.plugin.js";
|
|
13
|
+
export { runWorker } from "./worker-bootstrap.js";
|
|
14
|
+
export { getQueueService } from "./queue-service-holder.js";
|
|
15
|
+
export { queueMetadataStorage } from "./metadata/queue-storage.js";
|
package/dist/index.js
ADDED
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
export { MemoryAdapter } from "./adapters/memory.adapter.js";
|
|
2
|
+
export { RedisAdapter } from "./adapters/redis.adapter.js";
|
|
3
|
+
export { Job } from "./decorators/job.js";
|
|
4
|
+
export { Processor } from "./decorators/processor.js";
|
|
5
|
+
export { QueueService } from "./services/queue.service.js";
|
|
6
|
+
export { WorkerService } from "./services/worker.service.js";
|
|
7
|
+
export { QueueModule } from "./queue.module.js";
|
|
8
|
+
export { QueuePlugin } from "./queue.plugin.js";
|
|
9
|
+
export { runWorker } from "./worker-bootstrap.js";
|
|
10
|
+
export { getQueueService } from "./queue-service-holder.js";
|
|
11
|
+
export { queueMetadataStorage } from "./metadata/queue-storage.js";
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
export interface Job<T = unknown> {
|
|
2
|
+
id: string;
|
|
3
|
+
name: string;
|
|
4
|
+
data: T;
|
|
5
|
+
attempts: number;
|
|
6
|
+
maxAttempts?: number;
|
|
7
|
+
priority?: number;
|
|
8
|
+
delay?: number;
|
|
9
|
+
createdAt: Date;
|
|
10
|
+
processedAt?: Date;
|
|
11
|
+
completedAt?: Date;
|
|
12
|
+
failedAt?: Date;
|
|
13
|
+
error?: string;
|
|
14
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
import type { Job } from "./job.js";
|
|
2
|
+
import type { JobOptions } from "./job-options.js";
|
|
3
|
+
export interface QueueAdapter {
|
|
4
|
+
connect(): Promise<void>;
|
|
5
|
+
disconnect(): Promise<void>;
|
|
6
|
+
enqueue<T>(queueName: string, jobName: string, data: T, options?: JobOptions): Promise<Job<T>>;
|
|
7
|
+
process(queueName: string, jobName: string, handler: (job: Job) => Promise<void>): void;
|
|
8
|
+
getJob(queueName: string, id: string): Promise<Job | null>;
|
|
9
|
+
removeJob(queueName: string, id: string): Promise<void>;
|
|
10
|
+
startProcessing(): Promise<void>;
|
|
11
|
+
stopProcessing(): Promise<void>;
|
|
12
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export {};
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
export interface ProcessorMetadata {
|
|
2
|
+
target: Function;
|
|
3
|
+
queueName: string;
|
|
4
|
+
}
|
|
5
|
+
export interface JobMetadata {
|
|
6
|
+
target: Function;
|
|
7
|
+
methodKey: string;
|
|
8
|
+
jobName: string;
|
|
9
|
+
}
|
|
10
|
+
declare class QueueMetadataStorage {
|
|
11
|
+
private processors;
|
|
12
|
+
private jobs;
|
|
13
|
+
setProcessor(target: Function, queueName: string): void;
|
|
14
|
+
getProcessor(target: Function): ProcessorMetadata | undefined;
|
|
15
|
+
getAllProcessors(): ProcessorMetadata[];
|
|
16
|
+
addJob(target: Function, methodKey: string, jobName: string): void;
|
|
17
|
+
getJobs(target: Function): JobMetadata[];
|
|
18
|
+
reset(): void;
|
|
19
|
+
}
|
|
20
|
+
export declare const queueMetadataStorage: QueueMetadataStorage;
|
|
21
|
+
export {};
|
|
@@ -0,0 +1,24 @@
|
|
|
1
|
+
class QueueMetadataStorage {
|
|
2
|
+
processors = new Map();
|
|
3
|
+
jobs = [];
|
|
4
|
+
setProcessor(target, queueName) {
|
|
5
|
+
this.processors.set(target, { target, queueName });
|
|
6
|
+
}
|
|
7
|
+
getProcessor(target) {
|
|
8
|
+
return this.processors.get(target);
|
|
9
|
+
}
|
|
10
|
+
getAllProcessors() {
|
|
11
|
+
return Array.from(this.processors.values());
|
|
12
|
+
}
|
|
13
|
+
addJob(target, methodKey, jobName) {
|
|
14
|
+
this.jobs.push({ target, methodKey, jobName });
|
|
15
|
+
}
|
|
16
|
+
getJobs(target) {
|
|
17
|
+
return this.jobs.filter((j) => j.target === target);
|
|
18
|
+
}
|
|
19
|
+
reset() {
|
|
20
|
+
this.processors.clear();
|
|
21
|
+
this.jobs = [];
|
|
22
|
+
}
|
|
23
|
+
}
|
|
24
|
+
export const queueMetadataStorage = new QueueMetadataStorage();
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Holder for QueueService instance.
|
|
3
|
+
* Set by QueuePlugin on init. Use getQueueService() to obtain the service
|
|
4
|
+
* in controllers (since constructor injection runs before the plugin initializes).
|
|
5
|
+
*/
|
|
6
|
+
let queueServiceInstance = null;
|
|
7
|
+
export function setQueueService(service) {
|
|
8
|
+
queueServiceInstance = service;
|
|
9
|
+
}
|
|
10
|
+
export function getQueueService() {
|
|
11
|
+
return queueServiceInstance;
|
|
12
|
+
}
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
import type { QueueModuleOptions } from "./interfaces/queue-options.js";
|
|
2
|
+
import type { QueueAdapter } from "./interfaces/queue-adapter.js";
|
|
3
|
+
export declare class QueueModule {
|
|
4
|
+
static forRoot(options: QueueModuleOptions): Function;
|
|
5
|
+
static getOptions(): QueueModuleOptions;
|
|
6
|
+
static createAdapter(): QueueAdapter;
|
|
7
|
+
}
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
var __decorate = (this && this.__decorate) || function (decorators, target, key, desc) {
|
|
2
|
+
var c = arguments.length, r = c < 3 ? target : desc === null ? desc = Object.getOwnPropertyDescriptor(target, key) : desc, d;
|
|
3
|
+
if (typeof Reflect === "object" && typeof Reflect.decorate === "function") r = Reflect.decorate(decorators, target, key, desc);
|
|
4
|
+
else for (var i = decorators.length - 1; i >= 0; i--) if (d = decorators[i]) r = (c < 3 ? d(r) : c > 3 ? d(target, key, r) : d(target, key)) || r;
|
|
5
|
+
return c > 3 && r && Object.defineProperty(target, key, r), r;
|
|
6
|
+
};
|
|
7
|
+
import { Module } from "@cinnabun/core";
|
|
8
|
+
import { MemoryAdapter } from "./adapters/memory.adapter.js";
|
|
9
|
+
import { RedisAdapter } from "./adapters/redis.adapter.js";
|
|
10
|
+
let moduleOptions = null;
|
|
11
|
+
export class QueueModule {
|
|
12
|
+
static forRoot(options) {
|
|
13
|
+
moduleOptions = options;
|
|
14
|
+
let QueueDynamicModule = class QueueDynamicModule {
|
|
15
|
+
};
|
|
16
|
+
QueueDynamicModule = __decorate([
|
|
17
|
+
Module({
|
|
18
|
+
imports: [],
|
|
19
|
+
controllers: [],
|
|
20
|
+
providers: [],
|
|
21
|
+
exports: [],
|
|
22
|
+
})
|
|
23
|
+
], QueueDynamicModule);
|
|
24
|
+
return QueueDynamicModule;
|
|
25
|
+
}
|
|
26
|
+
static getOptions() {
|
|
27
|
+
if (!moduleOptions) {
|
|
28
|
+
throw new Error("QueueModule not initialized. Call QueueModule.forRoot() first.");
|
|
29
|
+
}
|
|
30
|
+
return moduleOptions;
|
|
31
|
+
}
|
|
32
|
+
static createAdapter() {
|
|
33
|
+
const options = QueueModule.getOptions();
|
|
34
|
+
const opts = options.defaultJobOptions ?? {};
|
|
35
|
+
if (options.adapter === "memory") {
|
|
36
|
+
return new MemoryAdapter(opts);
|
|
37
|
+
}
|
|
38
|
+
if (options.adapter === "redis" && options.redis) {
|
|
39
|
+
return new RedisAdapter(options.redis, opts);
|
|
40
|
+
}
|
|
41
|
+
throw new Error("QueueModule: redis adapter requires redis config (host, port)");
|
|
42
|
+
}
|
|
43
|
+
}
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import type { CinnabunPlugin, PluginContext } from "@cinnabun/core";
|
|
2
|
+
export declare class QueuePlugin implements CinnabunPlugin {
|
|
3
|
+
name: string;
|
|
4
|
+
private workerService;
|
|
5
|
+
private adapter;
|
|
6
|
+
onInit(context: PluginContext): Promise<void>;
|
|
7
|
+
onReady(_context: PluginContext): Promise<void>;
|
|
8
|
+
onShutdown(_context: PluginContext): Promise<void>;
|
|
9
|
+
}
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
import { Logger } from "@cinnabun/core";
|
|
2
|
+
import { QueueModule } from "./queue.module.js";
|
|
3
|
+
import { WorkerService } from "./services/worker.service.js";
|
|
4
|
+
import { QueueService } from "./services/queue.service.js";
|
|
5
|
+
import { setQueueService } from "./queue-service-holder.js";
|
|
6
|
+
export class QueuePlugin {
|
|
7
|
+
name = "QueuePlugin";
|
|
8
|
+
workerService = null;
|
|
9
|
+
adapter = null;
|
|
10
|
+
async onInit(context) {
|
|
11
|
+
const logger = new Logger("QueuePlugin");
|
|
12
|
+
const options = QueueModule.getOptions();
|
|
13
|
+
const container = context.container;
|
|
14
|
+
const adapter = QueueModule.createAdapter();
|
|
15
|
+
await adapter.connect();
|
|
16
|
+
this.adapter = adapter;
|
|
17
|
+
const workerService = new WorkerService(adapter, container);
|
|
18
|
+
workerService.registerHandlers();
|
|
19
|
+
this.workerService = workerService;
|
|
20
|
+
const jobToQueue = workerService.getJobToQueueMap();
|
|
21
|
+
const queueService = new QueueService(adapter, jobToQueue);
|
|
22
|
+
container.registerInstance(QueueService, queueService);
|
|
23
|
+
setQueueService(queueService);
|
|
24
|
+
logger.info(`Queue module initialized (adapter: ${options.adapter})`);
|
|
25
|
+
}
|
|
26
|
+
async onReady(_context) {
|
|
27
|
+
if (this.workerService) {
|
|
28
|
+
await this.workerService.start();
|
|
29
|
+
}
|
|
30
|
+
}
|
|
31
|
+
async onShutdown(_context) {
|
|
32
|
+
if (this.workerService) {
|
|
33
|
+
await this.workerService.stop();
|
|
34
|
+
this.workerService = null;
|
|
35
|
+
}
|
|
36
|
+
if (this.adapter) {
|
|
37
|
+
await this.adapter.disconnect();
|
|
38
|
+
this.adapter = null;
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
}
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
import type { Job } from "../interfaces/job.js";
|
|
2
|
+
import type { JobOptions } from "../interfaces/job-options.js";
|
|
3
|
+
import type { QueueAdapter } from "../interfaces/queue-adapter.js";
|
|
4
|
+
export declare class QueueService {
|
|
5
|
+
private readonly adapter;
|
|
6
|
+
private readonly jobToQueue;
|
|
7
|
+
private readonly jobIdToQueue;
|
|
8
|
+
constructor(adapter: QueueAdapter, jobToQueue: Map<string, string>);
|
|
9
|
+
enqueue<T>(jobName: string, data: T, options?: JobOptions): Promise<Job<T>>;
|
|
10
|
+
getJob(id: string): Promise<Job | null>;
|
|
11
|
+
removeJob(id: string): Promise<void>;
|
|
12
|
+
}
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
export class QueueService {
|
|
2
|
+
adapter;
|
|
3
|
+
jobToQueue;
|
|
4
|
+
jobIdToQueue = new Map();
|
|
5
|
+
constructor(adapter, jobToQueue) {
|
|
6
|
+
this.adapter = adapter;
|
|
7
|
+
this.jobToQueue = jobToQueue;
|
|
8
|
+
}
|
|
9
|
+
async enqueue(jobName, data, options) {
|
|
10
|
+
const queueName = this.jobToQueue.get(jobName);
|
|
11
|
+
if (!queueName) {
|
|
12
|
+
throw new Error(`No queue registered for job "${jobName}". Add a @Processor with @Job("${jobName}") handler.`);
|
|
13
|
+
}
|
|
14
|
+
const job = await this.adapter.enqueue(queueName, jobName, data, options);
|
|
15
|
+
this.jobIdToQueue.set(job.id, queueName);
|
|
16
|
+
return job;
|
|
17
|
+
}
|
|
18
|
+
async getJob(id) {
|
|
19
|
+
const queueName = this.jobIdToQueue.get(id);
|
|
20
|
+
if (!queueName)
|
|
21
|
+
return null;
|
|
22
|
+
return this.adapter.getJob(queueName, id);
|
|
23
|
+
}
|
|
24
|
+
async removeJob(id) {
|
|
25
|
+
const queueName = this.jobIdToQueue.get(id);
|
|
26
|
+
if (queueName) {
|
|
27
|
+
await this.adapter.removeJob(queueName, id);
|
|
28
|
+
this.jobIdToQueue.delete(id);
|
|
29
|
+
}
|
|
30
|
+
}
|
|
31
|
+
}
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
import type { Container } from "@cinnabun/core";
|
|
2
|
+
import type { QueueAdapter } from "../interfaces/queue-adapter.js";
|
|
3
|
+
export declare class WorkerService {
|
|
4
|
+
private readonly adapter;
|
|
5
|
+
private readonly container;
|
|
6
|
+
private readonly jobToQueue;
|
|
7
|
+
private started;
|
|
8
|
+
constructor(adapter: QueueAdapter, container: Container);
|
|
9
|
+
registerHandlers(): void;
|
|
10
|
+
getJobToQueueMap(): Map<string, string>;
|
|
11
|
+
start(): Promise<void>;
|
|
12
|
+
stop(): Promise<void>;
|
|
13
|
+
}
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
import { queueMetadataStorage } from "../metadata/queue-storage.js";
|
|
2
|
+
export class WorkerService {
|
|
3
|
+
adapter;
|
|
4
|
+
container;
|
|
5
|
+
jobToQueue = new Map();
|
|
6
|
+
started = false;
|
|
7
|
+
constructor(adapter, container) {
|
|
8
|
+
this.adapter = adapter;
|
|
9
|
+
this.container = container;
|
|
10
|
+
}
|
|
11
|
+
registerHandlers() {
|
|
12
|
+
const processors = queueMetadataStorage.getAllProcessors();
|
|
13
|
+
for (const { target: processorClass } of processors) {
|
|
14
|
+
let instance;
|
|
15
|
+
try {
|
|
16
|
+
instance = this.container.resolve(processorClass);
|
|
17
|
+
}
|
|
18
|
+
catch {
|
|
19
|
+
continue;
|
|
20
|
+
}
|
|
21
|
+
const processorMeta = queueMetadataStorage.getProcessor(processorClass);
|
|
22
|
+
if (!processorMeta)
|
|
23
|
+
continue;
|
|
24
|
+
const queueName = processorMeta.queueName;
|
|
25
|
+
const jobs = queueMetadataStorage.getJobs(processorClass);
|
|
26
|
+
for (const jobMeta of jobs) {
|
|
27
|
+
this.jobToQueue.set(jobMeta.jobName, queueName);
|
|
28
|
+
const method = instance[jobMeta.methodKey];
|
|
29
|
+
if (typeof method !== "function")
|
|
30
|
+
continue;
|
|
31
|
+
const handler = (job) => method.call(instance, job);
|
|
32
|
+
this.adapter.process(queueName, jobMeta.jobName, handler);
|
|
33
|
+
}
|
|
34
|
+
}
|
|
35
|
+
}
|
|
36
|
+
getJobToQueueMap() {
|
|
37
|
+
return new Map(this.jobToQueue);
|
|
38
|
+
}
|
|
39
|
+
async start() {
|
|
40
|
+
if (this.started)
|
|
41
|
+
return;
|
|
42
|
+
this.registerHandlers();
|
|
43
|
+
await this.adapter.startProcessing();
|
|
44
|
+
this.started = true;
|
|
45
|
+
}
|
|
46
|
+
async stop() {
|
|
47
|
+
if (!this.started)
|
|
48
|
+
return;
|
|
49
|
+
await this.adapter.stopProcessing();
|
|
50
|
+
this.started = false;
|
|
51
|
+
}
|
|
52
|
+
}
|
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
import { metadataStorage, } from "@cinnabun/core";
|
|
2
|
+
import { scanComponents } from "@cinnabun/core";
|
|
3
|
+
import { resolveModuleTree } from "@cinnabun/core";
|
|
4
|
+
import { CinnabunApplication } from "@cinnabun/core";
|
|
5
|
+
import { FileConfigService } from "@cinnabun/core";
|
|
6
|
+
import { loadConfig } from "@cinnabun/core";
|
|
7
|
+
import { Logger } from "@cinnabun/core";
|
|
8
|
+
import { validateDependencies } from "@cinnabun/core";
|
|
9
|
+
/**
|
|
10
|
+
* Bootstraps the app in worker-only mode (no HTTP server).
|
|
11
|
+
* Replicates CinnabunFactory bootstrap but skips listen().
|
|
12
|
+
*/
|
|
13
|
+
export async function runWorker(appClass) {
|
|
14
|
+
const logger = new Logger("QueueWorker");
|
|
15
|
+
const meta = metadataStorage.getAppMetadata(appClass);
|
|
16
|
+
if (!meta) {
|
|
17
|
+
throw new Error(`${appClass.name} is not decorated with @CinnabunApplication().`);
|
|
18
|
+
}
|
|
19
|
+
const rawConfig = await loadConfig();
|
|
20
|
+
const configService = new FileConfigService(rawConfig);
|
|
21
|
+
logger.info("Scanning components...");
|
|
22
|
+
const scanned = await scanComponents(meta.scanPaths);
|
|
23
|
+
const moduleProviders = [];
|
|
24
|
+
const moduleControllers = [];
|
|
25
|
+
for (const mod of [...meta.imports, ...scanned.modules]) {
|
|
26
|
+
const resolved = await resolveModuleTree(mod);
|
|
27
|
+
moduleProviders.push(...resolved.providers);
|
|
28
|
+
moduleControllers.push(...resolved.controllers);
|
|
29
|
+
}
|
|
30
|
+
const controllers = [...moduleControllers, ...scanned.controllers];
|
|
31
|
+
const providers = [...moduleProviders, ...scanned.providers];
|
|
32
|
+
if (metadataStorage.controllerPaths.has(appClass)) {
|
|
33
|
+
controllers.push(appClass);
|
|
34
|
+
}
|
|
35
|
+
try {
|
|
36
|
+
validateDependencies(providers);
|
|
37
|
+
}
|
|
38
|
+
catch (error) {
|
|
39
|
+
logger.error("Dependency validation failed");
|
|
40
|
+
throw error;
|
|
41
|
+
}
|
|
42
|
+
const plugins = meta.plugins ?? [];
|
|
43
|
+
const app = await CinnabunApplication.create({
|
|
44
|
+
controllers,
|
|
45
|
+
providers,
|
|
46
|
+
middleware: meta.middleware,
|
|
47
|
+
websocket: undefined,
|
|
48
|
+
plugins,
|
|
49
|
+
timeout: meta.timeout,
|
|
50
|
+
shutdownTimeout: meta.shutdownTimeout,
|
|
51
|
+
preRegister: (container) => {
|
|
52
|
+
container.registerInstance(FileConfigService, configService);
|
|
53
|
+
},
|
|
54
|
+
});
|
|
55
|
+
const pluginContext = {
|
|
56
|
+
container: app.getContainer(),
|
|
57
|
+
config: configService,
|
|
58
|
+
};
|
|
59
|
+
app.setPluginContext(pluginContext);
|
|
60
|
+
for (const plugin of plugins) {
|
|
61
|
+
if (plugin.onInit) {
|
|
62
|
+
logger.info(`Plugin "${plugin.name}" initializing...`);
|
|
63
|
+
await plugin.onInit(pluginContext);
|
|
64
|
+
}
|
|
65
|
+
}
|
|
66
|
+
for (const plugin of plugins) {
|
|
67
|
+
if (plugin.onReady) {
|
|
68
|
+
await plugin.onReady(pluginContext);
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
logger.info("Worker started, processing jobs...");
|
|
72
|
+
const shutdown = async (signal) => {
|
|
73
|
+
logger.info(`Received ${signal}, shutting down...`);
|
|
74
|
+
await app.close();
|
|
75
|
+
process.exit(0);
|
|
76
|
+
};
|
|
77
|
+
process.on("SIGINT", () => shutdown("SIGINT"));
|
|
78
|
+
process.on("SIGTERM", () => shutdown("SIGTERM"));
|
|
79
|
+
}
|
package/package.json
ADDED
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "@cinnabun/queue",
|
|
3
|
+
"version": "0.0.1",
|
|
4
|
+
"description": "Job queue system for Cinnabun with Redis and in-memory adapters",
|
|
5
|
+
"type": "module",
|
|
6
|
+
"main": "dist/index.js",
|
|
7
|
+
"types": "dist/index.d.ts",
|
|
8
|
+
"files": ["dist"],
|
|
9
|
+
"exports": {
|
|
10
|
+
".": "./dist/index.js",
|
|
11
|
+
"./cli": "./dist/cli/register-queue-commands.js"
|
|
12
|
+
},
|
|
13
|
+
"scripts": {
|
|
14
|
+
"test": "bun test",
|
|
15
|
+
"build": "tsc",
|
|
16
|
+
"prepublishOnly": "bun run build"
|
|
17
|
+
},
|
|
18
|
+
"keywords": ["cinnabun", "queue", "jobs", "bullmq", "redis"],
|
|
19
|
+
"peerDependencies": {
|
|
20
|
+
"@cinnabun/core": "^0.0.3",
|
|
21
|
+
"bullmq": ">=5.0.0"
|
|
22
|
+
},
|
|
23
|
+
"peerDependenciesMeta": {
|
|
24
|
+
"bullmq": {
|
|
25
|
+
"optional": true
|
|
26
|
+
}
|
|
27
|
+
},
|
|
28
|
+
"devDependencies": {
|
|
29
|
+
"@cinnabun/core": "workspace:*",
|
|
30
|
+
"@types/bun": "latest",
|
|
31
|
+
"reflect-metadata": "^0.2.2",
|
|
32
|
+
"typescript": "^5.9.3"
|
|
33
|
+
},
|
|
34
|
+
"dependencies": {
|
|
35
|
+
"commander": "^14.0.3"
|
|
36
|
+
}
|
|
37
|
+
}
|