consumer-pgmq 1.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.github/workflows/release.yml +44 -0
- package/README.md +31 -0
- package/examples/consumerPostgresDriver.ts +72 -0
- package/examples/consumerSupabaseDriver.ts +84 -0
- package/jest.config.js +7 -0
- package/package.json +34 -0
- package/src/consumer.ts +141 -0
- package/src/index.ts +3 -0
- package/src/queueDriver/PostgresQueueDriver.ts +88 -0
- package/src/queueDriver/SupabaseQueueDriver.ts +57 -0
- package/src/type.ts +97 -0
- package/tests/consumer.spec.ts +237 -0
- package/tsconfig.json +21 -0
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
name: Release
|
|
2
|
+
on:
|
|
3
|
+
push:
|
|
4
|
+
branches:
|
|
5
|
+
- master
|
|
6
|
+
|
|
7
|
+
permissions:
|
|
8
|
+
contents: read
|
|
9
|
+
|
|
10
|
+
jobs:
|
|
11
|
+
release:
|
|
12
|
+
name: Release
|
|
13
|
+
runs-on: ubuntu-latest
|
|
14
|
+
permissions:
|
|
15
|
+
contents: write
|
|
16
|
+
issues: write
|
|
17
|
+
pull-requests: write
|
|
18
|
+
id-token: write
|
|
19
|
+
steps:
|
|
20
|
+
|
|
21
|
+
- name: Checkout
|
|
22
|
+
uses: actions/checkout@v4
|
|
23
|
+
with:
|
|
24
|
+
fetch-depth: 0
|
|
25
|
+
|
|
26
|
+
- name: Setup Node.js
|
|
27
|
+
uses: actions/setup-node@v4
|
|
28
|
+
with:
|
|
29
|
+
node-version: "lts/*"
|
|
30
|
+
|
|
31
|
+
- name: Setup pnpm
|
|
32
|
+
uses: pnpm/action-setup@v3
|
|
33
|
+
|
|
34
|
+
- name: Install dependencies
|
|
35
|
+
run: pnpm install
|
|
36
|
+
|
|
37
|
+
- name: Verify the integrity of provenance attestations and registry signatures for installed dependencies
|
|
38
|
+
run: pnpm audit signatures
|
|
39
|
+
|
|
40
|
+
- name: Release
|
|
41
|
+
env:
|
|
42
|
+
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
|
|
43
|
+
NPM_TOKEN: ${{ secrets.NPM_TOKEN }}
|
|
44
|
+
run: npx semantic-release
|
package/README.md
ADDED
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
## About
|
|
2
|
+
|
|
3
|
+
This project is a consumer of Supabase/Postgresql queue(using pgmq extension) to simplify the process of consuming messages.
|
|
4
|
+
|
|
5
|
+
## Features
|
|
6
|
+
|
|
7
|
+
- Consumer message from Supabase queue. PS: instructions to setup https://supabase.com/blog/supabase-queues
|
|
8
|
+
- Consumer message from Postgresql queue. PS: instructions to setup https://github.com/pgmq/pgmq
|
|
9
|
+
- Support for both read and pop consume types
|
|
10
|
+
- Read consume type is when the consumer gets the message and the message is not deleted from queue until the callback is executed with success.
|
|
11
|
+
- Pop consume type is when the consumer gets the message and the message is deleted from queue.
|
|
12
|
+
- Support for both Supabase and Postgresql
|
|
13
|
+
- Support for both visibility time and pool size
|
|
14
|
+
|
|
15
|
+
## Installation
|
|
16
|
+
|
|
17
|
+
- Using pnpm
|
|
18
|
+
```bash
|
|
19
|
+
pnpm install consumer-pgmq
|
|
20
|
+
```
|
|
21
|
+
|
|
22
|
+
- Using npm
|
|
23
|
+
```bash
|
|
24
|
+
npm install consumer-pgmq
|
|
25
|
+
```
|
|
26
|
+
- Using yarn
|
|
27
|
+
```bash
|
|
28
|
+
yarn add consumer-pgmq
|
|
29
|
+
```
|
|
30
|
+
|
|
31
|
+
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
import { config } from "dotenv"
|
|
2
|
+
config()
|
|
3
|
+
|
|
4
|
+
import Consumer from '../src/consumer';
|
|
5
|
+
import PostgresQueueDriver from '../src/queueDriver/PostgresQueueDriver';
|
|
6
|
+
import timersPromises from "node:timers/promises";
|
|
7
|
+
import knex from 'knex'
|
|
8
|
+
|
|
9
|
+
async function start() {
|
|
10
|
+
const connection = knex({
|
|
11
|
+
client: 'pg',
|
|
12
|
+
connection: {
|
|
13
|
+
host: process.env.POSTGRES_HOST,
|
|
14
|
+
database: process.env.POSTGRES_DATABASE,
|
|
15
|
+
password: process.env.POSTGRES_PASSWORD,
|
|
16
|
+
port: Number(process.env.POSTGRES_PORT),
|
|
17
|
+
user: process.env.POSTGRES_USER,
|
|
18
|
+
ssl: false
|
|
19
|
+
}
|
|
20
|
+
});
|
|
21
|
+
|
|
22
|
+
const postgresQueueDriver = new PostgresQueueDriver(connection)
|
|
23
|
+
|
|
24
|
+
const consumer = new Consumer(
|
|
25
|
+
{
|
|
26
|
+
queueName: 'subscriptions',
|
|
27
|
+
visibilityTime: 15,
|
|
28
|
+
consumeType: "read",
|
|
29
|
+
poolSize: 4,
|
|
30
|
+
timeMsWaitBeforeNextPolling: 1000
|
|
31
|
+
},
|
|
32
|
+
async function (message: { [key: string]: any }, signal): Promise<void> {
|
|
33
|
+
try {
|
|
34
|
+
console.log(message)
|
|
35
|
+
const url = "https://jsonplaceholder.typicode.com/todos/1";
|
|
36
|
+
await timersPromises.setTimeout(100, null, { signal });
|
|
37
|
+
console.log("Fetching data...");
|
|
38
|
+
const response = await fetch(url, { signal });
|
|
39
|
+
const todo = await response.json();
|
|
40
|
+
console.log("Todo:", todo);
|
|
41
|
+
} catch (error: any) {
|
|
42
|
+
if (error.name === "AbortError") {
|
|
43
|
+
console.log("Operation aborted");
|
|
44
|
+
} else {
|
|
45
|
+
console.error("Error:", error);
|
|
46
|
+
}
|
|
47
|
+
}
|
|
48
|
+
},
|
|
49
|
+
postgresQueueDriver
|
|
50
|
+
);
|
|
51
|
+
|
|
52
|
+
consumer.on('finish', (message: { [key: string]: any }) => {
|
|
53
|
+
console.log('Consumed message =>', message);
|
|
54
|
+
});
|
|
55
|
+
|
|
56
|
+
consumer.on("abort-error", (err) => {
|
|
57
|
+
console.log("Abort error =>", err)
|
|
58
|
+
})
|
|
59
|
+
|
|
60
|
+
consumer.on('error', (err: Error) => {
|
|
61
|
+
if (err.message.includes("TypeError: fetch failed")) {
|
|
62
|
+
console.log(err)
|
|
63
|
+
process.exit(1);
|
|
64
|
+
}
|
|
65
|
+
console.error('Error consuming message:', err.message);
|
|
66
|
+
});
|
|
67
|
+
|
|
68
|
+
consumer.start();
|
|
69
|
+
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
start()
|
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
import { config } from "dotenv"
|
|
2
|
+
config()
|
|
3
|
+
|
|
4
|
+
import Consumer from '../src/consumer';
|
|
5
|
+
import { createClient, SupabaseClient } from '@supabase/supabase-js';
|
|
6
|
+
import SupabaseQueueDriver from '../src/queueDriver/SupabaseQueueDriver';
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
const supabase = createClient(
|
|
10
|
+
// @ts-ignore
|
|
11
|
+
process.env.SUPABASE_URL,
|
|
12
|
+
process.env.SUPABASE_ANON_KEY,
|
|
13
|
+
{
|
|
14
|
+
db: {
|
|
15
|
+
schema: 'pgmq_public'
|
|
16
|
+
}
|
|
17
|
+
}
|
|
18
|
+
);
|
|
19
|
+
|
|
20
|
+
const supabaseQueueDriver = new SupabaseQueueDriver(
|
|
21
|
+
supabase as unknown as SupabaseClient
|
|
22
|
+
)
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
import timersPromises from "node:timers/promises";
|
|
26
|
+
|
|
27
|
+
async function start() {
|
|
28
|
+
for (let i = 0; i < 200; i++) {
|
|
29
|
+
await supabase.rpc("send", {
|
|
30
|
+
queue_name: "subscriptions",
|
|
31
|
+
message: { "message": `Message triggered at ${Date.now()}` }
|
|
32
|
+
});
|
|
33
|
+
}
|
|
34
|
+
console.log("Total messages sent: ", 200)
|
|
35
|
+
|
|
36
|
+
const consumer = new Consumer(
|
|
37
|
+
{
|
|
38
|
+
queueName: 'subscriptions',
|
|
39
|
+
visibilityTime: 15,
|
|
40
|
+
consumeType: "read",
|
|
41
|
+
poolSize: 4,
|
|
42
|
+
timeMsWaitBeforeNextPolling: 1000
|
|
43
|
+
},
|
|
44
|
+
async function (message: { [key: string]: any }, signal): Promise<void> {
|
|
45
|
+
try {
|
|
46
|
+
console.log(message)
|
|
47
|
+
const url = "https://jsonplaceholder.typicode.com/todos/1";
|
|
48
|
+
await timersPromises.setTimeout(100, null, { signal });
|
|
49
|
+
console.log("Fetching data...");
|
|
50
|
+
const response = await fetch(url, { signal });
|
|
51
|
+
const todo = await response.json();
|
|
52
|
+
console.log("Todo:", todo);
|
|
53
|
+
} catch (error: any) {
|
|
54
|
+
if (error.name === "AbortError") {
|
|
55
|
+
console.log("Operation aborted");
|
|
56
|
+
} else {
|
|
57
|
+
console.error("Error:", error);
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
},
|
|
61
|
+
supabaseQueueDriver
|
|
62
|
+
);
|
|
63
|
+
|
|
64
|
+
consumer.on('finish', (message: { [key: string]: any }) => {
|
|
65
|
+
console.log('Consumed message =>', message);
|
|
66
|
+
});
|
|
67
|
+
|
|
68
|
+
consumer.on("abort-error", (err) => {
|
|
69
|
+
console.log("Abort error =>", err)
|
|
70
|
+
})
|
|
71
|
+
|
|
72
|
+
consumer.on('error', (err: Error) => {
|
|
73
|
+
if (err.message.includes("TypeError: fetch failed")) {
|
|
74
|
+
console.log(err)
|
|
75
|
+
process.exit(1);
|
|
76
|
+
}
|
|
77
|
+
console.error('Error consuming message:', err.message);
|
|
78
|
+
});
|
|
79
|
+
|
|
80
|
+
consumer.start();
|
|
81
|
+
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
start()
|
package/jest.config.js
ADDED
package/package.json
ADDED
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "consumer-pgmq",
|
|
3
|
+
"version": "1.0.0",
|
|
4
|
+
"description": "The consumer of Supabase pgmq",
|
|
5
|
+
"main": "dist/index.js",
|
|
6
|
+
"type": "commonjs",
|
|
7
|
+
"scripts": {
|
|
8
|
+
"start:example:postgres": "ts-node ./examples/consumerPostgresDriver.ts",
|
|
9
|
+
"start:example:supabase": "ts-node ./examples/consumerSupabaseDriver.ts",
|
|
10
|
+
"start:dev": "ts-node-dev ./src/index.ts",
|
|
11
|
+
"test": "jest ./tests/*.spec.ts --detectOpenHandles"
|
|
12
|
+
},
|
|
13
|
+
"keywords": [
|
|
14
|
+
"supabase",
|
|
15
|
+
"pgmq",
|
|
16
|
+
"consumer"
|
|
17
|
+
],
|
|
18
|
+
"author": "Tiago Rosa da costa<tiagorosadacosta@gmail.com>",
|
|
19
|
+
"license": "MIT",
|
|
20
|
+
"packageManager": "pnpm@10.7.1",
|
|
21
|
+
"devDependencies": {
|
|
22
|
+
"@types/jest": "^30.0.0",
|
|
23
|
+
"ts-jest": "^29.4.1",
|
|
24
|
+
"ts-node-dev": "^2.0.0",
|
|
25
|
+
"typescript": "^5.9.2"
|
|
26
|
+
},
|
|
27
|
+
"dependencies": {
|
|
28
|
+
"@supabase/supabase-js": "^2.56.0",
|
|
29
|
+
"dotenv": "^17.2.1",
|
|
30
|
+
"jest": "^30.1.1",
|
|
31
|
+
"knex": "^3.1.0",
|
|
32
|
+
"pg": "^8.16.3"
|
|
33
|
+
}
|
|
34
|
+
}
|
package/src/consumer.ts
ADDED
|
@@ -0,0 +1,141 @@
|
|
|
1
|
+
|
|
2
|
+
import { EventEmitter } from 'events';
|
|
3
|
+
import { HandlerCallback, Message, Options, QueueDriver } from './type';
|
|
4
|
+
|
|
5
|
+
const READ = "read";
|
|
6
|
+
|
|
7
|
+
/**
|
|
8
|
+
* The consumer class.
|
|
9
|
+
*/
|
|
10
|
+
class Consumer extends EventEmitter {
|
|
11
|
+
|
|
12
|
+
/**
|
|
13
|
+
* The callback function to handle the message.
|
|
14
|
+
*/
|
|
15
|
+
private callback: HandlerCallback
|
|
16
|
+
/**
|
|
17
|
+
* The options to configure the consumer.
|
|
18
|
+
*/
|
|
19
|
+
private options: Options
|
|
20
|
+
/**
|
|
21
|
+
* The supabase client.
|
|
22
|
+
*/
|
|
23
|
+
private client: QueueDriver
|
|
24
|
+
|
|
25
|
+
constructor(
|
|
26
|
+
options: Options,
|
|
27
|
+
callback: HandlerCallback,
|
|
28
|
+
client: QueueDriver,
|
|
29
|
+
) {
|
|
30
|
+
super();
|
|
31
|
+
this.options = options;
|
|
32
|
+
this.callback = callback;
|
|
33
|
+
this.client = client;
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
/**
|
|
37
|
+
* Get the message
|
|
38
|
+
* @returns Promise<{ data: Message[], error: any }>
|
|
39
|
+
* @private
|
|
40
|
+
*/
|
|
41
|
+
private async getMessage(): Promise<{ data: Message[], error: any }> {
|
|
42
|
+
if (this.options.consumeType === READ) {
|
|
43
|
+
if (!this.options.visibilityTime) {
|
|
44
|
+
throw new Error("visibilityTime is required for read");
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
const { data, error } = await this.client.get(
|
|
48
|
+
this.options.queueName, this.options.visibilityTime,
|
|
49
|
+
this.options.poolSize || 1
|
|
50
|
+
);
|
|
51
|
+
|
|
52
|
+
return { data: data as Message[], error };
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
const { data, error } = await this.client.pop(this.options.queueName);
|
|
56
|
+
return { data: data as Message[], error };
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
/**
|
|
60
|
+
* @param data
|
|
61
|
+
* @returns Promise<void>
|
|
62
|
+
* @private
|
|
63
|
+
*/
|
|
64
|
+
private async deleteMessage(data: Message, signal: AbortSignal) {
|
|
65
|
+
if (signal.aborted) {
|
|
66
|
+
return;
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
if (this.options.consumeType === READ) {
|
|
70
|
+
const deleteMessage = await this.client.delete(
|
|
71
|
+
this.options.queueName,
|
|
72
|
+
data.msg_id
|
|
73
|
+
);
|
|
74
|
+
if (deleteMessage.error) {
|
|
75
|
+
this.emit('error', deleteMessage.error);
|
|
76
|
+
return;
|
|
77
|
+
}
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
/**
|
|
82
|
+
* Poll the message
|
|
83
|
+
* @returns Promise<void>
|
|
84
|
+
* @private
|
|
85
|
+
*/
|
|
86
|
+
private async pollMessage() {
|
|
87
|
+
let promises: Promise<any>[] = [];
|
|
88
|
+
|
|
89
|
+
try {
|
|
90
|
+
const { data, error } = await this.getMessage();
|
|
91
|
+
if (error) {
|
|
92
|
+
throw error;
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
if (data.length === 0) {
|
|
96
|
+
setTimeout(
|
|
97
|
+
() => this.pollMessage(),
|
|
98
|
+
(this.options.timeMsWaitBeforeNextPolling || 1000) * 10
|
|
99
|
+
);
|
|
100
|
+
return;
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
const controller = new AbortController();
|
|
105
|
+
const signal = controller.signal;
|
|
106
|
+
|
|
107
|
+
for (let i = 0; i < (data.length || 1); i++) {
|
|
108
|
+
promises.push(
|
|
109
|
+
this.callback(data[i].message, signal).then(async () => {
|
|
110
|
+
await this.deleteMessage(data[i], signal);
|
|
111
|
+
this.emit('finish', data[i]);
|
|
112
|
+
})
|
|
113
|
+
);
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
setTimeout(() => controller.abort(), (this.options.visibilityTime || 1) * 1000);
|
|
117
|
+
await Promise.all(promises);
|
|
118
|
+
promises = [];
|
|
119
|
+
} catch (err: any) {
|
|
120
|
+
if (err.name === "AbortError") {
|
|
121
|
+
this.emit("abort-error", err)
|
|
122
|
+
} else {
|
|
123
|
+
this.emit('error', err);
|
|
124
|
+
}
|
|
125
|
+
} finally {
|
|
126
|
+
setTimeout(() => this.pollMessage(), this.options.timeMsWaitBeforeNextPolling || 1000);
|
|
127
|
+
}
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
/**
|
|
131
|
+
* Start the consumer
|
|
132
|
+
* @returns Promise<void>
|
|
133
|
+
* @public
|
|
134
|
+
*/
|
|
135
|
+
async start() {
|
|
136
|
+
await this.pollMessage();
|
|
137
|
+
}
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
export default Consumer;
|
package/src/index.ts
ADDED
|
@@ -0,0 +1,88 @@
|
|
|
1
|
+
import { Knex } from "knex";
|
|
2
|
+
import { Message, QueueDriver } from "../type";
|
|
3
|
+
|
|
4
|
+
class PostgresQueueDriver implements QueueDriver {
|
|
5
|
+
|
|
6
|
+
constructor(
|
|
7
|
+
private connection: Knex
|
|
8
|
+
) { }
|
|
9
|
+
|
|
10
|
+
/**
|
|
11
|
+
* Get the message
|
|
12
|
+
* @param queueName The name of the queue
|
|
13
|
+
* @param visibilityTime The visibility time of the message
|
|
14
|
+
* @param totalMessages The total messages to get
|
|
15
|
+
* @returns Promise<{ data: Message[], error: any }>
|
|
16
|
+
*/
|
|
17
|
+
async get(queueName: string, visibilityTime: number, totalMessages: number): Promise<{ data: Message[]; error: any; }> {
|
|
18
|
+
try {
|
|
19
|
+
const register = await this.connection.raw(`
|
|
20
|
+
SELECT * FROM pgmq.read(
|
|
21
|
+
queue_name => ?,
|
|
22
|
+
vt => ?,
|
|
23
|
+
qty => ?
|
|
24
|
+
);
|
|
25
|
+
`, [queueName, visibilityTime, totalMessages]
|
|
26
|
+
)
|
|
27
|
+
|
|
28
|
+
if (!register.rows) {
|
|
29
|
+
return { data: [], error: null };
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
return { data: register.rows as Message[], error: null };
|
|
33
|
+
} catch (error) {
|
|
34
|
+
return { data: [], error };
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
/**
|
|
40
|
+
* Pop the message
|
|
41
|
+
* @param queueName The name of the queue
|
|
42
|
+
* @returns Promise<{ data: Message[], error: any }>
|
|
43
|
+
*/
|
|
44
|
+
async pop(queueName: string): Promise<{ data: Message[]; error: any; }> {
|
|
45
|
+
try {
|
|
46
|
+
const register = await this.connection.raw(`
|
|
47
|
+
SELECT * FROM pgmq.pop(
|
|
48
|
+
queue_name => ?
|
|
49
|
+
);
|
|
50
|
+
`, [queueName]
|
|
51
|
+
)
|
|
52
|
+
|
|
53
|
+
if (!register.rows) {
|
|
54
|
+
return { data: [], error: null };
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
return { data: register.rows as Message[], error: null };
|
|
58
|
+
} catch (error) {
|
|
59
|
+
return { data: [], error };
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
/**
|
|
65
|
+
* Delete the message
|
|
66
|
+
* @param queueName The name of the queue
|
|
67
|
+
* @param messageID The message ID
|
|
68
|
+
* @returns Promise<{ error: any }>
|
|
69
|
+
*/
|
|
70
|
+
async delete(queueName: string, messageID: number): Promise<{ error: any; }> {
|
|
71
|
+
try {
|
|
72
|
+
await this.connection.raw(`
|
|
73
|
+
SELECT * FROM pgmq.delete(
|
|
74
|
+
queue_name => ?,
|
|
75
|
+
msg_id => ?
|
|
76
|
+
);
|
|
77
|
+
`, [queueName, messageID]
|
|
78
|
+
)
|
|
79
|
+
|
|
80
|
+
return { error: null };
|
|
81
|
+
} catch (error) {
|
|
82
|
+
return { error };
|
|
83
|
+
}
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
export default PostgresQueueDriver;
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
import { Message, QueueDriver } from "../type";
|
|
2
|
+
import { SupabaseClient } from '@supabase/supabase-js';
|
|
3
|
+
|
|
4
|
+
class SupabaseQueueDriver implements QueueDriver {
|
|
5
|
+
|
|
6
|
+
constructor(
|
|
7
|
+
private supabase: SupabaseClient
|
|
8
|
+
) { }
|
|
9
|
+
|
|
10
|
+
/**
|
|
11
|
+
* Get the message
|
|
12
|
+
* @param queueName The name of the queue
|
|
13
|
+
* @param visibilityTime The visibility time of the message
|
|
14
|
+
* @param totalMessages The total messages to get
|
|
15
|
+
* @returns Promise<{ data: Message[], error: any }>
|
|
16
|
+
*/
|
|
17
|
+
async get(queueName: string, visibilityTime: number, totalMessages: number): Promise<{ data: Message[]; error: any; }> {
|
|
18
|
+
const { data, error } = await this.supabase.rpc("read", {
|
|
19
|
+
queue_name: queueName,
|
|
20
|
+
sleep_seconds: visibilityTime,
|
|
21
|
+
n: totalMessages
|
|
22
|
+
});
|
|
23
|
+
|
|
24
|
+
return { data: data as Message[], error };
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
/**
|
|
28
|
+
* Pop the message
|
|
29
|
+
* @param queueName The name of the queue
|
|
30
|
+
* @returns Promise<{ data: Message[], error: any }>
|
|
31
|
+
*/
|
|
32
|
+
async pop(queueName: string): Promise<{ data: Message[]; error: any; }> {
|
|
33
|
+
const { data, error } = await this.supabase.rpc("pop", {
|
|
34
|
+
queue_name: queueName,
|
|
35
|
+
});
|
|
36
|
+
|
|
37
|
+
return { data: data as Message[], error };
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
/**
|
|
41
|
+
* Delete the message
|
|
42
|
+
* @param queueName The name of the queue
|
|
43
|
+
* @param messageID The message ID
|
|
44
|
+
* @returns Promise<{ error: any }>
|
|
45
|
+
*/
|
|
46
|
+
async delete(queueName: string, messageID: number): Promise<{ error: any; }> {
|
|
47
|
+
const { error } = await this.supabase.rpc("delete", {
|
|
48
|
+
queue_name: queueName,
|
|
49
|
+
message_id: messageID
|
|
50
|
+
});
|
|
51
|
+
|
|
52
|
+
return { error };
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
export default SupabaseQueueDriver;
|
package/src/type.ts
ADDED
|
@@ -0,0 +1,97 @@
|
|
|
1
|
+
|
|
2
|
+
/*
|
|
3
|
+
* The type of consume. PS: "pop" get the message and delete it from database, "read" get the message and keep it on database
|
|
4
|
+
*/
|
|
5
|
+
type CONSUME_TYPE = "pop" | "read";
|
|
6
|
+
|
|
7
|
+
interface Options {
|
|
8
|
+
/**
|
|
9
|
+
* The name of the queue
|
|
10
|
+
*/
|
|
11
|
+
queueName: string;
|
|
12
|
+
/**
|
|
13
|
+
* The visibility time of the message. PS: is the time in seconds that the message will be invisible to other consumers
|
|
14
|
+
*/
|
|
15
|
+
visibilityTime?: number;
|
|
16
|
+
/**
|
|
17
|
+
* The type of consume.
|
|
18
|
+
*/
|
|
19
|
+
consumeType: CONSUME_TYPE;
|
|
20
|
+
/**
|
|
21
|
+
* The pool size. PS: the number of messages that will be consumed at the same time
|
|
22
|
+
*/
|
|
23
|
+
poolSize?: number;
|
|
24
|
+
/**
|
|
25
|
+
* The time to wait before next polling. PS: the time in milliseconds
|
|
26
|
+
*/
|
|
27
|
+
timeMsWaitBeforeNextPolling?: number;
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
/**
|
|
31
|
+
* The callback function to handle the message
|
|
32
|
+
* @param message The message
|
|
33
|
+
* @param signal The abort signal
|
|
34
|
+
* @returns Promise<void>
|
|
35
|
+
*/
|
|
36
|
+
type HandlerCallback = (message: { [key: string]: any }, signal: AbortSignal) => Promise<void>
|
|
37
|
+
|
|
38
|
+
interface Message {
|
|
39
|
+
/**
|
|
40
|
+
* The message ID
|
|
41
|
+
*/
|
|
42
|
+
msg_id: number;
|
|
43
|
+
/**
|
|
44
|
+
* The number of times the message has been read
|
|
45
|
+
*/
|
|
46
|
+
read_ct: number;
|
|
47
|
+
/**
|
|
48
|
+
* The time the message was enqueued
|
|
49
|
+
*/
|
|
50
|
+
enqueued_at: string;
|
|
51
|
+
/**
|
|
52
|
+
* The visibility time of the message
|
|
53
|
+
*/
|
|
54
|
+
vt: string;
|
|
55
|
+
/**
|
|
56
|
+
* The message content
|
|
57
|
+
*/
|
|
58
|
+
message: { [key: string]: any };
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
interface QueueDriver {
|
|
62
|
+
|
|
63
|
+
/**
|
|
64
|
+
* Get the message
|
|
65
|
+
* @param queueName The name of the queue
|
|
66
|
+
* @param visibilityTime The visibility time of the message
|
|
67
|
+
* @param totalMessages The total messages to get
|
|
68
|
+
* @returns Promise<{ data: Message[], error: any }>
|
|
69
|
+
*/
|
|
70
|
+
get(
|
|
71
|
+
queueName: string,
|
|
72
|
+
visibilityTime: number,
|
|
73
|
+
totalMessages: number
|
|
74
|
+
): Promise<{ data: Message[], error: any }>;
|
|
75
|
+
|
|
76
|
+
/**
|
|
77
|
+
* Pop the message
|
|
78
|
+
* @param queueName The name of the queue
|
|
79
|
+
* @returns Promise<{ data: Message[], error: any }>
|
|
80
|
+
*/
|
|
81
|
+
pop(
|
|
82
|
+
queueName: string,
|
|
83
|
+
): Promise<{ data: Message[], error: any }>;
|
|
84
|
+
|
|
85
|
+
/**
|
|
86
|
+
* Delete the message
|
|
87
|
+
* @param queueName The name of the queue
|
|
88
|
+
* @param messageID The message ID
|
|
89
|
+
* @returns Promise<{ error: any }>
|
|
90
|
+
*/
|
|
91
|
+
delete(
|
|
92
|
+
queueName: string,
|
|
93
|
+
messageID: number
|
|
94
|
+
): Promise<{ error: any }>;
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
export type { Options, HandlerCallback, Message, QueueDriver }
|
|
@@ -0,0 +1,237 @@
|
|
|
1
|
+
import Consumer from '../src/consumer'
|
|
2
|
+
import { Message, QueueDriver, Options } from '../src/type'
|
|
3
|
+
import timersPromises from 'node:timers/promises'
|
|
4
|
+
|
|
5
|
+
describe('Consumer', () => {
|
|
6
|
+
let queueDriver: jest.Mocked<QueueDriver>
|
|
7
|
+
|
|
8
|
+
const message: Message = ({
|
|
9
|
+
msg_id: 1,
|
|
10
|
+
read_ct: 0,
|
|
11
|
+
enqueued_at: new Date().toISOString(),
|
|
12
|
+
vt: new Date().toISOString(),
|
|
13
|
+
message: { foo: 'bar' },
|
|
14
|
+
})
|
|
15
|
+
|
|
16
|
+
beforeEach(() => {
|
|
17
|
+
queueDriver = {
|
|
18
|
+
get: jest.fn(),
|
|
19
|
+
pop: jest.fn(),
|
|
20
|
+
delete: jest.fn(),
|
|
21
|
+
}
|
|
22
|
+
jest.useFakeTimers();
|
|
23
|
+
})
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
afterEach(() => {
|
|
27
|
+
jest.clearAllMocks()
|
|
28
|
+
})
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
it('Should not process message if read method does not return any message second polling', async () => {
|
|
33
|
+
queueDriver.get.mockResolvedValueOnce({ data: [message], error: null })
|
|
34
|
+
queueDriver.delete.mockResolvedValueOnce({ error: null })
|
|
35
|
+
queueDriver.get.mockResolvedValueOnce({ data: [], error: null })
|
|
36
|
+
|
|
37
|
+
const handler = jest.fn(async () => { })
|
|
38
|
+
const consumer = new Consumer(
|
|
39
|
+
{
|
|
40
|
+
queueName: 'q',
|
|
41
|
+
consumeType: 'read',
|
|
42
|
+
visibilityTime: 1,
|
|
43
|
+
poolSize: 1,
|
|
44
|
+
timeMsWaitBeforeNextPolling: 1
|
|
45
|
+
},
|
|
46
|
+
handler,
|
|
47
|
+
queueDriver
|
|
48
|
+
)
|
|
49
|
+
|
|
50
|
+
const onFinish = jest.fn()
|
|
51
|
+
consumer.on('finish', onFinish)
|
|
52
|
+
|
|
53
|
+
await consumer.start()
|
|
54
|
+
|
|
55
|
+
expect(handler).toHaveBeenCalledTimes(1)
|
|
56
|
+
expect(onFinish).toHaveBeenCalledTimes(1)
|
|
57
|
+
expect(queueDriver.delete).toHaveBeenCalledTimes(1)
|
|
58
|
+
|
|
59
|
+
})
|
|
60
|
+
|
|
61
|
+
it('Should not process message if pop method does not return any message second polling', async () => {
|
|
62
|
+
queueDriver.pop.mockResolvedValueOnce({ data: [message], error: null })
|
|
63
|
+
queueDriver.pop.mockResolvedValueOnce({ data: [], error: null })
|
|
64
|
+
|
|
65
|
+
const handler = jest.fn(async () => { })
|
|
66
|
+
const consumer = new Consumer(
|
|
67
|
+
{
|
|
68
|
+
queueName: 'q',
|
|
69
|
+
consumeType: 'pop',
|
|
70
|
+
visibilityTime: 1,
|
|
71
|
+
poolSize: 1,
|
|
72
|
+
timeMsWaitBeforeNextPolling: 1
|
|
73
|
+
},
|
|
74
|
+
handler,
|
|
75
|
+
queueDriver
|
|
76
|
+
)
|
|
77
|
+
|
|
78
|
+
const onFinish = jest.fn()
|
|
79
|
+
consumer.on('finish', onFinish)
|
|
80
|
+
|
|
81
|
+
await consumer.start()
|
|
82
|
+
|
|
83
|
+
expect(handler).toHaveBeenCalledTimes(1)
|
|
84
|
+
expect(onFinish).toHaveBeenCalledTimes(1)
|
|
85
|
+
})
|
|
86
|
+
|
|
87
|
+
|
|
88
|
+
it('Should not process message if pop method does not return any message', async () => {
|
|
89
|
+
queueDriver.pop.mockResolvedValue({ data: [], error: null })
|
|
90
|
+
|
|
91
|
+
const handler = jest.fn(async () => { })
|
|
92
|
+
const consumer = new Consumer(
|
|
93
|
+
{
|
|
94
|
+
queueName: 'q',
|
|
95
|
+
consumeType: 'pop',
|
|
96
|
+
visibilityTime: 1,
|
|
97
|
+
poolSize: 1,
|
|
98
|
+
timeMsWaitBeforeNextPolling: 1
|
|
99
|
+
},
|
|
100
|
+
handler,
|
|
101
|
+
queueDriver
|
|
102
|
+
)
|
|
103
|
+
|
|
104
|
+
const onFinish = jest.fn()
|
|
105
|
+
consumer.on('finish', onFinish)
|
|
106
|
+
|
|
107
|
+
await consumer.start()
|
|
108
|
+
|
|
109
|
+
expect(handler).not.toHaveBeenCalled()
|
|
110
|
+
expect(onFinish).not.toHaveBeenCalled()
|
|
111
|
+
})
|
|
112
|
+
|
|
113
|
+
it('Should not process message if get method does not return any message', async () => {
|
|
114
|
+
queueDriver.get.mockResolvedValue({ data: [], error: null })
|
|
115
|
+
queueDriver.delete.mockResolvedValue({ error: null })
|
|
116
|
+
|
|
117
|
+
const handler = jest.fn(async () => { })
|
|
118
|
+
const consumer = new Consumer(
|
|
119
|
+
{
|
|
120
|
+
queueName: 'q',
|
|
121
|
+
consumeType: 'read',
|
|
122
|
+
visibilityTime: 1,
|
|
123
|
+
poolSize: 1,
|
|
124
|
+
timeMsWaitBeforeNextPolling: 1
|
|
125
|
+
},
|
|
126
|
+
handler,
|
|
127
|
+
queueDriver
|
|
128
|
+
)
|
|
129
|
+
|
|
130
|
+
const onFinish = jest.fn()
|
|
131
|
+
consumer.on('finish', onFinish)
|
|
132
|
+
|
|
133
|
+
await consumer.start()
|
|
134
|
+
|
|
135
|
+
expect(handler).not.toHaveBeenCalled()
|
|
136
|
+
expect(queueDriver.delete).not.toHaveBeenCalled()
|
|
137
|
+
expect(onFinish).not.toHaveBeenCalled()
|
|
138
|
+
})
|
|
139
|
+
|
|
140
|
+
|
|
141
|
+
it('Should process message, delete it, emit finish', async () => {
|
|
142
|
+
const msg = message
|
|
143
|
+
queueDriver.get.mockResolvedValue({ data: [msg], error: null })
|
|
144
|
+
queueDriver.delete.mockResolvedValue({ error: null })
|
|
145
|
+
|
|
146
|
+
const handler = jest.fn(async () => { })
|
|
147
|
+
const consumer = new Consumer(
|
|
148
|
+
{
|
|
149
|
+
queueName: 'q',
|
|
150
|
+
consumeType: 'read',
|
|
151
|
+
visibilityTime: 1,
|
|
152
|
+
poolSize: 1,
|
|
153
|
+
timeMsWaitBeforeNextPolling: 1
|
|
154
|
+
},
|
|
155
|
+
handler,
|
|
156
|
+
queueDriver
|
|
157
|
+
)
|
|
158
|
+
|
|
159
|
+
const onFinish = jest.fn()
|
|
160
|
+
consumer.on('finish', onFinish)
|
|
161
|
+
|
|
162
|
+
await consumer.start()
|
|
163
|
+
|
|
164
|
+
expect(handler).toHaveBeenCalledWith(msg.message, expect.any(AbortSignal))
|
|
165
|
+
expect(queueDriver.delete).toHaveBeenCalledWith('q', msg.msg_id)
|
|
166
|
+
expect(onFinish).toHaveBeenCalledWith(msg)
|
|
167
|
+
})
|
|
168
|
+
|
|
169
|
+
|
|
170
|
+
it('Should process 4 messages using consumeType get', async () => {
|
|
171
|
+
const msgs = [message, message, message, message]
|
|
172
|
+
queueDriver.get.mockResolvedValue({ data: msgs, error: null })
|
|
173
|
+
queueDriver.delete.mockResolvedValue({ error: null })
|
|
174
|
+
|
|
175
|
+
const handler = jest.fn(async () => { })
|
|
176
|
+
const consumer = new Consumer(
|
|
177
|
+
{
|
|
178
|
+
queueName: 'q',
|
|
179
|
+
consumeType: 'read',
|
|
180
|
+
visibilityTime: 1,
|
|
181
|
+
poolSize: 4,
|
|
182
|
+
timeMsWaitBeforeNextPolling: 1
|
|
183
|
+
},
|
|
184
|
+
handler,
|
|
185
|
+
queueDriver
|
|
186
|
+
)
|
|
187
|
+
|
|
188
|
+
const onFinish = jest.fn()
|
|
189
|
+
consumer.on('finish', onFinish)
|
|
190
|
+
|
|
191
|
+
await consumer.start()
|
|
192
|
+
|
|
193
|
+
expect(handler).toHaveBeenCalledWith(msgs[0].message, expect.any(AbortSignal))
|
|
194
|
+
expect(queueDriver.delete).toHaveBeenCalledWith('q', msgs[0].msg_id)
|
|
195
|
+
expect(onFinish).toHaveBeenCalledWith(msgs[0])
|
|
196
|
+
expect(onFinish).toHaveBeenCalledTimes(4)
|
|
197
|
+
})
|
|
198
|
+
|
|
199
|
+
it('Should process message using consumeType is pop emits finish', async () => {
|
|
200
|
+
const msg = message
|
|
201
|
+
queueDriver.pop.mockResolvedValue({ data: [msg], error: null })
|
|
202
|
+
|
|
203
|
+
const handler = jest.fn(async () => { })
|
|
204
|
+
const consumer = new Consumer(
|
|
205
|
+
{ queueName: 'q', consumeType: 'pop', timeMsWaitBeforeNextPolling: 1 } as Options,
|
|
206
|
+
handler,
|
|
207
|
+
queueDriver
|
|
208
|
+
)
|
|
209
|
+
|
|
210
|
+
const onFinish = jest.fn()
|
|
211
|
+
consumer.on('finish', onFinish)
|
|
212
|
+
|
|
213
|
+
await consumer.start()
|
|
214
|
+
|
|
215
|
+
expect(handler).toHaveBeenCalledWith(msg.message, expect.any(AbortSignal))
|
|
216
|
+
expect(queueDriver.delete).not.toHaveBeenCalled()
|
|
217
|
+
expect(onFinish).toHaveBeenCalledWith(msg)
|
|
218
|
+
expect(queueDriver.delete).toHaveBeenCalledTimes(0)
|
|
219
|
+
})
|
|
220
|
+
|
|
221
|
+
it('Should emit error when consumeType is read without visibilityTime', async () => {
|
|
222
|
+
const consumer = new Consumer(
|
|
223
|
+
{ queueName: 'q', consumeType: 'read' } as Options,
|
|
224
|
+
async () => { },
|
|
225
|
+
queueDriver
|
|
226
|
+
)
|
|
227
|
+
|
|
228
|
+
const onError = jest.fn()
|
|
229
|
+
consumer.on('error', onError)
|
|
230
|
+
|
|
231
|
+
await consumer.start()
|
|
232
|
+
|
|
233
|
+
expect(onError).toHaveBeenCalled()
|
|
234
|
+
expect((onError.mock.calls[0][0] as Error).message)
|
|
235
|
+
.toBe('visibilityTime is required for read')
|
|
236
|
+
})
|
|
237
|
+
})
|
package/tsconfig.json
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
{
|
|
2
|
+
"compilerOptions": {
|
|
3
|
+
"target": "es2020",
|
|
4
|
+
"module": "commonjs",
|
|
5
|
+
"outDir": "./dist",
|
|
6
|
+
"strict": true,
|
|
7
|
+
"esModuleInterop": true,
|
|
8
|
+
"skipLibCheck": true,
|
|
9
|
+
"forceConsistentCasingInFileNames": true
|
|
10
|
+
},
|
|
11
|
+
"include": ["src/**/*.ts"],
|
|
12
|
+
"exclude": ["node_modules"],
|
|
13
|
+
"ts-node": {
|
|
14
|
+
"transpileOnly": true,
|
|
15
|
+
"files": true,
|
|
16
|
+
"compilerOptions": {
|
|
17
|
+
"rootDir": "."
|
|
18
|
+
}
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
}
|