consumer-pgmq 2.0.1 → 3.1.0

package/README.md

@@ -11,6 +11,7 @@ This project is a consumer of Supabase/Postgresql queue(using pgmq extension) to
    - Pop consume type is when the consumer gets the message and the message is deleted from queue.
 - Support for both Supabase and Postgresql
 - Support for both visibility time and pool size
+- Support to control the limit of consumers running at once. PS: is focus on Postgres queue driver, is helpful when you have a weak database where no allow to have a lot of connections.
 
 ## Installation
 
@@ -40,7 +41,19 @@ yarn add consumer-pgmq
 - poolSize: The number of consumers. PS: this is the number of consumers that will be created to consume the messages and 
 if you use read consume type, the pool size is the number of messages will get at the same time.
 - timeMsWaitBeforeNextPolling: The time in milliseconds to wait before the next polling
-- enabledPolling: The enabled polling. PS: if true, the consumer will poll the message, if false, the consumer will consume the message one time and stop. PS: is required to the versions more than 1.0.5
+- enabledPolling: The enabled polling. PS: if true, the consumer will poll the message, if false, the consumer will consume the message one time and stop. PS: is required to the versions more than 1.0.5.
+- queueNameDlq: The name of the dead letter queue. PS: recommended to set the same name of the queue, but suffix with '_dlq'. For example: **messages_dlq**
+- totalRetriesBeforeSendToDlq: The total retries before send to dlq. For example: if you set totalRetriesBeforeSendToDlq equal 2, the message will be sent to dlq if the handler fails 2 times, so the third time the message will be sent to dlq and remove the main queue to avoid infinite retries.
+- enableControlConsumer: The enable control consumer. PS: if true, you can control the number of consumers running at once, because your Postgres database is weak and can be a problem to have a lot of connections. 
+    - Warning:
+        - You need to execute SQL to create the table 'workers' and you can use **schema.sql** file for it.
+        -  If you need to increase the number of workers you can insert new registers on table 'workers'.
+        - Each register on table workers will represent a process running as consumer(worker). PS: status column can be 'idle' or 'working'. The **idle** means is waiting to be allocate by a consumer and the **working** means was allocated by a consumer.
+
+## Extra points to know when use the dlq feature
+- The dead letter queue no work If you setted the consumerType option with value 'pop', because the pop get the message and remove from queue at same time, so if failed when you are processing you lose the message.
+- Recommendation no set lower value to the option 'visibilityTime' if you are using the dead letter queue feature. For example: set visibilityTime value lower than 30 seconds, because if the message wasn't delete and the message be available again the consumer application can consume the message again.
+- On Postgresql queue driver when you enable the option 'isCustomQueueImplementation', means you created the custom table to work as queue. PS: in this case you need to use the **schema.sql** file to create a queue table, if you dont want jobs and jobs_dlq as table name you can change in the **schema.sql** file.
 
 ## Events
 
@@ -89,11 +102,13 @@ async function start() {
     const consumer = new Consumer(
         {
             queueName: 'subscriptions',
-            visibilityTime: 15,
+            visibilityTime: 30,
             consumeType: "read",
-            poolSize: 4,
+            poolSize: 8,
             timeMsWaitBeforeNextPolling: 1000,
-            enabledPolling: false
+            enabledPolling: true,
+            queueNameDlq: "subscriptions_dlq",
+            totalRetriesBeforeSendToDlq: 2
         },
         async function (message: { [key: string]: any }, signal): Promise<void> {
             try {
@@ -143,33 +158,39 @@ start()
 import { config } from "dotenv"
 config()
 
-import { Consumer, PostgresQueueDriver } from "consumer-pgmq"
-import timersPromises from "node:timers/promises";
-import knex from 'knex'
+import Consumer from '../src/consumer';
+import PostgresQueueDriver from '../src/queueDriver/PostgresQueueDriver';
+
+import { Client } from 'pg'
 
 async function start() {
-    const connection = knex({
-        client: 'pg',
-        connection: {
-            host: process.env.POSTGRES_HOST,
-            database: process.env.POSTGRES_DATABASE,
-            password: process.env.POSTGRES_PASSWORD,
-            port: Number(process.env.POSTGRES_PORT),
-            user: process.env.POSTGRES_USER,
-            ssl: false
-        }
-    });
 
-    const postgresQueueDriver = new PostgresQueueDriver(connection, "schema_name_here")
+    const pgClient = new Client({
+        host: process.env.POSTGRES_HOST,
+        database: process.env.POSTGRES_DATABASE,
+        password: process.env.POSTGRES_PASSWORD,
+        port: Number(process.env.POSTGRES_PORT),
+        user: process.env.POSTGRES_USER,
+        ssl: false,
+    })
+
+    await pgClient.connect()
+
+
+    const postgresQueueDriver = new PostgresQueueDriver(
+        pgClient, "pgmq"
+    )
 
     const consumer = new Consumer(
         {
             queueName: 'subscriptions',
-            visibilityTime: 15,
+            visibilityTime: 30,
             consumeType: "read",
-            poolSize: 4,
+            poolSize: 8,
             timeMsWaitBeforeNextPolling: 1000,
-            enabledPolling: false
+            enabledPolling: true,
+            queueNameDlq: "subscriptions_dlq",
+            totalRetriesBeforeSendToDlq: 2
         },
         async function (message: { [key: string]: any }, signal): Promise<void> {
             try {
@@ -214,6 +235,82 @@ async function start() {
 start()
 ```
 
+- Consuming messages from Postgresql queue has option enabledControlConsumer and isCustomQueueImplementation enable on Postgres queue driver:
+
+```js
+import { config } from "dotenv"
+config()
+
+import Consumer from '../src/consumer';
+import PostgresQueueDriver from '../src/queueDriver/PostgresQueueDriver';
+
+import { Client } from 'pg'
+
+async function start() {
+
+    const pgClient = new Client({
+        host: process.env.POSTGRES_HOST,
+        database: process.env.POSTGRES_DATABASE,
+        password: process.env.POSTGRES_PASSWORD,
+        port: Number(process.env.POSTGRES_PORT),
+        user: process.env.POSTGRES_USER,
+        ssl: true,
+    })
+
+    await pgClient.connect()
+
+
+    const postgresQueueDriver = new PostgresQueueDriver(
+        pgClient, "public", true
+    )
+
+    const consumer = new Consumer(
+        {
+            queueName: 'jobs',
+            visibilityTime: 30,
+            consumeType: "read",
+            poolSize: 8,
+            timeMsWaitBeforeNextPolling: 1000,
+            enabledPolling: true,
+            queueNameDlq: "jobs_dlq",
+            totalRetriesBeforeSendToDlq: 2,
+            enableControlConsumer: true
+        },
+        async function (message: { [key: string]: any }, signal): Promise<void> {
+            console.log(message)
+        },
+        postgresQueueDriver
+    );
+
+    for (let index = 0; index < 100; index++) {
+        await postgresQueueDriver.send("jobs", {
+            message: `Message ${index}`,
+            id: index
+        })
+    }
+
+    consumer.on("send-to-dlq", (message: { [key: string]: any }) => {
+        console.log("Send to DLQ =>", message)
+    })
+
+    consumer.on('error', (err: Error) => {
+        console.error('Error consuming message:', err.message);
+    });
+
+    await consumer.start();
+
+    process.on("SIGINT", async () => {
+        await consumer.freeConsumer()
+        await pgClient.end()
+        process.exit(0)
+    })
+
+}
+
+start()
+```
+
+
 
 
 

package/dist/consumer.js

@@ -8,9 +8,18 @@ const READ = "read";
 class Consumer extends events_1.EventEmitter {
     constructor(options, callback, client) {
         super();
+        this.setTimeoutId = null;
+        this.id = null;
         this.options = options;
         this.callback = callback;
         this.client = client;
+        this.valideOptions();
+        this.setTimeoutId = null;
+    }
+    valideOptions() {
+        if (this.options.queueNameDlq && !this.options.totalRetriesBeforeSendToDlq) {
+            throw new Error("The option totalRetriesBeforeSendToDlq is required when queueNameDlq is set");
+        }
     }
     /**
      * Get the message
@@ -51,6 +60,9 @@ class Consumer extends events_1.EventEmitter {
      * @private
      */
     async pollMessage() {
+        if (this.setTimeoutId) {
+            clearTimeout(this.setTimeoutId);
+        }
         let promises = [];
         try {
             const { data, error } = await this.getMessage();
@@ -58,19 +70,35 @@ class Consumer extends events_1.EventEmitter {
                 throw error;
             }
             if (data.length === 0 && this.options.enabledPolling) {
-                setTimeout(() => this.pollMessage(), (this.options.timeMsWaitBeforeNextPolling || 1000) * 10);
+                this.setTimeoutId = setTimeout(() => this.pollMessage(), (this.options.timeMsWaitBeforeNextPolling || 1000) * 10);
                 return;
             }
             const controller = new AbortController();
             const signal = controller.signal;
-            for (let i = 0; i < (data.length || 1); i++) {
+            for (let i = 0; i < data.length; i++) {
+                const hasSendToDlq = data[i] &&
+                    this.options.queueNameDlq &&
+                    this.options.totalRetriesBeforeSendToDlq &&
+                    data[i].read_ct > this.options.totalRetriesBeforeSendToDlq;
+                if (hasSendToDlq) {
+                    promises.push(this.client.send(
+                    // @ts-ignore
+                    this.options.queueNameDlq, data[i].message, signal).then(async () => {
+                        await this.deleteMessage(data[i], signal);
+                        this.emit('send-to-dlq', data[i]);
+                    }));
+                    continue;
+                }
                 promises.push(this.callback(data[i].message, signal).then(async () => {
                     await this.deleteMessage(data[i], signal);
                     this.emit('finish', data[i]);
                 }));
             }
-            setTimeout(() => controller.abort(), (this.options.visibilityTime || 1) * 1000);
-            await Promise.allSettled(promises);
+            const timeoutId = setTimeout(() => controller.abort(), (this.options.visibilityTime || 1) * 1000);
+            if (promises.length > 0) {
+                await Promise.allSettled(promises);
+            }
+            clearTimeout(timeoutId);
             promises = [];
         }
         catch (err) {
@@ -85,7 +113,7 @@ class Consumer extends events_1.EventEmitter {
             if (!this.options.enabledPolling) {
                 return;
             }
-            setTimeout(() => this.pollMessage(), this.options.timeMsWaitBeforeNextPolling || 1000);
+            this.setTimeoutId = setTimeout(() => this.pollMessage(), this.options.timeMsWaitBeforeNextPolling || 1000);
         }
     }
     /**
@@ -94,7 +122,17 @@ class Consumer extends events_1.EventEmitter {
      * @public
      */
     async start() {
+        if (this.options.enableControlConsumer) {
+            const { id } = await this.client.allocateConsumer();
+            this.id = id;
+        }
         await this.pollMessage();
     }
+    async freeConsumer() {
+        if (this.id) {
+            console.log("passed on here");
+            await this.client.freeConsumer(this.id);
+        }
+    }
 }
 exports.default = Consumer;

package/dist/queueDriver/PostgresQueueDriver.js

@@ -1,9 +1,123 @@
 "use strict";
 Object.defineProperty(exports, "__esModule", { value: true });
 class PostgresQueueDriver {
-    constructor(connection, schema = "public") {
+    constructor(connection, schema = "public", isCustomQueueImplementation = true) {
         this.connection = connection;
         this.schema = schema;
+        this.isCustomQueueImplementation = isCustomQueueImplementation;
+    }
+    /**
+     * Allocate the consumer
+     * @returns Promise<{ id: string; }>
+     */
+    async allocateConsumer() {
+        const register = await this.connection.query(`
+            WITH next_workers AS (
+            SELECT id
+            FROM workers as w
+            WHERE
+                status = 'idle'
+            LIMIT 1
+            FOR UPDATE SKIP LOCKED
+        )
+        UPDATE workers
+        SET status = 'working',
+            updated_at = now()
+        FROM next_workers
+        WHERE workers.id = next_workers.id   
+        RETURNING workers.*;    
+        `);
+        if (register.rows.length == 0) {
+            throw new Error("No available consumer(worker) to allocate");
+        }
+        return { id: register.rows[0].id };
+    }
+    async freeConsumer(id) {
+        try {
+            await this.connection.query(`UPDATE workers set status = 'idle' WHERE id = $1`, [id]);
+        }
+        catch (error) {
+            throw error;
+        }
+    }
+    async sendCustomQueue(queueName, message) {
+        try {
+            const query = `INSERT INTO ${this.schema}.${queueName}(payload) VALUES($1)`;
+            await this.connection.query(query, [JSON.stringify(message)]);
+            return { error: null };
+        }
+        catch (error) {
+            return { error };
+        }
+    }
+    /**
+     * Send the message
+     * @param queueName The name of the queue
+     * @param message The message
+     * @returns Promise<{ error: any }>
+     */
+    async send(queueName, message) {
+        if (this.isCustomQueueImplementation) {
+            return this.sendCustomQueue(queueName, message);
+        }
+        try {
+            await this.connection.query(`
+            SELECT * FROM ${this.schema}.send(
+                queue_name => $1,
+                msg        => $2,
+                delay      => $3
+            );
+            `, [queueName, message, 1]);
+            return { error: null };
+        }
+        catch (error) {
+            return { error };
+        }
+    }
+    async getCustomQueue(queueName, visibilityTime, totalMessages) {
+        try {
+            const register = await this.connection.query(`
+                WITH next_job AS (
+                SELECT id
+                FROM ${queueName} as jobs
+                WHERE
+                    (
+                        status = 'pending'
+                        OR (status = 'in_progress' AND visible_at <= now())
+                    )
+                ORDER BY created_at
+                LIMIT $1
+                FOR UPDATE SKIP LOCKED
+            )
+            UPDATE jobs
+            SET status = 'in_progress',
+                updated_at = now(),
+                visible_at = now() + interval '${visibilityTime} seconds',
+                retry_count = retry_count + 1
+            FROM next_job
+            WHERE jobs.id = next_job.id   
+            RETURNING jobs.*;
+        `, [
+                totalMessages,
+            ]);
+            if (!register.rows) {
+                return { data: [], error: null };
+            }
+            const items = [];
+            for (const row of register.rows) {
+                items.push({
+                    msg_id: row.id,
+                    read_ct: row.retry_count,
+                    enqueued_at: row.created_at,
+                    vt: row.visible_at,
+                    message: row.payload, // Assuming the message content is stored in a column named 'payload' or '
+                });
+            }
+            return { data: items, error: null };
+        }
+        catch (error) {
+            return { data: [], error };
+        }
     }
     /**
      * Get the message
@@ -13,12 +127,15 @@ class PostgresQueueDriver {
      * @returns Promise<{ data: Message[], error: any }>
      */
     async get(queueName, visibilityTime, totalMessages) {
+        if (this.isCustomQueueImplementation) {
+            return this.getCustomQueue(queueName, visibilityTime, totalMessages);
+        }
         try {
-            const register = await this.connection.raw(`
+            const register = await this.connection.query(`
                 SELECT * FROM ${this.schema}.read(
-                    queue_name => ?,
-                    vt         => ?,
-                    qty        => ?
+                    queue_name => $1,
+                    vt         => $2,
+                    qty        => $3
                 );
                 `, [queueName, visibilityTime, totalMessages]);
             if (!register.rows) {
@@ -36,11 +153,18 @@ class PostgresQueueDriver {
      * @returns Promise<{ data: Message[], error: any }>
      */
     async pop(queueName) {
+        if (this.isCustomQueueImplementation) {
+            const result = await this.getCustomQueue(queueName, 30, 1);
+            if (result.data && result.data[0]) {
+                await this.delete(queueName, result.data[0].msg_id);
+            }
+            return result;
+        }
         try {
-            const register = await this.connection.raw(`
+            const register = await this.connection.query(`
                 SELECT * FROM ${this.schema}.pop(
-                    queue_name => ?
-                );
+                    queue_name => $1
+                )
                 `, [queueName]);
             if (!register.rows) {
                 return { data: [], error: null };
@@ -51,6 +175,12 @@ class PostgresQueueDriver {
             return { data: [], error };
         }
     }
+    async deleteCustomQueue(queueName, messageID) {
+        await this.connection.query(`
+            DELETE FROM     ${queueName}
+            WHERE id = $1;`, [messageID]);
+        return { error: null };
+    }
     /**
      * Delete the message
      * @param queueName The name of the queue
@@ -59,10 +189,13 @@ class PostgresQueueDriver {
      */
     async delete(queueName, messageID) {
         try {
-            await this.connection.raw(`
+            if (this.isCustomQueueImplementation) {
+                return this.deleteCustomQueue(queueName, messageID);
+            }
+            await this.connection.query(`
             SELECT * FROM ${this.schema}.delete(
-                queue_name => ?,
-                msg_id     => ?
+                queue_name => $1,
+                msg_id     => $2
             );
             `, [queueName, messageID]);
             return { error: null };

package/dist/queueDriver/SupabaseQueueDriver.js

@@ -4,6 +4,19 @@ class SupabaseQueueDriver {
     constructor(supabase) {
         this.supabase = supabase;
     }
+    /**
+     * Send the message
+     * @param queueName The name of the queue
+     * @param message The message
+     * @returns Promise<{ error: any }>
+     */
+    async send(queueName, message) {
+        const { error } = await this.supabase.rpc("send", {
+            queue_name: queueName,
+            message: message
+        });
+        return { error };
+    }
     /**
      * Get the message
      * @param queueName The name of the queue
@@ -43,5 +56,11 @@ class SupabaseQueueDriver {
         });
         return { error };
     }
+    allocateConsumer() {
+        throw new Error("method logic no implemented");
+    }
+    freeConsumer(id) {
+        throw new Error("method logic no implemented");
+    }
 }
 exports.default = SupabaseQueueDriver;

package/examples/consumerPostgresDriver.ts

@@ -3,70 +3,69 @@ config()
 
 import Consumer from '../src/consumer';
 import PostgresQueueDriver from '../src/queueDriver/PostgresQueueDriver';
-import timersPromises from "node:timers/promises";
-import knex from 'knex'
+
+import { Client } from 'pg'
 
 async function start() {
-    const connection = knex({
-        client: 'pg',
-        connection: {
-            host: process.env.POSTGRES_HOST,
-            database: process.env.POSTGRES_DATABASE,
-            password: process.env.POSTGRES_PASSWORD,
-            port: Number(process.env.POSTGRES_PORT),
-            user: process.env.POSTGRES_USER,
-            ssl: false,
-        }
-    });
 
-    const postgresQueueDriver = new PostgresQueueDriver(connection, "pgmq")
+    const pgClient = new Client({
+        host: process.env.POSTGRES_HOST,
+        database: process.env.POSTGRES_DATABASE,
+        password: process.env.POSTGRES_PASSWORD,
+        port: Number(process.env.POSTGRES_PORT),
+        user: process.env.POSTGRES_USER,
+        ssl: true,
+    })
+
+    await pgClient.connect()
+
+
+    const postgresQueueDriver = new PostgresQueueDriver(
+        pgClient, "public", true
+    )
 
     const consumer = new Consumer(
         {
-            queueName: 'subscriptions',
-            visibilityTime: 15,
+            queueName: 'jobs',
+            visibilityTime: 30,
             consumeType: "read",
-            poolSize: 4,
+            poolSize: 8,
             timeMsWaitBeforeNextPolling: 1000,
-            enabledPolling: true
+            enabledPolling: true,
+            queueNameDlq: "jobs_dlq",
+            totalRetriesBeforeSendToDlq: 2,
+            enableControlConsumer: true
         },
         async function (message: { [key: string]: any }, signal): Promise<void> {
-            try {
-                console.log(message)
-                // const url = "https://jsonplaceholder.typicode.com/todos/1";
-                // await timersPromises.setTimeout(100, null, { signal });
-                // console.log("Fetching data...");
-                // const response = await fetch(url, { signal });
-                // const todo = await response.json();
-                // console.log("Todo:", todo);
-            } catch (error: any) {
-                if (error.name === "AbortError") {
-                    console.log("Operation aborted");
-                } else {
-                    console.error("Error:", error);
-                }
-            }
+            console.log(message)
         },
         postgresQueueDriver
     );
 
-    // consumer.on('finish', (message: { [key: string]: any }) => {
-    //     console.log('Consumed message =>', message);
-    // });
+    for (let index = 0; index < 100; index++) {
+        await postgresQueueDriver.send("jobs", {
+            message: `Message ${index}`,
+            id: index
+        })
+    }
+
+    console.log("@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@")
 
-    consumer.on("abort-error", (err) => {
-        console.log("Abort error =>", err)
+    consumer.on("send-to-dlq", (message: { [key: string]: any }) => {
+        console.log("Send to DLQ =>", message)
     })
 
     consumer.on('error', (err: Error) => {
-        if (err.message.includes("TypeError: fetch failed")) {
-            console.log(err)
-            process.exit(1);
-        }
         console.error('Error consuming message:', err.message);
     });
 
-    consumer.start();
+    await consumer.start();
+
+    process.on("SIGINT", async () => {
+        await consumer.freeConsumer()
+        await pgClient.end()
+        process.exit(0)
+    })
 
 }
 

package/examples/consumerSupabaseDriver.ts

@@ -21,33 +21,30 @@ const supabaseQueueDriver = new SupabaseQueueDriver(
     supabase as unknown as SupabaseClient
 )
 
-
-import timersPromises from "node:timers/promises";
-
 async function start() {
-    // for (let i = 0; i < 200; i++) {
-    //     await supabase.rpc("send", {
-    //         queue_name: "subscriptions",
-    //         message: { "message": `Message triggered at ${Date.now()}` }
-    //     });
-    // }
-    // console.log("Total messages sent: ", 200)
+    for (let i = 0; i < 50; i++) {
+        await supabase.rpc("send", {
+            queue_name: "subscriptions",
+            message: { "message": `Message triggered at ${Date.now()}` }
+        });
+    }
+    console.log("Total messages sent: ", 50)
 
     const consumer = new Consumer(
         {
             queueName: 'subscriptions',
-            visibilityTime: 15,
+            visibilityTime: 30,
             consumeType: "read",
             poolSize: 8,
             timeMsWaitBeforeNextPolling: 1000,
-            enabledPolling: false
+            enabledPolling: true,
+            queueNameDlq: "subscriptions_dlq",
+            totalRetriesBeforeSendToDlq: 2
         },
         async function (message: { [key: string]: any }, signal): Promise<void> {
             try {
-                if (message.error) {
-                    throw new Error("Error in message")
-                }
                 console.log(message)
+                throw new Error("Error in message")
             } catch (error: any) {
                 throw error
             }
@@ -59,6 +56,9 @@ async function start() {
     //     console.log('Consumed message =>', message);
     // });
 
+    consumer.on("send-to-dlq", (message: { [key: string]: any }) => {
+        console.log("Send to DLQ =>", message)
+    })
     consumer.on("abort-error", (err) => {
         console.log("Abort error =>", err)
     })