@sera4/essentia 3.0.13 → 3.1.0-rc.1

package/dist/ts/queue/index.d.ts

@@ -0,0 +1,232 @@
+interface ConnectionOptions {
+    connectionUrl: string;
+    logEnabled?: boolean;
+    retryInterval?: number;
+}
+interface Exchange {
+    name: string;
+    type: string;
+    options?: any;
+}
+interface PublisherOptions {
+    exchange: Exchange;
+}
+interface SubscriberOptions {
+    exchange: Exchange;
+    queue: {
+        name: string;
+        options?: any;
+    };
+    key: string;
+}
+interface UnregisterSubscriberOptions {
+    exchange: Exchange;
+    queue: {
+        name: string;
+    };
+    key: string;
+}
+interface FanoutQueueConfig {
+    name: string;
+    routingKey?: string;
+    options?: any;
+}
+interface FanoutConfiguration {
+    exchange: {
+        name: string;
+        type: "fanout";
+        options?: any;
+    };
+    queues: FanoutQueueConfig[];
+}
+type MessageListener = (payload: any, error?: any) => void;
+/**
+ * An interface to AMQP service
+ */
+declare class S4Queue {
+    private connection;
+    private connectionTestChannel;
+    private retry;
+    private pubs;
+    private subs;
+    private fanouts;
+    constructor();
+    /**
+     * Tests the connection to the service.
+     * Internally, the same connection is used for test
+     * and production code, but a separate channel is opened.
+     * Once the channel is obtained, there is an assert on the
+     * existence of a topic exchange called '--health-check--'.
+     * The following must be satisfied for the test to pass
+     * 1. must be connected
+     * 2. must have a test channel
+     * 3. must assert the exchange
+     */
+    testConnection(): Promise<void>;
+    /**
+     * Opens a connection to AMQP server.
+     * @param options Connection parameters. At the very least options.connectionUrl must be present.
+     */
+    openConnection(options: ConnectionOptions): Promise<void>;
+    /**
+     * Closes the connection to AMQP server
+     * and clears the registered publishers
+     * and subscribers
+     */
+    closeConnection(): Promise<void>;
+    /**
+     * Returns whether we have an active connection
+     * or not.
+     */
+    isConnected(): boolean;
+    /**
+     * @deprecated Use registerPublisher
+     * Registers a new direct publisher on the given exchange.
+     * If exchange doesn't exist, one will be created.
+     * @param exchange the name of the exchange on AMQP service.
+     */
+    registerDirectPublisher(exchange: string): Promise<void>;
+    /**
+     * @deprecated Use registerPublisher
+     * Registers a new topic publisher on the given exchange.
+     * If exchange doesn't exist, one will be created.
+     * @param exchange the name of the exchange on AMQP service.
+     */
+    registerTopicPublisher(exchange: string): Promise<void>;
+    /**
+     * @deprecated Use registerPublisher
+     * Registers a new fanout publisher on the given exchange.
+     * If exchange doesn't exist, one will be created.
+     * @param exchange the name of the exchange on AMQP service.
+     */
+    registerFanoutPublisher(exchange: string): Promise<void>;
+    /**
+     * Registers a new publisher
+     * @param options connection options.
+     */
+    registerPublisher(options: PublisherOptions): Promise<void>;
+    /**
+     * Checks whether or not a publisher with given parameters
+     * has been registered already
+     * @param exchange the name of the exchange.
+     * @returns true if registered already, false otherwise
+     */
+    hasPublisher(exchange: string): boolean;
+    /**
+     * Checks whether or not a fanout with given exchange name
+     * has been registered already
+     * @param exchange the name of the exchange.
+     * @returns true if registered already, false otherwise
+     */
+    hasFanout(exchange: string): boolean;
+    /**
+     * Removes a previously registered publisher
+     * @param options
+     */
+    unregisterPublisher(options: PublisherOptions): Promise<void>;
+    /**
+     * @deprecated Use registerSubscriber
+     * Registers a new direct subscriber on a given exchange and queue.
+     * If exchange and queue don't exists, they'll be created
+     * and bound through the provided routing key.
+     * @param exchange the name of the exchange.
+     * @param queue the name of the queue.
+     * @param key the name of the routing key.
+     */
+    registerDirectSubscriber(exchange: string, queue: string, key: string): Promise<void>;
+    /**
+     * @deprecated Use registerSubscriber
+     * Registers a new topic subscriber on a given exchange and queue.
+     * If exchange and queue don't exists, they'll be created
+     * and bound through the provided routing key.
+     * @param exchange the name of the exchange.
+     * @param queue the name of the queue.
+     * @param key the name of the routing key.
+     */
+    registerTopicSubscriber(exchange: string, queue: string, key: string): Promise<void>;
+    /**
+     * An overloaded version of registering a subscriber
+     * allowing for more fine tuned parameters
+     * @param options subscribing options
+     */
+    registerSubscriber(options: SubscriberOptions): Promise<void>;
+    /**
+     * Checks whether or not a subscriber with given parameters
+     * has been registered already
+     * @param exchange the name of the exchange.
+     * @param queue the name of the queue.
+     * @param key the name of the routing key.
+     * @returns true if registered already, false otherwise
+     */
+    hasSubscriber(exchange: string, queue: string, key: string): boolean;
+    /**
+     * Removes a previously registered subscriber
+     * @param options
+     */
+    unregisterSubscriber(options: UnregisterSubscriberOptions): Promise<void>;
+    /**
+     * Publishes a message on a given exchange with routing key.
+     * @param exchange the name of the exchange.
+     * @param key the name of the routing key.
+     * @param msg a JSON object as the message.
+     */
+    publishDirectMessage(exchange: string, key: string, msg: any): Promise<string>;
+    /**
+     * Publishes a message on a given exchange with a topic.
+     * @param exchange the name of the exchange.
+     * @param topic the name of the topic.
+     * @param msg a JSON object as the message.
+     */
+    publishTopicMessage(exchange: string, topic: string, msg: any): Promise<string>;
+    /**
+     * Publish a message to the exchange
+     * @param exchange name of exchange
+     * @param key name of the key or topic
+     * @param msg payload to be published
+     */
+    publishMessage(exchange: string, key: string, msg: any): Promise<string>;
+    /**
+     * Binds an exchange to a queue
+     * @param exchange the name of the exchange.
+     * @param queue the name of the queue.
+     * @param pattern (optional) pattern for routing key
+     */
+    bindQueueToExchange(exchange: string, queue: string, pattern?: string): Promise<void>;
+    /**
+     * Adds a listener for a specific key.
+     * If a subscriber that was registered was found
+     * with that key, the listener is added, otherwise
+     * it's ignored
+     * @param key AMQP routing key
+     * @param listener callback
+     */
+    addMessageListener(key: string, listener: MessageListener): void;
+    /**
+     * Registers a fanout exchange with multiple bound queues.
+     * Creates the fanout exchange if it doesn't exist, then creates and binds
+     * all specified queues to the exchange.
+     * @param config Fanout configuration with exchange and queue definitions
+     */
+    registerFanout(config: FanoutConfiguration): Promise<void>;
+    /**
+     * Unregisters a fanout configuration by removing all associated subscribers
+     * and optionally the publisher/exchange.
+     * @param config Fanout configuration to unregister
+     * @param removeExchange Whether to also remove the exchange (default: false)
+     */
+    unregisterFanout(config: FanoutConfiguration, removeExchange?: boolean): Promise<void>;
+    /**
+     * Updates the connection reference to all the previously
+     * registered publishers, subscribers, and fanouts.
+     * After we successfully establish a connection and register pubs/subs/fanouts,
+     * it's possible for the connection to break. This object will try
+     * to re-establish a new connection every 3 seconds (or whatever configuration is set)
+     * and if a new connection is established, we need to pass that connection to
+     * all our pubs/subs, otherwise they'll be operating with old/stale references
+     * @param connRef a new connection reference to the AMQP service
+     */
+    updatePubsAndSubsConnectionRef(connRef: any): void;
+}
+declare const queue: S4Queue;
+export default queue;
+export type { Exchange, FanoutConfiguration, FanoutQueueConfig, PublisherOptions, SubscriberOptions };

package/dist/ts/queue/index.js

@@ -0,0 +1,446 @@
+import amqp from "amqplib";
+import urlParse from "url-parse";
+import { Publisher } from "./publisher.js";
+import logger from "./queue-logger.js";
+import { Subscriber } from "./subscriber.js";
+/**
+ * An interface to AMQP service
+ */
+class S4Queue {
+    connection;
+    connectionTestChannel;
+    retry;
+    pubs;
+    subs;
+    fanouts;
+    constructor() {
+        this.connection = null;
+        this.connectionTestChannel = null;
+        this.retry = 10;
+        this.pubs = {};
+        this.subs = {};
+        this.fanouts = {};
+    }
+    /**
+     * Tests the connection to the service.
+     * Internally, the same connection is used for test
+     * and production code, but a separate channel is opened.
+     * Once the channel is obtained, there is an assert on the
+     * existence of a topic exchange called '--health-check--'.
+     * The following must be satisfied for the test to pass
+     * 1. must be connected
+     * 2. must have a test channel
+     * 3. must assert the exchange
+     */
+    async testConnection() {
+        if (!this.isConnected()) {
+            this.connectionTestChannel = null;
+            throw new Error("queue service not connected");
+        }
+        if (!this.connectionTestChannel) {
+            const ch = await this.connection.createChannel();
+            if (!ch) {
+                throw new Error("queue service failed to obtain a channel");
+            }
+            ch.on("error", () => { this.connectionTestChannel = null; });
+            ch.on("close", () => { this.connectionTestChannel = null; });
+            this.connectionTestChannel = ch;
+        }
+        await this.connectionTestChannel.assertExchange("--health-check--", "topic", { durable: false });
+    }
+    /**
+     * Opens a connection to AMQP server.
+     * @param options Connection parameters. At the very least options.connectionUrl must be present.
+     */
+    async openConnection(options) {
+        logger.setEnabled(options.logEnabled || false);
+        if (this.connection) {
+            logger.warn("Already initialized");
+            return;
+        }
+        const retry = parseInt(options.retryInterval?.toString() || "10", 10);
+        this.retry = retry || 10;
+        // We may run into issues with SNI TLS if
+        // we host with 3rd parties using our own certs.
+        // To avoid any issues, let's parse the url
+        // and provide the server name in the socket options
+        const urlParts = urlParse(options.connectionUrl);
+        const socketOptions = { servername: urlParts.hostname };
+        return new Promise((resolve) => {
+            const attempt = async () => {
+                logger.debug("Attempting connection...");
+                try {
+                    const conn = await amqp.connect(options.connectionUrl, socketOptions);
+                    this.connection = conn;
+                    this.updatePubsAndSubsConnectionRef(conn);
+                    this.connection.on("error", (err) => {
+                        logger.error("connection error", err);
+                        // Do not attempt to reconnect as the
+                        // 'close' event will also be emitted
+                        // if this error closes the connection
+                    });
+                    this.connection.on("close", (err) => {
+                        logger.error("connection closed");
+                        this.connection = null;
+                        this.connectionTestChannel = null;
+                        if (err) {
+                            logger.error("connection closed due to error:", err);
+                            logger.error("Reconnecting in", this.retry, "seconds");
+                            setTimeout(attempt, this.retry * 1000);
+                        }
+                        else {
+                            logger.debug("Will not re-attempt");
+                        }
+                    });
+                    logger.debug("Connected");
+                    resolve();
+                }
+                catch (err) {
+                    logger.error("Error while connecting", err);
+                    logger.error("Attempting again in", this.retry, "seconds");
+                    setTimeout(attempt, this.retry * 1000);
+                }
+            };
+            attempt();
+        });
+    }
+    /**
+     * Closes the connection to AMQP server
+     * and clears the registered publishers
+     * and subscribers
+     */
+    async closeConnection() {
+        if (!this.connection)
+            return;
+        await this.connection.close();
+        this.subs = {};
+        this.pubs = {};
+        this.connectionTestChannel = null;
+        this.connection = null;
+    }
+    /**
+     * Returns whether we have an active connection
+     * or not.
+     */
+    isConnected() {
+        return this.connection != null && typeof this.connection !== "undefined";
+    }
+    /**
+     * @deprecated Use registerPublisher
+     * Registers a new direct publisher on the given exchange.
+     * If exchange doesn't exist, one will be created.
+     * @param exchange the name of the exchange on AMQP service.
+     */
+    async registerDirectPublisher(exchange) {
+        return this.registerPublisher({ exchange: { name: exchange, type: "direct" } });
+    }
+    /**
+     * @deprecated Use registerPublisher
+     * Registers a new topic publisher on the given exchange.
+     * If exchange doesn't exist, one will be created.
+     * @param exchange the name of the exchange on AMQP service.
+     */
+    async registerTopicPublisher(exchange) {
+        return this.registerPublisher({ exchange: { name: exchange, type: "topic" } });
+    }
+    /**
+     * @deprecated Use registerPublisher
+     * Registers a new fanout publisher on the given exchange.
+     * If exchange doesn't exist, one will be created.
+     * @param exchange the name of the exchange on AMQP service.
+     */
+    async registerFanoutPublisher(exchange) {
+        return this.registerPublisher({ exchange: { name: exchange, type: "fanout" } });
+    }
+    /**
+     * Registers a new publisher
+     * @param options connection options.
+     */
+    async registerPublisher(options) {
+        const { exchange } = options;
+        const { name } = exchange;
+        if (Object.keys(this.pubs).includes(name)) {
+            throw new Error(`Publisher ${name} already registered`);
+        }
+        const p = new Publisher(exchange, name);
+        await p.init(this.connection);
+        this.pubs[name] = p;
+    }
+    /**
+     * Checks whether or not a publisher with given parameters
+     * has been registered already
+     * @param exchange the name of the exchange.
+     * @returns true if registered already, false otherwise
+     */
+    hasPublisher(exchange) {
+        return Object.keys(this.pubs).includes(exchange);
+    }
+    /**
+     * Checks whether or not a fanout with given exchange name
+     * has been registered already
+     * @param exchange the name of the exchange.
+     * @returns true if registered already, false otherwise
+     */
+    hasFanout(exchange) {
+        return Object.keys(this.fanouts).includes(exchange);
+    }
+    /**
+     * Removes a previously registered publisher
+     * @param options
+     */
+    async unregisterPublisher(options) {
+        const { exchange } = options;
+        const { name } = exchange;
+        if (!Object.keys(this.pubs).includes(name)) {
+            throw new Error(`Publisher ${name} not registered`);
+        }
+        const p = this.pubs[name];
+        await p.closeChannel();
+        delete this.pubs[name];
+        logger.debug("Unregistered publisher", name);
+    }
+    /**
+     * @deprecated Use registerSubscriber
+     * Registers a new direct subscriber on a given exchange and queue.
+     * If exchange and queue don't exists, they'll be created
+     * and bound through the provided routing key.
+     * @param exchange the name of the exchange.
+     * @param queue the name of the queue.
+     * @param key the name of the routing key.
+     */
+    async registerDirectSubscriber(exchange, queue, key) {
+        const options = {
+            key,
+            exchange: { name: exchange, type: "direct" },
+            queue: { name: queue },
+        };
+        return this.registerSubscriber(options);
+    }
+    /**
+     * @deprecated Use registerSubscriber
+     * Registers a new topic subscriber on a given exchange and queue.
+     * If exchange and queue don't exists, they'll be created
+     * and bound through the provided routing key.
+     * @param exchange the name of the exchange.
+     * @param queue the name of the queue.
+     * @param key the name of the routing key.
+     */
+    async registerTopicSubscriber(exchange, queue, key) {
+        const options = {
+            key,
+            exchange: { name: exchange, type: "topic" },
+            queue: { name: queue },
+        };
+        return this.registerSubscriber(options);
+    }
+    /**
+     * An overloaded version of registering a subscriber
+     * allowing for more fine tuned parameters
+     * @param options subscribing options
+     */
+    async registerSubscriber(options) {
+        const { exchange, queue, key } = options;
+        const name = `${exchange.name}-${queue.name}-${key}`;
+        if (Object.keys(this.subs).includes(name)) {
+            throw new Error(`Subscriber ${name} already registered`);
+        }
+        const s = new Subscriber(exchange, queue, key, name);
+        await s.init(this.connection);
+        this.subs[name] = s;
+    }
+    /**
+     * Checks whether or not a subscriber with given parameters
+     * has been registered already
+     * @param exchange the name of the exchange.
+     * @param queue the name of the queue.
+     * @param key the name of the routing key.
+     * @returns true if registered already, false otherwise
+     */
+    hasSubscriber(exchange, queue, key) {
+        const name = `${exchange}-${queue}-${key}`;
+        return Object.keys(this.subs).includes(name);
+    }
+    /**
+     * Removes a previously registered subscriber
+     * @param options
+     */
+    async unregisterSubscriber(options) {
+        const { exchange, queue, key } = options;
+        const name = `${exchange.name}-${queue.name}-${key}`;
+        if (!Object.keys(this.subs).includes(name)) {
+            throw new Error(`Subscriber ${name} not registered`);
+        }
+        const s = this.subs[name];
+        await s.closeChannel();
+        delete this.subs[name];
+        logger.debug("Unregistered subscriber", name);
+    }
+    /**
+     * Publishes a message on a given exchange with routing key.
+     * @param exchange the name of the exchange.
+     * @param key the name of the routing key.
+     * @param msg a JSON object as the message.
+     */
+    async publishDirectMessage(exchange, key, msg) {
+        return this.publishMessage(exchange, key, msg);
+    }
+    /**
+     * Publishes a message on a given exchange with a topic.
+     * @param exchange the name of the exchange.
+     * @param topic the name of the topic.
+     * @param msg a JSON object as the message.
+     */
+    async publishTopicMessage(exchange, topic, msg) {
+        return this.publishMessage(exchange, topic, msg);
+    }
+    /**
+     * Publish a message to the exchange
+     * @param exchange name of exchange
+     * @param key name of the key or topic
+     * @param msg payload to be published
+     */
+    async publishMessage(exchange, key, msg) {
+        if (!key) {
+            throw new Error(`Key argument not defined`);
+        }
+        const p = this.pubs[exchange];
+        if (!p) {
+            throw new Error(`No PUB for ${exchange} found`);
+        }
+        let content = msg;
+        if (content.constructor !== {}.constructor) {
+            // if not in JSON format, let's make it so
+            content = {
+                msg: msg,
+            };
+        }
+        return await p.publish({ key, content });
+    }
+    /**
+     * Binds an exchange to a queue
+     * @param exchange the name of the exchange.
+     * @param queue the name of the queue.
+     * @param pattern (optional) pattern for routing key
+     */
+    async bindQueueToExchange(exchange, queue, pattern) {
+        const p = this.pubs[exchange];
+        if (!p) {
+            throw new Error(`No PUB for ${exchange} found`);
+        }
+        if (!queue) {
+            throw new Error(`Binding a queue requires a queue name`);
+        }
+        return await p.bindQueue(queue, pattern || "");
+    }
+    /**
+     * Adds a listener for a specific key.
+     * If a subscriber that was registered was found
+     * with that key, the listener is added, otherwise
+     * it's ignored
+     * @param key AMQP routing key
+     * @param listener callback
+     */
+    addMessageListener(key, listener) {
+        Object.keys(this.subs).forEach((k) => {
+            const s = this.subs[k];
+            if (s.key === key) {
+                s.registerCallback(listener);
+            }
+        });
+    }
+    /**
+     * Registers a fanout exchange with multiple bound queues.
+     * Creates the fanout exchange if it doesn't exist, then creates and binds
+     * all specified queues to the exchange.
+     * @param config Fanout configuration with exchange and queue definitions
+     */
+    async registerFanout(config) {
+        const { exchange, queues } = config;
+        logger.debug(`Registering fanout configuration for exchange: ${exchange.name}`);
+        // First, register the fanout publisher (this creates the exchange)
+        await this.registerPublisher({ exchange });
+        // Then register subscribers for each queue
+        for (const queueConfig of queues) {
+            const subscriberOptions = {
+                exchange,
+                queue: {
+                    name: queueConfig.name,
+                    options: queueConfig.options,
+                },
+                key: queueConfig.routingKey || "", // Fanout exchanges ignore routing keys, but we need it for the interface
+            };
+            await this.registerSubscriber(subscriberOptions);
+            logger.debug(`Registered fanout subscriber for queue: ${queueConfig.name}${queueConfig.routingKey ? ` with routing key: ${queueConfig.routingKey}` : " (no routing key - fanout ignores it)"}`);
+        }
+        // Store the fanout configuration
+        this.fanouts[exchange.name] = config;
+        logger.info(`Successfully registered fanout configuration with ${queues.length} queues`);
+    }
+    /**
+     * Unregisters a fanout configuration by removing all associated subscribers
+     * and optionally the publisher/exchange.
+     * @param config Fanout configuration to unregister
+     * @param removeExchange Whether to also remove the exchange (default: false)
+     */
+    async unregisterFanout(config, removeExchange = false) {
+        const { exchange, queues } = config;
+        logger.debug(`Unregistering fanout configuration for exchange: ${exchange.name}`);
+        // Unregister all subscribers
+        for (const queueConfig of queues) {
+            const unregisterOptions = {
+                exchange,
+                queue: {
+                    name: queueConfig.name,
+                },
+                key: queueConfig.routingKey || "",
+            };
+            try {
+                await this.unregisterSubscriber(unregisterOptions);
+                logger.debug(`Unregistered fanout subscriber for queue: ${queueConfig.name}${queueConfig.routingKey ? ` with routing key: ${queueConfig.routingKey}` : " (no routing key)"}`);
+            }
+            catch (error) {
+                logger.warn(`Failed to unregister subscriber for queue ${queueConfig.name}:`, error);
+            }
+        }
+        // Optionally remove the exchange/publisher
+        if (removeExchange) {
+            try {
+                await this.unregisterPublisher({ exchange });
+                logger.debug(`Unregistered fanout publisher for exchange: ${exchange.name}`);
+            }
+            catch (error) {
+                logger.warn(`Failed to unregister publisher for exchange ${exchange.name}:`, error);
+            }
+        }
+        // Remove the fanout configuration from storage
+        delete this.fanouts[exchange.name];
+        logger.info(`Successfully unregistered fanout configuration`);
+    }
+    /**
+     * Updates the connection reference to all the previously
+     * registered publishers, subscribers, and fanouts.
+     * After we successfully establish a connection and register pubs/subs/fanouts,
+     * it's possible for the connection to break. This object will try
+     * to re-establish a new connection every 3 seconds (or whatever configuration is set)
+     * and if a new connection is established, we need to pass that connection to
+     * all our pubs/subs, otherwise they'll be operating with old/stale references
+     * @param connRef a new connection reference to the AMQP service
+     */
+    updatePubsAndSubsConnectionRef(connRef) {
+        Object.keys(this.pubs).forEach((k) => {
+            logger.debug(`Updating publisher ${k} with a new connection object`);
+            const p = this.pubs[k];
+            p.init(connRef);
+        });
+        Object.keys(this.subs).forEach((k) => {
+            logger.debug(`Updating subscriber ${k} with a new connection object`);
+            const s = this.subs[k];
+            s.init(connRef);
+        });
+        // Note: Fanouts don't need connection updates as they use the same pubs/subs
+        logger.debug(`Updated connection for ${Object.keys(this.pubs).length} publishers and ${Object.keys(this.subs).length} subscribers`);
+    }
+}
+const queue = new S4Queue();
+export default queue;
+//# sourceMappingURL=index.js.map