@6thbridge/utils 0.1.16

package/lib/brokers/kafka.js

@@ -0,0 +1,265 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.KafkaJS = void 0;
+const kafkajs_1 = require("kafkajs");
+const logger_1 = require("../logger/logger");
+class KafkaJS {
+    constructor(options) {
+        this.connection = null;
+        this.retryCountBeforeExit = 10;
+        this.admin = null;
+        this.connection = null;
+        this.init(options || {});
+    }
+    /**
+     *
+     * @param {Object} options
+     * @param {string} options.url
+     * @param {number} options.retryCountBeforeExit
+     * @param {number} options.heartbeat
+     * @return {Promise<null|*|undefined>}
+     */
+    init(options) {
+        options.url = options.url || process.env.KAFKA_CLUSTER_URL;
+        options.name = options.name || process.env.APP_NAME;
+        options.connectionTimeout = options.connectionTimeout || parseInt(process.env.KAFKA_CONNECTION_TIMEOUT) || 3000;
+        options.requestTimeout = options.requestTimeout || parseInt(process.env.KAFKA_REQUEST_TIMEOUT) || 25000;
+        options.logLevel = options.logLevel === 0 ? options.logLevel : parseInt(process.env.KAFKA_LOG_LEVEL) || kafkajs_1.logLevel.ERROR;
+        options.saslUsername = options.saslUsername || process.env.KAFKA_SASL_USERNAME;
+        options.saslPassword = options.saslPassword || process.env.KAFKA_SASL_PASSWORD;
+        options.saslMechanism = options.saslMechanism || "plain";
+        try {
+            if (!options.url)
+                throw Error("KAFKA URL is required");
+            //We are splitting the URL because of clusters
+            const kafkaConfig = {
+                clientId: options.name,
+                brokers: options.url.split(","),
+                connectionTimeout: options.connectionTimeout,
+                requestTimeout: options.requestTimeout,
+                logLevel: options.logLevel,
+                retry: {
+                    initialRetryTime: options.initialRetryTime || 100,
+                    retries: options.retries || 10,
+                }
+            };
+            if (options.saslMechanism && options.saslPassword) {
+                kafkaConfig.ssl = true;
+                kafkaConfig.sasl = {
+                    mechanism: options.saslMechanism, // scram-sha-256 or scram-sha-512
+                    username: options.saslUsername,
+                    password: options.saslPassword
+                };
+            }
+            this.connection = new kafkajs_1.Kafka(kafkaConfig);
+            return this.connection;
+        }
+        catch (e) {
+            if (!this.retryCountBeforeExit) {
+                console.log("Exiting....");
+                process.exit(1);
+            }
+            this.retryCountBeforeExit -= 1;
+            console.log("retryCountBeforeExit", this.retryCountBeforeExit);
+            console.log("Reconnecting Kafka", e);
+            return this.init(options);
+        }
+    }
+    /**
+     * @typedef {object} Topic
+     * @property {string} name
+     * @property {number} numberOfPartitions
+     * @property {number} replicationFactor
+     * @property {string} retentionPolicy
+     * @property {string} cleanUpPolicy
+     *
+     */
+    /**
+     *
+     * @param {Topic[]} topics
+     * @return {Promise<{error: string}|{data: boolean}|{error}>}
+     */
+    createTopics(topics) {
+        return __awaiter(this, void 0, void 0, function* () {
+            if (!this.connection)
+                throw Error("Connection has not been initialized. Call the init function first");
+            try {
+                const admin = this.connection.admin();
+                yield admin.connect();
+                if (!Array.isArray(topics)) {
+                    topics = [topics];
+                }
+                const kafkaTopics = [];
+                for (let topic of topics) {
+                    if (!topic.name || topic.name.trim() === "")
+                        return { error: "KafkaJS: Topic Name Cannot Be Empty" };
+                    kafkaTopics.push({
+                        topic: topic.name,
+                        numPartitions: topic.numberOfPartitions || 1,
+                        replicationFactor: topic.replicationFactor || 1,
+                        configEntries: [{
+                                name: 'cleanup.policy',
+                                value: topic.cleanUpPolicy || 'compact'
+                            }, {
+                                name: "retention.ms",
+                                value: topic.retentionPolicy || "-1"
+                            }]
+                    });
+                }
+                const response = yield admin.createTopics({
+                    topics: kafkaTopics,
+                });
+                console.log("Kafka: Topics Created", response);
+                return { data: true };
+            }
+            catch (e) {
+                console.error(e);
+                return { error: e.message };
+            }
+        });
+    }
+    createProducer() {
+        return __awaiter(this, void 0, void 0, function* () {
+            if (!this.connection)
+                throw Error("Connection has not been made. Call the init function first");
+            try {
+                this.producer = this.connection.producer();
+                yield this.producer.connect();
+                return { data: this.producer };
+            }
+            catch (e) {
+                console.error(e);
+                return { error: e.message };
+            }
+        });
+    }
+    /**
+     *
+     * @param {string} topic
+     * @param {string} key
+     * @param {any} payload
+     * @return Promise<*>
+     */
+    publish(topic, key, payload) {
+        return __awaiter(this, void 0, void 0, function* () {
+            try {
+                key = key || topic;
+                return {
+                    data: yield this.producer.send({
+                        topic,
+                        messages: [{ value: JSON.stringify(payload), key }]
+                    })
+                };
+            }
+            catch (e) {
+                console.error(e);
+                return { error: e.message };
+            }
+        });
+    }
+    /**
+     *
+     * @param {string[]} topics
+     * @param {string} groupId
+     * @param {function} callback
+     * @param fromBeginning
+     * @param {number} prefetch
+     * @return {null}
+     */
+    listen(topics_1, groupId_1, callback_1) {
+        return __awaiter(this, arguments, void 0, function* (topics, groupId, callback, fromBeginning = false, prefetch = 1) {
+            try {
+                if (typeof callback != "function")
+                    throw new Error("Callback must be a function");
+                this.consumer = this.connection.consumer({ groupId });
+                yield this.consumer.connect();
+                for (let topic of topics) {
+                    yield this.consumer.subscribe({ topic, fromBeginning });
+                }
+                yield this.consumer.run({
+                    partitionsConsumedConcurrently: prefetch,
+                    eachMessage: (_a) => __awaiter(this, [_a], void 0, function* ({ topic, partition, message }) {
+                        var _b, _c;
+                        try {
+                            logger_1.logger.out({
+                                topic,
+                                partition,
+                                key: (_b = message === null || message === void 0 ? void 0 : message.key) === null || _b === void 0 ? void 0 : _b.toString(),
+                                value: message.value.toString(),
+                                headers: message.headers,
+                            });
+                            return callback(null, message.value.toString(), {
+                                topic,
+                                partition,
+                                key: (_c = message === null || message === void 0 ? void 0 : message.key) === null || _c === void 0 ? void 0 : _c.toString(),
+                                value: message.value.toString(),
+                                headers: message.headers,
+                            });
+                        }
+                        catch (e) {
+                            console.log("Kafka consumer error", e);
+                            this.consumer.pause([{ topic }]);
+                            setTimeout(() => this.consumer.resume([{ topic }]), (e.retryAfter || 50) * 1000);
+                            return callback(e.message);
+                        }
+                    }),
+                });
+            }
+            catch (e) {
+                callback(e.message);
+            }
+        });
+    }
+    listenInBatch(topics_1, groupId_1, callback_1) {
+        return __awaiter(this, arguments, void 0, function* (topics, groupId, callback, options = {
+            fromBeginning: false,
+            prefetch: 1
+        }) {
+            try {
+                if (typeof callback != "function")
+                    throw new Error("Callback must be a function");
+                this.consumer = this.connection.consumer({ groupId });
+                yield this.consumer.connect();
+                for (let topic of topics) {
+                    yield this.consumer.subscribe({ topic, fromBeginning: options.fromBeginning });
+                }
+                yield this.consumer.run({
+                    eachBatchAutoResolve: options.eachBatchAutoResolve || true,
+                    partitionsConsumedConcurrently: options.prefetch,
+                    eachBatch: (batchPayload) => __awaiter(this, void 0, void 0, function* () {
+                        yield callback(null, batchPayload.batch.messages.map(message => { var _a; return (_a = message === null || message === void 0 ? void 0 : message.value) === null || _a === void 0 ? void 0 : _a.toString(); }).filter(Boolean), batchPayload);
+                        yield batchPayload.heartbeat();
+                    }),
+                });
+            }
+            catch (e) {
+                callback(e.message);
+            }
+        });
+    }
+    close() {
+        setTimeout(() => {
+            if (this.connection) {
+                console.log("Closing Kafka Connection");
+                try {
+                    return this.connection.close();
+                }
+                catch (alreadyClosed) {
+                    console.log(alreadyClosed.stackAtStateChange);
+                }
+            }
+            return null;
+        }, 5000);
+    }
+}
+exports.KafkaJS = KafkaJS;
+//# sourceMappingURL=kafka.js.map

package/lib/brokers/kafka.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"kafka.js","sourceRoot":"","sources":["../../src/brokers/kafka.ts"],"names":[],"mappings":"AAAA,YAAY,CAAC;;;;;;;;;;;;AACb,qCAAoG;AACpG,6CAAwC;AAexC,MAAa,OAAO;IAOhB,YAAY,OAAmB;QANvB,eAAU,GAAQ,IAAI,CAAC;QACvB,yBAAoB,GAAW,EAAE,CAAC;QAGlC,UAAK,GAAgB,IAAI,CAAC;QAG9B,IAAI,CAAC,UAAU,GAAG,IAAI,CAAC;QACvB,IAAI,CAAC,IAAI,CAAC,OAAO,IAAI,EAAE,CAAC,CAAC;IAC7B,CAAC;IAED;;;;;;;OAOG;IACH,IAAI,CAAC,OAAkB;QACnB,OAAO,CAAC,GAAG,GAAG,OAAO,CAAC,GAAG,IAAI,OAAO,CAAC,GAAG,CAAC,iBAAiB,CAAC;QAC3D,OAAO,CAAC,IAAI,GAAG,OAAO,CAAC,IAAI,IAAI,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC;QACpD,OAAO,CAAC,iBAAiB,GAAG,OAAO,CAAC,iBAAiB,IAAI,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,wBAAkC,CAAC,IAAI,IAAI,CAAC;QAC1H,OAAO,CAAC,cAAc,GAAG,OAAO,CAAC,cAAc,IAAI,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,qBAA+B,CAAC,IAAI,KAAK,CAAC;QAClH,OAAO,CAAC,QAAQ,GAAG,OAAO,CAAC,QAAQ,KAAK,CAAC,CAAC,CAAC,CAAC,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,QAAQ,CAAC,OAAO,CAAC,GAAG,CAAC,eAAyB,CAAC,IAAI,kBAAQ,CAAC,KAAK,CAAC;QACjI,OAAO,CAAC,YAAY,GAAG,OAAO,CAAC,YAAY,IAAI,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC;QAC/E,OAAO,CAAC,YAAY,GAAG,OAAO,CAAC,YAAY,IAAI,OAAO,CAAC,GAAG,CAAC,mBAAmB,CAAC;QAC/E,OAAO,CAAC,aAAa,GAAG,OAAO,CAAC,aAAa,IAAI,OAAO,CAAC;QAEzD,IAAI,CAAC;YACD,IAAI,CAAC,OAAO,CAAC,GAAG;gBAAE,MAAM,KAAK,CAAC,uBAAuB,CAAC,CAAC;YAGvD,8CAA8C;YAC9C,MAAM,WAAW,GAAQ;gBACrB,QAAQ,EAAE,OAAO,CAAC,IAAI;gBACtB,OAAO,EAAE,OAAO,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC;gBAC/B,iBAAiB,EAAE,OAAO,CAAC,iBAAiB;gBAC5C,cAAc,EAAE,OAAO,CAAC,cAAc;gBACtC,QAAQ,EAAE,OAAO,CAAC,QAAQ;gBAC1B,KAAK,EAAE;oBACH,gBAAgB,EAAE,OAAO,CAAC,gBAAgB,IAAI,GAAG;oBACjD,OAAO,EAAE,OAAO,CAAC,OAAO,IAAI,EAAE;iBACjC;aACJ,CAAC;YAEF,IAAI,OAAO,CAAC,aAAa,IAAI,OAAO,CAAC,YAAY,EAAE,CAAC;gBAChD,WAAW,CAAC,GAAG,GAAG,IAAI,CAAC;gBACvB,WAAW,CAAC,IAAI,GAAG;oBACf,SAAS,EAAE,OAAO,CAAC,aAAa,EAAE,iCAAiC;oBACnE,QAAQ,EAAE,OAAO,CAAC,YAAY;oBAC9B,QAAQ,EAAE,OAAO,CAAC,YAAY;iBACjC,CAAC;YACN,CAAC;YAED,IAAI,CAAC,UAAU,GAAG,IAAI,eAAK,CAAC,WAAW,CAAC,CAAA;YAExC,OAAO,IAAI,CAAC,UAAU,CAAC;QAC3B,CAAC;QAAC,OAAO,CAAM,EAAE,CAAC;YACd,IAAI,CAAC,IAAI,CAAC,oBAAoB,EAAE,CAAC;gBAC7B,OAAO,CAAC,GAAG,CAAC,aAAa,CAAC,CAAC;gBAC3B,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;YACpB,CAAC;YACD,IAAI,CAAC,oBAAoB,IAAI,CAAC,CAAC;YAC/B,OAAO,CAAC,GAAG,CAAC,sBAAsB,EAAE,IAAI,CAAC,oBAAoB,CAAC,CAAC;YAC/D,OAAO,CAAC,GAAG,CAAC,oBAAoB,EAAE,CAAC,CAAC,CAAC;YACrC,OAAO,IAAI,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QAC9B,CAAC;IACL,CAAC;IAED;;;;;;;;OAQG;IACH;;;;OAIG;IACG,YAAY,CAAC,MAAa;;YAC5B,IAAI,CAAC,IAAI,CAAC,UAAU;gBAChB,MAAM,KAAK,CAAC,mEAAmE,CAAC,CAAC;YAErF,IAAI,CAAC;gBACD,MAAM,KAAK,GAAG,IAAI,CAAC,UAAU,CAAC,KAAK,EAAE,CAAC;gBACtC,MAAM,KAAK,CAAC,OAAO,EAAE,CAAC;gBAEtB,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,EAAE,CAAC;oBACzB,MAAM,GAAG,CAAC,MAAM,CAAC,CAAC;gBACtB,CAAC;gBAED,MAAM,WAAW,GAAG,EAAE,CAAC;gBACvB,KAAK,IAAI,KAAK,IAAI,MAAM,EAAE,CAAC;oBACvB,IAAI,CAAC,KAAK,CAAC,IAAI,IAAI,KAAK,CAAC,IAAI,CAAC,IAAI,EAAE,KAAK,EAAE;wBAAE,OAAO,EAAC,KAAK,EAAE,qCAAqC,EAAC,CAAC;oBACnG,WAAW,CAAC,IAAI,CAAC;wBACb,KAAK,EAAE,KAAK,CAAC,IAAI;wBACjB,aAAa,EAAE,KAAK,CAAC,kBAAkB,IAAI,CAAC;wBAC5C,iBAAiB,EAAE,KAAK,CAAC,iBAAiB,IAAI,CAAC;wBAC/C,aAAa,EAAE,CAAC;gCACZ,IAAI,EAAE,gBAAgB;gCACtB,KAAK,EAAE,KAAK,CAAC,aAAa,IAAI,SAAS;6BAC1C,EAAE;gCACC,IAAI,EAAE,cAAc;gCACpB,KAAK,EAAE,KAAK,CAAC,eAAe,IAAI,IAAI;6BACvC,CAAC;qBACL,CAAC,CAAA;gBACN,CAAC;gBAED,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,YAAY,CAAC;oBACtC,MAAM,EAAE,WAAW;iBACtB,CAAC,CAAC;gBAEH,OAAO,CAAC,GAAG,CAAC,uBAAuB,EAAE,QAAQ,CAAC,CAAC;gBAC/C,OAAO,EAAC,IAAI,EAAE,IAAI,EAAC,CAAC;YAExB,CAAC;YAAC,OAAO,CAAM,EAAE,CAAC;gBACd,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;gBACjB,OAAO,EAAC,KAAK,EAAE,CAAC,CAAC,OAAO,EAAC,CAAC;YAC9B,CAAC;QACL,CAAC;KAAA;IAEK,cAAc;;YAEhB,IAAI,CAAC,IAAI,CAAC,UAAU;gBAChB,MAAM,KAAK,CAAC,4DAA4D,CAAC,CAAC;YAE9E,IAAI,CAAC;gBACD,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,UAAU,CAAC,QAAQ,EAAE,CAAA;gBAC1C,MAAM,IAAI,CAAC,QAAQ,CAAC,OAAO,EAAE,CAAA;gBAC7B,OAAO,EAAC,IAAI,EAAE,IAAI,CAAC,QAAQ,EAAC,CAAC;YACjC,CAAC;YAAC,OAAO,CAAM,EAAE,CAAC;gBACd,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;gBACjB,OAAO,EAAC,KAAK,EAAE,CAAC,CAAC,OAAO,EAAC,CAAC;YAC9B,CAAC;QACL,CAAC;KAAA;IAGD;;;;;;OAMG;IACG,OAAO,CAAC,KAAa,EAAE,GAAW,EAAE,OAAY;;YAClD,IAAI,CAAC;gBACD,GAAG,GAAG,GAAG,IAAI,KAAK,CAAC;gBACnB,OAAO;oBACH,IAAI,EAAE,MAAM,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC;wBAC3B,KAAK;wBACL,QAAQ,EAAE,CAAC,EAAC,KAAK,EAAE,IAAI,CAAC,SAAS,CAAC,OAAO,CAAC,EAAE,GAAG,EAAC,CAAC;qBACpD,CAAC;iBACL,CAAC;YACN,CAAC;YAAC,OAAO,CAAM,EAAE,CAAC;gBACd,OAAO,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC;gBAEjB,OAAO,EAAC,KAAK,EAAE,CAAC,CAAC,OAAO,EAAC,CAAC;YAC9B,CAAC;QACL,CAAC;KAAA;IAED;;;;;;;;OAQG;IACG,MAAM;6DAAC,MAAa,EAAE,OAAe,EAAE,QAAa,EAAE,gBAAyB,KAAK,EAAE,WAAmB,CAAC;YAC5G,IAAI,CAAC;gBACD,IAAI,OAAO,QAAQ,IAAI,UAAU;oBAC7B,MAAO,IAAI,KAAK,CAAC,6BAA6B,CAAC,CAAC;gBAGpD,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,EAAC,OAAO,EAAC,CAAC,CAAC;gBACpD,MAAM,IAAI,CAAC,QAAQ,CAAC,OAAO,EAAE,CAAA;gBAE7B,KAAK,IAAI,KAAK,IAAI,MAAM,EAAE,CAAC;oBACvB,MAAM,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAC,KAAK,EAAE,aAAa,EAAC,CAAC,CAAC;gBAC1D,CAAC;gBAGD,MAAM,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC;oBACpB,8BAA8B,EAAE,QAAQ;oBACxC,WAAW,EAAE,KAAyC,EAAE,0CAApC,EAAC,KAAK,EAAE,SAAS,EAAE,OAAO,EAAM;;wBAChD,IAAI,CAAC;4BACD,eAAM,CAAC,GAAG,CAAC;gCACP,KAAK;gCACL,SAAS;gCACT,GAAG,EAAE,MAAA,OAAO,aAAP,OAAO,uBAAP,OAAO,CAAE,GAAG,0CAAE,QAAQ,EAAE;gCAC7B,KAAK,EAAE,OAAO,CAAC,KAAK,CAAC,QAAQ,EAAE;gCAC/B,OAAO,EAAE,OAAO,CAAC,OAAO;6BAC3B,CAAC,CAAC;4BACH,OAAO,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAC,KAAK,CAAC,QAAQ,EAAE,EAAE;gCAC5C,KAAK;gCACL,SAAS;gCACT,GAAG,EAAE,MAAA,OAAO,aAAP,OAAO,uBAAP,OAAO,CAAE,GAAG,0CAAE,QAAQ,EAAE;gCAC7B,KAAK,EAAE,OAAO,CAAC,KAAK,CAAC,QAAQ,EAAE;gCAC/B,OAAO,EAAE,OAAO,CAAC,OAAO;6BAC3B,CAAC,CAAC;wBACP,CAAC;wBAAC,OAAO,CAAM,EAAE,CAAC;4BACd,OAAO,CAAC,GAAG,CAAC,sBAAsB,EAAE,CAAC,CAAC,CAAA;4BACtC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,EAAC,KAAK,EAAC,CAAC,CAAC,CAAA;4BAC9B,UAAU,CAAC,GAAG,EAAE,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC,EAAC,KAAK,EAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,UAAU,IAAI,EAAE,CAAC,GAAG,IAAI,CAAC,CAAC;4BAC/E,OAAO,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;wBAC/B,CAAC;oBAEL,CAAC,CAAA;iBACJ,CAAC,CAAC;YAEP,CAAC;YAAC,OAAO,CAAM,EAAE,CAAC;gBACd,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;YACxB,CAAC;QACL,CAAC;KAAA;IAEK,aAAa;6DAAC,MAAa,EAAE,OAAe,EAAE,QAAa,EAAE,UAAe;YAC9E,aAAa,EAAE,KAAK;YACpB,QAAQ,EAAE,CAAC;SACd;YACG,IAAI,CAAC;gBACD,IAAI,OAAO,QAAQ,IAAI,UAAU;oBAC7B,MAAO,IAAI,KAAK,CAAC,6BAA6B,CAAC,CAAC;gBAGpD,IAAI,CAAC,QAAQ,GAAG,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,EAAC,OAAO,EAAC,CAAC,CAAC;gBACpD,MAAM,IAAI,CAAC,QAAQ,CAAC,OAAO,EAAE,CAAA;gBAE7B,KAAK,IAAI,KAAK,IAAI,MAAM,EAAE,CAAC;oBACvB,MAAM,IAAI,CAAC,QAAQ,CAAC,SAAS,CAAC,EAAC,KAAK,EAAE,aAAa,EAAE,OAAO,CAAC,aAAa,EAAC,CAAC,CAAC;gBACjF,CAAC;gBAGD,MAAM,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC;oBACpB,oBAAoB,EAAE,OAAO,CAAC,oBAAoB,IAAI,IAAI;oBAC1D,8BAA8B,EAAE,OAAO,CAAC,QAAQ;oBAChD,SAAS,EAAE,CAAO,YAA8B,EAAE,EAAE;wBAChD,MAAM,QAAQ,CAAC,IAAI,EAAE,YAAY,CAAC,KAAK,CAAC,QAAQ,CAAC,GAAG,CAAC,OAAO,CAAC,EAAE,WAAC,OAAA,MAAA,OAAO,aAAP,OAAO,uBAAP,OAAO,CAAE,KAAK,0CAAE,QAAQ,EAAE,CAAA,EAAA,CAAC,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,YAAY,CAAC,CAAC;wBAC3H,MAAM,YAAY,CAAC,SAAS,EAAE,CAAC;oBACnC,CAAC,CAAA;iBACJ,CAAC,CAAA;YAEN,CAAC;YAAC,OAAO,CAAM,EAAE,CAAC;gBACd,QAAQ,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;YACxB,CAAC;QACL,CAAC;KAAA;IAED,KAAK;QACD,UAAU,CAAC,GAAG,EAAE;YACZ,IAAI,IAAI,CAAC,UAAU,EAAE,CAAC;gBAClB,OAAO,CAAC,GAAG,CAAC,0BAA0B,CAAC,CAAC;gBACxC,IAAI,CAAC;oBACD,OAAO,IAAI,CAAC,UAAU,CAAC,KAAK,EAAE,CAAC;gBACnC,CAAC;gBAAC,OAAO,aAAkB,EAAE,CAAC;oBAC1B,OAAO,CAAC,GAAG,CAAC,aAAa,CAAC,kBAAkB,CAAC,CAAC;gBAClD,CAAC;YACL,CAAC;YACD,OAAO,IAAI,CAAC;QAChB,CAAC,EAAE,IAAI,CAAC,CAAC;IACb,CAAC;CAEJ;AA3QD,0BA2QC"}

package/lib/brokers/kafka.test.d.ts

@@ -0,0 +1 @@
+export {};

package/lib/brokers/kafka.test.js

@@ -0,0 +1,118 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const kafka_1 = require("./kafka");
+const kafkajs_1 = require("kafkajs");
+try {
+    const kafkaInit = {
+        url: "pkc-6ojv2.us-west4.gcp.confluent.cloud:9092",
+        name: "my-app",
+        logLevel: kafkajs_1.logLevel.DEBUG,
+        saslUsername: "RFYODITNTCAMOUHA",
+        saslPassword: "+2RHsqskk/7iEQEi93b5J3o8pJbCPQE0/NO73ePDN6IYZdYQI24O4qSOKtRfETqn",
+        saslMechanism: "plain"
+    };
+    const kafka = new kafka_1.KafkaJS(kafkaInit);
+    // async function produce() {
+    //     const admin = kafka.admin()
+    //     await admin.connect();
+    //     // const topics =
+    //     await admin.createTopics({
+    //         waitForLeaders: true,
+    //         topics: [
+    //             {
+    //                 topic: 'user.created.test.1',
+    //                 numPartitions: 1,
+    //                 replicationFactor:3,
+    //                 configEntries: [{
+    //                     name: 'cleanup.policy',
+    //                     value: 'delete'
+    //                 }, {
+    //                     name: "retention.ms",
+    //                     value: "10000"
+    //                 }]
+    //             },
+    //         ],
+    //     })
+    //     const producer = kafka.producer()
+    //
+    //     await producer.connect()
+    //
+    //     for (let i = 0; i < 10; i++) {
+    //         const res = await producer.send({
+    //             topic: 'user.created',
+    //             messages: [
+    //                 {value: 'Hello KafkaJS user! ' + (i + 10), key: "users"},
+    //             ],
+    //         });
+    //
+    //         console.log(res, i)
+    //
+    //     }
+    //
+    //     await producer.disconnect()
+    // }
+    function consume() {
+        return __awaiter(this, void 0, void 0, function* () {
+            try {
+                console.log("init kafka");
+                yield kafka.init(kafkaInit);
+                console.log("init kafka listener");
+                yield kafka.listen(["request.logs"], "my-app", (...args) => {
+                    console.log("Incoming Arg", JSON.stringify(args));
+                }, true);
+            }
+            catch (e) {
+            }
+            // const consumer = kafka.consumer({groupId: 'my-group-2'})
+            // await consumer.connect()
+            //
+            // await consumer.subscribe({topic: 'user.created', fromBeginning: true})
+            //
+            // await consumer.run({
+            //     eachMessage: async ({topic, partition, message}) => {
+            //         console.log({
+            //             h: "here",
+            //             topic,
+            //             partition,
+            //             key: message.key.toString(),
+            //             value: message.value.toString(),
+            //             headers: message.headers,
+            //         })
+            //     },
+            // })
+        });
+    }
+    function consumeBatch() {
+        return __awaiter(this, void 0, void 0, function* () {
+            try {
+                console.log("init kafka listener");
+                yield kafka.listenInBatch(["request.logs"], "test-my-app", (error, messages, batchPayload) => {
+                    console.log("Incoming Arg", error, messages.length, batchPayload.batch.topic);
+                    console.log("Incoming Arg len", batchPayload.batch.messages.length);
+                }, {
+                    fromBeginning: false,
+                    prefetch: 100
+                });
+            }
+            catch (e) {
+                console.log("Error", e);
+            }
+        });
+    }
+    // produce();
+    consume();
+    // consumeBatch();
+}
+catch (e) {
+    console.log("e", e);
+}
+//# sourceMappingURL=kafka.test.js.map

package/lib/brokers/kafka.test.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"kafka.test.js","sourceRoot":"","sources":["../../src/brokers/kafka.test.ts"],"names":[],"mappings":";;;;;;;;;;;AAAA,mCAA0C;AAC1C,qCAAmD;AAGnD,IAAG,CAAC;IACA,MAAM,SAAS,GAAc;QACzB,GAAG,EAAE,6CAA6C;QAClD,IAAI,EAAE,QAAQ;QACd,QAAQ,EAAE,kBAAQ,CAAC,KAAK;QACxB,YAAY,EAAE,kBAAkB;QAChC,YAAY,EAAE,kEAAkE;QAChF,aAAa,EAAE,OAAO;KACzB,CAAC;IACF,MAAM,KAAK,GAAG,IAAI,eAAO,CAAC,SAAS,CAAC,CAAC;IAGzC,6BAA6B;IAC7B,kCAAkC;IAClC,6BAA6B;IAC7B,wBAAwB;IACxB,iCAAiC;IACjC,gCAAgC;IAChC,oBAAoB;IACpB,gBAAgB;IAChB,gDAAgD;IAChD,oCAAoC;IACpC,uCAAuC;IACvC,oCAAoC;IACpC,8CAA8C;IAC9C,sCAAsC;IACtC,uBAAuB;IACvB,4CAA4C;IAC5C,qCAAqC;IACrC,qBAAqB;IACrB,iBAAiB;IACjB,aAAa;IACb,SAAS;IACT,wCAAwC;IACxC,EAAE;IACF,+BAA+B;IAC/B,EAAE;IACF,qCAAqC;IACrC,4CAA4C;IAC5C,qCAAqC;IACrC,0BAA0B;IAC1B,4EAA4E;IAC5E,iBAAiB;IACjB,cAAc;IACd,EAAE;IACF,8BAA8B;IAC9B,EAAE;IACF,QAAQ;IACR,EAAE;IACF,kCAAkC;IAClC,IAAI;IAEA,SAAe,OAAO;;YAClB,IAAG,CAAC;gBACA,OAAO,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC;gBAC1B,MAAM,KAAK,CAAC,IAAI,CAAC,SAAS,CAAC,CAAC;gBAC5B,OAAO,CAAC,GAAG,CAAC,qBAAqB,CAAC,CAAC;gBACnC,MAAM,KAAK,CAAC,MAAM,CAAC,CAAC,cAAc,CAAC,EAAE,QAAQ,EAAE,CAAC,GAAG,IAAS,EAAE,EAAE;oBAC5D,OAAO,CAAC,GAAG,CAAC,cAAc,EAAE,IAAI,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC,CAAC;gBACtD,CAAC,EAAE,IAAI,CAAC,CAAC;YACb,CAAC;YACD,OAAO,CAAC,EAAC,CAAC;YAEV,CAAC;YACD,2DAA2D;YAC3D,2BAA2B;YAC3B,EAAE;YACF,yEAAyE;YACzE,EAAE;YACF,uBAAuB;YACvB,4DAA4D;YAC5D,wBAAwB;YACxB,yBAAyB;YACzB,qBAAqB;YACrB,yBAAyB;YACzB,2CAA2C;YAC3C,+CAA+C;YAC/C,wCAAwC;YACxC,aAAa;YACb,SAAS;YACT,KAAK;QACT,CAAC;KAAA;IACD,SAAe,YAAY;;YACvB,IAAG,CAAC;gBACA,OAAO,CAAC,GAAG,CAAC,qBAAqB,CAAC,CAAC;gBACnC,MAAM,KAAK,CAAC,aAAa,CAAC,CAAC,cAAc,CAAC,EAAE,aAAa,EAAE,CAAC,KAAU,EAAE,QAAe,EAAE,YAA8B,EAAE,EAAE;oBACvH,OAAO,CAAC,GAAG,CAAC,cAAc,EAAC,KAAK,EAAE,QAAQ,CAAC,MAAM,EAAE,YAAY,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;oBAC7E,OAAO,CAAC,GAAG,CAAC,kBAAkB,EAAE,YAAY,CAAC,KAAK,CAAC,QAAQ,CAAC,MAAM,CAAC,CAAC;gBACxE,CAAC,EAAE;oBACC,aAAa,EAAE,KAAK;oBACpB,QAAQ,EAAE,GAAG;iBAChB,CAAC,CAAC;YACP,CAAC;YACD,OAAO,CAAC,EAAC,CAAC;gBACN,OAAO,CAAC,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;YAC5B,CAAC;QACL,CAAC;KAAA;IAEL,aAAa;IACT,OAAO,EAAE,CAAC;IACV,kBAAkB;AACtB,CAAC;AAAA,OAAO,CAAC,EAAC,CAAC;IACP,OAAO,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC;AACxB,CAAC"}

package/lib/brokers/rabbitmq.d.ts

@@ -0,0 +1,108 @@
+import { BrokerDeadLetterPayloadEntity, BrokerRetryEntity } from "../types";
+import { IAmqpConnectionManager } from "amqp-connection-manager/dist/types/AmqpConnectionManager";
+export type RabbitMQInit = {
+    url?: string;
+    heartbeat?: number;
+    retryCountBeforeExit?: number;
+    [x: string]: any;
+};
+export declare class RabbitMQ {
+    private channel;
+    private connection;
+    private retryCountBeforeExit;
+    constructor(options?: RabbitMQInit);
+    /**
+     *
+     * @param {Object} options
+     * @param {string} options.url
+     * @param {number} options.retryCountBeforeExit
+     * @param {number} options.heartbeat
+     * @return {Promise<null|*|undefined>}
+     */
+    init(options?: RabbitMQInit): Promise<IAmqpConnectionManager>;
+    createChannel(callback: any): Promise<{
+        data: any;
+        error?: undefined;
+    } | {
+        error: any;
+        data?: undefined;
+    }>;
+    /**
+     *
+     * @param {string} queueName
+     * @param {Object} options
+     * @return {Promise<null|{data: (Promise<*>|{durable: (boolean|*), ticket: number, autoDelete, exclusive, arguments: any, passive: boolean, queue: *, nowait: boolean})}|{error: *}>}
+     */
+    createQueue(queueName: string | string[], options?: any): Promise<{
+        data: boolean;
+        error?: undefined;
+    } | {
+        error: any;
+        data?: undefined;
+    }>;
+    sendToQueue(channelName: string, payload: any, options?: {
+        persistent: boolean;
+    }): Promise<{
+        data: any;
+        error?: undefined;
+    } | {
+        error: any;
+        data?: undefined;
+    }>;
+    /**
+     *
+     * @param {string} exchangeName
+     * @param {string} exchangeType (fanout, direct, topic)
+     * @param {Object} option
+     * @return {Promise<{error: *}|{data: ({data}|{error: *})}>}
+     */
+    assertExchange(exchangeName: string, exchangeType?: string, option?: any): Promise<{
+        data: any;
+        error?: undefined;
+    } | {
+        error: any;
+        data?: undefined;
+    }>;
+    /***
+     *
+     * @param {string} exchangeName
+     * @param {string} queueName
+     * @param {Object} queueOption
+     * @param {string} bindKey
+     * @return {Promise<{data: *}|{error: *}>}
+     */
+    assertQueue(exchangeName: string, queueName?: string, bindKey?: string, queueOption?: any): Promise<{
+        data: any;
+        error?: undefined;
+    } | {
+        error: any;
+        data?: undefined;
+    }>;
+    /**
+     *
+     * @param {string} exchangeName
+     * @param {string} routeKey
+     * @param {string} payload
+     * @return Promise<*>
+     */
+    publishToExchange(exchangeName: string, routeKey: string | undefined, payload: any): Promise<{
+        data: any;
+        error?: undefined;
+    } | {
+        error: any;
+        data?: undefined;
+    }>;
+    /**
+     *
+     * @param {string} channelName
+     * @param {Object} options
+     * @param {function} callback
+     * @param {number} prefetch
+     * @return {null}
+     */
+    listen(channelName: string, options: any, callback: any): Promise<void>;
+    sendToDeadLetter(deadLetterEntity: BrokerDeadLetterPayloadEntity): Promise<void>;
+    sendToRetry(retryEntity: BrokerRetryEntity): Promise<void>;
+    requeueDeadLetter(deadLetterQueueName: string, queueOptions?: any): Promise<void>;
+    close(): void;
+}