kafka-console 1.1.0

package/README.md

@@ -0,0 +1,88 @@
+# Kafka CLI tool
+
+Command line tool to sufficiently and easy work with Kafka
+
+[![NPM version][npm-image]][npm-url]
+[![Downloads][downloads-image]][npm-url]
+
+
+## Installation
+
+```sh
+npm install -g kafka-console
+```
+
+## Usage
+
+### Consumer usage
+
+General usage with authentication
+```sh
+kcli consume $KAFKA_TOPIC -g $KAFKA_TOPIC_GROUP -b $KAFKA_BROKERS --ssl --mechanism plain --username $KAFKA_USERNAME --password $KAFKA_PASSWORD
+```
+
+Stdout `jq` example
+```sh
+kcli consume $KAFKA_TOPIC | jq .value
+```
+
+Custom data formatter example
+```sh
+kcli consume $KAFKA_TOPIC --format ./formatter/avro.js | jq
+```
+
+### Producer usage
+
+General usage
+```sh
+kcli produce $KAFKA_TOPIC -b $KAFKA_BROKERS --ssl --mechanism plain --username $KAFKA_USERNAME --password $KAFKA_PASSWORD
+```
+
+Produce a json data from stdin with custom formatter
+```sh
+cat payload.txt|kcli produce $KAFKA_TOPIC --format ./formatter/avro.js
+```
+
+Produce a json data from stdin
+```sh
+node payloadGenerator.js|kcli produce $KAFKA_TOPIC
+```
+
+Produce a json array data from stdin
+```sh
+cat payload.json|jq -r -c .[]|kcli produce $KAFKA_TOPIC
+```
+
+Payload single message input interface
+```typescript
+interface Payload {
+  key?: string; // kafka 
+  value: any;
+  headers?: { [key: string]: value };
+}
+```
+
+### Formatters
+
+```typescript
+export interface Encoder<T> {
+  (value: T): Promise<string | Buffer> | string | Buffer;
+}
+
+export interface Decoder<T> {
+  (value: string | Buffer): Promise<T> | T;
+}
+
+export interface Formatter<T> {
+  encode: Encoder<T>;
+  decode: Decoder<T>;
+}
+```
+
+## License
+License [The MIT License](http://opensource.org/licenses/MIT)
+Copyright (c) 2020 Ivan Zakharchanka
+
+[npm-url]: https://www.npmjs.com/package/kafka-console
+[downloads-image]: https://img.shields.io/npm/dw/kafka-console.svg?maxAge=43200
+[npm-image]: https://img.shields.io/npm/v/kafka-console.svg?maxAge=43200

package/build/cli.js

@@ -0,0 +1,4 @@
+#!/usr/bin/env node
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+require("./index");

package/build/commands/config.js

@@ -0,0 +1,45 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __rest = (this && this.__rest) || function (s, e) {
+    var t = {};
+    for (var p in s) if (Object.prototype.hasOwnProperty.call(s, p) && e.indexOf(p) < 0)
+        t[p] = s[p];
+    if (s != null && typeof Object.getOwnPropertySymbols === "function")
+        for (var i = 0, p = Object.getOwnPropertySymbols(s); i < p.length; i++) {
+            if (e.indexOf(p[i]) < 0 && Object.prototype.propertyIsEnumerable.call(s, p[i]))
+                t[p[i]] = s[p[i]];
+        }
+    return t;
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const kafka_1 = require("../utils/kafka");
+function config(opts, { parent }) {
+    return __awaiter(this, void 0, void 0, function* () {
+        const _a = Object.assign(Object.assign({}, parent.opts()), opts), { resource, resourceName: name, brokers, logLevel, ssl } = _a, rest = __rest(_a, ["resource", "resourceName", "brokers", "logLevel", "ssl"]);
+        const sasl = (0, kafka_1.getSASL)(rest);
+        const client = (0, kafka_1.createClient)(brokers, ssl, sasl, logLevel);
+        const admin = yield (0, kafka_1.createAdmin)(client);
+        const { resources } = yield admin.describeConfigs({
+            includeSynonyms: true,
+            resources: [
+                {
+                    type: resource,
+                    name,
+                },
+            ],
+        });
+        for (let resource of resources) {
+            console.log(resource);
+        }
+        yield admin.disconnect();
+    });
+}
+exports.default = config;

package/build/commands/consume.js

@@ -0,0 +1,80 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    Object.defineProperty(o, k2, { enumerable: true, get: function() { return m[k]; } });
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __rest = (this && this.__rest) || function (s, e) {
+    var t = {};
+    for (var p in s) if (Object.prototype.hasOwnProperty.call(s, p) && e.indexOf(p) < 0)
+        t[p] = s[p];
+    if (s != null && typeof Object.getOwnPropertySymbols === "function")
+        for (var i = 0, p = Object.getOwnPropertySymbols(s); i < p.length; i++) {
+            if (e.indexOf(p[i]) < 0 && Object.prototype.propertyIsEnumerable.call(s, p[i]))
+                t[p[i]] = s[p[i]];
+        }
+    return t;
+};
+var __asyncValues = (this && this.__asyncValues) || function (o) {
+    if (!Symbol.asyncIterator) throw new TypeError("Symbol.asyncIterator is not defined.");
+    var m = o[Symbol.asyncIterator], i;
+    return m ? m.call(o) : (o = typeof __values === "function" ? __values(o) : o[Symbol.iterator](), i = {}, verb("next"), verb("throw"), verb("return"), i[Symbol.asyncIterator] = function () { return this; }, i);
+    function verb(n) { i[n] = o[n] && function (v) { return new Promise(function (resolve, reject) { v = o[n](v), settle(resolve, reject, v.done, v.value); }); }; }
+    function settle(resolve, reject, d, v) { Promise.resolve(v).then(function(v) { resolve({ value: v, done: d }); }, reject); }
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const Fs = __importStar(require("fs"));
+const kafka_1 = require("../utils/kafka");
+const formatters_1 = require("../utils/formatters");
+function consume(topic, opts, { parent }) {
+    var e_1, _a;
+    return __awaiter(this, void 0, void 0, function* () {
+        const _b = Object.assign(Object.assign({}, parent.opts()), opts), { group, format, fromBeginning, filename, brokers, logLevel, ssl } = _b, rest = __rest(_b, ["group", "format", "fromBeginning", "filename", "brokers", "logLevel", "ssl"]);
+        const sasl = (0, kafka_1.getSASL)(rest);
+        const client = (0, kafka_1.createClient)(brokers, ssl, sasl, logLevel);
+        const output = filename ? Fs.createWriteStream(filename) : process.stdout;
+        const consumer = yield (0, kafka_1.createConsumer)(client, group, topic, fromBeginning);
+        const formatter = (0, formatters_1.getFormatter)(format);
+        try {
+            for (var consumer_1 = __asyncValues(consumer), consumer_1_1; consumer_1_1 = yield consumer_1.next(), !consumer_1_1.done;) {
+                let { message: { headers, key, value } } = consumer_1_1.value;
+                const parsedHeaders = Object.entries(headers).reduce((result, [key, value]) => {
+                    return Object.assign(Object.assign({}, result), { [key]: value.toString() });
+                }, {});
+                const message = { headers: parsedHeaders, key: key === null || key === void 0 ? void 0 : key.toString(), value: yield formatter.decode(value) };
+                output.write(JSON.stringify(message, null, '  ') + '\n');
+            }
+        }
+        catch (e_1_1) { e_1 = { error: e_1_1 }; }
+        finally {
+            try {
+                if (consumer_1_1 && !consumer_1_1.done && (_a = consumer_1.return)) yield _a.call(consumer_1);
+            }
+            finally { if (e_1) throw e_1.error; }
+        }
+    });
+}
+exports.default = consume;

package/build/commands/createTopic.js

@@ -0,0 +1,37 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __rest = (this && this.__rest) || function (s, e) {
+    var t = {};
+    for (var p in s) if (Object.prototype.hasOwnProperty.call(s, p) && e.indexOf(p) < 0)
+        t[p] = s[p];
+    if (s != null && typeof Object.getOwnPropertySymbols === "function")
+        for (var i = 0, p = Object.getOwnPropertySymbols(s); i < p.length; i++) {
+            if (e.indexOf(p[i]) < 0 && Object.prototype.propertyIsEnumerable.call(s, p[i]))
+                t[p[i]] = s[p[i]];
+        }
+    return t;
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const kafka_1 = require("../utils/kafka");
+function createTopic(topic, opts, { parent }) {
+    return __awaiter(this, void 0, void 0, function* () {
+        const _a = Object.assign(Object.assign({}, parent.opts()), opts), { brokers, logLevel, ssl } = _a, rest = __rest(_a, ["brokers", "logLevel", "ssl"]);
+        const sasl = (0, kafka_1.getSASL)(rest);
+        const client = (0, kafka_1.createClient)(brokers, ssl, sasl, logLevel);
+        const admin = yield (0, kafka_1.createAdmin)(client);
+        const topics = yield admin.createTopics({
+            topics: [{ topic }],
+        });
+        console.log(topics);
+        yield admin.disconnect();
+    });
+}
+exports.default = createTopic;

package/build/commands/deleteTopic.js

@@ -0,0 +1,36 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __rest = (this && this.__rest) || function (s, e) {
+    var t = {};
+    for (var p in s) if (Object.prototype.hasOwnProperty.call(s, p) && e.indexOf(p) < 0)
+        t[p] = s[p];
+    if (s != null && typeof Object.getOwnPropertySymbols === "function")
+        for (var i = 0, p = Object.getOwnPropertySymbols(s); i < p.length; i++) {
+            if (e.indexOf(p[i]) < 0 && Object.prototype.propertyIsEnumerable.call(s, p[i]))
+                t[p[i]] = s[p[i]];
+        }
+    return t;
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const kafka_1 = require("../utils/kafka");
+function deleteTopic(topic, opts, { parent }) {
+    return __awaiter(this, void 0, void 0, function* () {
+        const _a = Object.assign(Object.assign({}, parent.opts()), opts), { brokers, logLevel, ssl } = _a, rest = __rest(_a, ["brokers", "logLevel", "ssl"]);
+        const sasl = (0, kafka_1.getSASL)(rest);
+        const client = (0, kafka_1.createClient)(brokers, ssl, sasl, logLevel);
+        const admin = yield (0, kafka_1.createAdmin)(client);
+        yield admin.deleteTopics({
+            topics: [topic],
+        });
+        yield admin.disconnect();
+    });
+}
+exports.default = deleteTopic;

package/build/commands/list.js

@@ -0,0 +1,39 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __rest = (this && this.__rest) || function (s, e) {
+    var t = {};
+    for (var p in s) if (Object.prototype.hasOwnProperty.call(s, p) && e.indexOf(p) < 0)
+        t[p] = s[p];
+    if (s != null && typeof Object.getOwnPropertySymbols === "function")
+        for (var i = 0, p = Object.getOwnPropertySymbols(s); i < p.length; i++) {
+            if (e.indexOf(p[i]) < 0 && Object.prototype.propertyIsEnumerable.call(s, p[i]))
+                t[p[i]] = s[p[i]];
+        }
+    return t;
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const kafka_1 = require("../utils/kafka");
+function list(opts, { parent }) {
+    return __awaiter(this, void 0, void 0, function* () {
+        const _a = Object.assign(Object.assign({}, parent.opts()), opts), { all, brokers, logLevel, ssl } = _a, rest = __rest(_a, ["all", "brokers", "logLevel", "ssl"]);
+        const sasl = (0, kafka_1.getSASL)(rest);
+        const client = (0, kafka_1.createClient)(brokers, ssl, sasl, logLevel);
+        const cluster = yield (0, kafka_1.createCluster)(client);
+        const topics = (yield cluster.metadata()).topicMetadata
+            .filter(topic => all || !topic.isInternal)
+            .map(({ topic }) => topic);
+        for (const topic of topics) {
+            console.log(topic);
+        }
+        yield cluster.disconnect();
+    });
+}
+exports.default = list;

package/build/commands/metadata.js

@@ -0,0 +1,35 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __rest = (this && this.__rest) || function (s, e) {
+    var t = {};
+    for (var p in s) if (Object.prototype.hasOwnProperty.call(s, p) && e.indexOf(p) < 0)
+        t[p] = s[p];
+    if (s != null && typeof Object.getOwnPropertySymbols === "function")
+        for (var i = 0, p = Object.getOwnPropertySymbols(s); i < p.length; i++) {
+            if (e.indexOf(p[i]) < 0 && Object.prototype.propertyIsEnumerable.call(s, p[i]))
+                t[p[i]] = s[p[i]];
+        }
+    return t;
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const kafka_1 = require("../utils/kafka");
+function metadata(opts, { parent }) {
+    return __awaiter(this, void 0, void 0, function* () {
+        const _a = Object.assign(Object.assign({}, parent.opts()), opts), { brokers, logLevel, ssl } = _a, rest = __rest(_a, ["brokers", "logLevel", "ssl"]);
+        const sasl = (0, kafka_1.getSASL)(rest);
+        const client = (0, kafka_1.createClient)(brokers, ssl, sasl, logLevel);
+        const cluster = yield (0, kafka_1.createCluster)(client);
+        const metadata = yield cluster.metadata();
+        console.log(JSON.stringify(metadata, null, '  '));
+        yield cluster.disconnect();
+    });
+}
+exports.default = metadata;

package/build/commands/produce.js

@@ -0,0 +1,118 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    Object.defineProperty(o, k2, { enumerable: true, get: function() { return m[k]; } });
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || function (mod) {
+    if (mod && mod.__esModule) return mod;
+    var result = {};
+    if (mod != null) for (var k in mod) if (k !== "default" && Object.prototype.hasOwnProperty.call(mod, k)) __createBinding(result, mod, k);
+    __setModuleDefault(result, mod);
+    return result;
+};
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __rest = (this && this.__rest) || function (s, e) {
+    var t = {};
+    for (var p in s) if (Object.prototype.hasOwnProperty.call(s, p) && e.indexOf(p) < 0)
+        t[p] = s[p];
+    if (s != null && typeof Object.getOwnPropertySymbols === "function")
+        for (var i = 0, p = Object.getOwnPropertySymbols(s); i < p.length; i++) {
+            if (e.indexOf(p[i]) < 0 && Object.prototype.propertyIsEnumerable.call(s, p[i]))
+                t[p[i]] = s[p[i]];
+        }
+    return t;
+};
+var __asyncValues = (this && this.__asyncValues) || function (o) {
+    if (!Symbol.asyncIterator) throw new TypeError("Symbol.asyncIterator is not defined.");
+    var m = o[Symbol.asyncIterator], i;
+    return m ? m.call(o) : (o = typeof __values === "function" ? __values(o) : o[Symbol.iterator](), i = {}, verb("next"), verb("throw"), verb("return"), i[Symbol.asyncIterator] = function () { return this; }, i);
+    function verb(n) { i[n] = o[n] && function (v) { return new Promise(function (resolve, reject) { v = o[n](v), settle(resolve, reject, v.done, v.value); }); }; }
+    function settle(resolve, reject, d, v) { Promise.resolve(v).then(function(v) { resolve({ value: v, done: d }); }, reject); }
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const Fs = __importStar(require("fs"));
+const stream_1 = require("stream");
+const readline_1 = require("readline");
+const util_1 = require("util");
+const pool_1 = __importDefault(require("../utils/pool"));
+const kafka_1 = require("../utils/kafka");
+const formatters_1 = require("../utils/formatters");
+const readFile = (0, util_1.promisify)(Fs.readFile);
+function getInput(filename) {
+    return __awaiter(this, void 0, void 0, function* () {
+        if (filename) {
+            const content = yield readFile(filename, 'utf8');
+            const result = JSON.parse(content);
+            return new pool_1.default(result).done();
+        }
+        else {
+            const input = new stream_1.PassThrough();
+            const readLine = (0, readline_1.createInterface)({ input });
+            const dataPool = new pool_1.default();
+            readLine.on('line', (line) => __awaiter(this, void 0, void 0, function* () {
+                dataPool.push(yield JSON.parse(line));
+            }));
+            readLine.on('close', () => dataPool.done());
+            process.stdin.pipe(input);
+            return dataPool;
+        }
+    });
+}
+function produce(topic, opts, { parent }) {
+    var e_1, _a;
+    return __awaiter(this, void 0, void 0, function* () {
+        const _b = Object.assign(Object.assign({}, parent.opts()), opts), { format, header, input: filename, delay, brokers, logLevel, ssl } = _b, rest = __rest(_b, ["format", "header", "input", "delay", "brokers", "logLevel", "ssl"]);
+        const sasl = (0, kafka_1.getSASL)(rest);
+        const client = (0, kafka_1.createClient)(brokers, ssl, sasl, logLevel);
+        const producer = yield (0, kafka_1.createProducer)(client, topic);
+        const staticHeaders = header.reduce((result, header) => {
+            try {
+                const [, key, value] = header.match(/^([^:]+):(.*)$/);
+                return Object.assign(Object.assign({}, result), { [key.trim()]: value.trim() });
+            }
+            catch (e) {
+                console.error(e);
+            }
+            return result;
+        }, {});
+        const formatter = (0, formatters_1.getFormatter)(format);
+        const input = yield getInput(filename);
+        try {
+            for (var input_1 = __asyncValues(input), input_1_1; input_1_1 = yield input_1.next(), !input_1_1.done;) {
+                let { key, value, headers } = input_1_1.value;
+                const encodedValue = yield formatter.encode(value);
+                const message = { key, value: encodedValue, headers: Object.assign(Object.assign({}, staticHeaders), headers) };
+                producer.push(message);
+                yield new Promise(resolve => setTimeout(resolve, delay));
+            }
+        }
+        catch (e_1_1) { e_1 = { error: e_1_1 }; }
+        finally {
+            try {
+                if (input_1_1 && !input_1_1.done && (_a = input_1.return)) yield _a.call(input_1);
+            }
+            finally { if (e_1) throw e_1.error; }
+        }
+        producer.done();
+    });
+}
+exports.default = produce;

package/build/index.js

@@ -0,0 +1,93 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+const commander_1 = require("commander");
+const kafka_1 = require("./utils/kafka");
+const consume_1 = __importDefault(require("./commands/consume"));
+const produce_1 = __importDefault(require("./commands/produce"));
+const metadata_1 = __importDefault(require("./commands/metadata"));
+const list_1 = __importDefault(require("./commands/list"));
+const config_1 = __importDefault(require("./commands/config"));
+const createTopic_1 = __importDefault(require("./commands/createTopic"));
+const deleteTopic_1 = __importDefault(require("./commands/deleteTopic"));
+const { version } = require('../package.json');
+function collect(value, result) {
+    return result.concat([value]);
+}
+const commander = new commander_1.Command();
+commander
+    .option('-b, --brokers <brokers>', 'bootstrap server host', process.env.KAFKA_BROKERS || 'localhost:9092')
+    .option('-l, --log-level <logLevel>', 'log level')
+    .option('--ssl', 'enable ssl', false)
+    .option('--mechanism <mechanism>', 'sasl mechanism', process.env.KAFKA_MECHANISM)
+    .option('--username <username>', 'sasl username', process.env.KAFKA_USERNAME)
+    .option('--password <password>', 'sasl password', process.env.KAFKA_PASSWORD)
+    .option('--auth-id <authId>', 'sasl aws authorization identity', process.env.KAFKA_AUTH_ID)
+    .option('--access-key-id <accessKeyId>', 'sasl aws access key id', process.env.KAFKA_ACCESS_KEY_ID)
+    .option('--secret-access-key <secretAccessKey>', 'sasl aws secret access key', process.env.KAFKA_SECRET_ACCESS_KEY)
+    .option('--session-token <seccionToken>', 'sasl aws session token', process.env.KAFKA_SESSION_TOKEN)
+    .option('--oauth-bearer <oauthBearer>', 'sasl oauth bearer token', process.env.KAFKA_OAUTH_BEARER)
+    .version(version);
+commander
+    .command('consume <topic>')
+    .requiredOption('-g, --group <group>', 'consumer group name')
+    .option('-f, --format <format>', 'message type decoding', 'json')
+    .option('-o, --output <filename>', 'write output to specified filename')
+    .option('-a, --from-beginning', 'read messages from the beginning', false)
+    .description('Consume kafka topic events')
+    .action(consume_1.default);
+commander
+    .command('produce <topic>')
+    .option('-f, --format <format>', 'message format encoding', 'json')
+    .option('-i, --input <filename>', 'input filename')
+    .option('-d, --delay <delay>', 'delay in ms after event emitting', parseInt, 0)
+    .option('-h, --header <header>', 'set a static header', collect, [])
+    .description('Produce kafka topic events')
+    .action(produce_1.default);
+commander
+    .command('metadata')
+    .description('Displays kafka server metadata')
+    .action(metadata_1.default);
+commander
+    .command('list')
+    .alias('ls')
+    .option('-a, --all', 'include internal topics')
+    .description('Lists kafka topics')
+    .action(list_1.default);
+commander
+    .command('config')
+    .requiredOption('-r, --resource <resource>', 'resource', kafka_1.resourceParser)
+    .requiredOption('-n, --resourceName <resourceName>', 'resource name')
+    .description('Describes config for specific resource')
+    .action(config_1.default);
+commander
+    .command('create <topic>')
+    .description('Creates kafka topic')
+    .action(createTopic_1.default);
+commander
+    .command('delete <topic>')
+    .description('Deletes kafka topic')
+    .action(deleteTopic_1.default);
+commander.on('--help', function () {
+    [
+        '',
+        'Examples:',
+        '',
+        '  General consumer usage',
+        '  $ kcli consume $KAFKA_TOPIC -g $KAFKA_TOPIC_GROUP -b $KAFKA_BROKERS --ssl --mechanism plain --username $KAFKA_USERNAME --password $KAFKA_PASSWORD',
+        '',
+        '  Extracting consumer output with jq',
+        '  $ kcli consume $KAFKA_TOPIC -g $KAFKA_TOPIC_GROUP --f ./formatter/avro.js | jq .value',
+        '',
+        '  General producer usage',
+        '  $ kcli produce $KAFKA_TOPIC -b $KAFKA_BROKERS --ssl --mechanism plain --username $KAFKA_USERNAME --password $KAFKA_PASSWORD',
+        '',
+        '  Preparing producer payload json data with jq',
+        '  $ cat payload.json|jq -r -c .[]|kcli produce $KAFKA_TOPIC -f ./formatter/avro.js',
+        '',
+    ].forEach(msg => console.log(msg));
+});
+commander.parse(process.argv);
+exports.default = commander;

package/build/utils/formatters.js

@@ -0,0 +1,28 @@
+"use strict";
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.getFormatter = exports.json = exports.js = void 0;
+const path_1 = __importDefault(require("path"));
+const vm_1 = require("vm");
+exports.js = {
+    encode: (value) => JSON.stringify(value, null, '  '),
+    decode: (value) => (0, vm_1.runInNewContext)(value.toString(), { module: {} }),
+};
+exports.json = {
+    encode: (value) => JSON.stringify(value),
+    decode: (value) => JSON.parse(value.toString()),
+};
+function getFormatter(format) {
+    switch (format) {
+        case 'json':
+            return exports.json;
+        case 'js':
+            return exports.js;
+        default:
+            const modulePath = path_1.default.resolve(process.cwd(), format);
+            return require(modulePath);
+    }
+}
+exports.getFormatter = getFormatter;

package/build/utils/kafka.js

@@ -0,0 +1,166 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __asyncValues = (this && this.__asyncValues) || function (o) {
+    if (!Symbol.asyncIterator) throw new TypeError("Symbol.asyncIterator is not defined.");
+    var m = o[Symbol.asyncIterator], i;
+    return m ? m.call(o) : (o = typeof __values === "function" ? __values(o) : o[Symbol.iterator](), i = {}, verb("next"), verb("throw"), verb("return"), i[Symbol.asyncIterator] = function () { return this; }, i);
+    function verb(n) { i[n] = o[n] && function (v) { return new Promise(function (resolve, reject) { v = o[n](v), settle(resolve, reject, v.done, v.value); }); }; }
+    function settle(resolve, reject, d, v) { Promise.resolve(v).then(function(v) { resolve({ value: v, done: d }); }, reject); }
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.createProducer = exports.createConsumer = exports.createAdmin = exports.createCluster = exports.createClient = exports.getSASL = exports.resourceParser = exports.logLevelParser = void 0;
+const kafkajs_1 = require("kafkajs");
+const pool_1 = __importDefault(require("./pool"));
+function logLevelParser(level) {
+    if (/error/.test(level)) {
+        return kafkajs_1.logLevel.ERROR;
+    }
+    if (/warn/.test(level)) {
+        return kafkajs_1.logLevel.WARN;
+    }
+    if (/info/.test(level)) {
+        return kafkajs_1.logLevel.INFO;
+    }
+    if (/debug/.test(level)) {
+        return kafkajs_1.logLevel.DEBUG;
+    }
+    return kafkajs_1.logLevel.NOTHING;
+}
+exports.logLevelParser = logLevelParser;
+function resourceParser(resource) {
+    if (/^any$/i.test(resource)) {
+        return kafkajs_1.ResourceTypes.ANY;
+    }
+    if (/^topic$/i.test(resource)) {
+        return kafkajs_1.ResourceTypes.TOPIC;
+    }
+    if (/^group$/i.test(resource)) {
+        return kafkajs_1.ResourceTypes.GROUP;
+    }
+    if (/^cluster$/i.test(resource)) {
+        return kafkajs_1.ResourceTypes.CLUSTER;
+    }
+    if (/^transactional.?id$/i.test(resource)) {
+        return kafkajs_1.ResourceTypes.TRANSACTIONAL_ID;
+    }
+    if (/^delegation.?token$/i.test(resource)) {
+        return kafkajs_1.ResourceTypes.DELEGATION_TOKEN;
+    }
+    return kafkajs_1.ResourceTypes.UNKNOWN;
+}
+exports.resourceParser = resourceParser;
+const SASLMap = {
+    'plain': (username, password) => ({ username, password }),
+    'scram-sha-256': (username, password) => ({ username, password }),
+    'scram-sha-512': (username, password) => ({ username, password }),
+};
+function getSASL({ mechanism, username, password, authorizationIdentity, accessKeyId, secretAccessKey, sessionToken, oauthBearer, }) {
+    switch (mechanism) {
+        case 'plain':
+        case 'scram-sha-256':
+        case 'scram-sha-512':
+            return { mechanism, username, password };
+        case 'aws':
+            return { mechanism, authorizationIdentity, accessKeyId, secretAccessKey, sessionToken };
+        case 'oauthbearer':
+            const oauthBearerProvider = () => __awaiter(this, void 0, void 0, function* () { return ({ value: oauthBearer }); });
+            return { mechanism, oauthBearerProvider };
+    }
+}
+exports.getSASL = getSASL;
+function createClient(bootstrapServer, ssl, sasl, level) {
+    const options = {
+        clientId: 'Kafka CLI',
+        brokers: bootstrapServer.split(','),
+        ssl,
+        logLevel: logLevelParser(level),
+        sasl,
+    };
+    return new kafkajs_1.Kafka(options);
+}
+exports.createClient = createClient;
+function createCluster(client) {
+    return __awaiter(this, void 0, void 0, function* () {
+        const createClusterKey = Reflect.ownKeys(client).find(key => /createCluster/.test(key.toString()));
+        const createCluster = Reflect.get(client, createClusterKey);
+        const cluster = createCluster({});
+        yield cluster.connect();
+        return cluster;
+    });
+}
+exports.createCluster = createCluster;
+function createAdmin(client) {
+    return __awaiter(this, void 0, void 0, function* () {
+        const admin = client.admin();
+        yield admin.connect();
+        return admin;
+    });
+}
+exports.createAdmin = createAdmin;
+function createConsumer(client, group, topic, fromBeginning = false) {
+    return __awaiter(this, void 0, void 0, function* () {
+        const consumerConfig = {
+            groupId: group,
+        };
+        const consumerOptions = {
+            topic,
+            fromBeginning,
+        };
+        const consumer = client.consumer(consumerConfig);
+        yield consumer.connect();
+        yield consumer.subscribe(consumerOptions);
+        const pool = new pool_1.default();
+        pool.onDone(() => {
+            consumer.disconnect();
+        });
+        consumer.run({
+            eachMessage: (payload) => __awaiter(this, void 0, void 0, function* () {
+                pool.push(payload);
+            }),
+        }).catch(e => console.error(e));
+        return pool;
+    });
+}
+exports.createConsumer = createConsumer;
+function createProducer(client, topic) {
+    return __awaiter(this, void 0, void 0, function* () {
+        const producer = client.producer();
+        yield producer.connect();
+        const pool = new pool_1.default();
+        pool.onDone(() => {
+            producer.disconnect();
+        });
+        (() => __awaiter(this, void 0, void 0, function* () {
+            var e_1, _a;
+            try {
+                for (var pool_2 = __asyncValues(pool), pool_2_1; pool_2_1 = yield pool_2.next(), !pool_2_1.done;) {
+                    let message = pool_2_1.value;
+                    yield producer.send({
+                        topic,
+                        messages: [message],
+                    });
+                }
+            }
+            catch (e_1_1) { e_1 = { error: e_1_1 }; }
+            finally {
+                try {
+                    if (pool_2_1 && !pool_2_1.done && (_a = pool_2.return)) yield _a.call(pool_2);
+                }
+                finally { if (e_1) throw e_1.error; }
+            }
+        }))().catch(e => console.error(e));
+        return pool;
+    });
+}
+exports.createProducer = createProducer;

package/build/utils/pool.js

@@ -0,0 +1,99 @@
+"use strict";
+var __awaiter = (this && this.__awaiter) || function (thisArg, _arguments, P, generator) {
+    function adopt(value) { return value instanceof P ? value : new P(function (resolve) { resolve(value); }); }
+    return new (P || (P = Promise))(function (resolve, reject) {
+        function fulfilled(value) { try { step(generator.next(value)); } catch (e) { reject(e); } }
+        function rejected(value) { try { step(generator["throw"](value)); } catch (e) { reject(e); } }
+        function step(result) { result.done ? resolve(result.value) : adopt(result.value).then(fulfilled, rejected); }
+        step((generator = generator.apply(thisArg, _arguments || [])).next());
+    });
+};
+var __asyncValues = (this && this.__asyncValues) || function (o) {
+    if (!Symbol.asyncIterator) throw new TypeError("Symbol.asyncIterator is not defined.");
+    var m = o[Symbol.asyncIterator], i;
+    return m ? m.call(o) : (o = typeof __values === "function" ? __values(o) : o[Symbol.iterator](), i = {}, verb("next"), verb("throw"), verb("return"), i[Symbol.asyncIterator] = function () { return this; }, i);
+    function verb(n) { i[n] = o[n] && function (v) { return new Promise(function (resolve, reject) { v = o[n](v), settle(resolve, reject, v.done, v.value); }); }; }
+    function settle(resolve, reject, d, v) { Promise.resolve(v).then(function(v) { resolve({ value: v, done: d }); }, reject); }
+};
+var __importDefault = (this && this.__importDefault) || function (mod) {
+    return (mod && mod.__esModule) ? mod : { "default": mod };
+};
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.CancelToken = exports.createItem = void 0;
+const evnty_1 = __importDefault(require("evnty"));
+function createItem() {
+    const result = {};
+    result.promise = new Promise((resolve, reject) => {
+        result.resolve = resolve;
+        result.reject = reject;
+    });
+    return result;
+}
+exports.createItem = createItem;
+class CancelToken {
+}
+exports.CancelToken = CancelToken;
+class Pool {
+    constructor(values = []) {
+        this.pool = [createItem()];
+        this.doneEvent = (0, evnty_1.default)();
+        values.forEach(value => this.push(value));
+    }
+    push(value) {
+        this.pool[this.pool.length - 1].resolve(value);
+        this.pool.push(createItem());
+        return this;
+    }
+    done() {
+        this.pool[this.pool.length - 1].resolve(new CancelToken());
+        return this;
+    }
+    onDone(callback) {
+        this.doneEvent.on(callback);
+        return this;
+    }
+    toArray() {
+        var e_1, _a;
+        return __awaiter(this, void 0, void 0, function* () {
+            const result = [];
+            try {
+                for (var _b = __asyncValues(this), _c; _c = yield _b.next(), !_c.done;) {
+                    let item = _c.value;
+                    result.push(item);
+                }
+            }
+            catch (e_1_1) { e_1 = { error: e_1_1 }; }
+            finally {
+                try {
+                    if (_c && !_c.done && (_a = _b.return)) yield _a.call(_b);
+                }
+                finally { if (e_1) throw e_1.error; }
+            }
+            return result;
+        });
+    }
+    [Symbol.asyncIterator]() {
+        let i = 0;
+        return {
+            i,
+            next: () => __awaiter(this, void 0, void 0, function* () {
+                const value = yield this.pool[0].promise;
+                this.pool.shift();
+                if (value instanceof CancelToken) {
+                    this.doneEvent();
+                    return {
+                        i: ++i,
+                        done: true,
+                    };
+                }
+                return {
+                    i: ++i,
+                    value,
+                    done: false,
+                };
+            }),
+        };
+    }
+    ;
+}
+exports.default = Pool;

package/package.json

@@ -0,0 +1,44 @@
+{
+  "name": "kafka-console",
+  "version": "1.1.0",
+  "description": "Kafka CLI tool",
+  "main": "index.js",
+  "bin": {
+    "kcli": "build/cli.js",
+    "kafka-console": "build/cli.js"
+  },
+  "scripts": {
+    "build": "rm -rf build && tsc",
+    "test": "NODE_ENV=test jest",
+    "test:cov": "COVERAGE=1 NODE_ENV=test jest",
+    "kcli": "ts-node src/index.ts"
+  },
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/3axap4eHko/kafka-console.git"
+  },
+  "keywords": [],
+  "author": "Ivan Zakharchanka",
+  "license": "MIT",
+  "bugs": {
+    "url": "https://github.com/3axap4eHko/kafka-console/issues"
+  },
+  "homepage": "https://github.com/3axap4eHko/kafka-console#readme",
+  "devDependencies": {
+    "@types/commander": "^2.12.2",
+    "@types/jest": "^27.0.1",
+    "@types/js-yaml": "^4.0.3",
+    "@types/node": "^16.9.2",
+    "@types/winston": "^2.4.4",
+    "dotenv": "^10.0.0",
+    "jest": "^27.2.0",
+    "ts-jest": "^27.0.5",
+    "ts-node": "^10.2.1",
+    "typescript": "^4.4.3"
+  },
+  "dependencies": {
+    "commander": "^8.2.0",
+    "evnty": "^0.6.15",
+    "kafkajs": "^1.15.0"
+  }
+}