kafka-console 1.1.16 → 1.2.3

package/LICENSE

@@ -0,0 +1,24 @@
+The MIT License
+
+Copyright (c) 2021 Ivan Zakharchanka (https://github.com/3axap4eHko)
+
+Permission is hereby granted, free of charge,
+to any person obtaining a copy of this software and
+associated documentation files (the "Software"), to
+deal in the Software without restriction, including
+without limitation the rights to use, copy, modify,
+merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom
+the Software is furnished to do so,
+subject to the following conditions:
+
+The above copyright notice and this permission notice
+shall be included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
+OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
+IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR
+ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
+TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
+SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

package/README.md

@@ -5,16 +5,81 @@ Command line tool to sufficiently and easy work with Kafka
 [![NPM version][npm-image]][npm-url]
 [![Downloads][downloads-image]][npm-url]
 
-
-## Installation
+## Table of Contents
+
+  - [Features](#features)
+  - [Installing](#installing)
+  - [Examples](#examples)
+    - [Consumer](#consumer)
+    - [Producer](#producer)
+    - [Formatters](#formatters)
+  - [Environment](#environment)
+  - [License](#license)
+
+## Features
+
+- Producer
+- Consumer groups with seek and timeout
+- Built-in message encoders/decoders with types: json, js, raw
+- Custom message encoders/decoders as a js module
+- Message headers
+- GZIP compression
+- Plain, SSL and SASL_SSL implementations
+- Admin client
+- TypeScript support
+
+## Installing
 
 ```sh
 npm install -g kafka-console
 ```
 
-## Usage
+## Examples
+
+### Common options
+```
+  -b, --brokers <brokers>                bootstrap server host (default: "localhost:9092")
+  -l, --log-level <logLevel>             log level
+  -t, --timeout <timeout>                set a timeout of operation (default: "0")
+  --ssl                                  enable ssl (default: false)
+  --mechanism <mechanism>                sasl mechanism
+  --username <username>                  sasl username
+  --password <password>                  sasl password
+  --auth-id <authId>                     sasl aws authorization identity
+  --access-key-id <accessKeyId>          sasl aws access key id
+  --secret-access-key <secretAccessKey>  sasl aws secret access key
+  --session-token <seccionToken>         sasl aws session token
+  --oauth-bearer <oauthBearer>           sasl oauth bearer token
+  -V, --version                          output the version number
+  -h, --help                             display help for command
+```
+
+### Commands
+```
+  consume [options] <topic>              Consume kafka topic events
+  produce [options] <topic>              Produce kafka topic events
+  metadata                               Displays kafka server metadata
+  list|ls [options]                      Lists kafka topics
+  config [options]                       Describes config for specific resource
+  create <topic>                         Creates kafka topic
+  delete <topic>                         Deletes kafka topic
+  help [command]                         display help for command
+```
+
+### Consumer
 
-### Consumer usage
+`kcli consume [options] <topic>`
+
+#### Options
+```
+  -g, --group <group>      consumer group name
+  -f, --format <format>    message type decoding json, js, raw (default: "json")
+  -o, --output <filename>  write output to specified filename
+  -a, --from-beginning     read messages from the beginning (default: false)
+  -c, --count <count>      a number of messages to read (default: null)
+  -s, --skip <skip>        a number of messages to skip (default: 0)
+  -h, --help               display help for command
+```
 
 General usage with authentication
 ```sh
@@ -31,7 +96,18 @@ Custom data formatter example
 kcli consume $KAFKA_TOPIC --format ./formatter/avro.js | jq
 ```
 
-### Producer usage
+### Producer
+
+`kcli produce [options] <topic>`
+
+#### Options
+```
+  -f, --format <format>   message format encoding json, js, raw (default: "json")
+  -i, --input <filename>  input filename
+  -d, --delay <delay>     delay in ms after event emitting (default: 0)
+  -h, --header <header>   set a static header (default: [])
+  --help                  display help for command
+```
 
 General usage
 ```sh
@@ -70,7 +146,7 @@ export interface Encoder<T> {
 }
 
 export interface Decoder<T> {
-  (value: string | Buffer): Promise<T> | T;
+  (value: Buffer): Promise<T> | T;
 }
 
 export interface Formatter<T> {
@@ -79,6 +155,19 @@ export interface Formatter<T> {
 }
 ```
 
+## Environment
+
+ - KAFKA_BROKERS
+ - KAFKA_TIMEOUT
+ - KAFKA_MECHANISM
+ - KAFKA_USERNAME
+ - KAFKA_PASSWORD
+ - KAFKA_AUTH_ID
+ - KAFKA_ACCESS_KEY_ID
+ - KAFKA_SECRET_ACCESS_KEY
+ - KAFKA_SESSION_TOKEN
+ - KAFKA_OAUTH_BEARER
+
 ## License
 License [The MIT License](http://opensource.org/licenses/MIT)
 Copyright (c) 2021 Ivan Zakharchanka

package/build/commands/consume.js

@@ -52,11 +52,17 @@ const formatters_1 = require("../utils/formatters");
 function consume(topic, opts, { parent }) {
     var e_1, _a;
     return __awaiter(this, void 0, void 0, function* () {
-        const _b = Object.assign(Object.assign({}, parent.opts()), opts), { group, format, fromBeginning, filename, brokers, logLevel, ssl } = _b, rest = __rest(_b, ["group", "format", "fromBeginning", "filename", "brokers", "logLevel", "ssl"]);
-        const sasl = (0, kafka_1.getSASL)(rest);
+        const _b = Object.assign(Object.assign({}, parent.opts()), opts), { group, format, fromBeginning, count, skip, filename, brokers, logLevel, timeout, ssl } = _b, saslOptions = __rest(_b, ["group", "format", "fromBeginning", "count", "skip", "filename", "brokers", "logLevel", "timeout", "ssl"]);
+        const sasl = (0, kafka_1.getSASL)(saslOptions);
         const client = (0, kafka_1.createClient)(brokers, ssl, sasl, logLevel);
         const output = filename ? Fs.createWriteStream(filename) : process.stdout;
-        const consumer = yield (0, kafka_1.createConsumer)(client, group, topic, fromBeginning);
+        const consumer = yield (0, kafka_1.createConsumer)(client, group, topic, fromBeginning, { skip, count, timeout });
+        consumer.onDone((timeouted) => {
+            if (timeouted) {
+                console.log("TIMEOUT");
+                process.exit(1);
+            }
+        });
         const formatter = (0, formatters_1.getFormatter)(format);
         try {
             for (var consumer_1 = __asyncValues(consumer), consumer_1_1; consumer_1_1 = yield consumer_1.next(), !consumer_1_1.done;) {

package/build/index.js

@@ -3,7 +3,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.collect = void 0;
+exports.toInt = exports.collect = void 0;
 const commander_1 = require("commander");
 const kafka_1 = require("./utils/kafka");
 const consume_1 = __importDefault(require("./commands/consume"));
@@ -18,10 +18,15 @@ function collect(value, result) {
     return result.concat([value]);
 }
 exports.collect = collect;
+function toInt(value, result) {
+    return parseInt(value, 10);
+}
+exports.toInt = toInt;
 const commander = new commander_1.Command();
 commander
     .option('-b, --brokers <brokers>', 'bootstrap server host', process.env.KAFKA_BROKERS || 'localhost:9092')
     .option('-l, --log-level <logLevel>', 'log level')
+    .option('-t, --timeout <timeout>', 'set a timeout of operation', toInt, process.env.KAFKA_TIMEOUT || '0')
     .option('--ssl', 'enable ssl', false)
     .option('--mechanism <mechanism>', 'sasl mechanism', process.env.KAFKA_MECHANISM)
     .option('--username <username>', 'sasl username', process.env.KAFKA_USERNAME)
@@ -35,16 +40,18 @@ commander
 commander
     .command('consume <topic>')
     .requiredOption('-g, --group <group>', 'consumer group name')
-    .option('-f, --format <format>', 'message type decoding', 'json')
+    .option('-f, --format <format>', 'message type decoding json, js, raw', 'json')
     .option('-o, --output <filename>', 'write output to specified filename')
     .option('-a, --from-beginning', 'read messages from the beginning', false)
+    .option('-c, --count <count>', 'a number of messages to read', toInt, Infinity)
+    .option('-s, --skip <skip>', 'a number of messages to skip', toInt, 0)
     .description('Consume kafka topic events')
     .action(consume_1.default);
 commander
     .command('produce <topic>')
-    .option('-f, --format <format>', 'message format encoding', 'json')
+    .option('-f, --format <format>', 'message format encoding json, js, raw', 'json')
     .option('-i, --input <filename>', 'input filename')
-    .option('-d, --delay <delay>', 'delay in ms after event emitting', parseInt, 0)
+    .option('-d, --delay <delay>', 'delay in ms after event emitting', toInt, 0)
     .option('-h, --header <header>', 'set a static header', collect, [])
     .description('Produce kafka topic events')
     .action(produce_1.default);

package/build/utils/formatters.js

@@ -3,23 +3,33 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.getFormatter = exports.json = exports.js = void 0;
+exports.getFormatter = exports.raw = exports.js = exports.json = void 0;
 const path_1 = __importDefault(require("path"));
 const vm_1 = require("vm");
-exports.js = {
-    encode: (value) => JSON.stringify(value, null, '  '),
-    decode: (value) => (0, vm_1.runInNewContext)(value.toString(), { module: {} }),
-};
 exports.json = {
     encode: (value) => JSON.stringify(value),
     decode: (value) => JSON.parse(value.toString()),
 };
+exports.js = {
+    encode: (value) => JSON.stringify(value, null, '  '),
+    decode: (value) => {
+        const m = { exports: {} };
+        (0, vm_1.runInNewContext)(value.toString(), { module: m });
+        return m.exports;
+    },
+};
+exports.raw = {
+    encode: (value) => value,
+    decode: (value) => value.toString('utf8'),
+};
 function getFormatter(format) {
     switch (format) {
         case 'json':
             return exports.json;
         case 'js':
             return exports.js;
+        case 'raw':
+            return exports.raw;
         default:
             const modulePath = path_1.default.resolve(process.cwd(), format);
             return require(modulePath);

package/build/utils/kafka.js

@@ -108,7 +108,7 @@ function createAdmin(client) {
     });
 }
 exports.createAdmin = createAdmin;
-function createConsumer(client, group, topic, fromBeginning = false) {
+function createConsumer(client, group, topic, fromBeginning = false, poolOptions = {}) {
     return __awaiter(this, void 0, void 0, function* () {
         const consumerConfig = {
             groupId: group,
@@ -120,7 +120,7 @@ function createConsumer(client, group, topic, fromBeginning = false) {
         const consumer = client.consumer(consumerConfig);
         yield consumer.connect();
         yield consumer.subscribe(consumerOptions);
-        const pool = new pool_1.default();
+        const pool = new pool_1.default([], poolOptions);
         pool.onDone(() => {
             consumer.disconnect();
         });

package/build/utils/pool.js

@@ -31,21 +31,36 @@ function createItem() {
 }
 exports.createItem = createItem;
 class CancelToken {
+    constructor(timeout) {
+        this.timeout = timeout;
+    }
 }
 exports.CancelToken = CancelToken;
 class Pool {
-    constructor(values = []) {
+    constructor(values = [], { skip = 0, count = Infinity, timeout } = {}) {
         this.pool = [createItem()];
         this.doneEvent = (0, evnty_1.default)();
-        values.forEach(value => this.push(value));
+        this.index = 0;
+        this.skip = skip;
+        this.count = count + skip;
+        if (typeof timeout === 'number') {
+            this.timerId = setTimeout(() => this.done(true), timeout);
+        }
+        values.forEach((value) => this.push(value));
     }
     push(value) {
-        this.pool[this.pool.length - 1].resolve(value);
-        this.pool.push(createItem());
+        if (this.index >= this.skip && this.index < this.count) {
+            this.pool[this.pool.length - 1].resolve(value);
+            this.pool.push(createItem());
+        }
+        this.index++;
+        if (this.index >= this.count) {
+            this.done();
+        }
         return this;
     }
-    done() {
-        this.pool[this.pool.length - 1].resolve(new CancelToken());
+    done(timeout = false) {
+        this.pool[this.pool.length - 1].resolve(new CancelToken(timeout));
         return this;
     }
     onDone(callback) {
@@ -80,7 +95,8 @@ class Pool {
                 const value = yield this.pool[0].promise;
                 this.pool.shift();
                 if (value instanceof CancelToken) {
-                    this.doneEvent();
+                    clearTimeout(this.timerId);
+                    this.doneEvent(value.timeout);
                     return {
                         i: ++i,
                         done: true,
@@ -94,6 +110,5 @@ class Pool {
             }),
         };
     }
-    ;
 }
 exports.default = Pool;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "kafka-console",
-  "version": "1.1.16",
+  "version": "1.2.3",
   "description": "Kafka CLI tool",
   "main": "index.js",
   "bin": {
@@ -17,7 +17,14 @@
     "type": "git",
     "url": "git+https://github.com/3axap4eHko/kafka-console.git"
   },
-  "keywords": [],
+  "keywords": [
+    "kafka",
+    "cli",
+    "commands",
+    "sasl",
+    "consumer",
+    "producer"
+  ],
   "author": "Ivan Zakharchanka",
   "license": "MIT",
   "bugs": {
@@ -26,19 +33,19 @@
   "homepage": "https://github.com/3axap4eHko/kafka-console#readme",
   "devDependencies": {
     "@types/commander": "^2.12.2",
-    "@types/jest": "^27.0.2",
-    "@types/js-yaml": "^4.0.4",
-    "@types/node": "^16.11.6",
+    "@types/jest": "^27.0.3",
+    "@types/js-yaml": "^4.0.5",
+    "@types/node": "^17.0.2",
     "@types/winston": "^2.4.4",
     "dotenv": "^10.0.0",
-    "jest": "^27.3.1",
-    "ts-jest": "^27.0.7",
+    "jest": "^27.4.5",
+    "ts-jest": "^27.1.2",
     "ts-node": "^10.4.0",
-    "typescript": "^4.4.4"
+    "typescript": "^4.5.4"
   },
   "dependencies": {
     "commander": "^8.3.0",
-    "evnty": "^0.6.16",
+    "evnty": "^0.6.19",
     "kafkajs": "^1.15.0"
   }
 }