kafka-console 1.4.50 → 2.0.1

package/README.md

@@ -41,7 +41,7 @@ npm install -g kafka-console
   -b, --brokers <brokers>                bootstrap server host (default: "localhost:9092")
   -l, --log-level <logLevel>             log level
   -t, --timeout <timeout>                set a timeout of operation (default: "0")
-  -p, --pretty                           pretty print json
+  -p, --pretty                           pretty print (default: false)
   --ssl                                  enable ssl (default: false)
   --mechanism <mechanism>                sasl mechanism
   --username <username>                  sasl username
@@ -62,8 +62,9 @@ npm install -g kafka-console
   metadata                               Displays kafka server metadata
   list|ls [options]                      Lists kafka topics
   config [options]                       Describes config for specific resource
-  create <topic>                         Creates kafka topic
-  delete <topic>                         Deletes kafka topic
+  topic:create <topic>                   Creates kafka topic
+  topic:delete <topic>                   Deletes kafka topic
+  topic:offsets <topic> [timestamp]      Shows kafka topic offsets
   help [command]                         display help for command
 ```
 
@@ -73,13 +74,13 @@ npm install -g kafka-console
 
 #### Options
 ```
-  -g, --group <group>      consumer group name
-  -f, --format <format>    message type decoding json, js, raw (default: "json")
-  -o, --output <filename>  write output to specified filename
-  -a, --from-beginning     read messages from the beginning (default: false)
-  -c, --count <count>      a number of messages to read (default: null)
-  -s, --skip <skip>        a number of messages to skip (default: 0)
-  -h, --help               display help for command
+  -g, --group <group>              consumer group name (default: "kafka-console-consumer-TIMESTAMP")
+  -d, --data-format <data-format>  messages data-format: json, js, raw (default: "json")
+  -o, --output <filename>          write output to specified filename
+  -f, --from <from>                read messages from the specific timestamp in milliseconds or ISO 8601 format. Set 0 to read from the beginning
+  -c, --count <count>              a number of messages to read (default: null)
+  -s, --skip <skip>                a number of messages to skip (default: 0)
+  -h, --help                       display help for command
 ```
 
 General usage with authentication
@@ -87,14 +88,14 @@ General usage with authentication
 kcli consume $KAFKA_TOPIC -g $KAFKA_TOPIC_GROUP -b $KAFKA_BROKERS --ssl --mechanism plain --username $KAFKA_USERNAME --password $KAFKA_PASSWORD
 ```
 
-Stdout `jq` example
+Stdout from timestamp `jq` example
 ```sh
-kcli consume $KAFKA_TOPIC | jq .value
+kcli consume $KAFKA_TOPIC --from '1970-01-01T00:00:00.000Z' | jq .value
 ```
 
 Custom data formatter example
 ```sh
-kcli consume $KAFKA_TOPIC --format ./formatter/avro.js | jq
+kcli consume $KAFKA_TOPIC --data-format ./formatter/avro.js | jq
 ```
 
 ### Producer
@@ -103,11 +104,11 @@ kcli consume $KAFKA_TOPIC --format ./formatter/avro.js | jq
 
 #### Options
 ```
-  -f, --format <format>   message format encoding json, js, raw (default: "json")
-  -i, --input <filename>  input filename
-  -d, --delay <delay>     delay in ms after event emitting (default: 0)
-  -h, --header <header>   set a static header (default: [])
-  --help                  display help for command
+  -d, --data-format <data-format>  messages data-format: json, js, raw (default: "json")
+  -i, --input <filename>           input filename
+  -w, --wait <wait>                wait the time in ms after sending a message (default: 0)
+  -h, --header <header>            set a static header (default: [])
+  --help                           display help for command
 ```
 
 General usage
@@ -117,7 +118,7 @@ kcli produce $KAFKA_TOPIC -b $KAFKA_BROKERS --ssl --mechanism plain --username $
 
 Produce a json data from stdin with custom formatter
 ```sh
-cat payload.txt|kcli produce $KAFKA_TOPIC --format ./formatter/avro.js
+cat payload.txt|kcli produce $KAFKA_TOPIC --data-format ./formatter/avro.js
 ```
 
 Produce a json data from stdin

package/build/commands/config.js

@@ -20,7 +20,6 @@ var __rest = (this && this.__rest) || function (s, e) {
     return t;
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.default = config;
 const kafka_1 = require("../utils/kafka");
 function config(opts_1, _a) {
     return __awaiter(this, arguments, void 0, function* (opts, { parent }) {
@@ -44,3 +43,4 @@ function config(opts_1, _a) {
         yield admin.disconnect();
     });
 }
+exports.default = config;

package/build/commands/consume.js

@@ -50,25 +50,24 @@ var __asyncValues = (this && this.__asyncValues) || function (o) {
     function settle(resolve, reject, d, v) { Promise.resolve(v).then(function(v) { resolve({ value: v, done: d }); }, reject); }
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.default = consume;
 const Fs = __importStar(require("fs"));
 const kafka_1 = require("../utils/kafka");
 const formatters_1 = require("../utils/formatters");
 function consume(topic_1, opts_1, _a) {
     return __awaiter(this, arguments, void 0, function* (topic, opts, { parent }) {
         var _b, e_1, _c, _d;
-        const _e = Object.assign(Object.assign({}, parent.opts()), opts), { group, format, fromBeginning, count, skip, filename, brokers, logLevel, timeout, ssl, pretty } = _e, saslOptions = __rest(_e, ["group", "format", "fromBeginning", "count", "skip", "filename", "brokers", "logLevel", "timeout", "ssl", "pretty"]);
+        const _e = Object.assign(Object.assign({}, parent.opts()), opts), { group, dataFormat, from, count, skip, filename, brokers, logLevel, timeout, ssl, pretty } = _e, saslOptions = __rest(_e, ["group", "dataFormat", "from", "count", "skip", "filename", "brokers", "logLevel", "timeout", "ssl", "pretty"]);
         const sasl = (0, kafka_1.getSASL)(saslOptions);
         const client = (0, kafka_1.createClient)(brokers, ssl, sasl, logLevel);
         const output = filename ? Fs.createWriteStream(filename) : process.stdout;
-        const consumer = yield (0, kafka_1.createConsumer)(client, group, topic, fromBeginning, { skip, count, timeout });
+        const consumer = yield (0, kafka_1.createConsumer)(client, group, topic, from, { skip, count, timeout });
         consumer.onDone((timeouted) => {
             if (timeouted) {
-                console.log("TIMEOUT");
+                console.error("TIMEOUT");
                 process.exit(1);
             }
         });
-        const formatter = (0, formatters_1.getFormatter)(format);
+        const formatter = (0, formatters_1.getFormatter)(dataFormat);
         try {
             for (var _f = true, consumer_1 = __asyncValues(consumer), consumer_1_1; consumer_1_1 = yield consumer_1.next(), _b = consumer_1_1.done, !_b; _f = true) {
                 _d = consumer_1_1.value;
@@ -91,3 +90,4 @@ function consume(topic_1, opts_1, _a) {
         }
     });
 }
+exports.default = consume;

package/build/commands/createTopic.js

@@ -20,7 +20,6 @@ var __rest = (this && this.__rest) || function (s, e) {
     return t;
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.default = createTopic;
 const kafka_1 = require("../utils/kafka");
 function createTopic(topic_1, opts_1, _a) {
     return __awaiter(this, arguments, void 0, function* (topic, opts, { parent }) {
@@ -36,3 +35,4 @@ function createTopic(topic_1, opts_1, _a) {
         yield admin.disconnect();
     });
 }
+exports.default = createTopic;

package/build/commands/deleteTopic.js

@@ -20,7 +20,6 @@ var __rest = (this && this.__rest) || function (s, e) {
     return t;
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.default = deleteTopic;
 const kafka_1 = require("../utils/kafka");
 function deleteTopic(topic_1, opts_1, _a) {
     return __awaiter(this, arguments, void 0, function* (topic, opts, { parent }) {
@@ -34,3 +33,4 @@ function deleteTopic(topic_1, opts_1, _a) {
         yield admin.disconnect();
     });
 }
+exports.default = deleteTopic;

package/build/commands/fetchTopicOffsets.js

@@ -20,7 +20,6 @@ var __rest = (this && this.__rest) || function (s, e) {
     return t;
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.default = fetchTopicOffset;
 const kafka_1 = require("../utils/kafka");
 function fetchTopicOffset(topic_1, timestamp_1, opts_1, _a) {
     return __awaiter(this, arguments, void 0, function* (topic, timestamp, opts, { parent }) {
@@ -44,3 +43,4 @@ function fetchTopicOffset(topic_1, timestamp_1, opts_1, _a) {
         yield admin.disconnect();
     });
 }
+exports.default = fetchTopicOffset;

package/build/commands/list.js

@@ -20,7 +20,6 @@ var __rest = (this && this.__rest) || function (s, e) {
     return t;
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.default = list;
 const kafka_1 = require("../utils/kafka");
 function list(opts_1, _a) {
     return __awaiter(this, arguments, void 0, function* (opts, { parent }) {
@@ -36,3 +35,4 @@ function list(opts_1, _a) {
         yield cluster.disconnect();
     });
 }
+exports.default = list;

package/build/commands/metadata.js

@@ -20,7 +20,6 @@ var __rest = (this && this.__rest) || function (s, e) {
     return t;
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.default = metadata;
 const kafka_1 = require("../utils/kafka");
 function metadata(opts_1, _a) {
     return __awaiter(this, arguments, void 0, function* (opts, { parent }) {
@@ -34,3 +33,4 @@ function metadata(opts_1, _a) {
         yield cluster.disconnect();
     });
 }
+exports.default = metadata;

package/build/commands/produce.js

@@ -53,7 +53,6 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.default = produce;
 const Fs = __importStar(require("fs"));
 const stream_1 = require("stream");
 const readline_1 = require("readline");
@@ -85,7 +84,7 @@ function getInput(filename) {
 function produce(topic_1, opts_1, _a) {
     return __awaiter(this, arguments, void 0, function* (topic, opts, { parent }) {
         var _b, e_1, _c, _d;
-        const _e = Object.assign(Object.assign({}, parent.opts()), opts), { format, header, input: filename, delay, brokers, logLevel, ssl } = _e, rest = __rest(_e, ["format", "header", "input", "delay", "brokers", "logLevel", "ssl"]);
+        const _e = Object.assign(Object.assign({}, parent.opts()), opts), { dataFormat, header, input: filename, wait, brokers, logLevel, ssl } = _e, rest = __rest(_e, ["dataFormat", "header", "input", "wait", "brokers", "logLevel", "ssl"]);
         const sasl = (0, kafka_1.getSASL)(rest);
         const client = (0, kafka_1.createClient)(brokers, ssl, sasl, logLevel);
         const producer = yield (0, kafka_1.createProducer)(client, topic);
@@ -99,7 +98,7 @@ function produce(topic_1, opts_1, _a) {
             }
             return result;
         }, {});
-        const formatter = (0, formatters_1.getFormatter)(format);
+        const formatter = (0, formatters_1.getFormatter)(dataFormat);
         const input = yield getInput(filename);
         try {
             for (var _f = true, input_1 = __asyncValues(input), input_1_1; input_1_1 = yield input_1.next(), _b = input_1_1.done, !_b; _f = true) {
@@ -109,7 +108,7 @@ function produce(topic_1, opts_1, _a) {
                 const encodedValue = yield formatter.encode(value);
                 const message = { key, value: encodedValue, headers: Object.assign(Object.assign({}, staticHeaders), headers) };
                 producer.push(message);
-                yield new Promise(resolve => setTimeout(resolve, delay));
+                yield new Promise(resolve => setTimeout(resolve, wait));
             }
         }
         catch (e_1_1) { e_1 = { error: e_1_1 }; }
@@ -122,3 +121,4 @@ function produce(topic_1, opts_1, _a) {
         producer.done();
     });
 }
+exports.default = produce;

package/build/index.js

@@ -3,8 +3,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.collect = collect;
-exports.toInt = toInt;
+exports.toInt = exports.collect = void 0;
 const commander_1 = require("commander");
 const kafka_1 = require("./utils/kafka");
 const consume_1 = __importDefault(require("./commands/consume"));
@@ -19,9 +18,11 @@ const { version } = require('../package.json');
 function collect(value, result) {
     return result.concat([value]);
 }
+exports.collect = collect;
 function toInt(value, result) {
     return parseInt(value, 10);
 }
+exports.toInt = toInt;
 const commander = new commander_1.Command();
 commander
     .option('-b, --brokers <brokers>', 'bootstrap server host', process.env.KAFKA_BROKERS || 'localhost:9092')
@@ -40,19 +41,19 @@ commander
     .version(version);
 commander
     .command('consume <topic>')
-    .requiredOption('-g, --group <group>', 'consumer group name')
-    .option('-f, --format <format>', 'message type decoding json, js, raw', 'json')
+    .requiredOption('-g, --group <group>', 'consumer group name', `kafka-console-consumer-${Date.now()}`)
+    .option('-d, --data-format <data-format>', 'messages data-format: json, js, raw', 'json')
     .option('-o, --output <filename>', 'write output to specified filename')
-    .option('-a, --from-beginning', 'read messages from the beginning', false)
+    .option('-f, --from <from>', 'read messages from the specific timestamp in milliseconds or ISO 8601 format. Set 0 to read from the beginning')
     .option('-c, --count <count>', 'a number of messages to read', toInt, Infinity)
     .option('-s, --skip <skip>', 'a number of messages to skip', toInt, 0)
     .description('Consume kafka topic events')
     .action(consume_1.default);
 commander
     .command('produce <topic>')
-    .option('-f, --format <format>', 'message format encoding json, js, raw', 'json')
+    .option('-d, --data-format <data-format>', 'messages data-format: json, js, raw', 'json')
     .option('-i, --input <filename>', 'input filename')
-    .option('-d, --delay <delay>', 'delay in ms after event emitting', toInt, 0)
+    .option('-w, --wait <wait>', 'wait the time in ms after sending a message', toInt, 0)
     .option('-h, --header <header>', 'set a static header', collect, [])
     .description('Produce kafka topic events')
     .action(produce_1.default);
@@ -90,16 +91,16 @@ commander.on('--help', function () {
         'Examples:',
         '',
         '  General consumer usage',
-        '  $ kcli consume $KAFKA_TOPIC -g $KAFKA_TOPIC_GROUP -b $KAFKA_BROKERS --ssl --mechanism plain --username $KAFKA_USERNAME --password $KAFKA_PASSWORD',
+        '  $ npx kafka-console -b $KAFKA_BROKERS consume $KAFKA_TOPIC -g $KAFKA_TOPIC_GROUP --ssl --mechanism plain --username $KAFKA_USERNAME --password $KAFKA_PASSWORD',
         '',
         '  Extracting consumer output with jq',
-        '  $ kcli consume $KAFKA_TOPIC -g $KAFKA_TOPIC_GROUP --f ./formatter/avro.js | jq .value',
+        '  $ npx kafka-console consume $KAFKA_TOPIC -g $KAFKA_TOPIC_GROUP --f ./formatter/avro.js | jq .value',
         '',
         '  General producer usage',
-        '  $ kcli produce $KAFKA_TOPIC -b $KAFKA_BROKERS --ssl --mechanism plain --username $KAFKA_USERNAME --password $KAFKA_PASSWORD',
+        '  $ npx kafka-console produce $KAFKA_TOPIC -b $KAFKA_BROKERS --ssl --mechanism plain --username $KAFKA_USERNAME --password $KAFKA_PASSWORD',
         '',
         '  Preparing producer payload json data with jq',
-        '  $ cat payload.json|jq -r -c .[]|kcli produce $KAFKA_TOPIC -f ./formatter/avro.js',
+        '  $ cat payload.json|jq -r -c .[]|npx kafka-console produce $KAFKA_TOPIC -f ./formatter/avro.js',
         '',
     ].forEach(msg => console.log(msg));
 });

package/build/utils/formatters.js

@@ -3,8 +3,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.raw = exports.js = exports.json = void 0;
-exports.getFormatter = getFormatter;
+exports.getFormatter = exports.raw = exports.js = exports.json = void 0;
 const path_1 = __importDefault(require("path"));
 const vm_1 = require("vm");
 exports.json = {
@@ -36,3 +35,4 @@ function getFormatter(format) {
             return require(modulePath);
     }
 }
+exports.getFormatter = getFormatter;

package/build/utils/kafka.js

@@ -19,14 +19,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.logLevelParser = logLevelParser;
-exports.resourceParser = resourceParser;
-exports.getSASL = getSASL;
-exports.createClient = createClient;
-exports.createCluster = createCluster;
-exports.createAdmin = createAdmin;
-exports.createConsumer = createConsumer;
-exports.createProducer = createProducer;
+exports.createProducer = exports.createConsumer = exports.createAdmin = exports.createCluster = exports.createClient = exports.getSASL = exports.resourceParser = exports.logLevelParser = void 0;
 const kafkajs_1 = require("kafkajs");
 const pool_1 = __importDefault(require("./pool"));
 function logLevelParser(level) {
@@ -44,6 +37,7 @@ function logLevelParser(level) {
     }
     return kafkajs_1.logLevel.NOTHING;
 }
+exports.logLevelParser = logLevelParser;
 function resourceParser(resource) {
     if (/^any$/i.test(resource)) {
         return kafkajs_1.ConfigResourceTypes.UNKNOWN;
@@ -62,6 +56,7 @@ function resourceParser(resource) {
     }
     return kafkajs_1.ConfigResourceTypes.UNKNOWN;
 }
+exports.resourceParser = resourceParser;
 const SASLMap = {
     'plain': (username, password) => ({ username, password }),
     'scram-sha-256': (username, password) => ({ username, password }),
@@ -80,6 +75,7 @@ function getSASL({ mechanism, username, password, authorizationIdentity, accessK
             return { mechanism, oauthBearerProvider };
     }
 }
+exports.getSASL = getSASL;
 function createClient(bootstrapServer, ssl, sasl, level) {
     const options = {
         clientId: 'Kafka CLI',
@@ -90,6 +86,7 @@ function createClient(bootstrapServer, ssl, sasl, level) {
     };
     return new kafkajs_1.Kafka(options);
 }
+exports.createClient = createClient;
 function createCluster(client) {
     return __awaiter(this, void 0, void 0, function* () {
         const createClusterKey = Reflect.ownKeys(client).find(key => /createCluster/.test(key.toString()));
@@ -99,6 +96,7 @@ function createCluster(client) {
         return cluster;
     });
 }
+exports.createCluster = createCluster;
 function createAdmin(client) {
     return __awaiter(this, void 0, void 0, function* () {
         const admin = client.admin();
@@ -106,15 +104,30 @@ function createAdmin(client) {
         return admin;
     });
 }
-function createConsumer(client_1, group_1, topic_1) {
-    return __awaiter(this, arguments, void 0, function* (client, group, topic, fromBeginning = false, poolOptions = {}) {
+exports.createAdmin = createAdmin;
+function createConsumer(client_1, group_1, topic_1, from_1) {
+    return __awaiter(this, arguments, void 0, function* (client, group, topic, from, poolOptions = {}) {
         const consumerConfig = {
             groupId: group,
         };
         const consumerOptions = {
             topics: [topic],
-            fromBeginning,
+            fromBeginning: from === '0',
         };
+        const offsets = [];
+        if (from && from !== '0') {
+            const timestamp = /^\d+$/.test(from)
+                ? new Date(parseInt(from, 10)).getTime()
+                : Date.parse(from);
+            if (Number.isNaN(timestamp)) {
+                throw new Error(`Invalid timestamp "${from}"`);
+            }
+            const admin = yield createAdmin(client);
+            const partitionOffsets = yield admin.fetchTopicOffsetsByTimestamp(topic, timestamp);
+            for (const { partition, offset } of partitionOffsets) {
+                offsets.push({ topic, partition, offset });
+            }
+        }
         const consumer = client.consumer(consumerConfig);
         yield consumer.connect();
         yield consumer.subscribe(consumerOptions);
@@ -123,6 +136,9 @@ function createConsumer(client_1, group_1, topic_1) {
                 pool.push(payload);
             }),
         }).catch(e => console.error(e));
+        for (const offset of offsets) {
+            consumer.seek(offset);
+        }
         const pool = new pool_1.default([], poolOptions);
         pool.onDone(() => {
             consumer.disconnect();
@@ -130,6 +146,7 @@ function createConsumer(client_1, group_1, topic_1) {
         return pool;
     });
 }
+exports.createConsumer = createConsumer;
 function createProducer(client, topic) {
     return __awaiter(this, void 0, void 0, function* () {
         const producer = client.producer();
@@ -162,3 +179,4 @@ function createProducer(client, topic) {
         return pool;
     });
 }
+exports.createProducer = createProducer;

package/build/utils/pool.js

@@ -19,8 +19,7 @@ var __importDefault = (this && this.__importDefault) || function (mod) {
     return (mod && mod.__esModule) ? mod : { "default": mod };
 };
 Object.defineProperty(exports, "__esModule", { value: true });
-exports.CancelToken = void 0;
-exports.createItem = createItem;
+exports.CancelToken = exports.createItem = void 0;
 const evnty_1 = __importDefault(require("evnty"));
 function createItem() {
     const result = {};
@@ -30,6 +29,7 @@ function createItem() {
     });
     return result;
 }
+exports.createItem = createItem;
 class CancelToken {
     constructor(timeout) {
         this.timeout = timeout;

package/package.json

@@ -1,18 +1,12 @@
 {
   "name": "kafka-console",
-  "version": "1.4.50",
+  "version": "2.0.1",
   "description": "Kafka CLI tool",
   "main": "index.js",
   "bin": {
     "kcli": "build/cli.js",
     "kafka-console": "build/cli.js"
   },
-  "scripts": {
-    "build": "rm -rf build && tsc",
-    "test": "NODE_ENV=test jest",
-    "test:cov": "COVERAGE=1 NODE_ENV=test jest",
-    "kcli": "ts-node src/index.ts"
-  },
   "repository": {
     "type": "git",
     "url": "git+https://github.com/3axap4eHko/kafka-console.git"
@@ -45,5 +39,11 @@
     "commander": "^12.1.0",
     "evnty": "^2.1.106",
     "kafkajs": "^2.2.4"
+  },
+  "scripts": {
+    "build": "rm -rf build && tsc",
+    "test": "NODE_ENV=test jest",
+    "test:cov": "COVERAGE=1 NODE_ENV=test jest",
+    "kcli": "ts-node src/index.ts"
   }
-}
+}