kafka-console 3.0.0 → 3.1.0

package/build/commands/consume.js

@@ -1,7 +1,7 @@
 import * as Fs from 'fs';
 import { finished } from 'node:stream/promises';
-import { Consumer, stringDeserializers, MessagesStreamModes, ListOffsetTimestamps } from '@platformatic/kafka';
-import { getClientConfigFromOpts } from "../utils/kafka.js";
+import { Consumer, stringDeserializers, ListOffsetTimestamps } from '@platformatic/kafka';
+import { getClientConfigFromOpts, resolveConsumeMode } from "../utils/kafka.js";
 import { getFormatter } from "../utils/formatters.js";
 function toError(error) {
     return error instanceof Error ? error : new Error(String(error));
@@ -43,38 +43,15 @@ export default async function consume(topic, opts, { parent }) {
     let cleanupError;
     let aborted = false;
     try {
-        let mode = MessagesStreamModes.LATEST;
-        let offsets;
-        if (opts.snapshot && !opts.from) {
-            mode = MessagesStreamModes.EARLIEST;
-        }
-        else if (opts.from === '0') {
-            mode = MessagesStreamModes.EARLIEST;
-        }
-        else if (opts.from) {
-            const ts = /^\d+$/.test(opts.from) ? parseInt(opts.from, 10) : Date.parse(opts.from);
-            if (Number.isNaN(ts)) {
-                throw new Error(`Invalid timestamp "${opts.from}"`);
-            }
-            const offsetsMap = await consumer.listOffsets({ topics: [topic], timestamp: BigInt(ts) });
-            const partitionOffsets = offsetsMap.get(topic);
-            if (partitionOffsets) {
-                offsets = [];
-                for (let partition = 0; partition < partitionOffsets.length; partition++) {
-                    offsets.push({ topic, partition, offset: partitionOffsets[partition] });
-                }
-                mode = MessagesStreamModes.MANUAL;
-            }
-        }
+        const fromArg = opts.snapshot && !opts.from ? '0' : opts.from;
+        const { mode, offsets } = await resolveConsumeMode(consumer, topic, fromArg);
         const consumeOptions = {
             topics: [topic],
             mode,
+            offsets,
             sessionTimeout: 30000,
             heartbeatInterval: 1000,
         };
-        if (offsets) {
-            consumeOptions.offsets = offsets;
-        }
         try {
             stream = await consumer.consume(consumeOptions);
         }

package/build/commands/copyTopic.js

@@ -0,0 +1,81 @@
+import { Consumer, Producer, stringDeserializers, stringSerializers, ListOffsetTimestamps } from '@platformatic/kafka';
+import { getClientConfigFromOpts, resolveConsumeMode } from "../utils/kafka.js";
+export default async function copyTopic(source, dest, opts, { parent }) {
+    const config = getClientConfigFromOpts(parent.opts());
+    const consumer = new Consumer({
+        ...config,
+        groupId: opts.group,
+        deserializers: stringDeserializers,
+        autocommit: true,
+    });
+    const producer = new Producer({
+        ...config,
+        serializers: stringSerializers,
+        compression: opts.compression,
+    });
+    let stream;
+    let interrupted = false;
+    const handleSignal = () => {
+        interrupted = true;
+        if (stream)
+            void stream.close();
+    };
+    process.once('SIGINT', handleSignal);
+    process.once('SIGTERM', handleSignal);
+    try {
+        const fromArg = opts.from ?? '0';
+        const { mode, offsets } = await resolveConsumeMode(consumer, source, fromArg);
+        const latestMap = await consumer.listOffsets({ topics: [source], timestamp: ListOffsetTimestamps.LATEST });
+        const highWatermarks = latestMap.get(source) || [];
+        const remaining = new Set();
+        for (let p = 0; p < highWatermarks.length; p++) {
+            if (highWatermarks[p] > 0n)
+                remaining.add(p);
+        }
+        if (remaining.size === 0) {
+            console.error('0 messages copied');
+            return;
+        }
+        stream = await consumer.consume({
+            topics: [source],
+            mode,
+            offsets,
+            sessionTimeout: 30000,
+            heartbeatInterval: 1000,
+        });
+        let batch = [];
+        let total = 0;
+        for await (const message of stream) {
+            if (interrupted)
+                break;
+            batch.push({
+                topic: dest,
+                key: message.key ?? undefined,
+                value: message.value ?? undefined,
+                headers: message.headers,
+            });
+            total++;
+            if (message.offset >= highWatermarks[message.partition] - 1n) {
+                remaining.delete(message.partition);
+            }
+            if (batch.length >= opts.batchSize || remaining.size === 0 || total >= opts.count) {
+                await producer.send({ messages: batch });
+                batch = [];
+                if (remaining.size === 0 || total >= opts.count)
+                    break;
+            }
+        }
+        if (batch.length > 0) {
+            await producer.send({ messages: batch });
+        }
+        console.error(`${total} messages copied`);
+    }
+    finally {
+        process.off('SIGINT', handleSignal);
+        process.off('SIGTERM', handleSignal);
+        if (stream)
+            await stream.close().catch(() => { });
+        await consumer.close().catch(() => { });
+        await producer.close().catch(() => { });
+    }
+}

package/build/commands/dumpTopic.js

@@ -0,0 +1,75 @@
+import * as Fs from 'fs';
+import { finished } from 'node:stream/promises';
+import { Consumer, stringDeserializers, ListOffsetTimestamps } from '@platformatic/kafka';
+import { getClientConfigFromOpts, resolveConsumeMode } from "../utils/kafka.js";
+export default async function dumpTopic(topic, opts, { parent }) {
+    const config = getClientConfigFromOpts(parent.opts());
+    const output = Fs.createWriteStream(opts.output);
+    const consumer = new Consumer({
+        ...config,
+        groupId: opts.group,
+        deserializers: stringDeserializers,
+        autocommit: true,
+    });
+    let stream;
+    let interrupted = false;
+    const handleSignal = () => {
+        interrupted = true;
+        if (stream)
+            void stream.close();
+    };
+    process.once('SIGINT', handleSignal);
+    process.once('SIGTERM', handleSignal);
+    try {
+        const fromArg = opts.from ?? '0';
+        const { mode, offsets } = await resolveConsumeMode(consumer, topic, fromArg);
+        const latestMap = await consumer.listOffsets({ topics: [topic], timestamp: ListOffsetTimestamps.LATEST });
+        const highWatermarks = latestMap.get(topic) || [];
+        const remaining = new Set();
+        for (let p = 0; p < highWatermarks.length; p++) {
+            if (highWatermarks[p] > 0n)
+                remaining.add(p);
+        }
+        if (remaining.size === 0) {
+            console.error(`0 messages dumped to ${opts.output}`);
+            return;
+        }
+        stream = await consumer.consume({
+            topics: [topic],
+            mode,
+            offsets,
+            sessionTimeout: 30000,
+            heartbeatInterval: 1000,
+        });
+        let total = 0;
+        for await (const message of stream) {
+            if (interrupted)
+                break;
+            const msg = {
+                partition: message.partition,
+                offset: message.offset.toString(),
+                timestamp: message.timestamp.toString(),
+                headers: Object.fromEntries(message.headers),
+                key: message.key,
+                value: message.value,
+            };
+            output.write(JSON.stringify(msg) + '\n');
+            total++;
+            if (message.offset >= highWatermarks[message.partition] - 1n) {
+                remaining.delete(message.partition);
+            }
+            if (remaining.size === 0 || total >= opts.count)
+                break;
+        }
+        console.error(`${total} messages dumped to ${opts.output}`);
+    }
+    finally {
+        process.off('SIGINT', handleSignal);
+        process.off('SIGTERM', handleSignal);
+        if (stream)
+            await stream.close().catch(() => { });
+        await consumer.close().catch(() => { });
+        output.end();
+        await finished(output).catch(() => { });
+    }
+}

package/build/commands/produce.js

@@ -55,6 +55,7 @@ export default async function produce(topic, opts, { parent }) {
     const producer = new Producer({
         ...config,
         serializers: stringSerializers,
+        compression: opts.compression,
     });
     const staticHeaders = {};
     for (const h of opts.header) {

package/build/index.js

@@ -8,6 +8,8 @@ import configCommand from "./commands/config.js";
 import createTopicCommand from "./commands/createTopic.js";
 import deleteTopicCommand from "./commands/deleteTopic.js";
 import fetchTopicOffsets from "./commands/fetchTopicOffsets.js";
+import copyTopicCommand from "./commands/copyTopic.js";
+import dumpTopicCommand from "./commands/dumpTopic.js";
 import { createRequire } from 'node:module';
 const { version } = createRequire(import.meta.url)('../package.json');
 export function collect(value, previous) {
@@ -47,6 +49,7 @@ commander
     .option('-i, --input <filename>', 'read messages from a JSON array file instead of stdin')
     .option('-w, --wait <wait>', 'delay in milliseconds between sending each message', toInt, 0)
     .option('-H, --header <header>', 'static header added to every message (format: key:value), repeatable', collect, [])
+    .option('-C, --compression <algorithm>', 'compression algorithm: none, gzip, snappy, lz4, zstd')
     .action(produceCommand);
 commander.command('metadata').description('Display cluster metadata: broker list, controller, topic partitions, replicas, and ISR').action(metadataCommand);
 commander
@@ -73,6 +76,23 @@ commander
     .description('Show topic partition offsets. Without arguments shows high/low watermarks. With a timestamp (ms or ISO 8601) shows offsets at that point. With --group shows committed offsets for a consumer group.')
     .option('-g, --group <group>', 'show committed offsets for this consumer group instead of watermarks')
     .action(fetchTopicOffsets);
+commander
+    .command('topic:copy <source> <dest>')
+    .description('Copy messages from one topic to another. Reads all messages from <source> (snapshot mode) and writes them to <dest> in batches. Preserves keys, values, and headers. Exits when all existing messages are copied.')
+    .option('-g, --group <group>', 'consumer group name', `kafka-console-copy-${Date.now()}`)
+    .option('-f, --from <from>', 'start from a timestamp (ms), ISO 8601 date, or 0 for the beginning', '0')
+    .option('-c, --count <count>', 'maximum number of messages to copy', toInt, Infinity)
+    .option('--batch-size <size>', 'number of messages per producer send call', toInt, 500)
+    .option('-C, --compression <algorithm>', 'producer compression: none, gzip, snappy, lz4, zstd')
+    .action(copyTopicCommand);
+commander
+    .command('topic:dump <topic>')
+    .description('Dump messages from a topic to a JSONL file. Each line contains partition, offset, timestamp, headers, key, and the raw value string. Exits when all existing messages are written.')
+    .requiredOption('-o, --output <filename>', 'output file path')
+    .option('-g, --group <group>', 'consumer group name', `kafka-console-dump-${Date.now()}`)
+    .option('-f, --from <from>', 'start from a timestamp (ms), ISO 8601 date, or 0 for the beginning', '0')
+    .option('-c, --count <count>', 'maximum number of messages to dump', toInt, Infinity)
+    .action(dumpTopicCommand);
 commander.parseAsync(process.argv).catch((e) => {
     console.error(e.message);
     process.exit(1);

package/build/utils/kafka.js

@@ -1,4 +1,4 @@
-import { Admin, ConfigResourceTypes, Consumer, stringDeserializers, ListOffsetTimestamps } from '@platformatic/kafka';
+import { Admin, ConfigResourceTypes, Consumer, stringDeserializers, ListOffsetTimestamps, MessagesStreamModes } from '@platformatic/kafka';
 export function resourceParser(resource) {
     if (/^any$/i.test(resource)) {
         return 'UNKNOWN';
@@ -94,6 +94,25 @@ export async function fetchTopicOffsetsByTimestamp(config, topic, timestamp) {
         await consumer.close();
     }
 }
+export async function resolveConsumeMode(consumer, topic, from) {
+    if (!from || from === '0') {
+        return { mode: from === '0' ? MessagesStreamModes.EARLIEST : MessagesStreamModes.LATEST };
+    }
+    const ts = /^\d+$/.test(from) ? parseInt(from, 10) : Date.parse(from);
+    if (Number.isNaN(ts)) {
+        throw new Error(`Invalid timestamp "${from}"`);
+    }
+    const offsetsMap = await consumer.listOffsets({ topics: [topic], timestamp: BigInt(ts) });
+    const partitionOffsets = offsetsMap.get(topic);
+    if (!partitionOffsets) {
+        return { mode: MessagesStreamModes.LATEST };
+    }
+    const offsets = [];
+    for (let partition = 0; partition < partitionOffsets.length; partition++) {
+        offsets.push({ topic, partition, offset: partitionOffsets[partition] });
+    }
+    return { mode: MessagesStreamModes.MANUAL, offsets };
+}
 export async function fetchConsumerGroupOffsets(config, groupId, topics) {
     const admin = new Admin(config);
     let meta;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "kafka-console",
-  "version": "3.0.0",
+  "version": "3.1.0",
   "description": "Kafka CLI tool",
   "type": "module",
   "bin": {
@@ -29,12 +29,12 @@
     "@eslint/js": "^10.0.1",
     "@types/debug": "^4.1.13",
     "@types/node": "^25.5.0",
-    "@vitest/coverage-v8": "^4.1.0",
-    "eslint": "^10.0.3",
+    "@vitest/coverage-v8": "^4.1.2",
+    "eslint": "^10.1.0",
     "eslint-plugin-prettier": "^5.5.5",
-    "typescript": "^5.9.3",
-    "typescript-eslint": "^8.57.1",
-    "vitest": "^4.1.0"
+    "typescript": "^6.0.2",
+    "typescript-eslint": "^8.57.2",
+    "vitest": "^4.1.2"
   },
   "dependencies": {
     "@platformatic/kafka": "^1.31.0",