apache-iggy 0.5.2 → 0.6.0-edge.2

package/dist/client/client.connection.js

@@ -125,43 +125,39 @@ export class IggyConnection extends EventEmitter {
     }
     _onData(data) {
         debug('ONDATA', typeof data, Buffer.isBuffer(data), data?.length, this.waitingResponseEnd);
-        if (this.waitingResponseEnd) {
+        // Append new data to any buffered data
+        if (this.waitingResponseEnd && this.readBuffers.length > 0) {
             data = Buffer.concat([this.readBuffers, data]);
         }
-        if (data.length < 8) {
-            this.socket.unshift(data);
-            return;
-        }
-        /**
-         * data[0:4] hold response status code
-         * const status = data.readUInt32LE(0);
-         * data[4:8] hold response length
-         */
-        const responseSize = data.readUInt32LE(4);
-        /** response is just head, no payload */
-        if (responseSize === 0 && data.length === 8) {
-            this.emit('response', data);
-            return;
-        }
-        const payloadLength = data.length - 8;
-        /** payload is complete */
-        if (payloadLength === responseSize) {
-            this.emit('response', data);
-            this._endResponseWait();
-            return;
-        }
-        /** payload is incomplete, buffering until next data event */
-        if (payloadLength < responseSize) {
-            this.waitingResponseEnd = true;
-            this.readBuffers = data;
-            return;
-        }
-        /** payload is bigger than expected, cut response and unshift extra data  */
-        if (payloadLength > responseSize) {
-            this.emit('response', data.subarray(0, 8 + responseSize));
-            this._endResponseWait();
-            this.socket.unshift(data.subarray(8 + responseSize));
+        // Keep processing while we have enough data
+        let offset = 0;
+        while (offset < data.length) {
+            const remaining = data.length - offset;
+            // Need at least 8 bytes for the header (4 bytes status + 4 bytes length)
+            if (remaining < 8) {
+                // Buffer the incomplete header and wait for more data
+                this.waitingResponseEnd = true;
+                this.readBuffers = data.subarray(offset);
+                return;
+            }
+            // Read the header
+            const responseSize = data.readUInt32LE(offset + 4);
+            const totalSize = 8 + responseSize;
+            // Check if we have the complete response (header + payload)
+            if (remaining < totalSize) {
+                // Buffer the incomplete response and wait for more data
+                this.waitingResponseEnd = true;
+                this.readBuffers = data.subarray(offset);
+                return;
+            }
+            // We have a complete response, extract it and emit
+            const response = data.subarray(offset, offset + totalSize);
+            this.emit('response', response);
+            // Move to the next response
+            offset += totalSize;
         }
+        // All data processed, reset buffers
+        this._endResponseWait();
     }
     writeCommand(command, payload) {
         const cmd = serializeCommand(command, payload);

package/dist/client/client.socket.js

@@ -56,7 +56,7 @@ export class CommandResponseStream extends EventEmitter {
             await this.connection.connect();
         if (!this.isAuthenticated && !UNLOGGED_COMMAND_CODE.includes(command))
             await this.authenticate(this.options.credentials);
-        return new Promise(async (resolve, reject) => {
+        return new Promise((resolve, reject) => {
             if (last)
                 this._execQueue.push({ command, payload, resolve, reject });
             else

package/dist/debug-send.js

@@ -20,17 +20,15 @@ import assert from 'node:assert/strict';
 import { SingleClient } from './client/index.js';
 import { groupConsumerStream } from './stream/consumer-stream.js';
 import { PollingStrategy } from './wire/index.js';
-const streamId = 321;
-const topicId = 543;
-const groupId = 531231;
+const streamName = 'debug-snd-stream';
+const topicName = 'debug-snd-topic';
+const groupName = 'debug-snd-group';
 const stream = {
-    streamId,
-    name: 'debug2-send-message-stream'
+    name: streamName
 };
 const topic = {
-    streamId,
-    topicId,
-    name: 'debug2-send-message-topic',
+    streamId: streamName,
+    name: topicName,
     partitionCount: 1,
     compressionAlgorithm: 1
 };
@@ -51,13 +49,13 @@ const opt = {
 };
 const c = new SingleClient(opt);
 const cleanup = async () => {
-    assert.ok(await c.stream.delete(stream));
+    assert.ok(await c.stream.delete({ streamId: streamName }));
     assert.ok(await c.session.logout());
     c.destroy();
 };
 const msg = {
-    streamId,
-    topicId,
+    streamId: streamName,
+    topicId: topicName,
     messages: [
         { payload: 'yolo msg 2' },
         { payload: 'yolo msg v4' },
@@ -66,9 +64,9 @@ const msg = {
 };
 try {
     await c.stream.create(stream);
-    console.log('server stream CREATED::', { streamId });
+    console.log('server stream CREATED::', { streamName });
     await c.topic.create(topic);
-    console.log('server topic CREATED::', { topicId });
+    console.log('server topic CREATED::', { topicName });
     // send
     setInterval(async () => {
         assert.ok(await c.message.send(msg));
@@ -76,9 +74,9 @@ try {
     }, 3000);
     // POLL MESSAGE
     const pollReq = {
-        groupId,
-        streamId,
-        topicId,
+        groupName,
+        streamId: streamName,
+        topicId: topicName,
         pollingStrategy: PollingStrategy.Next,
         count: 1,
         interval: 5000

package/dist/debug.js

@@ -21,17 +21,15 @@ import { Client } from './client/index.js';
 import { uuidv7, uuidv4 } from 'uuidv7';
 import { groupConsumerStream } from './stream/consumer-stream.js';
 import { PollingStrategy } from './wire/index.js';
-const streamId = 123;
-const topicId = 345;
-const groupId = 12353;
+const streamName = 'debug-stream';
+const topicName = 'debug-topic';
+const groupName = 'debug-group';
 const stream = {
-    streamId,
     name: 'debug-send-message-stream'
 };
 const topic = {
-    streamId,
-    topicId,
-    name: 'debug-send-message-topic',
+    streamId: streamName,
+    name: topicName,
     partitionCount: 1,
     compressionAlgorithm: 1
 };
@@ -48,13 +46,13 @@ const opt = {
 };
 const c = new Client(opt);
 const cleanup = async () => {
-    assert.ok(await c.stream.delete(stream));
+    assert.ok(await c.stream.delete({ streamId: streamName }));
     assert.ok(await c.session.logout());
     c.destroy();
 };
 const msg = {
-    streamId,
-    topicId,
+    streamId: streamName,
+    topicId: topicName,
     messages: [
         { payload: 'yolo msg 2' },
         { id: 0, payload: 'nooooooooo' },
@@ -65,17 +63,17 @@ const msg = {
 };
 try {
     await c.stream.create(stream);
-    console.log('server stream CREATED::', { streamId });
+    console.log('server stream CREATED::', { streamName });
     await c.topic.create(topic);
-    console.log('server topic CREATED::', { topicId });
+    console.log('server topic CREATED::', { topicName });
     // send
     assert.ok(await c.message.send(msg));
     console.log('message SEND::', { msg });
     // POLL MESSAGE
     const pollReq = {
-        groupId,
-        streamId,
-        topicId,
+        groupName,
+        streamId: streamName,
+        topicId: topicName,
         pollingStrategy: PollingStrategy.Next,
         count: 5,
         interval: 5000

package/dist/e2e/tcp.cluster.e2e.d.ts

@@ -0,0 +1,20 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+export {};
+//# sourceMappingURL=tcp.cluster.e2e.d.ts.map

package/dist/e2e/tcp.cluster.e2e.js

@@ -0,0 +1,55 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+import { after, describe, it } from 'node:test';
+import assert from 'node:assert/strict';
+import { getTestClient } from './test-client.utils.js';
+// cluster mode still in dev atm
+// response is mocked from /core/configs/server.toml
+const expectedMeta = {
+    "name": "iggy-cluster",
+    "id": 1,
+    "transport": "TCP",
+    "nodes": [
+        {
+            "id": 1,
+            "name": "iggy-node-1",
+            "address": "127.0.0.1:8090",
+            "role": "Leader",
+            "status": "Healthy"
+        },
+        {
+            "id": 2,
+            "name": "iggy-node-2",
+            "address": "127.0.0.1:8091",
+            "role": "Follower",
+            "status": "Healthy"
+        }
+    ]
+};
+describe('e2e -> system', async () => {
+    const c = getTestClient();
+    it('e2e -> cluster::getClusterMetadata', async () => {
+        const meta = await c.cluster.getClusterMetadata();
+        assert.deepEqual(meta, expectedMeta);
+    });
+    after(() => {
+        c.destroy();
+    });
+});
+//# sourceMappingURL=tcp.cluster.e2e.js.map

package/dist/e2e/tcp.consumer-group.e2e.js

@@ -29,43 +29,49 @@ describe('e2e -> consumer-group', async () => {
         options: { host, port },
         credentials: { username: 'iggy', password: 'iggy' }
     });
-    const streamId = 555;
-    const topicId = 666;
-    const stream = {
-        streamId,
-        name: 'e2e-consumer-group-stream'
-    };
+    const streamName = 'e2e-consumer-group-stream-543';
+    const topicName = 'e2e-consumer-group-topic-543';
+    const groupName = 'e2e-cg-cg-1';
+    let payloadLength = 0;
+    await c.stream.create({ name: streamName });
     const topic = {
-        streamId,
-        topicId,
-        name: 'e2e-consumer-group-topic',
+        streamId: streamName,
+        name: topicName,
         partitionCount: 3,
         compressionAlgorithm: 1
     };
-    let payloadLength = 0;
-    await c.stream.create(stream);
     await c.topic.create(topic);
-    const groupId = 333;
-    const group = { streamId, topicId, groupId, name: 'e2e-cg-1' };
+    const group = {
+        streamId: streamName, topicId: topicName, name: groupName
+    };
     it('e2e -> consumer-group::create', async () => {
-        const r = await c.group.create(group);
-        assert.deepEqual(r, { id: groupId, name: 'e2e-cg-1', partitionsCount: 3, membersCount: 0 });
+        const g = await c.group.create(group);
+        // eslint-disable-next-line @typescript-eslint/no-unused-vars
+        const { id, ...rest } = g;
+        assert.deepEqual(rest, { name: groupName, partitionsCount: 3, membersCount: 0 });
     });
     it('e2e -> consumer-group::get', async () => {
-        const gg = await c.group.get({ streamId, topicId, groupId });
-        assert.deepEqual(gg, { id: groupId, name: 'e2e-cg-1', partitionsCount: 3, membersCount: 0 });
+        const gg = await c.group.get({
+            streamId: streamName, topicId: topicName, groupId: groupName
+        });
+        assert.ok(gg);
+        // eslint-disable-next-line @typescript-eslint/no-unused-vars
+        const { id, ...rest } = gg;
+        assert.deepEqual(rest, { name: groupName, partitionsCount: 3, membersCount: 0 });
     });
     it('e2e -> consumer-group::list', async () => {
-        const lg = await c.group.list({ streamId, topicId });
-        assert.deepEqual(lg, [{ id: groupId, name: 'e2e-cg-1', partitionsCount: 3, membersCount: 0 }]);
+        const lg = await c.group.list({ streamId: streamName, topicId: topicName });
+        // eslint-disable-next-line @typescript-eslint/no-unused-vars
+        const [{ id, ...rest }] = lg;
+        assert.deepEqual([rest], [{ name: groupName, partitionsCount: 3, membersCount: 0 }]);
     });
     it('e2e -> consumer-stream::send-messages', async () => {
         const ct = 1000;
         const mn = 200;
         for (let i = 0; i <= ct; i += mn) {
             assert.ok(await c.message.send({
-                streamId,
-                topicId,
+                streamId: streamName,
+                topicId: topicName,
                 messages: generateMessages(mn),
                 partition: Partitioning.MessageKey(`key-${i % 300}`)
             }));
@@ -73,14 +79,16 @@ describe('e2e -> consumer-group', async () => {
         payloadLength = ct;
     });
     it('e2e -> consumer-group::join', async () => {
-        assert.ok(await c.group.join({ streamId, topicId, groupId }));
+        assert.ok(await c.group.join({
+            streamId: streamName, topicId: topicName, groupId: groupName
+        }));
     });
     it('e2e -> consumer-group::poll', async () => {
         const pollReq = {
-            streamId,
-            topicId,
-            consumer: Consumer.Group(groupId),
-            partitionId: 0,
+            streamId: streamName,
+            topicId: topicName,
+            consumer: Consumer.Group(groupName),
+            partitionId: null,
             pollingStrategy: PollingStrategy.Next,
             count: 100,
             autocommit: true
@@ -97,13 +105,17 @@ describe('e2e -> consumer-group', async () => {
         assert.equal(count, 0);
     });
     it('e2e -> consumer-group::leave', async () => {
-        assert.ok(await c.group.leave({ streamId, topicId, groupId }));
+        assert.ok(await c.group.leave({
+            streamId: streamName, topicId: topicName, groupId: groupName
+        }));
     });
     it('e2e -> consumer-group::delete', async () => {
-        assert.ok(await c.group.delete({ streamId, topicId, groupId }));
+        assert.ok(await c.group.delete({
+            streamId: streamName, topicId: topicName, groupId: groupName
+        }));
     });
     after(async () => {
-        assert.ok(await c.stream.delete(stream));
+        assert.ok(await c.stream.delete({ streamId: streamName }));
         assert.ok(await c.session.logout());
         c.destroy();
     });

package/dist/e2e/tcp.consumer-stream.e2e.js

@@ -31,29 +31,28 @@ describe('e2e -> consumer-stream', async () => {
         credentials
     };
     const c = new Client(opt);
-    const streamId = 137;
-    const topicId = 123;
-    const groupId = 12;
-    await c.stream.create({ streamId, name: `e2e-stream-${streamId}` });
+    const groupName = 'cgtest-12';
+    const streamName = `e2e-stream-consumer-stream`;
+    const topicName = 'test-topic-stream-cg';
+    await c.stream.create({ name: streamName });
     await c.topic.create({
-        streamId,
-        topicId,
-        name: 'test-topic-stream-cg',
+        streamId: streamName,
+        name: topicName,
         partitionCount: 3,
         compressionAlgorithm: 1
     });
     const ct = 1000;
     it('e2e -> consumer-stream::send-messages', async () => {
         for (let i = 0; i < ct; i += 100) {
-            await sendSomeMessages(c.clientProvider)(streamId, topicId, Partitioning.MessageKey(`k-${i % 300}`));
+            await sendSomeMessages(c.clientProvider)(streamName, topicName, Partitioning.MessageKey(`k-${i % 300}`));
         }
     });
     it('e2e -> consumer-stream::poll-stream', async () => {
         // POLL MESSAGE
         const pollReq = {
-            groupId,
-            streamId,
-            topicId,
+            groupName,
+            streamId: streamName,
+            topicId: topicName,
             pollingStrategy: PollingStrategy.Next,
             count: 100,
             interval: 500
@@ -83,7 +82,7 @@ describe('e2e -> consumer-stream', async () => {
             });
         });
         after(async () => {
-            assert.ok(await c.stream.delete({ streamId }));
+            assert.ok(await c.stream.delete({ streamId: streamName }));
             assert.ok(await c.session.logout());
             await c.destroy();
         });

package/dist/e2e/tcp.send-message.e2e.js

@@ -23,35 +23,30 @@ import { generateMessages } from '../tcp.sm.utils.js';
 import { getTestClient } from './test-client.utils.js';
 describe('e2e -> message', async () => {
     const c = getTestClient();
-    const streamId = 934;
-    const topicId = 832;
-    const partitionId = 1;
-    const stream = {
-        streamId,
-        name: 'e2e-send-message-stream'
-    };
+    const streamName = 'e2e-stream-934';
+    const topicName = 'e2e-topic-832';
+    const partitionId = 0;
+    await c.stream.create({ name: streamName });
     const topic = {
-        streamId,
-        topicId,
-        name: 'e2e-send-message-topic',
+        streamId: streamName,
+        name: topicName,
         partitionCount: 1,
         compressionAlgorithm: 1
     };
+    await c.topic.create(topic);
     const msg = {
-        streamId,
-        topicId,
+        streamId: streamName,
+        topicId: topicName,
         messages: generateMessages(6),
         partition: Partitioning.PartitionId(partitionId)
     };
-    await c.stream.create(stream);
-    await c.topic.create(topic);
     it('e2e -> message::send', async () => {
         assert.ok(await c.message.send(msg));
     });
     it('e2e -> message::poll/last', async () => {
         const pollReq = {
-            streamId,
-            topicId,
+            streamId: streamName,
+            topicId: topicName,
             consumer: Consumer.Single,
             partitionId,
             pollingStrategy: PollingStrategy.Last,
@@ -64,8 +59,8 @@ describe('e2e -> message', async () => {
     });
     it('e2e -> message::poll/first', async () => {
         const pollReq = {
-            streamId,
-            topicId,
+            streamId: streamName,
+            topicId: topicName,
             consumer: Consumer.Single,
             partitionId,
             pollingStrategy: PollingStrategy.First,
@@ -78,8 +73,8 @@ describe('e2e -> message', async () => {
     });
     it('e2e -> message::poll/next', async () => {
         const pollReq = {
-            streamId,
-            topicId,
+            streamId: streamName,
+            topicId: topicName,
             consumer: Consumer.Single,
             partitionId,
             pollingStrategy: PollingStrategy.Next,
@@ -92,8 +87,8 @@ describe('e2e -> message', async () => {
     });
     it('e2e -> message::poll/next+commit', async () => {
         const pollReq = {
-            streamId,
-            topicId,
+            streamId: streamName,
+            topicId: topicName,
             consumer: Consumer.Single,
             partitionId,
             pollingStrategy: PollingStrategy.Next,
@@ -107,8 +102,50 @@ describe('e2e -> message', async () => {
         assert.equal(r2.count, 0);
         assert.equal(r2.messages.length, 0);
     });
+    it('e2e -> message::getOffset', async () => {
+        const offset = await c.offset.get({
+            streamId: streamName,
+            topicId: topicName,
+            consumer: Consumer.Single,
+            partitionId
+        });
+        assert.deepEqual(offset, { partitionId: 0, currentOffset: 5n, storedOffset: 5n });
+    });
+    it('e2e -> message::deleteOffset', async () => {
+        const r = await c.offset.delete({
+            streamId: streamName,
+            topicId: topicName,
+            consumer: Consumer.Single,
+            partitionId
+        });
+        assert.ok(r);
+        const offset = await c.offset.get({
+            streamId: streamName,
+            topicId: topicName,
+            consumer: Consumer.Single,
+            partitionId
+        });
+        assert.equal(offset, null);
+    });
+    it('e2e -> message::storeOffset', async () => {
+        const r = await c.offset.store({
+            streamId: streamName,
+            topicId: topicName,
+            consumer: Consumer.Single,
+            partitionId,
+            offset: 2n
+        });
+        assert.ok(r);
+        const offset = await c.offset.get({
+            streamId: streamName,
+            topicId: topicName,
+            consumer: Consumer.Single,
+            partitionId
+        });
+        assert.deepEqual(offset, { partitionId: 0, currentOffset: 5n, storedOffset: 2n });
+    });
     it('e2e -> message::cleanup', async () => {
-        assert.ok(await c.stream.delete(stream));
+        assert.ok(await c.stream.delete({ streamId: streamName }));
         assert.ok(await c.session.logout());
     });
     after(() => {

package/dist/e2e/tcp.stream.e2e.js

@@ -21,11 +21,10 @@ import assert from 'node:assert/strict';
 import { getTestClient } from './test-client.utils.js';
 describe('e2e -> stream', async () => {
     const c = getTestClient();
-    const streamId = 164;
     const name = 'e2e-tcp-stream';
     const name2 = `${name}-updated`;
     it('e2e -> stream::create', async () => {
-        const stream = await c.stream.create({ streamId, name });
+        const stream = await c.stream.create({ name });
         assert.ok(stream);
     });
     it('e2e -> stream::list', async () => {
@@ -33,21 +32,21 @@ describe('e2e -> stream', async () => {
         assert.ok(streams.length > 0);
     });
     it('e2e -> stream::get', async () => {
-        const stream = await c.stream.get({ streamId });
+        const stream = await c.stream.get({ streamId: name });
         assert.ok(stream);
     });
     it('e2e -> stream::update', async () => {
         const stream = await c.stream.update({
-            streamId,
+            streamId: name,
             name: name2
         });
         assert.ok(stream);
     });
     it('e2e -> stream::purge', async () => {
-        assert.ok(await c.stream.purge({ streamId }));
+        assert.ok(await c.stream.purge({ streamId: name2 }));
     });
     it('e2e -> stream::delete', async () => {
-        assert.ok(await c.stream.delete({ streamId }));
+        assert.ok(await c.stream.delete({ streamId: name2 }));
     });
     it('e2e -> stream::cleanup', async () => {
         assert.ok(await c.session.logout());

package/dist/e2e/tcp.system.e2e.js

@@ -25,7 +25,7 @@ describe('e2e -> system', async () => {
         assert.ok(await c.system.ping());
     });
     it('e2e -> system::login', async () => {
-        assert.deepEqual(await c.session.login({ username: 'iggy', password: 'iggy' }), { userId: 1 });
+        assert.deepEqual(await c.session.login({ username: 'iggy', password: 'iggy' }), { userId: 0 });
     });
     it('e2e -> system::getStat', async () => {
         assert.deepEqual(Object.keys(await c.system.getStats()), [