apache-iggy 0.10.0 → 0.10.1-edge.1

package/README.md

@@ -10,7 +10,7 @@
 
 Apache Iggy Node.js client written in typescript, it currently only supports tcp & tls transports.
 
-diclaimer: although all iggy commands & basic client/stream are implemented this is still a WIP, provided as is, and has still a long way to go to be considered "battle tested".
+Disclaimer: although all iggy commands & basic client/stream are implemented this is still a WIP, provided as is, and has still a long way to go to be considered "battle tested".
 
 note: This lib started as _iggy-bin_ ( [github](https://github.com/T1B0/iggy-bin) / [npm](https://www.npmjs.com/package/iggy-bin)) before migrating under iggy-rs org. package iggy-bin@v1.3.4 is equivalent to @iggy.rs/sdk@v1.0.3 and migrating again under apache iggy monorepo ( [github](https://github.com/apache/iggy/tree/master/foreign/node) and is now published on npmjs as apache-iggy
 

package/dist/e2e/tcp.parallel.e2e.js

@@ -43,7 +43,10 @@ describe('e2e -> parallel', async () => {
             'availableMemory', 'runTime', 'startTime', 'readBytes', 'writtenBytes',
             'messagesSizeBytes', 'streamsCount', 'topicsCount', 'partitionsCount',
             'segmentsCount', 'messagesCount', 'clientsCount', 'consumersGroupsCount',
-            'hostname', 'osName', 'osVersion', 'kernelVersion'
+            'hostname', 'osName', 'osVersion', 'kernelVersion',
+            'iggyServerVersion', 'iggyServerSemver', 'cacheMetrics',
+            'threadsCount', 'freeDiskSpace', 'totalDiskSpace',
+            'openFilesCount', 'openFilesLimit'
         ]);
         resp.forEach(r => assert.ok(r));
     });

package/dist/e2e/tcp.system.e2e.js

@@ -31,7 +31,10 @@ describe('e2e -> system', async () => {
             'availableMemory', 'runTime', 'startTime', 'readBytes', 'writtenBytes',
             'messagesSizeBytes', 'streamsCount', 'topicsCount', 'partitionsCount',
             'segmentsCount', 'messagesCount', 'clientsCount', 'consumersGroupsCount',
-            'hostname', 'osName', 'osVersion', 'kernelVersion'
+            'hostname', 'osName', 'osVersion', 'kernelVersion',
+            'iggyServerVersion', 'iggyServerSemver', 'cacheMetrics',
+            'threadsCount', 'freeDiskSpace', 'totalDiskSpace',
+            'openFilesCount', 'openFilesLimit'
         ]);
     });
     it('e2e -> system::logout', async () => {

package/dist/wire/command-set.d.ts

@@ -76,7 +76,6 @@ type OffsetAPI = ReturnType<typeof offsetAPI>;
 declare const messageAPI: (c: ClientProvider) => {
     poll: (request: import("./message/poll-messages.command.js").PollMessages) => Promise<import("./index.js").PollMessagesResponse>;
     send: (arg: import("./message/send-messages.command.js").SendMessages) => Promise<import("./message/send-messages.command.js").SendMessagesResponse>;
-    flushUnsavedBuffers: (arg: import("./message/flush-unsaved-buffers.command.js").FlushUnsavedBuffer) => Promise<boolean>;
 };
 type MessageAPI = ReturnType<typeof messageAPI>;
 declare const systemAPI: (c: ClientProvider) => {

package/dist/wire/command-set.js

@@ -41,7 +41,6 @@ import { storeOffset } from './offset/store-offset.command.js';
 import { deleteOffset } from './offset/delete-offset.command.js';
 import { sendMessages } from './message/send-messages.command.js';
 import { pollMessages } from './message/poll-messages.command.js';
-import { flushUnsavedBuffers } from './message/flush-unsaved-buffers.command.js';
 import { createStream } from './stream/create-stream.command.js';
 import { updateStream } from './stream/update-stream.command.js';
 import { getStream } from './stream/get-stream.command.js';
@@ -132,8 +131,7 @@ const offsetAPI = (c) => ({
 });
 const messageAPI = (c) => ({
     poll: pollMessages(c),
-    send: sendMessages(c),
-    flushUnsavedBuffers: flushUnsavedBuffers(c)
+    send: sendMessages(c)
 });
 const systemAPI = (c) => ({
     ping: ping(c),

package/dist/wire/command.code.d.ts

@@ -25,7 +25,6 @@ export declare const COMMAND_CODE: {
     LoginRegisterWithAccessToken: number;
     PollMessages: number;
     SendMessages: number;
-    FlushUnsavedBuffers: number;
     GetPollRouting: number;
     PollMessagesOnPrimary: number;
     GetOffset: number;

package/dist/wire/command.code.js

@@ -42,7 +42,6 @@ export const COMMAND_CODE = {
     LoginRegisterWithAccessToken: 45,
     PollMessages: 100,
     SendMessages: 101,
-    FlushUnsavedBuffers: 102,
     GetPollRouting: 103,
     PollMessagesOnPrimary: 104,
     GetOffset: 120,

package/dist/wire/error.code.js

@@ -84,6 +84,7 @@ export const translateErrorCode = (code) => {
         case '82': return "Invalid number encoding";
         case '83': return "Invalid boolean value";
         case '84': return "Invalid number value";
+        case '85': return "Request is below the deduplication window; outcome unknown, resending may duplicate the write";
         case '100': return "Client with ID: {0} was not found.";
         case '101': return "Invalid client ID";
         case '206': return "Connection closed";
@@ -140,6 +141,7 @@ export const translateErrorCode = (code) => {
         case '2019': return "Invalid partitions count";
         case '2020': return "Topic directory: {0} not found";
         case '2021': return "Too many topics";
+        case '2022': return "Partitions limit reached, raise [metadata] partitions_max";
         // TOPIC
         case '3000': return "Cannot create partition with ID: {0} for stream with ID: {1} and topic with ID: {2}";
         case '3001': return "Failed to create directory for partitions for stream with ID: {0} and topic with ID: {1}";

package/dist/wire/error.code.test.js

@@ -17,6 +17,12 @@
 import assert from 'node:assert/strict';
 import { it } from 'node:test';
 import { translateErrorCode } from './error.code.js';
+it('translates the partitions capacity error', () => {
+    assert.equal(translateErrorCode(2022), 'Partitions limit reached, raise [metadata] partitions_max');
+});
+it('reports an unknown outcome for an aged-out request', () => {
+    assert.equal(translateErrorCode(85), 'Request is below the deduplication window; outcome unknown, resending may duplicate the write');
+});
 it('translates the consumer-offset capacity error', () => {
     assert.equal(translateErrorCode(3024), 'Consumer offset limit reached for partition, raise [partition] consumer_offsets_max');
 });

package/dist/wire/message/message.utils.d.ts

@@ -81,14 +81,4 @@ export declare const encodeMessagesBatch: (messages: MessageToEncode[]) => Buffe
  * @throws Error if the message array is empty
  */
 export declare const serializeSendMessages: (streamId: Id, topicId: Id, messages: CreateMessage[], partitioning?: Partitioning) => Buffer<ArrayBuffer>;
-/**
- * Serializes a flush unsaved buffers command payload.
- *
- * @param streamId - Stream identifier
- * @param topicId - Topic identifier
- * @param partitionId - Partition ID to flush
- * @param fsync - Whether to force sync to disk
- * @returns Serialized command payload
- */
-export declare const serializeFlushUnsavedBuffers: (streamId: Id, topicId: Id, partitionId: number, fsync?: boolean) => Buffer<ArrayBuffer>;
 //# sourceMappingURL=message.utils.d.ts.map

package/dist/wire/message/message.utils.js

@@ -15,7 +15,7 @@
 // specific language governing permissions and limitations
 // under the License.
 import { randomFillSync } from 'node:crypto';
-import { uint32ToBuf, u128ToBuf, uint8ToBuf } from '../number.utils.js';
+import { uint32ToBuf, u128ToBuf } from '../number.utils.js';
 import { serializeHeaders } from './header.utils.js';
 import { serializeIdentifier } from '../identifier.utils.js';
 import { serializePartitioning } from './partitioning.utils.js';
@@ -198,25 +198,4 @@ export const serializeSendMessages = (streamId, topicId, messages, partitioning)
         bBatch
     ]);
 };
-/**
- * Serializes a flush unsaved buffers command payload.
- *
- * @param streamId - Stream identifier
- * @param topicId - Topic identifier
- * @param partitionId - Partition ID to flush
- * @param fsync - Whether to force sync to disk
- * @returns Serialized command payload
- */
-export const serializeFlushUnsavedBuffers = (streamId, topicId, partitionId, fsync = false) => {
-    const streamIdentifier = serializeIdentifier(streamId);
-    const topicIdentifier = serializeIdentifier(topicId);
-    const bPartitionId = uint32ToBuf(partitionId);
-    const bFSync = uint8ToBuf(fsync ? 1 : 0);
-    return Buffer.concat([
-        streamIdentifier,
-        topicIdentifier,
-        bPartitionId,
-        bFSync
-    ]);
-};
 //# sourceMappingURL=message.utils.js.map

package/dist/wire/system/get-stats.command.d.ts

@@ -1,4 +1,12 @@
 import type { CommandResponse } from '../../client/index.js';
+export type CacheMetrics = {
+    streamId: number;
+    topicId: number;
+    partitionId: number;
+    hits: bigint;
+    misses: bigint;
+    hitRatio: number;
+};
 export type Stats = {
     processId: number;
     cpuUsage: number;
@@ -22,6 +30,15 @@ export type Stats = {
     osName: string;
     osVersion: string;
     kernelVersion: string;
+    iggyServerVersion: string;
+    /** 0 when the server does not report it. */
+    iggyServerSemver: number;
+    cacheMetrics: CacheMetrics[];
+    threadsCount: number;
+    freeDiskSpace: bigint;
+    totalDiskSpace: bigint;
+    openFilesCount: bigint;
+    openFilesLimit: bigint;
 };
 export declare const GET_STATS: {
     code: number;

package/dist/wire/system/get-stats.command.js

@@ -16,7 +16,36 @@
 // under the License.
 import { COMMAND_CODE } from '../command.code.js';
 import { wrapCommand } from '../command.utils.js';
+import { deserializeError } from '../error.utils.js';
+// process_id through consumer_groups_count
+const FIXED_HEAD_SIZE = 108;
+// stream_id, topic_id, partition_id, hits, misses, hit_ratio
+const CACHE_METRIC_SIZE = 4 + 4 + 4 + 8 + 8 + 4;
+// threads_count, free_disk_space, total_disk_space
+const THREADS_AND_DISK_SIZE = 4 + 8 + 8;
+// open_files_count, open_files_limit
+const OPEN_FILES_SIZE = 8 + 8;
+const ensureLength = (b, end) => {
+    if (end > b.length)
+        deserializeError('stats', end, b.length);
+};
+/** A u32 length, then that many bytes of text. */
+const deserializeString = (b, position) => {
+    ensureLength(b, position + 4);
+    const end = position + 4 + b.readUInt32LE(position);
+    ensureLength(b, end);
+    return { value: b.subarray(position + 4, end).toString(), end };
+};
+const deserializeCacheMetric = (b, position) => ({
+    streamId: b.readUInt32LE(position),
+    topicId: b.readUInt32LE(position + 4),
+    partitionId: b.readUInt32LE(position + 8),
+    hits: b.readBigUInt64LE(position + 12),
+    misses: b.readBigUInt64LE(position + 20),
+    hitRatio: b.readFloatLE(position + 28)
+});
 const deserializeGetStats = (b) => {
+    ensureLength(b, FIXED_HEAD_SIZE);
     const processId = b.readUInt32LE(0);
     const cpuUsage = b.readFloatLE(4);
     const totalCpuUsage = b.readFloatLE(8);
@@ -35,18 +64,34 @@ const deserializeGetStats = (b) => {
     const messagesCount = b.readBigUInt64LE(92);
     const clientsCount = b.readUInt32LE(100);
     const consumersGroupsCount = b.readUInt32LE(104);
-    let position = 104 + 4;
-    const hostnameLength = b.readUInt32LE(position);
-    const hostname = b.subarray(position + 4, position + 4 + hostnameLength).toString();
-    position += 4 + hostnameLength;
-    const osNameLength = b.readUInt32LE(position);
-    const osName = b.subarray(position + 4, position + 4 + osNameLength).toString();
-    position += 4 + osNameLength;
-    const osVersionLength = b.readUInt32LE(position);
-    const osVersion = b.subarray(position + 4, position + 4 + osVersionLength).toString();
-    position += 4 + osVersionLength;
-    const kernelVersionLength = b.readUInt32LE(position);
-    const kernelVersion = b.subarray(position + 4, position + 4 + kernelVersionLength).toString();
+    const hostname = deserializeString(b, FIXED_HEAD_SIZE);
+    const osName = deserializeString(b, hostname.end);
+    const osVersion = deserializeString(b, osName.end);
+    const kernelVersion = deserializeString(b, osVersion.end);
+    const iggyServerVersion = deserializeString(b, kernelVersion.end);
+    let position = iggyServerVersion.end;
+    ensureLength(b, position + 4 + 4);
+    const iggyServerSemver = b.readUInt32LE(position);
+    const cacheMetricsCount = b.readUInt32LE(position + 4);
+    position += 4 + 4;
+    ensureLength(b, position + cacheMetricsCount * CACHE_METRIC_SIZE + THREADS_AND_DISK_SIZE);
+    const cacheMetrics = [];
+    for (let index = 0; index < cacheMetricsCount; index += 1) {
+        cacheMetrics.push(deserializeCacheMetric(b, position));
+        position += CACHE_METRIC_SIZE;
+    }
+    const threadsCount = b.readUInt32LE(position);
+    const freeDiskSpace = b.readBigUInt64LE(position + 4);
+    const totalDiskSpace = b.readBigUInt64LE(position + 12);
+    position += THREADS_AND_DISK_SIZE;
+    // Servers that predate the open-files fields end the reply here.
+    let openFilesCount = 0n;
+    let openFilesLimit = 0n;
+    if (b.length > position) {
+        ensureLength(b, position + OPEN_FILES_SIZE);
+        openFilesCount = b.readBigUInt64LE(position);
+        openFilesLimit = b.readBigUInt64LE(position + 8);
+    }
     return {
         processId,
         cpuUsage,
@@ -66,10 +111,18 @@ const deserializeGetStats = (b) => {
         messagesCount,
         clientsCount,
         consumersGroupsCount,
-        hostname,
-        osName,
-        osVersion,
-        kernelVersion
+        hostname: hostname.value,
+        osName: osName.value,
+        osVersion: osVersion.value,
+        kernelVersion: kernelVersion.value,
+        iggyServerVersion: iggyServerVersion.value,
+        iggyServerSemver,
+        cacheMetrics,
+        threadsCount,
+        freeDiskSpace,
+        totalDiskSpace,
+        openFilesCount,
+        openFilesLimit
     };
 };
 export const GET_STATS = {

package/dist/wire/system/get-stats.command.test.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=get-stats.command.test.d.ts.map

package/dist/wire/system/get-stats.command.test.js

@@ -0,0 +1,97 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+import { describe, it } from 'node:test';
+import assert from 'node:assert/strict';
+import { GET_STATS } from './get-stats.command.js';
+import { DeserializeError } from '../error.utils.js';
+const u32 = (value) => {
+    const b = Buffer.alloc(4);
+    b.writeUInt32LE(value);
+    return b;
+};
+const u64 = (value) => {
+    const b = Buffer.alloc(8);
+    b.writeBigUInt64LE(value);
+    return b;
+};
+const f32 = (value) => {
+    const b = Buffer.alloc(4);
+    b.writeFloatLE(value);
+    return b;
+};
+const str = (value) => Buffer.concat([u32(Buffer.byteLength(value)), Buffer.from(value)]);
+const cacheMetric = (streamId, topicId, partitionId) => Buffer.concat([
+    u32(streamId), u32(topicId), u32(partitionId),
+    u64(1000n), u64(50n), f32(0.95)
+]);
+const buildStatsPayload = (openFiles) => Buffer.concat([
+    u32(1234), f32(25.5), f32(50),
+    u64(1073741824n), u64(8589934592n), u64(4294967296n),
+    u64(3600n), u64(1710000000000n), u64(1000000n), u64(500000n),
+    u64(2000000n),
+    u32(3), u32(10), u32(30), u32(90), u64(50000n), u32(5), u32(2),
+    str('node-1'), str('Linux'), str('6.1'), str('6.1.0'), str('0.6.0'),
+    u32(600),
+    u32(2), cacheMetric(1, 1, 0), cacheMetric(2, 3, 1),
+    u32(16), u64(107374182400n), u64(512110190592n),
+    ...(openFiles ? [u64(openFiles.count), u64(openFiles.limit)] : [])
+]);
+const deserialize = (data) => GET_STATS.deserialize({ status: 0, length: data.length, data });
+describe('GetStats Command', () => {
+    it('deserializes open files count and limit', () => {
+        const stats = deserialize(buildStatsPayload({ count: 1234n, limit: 1048576n }));
+        assert.equal(stats.processId, 1234);
+        assert.equal(stats.kernelVersion, '6.1.0');
+        assert.equal(stats.openFilesCount, 1234n);
+        assert.equal(stats.openFilesLimit, 1048576n);
+    });
+    it('deserializes server version, cache metrics, threads and disk space', () => {
+        const stats = deserialize(buildStatsPayload());
+        assert.equal(stats.iggyServerVersion, '0.6.0');
+        assert.equal(stats.iggyServerSemver, 600);
+        assert.deepEqual(stats.cacheMetrics, [
+            {
+                streamId: 1, topicId: 1, partitionId: 0,
+                hits: 1000n, misses: 50n, hitRatio: Math.fround(0.95)
+            },
+            {
+                streamId: 2, topicId: 3, partitionId: 1,
+                hits: 1000n, misses: 50n, hitRatio: Math.fround(0.95)
+            }
+        ]);
+        assert.equal(stats.threadsCount, 16);
+        assert.equal(stats.freeDiskSpace, 107374182400n);
+        assert.equal(stats.totalDiskSpace, 512110190592n);
+    });
+    it('reads open files fields as 0 when the reply ends at total_disk_space', () => {
+        const stats = deserialize(buildStatsPayload());
+        assert.equal(stats.kernelVersion, '6.1.0');
+        assert.equal(stats.openFilesCount, 0n);
+        assert.equal(stats.openFilesLimit, 0n);
+    });
+    it('throws on truncated open files fields', () => {
+        const payload = buildStatsPayload({ count: 1234n, limit: 1048576n });
+        for (let cut = 1; cut < 16; cut += 1)
+            assert.throws(() => deserialize(payload.subarray(0, payload.length - cut)), DeserializeError);
+    });
+    it('throws on a reply truncated before total_disk_space ends', () => {
+        const payload = buildStatsPayload();
+        for (let length = 0; length < payload.length; length += 1)
+            assert.throws(() => deserialize(payload.subarray(0, length)), DeserializeError, `a reply cut to ${length} bytes must not decode`);
+    });
+});
+//# sourceMappingURL=get-stats.command.test.js.map

package/dist/wire/vsr/index.js

@@ -61,9 +61,8 @@ export class VsrSession {
         else {
             if (this.state.session === null)
                 throw responseError(command, UNAUTHENTICATED);
-            // Partition ops consume an id too, even though no partition-plane dedup
-            // exists yet: dedup needs each send to carry a distinct number, and the
-            // metadata watermark tolerates the gaps.
+            // Partition ops consume an id too: dedup needs each send to carry a
+            // distinct number, and the metadata watermark tolerates the gaps.
             request = this.state.nextRequestId();
             session = this.state.session;
         }

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "apache-iggy",
   "type": "module",
-  "version": "0.10.0",
+  "version": "0.10.1-edge.1",
   "description": "Official Apache Iggy NodeJS SDK",
   "keywords": [
     "iggy",
@@ -50,18 +50,18 @@
   "author": "github.com/T1B0",
   "license": "Apache-2.0",
   "dependencies": {
-    "@node-rs/xxhash": "1.7.7",
+    "@node-rs/xxhash": "1.7.8",
     "debug": "4.4.3",
     "generic-pool": "3.9.0",
     "uuidv7": "1.2.1"
   },
   "devDependencies": {
-    "@commitlint/cli": "21.2.2",
-    "@commitlint/config-conventional": "21.2.2",
+    "@commitlint/cli": "21.2.3",
+    "@commitlint/config-conventional": "21.2.3",
     "@cucumber/cucumber": "13.2.1",
     "@swc-node/register": "1.12.1",
     "@types/debug": "4.1.13",
-    "@types/node": "26.4.1",
+    "@types/node": "26.6.2",
     "c8": "^12.0.0",
     "husky": "9.1.7",
     "typescript": "6.0.3",

package/dist/wire/message/flush-unsaved-buffers.command.d.ts

@@ -1,26 +0,0 @@
-import { type Id } from '../identifier.utils.js';
-/**
- * Parameters for the flush unsaved buffers command.
- */
-export type FlushUnsavedBuffer = {
-    /** Stream identifier */
-    streamId: Id;
-    /** Topic identifier */
-    topicId: Id;
-    /** Partition ID to flush */
-    partitionId: number;
-};
-/**
- * Flush unsaved buffers command definition.
- * Forces buffered messages to be persisted to disk.
- */
-export declare const FLUSH_UNSAVED_BUFFERS: {
-    code: number;
-    serialize: ({ streamId, topicId, partitionId }: FlushUnsavedBuffer) => Buffer<ArrayBuffer>;
-    deserialize: (r: import("../../index.js").CommandResponse) => boolean;
-};
-/**
- * Executable flush unsaved buffers command function.
- */
-export declare const flushUnsavedBuffers: (getClient: import("../../index.js").ClientProvider) => (arg: FlushUnsavedBuffer) => Promise<boolean>;
-//# sourceMappingURL=flush-unsaved-buffers.command.d.ts.map

package/dist/wire/message/flush-unsaved-buffers.command.js

@@ -1,36 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements.  See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership.  The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License.  You may obtain a copy of the License at
-//
-//   http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied.  See the License for the
-// specific language governing permissions and limitations
-// under the License.
-import { serializeFlushUnsavedBuffers } from './message.utils.js';
-import { deserializeVoidResponse } from '../../client/client.utils.js';
-import { wrapCommand } from '../command.utils.js';
-import { COMMAND_CODE } from '../command.code.js';
-/**
- * Flush unsaved buffers command definition.
- * Forces buffered messages to be persisted to disk.
- */
-export const FLUSH_UNSAVED_BUFFERS = {
-    code: COMMAND_CODE.FlushUnsavedBuffers,
-    serialize: ({ streamId, topicId, partitionId }) => {
-        return serializeFlushUnsavedBuffers(streamId, topicId, partitionId);
-    },
-    deserialize: deserializeVoidResponse
-};
-/**
- * Executable flush unsaved buffers command function.
- */
-export const flushUnsavedBuffers = wrapCommand(FLUSH_UNSAVED_BUFFERS);
-//# sourceMappingURL=flush-unsaved-buffers.command.js.map