apache-iggy 0.10.0-edge.7 → 0.10.0

package/README.md

@@ -19,9 +19,11 @@ note: previous works on node.js http client has been moved to [iggy-node-http-cl
 ## install
 
 ```bash
-npm i --save apache-iggy
+npm i --save apache-iggy@edge
 ```
 
+Use the `edge` package with server 0.9.0 or `edge`. Stable Node SDK 0.8.0 uses the older wire protocol. For a local server with the example credentials, follow the [example setup](../../examples/node/README.md#running-examples).
+
 ## basic usage
 
 ### Response frame limit
@@ -49,12 +51,9 @@ Codes absent from the SDK command table use `Operation::NonReplicated` and
 carry the command code in the request header's reserved field. The server
 remains authoritative for classifying or rejecting extension commands.
 
-Sends must use explicit `Partitioning.PartitionId` partitioning: the client
-routes each request to a partition-scoped namespace, so broker-side balancing
-(`Partitioning.Balanced`) and key hashing (`Partitioning.MessageKey`) are
-rejected before the request is sent.
-<!-- TODO(hubcio): Balanced and MessageKey partitioning to be implemented;
-not decided yet whether it'll be on server side or client side. -->
+Sends encode `Partitioning.PartitionId`, `Partitioning.Balanced` or
+`Partitioning.MessageKey` in the payload. The server resolves the target
+partition at admission.
 
 VSR works over TCP and TLS. It restricts `Client` to one pooled connection because authentication, request sequencing, and consumer-group assignments belong to one consensus session. Configurations requesting more than one pooled connection fail before a socket is opened.
 
@@ -69,8 +68,9 @@ The client pings every `heartbeatInterval` milliseconds, 5000 by default, which
 keeps an idle session alive when the server's `[heartbeat]` eviction is enabled.
 `heartbeatInterval` also accepts a duration expression such as `"10s"` or
 `"1h 30m"`, like the Rust SDK.
-The server evicts a connection silent for 36 s, which is 1.2 x its 30 s
-heartbeat interval. Raising the client interval past that window, or setting it
+With the default heartbeat settings, a group member becomes eligible for
+eviction after 36 s of silence (1.2 x the 30 s interval); the verifier checks
+once per interval. Raising the client interval past that window, or setting it
 to 0 to disable client heartbeats, exposes an idle consumer-group member to
 eviction; a connection holding no group membership is left alone. Any other
 unusable value is rejected instead of silently disabling the heartbeat.
@@ -146,6 +146,13 @@ Durations accept the same expressions as the Rust SDK, for example `500ms`,
 
 ## use sources
 
+Cluster auto-commit polling over TCP/TLS keeps group membership on the coordinator
+and uses separate connections to partition primaries. It requires server support
+for binary commands 14, 103 and 104. Pause binary auto-commit consumers for the
+whole upgrade: upgrade every server first, then the SDKs, and restart consumers
+so they rejoin their groups. Older SDKs can lose membership when a backup refuses
+an offset commit; the new SDK does not fall back to legacy polling.
+
 ### Install
 
 ```bash

package/dist/client/client.connection.d.ts

@@ -68,7 +68,7 @@ export declare class IggyConnection extends EventEmitter {
      *
      * @returns Promise that resolves when connected
      */
-    connect(): Promise<this>;
+    connect(boundDial?: boolean): Promise<this>;
     /**
      * Waits for one dial, bounded while other endpoints are queued behind it.
      *

package/dist/client/client.connection.js

@@ -229,7 +229,7 @@ export class IggyConnection extends EventEmitter {
      *
      * @returns Promise that resolves when connected
      */
-    connect() {
+    connect(boundDial = false) {
         if (this.ending)
             return Promise.reject(new Error('connection is closed'));
         if (this.connected)
@@ -245,7 +245,7 @@ export class IggyConnection extends EventEmitter {
         // Bounded here too when the client knows somewhere else to go: this dial
         // is not part of a pass, so an endpoint that never becomes usable would
         // hold it with no timer of its own and the redial pass would never start.
-        const connectPromise = this._dialWithin(socket, this._redialCandidates().length > 1);
+        const connectPromise = this._dialWithin(socket, boundDial || this._redialCandidates().length > 1);
         this.connectPromise = connectPromise;
         const clearConnectPromise = () => {
             if (this.connectPromise === connectPromise)

package/dist/client/client.socket.d.ts

@@ -54,6 +54,12 @@ export declare class CommandResponseStream extends EventEmitter {
     heartbeatIntervalHandler?: NodeJS.Timeout;
     /** Whether a heartbeat ping is still awaiting its response */
     private heartbeatInFlight;
+    private rememberedCredentials?;
+    private clustered?;
+    private metadataWatermark;
+    private routingGeneration;
+    private pollRoutes;
+    private pollConnections;
     /**
      * Creates a new CommandResponseStream.
      *
@@ -109,6 +115,12 @@ export declare class CommandResponseStream extends EventEmitter {
      */
     private _releaseUndecidedMove;
     private _rememberRoster;
+    private _pollOnPrimary;
+    private _pollRoutingControl;
+    private _observeMetadataReply;
+    private _rememberCredentials;
+    private _signInCredentials;
+    private _clearPollRouting;
     /**
      * Processes queued commands sequentially.
      * Emits 'finishQueue' when all commands are processed.

package/dist/client/client.socket.js

@@ -20,8 +20,12 @@ import { debug } from './client.debug.js';
 import { endpointKey, IggyConnection } from './client.connection.js';
 import { LOGIN, LOGIN_WITH_TOKEN, LOGOUT, PING } from '../wire/index.js';
 import { GET_CLUSTER_METADATA } from '../wire/cluster/get-cluster-metadata.command.js';
+import { deserializeNode } from '../wire/cluster/cluster.utils.js';
 import { COMMAND_CODE } from '../wire/command.code.js';
-import { decodeVsrResponse, prepareVsrCommand, readRegisteredSession, VsrEvictionError, VsrSession, } from '../wire/vsr/index.js';
+import { serializeIdentifier } from '../wire/identifier.utils.js';
+import { Command, HEADER_SIZE, REPLY_OFFSET, peekCommand, readReplyOperation, readStatus } from '../wire/vsr/header.js';
+import { Operation, isKnownOperation } from '../wire/vsr/operation.js';
+import { decodeVsrResponse, prepareVsrCommand, readRegisteredSession, readWireName, VsrEvictionError, VsrSession, } from '../wire/vsr/index.js';
 import { normalizeClientConfig } from './client.config.js';
 const VSR_RESPONSE_TIMEOUT_MS = 30_000;
 const VSR_RETRY_INTERVAL_MS = 50;
@@ -30,6 +34,14 @@ const LEADERLESS_POLL_INTERVAL_MS = 250;
 const MAX_LEADER_REDIRECTS = 3;
 const TRANSIENT_NOT_COMMITTED = 57;
 const TRANSIENT_NOT_ACCEPTED = 58;
+const UNAUTHENTICATED = 40;
+const STALE_CLIENT = 30;
+const FEATURE_UNAVAILABLE = 5;
+const MAX_POLL_ROUTES = 4096;
+const MAX_POLL_CONNECTIONS = 256;
+const CONSUMER_SESSION_SIZE = 32;
+const METADATA_WATERMARK_OFFSET = 24;
+const POLL_OPTIONS_SIZE = 9 + 4 + 1;
 /**
  * How long a `TRANSIENT_NOT_ACCEPTED` request replays on the same connection
  * before the roster is re-read. A node that stopped being primary refuses
@@ -121,6 +133,12 @@ export class CommandResponseStream extends EventEmitter {
     heartbeatIntervalHandler;
     /** Whether a heartbeat ping is still awaiting its response */
     heartbeatInFlight;
+    rememberedCredentials;
+    clustered;
+    metadataWatermark = 0n;
+    routingGeneration = 0;
+    pollRoutes = new Map();
+    pollConnections = new Map();
     /**
      * Creates a new CommandResponseStream.
      *
@@ -209,10 +227,26 @@ export class CommandResponseStream extends EventEmitter {
         this.pendingSubmissions += 1;
         try {
             const { handleResponse = true, last = true, followsLeaderMoves = true } = options;
+            const autoCommitPoll = command === COMMAND_CODE.PollMessages &&
+                payload.length > POLL_OPTIONS_SIZE && payload.at(-1) === 1;
+            const pollDeadline = autoCommitPoll
+                ? options.deadline ?? Date.now() + VSR_RESPONSE_TIMEOUT_MS
+                : undefined;
             if (!this.connection.connected)
-                await this.connection.connect();
+                await (pollDeadline === undefined ? this.connection.connect()
+                    : withinDeadline(this.connection.connect(), pollDeadline));
             if (!this.isAuthenticated && !this.isUnloggedCommand(command))
-                await this.authenticate(this.options.credentials);
+                await (pollDeadline === undefined ? this.authenticate(this._signInCredentials())
+                    : withinDeadline(this.authenticate(this._signInCredentials()), pollDeadline));
+            if (pollDeadline !== undefined) {
+                const deadline = pollDeadline;
+                if (this.clustered === undefined)
+                    await withinDeadline(this.sendCommand(GET_CLUSTER_METADATA.code, GET_CLUSTER_METADATA.serialize(), { deadline, followsLeaderMoves: false }), deadline);
+                if (this.clustered === undefined)
+                    throw new Error('cannot determine the poll routing topology');
+                if (this.clustered)
+                    return await this._pollOnPrimary(payload, deadline, handleResponse);
+            }
             // The roster read is itself a queued command and the queue is
             // single-flighted, so the leader re-check cannot happen inside
             // `_processVsr`. The refusal comes back out here instead, where the
@@ -289,7 +323,7 @@ export class CommandResponseStream extends EventEmitter {
                     // command fails client-side, a non-replicated one goes out with
                     // session 0.
                     if (!this.isAuthenticated && !this.isUnloggedCommand(command))
-                        await this.authenticate(this.options.credentials);
+                        await this.authenticate(this._signInCredentials());
                 }
             }
             if (!isLoginCommand(command) || this.settlingLeader)
@@ -424,6 +458,9 @@ export class CommandResponseStream extends EventEmitter {
     _rememberRoster(response) {
         try {
             const metadata = GET_CLUSTER_METADATA.deserialize(response);
+            if (metadata.nodes.length === 0)
+                return;
+            this.clustered = metadata.nodes.length > 1;
             this.connection.rememberRoster(metadata.nodes
                 .filter((node) => node.endpoints.tcp !== 0)
                 .map((node) => ({ host: node.ip, port: node.endpoints.tcp })));
@@ -432,6 +469,201 @@ export class CommandResponseStream extends EventEmitter {
             debug('an unreadable roster leaves the redial candidates as they are', error);
         }
     }
+    async _pollOnPrimary(payload, deadline, handleResponse) {
+        const key = payload.subarray(0, -POLL_OPTIONS_SIZE).toString('hex');
+        while (Date.now() < deadline && !this.connection.ending) {
+            try {
+                const generation = this.routingGeneration;
+                let route = this.pollRoutes.get(key);
+                if (!route ||
+                    route.attachment.readBigUInt64LE(METADATA_WATERMARK_OFFSET) <
+                        this.metadataWatermark) {
+                    const response = await withinDeadline(this._pollRoutingControl(payload, deadline), deadline);
+                    if (generation !== this.routingGeneration)
+                        throw responseError(COMMAND_CODE.PollMessages, TRANSIENT_NOT_ACCEPTED);
+                    if (response.data.length < CONSUMER_SESSION_SIZE)
+                        throw new Error('poll routing response is incomplete');
+                    const node = deserializeNode(response.data, CONSUMER_SESSION_SIZE);
+                    if (node.length + CONSUMER_SESSION_SIZE !== response.data.length || !node.data.ip)
+                        throw new Error('poll routing response has no valid TCP endpoint');
+                    if (node.data.endpoints.tcp === 0)
+                        throw responseError(COMMAND_CODE.PollMessages, FEATURE_UNAVAILABLE);
+                    const attachment = Buffer.from(response.data.subarray(0, CONSUMER_SESSION_SIZE));
+                    if (attachment.readBigUInt64LE(METADATA_WATERMARK_OFFSET) < this.metadataWatermark)
+                        attachment.writeBigUInt64LE(this.metadataWatermark, METADATA_WATERMARK_OFFSET);
+                    route = {
+                        endpoint: { host: node.data.ip, port: node.data.endpoints.tcp },
+                        attachment
+                    };
+                    if (this.pollRoutes.size >= MAX_POLL_ROUTES)
+                        this.pollRoutes.clear();
+                    this.pollRoutes.set(key, route);
+                }
+                const endpoint = endpointKey(route.endpoint);
+                let entry = this.pollConnections.get(endpoint);
+                if (!entry) {
+                    if (this.pollConnections.size >= MAX_POLL_CONNECTIONS) {
+                        const idle = Array.from(this.pollConnections).find(([, value]) => !value.busy);
+                        if (!idle)
+                            throw responseError(COMMAND_CODE.PollMessages, TRANSIENT_NOT_ACCEPTED);
+                        this.pollConnections.delete(idle[0]);
+                        idle[1].client.destroy();
+                    }
+                    entry = {
+                        client: new CommandResponseStream({
+                            ...this.options,
+                            options: { ...this.options.options, ...route.endpoint },
+                            heartbeatInterval: 0,
+                            reconnect: { enabled: false, interval: 0, maxRetries: 0 }
+                        })
+                    };
+                    this.pollConnections.set(endpoint, entry);
+                }
+                while (entry.busy)
+                    await withinDeadline(entry.busy, deadline);
+                if (generation !== this.routingGeneration ||
+                    this.pollConnections.get(endpoint) !== entry ||
+                    route.attachment.readBigUInt64LE(METADATA_WATERMARK_OFFSET) <
+                        this.metadataWatermark)
+                    throw responseError(COMMAND_CODE.PollMessages, TRANSIENT_NOT_ACCEPTED);
+                let release;
+                entry.busy = new Promise((resolve) => { release = resolve; });
+                let polling = false;
+                try {
+                    if (!entry.client.isAuthenticated) {
+                        entry.attachment = undefined;
+                        await withinDeadline(entry.client.connection.connect(true), deadline, () => entry.client.destroy());
+                        const credentials = this._signInCredentials();
+                        const login = 'token' in credentials ? LOGIN_WITH_TOKEN : LOGIN;
+                        const loginPayload = 'token' in credentials
+                            ? LOGIN_WITH_TOKEN.serialize(credentials)
+                            : LOGIN.serialize(credentials);
+                        await entry.client._queueCommand(login.code, loginPayload, true, true, false, deadline);
+                    }
+                    if (!entry.attachment?.equals(route.attachment)) {
+                        await entry.client._queueCommand(COMMAND_CODE.AttachConsumerSession, route.attachment, true, true, false, deadline);
+                        entry.attachment = route.attachment;
+                    }
+                    if (generation !== this.routingGeneration ||
+                        route.attachment.readBigUInt64LE(METADATA_WATERMARK_OFFSET) <
+                            this.metadataWatermark)
+                        throw responseError(COMMAND_CODE.PollMessages, TRANSIENT_NOT_ACCEPTED);
+                    polling = true;
+                    return await entry.client._queueCommand(COMMAND_CODE.PollMessagesOnPrimary, payload, handleResponse, true, false, deadline);
+                }
+                catch (error) {
+                    if (error instanceof ResponseError &&
+                        error.errorCode === TRANSIENT_NOT_ACCEPTED) {
+                        entry.attachment = undefined;
+                    }
+                    else {
+                        if (this.pollConnections.get(endpoint) === entry)
+                            this.pollConnections.delete(endpoint);
+                        entry.client.destroy();
+                    }
+                    if (!(error instanceof ResponseError) || error instanceof VsrEvictionError ||
+                        error.errorCode === UNAUTHENTICATED || error.errorCode === STALE_CLIENT)
+                        throw responseError(COMMAND_CODE.PollMessages, polling ? TRANSIENT_NOT_COMMITTED : TRANSIENT_NOT_ACCEPTED);
+                    throw error;
+                }
+                finally {
+                    entry.busy = undefined;
+                    release();
+                }
+            }
+            catch (error) {
+                this.pollRoutes.delete(key);
+                if (error instanceof VsrResponseTimeoutError)
+                    throw responseError(COMMAND_CODE.PollMessages, TRANSIENT_NOT_COMMITTED);
+                if (!(error instanceof ResponseError) ||
+                    error.errorCode !== TRANSIENT_NOT_ACCEPTED)
+                    throw error instanceof ResponseError
+                        ? responseError(COMMAND_CODE.PollMessages, error.errorCode)
+                        : error;
+                if (!worthAnotherAttempt(deadline))
+                    throw responseError(COMMAND_CODE.PollMessages, TRANSIENT_NOT_ACCEPTED);
+                await delay(VSR_RETRY_INTERVAL_MS);
+            }
+        }
+        throw responseError(COMMAND_CODE.PollMessages, TRANSIENT_NOT_ACCEPTED);
+    }
+    async _pollRoutingControl(payload, deadline) {
+        const options = { deadline, followsLeaderMoves: false };
+        try {
+            return await this.sendCommand(COMMAND_CODE.GetPollRouting, payload, options);
+        }
+        catch (error) {
+            if (error instanceof ResponseError &&
+                error.errorCode !== UNAUTHENTICATED && error.errorCode !== STALE_CLIENT)
+                throw error;
+            if (Date.now() >= deadline)
+                throw error;
+            // Only a failed coordinator exchange uses normal session recovery.
+            // A healthy coordinator refusing a route must retain its group membership.
+            if (error instanceof ResponseError) {
+                this.connection.abort();
+                this.connection.connected = false;
+                this._resetSession();
+            }
+            await this.sendCommand(PING.code, PING.serialize(), { deadline });
+            return this.sendCommand(COMMAND_CODE.GetPollRouting, payload, options);
+        }
+    }
+    _observeMetadataReply(frame) {
+        if (frame.length < HEADER_SIZE || peekCommand(frame) !== Command.Reply)
+            return;
+        const operation = readReplyOperation(frame);
+        if (!isKnownOperation(operation) || operation === Operation.NonReplicated ||
+            operation === Operation.SendMessages || operation === Operation.StoreConsumerOffset ||
+            operation === Operation.DeleteConsumerOffset)
+            return;
+        const commit = frame.readBigUInt64LE(REPLY_OFFSET.commit);
+        if (commit > this.metadataWatermark)
+            this.metadataWatermark = commit;
+    }
+    _rememberCredentials(command, payload) {
+        if (command === LOGIN.code) {
+            const username = readWireName(payload, 0);
+            this.rememberedCredentials = {
+                username: username.value, password: readWireName(payload, username.next).value
+            };
+        }
+        else if (command === LOGIN_WITH_TOKEN.code) {
+            this.rememberedCredentials = { token: readWireName(payload, 0).value };
+        }
+        else if ((command === COMMAND_CODE.ChangePassword || command === COMMAND_CODE.UpdateUser) &&
+            this.userId !== undefined && this.rememberedCredentials &&
+            'username' in this.rememberedCredentials) {
+            const credentials = this.rememberedCredentials;
+            const identifier = [serializeIdentifier(this.userId), serializeIdentifier(credentials.username)]
+                .find((encoded) => payload.subarray(0, encoded.length).equals(encoded));
+            if (!identifier)
+                return;
+            if (command === COMMAND_CODE.ChangePassword) {
+                const currentPassword = readWireName(payload, identifier.length);
+                const password = readWireName(payload, currentPassword.next).value;
+                this.rememberedCredentials = { ...credentials, password };
+                if ('username' in this.options.credentials && this.options.credentials.username === credentials.username)
+                    this.options.credentials = { ...this.options.credentials, password };
+            }
+            else if (payload[identifier.length] === 1) {
+                const username = readWireName(payload, identifier.length + 1).value;
+                this.rememberedCredentials = { ...credentials, username };
+                if ('username' in this.options.credentials && this.options.credentials.username === credentials.username)
+                    this.options.credentials = { ...this.options.credentials, username };
+            }
+        }
+    }
+    _signInCredentials() {
+        return this.rememberedCredentials ?? this.options.credentials;
+    }
+    _clearPollRouting() {
+        this.routingGeneration += 1;
+        this.pollRoutes.clear();
+        for (const entry of this.pollConnections.values())
+            entry.client.destroy();
+        this.pollConnections.clear();
+    }
     /**
      * Processes queued commands sequentially.
      * Emits 'finishQueue' when all commands are processed.
@@ -535,8 +767,15 @@ export class CommandResponseStream extends EventEmitter {
                 finally {
                     requestWritten ||= exchangeState.written;
                 }
-                if (!handleResp)
+                this._observeMetadataReply(response);
+                if (!handleResp) {
+                    // Routing still needs refusals when the caller decodes the frame.
+                    if (command === COMMAND_CODE.PollMessagesOnPrimary &&
+                        peekCommand(response) === Command.Reply &&
+                        readStatus(response) === TRANSIENT_NOT_ACCEPTED)
+                        throw responseError(command, TRANSIENT_NOT_ACCEPTED);
                     return response;
+                }
                 try {
                     parsed = decodeVsrResponse(response, command);
                     break;
@@ -545,6 +784,10 @@ export class CommandResponseStream extends EventEmitter {
                     if (!(error instanceof ResponseError) ||
                         !isTransientVsrError(error.errorCode))
                         throw error;
+                    if (command === COMMAND_CODE.PollMessagesOnPrimary ||
+                        command === COMMAND_CODE.GetPollRouting ||
+                        command === COMMAND_CODE.AttachConsumerSession)
+                        throw error;
                     lastTransientError = error;
                     // A not-admitted refusal is a statement about who leads, not about
                     // load: a node that stopped being primary refuses forever, so
@@ -568,7 +811,9 @@ export class CommandResponseStream extends EventEmitter {
                 this.isAuthenticated = true;
                 this.userId = parsed.data.readUInt32LE(0);
             }
+            this._rememberCredentials(command, payload);
             if (prepared.command === COMMAND_CODE.LogoutUser) {
+                this.rememberedCredentials = undefined;
                 this._resetSession();
             }
             // Every roster read feeds the redial candidates, whoever asked for it
@@ -735,6 +980,8 @@ export class CommandResponseStream extends EventEmitter {
         this._execQueue = [];
     }
     _resetSession() {
+        this._clearPollRouting();
+        this.clustered = undefined;
         if (!this.isAuthenticated &&
             this.userId === undefined &&
             !this.vsrSession.hasActivity)
@@ -855,6 +1102,8 @@ export class CommandResponseStream extends EventEmitter {
      * Stops heartbeat and destroys the connection.
      */
     destroy() {
+        this.rememberedCredentials = undefined;
+        this._clearPollRouting();
         if (this.heartbeatIntervalHandler)
             clearInterval(this.heartbeatIntervalHandler);
         return this.connection._destroy();
@@ -873,6 +1122,23 @@ export function getRawClient(options) {
 const isLoginCommand = (command) => command === COMMAND_CODE.LoginUser ||
     command === COMMAND_CODE.LoginWithAccessToken;
 const delay = (milliseconds) => new Promise((resolve) => setTimeout(resolve, milliseconds));
+const withinDeadline = async (pending, deadline, expired) => {
+    let timer;
+    try {
+        return await Promise.race([
+            pending,
+            new Promise((_resolve, reject) => {
+                timer = setTimeout(() => {
+                    expired?.();
+                    reject(new VsrResponseTimeoutError(VSR_RESPONSE_TIMEOUT_MS));
+                }, Math.max(0, deadline - Date.now()));
+            })
+        ]);
+    }
+    finally {
+        clearTimeout(timer);
+    }
+};
 const isTransientVsrError = (errorCode) => errorCode === TRANSIENT_NOT_COMMITTED ||
     errorCode === TRANSIENT_NOT_ACCEPTED;
 //# sourceMappingURL=client.socket.js.map