@cratis/chronicle 1.0.2 → 2.1.0

package/connection/LeastConnectionsLoadBalancerStrategy.ts

@@ -0,0 +1,117 @@
+// Copyright (c) Cratis. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+import { Agent } from 'undici';
+import type { ChronicleServerAddress } from './ChronicleConnectionString';
+import { formatServerAddress } from './formatServerAddress';
+import type { ILoadBalancerStrategy } from './ILoadBalancerStrategy';
+
+const DEFAULT_MAX_JITTER_MS = 250;
+const PROBE_TIMEOUT_MS = 2000;
+
+// Node's global fetch types its `dispatcher` option against `undici-types` (bundled with
+// @types/node), which is structurally close to but not identical to the real `undici`
+// package's own `Agent`/`Dispatcher` types - hence the assertion through `unknown` at every
+// use below.
+type FetchDispatcher = NonNullable<RequestInit['dispatcher']>;
+
+/**
+ * Selects the least-loaded candidate by probing each one's current connection count over
+ * HTTP and picking the minimum, breaking ties randomly. This is the default strategy.
+ *
+ * Before every probe attempt (not just the first), waits a random jitter to avoid a
+ * thundering herd of clients probing every candidate at the exact same instant. A single
+ * candidate is returned immediately without probing, since there is nothing to choose
+ * between.
+ */
+export class LeastConnectionsLoadBalancerStrategy implements ILoadBalancerStrategy {
+    private readonly _dispatcher: Agent;
+
+    /**
+     * Initializes a new instance of {@link LeastConnectionsLoadBalancerStrategy}.
+     * @param skipTlsValidation - Whether to skip TLS certificate validation on the
+     * `/connections/count` and `/connections/reserve` probe requests, matching the toggle
+     * used for the gRPC channel itself.
+     * @param maxJitterMs - The maximum jitter, in milliseconds, to wait before every probe
+     * attempt. Defaults to 250ms. 0 disables jitter entirely.
+     */
+    constructor(
+        private readonly _skipTlsValidation: boolean,
+        private readonly _maxJitterMs: number = DEFAULT_MAX_JITTER_MS
+    ) {
+        this._dispatcher = new Agent({ connect: { rejectUnauthorized: !this._skipTlsValidation } });
+    }
+
+    async select(candidates: ChronicleServerAddress[]): Promise<ChronicleServerAddress> {
+        if (candidates.length === 0) {
+            throw new Error('Cannot select a server address from an empty candidate list');
+        }
+
+        if (candidates.length === 1) {
+            return candidates[0];
+        }
+
+        await this.jitter();
+
+        const connectionCounts = await Promise.all(candidates.map(candidate => this.probeConnectionCount(candidate)));
+        const minimumCount = Math.min(...connectionCounts);
+        const minimumIndices = connectionCounts.reduce<number[]>((indices, count, index) => {
+            if (count === minimumCount) {
+                indices.push(index);
+            }
+            return indices;
+        }, []);
+
+        const selected = candidates[minimumIndices[Math.floor(Math.random() * minimumIndices.length)]];
+
+        await this.reserve(selected);
+
+        return selected;
+    }
+
+    private async jitter(): Promise<void> {
+        if (this._maxJitterMs <= 0) {
+            return;
+        }
+
+        const delayMs = Math.floor(Math.random() * this._maxJitterMs);
+        if (delayMs <= 0) {
+            return;
+        }
+
+        await new Promise(resolve => setTimeout(resolve, delayMs));
+    }
+
+    private async probeConnectionCount(candidate: ChronicleServerAddress): Promise<number> {
+        try {
+            const response = await fetch(`https://${formatServerAddress(candidate)}/connections/count`, {
+                signal: AbortSignal.timeout(PROBE_TIMEOUT_MS),
+                dispatcher: this._dispatcher as unknown as FetchDispatcher
+            });
+
+            if (!response.ok) {
+                return Number.MAX_SAFE_INTEGER;
+            }
+
+            const body = (await response.text()).trim();
+            const count = Number(body);
+            return body.length > 0 && Number.isFinite(count) ? count : Number.MAX_SAFE_INTEGER;
+        } catch {
+            // A candidate that cannot be reached or answers with garbage is never preferred
+            // over one that responds normally, but must not fail the whole selection round.
+            return Number.MAX_SAFE_INTEGER;
+        }
+    }
+
+    private async reserve(candidate: ChronicleServerAddress): Promise<void> {
+        try {
+            await fetch(`https://${formatServerAddress(candidate)}/connections/reserve`, {
+                method: 'POST',
+                signal: AbortSignal.timeout(PROBE_TIMEOUT_MS),
+                dispatcher: this._dispatcher as unknown as FetchDispatcher
+            });
+        } catch {
+            // Best-effort reservation; the selected server is still used even if this fails.
+        }
+    }
+}

package/connection/LoadBalancerMode.ts

@@ -0,0 +1,25 @@
+// Copyright (c) Cratis. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+/**
+ * The strategy used to select one server address from a multi-host connection string
+ * (or a resolved set of `chronicle+srv://` addresses) for each connect/reconnect attempt.
+ */
+export enum LoadBalancerMode {
+    /**
+     * Probes every candidate's current connection count and picks the least-loaded one,
+     * breaking ties randomly. The default strategy.
+     */
+    LeastConnections = 'least-connections',
+
+    /**
+     * Cycles through candidates in order, starting from a random offset chosen once per
+     * strategy instance.
+     */
+    RoundRobin = 'round-robin',
+
+    /**
+     * Picks a uniformly random candidate on every call.
+     */
+    Random = 'random'
+}

package/connection/LoadBalancerStrategyFactory.ts

@@ -0,0 +1,28 @@
+// Copyright (c) Cratis. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+import type { ILoadBalancerStrategy } from './ILoadBalancerStrategy';
+import { LeastConnectionsLoadBalancerStrategy } from './LeastConnectionsLoadBalancerStrategy';
+import { LoadBalancerMode } from './LoadBalancerMode';
+import { RandomLoadBalancerStrategy } from './RandomLoadBalancerStrategy';
+import { RoundRobinLoadBalancerStrategy } from './RoundRobinLoadBalancerStrategy';
+
+/**
+ * Creates the {@link ILoadBalancerStrategy} instance matching a {@link LoadBalancerMode}.
+ * @param mode - The load balancer mode to create a strategy for.
+ * @param skipTlsValidation - Whether to skip TLS certificate validation for strategies that
+ * probe candidates over HTTP (currently only {@link LoadBalancerMode.LeastConnections}).
+ * @returns The load balancer strategy instance.
+ */
+export function createLoadBalancerStrategy(mode: LoadBalancerMode, skipTlsValidation: boolean): ILoadBalancerStrategy {
+    switch (mode) {
+        case LoadBalancerMode.RoundRobin:
+            return new RoundRobinLoadBalancerStrategy();
+        case LoadBalancerMode.Random:
+            return new RandomLoadBalancerStrategy();
+        case LoadBalancerMode.LeastConnections:
+            return new LeastConnectionsLoadBalancerStrategy(skipTlsValidation);
+        default:
+            throw new Error(`Unknown load balancer mode: ${mode}`);
+    }
+}

package/connection/RandomLoadBalancerStrategy.spec.ts

@@ -0,0 +1,50 @@
+// Copyright (c) Cratis. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+import { describe, expect, it } from 'vitest';
+import type { ChronicleServerAddress } from './ChronicleConnectionString';
+import { RandomLoadBalancerStrategy } from './RandomLoadBalancerStrategy';
+
+describe('RandomLoadBalancerStrategy', () => {
+    describe('when selecting from an empty candidate list', () => {
+        const strategy = new RandomLoadBalancerStrategy();
+
+        it('should throw', async () => {
+            await expect(strategy.select([])).rejects.toThrow();
+        });
+    });
+
+    describe('when selecting from a single candidate', () => {
+        const strategy = new RandomLoadBalancerStrategy();
+        const candidate: ChronicleServerAddress = { host: 'host1', port: 35000 };
+
+        it('should return that candidate', async () => {
+            expect(await strategy.select([candidate])).toEqual(candidate);
+        });
+    });
+
+    describe('when selecting from many candidates repeatedly', () => {
+        const strategy = new RandomLoadBalancerStrategy();
+        const candidates: ChronicleServerAddress[] = [
+            { host: 'host1', port: 35000 },
+            { host: 'host2', port: 35000 },
+            { host: 'host3', port: 35000 }
+        ];
+
+        it('should always return one of the candidates', async () => {
+            for (let attempt = 0; attempt < 25; attempt++) {
+                const selected = await strategy.select(candidates);
+                expect(candidates).toContainEqual(selected);
+            }
+        });
+
+        it('should eventually select more than one distinct candidate', async () => {
+            const selections = new Set<string>();
+            for (let attempt = 0; attempt < 50; attempt++) {
+                const selected = await strategy.select(candidates);
+                selections.add(`${selected.host}:${selected.port}`);
+            }
+            expect(selections.size).toBeGreaterThan(1);
+        });
+    });
+});

package/connection/RandomLoadBalancerStrategy.ts

@@ -0,0 +1,19 @@
+// Copyright (c) Cratis. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+import type { ChronicleServerAddress } from './ChronicleConnectionString';
+import type { ILoadBalancerStrategy } from './ILoadBalancerStrategy';
+
+/**
+ * Selects a uniformly random candidate on every call.
+ */
+export class RandomLoadBalancerStrategy implements ILoadBalancerStrategy {
+    async select(candidates: ChronicleServerAddress[]): Promise<ChronicleServerAddress> {
+        if (candidates.length === 0) {
+            throw new Error('Cannot select a server address from an empty candidate list');
+        }
+
+        const index = Math.floor(Math.random() * candidates.length);
+        return candidates[index];
+    }
+}

package/connection/RoundRobinLoadBalancerStrategy.spec.ts

@@ -0,0 +1,61 @@
+// Copyright (c) Cratis. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+import { describe, expect, it } from 'vitest';
+import type { ChronicleServerAddress } from './ChronicleConnectionString';
+import { RoundRobinLoadBalancerStrategy } from './RoundRobinLoadBalancerStrategy';
+
+describe('RoundRobinLoadBalancerStrategy', () => {
+    describe('when selecting from an empty candidate list', () => {
+        const strategy = new RoundRobinLoadBalancerStrategy();
+
+        it('should throw', async () => {
+            await expect(strategy.select([])).rejects.toThrow();
+        });
+    });
+
+    describe('when selecting repeatedly from the same strategy instance', () => {
+        const strategy = new RoundRobinLoadBalancerStrategy();
+        const candidates: ChronicleServerAddress[] = [
+            { host: 'host1', port: 35000 },
+            { host: 'host2', port: 35000 },
+            { host: 'host3', port: 35000 }
+        ];
+
+        it('should cycle through every candidate exactly once per full cycle', async () => {
+            const first = await strategy.select(candidates);
+            const second = await strategy.select(candidates);
+            const third = await strategy.select(candidates);
+            const fourth = await strategy.select(candidates);
+
+            const firstCycle = [first, second, third];
+            expect(firstCycle).toContainEqual(candidates[0]);
+            expect(firstCycle).toContainEqual(candidates[1]);
+            expect(firstCycle).toContainEqual(candidates[2]);
+
+            // The cycle wraps back to whichever candidate started it.
+            expect(fourth).toEqual(first);
+        });
+    });
+
+    describe('when two independent strategy instances select from the same candidates', () => {
+        const candidates: ChronicleServerAddress[] = [
+            { host: 'host1', port: 35000 },
+            { host: 'host2', port: 35000 },
+            { host: 'host3', port: 35000 }
+        ];
+
+        it('should not always start at the same candidate', async () => {
+            const startingPoints = new Set<string>();
+            for (let attempt = 0; attempt < 25; attempt++) {
+                const strategy = new RoundRobinLoadBalancerStrategy();
+                const selected = await strategy.select(candidates);
+                startingPoints.add(`${selected.host}:${selected.port}`);
+            }
+
+            // With a random starting offset across many fresh instances, at least one run
+            // should land somewhere other than the first candidate.
+            expect(startingPoints.size).toBeGreaterThan(1);
+        });
+    });
+});

package/connection/RoundRobinLoadBalancerStrategy.ts

@@ -0,0 +1,29 @@
+// Copyright (c) Cratis. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+import type { ChronicleServerAddress } from './ChronicleConnectionString';
+import type { ILoadBalancerStrategy } from './ILoadBalancerStrategy';
+
+/**
+ * Cycles through candidates in order, starting from a random offset chosen once per
+ * strategy instance so that multiple clients don't all start at the same candidate.
+ */
+export class RoundRobinLoadBalancerStrategy implements ILoadBalancerStrategy {
+    private _startOffset?: number;
+    private _callCount = 0;
+
+    async select(candidates: ChronicleServerAddress[]): Promise<ChronicleServerAddress> {
+        if (candidates.length === 0) {
+            throw new Error('Cannot select a server address from an empty candidate list');
+        }
+
+        if (this._startOffset === undefined) {
+            this._startOffset = Math.floor(Math.random() * candidates.length);
+        }
+
+        const index = (this._startOffset + this._callCount) % candidates.length;
+        this._callCount++;
+
+        return candidates[index];
+    }
+}

package/connection/TokenProvider.ts

@@ -53,7 +53,8 @@ export class OAuthTokenProvider implements ITokenProvider {
     constructor(
         private readonly _tokenEndpoint: string,
         private readonly _clientId: string,
-        private readonly _clientSecret: string
+        private readonly _clientSecret: string,
+        private readonly _skipTlsValidation: boolean = true
     ) {}
 
     async getAccessToken(): Promise<string | undefined> {
@@ -89,14 +90,18 @@ export class OAuthTokenProvider implements ITokenProvider {
 
         return new Promise((resolve, reject) => {
             const url = new URL(this._tokenEndpoint);
-            const httpModule = url.protocol === 'https:' ? https : http;
+            const isHttps = url.protocol === 'https:';
+            const httpModule = isHttps ? https : http;
 
             const req = httpModule.request(url, {
                 method: 'POST',
                 headers: {
                     'Content-Type': 'application/x-www-form-urlencoded',
                     'Content-Length': Buffer.byteLength(body)
-                }
+                },
+                // Chain validation is skipped only when skipTlsValidation is explicitly set,
+                // matching the gRPC channel's credentials for the same connection string.
+                ...(isHttps && this._skipTlsValidation ? { rejectUnauthorized: false } : {})
             }, response => {
                 let data = '';
 

package/connection/formatServerAddress.ts

@@ -0,0 +1,16 @@
+// Copyright (c) Cratis. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+import type { ChronicleServerAddress } from './ChronicleConnectionString';
+
+/**
+ * Formats a {@link ChronicleServerAddress} as a `host:port` string, wrapping IPv6 host
+ * literals in bracket notation (e.g. `[::1]:35000`) as required by both the connection
+ * string grammar and gRPC target grammar.
+ * @param address - The address to format.
+ * @returns The formatted `host:port` string.
+ */
+export function formatServerAddress(address: ChronicleServerAddress): string {
+    const host = address.host.includes(':') ? `[${address.host}]` : address.host;
+    return `${host}:${address.port}`;
+}

package/dist/ChronicleClient.d.ts

@@ -19,6 +19,7 @@ import { IEventStore } from './IEventStore';
 export declare class ChronicleClient implements IChronicleClient {
     readonly options: ChronicleOptions;
     private static readonly _healthCheckIntervalMs;
+    private static readonly _maxBackoffMs;
     private readonly _connection;
     private readonly _stores;
     private readonly _lifecycle;
@@ -29,6 +30,7 @@ export declare class ChronicleClient implements IChronicleClient {
     private _discoveryOperation?;
     private _isDisposed;
     private _keepAliveAbortController?;
+    private _healthCheckInFlight;
     /**
      * Creates a new {@link ChronicleClient} using the provided options.
      * @param options - The options to configure the client, including the connection string.
@@ -43,13 +45,18 @@ export declare class ChronicleClient implements IChronicleClient {
     private connectWithRetry;
     private ensureConnected;
     private reconnect;
+    /**
+     * Waits out the exponential backoff for a failed attempt and reports why.
+     * The delay is jittered so a fleet of clients that lost the same kernel does
+     * not come back in lockstep and knock it over again.
+     */
+    private backOff;
     private withReconnect;
     private shouldReconnect;
     private toErrorMessage;
     private startConnectionWatchdog;
     private runHealthCheck;
     private startKernelKeepAlive;
-    private runKeepAliveLoop;
     private registerArtifactsForStore;
 }
 //# sourceMappingURL=ChronicleClient.d.ts.map

package/dist/ChronicleClient.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"ChronicleClient.d.ts","sourceRoot":"","sources":["../ChronicleClient.ts"],"names":[],"mappings":"AAKA,OAAO,EAAE,gBAAgB,EAAE,MAAM,oBAAoB,CAAC;AAItD,OAAO,EAAE,cAAc,EAAE,MAAM,kBAAkB,CAAC;AAClD,OAAO,EAAE,uBAAuB,EAAE,MAAM,2BAA2B,CAAC;AACpE,OAAO,EAAE,gBAAgB,EAAE,MAAM,oBAAoB,CAAC;AACtD,OAAO,EAAE,WAAW,EAAE,MAAM,eAAe,CAAC;AAK5C;;;;;;;;;;;;GAYG;AACH,qBAAa,eAAgB,YAAW,gBAAgB;IAsBxC,QAAQ,CAAC,OAAO,EAAE,gBAAgB;IArB9C,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,sBAAsB,CAAQ;IAEtD,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAsB;IAClD,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAsC;IAC9D,OAAO,CAAC,QAAQ,CAAC,UAAU,CAA6B;IAExD,OAAO,CAAC,QAAQ,CAAC,OAAO,CAErB;IAEH,OAAO,CAAC,eAAe,CAAC,CAAiC;IACzD,OAAO,CAAC,iBAAiB,CAAC,CAAgB;IAC1C,OAAO,CAAC,mBAAmB,CAAC,CAAgB;IAC5C,OAAO,CAAC,mBAAmB,CAAC,CAAgB;IAC5C,OAAO,CAAC,WAAW,CAAS;IAC5B,OAAO,CAAC,yBAAyB,CAAC,CAAkB;IAEpD;;;OAGG;gBACkB,OAAO,EAAE,gBAAgB;IAgD9C,kBAAkB;IACZ,aAAa,CAAC,IAAI,EAAE,MAAM,GAAG,cAAc,EAAE,SAAS,CAAC,EAAE,MAAM,GAAG,uBAAuB,GAAG,OAAO,CAAC,WAAW,CAAC;IA0DtH,kBAAkB;IACZ,cAAc,IAAI,OAAO,CAAC,cAAc,EAAE,CAAC;IA0BjD,kBAAkB;IAClB,OAAO,IAAI,IAAI;YAsBD,gBAAgB;YA8ChB,eAAe;YAkBf,SAAS;YAgDT,aAAa;IAa3B,OAAO,CAAC,eAAe;IAwBvB,OAAO,CAAC,cAAc;IAQtB,OAAO,CAAC,uBAAuB;YAQjB,cAAc;YAad,oBAAoB;YAuBpB,gBAAgB;YAuBhB,yBAAyB;CAmB1C"}
+{"version":3,"file":"ChronicleClient.d.ts","sourceRoot":"","sources":["../ChronicleClient.ts"],"names":[],"mappings":"AAMA,OAAO,EAAE,gBAAgB,EAAE,MAAM,oBAAoB,CAAC;AAKtD,OAAO,EAAE,cAAc,EAAE,MAAM,kBAAkB,CAAC;AAClD,OAAO,EAAE,uBAAuB,EAAE,MAAM,2BAA2B,CAAC;AACpE,OAAO,EAAE,gBAAgB,EAAE,MAAM,oBAAoB,CAAC;AACtD,OAAO,EAAE,WAAW,EAAE,MAAM,eAAe,CAAC;AAK5C;;;;;;;;;;;;GAYG;AACH,qBAAa,eAAgB,YAAW,gBAAgB;IAwBxC,QAAQ,CAAC,OAAO,EAAE,gBAAgB;IAvB9C,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,sBAAsB,CAAQ;IACtD,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,aAAa,CAAU;IAE/C,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAsB;IAClD,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAsC;IAC9D,OAAO,CAAC,QAAQ,CAAC,UAAU,CAA6B;IAExD,OAAO,CAAC,QAAQ,CAAC,OAAO,CAErB;IAEH,OAAO,CAAC,eAAe,CAAC,CAAiC;IACzD,OAAO,CAAC,iBAAiB,CAAC,CAAgB;IAC1C,OAAO,CAAC,mBAAmB,CAAC,CAAgB;IAC5C,OAAO,CAAC,mBAAmB,CAAC,CAAgB;IAC5C,OAAO,CAAC,WAAW,CAAS;IAC5B,OAAO,CAAC,yBAAyB,CAAC,CAAkB;IACpD,OAAO,CAAC,oBAAoB,CAAS;IAErC;;;OAGG;gBACkB,OAAO,EAAE,gBAAgB;IAgD9C,kBAAkB;IACZ,aAAa,CAAC,IAAI,EAAE,MAAM,GAAG,cAAc,EAAE,SAAS,CAAC,EAAE,MAAM,GAAG,uBAAuB,GAAG,OAAO,CAAC,WAAW,CAAC;IA0DtH,kBAAkB;IACZ,cAAc,IAAI,OAAO,CAAC,cAAc,EAAE,CAAC;IA0BjD,kBAAkB;IAClB,OAAO,IAAI,IAAI;YAsBD,gBAAgB;YA0ChB,eAAe;YAkBf,SAAS;IA0CvB;;;;OAIG;YACW,OAAO;YAcP,aAAa;IAa3B,OAAO,CAAC,eAAe;IAwBvB,OAAO,CAAC,cAAc;IAQtB,OAAO,CAAC,uBAAuB;YAQjB,cAAc;YAkBd,oBAAoB;YAqCpB,yBAAyB;CAmB1C"}

package/dist/ChronicleClient.js

@@ -1,9 +1,11 @@
 // Copyright (c) Cratis. All rights reserved.
 // Licensed under the MIT license. See LICENSE file in the project root for full license information.
+import * as os from 'os';
 import { diag } from '@opentelemetry/api';
 import { SpanStatusCode } from '@opentelemetry/api';
 import { ChronicleConnection } from './connection';
 import { ConnectionLifecycle } from './connection/ConnectionLifecycle';
+import { KernelKeepAlive } from './connection/KernelKeepAlive';
 import { EventStore } from './EventStore';
 import { EventStoreName } from './EventStoreName';
 import { EventStoreNamespaceName } from './EventStoreNamespaceName';
@@ -26,6 +28,7 @@ import { TypeDiscoverer } from './types';
 export class ChronicleClient {
     options;
     static _healthCheckIntervalMs = 5000;
+    static _maxBackoffMs = 30_000;
     _connection;
     _stores = new Map();
     _lifecycle = new ConnectionLifecycle();
@@ -38,6 +41,7 @@ export class ChronicleClient {
     _discoveryOperation;
     _isDisposed = false;
     _keepAliveAbortController;
+    _healthCheckInFlight = false;
     /**
      * Creates a new {@link ChronicleClient} using the provided options.
      * @param options - The options to configure the client, including the connection string.
@@ -188,15 +192,16 @@ export class ChronicleClient {
         let attempt = 0;
         while (!this._isDisposed) {
             try {
-                if (attempt > 0) {
-                    // Recreate the gRPC channel so we start from IDLE. A failed
-                    // probe can leave the channel in TRANSIENT_FAILURE, which gRPC
-                    // won't recover without a fresh channel. The contracts connect()
-                    // is also bypassed here — it uses watchConnectivityState and
-                    // rejects as soon as the state changes to CONNECTING (not READY),
-                    // making it unreliable for initial connection establishment.
-                    this._connection.resetChannel();
-                }
+                // Resolve (DNS SRV, when applicable) and select (load balancer strategy) a
+                // server address and rebuild the gRPC channel from scratch on every attempt,
+                // including the first — not just once at startup — so membership and load
+                // changes are always picked up. This also guarantees a fresh IDLE channel: a
+                // failed probe can leave a channel in TRANSIENT_FAILURE, which gRPC won't
+                // recover from without a new channel. The contracts connect() is bypassed
+                // here — it uses watchConnectivityState and rejects as soon as the state
+                // changes to CONNECTING (not READY), making it unreliable for initial
+                // connection establishment.
+                await this._connection.resetChannel();
                 this._logger.debug('Connecting to Chronicle kernel', { attempt: attempt + 1 });
                 // Probe with a real RPC call. gRPC connects lazily on the first call,
                 // so this effectively waits until the channel reaches READY or fails.
@@ -212,13 +217,8 @@ export class ChronicleClient {
             }
             catch (error) {
                 attempt++;
-                const delayMs = Math.min(1000 * Math.pow(2, attempt - 1), 30_000);
-                this._logger.warn('Connection attempt failed, retrying', {
-                    attempt,
-                    delayMs,
-                    error: this.toErrorMessage(error)
-                });
-                await new Promise(resolve => setTimeout(resolve, delayMs));
+                const delayMs = await this.backOff(attempt, 'Connection attempt failed, retrying', error);
+                this._logger.verbose('Backed off before next connection attempt', { attempt, delayMs });
             }
         }
         throw new Error('ChronicleClient was disposed during connection attempt.');
@@ -254,7 +254,7 @@ export class ChronicleClient {
                 let attempt = 0;
                 while (!this._isDisposed) {
                     try {
-                        this._connection.resetChannel();
+                        await this._connection.resetChannel();
                         await this._connection.server.getVersionInfo({}, { signal: AbortSignal.timeout(10_000) });
                         this._logger.info('Reconnected to Chronicle kernel', { attempt: attempt + 1 });
                         await this.startKernelKeepAlive();
@@ -267,13 +267,7 @@ export class ChronicleClient {
                     }
                     catch (reconnectError) {
                         attempt++;
-                        const delayMs = Math.min(1000 * Math.pow(2, attempt - 1), 30_000);
-                        this._logger.warn('Reconnect attempt failed, retrying', {
-                            attempt,
-                            delayMs,
-                            error: this.toErrorMessage(reconnectError)
-                        });
-                        await new Promise(resolve => setTimeout(resolve, delayMs));
+                        await this.backOff(attempt, 'Reconnect attempt failed, retrying', reconnectError);
                     }
                 }
             })().finally(() => {
@@ -282,6 +276,22 @@ export class ChronicleClient {
         }
         await this._reconnectOperation;
     }
+    /**
+     * Waits out the exponential backoff for a failed attempt and reports why.
+     * The delay is jittered so a fleet of clients that lost the same kernel does
+     * not come back in lockstep and knock it over again.
+     */
+    async backOff(attempt, message, error) {
+        const ceiling = Math.min(1000 * Math.pow(2, attempt - 1), ChronicleClient._maxBackoffMs);
+        const delayMs = Math.round(ceiling / 2 + Math.random() * (ceiling / 2));
+        this._logger.warn(message, {
+            attempt,
+            delayMs,
+            error: this.toErrorMessage(error)
+        });
+        await new Promise(resolve => setTimeout(resolve, delayMs));
+        return delayMs;
+    }
     async withReconnect(operation, action) {
         try {
             return await action();
@@ -327,54 +337,51 @@ export class ChronicleClient {
         this._watchdogHandle.unref?.();
     }
     async runHealthCheck() {
-        if (this._isDisposed || !this._lifecycle.isConnected) {
+        // A black-holed connection makes the probe itself hang, so without the
+        // in-flight guard every tick would stack another one that never returns.
+        if (this._isDisposed || !this._lifecycle.isConnected || this._healthCheckInFlight) {
             return;
         }
+        this._healthCheckInFlight = true;
         try {
-            await this._connection.server.getVersionInfo({});
+            await this._connection.server.getVersionInfo({}, { signal: AbortSignal.timeout(10_000) });
             this._logger.verbose('Connection health check passed');
         }
         catch (error) {
             await this.reconnect('watchdog-health-check', error);
         }
+        finally {
+            this._healthCheckInFlight = false;
+        }
     }
     async startKernelKeepAlive() {
         this._keepAliveAbortController?.abort();
         this._keepAliveAbortController = new AbortController();
         const { signal } = this._keepAliveAbortController;
-        const keepAliveStream = this._connection.connections.connect({ ConnectionId: this._lifecycle.connectionId, ClientVersion: '1.0.0', IsRunningWithDebugger: false }, { signal });
-        const iterator = keepAliveStream[Symbol.asyncIterator]();
-        const firstResult = await iterator.next();
-        if (firstResult.done) {
-            throw new Error('Connection service stream ended before sending first keep-alive');
-        }
-        await this._connection.connections.connectionKeepAlive(firstResult.value);
+        // Losing the keep-alive means the kernel has stopped counting us as
+        // connected, so observers are already being torn down server-side.
+        // Reconnecting is the only way back — without this the client sits on a
+        // dead session, appends keep working, and reactors never fire again.
+        const keepAlive = new KernelKeepAlive(this._connection.connections, (reason, error) => {
+            if (this._isDisposed || signal.aborted) {
+                return;
+            }
+            void this.reconnect(reason, error);
+        });
+        await keepAlive.start({
+            ConnectionId: this._lifecycle.connectionId,
+            // TODO: Not derived from this package's own version anywhere yet; kept as
+            // the pre-existing hardcoded placeholder until such a mechanism exists.
+            ClientVersion: '1.0.0',
+            IsRunningWithDebugger: false,
+            ProcessId: process.pid,
+            ProcessPath: process.execPath,
+            MachineName: os.hostname(),
+            ClientType: 'TypeScript'
+        }, signal);
         this._logger.info('Client registered with kernel keep-alive mechanism', {
             connectionId: this._lifecycle.connectionId
         });
-        void this.runKeepAliveLoop(iterator, signal);
-    }
-    async runKeepAliveLoop(iterator, signal) {
-        try {
-            while (!signal.aborted) {
-                const result = await iterator.next();
-                if (result.done) {
-                    this._logger.info('Keep-alive stream ended');
-                    break;
-                }
-                await this._connection.connections.connectionKeepAlive(result.value);
-                this._logger.verbose('Keep-alive ping responded', {
-                    connectionId: this._lifecycle.connectionId
-                });
-            }
-        }
-        catch (err) {
-            if (!signal.aborted) {
-                this._logger.warn('Keep-alive loop ended with error', {
-                    error: this.toErrorMessage(err)
-                });
-            }
-        }
     }
     async registerArtifactsForStore(store, reason) {
         this._logger.debug('Registering artifacts for event store', {