@cratis/chronicle 5.1.0 → 6.1.0

package/ChronicleClient.ts

@@ -6,6 +6,7 @@ import { diag } from '@opentelemetry/api';
 import { SpanStatusCode } from '@opentelemetry/api';
 import { ChronicleOptions } from './ChronicleOptions';
 import { ChronicleConnection } from './connection';
+import { IncompatibleChronicleServer } from './connection/IncompatibleChronicleServer';
 import { ensureCommandSuccess, ensureQuerySuccess } from './connection/callResults';
 import { ConnectionLifecycle } from './connection/ConnectionLifecycle';
 import { KernelKeepAlive } from './connection/KernelKeepAlive';
@@ -50,6 +51,7 @@ export class ChronicleClient implements IChronicleClient {
     private _isDisposed = false;
     private _keepAliveAbortController?: AbortController;
     private _healthCheckInFlight = false;
+    private _connectionFailure?: Error;
 
     /**
      * Creates a new {@link ChronicleClient} using the provided options.
@@ -219,14 +221,10 @@ export class ChronicleClient implements IChronicleClient {
             try {
                 // Resolve (DNS SRV, when applicable) and select (load balancer strategy) a
                 // server address and rebuild the gRPC channel from scratch on every attempt,
-                // including the first — not just once at startup — so membership and load
-                // changes are always picked up. This also guarantees a fresh IDLE channel: a
-                // failed probe can leave a channel in TRANSIENT_FAILURE, which gRPC won't
-                // recover from without a new channel. The contracts connect() is bypassed
-                // here — it uses watchConnectivityState and rejects as soon as the state
-                // changes to CONNECTING (not READY), making it unreliable for initial
-                // connection establishment.
+                // including the first — so membership and load changes are picked up. Verify
+                // the new channel's descriptor before registration or append can produce effects.
                 await this._connection.resetChannel();
+                await this._connection.connect();
 
                 this._logger.debug('Connecting to Chronicle kernel', { attempt: attempt + 1 });
 
@@ -245,6 +243,10 @@ export class ChronicleClient implements IChronicleClient {
 
 
             } catch (error) {
+                if (error instanceof IncompatibleChronicleServer) {
+                    this.failConnection(error);
+                    throw error;
+                }
                 attempt++;
                 const delayMs = await this.backOff(attempt, 'Connection attempt failed, retrying', error);
                 this._logger.verbose('Backed off before next connection attempt', { attempt, delayMs });
@@ -259,6 +261,11 @@ export class ChronicleClient implements IChronicleClient {
             throw new Error('ChronicleClient is disposed. Create a new client instance before making calls.');
         }
 
+        if (this._connectionFailure) throw this._connectionFailure;
+
+        // Callers arriving during recovery must share its verdict, not create a competing channel.
+        if (this._reconnectOperation) await this._reconnectOperation;
+
         if (this._lifecycle.isConnected) {
             return;
         }
@@ -273,6 +280,8 @@ export class ChronicleClient implements IChronicleClient {
     }
 
     private async reconnect(reason: string, error: unknown): Promise<void> {
+        if (this._connectionFailure) throw this._connectionFailure;
+
         if (!this._reconnectOperation) {
             this._reconnectOperation = (async () => {
                 this._logger.warn('Reconnecting to Chronicle kernel', {
@@ -292,6 +301,7 @@ export class ChronicleClient implements IChronicleClient {
                 while (!this._isDisposed) {
                     try {
                         await this._connection.resetChannel();
+                        await this._connection.connect();
                         await this._connection.server.getVersionInfo({}, { signal: AbortSignal.timeout(10_000) });
                         this._logger.info('Reconnected to Chronicle kernel', { attempt: attempt + 1 });
                         await this.startKernelKeepAlive();
@@ -302,6 +312,10 @@ export class ChronicleClient implements IChronicleClient {
                         });
                         return;
                     } catch (reconnectError) {
+                        if (reconnectError instanceof IncompatibleChronicleServer) {
+                            this.failConnection(reconnectError);
+                            throw reconnectError;
+                        }
                         attempt++;
                         await this.backOff(attempt, 'Reconnect attempt failed, retrying', reconnectError);
                     }
@@ -347,6 +361,7 @@ export class ChronicleClient implements IChronicleClient {
     }
 
     private shouldReconnect(error: unknown): boolean {
+        if (error instanceof IncompatibleChronicleServer) return false;
         const code = Number((error as { code?: number })?.code ?? -1);
         const details = String((error as { details?: string })?.details ?? '');
         const message = this.toErrorMessage(error);
@@ -380,7 +395,7 @@ export class ChronicleClient implements IChronicleClient {
 
     private startConnectionWatchdog(): void {
         this._watchdogHandle = setInterval(() => {
-            void this.runHealthCheck();
+            void this.runHealthCheck().catch(error => this.backgroundConnectionFailed('watchdog-health-check', error));
         }, ChronicleClient._healthCheckIntervalMs);
 
         this._watchdogHandle.unref?.();
@@ -418,7 +433,7 @@ export class ChronicleClient implements IChronicleClient {
                 return;
             }
 
-            void this.reconnect(reason, error);
+            void this.reconnect(reason, error).catch(failure => this.backgroundConnectionFailed(reason, failure));
         });
 
         await keepAlive.start(
@@ -441,6 +456,24 @@ export class ChronicleClient implements IChronicleClient {
         });
     }
 
+    private failConnection(error: Error): void {
+        this._connectionFailure = error;
+        this._keepAliveAbortController?.abort();
+        this._connection.disconnect();
+        if (this._watchdogHandle) {
+            clearInterval(this._watchdogHandle);
+            this._watchdogHandle = undefined;
+        }
+    }
+
+    private backgroundConnectionFailed(reason: string, error: unknown): void {
+        this.failConnection(error instanceof Error ? error : new Error(String(error)));
+        this._logger.error('Background connection recovery failed; create a new client after correcting the server', {
+            reason,
+            error: this.toErrorMessage(error)
+        });
+    }
+
     private async registerArtifactsForStore(store: EventStore, reason: string): Promise<void> {
         this._logger.debug('Registering artifacts for event store', {
             eventStore: store.name.value,

package/connection/ChronicleConnection.ts

@@ -26,7 +26,7 @@ import {
     type ConnectionServiceClient
 } from '@cratis/chronicle.contracts';
 import { ComplianceDefinition } from '../compliance/ComplianceContracts';
-import { createChannel, createClientFactory, waitForChannelReady } from 'nice-grpc';
+import { createChannel, createClientFactory } from 'nice-grpc';
 import type { ClientMiddleware } from 'nice-grpc-common';
 import { Metadata } from 'nice-grpc-common';
 import { EventStoreSubscriptionsDefinition } from '../eventStoreSubscriptions/contracts';
@@ -34,6 +34,7 @@ import { ExternalServicesDefinition } from '../externalServices/ExternalServices
 import { AuthenticationMode, ChronicleConnectionString } from './ChronicleConnectionString';
 import { ChronicleServerAddressResolver } from './ChronicleServerAddressResolver';
 import { ChronicleServices } from './ChronicleServices';
+import { CompatibilityPreflight } from './CompatibilityPreflight';
 import { formatServerAddress } from './formatServerAddress';
 import type { ILoadBalancerStrategy } from './ILoadBalancerStrategy';
 import { createLoadBalancerStrategy } from './LoadBalancerStrategyFactory';
@@ -91,6 +92,7 @@ export class ChronicleConnection implements ChronicleServices {
     private _channel!: Channel;
     private _services!: ChronicleServices;
     private _connections!: ConnectionServiceClient;
+    private _compatibility!: CompatibilityPreflight;
     private readonly _connectionString: ChronicleConnectionString;
     private readonly _tokenProvider: ITokenProvider;
     private readonly _addressResolver: ChronicleServerAddressResolver;
@@ -223,8 +225,8 @@ export class ChronicleConnection implements ChronicleServices {
 
     async connect(): Promise<void> {
         await this._clientsReady;
-        const deadline = new Date(Date.now() + (this._options.connectTimeout ?? 10_000));
-        await waitForChannelReady(this._channel, deadline);
+        // The bounded compatibility RPC establishes transport readiness and contract support together.
+        await this._compatibility.verify();
         this._isConnected = true;
     }
 
@@ -276,12 +278,15 @@ export class ChronicleConnection implements ChronicleServices {
         this._channel = createChannel(serverAddress, credentials, channelOptions);
 
         const factory = createClientFactory().use(this.createAuthMiddleware());
+        this._connections = factory.create(ConnectionServiceDefinition, this._channel);
+        this._compatibility = new CompatibilityPreflight(this._connections, this._options.connectTimeout ?? 10_000);
+        const eventSequenceFactory = factory.use(this._compatibility.middleware());
         this._services = {
             eventStores: factory.create(EventStoresDefinition, this._channel),
             namespaces: factory.create(NamespacesDefinition, this._channel),
             recommendations: factory.create(RecommendationsDefinition, this._channel),
             identities: factory.create(IdentitiesDefinition, this._channel),
-            eventSequences: factory.create(EventSequencesDefinition, this._channel),
+            eventSequences: eventSequenceFactory.create(EventSequencesDefinition, this._channel),
             eventTypes: factory.create(EventTypesDefinition, this._channel),
             constraints: factory.create(ConstraintsDefinition, this._channel),
             observers: factory.create(ObserversDefinition, this._channel),
@@ -300,7 +305,6 @@ export class ChronicleConnection implements ChronicleServices {
             compliance: factory.create(ComplianceDefinition as any, this._channel) as any,
             externalServices: factory.create(ExternalServicesDefinition, this._channel)
         };
-        this._connections = factory.create(ConnectionServiceDefinition, this._channel);
     }
 
     private createTokenProvider(): ITokenProvider {

package/connection/CompatibilityPreflight.ts

@@ -0,0 +1,62 @@
+// Copyright (c) Cratis. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+import { createRequire } from 'node:module';
+import { chronicleDescriptorSet, type ConnectionServiceClient } from '@cratis/chronicle.contracts';
+import { ClientError, Status, type ClientMiddleware } from 'nice-grpc-common';
+import { IncompatibleChronicleServer } from './IncompatibleChronicleServer';
+
+const require = createRequire(import.meta.url);
+const clientVersion = (require('@cratis/chronicle/package.json') as { version: string }).version;
+const protocolVersion = (require('@cratis/chronicle.contracts/package.json') as { version: string }).version;
+
+/** Verifies the installed wire contract before a channel can perform event-sequence operations. */
+export class CompatibilityPreflight {
+    private _verification?: Promise<void>;
+
+    constructor(private readonly _connections: ConnectionServiceClient, private readonly _timeout: number) {}
+
+    /** Shares in-flight checks and caches verdicts for this channel; transient failures can be retried. */
+    verify(): Promise<void> {
+        if (!this._verification) {
+            const verification = this.check().catch(error => {
+                if (!(error instanceof IncompatibleChronicleServer) && this._verification === verification) {
+                    this._verification = undefined;
+                }
+                throw error;
+            });
+            this._verification = verification;
+        }
+        return this._verification;
+    }
+
+    /** Gates even direct service calls that did not first call connect(). */
+    middleware(): ClientMiddleware {
+        const preflight = this;
+        return async function* compatibilityMiddleware(call, options) {
+            await preflight.verify();
+            return yield* call.next(call.request, options);
+        };
+    }
+
+    private async check(): Promise<void> {
+        if (!chronicleDescriptorSet?.length) {
+            throw new IncompatibleChronicleServer('The installed Chronicle contracts have no descriptor set; compatibility cannot be verified.');
+        }
+        const response = await this._connections.checkCompatibility({
+            ClientType: 'TypeScript',
+            ClientVersion: clientVersion,
+            ProtocolVersion: protocolVersion,
+            DescriptorSet: Buffer.from(chronicleDescriptorSet)
+        }, { signal: AbortSignal.timeout(this._timeout) }).catch(error => {
+            if (error instanceof ClientError && error.code === Status.UNIMPLEMENTED) {
+                throw new IncompatibleChronicleServer('The Chronicle server does not support contract compatibility checks; upgrade the kernel before connecting this client.');
+            }
+            throw error;
+        });
+        if (response?.IsCompatible !== true || !Array.isArray(response.Incompatibilities) || response.Incompatibilities.length !== 0) {
+            const reasons = response?.Incompatibilities?.join('; ') || 'missing, negative, or inconsistent compatibility verdict';
+            throw new IncompatibleChronicleServer(`Chronicle server ${response?.ServerVersion ?? 'unknown'} is incompatible with contracts ${protocolVersion}: ${reasons}`);
+        }
+    }
+}

package/connection/IncompatibleChronicleServer.ts

@@ -0,0 +1,10 @@
+// Copyright (c) Cratis. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+/** A permanent contract mismatch; reconnect backoff cannot make this operation safe. */
+export class IncompatibleChronicleServer extends Error {
+    constructor(message: string) {
+        super(message);
+        this.name = 'IncompatibleChronicleServer';
+    }
+}

package/connection/callResults.spec.ts

@@ -23,6 +23,12 @@ describe('callResults', () => {
         });
     });
 
+    describe('when the server explicitly denies authorization without a message', () => {
+        it('should report failure', () => {
+            expect(isCallSuccess({ ...successfulResult, IsAuthorized: false })).toBe(false);
+        });
+    });
+
     describe('when checking a result with validation results', () => {
         it('should report failure', () => {
             expect(isCallSuccess({ ...successfulResult, ValidationResults: [{ Message: 'invalid', Members: [] }] })).toBe(false);

package/connection/callResults.ts

@@ -12,14 +12,12 @@ export interface CallValidationResult {
 /**
  * Structural shape shared by every Chronicle command and query result envelope.
  *
- * The envelopes also carry an `IsAuthorized` boolean, but it is deliberately not part of
- * this shape: the kernel declares it with a protobuf-net default of `true`, which means
- * the field is omitted from the wire whenever it is true. A proto3 client decodes that
- * absence as `false`, so the boolean always reads as `false` and carries no signal.
- * Authorization failures are reported through `AuthorizationFailureReason` and gRPC
- * status codes instead.
+ * Kernels from 18.4.1 transmit authorization verdicts explicitly, including true values.
+ * Older kernels omitted true values; the connection compatibility check must reject those kernels
+ * before operations rather than ignoring an authorization failure.
  */
 export interface CallResultLike {
+    IsAuthorized?: boolean;
     ValidationResults: CallValidationResult[];
     ExceptionMessages: string[];
     AuthorizationFailureReason?: string;
@@ -72,7 +70,7 @@ export class ChronicleCallFailed extends Error {
  * @returns True when the call succeeded, false otherwise.
  */
 export function isCallSuccess(result: CallResultLike): boolean {
-    return !result.AuthorizationFailureReason &&
+    return result.IsAuthorized !== false && !result.AuthorizationFailureReason &&
         (result.ValidationResults ?? []).length === 0 &&
         (result.ExceptionMessages ?? []).length === 0;
 }

package/connection/for_ChronicleConnection/when_checking_compatibility/before_append.spec.ts

@@ -0,0 +1,148 @@
+// Copyright (c) Cratis. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+import { createRequire } from 'node:module';
+import { afterEach, beforeEach, chai, describe, it, vi } from 'vitest';
+import { createServer, ServerError, Status, type CallContext, type ServiceImplementation } from 'nice-grpc';
+import { chronicleDescriptorSet, ConnectionServiceDefinition, EventSequencesDefinition, type CompatibilityRequest } from '@cratis/chronicle.contracts';
+import { ChronicleConnection } from '../../ChronicleConnection';
+import { ChronicleClient } from '../../../ChronicleClient';
+import { ChronicleOptions } from '../../../ChronicleOptions';
+import { IncompatibleChronicleServer } from '../../IncompatibleChronicleServer';
+
+chai.should();
+const installedContractsVersion = (createRequire(import.meta.url)('@cratis/chronicle.contracts/package.json') as { version: string }).version;
+
+describe('when checking server compatibility before append', () => {
+    let server: ReturnType<typeof createServer>;
+    let connection: ChronicleConnection;
+    let compatible: boolean;
+    let address: string;
+    let check: ReturnType<typeof vi.fn>;
+    let append: ReturnType<typeof vi.fn>;
+    let appendMany: ReturnType<typeof vi.fn>;
+    beforeEach(async () => {
+        compatible = true;
+        server = createServer();
+        check = vi.fn(async (_request: CompatibilityRequest, _context: CallContext) => ({
+            IsCompatible: compatible, Incompatibilities: compatible ? [] : ['missing field'], ServerVersion: 'server'
+        }));
+        append = vi.fn(async () => ({ Response: { SequenceNumber: 0n } }));
+        appendMany = vi.fn(async () => ({ Response: { SequenceNumbers: [0n] } }));
+        const unsupported = async () => { throw new ServerError(Status.UNIMPLEMENTED, 'Unexpected RPC in specification'); };
+        const connectionMethods = Object.fromEntries(Object.keys(ConnectionServiceDefinition.methods).map(name => [name, unsupported]));
+        const eventSequenceMethods = Object.fromEntries(Object.keys(EventSequencesDefinition.methods).map(name => [name, unsupported]));
+        server.add(ConnectionServiceDefinition, { ...connectionMethods, checkCompatibility: check } as ServiceImplementation<typeof ConnectionServiceDefinition>);
+        server.add(EventSequencesDefinition, { ...eventSequenceMethods, append, appendManyForEventSources: appendMany } as ServiceImplementation<typeof EventSequencesDefinition>);
+        const port = await server.listen('127.0.0.1:0');
+        address = `chronicle://127.0.0.1:${port}/?disableTls=true&apiKey=test-key`;
+        connection = new ChronicleConnection({ connectionString: address });
+    });
+    afterEach(async () => {
+        connection?.dispose();
+        await server.shutdown();
+    });
+    it('should send the installed descriptor and share one check across concurrent writes', async () => {
+        await connection.connect();
+        await Promise.all([connection.eventSequences.append({}), connection.eventSequences.appendManyForEventSources({})]);
+        check.mock.calls.should.have.lengthOf(1);
+        check.mock.calls[0][0].DescriptorSet.should.deep.equal(Buffer.from(chronicleDescriptorSet));
+        check.mock.calls[0][0].ProtocolVersion.should.equal(installedContractsVersion);
+        append.mock.calls.should.have.lengthOf(1);
+        appendMany.mock.calls.should.have.lengthOf(1);
+    });
+    it('should reject both append paths before writing even without connect', async () => {
+        compatible = false;
+        await connection.resetChannel();
+        const results = await Promise.allSettled([connection.eventSequences.append({}), connection.eventSequences.appendManyForEventSources({})]);
+        results.map(result => result.status).should.deep.equal(['rejected', 'rejected']);
+        check.mock.calls.should.have.lengthOf(1);
+        append.mock.calls.should.have.lengthOf(0);
+        appendMany.mock.calls.should.have.lengthOf(0);
+        connection.isConnected.should.equal(false);
+    });
+    it('should reject an unavailable compatibility endpoint without trying the append', async () => {
+        check.mockRejectedValue(new ServerError(Status.UNAVAILABLE, 'compatibility endpoint unavailable'));
+        await connection.resetChannel();
+        const results = await Promise.allSettled([connection.eventSequences.append({})]);
+        results[0].status.should.equal('rejected');
+        append.mock.calls.should.have.lengthOf(0);
+    });
+    it('should retry a transient failed check before allowing a later write', async () => {
+        check.mockRejectedValueOnce(new ServerError(Status.UNAVAILABLE, 'temporarily unavailable'));
+        await connection.resetChannel();
+        const failed = await Promise.allSettled([connection.eventSequences.append({})]);
+        failed[0].status.should.equal('rejected');
+        append.mock.calls.should.have.lengthOf(0);
+        await connection.eventSequences.append({});
+        check.mock.calls.should.have.lengthOf(2);
+        append.mock.calls.should.have.lengthOf(1);
+    });
+    it('should fail a high-level client connection instead of retrying an incompatible server forever', async () => {
+        compatible = false;
+        const client = new ChronicleClient(ChronicleOptions.fromConnectionString(address, { discoveryPatterns: [] }));
+        try {
+            const results = await Promise.allSettled([client.getEventStore('store')]);
+            results[0].status.should.equal('rejected');
+            if (results[0].status === 'rejected') results[0].reason.should.be.instanceOf(IncompatibleChronicleServer);
+            check.mock.calls.should.have.lengthOf(1);
+            append.mock.calls.should.have.lengthOf(0);
+        } finally {
+            client.dispose();
+        }
+    });
+    it('should reject a missing compatibility RPC permanently rather than reconnect forever', async () => {
+        check.mockRejectedValue(new ServerError(Status.UNIMPLEMENTED, 'compatibility not supported'));
+        const client = new ChronicleClient(ChronicleOptions.fromConnectionString(address, { discoveryPatterns: [] }));
+        try {
+            const results = await Promise.allSettled([client.getEventStore('store')]);
+            results[0].status.should.equal('rejected');
+            if (results[0].status === 'rejected') results[0].reason.should.be.instanceOf(IncompatibleChronicleServer);
+            check.mock.calls.should.have.lengthOf(1);
+            append.mock.calls.should.have.lengthOf(0);
+        } finally {
+            client.dispose();
+        }
+    });
+    it('should retain an incompatible verdict until the channel is replaced', async () => {
+        compatible = false;
+        await connection.resetChannel();
+        await Promise.allSettled([connection.eventSequences.append({})]);
+        const results = await Promise.allSettled([connection.eventSequences.append({})]);
+        results[0].status.should.equal('rejected');
+        check.mock.calls.should.have.lengthOf(1);
+        append.mock.calls.should.have.lengthOf(0);
+        compatible = true;
+        await connection.reconnect();
+        await connection.eventSequences.append({});
+        check.mock.calls.should.have.lengthOf(2);
+        append.mock.calls.should.have.lengthOf(1);
+    });
+    it('should bound a stalled compatibility check without allowing writes', async () => {
+        check.mockImplementation((_request, context) => new Promise((_resolve, reject) => {
+            context.signal.addEventListener('abort', () => reject(new ServerError(Status.CANCELLED, 'cancelled')), { once: true });
+        }));
+        connection.dispose();
+        connection = new ChronicleConnection({ connectionString: address, connectTimeout: 50 });
+        const results = await Promise.allSettled([connection.connect()]);
+        results[0].status.should.equal('rejected');
+        connection.isConnected.should.be.false;
+        append.mock.calls.should.have.lengthOf(0);
+    });
+    it('should reject a contradictory compatibility verdict', async () => {
+        check.mockResolvedValue({ IsCompatible: true, Incompatibilities: ['missing field'], ServerVersion: 'server' });
+        await connection.resetChannel();
+        const failed = await Promise.allSettled([connection.eventSequences.append({})]);
+        failed[0].status.should.equal('rejected');
+        append.mock.calls.should.have.lengthOf(0);
+    });
+    it('should recheck after reconnect and refuse a newly incompatible channel', async () => {
+        await connection.connect();
+        compatible = false;
+        const results = await Promise.allSettled([connection.reconnect()]);
+        results[0].status.should.equal('rejected');
+        check.mock.calls.should.have.lengthOf(2);
+        connection.isConnected.should.equal(false);
+        append.mock.calls.should.have.lengthOf(0);
+    });
+});

package/connection/for_CompatibilityPreflight/when_verifying/with_concurrent_calls.spec.ts

@@ -0,0 +1,38 @@
+// Copyright (c) Cratis. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+import { beforeEach, chai, describe, it, vi } from 'vitest';
+import { CompatibilityResponse, type ConnectionServiceClient } from '@cratis/chronicle.contracts';
+import { CompatibilityPreflight } from '../../CompatibilityPreflight';
+
+chai.should();
+
+describe('when verifying with concurrent calls', () => {
+    let check: ReturnType<typeof vi.fn>;
+    let preflight: CompatibilityPreflight;
+    let first: Promise<void>;
+    let second: Promise<void>;
+    let complete: (response: CompatibilityResponse) => void;
+
+    beforeEach(() => {
+        const response = new Promise<CompatibilityResponse>(resolve => { complete = resolve; });
+        check = vi.fn(() => response);
+        preflight = new CompatibilityPreflight({ checkCompatibility: check } as unknown as ConnectionServiceClient, 1000);
+        first = preflight.verify();
+        second = preflight.verify();
+    });
+
+    it('should share the same flight and issue exactly one RPC', async () => {
+        first.should.equal(second);
+        check.mock.calls.should.have.lengthOf(1);
+        complete(CompatibilityResponse.create({ IsCompatible: true }));
+        await Promise.all([first, second]);
+    });
+
+    it('should retain the successful verdict for later calls', async () => {
+        complete(CompatibilityResponse.create({ IsCompatible: true }));
+        await Promise.all([first, second]);
+        await preflight.verify();
+        check.mock.calls.should.have.lengthOf(1);
+    });
+});

package/dist/ChronicleClient.d.ts

@@ -31,6 +31,7 @@ export declare class ChronicleClient implements IChronicleClient {
     private _isDisposed;
     private _keepAliveAbortController?;
     private _healthCheckInFlight;
+    private _connectionFailure?;
     /**
      * Creates a new {@link ChronicleClient} using the provided options.
      * @param options - The options to configure the client, including the connection string.
@@ -57,6 +58,8 @@ export declare class ChronicleClient implements IChronicleClient {
     private startConnectionWatchdog;
     private runHealthCheck;
     private startKernelKeepAlive;
+    private failConnection;
+    private backgroundConnectionFailed;
     private registerArtifactsForStore;
 }
 //# sourceMappingURL=ChronicleClient.d.ts.map

package/dist/ChronicleClient.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"ChronicleClient.d.ts","sourceRoot":"","sources":["../ChronicleClient.ts"],"names":[],"mappings":"AAMA,OAAO,EAAE,gBAAgB,EAAE,MAAM,oBAAoB,CAAC;AAMtD,OAAO,EAAE,cAAc,EAAE,MAAM,kBAAkB,CAAC;AAClD,OAAO,EAAE,uBAAuB,EAAE,MAAM,2BAA2B,CAAC;AACpE,OAAO,EAAE,gBAAgB,EAAE,MAAM,oBAAoB,CAAC;AACtD,OAAO,EAAE,WAAW,EAAE,MAAM,eAAe,CAAC;AAK5C;;;;;;;;;;;;GAYG;AACH,qBAAa,eAAgB,YAAW,gBAAgB;IAwBxC,QAAQ,CAAC,OAAO,EAAE,gBAAgB;IAvB9C,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,sBAAsB,CAAQ;IACtD,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,aAAa,CAAU;IAE/C,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAsB;IAClD,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAsC;IAC9D,OAAO,CAAC,QAAQ,CAAC,UAAU,CAA6B;IAExD,OAAO,CAAC,QAAQ,CAAC,OAAO,CAErB;IAEH,OAAO,CAAC,eAAe,CAAC,CAAiC;IACzD,OAAO,CAAC,iBAAiB,CAAC,CAAgB;IAC1C,OAAO,CAAC,mBAAmB,CAAC,CAAgB;IAC5C,OAAO,CAAC,mBAAmB,CAAC,CAAgB;IAC5C,OAAO,CAAC,WAAW,CAAS;IAC5B,OAAO,CAAC,yBAAyB,CAAC,CAAkB;IACpD,OAAO,CAAC,oBAAoB,CAAS;IAErC;;;OAGG;gBACkB,OAAO,EAAE,gBAAgB;IAgD9C,kBAAkB;IACZ,aAAa,CAAC,IAAI,EAAE,MAAM,GAAG,cAAc,EAAE,SAAS,CAAC,EAAE,MAAM,GAAG,uBAAuB,GAAG,OAAO,CAAC,WAAW,CAAC;IA0DtH,kBAAkB;IACZ,cAAc,IAAI,OAAO,CAAC,cAAc,EAAE,CAAC;IA0BjD,kBAAkB;IAClB,OAAO,IAAI,IAAI;YAsBD,gBAAgB;YA0ChB,eAAe;YAkBf,SAAS;IA0CvB;;;;OAIG;YACW,OAAO;YAcP,aAAa;IAa3B,OAAO,CAAC,eAAe;IAwBvB,OAAO,CAAC,cAAc;IAQtB,OAAO,CAAC,uBAAuB;YAQjB,cAAc;YAkBd,oBAAoB;YAqCpB,yBAAyB;CAmB1C"}
+{"version":3,"file":"ChronicleClient.d.ts","sourceRoot":"","sources":["../ChronicleClient.ts"],"names":[],"mappings":"AAMA,OAAO,EAAE,gBAAgB,EAAE,MAAM,oBAAoB,CAAC;AAOtD,OAAO,EAAE,cAAc,EAAE,MAAM,kBAAkB,CAAC;AAClD,OAAO,EAAE,uBAAuB,EAAE,MAAM,2BAA2B,CAAC;AACpE,OAAO,EAAE,gBAAgB,EAAE,MAAM,oBAAoB,CAAC;AACtD,OAAO,EAAE,WAAW,EAAE,MAAM,eAAe,CAAC;AAK5C;;;;;;;;;;;;GAYG;AACH,qBAAa,eAAgB,YAAW,gBAAgB;IAyBxC,QAAQ,CAAC,OAAO,EAAE,gBAAgB;IAxB9C,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,sBAAsB,CAAQ;IACtD,OAAO,CAAC,MAAM,CAAC,QAAQ,CAAC,aAAa,CAAU;IAE/C,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAsB;IAClD,OAAO,CAAC,QAAQ,CAAC,OAAO,CAAsC;IAC9D,OAAO,CAAC,QAAQ,CAAC,UAAU,CAA6B;IAExD,OAAO,CAAC,QAAQ,CAAC,OAAO,CAErB;IAEH,OAAO,CAAC,eAAe,CAAC,CAAiC;IACzD,OAAO,CAAC,iBAAiB,CAAC,CAAgB;IAC1C,OAAO,CAAC,mBAAmB,CAAC,CAAgB;IAC5C,OAAO,CAAC,mBAAmB,CAAC,CAAgB;IAC5C,OAAO,CAAC,WAAW,CAAS;IAC5B,OAAO,CAAC,yBAAyB,CAAC,CAAkB;IACpD,OAAO,CAAC,oBAAoB,CAAS;IACrC,OAAO,CAAC,kBAAkB,CAAC,CAAQ;IAEnC;;;OAGG;gBACkB,OAAO,EAAE,gBAAgB;IAgD9C,kBAAkB;IACZ,aAAa,CAAC,IAAI,EAAE,MAAM,GAAG,cAAc,EAAE,SAAS,CAAC,EAAE,MAAM,GAAG,uBAAuB,GAAG,OAAO,CAAC,WAAW,CAAC;IA0DtH,kBAAkB;IACZ,cAAc,IAAI,OAAO,CAAC,cAAc,EAAE,CAAC;IA0BjD,kBAAkB;IAClB,OAAO,IAAI,IAAI;YAsBD,gBAAgB;YA0ChB,eAAe;YAuBf,SAAS;IAiDvB;;;;OAIG;YACW,OAAO;YAcP,aAAa;IAa3B,OAAO,CAAC,eAAe;IAyBvB,OAAO,CAAC,cAAc;IAQtB,OAAO,CAAC,uBAAuB;YAQjB,cAAc;YAkBd,oBAAoB;IAqClC,OAAO,CAAC,cAAc;IAUtB,OAAO,CAAC,0BAA0B;YAQpB,yBAAyB;CAmB1C"}