@cratis/chronicle 2.0.0 → 2.1.1

package/connection/LeastConnectionsLoadBalancerStrategy.ts

@@ -0,0 +1,117 @@
+// Copyright (c) Cratis. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+import { Agent } from 'undici';
+import type { ChronicleServerAddress } from './ChronicleConnectionString';
+import { formatServerAddress } from './formatServerAddress';
+import type { ILoadBalancerStrategy } from './ILoadBalancerStrategy';
+
+const DEFAULT_MAX_JITTER_MS = 250;
+const PROBE_TIMEOUT_MS = 2000;
+
+// Node's global fetch types its `dispatcher` option against `undici-types` (bundled with
+// @types/node), which is structurally close to but not identical to the real `undici`
+// package's own `Agent`/`Dispatcher` types - hence the assertion through `unknown` at every
+// use below.
+type FetchDispatcher = NonNullable<RequestInit['dispatcher']>;
+
+/**
+ * Selects the least-loaded candidate by probing each one's current connection count over
+ * HTTP and picking the minimum, breaking ties randomly. This is the default strategy.
+ *
+ * Before every probe attempt (not just the first), waits a random jitter to avoid a
+ * thundering herd of clients probing every candidate at the exact same instant. A single
+ * candidate is returned immediately without probing, since there is nothing to choose
+ * between.
+ */
+export class LeastConnectionsLoadBalancerStrategy implements ILoadBalancerStrategy {
+    private readonly _dispatcher: Agent;
+
+    /**
+     * Initializes a new instance of {@link LeastConnectionsLoadBalancerStrategy}.
+     * @param skipTlsValidation - Whether to skip TLS certificate validation on the
+     * `/connections/count` and `/connections/reserve` probe requests, matching the toggle
+     * used for the gRPC channel itself.
+     * @param maxJitterMs - The maximum jitter, in milliseconds, to wait before every probe
+     * attempt. Defaults to 250ms. 0 disables jitter entirely.
+     */
+    constructor(
+        private readonly _skipTlsValidation: boolean,
+        private readonly _maxJitterMs: number = DEFAULT_MAX_JITTER_MS
+    ) {
+        this._dispatcher = new Agent({ connect: { rejectUnauthorized: !this._skipTlsValidation } });
+    }
+
+    async select(candidates: ChronicleServerAddress[]): Promise<ChronicleServerAddress> {
+        if (candidates.length === 0) {
+            throw new Error('Cannot select a server address from an empty candidate list');
+        }
+
+        if (candidates.length === 1) {
+            return candidates[0];
+        }
+
+        await this.jitter();
+
+        const connectionCounts = await Promise.all(candidates.map(candidate => this.probeConnectionCount(candidate)));
+        const minimumCount = Math.min(...connectionCounts);
+        const minimumIndices = connectionCounts.reduce<number[]>((indices, count, index) => {
+            if (count === minimumCount) {
+                indices.push(index);
+            }
+            return indices;
+        }, []);
+
+        const selected = candidates[minimumIndices[Math.floor(Math.random() * minimumIndices.length)]];
+
+        await this.reserve(selected);
+
+        return selected;
+    }
+
+    private async jitter(): Promise<void> {
+        if (this._maxJitterMs <= 0) {
+            return;
+        }
+
+        const delayMs = Math.floor(Math.random() * this._maxJitterMs);
+        if (delayMs <= 0) {
+            return;
+        }
+
+        await new Promise(resolve => setTimeout(resolve, delayMs));
+    }
+
+    private async probeConnectionCount(candidate: ChronicleServerAddress): Promise<number> {
+        try {
+            const response = await fetch(`https://${formatServerAddress(candidate)}/connections/count`, {
+                signal: AbortSignal.timeout(PROBE_TIMEOUT_MS),
+                dispatcher: this._dispatcher as unknown as FetchDispatcher
+            });
+
+            if (!response.ok) {
+                return Number.MAX_SAFE_INTEGER;
+            }
+
+            const body = (await response.text()).trim();
+            const count = Number(body);
+            return body.length > 0 && Number.isFinite(count) ? count : Number.MAX_SAFE_INTEGER;
+        } catch {
+            // A candidate that cannot be reached or answers with garbage is never preferred
+            // over one that responds normally, but must not fail the whole selection round.
+            return Number.MAX_SAFE_INTEGER;
+        }
+    }
+
+    private async reserve(candidate: ChronicleServerAddress): Promise<void> {
+        try {
+            await fetch(`https://${formatServerAddress(candidate)}/connections/reserve`, {
+                method: 'POST',
+                signal: AbortSignal.timeout(PROBE_TIMEOUT_MS),
+                dispatcher: this._dispatcher as unknown as FetchDispatcher
+            });
+        } catch {
+            // Best-effort reservation; the selected server is still used even if this fails.
+        }
+    }
+}

package/connection/LoadBalancerMode.ts

@@ -0,0 +1,25 @@
+// Copyright (c) Cratis. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+/**
+ * The strategy used to select one server address from a multi-host connection string
+ * (or a resolved set of `chronicle+srv://` addresses) for each connect/reconnect attempt.
+ */
+export enum LoadBalancerMode {
+    /**
+     * Probes every candidate's current connection count and picks the least-loaded one,
+     * breaking ties randomly. The default strategy.
+     */
+    LeastConnections = 'least-connections',
+
+    /**
+     * Cycles through candidates in order, starting from a random offset chosen once per
+     * strategy instance.
+     */
+    RoundRobin = 'round-robin',
+
+    /**
+     * Picks a uniformly random candidate on every call.
+     */
+    Random = 'random'
+}

package/connection/LoadBalancerStrategyFactory.ts

@@ -0,0 +1,28 @@
+// Copyright (c) Cratis. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+import type { ILoadBalancerStrategy } from './ILoadBalancerStrategy';
+import { LeastConnectionsLoadBalancerStrategy } from './LeastConnectionsLoadBalancerStrategy';
+import { LoadBalancerMode } from './LoadBalancerMode';
+import { RandomLoadBalancerStrategy } from './RandomLoadBalancerStrategy';
+import { RoundRobinLoadBalancerStrategy } from './RoundRobinLoadBalancerStrategy';
+
+/**
+ * Creates the {@link ILoadBalancerStrategy} instance matching a {@link LoadBalancerMode}.
+ * @param mode - The load balancer mode to create a strategy for.
+ * @param skipTlsValidation - Whether to skip TLS certificate validation for strategies that
+ * probe candidates over HTTP (currently only {@link LoadBalancerMode.LeastConnections}).
+ * @returns The load balancer strategy instance.
+ */
+export function createLoadBalancerStrategy(mode: LoadBalancerMode, skipTlsValidation: boolean): ILoadBalancerStrategy {
+    switch (mode) {
+        case LoadBalancerMode.RoundRobin:
+            return new RoundRobinLoadBalancerStrategy();
+        case LoadBalancerMode.Random:
+            return new RandomLoadBalancerStrategy();
+        case LoadBalancerMode.LeastConnections:
+            return new LeastConnectionsLoadBalancerStrategy(skipTlsValidation);
+        default:
+            throw new Error(`Unknown load balancer mode: ${mode}`);
+    }
+}

package/connection/RandomLoadBalancerStrategy.spec.ts

@@ -0,0 +1,50 @@
+// Copyright (c) Cratis. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+import { describe, expect, it } from 'vitest';
+import type { ChronicleServerAddress } from './ChronicleConnectionString';
+import { RandomLoadBalancerStrategy } from './RandomLoadBalancerStrategy';
+
+describe('RandomLoadBalancerStrategy', () => {
+    describe('when selecting from an empty candidate list', () => {
+        const strategy = new RandomLoadBalancerStrategy();
+
+        it('should throw', async () => {
+            await expect(strategy.select([])).rejects.toThrow();
+        });
+    });
+
+    describe('when selecting from a single candidate', () => {
+        const strategy = new RandomLoadBalancerStrategy();
+        const candidate: ChronicleServerAddress = { host: 'host1', port: 35000 };
+
+        it('should return that candidate', async () => {
+            expect(await strategy.select([candidate])).toEqual(candidate);
+        });
+    });
+
+    describe('when selecting from many candidates repeatedly', () => {
+        const strategy = new RandomLoadBalancerStrategy();
+        const candidates: ChronicleServerAddress[] = [
+            { host: 'host1', port: 35000 },
+            { host: 'host2', port: 35000 },
+            { host: 'host3', port: 35000 }
+        ];
+
+        it('should always return one of the candidates', async () => {
+            for (let attempt = 0; attempt < 25; attempt++) {
+                const selected = await strategy.select(candidates);
+                expect(candidates).toContainEqual(selected);
+            }
+        });
+
+        it('should eventually select more than one distinct candidate', async () => {
+            const selections = new Set<string>();
+            for (let attempt = 0; attempt < 50; attempt++) {
+                const selected = await strategy.select(candidates);
+                selections.add(`${selected.host}:${selected.port}`);
+            }
+            expect(selections.size).toBeGreaterThan(1);
+        });
+    });
+});

package/connection/RandomLoadBalancerStrategy.ts

@@ -0,0 +1,19 @@
+// Copyright (c) Cratis. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+import type { ChronicleServerAddress } from './ChronicleConnectionString';
+import type { ILoadBalancerStrategy } from './ILoadBalancerStrategy';
+
+/**
+ * Selects a uniformly random candidate on every call.
+ */
+export class RandomLoadBalancerStrategy implements ILoadBalancerStrategy {
+    async select(candidates: ChronicleServerAddress[]): Promise<ChronicleServerAddress> {
+        if (candidates.length === 0) {
+            throw new Error('Cannot select a server address from an empty candidate list');
+        }
+
+        const index = Math.floor(Math.random() * candidates.length);
+        return candidates[index];
+    }
+}

package/connection/RoundRobinLoadBalancerStrategy.spec.ts

@@ -0,0 +1,61 @@
+// Copyright (c) Cratis. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+import { describe, expect, it } from 'vitest';
+import type { ChronicleServerAddress } from './ChronicleConnectionString';
+import { RoundRobinLoadBalancerStrategy } from './RoundRobinLoadBalancerStrategy';
+
+describe('RoundRobinLoadBalancerStrategy', () => {
+    describe('when selecting from an empty candidate list', () => {
+        const strategy = new RoundRobinLoadBalancerStrategy();
+
+        it('should throw', async () => {
+            await expect(strategy.select([])).rejects.toThrow();
+        });
+    });
+
+    describe('when selecting repeatedly from the same strategy instance', () => {
+        const strategy = new RoundRobinLoadBalancerStrategy();
+        const candidates: ChronicleServerAddress[] = [
+            { host: 'host1', port: 35000 },
+            { host: 'host2', port: 35000 },
+            { host: 'host3', port: 35000 }
+        ];
+
+        it('should cycle through every candidate exactly once per full cycle', async () => {
+            const first = await strategy.select(candidates);
+            const second = await strategy.select(candidates);
+            const third = await strategy.select(candidates);
+            const fourth = await strategy.select(candidates);
+
+            const firstCycle = [first, second, third];
+            expect(firstCycle).toContainEqual(candidates[0]);
+            expect(firstCycle).toContainEqual(candidates[1]);
+            expect(firstCycle).toContainEqual(candidates[2]);
+
+            // The cycle wraps back to whichever candidate started it.
+            expect(fourth).toEqual(first);
+        });
+    });
+
+    describe('when two independent strategy instances select from the same candidates', () => {
+        const candidates: ChronicleServerAddress[] = [
+            { host: 'host1', port: 35000 },
+            { host: 'host2', port: 35000 },
+            { host: 'host3', port: 35000 }
+        ];
+
+        it('should not always start at the same candidate', async () => {
+            const startingPoints = new Set<string>();
+            for (let attempt = 0; attempt < 25; attempt++) {
+                const strategy = new RoundRobinLoadBalancerStrategy();
+                const selected = await strategy.select(candidates);
+                startingPoints.add(`${selected.host}:${selected.port}`);
+            }
+
+            // With a random starting offset across many fresh instances, at least one run
+            // should land somewhere other than the first candidate.
+            expect(startingPoints.size).toBeGreaterThan(1);
+        });
+    });
+});

package/connection/RoundRobinLoadBalancerStrategy.ts

@@ -0,0 +1,29 @@
+// Copyright (c) Cratis. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+import type { ChronicleServerAddress } from './ChronicleConnectionString';
+import type { ILoadBalancerStrategy } from './ILoadBalancerStrategy';
+
+/**
+ * Cycles through candidates in order, starting from a random offset chosen once per
+ * strategy instance so that multiple clients don't all start at the same candidate.
+ */
+export class RoundRobinLoadBalancerStrategy implements ILoadBalancerStrategy {
+    private _startOffset?: number;
+    private _callCount = 0;
+
+    async select(candidates: ChronicleServerAddress[]): Promise<ChronicleServerAddress> {
+        if (candidates.length === 0) {
+            throw new Error('Cannot select a server address from an empty candidate list');
+        }
+
+        if (this._startOffset === undefined) {
+            this._startOffset = Math.floor(Math.random() * candidates.length);
+        }
+
+        const index = (this._startOffset + this._callCount) % candidates.length;
+        this._callCount++;
+
+        return candidates[index];
+    }
+}

package/connection/TokenProvider.spec.ts

@@ -0,0 +1,159 @@
+// Copyright (c) Cratis. All rights reserved.
+// Licensed under the MIT license. See LICENSE file in the project root for full license information.
+
+import { afterEach, describe, expect, it, vi } from 'vitest';
+import type { OAuthTokenResponse } from './fetchOAuthAccessToken';
+import { OAuthTokenProvider } from './TokenProvider';
+
+// Long enough to stay outside the 60s refresh margin for the whole spec.
+const longLifetime = 3600;
+// Short enough to be inside the refresh margin immediately.
+const shortLifetime = 30;
+
+const token = (value: string, expiresIn?: number | string): OAuthTokenResponse =>
+    ({ access_token: value, ...(expiresIn === undefined ? {} : { expires_in: expiresIn }) });
+
+/**
+ * Creates a provider whose token fetches are served from the given script of
+ * responses — the last one repeats for any further fetches.
+ */
+function createProvider(responses: Array<OAuthTokenResponse | Error>) {
+    let call = 0;
+    const fetchToken = vi.fn(() => {
+        const response = responses[Math.min(call++, responses.length - 1)];
+        return response instanceof Error ? Promise.reject(response) : Promise.resolve(response);
+    });
+
+    const provider = new OAuthTokenProvider('https://localhost:35000/connect/token', 'client', 'secret', true, fetchToken);
+    return { provider, fetchToken };
+}
+
+describe('OAuthTokenProvider', () => {
+    afterEach(() => {
+        vi.useRealTimers();
+    });
+
+    describe('when requesting the first token', () => {
+        it('should fetch it lazily and return it', async () => {
+            const { provider, fetchToken } = createProvider([token('token-1', longLifetime)]);
+
+            expect(await provider.getAccessToken()).toBe('token-1');
+            expect(fetchToken).toHaveBeenCalledTimes(1);
+        });
+    });
+
+    describe('when the cached token is fresh', () => {
+        it('should serve it without fetching again', async () => {
+            const { provider, fetchToken } = createProvider([token('token-1', longLifetime), token('token-2', longLifetime)]);
+
+            expect(await provider.getAccessToken()).toBe('token-1');
+            expect(await provider.getAccessToken()).toBe('token-1');
+            expect(fetchToken).toHaveBeenCalledTimes(1);
+        });
+    });
+
+    describe('when the token response has no expires_in', () => {
+        it('should assume the default lifetime and cache the token', async () => {
+            const { provider, fetchToken } = createProvider([token('token-1')]);
+
+            expect(await provider.getAccessToken()).toBe('token-1');
+            expect(await provider.getAccessToken()).toBe('token-1');
+            expect(fetchToken).toHaveBeenCalledTimes(1);
+        });
+    });
+
+    describe('when the token response sends expires_in as a string', () => {
+        it('should parse it and cache the token', async () => {
+            const { provider, fetchToken } = createProvider([token('token-1', '3600')]);
+
+            expect(await provider.getAccessToken()).toBe('token-1');
+            expect(await provider.getAccessToken()).toBe('token-1');
+            expect(fetchToken).toHaveBeenCalledTimes(1);
+        });
+    });
+
+    describe('when the token enters the refresh margin', () => {
+        it('should refresh ahead of expiry', async () => {
+            const { provider, fetchToken } = createProvider([token('token-1', shortLifetime), token('token-2', shortLifetime)]);
+
+            expect(await provider.getAccessToken()).toBe('token-1');
+
+            // The short lifetime is already inside the margin, so the next request
+            // refreshes even though the first token has not expired yet.
+            expect(await provider.getAccessToken()).toBe('token-2');
+            expect(fetchToken).toHaveBeenCalledTimes(2);
+        });
+    });
+
+    describe('when a refresh fails while the cached token is still valid', () => {
+        it('should keep serving the cached token', async () => {
+            const { provider } = createProvider([token('token-1', shortLifetime), new Error('unavailable')]);
+
+            expect(await provider.getAccessToken()).toBe('token-1');
+
+            // Refresh is due (inside the margin) and fails — the token is still valid
+            // for another 30s, so it must keep flowing rather than dropping auth.
+            expect(await provider.getAccessToken()).toBe('token-1');
+        });
+    });
+
+    describe('when no token can be fetched', () => {
+        it('should return undefined instead of rejecting', async () => {
+            const { provider } = createProvider([new Error('unavailable')]);
+
+            // The RPC proceeds and fails with the server's auth rejection — that is
+            // the session machinery's problem, not the token provider's.
+            expect(await provider.getAccessToken()).toBeUndefined();
+        });
+
+        it('should throttle further fetch attempts', async () => {
+            const { provider, fetchToken } = createProvider([new Error('unavailable')]);
+
+            expect(await provider.getAccessToken()).toBeUndefined();
+            expect(await provider.getAccessToken()).toBeUndefined();
+
+            // The second request arrives well inside the retry delay — one attempt,
+            // not one per RPC (the session answers a keepalive every second).
+            expect(fetchToken).toHaveBeenCalledTimes(1);
+        });
+
+        it('should try again once the retry delay has passed', async () => {
+            vi.useFakeTimers();
+            const { provider, fetchToken } = createProvider([new Error('unavailable'), token('token-1', longLifetime)]);
+
+            expect(await provider.getAccessToken()).toBeUndefined();
+            vi.advanceTimersByTime(5000);
+
+            expect(await provider.getAccessToken()).toBe('token-1');
+            expect(fetchToken).toHaveBeenCalledTimes(2);
+        });
+    });
+
+    describe('when multiple requests race', () => {
+        it('should share a single fetch', async () => {
+            let resolveFetch!: (response: OAuthTokenResponse) => void;
+            const fetchToken = vi.fn(() => new Promise<OAuthTokenResponse>(resolve => {
+                resolveFetch = resolve;
+            }));
+            const provider = new OAuthTokenProvider('https://localhost:35000/connect/token', 'client', 'secret', true, fetchToken);
+
+            const first = provider.getAccessToken();
+            const second = provider.getAccessToken();
+            resolveFetch(token('token-1', longLifetime));
+
+            expect(await first).toBe('token-1');
+            expect(await second).toBe('token-1');
+            expect(fetchToken).toHaveBeenCalledTimes(1);
+        });
+    });
+
+    describe('when a refresh is forced', () => {
+        it('should discard the cached token and fetch a new one', async () => {
+            const { provider, fetchToken } = createProvider([token('token-1', longLifetime), token('token-2', longLifetime)]);
+
+            expect(await provider.getAccessToken()).toBe('token-1');
+            expect(await provider.refresh()).toBe('token-2');
+            expect(fetchToken).toHaveBeenCalledTimes(2);
+        });
+    });
+});