@interopio/bridge 0.0.1-alpha → 0.0.3-beta

package/src/net/OutboundConnection.ts

@@ -1,1141 +0,0 @@
-import {Socket} from 'node:net';
-import {type Address, addressAsString} from '../cluster/Address.ts';
-import {approximateClock, type Clock} from '../utils/clock.ts';
-import {deferred, type DeferredPromise} from '../utils/promise.ts';
-import {ByteBufferOutput, type DataOutput} from '../io/VersionedSerializer.ts';
-import {type Message, serializer} from './Message.ts';
-import getLogger from '../logging.ts';
-import {
-    initiateMessaging,
-    type MessagingSuccess,
-    type OutboundSocketInitiatorResult
-} from './OutboundConnectionInitiator.ts';
-import {writeAndFlush} from './internal/AsyncSocketPromise.ts';
-import {type Payload, type PayloadAllocator} from './FrameEncoder.ts';
-import {MessagingService} from '../gossip/Messenger.ts';
-import {minn} from '../utils/bigint.ts';
-import {fromId} from './Action.ts';
-import {convert, type Unit} from '../config/Duration.ts';
-
-const logger = getLogger('net.outbound');
-
-class State {
-    readonly kind: 'established' | 'connecting' | 'paused' | 'closed';
-
-    constructor(kind: 'established' | 'connecting' | 'paused' | 'closed') {
-        this.kind = kind;
-    }
-
-    static readonly CLOSED = new State('closed');
-
-    isEstablished(): this is Established {
-        return this.kind === 'established';
-    }
-
-    isConnecting(): this is Connecting {
-        return this.kind === 'connecting';
-    }
-
-    isDisconnected(): this is Disconnected | Connecting {
-        return this.kind === 'connecting' || this.kind === 'paused';
-    }
-
-    isClosed(): boolean {
-        return this.kind === 'closed';
-    }
-}
-
-class Established extends State {
-    readonly socket: Socket;
-    readonly version: number;
-    readonly allocator: PayloadAllocator;
-    readonly settings: OutboundConnectionSettings;
-    constructor(version: number, socket: Socket, allocator: PayloadAllocator, settings: OutboundConnectionSettings) {
-        super('established');
-        this.version = version;
-        this.socket = socket;
-        this.allocator = allocator;
-        this.settings = settings;
-    }
-
-    isConnected(): boolean {
-        return !this.socket.closed;
-    }
-}
-
-class Disconnected extends State {
-    readonly maintenance: ReturnType<typeof setInterval>;
-
-    constructor(kind: 'connecting' | 'paused', maintenance: ReturnType<typeof setInterval>) {
-        super(kind);
-        this.maintenance = maintenance;
-    }
-
-    static paused(maintenance: ReturnType<typeof setInterval>) {
-        return new Disconnected('paused', maintenance);
-    }
-}
-
-class Connecting extends Disconnected {
-    readonly failingToConnect: boolean;
-    readonly attempt: DeferredPromise<OutboundSocketInitiatorResult<MessagingSuccess>>;
-
-    readonly scheduled?: ReturnType<typeof setTimeout>;
-
-    constructor(previous: Disconnected, attempt: DeferredPromise<OutboundSocketInitiatorResult<MessagingSuccess>>, scheduled?: ReturnType<typeof setTimeout>) {
-        super('connecting', previous.maintenance);
-        this.scheduled = scheduled;
-        this.attempt = attempt;
-        this.failingToConnect = (scheduled !== undefined) || (previous.isConnecting() && previous.failingToConnect);
-    }
-
-    cancel() {
-        if (this.scheduled) {
-            clearTimeout(this.scheduled);
-        }
-        this.attempt.cancel();
-
-    }
-}
-
-type Removed = { message?: Message<unknown>, next?: Removed };
-
-class RemoveRunner {
-    removed = new Set<Message<unknown>>();
-    readonly ready: Promise<void>;
-    private done: () => void;
-    private value?: Removed = {};
-    private readonly queue: OutboundQueue;
-
-    constructor(queue: OutboundQueue) {
-        this.queue = queue;
-        this.ready = new Promise<void>((resolve) => {
-            this.done = resolve;
-        });
-    }
-
-    undo(message: Message<unknown>): boolean {
-        const prev = this.value;
-        if (prev !== undefined) {
-            this.value = {message, next: prev};
-        }
-        return this.value !== undefined;
-    }
-
-    run() {
-        const remove = new Set<Message<unknown>>();
-        delete this.queue.removeRunner;
-        let undo = this.value;
-        delete this.value;
-        while (undo.message !== undefined) {
-            remove.add(undo.message);
-            undo = undo.next;
-        }
-
-
-        this.queue.drainExternalIntoInternal();
-        let earliest: bigint = undefined;
-        const size = this.queue.internal.length;
-        for (let i = 0; i < size; i++) {
-            const message = this.queue.internal[i];
-            if (!remove.has(message)) {
-                if (earliest === undefined || message.header.expiresAtNanos < earliest) {
-                    earliest = message.header.expiresAtNanos;
-                }
-                continue;
-            }
-            this.queue.internal.splice(i, 1);
-            this.removed.add(message);
-        }
-        const now = this.queue.clock.now();
-        this.queue.maybeUpdateNextExpireTime(now, this.queue.maybeUpdateEarliestExpire(now, earliest));
-
-        this.done();
-    }
-}
-
-class Locked {
-    readonly _run: () => void;
-    readonly next?: Locked;
-    constructor(run: () => void, next?: Locked) {
-        this._run = run;
-        this.next = next;
-    }
-
-    andThen(next: () => void): Locked {
-        return new Locked(next, this);
-    }
-
-    run() {
-        let current: Locked = this;
-        while (current !== undefined) {
-            try {
-                current._run();
-            }
-            catch (e) {
-                logger.error(`error while executing deferred functions`, e);
-            }
-            current = current.next;
-        }
-    }
-}
-
-const LOCKED = new Locked(() => {});
-
-class OutboundQueue {
-    readonly external: Message<unknown>[] = [];
-    readonly internal: Message<unknown>[] = [];
-    private earliestExpire?: bigint;
-    nextExpireTime?: bigint;
-    readonly onExpired: (message: Message<unknown>) => void;
-    private locked?: Locked;
-
-    readonly clock?: Clock;
-
-    constructor(clock: Clock, onExpired:(message: Message<unknown>) => void) {
-        this.clock = clock;
-        this.onExpired = onExpired;
-    }
-
-    add<T>(message: Message<T>): void {
-        this.pruneExpired();
-        this.external.push(message);
-        this.maybeUpdateEarliestExpire(this.clock.now(), message.header.expiresAtNanos);
-    }
-
-    private tryLock() {
-        if (this.locked === undefined) {
-            this.locked = LOCKED;
-            return true;
-        }
-        return false;
-    }
-
-    lockOrCallback(nowNs: bigint, callbackIfDeferred: () => void): WithLock | undefined {
-        if (!this.doLockOrCallback(callbackIfDeferred)) {
-            return;
-        }
-
-        return new WithLock(this, nowNs);
-    }
-
-    runEventually(runEventually:(withLock: WithLock) => void) {
-        const withLock = this.lockOrCallback(this.clock.now(), () => {
-            this.runEventually(runEventually);
-        });
-        try {
-            if (withLock) {
-                runEventually(withLock);
-            }
-        }
-        finally {
-            if (withLock) {
-                withLock.close();
-            }
-        }
-    }
-
-    unlock() {
-        const l = this.locked;
-        delete this.locked;
-        if (l) {
-            l.run();
-        }
-    }
-
-    private doLockOrCallback(callbackIfDeferred: () => void): boolean {
-        if (callbackIfDeferred === undefined) {
-            return this.tryLock();
-        }
-
-        while (true)
-        {
-            const current = this.locked;
-            if (current === undefined) {
-                this.locked = LOCKED;
-                try {
-                    throw new Error();
-                }
-                catch (e) {
-                    LOCKED['owner'] = e;
-                }
-                return true;
-            } else if (this.locked === current) {
-                this.locked = current.andThen(callbackIfDeferred);
-                return false;
-            }
-        }
-    }
-
-    private tryRun(runIfAvailable: () => void) {
-        if (!this.tryLock()) {
-            return false;
-        }
-        try {
-            runIfAvailable();
-            return true;
-        }
-        catch (e) {
-            this.unlock();
-        }
-    }
-
-    pruneExpired(now?: bigint) {
-        now = now ?? this.clock.now();
-        if (this.nextExpireTime !== undefined && !this.clock.isBefore(this.nextExpireTime, now)) {
-            return this.tryRun(() => this.pruneWithLock(now));
-        }
-        return false;
-    }
-
-    drainExternalIntoInternal() {
-        while (true) {
-            const message = this.external.shift();
-            if (!message) {
-                break;
-            }
-            this.internal.push(message);
-        }
-    }
-
-    static shouldSend(message: Message<unknown>, clock: Clock, now: bigint): boolean {
-        return clock.isBefore(message.header.expiresAtNanos, now);
-    }
-
-    pruneInternalWithLock(now: bigint) {
-        let earliest: bigint = undefined;
-        const size = this.internal.length;
-        for (let i = 0; i < size; i++) {
-            const message = this.internal[i];
-            if (OutboundQueue.shouldSend(message, this.clock, now)) {
-                if (earliest === undefined || message.header.expiresAtNanos < earliest) {
-                    earliest = message.header.expiresAtNanos;
-                }
-            } else {
-                this.internal.splice(i, 1);
-                this.onExpired(message);
-            }
-        }
-
-        this.maybeUpdateNextExpireTime(now, this.maybeUpdateEarliestExpire(now, earliest));
-    }
-
-    private pruneWithLock(now: bigint) {
-        this.drainExternalIntoInternal();
-        this.pruneInternalWithLock(now);
-    }
-
-    maybeUpdateNextExpireTime(now: bigint, deadline: bigint): bigint {
-        if (this.nextExpireTime === undefined || !this.clock.isBefore(this.nextExpireTime, now)) {
-            this.nextExpireTime = deadline;
-        } else {
-            this.nextExpireTime = minn(this.nextExpireTime, deadline);
-        }
-        return this.nextExpireTime;
-    }
-
-    maybeUpdateEarliestExpire(now: bigint, time: bigint): bigint {
-        if (this.earliestExpire === undefined || !this.clock.isBefore(this.earliestExpire, now)) {
-            this.earliestExpire = time;
-        } else {
-            this.earliestExpire = minn(this.earliestExpire, time);
-        }
-        return this.earliestExpire;
-    }
-
-    removeRunner?: RemoveRunner;
-
-    async remove(remove: Message<unknown>): Promise<boolean> {
-        if (!remove) {
-            throw new Error('no message');
-        }
-        let runner: RemoveRunner;
-        while (true) {
-            runner = this.removeRunner;
-            if (runner && runner.undo(remove)) {
-                break;
-            }
-            if (runner === undefined) {
-                this.removeRunner = runner = new RemoveRunner(this);
-                runner.undo(remove);
-                runner.run();
-                break;
-            }
-        }
-        await runner.ready;
-        return runner.removed.has(remove)
-    }
-}
-class WithLock {
-    private readonly queue: OutboundQueue;
-    private readonly nowNs: bigint;
-    constructor(queue: OutboundQueue, nowNs?: bigint) {
-        this.queue = queue;
-        this.nowNs = nowNs;
-        this.queue.drainExternalIntoInternal();
-    }
-
-    removeHead(expected: Message<unknown>): void {
-        if (expected !== this.queue.internal[0]) {
-            throw new Error('unexpected head');
-        }
-        this.queue.internal.shift();
-    }
-
-    peek(): Message<unknown> {
-        let m: Message<unknown>;
-        while (undefined !== (m = this.queue.internal[0])) {
-            if (OutboundQueue.shouldSend(m, this.queue.clock, this.nowNs)) {
-                break;
-            }
-            this.queue.internal.shift();
-            this.queue.onExpired(m);
-        }
-
-        return m;
-    }
-
-    poll(): Message<unknown> {
-        let m: Message<unknown>;
-        while (undefined !== (m = this.queue.internal.shift())) {
-            if (OutboundQueue.shouldSend(m, this.queue.clock, this.nowNs)) {
-                break;
-            }
-            this.queue.onExpired(m);
-        }
-
-        return m;
-    }
-
-    consume(consumer: (msg: Message<unknown>) => void): void {
-        let m: Message<unknown>;
-        while (undefined !== (m = this.poll())) {
-            consumer(m);
-        }
-    }
-
-    close() {
-        try {
-            if (!this.queue.clock.isBefore(this.queue.nextExpireTime, this.nowNs)) {
-                this.queue.pruneInternalWithLock(this.nowNs);
-            }
-        }
-        finally {
-            this.queue.unlock();
-        }
-    }
-}
-
-function andThen(a: Runnable | undefined, b: Runnable): Runnable  {
-    if (a === undefined) {
-        return b;
-    }
-    return () => {a(); b(); return;};
-}
-
-function valueOf(value: number): keyof typeof DeliveryState {
-    switch (value) {
-        case DeliveryState.STOPPED:
-            return 'STOPPED';
-        case DeliveryState.EXECUTING:
-            return 'EXECUTING';
-        case DeliveryState.EXECUTE_AGAIN:
-            return 'EXECUTE_AGAIN';
-        case DeliveryState.EXECUTING_AGAIN:
-            return 'EXECUTING_AGAIN';
-        case DeliveryState.WAITING_TO_EXECUTE:
-            return 'WAITING_TO_EXECUTE';
-        case DeliveryState.EXECUTING_AND_WAITING_TO_EXECUTE:
-            return 'EXECUTING_AND_WAITING_TO_EXECUTE';
-        default:
-            throw new Error('Invalid value');
-    }
-}
-
-const DeliveryState = {
-    /*0x0000*/STOPPED: 0,
-    /*0x0001*/EXECUTING: 1,
-    /*0x0010*/EXECUTE_AGAIN: 2,
-    /*0x0011*/EXECUTING_AGAIN: 1 /*EXECUTING*/ | 2 /*EXECUTE_AGAIN*/,
-    /*0x0100*/WAITING_TO_EXECUTE: 4,
-    /*0x0101*/EXECUTING_AND_WAITING_TO_EXECUTE: 1 /*EXECUTING*/ | 4 /*WAITING_TO_EXECUTE*/,
-}
-
-class Initiate {
-    private readonly connection: OutboundConnection;
-    private retryRateMs = 100;
-    private connectionAttempts = 0;
-    private successfullConnections = 0;
-    private settings: Required<OutboundConnectionSettings>;
-
-    constructor(connection: OutboundConnection) {
-        this.connection = connection;
-    }
-
-    private onFailure(err?: Error) {
-        const state = this.connection.state as Disconnected;
-        if (state.isConnecting()) {
-            state.attempt.promise.catch(err => {
-                return;
-            });
-            // await state.attempt.catch((err) => {
-            //     logger.debug(`handle connection failure`, err);
-            // });
-        }
-
-        if (this.connection.hasPending()) {
-            const result = deferred<OutboundSocketInitiatorResult<MessagingSuccess>>();
-            this.connection.state = new Connecting(state, result, setTimeout(async () => {
-                this.attempt(result);
-            }, Math.max(100, this.retryRateMs)));
-            this.retryRateMs = Math.min(1000, this.retryRateMs * 2);
-        } else {
-            this.connection.state = Disconnected.paused(state.maintenance);
-        }
-    }
-
-    private onSuccess(result: OutboundSocketInitiatorResult<MessagingSuccess>) {
-        switch (result.outcome) {
-            case "success": {
-                if (this.connection.state.isClosed()) {
-                    throw new Error('close did not canceled this. this is a bug');
-                }
-                clearInterval((this.connection.state as Disconnected).maintenance);
-                const allocator = result.allocator;
-                const socket = result.socket;
-                const established = new Established(result.version, socket, allocator, this.settings);
-                this.connection.state = established;
-                socket.on('error', (err: Error) => {
-                    try {
-                        this.connection.invalidateSocket(established, err);
-                    } catch (e) {
-                        logger.error(`unhandled exception in outbound connection`, e);
-                    }
-                });
-
-                ++this.successfullConnections;
-                logger.info(`successfully connected, version = ${result.version}`);
-                break;
-            }
-            case 'retry': {
-                this.initiate();
-                break;
-            }
-            case 'incompatible': {
-                this.onFailure(new Error('Incompatible connection'));
-                break;
-            }
-            default: {
-                throw new Error();
-            }
-        }
-    }
-
-    private attempt(result: DeferredPromise<OutboundSocketInitiatorResult<MessagingSuccess>>): void {
-        this.connectionAttempts++;
-        this.settings = this.connection.template;
-        initiateMessaging(this.settings, result)
-            .promise
-            .then((v) => this.onSuccess(v))
-            .catch((err?) => this.onFailure(err));
-    }
-
-    initiate(): Promise<OutboundSocketInitiatorResult<MessagingSuccess>> {
-        const def = deferred<OutboundSocketInitiatorResult<MessagingSuccess>>();
-        this.connection.state = new Connecting(this.connection.state as Disconnected, def);
-        this.attempt(def);
-        return def.promise;
-    }
-}
-
-type Runnable = () => void;
-
-abstract class Delivery {
-    private terminated = false;
-    private inProgress = false;
-    private _value: keyof typeof DeliveryState = 'STOPPED';
-
-    private _stopAndRun?: Runnable;
-    protected readonly connection: OutboundConnection;
-    set value(v) {
-        // logger.trace(`set value ${this._value} => ${v}}`);
-        this._value = v;
-    }
-    get value() {
-        return this._value;
-    }
-
-    /*protected */constructor(connection: OutboundConnection) {
-        this.connection = connection;
-    }
-
-    execute() {
-        const state = this.value;
-        if (DeliveryState[state] < DeliveryState.EXECUTE_AGAIN) {
-            this.value = (state === 'STOPPED') ? 'EXECUTING' : valueOf(DeliveryState[state] | DeliveryState.EXECUTE_AGAIN);
-            if (state === 'STOPPED') {
-                setImmediate(() => this.run());
-            }
-        }
-    }
-
-    private isExecuting(state: keyof typeof DeliveryState): boolean {
-        return (DeliveryState[state] & DeliveryState.EXECUTING) !== 0;
-    }
-
-    executeAgain(): void {
-        const state = this.value;
-        this.value = !this.isExecuting(state) ? 'EXECUTING' : 'EXECUTE_AGAIN';
-        if (!this.isExecuting(state)) {
-            setImmediate(() => this.run());
-        }
-    }
-
-    promiseToExecute(): void {
-        this.value = 'EXECUTING_AND_WAITING_TO_EXECUTE';
-    }
-
-    private maybeExecuteAgain() {
-        const state = this.value;
-        this.value = state === 'EXECUTING_AGAIN' ? 'EXECUTING' : valueOf(DeliveryState[state] & ~DeliveryState.EXECUTING);
-        if (state === 'EXECUTING_AGAIN') {
-            setImmediate(() => this.run());
-        }
-    }
-
-    terminate() {
-        this.terminated = true;
-    }
-
-    setInProgress(inProgress: boolean): void {
-        const wasInProgress = this.inProgress;
-        this.inProgress = inProgress;
-        if (wasInProgress && !inProgress) {
-            this.executeAgain();
-        }
-    }
-
-    run(): void {
-        try {
-            logger.error("START DELIVERY TASK");
-            while (true) {
-                if (this.terminated) {
-                    return;
-                }
-                if (this._stopAndRun) {
-                    if (this.inProgress) {
-                        this.promiseToExecute();
-                        break;
-                    }
-                    const stopAndRun = this._stopAndRun;
-                    this._stopAndRun = undefined;
-                    stopAndRun();
-                }
-                const state = this.connection.state;
-                if (!state.isEstablished() || !state.isConnected()) {
-                    if (this.connection.hasPending() || this._stopAndRun !== undefined) {
-                        this.promiseToExecute();
-                        this.connection.requestConnect().finally(() => {
-                            this.executeAgain();
-                        });
-                    }
-                    break;
-                }
-                if (!this.doRun(state)) {
-                    break;
-                }
-            }
-        } catch (e) {
-            logger.error(`error while executing outbound connection`, e);
-        } finally {
-            logger.error('END DELIVERY TASK');
-            this.maybeExecuteAgain();
-        }
-    }
-
-    protected abstract doRun(established: Established): boolean;
-
-    stopAndRun(stopAndRun: Runnable): void {
-        this._stopAndRun = andThen(this._stopAndRun, stopAndRun);
-        this.execute();
-    }
-
-    abstract stopAndRunOnEventLoop(stopAndRun: Runnable): void
-
-}
-
-class EventLoopDelivery extends Delivery {
-    private flushingBytes: number = 0;
-    private isWriteable: boolean = true;
-
-    constructor(connection: OutboundConnection) {
-        super(connection);
-    }
-
-    doRun(established: Established): boolean {
-        if (!this.isWriteable) {
-            return false;
-        }
-        const maxSendBytes = Math.min(this.connection.pendingBytes - this.flushingBytes, 64*1024);
-        if (maxSendBytes == 0) {
-            return false;
-        }
-
-        const settings = established.settings;
-        const version: number = established.version;
-
-        let payload: Payload = undefined;
-        let canonicalSize = 0;
-        const sending = {count: 0, bytes: 0};
-
-        const lock = this.connection.queue.lockOrCallback(approximateClock().now(), () => this.execute());
-
-        try {
-            if (!lock) {
-                return false;
-            }
-            payload = established.allocator(maxSendBytes);
-            let out: DataOutput = new ByteBufferOutput(payload.buffer);
-
-            let next: Message<unknown>;
-            while (undefined !== (next = lock.peek())) {
-                try {
-
-                    const messageSize = next.serializedSize(version);
-                    // todo: check message size limit
-                    if (messageSize > payload.remaining) {
-                        if (sending.bytes > 0) {
-                            break;
-                        }
-                        payload.release();
-                        payload = undefined;
-                        payload = established.allocator(messageSize);
-
-                        out = new ByteBufferOutput(payload.buffer);
-                    }
-                    serializer.serialize(next, out, version);
-                    if (payload.length != sending.bytes + messageSize) {
-                        throw new Error('Invalid payload length');
-                    }
-                    canonicalSize += this.connection.canonicalSize(next);
-                    sending.count += 1;
-                    sending.bytes += messageSize;
-
-                } catch (e) {
-                    this.connection.onFailedSerialize(next, version, 0, e);
-                    payload.trim(sending.bytes);
-                }
-                lock.removeHead(next);
-            }
-
-            if (sending.bytes == 0) {
-                return false;
-            }
-            payload.finish();
-
-            const flushResult = writeAndFlush(established.socket, payload);
-            payload = undefined;
-            if (flushResult.success) {
-                this.connection.sent.count += sending.count;
-                this.connection.sent.bytes += sending.bytes;
-            }
-            else {
-                this.flushingBytes += canonicalSize;
-                this.setInProgress(true);
-                const overflowed = this.flushingBytes >= settings.flushHighWaterMark
-                if (overflowed) {
-                    this.isWriteable = false;
-                    this.promiseToExecute();
-                }
-                const releaseBytes = canonicalSize;
-                const sendingBytes = sending.bytes;
-                const sendingCount = sending.count;
-                flushResult.promise
-                    .then(() => {return {success: true}})
-                    .catch((error?: Error) => {return {success: false, error}})
-                    .then(future => {
-                        this.connection.releaseCapacity(sendingCount, releaseBytes);
-                        this.flushingBytes -= releaseBytes;
-                        if (this.flushingBytes === 0) {
-                            this.setInProgress(false);
-                        }
-
-                        if (!this.isWriteable && this.flushingBytes <= settings.flushLowWaterMark) {
-                            this.isWriteable = true;
-                            this.executeAgain();
-                        }
-                        if (future.success) {
-                            this.connection.sent.count += sendingCount;
-                            this.connection.sent.bytes += sendingBytes;
-                        } else {
-                            this.connection.error.count += sendingCount;
-                            this.connection.error.bytes += sendingBytes;
-                            this.connection.invalidateSocket(established, future['error']);
-                        }
-                    });
-                canonicalSize = 0;
-            }
-        } catch (e) {
-            this.connection.error.count += sending.count;
-            this.connection.error.bytes += sending.bytes;
-            this.connection.invalidateSocket(established, e);
-        }
-        finally {
-            if (lock) {
-                lock.close();
-            }
-            if (canonicalSize > 0) {
-                this.connection.releaseCapacity(sending.count, canonicalSize);
-            }
-
-            if (payload) {
-                payload.release();
-            }
-            if (this.connection.pendingBytes > this.flushingBytes && this.isWriteable) {
-                this.execute();
-            }
-        }
-
-        return false;
-    }
-
-    stopAndRunOnEventLoop(stopAndRun: Runnable): void {
-        this.stopAndRun(stopAndRun);
-    }
-
-}
-
-export interface OutboundMessageCallbacks {
-
-    onExpired(message: Message<unknown>, peer: Address): void;
-    onFailedSerialize(message: Message<unknown>, peer: Address, bytesWritten: number, error: Error): void;
-    onDiscardOnClose(message: Message<unknown>, peer: Address): void;
-}
-
-export class OutboundConnectionSettings {
-    readonly acceptVersions = {min: MessagingService.CURRENT_VERSION, max: 13};
-    readonly from: Address;
-    readonly to: Address;
-    readonly connectTo?: Address;
-    readonly flushLowWaterMark: number;
-    readonly flushHighWaterMark: number;
-    readonly tcpNoDelay?: boolean;
-    readonly callbacks: OutboundMessageCallbacks;
-
-    constructor(from: Address, to: Address, callbacks: OutboundMessageCallbacks, flushLowWaterMark = 1<<15, flushHighWaterMark = 1 << 16,  connectTo?: Address, tcpNoDelay?: boolean) {
-        this.from = from;
-        this.to = to;
-        this.connectTo = connectTo;
-        this.tcpNoDelay = tcpNoDelay;
-        this.flushLowWaterMark = flushLowWaterMark;
-        this.flushHighWaterMark = flushHighWaterMark;
-        this.callbacks = callbacks;
-    }
-
-    get connectToId(): string {
-        const connectTo = this.connectTo;
-        if (connectTo === undefined) {
-            return addressAsString(this.to);
-        }
-        return `${addressAsString(this.to)} (${addressAsString(connectTo)})`;
-    }
-
-    getConnectTo(): Address {
-        const connectTo = this.connectTo;
-        if (connectTo === undefined) {
-            return this.to;
-        }
-        return connectTo;
-    }
-
-    getTcpNoDelay(): boolean {
-        const tcpNoDelay = this.tcpNoDelay;
-        if (tcpNoDelay === undefined) {
-            return true;
-        }
-        return tcpNoDelay;
-    }
-
-    withDefaults(): Required<OutboundConnectionSettings> {
-        return new OutboundConnectionSettings(this.from, this.to, this.callbacks, this.flushLowWaterMark, this.flushHighWaterMark, this.getConnectTo(), this.getTcpNoDelay()) as Required<OutboundConnectionSettings>;
-    }
-
-    toString() {
-        return `peer: (${addressAsString(this.to)}, ${addressAsString(this.connectTo)})`;
-    }
-}
-
-export class OutboundConnection {
-    state: State;
-
-    queue: OutboundQueue;
-    private readonly delivery = new EventLoopDelivery(this);
-    private closing?: DeferredPromise<void>;
-    private scheduledClose?: DeferredPromise<void>;
-    private readonly maxMessageSize: number = 1024 * 1024;
-    template: Required<OutboundConnectionSettings>;
-    private readonly callbacks: OutboundMessageCallbacks;
-
-    constructor(settings: OutboundConnectionSettings) {
-        this.template = settings.withDefaults();
-        this.callbacks = this.template.callbacks;
-        this.queue = new OutboundQueue(approximateClock(), (msg)=> this.onExpired(msg));
-        this.setDisconnected();
-    }
-
-    private onExpired(message: Message<unknown>) {
-        if (logger.enabledFor('debug')) {
-            const rpcTimeout = 200;
-            logger.debug(`dropping ${fromId(message.header.verb().id)} with payload ${message.payload} whose timeout (${rpcTimeout}) expired before reaching network.`);
-        }
-
-        const expiredBytes = this.canonicalSize(message);
-        this.releaseCapacity(1, expiredBytes);
-        this.expired.count += 1;
-        this.expired.bytes += expiredBytes;
-        this.callbacks.onExpired(message, this.template.to);
-    }
-
-    onFailedSerialize(message: Message<unknown>, version: number, bytesWritten:number, error: Error) {
-        const bytes = message.serializedSize(version);
-        this.releaseCapacity(1, bytes);
-        this.error.count += 1;
-        this.error.bytes += bytes;
-        this.callbacks.onFailedSerialize(message, this.template.to, bytesWritten, error);
-    }
-
-    onClosed(message: Message<unknown>) {
-        this.releaseCapacity(1, this.canonicalSize(message));
-        this.callbacks.onDiscardOnClose(message, this.template.to);
-    }
-
-    enqueue<T>(message: Message<T>): void {
-        if (this.isClosing()) {
-            throw new Error('socket closed');
-        }
-        const canonicalSize = this.canonicalSize(message);
-        this.acquireCapacity(1, canonicalSize);
-        this.queue.add(message);
-        this.delivery.execute();
-
-        // if (this.isClosing() && await this.queue.remove(message)) {
-        //     this.releaseCapacity(1, canonicalSize);
-        //     throw new Error('socket closing');
-        // }
-    }
-
-    private readonly pending = {count: 0, bytes: 0};
-    private readonly expired = {count: 0, bytes: 0};
-    readonly error = {count: 0, bytes: 0};
-    readonly sent = {count: 0, bytes: 0};
-
-    get pendingBytes(): number {
-        return this.pending.bytes;
-    }
-
-    private acquireCapacity(count: number = 1, bytes: number): 'success' {
-        this.pending.count += count;
-        this.pending.bytes += bytes;
-        return 'success';
-    }
-
-    releaseCapacity(count: number, bytes: number): number {
-        this.pending.count -= count;
-        this.pending.bytes -= bytes;
-        return this.pending.count;
-    }
-
-    canonicalSize(message: Message<unknown>): number {
-        return message.serializedSize(MessagingService.CURRENT_VERSION);
-    }
-
-    invalidateSocket(established: Established, err?: Error): void {
-        if (this.state !== established) {
-            return; // do nothing; socket already closed
-        }
-        this.disconnectNow(established);
-    }
-
-    get connected() {
-        return this.state.isEstablished() && this.state.isConnected();
-    }
-
-    private isClosing(): boolean {
-        return this.closing !== undefined;
-    }
-
-    private setDisconnected() {
-        if (this.state !== undefined && !this.state.isEstablished()) {
-            throw new Error();
-        }
-        this.state = Disconnected.paused(setInterval(this.queue.pruneExpired.bind(this.queue), 100));
-    }
-
-    scheduleClose(time: number, unit: Unit, flushQueue: boolean): DeferredPromise<void> {
-        if (this.scheduledClose === undefined) {
-            let timeoutId: ReturnType<typeof setTimeout>;
-
-            this.scheduledClose = deferred<void>((cb) => {
-                clearTimeout(timeoutId);
-                return true;
-            });
-
-            timeoutId = setTimeout(() => {
-                this.close(flushQueue).then(() => {
-                    this.scheduledClose.success();
-                }).catch(error => {
-                    this.scheduledClose.failure(error);
-                });
-            }, Number(convert('milliseconds', time, unit)));
-        }
-        return this.scheduledClose;
-    }
-
-    close(flushQueue: boolean): Promise<void> {
-        if (this.closing === undefined) {
-            this.closing = deferred<void>((cb) => {
-                return true;
-            });
-
-            const eventLoopCleanup = () => {
-                const onceNotConnecting = () => {
-                    const state = this.state;
-                    this.state = State.CLOSED;
-
-                    try {
-                        this.delivery.terminate();
-
-                        if (state.isDisconnected()) {
-                            clearInterval(state.maintenance);
-                            this.closing.success();
-                        }
-                        else {
-                            if (!state.isEstablished()) {
-                                throw new Error(`Invalid state: ${state}`);
-                            }
-                            state.socket.once('close', (hadError) => {
-                                this.closing.success();
-                            });
-                            state.socket.destroy();
-
-                        }
-                    } catch (ex) {
-                        this.closing.success();
-
-                        try {
-                            if (state.isEstablished()) {
-                                state.socket.destroy();
-                            }
-                        } catch (e) {
-                            logger.error('Failed to close the connection cleanly', e);
-                            logger.error('suppressed', ex);
-                        }
-                        throw ex;
-                    }
-                }
-
-                if (this.state.isConnecting()) {
-                    this.state.cancel();
-                    this.state.attempt.promise.catch(e => {}).finally(() => onceNotConnecting());
-                }
-                else {
-                    onceNotConnecting();
-                }
-
-            }
-
-            const clearQueue = async () => {
-                await new Promise<void>(resolve => {
-                    this.queue.runEventually((withLock) => {
-                        withLock.consume((msg) => this.onClosed(msg));
-                        resolve();
-                    });
-                });
-            };
-
-            if(flushQueue)
-            {
-                const finishDelivery = async () => {
-                    if (!this.hasPending()) {
-                        this.delivery.stopAndRunOnEventLoop(async () => await eventLoopCleanup());
-                    }
-                    else {
-                        this.delivery.stopAndRun(async () => {
-                            if (this.state.isConnecting() && this.state.failingToConnect) {
-                                await clearQueue();
-                            }
-                            await finishDelivery();
-                        });
-                    }
-                };
-
-                this.delivery.stopAndRun(finishDelivery);
-            }
-            else {
-                this.delivery.stopAndRunOnEventLoop(async () => {
-                    await clearQueue();
-                    eventLoopCleanup();
-                });
-            }
-        }
-        return this.closing.promise;
-    }
-
-    initiate(): Promise<unknown> {
-        return new Initiate(this).initiate();
-    }
-
-    requestConnect(): Promise<unknown> {
-        if (this.state?.isConnecting()) {
-            return this.state.attempt.promise
-        }
-        return new Promise<void>((resolve, reject) => {
-            setImmediate(() => {
-                if (this.state?.isClosed()) {
-                    reject(new Error('socket closed'));
-                } else if (this.state.isEstablished() && this.state.isConnected()) {
-                    resolve();
-                } else {
-                    if (this.state.isEstablished()) {
-                        this.setDisconnected();
-                    }
-
-                    if (!this.state.isConnecting()) {
-                        this.initiate().then(resolve).catch(reject);
-                    } else {
-                        this.state.attempt.promise.then(() => resolve()).catch(reject);
-                    }
-                }
-
-            });
-        });
-    }
-
-    private disconnectNow(closeIfIs: Established): Promise<void> {
-        return new Promise<void>(resolve => {
-            setImmediate(() => {
-                if (this.state === closeIfIs) {
-                    this.setDisconnected();
-                    if (this.hasPending()) {
-                        this.delivery.execute();
-                    }
-                    closeIfIs.socket.once('close', (hadError: boolean) => {
-                        if (hadError) {
-
-                        }
-                    });
-                    closeIfIs.socket.destroy();
-                    resolve();
-                }
-            });
-        });
-    }
-
-    hasPending() {
-        return this.pending.count !== 0 && this.pending.bytes !== 0;
-    }
-
-    /*testing*/ unsafeRunDelivery(run : () => void): void {
-        this.delivery.stopAndRun(run);
-    }
-}