@peerbit/stream 1.0.20 → 2.0.2

package/lib/esm/index.js

@@ -1,23 +1,40 @@
-import { EventEmitter, CustomEvent } from "@libp2p/interface/events";
+import { TypedEventEmitter, CustomEvent } from "@libp2p/interface";
 import { pipe } from "it-pipe";
 import Queue from "p-queue";
 import { pushable } from "it-pushable";
 import { Uint8ArrayList } from "uint8arraylist";
 import { abortableSource } from "abortable-iterator";
 import * as lp from "it-length-prefixed";
-import { Routes } from "./routes.js";
+import { MAX_ROUTE_DISTANCE, Routes } from "./routes.js";
+import pDefer from "p-defer";
+import { AbortError, delay, TimeoutError, waitFor } from "@peerbit/time";
+import { getKeypairFromPeerId, getPublicKeyFromPeerId, PublicSignKey, sha256Base64, toBase64 } from "@peerbit/crypto";
 import { multiaddr } from "@multiformats/multiaddr";
-import { delay, TimeoutError, waitFor, waitForAsync } from "@peerbit/time";
-import { getKeypairFromPeerId, getPublicKeyFromPeerId, PublicSignKey } from "@peerbit/crypto";
 import { logger } from "./logger.js";
-import { Cache } from "@peerbit/cache";
 export { logger };
-import { Message as Message, Goodbye, Hello, DataMessage, Ping, PingPong, Pong, getMsgId } from "@peerbit/stream-interface";
+import { Cache } from "@peerbit/cache";
+import { ready } from "@peerbit/crypto";
+import { Message as Message, DataMessage, getMsgId, ACK, SeekDelivery, AcknowledgeDelivery, SilentDelivery, MessageHeader, Goodbye, TracedDelivery, AnyWhere } from "@peerbit/stream-interface";
+import { MultiAddrinfo } from "@peerbit/stream-interface";
+import { MovingAverageTracker } from "./metrics.js";
+const logError = (e) => {
+    return logger.error(e?.message);
+};
+const DEFAULT_SEEK_MESSAGE_REDUDANCY = 2;
+const DEFAULT_SILENT_MESSAGE_REDUDANCY = 1;
 const isWebsocketConnection = (c) => c.remoteAddr.protoNames().find((x) => x === "ws" || x === "wss");
+const SEEK_DELIVERY_TIMEOUT = 15e3;
+const MAX_DATA_LENGTH = 1e7 + 1000; // 10 mb and some metadata
+const MAX_QUEUED_BYTES = MAX_DATA_LENGTH * 50;
+const DEFAULT_PRUNE_CONNECTIONS_INTERVAL = 2e4;
+const DEFAULT_MIN_CONNECTIONS = 2;
+const DEFAULT_MAX_CONNECTIONS = 300;
+const DEFAULT_PRUNED_CONNNECTIONS_TIMEOUT = 30 * 1000;
+const ROUTE_UPDATE_DELAY_FACTOR = 3e4;
 /**
  * Thin wrapper around a peer's inbound / outbound pubsub streams
  */
-export class PeerStreams extends EventEmitter {
+export class PeerStreams extends TypedEventEmitter {
     counter = 0;
     peerId;
     publicKey;
@@ -43,9 +60,9 @@ export class PeerStreams extends EventEmitter {
      */
     inboundAbortController;
     closed;
-    pingJob;
-    pingLatency;
     connId;
+    seekedOnce;
+    usedBandWidthTracker;
     constructor(init) {
         super();
         this.peerId = init.peerId;
@@ -55,6 +72,7 @@ export class PeerStreams extends EventEmitter {
         this.closed = false;
         this.connId = init.connId;
         this.counter = 1;
+        this.usedBandWidthTracker = new MovingAverageTracker();
     }
     /**
      * Do we have a connection to read from?
@@ -68,6 +86,9 @@ export class PeerStreams extends EventEmitter {
     get isWritable() {
         return Boolean(this.outboundStream);
     }
+    get usedBandwidth() {
+        return this.usedBandWidthTracker.value;
+    }
     /**
      * Send a message to this peer.
      * Throws if there is no `stream` to write to available.
@@ -77,9 +98,14 @@ export class PeerStreams extends EventEmitter {
             logger.error("No writable connection to " + this.peerId.toString());
             throw new Error("No writable connection to " + this.peerId.toString());
         }
-        this.outboundStream.push(data instanceof Uint8Array ? new Uint8ArrayList(data) : data);
+        this.usedBandWidthTracker.add(data.byteLength);
+        this.outboundStream.push(data instanceof Uint8Array ? data : data.subarray());
     }
     async waitForWrite(bytes) {
+        if (this.closed) {
+            logger.error("Failed to send to stream: " + this.peerId + ". Closed");
+            return;
+        }
         if (!this.isWritable) {
             // Catch the event where the outbound stream is attach, but also abort if we shut down
             const outboundPromise = new Promise((rs, rj) => {
@@ -133,7 +159,7 @@ export class PeerStreams extends EventEmitter {
         // - abortable, set to only return on abort, rather than throw
         // - transformed with length-prefix transform
         this._rawInboundStream = stream;
-        this.inboundStream = abortableSource(pipe(this._rawInboundStream, (source) => lp.decode(source, { maxDataLength: 100001000 })), this.inboundAbortController.signal, {
+        this.inboundStream = abortableSource(pipe(this._rawInboundStream, (source) => lp.decode(source, { maxDataLength: MAX_DATA_LENGTH })), this.inboundAbortController.signal, {
             returnOnAbort: true,
             onReturnError: (err) => {
                 logger.error("Inbound stream error", err?.message);
@@ -149,9 +175,8 @@ export class PeerStreams extends EventEmitter {
         // If an outbound stream already exists, gently close it
         const _prevStream = this.outboundStream;
         this._rawOutboundStream = stream;
-        this.pingJob?.abort();
         this.outboundStream = pushable({
-            objectMode: true,
+            objectMode: false,
             onEnd: () => {
                 return stream.close().then(() => {
                     if (this._rawOutboundStream === stream) {
@@ -162,9 +187,7 @@ export class PeerStreams extends EventEmitter {
                 });
             }
         });
-        pipe(this.outboundStream, (source) => lp.encode(source), this._rawOutboundStream).catch((err) => {
-            logger.error(err);
-        });
+        pipe(this.outboundStream, (source) => lp.encode(source), this._rawOutboundStream).catch(logError);
         // Emit if the connection is new
         this.dispatchEvent(new CustomEvent("stream:outbound"));
         if (_prevStream != null) {
@@ -191,19 +214,16 @@ export class PeerStreams extends EventEmitter {
             this.inboundAbortController.abort();
             await this._rawInboundStream?.close();
         }
-        this.pingJob?.abort();
-        this.pingLatency = undefined;
-        //this.dispatchEvent(new CustomEvent('close'))
+        this.dispatchEvent(new CustomEvent("close"));
         this._rawOutboundStream = undefined;
         this.outboundStream = undefined;
         this._rawInboundStream = undefined;
         this.inboundStream = undefined;
     }
 }
-export class DirectStream extends EventEmitter {
+export class DirectStream extends TypedEventEmitter {
     components;
     peerId;
-    peerIdStr;
     publicKey;
     publicKeyHash;
     sign;
@@ -213,7 +233,6 @@ export class DirectStream extends EventEmitter {
      */
     peers;
     peerKeyHashToPublicKey;
-    peerIdToPublicKey;
     routes;
     /**
      * If router can relay received messages, even if not subscribed
@@ -223,82 +242,110 @@ export class DirectStream extends EventEmitter {
      * if publish should emit to self, if subscribed
      */
     signaturePolicy;
-    emitSelf;
     queue;
     multicodecs;
     seenCache;
-    earlyGoodbyes;
-    helloMap; // key is hash of publicKey, value is map whey key is hash of signature bytes, and value is latest Hello
-    multiaddrsMap;
     _registrarTopologyIds;
     maxInboundStreams;
     maxOutboundStreams;
-    topology;
-    pingJobPromise;
-    pingJob;
-    pingInterval;
     connectionManagerOptions;
     recentDials;
+    healthChecks;
+    pruneConnectionsTimeout;
+    prunedConnectionsCache;
+    routeSeekInterval;
+    seekTimeout;
+    closeController;
+    _ackCallbacks = new Map();
     constructor(components, multicodecs, options) {
         super();
         this.components = components;
-        const { canRelayMessage = false, emitSelf = false, messageProcessingConcurrency = 10, pingInterval = 10 * 1000, maxInboundStreams, maxOutboundStreams, signaturePolicy = "StictSign", connectionManager = { autoDial: true } } = options || {};
+        const { canRelayMessage = false, messageProcessingConcurrency = 10, maxInboundStreams, maxOutboundStreams, signaturePolicy = "StictSign", connectionManager, routeSeekInterval = ROUTE_UPDATE_DELAY_FACTOR, seekTimeout = SEEK_DELIVERY_TIMEOUT } = options || {};
         const signKey = getKeypairFromPeerId(components.peerId);
+        this.seekTimeout = seekTimeout;
         this.sign = signKey.sign.bind(signKey);
         this.peerId = components.peerId;
-        this.peerIdStr = components.peerId.toString();
         this.publicKey = signKey.publicKey;
         this.publicKeyHash = signKey.publicKey.hashcode();
         this.multicodecs = multicodecs;
         this.started = false;
         this.peers = new Map();
-        this.helloMap = new Map();
-        this.multiaddrsMap = new Map();
         this.routes = new Routes(this.publicKeyHash);
         this.canRelayMessage = canRelayMessage;
-        this.emitSelf = emitSelf;
+        this.healthChecks = new Map();
+        this.routeSeekInterval = routeSeekInterval;
         this.queue = new Queue({ concurrency: messageProcessingConcurrency });
-        this.earlyGoodbyes = new Map();
         this.maxInboundStreams = maxInboundStreams;
         this.maxOutboundStreams = maxOutboundStreams;
-        this.seenCache = new Cache({ max: 1e3, ttl: 10 * 60 * 1e3 });
+        this.seenCache = new Cache({ max: 1e6, ttl: 10 * 60 * 1e3 });
         this.peerKeyHashToPublicKey = new Map();
-        this.peerIdToPublicKey = new Map();
-        this.pingInterval = pingInterval;
         this._onIncomingStream = this._onIncomingStream.bind(this);
         this.onPeerConnected = this.onPeerConnected.bind(this);
         this.onPeerDisconnected = this.onPeerDisconnected.bind(this);
         this.signaturePolicy = signaturePolicy;
-        this.connectionManagerOptions = connectionManager;
-        this.recentDials = new Cache({
-            ttl: connectionManager.retryDelay || 60 * 1000,
-            max: 1e3
-        });
+        if (connectionManager === false || connectionManager === null) {
+            this.connectionManagerOptions = {
+                maxConnections: Number.MAX_SAFE_INTEGER,
+                minConnections: 0,
+                dialer: undefined,
+                pruner: undefined
+            };
+        }
+        else {
+            this.connectionManagerOptions = {
+                maxConnections: DEFAULT_MAX_CONNECTIONS,
+                minConnections: DEFAULT_MIN_CONNECTIONS,
+                ...connectionManager,
+                dialer: connectionManager?.dialer !== false &&
+                    connectionManager?.dialer !== null
+                    ? { retryDelay: 60 * 1000, ...connectionManager?.dialer }
+                    : undefined,
+                pruner: connectionManager?.pruner !== false &&
+                    connectionManager?.pruner !== null
+                    ? {
+                        connectionTimeout: DEFAULT_PRUNED_CONNNECTIONS_TIMEOUT,
+                        interval: DEFAULT_PRUNE_CONNECTIONS_INTERVAL,
+                        maxBuffer: MAX_QUEUED_BYTES,
+                        ...connectionManager?.pruner
+                    }
+                    : undefined
+            };
+        }
+        this.recentDials = this.connectionManagerOptions.dialer
+            ? new Cache({
+                ttl: this.connectionManagerOptions.dialer.retryDelay,
+                max: 1e3
+            })
+            : undefined;
+        this.prunedConnectionsCache = this.connectionManagerOptions.pruner
+            ? new Cache({
+                max: 1e6,
+                ttl: this.connectionManagerOptions.pruner.connectionTimeout
+            })
+            : undefined;
     }
     async start() {
         if (this.started) {
             return;
         }
+        await ready;
+        this.closeController = new AbortController();
+        this.started = true;
         logger.debug("starting");
-        // register protocol with topology
-        // Topology callbacks called on connection manager changes
-        this._registrarTopologyIds = await Promise.all(this.multicodecs.map((multicodec) => this.components.registrar.register(multicodec, {
-            onConnect: this.onPeerConnected.bind(this),
-            onDisconnect: this.onPeerDisconnected.bind(this)
-        })));
         // Incoming streams
         // Called after a peer dials us
         await Promise.all(this.multicodecs.map((multicodec) => this.components.registrar.handle(multicodec, this._onIncomingStream, {
             maxInboundStreams: this.maxInboundStreams,
-            maxOutboundStreams: this.maxOutboundStreams
+            maxOutboundStreams: this.maxOutboundStreams,
+            runOnTransientConnection: false
+        })));
+        // register protocol with topology
+        // Topology callbacks called on connection manager changes
+        this._registrarTopologyIds = await Promise.all(this.multicodecs.map((multicodec) => this.components.registrar.register(multicodec, {
+            onConnect: this.onPeerConnected.bind(this),
+            onDisconnect: this.onPeerDisconnected.bind(this),
+            notifyOnTransient: false
         })));
-        // TODO remove/modify when https://github.com/libp2p/js-libp2p/issues/2036 is resolved
-        this.components.events.addEventListener("connection:open", (e) => {
-            if (e.detail.multiplexer === "/webrtc") {
-                this.onPeerConnected(e.detail.remotePeer, e.detail);
-            }
-        });
-        this.started = true;
         // All existing connections are like new ones for us. To deduplication on remotes so we only resuse one connection for this protocol (we could be connected with many connections)
         const peerToConnections = new Map();
         const connections = this.components.connectionManager.getConnections();
@@ -319,35 +366,24 @@ export class DirectStream extends EventEmitter {
                     break;
                 }
             }
-            await this.onPeerConnected(conn.remotePeer, conn, { fromExisting: true });
-        }
-        const pingJob = async () => {
-            // TODO don't use setInterval but waitFor previous done to be done
-            await this.pingJobPromise;
-            const promises = [];
-            this.peers.forEach((peer) => {
-                promises.push(this.ping(peer).catch((e) => {
-                    if (e instanceof TimeoutError) {
-                        // Ignore
-                    }
-                    else {
-                        logger.error(e);
-                    }
-                }));
-            });
-            promises.push(this.hello()); // Repetedly say hello to everyone to create traces in the network to measure latencies
-            this.pingJobPromise = Promise.all(promises)
-                .catch((e) => {
-                logger.error(e?.message);
-            })
-                .finally(() => {
-                if (!this.started || !this.pingInterval) {
+            await this.onPeerConnected(conn.remotePeer, conn);
+        }
+        if (this.connectionManagerOptions.pruner) {
+            const pruneConnectionsLoop = () => {
+                if (!this.connectionManagerOptions.pruner) {
                     return;
                 }
-                this.pingJob = setTimeout(pingJob, this.pingInterval);
-            });
-        };
-        pingJob();
+                this.pruneConnectionsTimeout = setTimeout(() => {
+                    this.maybePruneConnections().finally(() => {
+                        if (!this.started) {
+                            return;
+                        }
+                        pruneConnectionsLoop();
+                    });
+                }, this.connectionManagerOptions.pruner.interval);
+            };
+            pruneConnectionsLoop();
+        }
     }
     /**
      * Unregister the pubsub protocol and the streams with other peers will be closed.
@@ -357,26 +393,31 @@ export class DirectStream extends EventEmitter {
             return;
         }
         this.started = false;
-        clearTimeout(this.pingJob);
-        await this.pingJobPromise;
+        this.closeController.abort();
+        clearTimeout(this.pruneConnectionsTimeout);
         await Promise.all(this.multicodecs.map((x) => this.components.registrar.unhandle(x)));
         logger.debug("stopping");
         for (const peerStreams of this.peers.values()) {
             await peerStreams.close();
         }
+        for (const [k, v] of this.healthChecks) {
+            clearTimeout(v);
+        }
+        this.healthChecks.clear();
+        this.prunedConnectionsCache?.clear();
         // unregister protocol and handlers
         if (this._registrarTopologyIds != null) {
             this._registrarTopologyIds?.map((id) => this.components.registrar.unregister(id));
         }
         this.queue.clear();
-        this.helloMap.clear();
-        this.multiaddrsMap.clear();
-        this.earlyGoodbyes.clear();
         this.peers.clear();
         this.seenCache.clear();
         this.routes.clear();
         this.peerKeyHashToPublicKey.clear();
-        this.peerIdToPublicKey.clear();
+        for (const [k, v] of this._ackCallbacks) {
+            clearTimeout(v.timeout);
+        }
+        this._ackCallbacks.clear();
         logger.debug("stopped");
     }
     isStarted() {
@@ -396,65 +437,32 @@ export class DirectStream extends EventEmitter {
             return;
         }
         const publicKey = getPublicKeyFromPeerId(peerId);
+        if (this.prunedConnectionsCache?.has(publicKey.hashcode())) {
+            await connection.close();
+            await this.components.peerStore.delete(peerId);
+            return;
+        }
         const peer = this.addPeer(peerId, publicKey, stream.protocol, connection.id);
         const inboundStream = peer.attachInboundStream(stream);
-        this.processMessages(peerId, inboundStream, peer).catch((err) => {
-            logger.error(err);
-        });
+        this.processMessages(peer.publicKey, inboundStream, peer).catch(logError);
     }
     /**
      * Registrar notifies an established connection with protocol
      */
-    async onPeerConnected(peerId, conn, properties) {
-        if (conn.transient) {
-            return;
-        }
-        if (!this.isStarted() || conn.status !== "open") {
+    async onPeerConnected(peerId, connection) {
+        if (!this.isStarted() ||
+            connection.transient ||
+            connection.status !== "open") {
             return;
         }
-        try {
-            // TODO remove/modify when https://github.com/libp2p/js-libp2p/issues/2036 is resolved
-            let closeFn = undefined;
-            const close = () => {
-                closeFn?.();
-            };
-            this.addEventListener("close", close);
-            const result = await waitForAsync(async () => {
-                try {
-                    const hasProtocol = await this.components.peerStore
-                        .get(peerId)
-                        .then((x) => this.multicodecs.find((y) => x.protocols.includes(y)));
-                    if (!hasProtocol) {
-                        return;
-                    }
-                }
-                catch (error) {
-                    if (error.code === "ERR_NOT_FOUND") {
-                        return;
-                    }
-                    throw error;
-                }
-                return true;
-            }, {
-                delayInterval: 100,
-                timeout: 1e4,
-                stopperCallback: (cb) => {
-                    closeFn = cb;
-                }
-            }).finally(() => {
-                this.removeEventListener("close", close);
-            });
-            if (!result) {
-                return;
-            }
-        }
-        catch (error) {
-            return;
+        const peerKey = getPublicKeyFromPeerId(peerId);
+        if (this.prunedConnectionsCache?.has(peerKey.hashcode())) {
+            await connection.close();
+            await this.components.peerStore.delete(peerId);
+            return; // we recently pruned this connect, dont allow it to connect for a while
         }
         try {
-            const peerKey = getPublicKeyFromPeerId(peerId);
-            const peerKeyHash = peerKey.hashcode();
-            for (const existingStreams of conn.streams) {
+            for (const existingStreams of connection.streams) {
                 if (existingStreams.protocol &&
                     this.multicodecs.includes(existingStreams.protocol) &&
                     existingStreams.direction === "outbound") {
@@ -473,35 +481,20 @@ export class DirectStream extends EventEmitter {
                     return;
                 }
                 try {
-                    stream = await conn.newStream(this.multicodecs);
+                    stream = await connection.newStream(this.multicodecs);
                     if (stream.protocol == null) {
                         stream.abort(new Error("Stream was not multiplexed"));
                         return;
                     }
-                    peer = this.addPeer(peerId, peerKey, stream.protocol, conn.id); // TODO types
+                    peer = this.addPeer(peerId, peerKey, stream.protocol, connection.id); // TODO types
                     await peer.attachOutboundStream(stream);
-                    // TODO do we want to do this?
-                    /* if (!peer.inboundStream) {
-                        const inboundStream = conn.streams.find(
-                            (x) =>
-                                x.stat.protocol &&
-                                this.multicodecs.includes(x.stat.protocol) &&
-                                x.stat.direction === "inbound"
-                        );
-                        if (inboundStream) {
-                            this._onIncomingStream({
-                                connection: conn,
-                                stream: inboundStream,
-                            });
-                        }
-                    } */
                 }
                 catch (error) {
                     if (error.code === "ERR_UNSUPPORTED_PROTOCOL") {
                         await delay(100);
                         continue; // Retry
                     }
-                    if (conn.status !== "open" ||
+                    if (connection.status !== "open" ||
                         error?.message === "Muxer already closed" ||
                         error.code === "ERR_STREAM_RESET") {
                         return; // fail silenty
@@ -513,80 +506,12 @@ export class DirectStream extends EventEmitter {
             if (!stream) {
                 return;
             }
-            if (properties?.fromExisting) {
-                return; // we return here because we will enter this method once more once the protocol has been registered for the remote peer
-            }
-            // Add connection with assumed large latency
-            this.peerIdToPublicKey.set(peerId.toString(), peerKey);
-            const promises = [];
-            /* if (!existingStream)  */ {
-                this.addRouteConnection(this.publicKey, peerKey, Number.MAX_SAFE_INTEGER);
-                // Get accurate latency
-                promises.push(this.ping(peer));
-                // Say hello
-                promises.push(this.publishMessage(this.components.peerId, await new Hello({
-                    multiaddrs: this.components.addressManager
-                        .getAddresses()
-                        .map((x) => x.toString())
-                }).sign(this.sign), [peer]));
-                // Send my goodbye early if I disconnect for some reason, (so my peer can say goodbye for me)
-                // TODO add custom condition fn for doing below
-                promises.push(this.publishMessage(this.components.peerId, await new Goodbye({ early: true }).sign(this.sign), [peer]));
-                // replay all hellos
-                for (const [sender, hellos] of this.helloMap) {
-                    if (sender === peerKeyHash) {
-                        // Don't say hellos from sender to same sender (uneccessary)
-                        continue;
-                    }
-                    outer: for (const [key, hello] of hellos) {
-                        if (!hello.header.verify()) {
-                            hellos.delete(key);
-                        }
-                        for (const signer of hello.signatures.publicKeys) {
-                            if (!this.routes.hasNode(signer.hashcode())) {
-                                // purge this hello since it has travelled a path that no longer exist
-                                hellos.delete(key);
-                                continue outer;
-                            }
-                        }
-                        promises.push(this.publishMessage(this.components.peerId, hello, [peer]));
-                    }
-                }
-            }
-            const resolved = await Promise.all(promises);
-            return resolved;
+            this.addRouteConnection(this.publicKeyHash, peerKey.hashcode(), peerKey, 0, +new Date());
         }
         catch (err) {
             logger.error(err);
         }
     }
-    addRouteConnection(from, to, latency) {
-        this.peerKeyHashToPublicKey.set(from.hashcode(), from);
-        this.peerKeyHashToPublicKey.set(to.hashcode(), to);
-        const links = this.routes.addLink(from.hashcode(), to.hashcode(), latency);
-        for (const added of links) {
-            const key = this.peerKeyHashToPublicKey.get(added);
-            if (key?.equals(this.publicKey) === false) {
-                this.onPeerReachable(key);
-            }
-        }
-    }
-    removeRouteConnection(from, to) {
-        const has = this.routes.hasNode(to.hashcode());
-        if (!has) {
-            this.onPeerUnreachable(to);
-        }
-        else {
-            const links = this.routes.deleteLink(from.hashcode(), to.hashcode());
-            for (const deleted of links) {
-                const key = this.peerKeyHashToPublicKey.get(deleted);
-                this.peerKeyHashToPublicKey.delete(deleted);
-                if (key?.equals(this.publicKey) === false) {
-                    this.onPeerUnreachable(key);
-                }
-            }
-        }
-    }
     /**
      * Registrar notifies a closing connection with pubsub protocol
      */
@@ -608,19 +533,51 @@ export class DirectStream extends EventEmitter {
         }
         if (!this.publicKey.equals(peerKey)) {
             await this._removePeer(peerKey);
-            this.removeRouteConnection(this.publicKey, peerKey);
             // Notify network
-            const earlyGoodBye = this.earlyGoodbyes.get(peerKeyHash);
-            if (earlyGoodBye) {
-                earlyGoodBye.early = false;
-                await earlyGoodBye.sign(this.sign);
-                await this.publishMessage(this.components.peerId, earlyGoodBye);
-                this.earlyGoodbyes.delete(peerKeyHash);
+            const dependent = this.routes.getDependent(peerKeyHash);
+            this.removeRouteConnection(peerKeyHash, true);
+            if (dependent.length > 0) {
+                await this.publishMessage(this.publicKey, await new Goodbye({
+                    leaving: [peerKeyHash],
+                    header: new MessageHeader({
+                        mode: new SilentDelivery({ to: dependent, redundancy: 2 })
+                    })
+                }).sign(this.sign));
             }
-            this.peerIdToPublicKey.delete(peerId.toString());
         }
         logger.debug("connection ended:" + peerKey.toString());
     }
+    removeRouteConnection(hash, neigbour) {
+        const unreachable = neigbour
+            ? this.routes.removeNeighbour(hash)
+            : this.routes.removeTarget(hash);
+        for (const node of unreachable) {
+            this.onPeerUnreachable(node); // TODO types
+            this.peerKeyHashToPublicKey.delete(node);
+        }
+    }
+    addRouteConnection(from, neighbour, target, distance, session, pending = false) {
+        const targetHash = typeof target === "string" ? target : target.hashcode();
+        const wasReachable = from === this.publicKeyHash
+            ? this.routes.isReachable(from, targetHash)
+            : true;
+        if (pending) {
+            this.routes.addPendingRouteConnection(session, {
+                distance,
+                from,
+                neighbour,
+                target: targetHash
+            });
+        }
+        else {
+            this.routes.add(from, neighbour, targetHash, distance, session);
+        }
+        const newPeer = wasReachable === false && this.routes.isReachable(from, targetHash);
+        if (newPeer) {
+            this.peerKeyHashToPublicKey.set(target.hashcode(), target);
+            this.onPeerReachable(target); // TODO types
+        }
+    }
     /**
      * invoked when a new peer becomes reachable
      * @param publicKeyHash
@@ -633,17 +590,24 @@ export class DirectStream extends EventEmitter {
      * invoked when a new peer becomes unreachable
      * @param publicKeyHash
      */
-    onPeerUnreachable(publicKey) {
+    onPeerUnreachable(hash) {
         // override this fn
-        this.helloMap.delete(publicKey.hashcode());
-        this.multiaddrsMap.delete(publicKey.hashcode());
-        this.dispatchEvent(new CustomEvent("peer:unreachable", { detail: publicKey }));
+        this.dispatchEvent(
+        // TODO types
+        new CustomEvent("peer:unreachable", {
+            detail: this.peerKeyHashToPublicKey.get(hash)
+        }));
     }
     /**
      * Notifies the router that a peer has been connected
      */
     addPeer(peerId, publicKey, protocol, connId) {
         const publicKeyHash = publicKey.hashcode();
+        const hc = this.healthChecks.get(publicKeyHash);
+        if (hc) {
+            clearTimeout(hc);
+            this.healthChecks.delete(publicKeyHash);
+        }
         const existing = this.peers.get(publicKeyHash);
         // If peer streams already exists, do nothing
         if (existing != null) {
@@ -690,32 +654,7 @@ export class DirectStream extends EventEmitter {
         try {
             await pipe(stream, async (source) => {
                 for await (const data of source) {
-                    const msgId = await getMsgId(data);
-                    if (this.seenCache.has(msgId)) {
-                        // we got message that WE sent?
-                        /**
-                             * Most propobable reason why we arrive  here is a race condition/issue
-                            
-                            ┌─┐
-                            │0│
-                            └△┘
-                            ┌▽┐
-                            │1│
-                            └△┘
-                            ┌▽┐
-                            │2│
-                            └─┘
-                            
-                            from 2s perspective,
-    
-                            if everyone conents to each other at the same time, then 0 will say hello to 1 and 1 will save that hello to resend to 2 if 2 ever connects
-                            but two is already connected by onPeerConnected has not been invoked yet, so the hello message gets forwarded,
-                            and later onPeerConnected gets invoked on 1, and the same message gets resent to 2
-                             */
-                        continue;
-                    }
-                    this.seenCache.add(msgId);
-                    this.processRpc(peerId, peerStreams, data).catch((err) => logger.warn(err));
+                    this.processRpc(peerId, peerStreams, data).catch(logError);
                 }
             });
         }
@@ -731,10 +670,6 @@ export class DirectStream extends EventEmitter {
      * Handles an rpc request from a peer
      */
     async processRpc(from, peerStreams, message) {
-        if (!this.acceptFrom(from)) {
-            logger.debug("received message from unacceptable peer %p", from);
-            return false;
-        }
         // logger.debug("rpc from " + from + ", " + this.peerIdStr);
         if (message.length > 0) {
             //	logger.debug("messages from " + from);
@@ -747,18 +682,21 @@ export class DirectStream extends EventEmitter {
                     logger.error(err);
                 }
             })
-                .catch((err) => logger.warn(err));
+                .catch(logError);
         }
         return true;
     }
+    async modifySeenCache(message, getIdFn = getMsgId) {
+        const msgId = await getIdFn(message);
+        const seen = this.seenCache.get(msgId);
+        this.seenCache.add(msgId, seen ? seen + 1 : 1);
+        return seen || 0;
+    }
     /**
      * Handles a message from a peer
      */
     async processMessage(from, peerStream, msg) {
-        if (!from.publicKey) {
-            return;
-        }
-        if (this.components.peerId.equals(from) && !this.emitSelf) {
+        if (!this.started) {
             return;
         }
         // Ensure the message is valid before processing it
@@ -767,208 +705,226 @@ export class DirectStream extends EventEmitter {
             detail: message
         }));
         if (message instanceof DataMessage) {
-            await this.onDataMessage(from, peerStream, message);
+            // DONT await this since it might introduce a dead-lock
+            this._onDataMessage(from, peerStream, msg, message).catch(logError);
         }
-        else if (message instanceof Hello) {
-            await this.onHello(from, peerStream, message);
+        else {
+            if (message instanceof ACK) {
+                this.onAck(from, peerStream, msg, message).catch(logError);
+            }
+            else if (message instanceof Goodbye) {
+                this.onGoodBye(from, peerStream, msg, message).catch(logError);
+            }
+            else {
+                throw new Error("Unsupported message type");
+            }
         }
-        else if (message instanceof Goodbye) {
-            await this.onGoodbye(from, peerStream, message);
+    }
+    shouldIgnore(message, seenBefore) {
+        const fromMe = message.header.signatures?.publicKeys.find((x) => x.equals(this.publicKey));
+        if (fromMe) {
+            return true;
+        }
+        if ((seenBefore > 0 &&
+            message.header.mode instanceof SeekDelivery === false) ||
+            (message.header.mode instanceof SeekDelivery &&
+                seenBefore >= message.header.mode.redundancy)) {
+            return true;
+        }
+        return false;
+    }
+    async onDataMessage(from, peerStream, message, seenBefore) {
+        if (this.shouldIgnore(message, seenBefore)) {
+            return false;
         }
-        else if (message instanceof PingPong) {
-            await this.onPing(from, peerStream, message);
+        let isForMe = false;
+        if (message.header.mode instanceof AnyWhere) {
+            isForMe = true;
         }
         else {
-            throw new Error("Unsupported");
+            const isFromSelf = this.publicKey.equals(from);
+            if (!isFromSelf) {
+                isForMe =
+                    message.header.mode.to == null ||
+                        message.header.mode.to.find((x) => x === this.publicKeyHash) != null;
+            }
         }
-    }
-    async onDataMessage(from, peerStream, message) {
-        const isFromSelf = this.components.peerId.equals(from);
-        if (!isFromSelf || this.emitSelf) {
-            const isForAll = message.to.length === 0;
-            const isForMe = !isForAll && message.to.find((x) => x === this.publicKeyHash);
-            if (isForAll || isForMe) {
-                if (this.signaturePolicy === "StictSign" &&
-                    !(await message.verify(this.signaturePolicy === "StictSign"))) {
-                    // we don't verify messages we don't dispatch because of the performance penalty // TODO add opts for this
-                    logger.warn("Recieved message with invalid signature or timestamp");
-                    return false;
-                }
+        if (isForMe) {
+            if ((await this.maybeVerifyMessage(message)) === false) {
+                // we don't verify messages we don't dispatch because of the performance penalty // TODO add opts for this
+                logger.warn("Recieved message with invalid signature or timestamp");
+                return false;
+            }
+            await this.acknowledgeMessage(peerStream, message, seenBefore);
+            if (seenBefore === 0 && message.data) {
                 this.dispatchEvent(new CustomEvent("data", {
                     detail: message
                 }));
             }
-            if (isForMe && message.to.length === 1) {
+        }
+        if (message.header.mode instanceof SilentDelivery ||
+            message.header.mode instanceof AcknowledgeDelivery) {
+            if (message.header.mode.to &&
+                message.header.mode.to.length === 1 &&
+                message.header.mode.to[0] === this.publicKeyHash) {
                 // dont forward this message anymore because it was meant ONLY for me
                 return true;
             }
         }
         // Forward
-        await this.relayMessage(from, message);
-        return true;
-    }
-    async onHello(from, peerStream, message) {
-        if (!(await message.verify(false))) {
-            const a = message.header.verify();
-            const b = message.networkInfo.pingLatencies.length ===
-                message.signatures.signatures.length - 1;
-            logger.warn(`Recieved hello message that did not verify. Header: ${a}, Ping info ${b}, Signatures ${a && b}`);
-            return false;
-        }
-        const sender = message.sender?.hashcode();
-        if (!sender) {
-            logger.warn("Recieved hello without sender");
-            return false;
-        }
-        const signatures = message.signatures;
-        for (let i = 0; i < signatures.signatures.length - 1; i++) {
-            this.addRouteConnection(signatures.signatures[i].publicKey, signatures.signatures[i + 1].publicKey, message.networkInfo.pingLatencies[i]);
-        }
-        message.networkInfo.pingLatencies.push(peerStream.pingLatency || 4294967295); // TODO don't propagate if latency is high?
-        await message.sign(this.sign); // sign it so othere peers can now I have seen it (and can build a network graph from trace info)
-        let hellos = this.helloMap.get(sender);
-        if (!hellos) {
-            hellos = new Map();
-            this.helloMap.set(sender, hellos);
-        }
-        this.multiaddrsMap.set(sender, message.multiaddrs);
-        const helloSignaturHash = await message.signatures.hashPublicKeys();
-        const existingHello = hellos.get(helloSignaturHash);
-        if (existingHello) {
-            if (existingHello.header.expires < message.header.expires) {
-                hellos.set(helloSignaturHash, message);
+        if (message.header.mode instanceof SeekDelivery || seenBefore === 0) {
+            // DONT await this since it might introduce a dead-lock
+            if (message.header.mode instanceof SeekDelivery) {
+                if (seenBefore < message.header.mode.redundancy) {
+                    const to = [...this.peers.values()].filter((x) => !message.header.signatures?.publicKeys.find((y) => y.equals(x.publicKey)) && x != peerStream);
+                    if (to.length > 0) {
+                        this.relayMessage(from, message, to);
+                    }
+                }
+            }
+            else {
+                this.relayMessage(from, message);
             }
         }
-        else {
-            hellos.set(helloSignaturHash, message);
+    }
+    async maybeVerifyMessage(message) {
+        return message.verify(this.signaturePolicy === "StictSign");
+    }
+    async acknowledgeMessage(peerStream, message, seenBefore) {
+        if ((message.header.mode instanceof SeekDelivery ||
+            message.header.mode instanceof AcknowledgeDelivery) &&
+            seenBefore < message.header.mode.redundancy) {
+            await this.publishMessage(this.publicKey, await new ACK({
+                messageIdToAcknowledge: message.id,
+                seenCounter: seenBefore,
+                // TODO only give origin info to peers we want to connect to us
+                header: new MessageHeader({
+                    mode: new TracedDelivery(message.header.signatures.publicKeys.map((x) => x.hashcode())),
+                    // include our origin if message is SeekDelivery and we have not recently pruned a connection to this peer
+                    origin: message.header.mode instanceof SeekDelivery &&
+                        !message.header.signatures.publicKeys.find((x) => this.prunedConnectionsCache?.has(x.hashcode()))
+                        ? new MultiAddrinfo(this.components.addressManager
+                            .getAddresses()
+                            .map((x) => x.toString()))
+                        : undefined
+                })
+            }).sign(this.sign), [peerStream]);
         }
-        // Forward
-        await this.relayMessage(from, message);
-        return true;
     }
-    async onGoodbye(from, peerStream, message) {
-        if (!(await message.verify(false))) {
-            logger.warn("Recieved message with invalid signature or timestamp");
+    async _onDataMessage(from, peerStream, messageBytes, message) {
+        const seenBefore = await this.modifySeenCache(messageBytes instanceof Uint8ArrayList
+            ? messageBytes.subarray()
+            : messageBytes);
+        return this.onDataMessage(from, peerStream, message, seenBefore);
+    }
+    async onAck(publicKey, peerStream, messageBytes, message) {
+        const seenBefore = await this.modifySeenCache(messageBytes instanceof Uint8Array
+            ? messageBytes
+            : messageBytes.subarray(), (bytes) => sha256Base64(bytes));
+        if (seenBefore > 1) {
+            logger.debug("Received message already seen of type: " + message.constructor.name);
             return false;
         }
-        const sender = message.sender?.hashcode();
-        if (!sender) {
-            logger.warn("Recieved hello without sender");
+        if (!(await message.verify(true))) {
+            logger.warn(`Recieved ACK message that did not verify`);
             return false;
         }
-        const peerKey = getPublicKeyFromPeerId(from);
-        const peerKeyHash = peerKey.hashcode();
-        if (message.early) {
-            this.earlyGoodbyes.set(peerKeyHash, message);
+        const messageIdString = toBase64(message.messageIdToAcknowledge);
+        const myIndex = message.header.mode.trace.findIndex((x) => x === this.publicKeyHash);
+        const next = message.header.mode.trace[myIndex - 1];
+        const nextStream = next ? this.peers.get(next) : undefined;
+        this._ackCallbacks
+            .get(messageIdString)
+            ?.callback(message, peerStream, nextStream);
+        // relay ACK ?
+        // send exactly backwards same route we got this message
+        if (nextStream) {
+            await this.publishMessage(this.publicKey, message, [nextStream], true);
         }
         else {
-            const signatures = message.signatures;
-            /*  TODO Should we update routes on goodbye?
-            for (let i = 1; i < signatures.signatures.length - 1; i++) {
-                this.addRouteConnection(
-                    signatures.signatures[i].publicKey,
-                    signatures.signatures[i + 1].publicKey
-                );
+            // if origin exist (we can connect to remote peer) && we have autodialer turned on
+            if (message.header.origin && this.connectionManagerOptions.dialer) {
+                this.maybeConnectDirectly(message.header.signatures.publicKeys[0].hashcode(), message.header.origin);
             }
-            */
-            //let neighbour = message.trace[1] || this.peerIdStr;
-            this.removeRouteConnection(signatures.signatures[0].publicKey, signatures.signatures[1].publicKey || this.publicKey);
-            const relayToPeers = [];
-            for (const stream of this.peers.values()) {
-                if (stream.peerId.equals(from)) {
-                    continue;
-                }
-                relayToPeers.push(stream);
-            }
-            await message.sign(this.sign); // sign it so othere peers can now I have seen it (and can build a network graph from trace info)
-            const hellos = this.helloMap.get(sender);
-            if (hellos) {
-                const helloSignaturHash = await message.signatures.hashPublicKeys();
-                hellos.delete(helloSignaturHash);
-            }
-            // Forward
-            await this.relayMessage(from, message);
         }
-        return true;
     }
-    async onPing(from, peerStream, message) {
-        if (message instanceof Ping) {
-            // respond with pong
-            await this.publishMessage(this.components.peerId, new Pong(message.pingBytes), [peerStream]);
-        }
-        else if (message instanceof Pong) {
-            // Let the (waiting) thread know that we have received the pong
-            peerStream.pingJob?.resolve();
+    async onGoodBye(publicKey, peerStream, messageBytes, message) {
+        const seenBefore = await this.modifySeenCache(messageBytes instanceof Uint8Array
+            ? messageBytes
+            : messageBytes.subarray());
+        if (seenBefore > 0) {
+            logger.debug("Received message already seen of type: " + message.constructor.name);
+            return;
         }
-        else {
-            throw new Error("Unsupported");
+        if (!(await message.verify(true))) {
+            logger.warn(`Recieved ACK message that did not verify`);
+            return false;
         }
-    }
-    async ping(stream) {
-        return new Promise((resolve, reject) => {
-            stream.pingJob?.abort();
-            const ping = new Ping();
-            const start = +new Date();
-            const timeout = setTimeout(() => {
-                reject(new TimeoutError("Ping timed out"));
-            }, 10000);
-            const resolver = () => {
-                const end = +new Date();
-                clearTimeout(timeout);
-                // TODO what happens if a peer send a ping back then leaves? Any problems?
-                const latency = end - start;
-                stream.pingLatency = latency;
-                this.addRouteConnection(this.publicKey, stream.publicKey, latency);
-                resolve(undefined);
-            };
-            stream.pingJob = {
-                resolve: resolver,
-                abort: () => {
-                    clearTimeout(timeout);
-                    resolve(undefined);
+        const filteredLeaving = message.leaving.filter((x) => this.routes.hasTarget(x));
+        if (filteredLeaving.length > 0) {
+            this.publish(new Uint8Array(0), {
+                mode: new SeekDelivery({
+                    to: filteredLeaving,
+                    redundancy: DEFAULT_SEEK_MESSAGE_REDUDANCY
+                })
+            }).catch((e) => {
+                if (e instanceof TimeoutError || e instanceof AbortError) {
+                    // peer left or closed
                 }
-            };
-            this.publishMessage(this.components.peerId, ping, [stream]).catch((err) => {
-                clearTimeout(timeout);
-                reject(err);
-            });
-        });
-    }
-    /**
-     * Whether to accept a message from a peer
-     * Override to create a graylist
-     */
-    acceptFrom(id) {
-        return true;
+                else {
+                    throw e;
+                }
+            }); // this will remove the target if it is still not reable
+        }
+        for (const target of message.leaving) {
+            // relay message to every one who previously talked to this peer
+            const dependent = this.routes.getDependent(target);
+            message.header.mode.to = [...message.header.mode.to, ...dependent];
+            message.header.mode.to = message.header.mode.to.filter((x) => x !== this.publicKeyHash);
+            if (message.header.mode.to.length > 0) {
+                await this.publishMessage(publicKey, message, undefined, true);
+            }
+        }
     }
     async createMessage(data, options) {
         // dispatch the event if we are interested
-        let toHashes;
-        if (options?.to) {
-            if (options.to instanceof Set) {
-                toHashes = new Array(options.to.size);
-            }
-            else {
-                toHashes = new Array(options.to.length);
-            }
-            let i = 0;
-            for (const to of options.to) {
-                toHashes[i++] =
-                    to instanceof PublicSignKey
-                        ? to.hashcode()
-                        : typeof to === "string"
-                            ? to
-                            : getPublicKeyFromPeerId(to).hashcode();
+        let mode = options.mode
+            ? options.mode
+            : new SilentDelivery({
+                to: options.to,
+                redundancy: DEFAULT_SILENT_MESSAGE_REDUDANCY
+            });
+        if (mode instanceof AcknowledgeDelivery ||
+            mode instanceof SilentDelivery ||
+            mode instanceof SeekDelivery) {
+            if (mode.to?.find((x) => x === this.publicKeyHash)) {
+                mode.to = mode.to.filter((x) => x !== this.publicKeyHash);
             }
         }
-        else {
-            toHashes = [];
+        if (mode instanceof AcknowledgeDelivery || mode instanceof SilentDelivery) {
+            const now = +new Date();
+            for (const hash of mode.to) {
+                const neighbourRoutes = this.routes.routes
+                    .get(this.publicKeyHash)
+                    ?.get(hash);
+                if (!neighbourRoutes ||
+                    now - neighbourRoutes.session >
+                        neighbourRoutes.list.length * this.routeSeekInterval) {
+                    mode = new SeekDelivery({
+                        to: mode.to,
+                        redundancy: DEFAULT_SEEK_MESSAGE_REDUDANCY
+                    });
+                    break;
+                }
+            }
         }
         const message = new DataMessage({
             data: data instanceof Uint8ArrayList ? data.subarray() : data,
-            to: toHashes
+            header: new MessageHeader({ mode })
         });
-        if (this.signaturePolicy === "StictSign") {
+        if (this.signaturePolicy === "StictSign" ||
+            mode instanceof SeekDelivery ||
+            mode instanceof AcknowledgeDelivery) {
             await message.sign(this.sign);
         }
         return message;
@@ -976,122 +932,195 @@ export class DirectStream extends EventEmitter {
     /**
      * Publishes messages to all peers
      */
-    async publish(data, options) {
+    async publish(data, options = {
+        mode: new SeekDelivery({ redundancy: DEFAULT_SEEK_MESSAGE_REDUDANCY })
+    }) {
         if (!this.started) {
             throw new Error("Not started");
         }
         const message = await this.createMessage(data, options);
-        if (this.emitSelf) {
-            super.dispatchEvent(new CustomEvent("data", {
-                detail: message
-            }));
-        }
-        // send to all the other peers
-        await this.publishMessage(this.components.peerId, message, undefined);
+        await this.publishMessage(this.publicKey, message, undefined);
         return message.id;
     }
-    async hello(data) {
-        if (!this.started) {
-            return;
-        }
-        // send to all the other peers
-        await this.publishMessage(this.components.peerId, await new Hello({
-            multiaddrs: this.components.addressManager
-                .getAddresses()
-                .map((x) => x.toString()),
-            data
-        }).sign(this.sign.bind(this)));
-    }
     async relayMessage(from, message, to) {
         if (this.canRelayMessage) {
+            if (message instanceof DataMessage) {
+                if (message.header.mode instanceof AcknowledgeDelivery ||
+                    message.header.mode instanceof SeekDelivery) {
+                    await message.sign(this.sign);
+                }
+            }
             return this.publishMessage(from, message, to, true);
         }
         else {
-            logger.debug("received message we didn't subscribe to. Dropping.");
+            logger.debug("Received a message to relay but canRelayMessage is false");
         }
     }
-    async publishMessage(from, message, to, relayed) {
-        if (message instanceof DataMessage && !to) {
-            // message.to can be distant peers, but "to" are neighbours
-            const fanoutMap = new Map();
-            // Message to > 0
-            if (message.to.length > 0) {
-                const missingPathsFor = [];
-                for (const to of message.to) {
-                    const fromKey = this.peerIdToPublicKey
-                        .get(from.toString())
-                        ?.hashcode();
-                    if (to === this.publicKeyHash || fromKey === to) {
-                        continue; // don't send to me or backwards
-                    }
-                    const directStream = this.peers.get(to);
-                    if (directStream) {
-                        // always favor direct stream, even path seems longer
-                        const fanout = fanoutMap.get(to);
-                        if (!fanout) {
-                            fanoutMap.set(to, [to]);
-                        }
-                        else {
-                            fanout.push(to);
-                        }
-                        continue;
-                    }
-                    else {
-                        const fromMe = from.equals(this.components.peerId);
-                        const block = !fromMe ? fromKey : undefined;
-                        const path = this.routes.getPath(this.publicKeyHash, to, {
-                            block // prevent send message backwards
-                        });
-                        if (path && path.length > 0) {
-                            const fanout = fanoutMap.get(path[1]);
-                            if (!fanout) {
-                                fanoutMap.set(path[1], [to]);
-                                if (this.connectionManagerOptions.autoDial &&
-                                    path.length >= 3) {
-                                    // Dont await this even if it is async since this method can fail
-                                    // and might take some time to run
-                                    this.maybeConnectDirectly(path).catch((e) => {
-                                        logger.error("Failed to request direct connection: " + e.message);
-                                    });
-                                }
-                                continue;
-                            }
-                            else {
-                                fanout.push(to);
-                                continue;
+    async createDeliveryPromise(from, message, relayed) {
+        if (message.header.mode instanceof AnyWhere) {
+            return { promise: Promise.resolve() };
+        }
+        const idString = toBase64(message.id);
+        const existing = this._ackCallbacks.get(idString);
+        if (existing) {
+            return { promise: existing.promise };
+        }
+        const deliveryDeferredPromise = pDefer();
+        const fastestNodesReached = new Map();
+        const messageToSet = new Set();
+        if (message.header.mode.to) {
+            for (const to of message.header.mode.to) {
+                if (to === from.hashcode()) {
+                    continue;
+                }
+                messageToSet.add(to);
+                if (!relayed && !this.healthChecks.has(to)) {
+                    this.healthChecks.set(to, setTimeout(() => {
+                        this.removeRouteConnection(to, false);
+                    }, this.seekTimeout + 5000));
+                }
+            }
+        }
+        if (messageToSet.size === 0) {
+            deliveryDeferredPromise.resolve(); // we dont know how many answer to expect, just resolve immediately
+        }
+        const willGetAllAcknowledgements = !relayed; // Only the origin will get all acks
+        // Expected to receive at least 'filterMessageForSeenCounter' acknowledgements from each peer
+        const filterMessageForSeenCounter = relayed
+            ? undefined
+            : message.header.mode instanceof SeekDelivery
+                ? Math.min(this.peers.size, message.header.mode.redundancy)
+                : 1; /*  message.deliveryMode instanceof SeekDelivery ? Math.min(this.peers.size - (relayed ? 1 : 0), message.deliveryMode.redundancy) : 1 */
+        const finalize = () => {
+            this._ackCallbacks.delete(idString);
+            if (message.header.mode instanceof SeekDelivery) {
+                this.routes.commitPendingRouteConnection(session);
+            }
+        };
+        const uniqueAcks = new Set();
+        const session = +new Date();
+        const timeout = setTimeout(async () => {
+            let hasAll = true;
+            finalize();
+            // peer not reachable (?)!
+            for (const to of messageToSet) {
+                let foundNode = false;
+                if (fastestNodesReached.has(to)) {
+                    foundNode = true;
+                    break;
+                }
+                if (!foundNode && !relayed) {
+                    hasAll = false;
+                }
+            }
+            if (!hasAll && willGetAllAcknowledgements) {
+                deliveryDeferredPromise.reject(new TimeoutError(`${this.publicKeyHash} Failed to get message ${idString} ${filterMessageForSeenCounter} ${[
+                    ...messageToSet
+                ]} delivery acknowledges from all nodes (${fastestNodesReached.size}/${messageToSet.size}). Mode: ${message.header.mode.constructor.name}`));
+            }
+            else {
+                deliveryDeferredPromise.resolve();
+            }
+        }, this.seekTimeout);
+        this._ackCallbacks.set(idString, {
+            promise: deliveryDeferredPromise.promise,
+            callback: (ack, messageThrough, messageFrom) => {
+                const messageTarget = ack.header.signatures.publicKeys[0];
+                const messageTargetHash = messageTarget.hashcode();
+                const seenCounter = ack.seenCounter;
+                // remove the automatic removal of route timeout since we have observed lifesigns of a peer
+                const timer = this.healthChecks.get(messageTargetHash);
+                clearTimeout(timer);
+                this.healthChecks.delete(messageTargetHash);
+                // if the target is not inside the original message to, we still ad the target to our routes
+                // this because a relay might modify the 'to' list and we might receive more answers than initially set
+                if (message.header.mode instanceof SeekDelivery) {
+                    this.addRouteConnection(messageFrom?.publicKey.hashcode() || this.publicKeyHash, messageThrough.publicKey.hashcode(), messageTarget, seenCounter, session, true); // we assume the seenCounter = distance. The more the message has been seen by the target the longer the path is to the target
+                }
+                if (messageToSet.has(messageTargetHash)) {
+                    // Only keep track of relevant acks
+                    if (filterMessageForSeenCounter == null ||
+                        seenCounter < filterMessageForSeenCounter) {
+                        // TODO set limit correctly
+                        if (seenCounter < MAX_ROUTE_DISTANCE) {
+                            let arr = fastestNodesReached.get(messageTargetHash);
+                            if (!arr) {
+                                arr = [];
+                                fastestNodesReached.set(messageTargetHash, arr);
                             }
-                        }
-                        else {
-                            missingPathsFor.push(to);
+                            arr.push(seenCounter);
+                            uniqueAcks.add(messageTargetHash + seenCounter);
                         }
                     }
-                    // we can't find path, send message to all peers
-                    fanoutMap.clear();
-                    break;
                 }
-                // update to's
-                let sentOnce = false;
-                if (missingPathsFor.length === 0) {
-                    if (fanoutMap.size > 0) {
-                        for (const [neighbour, distantPeers] of fanoutMap) {
-                            message.to = distantPeers;
-                            const bytes = message.bytes();
-                            if (!sentOnce) {
-                                // if relayed = true, we have already added it to seenCache
-                                if (!relayed) {
-                                    this.seenCache.add(await getMsgId(bytes));
-                                }
-                                sentOnce = true;
-                            }
+                // This if clause should never enter for relayed connections, since we don't
+                // know how many ACKs we will get
+                if (filterMessageForSeenCounter != null &&
+                    uniqueAcks.size >= messageToSet.size * filterMessageForSeenCounter) {
+                    if (messageToSet.size > 0) {
+                        // this statement exist beacuse if we do SEEK and have to = [], then it means we try to reach as many as possible hence we never want to delete this ACK callback
+                        // only remove callback function if we actually expected a expected amount of responses
+                        clearTimeout(timeout);
+                        finalize();
+                    }
+                    deliveryDeferredPromise.resolve();
+                }
+            },
+            timeout
+        });
+        return deliveryDeferredPromise;
+    }
+    async publishMessage(from, message, to, relayed) {
+        let delivereyPromise = undefined;
+        if ((!message.header.signatures ||
+            message.header.signatures.publicKeys.length === 0) &&
+            message instanceof DataMessage &&
+            message.header.mode instanceof SilentDelivery === false) {
+            throw new Error("Missing signature");
+        }
+        /**
+         * Logic for handling acknowledge messages when we receive them (later)
+         */
+        if (message instanceof DataMessage &&
+            message.header.mode instanceof SeekDelivery &&
+            !relayed) {
+            to = this.peers; // seek delivery will not work unless we try all possible paths
+        }
+        if (message instanceof AcknowledgeDelivery) {
+            to = undefined;
+        }
+        if (message instanceof DataMessage &&
+            (message.header.mode instanceof SeekDelivery ||
+                message.header.mode instanceof AcknowledgeDelivery)) {
+            delivereyPromise = (await this.createDeliveryPromise(from, message, relayed)).promise;
+        }
+        const bytes = message.bytes();
+        if (!relayed) {
+            const bytesArray = bytes instanceof Uint8Array ? bytes : bytes.subarray();
+            await this.modifySeenCache(bytesArray);
+        }
+        /**
+         * For non SEEKing message delivery modes, use routing
+         */
+        if (message instanceof DataMessage) {
+            if ((message.header.mode instanceof AcknowledgeDelivery ||
+                message.header.mode instanceof SilentDelivery) &&
+                !to &&
+                message.header.mode.to) {
+                const fanout = this.routes.getFanout(from, message.header.mode.to, message.header.mode.redundancy);
+                if (fanout) {
+                    if (fanout.size > 0) {
+                        for (const [neighbour, _distantPeers] of fanout) {
                             const stream = this.peers.get(neighbour);
                             stream?.waitForWrite(bytes).catch((e) => {
                                 logger.error("Failed to publish message: " + e.message);
                             });
                         }
-                        return; // we are done sending the message in all direction with updates 'to' lists
+                        return delivereyPromise; // we are done sending the message in all direction with updates 'to' lists
                     }
-                    return;
+                    return; // we defintely that we should not forward the message anywhere
                 }
+                return;
                 // else send to all (fallthrough to code below)
             }
         }
@@ -1100,108 +1129,64 @@ export class DirectStream extends EventEmitter {
         if (peers == null ||
             (Array.isArray(peers) && peers.length === 0) ||
             (peers instanceof Map && peers.size === 0)) {
-            logger.debug("no peers are subscribed");
+            logger.debug("No peers to send to");
             return;
         }
-        const bytes = message.bytes();
         let sentOnce = false;
         for (const stream of peers.values()) {
             const id = stream;
-            if (id.peerId.equals(from)) {
+            if (id.publicKey.equals(from)) {
                 continue;
             }
-            if (!sentOnce) {
-                sentOnce = true;
-                if (!relayed) {
-                    // if relayed = true, we have already added it to seenCache
-                    const msgId = await getMsgId(bytes);
-                    this.seenCache.add(msgId);
-                }
-            }
+            sentOnce = true;
             id.waitForWrite(bytes).catch((e) => {
                 logger.error("Failed to publish message: " + e.message);
             });
         }
-        if (!sentOnce && !relayed) {
-            throw new Error("Message did not have any valid receivers. ");
+        if (!sentOnce) {
+            if (!relayed) {
+                throw new Error("Message did not have any valid receivers");
+            }
         }
+        return delivereyPromise;
     }
-    async maybeConnectDirectly(path) {
-        if (path.length < 3) {
-            return;
-        }
-        const toHash = path[path.length - 1];
-        if (this.peers.has(toHash)) {
+    async maybeConnectDirectly(toHash, origin) {
+        if (this.peers.has(toHash) || this.prunedConnectionsCache?.has(toHash)) {
             return; // TODO, is this expected, or are we to dial more addresses?
         }
-        // Try to either connect directly
-        if (!this.recentDials.has(toHash)) {
-            this.recentDials.add(toHash);
-            const addrs = this.multiaddrsMap.get(toHash);
+        const addresses = origin.multiaddrs
+            .filter((x) => {
+            const ret = !this.recentDials.has(x);
+            this.recentDials.add(x);
+            return ret;
+        })
+            .map((x) => multiaddr(x));
+        if (addresses.length > 0) {
             try {
-                if (addrs && addrs.length > 0) {
-                    await this.components.connectionManager.openConnection(addrs.map((x) => multiaddr(x)));
-                    return;
-                }
+                await this.components.connectionManager.openConnection(addresses);
             }
             catch (error) {
-                // continue regardless of error
-            }
-        }
-        // Connect through a closer relay that maybe does holepunch for us
-        const nextToHash = path[path.length - 2];
-        const routeKey = nextToHash + toHash;
-        if (!this.recentDials.has(routeKey)) {
-            this.recentDials.add(routeKey);
-            const to = this.peerKeyHashToPublicKey.get(toHash);
-            const toPeerId = await to.toPeerId();
-            const addrs = this.multiaddrsMap.get(path[path.length - 2]);
-            if (addrs && addrs.length > 0) {
-                const addressesToDial = addrs.sort((a, b) => {
-                    if (a.includes("/wss/")) {
-                        if (b.includes("/wss/")) {
-                            return 0;
-                        }
-                        return -1;
-                    }
-                    if (a.includes("/ws/")) {
-                        if (b.includes("/ws/")) {
-                            return 0;
-                        }
-                        if (b.includes("/wss/")) {
-                            return 1;
-                        }
-                        return -1;
-                    }
-                    return 0;
-                });
-                for (const addr of addressesToDial) {
-                    const circuitAddress = multiaddr(addr + "/p2p-circuit/webrtc/p2p/" + toPeerId.toString());
-                    try {
-                        await this.components.connectionManager.openConnection(circuitAddress);
-                        return;
-                    }
-                    catch (error) {
-                        logger.error("Failed to connect directly to: " +
-                            circuitAddress.toString() +
-                            ". " +
-                            error?.message);
-                    }
-                }
+                logger.info("Failed to connect directly to: " +
+                    JSON.stringify(addresses.map((x) => x.toString())) +
+                    ". " +
+                    error?.message);
             }
         }
     }
-    async waitFor(peer) {
+    async waitFor(peer, options) {
         const hash = (peer instanceof PublicSignKey ? peer : getPublicKeyFromPeerId(peer)).hashcode();
         try {
             await waitFor(() => {
                 if (!this.peers.has(hash)) {
                     return false;
                 }
-                if (!this.routes.hasLink(this.publicKeyHash, hash)) {
+                if (!this.routes.isReachable(this.publicKeyHash, hash)) {
                     return false;
                 }
                 return true;
+            }, {
+                signal: options?.signal,
+                timeout: 10 * 1000
             });
         }
         catch (error) {
@@ -1210,11 +1195,15 @@ export class DirectStream extends EventEmitter {
                 " does not exist. Connection exist: " +
                 this.peers.has(hash) +
                 ". Route exist: " +
-                this.routes.hasLink(this.publicKeyHash, hash));
+                this.routes.isReachable(this.publicKeyHash, hash));
         }
         const stream = this.peers.get(hash);
         try {
-            await waitFor(() => stream.isReadable && stream.isWritable);
+            // Dontwait for readlable https://github.com/libp2p/js-libp2p/issues/2321
+            await waitFor(() => /* stream.isReadable && */ stream.isWritable, {
+                signal: options?.signal,
+                timeout: 10 * 1000
+            });
         }
         catch (error) {
             throw new Error("Stream to " +
@@ -1225,6 +1214,58 @@ export class DirectStream extends EventEmitter {
                 stream.isWritable);
         }
     }
+    get pending() {
+        return this._ackCallbacks.size > 0;
+    }
+    lastQueuedBytes = 0;
+    // make this into a job? run every few ms
+    maybePruneConnections() {
+        if (this.connectionManagerOptions.pruner) {
+            if (this.connectionManagerOptions.pruner.bandwidth != null) {
+                let usedBandwidth = 0;
+                for (const [_k, v] of this.peers) {
+                    usedBandwidth += v.usedBandwidth;
+                }
+                usedBandwidth /= this.peers.size;
+                if (usedBandwidth > this.connectionManagerOptions.pruner.bandwidth) {
+                    // prune
+                    return this.pruneConnections();
+                }
+            }
+            else if (this.connectionManagerOptions.pruner.maxBuffer != null) {
+                const queuedBytes = this.getQueuedBytes();
+                if (queuedBytes > this.connectionManagerOptions.pruner.maxBuffer) {
+                    // prune
+                    return this.pruneConnections();
+                }
+            }
+        }
+        return Promise.resolve();
+    }
+    async pruneConnections() {
+        // TODO sort by bandwidth
+        if (this.peers.size <= this.connectionManagerOptions.minConnections) {
+            return;
+        }
+        const sorted = [...this.peers.values()]
+            .sort((x, y) => x.usedBandwidth - y.usedBandwidth)
+            .map((x) => x.publicKey.hashcode());
+        const prunables = this.routes.getPrunable(sorted);
+        if (prunables.length === 0) {
+            return;
+        }
+        const stream = this.peers.get(prunables[0]);
+        this.prunedConnectionsCache.add(stream.publicKey.hashcode());
+        await this.onPeerDisconnected(stream.peerId);
+        return this.components.connectionManager.closeConnections(stream.peerId);
+    }
+    getQueuedBytes() {
+        let sum = 0;
+        for (const peer of this.peers) {
+            sum += peer[1].outboundStream?.readableLength || 0;
+        }
+        return sum;
+    }
 }
 export const waitForPeers = async (...libs) => {
     for (let i = 0; i < libs.length; i++) {