@libp2p/autonat 2.0.12-b248eefc0 → 2.0.12-d19974d93

package/dist/src/autonat.js

@@ -1,30 +1,32 @@
-import { AbortError, serviceCapabilities, setMaxListeners } from '@libp2p/interface';
+import { serviceCapabilities, serviceDependencies, setMaxListeners } from '@libp2p/interface';
+import { peerSet } from '@libp2p/peer-collections';
 import { peerIdFromMultihash } from '@libp2p/peer-id';
-import { isPrivateIp } from '@libp2p/utils/private-ip';
+import { createScalableCuckooFilter } from '@libp2p/utils/filters';
+import { isPrivate } from '@libp2p/utils/multiaddr/is-private';
+import { PeerQueue } from '@libp2p/utils/peer-queue';
+import { repeatingTask } from '@libp2p/utils/repeating-task';
 import { multiaddr, protocols } from '@multiformats/multiaddr';
-import first from 'it-first';
-import * as lp from 'it-length-prefixed';
-import map from 'it-map';
-import parallel from 'it-parallel';
-import { pipe } from 'it-pipe';
+import { anySignal } from 'any-signal';
+import { pbStream } from 'it-protobuf-stream';
 import * as Digest from 'multiformats/hashes/digest';
-import { MAX_INBOUND_STREAMS, MAX_OUTBOUND_STREAMS, PROTOCOL_NAME, PROTOCOL_PREFIX, PROTOCOL_VERSION, REFRESH_INTERVAL, STARTUP_DELAY, TIMEOUT } from './constants.js';
+import { MAX_INBOUND_STREAMS, MAX_OUTBOUND_STREAMS, PROTOCOL_NAME, PROTOCOL_PREFIX, PROTOCOL_VERSION, TIMEOUT } from './constants.js';
 import { Message } from './pb/index.js';
 // if more than 3 peers manage to dial us on what we believe to be our external
 // IP then we are convinced that it is, in fact, our external IP
-// https://github.com/libp2p/specs/blob/master/autonat/README.md#autonat-protocol
+// https://github.com/libp2p/specs/blob/master/autonat/autonat-v1.md#autonat-protocol
 const REQUIRED_SUCCESSFUL_DIALS = 4;
 export class AutoNATService {
     components;
-    startupDelay;
-    refreshInterval;
     protocol;
     timeout;
     maxInboundStreams;
     maxOutboundStreams;
-    verifyAddressTimeout;
     started;
     log;
+    topologyId;
+    dialResults;
+    findPeers;
+    addressFilter;
     constructor(components, init) {
         this.components = components;
         this.log = components.logger.forComponent('libp2p:autonat');
@@ -33,14 +35,19 @@ export class AutoNATService {
         this.timeout = init.timeout ?? TIMEOUT;
         this.maxInboundStreams = init.maxInboundStreams ?? MAX_INBOUND_STREAMS;
         this.maxOutboundStreams = init.maxOutboundStreams ?? MAX_OUTBOUND_STREAMS;
-        this.startupDelay = init.startupDelay ?? STARTUP_DELAY;
-        this.refreshInterval = init.refreshInterval ?? REFRESH_INTERVAL;
-        this._verifyExternalAddresses = this._verifyExternalAddresses.bind(this);
+        this.dialResults = new Map();
+        this.findPeers = repeatingTask(this.findRandomPeers.bind(this), 60_000);
+        this.addressFilter = createScalableCuckooFilter(1024);
     }
     [Symbol.toStringTag] = '@libp2p/autonat';
     [serviceCapabilities] = [
         '@libp2p/autonat'
     ];
+    get [serviceDependencies]() {
+        return [
+            '@libp2p/identify'
+        ];
+    }
     isStarted() {
         return this.started;
     }
@@ -51,79 +58,94 @@ export class AutoNATService {
         await this.components.registrar.handle(this.protocol, (data) => {
             void this.handleIncomingAutonatStream(data)
                 .catch(err => {
-                this.log.error('error handling incoming autonat stream', err);
+                this.log.error('error handling incoming autonat stream - %e', err);
             });
         }, {
             maxInboundStreams: this.maxInboundStreams,
             maxOutboundStreams: this.maxOutboundStreams
         });
-        this.verifyAddressTimeout = setTimeout(this._verifyExternalAddresses, this.startupDelay);
+        this.topologyId = await this.components.registrar.register(this.protocol, {
+            onConnect: (peerId, connection) => {
+                this.verifyExternalAddresses(connection)
+                    .catch(err => {
+                    this.log.error('could not verify addresses - %e', err);
+                });
+            }
+        });
+        this.findPeers.start();
         this.started = true;
     }
     async stop() {
         await this.components.registrar.unhandle(this.protocol);
-        clearTimeout(this.verifyAddressTimeout);
+        if (this.topologyId != null) {
+            await this.components.registrar.unhandle(this.topologyId);
+        }
+        this.dialResults.clear();
+        this.findPeers.stop();
         this.started = false;
     }
+    allAddressesAreVerified() {
+        return this.components.addressManager.getAddressesWithMetadata().every(addr => addr.verified);
+    }
+    async findRandomPeers(options) {
+        // skip if all addresses are verified
+        if (this.allAddressesAreVerified()) {
+            return;
+        }
+        const signal = anySignal([
+            AbortSignal.timeout(10_000),
+            options?.signal
+        ]);
+        // spend a few seconds finding random peers - dial them which will run
+        // identify to trigger the topology callbacks and run AutoNAT
+        try {
+            this.log('starting random walk to find peers to run AutoNAT');
+            for await (const peer of this.components.randomWalk.walk({ signal })) {
+                if (!(await this.components.connectionManager.isDialable(peer.multiaddrs))) {
+                    this.log.trace('random peer %p was not dialable %s', peer.id, peer.multiaddrs.map(ma => ma.toString()).join(', '));
+                    // skip peers we can't dial
+                    continue;
+                }
+                try {
+                    this.log.trace('dial random peer %p', peer.id);
+                    await this.components.connectionManager.openConnection(peer.multiaddrs, {
+                        signal
+                    });
+                }
+                catch { }
+                if (this.allAddressesAreVerified()) {
+                    this.log('stopping random walk, all addresses are verified');
+                    return;
+                }
+            }
+        }
+        catch { }
+    }
     /**
      * Handle an incoming AutoNAT request
      */
     async handleIncomingAutonatStream(data) {
         const signal = AbortSignal.timeout(this.timeout);
-        const onAbort = () => {
-            data.stream.abort(new AbortError());
-        };
-        signal.addEventListener('abort', onAbort, { once: true });
-        // this controller may be used while dialing lots of peers so prevent MaxListenersExceededWarning
-        // appearing in the console
         setMaxListeners(Infinity, signal);
+        const messages = pbStream(data.stream).pb(Message);
         try {
-            const self = this;
-            await pipe(data.stream, (source) => lp.decode(source), async function* (stream) {
-                const buf = await first(stream);
-                if (buf == null) {
-                    self.log('no message received');
-                    yield Message.encode({
-                        type: Message.MessageType.DIAL_RESPONSE,
-                        dialResponse: {
-                            status: Message.ResponseStatus.E_BAD_REQUEST,
-                            statusText: 'No message was sent'
-                        }
-                    });
-                    return;
-                }
-                let request;
-                try {
-                    request = Message.decode(buf);
-                }
-                catch (err) {
-                    self.log.error('could not decode message', err);
-                    yield Message.encode({
-                        type: Message.MessageType.DIAL_RESPONSE,
-                        dialResponse: {
-                            status: Message.ResponseStatus.E_BAD_REQUEST,
-                            statusText: 'Could not decode message'
-                        }
-                    });
-                    return;
-                }
-                yield Message.encode(await self.handleAutonatMessage(request, data.connection, {
-                    signal
-                }));
-            }, (source) => lp.encode(source), data.stream);
+            const request = await messages.read({
+                signal
+            });
+            const response = await this.handleAutonatMessage(request, data.connection, {
+                signal
+            });
+            await messages.write(response, {
+                signal
+            });
+            await messages.unwrap().unwrap().close({
+                signal
+            });
         }
         catch (err) {
-            this.log.error('error handling incoming autonat stream', err);
+            this.log.error('error handling incoming autonat stream - %e', err);
+            data.stream.abort(err);
         }
-        finally {
-            signal.removeEventListener('abort', onAbort);
-        }
-    }
-    _verifyExternalAddresses() {
-        void this.verifyExternalAddresses()
-            .catch(err => {
-            this.log.error('error verifying external address', err);
-        });
     }
     async handleAutonatMessage(message, connection, options) {
         const ourHosts = this.components.addressManager.getAddresses()
@@ -156,7 +178,7 @@ export class AutoNATService {
             peerId = peerIdFromMultihash(digest);
         }
         catch (err) {
-            this.log.error('invalid PeerId', err);
+            this.log.error('invalid PeerId - %e', err);
             return {
                 type: Message.MessageType.DIAL_RESPONSE,
                 dialResponse: {
@@ -187,9 +209,8 @@ export class AutoNATService {
             return isFromSameHost;
         })
             .filter(ma => {
-            const host = ma.toOptions().host;
-            const isPublicIp = !(isPrivateIp(host) ?? false);
-            this.log.trace('host %s was public %s', host, isPublicIp);
+            const isPublicIp = !isPrivate(ma);
+            this.log.trace('%a was public %s', ma, isPublicIp);
             // don't try to dial private addresses
             return isPublicIp;
         })
@@ -236,7 +257,7 @@ export class AutoNATService {
                     this.log.error('tried to dial %a but dialed %a', multiaddr, connection.remoteAddr);
                     throw new Error('Unexpected remote address');
                 }
-                this.log('Success %p', peerId);
+                this.log('successfully dialed %p via %a', peerId, multiaddr);
                 return {
                     type: Message.MessageType.DIAL_RESPONSE,
                     dialResponse: {
@@ -246,7 +267,7 @@ export class AutoNATService {
                 };
             }
             catch (err) {
-                this.log('could not dial %p', peerId, err);
+                this.log('could not dial %p - %e', peerId, err);
                 errorMessage = err.message;
             }
             finally {
@@ -264,157 +285,234 @@ export class AutoNATService {
             }
         };
     }
+    /**
+     * The AutoNAT v1 server is only required to send us the address that it
+     * dialed successfully.
+     *
+     * When addresses fail, it can be because they are NATed, or because the peer
+     * did't support the transport, we have no way of knowing, so just send them
+     * one address so we can treat the response as:
+     *
+     * - OK - the dial request worked and the address is not NATed
+     * - E_DIAL_ERROR - the dial request failed and the address may be NATed
+     * - E_DIAL_REFUSED/E_BAD_REQUEST/E_INTERNAL_ERROR - the remote didn't dial the address
+     */
+    getFirstUnverifiedMultiaddr(segment) {
+        const addrs = this.components.addressManager.getAddressesWithMetadata()
+            .sort((a, b) => {
+            // sort addresses, prioritize DNS/IP mapped addresses over observed ones
+            if (a.type === 'dns-mapping') {
+                return -1;
+            }
+            if (b.type === 'dns-mapping') {
+                return 1;
+            }
+            if (a.type === 'ip-mapping') {
+                return -1;
+            }
+            if (b.type === 'ip-mapping') {
+                return 1;
+            }
+            return 0;
+        })
+            .filter(addr => {
+            if (addr.verified && addr.expires > Date.now()) {
+                // skip verified addresses within their TTL
+                return false;
+            }
+            if (isPrivate(addr.multiaddr)) {
+                // skip private addresses
+                return false;
+            }
+            return true;
+        });
+        for (const addr of addrs) {
+            const addrString = addr.multiaddr.toString();
+            let results = this.dialResults.get(addrString);
+            if (results != null) {
+                if (results.networkSegments.includes(segment)) {
+                    this.log.trace('%a already has a network segment result from %s', results.multiaddr, segment);
+                    // skip this address if we already have a dial result from the
+                    // network segment the peer is in
+                    continue;
+                }
+                if (results.queue.size > 10) {
+                    this.log.trace('%a already has enough peers queued', results.multiaddr);
+                    // already have enough peers verifying this address, skip on to the
+                    // next one
+                    continue;
+                }
+            }
+            // will include this multiaddr, ensure we have a results object
+            if (results == null) {
+                const needsRevalidating = addr.expires < Date.now();
+                // allow re-validating addresses that worked previously
+                if (needsRevalidating) {
+                    this.addressFilter.remove?.(addrString);
+                }
+                if (this.addressFilter.has(addrString)) {
+                    continue;
+                }
+                // only try to validate the address once
+                this.addressFilter.add(addrString);
+                this.log.trace('creating dial result %s %s', needsRevalidating ? 'to revalidate' : 'for', addrString);
+                results = {
+                    multiaddr: addr.multiaddr,
+                    success: 0,
+                    failure: 0,
+                    networkSegments: [],
+                    verifyingPeers: peerSet(),
+                    queue: new PeerQueue({
+                        concurrency: 3,
+                        maxSize: 50
+                    })
+                };
+                this.dialResults.set(addrString, results);
+            }
+            return results;
+        }
+    }
+    /**
+     * Removes any multiaddr result objects created for old multiaddrs that we are
+     * no longer waiting on
+     */
+    removeOutdatedMultiaddrResults() {
+        const unverifiedMultiaddrs = new Set(this.components.addressManager.getAddressesWithMetadata()
+            .filter(({ verified }) => !verified)
+            .map(({ multiaddr }) => multiaddr.toString()));
+        for (const multiaddr of this.dialResults.keys()) {
+            if (!unverifiedMultiaddrs.has(multiaddr)) {
+                this.log.trace('remove results for %a', multiaddr);
+                this.dialResults.delete(multiaddr);
+            }
+        }
+    }
     /**
      * Our multicodec topology noticed a new peer that supports autonat
      */
-    async verifyExternalAddresses() {
-        clearTimeout(this.verifyAddressTimeout);
-        // Do not try to push if we are not running
+    async verifyExternalAddresses(connection) {
+        // do nothing if we are not running
         if (!this.isStarted()) {
             return;
         }
-        const addressManager = this.components.addressManager;
-        const multiaddrs = addressManager.getObservedAddrs()
-            .filter(ma => {
-            const options = ma.toOptions();
-            return !(isPrivateIp(options.host) ?? false);
-        });
-        if (multiaddrs.length === 0) {
-            this.log('no public addresses found, not requesting verification');
-            this.verifyAddressTimeout = setTimeout(this._verifyExternalAddresses, this.refreshInterval);
+        // perform cleanup
+        this.removeOutdatedMultiaddrResults();
+        // get multiaddrs this peer is eligible to verify
+        const segment = this.getNetworkSegment(connection.remoteAddr);
+        let results = this.getFirstUnverifiedMultiaddr(segment);
+        if (results == null) {
+            this.log.trace('no unverified public addresses found for peer %p to verify, not requesting verification', connection.remotePeer);
             return;
         }
-        const signal = AbortSignal.timeout(this.timeout);
-        // this controller may be used while dialing lots of peers so prevent MaxListenersExceededWarning
-        // appearing in the console
-        setMaxListeners(Infinity, signal);
-        const self = this;
-        try {
-            this.log('verify multiaddrs %s', multiaddrs.map(ma => ma.toString()).join(', '));
-            const request = Message.encode({
-                type: Message.MessageType.DIAL,
-                dial: {
-                    peer: {
-                        id: this.components.peerId.toMultihash().bytes,
-                        addrs: multiaddrs.map(map => map.bytes)
-                    }
-                }
+        results.queue.add(async (options) => {
+            results = this.dialResults.get(options.multiaddr.toString());
+            if (results == null) {
+                this.log('%a was verified while %p was queued', options.multiaddr, connection.remotePeer);
+                return;
+            }
+            const signal = AbortSignal.timeout(this.timeout);
+            setMaxListeners(Infinity, signal);
+            this.log.trace('asking %p to verify multiaddr %s', connection.remotePeer, options.multiaddr);
+            const stream = await connection.newStream(this.protocol, {
+                signal
             });
-            const results = {};
-            const networkSegments = [];
-            const verifyAddress = async (peer) => {
-                let onAbort = () => { };
-                try {
-                    this.log('asking %p to verify multiaddr', peer.id);
-                    const connection = await self.components.connectionManager.openConnection(peer.id, {
-                        signal
-                    });
-                    const stream = await connection.newStream(this.protocol, {
-                        signal
-                    });
-                    onAbort = () => { stream.abort(new AbortError()); };
-                    signal.addEventListener('abort', onAbort, { once: true });
-                    const buf = await pipe([request], (source) => lp.encode(source), stream, (source) => lp.decode(source), async (stream) => first(stream));
-                    if (buf == null) {
-                        this.log('no response received from %p', connection.remotePeer);
-                        return undefined;
-                    }
-                    const response = Message.decode(buf);
-                    if (response.type !== Message.MessageType.DIAL_RESPONSE || response.dialResponse == null) {
-                        this.log('invalid autonat response from %p', connection.remotePeer);
-                        return undefined;
-                    }
-                    if (response.dialResponse.status === Message.ResponseStatus.OK) {
-                        // make sure we use different network segments
-                        const options = connection.remoteAddr.toOptions();
-                        let segment;
-                        if (options.family === 4) {
-                            const octets = options.host.split('.');
-                            segment = octets[0];
-                        }
-                        else if (options.family === 6) {
-                            const octets = options.host.split(':');
-                            segment = octets[0];
-                        }
-                        else {
-                            this.log('remote address "%s" was not IP4 or IP6?', options.host);
-                            return undefined;
-                        }
-                        if (networkSegments.includes(segment)) {
-                            this.log('already have response from network segment %d - %s', segment, options.host);
-                            return undefined;
+            try {
+                const messages = pbStream(stream).pb(Message);
+                const [, response] = await Promise.all([
+                    messages.write({
+                        type: Message.MessageType.DIAL,
+                        dial: {
+                            peer: {
+                                id: this.components.peerId.toMultihash().bytes,
+                                addrs: [options.multiaddr.bytes]
+                            }
                         }
-                        networkSegments.push(segment);
-                    }
-                    return response.dialResponse;
+                    }, { signal }),
+                    messages.read({ signal })
+                ]);
+                if (response.type !== Message.MessageType.DIAL_RESPONSE || response.dialResponse == null) {
+                    this.log('invalid autonat response from %p - %j', connection.remotePeer, response);
+                    return;
                 }
-                catch (err) {
-                    this.log.error('error asking remote to verify multiaddr', err);
+                const status = response.dialResponse.status;
+                this.log.trace('autonat response from %p for %a is %s', connection.remotePeer, options.multiaddr, status);
+                if (status !== Message.ResponseStatus.OK && status !== Message.ResponseStatus.E_DIAL_ERROR) {
+                    return;
                 }
-                finally {
-                    signal.removeEventListener('abort', onAbort);
+                results = this.dialResults.get(options.multiaddr.toString());
+                if (results == null) {
+                    this.log.trace('peer reported %a as %s but there is no result object', options.multiaddr, response.dialResponse.status);
+                    return;
                 }
-            };
-            // find some random peers
-            for await (const dialResponse of parallel(map(this.components.randomWalk.walk({
-                signal
-            }), (peer) => async () => verifyAddress(peer)), {
-                concurrency: REQUIRED_SUCCESSFUL_DIALS
-            })) {
+                if (results.networkSegments.includes(segment)) {
+                    this.log.trace('%a results included network segment %s', options.multiaddr, segment);
+                    return;
+                }
+                if (results.result != null) {
+                    this.log.trace('already resolved result for %a, ignoring response from', options.multiaddr, connection.remotePeer);
+                    return;
+                }
+                if (results.verifyingPeers.has(connection.remotePeer)) {
+                    this.log.trace('peer %p has already verified %a, ignoring response', connection.remotePeer, options.multiaddr);
+                    return;
+                }
+                results.verifyingPeers.add(connection.remotePeer);
+                results.networkSegments.push(segment);
+                if (status === Message.ResponseStatus.OK) {
+                    results.success++;
+                }
+                else if (status === Message.ResponseStatus.E_DIAL_ERROR) {
+                    results.failure++;
+                }
+                this.log('%a success %d failure %d', results.multiaddr, results.success, results.failure);
+                if (results.success === REQUIRED_SUCCESSFUL_DIALS) {
+                    // we are now convinced
+                    this.log('%a is externally dialable', results.multiaddr);
+                    this.components.addressManager.confirmObservedAddr(results.multiaddr);
+                    this.dialResults.delete(results.multiaddr.toString());
+                    // abort & remove any outstanding verification jobs for this multiaddr
+                    results.result = true;
+                    results.queue.abort();
+                }
+                if (results.failure === REQUIRED_SUCCESSFUL_DIALS) {
+                    // we are now unconvinced
+                    this.log('%a is not externally dialable', results.multiaddr);
+                    this.components.addressManager.removeObservedAddr(results.multiaddr);
+                    this.dialResults.delete(results.multiaddr.toString());
+                    // abort & remove any outstanding verification jobs for this multiaddr
+                    results.result = false;
+                    results.queue.abort();
+                }
+            }
+            finally {
                 try {
-                    if (dialResponse == null) {
-                        continue;
-                    }
-                    // they either told us which address worked/didn't work, or we only sent them one address
-                    const addr = dialResponse.addr == null ? multiaddrs[0] : multiaddr(dialResponse.addr);
-                    this.log('autonat response for %a is %s', addr, dialResponse.status);
-                    if (dialResponse.status === Message.ResponseStatus.E_BAD_REQUEST) {
-                        // the remote could not parse our request
-                        continue;
-                    }
-                    if (dialResponse.status === Message.ResponseStatus.E_DIAL_REFUSED) {
-                        // the remote could not honour our request
-                        continue;
-                    }
-                    if (dialResponse.addr == null && multiaddrs.length > 1) {
-                        // we sent the remote multiple addrs but they didn't tell us which ones worked/didn't work
-                        continue;
-                    }
-                    if (!multiaddrs.some(ma => ma.equals(addr))) {
-                        this.log('peer reported %a as %s but it was not in our observed address list', addr, dialResponse.status);
-                        continue;
-                    }
-                    const addrStr = addr.toString();
-                    if (results[addrStr] == null) {
-                        results[addrStr] = { success: 0, failure: 0 };
-                    }
-                    if (dialResponse.status === Message.ResponseStatus.OK) {
-                        results[addrStr].success++;
-                    }
-                    else if (dialResponse.status === Message.ResponseStatus.E_DIAL_ERROR) {
-                        results[addrStr].failure++;
-                    }
-                    if (results[addrStr].success === REQUIRED_SUCCESSFUL_DIALS) {
-                        // we are now convinced
-                        this.log('%a is externally dialable', addr);
-                        addressManager.confirmObservedAddr(addr);
-                        return;
-                    }
-                    if (results[addrStr].failure === REQUIRED_SUCCESSFUL_DIALS) {
-                        // we are now unconvinced
-                        this.log('%a is not externally dialable', addr);
-                        addressManager.removeObservedAddr(addr);
-                        return;
-                    }
+                    await stream.close({
+                        signal
+                    });
                 }
                 catch (err) {
-                    this.log.error('could not verify external address', err);
+                    stream.abort(err);
                 }
             }
+        }, {
+            peerId: connection.remotePeer,
+            multiaddr: results.multiaddr
+        })
+            .catch(err => {
+            if (results?.result == null) {
+                this.log.error('error from %p verifying address %a - %e', connection.remotePeer, results?.multiaddr, err);
+            }
+        });
+    }
+    getNetworkSegment(ma) {
+        // make sure we use different network segments
+        const options = ma.toOptions();
+        if (options.family === 4) {
+            const octets = options.host.split('.');
+            return octets[0].padStart(3, '0');
         }
-        finally {
-            this.verifyAddressTimeout = setTimeout(this._verifyExternalAddresses, this.refreshInterval);
-        }
+        const octets = options.host.split(':');
+        return octets[0].padStart(4, '0');
     }
 }
 //# sourceMappingURL=autonat.js.map