@peerbit/stream 4.3.10 → 4.4.0

package/dist/src/index.js

@@ -52,6 +52,25 @@ const getLaneFromPriority = (priority) => {
     }
     return 1;
 };
+// Hook for tests to override queued length measurement (peerStreams, default impl)
+export let measureOutboundQueuedBytes = (ps) => {
+    const active = ps._getActiveOutboundPushable();
+    if (!active)
+        return 0;
+    // Prefer lane-aware helper if present
+    // @ts-ignore - optional test helper
+    if (typeof active.getReadableLength === "function") {
+        try {
+            // lane 0 only
+            return active.getReadableLength(0) || 0;
+        }
+        catch {
+            // ignore
+        }
+    }
+    // @ts-ignore fallback for vanilla pushable
+    return active.readableLength || 0;
+};
 /**
  * Thin wrapper around a peer's inbound / outbound pubsub streams
  */
@@ -59,37 +78,95 @@ export class PeerStreams extends TypedEventEmitter {
     peerId;
     publicKey;
     protocol;
+    // Removed dedicated outboundStream; first element of outboundStreams[] is active
     /**
-     * Write stream - it's preferable to use the write method
-     */
-    outboundStream;
-    /**
-     * Read stream
+     * Backwards compatible single inbound references (points to first inbound candidate)
      */
     inboundStream;
-    /**
-     * The raw outbound stream, as retrieved from conn.newStream
-     */
-    rawOutboundStream;
-    /**
-     * The raw inbound stream, as retrieved from the callback from libp2p.handle
-     */
     rawInboundStream;
     /**
-     * An AbortController for controlled shutdown of the  treams
+     * Multiple inbound stream support (more permissive than outbound)
+     * We retain concurrent inbound streams to avoid races during migration; inactive ones can later be pruned.
      */
-    inboundAbortController;
+    inboundStreams = [];
+    _inboundPruneTimer;
+    static INBOUND_IDLE_MS = 10_000; // configurable grace for inactivity (made public for tests)
+    static MAX_INBOUND_STREAMS = 8; // sensible default to prevent flood
     outboundAbortController;
     closed;
     connId;
     seekedOnce;
     usedBandWidthTracker;
+    // Unified outbound streams list (during grace may contain >1; after pruning length==1)
+    outboundStreams = [];
+    // Public debug exposure of current raw outbound streams (during grace may contain >1)
+    get rawOutboundStreams() {
+        return this.outboundStreams.map((c) => c.raw);
+    }
+    _getActiveOutboundPushable() {
+        return this.outboundStreams[0]?.pushable;
+    }
+    _getOutboundCount() {
+        return this.outboundStreams.length;
+    }
+    _getInboundCount() {
+        return this.inboundStreams.length;
+    }
+    _debugInboundStats() {
+        return this.inboundStreams.map((c) => ({
+            id: c.raw.id,
+            created: c.created,
+            lastActivity: c.lastActivity,
+            bytesReceived: c.bytesReceived,
+        }));
+    }
+    _outboundPruneTimer;
+    static OUTBOUND_GRACE_MS = 500; // TODO configurable
+    _addOutboundCandidate(raw) {
+        const existing = this.outboundStreams.find((c) => c.raw === raw);
+        if (existing)
+            return existing;
+        const pushableInst = pushableLanes({
+            lanes: 2,
+            onPush: (val) => {
+                candidate.bytesDelivered += val.length || val.byteLength || 0;
+            },
+        });
+        const candidate = {
+            raw,
+            pushable: pushableInst,
+            created: Date.now(),
+            bytesDelivered: 0,
+            aborted: false,
+            existing: false,
+        };
+        pipe(pushableInst, (source) => lp.encode(source, { maxDataLength: MAX_DATA_LENGTH_OUT }), raw).catch((e) => {
+            candidate.aborted = true;
+            logError(e);
+        });
+        this.outboundStreams.push(candidate);
+        const origAbort = raw.abort?.bind(raw);
+        raw.abort = (err) => {
+            candidate.aborted = true;
+            return origAbort?.(err);
+        };
+        return candidate;
+    }
+    _scheduleOutboundPrune(reset = true) {
+        if (this.outboundStreams.length <= 1)
+            return;
+        if (reset && this._outboundPruneTimer) {
+            clearTimeout(this._outboundPruneTimer);
+        }
+        if (!this._outboundPruneTimer) {
+            this._outboundPruneTimer = setTimeout(() => this.pruneOutboundCandidates(), PeerStreams.OUTBOUND_GRACE_MS);
+        }
+    }
     constructor(init) {
         super();
         this.peerId = init.peerId;
         this.publicKey = init.publicKey;
         this.protocol = init.protocol;
-        this.inboundAbortController = new AbortController();
         this.outboundAbortController = new AbortController();
         this.closed = false;
         this.connId = init.connId;
@@ -100,13 +177,13 @@ export class PeerStreams extends TypedEventEmitter {
      * Do we have a connection to read from?
      */
     get isReadable() {
-        return Boolean(this.inboundStream);
+        return this.inboundStreams.length > 0;
     }
     /**
      * Do we have a connection to write on?
      */
     get isWritable() {
-        return Boolean(this.outboundStream);
+        return this.outboundStreams.length > 0;
     }
     get usedBandwidth() {
         return this.usedBandWidthTracker.value;
@@ -119,14 +196,56 @@ export class PeerStreams extends TypedEventEmitter {
         if (data.length > MAX_DATA_LENGTH_OUT) {
             throw new Error(`Message too large (${data.length * 1e-6}) mb). Needs to be less than ${MAX_DATA_LENGTH_OUT * 1e-6} mb`);
         }
-        if (this.outboundStream == null) {
+        if (!this.isWritable) {
             logger.error("No writable connection to " + this.peerId.toString());
             throw new Error("No writable connection to " + this.peerId.toString());
         }
         this.usedBandWidthTracker.add(data.byteLength);
-        this.outboundStream.push(data instanceof Uint8Array ? data : data.subarray(), this.outboundStream.getReadableLength(0) === 0
-            ? 0
-            : getLaneFromPriority(priority));
+        // Write to all current outbound streams (normally 1, but >1 during grace)
+        const payload = data instanceof Uint8Array ? data : data.subarray();
+        let successes = 0;
+        let failures = [];
+        const failed = [];
+        for (const c of this.outboundStreams) {
+            try {
+                c.pushable.push(payload, c.pushable.getReadableLength(0) === 0
+                    ? 0
+                    : getLaneFromPriority(priority));
+                successes++;
+            }
+            catch (e) {
+                failures.push(e);
+                failed.push(c);
+                logError(e);
+            }
+        }
+        if (successes === 0) {
+            throw new Error("All outbound writes failed (" +
+                failures.map((f) => f?.message).join(", ") +
+                ")");
+        }
+        if (failures.length > 0) {
+            logger.warn(`Partial outbound write failure: ${failures.length} failed, ${successes} succeeded`);
+            // Remove failed streams immediately (best-effort)
+            if (failed.length) {
+                this.outboundStreams = this.outboundStreams.filter((c) => !failed.includes(c));
+                for (const f of failed) {
+                    try {
+                        f.raw.abort?.(new AbortError("Failed write"));
+                    }
+                    catch { }
+                    try {
+                        f.raw.close?.();
+                    }
+                    catch { }
+                }
+                // If more than one remains schedule prune; else ensure outbound event raised
+                if (this.outboundStreams.length > 1)
+                    this._scheduleOutboundPrune(true);
+                else
+                    this.dispatchEvent(new CustomEvent("stream:outbound"));
+            }
+        }
     }
     /**
      * Write to the outbound stream, waiting until it becomes writable.
@@ -181,58 +300,207 @@ export class PeerStreams extends TypedEventEmitter {
      * Attach a raw inbound stream and setup a read stream
      */
     attachInboundStream(stream) {
-        // Create and attach a new inbound stream
-        // The inbound stream is:
-        // - abortable, set to only return on abort, rather than throw
-        // - transformed with length-prefix transform
-        this.rawInboundStream = stream;
-        this.inboundStream = abortableSource(pipe(this.rawInboundStream, (source) => lp.decode(source, { maxDataLength: MAX_DATA_LENGTH_IN })), this.inboundAbortController.signal, {
+        // Support multiple concurrent inbound streams with inactivity pruning.
+        // Enforce max inbound streams (drop least recently active)
+        if (this.inboundStreams.length >= PeerStreams.MAX_INBOUND_STREAMS) {
+            let dropIndex = 0;
+            for (let i = 1; i < this.inboundStreams.length; i++) {
+                const a = this.inboundStreams[i];
+                const b = this.inboundStreams[dropIndex];
+                if (a.lastActivity < b.lastActivity ||
+                    (a.lastActivity === b.lastActivity && a.created < b.created)) {
+                    dropIndex = i;
+                }
+            }
+            const [drop] = this.inboundStreams.splice(dropIndex, 1);
+            try {
+                drop.abortController.abort();
+            }
+            catch {
+                logger.error("Failed to abort inbound stream");
+            }
+            try {
+                drop.raw.close?.();
+            }
+            catch {
+                logger.error("Failed to close inbound stream");
+            }
+        }
+        const abortController = new AbortController();
+        const decoded = pipe(stream, (source) => lp.decode(source, { maxDataLength: MAX_DATA_LENGTH_IN }));
+        const iterable = abortableSource(decoded, abortController.signal, {
             returnOnAbort: true,
             onReturnError: (err) => {
                 logger.error("Inbound stream error", err?.message);
             },
         });
-        /* this.rawInboundStream = stream
-        this.inboundAbortController = new AbortController()
-        this.inboundAbortController.signal.addEventListener('abort', () => {
-            this.rawInboundStream!.close()
-                .catch(err => {
-                    this.rawInboundStream?.abort(err)
-                })
-        })
-
-        this.inboundStream = pipe(
-            this.rawInboundStream!,
-            (source) => lp.decode(source, { maxDataLength: MAX_DATA_LENGTH_IN }),
-        )
- */
+        const record = {
+            raw: stream,
+            iterable,
+            abortController,
+            created: Date.now(),
+            lastActivity: Date.now(),
+            bytesReceived: 0,
+        };
+        this.inboundStreams.push(record);
+        this._scheduleInboundPrune();
+        // Backwards compatibility: keep first inbound as public properties
+        if (this.inboundStreams.length === 1) {
+            this.rawInboundStream = stream;
+            this.inboundStream = iterable;
+        }
         this.dispatchEvent(new CustomEvent("stream:inbound"));
-        return this.inboundStream;
+        return record;
+    }
+    _scheduleInboundPrune() {
+        if (this._inboundPruneTimer)
+            return; // already scheduled
+        this._inboundPruneTimer = setTimeout(() => {
+            this._inboundPruneTimer = undefined;
+            this._pruneInboundInactive();
+            if (this.inboundStreams.length > 1) {
+                // schedule again if still multiple
+                this._scheduleInboundPrune();
+            }
+        }, PeerStreams.INBOUND_IDLE_MS);
+    }
+    _pruneInboundInactive() {
+        if (this.inboundStreams.length <= 1)
+            return;
+        const now = Date.now();
+        // Keep at least one (the most recently active)
+        this.inboundStreams.sort((a, b) => b.lastActivity - a.lastActivity);
+        const keep = this.inboundStreams[0];
+        const survivors = [keep];
+        for (let i = 1; i < this.inboundStreams.length; i++) {
+            const candidate = this.inboundStreams[i];
+            if (now - candidate.lastActivity <= PeerStreams.INBOUND_IDLE_MS) {
+                survivors.push(candidate);
+                continue; // still active
+            }
+            try {
+                candidate.abortController.abort();
+            }
+            catch { }
+            try {
+                candidate.raw.close?.();
+            }
+            catch { }
+        }
+        this.inboundStreams = survivors;
+        // update legacy references if they were pruned
+        if (!this.inboundStreams.includes(keep)) {
+            this.rawInboundStream = this.inboundStreams[0]?.raw;
+            this.inboundStream = this.inboundStreams[0]?.iterable;
+        }
+    }
+    forcePruneInbound() {
+        if (this._inboundPruneTimer) {
+            clearTimeout(this._inboundPruneTimer);
+            this._inboundPruneTimer = undefined;
+        }
+        this._pruneInboundInactive();
     }
     /**
      * Attach a raw outbound stream and setup a write stream
      */
     async attachOutboundStream(stream) {
-        // If an outbound stream already exists and is active, ignore the new one; if a raw stream exists, close it to avoid leaks
-        if (this.outboundStream != null &&
-            stream.id !== this.rawOutboundStream?.id) {
-            logger.info(`Outbound stream already exists for ${this.peerId.toString()}, ignoring additional stream`);
-            try {
-                await stream.abort?.(new AbortError("Superseded outbound stream"));
-            }
-            catch { }
+        if (this.outboundStreams[0] && stream.id === this.outboundStreams[0].raw.id)
+            return; // duplicate
+        this._addOutboundCandidate(stream);
+        if (this.outboundStreams.length === 1) {
+            this.dispatchEvent(new CustomEvent("stream:outbound"));
             return;
         }
-        this.rawOutboundStream = stream;
-        this.outboundStream = pushableLanes({ lanes: 2 });
-        this.outboundAbortController.signal.addEventListener("abort", () => {
-            this.rawOutboundStream?.close().catch((err) => {
-                this.rawOutboundStream?.abort(err);
-            });
-        });
-        pipe(this.outboundStream, (source) => lp.encode(source, { maxDataLength: MAX_DATA_LENGTH_OUT }), this.rawOutboundStream).catch(logError);
-        // Emit if the connection is new
-        this.dispatchEvent(new CustomEvent("stream:outbound"));
+        this._scheduleOutboundPrune(true);
+    }
+    pruneOutboundCandidates() {
+        try {
+            const candidates = this.outboundStreams;
+            if (!candidates.length)
+                return;
+            const now = Date.now();
+            const healthy = candidates.filter((c) => !c.aborted && c.bytesDelivered > 0);
+            let chosen;
+            if (healthy.length === 0) {
+                chosen = candidates.reduce((a, b) => b.created > a.created ? b : a);
+            }
+            else {
+                let bestScore = -Infinity;
+                for (const c of healthy) {
+                    const age = now - c.created || 1;
+                    const score = c.bytesDelivered / age;
+                    if (score > bestScore ||
+                        (score === bestScore && chosen && c.created > chosen.created)) {
+                        bestScore = score;
+                        chosen = c;
+                    }
+                }
+            }
+            if (!chosen)
+                return;
+            for (const c of candidates) {
+                if (c === chosen)
+                    continue; // never abort chosen
+                try {
+                    c.raw.abort?.(new AbortError("Replaced outbound stream"));
+                }
+                catch {
+                    logger.error("Failed to abort outbound stream");
+                }
+                try {
+                    c.pushable.return?.();
+                }
+                catch {
+                    logger.error("Failed to close outbound pushable");
+                }
+                try {
+                    c.raw.close?.();
+                }
+                catch {
+                    logger.error("Failed to close outbound stream");
+                }
+            }
+            this.outboundStreams = [chosen];
+        }
+        catch (e) {
+            logger.error("Error promoting outbound candidate: " + e?.message);
+        }
+        finally {
+            this.dispatchEvent(new CustomEvent("stream:outbound"));
+        }
+    }
+    forcePruneOutbound() {
+        if (this._outboundPruneTimer) {
+            clearTimeout(this._outboundPruneTimer);
+            this._outboundPruneTimer = undefined;
+        }
+        this.pruneOutboundCandidates();
+    }
+    /**
+     * Internal helper to perform the actual outbound replacement & piping.
+     */
+    // _replaceOutboundStream removed (legacy path)
+    // Debug/testing helper: list active outbound raw stream ids
+    _debugActiveOutboundIds() {
+        if (this.outboundStreams.length) {
+            return this.outboundStreams.map((c) => c.raw.id);
+        }
+        return this.outboundStreams.map((c) => c.raw.id);
+    }
+    _debugOutboundStats() {
+        if (this.outboundStreams.length) {
+            return this.outboundStreams.map((c) => ({
+                id: c.raw.id,
+                bytes: c.bytesDelivered,
+                aborted: !!c.aborted,
+            }));
+        }
+        return this.outboundStreams.map((c) => ({
+            id: c.raw.id,
+            bytes: c.bytesDelivered,
+            aborted: !!c.aborted,
+        }));
     }
     /**
      * Closes the open connection to peer
@@ -242,23 +510,47 @@ export class PeerStreams extends TypedEventEmitter {
             return;
         }
         this.closed = true;
+        if (this._outboundPruneTimer) {
+            clearTimeout(this._outboundPruneTimer);
+            this._outboundPruneTimer = undefined;
+        }
         // End the outbound stream
-        if (this.outboundStream != null) {
-            await this.outboundStream.return();
-            this.rawOutboundStream?.abort(new AbortError("Closed"));
+        if (this.outboundStreams.length) {
+            for (const c of this.outboundStreams) {
+                try {
+                    await c.pushable.return?.();
+                }
+                catch { }
+                try {
+                    c.raw.abort?.(new AbortError("Closed"));
+                }
+                catch { }
+            }
             this.outboundAbortController.abort();
         }
-        // End the inbound stream
-        if (this.inboundStream != null) {
-            this.inboundAbortController.abort();
-            await this.rawInboundStream?.close();
+        // End inbound streams
+        if (this.inboundStreams.length) {
+            for (const inbound of this.inboundStreams) {
+                try {
+                    inbound.abortController.abort();
+                }
+                catch {
+                    logger.error("Failed to abort inbound stream");
+                }
+                try {
+                    await inbound.raw.close?.();
+                }
+                catch {
+                    logger.error("Failed to close inbound stream");
+                }
+            }
         }
         this.usedBandWidthTracker.stop();
         this.dispatchEvent(new CustomEvent("close"));
-        this.rawOutboundStream = undefined;
-        this.outboundStream = undefined;
+        this.outboundStreams = [];
         this.rawInboundStream = undefined;
         this.inboundStream = undefined;
+        this.inboundStreams = [];
     }
 }
 export class DirectStream extends TypedEventEmitter {
@@ -304,12 +596,16 @@ export class DirectStream extends TypedEventEmitter {
     constructor(components, multicodecs, options) {
         super();
         this.components = components;
-        const { canRelayMessage = true, messageProcessingConcurrency = 10, maxInboundStreams, maxOutboundStreams, connectionManager, routeSeekInterval = ROUTE_UPDATE_DELAY_FACTOR, seekTimeout = SEEK_DELIVERY_TIMEOUT, routeMaxRetentionPeriod = ROUTE_MAX_RETANTION_PERIOD, } = options || {};
+        const { canRelayMessage = true, messageProcessingConcurrency = 10, maxInboundStreams, maxOutboundStreams, connectionManager, routeSeekInterval = ROUTE_UPDATE_DELAY_FACTOR, seekTimeout = SEEK_DELIVERY_TIMEOUT, routeMaxRetentionPeriod = ROUTE_MAX_RETANTION_PERIOD, inboundIdleTimeout, } = options || {};
         const signKey = getKeypairFromPrivateKey(components.privateKey);
         this.seekTimeout = seekTimeout;
         this.sign = signKey.sign.bind(signKey);
         this.peerId = components.peerId;
         this.publicKey = signKey.publicKey;
+        if (inboundIdleTimeout != null)
+            PeerStreams.INBOUND_IDLE_MS = inboundIdleTimeout;
+        if (maxInboundStreams != null)
+            PeerStreams.MAX_INBOUND_STREAMS = maxInboundStreams;
         this.publicKeyHash = signKey.publicKey.hashcode();
         this.multicodecs = multicodecs;
         this.started = false;
@@ -511,8 +807,8 @@ export class DirectStream extends TypedEventEmitter {
         }
         const peer = this.addPeer(peerId, publicKey, stream.protocol, connection.id);
         // handle inbound
-        const inboundStream = peer.attachInboundStream(stream);
-        this.processMessages(peer.publicKey, inboundStream, peer).catch(logError);
+        const inboundRecord = peer.attachInboundStream(stream);
+        this.processMessages(peer.publicKey, inboundRecord, peer).catch(logError);
         // try to create outbound stream
         await this.outboundInflightQueue.push({ peerId, connection });
     }
@@ -736,15 +1032,14 @@ export class DirectStream extends TypedEventEmitter {
     /**
      * Responsible for processing each RPC message received by other peers.
      */
-    async processMessages(peerId, stream, peerStreams) {
+    async processMessages(peerId, record, peerStreams) {
         try {
-            await pipe(stream, async (source) => {
-                for await (const data of source) {
-                    this.processRpc(peerId, peerStreams, data).catch((e) => {
-                        logError(e);
-                    });
-                }
-            });
+            for await (const data of record.iterable) {
+                const now = Date.now();
+                record.lastActivity = now;
+                record.bytesReceived += data.length || data.byteLength || 0;
+                this.processRpc(peerId, peerStreams, data).catch((e) => logError(e));
+            }
         }
         catch (err) {
             if (err?.code === "ERR_STREAM_RESET") {
@@ -1555,9 +1850,8 @@ export class DirectStream extends TypedEventEmitter {
     }
     getQueuedBytes() {
         let sum = 0;
-        for (const peer of this.peers) {
-            const out = peer[1].outboundStream;
-            sum += out ? out.readableLength : 0;
+        for (const [_k, ps] of this.peers) {
+            sum += measureOutboundQueuedBytes(ps); // cast to access hook
         }
         return sum;
     }