@decentnetwork/lan 0.1.150 → 0.1.151

package/dist/proxy/multi-exit-router.js

@@ -334,10 +334,17 @@ export function startMultiExitRouter(opts) {
     }
     // A tunnel established but stalled (delivered ~nothing) → count it; trip the
     // exit out of the pool once it stalls repeatedly, so a flaky node stops
-    // getting traffic instead of stalling segments.
+    // getting traffic instead of stalling segments. BUT never trip an exit the
+    // health PROBE still says is UP: under a busy live stream on a congested
+    // cross-border path, healthy exits routinely stall a segment or idle a small
+    // request — tripping them there collapses the pool (5↔3) and makes the stream
+    // LESS stable, not more. The probe (a real round-trip) is the down-detector;
+    // the stall breaker only fires on an exit that is ALSO failing its probe.
     function recordStall(exit, why) {
         exit.stalls++;
-        if (exit.stalls >= TRIP_AFTER_STALLS && exit.trippedUntil <= Date.now()) {
+        const probeAlive = exit.healthy ||
+            (exit.lastSuccessMs !== null && Date.now() - exit.lastSuccessMs < RECENT_SUCCESS_MS);
+        if (!probeAlive && exit.stalls >= TRIP_AFTER_STALLS && exit.trippedUntil <= Date.now()) {
             exit.trippedUntil = Date.now() + TRIP_COOLDOWN_MS;
             exit.healthy = false;
             log(`✗ ${exit.name} [${exit.region}] TRIPPED for ${TRIP_COOLDOWN_MS / 1000}s (${why}) — excluding from pool`);
@@ -413,7 +420,17 @@ export function startMultiExitRouter(opts) {
                 // problem — it fails identically on every exit, so tripping the exit for
                 // it churns the whole pool (a single dead host like sdc3.10086.cn or
                 // uem.migu.cn was knocking out all 6 China exits). Just fail over.
-                if (tripExit)
+                //
+                // AND: a connect-timeout to an exit the health probe STILL says is UP is
+                // transient cross-border congestion under load — NOT a dead exit (proven:
+                // the exit answers its probe at ~200ms and reaches the origin in ~40ms).
+                // Tripping it here removes it from the pool, piles load onto fewer exits,
+                // causes MORE timeouts → the pool collapses to 1-2 and the whole stream
+                // stalls (the high-bitrate live-video failure mode). Only trip an exit
+                // that is ALSO failing its probe (genuinely unreachable); a healthy exit
+                // just fails this ONE connection over and stays in the pool.
+                const probeAlive = exit.healthy || (exit.lastSuccessMs !== null && Date.now() - exit.lastSuccessMs < RECENT_SUCCESS_MS);
+                if (tripExit && !probeAlive)
                     recordStall(exit, why);
                 up.destroy();
                 onFail(why);
@@ -774,9 +791,20 @@ export function startMultiExitRouter(opts) {
                 ? () => tls.connect({ socket: tun, servername: host, rejectUnauthorized: false })
                 : () => tun;
             const up = reqFn({ host, port, path, method: req.method, headers, createConnection: conn }, (upRes) => {
+                log(`  MITM ${req.method} ${host}${(path || "/").slice(0, 80)} → ${upRes.statusCode}`);
                 const meter = new RateMeter();
-                upRes.on("data", (d) => meter.add(d.length, Date.now()));
-                upRes.once("end", () => sampleSpeed(exit, meter, Date.now() - t0));
+                // Sniff the stream-decision APIs' bodies (they say play:1+url vs play:0)
+                // so we can see WHY the app won't play even though every request is 200.
+                const sniff = /vdn\/live|cctv5\.json|getHttppdrp|\/vdn\//.test(path);
+                let sbuf = sniff ? Buffer.alloc(0) : null;
+                const enc = String(upRes.headers["content-encoding"] || "");
+                upRes.on("data", (d) => { meter.add(d.length, Date.now()); if (sbuf && sbuf.length < 1500)
+                    sbuf = Buffer.concat([sbuf, d]); });
+                upRes.once("end", () => {
+                    sampleSpeed(exit, meter, Date.now() - t0);
+                    if (sbuf)
+                        log(`    body[${host}${enc ? " " + enc : ""}]: ${sbuf.slice(0, 1000).toString("utf8").replace(/\s+/g, " ")}`);
+                });
                 res.writeHead(upRes.statusCode || 502, upRes.headers);
                 upRes.pipe(res);
             });
@@ -930,6 +958,11 @@ export function startMultiExitRouter(opts) {
                 }
                 fetchViaRegionBuffered(u, fwd, isPlaylist ? "playlist" : "segment")
                     .then((r) => {
+                    log(`  MITM ${method} ${host}${(path || "/").slice(0, 80)} → ${r.status} (${r.body.length}B)`);
+                    if (r.status >= 400) {
+                        const snip = r.body.slice(0, 300).toString("utf8").replace(/\s+/g, " ");
+                        log(`    body: ${snip}`);
+                    }
                     const outHeaders = { ...r.headers };
                     delete outHeaders["transfer-encoding"];
                     outHeaders["content-length"] = r.body.length;

package/dist/router/paced-forwarder.d.ts

@@ -25,9 +25,17 @@ export interface PacedForwarderStats {
 export declare class PacedForwarder {
     #private;
     constructor(opts: PacedForwarderOptions);
-    /** Set the paced target rate (bytes/sec) for a destination from a measured
-     *  delivered-rate estimate. */
+    /** Set the paced target rate (bytes/sec) for a destination directly. Kept for
+     *  tests / explicit control; the live path uses reportDelivered. */
     setRate(dstIp: string, rateBps: number): void;
+    /** Feed a peer's measured delivered-rate: bytes/sec of the traffic we sent
+     *  them that actually landed. Loss-driven: over a rolling window we compare it
+     *  to what we sent and only TIGHTEN the pace when the path is dropping a real
+     *  fraction of our egress; a clean path ramps the pace back toward uncapped.
+     *  This is what stops a bursty app (HLS video) — whose per-second delivered
+     *  rate swings between a full segment and an idle trough — from collapsing the
+     *  pace and never recovering. */
+    reportDelivered(dstIp: string, deliveredBps: number): void;
     /** Enqueue one IP packet for `dstIp` under flow `flowKey`. Never blocks; the
      *  drain loop releases it paced + fairly. Returns false if it was dropped
      *  (queue overflow). */

package/dist/router/paced-forwarder.js

@@ -31,6 +31,25 @@ const DEFAULT_MAX_FLOW_BYTES = 256 * 1024;
 const DEFAULT_MAX_DEST_BYTES = 4 * 1024 * 1024;
 const DEFAULT_BURST_SECONDS = 0.05; // 50ms of rate as burst — smooths without adding latency
 const MIN_RATE_BPS = 32 * 1024; // never pace below 256 kbps
+// LOSS-DRIVEN pacing. The old rule paced to `delivered × headroom`, but the
+// delivered rate is itself CAPPED by the pace — a classic self-limiting loop: a
+// single low sample (a bursty app's idle trough between HLS segments, or a slow
+// first segment) drags the pace down, which caps delivery, which keeps the next
+// sample low… and it never climbs back. That throttled healthy paths to a
+// fraction of capacity (measured: CCTV stuck at 0.3–0.5 Mbps even over an
+// all-local route with no congestion at all).
+//
+// Instead: judge the path over a rolling window by comparing what we SENT to
+// what the peer reports DELIVERED. Only real loss (a chunk of our egress not
+// landing) tightens the pace; a clean path ramps the pace back toward uncapped.
+// So we never throttle a healthy path, yet still back off the instant we
+// over-drive a congested one and induce drops. The window absorbs the timing
+// skew between our egress and the peer's 1s report so a burst isn't misread as
+// loss.
+const LOSS_WINDOW_MS = 2000;
+const LOSS_TRIGGER = 0.2; // >20% of sent bytes not landing ⇒ real congestion
+const PROBE_GAIN = 1.5; // clean path: ramp the pace up briskly toward uncapped
+const BACKOFF_GAIN = 1.1; // lossy path: pace just above what actually gets through
 class Dest {
     rateBps;
     tokens;
@@ -41,11 +60,19 @@ class Dest {
     cursor = 0;
     totalBytes = 0;
     draining = false;
+    // Loss-window accounting: bytes we've handed to the carrier and bytes the peer
+    // reports as delivered, since winStartMs. Compared once per LOSS_WINDOW_MS.
+    winSentBytes = 0;
+    winDeliveredBytes = 0;
+    winStartMs;
+    lastReportMs;
     constructor(rateBps, burstSeconds, nowMs) {
         this.rateBps = rateBps;
         this.burstBytes = Math.max(1500, rateBps * burstSeconds);
         this.tokens = this.burstBytes;
         this.lastRefillMs = nowMs;
+        this.winStartMs = nowMs;
+        this.lastReportMs = nowMs;
     }
     refill(nowMs) {
         const dt = nowMs - this.lastRefillMs;
@@ -77,12 +104,52 @@ export class PacedForwarder {
             schedule: opts.schedule ?? ((fn, ms) => { setTimeout(fn, ms).unref?.(); }),
         };
     }
-    /** Set the paced target rate (bytes/sec) for a destination from a measured
-     *  delivered-rate estimate. */
+    /** Set the paced target rate (bytes/sec) for a destination directly. Kept for
+     *  tests / explicit control; the live path uses reportDelivered. */
     setRate(dstIp, rateBps) {
         const d = this.#dest(dstIp);
         d.setRate(rateBps, this.#opts.burstSeconds);
     }
+    /** Feed a peer's measured delivered-rate: bytes/sec of the traffic we sent
+     *  them that actually landed. Loss-driven: over a rolling window we compare it
+     *  to what we sent and only TIGHTEN the pace when the path is dropping a real
+     *  fraction of our egress; a clean path ramps the pace back toward uncapped.
+     *  This is what stops a bursty app (HLS video) — whose per-second delivered
+     *  rate swings between a full segment and an idle trough — from collapsing the
+     *  pace and never recovering. */
+    reportDelivered(dstIp, deliveredBps) {
+        const d = this.#dest(dstIp);
+        const now = this.#opts.now();
+        const dt = now - d.lastReportMs;
+        d.lastReportMs = now;
+        if (dt > 0)
+            d.winDeliveredBytes += (deliveredBps * dt) / 1000;
+        const winMs = now - d.winStartMs;
+        if (winMs < LOSS_WINDOW_MS)
+            return;
+        const sent = d.winSentBytes;
+        const delivered = d.winDeliveredBytes;
+        const winSec = winMs / 1000;
+        d.winSentBytes = 0;
+        d.winDeliveredBytes = 0;
+        d.winStartMs = now;
+        // Too little egress this window to judge the path — leave the pace as is
+        // (don't let an idle stretch either shrink or grow it).
+        if (sent < MIN_RATE_BPS * winSec)
+            return;
+        const lossRatio = Math.max(0, 1 - delivered / sent);
+        const bs = this.#opts.burstSeconds;
+        if (lossRatio > LOSS_TRIGGER) {
+            // Congestion: a real fraction of what we sent didn't land. Pace down to
+            // the throughput that actually gets through (a hair above, to keep probing).
+            d.setRate(Math.max(MIN_RATE_BPS, (delivered / winSec) * BACKOFF_GAIN), bs);
+        }
+        else {
+            // Clean path: our egress is landing. Ramp the pace back up toward uncapped
+            // so we are never the bottleneck on a healthy path.
+            d.setRate(Math.min(this.#opts.defaultRateBps, d.rateBps * PROBE_GAIN), bs);
+        }
+    }
     /** Enqueue one IP packet for `dstIp` under flow `flowKey`. Never blocks; the
      *  drain loop releases it paced + fairly. Returns false if it was dropped
      *  (queue overflow). */
@@ -175,6 +242,7 @@ export class PacedForwarder {
             f.bytes -= size;
             d.totalBytes -= size;
             d.tokens -= size;
+            d.winSentBytes += size;
             this.#sent++;
             void this.#opts.send(dstIp, pkt).catch(() => { this.#dropped++; });
             d.cursor = (d.cursor + 1) % Math.max(1, d.order.length);

package/dist/router/packet-router.js

@@ -14,12 +14,9 @@ const SUBNET_PREFIX = "10.86.";
 // How often each node reports the delivered rate of a peer's inbound flow.
 const RATE_REPORT_INTERVAL_MS = 1000;
 // Only report for peers whose inbound this interval exceeds this — no point
-// pacing a trickle (SSH/keepalives), and it keeps reports sparse.
+// pacing a trickle (SSH/keepalives), and it keeps reports sparse. The pacer's
+// loss window derives its verdict from these reports vs. what it sent.
 const RATE_REPORT_MIN_BYTES = 32 * 1024; // ~256 kbps sustained
-// The sender paces to measured × headroom so throughput can still probe upward
-// when the path frees up; loss caps it back down via the next report.
-const RATE_REPORT_HEADROOM = 1.25;
-const RATE_REPORT_MIN_BPS = 64 * 1024;
 export class PacketRouter extends EventEmitter {
     tunDevice;
     peerManager;
@@ -71,15 +68,16 @@ export class PacketRouter extends EventEmitter {
                     this.stats.packetsForwarded++;
                 },
             });
-            // A peer's rate report tells us how fast our traffic to them is actually
-            // landing — pace our sends to their vIP toward it (with a little headroom
-            // so throughput can still climb when capacity frees up).
+            // A peer's rate report tells us how much of our traffic to them actually
+            // landed. Feed it to the loss-driven pacer: it compares this to what we
+            // sent over a rolling window and only tightens the pace on real loss — a
+            // clean path stays uncapped, so a bursty app (HLS video) is never
+            // throttled by us and an idle trough can't collapse the pace.
             this.peerManager.on("rate-report", (pubkey, bytesPerSec) => {
                 const mapping = this.ipam.resolveCarrierId(pubkey);
                 if (!mapping?.virtualIp || !this.pacer)
                     return;
-                const target = Math.max(RATE_REPORT_MIN_BPS, bytesPerSec * RATE_REPORT_HEADROOM);
-                this.pacer.setRate(mapping.virtualIp, target);
+                this.pacer.reportDelivered(mapping.virtualIp, bytesPerSec);
             });
         }
         this.setupCarrierHandlers();

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@decentnetwork/lan",
-  "version": "0.1.150",
+  "version": "0.1.151",
   "description": "Private virtual LAN for self-hosted services and AI agents, built on Elastos Carrier. NAT-traversal, name service, ACL, all over a peer-to-peer mesh — no public IP required.",
   "type": "module",
   "main": "./dist/index.js",