@seat-mesh/daemon 0.1.26 → 0.1.29

package/dist/connectivity/connectivity-recovery.js

@@ -0,0 +1,529 @@
+import { spawn } from "node:child_process";
+import fs from "node:fs";
+import path from "node:path";
+import { meshRuntimePaths, providerMonitoredForLimits, resolveConnectivityHooks, resolveUxConfig, uxLimitKinds, } from "@seat-mesh/core";
+import { capturePaneSnapshot, listMeshMonitorPanes } from "@seat-mesh/tmux";
+import { notifyConnectivityStatus } from "./oc-resume.js";
+const PROXY_UP_COOLDOWN_MS = 60_000;
+/** After an episode clears, ignore stale oc-connect re-arm unless streak re-confirms. */
+const PROXY_DOWN_REARM_COOLDOWN_MS = 90_000;
+const ROTATE_COOLDOWN_MS = 180_000;
+const IPIFY_POLL_MS = 30_000;
+/** During PROXY-DOWN episode, still cap sync ipify curls so /health stays responsive. */
+const IPIFY_EPISODE_MIN_MS = 15_000;
+const PANE_SCAN_BATCH = 4;
+/** Grace before clearing a rate-limit episode after the last limited pane clears (flicker). */
+const RATE_LIMIT_EPISODE_CLEAR_MS = 120_000;
+/** How long a PROXY-DOWN episode may run before operator gets a "still down" toast. */
+const PROXY_DOWN_STUCK_NOTIFY_MS = 120_000;
+/** Mirrors scripts/cpe-proxy-smart-restart.sh cooldown file (harness parity — do not fork). */
+const DEFAULT_SMART_RESTART_STAMP = "/tmp/cpe-proxy-smart-restart.last";
+const DEFAULT_SMART_RESTART_COOLDOWN_SEC = 1800;
+/** scripts/cpe-proxy-smart-restart.sh exit code for "cooldown still active, skipped". */
+const SMART_RESTART_COOLDOWN_EXIT_CODE = 2;
+/**
+ * Seconds left on the Proxy-SMART file cooldown (0 = none / expired).
+ * Reads the same stamp file cpe-proxy-smart-restart.sh writes — never the daemon's
+ * own state — so the daemon and any harness script agree on cooldown status.
+ */
+export function smartRestartCooldownLeftSec(stampPath = process.env.CPE_SMART_RESTART_STAMP || DEFAULT_SMART_RESTART_STAMP, cooldownSec = Number(process.env.CPE_SMART_RESTART_COOLDOWN_SEC || DEFAULT_SMART_RESTART_COOLDOWN_SEC), nowMs = Date.now()) {
+    try {
+        const last = Number(fs.readFileSync(stampPath, "utf8").trim()) || 0;
+        if (!last)
+            return 0;
+        const ageSec = Math.floor(nowMs / 1000) - last;
+        return ageSec >= cooldownSec ? 0 : cooldownSec - ageSec;
+    }
+    catch {
+        return 0;
+    }
+}
+/**
+ * Consecutive ipify probe failures before treating carrier as down (debounce flaky curls).
+ * Exported for unit testing without tmux/process spawning.
+ */
+export function updateIpifyProbeStreak(state, carrierOk, threshold) {
+    const need = Math.max(1, threshold);
+    if (carrierOk) {
+        state.ipifyFailStreak = 0;
+        return { confirmedDown: false, streak: 0 };
+    }
+    state.ipifyFailStreak += 1;
+    return {
+        confirmedDown: state.ipifyFailStreak >= need,
+        streak: state.ipifyFailStreak,
+    };
+}
+/**
+ * Pure rising-edge/episode reducer for the OC-LIMIT (rate-limit) state machine.
+ * Exported for unit testing without tmux/process spawning.
+ */
+export function updateRateLimitEpisode(state, anyRateLimitNow, nowMs = Date.now(), graceMs = RATE_LIMIT_EPISODE_CLEAR_MS) {
+    if (anyRateLimitNow) {
+        state.lastRateLimitSeenAt = nowMs;
+        if (!state.rateLimitEpisodeActive) {
+            state.rateLimitEpisodeActive = true;
+            return { action: "start" };
+        }
+        return { action: "continue" };
+    }
+    if (state.rateLimitEpisodeActive && nowMs - state.lastRateLimitSeenAt > graceMs) {
+        state.rateLimitEpisodeActive = false;
+        return { action: "clear" };
+    }
+    return { action: "none" };
+}
+/**
+ * Pure decision: after Proxy-SMART exits, should rotate-until run next?
+ * Mirrors inbox-server.mjs shouldRunRotateUntilAfterSmart — chain rotate-until
+ * when SMART was skipped (cooldown), failed, or the carrier IP did not change.
+ */
+export function shouldChainRotateUntilAfterSmart(input) {
+    const { exitCode, ipBefore, ipAfter } = input;
+    if (exitCode === SMART_RESTART_COOLDOWN_EXIT_CODE)
+        return true;
+    if (!ipAfter)
+        return true;
+    if (ipBefore && ipAfter && ipBefore === ipAfter)
+        return true;
+    return exitCode !== 0;
+}
+function wifiProbeLockActive(loaded) {
+    const lock = meshRuntimePaths(loaded).wifiProbeLock;
+    try {
+        return fs.existsSync(lock);
+    }
+    catch {
+        return false;
+    }
+}
+let cachedIpifyUp = null;
+let lastIpifyAt = 0;
+let paneScanOffset = 0;
+/** Last carrier IP actually observed while up — a live fetch during an outage always fails. */
+let lastKnownGoodIp = null;
+const paneLimitCache = new Map();
+/** Per-pane consecutive oc-connect observations (batch scan), not scrollback one-shots. */
+const paneConnectStreak = new Map();
+const paneCcLimitSeen = new Set();
+let lastProxyDownClearAt = 0;
+/** Exported for unit tests. */
+export function updatePaneConnectStreak(streakMap, paneId, sawConnect, threshold) {
+    const need = Math.max(1, threshold);
+    if (!sawConnect) {
+        streakMap.delete(paneId);
+        return { confirmed: false, streak: 0 };
+    }
+    const streak = (streakMap.get(paneId) ?? 0) + 1;
+    streakMap.set(paneId, streak);
+    return { confirmed: streak >= need, streak };
+}
+function anyRateLimitInCache() {
+    for (const flags of paneLimitCache.values()) {
+        if (flags.rate)
+            return true;
+    }
+    return false;
+}
+/** Drop OC-LIMIT border state for a pane once resume ack confirms it is working again. */
+export function clearOcLimitBannerForPane(state, paneId) {
+    const had = paneLimitCache.has(paneId) ||
+        state.ocLimited.has(paneId) ||
+        state.connectPanes.has(paneId);
+    paneLimitCache.delete(paneId);
+    paneConnectStreak.delete(paneId);
+    state.ocLimited.delete(paneId);
+    state.connectPanes.delete(paneId);
+    return had;
+}
+/** When no panes remain limited, end the OC-LIMIT episode so borders and proxy recovery unlock. */
+export function maybeEndRateLimitEpisode(state) {
+    if (anyRateLimitInCache())
+        return;
+    if (!state.rateLimitEpisodeActive && !state.rateLimitRecoveryStarted)
+        return;
+    state.rateLimitEpisodeActive = false;
+    state.rateLimitRecoveryStarted = false;
+    state.lastRateLimitSeenAt = 0;
+}
+export function newConnectivityRecoveryState() {
+    return {
+        ocLimited: new Set(),
+        connectPanes: new Set(),
+        proxyDownActive: false,
+        recoveryRunning: false,
+        lastProxyUpAt: 0,
+        lastRotateAt: 0,
+        carrierIpAtEpisodeStart: null,
+        rateLimitEpisodeActive: false,
+        lastRateLimitSeenAt: 0,
+        proxyDownEpisodeStartAt: 0,
+        proxyDownStuckNotified: false,
+        rateLimitRecoveryStarted: false,
+        ipifyFailStreak: 0,
+    };
+}
+/**
+ * Pure decision: has a PROXY-DOWN episode been running long enough, with no
+ * "still down" toast sent yet, to warrant telling operator it's stuck (not silent)?
+ * Exported for unit testing without tmux/process spawning.
+ */
+export function shouldNotifyProxyDownStuck(state, nowMs = Date.now(), thresholdMs = PROXY_DOWN_STUCK_NOTIFY_MS) {
+    if (state.proxyDownStuckNotified)
+        return false;
+    if (!state.proxyDownEpisodeStartAt)
+        return false;
+    return nowMs - state.proxyDownEpisodeStartAt >= thresholdMs;
+}
+let ipifyProbeInFlight = false;
+/** Non-blocking ipify refresh — poll tick and /health must never wait on curl. */
+function scheduleIpifyProbe(workspace, port) {
+    if (ipifyProbeInFlight)
+        return;
+    ipifyProbeInFlight = true;
+    const child = spawn("bash", [
+        "-c",
+        `HTTPS_PROXY=http://127.0.0.1:${port} HTTP_PROXY=http://127.0.0.1:${port} curl -4 -sS -m 3 https://api.ipify.org 2>/dev/null || true`,
+    ], { cwd: workspace, stdio: ["ignore", "pipe", "ignore"] });
+    let out = "";
+    child.stdout?.on("data", (c) => {
+        out += String(c);
+    });
+    child.on("close", () => {
+        ipifyProbeInFlight = false;
+        lastIpifyAt = Date.now();
+        const ip = out.trim();
+        cachedIpifyUp = Boolean(ip);
+        if (ip)
+            lastKnownGoodIp = ip;
+    });
+    child.on("error", () => {
+        ipifyProbeInFlight = false;
+        lastIpifyAt = Date.now();
+        cachedIpifyUp = false;
+    });
+}
+function carrierIpCached() {
+    return lastKnownGoodIp;
+}
+function ipifyUp(workspace, port, episode = false) {
+    const now = Date.now();
+    const minGap = episode ? IPIFY_EPISODE_MIN_MS : IPIFY_POLL_MS;
+    if (now - lastIpifyAt >= minGap) {
+        scheduleIpifyProbe(workspace, port);
+    }
+    if (cachedIpifyUp !== null)
+        return cachedIpifyUp;
+    return true;
+}
+function runScriptAsync(workspace, scriptPath, log, label, onDone, extraEnv = {}) {
+    if (!fs.existsSync(scriptPath)) {
+        log(`${label}: missing ${scriptPath}`);
+        return false;
+    }
+    const child = spawn("bash", [scriptPath], {
+        cwd: workspace,
+        stdio: "ignore",
+        detached: false,
+        env: { ...process.env, CPE_SKIP_DESKTOP_NOTIFY: "1", ...extraEnv },
+    });
+    child.on("error", (e) => {
+        log(`${label}: spawn error ${e.message}`);
+        onDone(1);
+    });
+    child.on("close", (code) => onDone(code ?? 1));
+    return true;
+}
+/** Scan all monitor panes; update limit sets; trigger async recovery on rising edges only. */
+export function pollConnectivityRecovery(input) {
+    const { loaded, registry, workspace, session, baseWindow, workersWindow, minisWindow, state, log, resumeOpenCodePanes, onCcLimitRise, } = input;
+    const conn = loaded.profile.connectivity;
+    if (!conn?.enabled)
+        return;
+    const proxyPort = conn.proxyPort ?? 18887;
+    const panes = listMeshMonitorPanes(session, baseWindow, workersWindow, minisWindow);
+    const liveIds = new Set(panes.map((p) => p.paneId));
+    for (const id of paneLimitCache.keys()) {
+        if (!liveIds.has(id))
+            paneLimitCache.delete(id);
+    }
+    for (const id of paneConnectStreak.keys()) {
+        if (!liveIds.has(id))
+            paneConnectStreak.delete(id);
+    }
+    for (const id of paneCcLimitSeen) {
+        if (!liveIds.has(id))
+            paneCcLimitSeen.delete(id);
+    }
+    const connectThreshold = conn.policy?.connectFailBeforeRecovery ??
+        conn.policy?.ipifyFailBeforeRecovery ??
+        15;
+    const prevConnect = new Set(state.connectPanes);
+    const prevLimited = new Set(state.ocLimited);
+    const batch = [];
+    if (panes.length) {
+        for (let i = 0; i < Math.min(PANE_SCAN_BATCH, panes.length); i++) {
+            batch.push(panes[(paneScanOffset + i) % panes.length]);
+        }
+        paneScanOffset = (paneScanOffset + batch.length) % panes.length;
+    }
+    const uxResolved = loaded.profile.ux !== undefined ? resolveUxConfig(loaded.profile.ux) : null;
+    const { proxyDownKinds, rateLimitKinds, limitProviders } = uxResolved
+        ? uxLimitKinds(uxResolved)
+        : {
+            proxyDownKinds: new Set(["oc-connect"]),
+            rateLimitKinds: new Set(["oc-limit"]),
+            limitProviders: new Set(["opencode"]),
+        };
+    for (const { paneId, label } of batch) {
+        const snap = capturePaneSnapshot(paneId);
+        if (!snap) {
+            paneLimitCache.delete(paneId);
+            continue;
+        }
+        const prov = registry.detect(snap);
+        if (!prov || !providerMonitoredForLimits(prov.id, limitProviders)) {
+            paneLimitCache.delete(paneId);
+            continue;
+        }
+        const st = prov.composerState(snap);
+        if (st.phase === "limit" && st.limitKind === "cc-limit" && prov.id === "claude") {
+            if (!paneCcLimitSeen.has(paneId)) {
+                paneCcLimitSeen.add(paneId);
+                onCcLimitRise?.(paneId, snap);
+                log(`CC-LIMIT rising ${label} ${paneId}`);
+            }
+            continue;
+        }
+        if (st.phase !== "limit" || !st.limitKind) {
+            paneLimitCache.delete(paneId);
+            paneConnectStreak.delete(paneId);
+            paneCcLimitSeen.delete(paneId);
+            continue;
+        }
+        const sawConnect = proxyDownKinds.has(st.limitKind);
+        const rate = rateLimitKinds.has(st.limitKind);
+        if (!sawConnect && !rate) {
+            paneLimitCache.delete(paneId);
+            paneConnectStreak.delete(paneId);
+            continue;
+        }
+        let connect = false;
+        if (sawConnect) {
+            const { confirmed, streak } = updatePaneConnectStreak(paneConnectStreak, paneId, true, connectThreshold);
+            connect = confirmed;
+            if (!confirmed) {
+                if (streak === 1) {
+                    log(`PROXY-DOWN defer ${label} ${paneId} (need ${connectThreshold} connect observations — stale scrollback filter)`);
+                }
+            }
+            else if (!prevConnect.has(paneId)) {
+                log(`PROXY-DOWN rising ${label} ${paneId} kind=${st.limitKind} streak=${streak}`);
+            }
+        }
+        else {
+            updatePaneConnectStreak(paneConnectStreak, paneId, false, connectThreshold);
+        }
+        if (connect || rate) {
+            paneLimitCache.set(paneId, { connect, rate });
+        }
+        else {
+            paneLimitCache.delete(paneId);
+        }
+        if (rate && !prevLimited.has(paneId)) {
+            log(`OC-LIMIT rising ${label} ${paneId} kind=${st.limitKind}`);
+        }
+    }
+    state.connectPanes.clear();
+    state.ocLimited.clear();
+    let anyConnect = false;
+    let anyRateLimit = false;
+    for (const [paneId, flags] of paneLimitCache) {
+        if (!liveIds.has(paneId))
+            continue;
+        state.ocLimited.add(paneId);
+        if (flags.connect) {
+            state.connectPanes.add(paneId);
+            anyConnect = true;
+        }
+        else if (flags.rate) {
+            anyRateLimit = true;
+        }
+    }
+    const hooks = resolveConnectivityHooks(loaded.profile, workspace);
+    if (wifiProbeLockActive(loaded)) {
+        if (state.proxyDownActive) {
+            log("PROXY-DOWN suppressed — cpe-wifi-probe.lock active");
+            state.proxyDownActive = false;
+        }
+        return;
+    }
+    const carrierOk = ipifyUp(workspace, proxyPort, anyConnect || state.proxyDownActive);
+    const ipifyThreshold = conn.policy?.ipifyFailBeforeRecovery ?? 15;
+    const { confirmedDown: ipifyConfirmedDown, streak: ipifyStreak } = updateIpifyProbeStreak(state, carrierOk, ipifyThreshold);
+    if (!carrierOk && !ipifyConfirmedDown && !anyConnect) {
+        log(`ipify probe fail streak ${ipifyStreak}/${ipifyThreshold} — defer PROXY-DOWN recovery`);
+    }
+    // Ipify blips alone must not start PROXY-DOWN / cpe-proxy-up — need OC connect errors and/or
+    // an active rate-limit episode before treating carrier ipify as down for recovery.
+    const ipifyDownForRecovery = ipifyConfirmedDown && (anyRateLimit || state.rateLimitEpisodeActive);
+    let proxyDownNow = anyConnect || ipifyDownForRecovery;
+    const wasDown = state.proxyDownActive;
+    const rearmBlocked = !wasDown &&
+        proxyDownNow &&
+        anyConnect &&
+        !ipifyDownForRecovery &&
+        carrierOk &&
+        Date.now() - lastProxyDownClearAt < PROXY_DOWN_REARM_COOLDOWN_MS;
+    if (rearmBlocked) {
+        log(`PROXY-DOWN re-arm suppressed (episode cooldown ${PROXY_DOWN_REARM_COOLDOWN_MS / 1000}s, ipify up, connect debounced)`);
+        proxyDownNow = false;
+    }
+    state.proxyDownActive = proxyDownNow;
+    if (proxyDownNow && !wasDown) {
+        state.carrierIpAtEpisodeStart = carrierIpCached();
+        state.proxyDownEpisodeStartAt = Date.now();
+        state.proxyDownStuckNotified = false;
+        log(`PROXY-DOWN episode start ipify=${carrierOk ? "up" : "down"} streak=${ipifyStreak}/${ipifyThreshold} connectPanes=${state.connectPanes.size} carrier=${state.carrierIpAtEpisodeStart ?? "?"}`);
+        notifyConnectivityStatus(workspace, "PROXY-DOWN", `Carrier ${state.carrierIpAtEpisodeStart ?? "?"} unreachable. Running cpe-proxy-up.sh now.`, "Wait for resume-sent then complete toasts — no action during starting.", "starting");
+        maybeRunProxyUp(hooks, workspace, state, log);
+    }
+    else if (!proxyDownNow && wasDown) {
+        const fromIp = state.carrierIpAtEpisodeStart;
+        const toIp = carrierIpCached();
+        log(`PROXY-DOWN episode clear -> resume wave carrier=${toIp ?? "?"}`);
+        state.carrierIpAtEpisodeStart = null;
+        state.proxyDownEpisodeStartAt = 0;
+        state.proxyDownStuckNotified = false;
+        state.ipifyFailStreak = 0;
+        lastProxyDownClearAt = Date.now();
+        paneConnectStreak.clear();
+        // Keep rate-limit cache entries — clearing all panes made OC-LIMIT episodes re-arm proxy restart.
+        for (const [paneId, flags] of [...paneLimitCache.entries()]) {
+            if (flags.rate) {
+                paneLimitCache.set(paneId, { connect: false, rate: true });
+            }
+            else {
+                paneLimitCache.delete(paneId);
+            }
+        }
+        setImmediate(() => resumeOpenCodePanes("proxy-up", { fromIp, toIp }));
+    }
+    else if (proxyDownNow && wasDown && shouldNotifyProxyDownStuck(state)) {
+        state.proxyDownStuckNotified = true;
+        const ageSec = Math.round((Date.now() - state.proxyDownEpisodeStartAt) / 1000);
+        log(`PROXY-DOWN still active after ${ageSec}s -> notify operator`);
+        notifyConnectivityStatus(workspace, "PROXY-DOWN", `Carrier still unreachable after ${ageSec}s. cpe-proxy-up.sh ran; on cooldown.`, "Check WiFi/CPE manually if this persists.", "incomplete");
+    }
+    const { action } = updateRateLimitEpisode(state, anyRateLimit);
+    if (action === "start") {
+        const fromIp = carrierIpCached() ?? state.carrierIpAtEpisodeStart ?? lastKnownGoodIp;
+        log(`OC-LIMIT rising-edge episode start carrier=${fromIp ?? "?"}`);
+        if (!state.rateLimitRecoveryStarted) {
+            notifyConnectivityStatus(workspace, "OC-LIMIT", `Carrier ${fromIp ?? "?"} rate-limited. One proxy recovery run for this episode (no repeat while banner persists).`, "Wait for resume-sent then complete toasts. Ack is automatic when limit screens clear.", "starting");
+            const rotateMax = conn.policy?.rotateMaxAttempts ?? 15;
+            maybeRunOcLimitRecovery(hooks, workspace, proxyPort, fromIp, state, log, resumeOpenCodePanes, rotateMax);
+        }
+        else {
+            log("OC-LIMIT episode active — recovery already ran this episode (skip proxy restart)");
+        }
+    }
+    else if (action === "clear") {
+        log("OC-LIMIT episode clear");
+        state.rateLimitRecoveryStarted = false;
+    }
+}
+function maybeRunProxyUp(hooks, workspace, state, log) {
+    const now = Date.now();
+    if (state.recoveryRunning)
+        return;
+    if (now - state.lastProxyUpAt < PROXY_UP_COOLDOWN_MS)
+        return;
+    const script = hooks?.up;
+    if (!script) {
+        log("PROXY-DOWN: connectivity.hooks.up not configured — skip");
+        return;
+    }
+    state.recoveryRunning = true;
+    state.lastProxyUpAt = now;
+    log(`PROXY-DOWN -> async ${path.basename(script)}`);
+    const started = runScriptAsync(workspace, script, log, "proxy-up", (code) => {
+        state.recoveryRunning = false;
+        log(`cpe-proxy-up exit=${code} — resume wave waits for PROXY-DOWN episode clear`);
+    });
+    if (!started)
+        state.recoveryRunning = false;
+}
+/**
+ * OC-LIMIT (rate-limit) recovery: Proxy-SMART restart first when off cooldown,
+ * else cpe-proxy-rotate-until.sh directly. Chains rotate-until after a SMART
+ * run that was skipped/failed/no-op, then notifies operator + resumes OC panes on
+ * any real carrier IP change. Mirrors inbox-server.mjs runOcLimitRecoveryAsync.
+ */
+function maybeRunOcLimitRecovery(hooks, workspace, proxyPort, fromIp, state, log, resumeOpenCodePanes, rotateMaxAttempts = 15) {
+    const now = Date.now();
+    if (state.recoveryRunning)
+        return;
+    if (state.rateLimitRecoveryStarted)
+        return;
+    if (now - state.lastRotateAt < ROTATE_COOLDOWN_MS)
+        return;
+    state.recoveryRunning = true;
+    state.rateLimitRecoveryStarted = true;
+    state.lastRotateAt = now;
+    const finish = (toIp, reason) => {
+        state.recoveryRunning = false;
+        const ipChanged = Boolean(toIp && fromIp && toIp !== fromIp);
+        if (!ipChanged) {
+            log(`OC-LIMIT recovery done reason=${reason} ip=${toIp ?? "?"} — skip resume wave (no carrier change)`);
+            return;
+        }
+        log(`OC-LIMIT recovery done -> resume wave reason=${reason} ${fromIp} -> ${toIp}`);
+        setImmediate(() => resumeOpenCodePanes(reason, { fromIp, toIp }));
+    };
+    const runRotateUntil = (reason) => {
+        const script = hooks?.rotate;
+        if (!script || !fs.existsSync(script)) {
+            log(`OC-LIMIT recovery: connectivity.hooks.rotate missing (${script ?? "unset"})`);
+            state.recoveryRunning = false;
+            return;
+        }
+        log(`OC-LIMIT -> async ${path.basename(script)} (${reason}) max=${rotateMaxAttempts}`);
+        const rotateEnv = { CPE_ROTATE_MAX_ATTEMPTS: String(rotateMaxAttempts) };
+        const started = runScriptAsync(workspace, script, log, "rotate-until", (code) => {
+            log(`rotate-until exit=${code}`);
+            scheduleIpifyProbe(workspace, proxyPort);
+            finish(carrierIpCached(), "rotate-until");
+        }, rotateEnv);
+        if (!started)
+            state.recoveryRunning = false;
+    };
+    const cooldownLeft = smartRestartCooldownLeftSec();
+    if (cooldownLeft > 0) {
+        log(`OC-LIMIT Proxy-SMART cooldown ${cooldownLeft}s -> rotate-until path`);
+        runRotateUntil("smart-cooldown");
+        return;
+    }
+    const smartScript = hooks?.smartRestart;
+    if (!smartScript || !fs.existsSync(smartScript)) {
+        log(`OC-LIMIT rising-edge but hooks.smartRestart missing -> rotate-until`);
+        runRotateUntil("no-smart-script");
+        return;
+    }
+    log(`OC-LIMIT rising-edge -> async ${path.basename(smartScript)}`);
+    const started = runScriptAsync(workspace, smartScript, log, "cpe-proxy-smart-restart", (code) => {
+        log(`cpe-proxy-smart-restart exit=${code}`);
+        scheduleIpifyProbe(workspace, proxyPort);
+        const ipAfter = carrierIpCached();
+        if (shouldChainRotateUntilAfterSmart({ exitCode: code, ipBefore: fromIp, ipAfter })) {
+            const chainReason = code === SMART_RESTART_COOLDOWN_EXIT_CODE ? "smart-cooldown" : "smart-unchanged";
+            notifyConnectivityStatus(workspace, "OC-LIMIT", `Proxy-SMART did not clear it (${chainReason}). Escalating to cpe-proxy-rotate-until.sh.`, "Wait for resume-sent then complete toasts.", "escalating");
+            runRotateUntil(chainReason);
+            return;
+        }
+        finish(ipAfter, "proxy-smart-restart");
+    });
+    if (!started)
+        state.recoveryRunning = false;
+}
+//# sourceMappingURL=connectivity-recovery.js.map

package/dist/connectivity/connectivity-recovery.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"connectivity-recovery.js","sourceRoot":"","sources":["../../src/connectivity/connectivity-recovery.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,oBAAoB,CAAC;AAC3C,OAAO,EAAE,MAAM,SAAS,CAAC;AACzB,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EACL,gBAAgB,EAChB,0BAA0B,EAC1B,wBAAwB,EACxB,eAAe,EACf,YAAY,GAIb,MAAM,iBAAiB,CAAC;AACzB,OAAO,EAAE,mBAAmB,EAAE,oBAAoB,EAAE,MAAM,iBAAiB,CAAC;AAC5E,OAAO,EAAE,wBAAwB,EAAuB,MAAM,gBAAgB,CAAC;AAE/E,MAAM,oBAAoB,GAAG,MAAM,CAAC;AACpC,yFAAyF;AACzF,MAAM,4BAA4B,GAAG,MAAM,CAAC;AAC5C,MAAM,kBAAkB,GAAG,OAAO,CAAC;AACnC,MAAM,aAAa,GAAG,MAAM,CAAC;AAC7B,yFAAyF;AACzF,MAAM,oBAAoB,GAAG,MAAM,CAAC;AACpC,MAAM,eAAe,GAAG,CAAC,CAAC;AAC1B,+FAA+F;AAC/F,MAAM,2BAA2B,GAAG,OAAO,CAAC;AAC5C,uFAAuF;AACvF,MAAM,0BAA0B,GAAG,OAAO,CAAC;AAC3C,+FAA+F;AAC/F,MAAM,2BAA2B,GAAG,mCAAmC,CAAC;AACxE,MAAM,kCAAkC,GAAG,IAAI,CAAC;AAChD,yFAAyF;AACzF,MAAM,gCAAgC,GAAG,CAAC,CAAC;AAE3C;;;;GAIG;AACH,MAAM,UAAU,2BAA2B,CACzC,YAAoB,OAAO,CAAC,GAAG,CAAC,uBAAuB,IAAI,2BAA2B,EACtF,cAAsB,MAAM,CAC1B,OAAO,CAAC,GAAG,CAAC,8BAA8B,IAAI,kCAAkC,CACjF,EACD,QAAgB,IAAI,CAAC,GAAG,EAAE;IAE1B,IAAI,CAAC;QACH,MAAM,IAAI,GAAG,MAAM,CAAC,EAAE,CAAC,YAAY,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,CAAC,CAAC;QACpE,IAAI,CAAC,IAAI;YAAE,OAAO,CAAC,CAAC;QACpB,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,GAAG,IAAI,CAAC,GAAG,IAAI,CAAC;QAC/C,OAAO,MAAM,IAAI,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,WAAW,GAAG,MAAM,CAAC;IAC1D,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,CAAC,CAAC;IACX,CAAC;AACH,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,sBAAsB,CACpC,KAAyD,EACzD,SAAkB,EAClB,SAAiB;IAEjB,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC;IACpC,IAAI,SAAS,EAAE,CAAC;QACd,KAAK,CAAC,eAAe,GAAG,CAAC,CAAC;QAC1B,OAAO,EAAE,aAAa,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC,EAAE,CAAC;IAC7C,CAAC;IACD,KAAK,CAAC,eAAe,IAAI,CAAC,CAAC;IAC3B,OAAO;QACL,aAAa,EAAE,KAAK,CAAC,eAAe,IAAI,IAAI;QAC5C,MAAM,EAAE,KAAK,CAAC,eAAe;KAC9B,CAAC;AACJ,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,sBAAsB,CACpC,KAAwF,EACxF,eAAwB,EACxB,QAAgB,IAAI,CAAC,GAAG,EAAE,EAC1B,UAAkB,2BAA2B;IAE7C,IAAI,eAAe,EAAE,CAAC;QACpB,KAAK,CAAC,mBAAmB,GAAG,KAAK,CAAC;QAClC,IAAI,CAAC,KAAK,CAAC,sBAAsB,EAAE,CAAC;YAClC,KAAK,CAAC,sBAAsB,GAAG,IAAI,CAAC;YACpC,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC;QAC7B,CAAC;QACD,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,CAAC;IAChC,CAAC;IACD,IAAI,KAAK,CAAC,sBAAsB,IAAI,KAAK,GAAG,KAAK,CAAC,mBAAmB,GAAG,OAAO,EAAE,CAAC;QAChF,KAAK,CAAC,sBAAsB,GAAG,KAAK,CAAC;QACrC,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC;IAC7B,CAAC;IACD,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC;AAC5B,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,gCAAgC,CAAC,KAIhD;IACC,MAAM,EAAE,QAAQ,EAAE,QAAQ,EAAE,OAAO,EAAE,GAAG,KAAK,CAAC;IAC9C,IAAI,QAAQ,KAAK,gCAAgC;QAAE,OAAO,IAAI,CAAC;IAC/D,IAAI,CAAC,OAAO;QAAE,OAAO,IAAI,CAAC;IAC1B,IAAI,QAAQ,IAAI,OAAO,IAAI,QAAQ,KAAK,OAAO;QAAE,OAAO,IAAI,CAAC;IAC7D,OAAO,QAAQ,KAAK,CAAC,CAAC;AACxB,CAAC;AAED,SAAS,mBAAmB,CAAC,MAAqB;IAChD,MAAM,IAAI,GAAG,gBAAgB,CAAC,MAAM,CAAC,CAAC,aAAa,CAAC;IACpD,IAAI,CAAC;QACH,OAAO,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC;IAC7B,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC;AAED,IAAI,aAAa,GAAmB,IAAI,CAAC;AACzC,IAAI,WAAW,GAAG,CAAC,CAAC;AACpB,IAAI,cAAc,GAAG,CAAC,CAAC;AACvB,+FAA+F;AAC/F,IAAI,eAAe,GAAkB,IAAI,CAAC;AAC1C,MAAM,cAAc,GAAG,IAAI,GAAG,EAA+C,CAAC;AAC9E,2FAA2F;AAC3F,MAAM,iBAAiB,GAAG,IAAI,GAAG,EAAkB,CAAC;AACpD,MAAM,eAAe,GAAG,IAAI,GAAG,EAAU,CAAC;AAC1C,IAAI,oBAAoB,GAAG,CAAC,CAAC;AAE7B,+BAA+B;AAC/B,MAAM,UAAU,uBAAuB,CACrC,SAA8B,EAC9B,MAAc,EACd,UAAmB,EACnB,SAAiB;IAEjB,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC;IACpC,IAAI,CAAC,UAAU,EAAE,CAAC;QAChB,SAAS,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QACzB,OAAO,EAAE,SAAS,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC,EAAE,CAAC;IACzC,CAAC;IACD,MAAM,MAAM,GAAG,CAAC,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC;IAChD,SAAS,CAAC,GAAG,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC;IAC9B,OAAO,EAAE,SAAS,EAAE,MAAM,IAAI,IAAI,EAAE,MAAM,EAAE,CAAC;AAC/C,CAAC;AAED,SAAS,mBAAmB;IAC1B,KAAK,MAAM,KAAK,IAAI,cAAc,CAAC,MAAM,EAAE,EAAE,CAAC;QAC5C,IAAI,KAAK,CAAC,IAAI;YAAE,OAAO,IAAI,CAAC;IAC9B,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,0FAA0F;AAC1F,MAAM,UAAU,yBAAyB,CACvC,KAAgC,EAChC,MAAc;IAEd,MAAM,GAAG,GACP,cAAc,CAAC,GAAG,CAAC,MAAM,CAAC;QAC1B,KAAK,CAAC,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC;QAC3B,KAAK,CAAC,YAAY,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;IACjC,cAAc,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;IAC9B,iBAAiB,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;IACjC,KAAK,CAAC,SAAS,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;IAC/B,KAAK,CAAC,YAAY,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;IAClC,OAAO,GAAG,CAAC;AACb,CAAC;AAED,mGAAmG;AACnG,MAAM,UAAU,wBAAwB,CAAC,KAAgC;IACvE,IAAI,mBAAmB,EAAE;QAAE,OAAO;IAClC,IAAI,CAAC,KAAK,CAAC,sBAAsB,IAAI,CAAC,KAAK,CAAC,wBAAwB;QAAE,OAAO;IAC7E,KAAK,CAAC,sBAAsB,GAAG,KAAK,CAAC;IACrC,KAAK,CAAC,wBAAwB,GAAG,KAAK,CAAC;IACvC,KAAK,CAAC,mBAAmB,GAAG,CAAC,CAAC;AAChC,CAAC;AAyBD,MAAM,UAAU,4BAA4B;IAC1C,OAAO;QACL,SAAS,EAAE,IAAI,GAAG,EAAE;QACpB,YAAY,EAAE,IAAI,GAAG,EAAE;QACvB,eAAe,EAAE,KAAK;QACtB,eAAe,EAAE,KAAK;QACtB,aAAa,EAAE,CAAC;QAChB,YAAY,EAAE,CAAC;QACf,uBAAuB,EAAE,IAAI;QAC7B,sBAAsB,EAAE,KAAK;QAC7B,mBAAmB,EAAE,CAAC;QACtB,uBAAuB,EAAE,CAAC;QAC1B,sBAAsB,EAAE,KAAK;QAC7B,wBAAwB,EAAE,KAAK;QAC/B,eAAe,EAAE,CAAC;KACnB,CAAC;AACJ,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,0BAA0B,CACxC,KAA4F,EAC5F,QAAgB,IAAI,CAAC,GAAG,EAAE,EAC1B,cAAsB,0BAA0B;IAEhD,IAAI,KAAK,CAAC,sBAAsB;QAAE,OAAO,KAAK,CAAC;IAC/C,IAAI,CAAC,KAAK,CAAC,uBAAuB;QAAE,OAAO,KAAK,CAAC;IACjD,OAAO,KAAK,GAAG,KAAK,CAAC,uBAAuB,IAAI,WAAW,CAAC;AAC9D,CAAC;AAED,IAAI,kBAAkB,GAAG,KAAK,CAAC;AAE/B,kFAAkF;AAClF,SAAS,kBAAkB,CAAC,SAAiB,EAAE,IAAY;IACzD,IAAI,kBAAkB;QAAE,OAAO;IAC/B,kBAAkB,GAAG,IAAI,CAAC;IAC1B,MAAM,KAAK,GAAG,KAAK,CACjB,MAAM,EACN;QACE,IAAI;QACJ,gCAAgC,IAAI,gCAAgC,IAAI,6DAA6D;KACtI,EACD,EAAE,GAAG,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC,QAAQ,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,CACxD,CAAC;IACF,IAAI,GAAG,GAAG,EAAE,CAAC;IACb,KAAK,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,EAAE,CAAC,CAAC,EAAE,EAAE;QAC7B,GAAG,IAAI,MAAM,CAAC,CAAC,CAAC,CAAC;IACnB,CAAC,CAAC,CAAC;IACH,KAAK,CAAC,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE;QACrB,kBAAkB,GAAG,KAAK,CAAC;QAC3B,WAAW,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACzB,MAAM,EAAE,GAAG,GAAG,CAAC,IAAI,EAAE,CAAC;QACtB,aAAa,GAAG,OAAO,CAAC,EAAE,CAAC,CAAC;QAC5B,IAAI,EAAE;YAAE,eAAe,GAAG,EAAE,CAAC;IAC/B,CAAC,CAAC,CAAC;IACH,KAAK,CAAC,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE;QACrB,kBAAkB,GAAG,KAAK,CAAC;QAC3B,WAAW,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACzB,aAAa,GAAG,KAAK,CAAC;IACxB,CAAC,CAAC,CAAC;AACL,CAAC;AAED,SAAS,eAAe;IACtB,OAAO,eAAe,CAAC;AACzB,CAAC;AAED,SAAS,OAAO,CAAC,SAAiB,EAAE,IAAY,EAAE,OAAO,GAAG,KAAK;IAC/D,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IACvB,MAAM,MAAM,GAAG,OAAO,CAAC,CAAC,CAAC,oBAAoB,CAAC,CAAC,CAAC,aAAa,CAAC;IAC9D,IAAI,GAAG,GAAG,WAAW,IAAI,MAAM,EAAE,CAAC;QAChC,kBAAkB,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC;IACtC,CAAC;IACD,IAAI,aAAa,KAAK,IAAI;QAAE,OAAO,aAAa,CAAC;IACjD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAS,cAAc,CACrB,SAAiB,EACjB,UAAkB,EAClB,GAA2B,EAC3B,KAAa,EACb,MAA8B,EAC9B,WAAmC,EAAE;IAErC,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC;QAC/B,GAAG,CAAC,GAAG,KAAK,aAAa,UAAU,EAAE,CAAC,CAAC;QACvC,OAAO,KAAK,CAAC;IACf,CAAC;IACD,MAAM,KAAK,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,UAAU,CAAC,EAAE;QACxC,GAAG,EAAE,SAAS;QACd,KAAK,EAAE,QAAQ;QACf,QAAQ,EAAE,KAAK;QACf,GAAG,EAAE,EAAE,GAAG,OAAO,CAAC,GAAG,EAAE,uBAAuB,EAAE,GAAG,EAAE,GAAG,QAAQ,EAAE;KACnE,CAAC,CAAC;IACH,KAAK,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,CAAC,EAAE,EAAE;QACtB,GAAG,CAAC,GAAG,KAAK,iBAAkB,CAAW,CAAC,OAAO,EAAE,CAAC,CAAC;QACrD,MAAM,CAAC,CAAC,CAAC,CAAC;IACZ,CAAC,CAAC,CAAC;IACH,KAAK,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,CAAC;IAC/C,OAAO,IAAI,CAAC;AACd,CAAC;AAiBD,8FAA8F;AAC9F,MAAM,UAAU,wBAAwB,CAAC,KAA4B;IACnE,MAAM,EACJ,MAAM,EACN,QAAQ,EACR,SAAS,EACT,OAAO,EACP,UAAU,EACV,aAAa,EACb,WAAW,EACX,KAAK,EACL,GAAG,EACH,mBAAmB,EACnB,aAAa,GACd,GAAG,KAAK,CAAC;IAEV,MAAM,IAAI,GAAG,MAAM,CAAC,OAAO,CAAC,YAAY,CAAC;IACzC,IAAI,CAAC,IAAI,EAAE,OAAO;QAAE,OAAO;IAE3B,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,IAAI,KAAK,CAAC;IAC1C,MAAM,KAAK,GAAG,oBAAoB,CAAC,OAAO,EAAE,UAAU,EAAE,aAAa,EAAE,WAAW,CAAC,CAAC;IACpF,MAAM,OAAO,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC;IACpD,KAAK,MAAM,EAAE,IAAI,cAAc,CAAC,IAAI,EAAE,EAAE,CAAC;QACvC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC;YAAE,cAAc,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;IAClD,CAAC;IACD,KAAK,MAAM,EAAE,IAAI,iBAAiB,CAAC,IAAI,EAAE,EAAE,CAAC;QAC1C,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC;YAAE,iBAAiB,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;IACrD,CAAC;IACD,KAAK,MAAM,EAAE,IAAI,eAAe,EAAE,CAAC;QACjC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC;YAAE,eAAe,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;IACnD,CAAC;IAED,MAAM,gBAAgB,GACpB,IAAI,CAAC,MAAM,EAAE,yBAAyB;QACtC,IAAI,CAAC,MAAM,EAAE,uBAAuB;QACpC,EAAE,CAAC;IAEL,MAAM,WAAW,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC;IAChD,MAAM,WAAW,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;IAE7C,MAAM,KAAK,GAAiB,EAAE,CAAC;IAC/B,IAAI,KAAK,CAAC,MAAM,EAAE,CAAC;QACjB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,eAAe,EAAE,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;YACjE,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,cAAc,GAAG,CAAC,CAAC,GAAG,KAAK,CAAC,MAAM,CAAE,CAAC,CAAC;QAC1D,CAAC;QACD,cAAc,GAAG,CAAC,cAAc,GAAG,KAAK,CAAC,MAAM,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC;IAClE,CAAC;IAED,MAAM,UAAU,GACd,MAAM,CAAC,OAAO,CAAC,EAAE,KAAK,SAAS,CAAC,CAAC,CAAC,eAAe,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IAC9E,MAAM,EAAE,cAAc,EAAE,cAAc,EAAE,cAAc,EAAE,GAAG,UAAU;QACnE,CAAC,CAAC,YAAY,CAAC,UAAU,CAAC;QAC1B,CAAC,CAAC;YACE,cAAc,EAAE,IAAI,GAAG,CAAC,CAAC,YAAY,CAAC,CAAC;YACvC,cAAc,EAAE,IAAI,GAAG,CAAC,CAAC,UAAU,CAAC,CAAC;YACrC,cAAc,EAAE,IAAI,GAAG,CAAC,CAAC,UAAU,CAAC,CAAC;SACtC,CAAC;IAEN,KAAK,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,IAAI,KAAK,EAAE,CAAC;QACtC,MAAM,IAAI,GAAG,mBAAmB,CAAC,MAAM,CAAC,CAAC;QACzC,IAAI,CAAC,IAAI,EAAE,CAAC;YACV,cAAc,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;YAC9B,SAAS;QACX,CAAC;QACD,MAAM,IAAI,GAAG,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;QACnC,IAAI,CAAC,IAAI,IAAI,CAAC,0BAA0B,CAAC,IAAI,CAAC,EAAE,EAAE,cAAc,CAAC,EAAE,CAAC;YAClE,cAAc,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;YAC9B,SAAS;QACX,CAAC;QACD,MAAM,EAAE,GAAG,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC;QACpC,IAAI,EAAE,CAAC,KAAK,KAAK,OAAO,IAAI,EAAE,CAAC,SAAS,KAAK,UAAU,IAAI,IAAI,CAAC,EAAE,KAAK,QAAQ,EAAE,CAAC;YAChF,IAAI,CAAC,eAAe,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC;gBACjC,eAAe,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;gBAC5B,aAAa,EAAE,CAAC,MAAM,EAAE,IAAI,CAAC,CAAC;gBAC9B,GAAG,CAAC,mBAAmB,KAAK,IAAI,MAAM,EAAE,CAAC,CAAC;YAC5C,CAAC;YACD,SAAS;QACX,CAAC;QACD,IAAI,EAAE,CAAC,KAAK,KAAK,OAAO,IAAI,CAAC,EAAE,CAAC,SAAS,EAAE,CAAC;YAC1C,cAAc,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;YAC9B,iBAAiB,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;YACjC,eAAe,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;YAC/B,SAAS;QACX,CAAC;QACD,MAAM,UAAU,GAAG,cAAc,CAAC,GAAG,CAAC,EAAE,CAAC,SAAS,CAAC,CAAC;QACpD,MAAM,IAAI,GAAG,cAAc,CAAC,GAAG,CAAC,EAAE,CAAC,SAAS,CAAC,CAAC;QAC9C,IAAI,CAAC,UAAU,IAAI,CAAC,IAAI,EAAE,CAAC;YACzB,cAAc,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;YAC9B,iBAAiB,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;YACjC,SAAS;QACX,CAAC;QACD,IAAI,OAAO,GAAG,KAAK,CAAC;QACpB,IAAI,UAAU,EAAE,CAAC;YACf,MAAM,EAAE,SAAS,EAAE,MAAM,EAAE,GAAG,uBAAuB,CACnD,iBAAiB,EACjB,MAAM,EACN,IAAI,EACJ,gBAAgB,CACjB,CAAC;YACF,OAAO,GAAG,SAAS,CAAC;YACpB,IAAI,CAAC,SAAS,EAAE,CAAC;gBACf,IAAI,MAAM,KAAK,CAAC,EAAE,CAAC;oBACjB,GAAG,CACD,oBAAoB,KAAK,IAAI,MAAM,UAAU,gBAAgB,kDAAkD,CAChH,CAAC;gBACJ,CAAC;YACH,CAAC;iBAAM,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC;gBACpC,GAAG,CAAC,qBAAqB,KAAK,IAAI,MAAM,SAAS,EAAE,CAAC,SAAS,WAAW,MAAM,EAAE,CAAC,CAAC;YACpF,CAAC;QACH,CAAC;aAAM,CAAC;YACN,uBAAuB,CAAC,iBAAiB,EAAE,MAAM,EAAE,KAAK,EAAE,gBAAgB,CAAC,CAAC;QAC9E,CAAC;QACD,IAAI,OAAO,IAAI,IAAI,EAAE,CAAC;YACpB,cAAc,CAAC,GAAG,CAAC,MAAM,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC;QAChD,CAAC;aAAM,CAAC;YACN,cAAc,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;QAChC,CAAC;QACD,IAAI,IAAI,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC;YACrC,GAAG,CAAC,mBAAmB,KAAK,IAAI,MAAM,SAAS,EAAE,CAAC,SAAS,EAAE,CAAC,CAAC;QACjE,CAAC;IACH,CAAC;IAED,KAAK,CAAC,YAAY,CAAC,KAAK,EAAE,CAAC;IAC3B,KAAK,CAAC,SAAS,CAAC,KAAK,EAAE,CAAC;IACxB,IAAI,UAAU,GAAG,KAAK,CAAC;IACvB,IAAI,YAAY,GAAG,KAAK,CAAC;IACzB,KAAK,MAAM,CAAC,MAAM,EAAE,KAAK,CAAC,IAAI,cAAc,EAAE,CAAC;QAC7C,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC;YAAE,SAAS;QACnC,KAAK,CAAC,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QAC5B,IAAI,KAAK,CAAC,OAAO,EAAE,CAAC;YAClB,KAAK,CAAC,YAAY,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;YAC/B,UAAU,GAAG,IAAI,CAAC;QACpB,CAAC;aAAM,IAAI,KAAK,CAAC,IAAI,EAAE,CAAC;YACtB,YAAY,GAAG,IAAI,CAAC;QACtB,CAAC;IACH,CAAC;IAED,MAAM,KAAK,GAAG,wBAAwB,CAAC,MAAM,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC;IAElE,IAAI,mBAAmB,CAAC,MAAM,CAAC,EAAE,CAAC;QAChC,IAAI,KAAK,CAAC,eAAe,EAAE,CAAC;YAC1B,GAAG,CAAC,oDAAoD,CAAC,CAAC;YAC1D,KAAK,CAAC,eAAe,GAAG,KAAK,CAAC;QAChC,CAAC;QACD,OAAO;IACT,CAAC;IAED,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,EAAE,SAAS,EAAE,UAAU,IAAI,KAAK,CAAC,eAAe,CAAC,CAAC;IACrF,MAAM,cAAc,GAAG,IAAI,CAAC,MAAM,EAAE,uBAAuB,IAAI,EAAE,CAAC;IAClE,MAAM,EAAE,aAAa,EAAE,kBAAkB,EAAE,MAAM,EAAE,WAAW,EAAE,GAAG,sBAAsB,CACvF,KAAK,EACL,SAAS,EACT,cAAc,CACf,CAAC;IACF,IAAI,CAAC,SAAS,IAAI,CAAC,kBAAkB,IAAI,CAAC,UAAU,EAAE,CAAC;QACrD,GAAG,CACD,2BAA2B,WAAW,IAAI,cAAc,8BAA8B,CACvF,CAAC;IACJ,CAAC;IACD,6FAA6F;IAC7F,mFAAmF;IACnF,MAAM,oBAAoB,GACxB,kBAAkB,IAAI,CAAC,YAAY,IAAI,KAAK,CAAC,sBAAsB,CAAC,CAAC;IACvE,IAAI,YAAY,GAAG,UAAU,IAAI,oBAAoB,CAAC;IACtD,MAAM,OAAO,GAAG,KAAK,CAAC,eAAe,CAAC;IACtC,MAAM,YAAY,GAChB,CAAC,OAAO;QACR,YAAY;QACZ,UAAU;QACV,CAAC,oBAAoB;QACrB,SAAS;QACT,IAAI,CAAC,GAAG,EAAE,GAAG,oBAAoB,GAAG,4BAA4B,CAAC;IACnE,IAAI,YAAY,EAAE,CAAC;QACjB,GAAG,CACD,kDAAkD,4BAA4B,GAAG,IAAI,iCAAiC,CACvH,CAAC;QACF,YAAY,GAAG,KAAK,CAAC;IACvB,CAAC;IACD,KAAK,CAAC,eAAe,GAAG,YAAY,CAAC;IAErC,IAAI,YAAY,IAAI,CAAC,OAAO,EAAE,CAAC;QAC7B,KAAK,CAAC,uBAAuB,GAAG,eAAe,EAAE,CAAC;QAClD,KAAK,CAAC,uBAAuB,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC3C,KAAK,CAAC,sBAAsB,GAAG,KAAK,CAAC;QACrC,GAAG,CACD,kCAAkC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,WAAW,WAAW,IAAI,cAAc,iBAAiB,KAAK,CAAC,YAAY,CAAC,IAAI,YAAY,KAAK,CAAC,uBAAuB,IAAI,GAAG,EAAE,CAC9L,CAAC;QACF,wBAAwB,CACtB,SAAS,EACT,YAAY,EACZ,WAAW,KAAK,CAAC,uBAAuB,IAAI,GAAG,4CAA4C,EAC3F,wEAAwE,EACxE,UAAU,CACX,CAAC;QACF,eAAe,CAAC,KAAK,EAAE,SAAS,EAAE,KAAK,EAAE,GAAG,CAAC,CAAC;IAChD,CAAC;SAAM,IAAI,CAAC,YAAY,IAAI,OAAO,EAAE,CAAC;QACpC,MAAM,MAAM,GAAG,KAAK,CAAC,uBAAuB,CAAC;QAC7C,MAAM,IAAI,GAAG,eAAe,EAAE,CAAC;QAC/B,GAAG,CAAC,mDAAmD,IAAI,IAAI,GAAG,EAAE,CAAC,CAAC;QACtE,KAAK,CAAC,uBAAuB,GAAG,IAAI,CAAC;QACrC,KAAK,CAAC,uBAAuB,GAAG,CAAC,CAAC;QAClC,KAAK,CAAC,sBAAsB,GAAG,KAAK,CAAC;QACrC,KAAK,CAAC,eAAe,GAAG,CAAC,CAAC;QAC1B,oBAAoB,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAClC,iBAAiB,CAAC,KAAK,EAAE,CAAC;QAC1B,kGAAkG;QAClG,KAAK,MAAM,CAAC,MAAM,EAAE,KAAK,CAAC,IAAI,CAAC,GAAG,cAAc,CAAC,OAAO,EAAE,CAAC,EAAE,CAAC;YAC5D,IAAI,KAAK,CAAC,IAAI,EAAE,CAAC;gBACf,cAAc,CAAC,GAAG,CAAC,MAAM,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;YAC7D,CAAC;iBAAM,CAAC;gBACN,cAAc,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;YAChC,CAAC;QACH,CAAC;QACD,YAAY,CAAC,GAAG,EAAE,CAAC,mBAAmB,CAAC,UAAU,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;IACxE,CAAC;SAAM,IAAI,YAAY,IAAI,OAAO,IAAI,0BAA0B,CAAC,KAAK,CAAC,EAAE,CAAC;QACxE,KAAK,CAAC,sBAAsB,GAAG,IAAI,CAAC;QACpC,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC,uBAAuB,CAAC,GAAG,IAAI,CAAC,CAAC;QAC/E,GAAG,CAAC,iCAAiC,MAAM,sBAAsB,CAAC,CAAC;QACnE,wBAAwB,CACtB,SAAS,EACT,YAAY,EACZ,mCAAmC,MAAM,sCAAsC,EAC/E,2CAA2C,EAC3C,YAAY,CACb,CAAC;IACJ,CAAC;IAED,MAAM,EAAE,MAAM,EAAE,GAAG,sBAAsB,CAAC,KAAK,EAAE,YAAY,CAAC,CAAC;IAC/D,IAAI,MAAM,KAAK,OAAO,EAAE,CAAC;QACvB,MAAM,MAAM,GAAG,eAAe,EAAE,IAAI,KAAK,CAAC,uBAAuB,IAAI,eAAe,CAAC;QACrF,GAAG,CAAC,8CAA8C,MAAM,IAAI,GAAG,EAAE,CAAC,CAAC;QACnE,IAAI,CAAC,KAAK,CAAC,wBAAwB,EAAE,CAAC;YACpC,wBAAwB,CACtB,SAAS,EACT,UAAU,EACV,WAAW,MAAM,IAAI,GAAG,2FAA2F,EACnH,uFAAuF,EACvF,UAAU,CACX,CAAC;YACF,MAAM,SAAS,GAAG,IAAI,CAAC,MAAM,EAAE,iBAAiB,IAAI,EAAE,CAAC;YACvD,uBAAuB,CACrB,KAAK,EACL,SAAS,EACT,SAAS,EACT,MAAM,EACN,KAAK,EACL,GAAG,EACH,mBAAmB,EACnB,SAAS,CACV,CAAC;QACJ,CAAC;aAAM,CAAC;YACN,GAAG,CAAC,kFAAkF,CAAC,CAAC;QAC1F,CAAC;IACH,CAAC;SAAM,IAAI,MAAM,KAAK,OAAO,EAAE,CAAC;QAC9B,GAAG,CAAC,wBAAwB,CAAC,CAAC;QAC9B,KAAK,CAAC,wBAAwB,GAAG,KAAK,CAAC;IACzC,CAAC;AACH,CAAC;AAED,SAAS,eAAe,CACtB,KAAkD,EAClD,SAAiB,EACjB,KAAgC,EAChC,GAA2B;IAE3B,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IACvB,IAAI,KAAK,CAAC,eAAe;QAAE,OAAO;IAClC,IAAI,GAAG,GAAG,KAAK,CAAC,aAAa,GAAG,oBAAoB;QAAE,OAAO;IAC7D,MAAM,MAAM,GAAG,KAAK,EAAE,EAAE,CAAC;IACzB,IAAI,CAAC,MAAM,EAAE,CAAC;QACZ,GAAG,CAAC,yDAAyD,CAAC,CAAC;QAC/D,OAAO;IACT,CAAC;IAED,KAAK,CAAC,eAAe,GAAG,IAAI,CAAC;IAC7B,KAAK,CAAC,aAAa,GAAG,GAAG,CAAC;IAC1B,GAAG,CAAC,uBAAuB,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;IACpD,MAAM,OAAO,GAAG,cAAc,CAAC,SAAS,EAAE,MAAM,EAAE,GAAG,EAAE,UAAU,EAAE,CAAC,IAAI,EAAE,EAAE;QAC1E,KAAK,CAAC,eAAe,GAAG,KAAK,CAAC;QAC9B,GAAG,CAAC,qBAAqB,IAAI,mDAAmD,CAAC,CAAC;IACpF,CAAC,CAAC,CAAC;IACH,IAAI,CAAC,OAAO;QAAE,KAAK,CAAC,eAAe,GAAG,KAAK,CAAC;AAC9C,CAAC;AAED;;;;;GAKG;AACH,SAAS,uBAAuB,CAC9B,KAAkD,EAClD,SAAiB,EACjB,SAAiB,EACjB,MAAqB,EACrB,KAAgC,EAChC,GAA2B,EAC3B,mBAAiE,EACjE,iBAAiB,GAAG,EAAE;IAEtB,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IACvB,IAAI,KAAK,CAAC,eAAe;QAAE,OAAO;IAClC,IAAI,KAAK,CAAC,wBAAwB;QAAE,OAAO;IAC3C,IAAI,GAAG,GAAG,KAAK,CAAC,YAAY,GAAG,kBAAkB;QAAE,OAAO;IAE1D,KAAK,CAAC,eAAe,GAAG,IAAI,CAAC;IAC7B,KAAK,CAAC,wBAAwB,GAAG,IAAI,CAAC;IACtC,KAAK,CAAC,YAAY,GAAG,GAAG,CAAC;IAEzB,MAAM,MAAM,GAAG,CAAC,IAAmB,EAAE,MAAc,EAAE,EAAE;QACrD,KAAK,CAAC,eAAe,GAAG,KAAK,CAAC;QAC9B,MAAM,SAAS,GAAG,OAAO,CAAC,IAAI,IAAI,MAAM,IAAI,IAAI,KAAK,MAAM,CAAC,CAAC;QAC7D,IAAI,CAAC,SAAS,EAAE,CAAC;YACf,GAAG,CACD,iCAAiC,MAAM,OAAO,IAAI,IAAI,GAAG,yCAAyC,CACnG,CAAC;YACF,OAAO;QACT,CAAC;QACD,GAAG,CAAC,gDAAgD,MAAM,IAAI,MAAM,OAAO,IAAI,EAAE,CAAC,CAAC;QACnF,YAAY,CAAC,GAAG,EAAE,CAAC,mBAAmB,CAAC,MAAM,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;IACpE,CAAC,CAAC;IAEF,MAAM,cAAc,GAAG,CAAC,MAAc,EAAE,EAAE;QACxC,MAAM,MAAM,GAAG,KAAK,EAAE,MAAM,CAAC;QAC7B,IAAI,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,MAAM,CAAC,EAAE,CAAC;YACtC,GAAG,CAAC,yDAAyD,MAAM,IAAI,OAAO,GAAG,CAAC,CAAC;YACnF,KAAK,CAAC,eAAe,GAAG,KAAK,CAAC;YAC9B,OAAO;QACT,CAAC;QACD,GAAG,CAAC,qBAAqB,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAK,MAAM,SAAS,iBAAiB,EAAE,CAAC,CAAC;QACvF,MAAM,SAAS,GAAG,EAAE,uBAAuB,EAAE,MAAM,CAAC,iBAAiB,CAAC,EAAE,CAAC;QACzE,MAAM,OAAO,GAAG,cAAc,CAC5B,SAAS,EACT,MAAM,EACN,GAAG,EACH,cAAc,EACd,CAAC,IAAI,EAAE,EAAE;YACP,GAAG,CAAC,qBAAqB,IAAI,EAAE,CAAC,CAAC;YACjC,kBAAkB,CAAC,SAAS,EAAE,SAAS,CAAC,CAAC;YACzC,MAAM,CAAC,eAAe,EAAE,EAAE,cAAc,CAAC,CAAC;QAC5C,CAAC,EACD,SAAS,CACV,CAAC;QACF,IAAI,CAAC,OAAO;YAAE,KAAK,CAAC,eAAe,GAAG,KAAK,CAAC;IAC9C,CAAC,CAAC;IAEF,MAAM,YAAY,GAAG,2BAA2B,EAAE,CAAC;IACnD,IAAI,YAAY,GAAG,CAAC,EAAE,CAAC;QACrB,GAAG,CAAC,iCAAiC,YAAY,wBAAwB,CAAC,CAAC;QAC3E,cAAc,CAAC,gBAAgB,CAAC,CAAC;QACjC,OAAO;IACT,CAAC;IAED,MAAM,WAAW,GAAG,KAAK,EAAE,YAAY,CAAC;IACxC,IAAI,CAAC,WAAW,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,WAAW,CAAC,EAAE,CAAC;QAChD,GAAG,CAAC,qEAAqE,CAAC,CAAC;QAC3E,cAAc,CAAC,iBAAiB,CAAC,CAAC;QAClC,OAAO;IACT,CAAC;IAED,GAAG,CAAC,iCAAiC,IAAI,CAAC,QAAQ,CAAC,WAAW,CAAC,EAAE,CAAC,CAAC;IACnE,MAAM,OAAO,GAAG,cAAc,CAAC,SAAS,EAAE,WAAW,EAAE,GAAG,EAAE,yBAAyB,EAAE,CAAC,IAAI,EAAE,EAAE;QAC9F,GAAG,CAAC,gCAAgC,IAAI,EAAE,CAAC,CAAC;QAC5C,kBAAkB,CAAC,SAAS,EAAE,SAAS,CAAC,CAAC;QACzC,MAAM,OAAO,GAAG,eAAe,EAAE,CAAC;QAClC,IAAI,gCAAgC,CAAC,EAAE,QAAQ,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC,EAAE,CAAC;YACpF,MAAM,WAAW,GAAG,IAAI,KAAK,gCAAgC,CAAC,CAAC,CAAC,gBAAgB,CAAC,CAAC,CAAC,iBAAiB,CAAC;YACrG,wBAAwB,CACtB,SAAS,EACT,UAAU,EACV,iCAAiC,WAAW,6CAA6C,EACzF,4CAA4C,EAC5C,YAAY,CACb,CAAC;YACF,cAAc,CAAC,WAAW,CAAC,CAAC;YAC5B,OAAO;QACT,CAAC;QACD,MAAM,CAAC,OAAO,EAAE,qBAAqB,CAAC,CAAC;IACzC,CAAC,CAAC,CAAC;IACH,IAAI,CAAC,OAAO;QAAE,KAAK,CAAC,eAAe,GAAG,KAAK,CAAC;AAC9C,CAAC"}

package/dist/connectivity/connectivity-recovery.test.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=connectivity-recovery.test.d.ts.map

package/dist/connectivity/connectivity-recovery.test.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"connectivity-recovery.test.d.ts","sourceRoot":"","sources":["../../src/connectivity/connectivity-recovery.test.ts"],"names":[],"mappings":""}