@seat-mesh/daemon 0.1.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (121) hide show
  1. package/dist/border-paint.d.ts +11 -0
  2. package/dist/border-paint.d.ts.map +1 -0
  3. package/dist/border-paint.js +163 -0
  4. package/dist/border-paint.js.map +1 -0
  5. package/dist/bullmq-runtime.d.ts +19 -0
  6. package/dist/bullmq-runtime.d.ts.map +1 -0
  7. package/dist/bullmq-runtime.js +78 -0
  8. package/dist/bullmq-runtime.js.map +1 -0
  9. package/dist/compose-gate.d.ts +18 -0
  10. package/dist/compose-gate.d.ts.map +1 -0
  11. package/dist/compose-gate.js +80 -0
  12. package/dist/compose-gate.js.map +1 -0
  13. package/dist/compose-gate.test.d.ts +2 -0
  14. package/dist/compose-gate.test.d.ts.map +1 -0
  15. package/dist/compose-gate.test.js +35 -0
  16. package/dist/compose-gate.test.js.map +1 -0
  17. package/dist/connectivity-recovery.d.ts +81 -0
  18. package/dist/connectivity-recovery.d.ts.map +1 -0
  19. package/dist/connectivity-recovery.js +454 -0
  20. package/dist/connectivity-recovery.js.map +1 -0
  21. package/dist/connectivity-recovery.test.d.ts +2 -0
  22. package/dist/connectivity-recovery.test.d.ts.map +1 -0
  23. package/dist/connectivity-recovery.test.js +147 -0
  24. package/dist/connectivity-recovery.test.js.map +1 -0
  25. package/dist/create-queue-store.d.ts +8 -0
  26. package/dist/create-queue-store.d.ts.map +1 -0
  27. package/dist/create-queue-store.js +13 -0
  28. package/dist/create-queue-store.js.map +1 -0
  29. package/dist/delivery-hold.d.ts +10 -0
  30. package/dist/delivery-hold.d.ts.map +1 -0
  31. package/dist/delivery-hold.js +21 -0
  32. package/dist/delivery-hold.js.map +1 -0
  33. package/dist/delivery-hold.test.d.ts +2 -0
  34. package/dist/delivery-hold.test.d.ts.map +1 -0
  35. package/dist/delivery-hold.test.js +35 -0
  36. package/dist/delivery-hold.test.js.map +1 -0
  37. package/dist/index.d.ts +3 -0
  38. package/dist/index.d.ts.map +1 -0
  39. package/dist/index.js +3 -0
  40. package/dist/index.js.map +1 -0
  41. package/dist/inject-delivery.d.ts +26 -0
  42. package/dist/inject-delivery.d.ts.map +1 -0
  43. package/dist/inject-delivery.js +108 -0
  44. package/dist/inject-delivery.js.map +1 -0
  45. package/dist/inject-delivery.test.d.ts +2 -0
  46. package/dist/inject-delivery.test.d.ts.map +1 -0
  47. package/dist/inject-delivery.test.js +14 -0
  48. package/dist/inject-delivery.test.js.map +1 -0
  49. package/dist/jsonl-dedupe.test.d.ts +2 -0
  50. package/dist/jsonl-dedupe.test.d.ts.map +1 -0
  51. package/dist/jsonl-dedupe.test.js +27 -0
  52. package/dist/jsonl-dedupe.test.js.map +1 -0
  53. package/dist/jsonl-store.d.ts +103 -0
  54. package/dist/jsonl-store.d.ts.map +1 -0
  55. package/dist/jsonl-store.js +235 -0
  56. package/dist/jsonl-store.js.map +1 -0
  57. package/dist/mesh-inbox-server.d.ts +3 -0
  58. package/dist/mesh-inbox-server.d.ts.map +1 -0
  59. package/dist/mesh-inbox-server.js +483 -0
  60. package/dist/mesh-inbox-server.js.map +1 -0
  61. package/dist/mesh-inbox-supervisor.d.ts +3 -0
  62. package/dist/mesh-inbox-supervisor.d.ts.map +1 -0
  63. package/dist/mesh-inbox-supervisor.js +227 -0
  64. package/dist/mesh-inbox-supervisor.js.map +1 -0
  65. package/dist/mesh-orchestrator.d.ts +32 -0
  66. package/dist/mesh-orchestrator.d.ts.map +1 -0
  67. package/dist/mesh-orchestrator.js +340 -0
  68. package/dist/mesh-orchestrator.js.map +1 -0
  69. package/dist/oc-resume-ack.d.ts +32 -0
  70. package/dist/oc-resume-ack.d.ts.map +1 -0
  71. package/dist/oc-resume-ack.js +89 -0
  72. package/dist/oc-resume-ack.js.map +1 -0
  73. package/dist/oc-resume-ack.test.d.ts +2 -0
  74. package/dist/oc-resume-ack.test.d.ts.map +1 -0
  75. package/dist/oc-resume-ack.test.js +23 -0
  76. package/dist/oc-resume-ack.test.js.map +1 -0
  77. package/dist/oc-resume.d.ts +25 -0
  78. package/dist/oc-resume.d.ts.map +1 -0
  79. package/dist/oc-resume.js +102 -0
  80. package/dist/oc-resume.js.map +1 -0
  81. package/dist/orchestrator.d.ts +17 -0
  82. package/dist/orchestrator.d.ts.map +1 -0
  83. package/dist/orchestrator.js +11 -0
  84. package/dist/orchestrator.js.map +1 -0
  85. package/dist/pane-ops-drain.d.ts +14 -0
  86. package/dist/pane-ops-drain.d.ts.map +1 -0
  87. package/dist/pane-ops-drain.js +53 -0
  88. package/dist/pane-ops-drain.js.map +1 -0
  89. package/dist/peer-backlog.d.ts +18 -0
  90. package/dist/peer-backlog.d.ts.map +1 -0
  91. package/dist/peer-backlog.js +137 -0
  92. package/dist/peer-backlog.js.map +1 -0
  93. package/dist/peer-comms-checkback.d.ts +5 -0
  94. package/dist/peer-comms-checkback.d.ts.map +1 -0
  95. package/dist/peer-comms-checkback.js +57 -0
  96. package/dist/peer-comms-checkback.js.map +1 -0
  97. package/dist/peer-skip.d.ts +10 -0
  98. package/dist/peer-skip.d.ts.map +1 -0
  99. package/dist/peer-skip.js +26 -0
  100. package/dist/peer-skip.js.map +1 -0
  101. package/dist/peer-skip.test.d.ts +2 -0
  102. package/dist/peer-skip.test.d.ts.map +1 -0
  103. package/dist/peer-skip.test.js +33 -0
  104. package/dist/peer-skip.test.js.map +1 -0
  105. package/dist/ppa-state.d.ts +32 -0
  106. package/dist/ppa-state.d.ts.map +1 -0
  107. package/dist/ppa-state.js +63 -0
  108. package/dist/ppa-state.js.map +1 -0
  109. package/dist/sqlite-store.d.ts +44 -0
  110. package/dist/sqlite-store.d.ts.map +1 -0
  111. package/dist/sqlite-store.js +422 -0
  112. package/dist/sqlite-store.js.map +1 -0
  113. package/dist/sqlite-store.test.d.ts +2 -0
  114. package/dist/sqlite-store.test.d.ts.map +1 -0
  115. package/dist/sqlite-store.test.js +63 -0
  116. package/dist/sqlite-store.test.js.map +1 -0
  117. package/dist/workers.d.ts +11 -0
  118. package/dist/workers.d.ts.map +1 -0
  119. package/dist/workers.js +12 -0
  120. package/dist/workers.js.map +1 -0
  121. package/package.json +28 -0
@@ -0,0 +1,454 @@
1
+ import { spawn } from "node:child_process";
2
+ import fs from "node:fs";
3
+ import path from "node:path";
4
+ import { meshRuntimePaths, providerMonitoredForLimits, resolveConnectivityHooks, resolveUxConfig, uxLimitKinds, } from "@seat-mesh/core";
5
+ import { capturePaneSnapshot, listMeshMonitorPanes } from "@seat-mesh/tmux";
6
+ import { notifyConnectivityStatus } from "./oc-resume.js";
7
+ const PROXY_UP_COOLDOWN_MS = 60_000;
8
+ const ROTATE_COOLDOWN_MS = 180_000;
9
+ const IPIFY_POLL_MS = 30_000;
10
+ /** During PROXY-DOWN episode, still cap sync ipify curls so /health stays responsive. */
11
+ const IPIFY_EPISODE_MIN_MS = 15_000;
12
+ const PANE_SCAN_BATCH = 4;
13
+ /** Grace before clearing a rate-limit episode after the last limited pane clears (flicker). */
14
+ const RATE_LIMIT_EPISODE_CLEAR_MS = 120_000;
15
+ /** How long a PROXY-DOWN episode may run before operator gets a "still down" toast. */
16
+ const PROXY_DOWN_STUCK_NOTIFY_MS = 120_000;
17
+ /** Mirrors scripts/cpe-proxy-smart-restart.sh cooldown file (harness parity — do not fork). */
18
+ const DEFAULT_SMART_RESTART_STAMP = "/tmp/cpe-proxy-smart-restart.last";
19
+ const DEFAULT_SMART_RESTART_COOLDOWN_SEC = 1800;
20
+ /** scripts/cpe-proxy-smart-restart.sh exit code for "cooldown still active, skipped". */
21
+ const SMART_RESTART_COOLDOWN_EXIT_CODE = 2;
22
+ /**
23
+ * Seconds left on the Proxy-SMART file cooldown (0 = none / expired).
24
+ * Reads the same stamp file cpe-proxy-smart-restart.sh writes — never the daemon's
25
+ * own state — so the daemon and any harness script agree on cooldown status.
26
+ */
27
+ export function smartRestartCooldownLeftSec(stampPath = process.env.CPE_SMART_RESTART_STAMP || DEFAULT_SMART_RESTART_STAMP, cooldownSec = Number(process.env.CPE_SMART_RESTART_COOLDOWN_SEC || DEFAULT_SMART_RESTART_COOLDOWN_SEC), nowMs = Date.now()) {
28
+ try {
29
+ const last = Number(fs.readFileSync(stampPath, "utf8").trim()) || 0;
30
+ if (!last)
31
+ return 0;
32
+ const ageSec = Math.floor(nowMs / 1000) - last;
33
+ return ageSec >= cooldownSec ? 0 : cooldownSec - ageSec;
34
+ }
35
+ catch {
36
+ return 0;
37
+ }
38
+ }
39
+ /**
40
+ * Consecutive ipify probe failures before treating carrier as down (debounce flaky curls).
41
+ * Exported for unit testing without tmux/process spawning.
42
+ */
43
+ export function updateIpifyProbeStreak(state, carrierOk, threshold) {
44
+ const need = Math.max(1, threshold);
45
+ if (carrierOk) {
46
+ state.ipifyFailStreak = 0;
47
+ return { confirmedDown: false, streak: 0 };
48
+ }
49
+ state.ipifyFailStreak += 1;
50
+ return {
51
+ confirmedDown: state.ipifyFailStreak >= need,
52
+ streak: state.ipifyFailStreak,
53
+ };
54
+ }
55
+ /**
56
+ * Pure rising-edge/episode reducer for the OC-LIMIT (rate-limit) state machine.
57
+ * Exported for unit testing without tmux/process spawning.
58
+ */
59
+ export function updateRateLimitEpisode(state, anyRateLimitNow, nowMs = Date.now(), graceMs = RATE_LIMIT_EPISODE_CLEAR_MS) {
60
+ if (anyRateLimitNow) {
61
+ state.lastRateLimitSeenAt = nowMs;
62
+ if (!state.rateLimitEpisodeActive) {
63
+ state.rateLimitEpisodeActive = true;
64
+ return { action: "start" };
65
+ }
66
+ return { action: "continue" };
67
+ }
68
+ if (state.rateLimitEpisodeActive && nowMs - state.lastRateLimitSeenAt > graceMs) {
69
+ state.rateLimitEpisodeActive = false;
70
+ return { action: "clear" };
71
+ }
72
+ return { action: "none" };
73
+ }
74
+ /**
75
+ * Pure decision: after Proxy-SMART exits, should rotate-until run next?
76
+ * Mirrors inbox-server.mjs shouldRunRotateUntilAfterSmart — chain rotate-until
77
+ * when SMART was skipped (cooldown), failed, or the carrier IP did not change.
78
+ */
79
+ export function shouldChainRotateUntilAfterSmart(input) {
80
+ const { exitCode, ipBefore, ipAfter } = input;
81
+ if (exitCode === SMART_RESTART_COOLDOWN_EXIT_CODE)
82
+ return true;
83
+ if (!ipAfter)
84
+ return true;
85
+ if (ipBefore && ipAfter && ipBefore === ipAfter)
86
+ return true;
87
+ return exitCode !== 0;
88
+ }
89
+ function wifiProbeLockActive(loaded) {
90
+ const lock = meshRuntimePaths(loaded).wifiProbeLock;
91
+ try {
92
+ return fs.existsSync(lock);
93
+ }
94
+ catch {
95
+ return false;
96
+ }
97
+ }
98
+ let cachedIpifyUp = null;
99
+ let lastIpifyAt = 0;
100
+ let paneScanOffset = 0;
101
+ /** Last carrier IP actually observed while up — a live fetch during an outage always fails. */
102
+ let lastKnownGoodIp = null;
103
+ const paneLimitCache = new Map();
104
+ function anyRateLimitInCache() {
105
+ for (const flags of paneLimitCache.values()) {
106
+ if (flags.rate)
107
+ return true;
108
+ }
109
+ return false;
110
+ }
111
+ /** Drop OC-LIMIT border state for a pane once resume ack confirms it is working again. */
112
+ export function clearOcLimitBannerForPane(state, paneId) {
113
+ const had = paneLimitCache.has(paneId) ||
114
+ state.ocLimited.has(paneId) ||
115
+ state.connectPanes.has(paneId);
116
+ paneLimitCache.delete(paneId);
117
+ state.ocLimited.delete(paneId);
118
+ state.connectPanes.delete(paneId);
119
+ return had;
120
+ }
121
+ /** When no panes remain limited, end the OC-LIMIT episode so borders and proxy recovery unlock. */
122
+ export function maybeEndRateLimitEpisode(state) {
123
+ if (anyRateLimitInCache())
124
+ return;
125
+ if (!state.rateLimitEpisodeActive && !state.rateLimitRecoveryStarted)
126
+ return;
127
+ state.rateLimitEpisodeActive = false;
128
+ state.rateLimitRecoveryStarted = false;
129
+ state.lastRateLimitSeenAt = 0;
130
+ }
131
+ export function newConnectivityRecoveryState() {
132
+ return {
133
+ ocLimited: new Set(),
134
+ connectPanes: new Set(),
135
+ proxyDownActive: false,
136
+ recoveryRunning: false,
137
+ lastProxyUpAt: 0,
138
+ lastRotateAt: 0,
139
+ carrierIpAtEpisodeStart: null,
140
+ rateLimitEpisodeActive: false,
141
+ lastRateLimitSeenAt: 0,
142
+ proxyDownEpisodeStartAt: 0,
143
+ proxyDownStuckNotified: false,
144
+ rateLimitRecoveryStarted: false,
145
+ ipifyFailStreak: 0,
146
+ };
147
+ }
148
+ /**
149
+ * Pure decision: has a PROXY-DOWN episode been running long enough, with no
150
+ * "still down" toast sent yet, to warrant telling operator it's stuck (not silent)?
151
+ * Exported for unit testing without tmux/process spawning.
152
+ */
153
+ export function shouldNotifyProxyDownStuck(state, nowMs = Date.now(), thresholdMs = PROXY_DOWN_STUCK_NOTIFY_MS) {
154
+ if (state.proxyDownStuckNotified)
155
+ return false;
156
+ if (!state.proxyDownEpisodeStartAt)
157
+ return false;
158
+ return nowMs - state.proxyDownEpisodeStartAt >= thresholdMs;
159
+ }
160
+ let ipifyProbeInFlight = false;
161
+ /** Non-blocking ipify refresh — poll tick and /health must never wait on curl. */
162
+ function scheduleIpifyProbe(workspace, port) {
163
+ if (ipifyProbeInFlight)
164
+ return;
165
+ ipifyProbeInFlight = true;
166
+ const child = spawn("bash", [
167
+ "-c",
168
+ `HTTPS_PROXY=http://127.0.0.1:${port} HTTP_PROXY=http://127.0.0.1:${port} curl -4 -sS -m 3 https://api.ipify.org 2>/dev/null || true`,
169
+ ], { cwd: workspace, stdio: ["ignore", "pipe", "ignore"] });
170
+ let out = "";
171
+ child.stdout?.on("data", (c) => {
172
+ out += String(c);
173
+ });
174
+ child.on("close", () => {
175
+ ipifyProbeInFlight = false;
176
+ lastIpifyAt = Date.now();
177
+ const ip = out.trim();
178
+ cachedIpifyUp = Boolean(ip);
179
+ if (ip)
180
+ lastKnownGoodIp = ip;
181
+ });
182
+ child.on("error", () => {
183
+ ipifyProbeInFlight = false;
184
+ lastIpifyAt = Date.now();
185
+ cachedIpifyUp = false;
186
+ });
187
+ }
188
+ function carrierIpCached() {
189
+ return lastKnownGoodIp;
190
+ }
191
+ function ipifyUp(workspace, port, episode = false) {
192
+ const now = Date.now();
193
+ const minGap = episode ? IPIFY_EPISODE_MIN_MS : IPIFY_POLL_MS;
194
+ if (now - lastIpifyAt >= minGap) {
195
+ scheduleIpifyProbe(workspace, port);
196
+ }
197
+ if (cachedIpifyUp !== null)
198
+ return cachedIpifyUp;
199
+ return true;
200
+ }
201
+ function runScriptAsync(workspace, scriptPath, log, label, onDone) {
202
+ if (!fs.existsSync(scriptPath)) {
203
+ log(`${label}: missing ${scriptPath}`);
204
+ return false;
205
+ }
206
+ const child = spawn("bash", [scriptPath], {
207
+ cwd: workspace,
208
+ stdio: "ignore",
209
+ detached: false,
210
+ env: { ...process.env, CPE_SKIP_DESKTOP_NOTIFY: "1" },
211
+ });
212
+ child.on("error", (e) => {
213
+ log(`${label}: spawn error ${e.message}`);
214
+ onDone(1);
215
+ });
216
+ child.on("close", (code) => onDone(code ?? 1));
217
+ return true;
218
+ }
219
+ /** Scan all monitor panes; update limit sets; trigger async recovery on rising edges only. */
220
+ export function pollConnectivityRecovery(input) {
221
+ const { loaded, registry, workspace, session, baseWindow, workersWindow, minisWindow, state, log, resumeOpenCodePanes, } = input;
222
+ const conn = loaded.profile.connectivity;
223
+ if (!conn?.enabled)
224
+ return;
225
+ const proxyPort = conn.proxyPort ?? 18887;
226
+ const panes = listMeshMonitorPanes(session, baseWindow, workersWindow, minisWindow);
227
+ const liveIds = new Set(panes.map((p) => p.paneId));
228
+ for (const id of paneLimitCache.keys()) {
229
+ if (!liveIds.has(id))
230
+ paneLimitCache.delete(id);
231
+ }
232
+ const prevConnect = new Set(state.connectPanes);
233
+ const prevLimited = new Set(state.ocLimited);
234
+ const batch = [];
235
+ if (panes.length) {
236
+ for (let i = 0; i < Math.min(PANE_SCAN_BATCH, panes.length); i++) {
237
+ batch.push(panes[(paneScanOffset + i) % panes.length]);
238
+ }
239
+ paneScanOffset = (paneScanOffset + batch.length) % panes.length;
240
+ }
241
+ const uxResolved = loaded.profile.ux !== undefined ? resolveUxConfig(loaded.profile.ux) : null;
242
+ const { proxyDownKinds, rateLimitKinds, limitProviders } = uxResolved
243
+ ? uxLimitKinds(uxResolved)
244
+ : {
245
+ proxyDownKinds: new Set(["oc-connect"]),
246
+ rateLimitKinds: new Set(["oc-limit"]),
247
+ limitProviders: new Set(["opencode"]),
248
+ };
249
+ for (const { paneId, label } of batch) {
250
+ const snap = capturePaneSnapshot(paneId);
251
+ if (!snap) {
252
+ paneLimitCache.delete(paneId);
253
+ continue;
254
+ }
255
+ const prov = registry.detect(snap);
256
+ if (!prov || !providerMonitoredForLimits(prov.id, limitProviders)) {
257
+ paneLimitCache.delete(paneId);
258
+ continue;
259
+ }
260
+ const st = prov.composerState(snap);
261
+ if (st.phase !== "limit" || !st.limitKind) {
262
+ paneLimitCache.delete(paneId);
263
+ continue;
264
+ }
265
+ const connect = proxyDownKinds.has(st.limitKind);
266
+ const rate = rateLimitKinds.has(st.limitKind);
267
+ if (!connect && !rate) {
268
+ paneLimitCache.delete(paneId);
269
+ continue;
270
+ }
271
+ paneLimitCache.set(paneId, { connect, rate });
272
+ if (connect && !prevConnect.has(paneId)) {
273
+ log(`PROXY-DOWN rising ${label} ${paneId} kind=${st.limitKind}`);
274
+ }
275
+ else if (rate && !prevLimited.has(paneId)) {
276
+ log(`OC-LIMIT rising ${label} ${paneId} kind=${st.limitKind}`);
277
+ }
278
+ }
279
+ state.connectPanes.clear();
280
+ state.ocLimited.clear();
281
+ let anyConnect = false;
282
+ let anyRateLimit = false;
283
+ for (const [paneId, flags] of paneLimitCache) {
284
+ if (!liveIds.has(paneId))
285
+ continue;
286
+ state.ocLimited.add(paneId);
287
+ if (flags.connect) {
288
+ state.connectPanes.add(paneId);
289
+ anyConnect = true;
290
+ }
291
+ else if (flags.rate) {
292
+ anyRateLimit = true;
293
+ }
294
+ }
295
+ const hooks = resolveConnectivityHooks(loaded.profile, workspace);
296
+ if (wifiProbeLockActive(loaded)) {
297
+ if (state.proxyDownActive) {
298
+ log("PROXY-DOWN suppressed — cpe-wifi-probe.lock active");
299
+ state.proxyDownActive = false;
300
+ }
301
+ return;
302
+ }
303
+ const carrierOk = ipifyUp(workspace, proxyPort, anyConnect || state.proxyDownActive);
304
+ const ipifyThreshold = conn.policy?.ipifyFailBeforeRecovery ?? 3;
305
+ const { confirmedDown: ipifyConfirmedDown, streak: ipifyStreak } = updateIpifyProbeStreak(state, carrierOk, ipifyThreshold);
306
+ if (!carrierOk && !ipifyConfirmedDown && !anyConnect) {
307
+ log(`ipify probe fail streak ${ipifyStreak}/${ipifyThreshold} — defer PROXY-DOWN recovery`);
308
+ }
309
+ const proxyDownNow = anyConnect || ipifyConfirmedDown;
310
+ const wasDown = state.proxyDownActive;
311
+ state.proxyDownActive = proxyDownNow;
312
+ if (proxyDownNow && !wasDown) {
313
+ state.carrierIpAtEpisodeStart = carrierIpCached();
314
+ state.proxyDownEpisodeStartAt = Date.now();
315
+ state.proxyDownStuckNotified = false;
316
+ log(`PROXY-DOWN episode start ipify=${carrierOk ? "up" : "down"} streak=${ipifyStreak}/${ipifyThreshold} connectPanes=${state.connectPanes.size} carrier=${state.carrierIpAtEpisodeStart ?? "?"}`);
317
+ notifyConnectivityStatus(workspace, "PROXY-DOWN", `Carrier ${state.carrierIpAtEpisodeStart ?? "?"} unreachable. Running cpe-proxy-up.sh now.`, "Wait for resume-sent then complete toasts — no action during starting.", "starting");
318
+ maybeRunProxyUp(hooks, workspace, state, log);
319
+ }
320
+ else if (!proxyDownNow && wasDown) {
321
+ const fromIp = state.carrierIpAtEpisodeStart;
322
+ const toIp = carrierIpCached();
323
+ log(`PROXY-DOWN episode clear -> resume wave carrier=${toIp ?? "?"}`);
324
+ state.carrierIpAtEpisodeStart = null;
325
+ state.proxyDownEpisodeStartAt = 0;
326
+ state.proxyDownStuckNotified = false;
327
+ state.ipifyFailStreak = 0;
328
+ // Keep rate-limit cache entries — clearing all panes made OC-LIMIT episodes re-arm proxy restart.
329
+ for (const [paneId, flags] of [...paneLimitCache.entries()]) {
330
+ if (flags.rate) {
331
+ paneLimitCache.set(paneId, { connect: false, rate: true });
332
+ }
333
+ else {
334
+ paneLimitCache.delete(paneId);
335
+ }
336
+ }
337
+ setImmediate(() => resumeOpenCodePanes("proxy-up", { fromIp, toIp }));
338
+ }
339
+ else if (proxyDownNow && wasDown && shouldNotifyProxyDownStuck(state)) {
340
+ state.proxyDownStuckNotified = true;
341
+ const ageSec = Math.round((Date.now() - state.proxyDownEpisodeStartAt) / 1000);
342
+ log(`PROXY-DOWN still active after ${ageSec}s -> notify operator`);
343
+ notifyConnectivityStatus(workspace, "PROXY-DOWN", `Carrier still unreachable after ${ageSec}s. cpe-proxy-up.sh ran; on cooldown.`, "Check WiFi/CPE manually if this persists.", "incomplete");
344
+ }
345
+ const { action } = updateRateLimitEpisode(state, anyRateLimit);
346
+ if (action === "start") {
347
+ const fromIp = carrierIpCached() ?? state.carrierIpAtEpisodeStart ?? lastKnownGoodIp;
348
+ log(`OC-LIMIT rising-edge episode start carrier=${fromIp ?? "?"}`);
349
+ if (!state.rateLimitRecoveryStarted) {
350
+ notifyConnectivityStatus(workspace, "OC-LIMIT", `Carrier ${fromIp ?? "?"} rate-limited. One proxy recovery run for this episode (no repeat while banner persists).`, "Wait for resume-sent then complete toasts. Ack is automatic when limit screens clear.", "starting");
351
+ maybeRunOcLimitRecovery(hooks, workspace, proxyPort, fromIp, state, log, resumeOpenCodePanes);
352
+ }
353
+ else {
354
+ log("OC-LIMIT episode active — recovery already ran this episode (skip proxy restart)");
355
+ }
356
+ }
357
+ else if (action === "clear") {
358
+ log("OC-LIMIT episode clear");
359
+ state.rateLimitRecoveryStarted = false;
360
+ }
361
+ }
362
+ function maybeRunProxyUp(hooks, workspace, state, log) {
363
+ const now = Date.now();
364
+ if (state.recoveryRunning)
365
+ return;
366
+ if (now - state.lastProxyUpAt < PROXY_UP_COOLDOWN_MS)
367
+ return;
368
+ const script = hooks?.up;
369
+ if (!script) {
370
+ log("PROXY-DOWN: connectivity.hooks.up not configured — skip");
371
+ return;
372
+ }
373
+ state.recoveryRunning = true;
374
+ state.lastProxyUpAt = now;
375
+ log(`PROXY-DOWN -> async ${path.basename(script)}`);
376
+ const started = runScriptAsync(workspace, script, log, "proxy-up", (code) => {
377
+ state.recoveryRunning = false;
378
+ log(`cpe-proxy-up exit=${code} — resume wave waits for PROXY-DOWN episode clear`);
379
+ });
380
+ if (!started)
381
+ state.recoveryRunning = false;
382
+ }
383
+ /**
384
+ * OC-LIMIT (rate-limit) recovery: Proxy-SMART restart first when off cooldown,
385
+ * else cpe-proxy-rotate-until.sh directly. Chains rotate-until after a SMART
386
+ * run that was skipped/failed/no-op, then notifies operator + resumes OC panes on
387
+ * any real carrier IP change. Mirrors inbox-server.mjs runOcLimitRecoveryAsync.
388
+ */
389
+ function maybeRunOcLimitRecovery(hooks, workspace, proxyPort, fromIp, state, log, resumeOpenCodePanes) {
390
+ const now = Date.now();
391
+ if (state.recoveryRunning)
392
+ return;
393
+ if (state.rateLimitRecoveryStarted)
394
+ return;
395
+ if (now - state.lastRotateAt < ROTATE_COOLDOWN_MS)
396
+ return;
397
+ state.recoveryRunning = true;
398
+ state.rateLimitRecoveryStarted = true;
399
+ state.lastRotateAt = now;
400
+ const finish = (toIp, reason) => {
401
+ state.recoveryRunning = false;
402
+ const ipChanged = Boolean(toIp && fromIp && toIp !== fromIp);
403
+ if (!ipChanged) {
404
+ log(`OC-LIMIT recovery done reason=${reason} ip=${toIp ?? "?"} — skip resume wave (no carrier change)`);
405
+ return;
406
+ }
407
+ log(`OC-LIMIT recovery done -> resume wave reason=${reason} ${fromIp} -> ${toIp}`);
408
+ setImmediate(() => resumeOpenCodePanes(reason, { fromIp, toIp }));
409
+ };
410
+ const runRotateUntil = (reason) => {
411
+ const script = hooks?.rotate;
412
+ if (!script || !fs.existsSync(script)) {
413
+ log(`OC-LIMIT recovery: connectivity.hooks.rotate missing (${script ?? "unset"})`);
414
+ state.recoveryRunning = false;
415
+ return;
416
+ }
417
+ log(`OC-LIMIT -> async ${path.basename(script)} (${reason})`);
418
+ const started = runScriptAsync(workspace, script, log, "rotate-until", (code) => {
419
+ log(`rotate-until exit=${code}`);
420
+ scheduleIpifyProbe(workspace, proxyPort);
421
+ finish(carrierIpCached(), "rotate-until");
422
+ });
423
+ if (!started)
424
+ state.recoveryRunning = false;
425
+ };
426
+ const cooldownLeft = smartRestartCooldownLeftSec();
427
+ if (cooldownLeft > 0) {
428
+ log(`OC-LIMIT Proxy-SMART cooldown ${cooldownLeft}s -> rotate-until path`);
429
+ runRotateUntil("smart-cooldown");
430
+ return;
431
+ }
432
+ const smartScript = hooks?.smartRestart;
433
+ if (!smartScript || !fs.existsSync(smartScript)) {
434
+ log(`OC-LIMIT rising-edge but hooks.smartRestart missing -> rotate-until`);
435
+ runRotateUntil("no-smart-script");
436
+ return;
437
+ }
438
+ log(`OC-LIMIT rising-edge -> async ${path.basename(smartScript)}`);
439
+ const started = runScriptAsync(workspace, smartScript, log, "cpe-proxy-smart-restart", (code) => {
440
+ log(`cpe-proxy-smart-restart exit=${code}`);
441
+ scheduleIpifyProbe(workspace, proxyPort);
442
+ const ipAfter = carrierIpCached();
443
+ if (shouldChainRotateUntilAfterSmart({ exitCode: code, ipBefore: fromIp, ipAfter })) {
444
+ const chainReason = code === SMART_RESTART_COOLDOWN_EXIT_CODE ? "smart-cooldown" : "smart-unchanged";
445
+ notifyConnectivityStatus(workspace, "OC-LIMIT", `Proxy-SMART did not clear it (${chainReason}). Escalating to cpe-proxy-rotate-until.sh.`, "Wait for resume-sent then complete toasts.", "escalating");
446
+ runRotateUntil(chainReason);
447
+ return;
448
+ }
449
+ finish(ipAfter, "proxy-smart-restart");
450
+ });
451
+ if (!started)
452
+ state.recoveryRunning = false;
453
+ }
454
+ //# sourceMappingURL=connectivity-recovery.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"connectivity-recovery.js","sourceRoot":"","sources":["../src/connectivity-recovery.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,MAAM,oBAAoB,CAAC;AAC3C,OAAO,EAAE,MAAM,SAAS,CAAC;AACzB,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EACL,gBAAgB,EAChB,0BAA0B,EAC1B,wBAAwB,EACxB,eAAe,EACf,YAAY,GAIb,MAAM,iBAAiB,CAAC;AACzB,OAAO,EAAE,mBAAmB,EAAE,oBAAoB,EAAE,MAAM,iBAAiB,CAAC;AAC5E,OAAO,EAAE,wBAAwB,EAAuB,MAAM,gBAAgB,CAAC;AAE/E,MAAM,oBAAoB,GAAG,MAAM,CAAC;AACpC,MAAM,kBAAkB,GAAG,OAAO,CAAC;AACnC,MAAM,aAAa,GAAG,MAAM,CAAC;AAC7B,yFAAyF;AACzF,MAAM,oBAAoB,GAAG,MAAM,CAAC;AACpC,MAAM,eAAe,GAAG,CAAC,CAAC;AAC1B,+FAA+F;AAC/F,MAAM,2BAA2B,GAAG,OAAO,CAAC;AAC5C,uFAAuF;AACvF,MAAM,0BAA0B,GAAG,OAAO,CAAC;AAC3C,+FAA+F;AAC/F,MAAM,2BAA2B,GAAG,mCAAmC,CAAC;AACxE,MAAM,kCAAkC,GAAG,IAAI,CAAC;AAChD,yFAAyF;AACzF,MAAM,gCAAgC,GAAG,CAAC,CAAC;AAE3C;;;;GAIG;AACH,MAAM,UAAU,2BAA2B,CACzC,YAAoB,OAAO,CAAC,GAAG,CAAC,uBAAuB,IAAI,2BAA2B,EACtF,cAAsB,MAAM,CAC1B,OAAO,CAAC,GAAG,CAAC,8BAA8B,IAAI,kCAAkC,CACjF,EACD,QAAgB,IAAI,CAAC,GAAG,EAAE;IAE1B,IAAI,CAAC;QACH,MAAM,IAAI,GAAG,MAAM,CAAC,EAAE,CAAC,YAAY,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC,IAAI,EAAE,CAAC,IAAI,CAAC,CAAC;QACpE,IAAI,CAAC,IAAI;YAAE,OAAO,CAAC,CAAC;QACpB,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,KAAK,GAAG,IAAI,CAAC,GAAG,IAAI,CAAC;QAC/C,OAAO,MAAM,IAAI,WAAW,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,WAAW,GAAG,MAAM,CAAC;IAC1D,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,CAAC,CAAC;IACX,CAAC;AACH,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,sBAAsB,CACpC,KAAyD,EACzD,SAAkB,EAClB,SAAiB;IAEjB,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,EAAE,SAAS,CAAC,CAAC;IACpC,IAAI,SAAS,EAAE,CAAC;QACd,KAAK,CAAC,eAAe,GAAG,CAAC,CAAC;QAC1B,OAAO,EAAE,aAAa,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC,EAAE,CAAC;IAC7C,CAAC;IACD,KAAK,CAAC,eAAe,IAAI,CAAC,CAAC;IAC3B,OAAO;QACL,aAAa,EAAE,KAAK,CAAC,eAAe,IAAI,IAAI;QAC5C,MAAM,EAAE,KAAK,CAAC,eAAe;KAC9B,CAAC;AACJ,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,sBAAsB,CACpC,KAAwF,EACxF,eAAwB,EACxB,QAAgB,IAAI,CAAC,GAAG,EAAE,EAC1B,UAAkB,2BAA2B;IAE7C,IAAI,eAAe,EAAE,CAAC;QACpB,KAAK,CAAC,mBAAmB,GAAG,KAAK,CAAC;QAClC,IAAI,CAAC,KAAK,CAAC,sBAAsB,EAAE,CAAC;YAClC,KAAK,CAAC,sBAAsB,GAAG,IAAI,CAAC;YACpC,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC;QAC7B,CAAC;QACD,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,CAAC;IAChC,CAAC;IACD,IAAI,KAAK,CAAC,sBAAsB,IAAI,KAAK,GAAG,KAAK,CAAC,mBAAmB,GAAG,OAAO,EAAE,CAAC;QAChF,KAAK,CAAC,sBAAsB,GAAG,KAAK,CAAC;QACrC,OAAO,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC;IAC7B,CAAC;IACD,OAAO,EAAE,MAAM,EAAE,MAAM,EAAE,CAAC;AAC5B,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,gCAAgC,CAAC,KAIhD;IACC,MAAM,EAAE,QAAQ,EAAE,QAAQ,EAAE,OAAO,EAAE,GAAG,KAAK,CAAC;IAC9C,IAAI,QAAQ,KAAK,gCAAgC;QAAE,OAAO,IAAI,CAAC;IAC/D,IAAI,CAAC,OAAO;QAAE,OAAO,IAAI,CAAC;IAC1B,IAAI,QAAQ,IAAI,OAAO,IAAI,QAAQ,KAAK,OAAO;QAAE,OAAO,IAAI,CAAC;IAC7D,OAAO,QAAQ,KAAK,CAAC,CAAC;AACxB,CAAC;AAED,SAAS,mBAAmB,CAAC,MAAqB;IAChD,MAAM,IAAI,GAAG,gBAAgB,CAAC,MAAM,CAAC,CAAC,aAAa,CAAC;IACpD,IAAI,CAAC;QACH,OAAO,EAAE,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC;IAC7B,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,KAAK,CAAC;IACf,CAAC;AACH,CAAC;AAED,IAAI,aAAa,GAAmB,IAAI,CAAC;AACzC,IAAI,WAAW,GAAG,CAAC,CAAC;AACpB,IAAI,cAAc,GAAG,CAAC,CAAC;AACvB,+FAA+F;AAC/F,IAAI,eAAe,GAAkB,IAAI,CAAC;AAC1C,MAAM,cAAc,GAAG,IAAI,GAAG,EAA+C,CAAC;AAE9E,SAAS,mBAAmB;IAC1B,KAAK,MAAM,KAAK,IAAI,cAAc,CAAC,MAAM,EAAE,EAAE,CAAC;QAC5C,IAAI,KAAK,CAAC,IAAI;YAAE,OAAO,IAAI,CAAC;IAC9B,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC;AAED,0FAA0F;AAC1F,MAAM,UAAU,yBAAyB,CACvC,KAAgC,EAChC,MAAc;IAEd,MAAM,GAAG,GACP,cAAc,CAAC,GAAG,CAAC,MAAM,CAAC;QAC1B,KAAK,CAAC,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC;QAC3B,KAAK,CAAC,YAAY,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;IACjC,cAAc,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;IAC9B,KAAK,CAAC,SAAS,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;IAC/B,KAAK,CAAC,YAAY,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;IAClC,OAAO,GAAG,CAAC;AACb,CAAC;AAED,mGAAmG;AACnG,MAAM,UAAU,wBAAwB,CAAC,KAAgC;IACvE,IAAI,mBAAmB,EAAE;QAAE,OAAO;IAClC,IAAI,CAAC,KAAK,CAAC,sBAAsB,IAAI,CAAC,KAAK,CAAC,wBAAwB;QAAE,OAAO;IAC7E,KAAK,CAAC,sBAAsB,GAAG,KAAK,CAAC;IACrC,KAAK,CAAC,wBAAwB,GAAG,KAAK,CAAC;IACvC,KAAK,CAAC,mBAAmB,GAAG,CAAC,CAAC;AAChC,CAAC;AAyBD,MAAM,UAAU,4BAA4B;IAC1C,OAAO;QACL,SAAS,EAAE,IAAI,GAAG,EAAE;QACpB,YAAY,EAAE,IAAI,GAAG,EAAE;QACvB,eAAe,EAAE,KAAK;QACtB,eAAe,EAAE,KAAK;QACtB,aAAa,EAAE,CAAC;QAChB,YAAY,EAAE,CAAC;QACf,uBAAuB,EAAE,IAAI;QAC7B,sBAAsB,EAAE,KAAK;QAC7B,mBAAmB,EAAE,CAAC;QACtB,uBAAuB,EAAE,CAAC;QAC1B,sBAAsB,EAAE,KAAK;QAC7B,wBAAwB,EAAE,KAAK;QAC/B,eAAe,EAAE,CAAC;KACnB,CAAC;AACJ,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,0BAA0B,CACxC,KAA4F,EAC5F,QAAgB,IAAI,CAAC,GAAG,EAAE,EAC1B,cAAsB,0BAA0B;IAEhD,IAAI,KAAK,CAAC,sBAAsB;QAAE,OAAO,KAAK,CAAC;IAC/C,IAAI,CAAC,KAAK,CAAC,uBAAuB;QAAE,OAAO,KAAK,CAAC;IACjD,OAAO,KAAK,GAAG,KAAK,CAAC,uBAAuB,IAAI,WAAW,CAAC;AAC9D,CAAC;AAED,IAAI,kBAAkB,GAAG,KAAK,CAAC;AAE/B,kFAAkF;AAClF,SAAS,kBAAkB,CAAC,SAAiB,EAAE,IAAY;IACzD,IAAI,kBAAkB;QAAE,OAAO;IAC/B,kBAAkB,GAAG,IAAI,CAAC;IAC1B,MAAM,KAAK,GAAG,KAAK,CACjB,MAAM,EACN;QACE,IAAI;QACJ,gCAAgC,IAAI,gCAAgC,IAAI,6DAA6D;KACtI,EACD,EAAE,GAAG,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC,QAAQ,EAAE,MAAM,EAAE,QAAQ,CAAC,EAAE,CACxD,CAAC;IACF,IAAI,GAAG,GAAG,EAAE,CAAC;IACb,KAAK,CAAC,MAAM,EAAE,EAAE,CAAC,MAAM,EAAE,CAAC,CAAC,EAAE,EAAE;QAC7B,GAAG,IAAI,MAAM,CAAC,CAAC,CAAC,CAAC;IACnB,CAAC,CAAC,CAAC;IACH,KAAK,CAAC,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE;QACrB,kBAAkB,GAAG,KAAK,CAAC;QAC3B,WAAW,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACzB,MAAM,EAAE,GAAG,GAAG,CAAC,IAAI,EAAE,CAAC;QACtB,aAAa,GAAG,OAAO,CAAC,EAAE,CAAC,CAAC;QAC5B,IAAI,EAAE;YAAE,eAAe,GAAG,EAAE,CAAC;IAC/B,CAAC,CAAC,CAAC;IACH,KAAK,CAAC,EAAE,CAAC,OAAO,EAAE,GAAG,EAAE;QACrB,kBAAkB,GAAG,KAAK,CAAC;QAC3B,WAAW,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QACzB,aAAa,GAAG,KAAK,CAAC;IACxB,CAAC,CAAC,CAAC;AACL,CAAC;AAED,SAAS,eAAe;IACtB,OAAO,eAAe,CAAC;AACzB,CAAC;AAED,SAAS,OAAO,CAAC,SAAiB,EAAE,IAAY,EAAE,OAAO,GAAG,KAAK;IAC/D,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IACvB,MAAM,MAAM,GAAG,OAAO,CAAC,CAAC,CAAC,oBAAoB,CAAC,CAAC,CAAC,aAAa,CAAC;IAC9D,IAAI,GAAG,GAAG,WAAW,IAAI,MAAM,EAAE,CAAC;QAChC,kBAAkB,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC;IACtC,CAAC;IACD,IAAI,aAAa,KAAK,IAAI;QAAE,OAAO,aAAa,CAAC;IACjD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,SAAS,cAAc,CACrB,SAAiB,EACjB,UAAkB,EAClB,GAA2B,EAC3B,KAAa,EACb,MAA8B;IAE9B,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,UAAU,CAAC,EAAE,CAAC;QAC/B,GAAG,CAAC,GAAG,KAAK,aAAa,UAAU,EAAE,CAAC,CAAC;QACvC,OAAO,KAAK,CAAC;IACf,CAAC;IACD,MAAM,KAAK,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,UAAU,CAAC,EAAE;QACxC,GAAG,EAAE,SAAS;QACd,KAAK,EAAE,QAAQ;QACf,QAAQ,EAAE,KAAK;QACf,GAAG,EAAE,EAAE,GAAG,OAAO,CAAC,GAAG,EAAE,uBAAuB,EAAE,GAAG,EAAE;KACtD,CAAC,CAAC;IACH,KAAK,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,CAAC,EAAE,EAAE;QACtB,GAAG,CAAC,GAAG,KAAK,iBAAkB,CAAW,CAAC,OAAO,EAAE,CAAC,CAAC;QACrD,MAAM,CAAC,CAAC,CAAC,CAAC;IACZ,CAAC,CAAC,CAAC;IACH,KAAK,CAAC,EAAE,CAAC,OAAO,EAAE,CAAC,IAAI,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,IAAI,CAAC,CAAC,CAAC,CAAC;IAC/C,OAAO,IAAI,CAAC;AACd,CAAC;AAeD,8FAA8F;AAC9F,MAAM,UAAU,wBAAwB,CAAC,KAA4B;IACnE,MAAM,EACJ,MAAM,EACN,QAAQ,EACR,SAAS,EACT,OAAO,EACP,UAAU,EACV,aAAa,EACb,WAAW,EACX,KAAK,EACL,GAAG,EACH,mBAAmB,GACpB,GAAG,KAAK,CAAC;IAEV,MAAM,IAAI,GAAG,MAAM,CAAC,OAAO,CAAC,YAAY,CAAC;IACzC,IAAI,CAAC,IAAI,EAAE,OAAO;QAAE,OAAO;IAE3B,MAAM,SAAS,GAAG,IAAI,CAAC,SAAS,IAAI,KAAK,CAAC;IAC1C,MAAM,KAAK,GAAG,oBAAoB,CAAC,OAAO,EAAE,UAAU,EAAE,aAAa,EAAE,WAAW,CAAC,CAAC;IACpF,MAAM,OAAO,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC;IACpD,KAAK,MAAM,EAAE,IAAI,cAAc,CAAC,IAAI,EAAE,EAAE,CAAC;QACvC,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,EAAE,CAAC;YAAE,cAAc,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;IAClD,CAAC;IAED,MAAM,WAAW,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,YAAY,CAAC,CAAC;IAChD,MAAM,WAAW,GAAG,IAAI,GAAG,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC;IAE7C,MAAM,KAAK,GAAiB,EAAE,CAAC;IAC/B,IAAI,KAAK,CAAC,MAAM,EAAE,CAAC;QACjB,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,GAAG,CAAC,eAAe,EAAE,KAAK,CAAC,MAAM,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC;YACjE,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC,cAAc,GAAG,CAAC,CAAC,GAAG,KAAK,CAAC,MAAM,CAAE,CAAC,CAAC;QAC1D,CAAC;QACD,cAAc,GAAG,CAAC,cAAc,GAAG,KAAK,CAAC,MAAM,CAAC,GAAG,KAAK,CAAC,MAAM,CAAC;IAClE,CAAC;IAED,MAAM,UAAU,GACd,MAAM,CAAC,OAAO,CAAC,EAAE,KAAK,SAAS,CAAC,CAAC,CAAC,eAAe,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;IAC9E,MAAM,EAAE,cAAc,EAAE,cAAc,EAAE,cAAc,EAAE,GAAG,UAAU;QACnE,CAAC,CAAC,YAAY,CAAC,UAAU,CAAC;QAC1B,CAAC,CAAC;YACE,cAAc,EAAE,IAAI,GAAG,CAAC,CAAC,YAAY,CAAC,CAAC;YACvC,cAAc,EAAE,IAAI,GAAG,CAAC,CAAC,UAAU,CAAC,CAAC;YACrC,cAAc,EAAE,IAAI,GAAG,CAAC,CAAC,UAAU,CAAC,CAAC;SACtC,CAAC;IAEN,KAAK,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,IAAI,KAAK,EAAE,CAAC;QACtC,MAAM,IAAI,GAAG,mBAAmB,CAAC,MAAM,CAAC,CAAC;QACzC,IAAI,CAAC,IAAI,EAAE,CAAC;YACV,cAAc,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;YAC9B,SAAS;QACX,CAAC;QACD,MAAM,IAAI,GAAG,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;QACnC,IAAI,CAAC,IAAI,IAAI,CAAC,0BAA0B,CAAC,IAAI,CAAC,EAAE,EAAE,cAAc,CAAC,EAAE,CAAC;YAClE,cAAc,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;YAC9B,SAAS;QACX,CAAC;QACD,MAAM,EAAE,GAAG,IAAI,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC;QACpC,IAAI,EAAE,CAAC,KAAK,KAAK,OAAO,IAAI,CAAC,EAAE,CAAC,SAAS,EAAE,CAAC;YAC1C,cAAc,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;YAC9B,SAAS;QACX,CAAC;QACD,MAAM,OAAO,GAAG,cAAc,CAAC,GAAG,CAAC,EAAE,CAAC,SAAS,CAAC,CAAC;QACjD,MAAM,IAAI,GAAG,cAAc,CAAC,GAAG,CAAC,EAAE,CAAC,SAAS,CAAC,CAAC;QAC9C,IAAI,CAAC,OAAO,IAAI,CAAC,IAAI,EAAE,CAAC;YACtB,cAAc,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;YAC9B,SAAS;QACX,CAAC;QACD,cAAc,CAAC,GAAG,CAAC,MAAM,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CAAC;QAC9C,IAAI,OAAO,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC;YACxC,GAAG,CAAC,qBAAqB,KAAK,IAAI,MAAM,SAAS,EAAE,CAAC,SAAS,EAAE,CAAC,CAAC;QACnE,CAAC;aAAM,IAAI,IAAI,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,MAAM,CAAC,EAAE,CAAC;YAC5C,GAAG,CAAC,mBAAmB,KAAK,IAAI,MAAM,SAAS,EAAE,CAAC,SAAS,EAAE,CAAC,CAAC;QACjE,CAAC;IACH,CAAC;IAED,KAAK,CAAC,YAAY,CAAC,KAAK,EAAE,CAAC;IAC3B,KAAK,CAAC,SAAS,CAAC,KAAK,EAAE,CAAC;IACxB,IAAI,UAAU,GAAG,KAAK,CAAC;IACvB,IAAI,YAAY,GAAG,KAAK,CAAC;IACzB,KAAK,MAAM,CAAC,MAAM,EAAE,KAAK,CAAC,IAAI,cAAc,EAAE,CAAC;QAC7C,IAAI,CAAC,OAAO,CAAC,GAAG,CAAC,MAAM,CAAC;YAAE,SAAS;QACnC,KAAK,CAAC,SAAS,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;QAC5B,IAAI,KAAK,CAAC,OAAO,EAAE,CAAC;YAClB,KAAK,CAAC,YAAY,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC;YAC/B,UAAU,GAAG,IAAI,CAAC;QACpB,CAAC;aAAM,IAAI,KAAK,CAAC,IAAI,EAAE,CAAC;YACtB,YAAY,GAAG,IAAI,CAAC;QACtB,CAAC;IACH,CAAC;IAED,MAAM,KAAK,GAAG,wBAAwB,CAAC,MAAM,CAAC,OAAO,EAAE,SAAS,CAAC,CAAC;IAElE,IAAI,mBAAmB,CAAC,MAAM,CAAC,EAAE,CAAC;QAChC,IAAI,KAAK,CAAC,eAAe,EAAE,CAAC;YAC1B,GAAG,CAAC,oDAAoD,CAAC,CAAC;YAC1D,KAAK,CAAC,eAAe,GAAG,KAAK,CAAC;QAChC,CAAC;QACD,OAAO;IACT,CAAC;IAED,MAAM,SAAS,GAAG,OAAO,CAAC,SAAS,EAAE,SAAS,EAAE,UAAU,IAAI,KAAK,CAAC,eAAe,CAAC,CAAC;IACrF,MAAM,cAAc,GAAG,IAAI,CAAC,MAAM,EAAE,uBAAuB,IAAI,CAAC,CAAC;IACjE,MAAM,EAAE,aAAa,EAAE,kBAAkB,EAAE,MAAM,EAAE,WAAW,EAAE,GAAG,sBAAsB,CACvF,KAAK,EACL,SAAS,EACT,cAAc,CACf,CAAC;IACF,IAAI,CAAC,SAAS,IAAI,CAAC,kBAAkB,IAAI,CAAC,UAAU,EAAE,CAAC;QACrD,GAAG,CACD,2BAA2B,WAAW,IAAI,cAAc,8BAA8B,CACvF,CAAC;IACJ,CAAC;IACD,MAAM,YAAY,GAAG,UAAU,IAAI,kBAAkB,CAAC;IACtD,MAAM,OAAO,GAAG,KAAK,CAAC,eAAe,CAAC;IACtC,KAAK,CAAC,eAAe,GAAG,YAAY,CAAC;IAErC,IAAI,YAAY,IAAI,CAAC,OAAO,EAAE,CAAC;QAC7B,KAAK,CAAC,uBAAuB,GAAG,eAAe,EAAE,CAAC;QAClD,KAAK,CAAC,uBAAuB,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;QAC3C,KAAK,CAAC,sBAAsB,GAAG,KAAK,CAAC;QACrC,GAAG,CACD,kCAAkC,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,MAAM,WAAW,WAAW,IAAI,cAAc,iBAAiB,KAAK,CAAC,YAAY,CAAC,IAAI,YAAY,KAAK,CAAC,uBAAuB,IAAI,GAAG,EAAE,CAC9L,CAAC;QACF,wBAAwB,CACtB,SAAS,EACT,YAAY,EACZ,WAAW,KAAK,CAAC,uBAAuB,IAAI,GAAG,4CAA4C,EAC3F,wEAAwE,EACxE,UAAU,CACX,CAAC;QACF,eAAe,CAAC,KAAK,EAAE,SAAS,EAAE,KAAK,EAAE,GAAG,CAAC,CAAC;IAChD,CAAC;SAAM,IAAI,CAAC,YAAY,IAAI,OAAO,EAAE,CAAC;QACpC,MAAM,MAAM,GAAG,KAAK,CAAC,uBAAuB,CAAC;QAC7C,MAAM,IAAI,GAAG,eAAe,EAAE,CAAC;QAC/B,GAAG,CAAC,mDAAmD,IAAI,IAAI,GAAG,EAAE,CAAC,CAAC;QACtE,KAAK,CAAC,uBAAuB,GAAG,IAAI,CAAC;QACrC,KAAK,CAAC,uBAAuB,GAAG,CAAC,CAAC;QAClC,KAAK,CAAC,sBAAsB,GAAG,KAAK,CAAC;QACrC,KAAK,CAAC,eAAe,GAAG,CAAC,CAAC;QAC1B,kGAAkG;QAClG,KAAK,MAAM,CAAC,MAAM,EAAE,KAAK,CAAC,IAAI,CAAC,GAAG,cAAc,CAAC,OAAO,EAAE,CAAC,EAAE,CAAC;YAC5D,IAAI,KAAK,CAAC,IAAI,EAAE,CAAC;gBACf,cAAc,CAAC,GAAG,CAAC,MAAM,EAAE,EAAE,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;YAC7D,CAAC;iBAAM,CAAC;gBACN,cAAc,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC;YAChC,CAAC;QACH,CAAC;QACD,YAAY,CAAC,GAAG,EAAE,CAAC,mBAAmB,CAAC,UAAU,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;IACxE,CAAC;SAAM,IAAI,YAAY,IAAI,OAAO,IAAI,0BAA0B,CAAC,KAAK,CAAC,EAAE,CAAC;QACxE,KAAK,CAAC,sBAAsB,GAAG,IAAI,CAAC;QACpC,MAAM,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,KAAK,CAAC,uBAAuB,CAAC,GAAG,IAAI,CAAC,CAAC;QAC/E,GAAG,CAAC,iCAAiC,MAAM,sBAAsB,CAAC,CAAC;QACnE,wBAAwB,CACtB,SAAS,EACT,YAAY,EACZ,mCAAmC,MAAM,sCAAsC,EAC/E,2CAA2C,EAC3C,YAAY,CACb,CAAC;IACJ,CAAC;IAED,MAAM,EAAE,MAAM,EAAE,GAAG,sBAAsB,CAAC,KAAK,EAAE,YAAY,CAAC,CAAC;IAC/D,IAAI,MAAM,KAAK,OAAO,EAAE,CAAC;QACvB,MAAM,MAAM,GAAG,eAAe,EAAE,IAAI,KAAK,CAAC,uBAAuB,IAAI,eAAe,CAAC;QACrF,GAAG,CAAC,8CAA8C,MAAM,IAAI,GAAG,EAAE,CAAC,CAAC;QACnE,IAAI,CAAC,KAAK,CAAC,wBAAwB,EAAE,CAAC;YACpC,wBAAwB,CACtB,SAAS,EACT,UAAU,EACV,WAAW,MAAM,IAAI,GAAG,2FAA2F,EACnH,uFAAuF,EACvF,UAAU,CACX,CAAC;YACF,uBAAuB,CAAC,KAAK,EAAE,SAAS,EAAE,SAAS,EAAE,MAAM,EAAE,KAAK,EAAE,GAAG,EAAE,mBAAmB,CAAC,CAAC;QAChG,CAAC;aAAM,CAAC;YACN,GAAG,CAAC,kFAAkF,CAAC,CAAC;QAC1F,CAAC;IACH,CAAC;SAAM,IAAI,MAAM,KAAK,OAAO,EAAE,CAAC;QAC9B,GAAG,CAAC,wBAAwB,CAAC,CAAC;QAC9B,KAAK,CAAC,wBAAwB,GAAG,KAAK,CAAC;IACzC,CAAC;AACH,CAAC;AAED,SAAS,eAAe,CACtB,KAAkD,EAClD,SAAiB,EACjB,KAAgC,EAChC,GAA2B;IAE3B,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IACvB,IAAI,KAAK,CAAC,eAAe;QAAE,OAAO;IAClC,IAAI,GAAG,GAAG,KAAK,CAAC,aAAa,GAAG,oBAAoB;QAAE,OAAO;IAC7D,MAAM,MAAM,GAAG,KAAK,EAAE,EAAE,CAAC;IACzB,IAAI,CAAC,MAAM,EAAE,CAAC;QACZ,GAAG,CAAC,yDAAyD,CAAC,CAAC;QAC/D,OAAO;IACT,CAAC;IAED,KAAK,CAAC,eAAe,GAAG,IAAI,CAAC;IAC7B,KAAK,CAAC,aAAa,GAAG,GAAG,CAAC;IAC1B,GAAG,CAAC,uBAAuB,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;IACpD,MAAM,OAAO,GAAG,cAAc,CAAC,SAAS,EAAE,MAAM,EAAE,GAAG,EAAE,UAAU,EAAE,CAAC,IAAI,EAAE,EAAE;QAC1E,KAAK,CAAC,eAAe,GAAG,KAAK,CAAC;QAC9B,GAAG,CAAC,qBAAqB,IAAI,mDAAmD,CAAC,CAAC;IACpF,CAAC,CAAC,CAAC;IACH,IAAI,CAAC,OAAO;QAAE,KAAK,CAAC,eAAe,GAAG,KAAK,CAAC;AAC9C,CAAC;AAED;;;;;GAKG;AACH,SAAS,uBAAuB,CAC9B,KAAkD,EAClD,SAAiB,EACjB,SAAiB,EACjB,MAAqB,EACrB,KAAgC,EAChC,GAA2B,EAC3B,mBAAiE;IAEjE,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IACvB,IAAI,KAAK,CAAC,eAAe;QAAE,OAAO;IAClC,IAAI,KAAK,CAAC,wBAAwB;QAAE,OAAO;IAC3C,IAAI,GAAG,GAAG,KAAK,CAAC,YAAY,GAAG,kBAAkB;QAAE,OAAO;IAE1D,KAAK,CAAC,eAAe,GAAG,IAAI,CAAC;IAC7B,KAAK,CAAC,wBAAwB,GAAG,IAAI,CAAC;IACtC,KAAK,CAAC,YAAY,GAAG,GAAG,CAAC;IAEzB,MAAM,MAAM,GAAG,CAAC,IAAmB,EAAE,MAAc,EAAE,EAAE;QACrD,KAAK,CAAC,eAAe,GAAG,KAAK,CAAC;QAC9B,MAAM,SAAS,GAAG,OAAO,CAAC,IAAI,IAAI,MAAM,IAAI,IAAI,KAAK,MAAM,CAAC,CAAC;QAC7D,IAAI,CAAC,SAAS,EAAE,CAAC;YACf,GAAG,CACD,iCAAiC,MAAM,OAAO,IAAI,IAAI,GAAG,yCAAyC,CACnG,CAAC;YACF,OAAO;QACT,CAAC;QACD,GAAG,CAAC,gDAAgD,MAAM,IAAI,MAAM,OAAO,IAAI,EAAE,CAAC,CAAC;QACnF,YAAY,CAAC,GAAG,EAAE,CAAC,mBAAmB,CAAC,MAAM,EAAE,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;IACpE,CAAC,CAAC;IAEF,MAAM,cAAc,GAAG,CAAC,MAAc,EAAE,EAAE;QACxC,MAAM,MAAM,GAAG,KAAK,EAAE,MAAM,CAAC;QAC7B,IAAI,CAAC,MAAM,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,MAAM,CAAC,EAAE,CAAC;YACtC,GAAG,CAAC,yDAAyD,MAAM,IAAI,OAAO,GAAG,CAAC,CAAC;YACnF,KAAK,CAAC,eAAe,GAAG,KAAK,CAAC;YAC9B,OAAO;QACT,CAAC;QACD,GAAG,CAAC,qBAAqB,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAK,MAAM,GAAG,CAAC,CAAC;QAC9D,MAAM,OAAO,GAAG,cAAc,CAAC,SAAS,EAAE,MAAM,EAAE,GAAG,EAAE,cAAc,EAAE,CAAC,IAAI,EAAE,EAAE;YAC9E,GAAG,CAAC,qBAAqB,IAAI,EAAE,CAAC,CAAC;YACjC,kBAAkB,CAAC,SAAS,EAAE,SAAS,CAAC,CAAC;YACzC,MAAM,CAAC,eAAe,EAAE,EAAE,cAAc,CAAC,CAAC;QAC5C,CAAC,CAAC,CAAC;QACH,IAAI,CAAC,OAAO;YAAE,KAAK,CAAC,eAAe,GAAG,KAAK,CAAC;IAC9C,CAAC,CAAC;IAEF,MAAM,YAAY,GAAG,2BAA2B,EAAE,CAAC;IACnD,IAAI,YAAY,GAAG,CAAC,EAAE,CAAC;QACrB,GAAG,CAAC,iCAAiC,YAAY,wBAAwB,CAAC,CAAC;QAC3E,cAAc,CAAC,gBAAgB,CAAC,CAAC;QACjC,OAAO;IACT,CAAC;IAED,MAAM,WAAW,GAAG,KAAK,EAAE,YAAY,CAAC;IACxC,IAAI,CAAC,WAAW,IAAI,CAAC,EAAE,CAAC,UAAU,CAAC,WAAW,CAAC,EAAE,CAAC;QAChD,GAAG,CAAC,qEAAqE,CAAC,CAAC;QAC3E,cAAc,CAAC,iBAAiB,CAAC,CAAC;QAClC,OAAO;IACT,CAAC;IAED,GAAG,CAAC,iCAAiC,IAAI,CAAC,QAAQ,CAAC,WAAW,CAAC,EAAE,CAAC,CAAC;IACnE,MAAM,OAAO,GAAG,cAAc,CAAC,SAAS,EAAE,WAAW,EAAE,GAAG,EAAE,yBAAyB,EAAE,CAAC,IAAI,EAAE,EAAE;QAC9F,GAAG,CAAC,gCAAgC,IAAI,EAAE,CAAC,CAAC;QAC5C,kBAAkB,CAAC,SAAS,EAAE,SAAS,CAAC,CAAC;QACzC,MAAM,OAAO,GAAG,eAAe,EAAE,CAAC;QAClC,IAAI,gCAAgC,CAAC,EAAE,QAAQ,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC,EAAE,CAAC;YACpF,MAAM,WAAW,GAAG,IAAI,KAAK,gCAAgC,CAAC,CAAC,CAAC,gBAAgB,CAAC,CAAC,CAAC,iBAAiB,CAAC;YACrG,wBAAwB,CACtB,SAAS,EACT,UAAU,EACV,iCAAiC,WAAW,6CAA6C,EACzF,4CAA4C,EAC5C,YAAY,CACb,CAAC;YACF,cAAc,CAAC,WAAW,CAAC,CAAC;YAC5B,OAAO;QACT,CAAC;QACD,MAAM,CAAC,OAAO,EAAE,qBAAqB,CAAC,CAAC;IACzC,CAAC,CAAC,CAAC;IACH,IAAI,CAAC,OAAO;QAAE,KAAK,CAAC,eAAe,GAAG,KAAK,CAAC;AAC9C,CAAC"}
@@ -0,0 +1,2 @@
1
+ export {};
2
+ //# sourceMappingURL=connectivity-recovery.test.d.ts.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"connectivity-recovery.test.d.ts","sourceRoot":"","sources":["../src/connectivity-recovery.test.ts"],"names":[],"mappings":""}
@@ -0,0 +1,147 @@
1
+ import { describe, expect, it } from "vitest";
2
+ import fs from "node:fs";
3
+ import os from "node:os";
4
+ import path from "node:path";
5
+ import { clearOcLimitBannerForPane, maybeEndRateLimitEpisode, newConnectivityRecoveryState, shouldChainRotateUntilAfterSmart, shouldNotifyProxyDownStuck, smartRestartCooldownLeftSec, updateIpifyProbeStreak, updateRateLimitEpisode, } from "./connectivity-recovery.js";
6
+ describe("updateIpifyProbeStreak (debounce shallow ipify failures)", () => {
7
+ it("does not confirm down on the first failure", () => {
8
+ const state = newConnectivityRecoveryState();
9
+ const r = updateIpifyProbeStreak(state, false, 3);
10
+ expect(r.confirmedDown).toBe(false);
11
+ expect(r.streak).toBe(1);
12
+ });
13
+ it("confirms down only after threshold consecutive failures", () => {
14
+ const state = newConnectivityRecoveryState();
15
+ updateIpifyProbeStreak(state, false, 3);
16
+ updateIpifyProbeStreak(state, false, 3);
17
+ const r = updateIpifyProbeStreak(state, false, 3);
18
+ expect(r.confirmedDown).toBe(true);
19
+ expect(r.streak).toBe(3);
20
+ });
21
+ it("resets streak when carrier is ok again", () => {
22
+ const state = newConnectivityRecoveryState();
23
+ updateIpifyProbeStreak(state, false, 3);
24
+ updateIpifyProbeStreak(state, true, 3);
25
+ expect(state.ipifyFailStreak).toBe(0);
26
+ const r = updateIpifyProbeStreak(state, false, 3);
27
+ expect(r.confirmedDown).toBe(false);
28
+ expect(r.streak).toBe(1);
29
+ });
30
+ });
31
+ describe("updateRateLimitEpisode (OC-LIMIT rising-edge/episode)", () => {
32
+ it("starts an episode on the first rate-limited pane", () => {
33
+ const state = newConnectivityRecoveryState();
34
+ const r = updateRateLimitEpisode(state, true, 1_000);
35
+ expect(r.action).toBe("start");
36
+ expect(state.rateLimitEpisodeActive).toBe(true);
37
+ expect(state.lastRateLimitSeenAt).toBe(1_000);
38
+ });
39
+ it("does not re-trigger while the episode is already active (no re-arm loop)", () => {
40
+ const state = newConnectivityRecoveryState();
41
+ updateRateLimitEpisode(state, true, 1_000);
42
+ const r = updateRateLimitEpisode(state, true, 2_000);
43
+ expect(r.action).toBe("continue");
44
+ expect(state.lastRateLimitSeenAt).toBe(2_000);
45
+ });
46
+ it("does not clear immediately when limited panes disappear (flicker grace)", () => {
47
+ const state = newConnectivityRecoveryState();
48
+ updateRateLimitEpisode(state, true, 1_000);
49
+ const r = updateRateLimitEpisode(state, false, 1_500, 120_000);
50
+ expect(r.action).toBe("none");
51
+ expect(state.rateLimitEpisodeActive).toBe(true);
52
+ });
53
+ it("clears the episode once the grace period has elapsed with no limited panes", () => {
54
+ const state = newConnectivityRecoveryState();
55
+ updateRateLimitEpisode(state, true, 1_000);
56
+ const r = updateRateLimitEpisode(state, false, 1_000 + 120_001, 120_000);
57
+ expect(r.action).toBe("clear");
58
+ expect(state.rateLimitEpisodeActive).toBe(false);
59
+ });
60
+ it("newConnectivityRecoveryState includes rateLimitRecoveryStarted false", () => {
61
+ expect(newConnectivityRecoveryState().rateLimitRecoveryStarted).toBe(false);
62
+ });
63
+ it("is a no-op when never armed and still no limit", () => {
64
+ const state = newConnectivityRecoveryState();
65
+ const r = updateRateLimitEpisode(state, false, 1_000);
66
+ expect(r.action).toBe("none");
67
+ });
68
+ });
69
+ describe("shouldChainRotateUntilAfterSmart (Proxy-SMART -> rotate-until chaining)", () => {
70
+ it("chains when Proxy-SMART skipped on cooldown (exit 2)", () => {
71
+ expect(shouldChainRotateUntilAfterSmart({ exitCode: 2, ipBefore: "1.2.3.4", ipAfter: "1.2.3.4" })).toBe(true);
72
+ });
73
+ it("chains when the carrier IP failed to resolve after SMART", () => {
74
+ expect(shouldChainRotateUntilAfterSmart({ exitCode: 0, ipBefore: "1.2.3.4", ipAfter: null })).toBe(true);
75
+ });
76
+ it("chains when the carrier IP is unchanged (SMART no-op)", () => {
77
+ expect(shouldChainRotateUntilAfterSmart({ exitCode: 0, ipBefore: "1.2.3.4", ipAfter: "1.2.3.4" })).toBe(true);
78
+ });
79
+ it("chains when SMART exited non-zero for any other reason", () => {
80
+ expect(shouldChainRotateUntilAfterSmart({ exitCode: 1, ipBefore: "1.2.3.4", ipAfter: "5.6.7.8" })).toBe(true);
81
+ });
82
+ it("does NOT chain when SMART succeeded and rotated the carrier IP", () => {
83
+ expect(shouldChainRotateUntilAfterSmart({ exitCode: 0, ipBefore: "1.2.3.4", ipAfter: "5.6.7.8" })).toBe(false);
84
+ });
85
+ });
86
+ describe("shouldNotifyProxyDownStuck (PROXY-DOWN silence fix)", () => {
87
+ it("does not notify before an episode has started", () => {
88
+ const state = newConnectivityRecoveryState();
89
+ expect(shouldNotifyProxyDownStuck(state, 1_000, 120_000)).toBe(false);
90
+ });
91
+ it("does not notify before the threshold elapses", () => {
92
+ const state = newConnectivityRecoveryState();
93
+ state.proxyDownEpisodeStartAt = 1_000;
94
+ expect(shouldNotifyProxyDownStuck(state, 1_000 + 119_999, 120_000)).toBe(false);
95
+ });
96
+ it("notifies once the threshold elapses", () => {
97
+ const state = newConnectivityRecoveryState();
98
+ state.proxyDownEpisodeStartAt = 1_000;
99
+ expect(shouldNotifyProxyDownStuck(state, 1_000 + 120_000, 120_000)).toBe(true);
100
+ });
101
+ it("never re-fires once already notified for this episode", () => {
102
+ const state = newConnectivityRecoveryState();
103
+ state.proxyDownEpisodeStartAt = 1_000;
104
+ state.proxyDownStuckNotified = true;
105
+ expect(shouldNotifyProxyDownStuck(state, 1_000 + 999_999, 120_000)).toBe(false);
106
+ });
107
+ });
108
+ describe("clearOcLimitBannerForPane", () => {
109
+ it("removes pane from ocLimited and ends episode when cache empty", () => {
110
+ const state = newConnectivityRecoveryState();
111
+ state.ocLimited.add("%9");
112
+ state.rateLimitEpisodeActive = true;
113
+ state.rateLimitRecoveryStarted = true;
114
+ expect(clearOcLimitBannerForPane(state, "%9")).toBe(true);
115
+ expect(state.ocLimited.has("%9")).toBe(false);
116
+ maybeEndRateLimitEpisode(state);
117
+ expect(state.rateLimitEpisodeActive).toBe(false);
118
+ expect(state.rateLimitRecoveryStarted).toBe(false);
119
+ });
120
+ });
121
+ describe("smartRestartCooldownLeftSec (harness-parity stamp file)", () => {
122
+ const tmpStamp = path.join(os.tmpdir(), `sm-test-smart-restart-${process.pid}.last`);
123
+ it("returns 0 when the stamp file does not exist", () => {
124
+ expect(smartRestartCooldownLeftSec(tmpStamp, 1800, 1_000_000)).toBe(0);
125
+ });
126
+ it("returns seconds left when within the cooldown window", () => {
127
+ const nowSec = 1_000_000;
128
+ fs.writeFileSync(tmpStamp, String(nowSec - 100));
129
+ try {
130
+ expect(smartRestartCooldownLeftSec(tmpStamp, 1800, nowSec * 1000)).toBe(1700);
131
+ }
132
+ finally {
133
+ fs.rmSync(tmpStamp, { force: true });
134
+ }
135
+ });
136
+ it("returns 0 once the cooldown window has fully elapsed", () => {
137
+ const nowSec = 1_000_000;
138
+ fs.writeFileSync(tmpStamp, String(nowSec - 1800));
139
+ try {
140
+ expect(smartRestartCooldownLeftSec(tmpStamp, 1800, nowSec * 1000)).toBe(0);
141
+ }
142
+ finally {
143
+ fs.rmSync(tmpStamp, { force: true });
144
+ }
145
+ });
146
+ });
147
+ //# sourceMappingURL=connectivity-recovery.test.js.map
@@ -0,0 +1 @@
1
+ {"version":3,"file":"connectivity-recovery.test.js","sourceRoot":"","sources":["../src/connectivity-recovery.test.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,QAAQ,CAAC;AAC9C,OAAO,EAAE,MAAM,SAAS,CAAC;AACzB,OAAO,EAAE,MAAM,SAAS,CAAC;AACzB,OAAO,IAAI,MAAM,WAAW,CAAC;AAC7B,OAAO,EACL,yBAAyB,EACzB,wBAAwB,EACxB,4BAA4B,EAC5B,gCAAgC,EAChC,0BAA0B,EAC1B,2BAA2B,EAC3B,sBAAsB,EACtB,sBAAsB,GACvB,MAAM,4BAA4B,CAAC;AAEpC,QAAQ,CAAC,0DAA0D,EAAE,GAAG,EAAE;IACxE,EAAE,CAAC,4CAA4C,EAAE,GAAG,EAAE;QACpD,MAAM,KAAK,GAAG,4BAA4B,EAAE,CAAC;QAC7C,MAAM,CAAC,GAAG,sBAAsB,CAAC,KAAK,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC;QAClD,MAAM,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACpC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAC3B,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,yDAAyD,EAAE,GAAG,EAAE;QACjE,MAAM,KAAK,GAAG,4BAA4B,EAAE,CAAC;QAC7C,sBAAsB,CAAC,KAAK,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC;QACxC,sBAAsB,CAAC,KAAK,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC;QACxC,MAAM,CAAC,GAAG,sBAAsB,CAAC,KAAK,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC;QAClD,MAAM,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QACnC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAC3B,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,wCAAwC,EAAE,GAAG,EAAE;QAChD,MAAM,KAAK,GAAG,4BAA4B,EAAE,CAAC;QAC7C,sBAAsB,CAAC,KAAK,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC;QACxC,sBAAsB,CAAC,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;QACvC,MAAM,CAAC,KAAK,CAAC,eAAe,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QACtC,MAAM,CAAC,GAAG,sBAAsB,CAAC,KAAK,EAAE,KAAK,EAAE,CAAC,CAAC,CAAC;QAClD,MAAM,CAAC,CAAC,CAAC,aAAa,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACpC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAC3B,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC;AAEH,QAAQ,CAAC,uDAAuD,EAAE,GAAG,EAAE;IACrE,EAAE,CAAC,kDAAkD,EAAE,GAAG,EAAE;QAC1D,MAAM,KAAK,GAAG,4BAA4B,EAAE,CAAC;QAC7C,MAAM,CAAC,GAAG,sBAAsB,CAAC,KAAK,EAAE,IAAI,EAAE,KAAK,CAAC,CAAC;QACrD,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QAC/B,MAAM,CAAC,KAAK,CAAC,sBAAsB,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAChD,MAAM,CAAC,KAAK,CAAC,mBAAmB,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAChD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,0EAA0E,EAAE,GAAG,EAAE;QAClF,MAAM,KAAK,GAAG,4BAA4B,EAAE,CAAC;QAC7C,sBAAsB,CAAC,KAAK,EAAE,IAAI,EAAE,KAAK,CAAC,CAAC;QAC3C,MAAM,CAAC,GAAG,sBAAsB,CAAC,KAAK,EAAE,IAAI,EAAE,KAAK,CAAC,CAAC;QACrD,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,UAAU,CAAC,CAAC;QAClC,MAAM,CAAC,KAAK,CAAC,mBAAmB,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAChD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,yEAAyE,EAAE,GAAG,EAAE;QACjF,MAAM,KAAK,GAAG,4BAA4B,EAAE,CAAC;QAC7C,sBAAsB,CAAC,KAAK,EAAE,IAAI,EAAE,KAAK,CAAC,CAAC;QAC3C,MAAM,CAAC,GAAG,sBAAsB,CAAC,KAAK,EAAE,KAAK,EAAE,KAAK,EAAE,OAAO,CAAC,CAAC;QAC/D,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;QAC9B,MAAM,CAAC,KAAK,CAAC,sBAAsB,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAClD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,4EAA4E,EAAE,GAAG,EAAE;QACpF,MAAM,KAAK,GAAG,4BAA4B,EAAE,CAAC;QAC7C,sBAAsB,CAAC,KAAK,EAAE,IAAI,EAAE,KAAK,CAAC,CAAC;QAC3C,MAAM,CAAC,GAAG,sBAAsB,CAAC,KAAK,EAAE,KAAK,EAAE,KAAK,GAAG,OAAO,EAAE,OAAO,CAAC,CAAC;QACzE,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;QAC/B,MAAM,CAAC,KAAK,CAAC,sBAAsB,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IACnD,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,sEAAsE,EAAE,GAAG,EAAE;QAC9E,MAAM,CAAC,4BAA4B,EAAE,CAAC,wBAAwB,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAC9E,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,gDAAgD,EAAE,GAAG,EAAE;QACxD,MAAM,KAAK,GAAG,4BAA4B,EAAE,CAAC;QAC7C,MAAM,CAAC,GAAG,sBAAsB,CAAC,KAAK,EAAE,KAAK,EAAE,KAAK,CAAC,CAAC;QACtD,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC;IAChC,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC;AAEH,QAAQ,CAAC,yEAAyE,EAAE,GAAG,EAAE;IACvF,EAAE,CAAC,sDAAsD,EAAE,GAAG,EAAE;QAC9D,MAAM,CACJ,gCAAgC,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAE,QAAQ,EAAE,SAAS,EAAE,OAAO,EAAE,SAAS,EAAE,CAAC,CAC3F,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACf,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,0DAA0D,EAAE,GAAG,EAAE;QAClE,MAAM,CACJ,gCAAgC,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAE,QAAQ,EAAE,SAAS,EAAE,OAAO,EAAE,IAAI,EAAE,CAAC,CACtF,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACf,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,uDAAuD,EAAE,GAAG,EAAE;QAC/D,MAAM,CACJ,gCAAgC,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAE,QAAQ,EAAE,SAAS,EAAE,OAAO,EAAE,SAAS,EAAE,CAAC,CAC3F,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACf,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,wDAAwD,EAAE,GAAG,EAAE;QAChE,MAAM,CACJ,gCAAgC,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAE,QAAQ,EAAE,SAAS,EAAE,OAAO,EAAE,SAAS,EAAE,CAAC,CAC3F,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACf,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,gEAAgE,EAAE,GAAG,EAAE;QACxE,MAAM,CACJ,gCAAgC,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAE,QAAQ,EAAE,SAAS,EAAE,OAAO,EAAE,SAAS,EAAE,CAAC,CAC3F,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAChB,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC;AAEH,QAAQ,CAAC,qDAAqD,EAAE,GAAG,EAAE;IACnE,EAAE,CAAC,+CAA+C,EAAE,GAAG,EAAE;QACvD,MAAM,KAAK,GAAG,4BAA4B,EAAE,CAAC;QAC7C,MAAM,CAAC,0BAA0B,CAAC,KAAK,EAAE,KAAK,EAAE,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IACxE,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,8CAA8C,EAAE,GAAG,EAAE;QACtD,MAAM,KAAK,GAAG,4BAA4B,EAAE,CAAC;QAC7C,KAAK,CAAC,uBAAuB,GAAG,KAAK,CAAC;QACtC,MAAM,CAAC,0BAA0B,CAAC,KAAK,EAAE,KAAK,GAAG,OAAO,EAAE,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAClF,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,qCAAqC,EAAE,GAAG,EAAE;QAC7C,MAAM,KAAK,GAAG,4BAA4B,EAAE,CAAC;QAC7C,KAAK,CAAC,uBAAuB,GAAG,KAAK,CAAC;QACtC,MAAM,CAAC,0BAA0B,CAAC,KAAK,EAAE,KAAK,GAAG,OAAO,EAAE,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACjF,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,uDAAuD,EAAE,GAAG,EAAE;QAC/D,MAAM,KAAK,GAAG,4BAA4B,EAAE,CAAC;QAC7C,KAAK,CAAC,uBAAuB,GAAG,KAAK,CAAC;QACtC,KAAK,CAAC,sBAAsB,GAAG,IAAI,CAAC;QACpC,MAAM,CAAC,0BAA0B,CAAC,KAAK,EAAE,KAAK,GAAG,OAAO,EAAE,OAAO,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IAClF,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC;AAEH,QAAQ,CAAC,2BAA2B,EAAE,GAAG,EAAE;IACzC,EAAE,CAAC,+DAA+D,EAAE,GAAG,EAAE;QACvE,MAAM,KAAK,GAAG,4BAA4B,EAAE,CAAC;QAC7C,KAAK,CAAC,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QAC1B,KAAK,CAAC,sBAAsB,GAAG,IAAI,CAAC;QACpC,KAAK,CAAC,wBAAwB,GAAG,IAAI,CAAC;QACtC,MAAM,CAAC,yBAAyB,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAC1D,MAAM,CAAC,KAAK,CAAC,SAAS,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QAC9C,wBAAwB,CAAC,KAAK,CAAC,CAAC;QAChC,MAAM,CAAC,KAAK,CAAC,sBAAsB,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;QACjD,MAAM,CAAC,KAAK,CAAC,wBAAwB,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,CAAC;IACrD,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC;AAEH,QAAQ,CAAC,yDAAyD,EAAE,GAAG,EAAE;IACvE,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,EAAE,CAAC,MAAM,EAAE,EAAE,yBAAyB,OAAO,CAAC,GAAG,OAAO,CAAC,CAAC;IAErF,EAAE,CAAC,8CAA8C,EAAE,GAAG,EAAE;QACtD,MAAM,CAAC,2BAA2B,CAAC,QAAQ,EAAE,IAAI,EAAE,SAAS,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACzE,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,sDAAsD,EAAE,GAAG,EAAE;QAC9D,MAAM,MAAM,GAAG,SAAS,CAAC;QACzB,EAAE,CAAC,aAAa,CAAC,QAAQ,EAAE,MAAM,CAAC,MAAM,GAAG,GAAG,CAAC,CAAC,CAAC;QACjD,IAAI,CAAC;YACH,MAAM,CAAC,2BAA2B,CAAC,QAAQ,EAAE,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;QAChF,CAAC;gBAAS,CAAC;YACT,EAAE,CAAC,MAAM,CAAC,QAAQ,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QACvC,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,EAAE,CAAC,sDAAsD,EAAE,GAAG,EAAE;QAC9D,MAAM,MAAM,GAAG,SAAS,CAAC;QACzB,EAAE,CAAC,aAAa,CAAC,QAAQ,EAAE,MAAM,CAAC,MAAM,GAAG,IAAI,CAAC,CAAC,CAAC;QAClD,IAAI,CAAC;YACH,MAAM,CAAC,2BAA2B,CAAC,QAAQ,EAAE,IAAI,EAAE,MAAM,GAAG,IAAI,CAAC,CAAC,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;QAC7E,CAAC;gBAAS,CAAC;YACT,EAAE,CAAC,MAAM,CAAC,QAAQ,EAAE,EAAE,KAAK,EAAE,IAAI,EAAE,CAAC,CAAC;QACvC,CAAC;IACH,CAAC,CAAC,CAAC;AACL,CAAC,CAAC,CAAC"}
@@ -0,0 +1,8 @@
1
+ import type { LoadedProfile } from "@seat-mesh/core";
2
+ import { JsonlStore } from "./jsonl-store.js";
3
+ import { SqliteStore } from "./sqlite-store.js";
4
+ export type { CheckbackRow, PeerRow, ToMasterRow, PeerKind } from "./jsonl-store.js";
5
+ export { isInboxDelivered, isPeerDelivered, dedupeJsonlRowsById } from "./jsonl-store.js";
6
+ export type QueueStore = JsonlStore | SqliteStore;
7
+ export declare function createQueueStore(loaded: LoadedProfile, log: (line: string) => void): QueueStore;
8
+ //# sourceMappingURL=create-queue-store.d.ts.map