instar 1.2.35 → 1.2.36
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/commands/server.d.ts.map +1 -1
- package/dist/commands/server.js +20 -67
- package/dist/commands/server.js.map +1 -1
- package/dist/tunnel/TunnelManager.d.ts +99 -62
- package/dist/tunnel/TunnelManager.d.ts.map +1 -1
- package/dist/tunnel/TunnelManager.js +340 -395
- package/dist/tunnel/TunnelManager.js.map +1 -1
- package/package.json +1 -1
- package/src/data/builtin-manifest.json +2 -2
- package/upgrades/1.2.36.md +111 -0
- package/upgrades/side-effects/tunnel-manager-rewrite.md +129 -0
|
@@ -1,473 +1,418 @@
|
|
|
1
1
|
/**
|
|
2
|
-
*
|
|
2
|
+
* TunnelManager — single owner of the tunnel lifecycle.
|
|
3
3
|
*
|
|
4
|
-
*
|
|
5
|
-
* ephemeral) and named tunnels (persistent, custom domain).
|
|
4
|
+
* Per spec specs/dev-infrastructure/tunnel-failure-resilience.md.
|
|
6
5
|
*
|
|
7
|
-
*
|
|
8
|
-
*
|
|
6
|
+
* Rewritten on top of the foundation modules:
|
|
7
|
+
* - TunnelProvider — backend abstraction; concrete providers under
|
|
8
|
+
* src/tunnel/Cloudflare*Provider.ts.
|
|
9
|
+
* - TunnelLifecycle — single-writer CAS-guarded state machine.
|
|
10
|
+
* - TunnelNotifier — two-channel routing (group / owner-DM) of
|
|
11
|
+
* transition events.
|
|
9
12
|
*
|
|
10
|
-
* The
|
|
11
|
-
*
|
|
12
|
-
*
|
|
13
|
-
*
|
|
14
|
-
*
|
|
15
|
-
*
|
|
13
|
+
* The manager is the SOLE owner of the detect → attempt → fall-back →
|
|
14
|
+
* notify → self-heal lifecycle. The previous server.ts startup-retry
|
|
15
|
+
* ladder + background-retry ladder + Lifeline failure message are
|
|
16
|
+
* retired in favor of routing all retry through here (one backoff
|
|
17
|
+
* engine, not two).
|
|
18
|
+
*
|
|
19
|
+
* Scope of this rewrite (PR 2 of the chain):
|
|
20
|
+
* - Tier-1 provider pool (Cloudflare named → quick) with internal
|
|
21
|
+
* backoff between retries within an episode.
|
|
22
|
+
* - Post-start reachability probe (HTTP /health through the public
|
|
23
|
+
* URL) before declaring `active` — prevents broadcasting a "back
|
|
24
|
+
* online" link that doesn't actually serve traffic.
|
|
25
|
+
* - Backward-compatible public API: start(), stop(), forceStop(),
|
|
26
|
+
* enableAutoReconnect(), disableAutoReconnect(), getExternalUrl(),
|
|
27
|
+
* url/isRunning/state. Plus the existing events.
|
|
28
|
+
* - Notifier sink optional; when telegram adapter is plumbed,
|
|
29
|
+
* transition events route to the group topic.
|
|
30
|
+
*
|
|
31
|
+
* Out of scope for THIS PR (future PRs in the chain):
|
|
32
|
+
* - Tier-2 consent flow + relay providers (PR 4).
|
|
33
|
+
* - Owner-DM channel + inline-button consent UX (PR 3).
|
|
34
|
+
* - authToken/PIN rotation + boot recovery (PR 5).
|
|
35
|
+
* - Self-heal probe with N-consecutive-success stability gate (PR 6).
|
|
36
|
+
* - The /tunnel route (PR 7).
|
|
37
|
+
*
|
|
38
|
+
* The lifecycle state machine already supports these states; the
|
|
39
|
+
* manager simply doesn't transition into them yet in PR 2.
|
|
16
40
|
*/
|
|
17
41
|
import { EventEmitter } from 'node:events';
|
|
18
|
-
import { spawn } from 'node:child_process';
|
|
19
42
|
import fs from 'node:fs';
|
|
20
43
|
import path from 'node:path';
|
|
21
|
-
import {
|
|
44
|
+
import { TunnelLifecycle, classifyFailure, } from './TunnelLifecycle.js';
|
|
45
|
+
import { CloudflareQuickProvider } from './CloudflareQuickProvider.js';
|
|
46
|
+
import { CloudflareNamedProvider } from './CloudflareNamedProvider.js';
|
|
47
|
+
import { TunnelNotifier } from './TunnelNotifier.js';
|
|
48
|
+
// ── Constants ───────────────────────────────────────────────────────
|
|
49
|
+
const BASE_BACKOFF_MS = 5_000;
|
|
50
|
+
const MAX_BACKOFF_MS = 5 * 60_000;
|
|
51
|
+
const MAX_BACKOFF_ATTEMPTS = 10;
|
|
52
|
+
const REACHABILITY_TIMEOUT_MS = 8_000;
|
|
53
|
+
/**
|
|
54
|
+
* Post-exhausted retry cadence — the minimum-viable "self-heal"
|
|
55
|
+
* placeholder for PR 2. After the bounded startup-reconnect ladder
|
|
56
|
+
* exhausts (MAX_BACKOFF_ATTEMPTS), the manager keeps probing the
|
|
57
|
+
* Tier-1 pool at this cadence indefinitely. This is intentionally
|
|
58
|
+
* crude; PR 6 replaces it with the spec's N-consecutive-success
|
|
59
|
+
* stability-gate probe per Part 5. Without this placeholder, the
|
|
60
|
+
* agent stays link-less after exhaustion until restart — which is
|
|
61
|
+
* the regression we explicitly need to avoid in the PR chain.
|
|
62
|
+
*/
|
|
63
|
+
const POST_EXHAUSTED_RETRY_INTERVAL_MS = 15 * 60_000;
|
|
22
64
|
// ── Manager ────────────────────────────────────────────────────────
|
|
23
65
|
export class TunnelManager extends EventEmitter {
|
|
24
66
|
config;
|
|
25
|
-
tunnel = null;
|
|
26
67
|
stateFile;
|
|
27
|
-
|
|
68
|
+
lifecycle;
|
|
69
|
+
providers;
|
|
70
|
+
notifier;
|
|
71
|
+
fetcher;
|
|
72
|
+
currentHandle = null;
|
|
73
|
+
currentProviderName = null;
|
|
74
|
+
_legacyState;
|
|
75
|
+
_autoReconnect = true; // always-on under the new design
|
|
28
76
|
_stopped = false;
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
static BASE_RECONNECT_DELAY_MS = 5_000;
|
|
35
|
-
static MAX_RECONNECT_DELAY_MS = 5 * 60_000;
|
|
36
|
-
constructor(config) {
|
|
77
|
+
_startPromise = null;
|
|
78
|
+
_backoffAttempt = 0;
|
|
79
|
+
_backoffTimer = null;
|
|
80
|
+
_postExhaustedTimer = null;
|
|
81
|
+
constructor(config, injections) {
|
|
37
82
|
super();
|
|
38
83
|
this.config = config;
|
|
39
84
|
this.stateFile = path.join(config.stateDir, 'tunnel.json');
|
|
40
|
-
this.
|
|
85
|
+
this.lifecycle = new TunnelLifecycle();
|
|
86
|
+
this.providers = injections?.providers ?? this.buildDefaultPool(config);
|
|
87
|
+
this.notifier = injections?.notifierSink
|
|
88
|
+
? new TunnelNotifier({ sink: injections.notifierSink })
|
|
89
|
+
: null;
|
|
90
|
+
this.fetcher = injections?.fetch ?? globalThis.fetch.bind(globalThis);
|
|
91
|
+
this._legacyState = {
|
|
41
92
|
url: null,
|
|
42
93
|
type: config.type,
|
|
43
94
|
startedAt: null,
|
|
44
95
|
};
|
|
96
|
+
// Route lifecycle transitions through the notifier.
|
|
97
|
+
this.lifecycle.on('transition', (e) => {
|
|
98
|
+
if (this.notifier)
|
|
99
|
+
void this.notifier.onTransition(e);
|
|
100
|
+
});
|
|
101
|
+
// Restore persisted snapshot (rotation-pending flag + consent cooldown).
|
|
102
|
+
this.restorePersisted();
|
|
45
103
|
}
|
|
46
|
-
|
|
47
|
-
get url() {
|
|
48
|
-
|
|
49
|
-
}
|
|
50
|
-
/**
|
|
51
|
-
get
|
|
52
|
-
|
|
53
|
-
|
|
54
|
-
|
|
55
|
-
get state() {
|
|
56
|
-
return { ...this._state };
|
|
104
|
+
// ── Public API (back-compat with the legacy manager) ────────────
|
|
105
|
+
get url() { return this._legacyState.url; }
|
|
106
|
+
get isRunning() { return this.currentHandle !== null && !this._stopped; }
|
|
107
|
+
get state() { return { ...this._legacyState }; }
|
|
108
|
+
/** Additive new accessor — lifecycle snapshot. */
|
|
109
|
+
get lifecycleState() {
|
|
110
|
+
const snap = this.lifecycle.snapshot();
|
|
111
|
+
snap.lastUrl = this._legacyState.url;
|
|
112
|
+
return snap;
|
|
57
113
|
}
|
|
58
114
|
/**
|
|
59
|
-
* Start the tunnel.
|
|
60
|
-
*
|
|
115
|
+
* Start the tunnel. Drives the full Tier-1 ladder internally — the
|
|
116
|
+
* caller does NOT wrap with its own retry loop (the old server.ts
|
|
117
|
+
* startup ladder is RETIRED).
|
|
118
|
+
*
|
|
119
|
+
* Resolves with the URL of the first provider that reaches `active`.
|
|
120
|
+
* Rejects when all Tier-1 providers fail and backoff is exhausted.
|
|
61
121
|
*/
|
|
62
122
|
async start() {
|
|
63
|
-
|
|
64
|
-
// This prevents concurrent start() calls from the SleepWake handler
|
|
65
|
-
// and auto-reconnect racing each other.
|
|
66
|
-
if (this._startPromise) {
|
|
123
|
+
if (this._startPromise)
|
|
67
124
|
return this._startPromise;
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
if (this.tunnel && this._state.url) {
|
|
71
|
-
return this._state.url;
|
|
72
|
-
}
|
|
125
|
+
if (this.currentHandle && this._legacyState.url)
|
|
126
|
+
return this._legacyState.url;
|
|
73
127
|
this._stopped = false;
|
|
74
|
-
|
|
75
|
-
try {
|
|
76
|
-
// Ensure cloudflared binary is installed
|
|
77
|
-
await this.ensureBinary();
|
|
78
|
-
// Start the appropriate tunnel type
|
|
79
|
-
if (this.config.type === 'named') {
|
|
80
|
-
if (!this.config.token && !this.config.configFile) {
|
|
81
|
-
throw new Error('Named tunnel requires either a token or a configFile. Set tunnel.token or tunnel.configFile in config.');
|
|
82
|
-
}
|
|
83
|
-
if (this.config.configFile) {
|
|
84
|
-
return await this.startConfigFileTunnel();
|
|
85
|
-
}
|
|
86
|
-
return await this.startNamedTunnel();
|
|
87
|
-
}
|
|
88
|
-
else {
|
|
89
|
-
return await this.startQuickTunnel();
|
|
90
|
-
}
|
|
91
|
-
}
|
|
92
|
-
finally {
|
|
93
|
-
this._startPromise = null;
|
|
94
|
-
}
|
|
95
|
-
};
|
|
96
|
-
this._startPromise = doStart();
|
|
128
|
+
this._startPromise = this.doStart().finally(() => { this._startPromise = null; });
|
|
97
129
|
return this._startPromise;
|
|
98
130
|
}
|
|
99
|
-
/**
|
|
100
|
-
* Stop the tunnel gracefully. Intentional stops disable auto-reconnect.
|
|
101
|
-
*/
|
|
102
131
|
async stop() {
|
|
103
132
|
this._stopped = true;
|
|
104
|
-
this.
|
|
105
|
-
|
|
106
|
-
|
|
107
|
-
|
|
133
|
+
this.clearBackoffTimer();
|
|
134
|
+
this.clearPostExhaustedTimer();
|
|
135
|
+
this._startPromise = null;
|
|
136
|
+
if (this.currentHandle) {
|
|
137
|
+
try {
|
|
138
|
+
await this.currentHandle.stop();
|
|
139
|
+
}
|
|
140
|
+
catch { /* handle may be dead */ }
|
|
141
|
+
this.currentHandle = null;
|
|
108
142
|
}
|
|
109
|
-
this.
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
143
|
+
this._legacyState.url = null;
|
|
144
|
+
this._legacyState.startedAt = null;
|
|
145
|
+
this.currentProviderName = null;
|
|
146
|
+
const from = this.lifecycle.state;
|
|
147
|
+
if (from !== 'idle') {
|
|
148
|
+
try {
|
|
149
|
+
this.lifecycle.transition(from, 'idle', { activeProvider: null });
|
|
150
|
+
}
|
|
151
|
+
catch { /* invalid pair — best-effort */ }
|
|
113
152
|
}
|
|
114
|
-
this.
|
|
115
|
-
this._state.startedAt = null;
|
|
116
|
-
this.saveState();
|
|
153
|
+
this.persist();
|
|
117
154
|
this.emit('stopped');
|
|
118
155
|
}
|
|
119
|
-
/**
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
this.
|
|
126
|
-
this.
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
156
|
+
/** Force-stop with escalation. Providers internally do SIGINT → SIGKILL. */
|
|
157
|
+
async forceStop(_timeoutMs) {
|
|
158
|
+
await this.stop();
|
|
159
|
+
}
|
|
160
|
+
enableAutoReconnect() { this._autoReconnect = true; }
|
|
161
|
+
disableAutoReconnect() {
|
|
162
|
+
this._autoReconnect = false;
|
|
163
|
+
this.clearBackoffTimer();
|
|
164
|
+
}
|
|
165
|
+
getExternalUrl(localPath) {
|
|
166
|
+
if (!this._legacyState.url)
|
|
167
|
+
return null;
|
|
168
|
+
const base = this._legacyState.url.replace(/\/$/, '');
|
|
169
|
+
const p = localPath.startsWith('/') ? localPath : `/${localPath}`;
|
|
170
|
+
return `${base}${p}`;
|
|
171
|
+
}
|
|
172
|
+
// ── Internals ──────────────────────────────────────────────────
|
|
173
|
+
buildDefaultPool(config) {
|
|
174
|
+
const pool = [];
|
|
175
|
+
if (config.token || config.configFile) {
|
|
176
|
+
pool.push(new CloudflareNamedProvider({
|
|
177
|
+
token: config.token,
|
|
178
|
+
configFile: config.configFile,
|
|
179
|
+
hostname: config.hostname,
|
|
180
|
+
}));
|
|
130
181
|
}
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
182
|
+
pool.push(new CloudflareQuickProvider({ port: config.port, stateDir: config.stateDir }));
|
|
183
|
+
return pool;
|
|
184
|
+
}
|
|
185
|
+
async doStart() {
|
|
186
|
+
if (!this.config.enabled)
|
|
187
|
+
throw new Error('tunnel.enabled is false');
|
|
188
|
+
if (this.lifecycle.state === 'idle' || this.lifecycle.state === 'active') {
|
|
189
|
+
this.lifecycle.startEpisode();
|
|
190
|
+
}
|
|
191
|
+
if (!this.lifecycle.transition('idle', 'starting')) {
|
|
192
|
+
if (this.lifecycle.state === 'starting' || this.lifecycle.state === 'active') {
|
|
193
|
+
if (this._legacyState.url)
|
|
194
|
+
return this._legacyState.url;
|
|
138
195
|
}
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
149
|
-
|
|
150
|
-
|
|
151
|
-
|
|
152
|
-
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
resolve(true);
|
|
162
|
-
}
|
|
163
|
-
}, 200);
|
|
164
|
-
});
|
|
165
|
-
if (!exited) {
|
|
166
|
-
console.warn(`[Tunnel] cloudflared (PID ${pid}) didn't exit after ${timeoutMs}ms SIGINT — sending SIGKILL`);
|
|
196
|
+
throw new Error(`cannot start: lifecycle in state ${this.lifecycle.state}`);
|
|
197
|
+
}
|
|
198
|
+
return this.driveTier1();
|
|
199
|
+
}
|
|
200
|
+
async driveTier1() {
|
|
201
|
+
let lastErr = null;
|
|
202
|
+
for (let i = 0; i < this.providers.length; i++) {
|
|
203
|
+
if (this._stopped)
|
|
204
|
+
throw new Error('tunnel start aborted: stopped');
|
|
205
|
+
const provider = this.providers[i];
|
|
206
|
+
if (!provider)
|
|
207
|
+
continue;
|
|
208
|
+
if (provider.tier !== 1)
|
|
209
|
+
continue; // Tier-2 deferred to PR 4
|
|
210
|
+
const available = await provider.isAvailable().catch(() => false);
|
|
211
|
+
if (!available)
|
|
212
|
+
continue;
|
|
213
|
+
try {
|
|
214
|
+
const handle = await provider.start(this.config.port);
|
|
215
|
+
// Reachability probe BEFORE declaring active.
|
|
216
|
+
const reachable = await this.probeReachability(handle.url).catch(() => false);
|
|
217
|
+
if (!reachable) {
|
|
167
218
|
try {
|
|
168
|
-
|
|
219
|
+
await handle.stop();
|
|
169
220
|
}
|
|
170
|
-
catch { /*
|
|
221
|
+
catch { /* best effort */ }
|
|
222
|
+
this.lifecycle.recordAttempt(provider.name, 'reachability-failed');
|
|
223
|
+
lastErr = new Error(`reachability-failed: ${provider.name} URL did not respond to /health`);
|
|
224
|
+
continue;
|
|
225
|
+
}
|
|
226
|
+
// Success.
|
|
227
|
+
this.currentHandle = handle;
|
|
228
|
+
this.currentProviderName = provider.name;
|
|
229
|
+
this._legacyState.url = handle.url;
|
|
230
|
+
this._legacyState.startedAt = new Date().toISOString();
|
|
231
|
+
const from = this.lifecycle.state;
|
|
232
|
+
if (from === 'starting' || from === 'retrying') {
|
|
233
|
+
this.lifecycle.transition(from, 'active', {
|
|
234
|
+
activeProvider: provider.name,
|
|
235
|
+
lastFailureReason: null,
|
|
236
|
+
});
|
|
237
|
+
}
|
|
238
|
+
// Persist AFTER the transition so the snapshot reflects 'active'.
|
|
239
|
+
this.persist();
|
|
240
|
+
this.emit('url', handle.url);
|
|
241
|
+
this._backoffAttempt = 0;
|
|
242
|
+
return handle.url;
|
|
243
|
+
}
|
|
244
|
+
catch (err) {
|
|
245
|
+
const reason = classifyFailure(err);
|
|
246
|
+
this.lifecycle.recordAttempt(provider.name, reason);
|
|
247
|
+
lastErr = err instanceof Error ? err : new Error(String(err));
|
|
248
|
+
if (this.lifecycle.state === 'starting') {
|
|
249
|
+
this.lifecycle.transition('starting', 'retrying', {
|
|
250
|
+
activeProvider: null,
|
|
251
|
+
lastFailureReason: reason,
|
|
252
|
+
});
|
|
171
253
|
}
|
|
172
254
|
}
|
|
173
|
-
this.tunnel = null;
|
|
174
255
|
}
|
|
175
|
-
this
|
|
176
|
-
this.
|
|
177
|
-
this.saveState();
|
|
178
|
-
this.emit('stopped');
|
|
179
|
-
}
|
|
180
|
-
/**
|
|
181
|
-
* Enable automatic reconnection when the tunnel disconnects unexpectedly.
|
|
182
|
-
* Uses exponential backoff: 5s, 10s, 20s, ... up to 5 minutes, max 10 attempts.
|
|
183
|
-
*/
|
|
184
|
-
enableAutoReconnect() {
|
|
185
|
-
this._autoReconnect = true;
|
|
256
|
+
// All Tier-1 providers exhausted in this attempt round.
|
|
257
|
+
return this.exhaustedOrBackoff(lastErr);
|
|
186
258
|
}
|
|
187
|
-
|
|
188
|
-
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
259
|
+
async exhaustedOrBackoff(lastErr) {
|
|
260
|
+
// Matches the legacy semantics: start() rejects after the FIRST
|
|
261
|
+
// round of provider attempts fails. The backoff retry runs in the
|
|
262
|
+
// background — the manager keeps trying without blocking the
|
|
263
|
+
// caller, and emits 'url' when a later attempt succeeds. This
|
|
264
|
+
// prevents start() from blocking server.ts boot for the full
|
|
265
|
+
// 25-minute worst-case exponential window. The original failing
|
|
266
|
+
// E2E (tests/e2e/tunnel-private-view.test.ts) timed out at the
|
|
267
|
+
// beforeAll 60s budget because the old draft kept retrying inside
|
|
268
|
+
// start().
|
|
269
|
+
const from = this.lifecycle.state;
|
|
270
|
+
if (from === 'retrying' || from === 'starting') {
|
|
271
|
+
this.lifecycle.transition(from, 'exhausted', { activeProvider: null });
|
|
192
272
|
}
|
|
193
|
-
|
|
273
|
+
const err = lastErr ?? new Error('all Tier-1 providers failed');
|
|
274
|
+
// Schedule background retry (bounded exponential ladder, then the
|
|
275
|
+
// indefinite post-exhausted placeholder once the ladder runs out).
|
|
276
|
+
// Fire-and-forget; if a later attempt succeeds, the 'url' event
|
|
277
|
+
// fires and downstream subscribers (notifier, dashboard
|
|
278
|
+
// broadcaster, log line) catch up.
|
|
279
|
+
if (this._autoReconnect && !this._stopped) {
|
|
280
|
+
this.scheduleBackgroundRetry();
|
|
281
|
+
}
|
|
282
|
+
this.emit('error', err);
|
|
283
|
+
throw err;
|
|
194
284
|
}
|
|
195
285
|
/**
|
|
196
|
-
*
|
|
286
|
+
* Background retry — runs the bounded exponential ladder off the
|
|
287
|
+
* start() promise so initial start() resolves/rejects fast. After
|
|
288
|
+
* the ladder exhausts, hands off to the indefinite post-exhausted
|
|
289
|
+
* placeholder (the PR 2 self-heal; PR 6 replaces with the spec's
|
|
290
|
+
* N-consecutive-success probe).
|
|
197
291
|
*/
|
|
198
|
-
|
|
199
|
-
if (
|
|
200
|
-
return;
|
|
201
|
-
// Skip if a start is already in progress (e.g., SleepWake handler is restarting)
|
|
202
|
-
if (this._startPromise)
|
|
292
|
+
scheduleBackgroundRetry() {
|
|
293
|
+
if (this._stopped || !this._autoReconnect)
|
|
203
294
|
return;
|
|
204
|
-
if (this.
|
|
205
|
-
|
|
206
|
-
|
|
295
|
+
if (this._backoffTimer)
|
|
296
|
+
return; // already scheduled
|
|
297
|
+
if (this._backoffAttempt >= MAX_BACKOFF_ATTEMPTS) {
|
|
298
|
+
this.schedulePostExhaustedRetry();
|
|
207
299
|
return;
|
|
208
300
|
}
|
|
209
|
-
const delay = Math.min(
|
|
210
|
-
this.
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
if (this._stopped)
|
|
301
|
+
const delay = Math.min(BASE_BACKOFF_MS * Math.pow(2, this._backoffAttempt), MAX_BACKOFF_MS);
|
|
302
|
+
this._backoffAttempt += 1;
|
|
303
|
+
this._backoffTimer = setTimeout(() => {
|
|
304
|
+
this._backoffTimer = null;
|
|
305
|
+
if (this._stopped || !this._autoReconnect)
|
|
214
306
|
return;
|
|
307
|
+
const from = this.lifecycle.state;
|
|
215
308
|
try {
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
309
|
+
if (from === 'exhausted') {
|
|
310
|
+
this.lifecycle.transition('exhausted', 'starting');
|
|
311
|
+
}
|
|
312
|
+
else if (from === 'idle') {
|
|
313
|
+
this.lifecycle.transition('idle', 'starting');
|
|
314
|
+
}
|
|
315
|
+
else if (from === 'retrying') {
|
|
316
|
+
this.lifecycle.transition('retrying', 'starting');
|
|
317
|
+
}
|
|
318
|
+
else {
|
|
319
|
+
return; // unexpected state; bail
|
|
320
|
+
}
|
|
219
321
|
}
|
|
220
|
-
catch
|
|
221
|
-
|
|
222
|
-
this.attemptReconnect();
|
|
322
|
+
catch {
|
|
323
|
+
return; // invalid transition; bail
|
|
223
324
|
}
|
|
325
|
+
void this.driveTier1().catch(() => { });
|
|
224
326
|
}, delay);
|
|
225
327
|
}
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
return null;
|
|
233
|
-
const base = this._state.url.replace(/\/$/, '');
|
|
234
|
-
const p = localPath.startsWith('/') ? localPath : `/${localPath}`;
|
|
235
|
-
return `${base}${p}`;
|
|
328
|
+
clearBackoffTimer() {
|
|
329
|
+
if (this._backoffTimer) {
|
|
330
|
+
clearTimeout(this._backoffTimer);
|
|
331
|
+
this._backoffTimer = null;
|
|
332
|
+
}
|
|
333
|
+
this._backoffAttempt = 0;
|
|
236
334
|
}
|
|
237
|
-
|
|
238
|
-
|
|
239
|
-
|
|
240
|
-
|
|
335
|
+
clearPostExhaustedTimer() {
|
|
336
|
+
if (this._postExhaustedTimer) {
|
|
337
|
+
clearTimeout(this._postExhaustedTimer);
|
|
338
|
+
this._postExhaustedTimer = null;
|
|
241
339
|
}
|
|
242
340
|
}
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
247
|
-
|
|
248
|
-
|
|
249
|
-
|
|
250
|
-
|
|
251
|
-
|
|
252
|
-
|
|
253
|
-
|
|
254
|
-
|
|
255
|
-
|
|
256
|
-
|
|
257
|
-
|
|
258
|
-
catch (err) {
|
|
259
|
-
reject(new Error(`Failed to start quick tunnel: ${err instanceof Error ? err.message : String(err)}`));
|
|
341
|
+
/**
|
|
342
|
+
* Minimum-viable post-exhausted retry (PR 2 self-heal placeholder).
|
|
343
|
+
* Schedules a single low-frequency probe; on completion (success or
|
|
344
|
+
* failure) it re-arms itself, so the agent keeps trying to recover
|
|
345
|
+
* even after the bounded startup-reconnect ladder gives up. PR 6
|
|
346
|
+
* replaces this with the spec's N-consecutive-success stability gate.
|
|
347
|
+
*/
|
|
348
|
+
schedulePostExhaustedRetry() {
|
|
349
|
+
if (this._stopped || !this._autoReconnect)
|
|
350
|
+
return;
|
|
351
|
+
if (this._postExhaustedTimer)
|
|
352
|
+
return; // already scheduled
|
|
353
|
+
this._postExhaustedTimer = setTimeout(async () => {
|
|
354
|
+
this._postExhaustedTimer = null;
|
|
355
|
+
if (this._stopped || !this._autoReconnect)
|
|
260
356
|
return;
|
|
261
|
-
|
|
262
|
-
|
|
263
|
-
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
reject(new Error('Tunnel connection timed out after 30 seconds'));
|
|
357
|
+
// Re-enter from exhausted → starting.
|
|
358
|
+
const from = this.lifecycle.state;
|
|
359
|
+
if (from === 'exhausted') {
|
|
360
|
+
try {
|
|
361
|
+
this.lifecycle.transition('exhausted', 'starting');
|
|
267
362
|
}
|
|
268
|
-
|
|
269
|
-
this.tunnel.once('url', (url) => {
|
|
270
|
-
if (resolved)
|
|
363
|
+
catch { /* invalid transition — bail */
|
|
271
364
|
return;
|
|
272
|
-
resolved = true;
|
|
273
|
-
clearTimeout(timeout);
|
|
274
|
-
this._state.url = url;
|
|
275
|
-
this._state.startedAt = new Date().toISOString();
|
|
276
|
-
this.saveState();
|
|
277
|
-
this.emit('url', url);
|
|
278
|
-
resolve(url);
|
|
279
|
-
});
|
|
280
|
-
this.tunnel.once('connected', (info) => {
|
|
281
|
-
this._state.connectionId = info.id;
|
|
282
|
-
this._state.connectionLocation = info.location;
|
|
283
|
-
this.saveState();
|
|
284
|
-
this.emit('connected', info);
|
|
285
|
-
});
|
|
286
|
-
this.tunnel.on('error', (err) => {
|
|
287
|
-
if (!resolved) {
|
|
288
|
-
resolved = true;
|
|
289
|
-
clearTimeout(timeout);
|
|
290
|
-
reject(err);
|
|
291
|
-
}
|
|
292
|
-
this.emit('error', err);
|
|
293
|
-
});
|
|
294
|
-
this.tunnel.on('exit', (code) => {
|
|
295
|
-
if (!this._stopped) {
|
|
296
|
-
this.tunnel = null;
|
|
297
|
-
this._state.url = null;
|
|
298
|
-
this.saveState();
|
|
299
|
-
this.emit('disconnected');
|
|
300
|
-
if (!resolved) {
|
|
301
|
-
resolved = true;
|
|
302
|
-
clearTimeout(timeout);
|
|
303
|
-
reject(new Error(`Tunnel process exited with code ${code}`));
|
|
304
|
-
}
|
|
305
|
-
else {
|
|
306
|
-
// Tunnel was running and disconnected unexpectedly — try to reconnect
|
|
307
|
-
this.attemptReconnect();
|
|
308
|
-
}
|
|
309
365
|
}
|
|
310
|
-
});
|
|
311
|
-
});
|
|
312
|
-
}
|
|
313
|
-
startNamedTunnel() {
|
|
314
|
-
return new Promise((resolve, reject) => {
|
|
315
|
-
try {
|
|
316
|
-
this.tunnel = Tunnel.withToken(this.config.token);
|
|
317
366
|
}
|
|
318
|
-
|
|
319
|
-
|
|
367
|
+
else if (from !== 'starting' && from !== 'idle') {
|
|
368
|
+
// State moved beyond our reach; let the active path handle it.
|
|
320
369
|
return;
|
|
321
370
|
}
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
325
|
-
resolved = true;
|
|
326
|
-
reject(new Error('Named tunnel connection timed out after 30 seconds'));
|
|
327
|
-
}
|
|
328
|
-
}, 30_000);
|
|
329
|
-
this.tunnel.once('url', (url) => {
|
|
330
|
-
if (resolved)
|
|
331
|
-
return;
|
|
332
|
-
resolved = true;
|
|
333
|
-
clearTimeout(timeout);
|
|
334
|
-
this._state.url = url;
|
|
335
|
-
this._state.startedAt = new Date().toISOString();
|
|
336
|
-
this.saveState();
|
|
337
|
-
this.emit('url', url);
|
|
338
|
-
resolve(url);
|
|
339
|
-
});
|
|
340
|
-
this.tunnel.once('connected', (info) => {
|
|
341
|
-
this._state.connectionId = info.id;
|
|
342
|
-
this._state.connectionLocation = info.location;
|
|
343
|
-
this.saveState();
|
|
344
|
-
this.emit('connected', info);
|
|
345
|
-
// For named tunnels, the URL may come from the connection info
|
|
346
|
-
// rather than the 'url' event, since the URL is pre-configured
|
|
347
|
-
if (!resolved && this._state.url) {
|
|
348
|
-
resolved = true;
|
|
349
|
-
clearTimeout(timeout);
|
|
350
|
-
resolve(this._state.url);
|
|
351
|
-
}
|
|
352
|
-
});
|
|
353
|
-
this.tunnel.on('error', (err) => {
|
|
354
|
-
if (!resolved) {
|
|
355
|
-
resolved = true;
|
|
356
|
-
clearTimeout(timeout);
|
|
357
|
-
reject(err);
|
|
358
|
-
}
|
|
359
|
-
this.emit('error', err);
|
|
360
|
-
});
|
|
361
|
-
this.tunnel.on('exit', (code) => {
|
|
362
|
-
if (!this._stopped) {
|
|
363
|
-
this.tunnel = null;
|
|
364
|
-
this._state.url = null;
|
|
365
|
-
this.saveState();
|
|
366
|
-
this.emit('disconnected');
|
|
367
|
-
if (!resolved) {
|
|
368
|
-
resolved = true;
|
|
369
|
-
clearTimeout(timeout);
|
|
370
|
-
reject(new Error(`Named tunnel process exited with code ${code}`));
|
|
371
|
-
}
|
|
372
|
-
else {
|
|
373
|
-
this.attemptReconnect();
|
|
374
|
-
}
|
|
375
|
-
}
|
|
376
|
-
});
|
|
377
|
-
});
|
|
371
|
+
this._backoffAttempt = 0;
|
|
372
|
+
this.driveTier1().then(() => { }, () => { });
|
|
373
|
+
}, POST_EXHAUSTED_RETRY_INTERVAL_MS);
|
|
378
374
|
}
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
const
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
}
|
|
387
|
-
// Spawn cloudflared directly with the config file
|
|
388
|
-
const child = spawn(bin, ['tunnel', '--config', configFile, 'run'], {
|
|
389
|
-
stdio: ['ignore', 'pipe', 'pipe'],
|
|
390
|
-
});
|
|
391
|
-
let resolved = false;
|
|
392
|
-
let stderrBuffer = '';
|
|
393
|
-
const timeout = setTimeout(() => {
|
|
394
|
-
if (!resolved) {
|
|
395
|
-
resolved = true;
|
|
396
|
-
// Named tunnels with config files don't emit a URL — the URL is the hostname
|
|
397
|
-
if (hostname) {
|
|
398
|
-
const url = `https://${hostname}`;
|
|
399
|
-
this._state.url = url;
|
|
400
|
-
this._state.startedAt = new Date().toISOString();
|
|
401
|
-
this.saveState();
|
|
402
|
-
this.emit('url', url);
|
|
403
|
-
resolve(url);
|
|
404
|
-
}
|
|
405
|
-
else {
|
|
406
|
-
reject(new Error('Named tunnel timed out and no hostname configured'));
|
|
407
|
-
}
|
|
408
|
-
}
|
|
409
|
-
}, 15_000);
|
|
410
|
-
// Watch stderr for connection established messages
|
|
411
|
-
child.stderr.on('data', (data) => {
|
|
412
|
-
const line = data.toString();
|
|
413
|
-
stderrBuffer += line;
|
|
414
|
-
// cloudflared logs connection info to stderr
|
|
415
|
-
if (line.includes('Registered tunnel connection') || line.includes('Connection registered')) {
|
|
416
|
-
if (!resolved && hostname) {
|
|
417
|
-
resolved = true;
|
|
418
|
-
clearTimeout(timeout);
|
|
419
|
-
const url = `https://${hostname}`;
|
|
420
|
-
this._state.url = url;
|
|
421
|
-
this._state.startedAt = new Date().toISOString();
|
|
422
|
-
this.saveState();
|
|
423
|
-
this.emit('url', url);
|
|
424
|
-
resolve(url);
|
|
425
|
-
}
|
|
426
|
-
}
|
|
427
|
-
});
|
|
428
|
-
child.on('error', (err) => {
|
|
429
|
-
if (!resolved) {
|
|
430
|
-
resolved = true;
|
|
431
|
-
clearTimeout(timeout);
|
|
432
|
-
reject(new Error(`Failed to start config-file tunnel: ${err.message}`));
|
|
433
|
-
}
|
|
434
|
-
});
|
|
435
|
-
child.on('exit', (code) => {
|
|
436
|
-
if (!this._stopped) {
|
|
437
|
-
this.tunnel = null;
|
|
438
|
-
this._state.url = null;
|
|
439
|
-
this.saveState();
|
|
440
|
-
this.emit('disconnected');
|
|
441
|
-
if (!resolved) {
|
|
442
|
-
resolved = true;
|
|
443
|
-
clearTimeout(timeout);
|
|
444
|
-
const errContext = stderrBuffer.slice(-500);
|
|
445
|
-
reject(new Error(`Config-file tunnel exited with code ${code}: ${errContext}`));
|
|
446
|
-
}
|
|
447
|
-
else {
|
|
448
|
-
this.attemptReconnect();
|
|
449
|
-
}
|
|
450
|
-
}
|
|
375
|
+
/** HTTP probe through the public URL — confirms the link actually serves. */
|
|
376
|
+
async probeReachability(url) {
|
|
377
|
+
try {
|
|
378
|
+
const controller = new AbortController();
|
|
379
|
+
const t = setTimeout(() => controller.abort(), REACHABILITY_TIMEOUT_MS);
|
|
380
|
+
const res = await this.fetcher(`${url.replace(/\/$/, '')}/health`, {
|
|
381
|
+
signal: controller.signal,
|
|
451
382
|
});
|
|
452
|
-
|
|
453
|
-
|
|
454
|
-
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
};
|
|
459
|
-
});
|
|
383
|
+
clearTimeout(t);
|
|
384
|
+
return res.ok;
|
|
385
|
+
}
|
|
386
|
+
catch {
|
|
387
|
+
return false;
|
|
388
|
+
}
|
|
460
389
|
}
|
|
461
|
-
|
|
390
|
+
// ── Persistence ──────────────────────────────────────────────────
|
|
391
|
+
persist() {
|
|
462
392
|
try {
|
|
463
393
|
const dir = path.dirname(this.stateFile);
|
|
464
|
-
if (!fs.existsSync(dir))
|
|
394
|
+
if (!fs.existsSync(dir))
|
|
465
395
|
fs.mkdirSync(dir, { recursive: true });
|
|
396
|
+
const snap = this.lifecycle.snapshot();
|
|
397
|
+
snap.lastUrl = this._legacyState.url;
|
|
398
|
+
fs.writeFileSync(this.stateFile, JSON.stringify(snap, null, 2));
|
|
399
|
+
}
|
|
400
|
+
catch {
|
|
401
|
+
// Non-critical.
|
|
402
|
+
}
|
|
403
|
+
}
|
|
404
|
+
restorePersisted() {
|
|
405
|
+
try {
|
|
406
|
+
if (!fs.existsSync(this.stateFile))
|
|
407
|
+
return;
|
|
408
|
+
const raw = fs.readFileSync(this.stateFile, 'utf-8');
|
|
409
|
+
const snap = JSON.parse(raw);
|
|
410
|
+
if (snap && typeof snap === 'object') {
|
|
411
|
+
this.lifecycle.restoreFrom(snap);
|
|
466
412
|
}
|
|
467
|
-
fs.writeFileSync(this.stateFile, JSON.stringify(this._state, null, 2));
|
|
468
413
|
}
|
|
469
414
|
catch {
|
|
470
|
-
//
|
|
415
|
+
// Corrupted state file — start fresh.
|
|
471
416
|
}
|
|
472
417
|
}
|
|
473
418
|
}
|