groove-dev 0.27.211 → 0.27.213
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CLAUDE.md +0 -8
- package/daemon-bridge.js +87 -0
- package/node_modules/@groove-dev/cli/bin/groove.js +20 -0
- package/node_modules/@groove-dev/cli/package.json +1 -1
- package/node_modules/@groove-dev/cli/src/client.js +9 -2
- package/node_modules/@groove-dev/cli/src/commands/ask.js +99 -0
- package/node_modules/@groove-dev/daemon/package.json +1 -1
- package/node_modules/@groove-dev/daemon/src/deliver.js +101 -1
- package/node_modules/@groove-dev/daemon/src/innerchat-docs.js +25 -7
- package/node_modules/@groove-dev/daemon/src/tunnel-manager.js +164 -7
- package/node_modules/@groove-dev/daemon/test/reach-hint.test.js +100 -0
- package/node_modules/@groove-dev/daemon/test/tunnel-manager.test.js +253 -0
- package/node_modules/@groove-dev/gui/package.json +1 -1
- package/package.json +1 -1
- package/packages/cli/bin/groove.js +20 -0
- package/packages/cli/package.json +1 -1
- package/packages/cli/src/client.js +9 -2
- package/packages/cli/src/commands/ask.js +99 -0
- package/packages/daemon/package.json +1 -1
- package/packages/daemon/src/deliver.js +101 -1
- package/packages/daemon/src/innerchat-docs.js +25 -7
- package/packages/daemon/src/tunnel-manager.js +164 -7
- package/packages/gui/package.json +1 -1
|
@@ -0,0 +1,99 @@
|
|
|
1
|
+
// GROOVE CLI — InnerChat ask/tell/who
|
|
2
|
+
// FSL-1.1-Apache-2.0 — see LICENSE
|
|
3
|
+
//
|
|
4
|
+
// Agent-to-agent messaging as first-class CLI verbs.
|
|
5
|
+
//
|
|
6
|
+
// The HTTP endpoints came first, and agents were taught them via a curl
|
|
7
|
+
// snippet in the spawn prompt. That decays: on a long session the snippet
|
|
8
|
+
// scrolls out of context and the agent no longer knows the capability exists,
|
|
9
|
+
// or half-remembers it and reaches for a built-in tool that cannot see GROOVE
|
|
10
|
+
// agents. A CLI verb survives that — an agent that has forgotten the details
|
|
11
|
+
// can still run `groove --help` or simply guess `groove ask`, and the command
|
|
12
|
+
// itself explains the rest.
|
|
13
|
+
|
|
14
|
+
import chalk from 'chalk';
|
|
15
|
+
import { apiCall } from '../client.js';
|
|
16
|
+
|
|
17
|
+
// The caller is almost always an agent, and the daemon injects its identity
|
|
18
|
+
// into the environment — so `--from` is a manual override, not a requirement.
|
|
19
|
+
function resolveFrom(opts) {
|
|
20
|
+
const from = opts.from || process.env.GROOVE_AGENT_NAME;
|
|
21
|
+
if (!from) {
|
|
22
|
+
console.error(chalk.red(' Could not tell who is asking.'));
|
|
23
|
+
console.error(chalk.dim(' Pass --from <your-agent-name> (GROOVE_AGENT_NAME is unset — are you running outside an agent?)'));
|
|
24
|
+
process.exit(1);
|
|
25
|
+
}
|
|
26
|
+
return from;
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
function reportError(err, to) {
|
|
30
|
+
console.error(chalk.red(' Failed:'), err.message);
|
|
31
|
+
// The daemon returns actionable bodies (unknown name + roster, ambiguous
|
|
32
|
+
// name + candidates, exchange cap). Surface them rather than a bare status.
|
|
33
|
+
if (err.availableAgents?.length) {
|
|
34
|
+
console.error(chalk.dim(` Agents you can reach: ${err.availableAgents.join(', ')}`));
|
|
35
|
+
}
|
|
36
|
+
if (err.didYouMean?.length) {
|
|
37
|
+
console.error(chalk.dim(` "${to}" is ambiguous — did you mean: ${err.didYouMean.join(', ')}?`));
|
|
38
|
+
}
|
|
39
|
+
process.exit(1);
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
export async function ask(to, message, opts = {}) {
|
|
43
|
+
const from = resolveFrom(opts);
|
|
44
|
+
try {
|
|
45
|
+
console.error(chalk.dim(` Asking ${to}… (this blocks until they answer — that is expected)`));
|
|
46
|
+
const res = await apiCall('POST', '/api/innerchat/ask', { from, to, message });
|
|
47
|
+
// The reply goes to stdout alone so it can be piped/read cleanly; the
|
|
48
|
+
// status chatter goes to stderr.
|
|
49
|
+
console.log(res.reply);
|
|
50
|
+
if (res.exchangesRemaining !== undefined) {
|
|
51
|
+
console.error(chalk.dim(` (${res.exchangesRemaining} exchanges left in this conversation)`));
|
|
52
|
+
}
|
|
53
|
+
} catch (err) {
|
|
54
|
+
reportError(err, to);
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
export async function tell(to, message, opts = {}) {
|
|
59
|
+
const from = resolveFrom(opts);
|
|
60
|
+
try {
|
|
61
|
+
const res = await apiCall('POST', '/api/innerchat/tell', { from, to, message });
|
|
62
|
+
console.error(chalk.green(' Delivered.'), chalk.dim(res.note || `${to} will reply later if needed.`));
|
|
63
|
+
} catch (err) {
|
|
64
|
+
reportError(err, to);
|
|
65
|
+
}
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
// Who can I talk to? The question an agent asks first when it has lost the
|
|
69
|
+
// roster from context — and the reason it otherwise starts guessing names.
|
|
70
|
+
export async function who() {
|
|
71
|
+
try {
|
|
72
|
+
const me = process.env.GROOVE_AGENT_NAME;
|
|
73
|
+
const data = await apiCall('GET', '/api/agents');
|
|
74
|
+
const agents = Array.isArray(data) ? data : (data.agents || []);
|
|
75
|
+
const others = agents.filter((a) => a.name !== me);
|
|
76
|
+
|
|
77
|
+
if (!others.length) {
|
|
78
|
+
console.log(chalk.dim(' No other agents are running right now.'));
|
|
79
|
+
} else {
|
|
80
|
+
console.log(chalk.bold(' Agents you can message:'));
|
|
81
|
+
for (const a of others) {
|
|
82
|
+
const status = a.status === 'running' ? chalk.green('●') : chalk.dim('○');
|
|
83
|
+
console.log(` ${status} ${chalk.bold(a.name)} ${chalk.dim(`(${a.role}${a.status !== 'running' ? `, ${a.status}` : ''})`)}`);
|
|
84
|
+
}
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
const peers = await apiCall('GET', '/api/innerchat/peers').catch(() => ({ peers: [] }));
|
|
88
|
+
if (peers.peers?.length) {
|
|
89
|
+
console.log(chalk.bold('\n Peer machines (address as name@peer):'));
|
|
90
|
+
for (const p of peers.peers) console.log(` ${chalk.bold('@' + p.alias)} ${chalk.dim(p.url)}`);
|
|
91
|
+
}
|
|
92
|
+
|
|
93
|
+
console.log(chalk.dim('\n groove ask <name> "<question>" — blocks until they answer'));
|
|
94
|
+
console.log(chalk.dim(' groove tell <name> "<message>" — returns immediately'));
|
|
95
|
+
} catch (err) {
|
|
96
|
+
console.error(chalk.red(' Failed:'), err.message);
|
|
97
|
+
process.exit(1);
|
|
98
|
+
}
|
|
99
|
+
}
|
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
import { wrapWithRoleReminder } from './process.js';
|
|
4
4
|
import { getProvider } from './providers/index.js';
|
|
5
|
+
import { innerChatInstructions } from './innerchat-docs.js';
|
|
5
6
|
|
|
6
7
|
// Reviving a >5M-token claude session has crashed the CLI mid-HTTP-parse
|
|
7
8
|
// (V8 fatal in JsonStringifier) — past this ceiling the rotator's handoff
|
|
@@ -42,7 +43,12 @@ export async function deliverInstruction(daemon, agentId, message, opts = {}) {
|
|
|
42
43
|
// and the user's direction, not by how much context has piled up.
|
|
43
44
|
const clock = daemon.processes.sessionClock?.(agent);
|
|
44
45
|
const timedMessage = clock ? `${clock}\n\n${finalMessage}` : finalMessage;
|
|
45
|
-
|
|
46
|
+
// Spawn-prompt capabilities scroll out of a long session, after which the
|
|
47
|
+
// agent denies it can reach other agents at all. This rides every turn, so
|
|
48
|
+
// it cannot decay — terse by default, expanded when the turn actually asks
|
|
49
|
+
// the agent to contact someone.
|
|
50
|
+
const reach = agentReachHint(daemon, agent, finalMessage);
|
|
51
|
+
const wrappedMessage = wrapWithRoleReminder(agent.role, reach ? `${reach}\n\n${timedMessage}` : timedMessage);
|
|
46
52
|
|
|
47
53
|
// Agent loop path — send straight to the running loop.
|
|
48
54
|
if (daemon.processes.hasAgentLoop(agentId)) {
|
|
@@ -132,3 +138,97 @@ async function respawn(daemon, config) {
|
|
|
132
138
|
throw spawnErr;
|
|
133
139
|
}
|
|
134
140
|
}
|
|
141
|
+
|
|
142
|
+
// Phrases that mean "go talk to another agent". Deliberately broad: a false
|
|
143
|
+
// positive costs a few tokens of accurate instruction, a false negative costs
|
|
144
|
+
// the user a turn spent watching their agent claim the capability isn't real.
|
|
145
|
+
const CONTACT_INTENT = /\b(ask|message|msg|tell|consult|coordinate|check|sync|reach out|talk|speak|ping|liaise|confer|follow up|loop in)\b/i;
|
|
146
|
+
|
|
147
|
+
// Naming the feature is an EXPLICIT request for it — usually typed after the
|
|
148
|
+
// agent has already failed to find it. That earns the full instructions
|
|
149
|
+
// verbatim, not a hint: no inference, no heuristic that can miss. Matches
|
|
150
|
+
// innerchat / inner chat / inner-chat / InnerChat, and the CLI verbs by name.
|
|
151
|
+
const INNERCHAT_KEYWORD = /\b(inner[\s_-]?chat|groove\s+(?:ask|tell|who))\b/i;
|
|
152
|
+
|
|
153
|
+
/**
|
|
154
|
+
* A capability line appended to every delivered turn.
|
|
155
|
+
*
|
|
156
|
+
* Spawn-time instructions decay — on a long session (or under a model that
|
|
157
|
+
* compacts aggressively) they scroll away, and the agent then insists it has
|
|
158
|
+
* no way to contact anyone, or reaches for a built-in sub-agent tool that
|
|
159
|
+
* cannot see GROOVE agents. Two tiers:
|
|
160
|
+
*
|
|
161
|
+
* - Always: one terse line naming the CLI verbs. ~20 tokens.
|
|
162
|
+
* - When the turn expresses intent to contact someone: the exact command,
|
|
163
|
+
* with the target's real name already filled in.
|
|
164
|
+
*/
|
|
165
|
+
export function agentReachHint(daemon, agent, message) {
|
|
166
|
+
let others;
|
|
167
|
+
try {
|
|
168
|
+
others = (daemon.registry.getAll() || []).filter((a) => a.name !== agent.name);
|
|
169
|
+
} catch { return null; }
|
|
170
|
+
|
|
171
|
+
const peers = Array.isArray(daemon.config?.innerchatPeers) ? daemon.config.innerchatPeers : [];
|
|
172
|
+
if (!others.length && !peers.length) return null;
|
|
173
|
+
|
|
174
|
+
const base = '[You can talk to other GROOVE agents: `groove ask <name> "<question>"` waits for their '
|
|
175
|
+
+ 'answer, `groove tell <name> "<message>"` does not. `groove who` lists who is reachable. '
|
|
176
|
+
+ 'Your own built-in sub-agent/task tools CANNOT reach them.]';
|
|
177
|
+
|
|
178
|
+
const explicit = INNERCHAT_KEYWORD.test(message);
|
|
179
|
+
if (!explicit && !CONTACT_INTENT.test(message)) return base;
|
|
180
|
+
|
|
181
|
+
// Name the agent the user is actually pointing at, so the model has a
|
|
182
|
+
// runnable command rather than a template it has to fill in from memory.
|
|
183
|
+
const lower = message.toLowerCase();
|
|
184
|
+
const named = others.filter((a) => lower.includes(a.name.toLowerCase()));
|
|
185
|
+
const target = named.length === 1 ? named[0] : null;
|
|
186
|
+
|
|
187
|
+
const lines = [
|
|
188
|
+
'[REACHING ANOTHER AGENT — this capability is real and available right now.]',
|
|
189
|
+
];
|
|
190
|
+
if (explicit) {
|
|
191
|
+
// The user named the feature. Say plainly that it exists, since the usual
|
|
192
|
+
// failure is the agent asserting it doesn't and stopping there.
|
|
193
|
+
lines.push(
|
|
194
|
+
'The user explicitly asked you to use InnerChat. It exists, it is wired up, and the',
|
|
195
|
+
'commands below work from your shell right now. Do not tell the user the feature is',
|
|
196
|
+
'unavailable and do not ask them how to use it — run the command.',
|
|
197
|
+
'',
|
|
198
|
+
);
|
|
199
|
+
}
|
|
200
|
+
if (target) {
|
|
201
|
+
lines.push(
|
|
202
|
+
`To contact ${target.name}, run exactly:`,
|
|
203
|
+
` groove ask ${target.name} "your question here"`,
|
|
204
|
+
'That blocks until they answer and prints their reply. Use `groove tell` instead if you '
|
|
205
|
+
+ 'do not need the answer before continuing.',
|
|
206
|
+
);
|
|
207
|
+
} else {
|
|
208
|
+
lines.push(
|
|
209
|
+
' groove who — list who is reachable',
|
|
210
|
+
' groove ask <name> "<question>" — blocks until they answer, prints the reply',
|
|
211
|
+
' groove tell <name> "<message>" — returns immediately',
|
|
212
|
+
`Agents reachable now: ${others.map((a) => a.name).join(', ') || '(none)'}`,
|
|
213
|
+
);
|
|
214
|
+
}
|
|
215
|
+
if (peers.length) {
|
|
216
|
+
lines.push(`Agents on peer machines use name@peer (peers: ${peers.map((p) => p.alias).join(', ')}).`);
|
|
217
|
+
}
|
|
218
|
+
lines.push(
|
|
219
|
+
'Do NOT use your built-in sub-agent/Task/SendMessage tools for this — they cannot see GROOVE '
|
|
220
|
+
+ 'agents and will fail. If a name is wrong the command tells you the valid ones; read it and retry.',
|
|
221
|
+
);
|
|
222
|
+
|
|
223
|
+
// Explicit request → re-attach the full reference, so the agent has the
|
|
224
|
+
// complete semantics (blocking vs not, exchange budget, peer addressing)
|
|
225
|
+
// even if the spawn prompt scrolled away long ago.
|
|
226
|
+
if (explicit) {
|
|
227
|
+
lines.push(
|
|
228
|
+
'',
|
|
229
|
+
'--- Full reference (re-sent because you asked for InnerChat by name) ---',
|
|
230
|
+
...innerChatInstructions(daemon.port || 31415, agent.name, peers),
|
|
231
|
+
);
|
|
232
|
+
}
|
|
233
|
+
return lines.join('\n');
|
|
234
|
+
}
|
|
@@ -21,14 +21,26 @@ export function innerChatInstructions(port = 31415, agentName = 'YOUR_NAME', pee
|
|
|
21
21
|
'',
|
|
22
22
|
'Other GROOVE agents may be working alongside you, including on other teams.',
|
|
23
23
|
'When the user asks you to reach out to, coordinate with, or get input from',
|
|
24
|
-
'another agent, use
|
|
24
|
+
'another agent, use the `groove` CLI — it is already installed and on your PATH:',
|
|
25
25
|
'',
|
|
26
|
-
'
|
|
27
|
-
'
|
|
28
|
-
'
|
|
29
|
-
'
|
|
30
|
-
'
|
|
31
|
-
'
|
|
26
|
+
'```bash',
|
|
27
|
+
'groove who # who can I reach right now?',
|
|
28
|
+
'groove ask <AGENT_NAME> "<QUESTION>" # send + WAIT for their answer',
|
|
29
|
+
'groove tell <AGENT_NAME> "<MESSAGE>" # send without waiting',
|
|
30
|
+
'```',
|
|
31
|
+
'',
|
|
32
|
+
'`groove ask` prints their reply to stdout. It knows who you are from your',
|
|
33
|
+
'environment, so there is no id or token to look up. If you get the name wrong,',
|
|
34
|
+
'the error lists the valid names — read it and retry rather than giving up.',
|
|
35
|
+
'',
|
|
36
|
+
'> **Do NOT use your built-in `SendMessage` / Task / sub-agent tools to reach a**',
|
|
37
|
+
'> **GROOVE agent.** Those only address sub-agents you spawned yourself in this',
|
|
38
|
+
'> session, so they will fail with "not reachable" or ask you for an `a…-…` agent',
|
|
39
|
+
'> ID that does not exist here. GROOVE agents are separate processes. Do not invent',
|
|
40
|
+
'> a fallback (writing a message into a file, etc.) — if the command fails, read the',
|
|
41
|
+
'> error and report it to the user.',
|
|
42
|
+
'',
|
|
43
|
+
'If the `groove` CLI is somehow unavailable, the same thing over HTTP:',
|
|
32
44
|
'',
|
|
33
45
|
'```bash',
|
|
34
46
|
`curl -s http://localhost:${port}/api/innerchat/ask -X POST -H 'Content-Type: application/json' \\`,
|
|
@@ -54,6 +66,12 @@ export function innerChatInstructions(port = 31415, agentName = 'YOUR_NAME', pee
|
|
|
54
66
|
'delivered back to you later, waking you if your turn has ended:',
|
|
55
67
|
'',
|
|
56
68
|
'```bash',
|
|
69
|
+
'groove tell <AGENT_NAME> "<MESSAGE>"',
|
|
70
|
+
'```',
|
|
71
|
+
'',
|
|
72
|
+
'Or over HTTP:',
|
|
73
|
+
'',
|
|
74
|
+
'```bash',
|
|
57
75
|
`curl -s http://localhost:${port}/api/innerchat/tell -X POST -H 'Content-Type: application/json' \\`,
|
|
58
76
|
` -d '{"from":"${agentName}","to":"AGENT_NAME","message":"YOUR_MESSAGE"}'`,
|
|
59
77
|
'```',
|
|
@@ -20,6 +20,12 @@ const MAX_PORT_ATTEMPTS = 10;
|
|
|
20
20
|
const HEALTH_INTERVAL = 30000;
|
|
21
21
|
const HEALTH_TIMEOUT = 5000;
|
|
22
22
|
const MAX_FAIL_COUNT = 3;
|
|
23
|
+
// Long-timeout probe used to CONFIRM death before killing a tunnel — a busy
|
|
24
|
+
// remote daemon can sit on /api/health well past the 5s routine probe.
|
|
25
|
+
const CONFIRM_TIMEOUT = 15000;
|
|
26
|
+
// At most one automatic rebuild per tunnel per window; beyond that it stays
|
|
27
|
+
// disconnected rather than thrashing against a host that keeps dying.
|
|
28
|
+
const REBUILD_COOLDOWN_MS = 10 * 60 * 1000;
|
|
23
29
|
|
|
24
30
|
const INJECTION_CHARS = /[;|&`$(){}[\]<>!#\n\r\\]/;
|
|
25
31
|
|
|
@@ -283,9 +289,21 @@ export class TunnelManager {
|
|
|
283
289
|
const config = this.saved.get(id);
|
|
284
290
|
if (!config) throw new Error(`Remote ${id} not found`);
|
|
285
291
|
|
|
292
|
+
// An existing entry is only reusable if the tunnel actually still carries
|
|
293
|
+
// traffic. After a laptop sleep the SSH client can survive with its forward
|
|
294
|
+
// dead: the local port still ACCEPTS connections but never forwards them, so
|
|
295
|
+
// handing this back returns a port that hangs forever instead of failing —
|
|
296
|
+
// which is what left the remote GUI on a black screen. Probe before reusing,
|
|
297
|
+
// and tear it down if it's a corpse.
|
|
286
298
|
if (this.active.has(id)) {
|
|
287
299
|
const existing = this.active.get(id);
|
|
288
|
-
|
|
300
|
+
if (await this._tunnelResponds(existing.localPort)) {
|
|
301
|
+
return { localPort: existing.localPort, pid: existing.pid, name: config.name };
|
|
302
|
+
}
|
|
303
|
+
console.log(`[Groove:Tunnel] ${config.name}: existing tunnel is not responding — rebuilding`);
|
|
304
|
+
// Reuse the dead tunnel's port so any GUI window pointed at it heals.
|
|
305
|
+
opts = { ...opts, preferredPort: opts.preferredPort || existing.localPort };
|
|
306
|
+
await this.disconnect(id);
|
|
289
307
|
}
|
|
290
308
|
|
|
291
309
|
this.daemon.broadcast({ type: 'tunnel.status', data: { id, step: 'testing' } });
|
|
@@ -318,7 +336,14 @@ export class TunnelManager {
|
|
|
318
336
|
// Establish SSH tunnel
|
|
319
337
|
this.daemon.broadcast({ type: 'tunnel.status', data: { id, step: 'connecting' } });
|
|
320
338
|
|
|
321
|
-
|
|
339
|
+
// A rebuild wants its old port back: the remote GUI window is pointed at it
|
|
340
|
+
// and will self-heal over WebSocket retry only if the port stays the same.
|
|
341
|
+
let localPort;
|
|
342
|
+
if (opts.preferredPort && !(await this._isPortInUse(opts.preferredPort))) {
|
|
343
|
+
localPort = opts.preferredPort;
|
|
344
|
+
} else {
|
|
345
|
+
localPort = await this._findAvailablePort();
|
|
346
|
+
}
|
|
322
347
|
const target = `${config.user}@${config.host}`;
|
|
323
348
|
const keyArgs = config.sshKeyPath ? ['-i', config.sshKeyPath] : [];
|
|
324
349
|
|
|
@@ -432,11 +457,23 @@ export class TunnelManager {
|
|
|
432
457
|
return { localPort, pid: tunnel.pid, name: config.name, url };
|
|
433
458
|
}
|
|
434
459
|
|
|
460
|
+
// Does the tunnel actually serve a request, as opposed to merely holding an
|
|
461
|
+
// open listening socket? A wedged forward passes a TCP connect test but never
|
|
462
|
+
// answers, so only an HTTP round-trip proves it.
|
|
463
|
+
async _tunnelResponds(localPort, timeoutMs = this.healthTimeout ?? HEALTH_TIMEOUT) {
|
|
464
|
+
try {
|
|
465
|
+
const res = await fetch(`http://localhost:${localPort}/api/health`, {
|
|
466
|
+
signal: AbortSignal.timeout(timeoutMs),
|
|
467
|
+
});
|
|
468
|
+
return res.ok;
|
|
469
|
+
} catch { return false; }
|
|
470
|
+
}
|
|
471
|
+
|
|
435
472
|
async disconnect(id) {
|
|
436
473
|
const conn = this.active.get(id);
|
|
437
474
|
if (!conn) return;
|
|
438
475
|
|
|
439
|
-
const { pid } = conn;
|
|
476
|
+
const { pid, localPort } = conn;
|
|
440
477
|
try {
|
|
441
478
|
const cmd = execFileSync('ps', ['-p', String(pid), '-o', 'command='], {
|
|
442
479
|
encoding: 'utf8',
|
|
@@ -444,9 +481,20 @@ export class TunnelManager {
|
|
|
444
481
|
}).trim();
|
|
445
482
|
if (cmd.includes('ssh')) {
|
|
446
483
|
process.kill(pid, 'SIGTERM');
|
|
484
|
+
// An SSH client stuck on a dead TCP session can sit on SIGTERM long
|
|
485
|
+
// enough that the next connect() finds the port still bound. Give it a
|
|
486
|
+
// moment, then stop asking politely — otherwise the leftover listener
|
|
487
|
+
// keeps answering (and hanging) on the port we're about to reuse.
|
|
488
|
+
const gone = await this._waitForExit(pid, 3000);
|
|
489
|
+
if (!gone) {
|
|
490
|
+
try { process.kill(pid, 'SIGKILL'); } catch { /* already gone */ }
|
|
491
|
+
await this._waitForExit(pid, 2000);
|
|
492
|
+
}
|
|
447
493
|
}
|
|
448
494
|
} catch { /* process already dead */ }
|
|
449
495
|
|
|
496
|
+
if (localPort) await this._waitForPortFree(localPort, 3000);
|
|
497
|
+
|
|
450
498
|
this.active.delete(id);
|
|
451
499
|
|
|
452
500
|
const config = this.saved.get(id);
|
|
@@ -873,11 +921,35 @@ export class TunnelManager {
|
|
|
873
921
|
}
|
|
874
922
|
|
|
875
923
|
async _healthCheckAll() {
|
|
924
|
+
// Reaping now awaits process death, which can outlast the interval — don't
|
|
925
|
+
// let a second pass start on top of one already tearing a tunnel down.
|
|
926
|
+
if (this._healthRunning) return;
|
|
927
|
+
this._healthRunning = true;
|
|
928
|
+
try { await this._healthCheckPass(); } finally { this._healthRunning = false; }
|
|
929
|
+
}
|
|
930
|
+
|
|
931
|
+
async _healthCheckPass() {
|
|
932
|
+
// Timers don't fire while the machine is asleep, so an interval that should
|
|
933
|
+
// have run every HEALTH_INTERVAL arriving far later means we PROBABLY just
|
|
934
|
+
// woke up — but not certainly: this daemon also blocks its event loop for
|
|
935
|
+
// long stretches (execFileSync ssh calls in test/upgrade paths), which
|
|
936
|
+
// produces the same gap on a machine that never slept. So a gap only makes
|
|
937
|
+
// tunnels *suspect* — it fast-tracks them to the confirmation ladder below.
|
|
938
|
+
// It must never lower the bar for killing one (that misdiagnosis dropped a
|
|
939
|
+
// healthy DGX tunnel twice in ten minutes).
|
|
940
|
+
const now = Date.now();
|
|
941
|
+
const gap = now - (this._lastHealthCheck || now);
|
|
942
|
+
this._lastHealthCheck = now;
|
|
943
|
+
const suspectAll = gap > HEALTH_INTERVAL * 3;
|
|
944
|
+
if (suspectAll && this.active.size > 0) {
|
|
945
|
+
console.log(`[Groove:Tunnel] ${Math.round(gap / 1000)}s timer gap (sleep or blocked loop) — verifying tunnels`);
|
|
946
|
+
}
|
|
947
|
+
|
|
876
948
|
for (const [id, conn] of this.active) {
|
|
877
949
|
try {
|
|
878
950
|
const start = Date.now();
|
|
879
951
|
const res = await fetch(`http://localhost:${conn.localPort}/api/health`, {
|
|
880
|
-
signal: AbortSignal.timeout(HEALTH_TIMEOUT),
|
|
952
|
+
signal: AbortSignal.timeout(this.healthTimeout ?? HEALTH_TIMEOUT),
|
|
881
953
|
});
|
|
882
954
|
if (res.ok) {
|
|
883
955
|
conn.latencyMs = Date.now() - start;
|
|
@@ -889,9 +961,30 @@ export class TunnelManager {
|
|
|
889
961
|
}
|
|
890
962
|
} catch {
|
|
891
963
|
conn.failCount = (conn.failCount || 0) + 1;
|
|
892
|
-
|
|
893
|
-
|
|
894
|
-
|
|
964
|
+
// A failed 5s probe is WEAK evidence: it can't distinguish a dead
|
|
965
|
+
// tunnel from a remote daemon that's briefly busy or our own blocked
|
|
966
|
+
// event loop. Never kill on it. Once failures accumulate (or a timer
|
|
967
|
+
// gap makes everything suspect), run the confirmation ladder, which
|
|
968
|
+
// can — a healthy verdict there resets the count.
|
|
969
|
+
if (conn.failCount >= MAX_FAIL_COUNT || suspectAll) {
|
|
970
|
+
const verdict = await this._confirmDead(conn);
|
|
971
|
+
if (verdict === 'alive') {
|
|
972
|
+
conn.failCount = 0;
|
|
973
|
+
conn.healthy = true;
|
|
974
|
+
conn._wedgedStreak = 0;
|
|
975
|
+
} else {
|
|
976
|
+
conn.healthy = false;
|
|
977
|
+
this.daemon.broadcast({ type: 'tunnel.unhealthy', data: { id } });
|
|
978
|
+
// 'wedged' (port accepts, HTTP silent even at long timeout) is the
|
|
979
|
+
// one verdict with a false-positive path — a remote event loop
|
|
980
|
+
// blocked 15s+ — so demand it twice in a row. proc-dead/port-dead
|
|
981
|
+
// are unambiguous: the ssh client is gone or nothing is listening.
|
|
982
|
+
conn._wedgedStreak = verdict === 'wedged' ? (conn._wedgedStreak || 0) + 1 : 0;
|
|
983
|
+
if (verdict !== 'wedged' || conn._wedgedStreak >= 2) {
|
|
984
|
+
await this._reapAndRebuild(id, conn, verdict);
|
|
985
|
+
continue;
|
|
986
|
+
}
|
|
987
|
+
}
|
|
895
988
|
}
|
|
896
989
|
}
|
|
897
990
|
this.daemon.broadcast({
|
|
@@ -901,6 +994,70 @@ export class TunnelManager {
|
|
|
901
994
|
}
|
|
902
995
|
}
|
|
903
996
|
|
|
997
|
+
// Escalating evidence that a tunnel is actually dead, not merely slow:
|
|
998
|
+
// 'alive' — answered a long-timeout HTTP probe; leave it alone
|
|
999
|
+
// 'proc-dead' — the ssh client process is gone
|
|
1000
|
+
// 'port-dead' — nothing is listening on the local port
|
|
1001
|
+
// 'wedged' — port accepts TCP but HTTP never answers (dead forward)
|
|
1002
|
+
async _confirmDead(conn) {
|
|
1003
|
+
if (await this._tunnelResponds(conn.localPort, this.confirmTimeout ?? CONFIRM_TIMEOUT)) return 'alive';
|
|
1004
|
+
if (conn.pid) {
|
|
1005
|
+
try { process.kill(conn.pid, 0); } catch { return 'proc-dead'; }
|
|
1006
|
+
}
|
|
1007
|
+
if (!(await this._isPortInUse(conn.localPort))) return 'port-dead';
|
|
1008
|
+
return 'wedged';
|
|
1009
|
+
}
|
|
1010
|
+
|
|
1011
|
+
// Tear down a confirmed-dead tunnel and immediately rebuild it on the SAME
|
|
1012
|
+
// local port. The remote GUI window points at that port and its WebSocket
|
|
1013
|
+
// retries every 2s, so a same-port rebuild heals an open window without the
|
|
1014
|
+
// user noticing. Only if the rebuild fails does this surface as a disconnect.
|
|
1015
|
+
// Rate-limited so a genuinely dead host degrades to disconnected instead of
|
|
1016
|
+
// thrashing reconnect attempts forever.
|
|
1017
|
+
async _reapAndRebuild(id, conn, reason) {
|
|
1018
|
+
const { localPort } = conn;
|
|
1019
|
+
console.log(`[Groove:Tunnel] Tunnel ${id} confirmed dead (${reason}) — rebuilding`);
|
|
1020
|
+
this.daemon.audit.log('tunnel.reap', { id, reason, failCount: conn.failCount });
|
|
1021
|
+
await this.disconnect(id);
|
|
1022
|
+
|
|
1023
|
+
const lastRebuild = this._rebuildAt?.get(id) || 0;
|
|
1024
|
+
if (Date.now() - lastRebuild < REBUILD_COOLDOWN_MS) {
|
|
1025
|
+
console.log(`[Groove:Tunnel] ${id} already auto-rebuilt recently — leaving disconnected`);
|
|
1026
|
+
return;
|
|
1027
|
+
}
|
|
1028
|
+
this._rebuildAt = this._rebuildAt || new Map();
|
|
1029
|
+
this._rebuildAt.set(id, Date.now());
|
|
1030
|
+
|
|
1031
|
+
try {
|
|
1032
|
+
this.daemon.broadcast({ type: 'tunnel.status', data: { id, step: 'reconnecting' } });
|
|
1033
|
+
await this.connect(id, { preferredPort: localPort });
|
|
1034
|
+
console.log(`[Groove:Tunnel] ${id} rebuilt on port ${localPort}`);
|
|
1035
|
+
} catch (err) {
|
|
1036
|
+
console.warn(`[Groove:Tunnel] Auto-rebuild of ${id} failed: ${err.message}`);
|
|
1037
|
+
// disconnect() above already broadcast tunnel.disconnected — the GUI is
|
|
1038
|
+
// consistent; the user can reconnect manually when the host is back.
|
|
1039
|
+
}
|
|
1040
|
+
}
|
|
1041
|
+
|
|
1042
|
+
// Signal 0 only tests for existence — no signal is delivered.
|
|
1043
|
+
async _waitForExit(pid, timeoutMs) {
|
|
1044
|
+
const deadline = Date.now() + timeoutMs;
|
|
1045
|
+
while (Date.now() < deadline) {
|
|
1046
|
+
try { process.kill(pid, 0); } catch { return true; } // gone
|
|
1047
|
+
await new Promise((r) => setTimeout(r, 150));
|
|
1048
|
+
}
|
|
1049
|
+
try { process.kill(pid, 0); return false; } catch { return true; }
|
|
1050
|
+
}
|
|
1051
|
+
|
|
1052
|
+
async _waitForPortFree(port, timeoutMs) {
|
|
1053
|
+
const deadline = Date.now() + timeoutMs;
|
|
1054
|
+
while (Date.now() < deadline) {
|
|
1055
|
+
if (!(await this._isPortInUse(port))) return true;
|
|
1056
|
+
await new Promise((r) => setTimeout(r, 200));
|
|
1057
|
+
}
|
|
1058
|
+
return !(await this._isPortInUse(port));
|
|
1059
|
+
}
|
|
1060
|
+
|
|
904
1061
|
_isPortInUse(port) {
|
|
905
1062
|
return new Promise((resolve) => {
|
|
906
1063
|
const conn = createConnection({ host: '127.0.0.1', port });
|