groove-dev 0.27.187 → 0.27.189

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (42) hide show
  1. package/innerchat/Screenshot_2026-07-23_at_1.30.16_PM.png +0 -0
  2. package/node_modules/@groove-dev/cli/package.json +1 -1
  3. package/node_modules/@groove-dev/daemon/package.json +1 -1
  4. package/node_modules/@groove-dev/daemon/src/index.js +6 -0
  5. package/node_modules/@groove-dev/daemon/src/innerchat-docs.js +18 -13
  6. package/node_modules/@groove-dev/daemon/src/process.js +10 -0
  7. package/node_modules/@groove-dev/daemon/src/teams.js +36 -4
  8. package/node_modules/@groove-dev/daemon/src/watcher.js +230 -104
  9. package/node_modules/@groove-dev/daemon/test/teams.test.js +48 -1
  10. package/node_modules/@groove-dev/daemon/test/watcher.test.js +68 -10
  11. package/node_modules/@groove-dev/gui/dist/assets/index-Cat5pJUx.css +1 -0
  12. package/node_modules/@groove-dev/gui/dist/assets/{index-DyI84i9Y.js → index-DTY0KsH9.js} +222 -222
  13. package/node_modules/@groove-dev/gui/dist/index.html +2 -2
  14. package/node_modules/@groove-dev/gui/package.json +1 -1
  15. package/node_modules/@groove-dev/gui/src/App.jsx +51 -2
  16. package/node_modules/@groove-dev/gui/src/components/agents/agent-feed.jsx +11 -2
  17. package/node_modules/@groove-dev/gui/src/components/agents/agent-panel.jsx +106 -10
  18. package/node_modules/@groove-dev/gui/src/lib/logpaths.js +1 -1
  19. package/node_modules/@groove-dev/gui/src/stores/groove.js +19 -6
  20. package/node_modules/@groove-dev/gui/src/stores/helpers.js +28 -0
  21. package/node_modules/@groove-dev/gui/src/stores/slices/agents-slice.js +20 -9
  22. package/package.json +1 -1
  23. package/packages/cli/package.json +1 -1
  24. package/packages/daemon/package.json +1 -1
  25. package/packages/daemon/src/index.js +6 -0
  26. package/packages/daemon/src/innerchat-docs.js +18 -13
  27. package/packages/daemon/src/process.js +10 -0
  28. package/packages/daemon/src/teams.js +36 -4
  29. package/packages/daemon/src/watcher.js +230 -104
  30. package/packages/gui/dist/assets/index-Cat5pJUx.css +1 -0
  31. package/packages/gui/dist/assets/{index-DyI84i9Y.js → index-DTY0KsH9.js} +222 -222
  32. package/packages/gui/dist/index.html +2 -2
  33. package/packages/gui/package.json +1 -1
  34. package/packages/gui/src/App.jsx +51 -2
  35. package/packages/gui/src/components/agents/agent-feed.jsx +11 -2
  36. package/packages/gui/src/components/agents/agent-panel.jsx +106 -10
  37. package/packages/gui/src/lib/logpaths.js +1 -1
  38. package/packages/gui/src/stores/groove.js +19 -6
  39. package/packages/gui/src/stores/helpers.js +28 -0
  40. package/packages/gui/src/stores/slices/agents-slice.js +20 -9
  41. package/node_modules/@groove-dev/gui/dist/assets/index-CU8L_r5f.css +0 -1
  42. package/packages/gui/dist/assets/index-CU8L_r5f.css +0 -1
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@groove-dev/cli",
3
- "version": "0.27.187",
3
+ "version": "0.27.189",
4
4
  "description": "GROOVE CLI — manage AI coding agents from your terminal",
5
5
  "license": "FSL-1.1-Apache-2.0",
6
6
  "type": "module",
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@groove-dev/daemon",
3
- "version": "0.27.187",
3
+ "version": "0.27.189",
4
4
  "description": "GROOVE daemon — agent orchestration engine",
5
5
  "license": "FSL-1.1-Apache-2.0",
6
6
  "type": "module",
@@ -638,6 +638,12 @@ export class Daemon {
638
638
  }
639
639
  }
640
640
 
641
+ // Reconnect to watches whose detached jobs survived this restart, so a
642
+ // "notify me when the training finishes" watch still fires afterward.
643
+ try { this.watcher.restore(); } catch (err) {
644
+ console.error('[startup] Failed to restore watches:', err.message);
645
+ }
646
+
641
647
  // Restore auth token from stored config so subscription polling works after restart
642
648
  const storedToken = this.skills.getToken();
643
649
  if (storedToken) {
@@ -70,27 +70,32 @@ export function innerChatInstructions(port = 31415, agentName = 'YOUR_NAME') {
70
70
  export function watchInstructions(port = 31415, agentName = 'YOUR_NAME') {
71
71
  const nameHint = agentName === 'YOUR_NAME' ? ' ($GROOVE_AGENT_NAME)' : '';
72
72
  return [
73
- '## Getting Notified When Something Finishes (Watch)',
73
+ '## Running Long Jobs & Getting Notified (Watch)',
74
74
  '',
75
- 'Your process ends when your turn ends, so a test suite or long job you kicked',
76
- 'off has nothing to notify you telling the user "I\'ll report back when it lands"',
77
- 'does not work on its own; the session is gone. Instead, set a Watch. The daemon',
78
- 'outlives your turn and RESUMES you with the result when it finishes:',
75
+ '> **A job you start yourself dies when your turn ends.** Anything you launch in Bash ',
76
+ "> even backgrounded with `&`, `nohup`, or `disown` is a child of your session process.",
77
+ '> When your turn ends or your context rotates, that process is torn down and the job dies',
78
+ '> with it. Do NOT start a long-running job (training, build, server, benchmark) directly',
79
+ "> in Bash and expect it to keep running. It won't.",
80
+ '',
81
+ 'Instead, hand the command to a Watch. The DAEMON runs it fully detached — independent of',
82
+ 'your turn, surviving turn-end, context rotation, and even a daemon restart — and RESUMES',
83
+ 'you with the exit code and output when it finishes:',
79
84
  '',
80
85
  '```bash',
81
86
  `curl -s http://localhost:${port}/api/watch -X POST -H 'Content-Type: application/json' \\`,
82
- ` -d '{"agent":"${agentName}","label":"test suite","command":"npm test"}'`,
87
+ ` -d '{"agent":"${agentName}","label":"model training","command":"python train.py"}'`,
83
88
  '```',
84
89
  '',
85
- 'This returns immediately with a `watchId`. **End your turn** — you will be woken',
86
- 'with the exit code and output tail when the command finishes.',
90
+ 'This returns immediately with a `watchId`. **End your turn** — the job keeps running under',
91
+ 'the daemon and you will be woken with the result when it completes.',
87
92
  '',
88
- '- Prefer `command` (let the daemon run it) over launching it yourself that way',
89
- ' it captures the real exit code and output.',
90
- '- For something ALREADY running, poll instead with `until` (fires when the check',
91
- ` exits 0): \`-d '{"agent":"${agentName}","label":"server up","until":"curl -sf localhost:3000/health"}'\`.`,
93
+ '- Use `command` for anything that must outlive your turn. Let the daemon run it — do not',
94
+ ' launch it yourself and then watch it; a self-launched job is already doomed.',
95
+ '- Use `until` ONLY for something the daemon or another durable process is already running',
96
+ ` (not a Bash job you started): \`-d '{"agent":"${agentName}","label":"server up","until":"curl -sf localhost:3000/health"}'\`.`,
92
97
  `- \`agent\` must be your own name${nameHint}.`,
93
- '- Watches time out (default 30 min). One clear task per watch; do not watch trivial commands you could just run.',
98
+ '- Watches time out (default 30 min, up to 24h pass `timeoutMs` for long trainings).',
94
99
  '- Do not poll or sleep-loop waiting yourself — set the watch and end the turn. That is the whole point.',
95
100
  ];
96
101
  }
@@ -2628,6 +2628,9 @@ After fixing all issues, run tests (npm test) and build (npm run build) to verif
2628
2628
  // Let the aborted process finish tearing down before re-spawning; resume()
2629
2629
  // kills the current handle and re-registers under a new agent id.
2630
2630
  setTimeout(() => {
2631
+ // The user may have paused/killed during this 500ms window — stop()/kill()
2632
+ // clear the pending message to cancel exactly this in-flight retry.
2633
+ if (!this._pendingUserMessage.has(agentId)) return;
2631
2634
  this.resume(agentId, pending.message).catch((err) => {
2632
2635
  this.daemon.broadcast({
2633
2636
  type: 'agent:output',
@@ -3060,6 +3063,13 @@ After fixing all issues, run tests (npm test) and build (npm run build) to verif
3060
3063
  const handle = this.handles.get(agentId);
3061
3064
  if (!handle) return;
3062
3065
 
3066
+ // A user pause aborts the current turn, which the CLI reports as a dropped
3067
+ // turn (isError, apiDurationMs === 0). Clear the pending message FIRST so
3068
+ // _retryDroppedTurn no-ops — otherwise the stop is auto-retried and the
3069
+ // agent resumes, forcing the user to click Pause twice.
3070
+ this._pendingUserMessage.delete(agentId);
3071
+ this._retryAttemptCarry.delete(agentId);
3072
+
3063
3073
  const { proc, loop } = handle;
3064
3074
 
3065
3075
  if (loop) {
@@ -230,6 +230,21 @@ export class Teams {
230
230
  originalWorkingDir: team.workingDir,
231
231
  };
232
232
  writeFileSync(resolve(archivePath, 'metadata.json'), JSON.stringify(metadata, null, 2));
233
+ } else if (
234
+ team.workingDir &&
235
+ team.workingDir !== this.daemon.projectDir &&
236
+ existsSync(team.workingDir) &&
237
+ this._dirHasExternalOccupants(team.workingDir)
238
+ ) {
239
+ // A moved agent still lives here — archive metadata only, leave the
240
+ // directory in place so that agent keeps working.
241
+ console.log(`[Groove:Teams] Archiving ${team.name} metadata only: agents from other teams still use its directory`);
242
+ mkdirSync(archivePath, { recursive: true });
243
+ writeFileSync(resolve(archivePath, 'metadata.json'), JSON.stringify({
244
+ originalName: team.name, originalId: team.id, mode: team.mode || 'sandbox',
245
+ deletedAt: new Date().toISOString(), agentCount: agents.length,
246
+ originalWorkingDir: team.workingDir, directoryRetained: true,
247
+ }, null, 2));
233
248
  } else if (
234
249
  team.workingDir &&
235
250
  team.workingDir !== this.daemon.projectDir &&
@@ -281,10 +296,16 @@ export class Teams {
281
296
  team.workingDir !== this.daemon.projectDir &&
282
297
  existsSync(team.workingDir)
283
298
  ) {
284
- try {
285
- rmSync(team.workingDir, { recursive: true, force: true });
286
- } catch (err) {
287
- console.log(`[Groove:Teams] Failed to delete directory: ${err.message}`);
299
+ if (this._dirHasExternalOccupants(team.workingDir)) {
300
+ // An agent moved to another team still lives here keep the directory
301
+ // so it doesn't lose its working dir. The team record is still removed.
302
+ console.log(`[Groove:Teams] Keeping ${team.workingDir}: agents from other teams still use it`);
303
+ } else {
304
+ try {
305
+ rmSync(team.workingDir, { recursive: true, force: true });
306
+ } catch (err) {
307
+ console.log(`[Groove:Teams] Failed to delete directory: ${err.message}`);
308
+ }
288
309
  }
289
310
  }
290
311
 
@@ -292,6 +313,17 @@ export class Teams {
292
313
  return true;
293
314
  }
294
315
 
316
+ // True if an agent OUTSIDE this team still lives in the given directory —
317
+ // i.e. an agent that was moved to another team but kept its working dir here.
318
+ // Destroying the directory would break it, so callers skip the removal.
319
+ // Call after _killAndRemoveAgents so this team's own agents are already gone.
320
+ _dirHasExternalOccupants(dir) {
321
+ if (!dir) return false;
322
+ const prefix = `${dir}/`;
323
+ return this.daemon.registry.getAll().some((a) =>
324
+ a.workingDir && (a.workingDir === dir || a.workingDir.startsWith(prefix)));
325
+ }
326
+
295
327
  _killAndRemoveAgents(teamId) {
296
328
  const agents = this.daemon.registry.getAll().filter((a) => a.teamId === teamId);
297
329
  for (const agent of agents) {
@@ -2,36 +2,52 @@
2
2
 
3
3
  import { spawn } from 'child_process';
4
4
  import { randomUUID } from 'crypto';
5
+ import {
6
+ mkdirSync, writeFileSync, readFileSync, existsSync, rmSync,
7
+ openSync, readSync, fstatSync, closeSync,
8
+ } from 'fs';
9
+ import { resolve } from 'path';
5
10
  import { deliverInstruction } from './deliver.js';
6
11
 
7
12
  // Watches are bounded so a wedged process or a condition that never comes true
8
13
  // can't poll forever or hold a slot indefinitely.
9
14
  const DEFAULT_TIMEOUT_MS = 30 * 60 * 1000;
10
- const MAX_TIMEOUT_MS = 6 * 60 * 60 * 1000;
15
+ const MAX_TIMEOUT_MS = 24 * 60 * 60 * 1000; // training can run for hours
11
16
  const DEFAULT_POLL_MS = 15 * 1000;
12
17
  const MIN_POLL_MS = 3 * 1000;
18
+ // Command mode just stats a sentinel file — nearly free, so poll it fast for a
19
+ // responsive wake. `until` mode spawns a shell each tick, hence the slower floor.
20
+ const COMMAND_POLL_MS = 500;
13
21
  const MAX_WATCHES_PER_AGENT = 5;
14
22
  const OUTPUT_TAIL = 4000;
23
+ const HISTORY_TTL_MS = 24 * 60 * 60 * 1000; // prune finished watches after a day
15
24
 
16
25
  /**
17
- * Wake-on-completion for agents.
26
+ * Wake-on-completion for agents — durable across daemon restarts.
18
27
  *
19
- * An agent promises to "report back when the tests finish", but its process
20
- * ends when the turn does so nothing is left to report. A Watch lives in the
21
- * daemon, not the agent's session: it outlives the turn, and when its target
22
- * finishes it resumes the agent (from `completed` if need be, via the same
23
- * pipe as user chat) with the outcome.
28
+ * An agent promises to "report back when the training finishes", but its
29
+ * process ends when the turn does, so nothing is left to report. A Watch lives
30
+ * in the daemon, not the agent's session: it outlives the turn, and when its
31
+ * target finishes it resumes the agent (from `completed` if need be, via the
32
+ * same pipe as user chat) with the outcome.
24
33
  *
25
- * Two flavours:
26
- * - command: the daemon runs a command detached and owns its lifecycle, so
27
- * it has the real exit code and output when it finishes.
28
- * - until: the daemon polls a check command for something already running,
29
- * and wakes the agent when the check first succeeds (exit 0).
34
+ * Crucially, watches survive a daemon restart. They are persisted to disk, and:
35
+ * - command: the daemon launches the command DETACHED, redirecting output to
36
+ * a file and writing its exit code to a sentinel file when it finishes. The
37
+ * job keeps running if the daemon dies; on restart the watcher re-polls the
38
+ * sentinel and fires when it appears. This is what makes "launch a training,
39
+ * get notified when done" reliable even if the harness restarts underneath.
40
+ * - until: the daemon polls a check command for something already running and
41
+ * wakes the agent when the check first succeeds (exit 0). Re-polled on
42
+ * restart the same way.
30
43
  */
31
44
  export class Watcher {
32
45
  constructor(daemon) {
33
46
  this.daemon = daemon;
34
47
  this.watches = new Map();
48
+ this.runsDir = resolve(daemon.grooveDir, 'watch-runs');
49
+ this.persistPath = resolve(daemon.grooveDir, 'watches.json');
50
+ try { mkdirSync(this.runsDir, { recursive: true }); } catch { /* exists */ }
35
51
  }
36
52
 
37
53
  create(agentId, opts = {}) {
@@ -61,84 +77,109 @@ export class Watcher {
61
77
  timeoutMs,
62
78
  status: 'active',
63
79
  createdAt: Date.now(),
64
- _child: null,
65
- _poll: null,
66
- _deadline: null,
80
+ // populated for command mode:
81
+ pid: null, outFile: null, statusFile: null,
82
+ // runtime-only (never persisted):
83
+ _poll: null, _deadline: null,
67
84
  };
68
85
 
69
86
  this.watches.set(watch.id, watch);
70
- watch._deadline = setTimeout(() => this._fireTimeout(watch), timeoutMs);
71
- if (watch.mode === 'command') this._runCommand(watch);
72
- else this._startPolling(watch);
87
+ if (watch.mode === 'command') this._launchDetached(watch);
88
+ this._arm(watch);
89
+ this._persist();
73
90
 
74
91
  this.daemon.audit?.log('watch.create', { id: watch.id, agent: agentId, mode: watch.mode });
75
92
  this.daemon.broadcast?.({ type: 'watch:created', data: this._public(watch) });
76
93
  return this._public(watch);
77
94
  }
78
95
 
79
- // ── command mode: daemon owns the process ─────────────────────
80
-
81
- _runCommand(watch) {
82
- const child = spawn('/bin/sh', ['-c', watch.command], {
83
- cwd: watch.cwd,
84
- env: process.env,
85
- detached: false,
86
- });
87
- watch._child = child;
96
+ // Start the timers for an active watch (used on both create and restore).
97
+ _arm(watch) {
98
+ // On restore the deadline may already be in the past — fire promptly rather
99
+ // than negative. The floor only guards against a 0/negative timer.
100
+ const remaining = Math.max(50, watch.timeoutMs - (Date.now() - watch.createdAt));
101
+ watch._deadline = setTimeout(() => this._fireTimeout(watch), remaining);
102
+ const interval = watch.mode === 'command' ? COMMAND_POLL_MS : watch.pollMs;
103
+ watch._poll = setInterval(() => this._tick(watch), interval);
104
+ // Check immediately — the job may have finished while the daemon was down,
105
+ // or the `until` condition may already hold.
106
+ this._tick(watch);
107
+ }
88
108
 
89
- const chunks = [];
90
- let bytes = 0;
91
- const collect = (buf) => {
92
- // Keep only the tail — a chatty build shouldn't grow this unbounded.
93
- chunks.push(buf);
94
- bytes += buf.length;
95
- while (bytes > OUTPUT_TAIL * 2 && chunks.length > 1) bytes -= chunks.shift().length;
96
- };
97
- child.stdout?.on('data', collect);
98
- child.stderr?.on('data', collect);
109
+ // ── command mode: detached job + exit sentinel ────────────────
110
+
111
+ _launchDetached(watch) {
112
+ const dir = resolve(this.runsDir, watch.id);
113
+ mkdirSync(dir, { recursive: true });
114
+ watch.outFile = resolve(dir, 'out.log');
115
+ watch.statusFile = resolve(dir, 'exit');
116
+ const scriptFile = resolve(dir, 'run.sh');
117
+
118
+ // Run the command in a SUBSHELL so its own `exit N` can't kill the wrapper
119
+ // before the sentinel is written; tee output to a file, then atomically
120
+ // publish the exit code so the poller never sees a half-written value.
121
+ const script = [
122
+ '#!/bin/sh',
123
+ '(',
124
+ watch.command,
125
+ `) > ${shq(watch.outFile)} 2>&1`,
126
+ `echo $? > ${shq(watch.statusFile)}.tmp && mv ${shq(watch.statusFile)}.tmp ${shq(watch.statusFile)}`,
127
+ '',
128
+ ].join('\n');
129
+ writeFileSync(scriptFile, script, { mode: 0o700 });
99
130
 
100
- child.on('error', (err) => {
101
- this._wake(watch, {
102
- outcome: 'error',
103
- summary: `Could not start the command: ${err.message}`,
131
+ try {
132
+ const child = spawn('/bin/sh', [scriptFile], {
133
+ cwd: watch.cwd,
134
+ env: process.env,
135
+ detached: true, // survive daemon exit
136
+ stdio: 'ignore',
104
137
  });
105
- });
138
+ child.unref();
139
+ watch.pid = child.pid;
140
+ } catch (err) {
141
+ // Fire synchronously via the poller path so the agent still hears about it.
142
+ watch._launchError = `Could not start the command: ${err.message}`;
143
+ }
144
+ }
145
+
146
+ // One poll iteration — checks for completion (sentinel for command mode, the
147
+ // `until` command for until mode) and wakes the agent when it's met.
148
+ _tick(watch) {
149
+ if (watch.status !== 'active') return;
150
+
151
+ if (watch._launchError) {
152
+ this._wake(watch, { outcome: 'error', summary: watch._launchError });
153
+ return;
154
+ }
106
155
 
107
- child.on('close', (code, signal) => {
108
- if (watch.status !== 'active') return; // already timed out / cancelled
109
- const output = Buffer.concat(chunks).toString('utf8').slice(-OUTPUT_TAIL).trim();
156
+ if (watch.mode === 'command') {
157
+ if (!existsSync(watch.statusFile)) return; // still running
158
+ let code = null;
159
+ try { code = parseInt(readFileSync(watch.statusFile, 'utf8').trim(), 10); } catch { return; }
160
+ if (Number.isNaN(code)) return;
110
161
  this._wake(watch, {
111
162
  outcome: code === 0 ? 'success' : 'failure',
112
163
  exitCode: code,
113
- signal,
114
- summary: signal
115
- ? `Terminated by signal ${signal}.`
116
- : `Finished with exit code ${code}.`,
117
- output,
164
+ summary: `Finished with exit code ${code}.`,
165
+ output: tailFile(watch.outFile, OUTPUT_TAIL),
118
166
  });
119
- });
120
- }
121
-
122
- // ── until mode: poll something already running ────────────────
167
+ return;
168
+ }
123
169
 
124
- _startPolling(watch) {
125
- const tick = () => {
170
+ // until mode — run the check command; exit 0 means the condition holds.
171
+ const check = spawn('/bin/sh', ['-c', watch.until], { cwd: watch.cwd, env: process.env });
172
+ const chunks = [];
173
+ check.stdout?.on('data', (b) => chunks.push(b));
174
+ check.stderr?.on('data', (b) => chunks.push(b));
175
+ check.on('error', () => { /* transient — try again next tick */ });
176
+ check.on('close', (code) => {
126
177
  if (watch.status !== 'active') return;
127
- const check = spawn('/bin/sh', ['-c', watch.until], { cwd: watch.cwd, env: process.env });
128
- const chunks = [];
129
- check.stdout?.on('data', (b) => chunks.push(b));
130
- check.stderr?.on('data', (b) => chunks.push(b));
131
- check.on('error', () => { /* transient — try again next tick */ });
132
- check.on('close', (code) => {
133
- if (watch.status !== 'active') return;
134
- if (code === 0) {
135
- const output = Buffer.concat(chunks).toString('utf8').slice(-OUTPUT_TAIL).trim();
136
- this._wake(watch, { outcome: 'success', summary: 'Your watch condition is now true.', output });
137
- }
138
- });
139
- };
140
- watch._poll = setInterval(tick, watch.pollMs);
141
- tick(); // check immediately — the condition may already hold
178
+ if (code === 0) {
179
+ const output = Buffer.concat(chunks).toString('utf8').slice(-OUTPUT_TAIL).trim();
180
+ this._wake(watch, { outcome: 'success', summary: 'Your watch condition is now true.', output });
181
+ }
182
+ });
142
183
  }
143
184
 
144
185
  // ── firing ────────────────────────────────────────────────────
@@ -157,34 +198,33 @@ export class Watcher {
157
198
  watch.status = 'fired';
158
199
  watch.firedAt = Date.now();
159
200
  watch.result = result;
160
- this._cleanup(watch);
201
+ this._disarm(watch);
161
202
 
162
203
  const message = this._composeMessage(watch, result);
163
204
 
164
- // Resolve by name, not the id we stored at creation: over a 30-minute
165
- // watch the agent has very likely been chatted with and rotated to a new
166
- // id. Names are stable across rotation, so this follows the agent; a
167
- // purged/gone agent simply doesn't resolve and the watch is undeliverable
168
- // rather than resurrecting a killed agent under a stale id.
205
+ // Resolve by name, not the id stored at creation: over a long watch the
206
+ // agent has very likely been chatted with and rotated to a new id. Names
207
+ // are stable across rotation, so this follows the agent; a purged agent
208
+ // simply doesn't resolve, rather than resurrecting a killed one.
169
209
  const target = this.daemon.registry.getAll().find((a) => a.name === watch.agentName);
170
210
  if (!target) {
171
211
  watch.status = 'undeliverable';
172
212
  watch.deliveryError = `Agent ${watch.agentName} no longer exists`;
173
213
  this.daemon.audit?.log('watch.undeliverable', { id: watch.id, reason: 'agent gone' });
174
- this.daemon.broadcast?.({ type: 'watch:fired', data: this._public(watch) });
175
- return;
176
- }
177
-
178
- try {
179
- const delivered = await deliverInstruction(this.daemon, target.id, message, { recordFeedback: false });
180
- watch.agentId = delivered.agentId;
181
- watch.status = 'delivered';
182
- } catch (err) {
183
- watch.status = 'undeliverable';
184
- watch.deliveryError = err.message;
185
- this.daemon.audit?.log('watch.undeliverable', { id: watch.id, error: err.message });
214
+ } else {
215
+ try {
216
+ const delivered = await deliverInstruction(this.daemon, target.id, message, { recordFeedback: false });
217
+ watch.agentId = delivered.agentId;
218
+ watch.status = 'delivered';
219
+ } catch (err) {
220
+ watch.status = 'undeliverable';
221
+ watch.deliveryError = err.message;
222
+ this.daemon.audit?.log('watch.undeliverable', { id: watch.id, error: err.message });
223
+ }
186
224
  }
187
225
 
226
+ this._cleanupRun(watch);
227
+ this._persist();
188
228
  this.daemon.broadcast?.({ type: 'watch:fired', data: this._public(watch) });
189
229
  }
190
230
 
@@ -197,6 +237,45 @@ export class Watcher {
197
237
  return lines.join('\n');
198
238
  }
199
239
 
240
+ // ── persistence & restart recovery ────────────────────────────
241
+
242
+ _persist() {
243
+ try {
244
+ const data = [...this.watches.values()].map((w) => this._serialize(w));
245
+ writeFileSync(this.persistPath, JSON.stringify(data, null, 2), { mode: 0o600 });
246
+ } catch { /* best effort — a lost persist just means one watch won't survive */ }
247
+ }
248
+
249
+ // Load persisted watches and re-arm the active ones. Called once at startup,
250
+ // after the registry is available. This is what lets a watch survive a daemon
251
+ // restart: the detached job kept running (or already finished), and here we
252
+ // reconnect to its sentinel.
253
+ restore() {
254
+ if (!existsSync(this.persistPath)) return 0;
255
+ let data;
256
+ try { data = JSON.parse(readFileSync(this.persistPath, 'utf8')); } catch { return 0; }
257
+ if (!Array.isArray(data)) return 0;
258
+
259
+ let rearmed = 0;
260
+ const now = Date.now();
261
+ for (const w of data) {
262
+ // Drop stale finished watches; keep recent ones for history.
263
+ if (w.status !== 'active') {
264
+ if (now - (w.firedAt || w.createdAt || 0) < HISTORY_TTL_MS) {
265
+ this.watches.set(w.id, { ...w, _poll: null, _deadline: null });
266
+ }
267
+ continue;
268
+ }
269
+ const watch = { ...w, _poll: null, _deadline: null };
270
+ this.watches.set(watch.id, watch);
271
+ this._arm(watch);
272
+ rearmed++;
273
+ }
274
+ if (rearmed > 0) console.log(`[Groove:Watcher] Restored ${rearmed} active watch(es) after restart`);
275
+ this._persist();
276
+ return rearmed;
277
+ }
278
+
200
279
  // ── lifecycle ─────────────────────────────────────────────────
201
280
 
202
281
  cancel(id, agentId = null) {
@@ -205,7 +284,10 @@ export class Watcher {
205
284
  if (agentId && watch.agentId !== agentId) return false;
206
285
  if (watch.status === 'active') {
207
286
  watch.status = 'cancelled';
208
- this._cleanup(watch);
287
+ this._disarm(watch);
288
+ this._killJob(watch);
289
+ this._cleanupRun(watch);
290
+ this._persist();
209
291
  this.daemon.broadcast?.({ type: 'watch:cancelled', data: this._public(watch) });
210
292
  }
211
293
  return true;
@@ -213,23 +295,35 @@ export class Watcher {
213
295
 
214
296
  // An agent that is killed/purged shouldn't leave watches firing into a void.
215
297
  cancelForAgent(agentId) {
298
+ let changed = false;
216
299
  for (const watch of this.watches.values()) {
217
300
  if (watch.agentId === agentId && watch.status === 'active') {
218
301
  watch.status = 'cancelled';
219
- this._cleanup(watch);
302
+ this._disarm(watch);
303
+ this._killJob(watch);
304
+ this._cleanupRun(watch);
305
+ changed = true;
220
306
  }
221
307
  }
308
+ if (changed) this._persist();
222
309
  }
223
310
 
224
- _cleanup(watch) {
311
+ _disarm(watch) {
225
312
  if (watch._deadline) { clearTimeout(watch._deadline); watch._deadline = null; }
226
313
  if (watch._poll) { clearInterval(watch._poll); watch._poll = null; }
227
- if (watch._child && watch.mode === 'command' && watch.status === 'cancelled') {
228
- // Only kill the process when the user/agent cancelled — a fired watch
229
- // means it exited on its own.
230
- try { watch._child.kill('SIGTERM'); } catch { /* already gone */ }
231
- }
232
- watch._child = null;
314
+ }
315
+
316
+ // Kill the detached job — only on explicit cancel. A fired watch means the
317
+ // job already exited on its own.
318
+ _killJob(watch) {
319
+ if (watch.mode !== 'command' || !watch.pid) return;
320
+ try { process.kill(-watch.pid, 'SIGTERM'); } // negative → the whole process group
321
+ catch { try { process.kill(watch.pid, 'SIGTERM'); } catch { /* already gone */ } }
322
+ }
323
+
324
+ _cleanupRun(watch) {
325
+ if (watch.mode !== 'command') return;
326
+ try { rmSync(resolve(this.runsDir, watch.id), { recursive: true, force: true }); } catch { /* ignore */ }
233
327
  }
234
328
 
235
329
  list(agentId = null) {
@@ -237,13 +331,21 @@ export class Watcher {
237
331
  return (agentId ? all.filter((w) => w.agentId === agentId) : all).map((w) => this._public(w));
238
332
  }
239
333
 
334
+ // Clear in-memory timers on shutdown. Deliberately does NOT kill the detached
335
+ // jobs — they must survive the restart; restore() reconnects to them.
240
336
  stop() {
241
- for (const watch of this.watches.values()) {
242
- if (watch._deadline) clearTimeout(watch._deadline);
243
- if (watch._poll) clearInterval(watch._poll);
244
- if (watch._child) { try { watch._child.kill('SIGTERM'); } catch { /* gone */ } }
245
- watch._deadline = watch._poll = watch._child = null;
246
- }
337
+ for (const watch of this.watches.values()) this._disarm(watch);
338
+ }
339
+
340
+ _serialize(w) {
341
+ return {
342
+ id: w.id, agentId: w.agentId, agentName: w.agentName, label: w.label,
343
+ mode: w.mode, command: w.command, until: w.until, cwd: w.cwd,
344
+ pollMs: w.pollMs, timeoutMs: w.timeoutMs, status: w.status,
345
+ createdAt: w.createdAt, firedAt: w.firedAt || null, result: w.result || null,
346
+ pid: w.pid || null, outFile: w.outFile || null, statusFile: w.statusFile || null,
347
+ deliveryError: w.deliveryError || null,
348
+ };
247
349
  }
248
350
 
249
351
  _public(w) {
@@ -255,4 +357,28 @@ export class Watcher {
255
357
  }
256
358
  }
257
359
 
360
+ // Single-quote for safe embedding in the runner script.
361
+ function shq(s) {
362
+ return `'${String(s).replace(/'/g, `'\\''`)}'`;
363
+ }
364
+
365
+ // Read the last `maxBytes` of a file without loading the whole thing — a
366
+ // training log can be huge.
367
+ function tailFile(path, maxBytes) {
368
+ if (!path || !existsSync(path)) return '';
369
+ let fd;
370
+ try {
371
+ fd = openSync(path, 'r');
372
+ const size = fstatSync(fd).size;
373
+ const len = Math.min(size, maxBytes);
374
+ const buf = Buffer.alloc(len);
375
+ readSync(fd, buf, 0, len, size - len);
376
+ return buf.toString('utf8').trim();
377
+ } catch {
378
+ return '';
379
+ } finally {
380
+ if (fd !== undefined) try { closeSync(fd); } catch { /* ignore */ }
381
+ }
382
+ }
383
+
258
384
  export { DEFAULT_TIMEOUT_MS, MAX_WATCHES_PER_AGENT };