groove-dev 0.27.187 → 0.27.189
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/innerchat/Screenshot_2026-07-23_at_1.30.16_PM.png +0 -0
- package/node_modules/@groove-dev/cli/package.json +1 -1
- package/node_modules/@groove-dev/daemon/package.json +1 -1
- package/node_modules/@groove-dev/daemon/src/index.js +6 -0
- package/node_modules/@groove-dev/daemon/src/innerchat-docs.js +18 -13
- package/node_modules/@groove-dev/daemon/src/process.js +10 -0
- package/node_modules/@groove-dev/daemon/src/teams.js +36 -4
- package/node_modules/@groove-dev/daemon/src/watcher.js +230 -104
- package/node_modules/@groove-dev/daemon/test/teams.test.js +48 -1
- package/node_modules/@groove-dev/daemon/test/watcher.test.js +68 -10
- package/node_modules/@groove-dev/gui/dist/assets/index-Cat5pJUx.css +1 -0
- package/node_modules/@groove-dev/gui/dist/assets/{index-DyI84i9Y.js → index-DTY0KsH9.js} +222 -222
- package/node_modules/@groove-dev/gui/dist/index.html +2 -2
- package/node_modules/@groove-dev/gui/package.json +1 -1
- package/node_modules/@groove-dev/gui/src/App.jsx +51 -2
- package/node_modules/@groove-dev/gui/src/components/agents/agent-feed.jsx +11 -2
- package/node_modules/@groove-dev/gui/src/components/agents/agent-panel.jsx +106 -10
- package/node_modules/@groove-dev/gui/src/lib/logpaths.js +1 -1
- package/node_modules/@groove-dev/gui/src/stores/groove.js +19 -6
- package/node_modules/@groove-dev/gui/src/stores/helpers.js +28 -0
- package/node_modules/@groove-dev/gui/src/stores/slices/agents-slice.js +20 -9
- package/package.json +1 -1
- package/packages/cli/package.json +1 -1
- package/packages/daemon/package.json +1 -1
- package/packages/daemon/src/index.js +6 -0
- package/packages/daemon/src/innerchat-docs.js +18 -13
- package/packages/daemon/src/process.js +10 -0
- package/packages/daemon/src/teams.js +36 -4
- package/packages/daemon/src/watcher.js +230 -104
- package/packages/gui/dist/assets/index-Cat5pJUx.css +1 -0
- package/packages/gui/dist/assets/{index-DyI84i9Y.js → index-DTY0KsH9.js} +222 -222
- package/packages/gui/dist/index.html +2 -2
- package/packages/gui/package.json +1 -1
- package/packages/gui/src/App.jsx +51 -2
- package/packages/gui/src/components/agents/agent-feed.jsx +11 -2
- package/packages/gui/src/components/agents/agent-panel.jsx +106 -10
- package/packages/gui/src/lib/logpaths.js +1 -1
- package/packages/gui/src/stores/groove.js +19 -6
- package/packages/gui/src/stores/helpers.js +28 -0
- package/packages/gui/src/stores/slices/agents-slice.js +20 -9
- package/node_modules/@groove-dev/gui/dist/assets/index-CU8L_r5f.css +0 -1
- package/packages/gui/dist/assets/index-CU8L_r5f.css +0 -1
|
Binary file
|
|
@@ -638,6 +638,12 @@ export class Daemon {
|
|
|
638
638
|
}
|
|
639
639
|
}
|
|
640
640
|
|
|
641
|
+
// Reconnect to watches whose detached jobs survived this restart, so a
|
|
642
|
+
// "notify me when the training finishes" watch still fires afterward.
|
|
643
|
+
try { this.watcher.restore(); } catch (err) {
|
|
644
|
+
console.error('[startup] Failed to restore watches:', err.message);
|
|
645
|
+
}
|
|
646
|
+
|
|
641
647
|
// Restore auth token from stored config so subscription polling works after restart
|
|
642
648
|
const storedToken = this.skills.getToken();
|
|
643
649
|
if (storedToken) {
|
|
@@ -70,27 +70,32 @@ export function innerChatInstructions(port = 31415, agentName = 'YOUR_NAME') {
|
|
|
70
70
|
export function watchInstructions(port = 31415, agentName = 'YOUR_NAME') {
|
|
71
71
|
const nameHint = agentName === 'YOUR_NAME' ? ' ($GROOVE_AGENT_NAME)' : '';
|
|
72
72
|
return [
|
|
73
|
-
'##
|
|
73
|
+
'## Running Long Jobs & Getting Notified (Watch)',
|
|
74
74
|
'',
|
|
75
|
-
'
|
|
76
|
-
|
|
77
|
-
'
|
|
78
|
-
'
|
|
75
|
+
'> **A job you start yourself dies when your turn ends.** Anything you launch in Bash —',
|
|
76
|
+
"> even backgrounded with `&`, `nohup`, or `disown` — is a child of your session process.",
|
|
77
|
+
'> When your turn ends or your context rotates, that process is torn down and the job dies',
|
|
78
|
+
'> with it. Do NOT start a long-running job (training, build, server, benchmark) directly',
|
|
79
|
+
"> in Bash and expect it to keep running. It won't.",
|
|
80
|
+
'',
|
|
81
|
+
'Instead, hand the command to a Watch. The DAEMON runs it fully detached — independent of',
|
|
82
|
+
'your turn, surviving turn-end, context rotation, and even a daemon restart — and RESUMES',
|
|
83
|
+
'you with the exit code and output when it finishes:',
|
|
79
84
|
'',
|
|
80
85
|
'```bash',
|
|
81
86
|
`curl -s http://localhost:${port}/api/watch -X POST -H 'Content-Type: application/json' \\`,
|
|
82
|
-
` -d '{"agent":"${agentName}","label":"
|
|
87
|
+
` -d '{"agent":"${agentName}","label":"model training","command":"python train.py"}'`,
|
|
83
88
|
'```',
|
|
84
89
|
'',
|
|
85
|
-
'This returns immediately with a `watchId`. **End your turn** —
|
|
86
|
-
'
|
|
90
|
+
'This returns immediately with a `watchId`. **End your turn** — the job keeps running under',
|
|
91
|
+
'the daemon and you will be woken with the result when it completes.',
|
|
87
92
|
'',
|
|
88
|
-
'-
|
|
89
|
-
' it
|
|
90
|
-
'-
|
|
91
|
-
`
|
|
93
|
+
'- Use `command` for anything that must outlive your turn. Let the daemon run it — do not',
|
|
94
|
+
' launch it yourself and then watch it; a self-launched job is already doomed.',
|
|
95
|
+
'- Use `until` ONLY for something the daemon or another durable process is already running',
|
|
96
|
+
` (not a Bash job you started): \`-d '{"agent":"${agentName}","label":"server up","until":"curl -sf localhost:3000/health"}'\`.`,
|
|
92
97
|
`- \`agent\` must be your own name${nameHint}.`,
|
|
93
|
-
'- Watches time out (default 30 min
|
|
98
|
+
'- Watches time out (default 30 min, up to 24h — pass `timeoutMs` for long trainings).',
|
|
94
99
|
'- Do not poll or sleep-loop waiting yourself — set the watch and end the turn. That is the whole point.',
|
|
95
100
|
];
|
|
96
101
|
}
|
|
@@ -2628,6 +2628,9 @@ After fixing all issues, run tests (npm test) and build (npm run build) to verif
|
|
|
2628
2628
|
// Let the aborted process finish tearing down before re-spawning; resume()
|
|
2629
2629
|
// kills the current handle and re-registers under a new agent id.
|
|
2630
2630
|
setTimeout(() => {
|
|
2631
|
+
// The user may have paused/killed during this 500ms window — stop()/kill()
|
|
2632
|
+
// clear the pending message to cancel exactly this in-flight retry.
|
|
2633
|
+
if (!this._pendingUserMessage.has(agentId)) return;
|
|
2631
2634
|
this.resume(agentId, pending.message).catch((err) => {
|
|
2632
2635
|
this.daemon.broadcast({
|
|
2633
2636
|
type: 'agent:output',
|
|
@@ -3060,6 +3063,13 @@ After fixing all issues, run tests (npm test) and build (npm run build) to verif
|
|
|
3060
3063
|
const handle = this.handles.get(agentId);
|
|
3061
3064
|
if (!handle) return;
|
|
3062
3065
|
|
|
3066
|
+
// A user pause aborts the current turn, which the CLI reports as a dropped
|
|
3067
|
+
// turn (isError, apiDurationMs === 0). Clear the pending message FIRST so
|
|
3068
|
+
// _retryDroppedTurn no-ops — otherwise the stop is auto-retried and the
|
|
3069
|
+
// agent resumes, forcing the user to click Pause twice.
|
|
3070
|
+
this._pendingUserMessage.delete(agentId);
|
|
3071
|
+
this._retryAttemptCarry.delete(agentId);
|
|
3072
|
+
|
|
3063
3073
|
const { proc, loop } = handle;
|
|
3064
3074
|
|
|
3065
3075
|
if (loop) {
|
|
@@ -230,6 +230,21 @@ export class Teams {
|
|
|
230
230
|
originalWorkingDir: team.workingDir,
|
|
231
231
|
};
|
|
232
232
|
writeFileSync(resolve(archivePath, 'metadata.json'), JSON.stringify(metadata, null, 2));
|
|
233
|
+
} else if (
|
|
234
|
+
team.workingDir &&
|
|
235
|
+
team.workingDir !== this.daemon.projectDir &&
|
|
236
|
+
existsSync(team.workingDir) &&
|
|
237
|
+
this._dirHasExternalOccupants(team.workingDir)
|
|
238
|
+
) {
|
|
239
|
+
// A moved agent still lives here — archive metadata only, leave the
|
|
240
|
+
// directory in place so that agent keeps working.
|
|
241
|
+
console.log(`[Groove:Teams] Archiving ${team.name} metadata only: agents from other teams still use its directory`);
|
|
242
|
+
mkdirSync(archivePath, { recursive: true });
|
|
243
|
+
writeFileSync(resolve(archivePath, 'metadata.json'), JSON.stringify({
|
|
244
|
+
originalName: team.name, originalId: team.id, mode: team.mode || 'sandbox',
|
|
245
|
+
deletedAt: new Date().toISOString(), agentCount: agents.length,
|
|
246
|
+
originalWorkingDir: team.workingDir, directoryRetained: true,
|
|
247
|
+
}, null, 2));
|
|
233
248
|
} else if (
|
|
234
249
|
team.workingDir &&
|
|
235
250
|
team.workingDir !== this.daemon.projectDir &&
|
|
@@ -281,10 +296,16 @@ export class Teams {
|
|
|
281
296
|
team.workingDir !== this.daemon.projectDir &&
|
|
282
297
|
existsSync(team.workingDir)
|
|
283
298
|
) {
|
|
284
|
-
|
|
285
|
-
|
|
286
|
-
|
|
287
|
-
console.log(`[Groove:Teams]
|
|
299
|
+
if (this._dirHasExternalOccupants(team.workingDir)) {
|
|
300
|
+
// An agent moved to another team still lives here — keep the directory
|
|
301
|
+
// so it doesn't lose its working dir. The team record is still removed.
|
|
302
|
+
console.log(`[Groove:Teams] Keeping ${team.workingDir}: agents from other teams still use it`);
|
|
303
|
+
} else {
|
|
304
|
+
try {
|
|
305
|
+
rmSync(team.workingDir, { recursive: true, force: true });
|
|
306
|
+
} catch (err) {
|
|
307
|
+
console.log(`[Groove:Teams] Failed to delete directory: ${err.message}`);
|
|
308
|
+
}
|
|
288
309
|
}
|
|
289
310
|
}
|
|
290
311
|
|
|
@@ -292,6 +313,17 @@ export class Teams {
|
|
|
292
313
|
return true;
|
|
293
314
|
}
|
|
294
315
|
|
|
316
|
+
// True if an agent OUTSIDE this team still lives in the given directory —
|
|
317
|
+
// i.e. an agent that was moved to another team but kept its working dir here.
|
|
318
|
+
// Destroying the directory would break it, so callers skip the removal.
|
|
319
|
+
// Call after _killAndRemoveAgents so this team's own agents are already gone.
|
|
320
|
+
_dirHasExternalOccupants(dir) {
|
|
321
|
+
if (!dir) return false;
|
|
322
|
+
const prefix = `${dir}/`;
|
|
323
|
+
return this.daemon.registry.getAll().some((a) =>
|
|
324
|
+
a.workingDir && (a.workingDir === dir || a.workingDir.startsWith(prefix)));
|
|
325
|
+
}
|
|
326
|
+
|
|
295
327
|
_killAndRemoveAgents(teamId) {
|
|
296
328
|
const agents = this.daemon.registry.getAll().filter((a) => a.teamId === teamId);
|
|
297
329
|
for (const agent of agents) {
|
|
@@ -2,36 +2,52 @@
|
|
|
2
2
|
|
|
3
3
|
import { spawn } from 'child_process';
|
|
4
4
|
import { randomUUID } from 'crypto';
|
|
5
|
+
import {
|
|
6
|
+
mkdirSync, writeFileSync, readFileSync, existsSync, rmSync,
|
|
7
|
+
openSync, readSync, fstatSync, closeSync,
|
|
8
|
+
} from 'fs';
|
|
9
|
+
import { resolve } from 'path';
|
|
5
10
|
import { deliverInstruction } from './deliver.js';
|
|
6
11
|
|
|
7
12
|
// Watches are bounded so a wedged process or a condition that never comes true
|
|
8
13
|
// can't poll forever or hold a slot indefinitely.
|
|
9
14
|
const DEFAULT_TIMEOUT_MS = 30 * 60 * 1000;
|
|
10
|
-
const MAX_TIMEOUT_MS =
|
|
15
|
+
const MAX_TIMEOUT_MS = 24 * 60 * 60 * 1000; // training can run for hours
|
|
11
16
|
const DEFAULT_POLL_MS = 15 * 1000;
|
|
12
17
|
const MIN_POLL_MS = 3 * 1000;
|
|
18
|
+
// Command mode just stats a sentinel file — nearly free, so poll it fast for a
|
|
19
|
+
// responsive wake. `until` mode spawns a shell each tick, hence the slower floor.
|
|
20
|
+
const COMMAND_POLL_MS = 500;
|
|
13
21
|
const MAX_WATCHES_PER_AGENT = 5;
|
|
14
22
|
const OUTPUT_TAIL = 4000;
|
|
23
|
+
const HISTORY_TTL_MS = 24 * 60 * 60 * 1000; // prune finished watches after a day
|
|
15
24
|
|
|
16
25
|
/**
|
|
17
|
-
* Wake-on-completion for agents.
|
|
26
|
+
* Wake-on-completion for agents — durable across daemon restarts.
|
|
18
27
|
*
|
|
19
|
-
* An agent promises to "report back when the
|
|
20
|
-
* ends when the turn does
|
|
21
|
-
* daemon, not the agent's session: it outlives the turn, and when its
|
|
22
|
-
* finishes it resumes the agent (from `completed` if need be, via the
|
|
23
|
-
* pipe as user chat) with the outcome.
|
|
28
|
+
* An agent promises to "report back when the training finishes", but its
|
|
29
|
+
* process ends when the turn does, so nothing is left to report. A Watch lives
|
|
30
|
+
* in the daemon, not the agent's session: it outlives the turn, and when its
|
|
31
|
+
* target finishes it resumes the agent (from `completed` if need be, via the
|
|
32
|
+
* same pipe as user chat) with the outcome.
|
|
24
33
|
*
|
|
25
|
-
*
|
|
26
|
-
* - command: the daemon
|
|
27
|
-
*
|
|
28
|
-
*
|
|
29
|
-
* and
|
|
34
|
+
* Crucially, watches survive a daemon restart. They are persisted to disk, and:
|
|
35
|
+
* - command: the daemon launches the command DETACHED, redirecting output to
|
|
36
|
+
* a file and writing its exit code to a sentinel file when it finishes. The
|
|
37
|
+
* job keeps running if the daemon dies; on restart the watcher re-polls the
|
|
38
|
+
* sentinel and fires when it appears. This is what makes "launch a training,
|
|
39
|
+
* get notified when done" reliable even if the harness restarts underneath.
|
|
40
|
+
* - until: the daemon polls a check command for something already running and
|
|
41
|
+
* wakes the agent when the check first succeeds (exit 0). Re-polled on
|
|
42
|
+
* restart the same way.
|
|
30
43
|
*/
|
|
31
44
|
export class Watcher {
|
|
32
45
|
constructor(daemon) {
|
|
33
46
|
this.daemon = daemon;
|
|
34
47
|
this.watches = new Map();
|
|
48
|
+
this.runsDir = resolve(daemon.grooveDir, 'watch-runs');
|
|
49
|
+
this.persistPath = resolve(daemon.grooveDir, 'watches.json');
|
|
50
|
+
try { mkdirSync(this.runsDir, { recursive: true }); } catch { /* exists */ }
|
|
35
51
|
}
|
|
36
52
|
|
|
37
53
|
create(agentId, opts = {}) {
|
|
@@ -61,84 +77,109 @@ export class Watcher {
|
|
|
61
77
|
timeoutMs,
|
|
62
78
|
status: 'active',
|
|
63
79
|
createdAt: Date.now(),
|
|
64
|
-
|
|
65
|
-
|
|
66
|
-
|
|
80
|
+
// populated for command mode:
|
|
81
|
+
pid: null, outFile: null, statusFile: null,
|
|
82
|
+
// runtime-only (never persisted):
|
|
83
|
+
_poll: null, _deadline: null,
|
|
67
84
|
};
|
|
68
85
|
|
|
69
86
|
this.watches.set(watch.id, watch);
|
|
70
|
-
watch.
|
|
71
|
-
|
|
72
|
-
|
|
87
|
+
if (watch.mode === 'command') this._launchDetached(watch);
|
|
88
|
+
this._arm(watch);
|
|
89
|
+
this._persist();
|
|
73
90
|
|
|
74
91
|
this.daemon.audit?.log('watch.create', { id: watch.id, agent: agentId, mode: watch.mode });
|
|
75
92
|
this.daemon.broadcast?.({ type: 'watch:created', data: this._public(watch) });
|
|
76
93
|
return this._public(watch);
|
|
77
94
|
}
|
|
78
95
|
|
|
79
|
-
//
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
|
|
96
|
+
// Start the timers for an active watch (used on both create and restore).
|
|
97
|
+
_arm(watch) {
|
|
98
|
+
// On restore the deadline may already be in the past — fire promptly rather
|
|
99
|
+
// than negative. The floor only guards against a 0/negative timer.
|
|
100
|
+
const remaining = Math.max(50, watch.timeoutMs - (Date.now() - watch.createdAt));
|
|
101
|
+
watch._deadline = setTimeout(() => this._fireTimeout(watch), remaining);
|
|
102
|
+
const interval = watch.mode === 'command' ? COMMAND_POLL_MS : watch.pollMs;
|
|
103
|
+
watch._poll = setInterval(() => this._tick(watch), interval);
|
|
104
|
+
// Check immediately — the job may have finished while the daemon was down,
|
|
105
|
+
// or the `until` condition may already hold.
|
|
106
|
+
this._tick(watch);
|
|
107
|
+
}
|
|
88
108
|
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
109
|
+
// ── command mode: detached job + exit sentinel ────────────────
|
|
110
|
+
|
|
111
|
+
_launchDetached(watch) {
|
|
112
|
+
const dir = resolve(this.runsDir, watch.id);
|
|
113
|
+
mkdirSync(dir, { recursive: true });
|
|
114
|
+
watch.outFile = resolve(dir, 'out.log');
|
|
115
|
+
watch.statusFile = resolve(dir, 'exit');
|
|
116
|
+
const scriptFile = resolve(dir, 'run.sh');
|
|
117
|
+
|
|
118
|
+
// Run the command in a SUBSHELL so its own `exit N` can't kill the wrapper
|
|
119
|
+
// before the sentinel is written; tee output to a file, then atomically
|
|
120
|
+
// publish the exit code so the poller never sees a half-written value.
|
|
121
|
+
const script = [
|
|
122
|
+
'#!/bin/sh',
|
|
123
|
+
'(',
|
|
124
|
+
watch.command,
|
|
125
|
+
`) > ${shq(watch.outFile)} 2>&1`,
|
|
126
|
+
`echo $? > ${shq(watch.statusFile)}.tmp && mv ${shq(watch.statusFile)}.tmp ${shq(watch.statusFile)}`,
|
|
127
|
+
'',
|
|
128
|
+
].join('\n');
|
|
129
|
+
writeFileSync(scriptFile, script, { mode: 0o700 });
|
|
99
130
|
|
|
100
|
-
|
|
101
|
-
|
|
102
|
-
|
|
103
|
-
|
|
131
|
+
try {
|
|
132
|
+
const child = spawn('/bin/sh', [scriptFile], {
|
|
133
|
+
cwd: watch.cwd,
|
|
134
|
+
env: process.env,
|
|
135
|
+
detached: true, // survive daemon exit
|
|
136
|
+
stdio: 'ignore',
|
|
104
137
|
});
|
|
105
|
-
|
|
138
|
+
child.unref();
|
|
139
|
+
watch.pid = child.pid;
|
|
140
|
+
} catch (err) {
|
|
141
|
+
// Fire synchronously via the poller path so the agent still hears about it.
|
|
142
|
+
watch._launchError = `Could not start the command: ${err.message}`;
|
|
143
|
+
}
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
// One poll iteration — checks for completion (sentinel for command mode, the
|
|
147
|
+
// `until` command for until mode) and wakes the agent when it's met.
|
|
148
|
+
_tick(watch) {
|
|
149
|
+
if (watch.status !== 'active') return;
|
|
150
|
+
|
|
151
|
+
if (watch._launchError) {
|
|
152
|
+
this._wake(watch, { outcome: 'error', summary: watch._launchError });
|
|
153
|
+
return;
|
|
154
|
+
}
|
|
106
155
|
|
|
107
|
-
|
|
108
|
-
if (watch.
|
|
109
|
-
|
|
156
|
+
if (watch.mode === 'command') {
|
|
157
|
+
if (!existsSync(watch.statusFile)) return; // still running
|
|
158
|
+
let code = null;
|
|
159
|
+
try { code = parseInt(readFileSync(watch.statusFile, 'utf8').trim(), 10); } catch { return; }
|
|
160
|
+
if (Number.isNaN(code)) return;
|
|
110
161
|
this._wake(watch, {
|
|
111
162
|
outcome: code === 0 ? 'success' : 'failure',
|
|
112
163
|
exitCode: code,
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
? `Terminated by signal ${signal}.`
|
|
116
|
-
: `Finished with exit code ${code}.`,
|
|
117
|
-
output,
|
|
164
|
+
summary: `Finished with exit code ${code}.`,
|
|
165
|
+
output: tailFile(watch.outFile, OUTPUT_TAIL),
|
|
118
166
|
});
|
|
119
|
-
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
// ── until mode: poll something already running ────────────────
|
|
167
|
+
return;
|
|
168
|
+
}
|
|
123
169
|
|
|
124
|
-
|
|
125
|
-
const
|
|
170
|
+
// until mode — run the check command; exit 0 means the condition holds.
|
|
171
|
+
const check = spawn('/bin/sh', ['-c', watch.until], { cwd: watch.cwd, env: process.env });
|
|
172
|
+
const chunks = [];
|
|
173
|
+
check.stdout?.on('data', (b) => chunks.push(b));
|
|
174
|
+
check.stderr?.on('data', (b) => chunks.push(b));
|
|
175
|
+
check.on('error', () => { /* transient — try again next tick */ });
|
|
176
|
+
check.on('close', (code) => {
|
|
126
177
|
if (watch.status !== 'active') return;
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
check.on('close', (code) => {
|
|
133
|
-
if (watch.status !== 'active') return;
|
|
134
|
-
if (code === 0) {
|
|
135
|
-
const output = Buffer.concat(chunks).toString('utf8').slice(-OUTPUT_TAIL).trim();
|
|
136
|
-
this._wake(watch, { outcome: 'success', summary: 'Your watch condition is now true.', output });
|
|
137
|
-
}
|
|
138
|
-
});
|
|
139
|
-
};
|
|
140
|
-
watch._poll = setInterval(tick, watch.pollMs);
|
|
141
|
-
tick(); // check immediately — the condition may already hold
|
|
178
|
+
if (code === 0) {
|
|
179
|
+
const output = Buffer.concat(chunks).toString('utf8').slice(-OUTPUT_TAIL).trim();
|
|
180
|
+
this._wake(watch, { outcome: 'success', summary: 'Your watch condition is now true.', output });
|
|
181
|
+
}
|
|
182
|
+
});
|
|
142
183
|
}
|
|
143
184
|
|
|
144
185
|
// ── firing ────────────────────────────────────────────────────
|
|
@@ -157,34 +198,33 @@ export class Watcher {
|
|
|
157
198
|
watch.status = 'fired';
|
|
158
199
|
watch.firedAt = Date.now();
|
|
159
200
|
watch.result = result;
|
|
160
|
-
this.
|
|
201
|
+
this._disarm(watch);
|
|
161
202
|
|
|
162
203
|
const message = this._composeMessage(watch, result);
|
|
163
204
|
|
|
164
|
-
// Resolve by name, not the id
|
|
165
|
-
//
|
|
166
|
-
//
|
|
167
|
-
//
|
|
168
|
-
// rather than resurrecting a killed agent under a stale id.
|
|
205
|
+
// Resolve by name, not the id stored at creation: over a long watch the
|
|
206
|
+
// agent has very likely been chatted with and rotated to a new id. Names
|
|
207
|
+
// are stable across rotation, so this follows the agent; a purged agent
|
|
208
|
+
// simply doesn't resolve, rather than resurrecting a killed one.
|
|
169
209
|
const target = this.daemon.registry.getAll().find((a) => a.name === watch.agentName);
|
|
170
210
|
if (!target) {
|
|
171
211
|
watch.status = 'undeliverable';
|
|
172
212
|
watch.deliveryError = `Agent ${watch.agentName} no longer exists`;
|
|
173
213
|
this.daemon.audit?.log('watch.undeliverable', { id: watch.id, reason: 'agent gone' });
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
|
|
181
|
-
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
watch.deliveryError = err.message;
|
|
185
|
-
this.daemon.audit?.log('watch.undeliverable', { id: watch.id, error: err.message });
|
|
214
|
+
} else {
|
|
215
|
+
try {
|
|
216
|
+
const delivered = await deliverInstruction(this.daemon, target.id, message, { recordFeedback: false });
|
|
217
|
+
watch.agentId = delivered.agentId;
|
|
218
|
+
watch.status = 'delivered';
|
|
219
|
+
} catch (err) {
|
|
220
|
+
watch.status = 'undeliverable';
|
|
221
|
+
watch.deliveryError = err.message;
|
|
222
|
+
this.daemon.audit?.log('watch.undeliverable', { id: watch.id, error: err.message });
|
|
223
|
+
}
|
|
186
224
|
}
|
|
187
225
|
|
|
226
|
+
this._cleanupRun(watch);
|
|
227
|
+
this._persist();
|
|
188
228
|
this.daemon.broadcast?.({ type: 'watch:fired', data: this._public(watch) });
|
|
189
229
|
}
|
|
190
230
|
|
|
@@ -197,6 +237,45 @@ export class Watcher {
|
|
|
197
237
|
return lines.join('\n');
|
|
198
238
|
}
|
|
199
239
|
|
|
240
|
+
// ── persistence & restart recovery ────────────────────────────
|
|
241
|
+
|
|
242
|
+
_persist() {
|
|
243
|
+
try {
|
|
244
|
+
const data = [...this.watches.values()].map((w) => this._serialize(w));
|
|
245
|
+
writeFileSync(this.persistPath, JSON.stringify(data, null, 2), { mode: 0o600 });
|
|
246
|
+
} catch { /* best effort — a lost persist just means one watch won't survive */ }
|
|
247
|
+
}
|
|
248
|
+
|
|
249
|
+
// Load persisted watches and re-arm the active ones. Called once at startup,
|
|
250
|
+
// after the registry is available. This is what lets a watch survive a daemon
|
|
251
|
+
// restart: the detached job kept running (or already finished), and here we
|
|
252
|
+
// reconnect to its sentinel.
|
|
253
|
+
restore() {
|
|
254
|
+
if (!existsSync(this.persistPath)) return 0;
|
|
255
|
+
let data;
|
|
256
|
+
try { data = JSON.parse(readFileSync(this.persistPath, 'utf8')); } catch { return 0; }
|
|
257
|
+
if (!Array.isArray(data)) return 0;
|
|
258
|
+
|
|
259
|
+
let rearmed = 0;
|
|
260
|
+
const now = Date.now();
|
|
261
|
+
for (const w of data) {
|
|
262
|
+
// Drop stale finished watches; keep recent ones for history.
|
|
263
|
+
if (w.status !== 'active') {
|
|
264
|
+
if (now - (w.firedAt || w.createdAt || 0) < HISTORY_TTL_MS) {
|
|
265
|
+
this.watches.set(w.id, { ...w, _poll: null, _deadline: null });
|
|
266
|
+
}
|
|
267
|
+
continue;
|
|
268
|
+
}
|
|
269
|
+
const watch = { ...w, _poll: null, _deadline: null };
|
|
270
|
+
this.watches.set(watch.id, watch);
|
|
271
|
+
this._arm(watch);
|
|
272
|
+
rearmed++;
|
|
273
|
+
}
|
|
274
|
+
if (rearmed > 0) console.log(`[Groove:Watcher] Restored ${rearmed} active watch(es) after restart`);
|
|
275
|
+
this._persist();
|
|
276
|
+
return rearmed;
|
|
277
|
+
}
|
|
278
|
+
|
|
200
279
|
// ── lifecycle ─────────────────────────────────────────────────
|
|
201
280
|
|
|
202
281
|
cancel(id, agentId = null) {
|
|
@@ -205,7 +284,10 @@ export class Watcher {
|
|
|
205
284
|
if (agentId && watch.agentId !== agentId) return false;
|
|
206
285
|
if (watch.status === 'active') {
|
|
207
286
|
watch.status = 'cancelled';
|
|
208
|
-
this.
|
|
287
|
+
this._disarm(watch);
|
|
288
|
+
this._killJob(watch);
|
|
289
|
+
this._cleanupRun(watch);
|
|
290
|
+
this._persist();
|
|
209
291
|
this.daemon.broadcast?.({ type: 'watch:cancelled', data: this._public(watch) });
|
|
210
292
|
}
|
|
211
293
|
return true;
|
|
@@ -213,23 +295,35 @@ export class Watcher {
|
|
|
213
295
|
|
|
214
296
|
// An agent that is killed/purged shouldn't leave watches firing into a void.
|
|
215
297
|
cancelForAgent(agentId) {
|
|
298
|
+
let changed = false;
|
|
216
299
|
for (const watch of this.watches.values()) {
|
|
217
300
|
if (watch.agentId === agentId && watch.status === 'active') {
|
|
218
301
|
watch.status = 'cancelled';
|
|
219
|
-
this.
|
|
302
|
+
this._disarm(watch);
|
|
303
|
+
this._killJob(watch);
|
|
304
|
+
this._cleanupRun(watch);
|
|
305
|
+
changed = true;
|
|
220
306
|
}
|
|
221
307
|
}
|
|
308
|
+
if (changed) this._persist();
|
|
222
309
|
}
|
|
223
310
|
|
|
224
|
-
|
|
311
|
+
_disarm(watch) {
|
|
225
312
|
if (watch._deadline) { clearTimeout(watch._deadline); watch._deadline = null; }
|
|
226
313
|
if (watch._poll) { clearInterval(watch._poll); watch._poll = null; }
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
|
|
232
|
-
watch.
|
|
314
|
+
}
|
|
315
|
+
|
|
316
|
+
// Kill the detached job — only on explicit cancel. A fired watch means the
|
|
317
|
+
// job already exited on its own.
|
|
318
|
+
_killJob(watch) {
|
|
319
|
+
if (watch.mode !== 'command' || !watch.pid) return;
|
|
320
|
+
try { process.kill(-watch.pid, 'SIGTERM'); } // negative → the whole process group
|
|
321
|
+
catch { try { process.kill(watch.pid, 'SIGTERM'); } catch { /* already gone */ } }
|
|
322
|
+
}
|
|
323
|
+
|
|
324
|
+
_cleanupRun(watch) {
|
|
325
|
+
if (watch.mode !== 'command') return;
|
|
326
|
+
try { rmSync(resolve(this.runsDir, watch.id), { recursive: true, force: true }); } catch { /* ignore */ }
|
|
233
327
|
}
|
|
234
328
|
|
|
235
329
|
list(agentId = null) {
|
|
@@ -237,13 +331,21 @@ export class Watcher {
|
|
|
237
331
|
return (agentId ? all.filter((w) => w.agentId === agentId) : all).map((w) => this._public(w));
|
|
238
332
|
}
|
|
239
333
|
|
|
334
|
+
// Clear in-memory timers on shutdown. Deliberately does NOT kill the detached
|
|
335
|
+
// jobs — they must survive the restart; restore() reconnects to them.
|
|
240
336
|
stop() {
|
|
241
|
-
for (const watch of this.watches.values())
|
|
242
|
-
|
|
243
|
-
|
|
244
|
-
|
|
245
|
-
|
|
246
|
-
|
|
337
|
+
for (const watch of this.watches.values()) this._disarm(watch);
|
|
338
|
+
}
|
|
339
|
+
|
|
340
|
+
_serialize(w) {
|
|
341
|
+
return {
|
|
342
|
+
id: w.id, agentId: w.agentId, agentName: w.agentName, label: w.label,
|
|
343
|
+
mode: w.mode, command: w.command, until: w.until, cwd: w.cwd,
|
|
344
|
+
pollMs: w.pollMs, timeoutMs: w.timeoutMs, status: w.status,
|
|
345
|
+
createdAt: w.createdAt, firedAt: w.firedAt || null, result: w.result || null,
|
|
346
|
+
pid: w.pid || null, outFile: w.outFile || null, statusFile: w.statusFile || null,
|
|
347
|
+
deliveryError: w.deliveryError || null,
|
|
348
|
+
};
|
|
247
349
|
}
|
|
248
350
|
|
|
249
351
|
_public(w) {
|
|
@@ -255,4 +357,28 @@ export class Watcher {
|
|
|
255
357
|
}
|
|
256
358
|
}
|
|
257
359
|
|
|
360
|
+
// Single-quote for safe embedding in the runner script.
|
|
361
|
+
function shq(s) {
|
|
362
|
+
return `'${String(s).replace(/'/g, `'\\''`)}'`;
|
|
363
|
+
}
|
|
364
|
+
|
|
365
|
+
// Read the last `maxBytes` of a file without loading the whole thing — a
|
|
366
|
+
// training log can be huge.
|
|
367
|
+
function tailFile(path, maxBytes) {
|
|
368
|
+
if (!path || !existsSync(path)) return '';
|
|
369
|
+
let fd;
|
|
370
|
+
try {
|
|
371
|
+
fd = openSync(path, 'r');
|
|
372
|
+
const size = fstatSync(fd).size;
|
|
373
|
+
const len = Math.min(size, maxBytes);
|
|
374
|
+
const buf = Buffer.alloc(len);
|
|
375
|
+
readSync(fd, buf, 0, len, size - len);
|
|
376
|
+
return buf.toString('utf8').trim();
|
|
377
|
+
} catch {
|
|
378
|
+
return '';
|
|
379
|
+
} finally {
|
|
380
|
+
if (fd !== undefined) try { closeSync(fd); } catch { /* ignore */ }
|
|
381
|
+
}
|
|
382
|
+
}
|
|
383
|
+
|
|
258
384
|
export { DEFAULT_TIMEOUT_MS, MAX_WATCHES_PER_AGENT };
|