groove-dev 0.27.207 → 0.27.208

package/node_modules/@groove-dev/daemon/src/watcher.js

@@ -59,6 +59,28 @@ export class Watcher {
     if (!command && !until) throw new Error('Provide either "command" (run it) or "until" (poll it)');
     if (command && until) throw new Error('Provide only one of "command" or "until"');
 
+    // Re-attach, never re-execute. A daemon restart resumes the agent mid-turn,
+    // so it re-issues the watch it thinks never completed — and a second copy of
+    // a long job (training, benchmark) launched against the same working dir
+    // corrupts the first one's output and competes for the GPU. If an identical
+    // command is still running for this agent, hand back the existing watch.
+    // This is the daemon-side equivalent of an flock, and it has to live here
+    // because the agent cannot know a duplicate already exists.
+    if (command) {
+      const dup = [...this.watches.values()].find(
+        (w) => w.status === 'active'
+          && w.mode === 'command'
+          && w.command === command
+          && (w.agentName === agent.name || w.agentId === agentId)
+          && this._jobAlive(w),
+      );
+      if (dup) {
+        this.daemon.audit?.log('watch.duplicate', { id: dup.id, agent: agentId, label: dup.label });
+        console.log(`[Groove:Watcher] Re-attached to running watch ${dup.id} instead of re-running: ${dup.label}`);
+        return { ...this._public(dup), reattached: true };
+      }
+    }
+
     const active = [...this.watches.values()].filter((w) => w.agentId === agentId && w.status === 'active');
     if (active.length >= MAX_WATCHES_PER_AGENT) {
       throw new Error(`You already have ${MAX_WATCHES_PER_AGENT} active watches — cancel one before adding another`);
@@ -118,11 +140,14 @@ export class Watcher {
     // Run the command in a SUBSHELL so its own `exit N` can't kill the wrapper
     // before the sentinel is written; tee output to a file, then atomically
     // publish the exit code so the poller never sees a half-written value.
+    // Append rather than truncate: if this script is ever run twice, `>` would
+    // erase the first run's record — exactly the evidence you need to work out
+    // what happened. Appending keeps both.
     const script = [
       '#!/bin/sh',
       '(',
       watch.command,
-      `) > ${shq(watch.outFile)} 2>&1`,
+      `) >> ${shq(watch.outFile)} 2>&1`,
       `echo $? > ${shq(watch.statusFile)}.tmp && mv ${shq(watch.statusFile)}.tmp ${shq(watch.statusFile)}`,
       '',
     ].join('\n');
@@ -143,6 +168,16 @@ export class Watcher {
     }
   }
 
+  // Is this watch's detached job still running? Signal 0 tests for existence
+  // without delivering anything. A finished job (sentinel present) is not alive
+  // even if the pid happens to have been recycled by another process.
+  _jobAlive(watch) {
+    if (watch.mode !== 'command') return false;
+    if (watch.statusFile && existsSync(watch.statusFile)) return false; // already exited
+    if (!watch.pid) return false;
+    try { process.kill(watch.pid, 0); return true; } catch { return false; }
+  }
+
   // One poll iteration — checks for completion (sentinel for command mode, the
   // `until` command for until mode) and wakes the agent when it's met.
   _tick(watch) {
@@ -257,6 +292,7 @@ export class Watcher {
     if (!Array.isArray(data)) return 0;
 
     let rearmed = 0;
+    let lost = 0;
     const now = Date.now();
     for (const w of data) {
       // Drop stale finished watches; keep recent ones for history.
@@ -268,10 +304,27 @@ export class Watcher {
       }
       const watch = { ...w, _poll: null, _deadline: null };
       this.watches.set(watch.id, watch);
+
+      // A command watch whose job is gone with no exit sentinel died while the
+      // daemon was down. Polling it would just burn until the timeout and then
+      // report "may still be running" — say what actually happened instead.
+      if (watch.mode === 'command' && watch.pid
+          && !existsSync(watch.statusFile || '') && !this._jobAlive(watch)) {
+        lost++;
+        this._wake(watch, {
+          outcome: 'error',
+          summary: `The job for "${watch.label}" is no longer running and never recorded an exit code — `
+            + 'it was lost while the daemon was down. Check its output before assuming it finished.',
+          output: tailFile(watch.outFile, OUTPUT_TAIL),
+        });
+        continue;
+      }
+
       this._arm(watch);
       rearmed++;
     }
-    if (rearmed > 0) console.log(`[Groove:Watcher] Restored ${rearmed} active watch(es) after restart`);
+    if (rearmed > 0) console.log(`[Groove:Watcher] Re-attached to ${rearmed} running watch(es) after restart`);
+    if (lost > 0) console.log(`[Groove:Watcher] ${lost} watch(es) lost their job while the daemon was down`);
     this._persist();
     return rearmed;
   }

package/node_modules/@groove-dev/daemon/test/axom-connector.test.js

@@ -34,6 +34,8 @@ class MockBridge {
     this.shutdowns = [];
     this.messages = [];
     this.sinceSeen = []; // ?since values observed on WS connects
+    this.epochsSeen = [];
+    this.epoch = 'epoch-A';
     this.sockets = new Set();
   }
 
@@ -50,8 +52,12 @@ class MockBridge {
         ws.on('close', () => this.sockets.delete(ws));
         const since = url.searchParams.get('since');
         this.sinceSeen.push(since);
+        this.epochsSeen.push(url.searchParams.get('epoch'));
+        // §16.4: hello first; a stale client epoch voids `since` (full replay).
+        ws.send(JSON.stringify({ kind: 'ws_hello', payload: { epoch: this.epoch, since_honored: url.searchParams.get('epoch') === this.epoch } }));
+        const staleEpoch = url.searchParams.get('epoch') && url.searchParams.get('epoch') !== this.epoch;
         // Ring replay: everything after `since`, then live.
-        const from = since ? parseInt(since.slice(3), 10) : 0;
+        const from = (since && !staleEpoch) ? parseInt(since.slice(3), 10) : 0;
         for (const e of session.events) {
           if (parseInt(e.id.slice(3), 10) > from) ws.send(JSON.stringify(e));
         }
@@ -435,6 +441,43 @@ describe('AxomConnector', () => {
     assert.equal(latest.data.endpoints[0].sessions[0].live, false);
   });
 
+  it('§16.4: a changed epoch resets the cursor — a runtime restart is replayed, never swallowed', async () => {
+    connect();
+    await waitFor(() => connector.status().endpoints[0]?.sessions[0]?.watching);
+    bridge.emit('s-test0001', envelope(1, 'pipeline_start'));
+    bridge.emit('s-test0001', envelope(2, 'thought'));
+    await waitFor(() => daemon.broadcasts.filter((b) => b.type === 'axom:event').length === 2);
+
+    // Runtime "restarts": new epoch, event ids reset to 1, fresh history.
+    bridge.epoch = 'epoch-B';
+    bridge.sessions['s-test0001'].events = [envelope(1, 'pipeline_start', { fresh: true }), envelope(2, 'narration', { text: 'post-restart' })];
+    bridge.sessions['s-test0001'].socket.terminate();
+
+    // Reconnect: our epoch-A + since goes up, hello says epoch-B → we reset
+    // and take the full replay. Without the reset, monotonic dedup would
+    // silently drop both replayed events (ids <= lastSeq).
+    await waitFor(() => daemon.broadcasts.some((b) => b.type === 'axom:session:reset'), 5000);
+    await waitFor(() => daemon.broadcasts.filter((b) => b.type === 'axom:event' && b.envelope.payload?.fresh).length === 1, 5000);
+    const s = connector.status().endpoints[0].sessions[0];
+    assert.equal(s.buffered, 2); // the ring holds ONLY post-restart history
+    assert.ok(bridge.epochsSeen.includes('epoch-A')); // we did present the old epoch
+  });
+
+  it('recheck collapses a stale connected state the moment the runtime is gone', async () => {
+    connect();
+    await waitFor(() => connector.status().endpoints[0]?.status === 'connected');
+    daemon.broadcasts.length = 0;
+
+    // Runtime dies (a deliberate stop); without recheck the endpoint would
+    // stay 'connected' until the next scheduled session poll failed.
+    await bridge.close();
+    connector.recheck('local');
+    await waitFor(() => connector.status().endpoints[0]?.status === 'error');
+    // The transition broadcast rode the recheck — the GUI card moves with
+    // the verb, not with a poll.
+    assert.ok(daemon.broadcasts.some((b) => b.type === 'axom:status'));
+  });
+
   it('reports an unreachable endpoint honestly and recovers by retry', async () => {
     const deadUrl = bridge.url;
     const port = Number(new URL(deadUrl).port);

package/node_modules/@groove-dev/daemon/test/axom-runtimes.test.js

@@ -0,0 +1,198 @@
+// GROOVE — Axom Runtime Model Tests
+// FSL-1.1-Apache-2.0 — see LICENSE
+//
+// The model is exercised with faked backends: what matters here is that verbs
+// dispatch on control mode, states derive honestly, and migration folds the
+// legacy keys without forking anyone's ledger.
+
+import { describe, it, beforeEach } from 'node:test';
+import assert from 'node:assert/strict';
+import { mkdtempSync } from 'fs';
+import { join } from 'path';
+import { tmpdir } from 'os';
+import { AxomRuntimes, validateRuntime } from '../src/axom-runtimes.js';
+
+function fakeDaemon() {
+  const calls = { remote: [], server: [], connector: [], nudges: [], rechecks: [] };
+  const broadcasts = [];
+  return {
+    calls,
+    broadcasts,
+    grooveDir: mkdtempSync(join(tmpdir(), 'groove-axrt-')),
+    config: { axom: {} },
+    broadcast(m) { broadcasts.push(m); },
+    audit: { log() {} },
+    axom: {
+      endpoints: new Map(),
+      configure(entries) { calls.connector.push(entries); },
+      nudge(name) { calls.nudges.push(name); },
+      recheck(name) { calls.rechecks.push(name); },
+    },
+    axomRemote: {
+      async status(cfg) { calls.remote.push(['status', cfg]); return fakeDaemon._remoteStatus || { running: false }; },
+      async start(cfg) { calls.remote.push(['start', cfg]); return { started: true }; },
+      async stop(o, cfg) { calls.remote.push(['stop', o, cfg]); return { stopped: true, via: 'shutdown' }; },
+      async ensureTunnel(cfg) { calls.remote.push(['tunnel', cfg]); return { tunneled: true }; },
+    },
+    axomServer: {
+      instances: [],
+      list() { return this.instances; },
+      async start(id, opts) { calls.server.push(['start', id, opts]); return { id, port: 8737, status: 'running' }; },
+      async stop(id) { calls.server.push(['stop', id]); },
+    },
+  };
+}
+
+const SSH_RT = {
+  id: 'spark', name: 'Spark', url: 'http://127.0.0.1:8737', control: 'ssh',
+  ssh: { host: 'spark.local', user: 'axom' },
+  launch: { command: 'cd /x && PYTHONPATH=model python3 -u -m axom.cli serve --cpu' },
+};
+
+describe('AxomRuntimes', () => {
+  let daemon, model;
+
+  beforeEach(() => {
+    daemon = fakeDaemon();
+    model = new AxomRuntimes(daemon);
+    // The real broadcastStatus probes runtime URLs; record the push instead so
+    // these tests stay hermetic. The genuine payload has its own test below.
+    model.broadcastStatus = async () => { daemon.broadcasts.push({ type: 'axom:runtimes' }); };
+  });
+
+  it('migrates the legacy endpoint+remote pair into one ssh runtime', () => {
+    daemon.config.axom = {
+      endpoints: [{ name: 'local', url: 'http://127.0.0.1:8737' }],
+      remote: { host: 'edgexpert-aaa6.local', user: 'axom', port: 8737, command: 'python3 -m axom.cli serve', logPath: '/x.log' },
+    };
+    assert.equal(model.migrate(), true);
+    const [rt] = model.list();
+    assert.equal(rt.control, 'ssh');
+    assert.equal(rt.url, 'http://127.0.0.1:8737');
+    assert.equal(rt.ssh.host, 'edgexpert-aaa6.local');
+    assert.equal(rt.launch.command, 'python3 -m axom.cli serve');
+    assert.equal(model.activeId(), rt.id);
+    assert.equal(model.migrate(), false); // idempotent
+  });
+
+  it('migrates a lone endpoint into a connect-only runtime', () => {
+    daemon.config.axom = { endpoints: [{ name: 'other', url: 'http://127.0.0.1:9999' }] };
+    model.migrate();
+    assert.equal(model.list()[0].control, 'none');
+  });
+
+  it('start dispatches by control: ssh starts remotely and heals the tunnel', async () => {
+    model.add(SSH_RT);
+    await model.startRuntime('spark');
+    const kinds = daemon.calls.remote.map((c) => c[0]);
+    assert.ok(kinds.includes('start'));
+    assert.ok(kinds.includes('tunnel')); // reachability follows lifecycle
+    const cfg = daemon.calls.remote.find((c) => c[0] === 'start')[1];
+    assert.equal(cfg.host, 'spark.local');
+    assert.equal(cfg.command, SSH_RT.launch.command); // spec passed VERBATIM
+  });
+
+  it('start dispatches by control: local spawns and adopts the resulting port as its URL', async () => {
+    model.add({ id: 'here', name: 'This machine', control: 'local', launch: { command: 'axom' }, dataDir: '/home/axom/axom-serve/data' });
+    await model.startRuntime('here');
+    const [, id, opts] = daemon.calls.server[0];
+    assert.equal(id, 'here');
+    assert.equal(opts.dataDir, '/home/axom/axom-serve/data'); // ADOPTS the ledger
+    assert.equal(model.get('here').url, 'http://127.0.0.1:8737');
+  });
+
+  it('refuses lifecycle verbs on connect-only runtimes with an honest message', async () => {
+    model.add({ id: 'theirs', name: 'Theirs', url: 'http://127.0.0.1:7000', control: 'none' });
+    await assert.rejects(() => model.startRuntime('theirs'), /not controlled by GROOVE/);
+    await assert.rejects(() => model.stopRuntime('theirs'), /not controlled by GROOVE/);
+    await assert.rejects(() => model.heal('theirs'), /only ssh runtimes/);
+  });
+
+  it('derives ssh states honestly: tunnel-down vs stopped vs host-unreachable', async () => {
+    model.add({ ...SSH_RT, url: 'http://127.0.0.1:1' }); // nothing listens on :1
+    fakeDaemon._remoteStatus = { running: true };
+    assert.equal((await model.state('spark')).state, 'unreachable'); // up on host, tunnel down
+    fakeDaemon._remoteStatus = { running: false };
+    assert.equal((await model.state('spark')).state, 'stopped');
+    fakeDaemon._remoteStatus = { running: null, error: 'no route to host' };
+    const s = await model.state('spark');
+    assert.equal(s.state, 'unreachable');
+    assert.match(s.detail, /no route/);
+    delete fakeDaemon._remoteStatus;
+  });
+
+  it('reports connected when the connector has the stream', async () => {
+    model.add(SSH_RT);
+    daemon.axom.endpoints.set('spark', { status: 'connected' });
+    assert.equal((await model.state('spark')).state, 'connected');
+  });
+
+  it('status exposes exactly the verbs each state supports', async () => {
+    model.add({ id: 'theirs', name: 'Theirs', url: 'http://127.0.0.1:1', control: 'none' });
+    const { runtimes } = await model.status();
+    const theirs = runtimes.find((r) => r.id === 'theirs');
+    assert.equal(theirs.canStart, false);
+    assert.equal(theirs.canStop, false);
+    assert.equal(theirs.canHeal, false);
+  });
+
+  it('keeps the connector in sync with the runtime list', () => {
+    model.add(SSH_RT);
+    model.remove('spark');
+    const last = daemon.calls.connector.pop();
+    assert.deepEqual(last, []);
+  });
+
+  // Axom-UX flag 1: the GUI's runtime cards must move on events, not a poll.
+  it('pushes fresh runtime state on every mutation and lifecycle verb', async () => {
+    const seen = () => daemon.broadcasts.filter((b) => b.type === 'axom:runtimes').length;
+    model.add(SSH_RT);
+    assert.ok(seen() >= 1);
+    let n = seen();
+    model.update('spark', { name: 'Spark 2' });
+    assert.ok(seen() > n);
+    n = seen();
+    model.activate('spark');
+    assert.ok(seen() > n);
+    n = seen();
+    await model.startRuntime('spark');
+    assert.ok(seen() > n);
+    // Axom-UX flag 3: start pulls the connector in NOW — 'running' must not
+    // linger for a backoff cycle before becoming 'connected'.
+    assert.deepEqual(daemon.calls.nudges, ['spark']);
+    n = seen();
+    await model.stopRuntime('spark');
+    assert.ok(seen() > n);
+    // ...and stop re-probes so a stale 'connected' collapses with the verb.
+    assert.deepEqual(daemon.calls.rechecks, ['spark']);
+    n = seen();
+    model.remove('spark');
+    assert.ok(seen() > n);
+  });
+
+  it('broadcastStatus emits the full axom:runtimes payload', async () => {
+    const d = fakeDaemon();
+    const m = new AxomRuntimes(d);
+    m.add({ id: 'here', name: 'This machine', control: 'local' }); // no URL → no probe
+    const deadline = Date.now() + 2000;
+    while (!d.broadcasts.some((b) => b.type === 'axom:runtimes')) {
+      if (Date.now() > deadline) throw new Error('no axom:runtimes broadcast');
+      await new Promise((r) => setTimeout(r, 10));
+    }
+    const msg = d.broadcasts.find((b) => b.type === 'axom:runtimes');
+    assert.equal(msg.data.activeRuntimeId, 'here');
+    assert.equal(msg.data.runtimes[0].state, 'stopped');
+    assert.equal(msg.data.runtimes[0].canStart, true);
+  });
+});
+
+describe('validateRuntime', () => {
+  it('accepts the three shapes and rejects garbage', () => {
+    assert.equal(validateRuntime(SSH_RT), null);
+    assert.equal(validateRuntime({ id: 'x', name: 'X', control: 'local' }), null); // url comes from spawn
+    assert.equal(validateRuntime({ id: 'y', name: 'Y', url: 'http://127.0.0.1:1', control: 'none' }), null);
+    assert.ok(validateRuntime({ id: 'z', name: 'Z', control: 'teleport' }));
+    assert.ok(validateRuntime({ id: 'z', name: 'Z', control: 'none' })); // none requires url
+    assert.ok(validateRuntime({ id: 'z', name: 'Z', control: 'ssh', url: 'http://127.0.0.1:1', ssh: { host: 'h;rm', user: 'u' } }));
+  });
+});

package/node_modules/@groove-dev/daemon/test/watcher.test.js

@@ -143,9 +143,11 @@ describe('Watcher', () => {
     assert.throws(() => watcher.create('nope', { command: 'x' }), /Agent not found/);
   });
 
+  // Distinct commands — identical ones now re-attach to the running job rather
+  // than stacking up, so they would never reach the cap.
   it('caps active watches per agent', () => {
-    for (let i = 0; i < 5; i++) watcher.create('a1', { command: 'sleep 5', label: `w${i}` });
-    assert.throws(() => watcher.create('a1', { command: 'sleep 5' }), /already have 5/);
+    for (let i = 0; i < 5; i++) watcher.create('a1', { command: `sleep 5 # ${i}`, label: `w${i}` });
+    assert.throws(() => watcher.create('a1', { command: 'sleep 5 # 6' }), /already have 5/);
   });
 
   it('cancels a watch and stops its process', async () => {
@@ -206,6 +208,70 @@ describe('Watcher', () => {
     watcher2.stop();
   });
 
+  // ── no double-launch (regression: daemon rebuild re-ran a live job) ──
+  //
+  // A daemon restart resumes the agent mid-turn, so it re-issues the watch it
+  // believes never completed. Launching a second copy of a long job corrupts
+  // the first one's output and competes for the same hardware.
+
+  it('re-attaches instead of launching a second copy of a running command', async () => {
+    const marker = resolve(grooveDir, 'runs.txt');
+    const cmd = `echo run >> ${marker}; sleep 3`;
+
+    const first = watcher.create('a1', { command: cmd, label: 'champion v11' });
+    await settle(300);
+
+    const second = watcher.create('a1', { command: cmd, label: 'champion v11' });
+    assert.equal(second.id, first.id, 'the same watch is returned, not a new one');
+    assert.equal(second.reattached, true, 'the caller is told it re-attached');
+    assert.equal(
+      watcher.list().filter((w) => w.status === 'active').length, 1,
+      'only one active watch exists',
+    );
+
+    await settle(400);
+    const runs = readFileSync(marker, 'utf8').trim().split('\n').filter(Boolean);
+    assert.equal(runs.length, 1, 'the command executed exactly once');
+  });
+
+  it('does not re-attach once the original job has exited', async () => {
+    const cmd = 'exit 0';
+    const first = watcher.create('a1', { command: cmd, label: 'short' });
+    await settle(500); // job finishes and the watch fires
+
+    const second = watcher.create('a1', { command: cmd, label: 'short again' });
+    assert.notEqual(second.id, first.id, 'a finished job may legitimately be re-run');
+    assert.ok(!second.reattached);
+  });
+
+  it('reports a job that vanished while the daemon was down instead of polling to timeout', async () => {
+    const w = watcher.create('a1', { command: 'sleep 30', label: 'doomed' });
+    await settle(200);
+    watcher._killJob(watcher.watches.get(w.id));   // job dies during the outage
+    watcher.stop();
+    await settle(200);
+
+    const daemon2 = makeDaemon(grooveDir);
+    wireDelivery(daemon2);
+    const watcher2 = new Watcher(daemon2);
+    daemon2.watcher = watcher2;
+    daemon2.registry.add({ id: 'a1', name: 'fullstack-1', role: 'fullstack', provider: 'claude-code' });
+
+    watcher2.restore();
+    await settle(120);
+    assert.equal(daemon2.delivered.length, 1, 'the agent is told the job was lost');
+    assert.match(daemon2.delivered[0].message, /lost while the daemon was down/);
+    watcher2.stop();
+  });
+
+  it('appends to the run log so a re-run cannot erase the record', () => {
+    const w = watcher.create('a1', { command: 'echo hi', label: 'log' });
+    const script = readFileSync(resolve(grooveDir, 'watch-runs', w.id, 'run.sh'), 'utf8');
+    const outLine = script.split('\n').find((l) => l.includes('out.log'));
+    assert.ok(outLine.includes('>>'), 'the output redirect appends');
+    assert.ok(!/[^>]>\s*'[^']*out\.log/.test(outLine), 'the output log is never truncated');
+  });
+
   it('persists watches to disk', () => {
     watcher.create('a1', { command: 'sleep 5', label: 'persisted' });
     const raw = JSON.parse(readFileSync(resolve(grooveDir, 'watches.json'), 'utf8'));