groove-dev 0.27.207 → 0.27.208

package/packages/daemon/src/axom-runtimes.js

@@ -0,0 +1,301 @@
+// GROOVE — Axom Runtime Model (the redesign's one entity)
+// FSL-1.1-Apache-2.0 — see LICENSE
+//
+// A `runtime` is the single thing the GUI reasons about: a named Axom with a
+// URL and a control mode. Everything else (connector endpoints, spawned
+// instances, SSH lifecycle, tunnels) is a backend behind this model.
+//
+//   control: 'local' — a process THIS daemon spawns/kills (AxomServerManager)
+//   control: 'ssh'   — start/stop over SSH on a machine the user owns
+//   control: 'none'  — someone else's runtime; connect-only
+//
+// States, each owning one recovery action (plans/axom-runtime-flow-redesign.md):
+//   connected → events flowing        (verb: stop / disconnect)
+//   running   → /about answers, WS catching up   (no verb needed)
+//   stopped   → reachable, no runtime            (verb: start, if controllable)
+//   unreachable → can't reach the URL/host       (verb: heal tunnel / retry)
+// `unknown` appears only when probing itself failed — never guessed away.
+
+import { validateEndpoint } from './axom-connector.js';
+import { validateRemote } from './axom-remote.js';
+import { saveConfig } from './firstrun.js';
+
+export function validateRuntime(rt) {
+  if (!rt || typeof rt !== 'object') return 'runtime must be an object';
+  if (!rt.id || !/^[a-zA-Z0-9_-]{1,40}$/.test(rt.id)) return 'invalid runtime id';
+  if (!rt.name || typeof rt.name !== 'string' || rt.name.length > 60) return 'invalid runtime name';
+  if (!['local', 'ssh', 'none'].includes(rt.control)) return 'control must be local, ssh, or none';
+  if (rt.control !== 'local' || rt.url) {
+    // local runtimes get their URL from the spawned port; others must have one
+    const problem = validateEndpoint({ name: rt.id, url: rt.url });
+    if (rt.control !== 'local' && problem) return problem;
+  }
+  if (rt.control === 'ssh') {
+    const problem = validateRemote({ port: undefined, ...rt.ssh });
+    if (problem) return `ssh config: ${problem}`;
+  }
+  if (rt.launch !== undefined) {
+    if (typeof rt.launch !== 'object' || typeof rt.launch.command !== 'string'
+      || rt.launch.command.length === 0 || rt.launch.command.length > 500) {
+      return 'launch.command must be a non-empty string of at most 500 chars';
+    }
+  }
+  return null;
+}
+
+export class AxomRuntimes {
+  constructor(daemon) {
+    this.daemon = daemon;
+  }
+
+  // ── Config ────────────────────────────────────────────────────────────────
+
+  _cfg() {
+    if (!this.daemon.config.axom) this.daemon.config.axom = {};
+    return this.daemon.config.axom;
+  }
+
+  list() {
+    return this._cfg().runtimes || [];
+  }
+
+  get(id) {
+    return this.list().find((r) => r.id === id) || null;
+  }
+
+  activeId() {
+    const cfg = this._cfg();
+    return cfg.activeRuntimeId && this.get(cfg.activeRuntimeId)
+      ? cfg.activeRuntimeId
+      : this.list()[0]?.id || null;
+  }
+
+  // One-time, idempotent: fold the four legacy keys into runtimes[]. Old keys
+  // are kept until the model proves out — compat routes still read them.
+  migrate() {
+    const cfg = this._cfg();
+    if (Array.isArray(cfg.runtimes)) return false;
+    const runtimes = [];
+    const remote = cfg.remote || null;
+    const ep = (cfg.endpoints || [])[0] || null;
+    if (ep && remote && ep.url.endsWith(`:${remote.port || 8737}`)) {
+      runtimes.push({
+        id: 'spark', name: remote.host.split('.')[0] || 'Remote', url: ep.url,
+        control: 'ssh',
+        ssh: { host: remote.host, user: remote.user, sshPort: remote.sshPort, autoTunnel: true },
+        launch: remote.command ? { command: remote.command } : undefined,
+        logPath: remote.logPath,
+      });
+    } else if (ep) {
+      let name = 'Axom';
+      try { name = new URL(ep.url).hostname; } catch { /* keep default */ }
+      runtimes.push({ id: ep.name || 'axom', name, url: ep.url, control: 'none' });
+    }
+    cfg.runtimes = runtimes;
+    if (runtimes.length && !cfg.activeRuntimeId) cfg.activeRuntimeId = runtimes[0].id;
+    return true;
+  }
+
+  _save() {
+    saveConfig(this.daemon.grooveDir, this.daemon.config);
+  }
+
+  start() {
+    this.migrate();
+    this._syncConnector();
+  }
+
+  // Every runtime with a URL becomes a connector endpoint — the connector
+  // stays the single owner of event streams.
+  _syncConnector() {
+    const entries = this.list()
+      .filter((r) => r.url)
+      .map((r) => ({ name: r.id, url: r.url }));
+    this.daemon.axom.configure(entries);
+  }
+
+  add(rt) {
+    const problem = validateRuntime(rt);
+    if (problem) throw new Error(problem);
+    if (this.get(rt.id)) throw new Error(`runtime "${rt.id}" already exists`);
+    this._cfg().runtimes = [...this.list(), rt];
+    if (!this._cfg().activeRuntimeId) this._cfg().activeRuntimeId = rt.id;
+    this._save();
+    this._syncConnector();
+    this.broadcastStatus();
+    return rt;
+  }
+
+  update(id, patch) {
+    const existing = this.get(id);
+    if (!existing) throw new Error(`no runtime "${id}"`);
+    const next = { ...existing, ...patch, id };
+    const problem = validateRuntime(next);
+    if (problem) throw new Error(problem);
+    this._cfg().runtimes = this.list().map((r) => (r.id === id ? next : r));
+    this._save();
+    this._syncConnector();
+    this.broadcastStatus();
+    return next;
+  }
+
+  remove(id) {
+    if (!this.get(id)) throw new Error(`no runtime "${id}"`);
+    this._cfg().runtimes = this.list().filter((r) => r.id !== id);
+    if (this._cfg().activeRuntimeId === id) {
+      this._cfg().activeRuntimeId = this.list()[0]?.id || null;
+    }
+    this._save();
+    this._syncConnector();
+    this.broadcastStatus();
+  }
+
+  activate(id) {
+    if (!this.get(id)) throw new Error(`no runtime "${id}"`);
+    this._cfg().activeRuntimeId = id;
+    this._save();
+    this.broadcastStatus();
+  }
+
+  // ── State derivation ──────────────────────────────────────────────────────
+
+  async state(id) {
+    const rt = this.get(id);
+    if (!rt) throw new Error(`no runtime "${id}"`);
+    const ep = rt.url ? this.daemon.axom.endpoints.get(rt.id) : null;
+
+    if (ep?.status === 'connected') {
+      return { state: 'connected', detail: null };
+    }
+    // Probe the URL directly — the connector's backoff may simply not have
+    // caught up yet, and "running" beats a stale "error".
+    if (rt.url) {
+      try {
+        const res = await fetch(`${rt.url}/about`, { signal: AbortSignal.timeout(3000) });
+        if (res.ok) {
+          // It answers — pull the connector in NOW instead of letting its
+          // backoff stretch the 'running' limbo into a felt stall.
+          this.daemon.axom.nudge?.(rt.id);
+          return { state: 'running', detail: 'connecting to event stream' };
+        }
+      } catch (err) {
+        const refused = /ECONNREFUSED/.test(err.cause?.code || err.message || '');
+        if (refused && rt.control !== 'ssh') {
+          return { state: 'stopped', detail: null };
+        }
+      }
+    }
+    if (rt.control === 'ssh') {
+      // The host knows more than the tunnel does.
+      const remote = await this.daemon.axomRemote.status(this._sshCfg(rt));
+      if (remote.running === true) return { state: 'unreachable', detail: 'runtime is up on the host — the tunnel is down' };
+      if (remote.running === false) return { state: 'stopped', detail: null };
+      return { state: 'unreachable', detail: remote.error || `can't reach ${rt.ssh.host}` };
+    }
+    if (rt.control === 'local') {
+      const inst = this.daemon.axomServer.list().find((i) => i.id === rt.id);
+      if (inst?.status === 'running') return { state: 'running', detail: 'connecting to event stream' };
+      return { state: 'stopped', detail: inst?.error || null };
+    }
+    return { state: 'unreachable', detail: 'nothing answers at this endpoint' };
+  }
+
+  _broadcasting = false;
+  async broadcastStatus() {
+    if (this._broadcasting) return; // status() probes; don't stampede
+    this._broadcasting = true;
+    try {
+      this.daemon.broadcast({ type: 'axom:runtimes', data: await this.status() });
+    } catch { /* next state change rebroadcasts */ } finally {
+      this._broadcasting = false;
+    }
+  }
+
+  async status() {
+    const runtimes = await Promise.all(this.list().map(async (rt) => {
+      let derived;
+      try {
+        derived = await this.state(rt.id);
+      } catch (err) {
+        derived = { state: 'unknown', detail: err.message };
+      }
+      const ep = this.daemon.axom.endpoints.get(rt.id);
+      return {
+        id: rt.id,
+        name: rt.name,
+        control: rt.control,
+        url: rt.url || null,
+        ...derived,
+        about: ep?.about || null,
+        error: ep?.error || null,
+        canStart: rt.control !== 'none' && derived.state === 'stopped',
+        canStop: rt.control !== 'none' && (derived.state === 'connected' || derived.state === 'running'),
+        canHeal: rt.control === 'ssh' && derived.state === 'unreachable',
+      };
+    }));
+    return { runtimes, activeRuntimeId: this.activeId() };
+  }
+
+  // ── Verbs — dispatch on control, never guess ─────────────────────────────
+
+  _sshCfg(rt) {
+    let port = 8737;
+    try { port = Number(new URL(rt.url).port) || 8737; } catch { /* default */ }
+    return {
+      ...rt.ssh,
+      port,
+      command: rt.launch?.command,
+      logPath: rt.logPath,
+    };
+  }
+
+  async startRuntime(id) {
+    const rt = this.get(id);
+    if (!rt) throw new Error(`no runtime "${id}"`);
+    if (rt.control === 'none') throw new Error(`"${rt.name}" is not controlled by GROOVE — start it where it runs`);
+    if (rt.control === 'ssh') {
+      const result = await this.daemon.axomRemote.start(this._sshCfg(rt));
+      if (rt.ssh?.autoTunnel !== false) await this.daemon.axomRemote.ensureTunnel(this._sshCfg(rt));
+      this._syncConnector();
+      this.daemon.axom.nudge?.(rt.id);
+      this.broadcastStatus();
+      return result;
+    }
+    // local: the spawned port becomes the runtime's URL.
+    const instance = await this.daemon.axomServer.start(rt.id, {
+      launch: rt.launch,
+      dataDir: rt.dataDir,
+    });
+    this.update(id, { url: `http://127.0.0.1:${instance.port}` });
+    this.daemon.axom.nudge?.(rt.id);
+    return { started: true, port: instance.port };
+  }
+
+  async stopRuntime(id, { force = false } = {}) {
+    const rt = this.get(id);
+    if (!rt) throw new Error(`no runtime "${id}"`);
+    if (rt.control === 'none') throw new Error(`"${rt.name}" is not controlled by GROOVE`);
+    let result;
+    if (rt.control === 'ssh') {
+      result = await this.daemon.axomRemote.stop({ force }, this._sshCfg(rt));
+    } else {
+      await this.daemon.axomServer.stop(rt.id);
+      result = { stopped: true };
+    }
+    // The connector still believes 'connected' until its next poll fails —
+    // re-probe now so the runtime card moves with the verb, not the poll.
+    this.daemon.axom.recheck?.(rt.id);
+    this.broadcastStatus();
+    return result;
+  }
+
+  async heal(id) {
+    const rt = this.get(id);
+    if (!rt) throw new Error(`no runtime "${id}"`);
+    if (rt.control !== 'ssh') throw new Error('only ssh runtimes have a tunnel to heal');
+    const result = await this.daemon.axomRemote.ensureTunnel(this._sshCfg(rt));
+    this.daemon.axom.nudge?.(rt.id);
+    this.broadcastStatus();
+    return result;
+  }
+}

package/packages/daemon/src/axom-server.js

@@ -106,7 +106,14 @@ export class AxomServerManager {
 
   // Start a local instance. `id` doubles as the data-dir name, so the same id
   // across restarts resumes the same sovereign memory.
-  async start(id = 'default') {
+  // opts.launch: {command, cwd?, env?} — a SPEC, not a binary name (source
+  // checkouts launch as `python3 -u -m axom.cli ...` with cwd+env; the
+  // installed default is just `axom`). Specs are verbatim: GROOVE never adds
+  // or removes flags (--cpu on the Spark is policy, not ours to "improve").
+  // opts.dataDir: adopt an existing sovereign ledger instead of minting one
+  // under .groove — one central ledger per user; sessions are recency
+  // scopes, never memory walls (SPARK_DEV_SETUP.md ruling).
+  async start(id = 'default', opts = {}) {
     if (!/^[a-zA-Z0-9_-]{1,40}$/.test(id)) throw new Error('invalid instance id');
     const existing = this.instances.get(id);
     if (existing && existing.status === 'running') return this.list().find((i) => i.id === id);
@@ -124,7 +131,7 @@ export class AxomServerManager {
     }
 
     const port = this._allocatePort();
-    const dataDir = join(this.daemon.grooveDir, 'axom', 'instances', id);
+    const dataDir = opts.dataDir || join(this.daemon.grooveDir, 'axom', 'instances', id);
     mkdirSync(dataDir, { recursive: true });
 
     const args = [
@@ -144,9 +151,21 @@ export class AxomServerManager {
     this.instances.set(id, instance);
     this._broadcast();
 
+    const launch = opts.launch || { command: this._command() };
     let proc;
     try {
-      proc = spawn(this._command(), args, { stdio: ['ignore', 'pipe', 'pipe'] });
+      if (launch.cwd || launch.env || /\s/.test(launch.command)) {
+        // Compound commands run through bash -lc — `nohup cd x && prog`-class
+        // failures taught us a spec is a shell line, not an argv[0].
+        const quotedArgs = args.map((a) => `'${String(a).replace(/'/g, `'\\''`)}'`).join(' ');
+        proc = spawn('bash', ['-lc', `exec ${launch.command} ${quotedArgs}`], {
+          stdio: ['ignore', 'pipe', 'pipe'],
+          cwd: launch.cwd || undefined,
+          env: launch.env ? { ...process.env, ...launch.env } : process.env,
+        });
+      } else {
+        proc = spawn(launch.command, args, { stdio: ['ignore', 'pipe', 'pipe'] });
+      }
     } catch (err) {
       instance.status = 'error';
       instance.error = err.message;
@@ -159,7 +178,7 @@ export class AxomServerManager {
     proc.on('error', (err) => {
       instance.status = 'error';
       instance.error = err.code === 'ENOENT'
-        ? `"${this._command()}" not found — install the Axom runtime first`
+        ? `"${launch.command}" not found — install the Axom runtime first`
         : err.message;
       this._broadcast();
     });

package/packages/daemon/src/index.js

@@ -58,6 +58,7 @@ import { AxomConnector } from './axom-connector.js';
 import { AxomServerManager } from './axom-server.js';
 import { AxomInstaller } from './axom-install.js';
 import { AxomRemote } from './axom-remote.js';
+import { AxomRuntimes } from './axom-runtimes.js';
 import { setProviderPaths } from './providers/index.js';
 
 const DEFAULT_PORT = 31415;
@@ -175,6 +176,7 @@ export class Daemon {
     this.axomServer = new AxomServerManager(this);
     this.axomInstaller = new AxomInstaller(this);
     this.axomRemote = new AxomRemote(this);
+    this.axomRuntimes = new AxomRuntimes(this);
     this.trajectoryCapture = null;
 
     // Hook teams.delete to clean up agent-loop session files
@@ -642,7 +644,7 @@ export class Daemon {
         this.orchestrator.start();
         this.timeline.start();
         this.gateways.start();
-        this.axom.start();
+        this.axomRuntimes.start();
         this.federation.initialize();
         this._startGarbageCollector();
 

package/packages/daemon/src/routes/axom.js

@@ -122,6 +122,79 @@ export function registerAxomRoutes(app, daemon) {
     }
   });
 
+  // ── Runtimes — the one entity (plans/axom-runtime-flow-redesign.md) ─────
+  // The GUI reasons about runtimes only; endpoints/instances/ssh are backends.
+
+  app.get('/api/axom/runtimes', async (req, res) => {
+    res.json(await daemon.axomRuntimes.status());
+  });
+
+  app.post('/api/axom/runtimes', (req, res) => {
+    try {
+      const rt = daemon.axomRuntimes.add(req.body);
+      if (req.body?.activate) daemon.axomRuntimes.activate(rt.id);
+      daemon.audit.log('axom.runtime.add', { id: rt.id, control: rt.control });
+      res.json(rt);
+    } catch (err) {
+      res.status(400).json({ error: err.message });
+    }
+  });
+
+  app.patch('/api/axom/runtimes/:id', (req, res) => {
+    try {
+      res.json(daemon.axomRuntimes.update(req.params.id, req.body || {}));
+    } catch (err) {
+      res.status(400).json({ error: err.message });
+    }
+  });
+
+  app.delete('/api/axom/runtimes/:id', (req, res) => {
+    try {
+      daemon.axomRuntimes.remove(req.params.id);
+      daemon.audit.log('axom.runtime.remove', { id: req.params.id });
+      res.json({ ok: true });
+    } catch (err) {
+      res.status(400).json({ error: err.message });
+    }
+  });
+
+  app.post('/api/axom/runtimes/:id/activate', (req, res) => {
+    try {
+      daemon.axomRuntimes.activate(req.params.id);
+      res.json({ ok: true, activeRuntimeId: req.params.id });
+    } catch (err) {
+      res.status(400).json({ error: err.message });
+    }
+  });
+
+  app.post('/api/axom/runtimes/:id/start', async (req, res) => {
+    try {
+      const result = await daemon.axomRuntimes.startRuntime(req.params.id);
+      daemon.audit.log('axom.runtime.start', { id: req.params.id });
+      res.json(result);
+    } catch (err) {
+      res.status(502).json({ error: err.message });
+    }
+  });
+
+  app.post('/api/axom/runtimes/:id/stop', async (req, res) => {
+    try {
+      const result = await daemon.axomRuntimes.stopRuntime(req.params.id, { force: !!req.body?.force });
+      daemon.audit.log('axom.runtime.stop', { id: req.params.id });
+      res.json(result);
+    } catch (err) {
+      res.status(502).json({ error: err.message });
+    }
+  });
+
+  app.post('/api/axom/runtimes/:id/heal', async (req, res) => {
+    try {
+      res.json(await daemon.axomRuntimes.heal(req.params.id));
+    } catch (err) {
+      res.status(502).json({ error: err.message });
+    }
+  });
+
   // ── Remote runtime control over SSH (manual only, never automatic) ──────
 
   app.get('/api/axom/remote', async (req, res) => {

package/packages/daemon/src/routes/watch.js

@@ -20,6 +20,17 @@ export function registerWatchRoutes(app, daemon) {
       if (!who) return res.status(404).json({ error: `Unknown agent: ${agent}` });
 
       const watch = daemon.watcher.create(who.id, { command, until, label, timeoutMs, intervalMs });
+      if (watch.reattached) {
+        return res.json({
+          ok: true,
+          watchId: watch.id,
+          reattached: true,
+          message: `That command is ALREADY RUNNING under watch ${watch.id} ("${watch.label}") — `
+            + 'this request re-attached to it instead of starting a second copy. '
+            + 'Do NOT launch it again by hand. You will be resumed with the result when it finishes; '
+            + 'you can end your turn now.',
+        });
+      }
       res.json({
         ok: true,
         watchId: watch.id,

package/packages/daemon/src/watcher.js

@@ -59,6 +59,28 @@ export class Watcher {
     if (!command && !until) throw new Error('Provide either "command" (run it) or "until" (poll it)');
     if (command && until) throw new Error('Provide only one of "command" or "until"');
 
+    // Re-attach, never re-execute. A daemon restart resumes the agent mid-turn,
+    // so it re-issues the watch it thinks never completed — and a second copy of
+    // a long job (training, benchmark) launched against the same working dir
+    // corrupts the first one's output and competes for the GPU. If an identical
+    // command is still running for this agent, hand back the existing watch.
+    // This is the daemon-side equivalent of an flock, and it has to live here
+    // because the agent cannot know a duplicate already exists.
+    if (command) {
+      const dup = [...this.watches.values()].find(
+        (w) => w.status === 'active'
+          && w.mode === 'command'
+          && w.command === command
+          && (w.agentName === agent.name || w.agentId === agentId)
+          && this._jobAlive(w),
+      );
+      if (dup) {
+        this.daemon.audit?.log('watch.duplicate', { id: dup.id, agent: agentId, label: dup.label });
+        console.log(`[Groove:Watcher] Re-attached to running watch ${dup.id} instead of re-running: ${dup.label}`);
+        return { ...this._public(dup), reattached: true };
+      }
+    }
+
     const active = [...this.watches.values()].filter((w) => w.agentId === agentId && w.status === 'active');
     if (active.length >= MAX_WATCHES_PER_AGENT) {
       throw new Error(`You already have ${MAX_WATCHES_PER_AGENT} active watches — cancel one before adding another`);
@@ -118,11 +140,14 @@ export class Watcher {
     // Run the command in a SUBSHELL so its own `exit N` can't kill the wrapper
     // before the sentinel is written; tee output to a file, then atomically
     // publish the exit code so the poller never sees a half-written value.
+    // Append rather than truncate: if this script is ever run twice, `>` would
+    // erase the first run's record — exactly the evidence you need to work out
+    // what happened. Appending keeps both.
     const script = [
       '#!/bin/sh',
       '(',
       watch.command,
-      `) > ${shq(watch.outFile)} 2>&1`,
+      `) >> ${shq(watch.outFile)} 2>&1`,
       `echo $? > ${shq(watch.statusFile)}.tmp && mv ${shq(watch.statusFile)}.tmp ${shq(watch.statusFile)}`,
       '',
     ].join('\n');
@@ -143,6 +168,16 @@ export class Watcher {
     }
   }
 
+  // Is this watch's detached job still running? Signal 0 tests for existence
+  // without delivering anything. A finished job (sentinel present) is not alive
+  // even if the pid happens to have been recycled by another process.
+  _jobAlive(watch) {
+    if (watch.mode !== 'command') return false;
+    if (watch.statusFile && existsSync(watch.statusFile)) return false; // already exited
+    if (!watch.pid) return false;
+    try { process.kill(watch.pid, 0); return true; } catch { return false; }
+  }
+
   // One poll iteration — checks for completion (sentinel for command mode, the
   // `until` command for until mode) and wakes the agent when it's met.
   _tick(watch) {
@@ -257,6 +292,7 @@ export class Watcher {
     if (!Array.isArray(data)) return 0;
 
     let rearmed = 0;
+    let lost = 0;
     const now = Date.now();
     for (const w of data) {
       // Drop stale finished watches; keep recent ones for history.
@@ -268,10 +304,27 @@ export class Watcher {
       }
       const watch = { ...w, _poll: null, _deadline: null };
       this.watches.set(watch.id, watch);
+
+      // A command watch whose job is gone with no exit sentinel died while the
+      // daemon was down. Polling it would just burn until the timeout and then
+      // report "may still be running" — say what actually happened instead.
+      if (watch.mode === 'command' && watch.pid
+          && !existsSync(watch.statusFile || '') && !this._jobAlive(watch)) {
+        lost++;
+        this._wake(watch, {
+          outcome: 'error',
+          summary: `The job for "${watch.label}" is no longer running and never recorded an exit code — `
+            + 'it was lost while the daemon was down. Check its output before assuming it finished.',
+          output: tailFile(watch.outFile, OUTPUT_TAIL),
+        });
+        continue;
+      }
+
       this._arm(watch);
       rearmed++;
     }
-    if (rearmed > 0) console.log(`[Groove:Watcher] Restored ${rearmed} active watch(es) after restart`);
+    if (rearmed > 0) console.log(`[Groove:Watcher] Re-attached to ${rearmed} running watch(es) after restart`);
+    if (lost > 0) console.log(`[Groove:Watcher] ${lost} watch(es) lost their job while the daemon was down`);
     this._persist();
     return rearmed;
   }