@monoes/monomindcli 2.7.13 → 2.7.15

package/dist/src/commands/org.js

@@ -1,11 +1,13 @@
 // packages/@monomind/cli/src/commands/org.ts
 import { readFileSync, writeFileSync, existsSync, unlinkSync, rmSync, readdirSync, mkdirSync } from 'node:fs';
-import { join, resolve } from 'node:path';
+import { join, resolve, dirname } from 'node:path';
+import { createHash } from 'node:crypto';
 import { output } from '../output.js';
 import { OrgDaemon } from '../orgrt/daemon.js';
 import { startOrgServer } from '../orgrt/server.js';
 import { ORG_DIR, OrgDefSchema } from '../orgrt/types.js';
 import { migrateOrgFile } from '../orgrt/migrate.js';
+import { readHistory, readRunEvents, summarizeRun } from '../orgrt/reporting.js';
 const log = (text) => { console.log(text); };
 /** Org names are used to build filesystem paths under .monomind/orgs — reject
  * anything that isn't a plain identifier to prevent path traversal (e.g.
@@ -44,6 +46,36 @@ export function listOrgConfigFiles(orgsDir) {
 export const clearStopfile = (cwd, name) => {
     rmSync(join(cwd, ORG_DIR, name, 'stop'), { force: true });
 };
+/** True when a pause sentinel exists for an org. */
+export const isOrgPaused = (cwd, name) => existsSync(join(cwd, ORG_DIR, name, 'pause'));
+const clearPausefile = (cwd, name) => {
+    rmSync(join(cwd, ORG_DIR, name, 'pause'), { force: true });
+};
+/** PID of a live `org serve` daemon for this project, or null.
+ *
+ *  The heartbeat file is written every 30s and removed on clean exit, but a
+ *  SIGKILLed daemon leaves it behind — so liveness is confirmed against the pid
+ *  itself, not the file's presence. A stale heartbeat must not make `org run`
+ *  post a runfile nobody will ever read. */
+function liveServeDaemonPid(cwd) {
+    try {
+        const hb = JSON.parse(readFileSync(join(cwd, '.monomind', 'serve-heartbeat.json'), 'utf8'));
+        if (typeof hb.pid !== 'number' || hb.pid === process.pid)
+            return null;
+        // Freshness as well as liveness. The daemon beats every 30s, so a stamp
+        // older than a few beats means it is gone or wedged — and a pid alone can
+        // be recycled onto an unrelated process, which would send the runfile to
+        // something that will never read it.
+        const age = Date.now() - Date.parse(hb.updatedAt ?? '');
+        if (!Number.isFinite(age) || age > 3 * 60_000)
+            return null;
+        process.kill(hb.pid, 0); // throws if the process is gone
+        return hb.pid;
+    }
+    catch {
+        return null;
+    }
+}
 const runAction = async (ctx) => {
     if (!ctx.args[0])
         return { success: false, message: 'org name required: monomind org run <name> [--task "..."]' };
@@ -96,6 +128,20 @@ const runAction = async (ctx) => {
             return { success: false, message: 'invalid org config' };
         }
     }
+    // A live `org serve` daemon already owns this project's orgs. Starting our
+    // own here would put two processes on one runtime.json and one broker lease,
+    // so hand the request to the daemon via its runfile instead of racing it.
+    const serveOwner = liveServeDaemonPid(ctx.cwd);
+    if (serveOwner != null) {
+        mkdirSync(join(orgsDir, name), { recursive: true });
+        // The task rides along in the runfile. Dropping it here would have made
+        // `org run <name> --task "..."` silently start a generic cycle — the flag
+        // accepted, the instruction discarded.
+        writeFileSync(join(orgsDir, name, 'run'), JSON.stringify({ ts: Date.now(), task: taskFlag ?? null }), 'utf8');
+        log(output.info(`org ${name}: start requested from the serve daemon (pid ${serveOwner}) — it picks this up within ~2s`));
+        log(output.dim(`  watch it with: monomind org logs ${name} --follow`));
+        return { success: true, message: 'start requested' };
+    }
     const crossProcess = ctx.flags['crossProcess'] !== false;
     const daemon = new OrgDaemon(ctx.cwd, { crossProcess });
     let srv;
@@ -193,6 +239,37 @@ const stopAction = async (ctx) => {
     log(output.info(`Stop requested for "${name}" (pid ${rt.pid}) — the daemon picks it up within ~2s.`));
     return { success: true, message: `stop requested for ${name} (daemon exits within 2s)` };
 };
+const pauseAction = async (ctx) => {
+    const validated = validateOrgName(ctx.args[0]);
+    if (!validated.ok)
+        return validated.result;
+    const name = validated.name;
+    if (!existsSync(join(ctx.cwd, ORG_DIR, `${name}.json`))) {
+        log(output.error(`Org not found: ${name}`));
+        return { success: false, message: 'org not found' };
+    }
+    if (isOrgPaused(ctx.cwd, name)) {
+        log(output.warning(`Org "${name}" is already paused.`));
+        return { success: true, message: 'already paused' };
+    }
+    mkdirSync(join(ctx.cwd, ORG_DIR, name), { recursive: true });
+    writeFileSync(join(ctx.cwd, ORG_DIR, name, 'pause'), new Date().toISOString());
+    log(output.info(`Org "${name}" paused — current turns will finish, no new cycles will start. Resume with: monomind org resume ${name}`));
+    return { success: true, message: `org ${name} paused` };
+};
+const resumeAction = async (ctx) => {
+    const validated = validateOrgName(ctx.args[0]);
+    if (!validated.ok)
+        return validated.result;
+    const name = validated.name;
+    if (!isOrgPaused(ctx.cwd, name)) {
+        log(output.warning(`Org "${name}" is not paused.`));
+        return { success: true, message: 'not paused' };
+    }
+    clearPausefile(ctx.cwd, name);
+    log(output.info(`Org "${name}" resumed — next scheduled tick will start a cycle.`));
+    return { success: true, message: `org ${name} resumed` };
+};
 const statusAction = async (ctx) => {
     let name;
     if (ctx.args[0]) {
@@ -247,10 +324,64 @@ const statusAction = async (ctx) => {
                 continue;
             }
         }
-        log(output.info(`${t}: ${state.status}${state.run ? ` (run ${state.run}, pid ${state.pid})` : ''}`));
+        const paused = isOrgPaused(ctx.cwd, t);
+        const statusLabel = paused && state.status === 'running' ? 'running (PAUSED)' : state.status;
+        const line = `${t}: ${statusLabel}${state.run ? ` (run ${state.run}, pid ${state.pid})` : ''}`;
+        // A role that never spawned is a silent capability hole — an org with no
+        // tester still reports a clean "running". Say it on the status line.
+        if (state.abandonedRoles?.length) {
+            log(output.warning(`${line} — DEGRADED: ${state.abandonedRoles.length} role(s) never spawned: ${state.abandonedRoles.join(', ')}`));
+        }
+        else {
+            log(output.info(line));
+        }
+        // Enriched progress for running orgs
+        if (state.status === 'running' && state.run) {
+            const events = readRunEvents(ctx.cwd, t, state.run);
+            if (events.length) {
+                const summary = summarizeRun(events);
+                const elapsed = summary.startedAt ? Date.now() - summary.startedAt : null;
+                const elapsedStr = elapsed !== null ? fmtDuration(elapsed) : '?';
+                const lastTs = events[events.length - 1].ts;
+                const quietMs = Date.now() - lastTs;
+                const quietStr = fmtDuration(quietMs);
+                const toolCalls = Object.values(summary.roles).reduce((a, r) => a + r.toolsAllowed + r.toolsDenied, 0);
+                const rolesUp = Object.keys(summary.roles).filter(r => r !== '(system)').length;
+                log(`  elapsed: ${elapsedStr} | events: ${summary.events} | messages: ${summary.messages} | tools: ${toolCalls}`);
+                log(`  roles active: ${rolesUp} | tokens: ${fmtNum(summary.totalTokens)} | cost: $${summary.totalCostUsd.toFixed(2)}`);
+                log(`  quiet since: ${new Date(lastTs).toISOString().slice(11, 19)} (${quietStr} ago)`);
+                if (summary.crashes.length)
+                    log(output.warning(`  crashes: ${summary.crashes.join(', ')}`));
+            }
+            // Previous cycle comparison from history
+            const history = readHistory(ctx.cwd, t);
+            const prev = history.filter(h => h.run !== state.run).at(-1);
+            if (prev) {
+                const dur = prev.durationMs !== null ? fmtDuration(prev.durationMs) : '?';
+                const outcome = prev.outcome?.status ?? (prev.crashes.length ? 'crashed' : 'completed');
+                log(`  prev cycle: ${dur}, ${outcome}, ${prev.events} events, ${fmtNum(prev.totalTokens)} tokens`);
+            }
+        }
     }
     return { success: true };
 };
+function fmtDuration(ms) {
+    const s = Math.floor(ms / 1000);
+    if (s < 60)
+        return `${s}s`;
+    const m = Math.floor(s / 60);
+    if (m < 60)
+        return `${m}m${s % 60}s`;
+    const h = Math.floor(m / 60);
+    return `${h}h${m % 60}m`;
+}
+function fmtNum(n) {
+    if (n >= 1_000_000)
+        return `${(n / 1_000_000).toFixed(1)}M`;
+    if (n >= 1_000)
+        return `${(n / 1_000).toFixed(1)}k`;
+    return String(n);
+}
 /** One pass of the `org serve` stopfile poll.
  *
  * `monomind org stop <name>` writes `.monomind/orgs/<name>/stop`. `org run` has always
@@ -279,6 +410,161 @@ export const pollStopfiles = async (cwd, daemon) => {
     }
     return stopped;
 };
+/** One pass of the `org serve` runfile poll — the mirror of pollStopfiles.
+ *
+ * A serve daemon owns its orgs, and nothing could ask it to start one off-cycle:
+ * a scheduled org simply waited for its next tick, and `org run` against a
+ * served org would spawn a second daemon competing for the same runtime.json
+ * and broker lease. "Run it now" therefore meant killing and restarting the
+ * daemon, which resets the schedule and drops any in-flight work.
+ *
+ * `.monomind/orgs/<name>/run` is the request. Consumed (deleted) before the
+ * start, so a crash mid-run cannot wedge the org into a restart loop, and an
+ * already-running org just clears it — "start now" on something already started
+ * is satisfied, not an error.
+ *
+ * Returns the names it started, so callers/tests don't have to guess. */
+export const pollRunfiles = async (cwd, daemon) => {
+    const started = [];
+    const orgDir = join(cwd, ORG_DIR);
+    if (!existsSync(orgDir))
+        return started;
+    for (const f of listOrgConfigFiles(orgDir)) {
+        const name = f.replace(/\.json$/, '');
+        const runfile = join(orgDir, name, 'run');
+        if (!existsSync(runfile))
+            continue;
+        // Read before consuming. Pre-runfile writers (and a hand-touched file) left
+        // a bare timestamp or nothing at all, so an unparseable body is a plain
+        // "start it" request, not an error.
+        let task;
+        try {
+            const body = JSON.parse(readFileSync(runfile, 'utf8'));
+            if (typeof body.task === 'string' && body.task.trim())
+                task = body.task;
+        }
+        catch { /* bare/empty runfile — start with the org's own goal */ }
+        try {
+            unlinkSync(runfile);
+        }
+        catch { /* already gone */ }
+        if (daemon.listRunning().includes(name))
+            continue; // already running — request satisfied
+        log(output.info(`org ${name}: run requested — starting now${task ? ' (with task)' : ''}`));
+        try {
+            await daemon.startOrg(name, task);
+            started.push(name);
+        }
+        catch (err) {
+            console.error(`org ${name}: requested start failed:`, err);
+        }
+    }
+    return started;
+};
+/**
+ * Emit a supervisor unit for `org serve`.
+ *
+ * Why this is an EXTERNAL supervisor and not an `--supervise` flag: the daemon
+ * already logs every death it can observe — signals, uncaught exceptions,
+ * unhandled rejections, and the event loop draining. The one death it cannot
+ * observe is SIGKILL, which is what the OOM killer sends, and which is the
+ * suspected cause of the reported disappearance (its org logs showed repeated
+ * low-memory warnings). No in-process handler survives SIGKILL, so a daemon
+ * that restarts itself is theatre for exactly the case that matters. Only
+ * something outside the process can bring it back.
+ *
+ * launchd and systemd both already do this well, so this generates a correct
+ * unit rather than reimplementing them.
+ */
+const supervisorAction = async (ctx) => {
+    const cwd = resolve(ctx.cwd || process.cwd());
+    const requested = String(ctx.flags.format ?? '').trim().toLowerCase();
+    const format = requested || (process.platform === 'darwin' ? 'launchd' : 'systemd');
+    if (format !== 'launchd' && format !== 'systemd') {
+        log(output.error(`Unknown --format "${requested}" — expected launchd or systemd.`));
+        return { success: false, message: 'unknown supervisor format' };
+    }
+    // argv[1] is this CLI's entry point — the same resolution `init` uses when it
+    // spawns a watcher. A supervisor must not depend on PATH or npx resolving to
+    // the same version later.
+    const cliEntry = process.argv[1] ? resolve(process.argv[1]) : 'monomind';
+    const node = process.execPath;
+    // Per-project identity. The unit bakes in a WorkingDirectory, so a constant
+    // Label and filename meant `--install` from a second project silently
+    // OVERWROTE the first project's unit — one file, the first daemon left
+    // unsupervised, no warning. Verified before fixing: installing from projA
+    // then projB left a single unit pointing at projB.
+    //
+    // The hash keeps it unique for two directories with the same basename; the
+    // basename keeps it recognisable in `launchctl list` / `systemctl --user`.
+    const slug = `${(cwd.split(/[\\/]/).pop() || 'org').replace(/[^A-Za-z0-9._-]/g, '-')}-${createHash('sha256').update(cwd).digest('hex').slice(0, 8)}`;
+    const label = `com.monomind.org-serve.${slug}`;
+    const logPath = join(cwd, '.monomind', 'org-serve.log');
+    const unit = format === 'launchd'
+        ? `<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+<plist version="1.0">
+<dict>
+  <key>Label</key><string>${label}</string>
+  <key>ProgramArguments</key>
+  <array>
+    <string>${node}</string>
+    <string>${cliEntry}</string>
+    <string>org</string>
+    <string>serve</string>
+  </array>
+  <key>WorkingDirectory</key><string>${cwd}</string>
+  <!-- KeepAlive restarts the daemon however it died, including SIGKILL. -->
+  <key>KeepAlive</key><true/>
+  <key>RunAtLoad</key><true/>
+  <key>StandardOutPath</key><string>${logPath}</string>
+  <key>StandardErrorPath</key><string>${logPath}</string>
+</dict>
+</plist>
+`
+        : `[Unit]
+Description=monomind org serve (${cwd})
+After=network.target
+
+[Service]
+Type=simple
+WorkingDirectory=${cwd}
+ExecStart=${node} ${cliEntry} org serve
+# Restart however it died, including an OOM kill.
+Restart=always
+RestartSec=5
+StandardOutput=append:${logPath}
+StandardError=append:${logPath}
+
+[Install]
+WantedBy=default.target
+`;
+    const target = format === 'launchd'
+        ? `~/Library/LaunchAgents/${label}.plist`
+        : `~/.config/systemd/user/monomind-org-serve-${slug}.service`;
+    if (ctx.flags.install === true) {
+        const home = process.env.HOME || process.env.USERPROFILE || '';
+        if (!home) {
+            log(output.error('Cannot resolve a home directory to install into — write the unit manually.'));
+            return { success: false, message: 'no home directory' };
+        }
+        const dest = format === 'launchd'
+            ? join(home, 'Library', 'LaunchAgents', `${label}.plist`)
+            : join(home, '.config', 'systemd', 'user', `monomind-org-serve-${slug}.service`);
+        mkdirSync(dirname(dest), { recursive: true });
+        writeFileSync(dest, unit);
+        log(output.success(`Wrote ${dest}`));
+        log(output.info(format === 'launchd'
+            ? `Load it with: launchctl load -w ${dest}`
+            : `Load it with: systemctl --user daemon-reload && systemctl --user enable --now monomind-org-serve-${slug}`));
+        return { success: true, message: `supervisor unit written to ${dest}` };
+    }
+    log(unit);
+    log(output.info(`Write this to ${target}, or re-run with --install to do it for you.`));
+    log(output.info('Why a supervisor: the daemon logs every death it can observe, but an OOM kill is SIGKILL — ' +
+        'uncatchable by design, so nothing in-process can restart after one.'));
+    return { success: true, message: `${format} unit emitted` };
+};
 const serveAction = async (ctx) => {
     const crossProcess = ctx.flags['crossProcess'] !== false;
     const daemon = new OrgDaemon(ctx.cwd, { crossProcess });
@@ -352,8 +638,17 @@ const serveAction = async (ctx) => {
     // schedule orgs whose definition declares an interval (e.g. "15m", "2h")
     const { OrgScheduler, parseSchedule } = await import('../orgrt/scheduler.js');
     const sched = new OrgScheduler(async (name, intervalMs) => {
+        if (isOrgPaused(ctx.cwd, name))
+            return;
+        // Only ever stop a run THIS tick started. The runfile poll can start an org
+        // out-of-band, and the scheduler has no visibility into that — so a tick
+        // landing on an already-running org threw "already running", fell into the
+        // finally, and stopped a healthy run that had nothing to do with it. The
+        // tick's job in that case is simply to yield.
+        let startedHere = false;
         try {
             await daemon.startOrg(name);
+            startedHere = true;
             // Scheduled iterations are time-bounded: agents' `done` promises only
             // resolve after stopOrg closes the mailboxes, so waiting on them alone
             // deadlocks. Race against a max-run timeout, then ALWAYS stopOrg
@@ -363,7 +658,16 @@ const serveAction = async (ctx) => {
                 ? Promise.allSettled([...org.agents.values()].map(a => a.done))
                 : Promise.resolve([]);
             const maxRun = org?.def?.run_config?.max_run;
-            const maxMs = parseSchedule(maxRun) ?? Math.min(intervalMs, 600_000); // cap: schedule interval or 10 min
+            // Default to the full interval. The old `min(interval, 10min)` clamp
+            // silently guillotined every org that didn't set max_run: real cycles
+            // here run 75-93 minutes, so a 2h-scheduled org was being force-stopped
+            // a twelfth of the way in, every time, with nothing saying so. Ten
+            // minutes was never a considered bound for agent work — it only looked
+            // safe because overrunning the interval used to cost a whole idle
+            // period. Now that a missed tick catches up the moment a run ends
+            // (OrgScheduler.pending), a run may safely use its whole interval.
+            // Set run_config.max_run to bound it tighter — or looser.
+            const maxMs = parseSchedule(maxRun) ?? intervalMs;
             let timer;
             await Promise.race([allDone, new Promise(r => {
                     timer = setTimeout(r, maxMs);
@@ -376,7 +680,12 @@ const serveAction = async (ctx) => {
             console.error(`org ${name}: scheduled run failed:`, err);
         }
         finally {
-            await daemon.stopOrg(name).catch(err => console.error(`org ${name}: stop failed:`, err));
+            // A deadline stop still lands on agents mid-tool-call. The 15s abort
+            // bound threw that work away; a minute is enough to finish an edit or a
+            // test run and flush, and still well inside any sane interval.
+            if (startedHere) {
+                await daemon.stopOrg(name, { drainMs: 60_000 }).catch(err => console.error(`org ${name}: stop failed:`, err));
+            }
         }
     });
     const orgDir = join(ctx.cwd, ORG_DIR);
@@ -390,8 +699,16 @@ const serveAction = async (ctx) => {
                     const stem = f.replace(/\.json$/, '');
                     if (def.name && def.name !== stem)
                         log(output.warning(`org file ${f}: def.name "${def.name}" differs from filename — scheduling as "${stem}"`));
-                    sched.add(stem, ms);
-                    log(output.info(`scheduled org ${stem} every ${Math.round(ms / 60_000)}m`));
+                    // Due = never run, or last run ended longer ago than the interval.
+                    // Without this, starting the daemon meant waiting a full period
+                    // before anything happened at all; gating on due-ness means a
+                    // restart doesn't stampede every scheduled org back into a run.
+                    const lastEnded = readHistory(ctx.cwd, stem).at(-1)?.endedAt ?? 0;
+                    const since = lastEnded ? Date.now() - lastEnded : undefined;
+                    const due = (since ?? Infinity) >= ms;
+                    sched.add(stem, ms, due, since);
+                    const waitMin = due ? 0 : Math.round((ms - (since ?? 0)) / 60_000);
+                    log(output.info(`scheduled org ${stem} every ${Math.round(ms / 60_000)}m${due ? ' — due now, starting first run' : ` — next run in ~${waitMin}m`}`));
                 }
             }
             catch (err) {
@@ -401,8 +718,11 @@ const serveAction = async (ctx) => {
     }
     const stopPoll = setInterval(() => { void pollStopfiles(ctx.cwd, daemon); }, 2000);
     stopPoll.unref?.();
+    const runPoll = setInterval(() => { void pollRunfiles(ctx.cwd, daemon); }, 2000);
+    runPoll.unref?.();
     await new Promise(r => { process.once('SIGINT', () => r()); process.once('SIGTERM', () => r()); });
     clearInterval(stopPoll);
+    clearInterval(runPoll);
     clearInterval(heartbeatInterval);
     sched.stop();
     await daemon.stopAll();
@@ -602,9 +922,15 @@ const markCompleteAction = async (ctx) => {
             auth = readFileSync(join(cwd, '.monomind', 'dashboard-token'), 'utf8').trim();
         }
         catch { /* server may be pre-auth */ }
+        // Bounded. Updating the dashboard is best-effort — the local state has
+        // already been cleared by this point — but the fetch had no timeout, so a
+        // dashboard that holds the port without answering (a wedged build from an
+        // earlier session; see the stale-dashboard issue) hung `mark-complete`
+        // indefinitely. "Unreachable" and "not answering" must cost the same.
         const res = await fetch(`${ctrlUrl}/api/orgs/${encodeURIComponent(orgName)}/mark-complete`, {
             method: 'POST',
             headers: auth ? { 'x-monomind-token': auth } : {},
+            signal: AbortSignal.timeout(5_000),
         });
         const body = await res.json().catch(() => ({}));
         if (!res.ok) {
@@ -671,6 +997,8 @@ export const orgCommand = {
             action: runAction,
         },
         { name: 'stop', description: 'Request a running org daemon to stop', action: stopAction },
+        { name: 'pause', description: 'Pause an org — current turns finish, no new cycles start', action: pauseAction },
+        { name: 'resume', description: 'Resume a paused org', action: resumeAction },
         { name: 'status', description: 'Show runtime state of orgs', action: statusAction },
         {
             name: 'serve', description: 'Start the daemon server only (hosts scheduled orgs)',
@@ -679,6 +1007,15 @@ export const orgCommand = {
             ],
             action: serveAction,
         },
+        {
+            name: 'supervisor',
+            description: 'Print (or --install) a launchd/systemd unit that keeps `org serve` running',
+            options: [
+                { name: 'format', description: 'launchd or systemd (default: platform)', type: 'string' },
+                { name: 'install', description: 'Write the unit into the per-user location', type: 'boolean' },
+            ],
+            action: supervisorAction,
+        },
         {
             name: 'test-loop', description: 'Run the org e2e verification loop N times',
             options: [{ name: 'times', short: 'n', description: 'Iterations', type: 'number', default: 5 }],