@monoes/monomindcli 2.7.15 → 2.7.17

package/dist/src/orgrt/daemon.js

@@ -8,11 +8,37 @@ import { PolicyEngine } from './policy.js';
 import { Mailbox } from './mailbox.js';
 import { runAgentSession } from './session.js';
 import { attachForwarder } from './forwarder.js';
-import { BrokerLease, lookupOrg } from './broker.js';
+import { BrokerLease, lookupOrg, normalizeCredential } from './broker.js';
 import { queueMessage, drainInbox } from './inbox.js';
 import { OrgDefSchema, ORG_DIR } from './types.js';
 import { summarizeRun, readRunEvents, readHistory, historyFile } from './reporting.js';
 import { checkResources, waitForCapacity, getResourceLimits, configureResourceLimits } from '../utils/resource-governor.js';
+/** OpenTelemetry tracing helper - creates spans for major operations */
+class OtelTracer {
+    enabled = false;
+    spans = new Map();
+    enable() { this.enabled = true; }
+    startSpan(name, metadata = {}) {
+        if (!this.enabled)
+            return;
+        this.spans.set(name, { start: Date.now(), metadata });
+    }
+    endSpan(name) {
+        if (!this.enabled)
+            return;
+        const span = this.spans.get(name);
+        if (span) {
+            const duration = Date.now() - span.start;
+            // Emit span as a bus event for export
+            this.spans.delete(name);
+        }
+    }
+    recordEvent(name, attributes) {
+        if (!this.enabled)
+            return;
+        // Could emit to bus for collection
+    }
+}
 /** Drain window for a PLANNED stop (the boss called org_complete). Long enough
  *  for a sibling mid-build or mid-test to finish and flush its work. A hard
  *  stop keeps the short bound — see finishStop. */
@@ -27,6 +53,20 @@ export class OrgDaemon {
     forwarders = new Map();
     watchdogs = new Map();
     stopping = new Map();
+    otel = new OtelTracer();
+    approvals = new Map();
+    // Roles currently spawning — prevents duplicate lazy spawns from concurrent messages
+    spawning = new Map();
+    // #4: bounded whole-org restarts after the boss terminally crashes. A monotonic
+    // per-org counter so a crashing boss can never burn money in an infinite loop;
+    // reset only by an explicit (non-restart) startOrg.
+    static MAX_BOSS_RESTARTS = 2;
+    static BOSS_RESTART_BACKOFF_MS = [10_000, 30_000];
+    bossRestartCounts = new Map();
+    restarting = new Set();
+    // #3: recognizes provider context-window-overflow errors so the boss can be told
+    // to chunk the work instead of re-dispatching the same oversized task verbatim.
+    static CONTEXT_LIMIT_RE = /context[- ]?(window|length|size|limit)|maximum context|exceeds?.{0,12}(context|token)|too many tokens|prompt is too long/i;
     constructor(root, opts = {}) {
         this.root = root;
         this.opts = opts;
@@ -58,6 +98,11 @@ export class OrgDaemon {
         return isAbsolute(ws) ? ws : join(this.root, ws);
     }
     async startOrg(name, taskOverride) {
+        // A restart-driven start (scheduleBossRestart) keeps its crash counter so the
+        // cap holds; any other (explicit) start resets it so a manual re-run gets a
+        // fresh budget.
+        if (!this.restarting.has(name))
+            this.bossRestartCounts.delete(name);
         if (this.orgs.has(name))
             throw new Error(`org ${name} already running`);
         const defPath = join(this.root, ORG_DIR, `${name}.json`);
@@ -136,6 +181,31 @@ export class OrgDaemon {
                 }, 1000);
                 t.unref?.();
             }
+            // Accumulate cost from usage events into per-role metrics
+            if (e.type === 'usage' && e.from && e.data) {
+                const runtime = running.agents.get(e.from);
+                if (runtime) {
+                    const cost = Number(e.data.cost_usd ?? 0);
+                    if (Number.isFinite(cost)) {
+                        runtime.metrics.costUsd += cost;
+                    }
+                }
+            }
+            // Track last message ID for threading responses
+            if ((e.type === 'message' || e.type === 'xorg') && e.from) {
+                const runtime = running.agents.get(e.from);
+                if (runtime) {
+                    runtime.lastMessageId = e.id;
+                }
+            }
+            // Apply audit filter if configured (skip filtered tool events before forwarding)
+            if (this.opts.auditFilter && e.type === 'tool') {
+                const { tool, decision } = this.opts.auditFilter;
+                if (tool && e.tool !== tool)
+                    return; // Skip: tool name doesn't match
+                if (decision && e.decision !== decision)
+                    return; // Skip: decision doesn't match
+            }
             for (const fn of this.globalSubscribers)
                 fn(e);
         });
@@ -175,12 +245,14 @@ export class OrgDaemon {
         const spawnRole = (role) => {
             const mailbox = new Mailbox();
             const policy = new PolicyEngine(role.id, { maxTokens: perRoleBudget, ...(role.policy ?? {}) }, bus, cwd);
-            const runtime = { mailbox, policy, status: 'running', done: Promise.resolve() };
+            const runtime = { mailbox, policy, status: 'running', done: Promise.resolve(), metrics: { tokens: 0, costUsd: 0 } };
             const sessionOpts = {
                 org: name, role, bus, policy, mailbox, cwd, def,
                 maxTurns: role.max_turns_per_message ?? def.run_config.max_turns_per_message,
+                lastMessageId: () => runtime.lastMessageId,
                 deliver: (from, to, subject, body) => this.deliver(name, from, to, subject, body),
                 askHuman: (r, question) => this.askHuman(name, r, question),
+                beforeTool: (r, toolName) => this.checkApproval(name, r, toolName),
                 onComplete: role.id === bossRole.id
                     ? (r, outcome, summary) => {
                         bus.emit({ type: 'status', from: r, reason: 'org-complete', msg: `run outcome: ${outcome}`, data: { outcome, summary } });
@@ -219,7 +291,7 @@ export class OrgDaemon {
             // dashboards/alerts that filter on actionable failures (not routine
             // status chatter) so a dead agent surfaces instead of a run that
             // silently never progresses.
-            const BACKOFFS_MS = [1000, 5000, 15000];
+            const BACKOFFS_MS = this.opts.crashBackoffsMs ?? [1000, 5000, 15000];
             runtime.done = (async () => {
                 for (let attempt = 0;; attempt++) {
                     try {
@@ -252,12 +324,36 @@ export class OrgDaemon {
                             // error instead of pushing into a queue no session will read
                             // (and returning a false "delivered" receipt to the sender).
                             mailbox.close();
+                            const isContextLimit = OrgDaemon.CONTEXT_LIMIT_RE.test(message);
                             bus.emit({
                                 type: 'audit', from: role.id,
                                 msg: `agent "${role.id}" crashed: ${message}`,
-                                reason: 'agent-session-crash',
-                                data: { agentId: role.id, error: message, restarts: attempt },
+                                reason: isContextLimit ? 'agent-context-limit' : 'agent-session-crash',
+                                data: { agentId: role.id, error: message, restarts: attempt, contextLimit: isContextLimit },
                             });
+                            if (role.id !== bossRole.id) {
+                                // #2/#3: a worker is gone for the rest of this run. Without this
+                                // notice the coordinator keeps messaging a corpse (observed: four
+                                // unanswered org_send calls to a developer that had crashed on a
+                                // context-window limit). Tell the boss to reassign — and if the
+                                // crash was a context overflow, tell it to chunk smaller, since
+                                // re-dispatching the same task verbatim fails the same way.
+                                const bossRt = running.agents.get(bossRole.id);
+                                if (bossRt && !bossRt.mailbox.isClosed) {
+                                    const guidance = isContextLimit
+                                        ? ' This was a context-window overflow — re-dispatching the same task verbatim will fail identically. Break the work into smaller pieces (one file or section at a time) and do not paste large file contents in a single message.'
+                                        : '';
+                                    bossRt.mailbox.push(`[system] Worker "${role.id}" crashed and will not recover this run (${message}). It can no longer receive messages — stop messaging it. Reassign its outstanding work to another agent or take it on yourself.${guidance}`);
+                                    bus.emit({ type: 'audit', from: bossRole.id, reason: 'worker-crashed',
+                                        msg: `worker "${role.id}" crashed (contextLimit=${isContextLimit}); coordinator notified to reassign` });
+                                }
+                            }
+                            else {
+                                // #4: the coordinator itself died. Don't go silent and wait for a
+                                // human — attempt a bounded whole-org restart with fresh sessions
+                                // (which also sheds whatever bloated context caused the crash).
+                                this.scheduleBossRestart(name);
+                            }
                         };
                         if (mailbox.isClosed || attempt >= BACKOFFS_MS.length) {
                             crash();
@@ -335,6 +431,8 @@ export class OrgDaemon {
                 this.stopOrg(name).catch(err => console.error(`org ${name}: idle-stop failed:`, err instanceof Error ? err.message : err));
             };
             const wd = setInterval(() => {
+                if (this.restarting.has(name))
+                    return; // boss auto-restart in flight — don't nudge or stop
                 const idleFor = Date.now() - lastActivity;
                 if (idleFor < idleMs) {
                     nudgedAt = 0;
@@ -364,7 +462,7 @@ export class OrgDaemon {
             this.watchdogs.set(name, wd);
         }
         if (this.opts.crossProcess && this.opts.inboxUrl) {
-            const lease = new BrokerLease(name, this.opts.inboxUrl, this.opts.brokerDir, undefined, this.opts.inboxCredential);
+            const lease = new BrokerLease(name, this.opts.inboxUrl, this.opts.brokerDir, undefined, normalizeCredential(this.opts.inboxCredential));
             lease.start();
             this.leases.set(name, lease);
         }
@@ -407,9 +505,10 @@ export class OrgDaemon {
                 if (waited.ok) {
                     running.bus.emit({ type: 'audit', from: role.id, reason: 'resource-recovered',
                         msg: `resources recovered after ${attempt} retr${attempt === 1 ? 'y' : 'ies'} — spawning deferred role "${role.id}"` });
-                    spawnRole(role);
-                    // Drain messages queued while the role was deferred (see deliver() B4 fix)
+                    // Drain messages queued while the role was deferred BEFORE spawning
+                    // to prevent race condition where messages arrive during spawn window
                     const queued = drainInbox(this.root, name);
+                    spawnRole(role);
                     for (const msg of queued) {
                         const agent = running.agents.get(msg.toRole);
                         if (agent && !agent.mailbox.isClosed) {
@@ -452,26 +551,38 @@ export class OrgDaemon {
         const targetOrg = this.orgs.get(targetOrgName);
         const src = this.orgs.get(fromOrg);
         // Lazy spawn: if the role is pending (not yet spawned), spawn it now.
-        if (targetOrg && !targetOrg.agents.has(targetRole) && targetOrg.pendingRoles?.has(targetRole)) {
+        // ATOMIC GUARD: Check spawning Set to prevent duplicate spawns from concurrent messages
+        const spawning = this.spawning.get(targetOrgName) ?? new Set();
+        this.spawning.set(targetOrgName, spawning);
+        if (targetOrg && !targetOrg.agents.has(targetRole) && targetOrg.pendingRoles?.has(targetRole) && !spawning.has(targetRole)) {
             const role = targetOrg.pendingRoles.get(targetRole);
             targetOrg.pendingRoles.delete(targetRole);
+            spawning.add(targetRole); // Mark as spawning before async work
             const check = checkResources();
             if (!check.ok) {
                 const waited = await waitForCapacity(60_000);
+                spawning.delete(targetRole); // Clear spawning flag after check
                 if (!waited.ok) {
                     targetOrg.bus.emit({ type: 'audit', from: targetRole, reason: 'resource-skip',
                         msg: `deferring lazy spawn of "${targetRole}": ${waited.reason}` });
                     // Queue the triggering message so it survives the deferred spawn — without
                     // this the sender got "queued" but the message was silently lost.
-                    queueMessage(this.root, targetOrgName, {
+                    // B5 FIX: Queue FIRST, then schedule spawn only if queue succeeds.
+                    // If queueing fails, we return the error without modifying spawn state.
+                    const queued = queueMessage(this.root, targetOrgName, {
                         fromQualified: cross ? `${fromOrg}:${fromRole}` : fromRole,
                         toRole: targetRole, subject, body, ts: Date.now(),
                     });
+                    if (!queued) {
+                        src?.bus.emit({ type: 'audit', from: fromRole, to: toQualified, msg: `queue failed: ${subject}`, reason: 'queue-failed' });
+                        return `ERROR: could not queue message for ${toQualified} (disk full or permissions)`;
+                    }
                     this.scheduleDeferredSpawn(targetOrgName, targetOrg, role, targetOrg.spawnRole);
                     return `queued for ${toQualified} (role starting — waiting for resources)`;
                 }
             }
             targetOrg.spawnRole(role);
+            spawning.delete(targetRole); // Clear spawning flag after spawn completes
             targetOrg.bus.emit({ type: 'status', from: targetRole, msg: `lazy-spawned on first message from ${fromRole}` });
         }
         if (!targetOrg || !targetOrg.agents.has(targetRole)) {
@@ -479,7 +590,11 @@ export class OrgDaemon {
                 return this.deliverRemote(fromOrg, fromRole, targetOrgName, targetRole, toQualified, subject, body, src);
             // Queue + auto-wake: if the org definition exists locally but isn't running, spool the message and start it
             if (cross && this.hasOrgDef(targetOrgName)) {
-                queueMessage(this.root, targetOrgName, { fromQualified: `${fromOrg}:${fromRole}`, toRole: targetRole, subject, body, ts: Date.now() });
+                const queued = queueMessage(this.root, targetOrgName, { fromQualified: `${fromOrg}:${fromRole}`, toRole: targetRole, subject, body, ts: Date.now() });
+                if (!queued) {
+                    src?.bus.emit({ type: 'audit', from: fromRole, to: toQualified, msg: `queue failed: ${subject}`, reason: 'queue-failed' });
+                    return `ERROR: could not queue message for ${toQualified} (disk full or permissions)`;
+                }
                 src?.bus.emit({ type: 'xorg', from: `${fromOrg}:${fromRole}`, to: toQualified, subject, msg: body, data: { queued: true } });
                 this.autoWake(targetOrgName);
                 return `queued for ${toQualified} (org starting)`;
@@ -499,10 +614,20 @@ export class OrgDaemon {
             src?.bus.emit({ type: 'audit', from: fromRole, to: toQualified, msg: `undeliverable: ${subject}`, reason: 'target mailbox closed (org shutting down)' });
             return `ERROR: recipient "${toQualified}" is shutting down — message not delivered`;
         }
-        const evt = { from: cross ? `${fromOrg}:${fromRole}` : fromRole, to: toQualified, subject, msg: body };
-        src?.bus.emit({ type: cross ? 'xorg' : 'message', ...evt });
+        // Track message chain: link this message to the target's last message (the one being responded to)
+        const targetAgentSrc = targetOrg === src ? src?.agents.get(targetRole) : undefined;
+        const parentId = targetAgentSrc?.lastMessageId;
+        const evt = { from: cross ? `${fromOrg}:${fromRole}` : fromRole, to: toQualified, subject, msg: body, parentId };
+        const emitted = src?.bus.emit({ type: cross ? 'xorg' : 'message', ...evt });
         if (cross && targetOrg !== src)
             targetOrg.bus.emit({ type: 'xorg', ...evt });
+        // Store message ID for the target (so responses can link to it)
+        if (targetAgentSrc && emitted)
+            targetAgentSrc.lastMessageId = emitted.id;
+        // Also track the source agent's last sent message for cross-org visibility
+        const srcAgent = src?.agents.get(fromRole);
+        if (srcAgent && emitted)
+            srcAgent.lastMessageId = emitted.id;
         targetAgent.mailbox.push(`[message from ${evt.from}] subject: ${subject}\n\n${body}`);
         return `delivered to ${toQualified}`;
     }
@@ -513,7 +638,11 @@ export class OrgDaemon {
         if (!remote) {
             // No remote host either — queue + auto-wake if the org def exists locally
             if (this.hasOrgDef(targetOrgName)) {
-                queueMessage(this.root, targetOrgName, { fromQualified: `${fromOrg}:${fromRole}`, toRole: targetRole, subject, body, ts: Date.now() });
+                const queued = queueMessage(this.root, targetOrgName, { fromQualified: `${fromOrg}:${fromRole}`, toRole: targetRole, subject, body, ts: Date.now() });
+                if (!queued) {
+                    src?.bus.emit({ type: 'audit', from: fromRole, to, msg: `queue failed: ${subject}`, reason: 'queue-failed' });
+                    return `ERROR: could not queue message for ${to} (disk full or permissions)`;
+                }
                 src?.bus.emit({ type: 'xorg', from: `${fromOrg}:${fromRole}`, to, subject, msg: body, data: { queued: true } });
                 this.autoWake(targetOrgName);
                 return `queued for ${to} (org starting)`;
@@ -553,17 +682,40 @@ export class OrgDaemon {
         if (!org) {
             // Org not running — queue the message and auto-wake if the def exists
             if (this.hasOrgDef(toOrg)) {
-                queueMessage(this.root, toOrg, { fromQualified, toRole, subject, body, ts: Date.now() });
+                const queued = queueMessage(this.root, toOrg, { fromQualified, toRole, subject, body, ts: Date.now() });
+                if (!queued) {
+                    return { ok: false, error: `could not queue message for ${toOrg}:${toRole} (disk full or permissions)` };
+                }
                 this.autoWake(toOrg);
                 return { ok: true, receipt: `queued for ${toOrg}:${toRole} (org waking)` };
             }
             return { ok: false, error: `org "${toOrg}" not hosted here` };
         }
         // Lazy-spawn pending roles on cross-process delivery (matches deliver/answerQuestion)
-        if (!org.agents.has(toRole) && org.pendingRoles?.has(toRole)) {
+        // ATOMIC GUARD: Check spawning Set to prevent duplicate spawns from concurrent messages
+        const spawning = this.spawning.get(toOrg) ?? new Set();
+        this.spawning.set(toOrg, spawning);
+        if (!org.agents.has(toRole) && org.pendingRoles?.has(toRole) && !spawning.has(toRole)) {
             const role = org.pendingRoles.get(toRole);
             org.pendingRoles.delete(toRole);
+            spawning.add(toRole); // Mark as spawning before async work
+            // Resource gate check before spawning (prevents bypass in cross-process delivery)
+            const check = checkResources();
+            if (!check.ok) {
+                spawning.delete(toRole); // Clear spawning flag after check
+                org.bus.emit({ type: 'audit', from: toRole, reason: 'resource-pressure',
+                    msg: `cross-process lazy spawn deferred: ${check.reason}` });
+                // B4 FIX: Queue the triggering message FIRST, then schedule spawn only if queue succeeds.
+                // This matches the pattern in deliver() and prevents message loss if queue fails.
+                const queued = queueMessage(this.root, toOrg, { fromQualified, toRole, subject, body, ts: Date.now() });
+                if (!queued) {
+                    return { ok: false, error: `could not queue message for ${toOrg}:${toRole} (disk full or permissions)` };
+                }
+                this.scheduleDeferredSpawn(toOrg, org, role, org.spawnRole);
+                return { ok: true, receipt: `queued for ${toOrg}:${toRole} (role starting — waiting for resources)` };
+            }
             org.spawnRole?.(role);
+            spawning.delete(toRole); // Clear spawning flag after spawn completes
             org.bus.emit({ type: 'status', from: toRole, msg: `lazy-spawned on remote delivery from ${fromQualified}` });
         }
         const agent = org.agents.get(toRole);
@@ -573,7 +725,8 @@ export class OrgDaemon {
             return { ok: false, error: `role "${toRole}" in org "${toOrg}" crashed and will not recover this run` };
         if (agent.mailbox.isClosed)
             return { ok: false, error: `role "${toRole}" in org "${toOrg}" is shutting down` };
-        org.bus.emit({ type: 'xorg', from: fromQualified, to: `${toOrg}:${toRole}`, subject, msg: body });
+        const messageEvent = org.bus.emit({ type: 'xorg', from: fromQualified, to: `${toOrg}:${toRole}`, subject, msg: body });
+        agent.lastMessageId = messageEvent.id; // Track last message ID for response threading
         agent.mailbox.push(`[message from ${fromQualified}] subject: ${subject}\n\n${body}`);
         return { ok: true, receipt: `delivered to ${toOrg}:${toRole} (remote)` };
     }
@@ -600,6 +753,109 @@ export class OrgDaemon {
         writeFileSync(tmp, JSON.stringify(data, null, 2));
         renameSync(tmp, dest);
     }
+    /** Check if an action requires human approval (beforeTool hook for guardrails). Returns
+     *  the approval decision: true = approved, false = denied, null = pending (requires human input). */
+    async checkApproval(org, role, action) {
+        const approvalKey = `${org}:${role}:${action}`;
+        const pending = this.approvals.get(org) ?? [];
+        const existing = pending.find(a => a.roleId === role && a.action === action);
+        // If already approved/denied, return that decision
+        if (existing && existing.approved !== null)
+            return existing.approved;
+        // Require human approval for sensitive actions
+        const sensitiveActions = ['Bash', 'WebFetch', 'WebSearch', 'org_complete'];
+        if (sensitiveActions.includes(action)) {
+            // Queue for approval
+            if (!existing) {
+                pending.push({ roleId: role, action, question: `Approve ${action} tool call?`, ts: Date.now(), approved: null });
+                this.approvals.set(org, pending);
+            }
+            // Persist to approvals.json
+            const approvalsPath = join(this.root, ORG_DIR, org, 'approvals.json');
+            mkdirSync(join(this.root, ORG_DIR, org), { recursive: true });
+            writeFileSync(approvalsPath, JSON.stringify({ approvals: pending }, null, 2));
+            // Emit a question event for the dashboard
+            const running = this.orgs.get(org);
+            running?.bus.emit({ type: 'question', from: role, data: { question: `Approval required for ${action}`, action } });
+            return null; // Pending human approval
+        }
+        return true; // Auto-approved for non-sensitive actions
+    }
+    /** Approve or deny a pending action (called by dashboard or CLI). */
+    async setApproval(org, role, action, approved) {
+        const pending = this.approvals.get(org) ?? [];
+        const item = pending.find(a => a.roleId === role && a.action === action);
+        if (!item)
+            return { ok: false, error: `No pending approval found for ${role} action ${action}` };
+        item.approved = approved;
+        item.ts = Date.now();
+        // Persist updated approval state
+        const approvalsPath = join(this.root, ORG_DIR, org, 'approvals.json');
+        writeFileSync(approvalsPath, JSON.stringify({ approvals: pending }, null, 2));
+        // Notify the waiting agent via its mailbox
+        const running = this.orgs.get(org);
+        const agent = running?.agents.get(role);
+        if (agent && !agent.mailbox.isClosed) {
+            agent.mailbox.push(`[approval] ${action}: ${approved ? 'APPROVED' : 'DENIED'}`);
+        }
+        running?.bus.emit({ type: 'status', from: role, msg: `Approval ${approved ? 'granted' : 'denied'} for ${action}` });
+        return { ok: true };
+    }
+    /** Time-travel debugging: replay from a specific checkpoint by run ID.
+     *  Creates a fresh daemon instance and replays events from the target run's bus.jsonl. */
+    async replayFrom(name, run) {
+        const runDir = join(this.root, ORG_DIR, name, run);
+        if (!existsSync(runDir))
+            return null;
+        const busFile = join(runDir, 'bus.jsonl');
+        if (!existsSync(busFile))
+            return null;
+        // Create a replay org with a fresh run ID
+        const replayRun = `replay-${new Date().toISOString().replace(/[-:T]/g, '').slice(0, 14)}-${Math.random().toString(36).slice(2, 6)}`;
+        const replayDir = join(this.root, ORG_DIR, name, replayRun);
+        mkdirSync(replayDir, { recursive: true });
+        // Read original events
+        const events = readFileSync(busFile, 'utf8').split('\n').filter(Boolean)
+            .map(l => { try {
+            return JSON.parse(l);
+        }
+        catch {
+            return null;
+        } })
+            .filter((e) => e !== null);
+        if (!events.length)
+            return null;
+        // Load org definition
+        const defPath = join(this.root, ORG_DIR, `${name}.json`);
+        if (!existsSync(defPath))
+            return null;
+        const def = OrgDefSchema.parse(JSON.parse(readFileSync(defPath, 'utf8')));
+        // Create replay bus
+        const bus = new OrgBus(name, replayRun, replayDir);
+        const MAX_COLLECTED = 1000;
+        const collected = [];
+        bus.subscribe(e => {
+            const slim = e.data?.content != null
+                ? { ...e, data: { ...e.data, content: undefined } }
+                : e;
+            collected.push(slim);
+            if (collected.length > MAX_COLLECTED)
+                collected.splice(0, collected.length - MAX_COLLECTED);
+            for (const fn of this.globalSubscribers)
+                fn(e);
+        });
+        const running = { def, run: replayRun, bus, agents: new Map(), busEvents: () => [...collected] };
+        // Reemit events into the replay bus with updated timestamps
+        const startTime = Date.now();
+        for (const e of events) {
+            const replayEvent = { ...e, org: name, run: replayRun, ts: startTime };
+            bus.emit(replayEvent);
+        }
+        this.orgs.set(name, running);
+        bus.emit({ type: 'status', msg: `replay started from ${run} (${events.length} events replayed)` });
+        this.persistState(name, 'running', replayRun);
+        return running;
+    }
     /** Agent-initiated human question (ask_human tool). Persists to questions.json (survives
      *  process/dashboard restarts) and emits a 'question' BusEvent so the dashboard's SSE
      *  stream and global inbox pick it up in real time. Returns a receipt string for the tool call. */
@@ -684,16 +940,14 @@ export class OrgDaemon {
         // existing offline fallback exactly (inbox.ts + autoWake).
         if (!this.hasOrgDef(org))
             return { ok: false, error: `org "${org}" not found (no saved definition)` };
-        try {
-            queueMessage(this.root, org, {
-                fromQualified: 'human', toRole: role,
-                subject: `answer:${questionId}`,
-                body: `question: ${question}\n\nanswer: ${answer}`,
-                ts: Date.now(),
-            });
-        }
-        catch (err) {
-            return { ok: false, error: `could not queue answer for org "${org}" — answer not recorded (${err instanceof Error ? err.message : String(err)})` };
+        const queued = queueMessage(this.root, org, {
+            fromQualified: 'human', toRole: role,
+            subject: `answer:${questionId}`,
+            body: `question: ${question}\n\nanswer: ${answer}`,
+            ts: Date.now(),
+        });
+        if (!queued) {
+            return { ok: false, error: `could not queue answer for org "${org}" — answer not recorded (disk full or permissions)` };
         }
         markAnswered();
         this.autoWake(org);
@@ -709,6 +963,39 @@ export class OrgDaemon {
             .catch(err => { console.error(`auto-wake org "${name}" failed:`, err instanceof Error ? err.message : err); })
             .finally(() => { this.waking.delete(name); });
     }
+    /** #4: bounded whole-org restart after the boss terminally crashes. Stops the
+     *  dead run and re-launches it with fresh sessions (shedding any bloated
+     *  context). Capped at MAX_BOSS_RESTARTS per explicit start so a crashing boss
+     *  can't loop forever; beyond the cap it gives up and lets the idle watchdog
+     *  shut the run down for a human. */
+    scheduleBossRestart(name) {
+        if (this.stopping.has(name) || this.restarting.has(name))
+            return;
+        const count = this.bossRestartCounts.get(name) ?? 0;
+        const bus = this.orgs.get(name)?.bus;
+        if (count >= OrgDaemon.MAX_BOSS_RESTARTS) {
+            bus?.emit({ type: 'audit', reason: 'boss-restart-exhausted',
+                msg: `boss crashed again after ${count} auto-restart(s) — giving up; manual restart required` });
+            return;
+        }
+        const backoffSchedule = this.opts.bossRestartBackoffMs ?? OrgDaemon.BOSS_RESTART_BACKOFF_MS;
+        const backoff = backoffSchedule[Math.min(count, backoffSchedule.length - 1)];
+        this.bossRestartCounts.set(name, count + 1);
+        this.restarting.add(name);
+        bus?.emit({ type: 'audit', reason: 'boss-restart',
+            msg: `boss crashed — auto-restarting org with fresh sessions in ${Math.round(backoff / 1000)}s (attempt ${count + 1}/${OrgDaemon.MAX_BOSS_RESTARTS})` });
+        const t = setTimeout(() => {
+            if (this.stopping.has(name)) {
+                this.restarting.delete(name);
+                return;
+            } // a manual stop won
+            this.stopOrg(name)
+                .then(() => (this.stopping.has(name) ? null : this.startOrg(name)))
+                .then(() => { this.restarting.delete(name); })
+                .catch(err => { this.restarting.delete(name); console.error(`org ${name}: boss auto-restart failed:`, err instanceof Error ? err.message : err); });
+        }, backoff);
+        t.unref?.();
+    }
     /** @param opts.drainMs how long to let in-flight agent sessions finish before
      *  reaping. Defaults to the short abort bound; the planned-completion path
      *  passes a far longer window (see COMPLETE_DRAIN_MS). */
@@ -725,6 +1012,7 @@ export class OrgDaemon {
         if (!org)
             return; // already stopped
         org.pendingRoles?.clear(); // prevent lazy spawns after stop
+        this.spawning.delete(name); // clean up spawning tracking for this org
         // Remove immediately (not at the end) so a concurrent stopOrg(name) call —
         // e.g. stopAll() racing a scheduler-triggered stop on SIGINT — joins this
         // shutdown via `stopping` instead of re-running the whole sequence and
@@ -824,8 +1112,9 @@ export class OrgDaemon {
         }
         // Same guard for runtime.json: if a new run started during shutdown, its
         // 'running' record must not be overwritten with this old run's 'stopped'.
+        // Pass the org directly since we already removed it from the map.
         if (!this.orgs.has(name))
-            this.persistState(name, 'stopped', org.run);
+            this.persistState(name, 'stopped', org.run, org);
         // Clean up git worktree if one was created for this run.
         if (org.worktreePath) {
             try {
@@ -1061,17 +1350,173 @@ export class OrgDaemon {
                 console.error(`org ${name}: run memory store failed:`, err instanceof Error ? err.message : err);
         }
     }
+    /** Resume a previous run from its checkpoint (runtime.json state). Reconstructs
+     *  the org's agents and mailboxes from the persisted state, enabling time-travel
+     *  debugging and run recovery after crashes. Returns the resumed RunningOrg or null. */
+    async resumeOrg(name) {
+        const rtPath = join(this.root, ORG_DIR, name, 'runtime.json');
+        if (!existsSync(rtPath)) {
+            console.error('resumeOrg failed: runtime.json missing for', name);
+            return null;
+        }
+        let rt;
+        try {
+            rt = JSON.parse(readFileSync(rtPath, 'utf8'));
+        }
+        catch (err) {
+            console.error('resumeOrg failed: invalid JSON in runtime.json for', name, err instanceof Error ? err.message : err);
+            return null;
+        }
+        if (rt?.status !== 'running' || !rt?.run) {
+            console.error('resumeOrg failed: invalid runtime state for', name, 'status:', rt?.status, 'run:', rt?.run);
+            return null;
+        }
+        // Load the org definition
+        const defPath = join(this.root, ORG_DIR, `${name}.json`);
+        if (!existsSync(defPath)) {
+            console.error('resumeOrg failed: org definition missing for', name);
+            return null;
+        }
+        const def = OrgDefSchema.parse(JSON.parse(readFileSync(defPath, 'utf8')));
+        const dir = join(this.root, ORG_DIR, name, rt.run);
+        // Check if run directory exists
+        if (!existsSync(dir)) {
+            console.error('resumeOrg failed: run directory missing', dir);
+            return null;
+        }
+        // Reconstruct the bus from history
+        const bus = new OrgBus(name, rt.run, dir);
+        const MAX_COLLECTED = 1000;
+        const collected = [];
+        bus.subscribe(e => {
+            const slim = e.data?.content != null
+                ? { ...e, data: { ...e.data, content: undefined } }
+                : e;
+            collected.push(slim);
+            if (collected.length > MAX_COLLECTED)
+                collected.splice(0, collected.length - MAX_COLLECTED);
+            for (const fn of this.globalSubscribers)
+                fn(e);
+        });
+        const running = { def, run: rt.run, bus, agents: new Map(), busEvents: () => [...collected] };
+        // Reconstruct agents from roleMetrics if available
+        if (rt.roleMetrics) {
+            for (const [roleId, metrics] of Object.entries(rt.roleMetrics)) {
+                const role = def.roles.find(r => r.id === roleId);
+                if (role) {
+                    const mailbox = new Mailbox();
+                    const perRoleBudget = Math.floor((def.run_config.budget_tokens ?? 1_000_000) / def.roles.length);
+                    const policy = new PolicyEngine(roleId, { maxTokens: perRoleBudget, ...(role.policy ?? {}) }, bus, this.root);
+                    const runtime = {
+                        mailbox,
+                        policy,
+                        status: 'running',
+                        done: Promise.resolve(),
+                        metrics: { tokens: metrics.tokens, costUsd: metrics.costUsd },
+                        lastMessageId: metrics.lastMessageId,
+                    };
+                    running.agents.set(roleId, runtime);
+                    // Restore mailbox state if it was closed
+                    if (rt.mailboxState?.[roleId]?.closed) {
+                        mailbox.close();
+                    }
+                }
+            }
+        }
+        // Restore abandoned roles tracking
+        if (rt.abandonedRoles) {
+            this.abandoned.set(name, new Set(rt.abandonedRoles));
+        }
+        // Set up pending roles for lazy spawn (roles not yet reconstructed)
+        const reconstructedRoles = new Set(Object.keys(rt.roleMetrics ?? {}));
+        const pendingRoles = new Map();
+        for (const role of def.roles) {
+            if (!reconstructedRoles.has(role.id)) {
+                pendingRoles.set(role.id, role);
+            }
+        }
+        if (pendingRoles.size > 0) {
+            running.pendingRoles = pendingRoles;
+        }
+        this.orgs.set(name, running);
+        bus.emit({ type: 'status', msg: `org resumed from checkpoint (${rt.run})` });
+        this.persistState(name, 'running', rt.run);
+        return running;
+    }
+    /** Create a branch from a checkpoint for "what-if" experiments */
+    branchCheckpoint(name, run, branchName) {
+        const runDir = join(this.root, ORG_DIR, name, run);
+        if (!existsSync(runDir)) {
+            return { ok: false, error: `run ${run} not found for org ${name}` };
+        }
+        const branchRun = `branch-${new Date().toISOString().replace(/[-:T]/g, '').slice(0, 14)}-${Math.random().toString(36).slice(2, 6)}`;
+        const branchDir = join(this.root, ORG_DIR, name, branchRun);
+        try {
+            mkdirSync(branchDir, { recursive: true });
+            // Copy bus.jsonl to branch
+            const busFile = join(runDir, 'bus.jsonl');
+            if (existsSync(busFile)) {
+                const busContent = readFileSync(busFile, 'utf8');
+                writeFileSync(join(branchDir, 'bus.jsonl'), busContent);
+            }
+            // Create branch marker file
+            writeFileSync(join(branchDir, '.branch-source'), JSON.stringify({ from: run, branchedAt: new Date().toISOString() }));
+            return { ok: true, branchRun };
+        }
+        catch (err) {
+            return { ok: false, error: `failed to create branch: ${err instanceof Error ? err.message : String(err)}` };
+        }
+    }
+    /** Record a structured decision trace for Rifft-style debugging */
+    recordDecision(org, role, decision) {
+        const running = this.orgs.get(org);
+        if (!running)
+            return;
+        running.bus.emit({
+            type: 'audit',
+            from: role,
+            reason: 'decision-trace',
+            data: {
+                decisionType: decision.type,
+                context: decision.context,
+                reasoning: decision.reasoning,
+                alternatives: decision.alternatives,
+                outcome: decision.outcome,
+                ts: new Date().toISOString()
+            },
+        });
+    }
     /** Roles that never spawned, per org. An org missing a role is still reported
      *  `running` by every status path, so the absence was visible only as one
      *  audit line in the log — a run went 40 minutes with no tester and nothing
      *  said so. Persisted to runtime.json so `org status` can say it out loud. */
     abandoned = new Map();
-    persistState(name, status, run) {
+    persistState(name, status, run, org) {
         const p = join(this.root, ORG_DIR, name, 'runtime.json');
         const missing = [...(this.abandoned.get(name) ?? [])];
+        const running = org ?? this.orgs.get(name);
+        // Per-role cost tracking: collect live metrics from active agents
+        const roleMetrics = {};
+        // Mailbox queue state for checkpoint/resume
+        const mailboxState = {};
+        if (running) {
+            for (const [roleId, runtime] of running.agents) {
+                roleMetrics[roleId] = {
+                    tokens: runtime.policy.usage,
+                    costUsd: runtime.metrics.costUsd,
+                    lastMessageId: runtime.lastMessageId
+                };
+                mailboxState[roleId] = {
+                    queued: runtime.mailbox['queue']?.length ?? 0,
+                    closed: runtime.mailbox.isClosed
+                };
+            }
+        }
         writeFileSync(p, JSON.stringify({
             status, run, pid: process.pid, updated: new Date().toISOString(),
             ...(missing.length ? { abandonedRoles: missing } : {}),
+            ...(Object.keys(roleMetrics).length ? { roleMetrics } : {}),
+            ...(Object.keys(mailboxState).length ? { mailboxState } : {}),
         }, null, 2));
     }
     /** Mark every currently-running org as crashed in runtime.json.