@monoes/monomindcli 2.12.0 → 2.14.0

package/dist/src/orgrt/daemon.js

@@ -25,9 +25,10 @@ import { fileToolRoots } from './file-roots.js';
 import { attachForwarder } from './forwarder.js';
 import { GrokAgentRunner } from './grok-runner.js';
 import { HermesAgentRunner } from './hermes-runner.js';
-import { clearIdleRecord, projectIdleStop, writeIdleRecord, } from './idle-deadline.js';
+import { advanceHold, clearIdleRecord, noProgressRoles, projectIdleStop, writeIdleRecord, } from './idle-deadline.js';
 import { drainInbox, newMessageId, queueMessage } from './inbox.js';
 import { KimiCodeAgentRunner } from './kimicode-runner.js';
+import { loadoutCatalog, resolveLoadout, sessionLoadoutFor, taskTag } from './loadouts.js';
 import { isRecoverableCloseReason, Mailbox } from './mailbox.js';
 import { OpencodeAgentRunner } from './opencode-runner.js';
 import * as orgMemory from './org-memory.js';
@@ -43,6 +44,7 @@ import { buildRespawnReceipt, computeReplacementBudget, mergeEffectiveRoleConfig
 import { buildRuntimeOptions } from './runtime-options.js';
 import * as scheduler from './scheduler-integration.js';
 import { runAgentSession } from './session.js';
+import { SessionLedger } from './session-ledger.js';
 import { TaskDag } from './task-dag.js';
 import { freshChainId, roleProviderPrefixes, ToolProviderHub, } from './tool-providers.js';
 import { ORG_DIR, OrgDefSchema, } from './types.js';
@@ -624,6 +626,22 @@ export class OrgDaemon {
             getResourceLimits().maxSdkProcesses < sessionRoleCount) {
             configureResourceLimits({ maxSdkProcesses: sessionRoleCount });
         }
+        // ADR-O001 D8: a cost tier that can't resolve a model for a role's
+        // provider must stop the run here. The alternative — resolving it at
+        // session start — would either crash one role ten minutes in or, worse,
+        // quietly leave that role on a different model than the tier claimed.
+        const { validateCostTiers } = await import('./cost-tier.js');
+        const tierErrors = validateCostTiers(def);
+        if (tierErrors.length) {
+            throw new Error(`org ${name}: ${tierErrors.join('; ')}`);
+        }
+        // ADR-O001 D7: an oversized or unresolvable loadout catalog stops the run
+        // here, for the same reason — not ten minutes in, at some role's spawn.
+        const { validateLoadouts } = await import('./loadouts.js');
+        const loadoutErrors = validateLoadouts(def, this.root).errors;
+        if (loadoutErrors.length) {
+            throw new Error(`org ${name}: ${loadoutErrors.join('; ')}`);
+        }
         // Validate per-role providers before spawning anything (fail-fast: a
         // missing env var discovered 10 minutes into a run wastes the entire run).
         const { resolveProviderEnv: validateProvider, resolveRoleProvider } = await import('./provider.js');
@@ -674,6 +692,13 @@ export class OrgDaemon {
         // boss that's genuinely out of ideas can loop forever making zero
         // progress without ever tripping the watchdog.
         let lastToolActivity = 0;
+        // ADR-O001 D4 (no-progress detector): the idle clock above is org-wide and
+        // says nothing at all while a hold is in force — the 8.3-hour stall looked
+        // perfectly healthy from it. Per-role last-activity is what tells a role
+        // that is nominally working but producing nothing from one that is simply
+        // waiting its turn.
+        const roleActivity = new Map();
+        const noProgressAlarmed = new Set();
         // Org-wide budget ceiling (bug 1): tracks whether run_config.budget_tokens
         // has already been enforced this run, so the close-all-mailboxes sweep
         // below only fires once instead of on every subsequent usage event.
@@ -683,12 +708,18 @@ export class OrgDaemon {
             collected.push(slim);
             if (collected.length > MAX_COLLECTED)
                 collected.splice(0, collected.length - MAX_COLLECTED);
-            // The watchdog's own nudge event must not count as org activity, or a
-            // hung boss would never trip the "nudge produced no activity" stop.
-            if (e.reason !== 'idle-nudge')
+            // The watchdog's own events must not count as org activity, or a hung
+            // boss would never trip the "nudge produced no activity" stop and a
+            // silent role would clear its own no-progress alarm.
+            const selfEmitted = e.reason === 'idle-nudge' || e.reason === 'no-progress' || e.reason === 'hold-expired';
+            if (!selfEmitted)
                 lastActivity = Date.now();
             if (e.type === 'tool')
                 lastToolActivity = Date.now();
+            if (e.from && !selfEmitted) {
+                roleActivity.set(e.from, Date.now());
+                noProgressAlarmed.delete(e.from);
+            }
             // org_complete IS the end of the run — self-stop instead of sitting
             // "running" forever after a recorded outcome. Deferred (unref'd) so the
             // tool call's receipt reaches the boss and its final turn text still
@@ -727,8 +758,11 @@ export class OrgDaemon {
                 const orgBudget = def.run_config.budget_tokens;
                 if (orgBudget != null) {
                     let orgUsage = 0;
+                    // ADR-O001 D1: budgetedUsage, not usage — the org-wide ceiling is
+                    // the same declared number as the per-role one and must be compared
+                    // on the same basis.
                     for (const rt of running.agents.values())
-                        orgUsage += rt.policy.usage;
+                        orgUsage += rt.policy.budgetedUsage;
                     // Mid-run role replacement retires a policy engine's usage into the
                     // slot instead of discarding it (see role-slot.ts / respawnRole) -
                     // include it here or a replacement could silently reset spend and
@@ -773,6 +807,7 @@ export class OrgDaemon {
         const running = {
             def,
             run,
+            sessionLedger: new SessionLedger(join(dir, 'sessions.json')),
             bus,
             agents: new Map(),
             roleSlots: new Map(),
@@ -1075,7 +1110,7 @@ export class OrgDaemon {
                         idleMs,
                         bossReachable: bossRt?.status === 'running' && !bossRt.mailbox.isClosed,
                     })).toISOString();
-                const key = `${at}|${hold}`;
+                const key = `${at}|${hold?.reason ?? null}|${hold?.until ?? null}`;
                 if (key === published)
                     return;
                 try {
@@ -1091,7 +1126,32 @@ export class OrgDaemon {
                     console.error(`org ${name}: could not write the idle deadline:`, err instanceof Error ? err.message : err);
                 }
             };
-            // The legitimate waits the watchdog never nudges or stops through.
+            // ADR-O001 D4 — Gas Town's "30 minutes hooked without progress" alarm.
+            // The org-wide idle clock is silent while a hold is in force, so a role
+            // that is nominally running and producing nothing gets its own, loud
+            // audit event. Once per spell: the bus subscriber clears the flag as
+            // soon as the role emits anything.
+            const alarmNoProgress = (now) => {
+                const stalled = noProgressRoles([...running.agents].map(([id, rt]) => ({
+                    id,
+                    working: rt.status === 'running' && !rt.mailbox.isClosed,
+                    lastActivity: roleActivity.get(id) ?? lastActivity,
+                    alarmed: noProgressAlarmed.has(id),
+                })), now);
+                for (const role of stalled) {
+                    noProgressAlarmed.add(role.id);
+                    bus.emit({
+                        type: 'audit',
+                        from: role.id,
+                        reason: 'no-progress',
+                        msg: `role "${role.id}" has been running for ${Math.round(role.silentMs / 60_000)}m ` +
+                            `without a single bus event — it is hooked but producing nothing`,
+                        data: { role: role.id, silentMinutes: Math.round(role.silentMs / 60_000) },
+                    });
+                }
+            };
+            // The legitimate waits the watchdog holds through — every one of them
+            // with a deadline attached by advanceHold (ADR-O001 D4).
             const holdReason = () => {
                 if (this.restarting.has(name))
                     return 'restarting'; // boss auto-restart in flight
@@ -1106,10 +1166,12 @@ export class OrgDaemon {
                 // agent. Without this check the watchdog nudges (and, after enough
                 // nudges, idle-stops) an org that's simply waiting on a human answer
                 // that's already on its way.
-                const pendingQuestions = questionOps
-                    .readQuestions(this.root, name)
-                    .questions.filter((q) => q.answer === null);
-                if (pendingQuestions.length > 0)
+                //
+                // ADR-O001 D4: only a question the asker declared BLOCKING counts. The
+                // 8.3-hour stall was held open by a question whose own text opened
+                // with "no answer needed for the run to continue; I am not blocking on
+                // this" — it suppressed the watchdog exactly like a real blocker.
+                if (questionOps.pendingBlockingQuestions(this.root, name).length > 0)
                     return 'pending-question';
                 // M1 (C-41): a pending tool approval is the same kind of legitimate
                 // wait — the role was told to wait for `org approve/deny`.
@@ -1118,23 +1180,28 @@ export class OrgDaemon {
                 // M2 (C-41): a delivered endpoint message whose reply is still due.
                 if (hasActiveEndpointWait(running))
                     return 'endpoint-reply-due';
+                // A task blocked on a real-world time still in the future is
+                // legitimate waiting, same as a pending gate — don't nudge about it.
+                // Its deadline is the time the asker actually named, not the default
+                // hold TTL, so a block set hours out is honoured exactly.
+                const blockedUntil = running.taskDag?.activeBlockUntil(Date.now()) ?? null;
+                if (blockedUntil !== null)
+                    return { reason: 'task-blocked', until: blockedUntil };
                 return null;
             };
-            const check = () => {
-                const hold = holdReason();
-                if (hold)
-                    return hold;
-                // Auto-resume any task whose org_task_block time has passed: flip it
-                // back to 'running' and re-push it into the assignee's mailbox, same
-                // as a fresh dispatch. This IS real activity, so fall through to the
-                // normal idleFor check below rather than returning early — an
-                // unblocked task should reset the idle clock, not just silently
-                // update state nobody notices until the next nudge.
+            // Auto-resume any task whose org_task_block time has passed: flip it
+            // back to 'running' and re-push it into the assignee's mailbox, same as
+            // a fresh dispatch. This IS real activity, so it feeds the normal
+            // idleFor check rather than short-circuiting it — an unblocked task
+            // should reset the idle clock, not just silently update state nobody
+            // notices until the next nudge. Runs on every tick, a hold in force
+            // included: a block that expires mid-wait is real work again.
+            const resumeExpiredBlocks = () => {
                 const unblocked = running.taskDag?.unblockExpired(Date.now()) ?? [];
                 for (const task of unblocked) {
                     const agent = running.agents.get(task.assignee);
                     if (agent && !agent.mailbox.isClosed) {
-                        agent.mailbox.push(`[task:${task.id}] Block expired — resuming: ${task.title}`);
+                        agent.mailbox.push(`${taskTag(task)} Block expired — resuming: ${task.title}`);
                     }
                     bus.emit({
                         type: 'status',
@@ -1144,20 +1211,20 @@ export class OrgDaemon {
                         data: { taskId: task.id, assignee: task.assignee },
                     });
                 }
-                // A task blocked on a real-world time still in the future is
-                // legitimate waiting, same as a pending gate — don't nudge about it.
-                if (running.taskDag?.hasActiveBlock(Date.now()))
-                    return 'task-blocked';
+            };
+            // The normal idle path: nudge the boss, then stop if the nudge produced
+            // nothing. Runs only when nothing (still) holds the watchdog.
+            const check = () => {
                 const idleFor = Date.now() - lastActivity;
                 if (idleFor < idleMs) {
                     nudges = resolvedIdleNudgeCount(nudgedAt, nudges, lastToolActivity);
                     nudgedAt = 0;
-                    return null;
+                    return;
                 }
                 if (nudgedAt === 0) {
                     if (nudges >= MAX_IDLE_NUDGES) {
                         idleStop(`org idle again after ${nudges} nudges — stopping run`);
-                        return null;
+                        return;
                     }
                     const bossRt = running.agents.get(bossRole.id);
                     // #205: a budget-exhausted boss closed its own mailbox on
@@ -1170,11 +1237,11 @@ export class OrgDaemon {
                         idleStop(`org idle for ${Math.round(idleFor / 60_000)}m and boss "${bossRole.id}" is over its ` +
                             `${budgetReason === 'token-budget' ? 'token' : 'USD'} budget — raise the role's ` +
                             `${budgetReason === 'token-budget' ? 'budget_tokens' : 'budget_usd'} (or run_config's) and resume from checkpoint — stopping run`);
-                        return null;
+                        return;
                     }
                     if (bossRt?.status !== 'running' || bossRt.mailbox.isClosed) {
                         idleStop(`org idle for ${Math.round(idleFor / 60_000)}m and boss "${bossRole.id}" is unreachable — stopping run`);
-                        return null;
+                        return;
                     }
                     nudges++;
                     nudgedAt = Date.now();
@@ -1193,10 +1260,35 @@ export class OrgDaemon {
                 else if (Date.now() - nudgedAt >= idleMs) {
                     idleStop(`nudge produced no activity for another ${Math.round(idleMs / 60_000)}m — boss appears hung, stopping run`);
                 }
+            };
+            // One watchdog tick. The hold is resolved FIRST and with a deadline, so
+            // a wait that outlives its deadline hands the run back to the idle path
+            // instead of suppressing it forever (ADR-O001 D4).
+            let holdTrack = null;
+            const tick = () => {
+                const now = Date.now();
+                alarmNoProgress(now);
+                resumeExpiredBlocks();
+                const step = advanceHold(holdTrack, holdReason(), now);
+                holdTrack = step.track;
+                if (step.expired) {
+                    bus.emit({
+                        type: 'audit',
+                        reason: 'hold-expired',
+                        msg: `the "${step.expired}" hold on org "${name}" outlived its deadline — the idle ` +
+                            `watchdog is running again and will nudge, then stop the run if nothing happens`,
+                        data: { hold: step.expired },
+                    });
+                }
+                if (step.hold)
+                    return step.hold;
+                check();
                 return null;
             };
-            publishDeadline(holdReason());
-            const wd = setInterval(() => publishDeadline(check()), Math.max(200, Math.min(idleMs / 2, 30_000)));
+            const initialHold = advanceHold(null, holdReason(), Date.now());
+            holdTrack = initialHold.track;
+            publishDeadline(initialHold.hold);
+            const wd = setInterval(() => publishDeadline(tick()), Math.max(200, Math.min(idleMs / 2, 30_000)));
             wd.unref?.();
             this.watchdogs.set(name, wd);
         }
@@ -1206,7 +1298,7 @@ export class OrgDaemon {
                     run,
                     idle_minutes: 0,
                     idle_stop_at: null,
-                    hold: 'disabled',
+                    hold: { reason: 'disabled', until: null },
                 });
             }
             catch (err) {
@@ -1344,6 +1436,11 @@ export class OrgDaemon {
         }
         const policy = new PolicyEngine(role.id, {
             maxTokens: role.budget_tokens ?? perRoleBudget,
+            // ADR-O001 D1: which basis that ceiling is enforced on. Defaults to
+            // the historical uncached basis so the honest (cache-aware) meter
+            // introduced alongside it cannot exhaust an existing budget_tokens —
+            // including the schema's 1M default — roughly 100x early.
+            maxTokensBasis: def.run_config.budget_tokens_basis ?? 'uncached',
             maxUsd: role.budget_usd,
             ...(role.policy ?? {}),
         }, bus, roleCwd, 
@@ -1356,7 +1453,13 @@ export class OrgDaemon {
             providerPrefixes: () => roleProviderPrefixes(role),
             trace: () => this.roleTrace(name, role.id),
         });
-        if (roleCheckpoint?.tokensUsed) {
+        // ADR-O001 D1: prefer the persisted four-quantity breakdown; a checkpoint
+        // written before it existed still resumes via the scalar, on the uncached
+        // basis it was recorded on.
+        if (roleCheckpoint?.tokenUsage) {
+            policy.setTokenUsage(roleCheckpoint.tokenUsage);
+        }
+        else if (roleCheckpoint?.tokensUsed) {
             policy.setUsage(roleCheckpoint.tokensUsed);
         }
         // ORG-7: restore accumulated USD spend across resume so a stop/resume
@@ -1364,6 +1467,31 @@ export class OrgDaemon {
         if (roleCheckpoint?.costUsd) {
             policy.setUsageUsd(roleCheckpoint.costUsd);
         }
+        // ADR-O001 D7: the loadout this incarnation's session is built with, fixed
+        // for its life. A checkpointed role keeps what it had (its SDK session was
+        // built with it); a replacement keeps its predecessor's; a new role takes
+        // the loadout of the first ready task it is being spawned for.
+        const loadoutName = roleCheckpoint
+            ? roleCheckpoint.loadout
+            : (existingSlot?.runtime?.loadout ?? sessionLoadoutFor(running.taskDag, role.id));
+        let loadout;
+        if (loadoutName) {
+            try {
+                loadout = resolveLoadout(def, loadoutName, this.root);
+            }
+            catch (err) {
+                // Validated at start, so only a config hot-reload or a deleted
+                // instructions_file lands here. Spawn without it, loudly: a role that
+                // fails to spawn would strand its task, which is worse.
+                bus.emit({
+                    type: 'audit',
+                    from: role.id,
+                    reason: 'loadout-unresolvable',
+                    msg: `role "${role.id}" spawned without loadout "${loadoutName}": ${err instanceof Error ? err.message : err}`,
+                    data: { loadout: loadoutName },
+                });
+            }
+        }
         const runtime = {
             mailbox,
             policy,
@@ -1375,6 +1503,7 @@ export class OrgDaemon {
             sessionId: roleCheckpoint?.sessionId,
             worktreePath: roleCwd !== cwd ? roleCwd : undefined,
             scrollback: new ScrollbackBuffer(),
+            ...(loadout ? { loadout: loadout.name } : {}),
         };
         if (roleCheckpoint?.scrollback?.length) {
             for (const line of roleCheckpoint.scrollback)
@@ -1412,13 +1541,34 @@ export class OrgDaemon {
             },
             maxTurns: role.max_turns_per_message ?? def.run_config.max_turns_per_message,
             resumeSessionId: roleCheckpoint?.sessionId,
+            sessionLedger: running.sessionLedger,
+            // ADR-O001 D3 x D7: a task-scoped session is built with its task's own
+            // recorded loadout. Unresolvable → no loadout, loudly (as at spawn).
+            loadoutFor: (taskId) => {
+                const name = running.taskDag?.get(taskId)?.loadout;
+                if (!name)
+                    return undefined;
+                try {
+                    return resolveLoadout(def, name, this.root);
+                }
+                catch (err) {
+                    bus.emit({
+                        type: 'audit',
+                        from: role.id,
+                        reason: 'loadout-unresolvable',
+                        msg: `task ${taskId} session built without loadout "${name}": ${err instanceof Error ? err.message : err}`,
+                        data: { loadout: name, taskId },
+                    });
+                    return undefined;
+                }
+            },
             lastMessageId: () => runtime.lastMessageId,
             onOutput: (line) => runtime.scrollback.push(line),
             onSessionId: (id) => {
                 runtime.sessionId = id;
             },
             deliver: (from, to, subject, body) => this.deliver(name, from, to, subject, body),
-            askHuman: (r, question) => this.askHuman(name, r, question),
+            askHuman: (r, question, blocking) => this.askHuman(name, r, question, blocking),
             onGate: (r, gateName, gateDesc) => this.createGate(name, r, gateName, gateDesc),
             circuitBreaker: (() => {
                 const cb = def.run_config.circuit_breaker;
@@ -1463,9 +1613,7 @@ export class OrgDaemon {
                     // unanswered question is the same "genuinely waiting on a
                     // human" fact either way.
                     const pendingHumanWaits = this.listGates(name, 'pending').length +
-                        questionOps
-                            .readQuestions(this.root, name)
-                            .questions.filter((q) => q.answer === null).length;
+                        questionOps.pendingBlockingQuestions(this.root, name).length;
                     return resolveOrgComplete(bus, r, outcome, summary, blocker, blockerDetail, {
                         mode: def.run_config.completion ?? 'boss',
                         maxBudgetFraction,
@@ -1535,12 +1683,24 @@ export class OrgDaemon {
                 });
                 return text;
             },
-            createTask: (r, title, assignee, deps) => {
-                return this.dagCreateTask(name, r, title, assignee, deps);
+            createTask: (r, title, assignee, deps, loadout) => {
+                return this.dagCreateTask(name, r, title, assignee, deps, loadout);
             },
-            completeTask: (r, taskId, result) => {
-                return this.dagCompleteTask(name, r, taskId, result);
+            // ADR-O001 D7: only an org with a catalog gets the `loadout` argument;
+            // the session itself is built with the loadout frozen above.
+            loadoutCatalog: loadoutCatalog(def),
+            loadout,
+            completeTask: (r, taskId, result, evidence) => {
+                return this.dagCompleteTask(name, r, taskId, result, evidence);
             },
+            // ADR-O001 D5: only an org that opted in advertises the evidence
+            // argument, so every other org's tool list stays byte-identical.
+            requireTaskEvidence: def.run_config.completion_evidence === true,
+            // ADR-O001 D6: only an org with an artifact-only reviewer gets org_review,
+            // so every other org's tool list stays byte-identical.
+            requestReview: def.roles.some((r) => r.review_input === 'artifact-only')
+                ? (r, taskId, reviewer, base) => this.dagRequestReview(name, r, taskId, reviewer, base)
+                : undefined,
             listTasks: () => {
                 const running = this.orgs.get(name);
                 return JSON.stringify(running?.taskDag?.all() ?? [], null, 2);
@@ -1986,7 +2146,10 @@ export class OrgDaemon {
         slot.queuedDuringSwap.push(...reclaimedQueue);
         // Step 9: retire accounting BEFORE replacing the runtime.
         slot.retiredUsage = {
-            tokens: slot.retiredUsage.tokens + oldRuntime.policy.usage,
+            // Budgeted basis: this total is summed with live policy.budgetedUsage
+            // against the org-wide budget_tokens ceiling (ADR-O001 D1), so the two
+            // terms must share a basis.
+            tokens: slot.retiredUsage.tokens + oldRuntime.policy.budgetedUsage,
             costUsd: slot.retiredUsage.costUsd + oldRuntime.metrics.costUsd,
         };
         // Step 10: spawn generation N+1 (generation already bumped in step 6).
@@ -2552,8 +2715,8 @@ export class OrgDaemon {
         return approvalOps.setApproval(this, org, role, action, approved, opts);
     }
     // questions.ts
-    async askHuman(org, role, question) {
-        return questionOps.askHuman(this, org, role, question);
+    async askHuman(org, role, question, blocking) {
+        return questionOps.askHuman(this, org, role, question, blocking);
     }
     async answerQuestion(org, role, questionId, answer, resolvedBy) {
         return questionOps.answerQuestion(this, org, role, questionId, answer, resolvedBy);
@@ -2571,11 +2734,15 @@ export class OrgDaemon {
     listGates(org, status) {
         return decisionOps.listGates(this, org, status);
     }
-    dagCreateTask(org, role, title, assignee, deps) {
-        return decisionOps.dagCreateTask(this, org, role, title, assignee, deps);
+    dagCreateTask(org, role, title, assignee, deps, loadout) {
+        return decisionOps.dagCreateTask(this, org, role, title, assignee, deps, loadout);
+    }
+    dagCompleteTask(org, role, taskId, result, evidence) {
+        return decisionOps.dagCompleteTask(this, org, role, taskId, result, evidence);
     }
-    dagCompleteTask(org, role, taskId, result) {
-        return decisionOps.dagCompleteTask(this, org, role, taskId, result);
+    /** ADR-O001 D6 — see decisions.ts's dagRequestReview. */
+    dagRequestReview(org, role, taskId, reviewer, base) {
+        return decisionOps.dagRequestReview(this, org, role, taskId, reviewer, base);
     }
     dagSplitTask(org, role, parentId, children) {
         return decisionOps.dagSplitTask(this, org, role, parentId, children);