omp-conductor 0.7.1 → 0.8.0

package/src/decisions.ts

@@ -17,7 +17,7 @@ import type { DecisionRecord, Store, Tracker } from "./types.ts";
 /**
  * A precondition whose truth this package can check on its own.
  *
- * Exactly three kinds, deliberately. Each one is a question the tracker or npm
+ * Exactly six kinds, deliberately. Each one is a question the tracker or npm
  * already answers, so the row moves from "parked" to "act on this" without a
  * human re-reading it. Anything richer — a label appearing, a workflow going
  * green — is a follow-on issue rather than a grammar nobody validated.
@@ -25,11 +25,22 @@ import type { DecisionRecord, Store, Tracker } from "./types.ts";
 export type DecisionCondition =
   | { kind: "pr-merged"; url: string }
   | { kind: "issue-closed"; issue: number }
-  | { kind: "npm-version"; spec: string };
+  | { kind: "npm-version"; spec: string }
+  | { kind: "pr-checks-green"; url: string }
+  | { kind: "pr-mergeable"; url: string }
+  | { kind: "rate-limit-reset" };
 
 /** `pkg@version`, including a scoped package (`@scope/pkg@1.2.3`). */
 const NPM_SPEC = /^(@[^/@\s]+\/)?[^@\s]+@[^\s]+$/;
 
+/**
+ * Check states that count as a green verdict for `pr-checks-green`, copied from
+ * the daemon's failure classifier (`failure-class.ts` `SUCCESS_CHECK_STATES`):
+ * both spellings are terminally successful, and anything else — a failure, a
+ * still-running/pending check, a cancelled or skipped runner — is not.
+ */
+const GREEN_CHECK_STATES: Record<string, true> = { success: true, neutral: true };
+
 /**
  * Parse a raw condition, or `undefined` when it is not one of the three forms.
  *
@@ -46,8 +57,8 @@ export function parseCondition(raw: string): DecisionCondition | undefined {
   const rest = text.slice(at + 1).trim();
   if (rest.length === 0) return undefined;
 
-  if (kind === "pr-merged") {
-    return rest.startsWith("https://") ? { kind: "pr-merged", url: rest } : undefined;
+  if (kind === "pr-merged" || kind === "pr-checks-green" || kind === "pr-mergeable") {
+    return rest.startsWith("https://") ? { kind, url: rest } : undefined;
   }
   if (kind === "issue-closed") {
     if (!/^\d+$/.test(rest)) return undefined;
@@ -57,14 +68,22 @@ export function parseCondition(raw: string): DecisionCondition | undefined {
   if (kind === "npm-version") {
     return NPM_SPEC.test(rest) ? { kind: "npm-version", spec: rest } : undefined;
   }
+  if (kind === "rate-limit-reset") {
+    // Spelled `rate-limit-reset:github`: the grammar requires a non-empty rest,
+    // and `github` names the one GraphQL provider the fleet is on.
+    return rest === "github" ? { kind: "rate-limit-reset" } : undefined;
+  }
   return undefined;
 }
 
-/** The three accepted forms, for a refusal that can be acted on in one turn. */
+/** The six accepted forms, for a refusal that can be acted on in one turn. */
 export const CONDITION_FORMS = [
   "pr-merged:https://github.com/owner/repo/pull/123",
   "issue-closed:123",
   "npm-version:omp-conductor@0.4.3",
+  "pr-checks-green:https://github.com/owner/repo/pull/123",
+  "pr-mergeable:https://github.com/owner/repo/pull/123",
+  "rate-limit-reset:github",
 ] as const;
 
 /** Probe for `npm-version`, injectable so tests never reach the network. */
@@ -90,6 +109,33 @@ export const probeNpmVersion: NpmProbe = async (spec) => {
   }
 };
 
+/** Probe for `rate-limit-reset`, injectable so tests never reach the network. */
+export type RateLimitProbe = () => Promise<boolean>;
+
+/**
+ * `gh api rate_limit` — any remaining GraphQL quota means the limit has reset
+ * (or never bound). Bounded at 10 s like {@link probeNpmVersion}, for the same
+ * reason: this runs inside a tick, and a hung `gh` must cost one unevaluated
+ * decision, not the tick.
+ */
+export const probeRateLimitReset: RateLimitProbe = async () => {
+  const proc = Bun.spawn(
+    ["gh", "api", "rate_limit", "--jq", ".resources.graphql.remaining"],
+    { stdout: "pipe", stderr: "ignore" },
+  );
+  const timer = setTimeout(() => {
+    proc.kill();
+  }, 10_000);
+  try {
+    const [text, code] = await Promise.all([new Response(proc.stdout).text(), proc.exited]);
+    return code === 0 && Number.parseInt(text.trim(), 10) > 0;
+  } catch {
+    return false;
+  } finally {
+    clearTimeout(timer);
+  }
+};
+
 /**
  * Check every open decision that carries a condition and has not met it yet.
  *
@@ -105,7 +151,7 @@ export async function evaluateDecisionConditions(
   store: Store,
   project: string,
   tracker: Tracker,
-  probeNpm: NpmProbe,
+  probes: { npm: NpmProbe; rateLimit: RateLimitProbe },
   now: () => number,
 ): Promise<DecisionRecord[]> {
   const met: DecisionRecord[] = [];
@@ -119,8 +165,22 @@ export async function evaluateDecisionConditions(
         satisfied = (await tracker.prState(condition.url)) === "merged";
       } else if (condition.kind === "issue-closed") {
         satisfied = (await tracker.issueState(condition.issue)) === "closed";
+      } else if (condition.kind === "npm-version") {
+        satisfied = await probes.npm(condition.spec);
+      } else if (condition.kind === "pr-checks-green") {
+        // The same conclusion values the daemon's failure classifier treats as
+        // a green verdict (`success` / `neutral`, lowercased): a non-empty list
+        // in which every check is terminally successful and none is failing or
+        // pending (#189).
+        const checks = await tracker.checkConclusions(condition.url);
+        satisfied =
+          checks.length > 0 && checks.every((c) => GREEN_CHECK_STATES[c.state.trim().toLowerCase()] === true);
+      } else if (condition.kind === "pr-mergeable") {
+        // `clean` is the tracker's "this PR can merge" literal; `unknown` and a
+        // conflict are both unsatisfied (#189).
+        satisfied = (await tracker.mergeable(condition.url)) === "clean";
       } else {
-        satisfied = await probeNpm(condition.spec);
+        satisfied = await probes.rateLimit();
       }
     } catch {
       continue;

package/src/fleet.ts

@@ -42,6 +42,8 @@ import {
   formatReleaseGrants,
   formatSalvagedRuns,
   isPaused,
+  pauseProvenance,
+  pausedAt,
   setPaused,
   statusSnapshot,
   type StatusSnapshot,
@@ -55,6 +57,7 @@ import {
   SYSTEMD_UNIT,
 } from "./lifecycle.ts";
 import { formatRss, rssBytesFromHealthz } from "./host.ts";
+import { fetchRateLimit } from "./tracker/github.ts";
 import {
   readTickConfig,
   readTickRuntimeStatus,
@@ -252,9 +255,9 @@ export interface HaltWithPaneResult extends HaltResult {
   pane: PaneStopResult;
 }
 
-export function hold(projectName?: string): HoldResult {
+export function hold(projectName?: string, source: string = "hold"): HoldResult {
   const wasPaused = isPaused();
-  setPaused(true);
+  setPaused(true, { source });
   return { wasPaused, disarmed: disarmTicks(projectName) };
 }
 
@@ -263,7 +266,7 @@ export function releaseHold(): void {
 }
 
 export async function halt(projectName?: string): Promise<HaltResult> {
-  const held = hold(projectName);
+  const held = hold(projectName, "halt");
   const stop = await stopDaemon();
   return { hold: held, stop };
 }
@@ -1019,8 +1022,27 @@ export function formatFleetStatus(
 
   const graphBlock = formatCodeGraphHealth(codeGraph, now);
 
+  // Pause provenance (`pause --reason`, an integrity/spend-cap/upgrade pause,
+  // a halt) answers "who stopped the fleet" without opening a file (#185). An
+  // unparseable sentinel — paused but with no datable line 1 — is itself news:
+  // it means a run admitted before an *unknown* pause cannot prove innocence
+  // (#174), so every mutation is refused.
+  const dispatchLine =
+    layers.dispatch === "paused"
+      ? (() => {
+          const prov = pauseProvenance();
+          if (prov !== undefined) {
+            const reason = prov.reason === undefined ? "" : ` — "${prov.reason}"`;
+            return `dispatch  paused (source: ${prov.source}${reason})`;
+          }
+          return isPaused() && pausedAt() === undefined
+            ? "dispatch  paused (unparseable sentinel — all mutations refused)"
+            : "dispatch  paused";
+        })()
+      : `dispatch  ${layers.dispatch}`;
+
   return [
-    `dispatch  ${layers.dispatch}`,
+    dispatchLine,
     tickLine,
     ...(nextTickLine === undefined ? [] : [nextTickLine]),
     paneLine,
@@ -1053,6 +1075,13 @@ function formatProjectBody(s: StatusSnapshot): string {
     // independent controls and an operator has to see which one stopped the
     // fleet (#110).
     `  plan usage         ${planUsageLine(s.planUsage)}`,
+    // The tracker's API budget, when the renderer could read it. Absent on a
+    // broken `gh`: one missing row, never a broken report (#188).
+    ...(s.github === undefined
+      ? []
+      : [
+          `  github             graphql ${s.github.graphql.remaining}/${s.github.graphql.limit}, core ${s.github.core.remaining}/${s.github.core.limit} (graphql resets ${Math.max(0, Math.round((s.github.graphql.reset * 1000 - Date.now()) / 60_000))}m)`,
+        ]),
     `  new worker turns   ${s.caps.workerMaxTurns}`,
     `  worker wall clock  ${Math.round(s.caps.workerWallClockMs / 60_000)}m`,
     `  failed attempts    ${s.caps.maxAttemptsPerIssue}`,
@@ -1097,18 +1126,22 @@ export async function renderStatus(projectName?: string): Promise<string> {
   const layers = fleetLayers(projectName);
   const project = findProject(loadConfig(), projectName);
   const rec = livingDaemon();
-  const [health, telegram, planUsage] = await Promise.all([
+  const [health, telegram, planUsage, github] = await Promise.all([
     rec === undefined ? undefined : healthCheck(rec.port),
     probeTelegramHealth(projectName),
     // Read here rather than in `statusSnapshot`, which is synchronous and used
     // by callers that must not shell out. An unmetered project never spawns
     // the provider at all.
     readPlanUsage(s.caps.planUsage, sharedUsageSource()),
+    // Same reasoning as `planUsage`: the snapshot is synchronous, this read is
+    // a shell-out, and undefined on any failure — one missing row, never a
+    // broken report (#188).
+    fetchRateLimit(),
   ]);
   const cached = codeGraphFromHealthz(health?.body, project.name);
   const codeGraph = cached ?? (await probeCodeGraph(project));
   return formatFleetStatus(
-    { ...s, planUsage },
+    { ...s, planUsage, github },
     layers,
     health,
     telegram,

package/src/lifecycle.ts

@@ -292,6 +292,13 @@ export type StopResult =
   | { kind: "stopped"; pid: number; via: "systemctl" | "signal" }
   | { kind: "not-running" };
 
+/** What {@link restartDaemon} returns: the pre-restart record, the new one, and the path taken. */
+export interface RestartResult {
+  previous: DaemonRecord | undefined;
+  record: DaemonRecord;
+  via: "systemctl" | "cli";
+}
+
 /**
  * Stops the daemon.
  *
@@ -365,7 +372,7 @@ export async function stopDaemon(o: { timeoutMs?: number } = {}): Promise<StopRe
  */
 export async function restartDaemon(
   o: { port?: number; project?: string; timeoutMs?: number } = {},
-): Promise<{ previous: DaemonRecord | undefined; record: DaemonRecord; via: "systemctl" | "cli" }> {
+): Promise<RestartResult> {
   const previous = livingDaemon();
   const ownership = probeUnit();
   if (ownership.kind === "unknown") {

package/src/orchestrator-tick.ts

@@ -70,6 +70,7 @@ import {
 import {
   DEFAULT_REPORT_SCOPE,
   DENIED_RELEASE_GRANTS,
+  type DispatchSummary,
   type FrictionSignal,
   type ReportScope,
   type ResolvedGrants,
@@ -132,6 +133,13 @@ const RETRY_OWNERSHIP_MS = 60_000;
 export const STALL_MARKER_FILE = ".conductor-stalled";
 export const STALL_TICKS = 2;
 
+/** A turn older than this has its remaining tool calls refused (#189). */
+const DEFAULT_TICK_BUDGET_SECONDS = 600;
+/** Grace before a queued operator message preempts the turn's tool calls. */
+const PENDING_MESSAGE_GRACE_MS = 60_000;
+/** Routable candidates below which the queue digest tells the orchestrator to groom (#181). */
+const DEFAULT_GROOM_BELOW = 4;
+
 /**
  * Written by herdr-conductor `recover.sh` *before* `agent start`, so a resumed
  * fleet can reconcile orphans without waiting a full `intervalSeconds`. Cleared
@@ -244,6 +252,8 @@ interface TickApi {
    */
   getActiveTools(): string[];
   on(event: "session_start", handler: (event: { type: "session_start" }, ctx: TickContext) => void): void;
+  on(event: "turn_start", handler: (event: { type: "turn_start" }, ctx: TickContext) => void): void;
+  on(event: "turn_end", handler: (event: { type: "turn_end" }, ctx: TickContext) => void): void;
   /**
    * `deliverAs: "followUp"` + `triggerTurn: true`, verified against
    * `AgentSession.sendCustomMessage` rather than assumed:
@@ -276,6 +286,13 @@ export interface TickConfig {
   armedFile?: string;
   accessFile?: string;
   message?: string;
+  /**
+   * Seconds a turn may run before the tick guard refuses its remaining tool
+   * calls (#189). Optional; defaults to {@link DEFAULT_TICK_BUDGET_SECONDS} (10
+   * minutes, at or below a normal tick interval so a runaway turn is caught
+   * before the next tick).
+   */
+  budgetSeconds?: number;
   /**
    * The herdr agent name this fleet's orchestrator pane is registered under, and
    * the whole of {@link resolveTickOwnership}'s identity test under herdr.
@@ -353,6 +370,32 @@ export function recoveryDigestLine(recovered: readonly RunRecord[]): string | un
   return `Auto-recovered since last tick: ${recovered.length} (${named}${rest}) — already handled, do not re-triage these.`;
 }
 
+/**
+ * One line telling the orchestrator the routable queue is running dry (#181),
+ * or `undefined` when healthy — no dispatch recorded yet, or the routable count
+ * is at/above the grooming trigger.
+ */
+export function queueDigestLine(
+  summary: DispatchSummary | undefined,
+  queueLabel: string,
+  labelPrefix: string,
+  groomBelow: number,
+): string | undefined {
+  if (summary === undefined) return undefined;
+  if (summary.ready === 0) {
+    return `Queue: empty — nothing carries "${queueLabel}". Groom the backlog (Duty 2): promote or file the next issues, or say in this tick's report why there is nothing to do.`;
+  }
+  if (summary.routed === 0) {
+    return `Queue: ${summary.ready} ready but 0 routable — each needs exactly one "${labelPrefix}<repo>" label before dispatch can ever see it.`;
+  }
+  if (summary.routed >= groomBelow) return undefined;
+  let line = `Queue: running low — ${summary.routed} routable candidate(s), below the grooming trigger of ${groomBelow}.`;
+  if (summary.admitted === 0 && summary.holds.length > 0) {
+    line += ` All held: ${summary.holds.map((h) => `${h.reason} ${h.count}`).join(", ")}.`;
+  }
+  return line;
+}
+
 export const TICK_SCOPE_CONSTRAINTS: { readonly [K in ReportScope]: string } = {
   material: "Report material events per your brief.",
   escalations:
@@ -614,6 +657,15 @@ export function readTickConfig(cwd: string): TickConfigResult {
     intervalSeconds = interval;
   }
 
+  // Tolerant like the other optional fields: an integer >= the minimum is the
+  // turn budget in seconds, anything else degrades to absent (the shipped
+  // default) rather than invalidating the config.
+  const budgetRaw = raw["budgetSeconds"];
+  const budgetSeconds =
+    typeof budgetRaw === "number" && Number.isInteger(budgetRaw) && budgetRaw >= MIN_INTERVAL_SECONDS
+      ? budgetRaw
+      : undefined;
+
   // Relative paths resolve against the session cwd, so the files can sit beside
   // the config that names them (`state/armed`) without hard-coding a deploy path.
   const armedFile = optionalPath(raw["armedFile"], "armedFile", cwd, problems);
@@ -646,6 +698,7 @@ export function readTickConfig(cwd: string): TickConfigResult {
     path,
     config: {
       intervalSeconds,
+      ...(budgetSeconds === undefined ? {} : { budgetSeconds }),
       ...(armedFile === undefined ? {} : { armedFile }),
       ...(accessFile === undefined ? {} : { accessFile }),
       ...(message === undefined ? {} : { message }),
@@ -1012,6 +1065,41 @@ export function tickDecision(input: {
   return { send: true, reason: "armed, nothing pending" };
 }
 
+/**
+ * The tick guard's decision, extracted pure from {@link armTickGuard} so it can
+ * be pinned without a fake host: whether a turn's remaining tool calls should be
+ * refused. Budget first, then a queued operator message; otherwise nothing is
+ * blocked (#189).
+ */
+export function tickGuardDecision(input: {
+  turnStartedAt: number | undefined;
+  now: number;
+  budgetMs: number;
+  hasPending: boolean;
+}): { block: true; reason: string } | undefined {
+  if (input.turnStartedAt === undefined) return undefined;
+  const elapsed = input.now - input.turnStartedAt;
+  if (elapsed > input.budgetMs) {
+    return {
+      block: true,
+      reason:
+        `Blocked: this turn has run ${Math.round(elapsed / 1000)}s, past the ${Math.round(input.budgetMs / 1000)}s tick budget (#189). ` +
+        `Finish the turn NOW with a short report. Park anything you were waiting on as a watch — ` +
+        `omp-conductor decision open --question "..." --resolves-when pr-checks-green:<url> | pr-mergeable:<url> | pr-merged:<url> | rate-limit-reset:github — ` +
+        `or hand it to a subagent; the daemon flags met conditions in your next tick digest.`,
+    };
+  }
+  if (input.hasPending && elapsed > PENDING_MESSAGE_GRACE_MS) {
+    return {
+      block: true,
+      reason:
+        `Blocked: an operator message is queued behind this turn (#189). End the turn now and answer it — the person is waiting. ` +
+        `Park any wait as a decision watch (--resolves-when) instead of polling.`,
+    };
+  }
+  return undefined;
+}
+
 /**
  * Whether the Telegram bridge can still reach a person: a bot token, enabled,
  * with exactly one paired owner.
@@ -1269,6 +1357,22 @@ function tick(pi: TickApi, ctx: TickContext, config: TickConfig, session: TickSe
           frictionStore.recoveredSince(scope.projectName, now - 2 * config.intervalSeconds * 1_000),
         );
         if (recovered !== undefined) content = `${content}\n${recovered}`;
+        // The dispatch pass already persists the numbers that answer "is the
+        // queue running dry", so this reads the store rather than asking the
+        // tracker (#181). A failed config read skips the line rather than
+        // wedging the tick.
+        try {
+          const project = findProject(loadConfig());
+          const queue = queueDigestLine(
+            frictionStore.latestDispatch(scope.projectName),
+            project.queueLabel,
+            project.routing.labelPrefix,
+            project.groomBelow ?? DEFAULT_GROOM_BELOW,
+          );
+          if (queue !== undefined) content = `${content}\n${queue}`;
+        } catch {
+          // unreadable config: no queue digest this tick
+        }
       } catch (err) {
         frictionStore?.close();
         frictionStore = undefined;
@@ -1409,6 +1513,43 @@ function armTickHeartbeat(pi: TickApi, ctx: TickContext, config: TickConfig, ses
   tick(pi, ctx, config, session);
 }
 
+/**
+ * The mechanical guard against a turn that never ends (#189).
+ *
+ * The harness delivers `turn_start`/`turn_end`, so the guard times a turn from
+ * the first to the last tool-eligible phase and refuses further tool calls once
+ * it exceeds `budgetMs` — or once an operator message has been queued behind it
+ * for longer than {@link PENDING_MESSAGE_GRACE_MS}. Refusing every tool forces
+ * the model to end the turn with text; nothing else unblocks the queue. `yield`
+ * stays callable so a session that is also a task can still return.
+ */
+function armTickGuard(pi: TickApi, ctx: TickContext, budgetMs: number): void {
+  let turnStartedAt: number | undefined;
+  pi.on("turn_start", () => {
+    turnStartedAt = Date.now();
+  });
+  pi.on("turn_end", () => {
+    turnStartedAt = undefined;
+  });
+  (pi as TickApi & {
+    on(
+      event: "tool_call",
+      handler: (
+        event: { toolName: string; input: Record<string, unknown> },
+        ctx: unknown,
+      ) => { block: true; reason: string } | undefined,
+    ): void;
+  }).on("tool_call", (event) => {
+    if (event.toolName === "yield") return undefined;
+    return tickGuardDecision({
+      turnStartedAt,
+      now: Date.now(),
+      budgetMs,
+      hasPending: ctx.hasPendingMessages(),
+    });
+  });
+}
+
 export default function orchestratorTickExtension(pi: TickApi): void {
   // Scoped to this registration rather than the module, so a second
   // `session_start` can neither install a second heartbeat on the same session
@@ -1426,6 +1567,7 @@ export default function orchestratorTickExtension(pi: TickApi): void {
     pendingSkips: 0,
   };
   let releaseGateArmed = false;
+  let guardArmed = false;
   // An activation file makes this a fleet directory before Herdr can prove
   // which pane owns it. The gate therefore starts closed and only honours a
   // configured grant after ownership is accepted.
@@ -1594,6 +1736,10 @@ export default function orchestratorTickExtension(pi: TickApi): void {
         if (next.note !== undefined) pi.logger.error(`[omp-conductor] tick ownership: ${next.note}`);
         releaseAuthorityAccepted = true;
         armTickHeartbeat(pi, ctx, config, session);
+        if (!guardArmed) {
+          guardArmed = true;
+          armTickGuard(pi, ctx, (config.budgetSeconds ?? DEFAULT_TICK_BUDGET_SECONDS) * 1000);
+        }
         pi.logger.info(`[omp-conductor] orchestrator tick active: ownership resolved on retry`, { agentName });
       }, retryMs);
       decided = true;
@@ -1608,6 +1754,10 @@ export default function orchestratorTickExtension(pi: TickApi): void {
     if (ownership.note !== undefined) pi.logger.error(`[omp-conductor] tick ownership: ${ownership.note}`);
     releaseAuthorityAccepted = true;
     armTickHeartbeat(pi, ctx, config, session);
+    if (!guardArmed) {
+      guardArmed = true;
+      armTickGuard(pi, ctx, (config.budgetSeconds ?? DEFAULT_TICK_BUDGET_SECONDS) * 1000);
+    }
     decided = true;
     // Both gates are named at startup: "why is it not ticking?" is answered by
     // looking at the files this line lists, and an unset channel gate on a fleet

package/src/plugin.ts

@@ -1358,7 +1358,7 @@ export default function conductorPlugin(pi: PluginApi): void {
           }
 
           case "pause":
-            setPaused(true);
+            setPaused(true, { source: "pause", reason: "via /conductor pause" });
             ctx.ui.notify("Paused claiming only — ticks keep firing if armed. Prefer /conductor hold.", "info");
             break;
 

package/src/setup.ts

@@ -1167,7 +1167,7 @@ export const AMEND_AREAS: {
       const spend =
         c.dailySpendUsd === null ? "no spend cap" : `$${c.dailySpendUsd}/day`;
       return (
-        `${c.maxConcurrentWorkers} workers, ${c.workerMaxTurns} turns, ` +
+        `${c.maxConcurrentWorkers} workers (${c.maxConcurrentWorkersPerRepo}/repo), ${c.workerMaxTurns} turns, ` +
         `${Math.round(c.workerWallClockMs / 60000)}m, ${spend}, ` +
         `${c.maxAttemptsPerIssue} failed attempt${c.maxAttemptsPerIssue === 1 ? "" : "s"}, ` +
         `${c.maxContinuationsPerIssue} continuation${c.maxContinuationsPerIssue === 1 ? "" : "s"}` +