@intentius/chant 0.124.13 → 0.124.15

package/dist/op/operator.js

@@ -0,0 +1,743 @@
+import { discoverOps } from "./discover.js";
+import { runOpLocally, OpRunFailure } from "./local-executor.js";
+import { acquireLease, releaseLease, stillHoldsLease, currentHolderId, readLease, DEFAULT_LEASE_TTL_MS } from "../lifecycle/lease.js";
+import { randomUUID } from "node:crypto";
+import { readRunLedger, runEnvOf } from "../lifecycle/run-ledger.js";
+import { readGateLedger, latestResolutionSince } from "../lifecycle/gate-ledger.js";
+import { enterStewardTurn } from "./steward-turn.js";
+import { stewardBesideOf, stewardLeaseName, stewardTurnLeaseName, stewardTurnOps } from "./steward.js";
+import { askReady, spawnBesideRun } from "./steward-beside.js";
+import { stewardWorkHolder } from "./work-lease-run.js";
+import { StaleLockError } from "../lifecycle/git.js";
+import { cronMatches, cronDueBetween } from "./cron.js";
+import { createChangeSignalGate, DEFAULT_SIGNAL_FLOOR_MS } from "./change-signal.js";
+/**
+ * Poll interval between rounds — the operator's own cadence, and the fallback
+ * for an Op that declares none. Since #2120 an Op can carry its own
+ * `schedule.cron` (`./composites/converge-op.ts`), which this loop honours per
+ * Op: the round still wakes every `intervalMs`, but an Op with a cron is
+ * ticked only on the minutes that cron fires. Chosen short enough to converge
+ * promptly, long enough not to hammer `chant lifecycle plan --live` every few
+ * seconds — and short enough that a per-minute cron is not missed.
+ */
+export const DEFAULT_OPERATOR_INTERVAL_MS = 60000;
+/** One ConvergeOp's own `labels.Env` (`./composites/converge-op.ts` always sets it). `undefined` for a non-ConvergeOp or a hand-built one missing it — filtered out by `isConvergeOp` before this is trusted. */
+function envOf(config) {
+    return config.labels?.Env;
+}
+function isConvergeOp(config) {
+    return config.labels?.Converge === "true";
+}
+/** Discover every `ConvergeOp` in the project (`labels.Converge === "true"`), optionally filtered to one environment. Ops that don't declare an `Env` label are excluded from an `--env`-filtered discovery (there's nothing to match), but included when no `env` filter is given. */
+export async function discoverConvergeOps(opts) {
+    const { ops, errors } = await discoverOps({ cwd: opts?.cwd });
+    const converge = [...ops.values()].filter((d) => isConvergeOp(d.config));
+    const filtered = opts?.env ? converge.filter((d) => envOf(d.config) === opts.env) : converge;
+    return { ops: filtered.sort((a, b) => a.config.name.localeCompare(b.config.name)), errors };
+}
+/** A fresh {@link BesideState}. */
+export function createBesideState() {
+    return { running: new Map(), started: new Map(), ended: [] };
+}
+/**
+ * Release the per-Op leases a stopped operator still holds (#2879), best
+ * effort: a lease that is gone or taken over since is left alone, and one that
+ * can't be released expires on its TTL as before.
+ */
+export async function releaseHeldLeases(held, holder, opts = {}) {
+    const entries = [...held];
+    held.clear();
+    await Promise.allSettled(entries.map(([op, token]) => releaseLease(op, holder, token, opts)));
+}
+/** Wait for every run in flight beside the turns to end. `chant operator --steward --once` does before it exits. */
+export async function waitForBesideRuns(state) {
+    await Promise.all([...state.running.values()].map((r) => r.handle.done));
+}
+/**
+ * Ask every run in flight beside the turns to stop, and wait up to `graceMs`
+ * for them to end. The operator does when it stops, so no run it started
+ * outlives it holding the Op's lease past its renewals.
+ */
+export async function stopBesideRuns(state, graceMs = 10000) {
+    const runs = [...state.running.values()];
+    for (const r of runs)
+        r.handle.stop();
+    let timer;
+    await Promise.race([
+        Promise.all(runs.map((r) => r.handle.done)),
+        new Promise((resolve) => {
+            timer = setTimeout(resolve, graceMs);
+            timer.unref?.();
+        }),
+    ]);
+    if (timer)
+        clearTimeout(timer);
+}
+/**
+ * The steward's Ops whose newest run is the steward's own and stopped on a
+ * decision point, with that question's state now. The questions are read
+ * from the workspace the way `points` reads them; a workspace that can't be
+ * read leaves every run waiting.
+ */
+async function stewardWaits(steward, opts) {
+    const out = new Map();
+    const runs = [];
+    for (const op of steward.ops) {
+        try {
+            const newest = (await readRunLedger(runEnvOf(op), op.name, { cwd: opts.cwd })).records.at(-1);
+            if (newest?.status === "waiting" && newest.point && newest.steward === steward.name) {
+                runs.push({ op: op.name, question: newest.point.id, point: newest.point.point, state: newest.point.state });
+            }
+        }
+        catch {
+            // A ledger that can't be read has no waiting run to resume.
+        }
+    }
+    if (runs.length === 0)
+        return out;
+    const states = await (opts.readQuestions ?? readQuestionStates)(opts.cwd ?? process.cwd());
+    for (const r of runs) {
+        const now = states?.get(r.question);
+        const answered = states !== null && (now === undefined || now === "answered");
+        out.set(r.op, { question: r.question, point: r.point, state: now ?? r.state, answered });
+    }
+    return out;
+}
+/**
+ * The steward's Ops whose newest run is the steward's own and stopped at a
+ * gate, and whether the gate has been approved since. A gate ledger that
+ * can't be read leaves the run where it stopped.
+ */
+async function stewardGates(steward, opts) {
+    const out = new Map();
+    for (const op of steward.ops) {
+        try {
+            const newest = (await readRunLedger(runEnvOf(op), op.name, { cwd: opts.cwd })).records.at(-1);
+            if (newest?.status !== "gated" || !newest.gate || newest.steward !== steward.name)
+                continue;
+            const gateOp = newest.gate.op ?? op.name;
+            const { resolutions } = await readGateLedger(gateOp, { cwd: opts.cwd });
+            const approved = latestResolutionSince(resolutions, newest.gate.name, newest.gate.since) !== undefined;
+            out.set(op.name, { gateOp, gate: newest.gate.name, approved });
+        }
+        catch {
+            // A ledger that can't be read has no gated run to resume.
+        }
+    }
+    return out;
+}
+/** Every question's state in the workspace, by id, or null when the workspace's points can't be read. */
+async function readQuestionStates(cwd) {
+    const { readQuestionStates: read } = await import("../workspace/points-cli.js");
+    return read(cwd);
+}
+/** Take or renew a local steward's own lease (#2731). */
+export async function acquireStewardLease(steward, holder, opts = {}) {
+    return acquireLease(stewardLeaseName(steward), holder, { ...opts, ttlMs: opts.ttlMs ?? DEFAULT_LEASE_TTL_MS });
+}
+/**
+ * How long a caller contending for a steward's turn lease
+ * (`acquireStewardTurn`) retries before giving up (#2750). Long enough to
+ * ride out a race with a turn that is just ending; short enough that a
+ * genuinely busy steward is reported back promptly. A round never waits
+ * (`waitMs` omitted there): it already runs on its own timer and simply
+ * tries the turn again next round, exactly as it already does for a busy
+ * per-op lease. `chant run <op>` (`../cli/handlers/run.ts`) is the caller
+ * that waits, since a human is standing at the terminal for the answer.
+ */
+export const STEWARD_TURN_WAIT_MS = 2000;
+/** How often {@link acquireStewardTurn} retries while it waits. */
+const STEWARD_TURN_POLL_MS = 150;
+/**
+ * Take the steward's turn lease (#2750): the lease every run of one of its
+ * Ops — a round's scheduled tick, or `chant run <op>` typed by hand — holds
+ * for exactly as long as that one run takes, so the two are never beside each
+ * other. Distinct from {@link acquireStewardLease}, which one `chant operator
+ * --steward` process holds for its whole life: this one is held per-run, so a
+ * hand run succeeds the moment a round's tick ends, not only once the daemon
+ * itself stops.
+ *
+ * Contention never queues past `waitMs` (0 by default): consistent with a
+ * round's own leases, which skip and report rather than wait (see this
+ * module's doc), and with the fountain form, whose busy teammate is likewise
+ * refused rather than retried. A caller that wants to ride out a race with a
+ * turn about to end passes `waitMs`, and this polls every
+ * {@link STEWARD_TURN_POLL_MS} until it acquires or the wait runs out.
+ */
+export async function acquireStewardTurn(steward, holder, opts = {}) {
+    const deadline = Date.now() + (opts.waitMs ?? 0);
+    for (;;) {
+        const result = await acquireLease(stewardTurnLeaseName(steward), holder, {
+            cwd: opts.cwd,
+            ttlMs: opts.ttlMs ?? DEFAULT_LEASE_TTL_MS,
+            now: opts.now,
+        });
+        if (result.acquired || result.reason !== "held" || Date.now() >= deadline)
+            return result;
+        await new Promise((r) => setTimeout(r, STEWARD_TURN_POLL_MS));
+    }
+}
+/** Release a steward's turn lease, best-effort — a run whose turn was never actually acquired (or already lost) has nothing to release. */
+async function releaseStewardTurn(steward, holder, token, opts = {}) {
+    await releaseLease(stewardTurnLeaseName(steward), holder, token, opts).catch(() => false);
+}
+/**
+ * The Ops one round runs as turns: a steward's Ops but those beside its turns
+ * (#2861), or every ConvergeOp. A steward's Op with no schedule is considered
+ * only to resume a run of the steward's that waits on a decision point.
+ */
+async function roundOps(opts) {
+    if (opts.steward)
+        return stewardTurnOps(opts.steward);
+    const { ops } = await discoverConvergeOps({ cwd: opts.cwd, env: opts.env });
+    return ops.map((d) => d.config);
+}
+/**
+ * Run exactly one round: for every discovered ConvergeOp (filtered per
+ * `opts.env`), try its lease; tick the ones this holder wins. Returns one
+ * event per discovered op, in discovery order — deterministic, so tests can
+ * assert on it directly without depending on log output.
+ */
+export async function runOperatorRound(opts) {
+    const holder = opts.holder ?? currentHolderId();
+    const events = [];
+    const steward = opts.steward;
+    const leaseOpts = { cwd: opts.cwd, ttlMs: opts.leaseTtlMs, now: opts.now };
+    // A local steward is one writer (#2731): its round runs only while it holds
+    // its own lease, renewed here and again after every tick, so a turn longer
+    // than the lease's TTL still keeps it.
+    const holdSteward = async () => {
+        if (!steward)
+            return true;
+        let acquired;
+        try {
+            acquired = await acquireStewardLease(steward.name, holder, leaseOpts);
+        }
+        catch (err) {
+            events.push({
+                kind: "lease-error",
+                op: stewardLeaseName(steward.name),
+                env: "-",
+                error: err instanceof Error ? err.message : String(err),
+            });
+            return false;
+        }
+        if (acquired.acquired)
+            return true;
+        events.push({
+            kind: "steward-busy",
+            op: stewardLeaseName(steward.name),
+            env: "-",
+            steward: steward.name,
+            ...(acquired.heldBy?.holder ? { heldBy: acquired.heldBy.holder } : {}),
+        });
+        return false;
+    };
+    if (!(await holdSteward()))
+        return events;
+    // The runs started beside the turns that ended since the last round (#2861).
+    const besideState = opts.besideState ?? createBesideState();
+    events.push(...besideState.ended.splice(0));
+    // A steward's Ops whose last run waits on a decision point (#2749), read
+    // once per round, and the state of those questions now.
+    const waiting = steward ? await stewardWaits(steward, opts) : new Map();
+    // And those whose last run stopped at a gate (#2779), with whether it has
+    // been approved since.
+    const gated = steward ? await stewardGates(steward, opts) : new Map();
+    let stewardLost = false;
+    for (const config of await roundOps(opts)) {
+        const env = steward ? runEnvOf(config) : (envOf(config) ?? "unknown");
+        const wait = waiting.get(config.name);
+        const stopped = gated.get(config.name);
+        let resumed;
+        let approved;
+        // A run waiting on a question that is now answered is resumed on this
+        // round, whatever its cron says. One whose question is still open is left
+        // until a person answers it, unless its cron fires, which asks again.
+        if (wait?.answered)
+            resumed = wait.question;
+        // The same for a run stopped at a gate that is now approved (#2779).
+        if (stopped?.approved)
+            approved = { op: stopped.gateOp, gate: stopped.gate };
+        const standing = wait
+            ? { kind: "waiting-on-point", op: config.name, env, point: wait.point, question: wait.question, state: wait.state }
+            : stopped
+                ? { kind: "waiting-on-gate", op: config.name, env, gateOp: stopped.gateOp, gate: stopped.gate }
+                : undefined;
+        const resuming = resumed !== undefined || approved !== undefined;
+        // An Op that declares its own cadence (#2120) is ticked on that cron
+        // rather than on every round. Level-triggered: the question is whether a
+        // firing minute has passed since this Op was last looked at, so a round
+        // that arrives late still owes exactly one tick — a converge tick
+        // re-observes everything anyway, and several missed fires are one tick's
+        // worth of work.
+        const cron = config.schedule?.cron;
+        if (cron) {
+            const now = (opts.now ?? (() => new Date()))();
+            const lastSeen = opts.scheduleState?.get(config.name);
+            const due = lastSeen === undefined ? cronMatches(cron, now) : cronDueBetween(cron, lastSeen, now);
+            opts.scheduleState?.set(config.name, now);
+            if (!due && !resuming) {
+                events.push(standing ?? { kind: "skipped-not-due", op: config.name, env, cron });
+                continue;
+            }
+        }
+        else if (steward && !resuming) {
+            // An unscheduled Op runs when someone asks for it; the steward runs it
+            // only to resume its own waiting or gated run.
+            if (standing)
+                events.push(standing);
+            continue;
+        }
+        let acquired;
+        try {
+            acquired = await acquireLease(config.name, holder, {
+                cwd: opts.cwd,
+                ttlMs: opts.leaseTtlMs ?? DEFAULT_LEASE_TTL_MS,
+                now: opts.now,
+            });
+        }
+        catch (err) {
+            // Not lease contention — the acquire attempt itself failed (most
+            // likely a StaleLockError, ./lifecycle/git.ts's diagnosable
+            // stand-in for "a previous operator was killed mid-write"). Report it
+            // for this op and move on to the next; never abort the whole round.
+            const message = err instanceof StaleLockError ? err.message : err instanceof Error ? err.message : String(err);
+            events.push({ kind: "lease-error", op: config.name, env, error: message });
+            continue;
+        }
+        if (!acquired.acquired) {
+            events.push({ kind: "skipped-lease-held", op: config.name, env, heldBy: acquired.heldBy?.holder });
+            continue;
+        }
+        const lease = acquired.lease;
+        opts.heldLeases?.set(config.name, lease.token);
+        // A local steward is one turn at a time (#2750): a hand run (`chant run
+        // <op>`) holds this same lease for as long as it runs, so a round never
+        // ticks beside one. It is one lease for the whole steward, not one per
+        // op, so finding it held stops the round outright — every op still ahead
+        // of it in `roundOps` would fail the identical check.
+        let turn;
+        if (steward) {
+            try {
+                turn = await acquireStewardTurn(steward.name, holder, { cwd: opts.cwd, ttlMs: opts.leaseTtlMs, now: opts.now });
+            }
+            catch (err) {
+                // Not turn contention — the acquire attempt itself failed (most
+                // likely a StaleLockError). Same shape as the per-op acquire's own
+                // catch above: report it and stop, rather than let it propagate and
+                // leak the per-op lease this round just took.
+                await releaseLease(config.name, holder, lease.token, { cwd: opts.cwd }).catch(() => false);
+                opts.heldLeases?.delete(config.name);
+                const message = err instanceof StaleLockError ? err.message : err instanceof Error ? err.message : String(err);
+                events.push({ kind: "lease-error", op: stewardTurnLeaseName(steward.name), env, error: message });
+                break;
+            }
+            if (!turn.acquired) {
+                await releaseLease(config.name, holder, lease.token, { cwd: opts.cwd }).catch(() => false);
+                opts.heldLeases?.delete(config.name);
+                events.push({ kind: "turn-busy", op: config.name, env, steward: steward.name, heldBy: turn.heldBy?.holder });
+                break;
+            }
+        }
+        // The run is the steward's turn (#2749): a decision point it asks names the
+        // steward and makes its model call through the steward's broker, and an
+        // answer from inside the turn is refused.
+        const runId = randomUUID();
+        const restoreTurn = steward
+            ? enterStewardTurn({ steward: steward.name, capabilities: steward.capabilities, vault: steward.vault, run: runId })
+            : undefined;
+        try {
+            const result = await runOpLocally(config, opts.activities, opts.profiles, opts.signal, {
+                runId,
+                // #2522: the environment the operator was started for (`--env`, or a
+                // steward's environment) reaches the activities through the run
+                // context, as `chant run --env` does.
+                ...(opts.env ?? opts.stewardEnv ? { env: opts.env ?? opts.stewardEnv } : {}),
+                ...(steward ? { steward: steward.name } : {}),
+                ledger: { cwd: opts.cwd },
+                // A steward's turn claims work leases as `<steward>/<op>@<holder>`
+                // (#2748), which is how `workspace status` finds the lease it holds.
+                ...(steward ? { work: { holder: stewardWorkHolder(steward.name, config.name, holder) } } : {}),
+                // #2301: without a sink, `settle` catches a failed ledger append and
+                // drops it. That is the failure this tick can least afford to lose —
+                // the message below points the reader at the ledger record, which is
+                // exactly the artifact a failed append means is not there.
+                onLedgerError: (err) => events.push({
+                    kind: "tick-failed",
+                    op: config.name,
+                    env,
+                    error: `Op "${config.name}" ran, but its record could not be appended to the run ledger: ` +
+                        `${err instanceof Error ? err.message : String(err)}`,
+                }),
+            });
+            const held = await stillHoldsLease(config.name, holder, lease.token, { cwd: opts.cwd });
+            events.push(held
+                ? {
+                    kind: "ticked",
+                    op: config.name,
+                    env,
+                    result,
+                    ...(resumed !== undefined ? { resumed } : {}),
+                    ...(approved !== undefined ? { approved } : {}),
+                }
+                : { kind: "fenced", op: config.name, env });
+        }
+        catch (err) {
+            // #2301: "see its ledger record" was the whole message, and it sent the
+            // reader to an artifact that a failed ledger append means is missing —
+            // while `err.result` was carrying the failing steps all along. Name the
+            // failing steps here, and fall back to `cause` for a failure that
+            // produced no step record at all.
+            const message = err instanceof OpRunFailure
+                ? failureDetail(config.name, err)
+                : err instanceof Error
+                    ? err.message
+                    : String(err);
+            events.push({ kind: "tick-failed", op: config.name, env, error: message });
+        }
+        finally {
+            restoreTurn?.();
+            if (steward && turn?.acquired)
+                await releaseStewardTurn(steward.name, holder, turn.lease.token, { cwd: opts.cwd });
+        }
+        if (!(await holdSteward())) {
+            stewardLost = true;
+            break;
+        }
+    }
+    // The Ops beside the turns (#2861) are considered whatever became of the
+    // turns: a turn in progress holds none of them. Only a steward that lost its
+    // own lease starts nothing.
+    if (steward && !stewardLost) {
+        await besideRound(steward, opts, holder, besideState, waiting, gated, events);
+    }
+    return events;
+}
+/**
+ * One round for a steward's Ops beside its turns (#2861). For each one not
+ * already running: start it when a run of the steward's waits on a question
+ * now answered or a gate now approved, when its cron fires, or when its ready
+ * step names work it has not started a run for. The run is started and not
+ * waited for.
+ */
+async function besideRound(steward, opts, holder, state, waiting, gated, events) {
+    for (const beside of stewardBesideOf(steward)) {
+        const config = steward.ops.find((op) => op.name === beside.op);
+        if (!config)
+            continue;
+        const env = runEnvOf(config);
+        // The cron is read whether or not a run is in flight, so a fire during a
+        // run is dropped (overlap "skip") rather than owed afterwards.
+        const cron = config.schedule?.cron;
+        let due = false;
+        if (cron) {
+            const now = (opts.now ?? (() => new Date()))();
+            const lastSeen = opts.scheduleState?.get(config.name);
+            due = lastSeen === undefined ? cronMatches(cron, now) : cronDueBetween(cron, lastSeen, now);
+            opts.scheduleState?.set(config.name, now);
+        }
+        const mine = state.running.get(config.name);
+        if (mine) {
+            events.push({ kind: "beside-running", op: config.name, env, heldBy: mine.holder });
+            continue;
+        }
+        // A run another process holds the lease for: a hand `chant run`, or one
+        // this steward started before its operator restarted.
+        let current;
+        try {
+            current = (await readLease(config.name, { cwd: opts.cwd })).record;
+        }
+        catch (err) {
+            events.push({ kind: "lease-error", op: config.name, env, error: err instanceof Error ? err.message : String(err) });
+            continue;
+        }
+        const now = (opts.now ?? (() => new Date()))();
+        if (current && new Date(current.expiresAt).getTime() > now.getTime()) {
+            events.push({ kind: "beside-running", op: config.name, env, heldBy: current.holder });
+            continue;
+        }
+        const wait = waiting.get(config.name);
+        const stopped = gated.get(config.name);
+        const standing = wait
+            ? { kind: "waiting-on-point", op: config.name, env, point: wait.point, question: wait.question, state: wait.state }
+            : stopped
+                ? { kind: "waiting-on-gate", op: config.name, env, gateOp: stopped.gateOp, gate: stopped.gate }
+                : undefined;
+        let why;
+        if (wait?.answered)
+            why = "resumed";
+        else if (stopped?.approved)
+            why = "approved";
+        else if (due)
+            why = "cron";
+        let keys;
+        if (!why && beside.ready) {
+            const answer = await askReady(beside.ready, opts.activities, opts.signal);
+            if ("error" in answer) {
+                events.push({ kind: "ready-failed", op: config.name, env, error: answer.error });
+                continue;
+            }
+            if (answer.ready) {
+                const seen = state.started.get(config.name) ?? new Set();
+                if (answer.keys.length === 0 || answer.keys.some((k) => !seen.has(k))) {
+                    why = "ready";
+                    keys = answer.keys;
+                }
+                else {
+                    events.push(standing ?? { kind: "skipped-not-ready", op: config.name, env, already: answer.keys });
+                    continue;
+                }
+            }
+            else {
+                events.push(standing ?? { kind: "skipped-not-ready", op: config.name, env });
+                continue;
+            }
+        }
+        if (!why) {
+            const idle = standing ?? (cron ? { kind: "skipped-not-due", op: config.name, env, cron } : undefined);
+            if (idle)
+                events.push(idle);
+            continue;
+        }
+        const runHolder = stewardWorkHolder(steward.name, config.name, holder);
+        let handle;
+        try {
+            handle = (opts.launchBeside ?? spawnBesideRun)({
+                op: config,
+                steward: steward.name,
+                ...(opts.stewardEnv !== undefined ? { env: opts.stewardEnv } : {}),
+                holder: runHolder,
+                cwd: opts.cwd ?? process.cwd(),
+                ...(opts.leaseTtlMs !== undefined ? { leaseTtlMs: opts.leaseTtlMs } : {}),
+            });
+        }
+        catch (err) {
+            events.push({ kind: "tick-failed", op: config.name, env, error: err instanceof Error ? err.message : String(err) });
+            continue;
+        }
+        if (keys) {
+            const seen = state.started.get(config.name) ?? new Set();
+            for (const k of keys)
+                seen.add(k);
+            state.started.set(config.name, seen);
+        }
+        state.running.set(config.name, { handle, holder: runHolder, why });
+        void handle.done.then((exit) => {
+            state.running.delete(config.name);
+            state.ended.push({ kind: "beside-ended", op: config.name, env, code: exit.code, ...(exit.error ? { error: exit.error } : {}) });
+        });
+        events.push({
+            kind: "beside-started",
+            op: config.name,
+            env,
+            why,
+            holder: runHolder,
+            ...(keys && keys.length > 0 ? { keys } : {}),
+            ...(why === "resumed" && wait ? { resumed: wait.question } : {}),
+            ...(why === "approved" && stopped ? { approved: { op: stopped.gateOp, gate: stopped.gate } } : {}),
+        });
+    }
+}
+/**
+ * What a failed tick says about itself (#2301).
+ *
+ * `chant operator` renders its own events rather than going through
+ * `renderHuman`, so the executor's step records reached nobody here even after
+ * the executor stopped dropping them. Every failing step's message, in order,
+ * or the underlying error when the run failed without producing one.
+ */
+function failureDetail(op, err) {
+    const failed = err.result.records.filter((r) => r.status === "fail" && r.error);
+    if (failed.length > 0) {
+        return `Op "${op}" failed: ${failed.map((r) => `${r.fn}: ${r.error}`).join("; ")}`;
+    }
+    const cause = err.cause;
+    if (cause !== undefined) {
+        return `Op "${op}" failed: ${cause instanceof Error ? cause.message : String(cause)}`;
+    }
+    return `Op "${op}" failed — see its ledger record for step-level detail`;
+}
+/** Abortable sleep — resolves early (without throwing) if `signal` fires mid-wait, so the operator loop can stop promptly on Ctrl-C rather than finishing out a long interval. */
+function sleepAbortable(ms, signal) {
+    return new Promise((resolve) => {
+        if (signal?.aborted)
+            return resolve();
+        const timer = setTimeout(() => {
+            signal?.removeEventListener("abort", onAbort);
+            resolve();
+        }, ms);
+        const onAbort = () => {
+            clearTimeout(timer);
+            resolve();
+        };
+        signal?.addEventListener("abort", onAbort, { once: true });
+    });
+}
+/** Render one signal event as `chant operator`'s log line shape. */
+export function formatSignalLine(event) {
+    switch (event.kind) {
+        case "subscribed":
+            return `operator: ${event.lexicon} change signal subscribed (the timer still runs)`;
+        case "subscribe-failed":
+            return `operator: ${event.lexicon} change signal unavailable, polling on the timer (${event.error})`;
+        case "subscription-lost":
+            return `operator: ${event.lexicon} change signal lost, polling on the timer until the next round re-subscribes (${event.error})`;
+        case "woken":
+            return `operator: woken by a change signal after ${event.afterMs}ms, running an ordinary round`;
+    }
+}
+/**
+ * Run rounds forever, sleeping `intervalMs` between them, until
+ * `opts.signal` aborts. The daemon is a convenience, never a requirement
+ * (issue: "if it dies, nothing breaks and nothing is lost") — this function
+ * is exactly `while (!aborted) { round(); sleep(); }`, nothing durable lives
+ * in its own memory that a restart would need to recover.
+ *
+ * With `subscribers` (#1981) the sleep gains a second wake source, and only a
+ * wake source. A signal aborts the current sleep; the round that follows is
+ * the same round the timer would have run, deriving everything from a fresh
+ * observation. Nothing a subscription reports reaches a tick, because the
+ * seam has nowhere to put it: `onChange` takes no arguments.
+ */
+export async function runOperatorForever(opts) {
+    const holder = opts.holder ?? currentHolderId();
+    // Per-op cron state (#2120) lives for the life of the loop. A restart starts
+    // it empty, which means the first round ticks a scheduled Op only if it
+    // lands on a firing minute — a fresh operator owes no catch-up for the ticks
+    // it was not running for.
+    const scheduleState = opts.scheduleState ?? new Map();
+    // The runs a steward starts beside its turns (#2861), for the life of the
+    // loop. When the loop stops, so do they.
+    const besideState = opts.besideState ?? createBesideState();
+    // The per-Op leases the rounds leave held (#2879), released when the loop stops.
+    const heldLeases = opts.heldLeases ?? new Map();
+    const intervalMs = opts.intervalMs ?? DEFAULT_OPERATOR_INTERVAL_MS;
+    const subscribers = opts.subscribers ?? [];
+    // Nothing to subscribe to: the loop below is the loop that shipped before
+    // #1981, and takes none of the machinery's cost.
+    if (subscribers.length === 0) {
+        try {
+            while (!opts.signal?.aborted) {
+                const events = await runOperatorRound({ ...opts, holder, scheduleState, besideState, heldLeases });
+                opts.onRound?.(events);
+                if (opts.signal?.aborted)
+                    break;
+                await sleepAbortable(intervalMs, opts.signal);
+            }
+        }
+        finally {
+            await stopBesideRuns(besideState);
+            await releaseHeldLeases(heldLeases, holder, { cwd: opts.cwd });
+        }
+        return;
+    }
+    const gate = createChangeSignalGate({ floorMs: opts.signalFloorMs ?? DEFAULT_SIGNAL_FLOOR_MS });
+    // The subscriptions the loop currently holds, by lexicon. A lexicon absent
+    // from this map is one to (re-)subscribe on the next round, which is how a
+    // killed watch comes back without a special retry path.
+    const live = new Map();
+    // Stops every subscription at once when the loop ends, whatever ends it.
+    const subscriptionsAbort = new AbortController();
+    const closeAll = async () => {
+        subscriptionsAbort.abort();
+        const open = [...live.values()];
+        live.clear();
+        // A close that throws must not mask why the loop is unwinding.
+        await Promise.allSettled(open.map((s) => s.close()));
+    };
+    const ensureSubscribed = async () => {
+        for (const subscriber of subscribers) {
+            if (live.has(subscriber.lexicon) || subscriptionsAbort.signal.aborted)
+                continue;
+            try {
+                const subscription = await subscriber.subscribe({
+                    onChange: () => gate.signal(),
+                    onError: (error) => {
+                        // Reported once, then forgotten: dropping it from `live` is what
+                        // makes the next round re-subscribe. The loop itself is untouched.
+                        if (!live.delete(subscriber.lexicon))
+                            return;
+                        opts.onSignalEvent?.({ kind: "subscription-lost", lexicon: subscriber.lexicon, error });
+                    },
+                    signal: subscriptionsAbort.signal,
+                });
+                live.set(subscriber.lexicon, subscription);
+                opts.onSignalEvent?.({ kind: "subscribed", lexicon: subscriber.lexicon });
+            }
+            catch (err) {
+                // A subscription that cannot be established is a slower loop, never a
+                // stopped one: report it and fall through to the timer.
+                opts.onSignalEvent?.({
+                    kind: "subscribe-failed",
+                    lexicon: subscriber.lexicon,
+                    error: err instanceof Error ? err.message : String(err),
+                });
+            }
+        }
+    };
+    try {
+        while (!opts.signal?.aborted) {
+            gate.roundStarted();
+            const startedAt = Date.now();
+            const events = await runOperatorRound({ ...opts, holder, scheduleState, besideState, heldLeases });
+            opts.onRound?.(events);
+            if (opts.signal?.aborted)
+                break;
+            await ensureSubscribed();
+            if (opts.signal?.aborted)
+                break;
+            const reason = await gate.wait(intervalMs, opts.signal);
+            if (reason === "signal") {
+                opts.onSignalEvent?.({ kind: "woken", afterMs: Date.now() - startedAt });
+            }
+        }
+    }
+    finally {
+        await closeAll();
+        await stopBesideRuns(besideState);
+        await releaseHeldLeases(heldLeases, holder, { cwd: opts.cwd });
+    }
+}
+/** Render one round's events as `chant operator`'s log line shape — one line per ticked/skipped/failed op, reusing each ticked op's own `convergeTick` log line (`OpRunResult`'s outcome attribute doesn't carry it, so this reads the tick's step record's `Remediated`... actually the human-readable summary comes from the ledger, not the run result — see `../cli/handlers/operator.ts` for where `chant operator status` reads it back). This formats what's available from the round itself: which op, which env, and whether it ticked, skipped, or failed. */
+export function formatRoundLine(event) {
+    switch (event.kind) {
+        case "ticked":
+            return `operator: ${event.op}@${event.env} ticked=1 status=${event.result.status}` +
+                (event.result.gate ? ` gate="${event.result.gate.gate}"` : "") +
+                (event.result.point ? ` point="${event.result.point.id}"` : "") +
+                (event.resumed ? ` resumed="${event.resumed}"` : "") +
+                (event.approved ? ` approved="${event.approved.op}/${event.approved.gate}"` : "");
+        case "waiting-on-point":
+            return `operator: ${event.op}@${event.env} waiting=1(point:${event.question}:${event.state})`;
+        case "waiting-on-gate":
+            return `operator: ${event.op}@${event.env} gated=1(gate:${event.gateOp}/${event.gate})`;
+        case "skipped-lease-held":
+            return `operator: ${event.op}@${event.env} skipped=1(lease-held${event.heldBy ? `:${event.heldBy}` : ""})`;
+        case "skipped-not-due":
+            return `operator: ${event.op}@${event.env} skipped=1(not-due:${event.cron})`;
+        case "tick-failed":
+            return `operator: ${event.op}@${event.env} failed=1 error="${event.error}"`;
+        case "fenced":
+            return `operator: ${event.op}@${event.env} fenced=1(lease lost mid-tick — ledger record still written)`;
+        case "lease-error":
+            return `operator: ${event.op}@${event.env} error=1(lease acquire failed — ${event.error})`;
+        case "steward-busy":
+            return `operator: steward ${event.steward} skipped=1(steward-lease-held${event.heldBy ? `:${event.heldBy}` : ""})`;
+        case "turn-busy":
+            return `operator: ${event.op}@${event.env} skipped=1(turn-held${event.heldBy ? `:${event.heldBy}` : ""}, steward "${event.steward}" is already mid-turn)`;
+        case "beside-started":
+            return `operator: ${event.op}@${event.env} started=1(beside:${event.why}) holder="${event.holder}"` +
+                (event.keys ? ` keys="${event.keys.join(",")}"` : "") +
+                (event.resumed ? ` resumed="${event.resumed}"` : "") +
+                (event.approved ? ` approved="${event.approved.op}/${event.approved.gate}"` : "");
+        case "beside-running":
+            return `operator: ${event.op}@${event.env} skipped=1(beside-running${event.heldBy ? `:${event.heldBy}` : ""})`;
+        case "beside-ended":
+            return `operator: ${event.op}@${event.env} ended=1(beside) exit=${event.code ?? "none"}` + (event.error ? ` error="${event.error}"` : "");
+        case "skipped-not-ready":
+            return `operator: ${event.op}@${event.env} skipped=1(not-ready${event.already ? `:already-started:${event.already.join(",")}` : ""})`;
+        case "ready-failed":
+            return `operator: ${event.op}@${event.env} failed=1(ready) error="${event.error}"`;
+    }
+}