@intentius/chant 0.49.0 → 0.51.0

package/src/op/operator.test.ts

@@ -0,0 +1,346 @@
+import { describe, test, expect } from "vitest";
+import { withTestDir } from "@intentius/chant-test-utils";
+import { spawnSync } from "node:child_process";
+import { writeFileSync, mkdirSync } from "node:fs";
+import { join } from "node:path";
+import type { ActivityFn, ActivityProfile } from "./activity-registry";
+import {
+  discoverConvergeOps,
+  runOperatorRound,
+  runOperatorForever,
+  formatRoundLine,
+  DEFAULT_OPERATOR_INTERVAL_MS,
+} from "./operator";
+import { readLease } from "../lifecycle/lease";
+import { appendConvergeRecord, readConvergeLedger } from "../lifecycle/converge-ledger";
+
+const PROFILES: Record<string, ActivityProfile> = {};
+
+function git(args: string[], cwd: string): { stdout: string; exitCode: number } {
+  const r = spawnSync("git", args, { cwd, encoding: "utf-8" });
+  return { stdout: r.stdout ?? "", exitCode: r.status ?? -1 };
+}
+
+async function initRepo(dir: string): Promise<void> {
+  git(["init", "-q", "-b", "main"], dir);
+  git(["config", "user.email", "test@chant.dev"], dir);
+  git(["config", "user.name", "Test"], dir);
+  writeFileSync(join(dir, "README.md"), "fixture\n");
+  git(["add", "README.md"], dir);
+  git(["commit", "-q", "-m", "init"], dir);
+}
+
+/**
+ * Write a minimal fixture ConvergeOp — the same `{ props: OpConfig }` shape
+ * `discoverOps` reads from a real `*.op.ts` file's default export
+ * (`../op/discover.ts`), hand-built here rather than going through the real
+ * `ConvergeOp()` composite (lexicons/temporal) so this suite tests the
+ * operator's own discovery/lease/tick-loop logic without depending on that
+ * lexicon at all. One "Converge" phase calling a single fake activity —
+ * `runOpLocally` doesn't care what the phases are named or how many there
+ * are, only that they run through the local executor.
+ */
+function writeFixtureConvergeOp(dir: string, opName: string, env: string): void {
+  const opsDir = join(dir, "ops");
+  mkdirSync(opsDir, { recursive: true });
+  writeFileSync(
+    join(opsDir, `${opName}.op.ts`),
+    `export default {
+  props: {
+    name: ${JSON.stringify(opName)},
+    overview: "fixture converge op",
+    phases: [{ name: "Converge", steps: [{ kind: "activity", fn: "fakeTick", args: {} }] }],
+    searchAttributes: { Converge: "true", Env: ${JSON.stringify(env)}, Dial: "observe" },
+  },
+};
+`,
+  );
+}
+
+function writeFixtureNonConvergeOp(dir: string, opName: string): void {
+  const opsDir = join(dir, "ops");
+  mkdirSync(opsDir, { recursive: true });
+  writeFileSync(
+    join(opsDir, `${opName}.op.ts`),
+    `export default {
+  props: {
+    name: ${JSON.stringify(opName)},
+    overview: "fixture ordinary op",
+    phases: [{ name: "Deploy", steps: [{ kind: "activity", fn: "fakeTick", args: {} }] }],
+  },
+};
+`,
+  );
+}
+
+/** A fake `convergeTick`-shaped activity that appends a real converge-ledger record — so a round's ledger effect is observable through the public reader, the same as the real activity's I/O shape. */
+function fakeTickActivities(dir: string, env: string, opName: string, opts?: { throws?: boolean }): Map<string, ActivityFn> {
+  return new Map<string, ActivityFn>([
+    [
+      "fakeTick",
+      async () => {
+        if (opts?.throws) throw new Error("boom: simulated tick failure");
+        await appendConvergeRecord(
+          {
+            op: opName,
+            env,
+            timestamp: new Date().toISOString(),
+            firedRuleIds: [],
+            outcomes: [],
+            summary: { drifted: 0, remediated: 0, reported: 0, skippedBudget: 0, skippedFlap: 0, unobserved: 0, adopted: 0 },
+            log: `converge(${env}): drifted=0 remediated=0 reported=0 skipped-budget=0 skipped-flap=0 unobserved=0 adopted=0`,
+          },
+          { cwd: dir },
+        );
+        return { ok: true };
+      },
+    ],
+  ]);
+}
+
+describe("discoverConvergeOps", () => {
+  test("finds only ops with searchAttributes.Converge === 'true'", async () => {
+    await withTestDir(async (dir) => {
+      await initRepo(dir);
+      writeFixtureConvergeOp(dir, "staging-converge", "staging");
+      writeFixtureNonConvergeOp(dir, "fountain-apply");
+
+      const { ops, errors } = await discoverConvergeOps({ cwd: dir });
+      expect(errors).toEqual([]);
+      expect(ops.map((d) => d.config.name)).toEqual(["staging-converge"]);
+    });
+  });
+
+  test("--env filters to ConvergeOps declaring that env", async () => {
+    await withTestDir(async (dir) => {
+      await initRepo(dir);
+      writeFixtureConvergeOp(dir, "staging-converge", "staging");
+      writeFixtureConvergeOp(dir, "prod-converge", "prod");
+
+      const { ops } = await discoverConvergeOps({ cwd: dir, env: "prod" });
+      expect(ops.map((d) => d.config.name)).toEqual(["prod-converge"]);
+    });
+  });
+
+  test("no env filter discovers every ConvergeOp, sorted by name", async () => {
+    await withTestDir(async (dir) => {
+      await initRepo(dir);
+      writeFixtureConvergeOp(dir, "staging-converge", "staging");
+      writeFixtureConvergeOp(dir, "prod-converge", "prod");
+
+      const { ops } = await discoverConvergeOps({ cwd: dir });
+      expect(ops.map((d) => d.config.name)).toEqual(["prod-converge", "staging-converge"]);
+    });
+  });
+});
+
+describe("runOperatorRound — lease + tick execution over a fixture ConvergeOp (offline)", () => {
+  test("acquires the lease, ticks, and records the tick as a ledger fact", async () => {
+    await withTestDir(async (dir) => {
+      await initRepo(dir);
+      writeFixtureConvergeOp(dir, "staging-converge", "staging");
+      const activities = fakeTickActivities(dir, "staging", "staging-converge");
+
+      const events = await runOperatorRound({ cwd: dir, holder: "op-a", activities, profiles: PROFILES });
+      expect(events).toEqual([{ kind: "ticked", op: "staging-converge", env: "staging", result: expect.objectContaining({ ok: true }) }]);
+
+      const { records } = await readConvergeLedger("staging", { cwd: dir });
+      expect(records).toHaveLength(1);
+      expect(records[0].op).toBe("staging-converge");
+    });
+  });
+
+  test("a second operator process (different holder) sees the lease held and skips — never queues", async () => {
+    await withTestDir(async (dir) => {
+      await initRepo(dir);
+      writeFixtureConvergeOp(dir, "staging-converge", "staging");
+      const activities = fakeTickActivities(dir, "staging", "staging-converge");
+
+      // holder-a acquires and is (simulated as) still ticking — hold the
+      // lease without releasing, by not letting a full round complete for it.
+      const { acquireLease } = await import("../lifecycle/lease");
+      await acquireLease("staging-converge", "holder-a", { cwd: dir, ttlMs: 60_000 });
+
+      const events = await runOperatorRound({ cwd: dir, holder: "holder-b", activities, profiles: PROFILES });
+      expect(events).toEqual([{ kind: "skipped-lease-held", op: "staging-converge", env: "staging", heldBy: "holder-a" }]);
+
+      // Nothing ticked — no ledger record.
+      const { records } = await readConvergeLedger("staging", { cwd: dir });
+      expect(records).toHaveLength(0);
+    });
+  });
+
+  test("a stale lease .lock file reports lease-error for that op — never a thrown exception out of the round, never misread as lease-held (#1959 finding 2)", async () => {
+    await withTestDir(async (dir) => {
+      await initRepo(dir);
+      writeFixtureConvergeOp(dir, "staging-converge", "staging");
+      const activities = fakeTickActivities(dir, "staging", "staging-converge");
+
+      // A previous `chant operator` acquired the lease, then was killed
+      // mid-renewal — its `.lock` file is still sitting there, blocking
+      // every future write to this op's lease ref.
+      const { acquireLease } = await import("../lifecycle/lease");
+      await acquireLease("staging-converge", "holder-a", { cwd: dir, ttlMs: 60_000 });
+      mkdirSync(join(dir, ".git", "refs", "chant", "lease"), { recursive: true });
+      writeFileSync(join(dir, ".git", "refs", "chant", "lease", "staging-converge.lock"), "");
+
+      // holder-a itself tries to renew next round — it owns the lease, so
+      // acquireLease attempts the write and hits the lock file.
+      const events = await runOperatorRound({ cwd: dir, holder: "holder-a", activities, profiles: PROFILES });
+      expect(events).toHaveLength(1);
+      expect(events[0]).toMatchObject({ kind: "lease-error", op: "staging-converge", env: "staging" });
+      expect((events[0] as { error: string }).error).toContain("staging-converge.lock");
+
+      // Nothing ticked.
+      const { records } = await readConvergeLedger("staging", { cwd: dir });
+      expect(records).toHaveLength(0);
+    });
+  });
+
+  test("a failing tick reports tick-failed, not a thrown exception out of the round", async () => {
+    await withTestDir(async (dir) => {
+      await initRepo(dir);
+      writeFixtureConvergeOp(dir, "staging-converge", "staging");
+      const activities = fakeTickActivities(dir, "staging", "staging-converge", { throws: true });
+
+      const events = await runOperatorRound({ cwd: dir, holder: "op-a", activities, profiles: PROFILES });
+      expect(events).toHaveLength(1);
+      expect(events[0].kind).toBe("tick-failed");
+    });
+  });
+
+  test("ticks every discovered ConvergeOp across environments in one round", async () => {
+    await withTestDir(async (dir) => {
+      await initRepo(dir);
+      writeFixtureConvergeOp(dir, "staging-converge", "staging");
+      writeFixtureConvergeOp(dir, "prod-converge", "prod");
+      const activities = new Map<string, ActivityFn>([
+        ...fakeTickActivities(dir, "staging", "staging-converge"),
+      ]);
+      // Both fixtures call the same "fakeTick" fn name; one shared fake that
+      // reads env/op from the running Op's own config isn't available to an
+      // ActivityFn (it only sees step args), so this test only asserts round
+      // shape/coverage, not each op's own ledger content — that's covered by
+      // the single-op test above.
+      activities.set("fakeTick", async () => ({ ok: true }));
+
+      const events = await runOperatorRound({ cwd: dir, holder: "op-a", activities, profiles: PROFILES });
+      expect(events.map((e) => e.op).sort()).toEqual(["prod-converge", "staging-converge"]);
+      expect(events.every((e) => e.kind === "ticked")).toBe(true);
+    });
+  });
+});
+
+describe("crash recovery — re-tick after a lease expires with no manual repair (#1485)", () => {
+  test("holder A ticks, 'crashes' (never renews); after the TTL passes, holder B re-acquires and re-ticks — the ledger shows continuity, not a gap", async () => {
+    await withTestDir(async (dir) => {
+      await initRepo(dir);
+      writeFixtureConvergeOp(dir, "staging-converge", "staging");
+      const activities = fakeTickActivities(dir, "staging", "staging-converge");
+
+      const t0 = new Date("2026-01-01T00:00:00.000Z");
+      const round1 = await runOperatorRound({
+        cwd: dir, holder: "operator-a", leaseTtlMs: 1_000, activities, profiles: PROFILES, now: () => t0,
+      });
+      expect(round1[0].kind).toBe("ticked");
+
+      // operator-a crashes: no release, no renewal. A restarted process (a
+      // new holder id, matching `currentHolderId()`'s per-process identity)
+      // tries again after the lease's TTL has passed.
+      const tAfterCrash = new Date("2026-01-01T00:00:05.000Z");
+      const round2 = await runOperatorRound({
+        cwd: dir, holder: "operator-a-restarted", leaseTtlMs: 1_000, activities, profiles: PROFILES, now: () => tAfterCrash,
+      });
+      expect(round2[0].kind).toBe("ticked");
+
+      const { records } = await readConvergeLedger("staging", { cwd: dir });
+      expect(records).toHaveLength(2);
+
+      const { record: lease } = await readLease("staging-converge", { cwd: dir });
+      expect(lease?.holder).toBe("operator-a-restarted");
+    });
+  });
+
+  test("re-acquiring before the TTL passes still fails — a live operator isn't preempted by a phantom crash", async () => {
+    await withTestDir(async (dir) => {
+      await initRepo(dir);
+      writeFixtureConvergeOp(dir, "staging-converge", "staging");
+      const activities = fakeTickActivities(dir, "staging", "staging-converge");
+
+      const t0 = new Date("2026-01-01T00:00:00.000Z");
+      await runOperatorRound({ cwd: dir, holder: "operator-a", leaseTtlMs: 60_000, activities, profiles: PROFILES, now: () => t0 });
+
+      const tSoonAfter = new Date("2026-01-01T00:00:05.000Z");
+      const events = await runOperatorRound({
+        cwd: dir, holder: "operator-b", leaseTtlMs: 60_000, activities, profiles: PROFILES, now: () => tSoonAfter,
+      });
+      expect(events[0].kind).toBe("skipped-lease-held");
+    });
+  });
+});
+
+describe("runOperatorForever — interval/timer behavior", () => {
+  // `sleepAbortable` (operator.ts) is driven by real `setTimeout`, but each
+  // round underneath it makes several *real* git subprocess calls (lease
+  // read/acquire, ledger append) — genuine process I/O that fake timers
+  // (`vi.useFakeTimers`) can't fast-forward through, only JS-scheduled
+  // callbacks. So this exercises the actual interval/abort behavior over
+  // real (small) wall-clock time instead: a short `intervalMs`, polling for
+  // each expected round count with a generous bound, never a bare sleep.
+  async function waitForRoundCount(rounds: unknown[], n: number, maxWaitMs = 5_000): Promise<void> {
+    const deadline = Date.now() + maxWaitMs;
+    while (rounds.length < n && Date.now() < deadline) {
+      await new Promise((r) => setTimeout(r, 10));
+    }
+    expect(rounds.length).toBeGreaterThanOrEqual(n);
+  }
+
+  test("runs a round immediately, then again every intervalMs, stopping on abort", async () => {
+    await withTestDir(async (dir) => {
+      await initRepo(dir);
+      writeFixtureConvergeOp(dir, "staging-converge", "staging");
+      const activities = fakeTickActivities(dir, "staging", "staging-converge");
+
+      const controller = new AbortController();
+      const rounds: number[] = [];
+      const loop = runOperatorForever({
+        cwd: dir,
+        holder: "op-a",
+        intervalMs: 40,
+        activities,
+        profiles: PROFILES,
+        signal: controller.signal,
+        onRound: () => rounds.push(Date.now()),
+      });
+
+      await waitForRoundCount(rounds, 1); // the immediate first round
+      await waitForRoundCount(rounds, 3); // at least two more, ~40ms apart
+
+      controller.abort();
+      const countAtAbort = rounds.length;
+      await new Promise((r) => setTimeout(r, 120)); // well past a couple more intervals, if any fired
+      expect(rounds.length).toBe(countAtAbort); // no further round after abort
+
+      await loop;
+    });
+  }, 10_000);
+
+  test("default interval is used when none is given", () => {
+    expect(DEFAULT_OPERATOR_INTERVAL_MS).toBeGreaterThan(0);
+  });
+});
+
+describe("formatRoundLine", () => {
+  test("renders each event kind as one line", () => {
+    expect(formatRoundLine({ kind: "ticked", op: "x", env: "staging", result: { op: "x", records: [], totalMs: 1, ok: true } }))
+      .toContain("ticked=1 ok=true");
+    expect(formatRoundLine({ kind: "skipped-lease-held", op: "x", env: "staging", heldBy: "holder-a" }))
+      .toContain("lease-held:holder-a");
+    expect(formatRoundLine({ kind: "tick-failed", op: "x", env: "staging", error: "boom" }))
+      .toContain('error="boom"');
+    expect(formatRoundLine({ kind: "fenced", op: "x", env: "staging" }))
+      .toContain("fenced=1");
+    expect(formatRoundLine({ kind: "lease-error", op: "x", env: "staging", error: "stale lock at .../x.lock" }))
+      .toContain("stale lock at .../x.lock");
+  });
+});

package/src/op/operator.ts

@@ -0,0 +1,213 @@
+/**
+ * `chant operator` (#1485, epic #1487) — native durable ticks without
+ * Temporal. A deliberately small daemon: a timer, a lease
+ * (../lifecycle/lease.ts), and the existing local executor
+ * (./local-executor.ts) in a loop. No new service, no new state store — the
+ * lease lives on a git ref, tick state lives on the `chant/lifecycle`
+ * orphan branch via the same `convergeTick` activity `chant run <op>`
+ * already calls for a single one-shot tick.
+ *
+ * One round: discover every `ConvergeOp` (searchAttributes `Converge:
+ * "true"`, optionally filtered to one env), and for each one, try to
+ * acquire (or renew) its lease. Win it — run one tick via `runOpLocally` on
+ * the Op's own Observe → Converge phases, the identical path `chant run
+ * <name>` takes for a single tick (issue: "chant run fountain-converge —
+ * one tick, by hand — unchanged, and also the test story"). Lose it — skip
+ * and report, never queue (issue: "a tick that finds the lease held skips
+ * and reports ... never queues").
+ *
+ * Crash recovery needs no special code: a killed operator simply stops
+ * renewing its lease, which expires on its own TTL; the next round (from
+ * this process restarted, a different machine's operator, or a bare `chant
+ * run <name>` invoked by cron) re-acquires and re-ticks. A converge tick
+ * re-observes and re-derives everything every time (ConvergeOp's own
+ * design, #1484) — re-ticking after a crash and ticking on a normal
+ * schedule are the same act, not two different code paths.
+ *
+ * One crash *does* need a distinct report, though (#1959 finding 2): if the
+ * killed operator's own `git update-ref` was interrupted mid-write, it can
+ * leave a stale `.lock` file behind that blocks every future acquire
+ * attempt for that op's lease until someone removes it — TTL expiry doesn't
+ * help, since the ref update itself can't land. `acquireLease` surfaces
+ * that case as a `StaleLockError` rather than ordinary "lease held by
+ * someone else" contention, and a round reports it per-op as
+ * `{ kind: "lease-error" }` (never aborting the whole round) so `chant
+ * operator`'s log names the fix instead of quietly skipping that op
+ * forever.
+ */
+import type { ActivityFn, ActivityProfile } from "./activity-registry";
+import { discoverOps, type DiscoveredOp } from "./discover";
+import { runOpLocally, OpRunFailure, LocalGateUnsupportedError, type OpRunResult } from "./local-executor";
+import { acquireLease, stillHoldsLease, currentHolderId, DEFAULT_LEASE_TTL_MS, type LeaseRecord, type AcquireLeaseResult } from "../lifecycle/lease";
+import { StaleLockError } from "../lifecycle/git";
+
+/** Poll interval between rounds — the operator's own cadence, distinct from a `ConvergeOp`'s Temporal `schedule` cron (that field drives the durable path's `TemporalSchedule`, not anything the local executor can read back at discovery time; see this module's doc). Chosen short enough to converge promptly, long enough not to hammer `chant lifecycle plan --live` every few seconds. */
+export const DEFAULT_OPERATOR_INTERVAL_MS = 60_000;
+
+/** One ConvergeOp's own `searchAttributes.Env` (`../../lexicons/temporal/src/composites/converge-op.ts` always sets it). `undefined` for a non-ConvergeOp or a hand-built one missing it — filtered out by `isConvergeOp` before this is trusted. */
+function envOf(config: DiscoveredOp["config"]): string | undefined {
+  return config.searchAttributes?.Env;
+}
+
+function isConvergeOp(config: DiscoveredOp["config"]): boolean {
+  return config.searchAttributes?.Converge === "true";
+}
+
+/** Discover every `ConvergeOp` in the project (`searchAttributes.Converge === "true"`), optionally filtered to one environment. Ops that don't declare an `Env` search attribute are excluded from an `--env`-filtered discovery (there's nothing to match), but included when no `env` filter is given. */
+export async function discoverConvergeOps(
+  opts?: { cwd?: string; env?: string },
+): Promise<{ ops: DiscoveredOp[]; errors: string[] }> {
+  const { ops, errors } = await discoverOps({ cwd: opts?.cwd });
+  const converge = [...ops.values()].filter((d) => isConvergeOp(d.config));
+  const filtered = opts?.env ? converge.filter((d) => envOf(d.config) === opts.env) : converge;
+  return { ops: filtered.sort((a, b) => a.config.name.localeCompare(b.config.name)), errors };
+}
+
+/** One discovered ConvergeOp's outcome for one round — what `chant operator`'s one-line-per-tick log (and its tests) key off of. */
+export type OperatorTickEvent =
+  | { kind: "ticked"; op: string; env: string; result: OpRunResult }
+  | { kind: "skipped-lease-held"; op: string; env: string; heldBy?: string }
+  | { kind: "tick-failed"; op: string; env: string; error: string }
+  /** The lease was lost between acquiring it and the tick finishing (e.g. this process stalled past its TTL and another operator reclaimed it) — the tick's own ledger record (written inside `convergeTick`) still landed, since a converge tick is idempotent by design; this event exists purely so `chant operator`'s log and the ledger-independent test surface can see the fencing violation happened. Never a hard failure. */
+  | { kind: "fenced"; op: string; env: string }
+  /**
+   * `acquireLease` itself threw — a `StaleLockError` (#1959 finding 2: a
+   * previous `chant operator` was killed mid-write and left a `.lock` file
+   * behind, so this op's lease can no longer be acquired or renewed by
+   * anyone until the file is removed) or any other unexpected failure. A
+   * distinct outcome from `skipped-lease-held` on purpose — this is not
+   * ordinary contention with a live holder, it's wreckage that needs a
+   * human, and reading it as "someone else has it" would make the operator
+   * back off silently forever instead of surfacing the fix. One op's lease
+   * error never aborts the round for every other op.
+   */
+  | { kind: "lease-error"; op: string; env: string; error: string };
+
+export interface OperatorRoundOptions {
+  cwd?: string;
+  /** Restrict ticking to ConvergeOps for this environment. Omit to tick every discovered ConvergeOp. */
+  env?: string;
+  /** @default DEFAULT_LEASE_TTL_MS */
+  leaseTtlMs?: number;
+  /** This process's lease identity. @default currentHolderId() — a fresh identity per process, deliberately: two ticks from what could plausibly be two different processes should behave like two different holders even in a test that reuses one process, unless the caller wants to simulate "same process, later round" by passing a stable value. */
+  holder?: string;
+  /** Injected activity implementations — the CLI loads the real ones the same way `chant run` does (`loadActivities`); tests inject fakes so a round never shells out. */
+  activities: Map<string, ActivityFn>;
+  profiles: Record<string, ActivityProfile>;
+  now?: () => Date;
+  signal?: AbortSignal;
+}
+
+/**
+ * Run exactly one round: for every discovered ConvergeOp (filtered per
+ * `opts.env`), try its lease; tick the ones this holder wins. Returns one
+ * event per discovered op, in discovery order — deterministic, so tests can
+ * assert on it directly without depending on log output.
+ */
+export async function runOperatorRound(opts: OperatorRoundOptions): Promise<OperatorTickEvent[]> {
+  const holder = opts.holder ?? currentHolderId();
+  const { ops } = await discoverConvergeOps({ cwd: opts.cwd, env: opts.env });
+  const events: OperatorTickEvent[] = [];
+
+  for (const { config } of ops) {
+    const env = envOf(config) ?? "unknown";
+
+    let acquired: AcquireLeaseResult;
+    try {
+      acquired = await acquireLease(config.name, holder, {
+        cwd: opts.cwd,
+        ttlMs: opts.leaseTtlMs ?? DEFAULT_LEASE_TTL_MS,
+        now: opts.now,
+      });
+    } catch (err) {
+      // Not lease contention — the acquire attempt itself failed (most
+      // likely a StaleLockError, ./lifecycle/git.ts's diagnosable
+      // stand-in for "a previous operator was killed mid-write"). Report it
+      // for this op and move on to the next; never abort the whole round.
+      const message =
+        err instanceof StaleLockError ? err.message : err instanceof Error ? err.message : String(err);
+      events.push({ kind: "lease-error", op: config.name, env, error: message });
+      continue;
+    }
+
+    if (!acquired.acquired) {
+      events.push({ kind: "skipped-lease-held", op: config.name, env, heldBy: acquired.heldBy?.holder });
+      continue;
+    }
+
+    const lease = acquired.lease as LeaseRecord;
+    try {
+      const result = await runOpLocally(config, opts.activities, opts.profiles, opts.signal);
+      const held = await stillHoldsLease(config.name, holder, lease.token, { cwd: opts.cwd });
+      events.push(held ? { kind: "ticked", op: config.name, env, result } : { kind: "fenced", op: config.name, env });
+    } catch (err) {
+      const message =
+        err instanceof OpRunFailure
+          ? `Op "${config.name}" failed — see its ledger record for step-level detail`
+          : err instanceof LocalGateUnsupportedError
+            ? err.message
+            : err instanceof Error
+              ? err.message
+              : String(err);
+      events.push({ kind: "tick-failed", op: config.name, env, error: message });
+    }
+  }
+
+  return events;
+}
+
+/** Abortable sleep — resolves early (without throwing) if `signal` fires mid-wait, so the operator loop can stop promptly on Ctrl-C rather than finishing out a long interval. */
+function sleepAbortable(ms: number, signal?: AbortSignal): Promise<void> {
+  return new Promise((resolve) => {
+    if (signal?.aborted) return resolve();
+    const timer = setTimeout(() => {
+      signal?.removeEventListener("abort", onAbort);
+      resolve();
+    }, ms);
+    const onAbort = () => {
+      clearTimeout(timer);
+      resolve();
+    };
+    signal?.addEventListener("abort", onAbort, { once: true });
+  });
+}
+
+export interface OperatorLoopOptions extends OperatorRoundOptions {
+  /** @default DEFAULT_OPERATOR_INTERVAL_MS */
+  intervalMs?: number;
+  /** Called after every round completes — the CLI's one-line-per-tick log lives here, not inside the loop itself, so a test can drive rounds without capturing stdout. */
+  onRound?: (events: OperatorTickEvent[]) => void;
+}
+
+/**
+ * Run rounds forever, sleeping `intervalMs` between them, until
+ * `opts.signal` aborts. The daemon is a convenience, never a requirement
+ * (issue: "if it dies, nothing breaks and nothing is lost") — this function
+ * is exactly `while (!aborted) { round(); sleep(); }`, nothing durable lives
+ * in its own memory that a restart would need to recover.
+ */
+export async function runOperatorForever(opts: OperatorLoopOptions): Promise<void> {
+  const holder = opts.holder ?? currentHolderId();
+  while (!opts.signal?.aborted) {
+    const events = await runOperatorRound({ ...opts, holder });
+    opts.onRound?.(events);
+    if (opts.signal?.aborted) break;
+    await sleepAbortable(opts.intervalMs ?? DEFAULT_OPERATOR_INTERVAL_MS, opts.signal);
+  }
+}
+
+/** Render one round's events as `chant operator`'s log line shape — one line per ticked/skipped/failed op, reusing each ticked op's own `convergeTick` log line (`OpRunResult`'s outcome attribute doesn't carry it, so this reads the tick's step record's `Remediated`... actually the human-readable summary comes from the ledger, not the run result — see `../cli/handlers/operator.ts` for where `chant operator status` reads it back). This formats what's available from the round itself: which op, which env, and whether it ticked, skipped, or failed. */
+export function formatRoundLine(event: OperatorTickEvent): string {
+  switch (event.kind) {
+    case "ticked":
+      return `operator: ${event.op}@${event.env} ticked=1 ok=${event.result.ok}`;
+    case "skipped-lease-held":
+      return `operator: ${event.op}@${event.env} skipped=1(lease-held${event.heldBy ? `:${event.heldBy}` : ""})`;
+    case "tick-failed":
+      return `operator: ${event.op}@${event.env} failed=1 error="${event.error}"`;
+    case "fenced":
+      return `operator: ${event.op}@${event.env} fenced=1(lease lost mid-tick — ledger record still written)`;
+    case "lease-error":
+      return `operator: ${event.op}@${event.env} error=1(lease acquire failed — ${event.error})`;
+  }
+}