@intentius/chant 0.94.0 → 0.95.0

package/src/op/operator.ts

@@ -38,12 +38,13 @@
 import type { ActivityFn, ActivityProfile } from "./activity-registry";
 import { discoverOps, type DiscoveredOp } from "./discover";
 import { runOpLocally, OpRunFailure, type OpRunResult } from "./local-executor";
-import { acquireLease, releaseLease, stillHoldsLease, currentHolderId, DEFAULT_LEASE_TTL_MS, type LeaseRecord, type AcquireLeaseResult } from "../lifecycle/lease";
+import { acquireLease, releaseLease, stillHoldsLease, currentHolderId, readLease, DEFAULT_LEASE_TTL_MS, type LeaseRecord, type AcquireLeaseResult } from "../lifecycle/lease";
 import { randomUUID } from "node:crypto";
 import { readRunLedger, runEnvOf } from "../lifecycle/run-ledger";
 import { readGateLedger, latestResolutionSince } from "../lifecycle/gate-ledger";
 import { enterStewardTurn } from "./steward-turn";
-import { stewardLeaseName, stewardTurnLeaseName, type StewardDeclaration } from "./steward";
+import { stewardBesideOf, stewardLeaseName, stewardTurnLeaseName, stewardTurnOps, type StewardDeclaration } from "./steward";
+import { askReady, spawnBesideRun, type BesideHandle, type BesideLauncher, type BesideWhy } from "./steward-beside";
 import { stewardWorkHolder } from "./work-lease-run";
 import { StaleLockError } from "../lifecycle/git";
 import { cronMatches, cronDueBetween } from "./cron";
@@ -146,7 +147,78 @@ export type OperatorTickEvent =
    * lease for the whole steward, not one per op). The next round tries
    * again.
    */
-  | { kind: "turn-busy"; op: string; env: string; steward: string; heldBy?: string };
+  | { kind: "turn-busy"; op: string; env: string; steward: string; heldBy?: string }
+  /**
+   * A steward's Op beside its turns (#2861) was started in a process of its
+   * own, and the round went on without waiting for it. `why` is its cron, its
+   * ready step (`keys` are the work keys it named), or a waiting run to
+   * resume (`resumed`, the question now answered) or a gated one (`approved`).
+   * `holder` is the Op's lease holder and its work lease holder.
+   */
+  | {
+      kind: "beside-started";
+      op: string;
+      env: string;
+      why: BesideWhy;
+      holder: string;
+      keys?: string[];
+      resumed?: string;
+      approved?: { op: string; gate: string };
+    }
+  /** A run of an Op beside the turns is in flight (#2861): this operator's, or one whose lease another holder has, such as a hand `chant run`. Nothing was started. */
+  | { kind: "beside-running"; op: string; env: string; heldBy?: string }
+  /** A run this operator started beside the turns ended (#2861), reported on the round after. `code` is `chant run`'s exit code: 0 ok, 3 gated or waiting, 1 failed. */
+  | { kind: "beside-ended"; op: string; env: string; code: number | null; error?: string }
+  /**
+   * An Op beside the turns whose ready step names no work (#2861), or only
+   * work this operator already started a run for (`already`).
+   */
+  | { kind: "skipped-not-ready"; op: string; env: string; already?: string[] }
+  /** An Op beside the turns whose ready step failed, or answered in a shape it can't read (#2861). Nothing was started. */
+  | { kind: "ready-failed"; op: string; env: string; error: string };
+
+/**
+ * What a local steward's operator keeps about the runs it starts beside its
+ * turns (#2861), across rounds: the ones in flight, the work keys a ready
+ * step named that it already started a run for, and the runs that ended since
+ * the last round, which the next round reports. `runOperatorForever` owns one
+ * for its life; a restarted operator starts empty and finds a run still in
+ * flight by its lease.
+ */
+export interface BesideState {
+  running: Map<string, { handle: BesideHandle; holder: string; why: BesideWhy }>;
+  started: Map<string, Set<string>>;
+  ended: OperatorTickEvent[];
+}
+
+/** A fresh {@link BesideState}. */
+export function createBesideState(): BesideState {
+  return { running: new Map(), started: new Map(), ended: [] };
+}
+
+/** Wait for every run in flight beside the turns to end. `chant operator --steward --once` does before it exits. */
+export async function waitForBesideRuns(state: BesideState): Promise<void> {
+  await Promise.all([...state.running.values()].map((r) => r.handle.done));
+}
+
+/**
+ * Ask every run in flight beside the turns to stop, and wait up to `graceMs`
+ * for them to end. The operator does when it stops, so no run it started
+ * outlives it holding the Op's lease past its renewals.
+ */
+export async function stopBesideRuns(state: BesideState, graceMs = 10_000): Promise<void> {
+  const runs = [...state.running.values()];
+  for (const r of runs) r.handle.stop();
+  let timer: ReturnType<typeof setTimeout> | undefined;
+  await Promise.race([
+    Promise.all(runs.map((r) => r.handle.done)),
+    new Promise<void>((resolve) => {
+      timer = setTimeout(resolve, graceMs);
+      timer.unref?.();
+    }),
+  ]);
+  if (timer) clearTimeout(timer);
+}
 
 export interface OperatorRoundOptions {
   cwd?: string;
@@ -185,6 +257,16 @@ export interface OperatorRoundOptions {
    * read. Defaults to the workspace's `points` read; a test injects one.
    */
   readQuestions?: (cwd: string) => Promise<Map<string, string> | null>;
+  /**
+   * The runs a steward's operator started beside its turns (#2861), kept
+   * across rounds. A round given none keeps its own, which it forgets when
+   * it returns.
+   */
+  besideState?: BesideState;
+  /** How a run beside the turns is started. Default: `chant run <op>` in a process of its own ({@link spawnBesideRun}). */
+  launchBeside?: BesideLauncher;
+  /** The environment the steward operator was started for (`--env`), passed on to a run it starts beside its turns. */
+  stewardEnv?: string;
 }
 
 /** A steward's Op whose last run waits on a decision point (#2749). */
@@ -336,12 +418,12 @@ async function releaseStewardTurn(steward: string, holder: string, token: string
 }
 
 /**
- * The Ops one round considers: a steward's Ops, or every ConvergeOp. A
- * steward's Op with no schedule is considered only to resume a run of the
- * steward's that waits on a decision point.
+ * The Ops one round runs as turns: a steward's Ops but those beside its turns
+ * (#2861), or every ConvergeOp. A steward's Op with no schedule is considered
+ * only to resume a run of the steward's that waits on a decision point.
  */
 async function roundOps(opts: OperatorRoundOptions): Promise<DiscoveredOp["config"][]> {
-  if (opts.steward) return [...opts.steward.ops];
+  if (opts.steward) return stewardTurnOps(opts.steward);
   const { ops } = await discoverConvergeOps({ cwd: opts.cwd, env: opts.env });
   return ops.map((d) => d.config);
 }
@@ -388,6 +470,10 @@ export async function runOperatorRound(opts: OperatorRoundOptions): Promise<Oper
 
   if (!(await holdSteward())) return events;
 
+  // The runs started beside the turns that ended since the last round (#2861).
+  const besideState = opts.besideState ?? createBesideState();
+  events.push(...besideState.ended.splice(0));
+
   // A steward's Ops whose last run waits on a decision point (#2749), read
   // once per round, and the state of those questions now.
   const waiting = steward ? await stewardWaits(steward, opts) : new Map<string, WaitState>();
@@ -395,6 +481,7 @@ export async function runOperatorRound(opts: OperatorRoundOptions): Promise<Oper
   // been approved since.
   const gated = steward ? await stewardGates(steward, opts) : new Map<string, GateWaitState>();
 
+  let stewardLost = false;
   for (const config of await roundOps(opts)) {
     const env = steward ? runEnvOf(config) : (envOf(config) ?? "unknown");
     const wait = waiting.get(config.name);
@@ -548,12 +635,152 @@ export async function runOperatorRound(opts: OperatorRoundOptions): Promise<Oper
       restoreTurn?.();
       if (steward && turn?.acquired) await releaseStewardTurn(steward.name, holder, turn.lease!.token, { cwd: opts.cwd });
     }
-    if (!(await holdSteward())) break;
+    if (!(await holdSteward())) {
+      stewardLost = true;
+      break;
+    }
+  }
+
+  // The Ops beside the turns (#2861) are considered whatever became of the
+  // turns: a turn in progress holds none of them. Only a steward that lost its
+  // own lease starts nothing.
+  if (steward && !stewardLost) {
+    await besideRound(steward, opts, holder, besideState, waiting, gated, events);
   }
 
   return events;
 }
 
+/**
+ * One round for a steward's Ops beside its turns (#2861). For each one not
+ * already running: start it when a run of the steward's waits on a question
+ * now answered or a gate now approved, when its cron fires, or when its ready
+ * step names work it has not started a run for. The run is started and not
+ * waited for.
+ */
+async function besideRound(
+  steward: StewardDeclaration,
+  opts: OperatorRoundOptions,
+  holder: string,
+  state: BesideState,
+  waiting: Map<string, WaitState>,
+  gated: Map<string, GateWaitState>,
+  events: OperatorTickEvent[],
+): Promise<void> {
+  for (const beside of stewardBesideOf(steward)) {
+    const config = steward.ops.find((op) => op.name === beside.op);
+    if (!config) continue;
+    const env = runEnvOf(config);
+
+    // The cron is read whether or not a run is in flight, so a fire during a
+    // run is dropped (overlap "skip") rather than owed afterwards.
+    const cron = config.schedule?.cron;
+    let due = false;
+    if (cron) {
+      const now = (opts.now ?? (() => new Date()))();
+      const lastSeen = opts.scheduleState?.get(config.name);
+      due = lastSeen === undefined ? cronMatches(cron, now) : cronDueBetween(cron, lastSeen, now);
+      opts.scheduleState?.set(config.name, now);
+    }
+
+    const mine = state.running.get(config.name);
+    if (mine) {
+      events.push({ kind: "beside-running", op: config.name, env, heldBy: mine.holder });
+      continue;
+    }
+    // A run another process holds the lease for: a hand `chant run`, or one
+    // this steward started before its operator restarted.
+    let current: LeaseRecord | undefined;
+    try {
+      current = (await readLease(config.name, { cwd: opts.cwd })).record;
+    } catch (err) {
+      events.push({ kind: "lease-error", op: config.name, env, error: err instanceof Error ? err.message : String(err) });
+      continue;
+    }
+    const now = (opts.now ?? (() => new Date()))();
+    if (current && new Date(current.expiresAt).getTime() > now.getTime()) {
+      events.push({ kind: "beside-running", op: config.name, env, heldBy: current.holder });
+      continue;
+    }
+
+    const wait = waiting.get(config.name);
+    const stopped = gated.get(config.name);
+    const standing: OperatorTickEvent | undefined = wait
+      ? { kind: "waiting-on-point", op: config.name, env, point: wait.point, question: wait.question, state: wait.state }
+      : stopped
+        ? { kind: "waiting-on-gate", op: config.name, env, gateOp: stopped.gateOp, gate: stopped.gate }
+        : undefined;
+
+    let why: BesideWhy | undefined;
+    if (wait?.answered) why = "resumed";
+    else if (stopped?.approved) why = "approved";
+    else if (due) why = "cron";
+
+    let keys: string[] | undefined;
+    if (!why && beside.ready) {
+      const answer = await askReady(beside.ready, opts.activities, opts.signal);
+      if ("error" in answer) {
+        events.push({ kind: "ready-failed", op: config.name, env, error: answer.error });
+        continue;
+      }
+      if (answer.ready) {
+        const seen = state.started.get(config.name) ?? new Set<string>();
+        if (answer.keys.length === 0 || answer.keys.some((k) => !seen.has(k))) {
+          why = "ready";
+          keys = answer.keys;
+        } else {
+          events.push(standing ?? { kind: "skipped-not-ready", op: config.name, env, already: answer.keys });
+          continue;
+        }
+      } else {
+        events.push(standing ?? { kind: "skipped-not-ready", op: config.name, env });
+        continue;
+      }
+    }
+    if (!why) {
+      const idle: OperatorTickEvent | undefined = standing ?? (cron ? { kind: "skipped-not-due", op: config.name, env, cron } : undefined);
+      if (idle) events.push(idle);
+      continue;
+    }
+
+    const runHolder = stewardWorkHolder(steward.name, config.name, holder);
+    let handle: BesideHandle;
+    try {
+      handle = (opts.launchBeside ?? spawnBesideRun)({
+        op: config,
+        steward: steward.name,
+        ...(opts.stewardEnv !== undefined ? { env: opts.stewardEnv } : {}),
+        holder: runHolder,
+        cwd: opts.cwd ?? process.cwd(),
+        ...(opts.leaseTtlMs !== undefined ? { leaseTtlMs: opts.leaseTtlMs } : {}),
+      });
+    } catch (err) {
+      events.push({ kind: "tick-failed", op: config.name, env, error: err instanceof Error ? err.message : String(err) });
+      continue;
+    }
+    if (keys) {
+      const seen = state.started.get(config.name) ?? new Set<string>();
+      for (const k of keys) seen.add(k);
+      state.started.set(config.name, seen);
+    }
+    state.running.set(config.name, { handle, holder: runHolder, why });
+    void handle.done.then((exit) => {
+      state.running.delete(config.name);
+      state.ended.push({ kind: "beside-ended", op: config.name, env, code: exit.code, ...(exit.error ? { error: exit.error } : {}) });
+    });
+    events.push({
+      kind: "beside-started",
+      op: config.name,
+      env,
+      why,
+      holder: runHolder,
+      ...(keys && keys.length > 0 ? { keys } : {}),
+      ...(why === "resumed" && wait ? { resumed: wait.question } : {}),
+      ...(why === "approved" && stopped ? { approved: { op: stopped.gateOp, gate: stopped.gate } } : {}),
+    });
+  }
+}
+
 /**
  * What a failed tick says about itself (#2301).
  *
@@ -673,17 +900,24 @@ export async function runOperatorForever(opts: OperatorLoopOptions): Promise<voi
   // lands on a firing minute — a fresh operator owes no catch-up for the ticks
   // it was not running for.
   const scheduleState = opts.scheduleState ?? new Map<string, Date>();
+  // The runs a steward starts beside its turns (#2861), for the life of the
+  // loop. When the loop stops, so do they.
+  const besideState = opts.besideState ?? createBesideState();
   const intervalMs = opts.intervalMs ?? DEFAULT_OPERATOR_INTERVAL_MS;
   const subscribers = opts.subscribers ?? [];
 
-  // Nothing to subscribe to: the loop below is byte for byte the loop that
-  // shipped before #1981, and takes none of the machinery's cost.
+  // Nothing to subscribe to: the loop below is the loop that shipped before
+  // #1981, and takes none of the machinery's cost.
   if (subscribers.length === 0) {
-    while (!opts.signal?.aborted) {
-      const events = await runOperatorRound({ ...opts, holder, scheduleState });
-      opts.onRound?.(events);
-      if (opts.signal?.aborted) break;
-      await sleepAbortable(intervalMs, opts.signal);
+    try {
+      while (!opts.signal?.aborted) {
+        const events = await runOperatorRound({ ...opts, holder, scheduleState, besideState });
+        opts.onRound?.(events);
+        if (opts.signal?.aborted) break;
+        await sleepAbortable(intervalMs, opts.signal);
+      }
+    } finally {
+      await stopBesideRuns(besideState);
     }
     return;
   }
@@ -736,7 +970,7 @@ export async function runOperatorForever(opts: OperatorLoopOptions): Promise<voi
     while (!opts.signal?.aborted) {
       gate.roundStarted();
       const startedAt = Date.now();
-      const events = await runOperatorRound({ ...opts, holder, scheduleState });
+      const events = await runOperatorRound({ ...opts, holder, scheduleState, besideState });
       opts.onRound?.(events);
       if (opts.signal?.aborted) break;
 
@@ -750,6 +984,7 @@ export async function runOperatorForever(opts: OperatorLoopOptions): Promise<voi
     }
   } finally {
     await closeAll();
+    await stopBesideRuns(besideState);
   }
 }
 
@@ -780,5 +1015,18 @@ export function formatRoundLine(event: OperatorTickEvent): string {
       return `operator: steward ${event.steward} skipped=1(steward-lease-held${event.heldBy ? `:${event.heldBy}` : ""})`;
     case "turn-busy":
       return `operator: ${event.op}@${event.env} skipped=1(turn-held${event.heldBy ? `:${event.heldBy}` : ""}, steward "${event.steward}" is already mid-turn)`;
+    case "beside-started":
+      return `operator: ${event.op}@${event.env} started=1(beside:${event.why}) holder="${event.holder}"` +
+        (event.keys ? ` keys="${event.keys.join(",")}"` : "") +
+        (event.resumed ? ` resumed="${event.resumed}"` : "") +
+        (event.approved ? ` approved="${event.approved.op}/${event.approved.gate}"` : "");
+    case "beside-running":
+      return `operator: ${event.op}@${event.env} skipped=1(beside-running${event.heldBy ? `:${event.heldBy}` : ""})`;
+    case "beside-ended":
+      return `operator: ${event.op}@${event.env} ended=1(beside) exit=${event.code ?? "none"}` + (event.error ? ` error="${event.error}"` : "");
+    case "skipped-not-ready":
+      return `operator: ${event.op}@${event.env} skipped=1(not-ready${event.already ? `:already-started:${event.already.join(",")}` : ""})`;
+    case "ready-failed":
+      return `operator: ${event.op}@${event.env} failed=1(ready) error="${event.error}"`;
   }
 }

package/src/op/steward-beside.test.ts

@@ -0,0 +1,267 @@
+/**
+ * An Op a steward runs beside its turns (#2861): the declaration, a long run
+ * that holds none of the steward's turns up, a hand run that takes the Op's
+ * lease and not the turn, and the operator stopping the runs it started.
+ */
+import { describe, test, expect, vi } from "vitest";
+import { withTestDir } from "@intentius/chant-test-utils";
+import { spawnSync } from "node:child_process";
+import { mkdirSync, writeFileSync } from "node:fs";
+import { join } from "node:path";
+import type { ActivityFn, ActivityProfile } from "./activity-registry";
+import type { ActivityStep, OpConfig } from "./types";
+import { declareSteward, readinessKeys, stewardBesideFor, stewardTurnOps, stewardTurnLeaseName } from "./steward";
+import { createBesideState, formatRoundLine, runOperatorForever, runOperatorRound, waitForBesideRuns, type OperatorTickEvent } from "./operator";
+import { askReady, holdBesideLease, inProcessBesideLauncher, type BesideLauncher } from "./steward-beside";
+import { stepOutput } from "./step-output-ref";
+import { readLease } from "../lifecycle/lease";
+import { readRunLedger } from "../lifecycle/run-ledger";
+
+const PROFILES: Record<string, ActivityProfile> = {};
+
+function op(name: string, cron?: string, fn = "tick"): OpConfig {
+  return {
+    name,
+    overview: `${name} fixture`,
+    phases: [{ name: "Run", steps: [{ kind: "activity", fn, args: {} }] }],
+    ...(cron ? { schedule: { cron, overlap: "skip" as const } } : {}),
+  };
+}
+
+const ready = (fn = "ready"): ActivityStep => ({ kind: "activity", fn, args: {} });
+
+function git(args: string[], cwd: string): void {
+  spawnSync("git", args, { cwd, encoding: "utf-8" });
+}
+
+function initRepo(dir: string): void {
+  git(["init", "-q", "-b", "main"], dir);
+  git(["config", "user.email", "test@chant.dev"], dir);
+  git(["config", "user.name", "Test"], dir);
+  mkdirSync(join(dir, "app"), { recursive: true });
+  writeFileSync(join(dir, "app", "index.ts"), "export const hello = 1;\n");
+  git(["add", "app/index.ts"], dir);
+  git(["commit", "-q", "-m", "init"], dir);
+}
+
+const kinds = (events: OperatorTickEvent[]) => events.map((e) => [e.kind, e.op]);
+
+describe("declareSteward's beside (#2861)", () => {
+  test("lists the Ops beside the turns after its turns' Ops, and names them", () => {
+    const steward = declareSteward({
+      name: "box-steward",
+      ops: [op("converge", "* * * * *")],
+      beside: [{ op: op("dispatch", undefined, "build"), ready: ready() }, op("rebuild", "0 3 * * *")],
+    });
+    expect(steward.ops.map((o) => o.name)).toEqual(["converge", "dispatch", "rebuild"]);
+    expect(steward.beside).toEqual([{ op: "dispatch", ready: ready() }, { op: "rebuild", ready: null }]);
+    expect(stewardTurnOps(steward).map((o) => o.name)).toEqual(["converge"]);
+    expect(stewardBesideFor(steward, "dispatch")?.ready).toEqual(ready());
+    expect(stewardBesideFor(steward, "converge")).toBeUndefined();
+    // A declaration from before #2861 has no beside, and every Op is a turn.
+    const { beside: _dropped, ...older } = steward;
+    expect(stewardTurnOps(older as typeof steward).map((o) => o.name)).toEqual(["converge", "dispatch", "rebuild"]);
+  });
+
+  test("refuses an Op both in ops and beside, a ready that is not one activity step, and one that reads a step's output", () => {
+    const converge = op("converge", "* * * * *");
+    expect(() => declareSteward({ name: "s", ops: [converge], beside: [converge] })).toThrow(/listed twice/);
+    expect(() =>
+      declareSteward({ name: "s", ops: [], beside: [{ op: op("dispatch"), ready: { kind: "gate", gate: "g" } as unknown as ActivityStep }] }),
+    ).toThrow(/ready is one activity step/);
+    expect(() =>
+      declareSteward({ name: "s", ops: [], beside: [{ op: op("dispatch"), ready: { kind: "activity", fn: "shellCmd", args: { cmd: stepOutput("pick", "json") } } }] }),
+    ).toThrow(/reads another step's output/);
+    // Started on its own, it can't be handed the work item a run names.
+    const leased: OpConfig = { ...op("dispatch"), changesCheckout: true, workLease: {} };
+    expect(() => declareSteward({ name: "s", ops: [], beside: [{ op: leased, ready: ready() }] })).toThrow(/workLease names no item/);
+  });
+
+  test("a ready step's answer: a list of keys, a string, true, or no work", () => {
+    expect(readinessKeys({ stdout: "", json: ["W-1:0", "intent:ws-1"] })).toEqual({ ready: true, keys: ["W-1:0", "intent:ws-1"] });
+    expect(readinessKeys([{ item: "W-2" }])).toEqual({ ready: true, keys: ['{"item":"W-2"}'] });
+    expect(readinessKeys("W-3")).toEqual({ ready: true, keys: ["W-3"] });
+    expect(readinessKeys(true)).toEqual({ ready: true, keys: [] });
+    for (const none of [false, null, "", [], { json: null }]) expect(readinessKeys(none)).toEqual({ ready: false, keys: [] });
+    expect(readinessKeys({ ready: [] })).toBeNull();
+  });
+
+  test("askReady reports a step that fails, answers in another shape, or isn't loaded", async () => {
+    const acts = new Map<string, ActivityFn>([
+      ["boom", async () => { throw new Error("no factory here"); }],
+      ["odd", async () => ({ ready: [{ item: "W-1" }], held: null })],
+    ]);
+    expect(await askReady(ready("boom"), acts)).toEqual({ error: "no factory here" });
+    expect(await askReady(ready("odd"), acts)).toMatchObject({ error: expect.stringContaining("neither true") });
+    expect(await askReady(ready("missing"), acts)).toMatchObject({ error: expect.stringContaining('"missing" is not loaded') });
+  });
+});
+
+describe("an Op beside a steward's turns (#2861)", () => {
+  test("a long run beside a per-minute converge: converge ticks every minute while it runs, and no work is started twice", async () => {
+    await withTestDir(async (dir) => {
+      initRepo(dir);
+      let finish: () => void = () => {};
+      const building = new Promise<void>((resolve) => { finish = resolve; });
+      const log: string[] = [];
+      const activities = new Map<string, ActivityFn>([
+        ["tick", async () => { log.push("converge"); return { ok: true }; }],
+        ["build", async () => { log.push("build:start"); await building; log.push("build:end"); return { built: true }; }],
+        ["ready", async () => ({ stdout: "", json: ["W-1:0"] })],
+      ]);
+      const steward = declareSteward({
+        name: "box-steward",
+        ops: [op("converge", "* * * * *")],
+        beside: [{ op: op("dispatch", undefined, "build"), ready: ready() }],
+      });
+      const besideState = createBesideState();
+      const scheduleState = new Map<string, Date>();
+      const launchBeside = inProcessBesideLauncher(activities, PROFILES);
+      // Cron minutes are local time, so the rounds are too. The leases use the
+      // wall clock, so a fixed `now` is only handed to the cron.
+      let minute = 1;
+      const round = () => {
+        const at = new Date(2026, 8, 25, 10, minute++, 0);
+        return runOperatorRound({ cwd: dir, holder: "op1", steward, activities, profiles: PROFILES, besideState, scheduleState, launchBeside, now: () => at });
+      };
+
+      // 10:01: converge ticks as a turn; the ready step names work, and dispatch starts beside it.
+      const first = await round();
+      expect(kinds(first)).toEqual([["ticked", "converge"], ["beside-started", "dispatch"]]);
+      expect(first[1]).toMatchObject({ why: "ready", keys: ["W-1:0"], holder: "box-steward/dispatch@op1" });
+      expect(formatRoundLine(first[1])).toBe('operator: dispatch@local started=1(beside:ready) holder="box-steward/dispatch@op1" keys="W-1:0"');
+      await vi.waitFor(() => expect(log).toContain("build:start"));
+
+      // Mid-build the run holds the Op's own lease, and not the steward's turn.
+      expect((await readLease("dispatch", { cwd: dir })).record?.holder).toBe("box-steward/dispatch@op1");
+      expect((await readLease(stewardTurnLeaseName("box-steward"), { cwd: dir })).record).toBeUndefined();
+
+      // 10:02 and 10:03: converge ticks each minute while the build runs, and no second run starts.
+      for (let i = 0; i < 2; i++) {
+        const mid = await round();
+        expect(kinds(mid)).toEqual([["ticked", "converge"], ["beside-running", "dispatch"]]);
+      }
+      expect(log.filter((l) => l === "converge")).toHaveLength(3);
+      expect(log).not.toContain("build:end");
+
+      finish();
+      await waitForBesideRuns(besideState);
+      expect((await readLease("dispatch", { cwd: dir })).record).toBeUndefined();
+      const run = (await readRunLedger("local", "dispatch", { cwd: dir })).records.at(-1)!;
+      expect(run).toMatchObject({ status: "ok", steward: "box-steward" });
+
+      // 10:04: the end is reported, and the same work still named is not started again.
+      const after = await round();
+      expect(after).toEqual([
+        { kind: "beside-ended", op: "dispatch", env: "local", code: 0 },
+        expect.objectContaining({ kind: "ticked", op: "converge" }),
+        { kind: "skipped-not-ready", op: "dispatch", env: "local", already: ["W-1:0"] },
+      ]);
+      expect(log.filter((l) => l === "build:start")).toHaveLength(1);
+    });
+  });
+
+  test("a hand run holds the Op's lease and not the turn: converge still ticks, and the operator starts no second run", async () => {
+    await withTestDir(async (dir) => {
+      initRepo(dir);
+      const log: string[] = [];
+      const activities = new Map<string, ActivityFn>([
+        ["tick", async () => { log.push("converge"); return { ok: true }; }],
+        ["ready", async () => true],
+      ]);
+      const steward = declareSteward({
+        name: "box-steward",
+        ops: [op("converge", "* * * * *")],
+        beside: [{ op: op("dispatch", undefined, "build"), ready: ready() }],
+      });
+      const launched: string[] = [];
+      const launchBeside: BesideLauncher = (start) => {
+        launched.push(start.op.name);
+        return { done: Promise.resolve({ code: 0 }), stop: () => {} };
+      };
+
+      // What `chant run dispatch` takes for an Op beside the turns.
+      const hand = await holdBesideLease("dispatch", "hand-run", { cwd: dir });
+      expect(hand.acquired).toBe(true);
+      expect((await readLease(stewardTurnLeaseName("box-steward"), { cwd: dir })).record).toBeUndefined();
+
+      const at = new Date(2026, 8, 25, 10, 1, 0);
+      const events = await runOperatorRound({ cwd: dir, holder: "op1", steward, activities, profiles: PROFILES, launchBeside, now: () => at });
+      expect(events.map((e) => e.kind)).toEqual(["ticked", "beside-running"]);
+      expect(events[1]).toEqual({ kind: "beside-running", op: "dispatch", env: "local", heldBy: "hand-run" });
+      expect(log).toEqual(["converge"]);
+      expect(launched).toEqual([]);
+
+      // A second hand run is refused while the first holds the lease.
+      expect(await holdBesideLease("dispatch", "hand-run-2", { cwd: dir })).toEqual({ acquired: false, heldBy: "hand-run" });
+      if (hand.acquired) await hand.release();
+      expect((await readLease("dispatch", { cwd: dir })).record).toBeUndefined();
+      const next = await runOperatorRound({ cwd: dir, holder: "op1", steward, activities, profiles: PROFILES, launchBeside, now: () => at });
+      expect(next.map((e) => e.kind)).toContain("beside-started");
+      expect(launched).toEqual(["dispatch"]);
+    });
+  });
+
+  test("the lease a long run holds is renewed while it runs", async () => {
+    await withTestDir(async (dir) => {
+      initRepo(dir);
+      const held = await holdBesideLease("dispatch", "h", { cwd: dir, ttlMs: 900 });
+      expect(held.acquired).toBe(true);
+      if (!held.acquired) return;
+      const first = held.lease.expiresAt;
+      await vi.waitFor(async () => {
+        const now = (await readLease("dispatch", { cwd: dir })).record;
+        expect(now?.token).toBe(held.lease.token);
+        expect(now!.expiresAt > first).toBe(true);
+      }, { timeout: 3000 });
+      await held.release();
+    });
+  });
+
+  test("a cron fire starts it, and one that fires while a run is in flight is dropped", async () => {
+    await withTestDir(async (dir) => {
+      initRepo(dir);
+      const steward = declareSteward({ name: "box-steward", ops: [], beside: [op("nightly", "0 3 * * *")] });
+      let end: () => void = () => {};
+      const launchBeside: BesideLauncher = () => ({ done: new Promise((resolve) => { end = () => resolve({ code: 0 }); }), stop: () => {} });
+      const besideState = createBesideState();
+      const scheduleState = new Map<string, Date>();
+      const round = (h: number, m: number) =>
+        runOperatorRound({ cwd: dir, holder: "op1", steward, activities: new Map(), profiles: PROFILES, besideState, scheduleState, launchBeside, now: () => new Date(2026, 8, 25, h, m, 0) });
+      expect(kinds(await round(2, 59))).toEqual([["skipped-not-due", "nightly"]]);
+      expect(await round(3, 0)).toEqual([expect.objectContaining({ kind: "beside-started", op: "nightly", why: "cron" })]);
+      expect(kinds(await round(3, 1))).toEqual([["beside-running", "nightly"]]);
+      end();
+      await waitForBesideRuns(besideState);
+      expect(kinds(await round(3, 2))).toEqual([["beside-ended", "nightly"], ["skipped-not-due", "nightly"]]);
+    });
+  });
+
+  test("the operator stops the runs it started when it stops", async () => {
+    await withTestDir(async (dir) => {
+      initRepo(dir);
+      const steward = declareSteward({ name: "box-steward", ops: [], beside: [{ op: op("dispatch"), ready: ready() }] });
+      const controller = new AbortController();
+      const stopped: string[] = [];
+      const launchBeside: BesideLauncher = (start) => {
+        let end: (code: number) => void = () => {};
+        const done = new Promise<{ code: number }>((resolve) => { end = (code) => resolve({ code }); });
+        return { done, stop: () => { stopped.push(start.op.name); end(1); } };
+      };
+      await runOperatorForever({
+        cwd: dir,
+        holder: "op1",
+        steward,
+        activities: new Map<string, ActivityFn>([["ready", async () => true]]),
+        profiles: PROFILES,
+        launchBeside,
+        intervalMs: 10,
+        signal: controller.signal,
+        onRound: (events) => {
+          if (events.some((e) => e.kind === "beside-started")) controller.abort();
+        },
+      });
+      expect(stopped).toEqual(["dispatch"]);
+    });
+  });
+});