omp-conductor 0.12.0 → 0.14.0

package/src/verbs/actions.ts

@@ -14,8 +14,9 @@
  * enough to actually write.
  */
 
-import { credentialedEnv, openRunPr, pushRunBranch, repoSlugFor } from "../gitops.ts";
-import type { ProjectConfig, ReleaseShape } from "../types.ts";
+import { credentialedEnv, openRunPr, pushRunBranch, repoSlugFor, scrubUserinfo } from "../gitops.ts";
+import { ensureMirror } from "../worktree.ts";
+import type { ProjectConfig, ReleaseShape, RepoTarget } from "../types.ts";
 import type { ActionOutcome, ReleaseExecution, VerbActions } from "./server.ts";
 
 /**
@@ -47,6 +48,25 @@ export type CommandRunner = (
   opts: { cwd?: string; env?: Record<string, string> },
 ) => Promise<CommandRun>;
 
+export type MirrorPreparer = (repo: RepoTarget, mirrorRoot: string) => Promise<string>;
+
+type CommitOutcome = { ok: true; sha: string } | { ok: false; stderr: string };
+
+const FULL_COMMIT = /^(?:[0-9a-f]{40}|[0-9a-f]{64})$/i;
+const RELEASE_TAG_CONFIG = [
+  "-c",
+  "user.name=conductor",
+  "-c",
+  "user.email=conductor@invalid",
+  "-c",
+  "tag.gpgSign=false",
+];
+
+function commitFrom(stdout: string): string | undefined {
+  const value = stdout.trim().split(/\s+/, 1)[0];
+  return value !== undefined && FULL_COMMIT.test(value) ? value.toLowerCase() : undefined;
+}
+
 const spawnCommand: CommandRunner = async (argv, opts) => {
   const proc = Bun.spawn(argv, {
     // Always a closed stream, matching the tracker adapter: a command that does
@@ -65,8 +85,11 @@ const spawnCommand: CommandRunner = async (argv, opts) => {
   return { ok: code === 0 && !proc.signalCode, stdout, stderr };
 };
 
-function failed(run: CommandRun, argv: string[]): ActionOutcome {
-  return { ok: false, stderr: run.stderr.trim() || `\`${argv.join(" ")}\` failed with no stderr` };
+function failed(run: CommandRun, argv: string[]): { ok: false; stderr: string } {
+  return {
+    ok: false,
+    stderr: scrubUserinfo(run.stderr.trim() || `\`${argv.join(" ")}\` failed with no stderr`),
+  };
 }
 
 /**
@@ -75,7 +98,11 @@ function failed(run: CommandRun, argv: string[]): ActionOutcome {
  * `run` is injected so the release and tracker paths can be exercised without
  * a live repository; production callers never pass it.
  */
-export function githubVerbActions(project: ProjectConfig, run: CommandRunner = spawnCommand): VerbActions {
+export function githubVerbActions(
+  project: ProjectConfig,
+  run: CommandRunner = spawnCommand,
+  prepareMirror: MirrorPreparer = ensureMirror,
+): VerbActions {
   const env = (): Record<string, string> => credentialedEnv();
 
   const gh = async (argv: string[], cwd?: string): Promise<ActionOutcome> => {
@@ -84,6 +111,69 @@ export function githubVerbActions(project: ProjectConfig, run: CommandRunner = s
     return result.ok ? { ok: true, detail: result.stdout.trim() || undefined } : failed(result, full);
   };
 
+  const git = async (mirror: string, args: string[]): Promise<{ argv: string[]; result: CommandRun }> => {
+    const argv = ["git", "-C", mirror, ...args];
+    return { argv, result: await run(argv, { env: env() }) };
+  };
+
+  const readCommit = async (mirror: string, args: string[], fact: string): Promise<CommitOutcome> => {
+    const { argv, result } = await git(mirror, args);
+    if (!result.ok) return failed(result, argv);
+    const sha = commitFrom(result.stdout);
+    return sha === undefined
+      ? { ok: false, stderr: `${fact} returned no full commit SHA` }
+      : { ok: true, sha };
+  };
+
+  const liveDefaultHead = (mirror: string, repo: RepoTarget): Promise<CommitOutcome> =>
+    readCommit(
+      mirror,
+      ["ls-remote", "origin", `refs/heads/${repo.defaultBranch}`],
+      `live ${repo.defaultBranch} lookup`,
+    );
+
+  const remoteTagCommit = async (
+    mirror: string,
+    tag: string,
+  ): Promise<{ ok: true; sha: string | undefined } | { ok: false; stderr: string }> => {
+    const directRef = `refs/tags/${tag}`;
+    const peeledRef = `${directRef}^{}`;
+    const { argv, result } = await git(mirror, ["ls-remote", "origin", directRef, peeledRef]);
+    if (!result.ok) return failed(result, argv);
+    if (result.stdout.trim() === "") return { ok: true, sha: undefined };
+
+    const refs = new Map(
+      result.stdout
+        .trim()
+        .split("\n")
+        .map((line) => line.trim().split(/\s+/, 2))
+        .filter((entry): entry is [string, string] => entry.length === 2)
+        .map(([sha, ref]) => [ref, sha] as const),
+    );
+    const sha = commitFrom(refs.get(peeledRef) ?? refs.get(directRef) ?? "");
+    return sha === undefined
+      ? { ok: false, stderr: `remote tag ${tag} lookup returned no full commit SHA` }
+      : { ok: true, sha };
+  };
+
+  const releaseTargetMoved = (repo: RepoTarget, target: string, live: string): { ok: false; stderr: string } => ({
+    ok: false,
+    stderr:
+      `refusing release: target ${target} is behind live ${repo.defaultBranch} ${live}. ` +
+      "Refresh the release status and retry so the tag contains every merged change.",
+  });
+
+  const releaseMirror = async (repo: RepoTarget): Promise<{ ok: true; path: string } | { ok: false; stderr: string }> => {
+    try {
+      return { ok: true, path: await prepareMirror(repo, project.mirrorRoot) };
+    } catch (err) {
+      return {
+        ok: false,
+        stderr: `could not refresh ${repo.name}'s release mirror: ${err instanceof Error ? err.message : String(err)}`,
+      };
+    }
+  };
+
   return {
     releasableShapes: GITHUB_RELEASABLE_SHAPES,
 
@@ -133,18 +223,158 @@ export function githubVerbActions(project: ProjectConfig, run: CommandRunner = s
       if (execution.shape === "github-release") {
         return gh(["release", "create", tag, "--repo", slug, "--generate-notes"]);
       }
-      // The two git shapes act on the project's mirror of the repo, which is
-      // the only checkout the daemon owns. A tag is cut against the mirror's
-      // view of the default branch and pushed from there.
-      const mirror = `${project.mirrorRoot}/${execution.repo.name}.git`;
+      // The two git shapes act on the project's serialized, freshly fetched
+      // mirror. The live read on either side of tag creation closes the stale
+      // mirror and branch-moved-during-release windows (#258); the push path
+      // repeats the guard because a branch may move between the two verbs.
+      const prepared = await releaseMirror(execution.repo);
+      if (!prepared.ok) return prepared;
+      const mirror = prepared.path;
       if (execution.shape === "git-tag") {
-        const argv = ["git", "-C", mirror, "tag", "-a", tag, "-m", `release ${tag}`, `origin/${execution.repo.defaultBranch}`];
-        const result = await run(argv, { env: env() });
-        return result.ok ? { ok: true, detail: `tagged ${tag}` } : failed(result, argv);
+        const target = await readCommit(
+          mirror,
+          ["rev-parse", `refs/remotes/origin/${execution.repo.defaultBranch}^{commit}`],
+          `refreshed ${execution.repo.defaultBranch}`,
+        );
+        if (!target.ok) return target;
+        const liveBefore = await liveDefaultHead(mirror, execution.repo);
+        if (!liveBefore.ok) return liveBefore;
+        if (target.sha !== liveBefore.sha) return releaseTargetMoved(execution.repo, target.sha, liveBefore.sha);
+
+        const localTag = await git(mirror, ["rev-parse", "-q", "--verify", `refs/tags/${tag}^{commit}`]);
+        let previousSha: string | undefined;
+        if (localTag.result.ok) {
+          previousSha = commitFrom(localTag.result.stdout);
+          if (previousSha === undefined) {
+            return { ok: false, stderr: `tag ${tag} returned no full commit SHA` };
+          }
+
+          const remoteTag = await remoteTagCommit(mirror, tag);
+          if (!remoteTag.ok) return remoteTag;
+          if (remoteTag.sha === target.sha) {
+            return {
+              ok: true,
+              sha: target.sha,
+              detail: `tag ${tag} already exists at ${target.sha} and is already on origin`,
+            };
+          }
+          if (remoteTag.sha !== undefined) {
+            return {
+              ok: false,
+              stderr:
+                `refusing release: ${tag} already exists on origin at ${remoteTag.sha}. ` +
+                "A published tag is never re-cut or moved; choose a new tag name.",
+            };
+          }
+        }
+
+        const repointed = previousSha !== undefined;
+        // An unattended daemon cannot depend on a host-level Git identity or a
+        // signing key with an interactive passphrase.
+        const tagged = await git(mirror, [
+          ...RELEASE_TAG_CONFIG,
+          "tag",
+          ...(repointed ? ["-f"] : []),
+          "-a",
+          tag,
+          "-m",
+          `release ${tag}`,
+          target.sha,
+        ]);
+        if (!tagged.result.ok) return failed(tagged.result, tagged.argv);
+
+        const liveAfter = await liveDefaultHead(mirror, execution.repo);
+        if (!liveAfter.ok || liveAfter.sha !== target.sha) {
+          const rollback = previousSha !== undefined
+            ? await git(mirror, [
+                ...RELEASE_TAG_CONFIG,
+                "tag",
+                "-f",
+                "-a",
+                tag,
+                "-m",
+                `release ${tag}`,
+                previousSha,
+              ])
+            : await git(mirror, ["tag", "-d", tag]);
+          const reason = liveAfter.ok
+            ? releaseTargetMoved(execution.repo, target.sha, liveAfter.sha).stderr
+            : `${repointed ? "re-pointed" : "created"} ${tag} at ${target.sha}, but could not verify the live default branch: ${liveAfter.stderr}`;
+          const rollbackDetail = repointed
+            ? rollback.result.ok
+              ? ` The tag was restored to ${previousSha}.`
+              : ` WARNING: the tag could not be restored to ${previousSha}: ${failed(rollback.result, rollback.argv).stderr}`
+            : rollback.result.ok
+              ? " The unpushed local tag was deleted."
+              : ` WARNING: the unpushed local tag could not be deleted: ${failed(rollback.result, rollback.argv).stderr}`;
+          return { ok: false, stderr: reason + rollbackDetail };
+        }
+        return {
+          ok: true,
+          sha: target.sha,
+          detail: repointed
+            ? `re-pointed unpushed ${tag} from ${previousSha} to ${target.sha}`
+            : `tagged ${tag} at ${target.sha}`,
+        };
       }
-      const argv = ["git", "-C", mirror, "push", "origin", `refs/tags/${tag}`];
-      const result = await run(argv, { env: env() });
-      return result.ok ? { ok: true, detail: `pushed refs/tags/${tag}` } : failed(result, argv);
+
+      const target = await readCommit(mirror, ["rev-parse", `refs/tags/${tag}^{commit}`], `tag ${tag}`);
+      if (!target.ok) return target;
+      let tagSha = target.sha;
+      const live = await liveDefaultHead(mirror, execution.repo);
+      if (!live.ok) return live;
+
+      const remoteTag = await remoteTagCommit(mirror, tag);
+      if (!remoteTag.ok) return remoteTag;
+      if (remoteTag.sha === tagSha) {
+        return { ok: true, sha: tagSha, detail: `refs/tags/${tag} already on origin at ${tagSha}` };
+      }
+      if (remoteTag.sha !== undefined) {
+        return {
+          ok: false,
+          stderr:
+            `refusing release: origin already has ${tag} at ${remoteTag.sha}, ` +
+            `which differs from the local tag at ${tagSha}. ` +
+            "A published tag is never force-moved; cut a new tag instead.",
+        };
+      }
+
+      let oldSha: string | undefined;
+      if (tagSha !== live.sha) {
+        const fresh = await readCommit(
+          mirror,
+          ["rev-parse", `refs/remotes/origin/${execution.repo.defaultBranch}^{commit}`],
+          `refreshed ${execution.repo.defaultBranch}`,
+        );
+        if (!fresh.ok) return fresh;
+        if (fresh.sha !== live.sha) return releaseTargetMoved(execution.repo, fresh.sha, live.sha);
+
+        oldSha = tagSha;
+        const retagged = await git(mirror, [
+          ...RELEASE_TAG_CONFIG,
+          "tag",
+          "-f",
+          "-a",
+          tag,
+          "-m",
+          `release ${tag}`,
+          live.sha,
+        ]);
+        if (!retagged.result.ok) return failed(retagged.result, retagged.argv);
+        tagSha = live.sha;
+      }
+
+      const pushed = await git(mirror, ["push", "origin", `refs/tags/${tag}`]);
+      return pushed.result.ok
+        ? {
+            ok: true,
+            sha: tagSha,
+            detail:
+              oldSha === undefined
+                ? `pushed refs/tags/${tag} at ${tagSha}`
+                : `re-pointed unpushed ${tag} from ${oldSha} to ${tagSha}; pushed refs/tags/${tag} at ${tagSha}`,
+          }
+        : failed(pushed.result, pushed.argv);
     },
   };
 }

package/src/verbs/protocol.ts

@@ -293,9 +293,10 @@ export const VERB_SPECS: Readonly<Record<VerbName, VerbSpec>> = {
     mutating: false,
     allowedRoles: ["worker", "orchestrator"],
     description:
-      "Read one pull request's live state: is it open, is its head still the sha you pushed, and are " +
-      "its checks green. This is the daemon's own merge-gate verdict, not a summary of it. A poll, not " +
-      "a watcher — call it again after waiting. A worker may omit prUrl and gets its own run's.",
+      "Read one pull request's live state and current head: is it open and are its checks green. " +
+      "Optionally compare that head with one the caller already observed. This is the daemon's own " +
+      "merge-gate verdict, not a summary of it. A poll, not a watcher — call it again after waiting. " +
+      "A worker may omit prUrl and gets its own run's.",
     args: {
       prUrl: {
         type: "string",
@@ -304,10 +305,10 @@ export const VERB_SPECS: Readonly<Record<VerbName, VerbSpec>> = {
       },
       headSha: {
         type: "string",
-        required: true,
+        required: false,
         description:
-          "The head you believe is on the branch — the sha conductor_push returned. If the branch has " +
-          "moved, the answer names both shas.",
+          "Optional: an expected head to compare against; omit it to read the current head. If the " +
+          "branch has moved, the answer names both shas.",
       },
     },
     roleRefusalText: (role) => `conductor_pr_status is not open to a ${role} session.`,

package/src/verbs/server.ts

@@ -265,6 +265,10 @@ export interface ReleaseFacts {
   openPrs: number;
   /** Queue depth, or `undefined` when the tracker could not be read. */
   queueDepth: number | undefined;
+  /** Current live-head workflow verdict for the released routed repository. */
+  baseCheck?: RunRecord["baseCheck"];
+  /** Evidence attached to a current red verdict. */
+  redBase?: string;
 }
 
 export function releaseRequirementRefusal(
@@ -286,6 +290,19 @@ export function releaseRequirementRefusal(
         return `policy.release.requires includes queue-drained and ${facts.queueDepth} issue(s) are still queued`;
       }
     }
+    if (requirement === "base-branch-green") {
+      if (facts.baseCheck === "green") continue;
+      if (facts.baseCheck === "pending") {
+        return "policy.release.requires includes base-branch-green and the newest merge's base workflows are still pending";
+      }
+      if (facts.baseCheck === "red" || facts.baseCheck === "red-preexisting") {
+        return (
+          "policy.release.requires includes base-branch-green and the newest observed base branch is red" +
+          (facts.redBase === undefined ? "" : `: ${facts.redBase}`)
+        );
+      }
+      return "policy.release.requires includes base-branch-green and no green base-branch verdict is available; refusing rather than assuming the base is healthy";
+    }
     if (requirement === "epic-children-closed") {
       return "policy.release.requires includes epic-children-closed, which nothing in a release request names an epic for. The daemon cannot settle it, so it refuses: take the requirement off this project's policy, or cut this release by hand.";
     }
@@ -423,19 +440,32 @@ async function decide(deps: VerbDeps, channel: VerbChannel, raw: unknown): Promi
     }
   }
 
-  // 4. Stop always wins, and is re-read here rather than remembered. Reads are
-  //    exempt: a run finishing its report during a `hold` still has to be able
-  //    to say what it saw, and refusing that would only make it guess. A run
-  //    admitted *before* the pause may finish its in-flight mutations — pause
-  //    stops new claims, not the work already running (#174) — so only a call
-  //    whose run started at or after the pause (or an orchestrator's call, with
-  //    no run row at all) is refused.
-  const stopped = spec.mutating ? deps.fleetStop() : undefined;
+  // 4. Stop always wins for work-starting mutations, and is re-read here
+  //    rather than remembered. Reads are exempt. A run admitted before the
+  //    pause may finish its own mutations (#174). Orchestrator completion
+  //    verbs resolve their target run inside the handler, where they can prove
+  //    that run also predates the pause (#251). Release remains available:
+  //    its authority, grant, and runs-settled gates still fail closed below.
+  const orchestratorCompletion =
+    channel.kind === "orchestrator" &&
+    (verb === "conductor_pr_merge" ||
+      verb === "conductor_pr_update_branch" ||
+      verb === "conductor_pr_update" ||
+      verb === "conductor_label");
+  const stopped =
+    spec.mutating && verb !== "conductor_release" && !orchestratorCompletion
+      ? deps.fleetStop()
+      : undefined;
   if (stopped !== undefined) {
     const at = deps.pausedAt();
     const admittedBeforePause = run !== undefined && at !== undefined && run.startedAt < at;
     if (!admittedBeforePause) {
-      return refuse("fleet-paused", `refused: ${stopped}. The pause stops new claims and new mutations; a run admitted before the pause may finish.`);
+      return refuse(
+        "fleet-paused",
+        `refused: ${stopped}. The pause refuses new claims and work-starting mutations. ` +
+          "Completion verbs for runs admitted before the pause (conductor_pr_merge, conductor_pr_update_branch, " +
+          "conductor_pr_update, conductor_label) and conductor_release remain available.",
+      );
     }
   }
 
@@ -599,7 +629,7 @@ async function prCreateVerb(
  * How far back to look is the recent-history cutoff `status` already uses: a
  * pull request older than that is not one an orchestrator is mid-flight on.
  */
-const PR_LOOKUP_WINDOW_MS = 30 * 24 * 60 * 60_000;
+export const PR_LOOKUP_WINDOW_MS = 30 * 24 * 60 * 60_000;
 
 function runForPr(deps: VerbDeps, project: string, prUrl: string): RunRecord | undefined {
   return deps.store
@@ -607,6 +637,31 @@ function runForPr(deps: VerbDeps, project: string, prUrl: string): RunRecord | u
     .find((candidate) => candidate.prUrl === prUrl);
 }
 
+/** Refuse an orchestrator completion mutation that cannot prove a pre-pause run. */
+function orchestratorPauseRefusal(
+  deps: VerbDeps,
+  channel: VerbChannel,
+  target: RunRecord | undefined,
+  refuse: Refuse,
+): Verdict | undefined {
+  if (channel.kind !== "orchestrator") return undefined;
+  const stopped = deps.fleetStop();
+  if (stopped === undefined) return undefined;
+  const at = deps.pausedAt();
+  if (target !== undefined && at !== undefined && target.startedAt < at) return undefined;
+  const why =
+    target === undefined
+      ? "this issue has no recorded run admitted before it."
+      : at === undefined
+        ? "the pause timestamp cannot prove when this run started."
+        : "this run started after it.";
+  return refuse(
+    "fleet-paused",
+    `refused: ${stopped}. Under pause, this verb applies only to runs admitted before the pause; ${why}`,
+    target?.issue,
+  );
+}
+
 async function prUpdateBranchVerb(
   deps: VerbDeps,
   project: ProjectConfig,
@@ -638,6 +693,9 @@ async function prUpdateBranchVerb(
     }
   }
 
+  const paused = orchestratorPauseRefusal(deps, channel, target, refuse);
+  if (paused !== undefined) return paused;
+
   let state: PrState | undefined;
   try {
     state = await deps.tracker.prState(prUrl);
@@ -691,6 +749,9 @@ async function prMergeVerb(
     return refuse("pr-not-this-run", `refused: ${prUrl} is not a pull request any run in ${project.name} opened.`);
   }
 
+  const paused = orchestratorPauseRefusal(deps, channel, target, refuse);
+  if (paused !== undefined) return paused;
+
   // Taken before any network call, so two concurrent callers contend here
   // rather than both spending a `gh` round trip and racing at the merge.
   const holderId = randomUUID();
@@ -814,6 +875,14 @@ async function labelVerb(
     );
   }
 
+  const paused = orchestratorPauseRefusal(
+    deps,
+    channel,
+    deps.store.latestRun(project.name, ref.issue),
+    refuse,
+  );
+  if (paused !== undefined) return paused;
+
   const label = String(args["label"]);
   const vocabulary = labelVocabulary(project);
   if (vocabulary.lifecycle.includes(label)) {
@@ -908,10 +977,18 @@ async function releaseVerb(
       queueDepth = undefined;
     }
   }
+  const wantsBase = policy.release.requires.includes("base-branch-green");
+  const health = wantsBase
+    ? deps.store.baseHealth(project.name).find((row) => row.repo === repoName)
+    : undefined;
   const unmet = releaseRequirementRefusal(policy.release.requires, {
     unsettledRuns: active.length,
     openPrs: active.filter((r) => r.prUrl !== undefined).length,
     queueDepth,
+    ...(health === undefined ? {} : { baseCheck: health.verdict }),
+    ...(health?.verdict === "red" && health.detail !== undefined
+      ? { redBase: health.detail }
+      : {}),
   });
   if (unmet !== undefined) return refuse("release-not-granted", `refused: ${unmet}`);
 
@@ -991,10 +1068,11 @@ async function prStatusVerb(
     issue = target.issue;
   }
 
-  const headSha = String(args["headSha"]);
+  const askedHead = args["headSha"];
+  const expectedHead = typeof askedHead === "string" ? askedHead : undefined;
   let verification: PrVerification | undefined;
   try {
-    verification = await deps.tracker.verifyPr(prUrl, headSha);
+    verification = await deps.tracker.verifyPr(prUrl, expectedHead);
   } catch (err) {
     verification = undefined;
     deps.log(`verb pr_status could not read ${prUrl}: ${err instanceof Error ? err.message : String(err)}`);
@@ -1007,7 +1085,7 @@ async function prStatusVerb(
       issue,
     );
   }
-  return allow(`${prUrl} at ${headSha}: ${verification.status} — ${verification.reason}`, undefined, issue);
+  return allow(`${prUrl} at ${verification.headSha}: ${verification.status} — ${verification.reason}`, undefined, issue);
 }
 
 /**
@@ -1054,6 +1132,9 @@ async function prUpdateVerb(
     if (target === undefined) {
       return refuse("pr-not-this-run", `refused: ${asked} is not a pull request any run in ${project.name} opened.`);
     }
+
+    const paused = orchestratorPauseRefusal(deps, channel, target, refuse);
+    if (paused !== undefined) return paused;
     prUrl = asked;
     issue = target.issue;
   }