@join-pantheon/cli 0.33.6 → 0.33.7

package/dist/worker.js

@@ -3,7 +3,7 @@ import { execFileSync } from "node:child_process";
 import { chmodSync, lstatSync, mkdirSync, writeFileSync, existsSync, readdirSync, readFileSync, readlinkSync, realpathSync, rmSync, statSync } from "node:fs";
 import { homedir } from "node:os";
 import { basename, dirname, isAbsolute, join, relative, resolve, sep } from "node:path";
-import { absentDeniedPaths, detectClaudeFeatures, describeFeatures, ensureRunTmp, NO_CLAUDE_FEATURES, removeSandboxLeftovers, runTmpDir } from "./claude-confine.js";
+import { absentDeniedPaths, describeKept, ownWorktreeConfig, detectClaudeFeatures, describeFeatures, ensureRunTmp, leavesSandboxMounts, NO_CLAUDE_FEATURES, removeSandboxLeftovers, runTmpDir } from "./claude-confine.js";
 import { snapshotHome, restoreHome, describeChange, trustWrite, askedHere } from "./home-guard.js";
 import { PantheonClient } from "./client.js";
 import { createHash, randomBytes } from "node:crypto";
@@ -12,9 +12,10 @@ import { loadPins, pinScope, savePins, teamPinId, writeWhole } from "./pins.js";
 import { answeredIn, markAnswered, noteNonce } from "./nonces.js";
 import { brokenStateFiles, brokenStateFix, pidAlive, readStateFile, withFileLock } from "./file-lock.js";
 import { mkdirInside, writeInside } from "./safe-write.js";
+import { corruptHeldFiles, MAX_HELD_PER_RUN, remoteTipIs, takeHeldFrames, updateHeldState, verifyHeldFrame, writeHoldConfig } from "./held-delivery.js";
 import { adoptedCommon, adoptWorktree, gitCallEnv, lfsInUse, repoHooksArgs, safeGitArgs, WorktreeTampered } from "./git-safe.js";
 import { trusted } from "./verify-act.js";
-import { headlessSpawn, materializeSpawnFiles, chooseAgent, agentById, describeChoice, unattendedRefusal, SANDBOXED_FOR_UNATTENDED, NOT_FOR_UNATTENDED_RUNS, claudeSandboxable, unattendedReadiness, anyUnattendedAgentOnPath, forgetMissingBinaries, quarantineRepoAgentConfig, AGENT_EDIT_SUFFIX, REPO_AGENT_CONFIG, COMMAND_GRANT_SCOPE, repoCommands, REPO_COMMANDS_FILE, claudeCommandRules, } from "./agents.js";
+import { headlessSpawn, materializeSpawnFiles, chooseAgent, agentById, describeChoice, unattendedRefusal, unattendedRefusalOf, SIGN_IN_CHECK, SANDBOXED_FOR_UNATTENDED, NOT_FOR_UNATTENDED_RUNS, claudeSandboxable, unattendedReadiness, anyUnattendedAgentOnPath, forgetMissingBinaries, quarantineRepoAgentConfig, AGENT_EDIT_SUFFIX, REPO_AGENT_CONFIG, COMMAND_GRANT_SCOPE, repoCommands, REPO_COMMANDS_FILE, claudeCommandRules, } from "./agents.js";
 import { activeOutages, detectOutage, earliest, whenAgain } from "./agent-outage.js";
 import { agentSpawn, CmdLineUnsafe } from "./which.js";
 import { VERSION } from "./version.js";
@@ -758,8 +759,8 @@ export function deliveryStep(branch, canPush, ref) {
     return canPush
         ? `If you wrote code: commit it on this branch, push it (git push origin ${branch} — this branch, never any other), then
     pantheon_send DELIVERED with${r} branch ${branch} and the commit sha, so your human and your peer can review it.
-    If the push is refused, send the DELIVERED anyway: the worker pushes this branch after you finish, and corrects
-    the DELIVERED on the ledger if it cannot.`
+    If the push is refused, send the DELIVERED anyway: Pantheon holds it until this branch is on origin — the worker
+    pushes it after you finish and sends your DELIVERED then, or tells your peer why it could not.`
         : `This run cannot commit or push, so never send DELIVERED — your peer could not fetch it. If you wrote code,
     pantheon_send NOTE with${r || " a ref,"} saying what you changed and that it waits on branch ${branch} on this machine
     for your human to review and push. If the request asked for code, that NOTE is your reply.`;
@@ -1005,6 +1006,8 @@ export function runInFlight(dir, now = Date.now()) {
         return null;
     return { pid: r.pid, at: r.at };
 }
+const HELD_BACKOFF_MS = 60_000;
+const HELD_BACKOFF_MAX_MS = 30 * 60_000;
 export const EXIT_GRACE_MS = 3_000;
 const CONTINUATION_HORIZON_MS = 24 * 60 * 60_000;
 const HELD_CONTINUATIONS_MAX = 20;
@@ -1163,7 +1166,7 @@ export class Worker {
         if (state === "idle" && this.readiness && !this.readiness.ok) {
             state = "unavailable";
             detail = "can't take requests yet";
-            more = { ...(this.readiness.code ? { reason: this.readiness.code } : {}) };
+            more = { ...(this.readiness.code ? { reason: this.readiness.pinned ? (this.readiness.code === "kiro_only" ? "pinned_kiro" : this.readiness.code === "unsandboxed_off" ? "pinned_unsandboxed" : this.readiness.code) : this.readiness.code } : {}) };
         }
         this.lastReport = { state, ...(detail ? { detail } : {}), ...(until ? { until } : {}), ...more, at: Date.now() };
         this.client?.reportWorkerState(state, detail, until, this.idleMs(), more);
@@ -1206,7 +1209,7 @@ export class Worker {
                 this.opts.log(`! teammates are told this machine can't take requests yet: ${r.short} — ${r.fix}`);
             if (r.ok && this.readiness && !this.readiness.ok)
                 this.opts.log(`  agent   ready again — teammates are told this machine takes requests`);
-            this.readiness = { ok: r.ok, short: r.short, ...(r.code ? { code: r.code } : {}) };
+            this.readiness = { ok: r.ok, short: r.short, ...(r.code ? { code: r.code } : {}), ...(r.pinned ? { pinned: true } : {}) };
         }
         catch {
             this.readiness = null;
@@ -1891,11 +1894,13 @@ export class Worker {
         mkdirSync(dir, { recursive: true, mode: 0o700 });
         const file = join(dir, "config.json");
         let written = cfg;
+        this.runKey = null;
         if (features.includes("delegate") && cfg.privateKey) {
             const run = newKeypair();
             const team = cfg.spaceId ?? this.client?.teamKey() ?? cfg.space;
             const expires = Date.now() + limitMs + 10 * 60_000;
             const cert = signDelegate(cfg.privateKey, team, cfg.member, run.publicKey, randomBytes(8).toString("hex"), expires);
+            this.runKey = { runId: cert.run, runPub: cert.pub };
             written = { ...cfg, privateKey: "", delegate: { privateKey: run.privateKey, cert }, ...(cfg.spaceId ? {} : { spaceId: team }) };
         }
         else if (features.includes("act-sig") || this.client?.signaturesExpected()) {
@@ -1914,6 +1919,7 @@ export class Worker {
         catch { }
         return dir;
     }
+    runKey = null;
     cleanupStep(what, step) {
         try {
             step();
@@ -2067,6 +2073,7 @@ export class Worker {
                     this.opts.log(`    ! ${p.name}: ${w}`);
         }
         this.recoverInterruptedRun();
+        await this.settleLeftHeld(client);
         this.reportStanding(true);
         if (this.opts.continuations !== false)
             await this.restoreHeldContinuations(client);
@@ -2094,6 +2101,9 @@ export class Worker {
             await this.maintain(client, arrived);
             if (this.running)
                 continue;
+            this.cleanPendingSandbox();
+            if (this.heldLeft && Date.now() >= this.heldRetryAt)
+                await this.settleLeftHeld(client);
             if (fleetLock && fleetLock !== this) {
                 this.writeState("watching", `waiting: a run for "${fleetLock.cfg.space}" is in progress`);
                 this.tickHold = { reason: "busy_elsewhere", detail: "another team's run is in progress on this machine" };
@@ -2502,7 +2512,7 @@ export class Worker {
         liveRuns.add(pid);
         installRunReaper();
         this.attemptPid = pid;
-        writeRunRecord(this.stateDir(), { pid, member: this.cfg.member, space: this.cfg.space, at: Date.now(), worker: process.pid, obligation: attempt.obligation, ...(attempt.spend ? { spend: attempt.spend } : {}), ...(attempt.limit ? { limit: attempt.limit } : {}) });
+        writeRunRecord(this.stateDir(), { pid, member: this.cfg.member, space: this.cfg.space, at: Date.now(), worker: process.pid, obligation: attempt.obligation, ...(attempt.spend ? { spend: attempt.spend } : {}), ...(attempt.limit ? { limit: attempt.limit } : {}), ...(attempt.sandbox ? { sandbox: attempt.sandbox } : {}) });
     }
     untrackRun(pid) {
         this.runPids.delete(pid);
@@ -2530,6 +2540,11 @@ export class Worker {
             return;
         if (Number.isInteger(r.pid) && typeof r.at === "number" && Date.now() - r.at <= recordLimitMs(r) && recordedRunAlive(r.pid))
             killRunTree(r.pid);
+        const sb = r.sandbox;
+        if (sb && typeof sb.common === "string" && Array.isArray(sb.absent)) {
+            this.sandboxPending = { pid: Number.isInteger(r.pid) ? r.pid : 0, common: sb.common, absent: sb.absent.filter((x) => typeof x === "string") };
+            this.cleanPendingSandbox();
+        }
         try {
             rmSync(f, { force: true });
         }
@@ -2552,6 +2567,21 @@ export class Worker {
     stateDir() {
         return this.opts.home ?? process.env.PANTHEON_HOME ?? join(homedir(), ".pantheon");
     }
+    sandboxPending = null;
+    cleanPendingSandbox() {
+        const p = this.sandboxPending;
+        if (!p)
+            return;
+        if (p.pid > 0 && recordedRunAlive(p.pid))
+            return;
+        this.sandboxPending = null;
+        try {
+            const gone = removeSandboxLeftovers(p.absent, p.common);
+            if (gone.length)
+                this.opts.log(`  confine removed ${gone.length} empty mount point${gone.length === 1 ? "" : "s"} the interrupted run's sandbox left: ${gone.map((g) => basename(g)).join(", ")}`);
+        }
+        catch { }
+    }
     brokenSaid = "";
     sayBrokenState(broken) {
         const key = broken.join(",");
@@ -2720,16 +2750,19 @@ export class Worker {
                 }
                 return { obligation: o, branch, worktree, ok: false, refused: true, detail: diagnosis.says, diagnosis, waitUntil: choice.outage.until };
             }
-            const refused = unattendedRefusal(choice, { unsandboxedOk: this.unsandboxedOk() });
-            if (refused) {
-                const noSandbox = !SANDBOXED_FOR_UNATTENDED.has(choice.backend.id) && !NOT_FOR_UNATTENDED_RUNS.has(choice.backend.id);
+            const refusal = unattendedRefusalOf(choice, { unsandboxedOk: this.unsandboxedOk() });
+            if (refusal) {
+                const refused = refusal.says;
+                const noSandbox = refusal.cause === "unsandboxed_off";
                 const diagnosis = noSandbox
                     ? this.noSandboxDiagnosis(choice)
-                    : {
-                        cause: "no_unattended_agent",
-                        says: `Teammates' requests wait for your next session here: ${refused}.`,
-                        fix: "Install and sign in to another agent CLI (claude, codex, cursor-agent or copilot) for woken runs; until then requests wait for a live session.",
-                    };
+                    : refusal.cause === "not_signed_in"
+                        ? { cause: "not_signed_in", says: `${refused}.`, fix: `Sign in once and requests are answered while you're away: ${SIGN_IN_CHECK[choice.backend.id].fix}` }
+                        : {
+                            cause: "no_unattended_agent",
+                            says: `Teammates' requests wait for your next session here: ${refused}.`,
+                            fix: "Install and sign in to another agent CLI (claude, codex, cursor-agent or copilot) for woken runs; until then requests wait for a live session.",
+                        };
                 if (noSandbox && !this.parkedNoSandbox.has(o.id))
                     this.parkedNoSandbox.set(o.id, { at: Date.now(), ...(policy.agent ? { agent: policy.agent } : {}) });
                 if (!this.skipLogged.has(`refused:${o.id}`)) {
@@ -2849,6 +2882,8 @@ export class Worker {
         let bashRules = [];
         let fenced = null;
         let sandboxAbsent = [];
+        let sandboxCommon = "";
+        let heldRun = null;
         if (!client.features.includes("delegate") && client.signaturesExpected()) {
             const says = "this team's relay signs messages but did not offer a key for the run, so the run would have needed your own key — it was not started";
             this.opts.log(`! ${says}`);
@@ -2876,6 +2911,33 @@ export class Worker {
                 for (const l of fenced.recovered.lines)
                     this.opts.log(`  config  ${l}`);
             }
+            {
+                let url = null;
+                if (canPush && this.runKey) {
+                    try {
+                        url = await gitOut(worktree, ["remote", "get-url", "origin"]);
+                    }
+                    catch {
+                        url = null;
+                    }
+                }
+                heldRun = url && this.runKey ? { team: this.pinTeam, runHome, ...this.runKey, obligation: o.id, ref: o.ref ?? null, goal: o.goal ?? null, branch, worktree, origin: url, at: Date.now() } : null;
+                try {
+                    if (heldRun) {
+                        const r = heldRun;
+                        this.heldState((st) => { st.runs = [...st.runs.filter((x) => !(x.team === r.team && x.obligation === r.obligation)), r]; });
+                    }
+                    writeHoldConfig(runHome, heldRun ? { origin: heldRun.origin, branch } : null);
+                }
+                catch (e) {
+                    heldRun = null;
+                    try {
+                        writeHoldConfig(runHome, null);
+                    }
+                    catch { }
+                    this.opts.log(`! could not set up holding this run's DELIVERED until it is pushed: ${e.message}`);
+                }
+            }
             const backend = this.opts.agentCmd ? { id: "claude-code", label: "Claude Code", binaries: ["claude"] } : choice.backend;
             const personHome = this.opts.personHome ?? homedir();
             if (features && allowBash && !ensureRunTmp(runTmpDir(runHome))) {
@@ -2888,16 +2950,21 @@ export class Worker {
                 ...(features ? { claude: { features, pantheonHome: this.stateDir(), home: personHome, userSettings: join(personHome, ".claude", "settings.json"), ...(() => { const c = adoptedCommon(worktree); return c ? { gitCommon: c } : {}; })() } } : {}),
             });
             plannedFiles = plan.writeFiles;
-            {
+            if (leavesSandboxMounts({ os: process.platform, agent: backend.id, sandbox: !!plan.osSandbox })) {
                 const gc = adoptedCommon(worktree);
                 if (gc) {
                     try {
                         sandboxAbsent = absentDeniedPaths(gc);
+                        sandboxCommon = gc;
+                        const own = ownWorktreeConfig(worktree, gc);
+                        if (own && !sandboxAbsent.includes(own))
+                            sandboxAbsent = [own, ...sandboxAbsent];
                     }
                     catch {
                         sandboxAbsent = [];
                     }
                 }
+                this.opts.log(`  confine Linux sandbox: watching ${sandboxAbsent.length} path${sandboxAbsent.length === 1 ? "" : "s"} it may leave behind${gc ? ` in ${gc}` : " (no common git dir known — none)"}`);
             }
             for (const n of plan.notes ?? [])
                 if (!this.notesSaid.has(n)) {
@@ -2932,6 +2999,12 @@ export class Worker {
                 for (const l of fenced.restore().lines)
                     this.opts.log(`  config  ${l}`); });
             this.cleanupStep("sealing the run home", () => this.sealRunHome(runHome));
+            this.cleanupStep("taking back the delivery hold", () => {
+                writeHoldConfig(runHome, null);
+                const r = heldRun;
+                if (r)
+                    this.heldState((st) => { st.runs = st.runs.filter((x) => !(x.team === r.team && x.obligation === r.obligation)); });
+            });
             const detail = `could not plan the run: ${e.message}`;
             this.opts.log(`! ${detail}`);
             recordRun({ at: Date.now(), goal: o.goal, obligation: o.id, ok: false, triggeredBy: (await this.actOf(client, o.ref))?.from ?? "unknown", summary: clip(o.body), space: this.spaceKey(), member: this.cfg.member }, this.opts.home);
@@ -2961,7 +3034,7 @@ export class Worker {
                     }
                     const pid = child.pid;
                     if (pid)
-                        this.trackRun(pid, { obligation: o.id, spend: spendId, limit: runLimit.ms });
+                        this.trackRun(pid, { obligation: o.id, spend: spendId, limit: runLimit.ms, ...(sandboxAbsent.length ? { sandbox: { common: sandboxCommon, absent: sandboxAbsent } } : {}) });
                     let head = "", tail = "";
                     const take = (b) => {
                         const s = b.toString();
@@ -3015,14 +3088,17 @@ export class Worker {
             }
             finally {
                 this.cleanupStep("removing the sandbox's empty mount points", () => {
-                    const gone = removeSandboxLeftovers(sandboxAbsent);
+                    const gone = sandboxAbsent.length ? removeSandboxLeftovers(sandboxAbsent, sandboxCommon) : [];
                     if (gone.length)
                         this.opts.log(`  confine removed ${gone.length} empty mount point${gone.length === 1 ? "" : "s"} the sandbox left: ${gone.map((g) => basename(g)).join(", ")}`);
+                    for (const k of describeKept(sandboxAbsent.filter((p) => !gone.includes(p))))
+                        this.opts.log(`  confine kept ${k} — not bubblewrap's empty mount point`);
                 });
                 this.cleanupStep("putting the repo's agent config back", () => { if (fenced)
                     for (const l of fenced.restore().lines)
                         this.opts.log(`  config  ${l}`); });
                 this.cleanupStep("sealing the run home", () => this.sealRunHome(runHome));
+                this.cleanupStep("taking back the delivery hold", () => writeHoldConfig(runHome, null));
                 this.cleanupStep("checking Pantheon's own files", () => this.restoreHomeAfterRun(homeBefore, o));
             }
             const logRoot = snap ? scratch : worktree;
@@ -3044,14 +3120,17 @@ export class Worker {
             }
             this.writeRunLog(logRoot, runLog, `obligation ${o.id}\nbranch ${branch}\n\n${detail}\n${extra}`);
             const exited = /^exit 0/.test(detail);
-            const sent = client.mineSince(startedAt, o.goal);
+            const held = await this.settleHeld(client, heldRun ? { run: heldRun, canPush: canPush && !massDelete, massDelete } : null);
+            const settled = [...held.sent, ...held.waiting];
+            const settledIds = new Set(settled.map((a) => a.id));
+            const sent = [...client.mineSince(startedAt, o.goal).filter((a) => !settledIds.has(a.id)), ...settled.filter((a) => o.goal == null || a.goal === o.goal)];
             const repliedWith = sent.find((a) => ["DELIVERED", "ANSWER", "ESCALATE", "ACCEPT", "OBJECT", "PROPOSE", "CONTRACT", "CLOSE", "WITHDRAW"].includes(a.act) || (a.act === "NOTE" && !!a.ref));
             const replied = requireReply ? client.repliedTo(o) || !!repliedWith : true;
             if (requireReply && replied && !client.repliedTo(o) && repliedWith) {
                 this.opts.log(`  reply   ${repliedWith.act} ${repliedWith.id} answers ${o.id}, though the relay still lists it as owed${repliedWith.ref ? "" : " (the act named no ref)"}`);
             }
             const ok = exited && replied;
-            await this.checkDeliveries(client, sent, { branch, worktree, canPush: canPush && !massDelete, goal: o.goal ?? null });
+            await this.checkDeliveries(client, sent.filter((a) => !settledIds.has(a.id)), { branch, worktree, canPush: canPush && !massDelete, goal: o.goal ?? null });
             this.runProject = null;
             if (snap) {
                 const ours = new Set((plannedFiles ?? []).map((f) => (isAbsolute(f.path) ? relative(worktree, f.path) : f.path).split("\\").join("/")));
@@ -3170,6 +3249,298 @@ export class Worker {
             catch { }
         }
     }
+    async pushRunBranch(worktree, br, commit) {
+        if (!validBase(br) || (await gitOut(worktree, ["merge-base", "--is-ancestor", commit, `refs/heads/${br}`])) === null)
+            return false;
+        const r = await gitOut(worktree, ["push", "--quiet", "origin", `refs/heads/${br}:refs/heads/${br}`], FETCH_TIMEOUT_MS);
+        return r !== null && await onRemote(worktree, br, commit);
+    }
+    heldLeft = false;
+    heldRetryAt = 0;
+    heldCorruptSaid = new Set();
+    heldState(fn) {
+        const { result, corrupt } = updateHeldState(this.stateDir(), fn);
+        if (corrupt)
+            this.sayHeldCorrupt(corrupt);
+        return result;
+    }
+    sayHeldCorrupt(file) {
+        if (this.heldCorruptSaid.has(file))
+            return;
+        this.heldCorruptSaid.add(file);
+        this.opts.log(`! the record of held deliveries could not be read — set aside as ${file}`);
+        try {
+            this.opts.notify?.({ kind: "worker", who: "Your worker", heading: "Held deliveries could not be read", context: this.cfg.space,
+                body: `A record of deliveries a woken run held until their branch was pushed could not be read, so they were not sent. It is kept at ${file}. Check with your teammates whether they got the work.`, ttl: 0, sound: true });
+        }
+        catch { }
+    }
+    async originHas(worktree, branch, commit) {
+        if (!validBase(branch))
+            return false;
+        let line;
+        try {
+            line = await gitOut(worktree, ["ls-remote", "origin", `refs/heads/${branch}`], FETCH_TIMEOUT_MS);
+        }
+        catch {
+            return null;
+        }
+        if (line === null)
+            return null;
+        const tip = line.split(/\s+/)[0] ?? "";
+        if (!tip)
+            return false;
+        if (tip.startsWith(commit.toLowerCase()) || commit.toLowerCase().startsWith(tip))
+            return true;
+        try {
+            return (await gitOut(worktree, ["merge-base", "--is-ancestor", commit, tip])) !== null;
+        }
+        catch {
+            return false;
+        }
+    }
+    async settleHeld(client, live) {
+        const team = this.pinTeam;
+        const sent = [], waiting = [];
+        const registered = this.heldState((st) => st.runs.filter((r) => r.team === team));
+        const runs = live ? [live.run, ...registered.filter((r) => r.runHome === live.run.runHome && r.obligation !== live.run.obligation)] : registered;
+        const homes = [...new Set(runs.map((r) => r.runHome))];
+        for (const home of homes) {
+            try {
+                const { corrupt, dropped } = takeHeldFrames(home, (frames) => {
+                    const add = [];
+                    for (const h of frames) {
+                        const teams = [this.cfg.spaceId, client.teamKey(), this.cfg.space];
+                        const r = runs.find((x) => x.runHome === home && verifyHeldFrame(h.frame, { member: this.cfg.member, memberPub: this.cfg.publicKey, teams, runId: x.runId, runPub: x.runPub }));
+                        if (!r) {
+                            this.opts.log(`! dropped a held DELIVERED that is not signed by a run of this worker's — nothing is sent, pushed or answered for it`);
+                            continue;
+                        }
+                        add.push({ ...h, team, obligation: r.obligation, ref: r.ref, goal: r.goal, branch: r.branch, worktree: r.worktree, origin: r.origin, tries: 0, firstTry: Date.now() });
+                    }
+                    this.heldState((st) => {
+                        st.held = [...st.held.filter((x) => !add.some((a) => a.frame.id === x.frame.id)), ...add];
+                        st.runs = st.runs.filter((x) => !(x.team === team && x.runHome === home));
+                    });
+                });
+                if (corrupt)
+                    this.sayHeldCorrupt(corrupt);
+                if (dropped)
+                    this.opts.log(`! a run held more than ${MAX_HELD_PER_RUN} deliveries — only its newest ${MAX_HELD_PER_RUN} are kept, ${dropped} dropped unsent`);
+                for (const f of corruptHeldFiles(home))
+                    this.sayHeldCorrupt(f);
+            }
+            catch (e) {
+                this.opts.log(`! could not read what the run held: ${e.message}`);
+            }
+        }
+        const all = this.heldState((st) => st.held.filter((x) => x.team === team)).sort((a, b) => a.at - b.at);
+        if (!all.length) {
+            this.heldLeft = false;
+            return { sent, waiting };
+        }
+        const drop = (e) => this.heldState((st) => { st.held = st.held.filter((x) => x.frame.id !== e.frame.id); });
+        const keep = (e, why) => this.heldState((st) => {
+            const x = st.held.find((y) => y.frame.id === e.frame.id);
+            if (x) {
+                x.tries += 1;
+                x.nextTry = Date.now() + Math.min(HELD_BACKOFF_MAX_MS, HELD_BACKOFF_MS * 2 ** Math.min(x.tries - 1, 20));
+                if (why)
+                    x.why = why;
+            }
+        });
+        const rec = (r, a) => ({ id: r.id, seq: r.seq, ts: Date.now(), space: this.cfg.space, from: this.cfg.member, to: a.to ?? null, goal: a.goal ?? null, act: a.act, body: a.body, ref: a.ref ?? null, ...(a.meta ? { meta: a.meta } : {}) });
+        const codeOf = (e) => /^([A-Z][A-Z_]{2,40}):/.exec(e.message)?.[1] ?? null;
+        const transient = (e) => !client.connected || /not connected to relay|did not answer/.test(e.message)
+            || ["RATE_LIMIT", "SPACE_FULL", "STALE_VIEW", "SPACE_UNREADABLE", "TOO_MANY_REFUSALS"].includes(codeOf(e) ?? "") || /^STORAGE/.test(codeOf(e) ?? "");
+        const note = async (e, summary, body) => {
+            const asker = e.ref ? (await this.actOf(client, e.ref))?.from : undefined;
+            const to = asker && asker !== this.cfg.member ? [asker] : undefined;
+            for (const where of [{ goal: e.goal, ref: e.ref, ...(to ? { to } : {}) }, ...(to ? [{ to }] : []), {}]) {
+                const a = { act: "NOTE", summary: clip(summary, 190), body, ...where };
+                try {
+                    const r = await client.send(a);
+                    return rec(r, a);
+                }
+                catch (err) {
+                    if (transient(err))
+                        return "later";
+                    this.opts.log(`  the note about held DELIVERED ${e.frame.id} was refused (${codeOf(err) ?? "refused"}) — trying it more plainly`);
+                }
+            }
+            return null;
+        };
+        const tell = (heading, body, e, branch) => {
+            try {
+                this.opts.notify?.({ kind: "worker", who: "Your worker", heading, context: this.cfg.space, body, ttl: 0, sound: false,
+                    ...(e && branch ? { fix: { label: "Push it", dir: e.worktree, lines: [`git push origin ${branch}`], why: `the run's delivery of ${branch} is not on origin` } } : {}) });
+            }
+            catch { }
+        };
+        const GIVE_UP_MS = 24 * 3_600_000;
+        const inGoal = new Map();
+        const mineInGoal = async (goal) => {
+            if (!goal)
+                return [];
+            if (!inGoal.has(goal)) {
+                let acts = [];
+                try {
+                    const res = await signedRead(this.cfg, "/history", { goal }, { timeoutMs: 10_000 });
+                    if (res.ok)
+                        acts = (await res.json()).acts ?? [];
+                }
+                catch { }
+                inGoal.set(goal, acts.filter((a) => a.goal === goal && a.from === this.cfg.member && a.act === "DELIVERED"));
+            }
+            return inGoal.get(goal);
+        };
+        for (const e of all) {
+            const t = { branch: e.frame.meta.branch, commit: e.frame.meta.commit };
+            const ours = !!live && e.obligation === live.run.obligation && e.worktree === live.run.worktree;
+            const pending = rec({ id: e.frame.id, seq: 0 }, e.frame);
+            const later = (why) => {
+                if (Date.now() - e.firstTry > GIVE_UP_MS) {
+                    drop(e);
+                    this.opts.log(`! gave up on the DELIVERED the run held for ${t.branch} at ${t.commit.slice(0, 7)}: it could not be settled in a day (${e.tries + 1} tries)`);
+                    tell("A delivery could not be sent", `A woken run finished work on ${t.branch} at ${t.commit.slice(0, 7)}, but Pantheon could not reach the relay or origin to tell your teammate, and has stopped trying. Tell them yourself where the work is.`);
+                    return;
+                }
+                keep(e, why);
+                if (ours)
+                    waiting.push(pending);
+            };
+            if (!ours && (e.nextTry ?? 0) > Date.now())
+                continue;
+            if (all.some((x) => x !== e && x.obligation === e.obligation && x.at > e.at)) {
+                this.opts.log(`  dropped an earlier DELIVERED the run held for ${t.branch} at ${t.commit.slice(0, 7)}: the same run held a later one`);
+                drop(e);
+                continue;
+            }
+            const sameAs = (r) => r.act === "DELIVERED"
+                && (e.frame.ref ? r.ref === e.frame.ref : r.goal === e.frame.goal && r.meta?.branch === t.branch);
+            const signedAt = e.frame.sig?.ts ?? e.at;
+            const laterSent = [...client.mineSince(e.at), ...await mineInGoal(e.frame.goal ?? null)]
+                .some((r) => r.id !== e.frame.id && (r.sig?.ts ?? r.ts) > signedAt && sameAs(r));
+            if (all.some((x) => x !== e && x.at > e.at && sameAs(x.frame)) || laterSent) {
+                this.opts.log(`  dropped the DELIVERED the run held for ${t.branch} at ${t.commit.slice(0, 7)}: a later DELIVERED for the same request went since`);
+                drop(e);
+                continue;
+            }
+            let there = ours || existsSync(e.worktree)
+                ? await this.originHas(e.worktree, t.branch, t.commit)
+                : await remoteTipIs(e.origin, t.branch, t.commit, FETCH_TIMEOUT_MS);
+            let why = null;
+            if (there === null && !ours) {
+                later();
+                continue;
+            }
+            if (!there) {
+                if (e.why)
+                    why = e.why;
+                else if (!ours)
+                    why = "stopped";
+                else if (t.branch !== e.branch)
+                    why = "not_own_branch";
+                else if (!live.canPush)
+                    why = live.massDelete ? "mass_delete" : "not_allowed";
+                else {
+                    there = await this.pushRunBranch(e.worktree, e.branch, t.commit).catch(() => false);
+                    if (there)
+                        this.opts.log(`* pushed ${e.branch} for the run: the DELIVERED it held named ${t.commit.slice(0, 7)}, which origin did not have — repo hooks not run on Pantheon's push${lfsInUse(e.worktree) ? "; LFS objects uploaded" : ""}`);
+                    else
+                        why = "push_refused";
+                }
+            }
+            if (there) {
+                try {
+                    const { after_seq: _seen, ...frame } = e.frame;
+                    const r = await client.postAct(frame);
+                    this.opts.log(`* sent the DELIVERED ${r.id} the run held: ${t.branch} at ${t.commit.slice(0, 7)} is on origin now`);
+                    sent.push(rec(r, e.frame));
+                    drop(e);
+                    if (e.ref)
+                        this.markDone(`handled:${e.ref}`);
+                    continue;
+                }
+                catch (err) {
+                    if (transient(err)) {
+                        this.opts.log(`  could not send the DELIVERED the run held yet (${codeOf(err) ?? "the relay is out of reach"}) — it is tried again`);
+                        later();
+                        continue;
+                    }
+                    if (codeOf(err) === "ID_TAKEN") {
+                        this.opts.log(`  the DELIVERED ${e.frame.id} the run held is on the ledger already`);
+                        sent.push(pending);
+                        drop(e);
+                        if (e.ref)
+                            this.markDone(`handled:${e.ref}`);
+                        continue;
+                    }
+                    const code = codeOf(err);
+                    const because = code === "BAD_ACT_SIGNATURE" ? "the run's key had expired" : code === "GOAL_CLOSED" ? "the goal was closed" : code === "NOT_IN_GOAL" ? "the goal no longer includes them" : "the relay would not take it now";
+                    this.opts.log(`! the relay refused the DELIVERED the run held (${code ?? "refused"}) — a NOTE says the branch is on origin instead`);
+                    const n = await note(e, `The work is on origin: ${t.branch} at ${t.commit.slice(0, 7)}`, [
+                        `${this.cfg.member}'s unattended run finished this work: ${t.branch} at ${t.commit.slice(0, 12)} is on origin — fetch it and check it.`,
+                        `Its DELIVERED could not be sent (${because}), so this note says it instead.`,
+                    ].join("\n"));
+                    if (n === "later") {
+                        later();
+                        continue;
+                    }
+                    if (n) {
+                        sent.push(n);
+                        if (e.ref)
+                            this.markDone(`handled:${e.ref}`);
+                    }
+                    drop(e);
+                    tell("A delivery went as a note", `A woken run's work on ${t.branch} at ${t.commit.slice(0, 7)} is on origin, but its DELIVERED could not be sent (${because}). ${n ? "Your teammate was told in a note." : "Your teammate could not be told — tell them yourself."}`);
+                    continue;
+                }
+            }
+            const words = why === "stopped" ? "the worker stopped before it could push it"
+                : why === "not_own_branch" ? `it names ${t.branch}, not this run's own branch ${e.branch}, which the worker will not push`
+                    : why === "mass_delete" ? "this branch deletes most of the files tracked at its base, so it is not pushed without a person looking"
+                        : why === "not_allowed" ? "this run was not allowed to push"
+                            : "the worker's push was refused too";
+            this.opts.log(`! the DELIVERED the run held names ${t.branch} at ${t.commit.slice(0, 7)}, which is not on origin — ${words}; not sent, a NOTE goes instead`);
+            const n = await note(e, `Not delivered yet: ${t.branch} could not be pushed`, [
+                `The work for this is on branch ${t.branch} at ${t.commit.slice(0, 12)} on ${this.cfg.member}'s machine, but it could not be pushed, so there is nothing to fetch yet.`,
+                `Why: ${words}. ${this.cfg.member}'s human has been told and will push it or say what happened.`,
+            ].join("\n"));
+            if (n === "later") {
+                this.opts.log(`  the note about it could not be sent yet — it is tried again`);
+                later(why ?? undefined);
+                continue;
+            }
+            if (n) {
+                sent.push(n);
+                if (e.ref)
+                    this.markDone(`handled:${e.ref}`);
+            }
+            drop(e);
+            tell("A delivery was not pushed", `A woken run finished work on ${t.branch} at ${t.commit.slice(0, 7)}, but it could not be pushed (${words}), so its DELIVERED was not sent; ${n ? "your teammate was told the work is here" : "your teammate could not be told — tell them yourself"}. Push it yourself: git push origin ${t.branch} — then tell them.`, e, t.branch);
+        }
+        const left = this.heldState((st) => st.held.filter((x) => x.team === team));
+        this.heldLeft = left.length > 0;
+        this.heldRetryAt = left.length ? Math.max(Date.now() + 5_000, Math.min(...left.map((x) => x.nextTry ?? 0))) : 0;
+        return { sent, waiting };
+    }
+    async settleLeftHeld(client) {
+        if (this.running || !client.connected)
+            return;
+        try {
+            const r = JSON.parse(readFileSync(join(this.stateDir(), RUN_RECORD), "utf8"));
+            if (r && r.space === this.cfg.space && Number.isInteger(r.worker) && r.worker !== process.pid && pidAlive(r.worker))
+                return;
+        }
+        catch { }
+        try {
+            await this.settleHeld(client, null);
+        }
+        catch (e) {
+            this.opts.log(`! could not settle what a run held: ${e.message}`);
+        }
+    }
     async checkDeliveries(client, sent, run) {
         const deliveries = sent.filter((a) => a.act === "DELIVERED");
         if (!deliveries.length || !run.branch)
@@ -3187,14 +3558,9 @@ export class Worker {
                 continue;
             if (await onRemote(run.worktree, br, commit))
                 continue;
-            let pushed = false;
-            if (run.canPush && br === run.branch && validBase(br)
-                && (await gitOut(run.worktree, ["merge-base", "--is-ancestor", commit, `refs/heads/${br}`])) !== null) {
-                const r = await gitOut(run.worktree, ["push", "--quiet", "origin", `refs/heads/${br}:refs/heads/${br}`], FETCH_TIMEOUT_MS);
-                pushed = r !== null && await onRemote(run.worktree, br, commit);
-                if (pushed)
-                    this.opts.log(`* pushed ${br} for the run: its DELIVERED ${d.id} named ${commit.slice(0, 7)}, which origin did not have — repo hooks not run on Pantheon's push${lfsInUse(run.worktree) ? "; LFS objects uploaded" : ""}`);
-            }
+            const pushed = run.canPush && br === run.branch && await this.pushRunBranch(run.worktree, br, commit);
+            if (pushed)
+                this.opts.log(`* pushed ${br} for the run: its DELIVERED ${d.id} named ${commit.slice(0, 7)}, which origin did not have — repo hooks not run on Pantheon's push${lfsInUse(run.worktree) ? "; LFS objects uploaded" : ""}`);
             if (pushed)
                 continue;
             const why = br !== run.branch ? `it names ${br}, not this run's own branch ${run.branch}, which the worker will not push`

package/helper/macos/prebuilt/manifest.json

@@ -1,6 +1,6 @@
 {
   "format": 1,
-  "built_at": "2026-10-11T16:51:24.237Z",
+  "built_at": "2026-10-13T06:04:13.698Z",
   "swiftc": "Apple Swift version 6.3.3 (swiftlang-6.3.3.1.3 clang-2100.1.1.101)",
   "helpers": {
     "pantheon-tray": {

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@join-pantheon/cli",
-  "version": "0.33.6",
+  "version": "0.33.7",
   "description": "Let your coding agent collaborate directly with your teammates' agents — ask, agree an interface, and deliver, without a human relaying messages.",
   "type": "module",
   "bin": {