@bivy/bivy 0.16.19-staging.2 → 0.16.19-staging.4

package/dist/control-plane-tasks.js

@@ -16,6 +16,8 @@
  * issues (and Slack) across many repos/nodes. Pure HTTP + claim/loop logic lives
  * here; the actual run is injected so the daemon keeps the agent wiring.
  */
+import { randomUUID } from "node:crypto";
+import { WorkResultOutbox } from "./work-result-outbox.js";
 import { RemoteSessionAdmissionError } from "./session/remote-session-admission.js";
 /** True when every required tag is among this node's declared capabilities.
  * A node that fails this must never attempt to claim the item — the hard
@@ -73,12 +75,26 @@ export function resolveControlPlaneTaskConfig(relay, env = process.env, nodeName
         capabilities,
     };
 }
-async function cp(cfg, method, path) {
+async function cp(cfg, method, path, claimToken, body, signal) {
+    const timeout = AbortSignal.timeout(10_000);
     return fetch(`${cfg.controlPlaneUrl}${path}`, {
         method,
-        headers: { authorization: `Bearer ${cfg.enrollmentToken}` },
+        signal: signal ? AbortSignal.any([signal, timeout]) : timeout,
+        headers: { authorization: `Bearer ${cfg.enrollmentToken}`, 'content-type': 'application/json', ...(claimToken ? { 'x-bivy-work-claim': claimToken } : {}) },
+        body: body === undefined ? undefined : JSON.stringify(body),
     });
 }
+async function sendWorkResult(cfg, result) {
+    try {
+        const res = await cp(cfg, 'POST', `/node/work/${encodeURIComponent(result.id)}/${result.action}`, result.claimToken);
+        if (res.ok)
+            return 'acknowledged';
+        if (res.status === 409 || res.status === 404)
+            return 'lost';
+    }
+    catch { /* Outbox retains the intent; retry delivery without agent work. */ }
+    return 'retry';
+}
 async function transitionWork(cfg, id, action) {
     // Best-effort — a dropped transition never loses the run itself — but NOT
     // silent: a swallowed `complete`/`fail`/`needs-attention` leaves the control
@@ -101,10 +117,13 @@ export async function fetchPendingWork(cfg) {
     const data = (await res.json().catch(() => ({})));
     return Array.isArray(data.items) ? data.items : [];
 }
-/** Atomically claim an item. Returns true only if THIS node won the claim. */
+/** Consume the authoritative claim snapshot, not the earlier queue listing. */
 export async function claimWork(cfg, id) {
-    const res = await cp(cfg, "POST", `/node/work/${encodeURIComponent(id)}/claim`);
-    return res.ok;
+    const res = await cp(cfg, "POST", `/node/work/${encodeURIComponent(id)}/claim`, randomUUID());
+    if (!res.ok)
+        return undefined;
+    const data = await res.json();
+    return data.item ?? { id }; // compatibility with older control planes
 }
 /** Renew ownership and retain the reason a renewal was rejected. Cancellation is
  *  intentionally distinct from a generic lost lease so an active agent can be
@@ -131,11 +150,12 @@ export async function needsAttentionWork(cfg, id) {
  *  Best-effort: a dropped report loses one evidence update, never the run
  *  itself. It is not throwing, but the failure is logged (A4) so a persistently
  *  failing evidence channel is visible in diagnostics instead of silent. */
-export async function reportEvidence(cfg, id, patch) {
+export async function reportEvidence(cfg, id, patch, claimToken) {
     try {
         const res = await fetch(`${cfg.controlPlaneUrl}/node/work/${encodeURIComponent(id)}/evidence`, {
             method: "POST",
-            headers: { authorization: `Bearer ${cfg.enrollmentToken}`, "content-type": "application/json" },
+            headers: { authorization: `Bearer ${cfg.enrollmentToken}`, "content-type": "application/json", ...(claimToken ? { 'x-bivy-work-claim': claimToken } : {}) },
+            signal: AbortSignal.timeout(10_000),
             body: JSON.stringify(patch),
         });
         if (!res.ok) {
@@ -158,6 +178,8 @@ export class ControlPlaneTaskPoller {
     policy;
     sleep;
     leaseHeartbeatMs;
+    outbox;
+    flushingResults;
     constructor(cfg, runItem, 
     /** Node's cap on concurrently-running queue sessions (0/undefined = unlimited).
      *  Read fresh each tick so the Settings → Nodes value takes effect live. */
@@ -168,8 +190,11 @@ export class ControlPlaneTaskPoller {
         this.policy = options.policy;
         this.sleep = options.sleep ?? defaultSleep;
         this.leaseHeartbeatMs = options.leaseHeartbeatMs ?? 30_000;
+        this.outbox = new WorkResultOutbox(options.resultDirectory, `${cfg.controlPlaneUrl}:${cfg.enrollmentToken}`);
     }
     start() {
+        if (this.timer)
+            return;
         void this.tick();
         this.timer = setInterval(() => void this.tick(), this.cfg.pollMs);
         this.timer.unref?.();
@@ -187,10 +212,10 @@ export class ControlPlaneTaskPoller {
         }
         void this.tick();
     }
-    /** Number of queue items currently running on this node. Lets an ephemeral
-     *  machine's self-teardown avoid exiting while it's mid-work. */
+    /** Include undelivered results so an ephemeral machine cannot tear down its
+     * only durable outbox while completion reconciliation is still pending. */
     inFlightCount() {
-        return this.inFlight.size;
+        return new Set([...this.inFlight.keys(), ...this.outbox.list().map(result => result.id)]).size;
     }
     /**
      * Replace the routing labels this live poller serves.
@@ -212,8 +237,10 @@ export class ControlPlaneTaskPoller {
     stop() {
         if (this.timer)
             clearInterval(this.timer);
+        this.timer = undefined;
     }
     async tick() {
+        await this.flushResults();
         let items;
         try {
             items = await fetchPendingWork(this.cfg);
@@ -224,7 +251,7 @@ export class ControlPlaneTaskPoller {
         const max = this.maxConcurrent?.() ?? 0;
         const running = [];
         for (const item of items) {
-            if (this.inFlight.has(item.id))
+            if (this.inFlight.has(item.id) || this.outbox.has(item.id))
                 continue;
             // Hard block: never contend for an item requiring a capability this
             // node hasn't declared. It stays pending for a node that has it (or
@@ -268,12 +295,18 @@ export class ControlPlaneTaskPoller {
             // Claim first so only one node runs it; skip if another node won (no
             // claim → not ours → don't run or complete it). A heartbeat keeps the
             // finite lease alive; process death stops it and makes the item reclaimable.
-            if (!(await claimWork(this.cfg, item.id)) || run.state !== "active")
+            const claimed = await claimWork(this.cfg, item.id);
+            if (!claimed || run.state !== 'active')
                 return;
+            item = { ...item, ...claimed };
+            run.claimToken = claimed.claimToken;
+            this.setLeaseDeadline(run, claimed.leaseExpiresAt);
             run.heartbeat = setInterval(() => void this.checkLease(item.id, run), this.leaseHeartbeatMs);
             run.heartbeat.unref?.();
-            const report = (patch) => reportEvidence(this.cfg, item.id, patch);
-            await transitionWork(this.cfg, item.id, "running");
+            const report = (patch) => reportEvidence(this.cfg, item.id, patch, run.claimToken);
+            const started = await cp(this.cfg, 'POST', `/node/work/${encodeURIComponent(item.id)}/running`, run.claimToken);
+            if (!started.ok)
+                return; // Never execute after a rejected lifecycle transition.
             if (run.state !== "active")
                 return;
             console.log(`[control-plane-tasks] running ${item.source} item ${item.id}: ${item.title}`);
@@ -286,14 +319,105 @@ export class ControlPlaneTaskPoller {
                 return;
             await this.runWithPolicy(item, report, run);
         }
+        catch (error) {
+            console.warn(`[control-plane-tasks] work ${item.id} orchestration failed:`, error);
+        }
         finally {
             if (run.heartbeat)
                 clearInterval(run.heartbeat);
+            if (run.deadline)
+                clearTimeout(run.deadline);
             // A stale completion must not remove a newer reservation for the same id.
             if (this.inFlight.get(item.id) === run)
                 this.inFlight.delete(item.id);
         }
     }
+    setLeaseDeadline(run, expiry) {
+        if (!expiry && !run.claimToken)
+            return; // legacy servers do not advertise a lease
+        const remaining = expiry ? Date.parse(expiry) - Date.now() : 0;
+        if (run.deadline)
+            clearTimeout(run.deadline);
+        const lose = () => {
+            if (run.state !== 'active')
+                return;
+            run.state = 'lost';
+            run.controller.abort(new Error('Run lease expired without confirmed renewal'));
+            if (run.heartbeat)
+                clearInterval(run.heartbeat);
+        };
+        if (!Number.isFinite(remaining) || remaining <= 0) {
+            lose();
+            return;
+        }
+        run.deadline = setTimeout(lose, Math.max(1, remaining - Math.min(1000, remaining / 10)));
+        run.deadline.unref?.();
+    }
+    flushResults() {
+        if (this.flushingResults)
+            return this.flushingResults;
+        this.flushingResults = (async () => {
+            for (const result of this.outbox.list()) {
+                // Active workers own their delivery/heartbeat loop.
+                if (this.inFlight.has(result.id))
+                    continue;
+                const status = await sendWorkResult(this.cfg, result);
+                if (status !== 'retry')
+                    this.outbox.remove(result.id);
+            }
+        })().catch(error => console.warn('[control-plane-tasks] outcome reconciliation failed:', error))
+            .finally(() => { this.flushingResults = undefined; });
+        return this.flushingResults;
+    }
+    async finishWork(id, action, run) {
+        const result = { id, action, claimToken: run.claimToken };
+        this.outbox.put(result);
+        // Keep ownership while retrying delivery, but never beyond its confirmed
+        // deadline. After a restart the outbox is reconciled before queue intake.
+        do {
+            const status = await sendWorkResult(this.cfg, result);
+            if (status !== 'retry') {
+                this.outbox.remove(id);
+                return;
+            }
+            if (!run.claimToken)
+                return; // legacy server: retain for the recovery poll
+            await this.sleep(1000);
+        } while (run.state === 'active');
+    }
+    /** Redeliver the same reservation after uncertain acknowledgement, without
+     * consuming another attempt or allowing agent work beyond our lease. */
+    async reserveAttempt(id, attempt, run) {
+        while (run.state === 'active') {
+            let rejected = false;
+            try {
+                const response = await cp(this.cfg, 'POST', `/node/work/${encodeURIComponent(id)}/attempt`, run.claimToken, { attempt }, run.controller.signal);
+                if (response.ok) {
+                    const data = await response.json();
+                    if (data.item?.attempt !== attempt + 1)
+                        throw new Error('Invalid attempt reservation acknowledgement');
+                    return run.state === 'active' ? data.item.attempt : undefined;
+                }
+                rejected = response.status !== 408 && response.status !== 429 && response.status < 500;
+            }
+            catch (error) {
+                if (run.state !== 'active')
+                    return undefined;
+                console.warn(`[control-plane-tasks] work ${id} attempt reservation failed:`, error instanceof Error ? error.message : error);
+            }
+            if (rejected) {
+                // A conflict may mean either exhausted budget or lost ownership.
+                // Confirm ownership before parking; cancelled/stale workers just stop.
+                await this.checkLease(id, run);
+                if (run.state === 'active')
+                    await this.finishWork(id, 'needs-attention', run);
+                return undefined;
+            }
+            if (run.state === 'active')
+                await this.sleep(1000);
+        }
+        return undefined;
+    }
     checkLease(id, run) {
         if (run.state !== "active")
             return Promise.resolve();
@@ -301,9 +425,17 @@ export class ControlPlaneTaskPoller {
             return run.leaseCheck;
         run.leaseCheck = (async () => {
             try {
-                const renewal = await renewWorkLease(this.cfg, id);
-                if (renewal === "renewed" || this.inFlight.get(id) !== run || run.state !== "active")
+                const response = await cp(this.cfg, 'POST', `/node/work/${encodeURIComponent(id)}/heartbeat`, run.claimToken);
+                const data = await response.json();
+                if (this.inFlight.get(id) !== run || run.state !== 'active')
+                    return;
+                if (response.ok) {
+                    this.setLeaseDeadline(run, data.leaseExpiresAt);
+                    return;
+                }
+                if (response.status !== 409 && response.status !== 404)
                     return;
+                const renewal = data.reason === 'cancelled' ? 'cancelled' : 'lost';
                 run.state = renewal;
                 if (run.heartbeat) {
                     clearInterval(run.heartbeat);
@@ -312,8 +444,7 @@ export class ControlPlaneTaskPoller {
                 run.controller.abort(new Error(renewal === "cancelled" ? "Run cancelled" : "Run lease lost"));
             }
             catch (error) {
-                // A transient network error does not prove ownership was lost. Keep the
-                // Run alive and let the next heartbeat retry.
+                // Transient failures may retry only until the last confirmed deadline.
                 console.warn(`[control-plane-tasks] work ${id} heartbeat failed:`, error instanceof Error ? error.message : error);
             }
             finally {
@@ -333,7 +464,7 @@ export class ControlPlaneTaskPoller {
      */
     async runWithPolicy(item, report, run) {
         let current = item;
-        let attempt = 1;
+        let attempt = Math.max(1, item.attempt ?? 1);
         let rerouteCount = 0;
         for (;;) {
             if (run.state !== "active")
@@ -344,12 +475,16 @@ export class ControlPlaneTaskPoller {
                 await this.runItem({ ...current, attempt }, report, run.controller.signal);
                 if (run.state !== "active")
                     return;
-                await completeWork(this.cfg, item.id);
+                await this.finishWork(item.id, 'complete', run);
                 return;
             }
             catch (error) {
                 // Abort errors are ordinary throws to the policy layer unless guarded.
                 // A cancelled/lost Run has no node-side terminal transition or retry.
+                if (this.outbox.has(item.id)) {
+                    console.error(`[control-plane-tasks] work ${item.id} result delivery/persistence failed; agent will not be retried:`, error);
+                    return;
+                }
                 if (run.state !== "active")
                     return;
                 // Deployment admission is not an agent failure. Never reroute/retry it
@@ -362,7 +497,7 @@ export class ControlPlaneTaskPoller {
                     });
                     if (run.state !== "active")
                         return;
-                    await needsAttentionWork(this.cfg, item.id);
+                    await this.finishWork(item.id, 'needs-attention', run);
                     return;
                 }
                 const policy = typeof this.policy === "function" ? this.policy(current) : this.policy;
@@ -381,11 +516,19 @@ export class ControlPlaneTaskPoller {
                     await report({ events: [{ at: new Date().toISOString(), kind: "needs_attention", summary, attempt }] });
                     if (run.state !== "active")
                         return;
-                    await needsAttentionWork(this.cfg, item.id);
+                    await this.finishWork(item.id, 'needs-attention', run);
                     return;
                 }
                 if (decision.action === "retry" || decision.action === "reroute") {
-                    attempt += 1;
+                    if (run.claimToken) {
+                        const reserved = await this.reserveAttempt(item.id, attempt, run);
+                        if (reserved === undefined)
+                            return;
+                        attempt = reserved;
+                    }
+                    else {
+                        attempt += 1;
+                    } // legacy control plane
                     const kind = decision.action === "retry" ? "retry" : "fallback";
                     console.warn(`[control-plane-tasks] item ${item.id} ${kind} (${decision.condition}): ${decision.summary}`);
                     await report({
@@ -425,13 +568,13 @@ export class ControlPlaneTaskPoller {
                     await report({ events: [{ at: new Date().toISOString(), kind: "needs_attention", summary: decision.summary }] });
                     if (run.state !== "active")
                         return;
-                    await needsAttentionWork(this.cfg, item.id);
+                    await this.finishWork(item.id, 'needs-attention', run);
                     return;
                 }
                 console.warn(`[control-plane-tasks] item ${item.id} failed:`, error);
                 if (run.state !== "active")
                     return;
-                await failWork(this.cfg, item.id);
+                await this.finishWork(item.id, 'fail', run);
                 return;
             }
         }

package/dist/server.js

@@ -5097,6 +5097,9 @@ async function executeWorkItem(item, report, signal) {
     const sandbox = safety.sandbox;
     const sessionOpts = {
         makeActive: false,
+        // The same durable Run adopts its branch after a retry/reclaim instead of
+        // producing a second randomly named branch/PR on a fresh process.
+        workBranch: `bivy/run-${createHash('sha256').update(item.id).digest('hex').slice(0, 24)}`,
         title: item.title,
         credentialLabels,
         runtimeId: item.runtimeId,
@@ -5210,6 +5213,7 @@ function startControlPlaneTasksIfConfigured() {
     // limit, park quota/auth/context); user-authored rulesets can add fallback
     // chains. Queue runs are unattended, so they act automatically within bounds.
     controlPlanePoller = new ControlPlaneTaskPoller(cfg, runWorkItem, nodeGithubMaxConcurrent, {
+        resultDirectory: path.join(appDir, 'work-results'),
         policy: (item) => {
             // Repository policy is version-controlled with the code and wins over the
             // node-global UI ruleset for this run. The shared clone exists by the time
@@ -8403,7 +8407,7 @@ async function createGitWorkspaceSession(repoDir, parsed, opts = {}) {
     // Start from an opaque, git-safe unique branch. The first user message then
     // triggers sessionNamer.maybeNameSession(), which renames both the session and local branch
     // before the first publish/PR attempt.
-    const branch = `bivy/session-${randomBytes(6).toString("hex")}`;
+    const branch = opts.workBranch ?? `bivy/session-${randomBytes(6).toString("hex")}`;
     const record = await createSession(repoDir, undefined, {
         worktree: { branch, base },
         source: `repo:${parsed.slug}`,

package/dist/work-result-outbox.js

@@ -0,0 +1,55 @@
+// SPDX-License-Identifier: AGPL-3.0-only
+import { closeSync, existsSync, fsyncSync, mkdirSync, openSync, readFileSync, renameSync, unlinkSync, writeFileSync } from "node:fs";
+import { createHash } from "node:crypto";
+import path from "node:path";
+/** Metadata only. Persist before delivery so restarting the daemon retries the
+ * acknowledgement, not the agent. Scope files to the enrolled control plane. */
+export class WorkResultOutbox {
+    pending = new Map();
+    file;
+    constructor(directory, scope = "") {
+        if (!directory)
+            return;
+        mkdirSync(directory, { recursive: true, mode: 0o700 });
+        this.file = path.join(directory, `${createHash("sha256").update(scope).digest("hex")}.json`);
+        if (existsSync(this.file)) {
+            const rows = JSON.parse(readFileSync(this.file, "utf8"));
+            if (!Array.isArray(rows) || rows.some(row => !row || typeof row.id !== 'string' || !['complete', 'fail', 'needs-attention'].includes(row.action)))
+                throw new Error('Invalid work result outbox; refusing to replay work');
+            for (const row of rows)
+                this.pending.set(row.id, row);
+        }
+    }
+    list() { return [...this.pending.values()]; }
+    has(id) { return this.pending.has(id); }
+    put(result) { this.pending.set(result.id, result); this.flush(); }
+    remove(id) { this.pending.delete(id); this.flush(); }
+    flush() {
+        if (!this.file)
+            return;
+        const temp = `${this.file}.tmp`;
+        const fd = openSync(temp, 'w', 0o600);
+        try {
+            writeFileSync(fd, JSON.stringify(this.list()));
+            fsyncSync(fd);
+        }
+        finally {
+            closeSync(fd);
+        }
+        try {
+            renameSync(temp, this.file);
+        }
+        catch (error) {
+            if (existsSync(temp))
+                unlinkSync(temp);
+            throw error;
+        }
+        const dir = openSync(path.dirname(this.file), 'r');
+        try {
+            fsyncSync(dir);
+        }
+        finally {
+            closeSync(dir);
+        }
+    }
+}

package/dist/worktree.js

@@ -52,7 +52,7 @@ function excludeMeshDir(repoRoot) {
     }
 }
 /**
- * Create a worktree for `repoDir` on a new branch. The worktree lives under
+ * Create or recover a worktree for `repoDir`, preserving existing work. It lives under
  * `<repoRoot>/.bivy/worktrees/<slug>` by default (excluded from git).
  */
 export async function createWorktree(opts) {
@@ -63,41 +63,38 @@ export async function createWorktree(opts) {
     const branch = opts.branch ?? `bivy/${slug}`;
     const root = opts.root ?? path.join(repoRoot, ".bivy", "worktrees");
     const wtPath = path.join(root, slug);
-    const base = opts.base ?? (await currentRef(repoRoot));
+    let base = opts.base ?? (await currentRef(repoRoot));
     excludeMeshDir(repoRoot);
     fs.mkdirSync(root, { recursive: true });
-    try {
-        await exec("git", ["-C", repoRoot, "worktree", "add", "-b", branch, wtPath, base]);
+    // Reuse the registered checkout in place. Recovery must not force-remove
+    // staged, unstaged, untracked or ignored work left by the previous attempt.
+    // Use Git's registry rather than merely accepting an existing directory.
+    const { stdout: registered } = await exec("git", ["-C", repoRoot, "worktree", "list", "--porcelain", "-z"]);
+    const existing = registered.split("\0\0").map(entry => entry.split("\0"))
+        .find(fields => fields.includes(`worktree ${wtPath}`));
+    if (existing && fs.existsSync(wtPath)) {
+        if (!existing.includes(`branch refs/heads/${branch}`))
+            throw new Error(`Worktree ${wtPath} is not on expected branch ${branch}`);
+        return { path: wtPath, branch, repoRoot };
     }
-    catch (error) {
-        // The branch already exists (e.g. a GitHub-issue pickup whose branch was
-        // pushed on an earlier run, now re-triggered after the session closed). Adopt
-        // it instead of hard-failing, which previously bubbled up and silently marked
-        // the work item "done" with nothing happening. Clear any stale worktree dir
-        // first, then check the existing branch out into a fresh worktree.
-        const localExists = await refExists(repoRoot, `refs/heads/${branch}`);
-        const remoteExists = !localExists && (await refExists(repoRoot, `refs/remotes/origin/${branch}`));
-        if (localExists || remoteExists) {
-            await removeWorktree(repoRoot, wtPath);
-            fs.rmSync(wtPath, { recursive: true, force: true });
-            if (localExists) {
-                await exec("git", ["-C", repoRoot, "worktree", "add", wtPath, branch]);
-            }
-            else {
-                // Recreate the local branch from origin, then check it out in the worktree.
-                await exec("git", ["-C", repoRoot, "worktree", "add", "-b", branch, wtPath, `origin/${branch}`]);
-            }
-        }
-        else if (base !== "HEAD" && !(await refExists(repoRoot, base))) {
-            // Defense in depth for stale internal metadata: callers should resolve a
-            // fork base first, but a missing ref must not prevent session stand-up.
-            await removeWorktree(repoRoot, wtPath);
-            fs.rmSync(wtPath, { recursive: true, force: true });
-            await exec("git", ["-C", repoRoot, "worktree", "add", "-b", branch, wtPath, "HEAD"]);
-        }
-        else {
-            throw error;
-        }
+    // A manually reaped directory can leave a stale registration. Remove only
+    // that registration, without --force (locked or newly dirty work stays safe).
+    if (existing)
+        await exec("git", ["-C", repoRoot, "worktree", "remove", wtPath]);
+    // A remote-only branch does NOT make `worktree add -b` fail. Resolve it
+    // before creation, otherwise a fresh Machine silently starts at default HEAD.
+    const localExists = await refExists(repoRoot, `refs/heads/${branch}`);
+    const remoteExists = !localExists && await refExists(repoRoot, `refs/remotes/origin/${branch}`);
+    if (localExists) {
+        await exec("git", ["-C", repoRoot, "worktree", "add", wtPath, branch]);
+    }
+    else {
+        if (remoteExists)
+            base = `refs/remotes/origin/${branch}`;
+        else if (base !== "HEAD" && !(await refExists(repoRoot, base)))
+            base = "HEAD";
+        // Fail closed on path collisions; never delete an unrelated directory.
+        await exec("git", ["-C", repoRoot, "worktree", "add", "-b", branch, wtPath, base]);
     }
     // Opportunistically reuse a sibling worktree's installed deps (node_modules,
     // target, .venv) via copy-on-write when the filesystem supports it, so a

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@bivy/bivy",
-  "version": "0.16.19-staging.2",
+  "version": "0.16.19-staging.4",
   "type": "module",
   "license": "AGPL-3.0-only",
   "description": "Run coding agents on machines you own. Open-source, self-hostable agent workspace.",