agent-relay-orchestrator 0.78.5 → 0.78.7

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "agent-relay-orchestrator",
-  "version": "0.78.5",
+  "version": "0.78.7",
   "description": "Agent Relay orchestrator — manages agent lifecycle across hosts",
   "type": "module",
   "bin": {
@@ -16,7 +16,7 @@
     "test": "bun test"
   },
   "dependencies": {
-    "agent-relay-sdk": "0.2.56"
+    "agent-relay-sdk": "0.2.57"
   },
   "devDependencies": {
     "@types/bun": "latest",

package/src/control.ts

@@ -95,6 +95,8 @@ export function createControlHandler(
           repoRoot: typeof command.params.repoRoot === "string" ? command.params.repoRoot : undefined,
           worktreePath: typeof command.params.worktreePath === "string" ? command.params.worktreePath : undefined,
           branch: typeof command.params.branch === "string" ? command.params.branch : undefined,
+          baseRef: typeof command.params.baseRef === "string" ? command.params.baseRef : undefined,
+          baseSha: typeof command.params.baseSha === "string" ? command.params.baseSha : undefined,
           deleteBranch: command.params.deleteBranch !== false,
           workspacesRoot: workspacesRoot(config.baseDir),
         });

package/src/quota-poller.ts

@@ -25,6 +25,8 @@ import { codexCommandFromEnv, providerHomeRootFromEnv, type OrchestratorConfig }
 
 const QUOTA_LEASE_TTL_MS = 90_000;
 const QUOTA_LEASE_RENEW_MS = 30_000;
+const QUOTA_RETRY_BACKOFF_MAX_MS = 15 * 60_000;
+const QUOTA_RETRY_BACKOFF_MIN_MS = 1_000;
 const CODEX_APP_SERVER_CONNECT_ATTEMPTS = 40;
 const CODEX_APP_SERVER_CONNECT_RETRY_MS = 250;
 
@@ -62,6 +64,7 @@ type QuotaPollState = {
   leaseExpiresAt?: number;
   nextPollAt?: number;
   lastAttemptAt?: number;
+  consecutiveFailures?: number;
   lastLog?: { key: string; at: number };
 };
 
@@ -115,14 +118,18 @@ export class OrchestratorQuotaPoller {
       const { candidates, skips } = await this.discoverCandidates();
       await this.releaseRemovedCandidates(candidates);
       for (const candidate of candidates) {
-        await this.processCandidate(candidate);
+        try {
+          await this.processCandidate(candidate);
+        } catch (error) {
+          await this.handleCandidateFailure(candidate, error);
+        }
       }
       for (const skip of skips) {
         await this.reportSkip(skip);
       }
     } finally {
       this.inFlight = false;
-      this.schedule(this.options.intervalMs ?? QUOTA_LEASE_RENEW_MS);
+      this.schedule(this.nextScheduleDelay(this.options.intervalMs ?? QUOTA_LEASE_RENEW_MS));
     }
   }
 
@@ -136,6 +143,18 @@ export class OrchestratorQuotaPoller {
     }, Math.max(1_000, delayMs));
   }
 
+  private nextScheduleDelay(defaultDelayMs: number): number {
+    const now = this.now();
+    let delayMs = defaultDelayMs;
+    for (const state of this.states.values()) {
+      if (state.nextPollAt !== undefined) delayMs = Math.min(delayMs, state.nextPollAt - now);
+      if (state.leaseToken && state.leaseExpiresAt !== undefined) {
+        delayMs = Math.min(delayMs, state.leaseExpiresAt - now - QUOTA_LEASE_RENEW_MS);
+      }
+    }
+    return Math.max(QUOTA_RETRY_BACKOFF_MIN_MS, delayMs);
+  }
+
   // Refresh per-provider quota config (#605). Best-effort: on failure we keep the
   // last known config (defaults for any unset provider), so a transient relay blip
   // never silently stops collection.
@@ -270,16 +289,14 @@ export class OrchestratorQuotaPoller {
       await this.relay.reportProviderQuota(update);
       state.lastAttemptAt = update.lastAttemptAt;
       state.nextPollAt = now + pollIntervalMs;
+      state.consecutiveFailures = 0;
     } catch (error) {
       const retryAfterMs = quotaRetryAfterMs(error);
       const lastError = providerQuotaErrorFromCollectorError(error, retryAfterMs);
-      const retryDelayMs = retryAfterMs ?? (state.lastAttemptAt ? pollIntervalMs : QUOTA_FAST_RETRY_MS);
+      const retryDelayMs = this.retryDelayMs(state, retryAfterMs);
       state.lastAttemptAt = lastAttemptAt;
       state.nextPollAt = now + retryDelayMs;
-      if (candidate.provider === "claude" && retryAfterMs !== undefined) {
-        this.logFailure(candidate, error, retryAfterMs);
-        return;
-      }
+      state.consecutiveFailures = (state.consecutiveFailures ?? 0) + 1;
       await this.relay.reportProviderQuota({
         provider: candidate.provider,
         accountKey: candidate.accountKey,
@@ -287,10 +304,37 @@ export class OrchestratorQuotaPoller {
         lastError,
         sourceAgentId: this.sourceAgentId(),
       }).catch((publishError) => this.log(`quota status publish failed: ${errMessage(publishError)}`));
-      this.logFailure(candidate, error, retryAfterMs);
+      this.logFailure(candidate, error, retryAfterMs, retryDelayMs);
     }
   }
 
+  private async handleCandidateFailure(candidate: QuotaCandidate, error: unknown): Promise<void> {
+    const state = this.stateFor(candidate);
+    const now = this.now();
+    const retryAfterMs = quotaRetryAfterMs(error);
+    const retryDelayMs = this.retryDelayMs(state, retryAfterMs);
+    state.lastAttemptAt = now;
+    state.nextPollAt = now + retryDelayMs;
+    state.consecutiveFailures = (state.consecutiveFailures ?? 0) + 1;
+    await this.relay.reportProviderQuota({
+      provider: candidate.provider,
+      accountKey: candidate.accountKey,
+      lastAttemptAt: now,
+      lastError: providerQuotaErrorFromCollectorError(error, retryAfterMs),
+      sourceAgentId: this.sourceAgentId(),
+    }).catch((publishError) => this.log(`quota status publish failed: ${errMessage(publishError)}`));
+    this.logFailure(candidate, error, retryAfterMs, retryDelayMs);
+  }
+
+  private retryDelayMs(state: QuotaPollState, retryAfterMs: number | undefined): number {
+    const baseDelayMs = retryAfterMs ?? QUOTA_FAST_RETRY_MS;
+    const multiplier = 2 ** Math.min(state.consecutiveFailures ?? 0, 10);
+    return Math.min(
+      QUOTA_RETRY_BACKOFF_MAX_MS,
+      Math.max(QUOTA_RETRY_BACKOFF_MIN_MS, Math.round(baseDelayMs * multiplier)),
+    );
+  }
+
   private async ensureLease(candidate: QuotaCandidate, state: QuotaPollState, now: number): Promise<boolean> {
     if (state.leaseToken && state.leaseExpiresAt && state.leaseExpiresAt - now > QUOTA_LEASE_RENEW_MS) return true;
     const result = await this.relay.acquireProviderQuotaLease(this.config.id, {
@@ -353,13 +397,13 @@ export class OrchestratorQuotaPoller {
     (this.options.log ?? ((line) => console.error(`[orchestrator] ${line}`)))(message);
   }
 
-  private logFailure(candidate: QuotaCandidate, error: unknown, retryAfterMs: number | undefined): void {
+  private logFailure(candidate: QuotaCandidate, error: unknown, retryAfterMs: number | undefined, retryDelayMs: number): void {
     const state = this.stateFor(candidate);
-    const key = retryAfterMs !== undefined ? `retry-after:${retryAfterMs}` : errMessage(error);
+    const key = retryAfterMs !== undefined ? `retry-after:${retryAfterMs}:delay:${retryDelayMs}` : `${errMessage(error)}:delay:${retryDelayMs}`;
     const now = this.now();
     if (state.lastLog?.key === key && now - state.lastLog.at < QUOTA_FAILURE_LOG_INTERVAL_MS) return;
     state.lastLog = { key, at: now };
-    const suffix = retryAfterMs !== undefined ? `; retrying in ${Math.round(retryAfterMs / 1000)}s` : "";
+    const suffix = `; retrying in ${Math.round(retryDelayMs / 1000)}s`;
     this.log(`quota refresh failed for ${candidate.provider}/${candidate.accountKey}${suffix}: ${errMessage(error)}`);
   }
 

package/src/workspace-probe/cleanup.ts

@@ -22,7 +22,7 @@ function owningRepoRoot(worktreePath: string, fallback: string): string {
   return existsSync(root) ? root : fallback;
 }
 
-export function cleanupWorkspace(workspace: { repoRoot?: string; worktreePath?: string; id?: string; branch?: string; deleteBranch?: boolean; workspacesRoot?: string }): { workspaceId?: string; removed: boolean; worktreePath?: string; branchDeleted?: boolean; containerRemoved?: boolean } {
+export function cleanupWorkspace(workspace: { repoRoot?: string; worktreePath?: string; id?: string; branch?: string; baseRef?: string; baseSha?: string; deleteBranch?: boolean; workspacesRoot?: string }): { workspaceId?: string; removed: boolean; worktreePath?: string; branchDeleted?: boolean; branchPreservedReason?: string; containerRemoved?: boolean } {
   if (!workspace.worktreePath) throw new Error("worktreePath required");
   const path = resolve(workspace.worktreePath);
   const recordedRepo = workspace.repoRoot ? resolve(workspace.repoRoot) : path;
@@ -38,14 +38,61 @@ export function cleanupWorkspace(workspace: { repoRoot?: string; worktreePath?:
     throw new Error(result.stderr || `worktree ${path} still present after \`git worktree remove\` (repo ${repo})`);
   }
   // Once the worktree is gone the agent/... branch is litter — delete it so
-  // branches don't accumulate. Best-effort: don't fail cleanup if it can't
-  // (e.g. branch already gone, or checked out elsewhere).
+  // branches don't accumulate. First prove it has no unlanded commits, because
+  // deleting the branch ref is what can make committed-but-unlanded work
+  // unreachable after a crashed worker (#614).
   let branchDeleted = false;
+  let branchPreservedReason: string | undefined;
   if (workspace.branch && workspace.deleteBranch !== false) {
-    branchDeleted = git(["branch", "-D", workspace.branch], repo).ok;
+    const safety = branchSafeToDelete(repo, workspace.branch, workspace.baseRef, workspace.baseSha);
+    if (safety.safe) {
+      branchDeleted = git(["branch", "-D", workspace.branch], repo).ok;
+      if (!branchDeleted) branchPreservedReason = "branch delete failed";
+    } else {
+      branchPreservedReason = safety.reason;
+    }
   }
   const containerRemoved = workspace.workspacesRoot ? removeEmptyContainer(dirname(path), resolve(workspace.workspacesRoot)) : false;
-  return { workspaceId: workspace.id, removed: true, worktreePath: path, branchDeleted, containerRemoved };
+  return { workspaceId: workspace.id, removed: true, worktreePath: path, branchDeleted, ...(branchPreservedReason ? { branchPreservedReason } : {}), containerRemoved };
+}
+
+function branchSafeToDelete(repo: string, branch: string, baseRef?: string, baseSha?: string): { safe: boolean; reason?: string } {
+  const branchRef = resolveCommit(repo, branch) ? branch : resolveCommit(repo, `refs/heads/${branch}`) ? `refs/heads/${branch}` : undefined;
+  if (!branchRef) return { safe: true };
+
+  const base = resolveCleanupBase(repo, baseRef, baseSha);
+  if (!base) return { safe: false, reason: "base ref unavailable" };
+
+  const counts = git(["rev-list", "--left-right", "--count", `${base}...${branchRef}`], repo);
+  if (!counts.ok || !counts.stdout) return { safe: false, reason: counts.stderr || "ahead count unavailable" };
+  const ahead = Number(counts.stdout.split(/\s+/)[1]);
+  if (!Number.isFinite(ahead)) return { safe: false, reason: "ahead count unavailable" };
+  if (ahead === 0) return { safe: true };
+
+  const cherryBase = upstreamRef(repo, base) ?? base;
+  if (git(["diff", "--quiet", cherryBase, branchRef], repo).ok) return { safe: true };
+  const cherry = git(["cherry", cherryBase, branchRef], repo);
+  if (!cherry.ok) return { safe: false, reason: cherry.stderr || "unmerged commit check unavailable" };
+  const unmergedAhead = cherry.stdout ? cherry.stdout.split("\n").filter((line) => line.startsWith("+")).length : 0;
+  return unmergedAhead === 0
+    ? { safe: true }
+    : { safe: false, reason: `${unmergedAhead} unlanded commit(s)` };
+}
+
+function resolveCleanupBase(repo: string, baseRef?: string, baseSha?: string): string | undefined {
+  for (const candidate of [baseRef, baseSha, "main", "master"]) {
+    if (candidate && resolveCommit(repo, candidate)) return candidate;
+  }
+  return undefined;
+}
+
+function resolveCommit(repo: string, ref: string): boolean {
+  return git(["rev-parse", "--verify", "--quiet", `${ref}^{commit}`], repo).ok;
+}
+
+function upstreamRef(repo: string, base: string): string | undefined {
+  const res = git(["rev-parse", "--abbrev-ref", `${base}@{upstream}`], repo);
+  return res.ok && res.stdout ? res.stdout : undefined;
 }
 
 export function sweepEmptyWorkspaceContainers(wsRoot: string): string[] {

package/src/workspace-probe/git-state.ts

@@ -138,7 +138,7 @@ export function reconcileWorkspace(workspace: { id?: string; repoRoot?: string;
   // detection can only under-report, so this never deletes unmerged work.
   const empty = gitState.error === undefined && gitState.dirtyCount === 0 && ((gitState.ahead ?? 0) === 0 || gitState.landed === true);
   if (empty) {
-    cleanupWorkspace({ id: workspace.id, repoRoot: workspace.repoRoot, worktreePath: workspace.worktreePath, branch: workspace.branch });
+    cleanupWorkspace({ id: workspace.id, repoRoot: workspace.repoRoot, worktreePath: workspace.worktreePath, branch: workspace.branch, baseRef: workspace.baseRef, baseSha: workspace.baseSha });
     return { workspaceId: workspace.id, removed: true, status: "cleaned", gitState };
   }
   return { workspaceId: workspace.id, removed: false, status: "review_requested", gitState };