agent-relay-orchestrator 0.129.10 → 0.129.12

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (47) hide show
  1. package/package.json +3 -3
  2. package/src/api.ts +20 -6
  3. package/src/async-guard.ts +42 -0
  4. package/src/command-poller.ts +2 -1
  5. package/src/command-result-report.ts +32 -0
  6. package/src/config.ts +47 -0
  7. package/src/control.ts +63 -13
  8. package/src/git.ts +6 -0
  9. package/src/host-admission.ts +426 -0
  10. package/src/index.ts +58 -29
  11. package/src/maintenance.ts +78 -24
  12. package/src/process.ts +83 -9
  13. package/src/provider-probe.ts +73 -6
  14. package/src/quota-poller.ts +2 -1
  15. package/src/registration.ts +144 -0
  16. package/src/relay.ts +322 -27
  17. package/src/repo-lock.ts +3 -2
  18. package/src/self-supervision.ts +8 -7
  19. package/src/self-upgrade-guard.ts +1717 -0
  20. package/src/self-upgrade.ts +451 -170
  21. package/src/shared-callmux.ts +47 -6
  22. package/src/spawn/command.ts +5 -0
  23. package/src/spawn/constants.ts +18 -1
  24. package/src/spawn/log-utils.ts +21 -0
  25. package/src/spawn/runtime.ts +440 -8
  26. package/src/spawn/sessions.ts +103 -22
  27. package/src/spawn/spawn-agent.ts +19 -1
  28. package/src/spawn/supervisor.ts +63 -8
  29. package/src/spawn/systemd.ts +15 -2
  30. package/src/spawn/terminal.ts +3 -4
  31. package/src/spawn/types.ts +6 -0
  32. package/src/terminal-stream.ts +12 -10
  33. package/src/wedged-session-reaper.ts +804 -0
  34. package/src/workspace-pr.ts +13 -0
  35. package/src/workspace-probe/base-sync.ts +408 -0
  36. package/src/workspace-probe/cleanup.ts +29 -29
  37. package/src/workspace-probe/content-landed.ts +108 -0
  38. package/src/workspace-probe/git-state.ts +90 -17
  39. package/src/workspace-probe/idle-refresh.ts +3 -1
  40. package/src/workspace-probe/index.ts +1 -0
  41. package/src/workspace-probe/integrated-land-gates.ts +758 -32
  42. package/src/workspace-probe/merge.ts +173 -243
  43. package/src/workspace-probe/names.ts +18 -0
  44. package/src/workspace-probe/plain-git.ts +96 -21
  45. package/src/workspace-probe/protected-tip.ts +57 -7
  46. package/src/workspace-probe/review-ref.ts +363 -0
  47. package/src/workspace-probe/types.ts +3 -0
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "agent-relay-orchestrator",
3
- "version": "0.129.10",
3
+ "version": "0.129.12",
4
4
  "description": "Agent Relay orchestrator — manages agent lifecycle across hosts",
5
5
  "type": "module",
6
6
  "bin": {
@@ -16,8 +16,8 @@
16
16
  "test": "bun test"
17
17
  },
18
18
  "dependencies": {
19
- "agent-relay-providers": "0.104.5",
20
- "agent-relay-sdk": "0.2.128",
19
+ "agent-relay-providers": "0.104.6",
20
+ "agent-relay-sdk": "0.2.130",
21
21
  "callmux": "0.24.2"
22
22
  },
23
23
  "devDependencies": {
package/src/api.ts CHANGED
@@ -3,8 +3,10 @@ import { errMessage, stringValue } from "agent-relay-sdk";
3
3
  import { basename, dirname, extname, isAbsolute, join, relative, resolve } from "node:path";
4
4
  import type { ServerWebSocket } from "bun";
5
5
  import { proxyArtifactRequest } from "./artifact-proxy";
6
+ import { fireAndForget } from "./async-guard";
6
7
  import type { OrchestratorConfig } from "./config";
7
8
  import type { ProviderProbeCache } from "./provider-probe";
9
+ import type { RegistrationDriver } from "./registration";
8
10
  import type { RelayClient } from "./relay";
9
11
  import { captureSession, captureSessionMirror, captureTerminal, createTerminalGuest, listSessions, sendTerminalInput, resizeTerminal, stopTerminalGuest, validateTerminalInputData, validateTerminalResize } from "./spawn";
10
12
  import { acquireTerminalStream, type TerminalStreamHandle, type TerminalStreamSubscriber } from "./terminal-stream";
@@ -390,7 +392,7 @@ export interface CodexResetConsumer {
390
392
  }>;
391
393
  }
392
394
 
393
- export function startApiServer(config: OrchestratorConfig, probeCache: ProviderProbeCache, relay?: RelayClient, codexResetConsumer?: CodexResetConsumer): { stop(): void; url: string } {
395
+ export function startApiServer(config: OrchestratorConfig, probeCache: ProviderProbeCache, relay?: RelayClient, codexResetConsumer?: CodexResetConsumer, registration?: Pick<RegistrationDriver, "getHealth">): { stop(): void; url: string } {
394
396
  const server = Bun.serve<TerminalSocketData>({
395
397
  port: config.apiPort,
396
398
  hostname: "0.0.0.0",
@@ -669,7 +671,19 @@ export function startApiServer(config: OrchestratorConfig, probeCache: ProviderP
669
671
  }
670
672
 
671
673
  if (req.method === "GET" && url.pathname === "/api/health") {
672
- return json({ status: "ok", ok: true, id: config.id, hostname: config.hostname });
674
+ // #1676 surface relay-peer and registration degradation here so a host stuck
675
+ // re-registering is distinguishable from a healthy one WITHOUT log archaeology
676
+ // (previously it looked identical, and spawns routed to it burned their full
677
+ // timeout). `status`/`ok` deliberately stay "ok": this probe is the
678
+ // Relay-INDEPENDENT local readiness signal the self-upgrade guard gates on, and a
679
+ // degraded peer must not make this process look un-upgradable. Read `degraded`.
680
+ const peer = relay?.getHealth();
681
+ const reg = registration?.getHealth();
682
+ return json({
683
+ status: "ok", ok: true, id: config.id, hostname: config.hostname,
684
+ degraded: Boolean(peer?.degraded || reg?.degraded),
685
+ ...(peer ? { relay: peer } : {}), ...(reg ? { registration: reg } : {}),
686
+ });
673
687
  }
674
688
 
675
689
  return error("Not found", 404);
@@ -739,7 +753,7 @@ function startTerminalSocket(ws: TerminalSocket): void {
739
753
  // Wait for the client's resize to size the pane before backfilling. Fall back to
740
754
  // the pane's current size if no resize arrives (e.g. a non-fitting client).
741
755
  ws.data.syncTimer = setTimeout(() => {
742
- if (!ws.data.synced) void syncAndBackfill(ws);
756
+ if (!ws.data.synced) void fireAndForget("Terminal socket sync/backfill", () => syncAndBackfill(ws));
743
757
  }, 700);
744
758
  } catch (e) {
745
759
  ws.send(JSON.stringify({ type: "error", error: errMessage(e) }));
@@ -826,7 +840,7 @@ function handleTerminalSocketMessage(ws: TerminalSocket, data: string | Buffer):
826
840
  // First resize sizes the pane and triggers the (size-matched) backfill;
827
841
  // later ones just reflow the live stream.
828
842
  if (!ws.data.synced) {
829
- void syncAndBackfill(ws, cols, rows);
843
+ void fireAndForget("Terminal socket sync/backfill", () => syncAndBackfill(ws, cols, rows));
830
844
  } else {
831
845
  ws.data.stream?.resize(cols, rows);
832
846
  }
@@ -840,10 +854,10 @@ function handleTerminalSocketMessage(ws: TerminalSocket, data: string | Buffer):
840
854
  if (wasPaused && !ws.data.paused && ws.data.synced) {
841
855
  const dropped = ws.data.droppedWhilePaused ?? 0;
842
856
  ws.data.droppedWhilePaused = 0;
843
- if (dropped > 0) void sendBackfill(ws, undefined, undefined, true);
857
+ if (dropped > 0) void fireAndForget("Terminal socket backfill", () => sendBackfill(ws, undefined, undefined, true));
844
858
  }
845
859
  } else if (frame.type === "refresh") {
846
- if (ws.data.synced) void sendBackfill(ws, undefined, undefined, true);
860
+ if (ws.data.synced) void fireAndForget("Terminal socket backfill", () => sendBackfill(ws, undefined, undefined, true));
847
861
  } else if (frame.type === "interactive") {
848
862
  // Read-only watchers must not reflow the shared tmux window (#273); the typist owns
849
863
  // sizing. The client reports its interactivity so the stream knows who may resize.
@@ -0,0 +1,42 @@
1
+ // #1676 — orchestrator adapter over the SHARED async guards in `agent-relay-sdk`.
2
+ //
3
+ // The primitives themselves live in `sdk/src/async-guard.ts` because #1665 is the same
4
+ // defect class in the runner: an unguarded `void <promise>` / async timer callback whose
5
+ // rejection reaches the runtime and (in Bun) exits the process. One implementation, two
6
+ // processes. Import from `agent-relay-sdk` directly if you are outside this package.
7
+ //
8
+ // All this file adds is the `[orchestrator]` log prefix, so the daemon's log format is
9
+ // unchanged and call sites pass a bare label ("Relay heartbeat", not the prefix).
10
+
11
+ import {
12
+ fireAndForget as sdkFireAndForget,
13
+ guardedInterval as sdkGuardedInterval,
14
+ guardedTimeout as sdkGuardedTimeout,
15
+ installUnhandledRejectionGuard as sdkInstallUnhandledRejectionGuard,
16
+ type GuardLog,
17
+ } from "agent-relay-sdk";
18
+
19
+ export { describeFailure, isAbortError, type GuardLog } from "agent-relay-sdk";
20
+
21
+ const PREFIX = "[orchestrator]";
22
+
23
+ /** See {@link import("agent-relay-sdk").fireAndForget}. Replaces bare `void promise()`. */
24
+ export function fireAndForget(label: string, work: Promise<unknown> | (() => Promise<unknown> | unknown), log?: GuardLog): Promise<void> {
25
+ return sdkFireAndForget(`${PREFIX} ${label}`, work, log);
26
+ }
27
+
28
+ /** See {@link import("agent-relay-sdk").guardedInterval}. Replaces `setInterval(asyncFn, ms)`. */
29
+ export function guardedInterval(label: string, intervalMs: number, fn: () => Promise<unknown> | unknown, log?: GuardLog): ReturnType<typeof setInterval> {
30
+ return sdkGuardedInterval(`${PREFIX} ${label}`, intervalMs, fn, log);
31
+ }
32
+
33
+ /** See {@link import("agent-relay-sdk").guardedTimeout}. Replaces `setTimeout(asyncFn, ms)`. */
34
+ export function guardedTimeout(label: string, delayMs: number, fn: () => Promise<unknown> | unknown, log?: GuardLog): ReturnType<typeof setTimeout> {
35
+ return sdkGuardedTimeout(`${PREFIX} ${label}`, delayMs, fn, log);
36
+ }
37
+
38
+ /** Install the non-fatal unhandled-rejection backstop. See the SDK function for the
39
+ * reasoning behind "log and survive" and why `uncaughtException` stays fatal. */
40
+ export function installUnhandledRejectionGuard(log?: GuardLog): () => void {
41
+ return sdkInstallUnhandledRejectionGuard(PREFIX, log);
42
+ }
@@ -1,3 +1,4 @@
1
+ import { fireAndForget } from "./async-guard";
1
2
  import type { RelayClient, RelayCommand } from "./relay";
2
3
 
3
4
  interface CommandPollerControl {
@@ -79,7 +80,7 @@ export function createCommandPoller({
79
80
  if (stopped) return;
80
81
  timer = setTimeout(() => {
81
82
  timer = undefined;
82
- void runCycle(generation);
83
+ void fireAndForget("Command poll cycle", () => runCycle(generation), log);
83
84
  }, delayMs);
84
85
  timer.unref?.();
85
86
  }
@@ -0,0 +1,32 @@
1
+ import type { WorkspaceMergeResult } from "agent-relay-sdk";
2
+
3
+ // #638 — settle a `workspace.merge` command on whether it made progress, not on whether
4
+ // the merge function threw. Every no-progress outcome (origin moved ahead, an unpredicted
5
+ // replay conflict, a push race) returns merged:false with an `error` set; every genuine
6
+ // land / recycle / noop-resolve / PR-opened leaves `error` undefined. Reporting `succeeded`
7
+ // for an unlanded branch is what let the auto-merge job believe it was progressing and
8
+ // busy-loop the repo's merge lease. Mirrors the pr-merge / pr-arm settle contract.
9
+ export function mergeCommandStatus(result: WorkspaceMergeResult): "succeeded" | "failed" {
10
+ return result.error ? "failed" : "succeeded";
11
+ }
12
+
13
+ // Command PATCH bodies are capped at 64 KiB by the relay. A failing full land gate
14
+ // may retain 256 KiB of output for artifact upload, so forwarding it verbatim can
15
+ // reject the *terminal* update and leave expiry to misreport a red gate as a timeout.
16
+ // Keep ordinary output intact; when it will not fit, retain the bounded tail that is
17
+ // already intended for the worker/coordinator notification.
18
+ const MAX_COMMAND_REPORT_CHARS = 60 * 1024;
19
+
20
+ export function prepareWorkspaceMergeResultForReport(result: WorkspaceMergeResult): WorkspaceMergeResult {
21
+ const gateFailure = result.gateFailure;
22
+ if (!gateFailure?.output) return result;
23
+ const body = JSON.stringify({ status: mergeCommandStatus(result), result, error: result.error });
24
+ if (body.length <= MAX_COMMAND_REPORT_CHARS) return result;
25
+ return {
26
+ ...result,
27
+ gateFailure: {
28
+ ...gateFailure,
29
+ output: gateFailure.outputTail,
30
+ },
31
+ };
32
+ }
package/src/config.ts CHANGED
@@ -32,6 +32,34 @@ export function gitShaFromEnv(): string | undefined {
32
32
  return process.env.AGENT_RELAY_GIT_SHA || process.env.GIT_SHA || undefined;
33
33
  }
34
34
 
35
+ // #1452 — host-side gate for REVIEW-REF PUBLISHING (the origin-mutating half). The host must
36
+ // decide this itself and NOT trust the relay/caller: a queued, replayed, or hand-crafted
37
+ // publish-review command must NOT mutate origin unless THIS host has the feature enabled. The
38
+ // operator sets AGENT_RELAY_REVIEW_PUBLISH=1 on both the relay AND the orchestrator to activate;
39
+ // unset here ⇒ the host no-ops every publish (fail-safe OFF). UNPUBLISH is deliberately NOT
40
+ // gated on this — a ref created during a prior ON era must still be cleanable after it flips OFF.
41
+ export function reviewPublishEnabled(): boolean {
42
+ return process.env.AGENT_RELAY_REVIEW_PUBLISH === "1";
43
+ }
44
+
45
+ // #1452 round-3 (H5) — a hard ceiling on every review-ref git op that touches `origin`. Short
46
+ // enough that a dead/hung remote fails fast instead of freezing the teardown/land/command loop that
47
+ // awaits it, generous enough for a real push over a slow link. Clamped to a sane minimum.
48
+ //
49
+ // #1452 round-12 — the ceiling is now LOAD-BEARING, not just a fail-fast bound. The push fence
50
+ // (fencePublishForPush) refreshes the relay's publish-command lease (WORKSPACE_COMMAND_TTL_MS = 40 min)
51
+ // immediately before the push, and its correctness relies on the git-timeout-bounded push completing
52
+ // WITHIN that fresh lease so the TTL sweep can never fire between the fence and the push. Without an
53
+ // UPPER clamp, an operator misconfig (AGENT_RELAY_REVIEW_REF_GIT_TIMEOUT_MS above the lease) would let a
54
+ // single git op outlast the lease → mid-op timeout → the later push races the reconciler's cleanup
55
+ // (also LOW-1's over-cleanup of a live push). The 5-min ceiling keeps "the lease comfortably exceeds
56
+ // every review-ref git op" a STRUCTURAL invariant — even a handful of sequential pre-push ops finish
57
+ // well inside 40 min. A guard test asserts ceiling << lease so the two constants can't silently drift.
58
+ export const REVIEW_REF_GIT_TIMEOUT_CEILING_MS = 5 * 60_000;
59
+ export function reviewRefGitTimeoutMs(): number {
60
+ return Math.min(REVIEW_REF_GIT_TIMEOUT_CEILING_MS, Math.max(1_000, Number(process.env.AGENT_RELAY_REVIEW_REF_GIT_TIMEOUT_MS) || 15_000));
61
+ }
62
+
35
63
  export function agentRelayHome(): string {
36
64
  return process.env.AGENT_RELAY_HOME || join(homedir(), ".agent-relay");
37
65
  }
@@ -90,6 +118,25 @@ export function tmuxSocketSweepIntervalMs(): number {
90
118
  return envNonNegativeMax("AGENT_RELAY_TMUX_SOCKET_SWEEP_INTERVAL_MS", 10 * 60 * 1000, 60_000);
91
119
  }
92
120
 
121
+ // #1514 — reaper for orphaned, idle managed tmux sessions (e.g. a headless Claude
122
+ // session wedged at the "Exit anyway / Move to background / Stay" dialog). On by
123
+ // default; set AGENT_RELAY_WEDGED_SESSION_REAP=0 to disable.
124
+ export function wedgedSessionReapEnabled(): boolean {
125
+ return process.env.AGENT_RELAY_WEDGED_SESSION_REAP !== "0";
126
+ }
127
+
128
+ export function wedgedSessionReapIntervalMs(): number {
129
+ return envNonNegativeMax("AGENT_RELAY_WEDGED_SESSION_REAP_INTERVAL_MS", 5 * 60 * 1000, 60_000);
130
+ }
131
+
132
+ // A session must show no tmux activity for at least this long before an unmanaged
133
+ // session is reaped. Generous so a brief spawn/registration race (record not yet
134
+ // written) or a momentary state-file gap can never clip a healthy session; a real
135
+ // wedge sits idle for hours/days.
136
+ export function wedgedSessionIdleThresholdMs(): number {
137
+ return envNonNegativeMax("AGENT_RELAY_WEDGED_SESSION_IDLE_THRESHOLD_MS", 30 * 60 * 1000, 5 * 60 * 1000);
138
+ }
139
+
93
140
  export const TERMINAL_FLUSH_MS = envNonNegativeMax("AGENT_RELAY_TERMINAL_FLUSH_MS", 6, 0);
94
141
  export const TERMINAL_FLUSH_MAX_BYTES = envNonNegativeMax("AGENT_RELAY_TERMINAL_FLUSH_MAX_BYTES", 65536, 4096);
95
142
  export const TERMINAL_BACKPRESSURE_MAX_BYTES = envNonNegativeMax("AGENT_RELAY_TERMINAL_BACKPRESSURE_MAX_BYTES", 8 << 20, 1 << 20);
package/src/control.ts CHANGED
@@ -1,26 +1,20 @@
1
1
  import { errMessage, isPathWithinBase, isRecord, normalizeAgentLifecycle, normalizeWorkspaceMode, Semaphore } from "agent-relay-sdk";
2
+ import { fireAndForget } from "./async-guard";
2
3
  import { getAllManifests, getManifest } from "agent-relay-providers";
3
4
  import { SCHEDULER_COMMAND_MAX_CONCURRENT, SCHEDULER_COMMAND_MAX_OUTPUT_BYTES, type OrchestratorConfig } from "./config";
5
+ import { checkHostHeadroom, isHostAdmissionDisabled, queuedSpawnFromCommand, resolveAdmissionMaxWaitMs, resolveAdmissionPollMs, resolveMinFreeMemMb, waitForHostHeadroom, type HostHeadroomResult } from "./host-admission";
4
6
  import type { ManagedAgentReport, RelayClient, RelayCommand } from "./relay";
7
+ import type { QueuedSpawn } from "agent-relay-sdk";
5
8
  import { handleSelfUpgrade } from "./self-upgrade";
6
9
  import { readLocalProviderConfigs } from "./provider-config-migration";
7
10
  import { findSessionRecord, isSessionAlive, readRunnerInfo, spawnAgent, stopSession, type SpawnOptions } from "./spawn";
8
- import { cleanupWorkspace, discardRecoveryBranch, idleRefreshWorktree, mergeWorkspace, mergeWorkspacePlainGit, pruneWorktrees, reconcileWorkspace, refreshWorkspaceDeps, workspacesRoot } from "./workspace-probe";
11
+ import { cleanupWorkspace, discardRecoveryBranch, handleReviewRefCommand, idleRefreshWorktree, mergeWorkspace, mergeWorkspacePlainGit, pruneWorktrees, reconcileWorkspace, refreshWorkspaceDeps, runClaimGatedPublishReview, workspacesRoot } from "./workspace-probe";
9
12
  import { withMergePhaseTimeout } from "./workspace-probe/merge-timeouts";
10
13
  import { withRepoLock } from "./repo-lock";
11
14
  import { armWorkspacePrAutoMerge, mergeWorkspacePr, refreshWorkspacePrBranch } from "./workspace-pr";
12
15
  import type { WorkspaceMergeResult } from "agent-relay-sdk";
13
16
  import { execProcess } from "./process";
14
-
15
- // #638 — settle a `workspace.merge` command on whether it made progress, not on whether
16
- // the merge function threw. Every no-progress outcome (origin moved ahead, an unpredicted
17
- // replay conflict, a push race) returns merged:false with an `error` set; every genuine
18
- // land / recycle / noop-resolve / PR-opened leaves `error` undefined. Reporting `succeeded`
19
- // for an unlanded branch is what let the auto-merge job believe it was progressing and
20
- // busy-loop the repo's merge lease. Mirrors the pr-merge / pr-arm settle contract.
21
- export function mergeCommandStatus(result: WorkspaceMergeResult): "succeeded" | "failed" {
22
- return result.error ? "failed" : "succeeded";
23
- }
17
+ import { mergeCommandStatus, prepareWorkspaceMergeResultForReport } from "./command-result-report";
24
18
 
25
19
  // #1020 (B2) — command types that mutate a physical repo's git refs / worktrees. These take the
26
20
  // per-repoRoot lock (see repo-lock.ts) so they can't interleave with the background spawn's
@@ -145,6 +139,12 @@ interface ControlHandlerDeps {
145
139
  registrationSettleMs?: number;
146
140
  // #1201 F2 — injectable safe-point re-check. Defaults to relay.recheckSelfRestartSafePoint.
147
141
  recheckSafePoint?: (agentId: string) => Promise<{ ok: boolean; reason?: string }>;
142
+ // #893 — injectable host headroom check. Defaults to a real os.freemem()/loadavg() read.
143
+ // Tests override it to drive the admission gate deterministically.
144
+ checkHostHeadroom?: () => HostHeadroomResult;
145
+ hostAdmissionDisabled?: boolean;
146
+ admissionMaxWaitMs?: number;
147
+ admissionPollMs?: number;
148
148
  }
149
149
 
150
150
  export function createControlHandler(
@@ -167,6 +167,12 @@ export function createControlHandler(
167
167
  return session ? isSessionAlive(session) : false;
168
168
  });
169
169
  const recheckSafePoint = deps.recheckSafePoint ?? ((agentId: string) => relay.recheckSelfRestartSafePoint(agentId));
170
+ // #893 — per-host memory/load admission gate. Runs once per spawn, right before launch, so a
171
+ // spawn that would tip the host into swap-thrash queues (bounded poll/retry) instead of firing.
172
+ const headroomCheck = deps.checkHostHeadroom ?? checkHostHeadroom;
173
+ const admissionDisabled = deps.hostAdmissionDisabled ?? isHostAdmissionDisabled();
174
+ const admissionMaxWaitMs = deps.admissionMaxWaitMs ?? resolveAdmissionMaxWaitMs();
175
+ const admissionPollMs = deps.admissionPollMs ?? resolveAdmissionPollMs();
170
176
  const schedulerCommandSlots = new Semaphore(deps.maxConcurrentSchedulerCommands ?? resolveMaxConcurrentSchedulerCommands());
171
177
  const schedulerCommandMaxOutputBytes = deps.schedulerCommandMaxOutputBytes ?? resolveSchedulerCommandMaxOutputBytes();
172
178
  // Backgrounded spawn tasks: each is claimed synchronously (status → accepted)
@@ -174,6 +180,18 @@ export function createControlHandler(
174
180
  // registration window. Tracked so tests/shutdown can observe and settle them.
175
181
  const backgroundSpawns = new Set<Promise<void>>();
176
182
  const backgroundCommands = new Set<Promise<void>>();
183
+ // #1513 — spawns currently held back by the host headroom admission gate, keyed by command id.
184
+ // Published on the roster PATCH so the dashboard shows a "queued: waiting for host memory" row and
185
+ // the relay notifies the spawner. Entries are added on the first queued poll and removed once the
186
+ // spawn is admitted or times out (see the onQueued/onSettled wiring in runSpawnToRegistration).
187
+ const queuedSpawns = new Map<string, QueuedSpawn>();
188
+ const minFreeMemMb = resolveMinFreeMemMb();
189
+
190
+ function publishQueuedSpawns(): void {
191
+ // Best-effort — a failed roster write is retried by the reconnect resync (which carries the held
192
+ // snapshot). Never let a roster-push rejection escape into the backgrounded spawn task.
193
+ void Promise.resolve(relay.updateManagedAgents(managedAgents, [], [...queuedSpawns.values()])).catch(() => undefined);
194
+ }
177
195
 
178
196
  async function spawnManagedAgent(opts: SpawnOptions, action = "Spawned"): Promise<ManagedAgentReport> {
179
197
  const agent = await dispatchSpawn(opts, config);
@@ -244,7 +262,7 @@ export function createControlHandler(
244
262
  console.error(`[orchestrator] backgrounded spawn task error (swallowed): ${errMessage(error)}`);
245
263
  });
246
264
  backgroundSpawns.add(task);
247
- void task.finally(() => backgroundSpawns.delete(task));
265
+ void fireAndForget("Background spawn bookkeeping", task.finally(() => backgroundSpawns.delete(task)));
248
266
  return true;
249
267
  }
250
268
 
@@ -260,6 +278,25 @@ export function createControlHandler(
260
278
  }
261
279
  await relay.updateCommand(command.id, "running");
262
280
  try {
281
+ // #893 — host memory/load admission gate; see host-admission.ts.
282
+ // #1513 — surface the queued state: publish a "queued for host memory" roster row (and let the
283
+ // relay notify the spawner) while held back, updating the reason/free-memory as headroom moves,
284
+ // and clear it once the spawn is admitted or the wait ceiling is exceeded.
285
+ const queuedAt = Date.now();
286
+ await waitForHostHeadroom({
287
+ disabled: admissionDisabled,
288
+ maxWaitMs: admissionMaxWaitMs,
289
+ pollMs: admissionPollMs,
290
+ check: headroomCheck,
291
+ log: (message) => console.error(`[orchestrator] ${message}`),
292
+ onQueued: (result) => {
293
+ queuedSpawns.set(command.id, queuedSpawnFromCommand(command.params, result, queuedAt, minFreeMemMb, admissionMaxWaitMs));
294
+ publishQueuedSpawns();
295
+ },
296
+ onSettled: () => {
297
+ if (queuedSpawns.delete(command.id)) publishQueuedSpawns();
298
+ },
299
+ });
263
300
  await spawnAndWaitForRegistration(spawnOptionsFromControl(command.params, config));
264
301
  launched = true;
265
302
  } catch (error) {
@@ -452,6 +489,11 @@ export function createControlHandler(
452
489
  }
453
490
 
454
491
  async function runNonSpawnCommand(command: RelayCommand): Promise<boolean> {
492
+ // #1452 round-11 HIGH#1 — the review publish force-pushes to origin, so it is gated on an ATOMIC
493
+ // claim (helper in workspace-probe/review-ref): push ONLY IF the claim won, so a reconciler cancel
494
+ // (round closed) provably precludes a late orphaning push. The ONLY command gated on the claim —
495
+ // all others keep the generic accepted→running flow, confining the CAS blast radius to publish.
496
+ if (command.type === "workspace.publish-review") return runClaimGatedPublishReview(command, relay, managedAgents);
455
497
  // #1201 F4 — at-most-once guard for restart/shutdown, mirroring runSpawnToRegistration's
456
498
  // terminal-status re-fetch. A redelivered agent.restart (e.g. relay reconcile after an
457
499
  // orchestrator bounce) must not spawn a SECOND successor sharing the predecessor's fresh
@@ -482,6 +524,7 @@ export function createControlHandler(
482
524
  reason: typeof command.params.reason === "string" ? command.params.reason : undefined,
483
525
  workspacesRoot: workspacesRoot(config.baseDir),
484
526
  });
527
+ // #1452 — the review ref (if any) is dropped inside cleanupWorkspace itself, host-side.
485
528
  await relay.updateCommand(command.id, "succeeded", result);
486
529
  } else if (command.type === "workspace.reconcile") {
487
530
  const result = await reconcileWorkspace({
@@ -501,6 +544,7 @@ export function createControlHandler(
501
544
  repoRoot: typeof command.params.repoRoot === "string" ? command.params.repoRoot : undefined,
502
545
  worktreePath: typeof command.params.worktreePath === "string" ? command.params.worktreePath : undefined,
503
546
  branch: typeof command.params.branch === "string" ? command.params.branch : undefined,
547
+ expectedHeadSha: typeof command.params.expectedHeadSha === "string" ? command.params.expectedHeadSha : undefined,
504
548
  baseRef: typeof command.params.baseRef === "string" ? command.params.baseRef : undefined,
505
549
  baseSha: typeof command.params.baseSha === "string" ? command.params.baseSha : undefined,
506
550
  strategy: command.params.strategy === "pr" || command.params.strategy === "rebase-ff" || command.params.strategy === "auto" ? command.params.strategy : undefined,
@@ -550,7 +594,8 @@ export function createControlHandler(
550
594
  // (recoverAgedInFlightWorkspaces, the #1025 A7 timed_out→succeeded correction) settles it.
551
595
  // Same belt-and-suspenders as the spawn path (#930, markSpawnCommandFailed).
552
596
  try {
553
- await relay.updateCommand(command.id, mergeCommandStatus(result), result as unknown as Record<string, unknown>, result.error);
597
+ const report = prepareWorkspaceMergeResultForReport(result);
598
+ await relay.updateCommand(command.id, mergeCommandStatus(report), report as unknown as Record<string, unknown>, report.error);
554
599
  } catch (reportError) {
555
600
  console.error(`[orchestrator] workspace.merge result report failed for ${command.id} (not re-stamping as failed — relay reconcile owns recovery): ${errMessage(reportError)}`);
556
601
  }
@@ -618,6 +663,11 @@ export function createControlHandler(
618
663
  repoRoot: typeof command.params.repoRoot === "string" ? command.params.repoRoot : undefined,
619
664
  });
620
665
  await relay.updateCommand(command.id, "succeeded", result);
666
+ } else if (command.type === "workspace.unpublish-review") {
667
+ // #1452 — REVIEW-ONLY unpublish of the worker tip's refs/review/* ref on origin (never
668
+ // refs/heads/*/main, never lands). PUBLISH is intercepted earlier and runs the claim+fence+token-
669
+ // gated path (runClaimGatedPublishReview); only unpublish reaches here. Safety is host-side.
670
+ await handleReviewRefCommand(command, relay);
621
671
  } else if (command.type === "orchestrator.migrate-provider-config") {
622
672
  // Report our host-local provider config files; the relay seeds the central
623
673
  // provider-config rows with its own authority (#465). We don't write anything.
package/src/git.ts CHANGED
@@ -11,6 +11,12 @@ interface GitOptions {
11
11
  timeoutMs?: number;
12
12
  timeoutLabel?: string;
13
13
  signal?: AbortSignal;
14
+ /** Override the child's environment (default: inherit `process.env`, the existing behavior for
15
+ * every caller that omits this). #1145 round-6 — the ONLY current use is the hermetic
16
+ * gitignore-guard materialization (integrated-land-gates.ts), which must run with every
17
+ * ambient git config source (global/system/command-scope) scrubbed; every other caller is
18
+ * unaffected by this option existing. */
19
+ env?: Record<string, string | undefined>;
14
20
  }
15
21
 
16
22
  /** Run `git -C cwd <args>` and capture trimmed stdout/stderr; never throws. */