agent-relay-orchestrator 0.129.10 → 0.129.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +3 -3
- package/src/api.ts +20 -6
- package/src/async-guard.ts +42 -0
- package/src/command-poller.ts +2 -1
- package/src/command-result-report.ts +32 -0
- package/src/config.ts +47 -0
- package/src/control.ts +63 -13
- package/src/git.ts +6 -0
- package/src/host-admission.ts +426 -0
- package/src/index.ts +58 -29
- package/src/maintenance.ts +78 -24
- package/src/process.ts +83 -9
- package/src/provider-probe.ts +73 -6
- package/src/quota-poller.ts +2 -1
- package/src/registration.ts +144 -0
- package/src/relay.ts +322 -27
- package/src/repo-lock.ts +3 -2
- package/src/self-supervision.ts +8 -7
- package/src/self-upgrade-guard.ts +1717 -0
- package/src/self-upgrade.ts +451 -170
- package/src/shared-callmux.ts +47 -6
- package/src/spawn/command.ts +5 -0
- package/src/spawn/constants.ts +18 -1
- package/src/spawn/log-utils.ts +21 -0
- package/src/spawn/runtime.ts +440 -8
- package/src/spawn/sessions.ts +103 -22
- package/src/spawn/spawn-agent.ts +19 -1
- package/src/spawn/supervisor.ts +63 -8
- package/src/spawn/systemd.ts +15 -2
- package/src/spawn/terminal.ts +3 -4
- package/src/spawn/types.ts +6 -0
- package/src/terminal-stream.ts +12 -10
- package/src/wedged-session-reaper.ts +804 -0
- package/src/workspace-pr.ts +13 -0
- package/src/workspace-probe/base-sync.ts +408 -0
- package/src/workspace-probe/cleanup.ts +29 -29
- package/src/workspace-probe/content-landed.ts +108 -0
- package/src/workspace-probe/git-state.ts +90 -17
- package/src/workspace-probe/idle-refresh.ts +3 -1
- package/src/workspace-probe/index.ts +1 -0
- package/src/workspace-probe/integrated-land-gates.ts +758 -32
- package/src/workspace-probe/merge.ts +173 -243
- package/src/workspace-probe/names.ts +18 -0
- package/src/workspace-probe/plain-git.ts +96 -21
- package/src/workspace-probe/protected-tip.ts +57 -7
- package/src/workspace-probe/review-ref.ts +363 -0
- package/src/workspace-probe/types.ts +3 -0
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "agent-relay-orchestrator",
|
|
3
|
-
"version": "0.129.
|
|
3
|
+
"version": "0.129.12",
|
|
4
4
|
"description": "Agent Relay orchestrator — manages agent lifecycle across hosts",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"bin": {
|
|
@@ -16,8 +16,8 @@
|
|
|
16
16
|
"test": "bun test"
|
|
17
17
|
},
|
|
18
18
|
"dependencies": {
|
|
19
|
-
"agent-relay-providers": "0.104.
|
|
20
|
-
"agent-relay-sdk": "0.2.
|
|
19
|
+
"agent-relay-providers": "0.104.6",
|
|
20
|
+
"agent-relay-sdk": "0.2.130",
|
|
21
21
|
"callmux": "0.24.2"
|
|
22
22
|
},
|
|
23
23
|
"devDependencies": {
|
package/src/api.ts
CHANGED
|
@@ -3,8 +3,10 @@ import { errMessage, stringValue } from "agent-relay-sdk";
|
|
|
3
3
|
import { basename, dirname, extname, isAbsolute, join, relative, resolve } from "node:path";
|
|
4
4
|
import type { ServerWebSocket } from "bun";
|
|
5
5
|
import { proxyArtifactRequest } from "./artifact-proxy";
|
|
6
|
+
import { fireAndForget } from "./async-guard";
|
|
6
7
|
import type { OrchestratorConfig } from "./config";
|
|
7
8
|
import type { ProviderProbeCache } from "./provider-probe";
|
|
9
|
+
import type { RegistrationDriver } from "./registration";
|
|
8
10
|
import type { RelayClient } from "./relay";
|
|
9
11
|
import { captureSession, captureSessionMirror, captureTerminal, createTerminalGuest, listSessions, sendTerminalInput, resizeTerminal, stopTerminalGuest, validateTerminalInputData, validateTerminalResize } from "./spawn";
|
|
10
12
|
import { acquireTerminalStream, type TerminalStreamHandle, type TerminalStreamSubscriber } from "./terminal-stream";
|
|
@@ -390,7 +392,7 @@ export interface CodexResetConsumer {
|
|
|
390
392
|
}>;
|
|
391
393
|
}
|
|
392
394
|
|
|
393
|
-
export function startApiServer(config: OrchestratorConfig, probeCache: ProviderProbeCache, relay?: RelayClient, codexResetConsumer?: CodexResetConsumer): { stop(): void; url: string } {
|
|
395
|
+
export function startApiServer(config: OrchestratorConfig, probeCache: ProviderProbeCache, relay?: RelayClient, codexResetConsumer?: CodexResetConsumer, registration?: Pick<RegistrationDriver, "getHealth">): { stop(): void; url: string } {
|
|
394
396
|
const server = Bun.serve<TerminalSocketData>({
|
|
395
397
|
port: config.apiPort,
|
|
396
398
|
hostname: "0.0.0.0",
|
|
@@ -669,7 +671,19 @@ export function startApiServer(config: OrchestratorConfig, probeCache: ProviderP
|
|
|
669
671
|
}
|
|
670
672
|
|
|
671
673
|
if (req.method === "GET" && url.pathname === "/api/health") {
|
|
672
|
-
|
|
674
|
+
// #1676 — surface relay-peer and registration degradation here so a host stuck
|
|
675
|
+
// re-registering is distinguishable from a healthy one WITHOUT log archaeology
|
|
676
|
+
// (previously it looked identical, and spawns routed to it burned their full
|
|
677
|
+
// timeout). `status`/`ok` deliberately stay "ok": this probe is the
|
|
678
|
+
// Relay-INDEPENDENT local readiness signal the self-upgrade guard gates on, and a
|
|
679
|
+
// degraded peer must not make this process look un-upgradable. Read `degraded`.
|
|
680
|
+
const peer = relay?.getHealth();
|
|
681
|
+
const reg = registration?.getHealth();
|
|
682
|
+
return json({
|
|
683
|
+
status: "ok", ok: true, id: config.id, hostname: config.hostname,
|
|
684
|
+
degraded: Boolean(peer?.degraded || reg?.degraded),
|
|
685
|
+
...(peer ? { relay: peer } : {}), ...(reg ? { registration: reg } : {}),
|
|
686
|
+
});
|
|
673
687
|
}
|
|
674
688
|
|
|
675
689
|
return error("Not found", 404);
|
|
@@ -739,7 +753,7 @@ function startTerminalSocket(ws: TerminalSocket): void {
|
|
|
739
753
|
// Wait for the client's resize to size the pane before backfilling. Fall back to
|
|
740
754
|
// the pane's current size if no resize arrives (e.g. a non-fitting client).
|
|
741
755
|
ws.data.syncTimer = setTimeout(() => {
|
|
742
|
-
if (!ws.data.synced) void syncAndBackfill(ws);
|
|
756
|
+
if (!ws.data.synced) void fireAndForget("Terminal socket sync/backfill", () => syncAndBackfill(ws));
|
|
743
757
|
}, 700);
|
|
744
758
|
} catch (e) {
|
|
745
759
|
ws.send(JSON.stringify({ type: "error", error: errMessage(e) }));
|
|
@@ -826,7 +840,7 @@ function handleTerminalSocketMessage(ws: TerminalSocket, data: string | Buffer):
|
|
|
826
840
|
// First resize sizes the pane and triggers the (size-matched) backfill;
|
|
827
841
|
// later ones just reflow the live stream.
|
|
828
842
|
if (!ws.data.synced) {
|
|
829
|
-
void syncAndBackfill(ws, cols, rows);
|
|
843
|
+
void fireAndForget("Terminal socket sync/backfill", () => syncAndBackfill(ws, cols, rows));
|
|
830
844
|
} else {
|
|
831
845
|
ws.data.stream?.resize(cols, rows);
|
|
832
846
|
}
|
|
@@ -840,10 +854,10 @@ function handleTerminalSocketMessage(ws: TerminalSocket, data: string | Buffer):
|
|
|
840
854
|
if (wasPaused && !ws.data.paused && ws.data.synced) {
|
|
841
855
|
const dropped = ws.data.droppedWhilePaused ?? 0;
|
|
842
856
|
ws.data.droppedWhilePaused = 0;
|
|
843
|
-
if (dropped > 0) void sendBackfill(ws, undefined, undefined, true);
|
|
857
|
+
if (dropped > 0) void fireAndForget("Terminal socket backfill", () => sendBackfill(ws, undefined, undefined, true));
|
|
844
858
|
}
|
|
845
859
|
} else if (frame.type === "refresh") {
|
|
846
|
-
if (ws.data.synced) void sendBackfill(ws, undefined, undefined, true);
|
|
860
|
+
if (ws.data.synced) void fireAndForget("Terminal socket backfill", () => sendBackfill(ws, undefined, undefined, true));
|
|
847
861
|
} else if (frame.type === "interactive") {
|
|
848
862
|
// Read-only watchers must not reflow the shared tmux window (#273); the typist owns
|
|
849
863
|
// sizing. The client reports its interactivity so the stream knows who may resize.
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
// #1676 — orchestrator adapter over the SHARED async guards in `agent-relay-sdk`.
|
|
2
|
+
//
|
|
3
|
+
// The primitives themselves live in `sdk/src/async-guard.ts` because #1665 is the same
|
|
4
|
+
// defect class in the runner: an unguarded `void <promise>` / async timer callback whose
|
|
5
|
+
// rejection reaches the runtime and (in Bun) exits the process. One implementation, two
|
|
6
|
+
// processes. Import from `agent-relay-sdk` directly if you are outside this package.
|
|
7
|
+
//
|
|
8
|
+
// All this file adds is the `[orchestrator]` log prefix, so the daemon's log format is
|
|
9
|
+
// unchanged and call sites pass a bare label ("Relay heartbeat", not the prefix).
|
|
10
|
+
|
|
11
|
+
import {
|
|
12
|
+
fireAndForget as sdkFireAndForget,
|
|
13
|
+
guardedInterval as sdkGuardedInterval,
|
|
14
|
+
guardedTimeout as sdkGuardedTimeout,
|
|
15
|
+
installUnhandledRejectionGuard as sdkInstallUnhandledRejectionGuard,
|
|
16
|
+
type GuardLog,
|
|
17
|
+
} from "agent-relay-sdk";
|
|
18
|
+
|
|
19
|
+
export { describeFailure, isAbortError, type GuardLog } from "agent-relay-sdk";
|
|
20
|
+
|
|
21
|
+
const PREFIX = "[orchestrator]";
|
|
22
|
+
|
|
23
|
+
/** See {@link import("agent-relay-sdk").fireAndForget}. Replaces bare `void promise()`. */
|
|
24
|
+
export function fireAndForget(label: string, work: Promise<unknown> | (() => Promise<unknown> | unknown), log?: GuardLog): Promise<void> {
|
|
25
|
+
return sdkFireAndForget(`${PREFIX} ${label}`, work, log);
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
/** See {@link import("agent-relay-sdk").guardedInterval}. Replaces `setInterval(asyncFn, ms)`. */
|
|
29
|
+
export function guardedInterval(label: string, intervalMs: number, fn: () => Promise<unknown> | unknown, log?: GuardLog): ReturnType<typeof setInterval> {
|
|
30
|
+
return sdkGuardedInterval(`${PREFIX} ${label}`, intervalMs, fn, log);
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
/** See {@link import("agent-relay-sdk").guardedTimeout}. Replaces `setTimeout(asyncFn, ms)`. */
|
|
34
|
+
export function guardedTimeout(label: string, delayMs: number, fn: () => Promise<unknown> | unknown, log?: GuardLog): ReturnType<typeof setTimeout> {
|
|
35
|
+
return sdkGuardedTimeout(`${PREFIX} ${label}`, delayMs, fn, log);
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
/** Install the non-fatal unhandled-rejection backstop. See the SDK function for the
|
|
39
|
+
* reasoning behind "log and survive" and why `uncaughtException` stays fatal. */
|
|
40
|
+
export function installUnhandledRejectionGuard(log?: GuardLog): () => void {
|
|
41
|
+
return sdkInstallUnhandledRejectionGuard(PREFIX, log);
|
|
42
|
+
}
|
package/src/command-poller.ts
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { fireAndForget } from "./async-guard";
|
|
1
2
|
import type { RelayClient, RelayCommand } from "./relay";
|
|
2
3
|
|
|
3
4
|
interface CommandPollerControl {
|
|
@@ -79,7 +80,7 @@ export function createCommandPoller({
|
|
|
79
80
|
if (stopped) return;
|
|
80
81
|
timer = setTimeout(() => {
|
|
81
82
|
timer = undefined;
|
|
82
|
-
void runCycle(generation);
|
|
83
|
+
void fireAndForget("Command poll cycle", () => runCycle(generation), log);
|
|
83
84
|
}, delayMs);
|
|
84
85
|
timer.unref?.();
|
|
85
86
|
}
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
import type { WorkspaceMergeResult } from "agent-relay-sdk";
|
|
2
|
+
|
|
3
|
+
// #638 — settle a `workspace.merge` command on whether it made progress, not on whether
|
|
4
|
+
// the merge function threw. Every no-progress outcome (origin moved ahead, an unpredicted
|
|
5
|
+
// replay conflict, a push race) returns merged:false with an `error` set; every genuine
|
|
6
|
+
// land / recycle / noop-resolve / PR-opened leaves `error` undefined. Reporting `succeeded`
|
|
7
|
+
// for an unlanded branch is what let the auto-merge job believe it was progressing and
|
|
8
|
+
// busy-loop the repo's merge lease. Mirrors the pr-merge / pr-arm settle contract.
|
|
9
|
+
export function mergeCommandStatus(result: WorkspaceMergeResult): "succeeded" | "failed" {
|
|
10
|
+
return result.error ? "failed" : "succeeded";
|
|
11
|
+
}
|
|
12
|
+
|
|
13
|
+
// Command PATCH bodies are capped at 64 KiB by the relay. A failing full land gate
|
|
14
|
+
// may retain 256 KiB of output for artifact upload, so forwarding it verbatim can
|
|
15
|
+
// reject the *terminal* update and leave expiry to misreport a red gate as a timeout.
|
|
16
|
+
// Keep ordinary output intact; when it will not fit, retain the bounded tail that is
|
|
17
|
+
// already intended for the worker/coordinator notification.
|
|
18
|
+
const MAX_COMMAND_REPORT_CHARS = 60 * 1024;
|
|
19
|
+
|
|
20
|
+
export function prepareWorkspaceMergeResultForReport(result: WorkspaceMergeResult): WorkspaceMergeResult {
|
|
21
|
+
const gateFailure = result.gateFailure;
|
|
22
|
+
if (!gateFailure?.output) return result;
|
|
23
|
+
const body = JSON.stringify({ status: mergeCommandStatus(result), result, error: result.error });
|
|
24
|
+
if (body.length <= MAX_COMMAND_REPORT_CHARS) return result;
|
|
25
|
+
return {
|
|
26
|
+
...result,
|
|
27
|
+
gateFailure: {
|
|
28
|
+
...gateFailure,
|
|
29
|
+
output: gateFailure.outputTail,
|
|
30
|
+
},
|
|
31
|
+
};
|
|
32
|
+
}
|
package/src/config.ts
CHANGED
|
@@ -32,6 +32,34 @@ export function gitShaFromEnv(): string | undefined {
|
|
|
32
32
|
return process.env.AGENT_RELAY_GIT_SHA || process.env.GIT_SHA || undefined;
|
|
33
33
|
}
|
|
34
34
|
|
|
35
|
+
// #1452 — host-side gate for REVIEW-REF PUBLISHING (the origin-mutating half). The host must
|
|
36
|
+
// decide this itself and NOT trust the relay/caller: a queued, replayed, or hand-crafted
|
|
37
|
+
// publish-review command must NOT mutate origin unless THIS host has the feature enabled. The
|
|
38
|
+
// operator sets AGENT_RELAY_REVIEW_PUBLISH=1 on both the relay AND the orchestrator to activate;
|
|
39
|
+
// unset here ⇒ the host no-ops every publish (fail-safe OFF). UNPUBLISH is deliberately NOT
|
|
40
|
+
// gated on this — a ref created during a prior ON era must still be cleanable after it flips OFF.
|
|
41
|
+
export function reviewPublishEnabled(): boolean {
|
|
42
|
+
return process.env.AGENT_RELAY_REVIEW_PUBLISH === "1";
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
// #1452 round-3 (H5) — a hard ceiling on every review-ref git op that touches `origin`. Short
|
|
46
|
+
// enough that a dead/hung remote fails fast instead of freezing the teardown/land/command loop that
|
|
47
|
+
// awaits it, generous enough for a real push over a slow link. Clamped to a sane minimum.
|
|
48
|
+
//
|
|
49
|
+
// #1452 round-12 — the ceiling is now LOAD-BEARING, not just a fail-fast bound. The push fence
|
|
50
|
+
// (fencePublishForPush) refreshes the relay's publish-command lease (WORKSPACE_COMMAND_TTL_MS = 40 min)
|
|
51
|
+
// immediately before the push, and its correctness relies on the git-timeout-bounded push completing
|
|
52
|
+
// WITHIN that fresh lease so the TTL sweep can never fire between the fence and the push. Without an
|
|
53
|
+
// UPPER clamp, an operator misconfig (AGENT_RELAY_REVIEW_REF_GIT_TIMEOUT_MS above the lease) would let a
|
|
54
|
+
// single git op outlast the lease → mid-op timeout → the later push races the reconciler's cleanup
|
|
55
|
+
// (also LOW-1's over-cleanup of a live push). The 5-min ceiling keeps "the lease comfortably exceeds
|
|
56
|
+
// every review-ref git op" a STRUCTURAL invariant — even a handful of sequential pre-push ops finish
|
|
57
|
+
// well inside 40 min. A guard test asserts ceiling << lease so the two constants can't silently drift.
|
|
58
|
+
export const REVIEW_REF_GIT_TIMEOUT_CEILING_MS = 5 * 60_000;
|
|
59
|
+
export function reviewRefGitTimeoutMs(): number {
|
|
60
|
+
return Math.min(REVIEW_REF_GIT_TIMEOUT_CEILING_MS, Math.max(1_000, Number(process.env.AGENT_RELAY_REVIEW_REF_GIT_TIMEOUT_MS) || 15_000));
|
|
61
|
+
}
|
|
62
|
+
|
|
35
63
|
export function agentRelayHome(): string {
|
|
36
64
|
return process.env.AGENT_RELAY_HOME || join(homedir(), ".agent-relay");
|
|
37
65
|
}
|
|
@@ -90,6 +118,25 @@ export function tmuxSocketSweepIntervalMs(): number {
|
|
|
90
118
|
return envNonNegativeMax("AGENT_RELAY_TMUX_SOCKET_SWEEP_INTERVAL_MS", 10 * 60 * 1000, 60_000);
|
|
91
119
|
}
|
|
92
120
|
|
|
121
|
+
// #1514 — reaper for orphaned, idle managed tmux sessions (e.g. a headless Claude
|
|
122
|
+
// session wedged at the "Exit anyway / Move to background / Stay" dialog). On by
|
|
123
|
+
// default; set AGENT_RELAY_WEDGED_SESSION_REAP=0 to disable.
|
|
124
|
+
export function wedgedSessionReapEnabled(): boolean {
|
|
125
|
+
return process.env.AGENT_RELAY_WEDGED_SESSION_REAP !== "0";
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
export function wedgedSessionReapIntervalMs(): number {
|
|
129
|
+
return envNonNegativeMax("AGENT_RELAY_WEDGED_SESSION_REAP_INTERVAL_MS", 5 * 60 * 1000, 60_000);
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
// A session must show no tmux activity for at least this long before an unmanaged
|
|
133
|
+
// session is reaped. Generous so a brief spawn/registration race (record not yet
|
|
134
|
+
// written) or a momentary state-file gap can never clip a healthy session; a real
|
|
135
|
+
// wedge sits idle for hours/days.
|
|
136
|
+
export function wedgedSessionIdleThresholdMs(): number {
|
|
137
|
+
return envNonNegativeMax("AGENT_RELAY_WEDGED_SESSION_IDLE_THRESHOLD_MS", 30 * 60 * 1000, 5 * 60 * 1000);
|
|
138
|
+
}
|
|
139
|
+
|
|
93
140
|
export const TERMINAL_FLUSH_MS = envNonNegativeMax("AGENT_RELAY_TERMINAL_FLUSH_MS", 6, 0);
|
|
94
141
|
export const TERMINAL_FLUSH_MAX_BYTES = envNonNegativeMax("AGENT_RELAY_TERMINAL_FLUSH_MAX_BYTES", 65536, 4096);
|
|
95
142
|
export const TERMINAL_BACKPRESSURE_MAX_BYTES = envNonNegativeMax("AGENT_RELAY_TERMINAL_BACKPRESSURE_MAX_BYTES", 8 << 20, 1 << 20);
|
package/src/control.ts
CHANGED
|
@@ -1,26 +1,20 @@
|
|
|
1
1
|
import { errMessage, isPathWithinBase, isRecord, normalizeAgentLifecycle, normalizeWorkspaceMode, Semaphore } from "agent-relay-sdk";
|
|
2
|
+
import { fireAndForget } from "./async-guard";
|
|
2
3
|
import { getAllManifests, getManifest } from "agent-relay-providers";
|
|
3
4
|
import { SCHEDULER_COMMAND_MAX_CONCURRENT, SCHEDULER_COMMAND_MAX_OUTPUT_BYTES, type OrchestratorConfig } from "./config";
|
|
5
|
+
import { checkHostHeadroom, isHostAdmissionDisabled, queuedSpawnFromCommand, resolveAdmissionMaxWaitMs, resolveAdmissionPollMs, resolveMinFreeMemMb, waitForHostHeadroom, type HostHeadroomResult } from "./host-admission";
|
|
4
6
|
import type { ManagedAgentReport, RelayClient, RelayCommand } from "./relay";
|
|
7
|
+
import type { QueuedSpawn } from "agent-relay-sdk";
|
|
5
8
|
import { handleSelfUpgrade } from "./self-upgrade";
|
|
6
9
|
import { readLocalProviderConfigs } from "./provider-config-migration";
|
|
7
10
|
import { findSessionRecord, isSessionAlive, readRunnerInfo, spawnAgent, stopSession, type SpawnOptions } from "./spawn";
|
|
8
|
-
import { cleanupWorkspace, discardRecoveryBranch, idleRefreshWorktree, mergeWorkspace, mergeWorkspacePlainGit, pruneWorktrees, reconcileWorkspace, refreshWorkspaceDeps, workspacesRoot } from "./workspace-probe";
|
|
11
|
+
import { cleanupWorkspace, discardRecoveryBranch, handleReviewRefCommand, idleRefreshWorktree, mergeWorkspace, mergeWorkspacePlainGit, pruneWorktrees, reconcileWorkspace, refreshWorkspaceDeps, runClaimGatedPublishReview, workspacesRoot } from "./workspace-probe";
|
|
9
12
|
import { withMergePhaseTimeout } from "./workspace-probe/merge-timeouts";
|
|
10
13
|
import { withRepoLock } from "./repo-lock";
|
|
11
14
|
import { armWorkspacePrAutoMerge, mergeWorkspacePr, refreshWorkspacePrBranch } from "./workspace-pr";
|
|
12
15
|
import type { WorkspaceMergeResult } from "agent-relay-sdk";
|
|
13
16
|
import { execProcess } from "./process";
|
|
14
|
-
|
|
15
|
-
// #638 — settle a `workspace.merge` command on whether it made progress, not on whether
|
|
16
|
-
// the merge function threw. Every no-progress outcome (origin moved ahead, an unpredicted
|
|
17
|
-
// replay conflict, a push race) returns merged:false with an `error` set; every genuine
|
|
18
|
-
// land / recycle / noop-resolve / PR-opened leaves `error` undefined. Reporting `succeeded`
|
|
19
|
-
// for an unlanded branch is what let the auto-merge job believe it was progressing and
|
|
20
|
-
// busy-loop the repo's merge lease. Mirrors the pr-merge / pr-arm settle contract.
|
|
21
|
-
export function mergeCommandStatus(result: WorkspaceMergeResult): "succeeded" | "failed" {
|
|
22
|
-
return result.error ? "failed" : "succeeded";
|
|
23
|
-
}
|
|
17
|
+
import { mergeCommandStatus, prepareWorkspaceMergeResultForReport } from "./command-result-report";
|
|
24
18
|
|
|
25
19
|
// #1020 (B2) — command types that mutate a physical repo's git refs / worktrees. These take the
|
|
26
20
|
// per-repoRoot lock (see repo-lock.ts) so they can't interleave with the background spawn's
|
|
@@ -145,6 +139,12 @@ interface ControlHandlerDeps {
|
|
|
145
139
|
registrationSettleMs?: number;
|
|
146
140
|
// #1201 F2 — injectable safe-point re-check. Defaults to relay.recheckSelfRestartSafePoint.
|
|
147
141
|
recheckSafePoint?: (agentId: string) => Promise<{ ok: boolean; reason?: string }>;
|
|
142
|
+
// #893 — injectable host headroom check. Defaults to a real os.freemem()/loadavg() read.
|
|
143
|
+
// Tests override it to drive the admission gate deterministically.
|
|
144
|
+
checkHostHeadroom?: () => HostHeadroomResult;
|
|
145
|
+
hostAdmissionDisabled?: boolean;
|
|
146
|
+
admissionMaxWaitMs?: number;
|
|
147
|
+
admissionPollMs?: number;
|
|
148
148
|
}
|
|
149
149
|
|
|
150
150
|
export function createControlHandler(
|
|
@@ -167,6 +167,12 @@ export function createControlHandler(
|
|
|
167
167
|
return session ? isSessionAlive(session) : false;
|
|
168
168
|
});
|
|
169
169
|
const recheckSafePoint = deps.recheckSafePoint ?? ((agentId: string) => relay.recheckSelfRestartSafePoint(agentId));
|
|
170
|
+
// #893 — per-host memory/load admission gate. Runs once per spawn, right before launch, so a
|
|
171
|
+
// spawn that would tip the host into swap-thrash queues (bounded poll/retry) instead of firing.
|
|
172
|
+
const headroomCheck = deps.checkHostHeadroom ?? checkHostHeadroom;
|
|
173
|
+
const admissionDisabled = deps.hostAdmissionDisabled ?? isHostAdmissionDisabled();
|
|
174
|
+
const admissionMaxWaitMs = deps.admissionMaxWaitMs ?? resolveAdmissionMaxWaitMs();
|
|
175
|
+
const admissionPollMs = deps.admissionPollMs ?? resolveAdmissionPollMs();
|
|
170
176
|
const schedulerCommandSlots = new Semaphore(deps.maxConcurrentSchedulerCommands ?? resolveMaxConcurrentSchedulerCommands());
|
|
171
177
|
const schedulerCommandMaxOutputBytes = deps.schedulerCommandMaxOutputBytes ?? resolveSchedulerCommandMaxOutputBytes();
|
|
172
178
|
// Backgrounded spawn tasks: each is claimed synchronously (status → accepted)
|
|
@@ -174,6 +180,18 @@ export function createControlHandler(
|
|
|
174
180
|
// registration window. Tracked so tests/shutdown can observe and settle them.
|
|
175
181
|
const backgroundSpawns = new Set<Promise<void>>();
|
|
176
182
|
const backgroundCommands = new Set<Promise<void>>();
|
|
183
|
+
// #1513 — spawns currently held back by the host headroom admission gate, keyed by command id.
|
|
184
|
+
// Published on the roster PATCH so the dashboard shows a "queued: waiting for host memory" row and
|
|
185
|
+
// the relay notifies the spawner. Entries are added on the first queued poll and removed once the
|
|
186
|
+
// spawn is admitted or times out (see the onQueued/onSettled wiring in runSpawnToRegistration).
|
|
187
|
+
const queuedSpawns = new Map<string, QueuedSpawn>();
|
|
188
|
+
const minFreeMemMb = resolveMinFreeMemMb();
|
|
189
|
+
|
|
190
|
+
function publishQueuedSpawns(): void {
|
|
191
|
+
// Best-effort — a failed roster write is retried by the reconnect resync (which carries the held
|
|
192
|
+
// snapshot). Never let a roster-push rejection escape into the backgrounded spawn task.
|
|
193
|
+
void Promise.resolve(relay.updateManagedAgents(managedAgents, [], [...queuedSpawns.values()])).catch(() => undefined);
|
|
194
|
+
}
|
|
177
195
|
|
|
178
196
|
async function spawnManagedAgent(opts: SpawnOptions, action = "Spawned"): Promise<ManagedAgentReport> {
|
|
179
197
|
const agent = await dispatchSpawn(opts, config);
|
|
@@ -244,7 +262,7 @@ export function createControlHandler(
|
|
|
244
262
|
console.error(`[orchestrator] backgrounded spawn task error (swallowed): ${errMessage(error)}`);
|
|
245
263
|
});
|
|
246
264
|
backgroundSpawns.add(task);
|
|
247
|
-
void task.finally(() => backgroundSpawns.delete(task));
|
|
265
|
+
void fireAndForget("Background spawn bookkeeping", task.finally(() => backgroundSpawns.delete(task)));
|
|
248
266
|
return true;
|
|
249
267
|
}
|
|
250
268
|
|
|
@@ -260,6 +278,25 @@ export function createControlHandler(
|
|
|
260
278
|
}
|
|
261
279
|
await relay.updateCommand(command.id, "running");
|
|
262
280
|
try {
|
|
281
|
+
// #893 — host memory/load admission gate; see host-admission.ts.
|
|
282
|
+
// #1513 — surface the queued state: publish a "queued for host memory" roster row (and let the
|
|
283
|
+
// relay notify the spawner) while held back, updating the reason/free-memory as headroom moves,
|
|
284
|
+
// and clear it once the spawn is admitted or the wait ceiling is exceeded.
|
|
285
|
+
const queuedAt = Date.now();
|
|
286
|
+
await waitForHostHeadroom({
|
|
287
|
+
disabled: admissionDisabled,
|
|
288
|
+
maxWaitMs: admissionMaxWaitMs,
|
|
289
|
+
pollMs: admissionPollMs,
|
|
290
|
+
check: headroomCheck,
|
|
291
|
+
log: (message) => console.error(`[orchestrator] ${message}`),
|
|
292
|
+
onQueued: (result) => {
|
|
293
|
+
queuedSpawns.set(command.id, queuedSpawnFromCommand(command.params, result, queuedAt, minFreeMemMb, admissionMaxWaitMs));
|
|
294
|
+
publishQueuedSpawns();
|
|
295
|
+
},
|
|
296
|
+
onSettled: () => {
|
|
297
|
+
if (queuedSpawns.delete(command.id)) publishQueuedSpawns();
|
|
298
|
+
},
|
|
299
|
+
});
|
|
263
300
|
await spawnAndWaitForRegistration(spawnOptionsFromControl(command.params, config));
|
|
264
301
|
launched = true;
|
|
265
302
|
} catch (error) {
|
|
@@ -452,6 +489,11 @@ export function createControlHandler(
|
|
|
452
489
|
}
|
|
453
490
|
|
|
454
491
|
async function runNonSpawnCommand(command: RelayCommand): Promise<boolean> {
|
|
492
|
+
// #1452 round-11 HIGH#1 — the review publish force-pushes to origin, so it is gated on an ATOMIC
|
|
493
|
+
// claim (helper in workspace-probe/review-ref): push ONLY IF the claim won, so a reconciler cancel
|
|
494
|
+
// (round closed) provably precludes a late orphaning push. The ONLY command gated on the claim —
|
|
495
|
+
// all others keep the generic accepted→running flow, confining the CAS blast radius to publish.
|
|
496
|
+
if (command.type === "workspace.publish-review") return runClaimGatedPublishReview(command, relay, managedAgents);
|
|
455
497
|
// #1201 F4 — at-most-once guard for restart/shutdown, mirroring runSpawnToRegistration's
|
|
456
498
|
// terminal-status re-fetch. A redelivered agent.restart (e.g. relay reconcile after an
|
|
457
499
|
// orchestrator bounce) must not spawn a SECOND successor sharing the predecessor's fresh
|
|
@@ -482,6 +524,7 @@ export function createControlHandler(
|
|
|
482
524
|
reason: typeof command.params.reason === "string" ? command.params.reason : undefined,
|
|
483
525
|
workspacesRoot: workspacesRoot(config.baseDir),
|
|
484
526
|
});
|
|
527
|
+
// #1452 — the review ref (if any) is dropped inside cleanupWorkspace itself, host-side.
|
|
485
528
|
await relay.updateCommand(command.id, "succeeded", result);
|
|
486
529
|
} else if (command.type === "workspace.reconcile") {
|
|
487
530
|
const result = await reconcileWorkspace({
|
|
@@ -501,6 +544,7 @@ export function createControlHandler(
|
|
|
501
544
|
repoRoot: typeof command.params.repoRoot === "string" ? command.params.repoRoot : undefined,
|
|
502
545
|
worktreePath: typeof command.params.worktreePath === "string" ? command.params.worktreePath : undefined,
|
|
503
546
|
branch: typeof command.params.branch === "string" ? command.params.branch : undefined,
|
|
547
|
+
expectedHeadSha: typeof command.params.expectedHeadSha === "string" ? command.params.expectedHeadSha : undefined,
|
|
504
548
|
baseRef: typeof command.params.baseRef === "string" ? command.params.baseRef : undefined,
|
|
505
549
|
baseSha: typeof command.params.baseSha === "string" ? command.params.baseSha : undefined,
|
|
506
550
|
strategy: command.params.strategy === "pr" || command.params.strategy === "rebase-ff" || command.params.strategy === "auto" ? command.params.strategy : undefined,
|
|
@@ -550,7 +594,8 @@ export function createControlHandler(
|
|
|
550
594
|
// (recoverAgedInFlightWorkspaces, the #1025 A7 timed_out→succeeded correction) settles it.
|
|
551
595
|
// Same belt-and-suspenders as the spawn path (#930, markSpawnCommandFailed).
|
|
552
596
|
try {
|
|
553
|
-
|
|
597
|
+
const report = prepareWorkspaceMergeResultForReport(result);
|
|
598
|
+
await relay.updateCommand(command.id, mergeCommandStatus(report), report as unknown as Record<string, unknown>, report.error);
|
|
554
599
|
} catch (reportError) {
|
|
555
600
|
console.error(`[orchestrator] workspace.merge result report failed for ${command.id} (not re-stamping as failed — relay reconcile owns recovery): ${errMessage(reportError)}`);
|
|
556
601
|
}
|
|
@@ -618,6 +663,11 @@ export function createControlHandler(
|
|
|
618
663
|
repoRoot: typeof command.params.repoRoot === "string" ? command.params.repoRoot : undefined,
|
|
619
664
|
});
|
|
620
665
|
await relay.updateCommand(command.id, "succeeded", result);
|
|
666
|
+
} else if (command.type === "workspace.unpublish-review") {
|
|
667
|
+
// #1452 — REVIEW-ONLY unpublish of the worker tip's refs/review/* ref on origin (never
|
|
668
|
+
// refs/heads/*/main, never lands). PUBLISH is intercepted earlier and runs the claim+fence+token-
|
|
669
|
+
// gated path (runClaimGatedPublishReview); only unpublish reaches here. Safety is host-side.
|
|
670
|
+
await handleReviewRefCommand(command, relay);
|
|
621
671
|
} else if (command.type === "orchestrator.migrate-provider-config") {
|
|
622
672
|
// Report our host-local provider config files; the relay seeds the central
|
|
623
673
|
// provider-config rows with its own authority (#465). We don't write anything.
|
package/src/git.ts
CHANGED
|
@@ -11,6 +11,12 @@ interface GitOptions {
|
|
|
11
11
|
timeoutMs?: number;
|
|
12
12
|
timeoutLabel?: string;
|
|
13
13
|
signal?: AbortSignal;
|
|
14
|
+
/** Override the child's environment (default: inherit `process.env`, the existing behavior for
|
|
15
|
+
* every caller that omits this). #1145 round-6 — the ONLY current use is the hermetic
|
|
16
|
+
* gitignore-guard materialization (integrated-land-gates.ts), which must run with every
|
|
17
|
+
* ambient git config source (global/system/command-scope) scrubbed; every other caller is
|
|
18
|
+
* unaffected by this option existing. */
|
|
19
|
+
env?: Record<string, string | undefined>;
|
|
14
20
|
}
|
|
15
21
|
|
|
16
22
|
/** Run `git -C cwd <args>` and capture trimmed stdout/stderr; never throws. */
|