agent-relay-orchestrator 0.118.0 → 0.118.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +2 -2
- package/src/api.ts +5 -5
- package/src/command-poller.ts +44 -1
- package/src/control.ts +34 -20
- package/src/git.ts +8 -24
- package/src/index.ts +4 -6
- package/src/process.ts +41 -0
- package/src/relay.ts +26 -3
- package/src/spawn/acquisition.ts +193 -0
- package/src/spawn/index.ts +1 -0
- package/src/spawn/spawn-agent.ts +8 -4
- package/src/spawn/supervisor.ts +18 -30
- package/src/spawn/systemd.ts +39 -0
- package/src/spawn/types.ts +12 -0
- package/src/workspace-pr.ts +20 -31
- package/src/workspace-probe/cleanup.ts +23 -23
- package/src/workspace-probe/deps.ts +11 -10
- package/src/workspace-probe/git-state.ts +31 -31
- package/src/workspace-probe/idle-refresh.ts +14 -14
- package/src/workspace-probe/land-gates-runner.ts +109 -10
- package/src/workspace-probe/merge.ts +134 -139
- package/src/workspace-probe/names.ts +9 -9
- package/src/workspace-probe/probe.ts +19 -17
- package/src/workspace-probe/recovery-branches.ts +19 -19
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "agent-relay-orchestrator",
|
|
3
|
-
"version": "0.118.
|
|
3
|
+
"version": "0.118.2",
|
|
4
4
|
"description": "Agent Relay orchestrator — manages agent lifecycle across hosts",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"bin": {
|
|
@@ -17,7 +17,7 @@
|
|
|
17
17
|
},
|
|
18
18
|
"dependencies": {
|
|
19
19
|
"agent-relay-providers": "0.104.1",
|
|
20
|
-
"agent-relay-sdk": "0.2.
|
|
20
|
+
"agent-relay-sdk": "0.2.101",
|
|
21
21
|
"callmux": "0.23.0"
|
|
22
22
|
},
|
|
23
23
|
"devDependencies": {
|
package/src/api.ts
CHANGED
|
@@ -479,7 +479,7 @@ export function startApiServer(config: OrchestratorConfig, probeCache: ProviderP
|
|
|
479
479
|
if (!authorized(req, config)) return error("unauthorized", 401);
|
|
480
480
|
try {
|
|
481
481
|
const { target } = resolveInsideBase(url.searchParams.get("path") || undefined, config.baseDir);
|
|
482
|
-
return json(workspaceGitState({
|
|
482
|
+
return json(await workspaceGitState({
|
|
483
483
|
worktreePath: target,
|
|
484
484
|
baseRef: url.searchParams.get("baseRef") || undefined,
|
|
485
485
|
baseSha: url.searchParams.get("baseSha") || undefined,
|
|
@@ -493,7 +493,7 @@ export function startApiServer(config: OrchestratorConfig, probeCache: ProviderP
|
|
|
493
493
|
if (!authorized(req, config)) return error("unauthorized", 401);
|
|
494
494
|
try {
|
|
495
495
|
const { target } = resolveInsideBase(url.searchParams.get("path") || undefined, config.baseDir);
|
|
496
|
-
return json(workspaceDiff({
|
|
496
|
+
return json(await workspaceDiff({
|
|
497
497
|
worktreePath: target,
|
|
498
498
|
baseRef: url.searchParams.get("baseRef") || undefined,
|
|
499
499
|
baseSha: url.searchParams.get("baseSha") || undefined,
|
|
@@ -506,17 +506,17 @@ export function startApiServer(config: OrchestratorConfig, probeCache: ProviderP
|
|
|
506
506
|
|
|
507
507
|
if (req.method === "GET" && url.pathname === "/api/workspace/merge-preview") {
|
|
508
508
|
if (!authorized(req, config)) return error("unauthorized", 401);
|
|
509
|
-
return mergePreviewResponse(url, config.baseDir);
|
|
509
|
+
return await mergePreviewResponse(url, config.baseDir);
|
|
510
510
|
}
|
|
511
511
|
|
|
512
512
|
if (req.method === "GET" && url.pathname === "/api/workspace/branch-merge-preview") {
|
|
513
513
|
if (!authorized(req, config)) return error("unauthorized", 401);
|
|
514
|
-
return branchMergePreviewResponse(url, config.baseDir);
|
|
514
|
+
return await branchMergePreviewResponse(url, config.baseDir);
|
|
515
515
|
}
|
|
516
516
|
|
|
517
517
|
if (req.method === "GET" && url.pathname === "/api/workspace/recovery-branches") {
|
|
518
518
|
if (!authorized(req, config)) return error("unauthorized", 401);
|
|
519
|
-
return recoveryBranchesResponse(url, config.baseDir);
|
|
519
|
+
return await recoveryBranchesResponse(url, config.baseDir);
|
|
520
520
|
}
|
|
521
521
|
|
|
522
522
|
if (req.method === "GET" && url.pathname === "/api/providers") {
|
package/src/command-poller.ts
CHANGED
|
@@ -8,14 +8,20 @@ interface CommandPollerOptions {
|
|
|
8
8
|
relay: Pick<RelayClient, "connected" | "pollCommands">;
|
|
9
9
|
control: CommandPollerControl;
|
|
10
10
|
log?: (message: string) => void;
|
|
11
|
+
intervalMs?: number;
|
|
12
|
+
errorBackoffMs?: number;
|
|
11
13
|
}
|
|
12
14
|
|
|
13
|
-
export function createCommandPoller({ relay, control, log = console.error }: CommandPollerOptions) {
|
|
15
|
+
export function createCommandPoller({ relay, control, log = console.error, intervalMs = 3_000, errorBackoffMs = 3_000 }: CommandPollerOptions) {
|
|
14
16
|
let inFlight = false;
|
|
17
|
+
let stopped = true;
|
|
18
|
+
let timer: ReturnType<typeof setTimeout> | undefined;
|
|
19
|
+
let lastTickErrored = false;
|
|
15
20
|
|
|
16
21
|
async function tick(): Promise<boolean> {
|
|
17
22
|
if (!relay.connected || inFlight) return false;
|
|
18
23
|
inFlight = true;
|
|
24
|
+
lastTickErrored = false;
|
|
19
25
|
try {
|
|
20
26
|
const commands = await relay.pollCommands();
|
|
21
27
|
if (commands.length > 0) {
|
|
@@ -27,6 +33,7 @@ export function createCommandPoller({ relay, control, log = console.error }: Com
|
|
|
27
33
|
}
|
|
28
34
|
return true;
|
|
29
35
|
} catch (err) {
|
|
36
|
+
lastTickErrored = true;
|
|
30
37
|
log(`[orchestrator] Poll error: ${err}`);
|
|
31
38
|
return false;
|
|
32
39
|
} finally {
|
|
@@ -34,8 +41,44 @@ export function createCommandPoller({ relay, control, log = console.error }: Com
|
|
|
34
41
|
}
|
|
35
42
|
}
|
|
36
43
|
|
|
44
|
+
function schedule(delayMs: number): void {
|
|
45
|
+
if (stopped) return;
|
|
46
|
+
timer = setTimeout(() => {
|
|
47
|
+
timer = undefined;
|
|
48
|
+
void runCycle();
|
|
49
|
+
}, delayMs);
|
|
50
|
+
timer.unref?.();
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
async function runCycle(): Promise<void> {
|
|
54
|
+
let errored = false;
|
|
55
|
+
try {
|
|
56
|
+
await tick();
|
|
57
|
+
errored = lastTickErrored;
|
|
58
|
+
} catch (err) {
|
|
59
|
+
errored = true;
|
|
60
|
+
log(`[orchestrator] Poll loop error: ${err}`);
|
|
61
|
+
} finally {
|
|
62
|
+
if (!stopped) schedule(errored ? errorBackoffMs : intervalMs);
|
|
63
|
+
}
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
function start(): void {
|
|
67
|
+
if (!stopped) return;
|
|
68
|
+
stopped = false;
|
|
69
|
+
schedule(0);
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
function stop(): void {
|
|
73
|
+
stopped = true;
|
|
74
|
+
if (timer) clearTimeout(timer);
|
|
75
|
+
timer = undefined;
|
|
76
|
+
}
|
|
77
|
+
|
|
37
78
|
return {
|
|
38
79
|
tick,
|
|
80
|
+
start,
|
|
81
|
+
stop,
|
|
39
82
|
get inFlight() {
|
|
40
83
|
return inFlight;
|
|
41
84
|
},
|
package/src/control.ts
CHANGED
|
@@ -33,16 +33,10 @@ export function createControlHandler(
|
|
|
33
33
|
|
|
34
34
|
async function handleSpawn(ctrl: Record<string, any>): Promise<boolean> {
|
|
35
35
|
const opts = spawnOptionsFromControl(ctrl, config);
|
|
36
|
-
|
|
37
|
-
|
|
38
|
-
|
|
39
|
-
|
|
40
|
-
console.error(`[orchestrator] Spawned ${opts.provider} agent: ${agent.tmuxSession}`);
|
|
41
|
-
return true;
|
|
42
|
-
} catch (err) {
|
|
43
|
-
console.error(`[orchestrator] Spawn failed: ${err}`);
|
|
44
|
-
return false;
|
|
45
|
-
}
|
|
36
|
+
const agent = await spawnAgent(opts, config);
|
|
37
|
+
managedAgents.push(agent);
|
|
38
|
+
console.error(`[orchestrator] Spawned ${opts.provider} agent: ${agent.tmuxSession}`);
|
|
39
|
+
return true;
|
|
46
40
|
}
|
|
47
41
|
|
|
48
42
|
async function handleShutdown(ctrl: Record<string, any>, restart = false): Promise<Record<string, unknown>> {
|
|
@@ -102,7 +96,7 @@ export function createControlHandler(
|
|
|
102
96
|
const result = await handleShutdown(command.params, command.type === "agent.restart");
|
|
103
97
|
await relay.updateCommand(command.id, "succeeded", result);
|
|
104
98
|
} else if (command.type === "workspace.cleanup") {
|
|
105
|
-
const result = cleanupWorkspace({
|
|
99
|
+
const result = await cleanupWorkspace({
|
|
106
100
|
id: typeof command.params.workspaceId === "string" ? command.params.workspaceId : undefined,
|
|
107
101
|
repoRoot: typeof command.params.repoRoot === "string" ? command.params.repoRoot : undefined,
|
|
108
102
|
worktreePath: typeof command.params.worktreePath === "string" ? command.params.worktreePath : undefined,
|
|
@@ -114,7 +108,7 @@ export function createControlHandler(
|
|
|
114
108
|
});
|
|
115
109
|
await relay.updateCommand(command.id, "succeeded", result);
|
|
116
110
|
} else if (command.type === "workspace.reconcile") {
|
|
117
|
-
const result = reconcileWorkspace({
|
|
111
|
+
const result = await reconcileWorkspace({
|
|
118
112
|
id: typeof command.params.workspaceId === "string" ? command.params.workspaceId : undefined,
|
|
119
113
|
repoRoot: typeof command.params.repoRoot === "string" ? command.params.repoRoot : undefined,
|
|
120
114
|
worktreePath: typeof command.params.worktreePath === "string" ? command.params.worktreePath : undefined,
|
|
@@ -124,7 +118,7 @@ export function createControlHandler(
|
|
|
124
118
|
});
|
|
125
119
|
await relay.updateCommand(command.id, "succeeded", result);
|
|
126
120
|
} else if (command.type === "workspace.merge") {
|
|
127
|
-
const result = mergeWorkspace({
|
|
121
|
+
const result = await mergeWorkspace({
|
|
128
122
|
id: typeof command.params.workspaceId === "string" ? command.params.workspaceId : undefined,
|
|
129
123
|
repoRoot: typeof command.params.repoRoot === "string" ? command.params.repoRoot : undefined,
|
|
130
124
|
worktreePath: typeof command.params.worktreePath === "string" ? command.params.worktreePath : undefined,
|
|
@@ -149,7 +143,7 @@ export function createControlHandler(
|
|
|
149
143
|
await relay.updateCommand(command.id, mergeCommandStatus(result), result as unknown as Record<string, unknown>, result.error);
|
|
150
144
|
} else if (command.type === "workspace.pr-arm-auto-merge") {
|
|
151
145
|
const rawPrNumber = command.params.prNumber;
|
|
152
|
-
const result = armWorkspacePrAutoMerge({
|
|
146
|
+
const result = await armWorkspacePrAutoMerge({
|
|
153
147
|
id: typeof command.params.workspaceId === "string" ? command.params.workspaceId : undefined,
|
|
154
148
|
repoRoot: typeof command.params.repoRoot === "string" ? command.params.repoRoot : undefined,
|
|
155
149
|
worktreePath: typeof command.params.worktreePath === "string" ? command.params.worktreePath : undefined,
|
|
@@ -160,7 +154,7 @@ export function createControlHandler(
|
|
|
160
154
|
await relay.updateCommand(command.id, result.autoMergeArmed ? "succeeded" : "failed", result as unknown as Record<string, unknown>, result.error);
|
|
161
155
|
} else if (command.type === "workspace.pr-merge") {
|
|
162
156
|
const rawPrNumber = command.params.prNumber;
|
|
163
|
-
const result = mergeWorkspacePr({
|
|
157
|
+
const result = await mergeWorkspacePr({
|
|
164
158
|
id: typeof command.params.workspaceId === "string" ? command.params.workspaceId : undefined,
|
|
165
159
|
repoRoot: typeof command.params.repoRoot === "string" ? command.params.repoRoot : undefined,
|
|
166
160
|
worktreePath: typeof command.params.worktreePath === "string" ? command.params.worktreePath : undefined,
|
|
@@ -171,7 +165,7 @@ export function createControlHandler(
|
|
|
171
165
|
await relay.updateCommand(command.id, result.relayMerged ? "succeeded" : "failed", result as unknown as Record<string, unknown>, result.error);
|
|
172
166
|
} else if (command.type === "workspace.pr-refresh") {
|
|
173
167
|
const rawPrNumber = command.params.prNumber;
|
|
174
|
-
const result = refreshWorkspacePrBranch({
|
|
168
|
+
const result = await refreshWorkspacePrBranch({
|
|
175
169
|
id: typeof command.params.workspaceId === "string" ? command.params.workspaceId : undefined,
|
|
176
170
|
repoRoot: typeof command.params.repoRoot === "string" ? command.params.repoRoot : undefined,
|
|
177
171
|
worktreePath: typeof command.params.worktreePath === "string" ? command.params.worktreePath : undefined,
|
|
@@ -181,14 +175,14 @@ export function createControlHandler(
|
|
|
181
175
|
});
|
|
182
176
|
await relay.updateCommand(command.id, result.prRefreshed ? "succeeded" : "failed", result as unknown as Record<string, unknown>, result.error);
|
|
183
177
|
} else if (command.type === "workspace.deps-refresh") {
|
|
184
|
-
const result = refreshWorkspaceDeps(
|
|
178
|
+
const result = await refreshWorkspaceDeps(
|
|
185
179
|
typeof command.params.repoRoot === "string" ? command.params.repoRoot : "",
|
|
186
180
|
typeof command.params.worktreePath === "string" ? command.params.worktreePath : "",
|
|
187
181
|
{ checkOnly: command.params.checkOnly === true },
|
|
188
182
|
);
|
|
189
183
|
await relay.updateCommand(command.id, "succeeded", { workspaceId: typeof command.params.workspaceId === "string" ? command.params.workspaceId : undefined, ...result });
|
|
190
184
|
} else if (command.type === "workspace.recovery-branch-discard") {
|
|
191
|
-
const result = discardRecoveryBranch({
|
|
185
|
+
const result = await discardRecoveryBranch({
|
|
192
186
|
repoRoot: typeof command.params.repoRoot === "string" ? command.params.repoRoot : undefined,
|
|
193
187
|
branch: typeof command.params.branch === "string" ? command.params.branch : undefined,
|
|
194
188
|
baseRef: typeof command.params.baseRef === "string" ? command.params.baseRef : undefined,
|
|
@@ -197,7 +191,7 @@ export function createControlHandler(
|
|
|
197
191
|
});
|
|
198
192
|
await relay.updateCommand(command.id, "succeeded", result as unknown as Record<string, unknown>);
|
|
199
193
|
} else if (command.type === "workspace.idle-refresh") {
|
|
200
|
-
const result = idleRefreshWorktree({
|
|
194
|
+
const result = await idleRefreshWorktree({
|
|
201
195
|
id: typeof command.params.workspaceId === "string" ? command.params.workspaceId : undefined,
|
|
202
196
|
repoRoot: typeof command.params.repoRoot === "string" ? command.params.repoRoot : undefined,
|
|
203
197
|
worktreePath: typeof command.params.worktreePath === "string" ? command.params.worktreePath : undefined,
|
|
@@ -207,7 +201,7 @@ export function createControlHandler(
|
|
|
207
201
|
});
|
|
208
202
|
await relay.updateCommand(command.id, result.error ? "failed" : "succeeded", result as unknown as Record<string, unknown>, result.error);
|
|
209
203
|
} else if (command.type === "workspace.prune") {
|
|
210
|
-
const result = pruneWorktrees({
|
|
204
|
+
const result = await pruneWorktrees({
|
|
211
205
|
repoRoot: typeof command.params.repoRoot === "string" ? command.params.repoRoot : undefined,
|
|
212
206
|
});
|
|
213
207
|
await relay.updateCommand(command.id, "succeeded", result);
|
|
@@ -297,6 +291,7 @@ function spawnOptionsFromRecord(source: Record<string, any>, config: Orchestrato
|
|
|
297
291
|
automationRunId: typeof source.automationRunId === "string" ? source.automationRunId : undefined,
|
|
298
292
|
requestedVia: typeof source.requestedVia === "string" ? source.requestedVia : undefined,
|
|
299
293
|
resumeWorkspace: parseResumeWorkspace(source.resumeWorkspace),
|
|
294
|
+
acquisition: parseProjectAcquisition(source.acquisition),
|
|
300
295
|
};
|
|
301
296
|
}
|
|
302
297
|
|
|
@@ -331,3 +326,22 @@ function parseResumeWorkspace(value: unknown): import("./workspace-probe/types")
|
|
|
331
326
|
baseSha: typeof value.baseSha === "string" ? value.baseSha : undefined,
|
|
332
327
|
};
|
|
333
328
|
}
|
|
329
|
+
|
|
330
|
+
function parseProjectAcquisition(value: unknown): import("./spawn/types").ProjectAcquisitionManifest | undefined {
|
|
331
|
+
if (!isRecord(value)) return undefined;
|
|
332
|
+
if (value.mode !== "project-root" || value.sync !== "ff-only") return undefined;
|
|
333
|
+
const projectId = typeof value.projectId === "string" ? value.projectId : undefined;
|
|
334
|
+
const rootPath = typeof value.rootPath === "string" ? value.rootPath : undefined;
|
|
335
|
+
const cwd = typeof value.cwd === "string" ? value.cwd : undefined;
|
|
336
|
+
const remoteUrl = typeof value.remoteUrl === "string" ? value.remoteUrl : undefined;
|
|
337
|
+
if (!projectId || !rootPath || !cwd || !remoteUrl) return undefined;
|
|
338
|
+
return {
|
|
339
|
+
mode: "project-root",
|
|
340
|
+
projectId,
|
|
341
|
+
rootPath,
|
|
342
|
+
cwd,
|
|
343
|
+
remoteUrl,
|
|
344
|
+
ref: typeof value.ref === "string" ? value.ref : undefined,
|
|
345
|
+
sync: "ff-only",
|
|
346
|
+
};
|
|
347
|
+
}
|
package/src/git.ts
CHANGED
|
@@ -2,6 +2,8 @@
|
|
|
2
2
|
// Extracted from workspace-probe.ts so that giant keeps shrinking (epic #291) and the
|
|
3
3
|
// `git -C` invocation lives in one place.
|
|
4
4
|
|
|
5
|
+
import { execProcess } from "./process";
|
|
6
|
+
|
|
5
7
|
interface GitResult {
|
|
6
8
|
ok: boolean;
|
|
7
9
|
stdout: string;
|
|
@@ -9,36 +11,18 @@ interface GitResult {
|
|
|
9
11
|
}
|
|
10
12
|
|
|
11
13
|
/** Run `git -C cwd <args>` and capture trimmed stdout/stderr; never throws. */
|
|
12
|
-
export function git(args: string[], cwd: string): GitResult {
|
|
13
|
-
|
|
14
|
-
stdin: "ignore",
|
|
15
|
-
stdout: "pipe",
|
|
16
|
-
stderr: "pipe",
|
|
17
|
-
});
|
|
18
|
-
return {
|
|
19
|
-
ok: proc.exitCode === 0,
|
|
20
|
-
stdout: proc.stdout.toString().trim(),
|
|
21
|
-
stderr: proc.stderr.toString().trim(),
|
|
22
|
-
};
|
|
14
|
+
export async function git(args: string[], cwd: string): Promise<GitResult> {
|
|
15
|
+
return await execProcess(["git", "-C", cwd, ...args]);
|
|
23
16
|
}
|
|
24
17
|
|
|
25
18
|
/** Run `git -C cwd <args>` and preserve stdout exactly for path-safe parsers. */
|
|
26
|
-
export function gitRaw(args: string[], cwd: string): GitResult {
|
|
27
|
-
|
|
28
|
-
stdin: "ignore",
|
|
29
|
-
stdout: "pipe",
|
|
30
|
-
stderr: "pipe",
|
|
31
|
-
});
|
|
32
|
-
return {
|
|
33
|
-
ok: proc.exitCode === 0,
|
|
34
|
-
stdout: proc.stdout.toString(),
|
|
35
|
-
stderr: proc.stderr.toString().trim(),
|
|
36
|
-
};
|
|
19
|
+
export async function gitRaw(args: string[], cwd: string): Promise<GitResult> {
|
|
20
|
+
return await execProcess(["git", "-C", cwd, ...args], { trimStdout: false });
|
|
37
21
|
}
|
|
38
22
|
|
|
39
23
|
/** Like {@link git} but throws on a non-zero exit, returning stdout on success. */
|
|
40
|
-
export function requireGit(args: string[], cwd: string): string {
|
|
41
|
-
const result = git(args, cwd);
|
|
24
|
+
export async function requireGit(args: string[], cwd: string): Promise<string> {
|
|
25
|
+
const result = await git(args, cwd);
|
|
42
26
|
if (!result.ok) throw new Error(result.stderr || `git ${args.join(" ")} failed`);
|
|
43
27
|
return result.stdout;
|
|
44
28
|
}
|
package/src/index.ts
CHANGED
|
@@ -60,7 +60,7 @@ const sharedCallmux = new SharedCallmuxSupervisor(config);
|
|
|
60
60
|
const POLL_INTERVAL_MS = 3_000;
|
|
61
61
|
const REGISTER_RETRY_MS = 5_000;
|
|
62
62
|
const GUEST_REAP_INTERVAL_MS = 60_000;
|
|
63
|
-
let
|
|
63
|
+
let commandPoller: ReturnType<typeof createCommandPoller> | null = null;
|
|
64
64
|
let healthCheckTimer: Timer | null = null;
|
|
65
65
|
let guestReaperTimer: Timer | null = null;
|
|
66
66
|
let apiServer: { stop(): void; url: string } | null = null;
|
|
@@ -124,10 +124,8 @@ async function startup(): Promise<void> {
|
|
|
124
124
|
}
|
|
125
125
|
|
|
126
126
|
function startPolling(): void {
|
|
127
|
-
|
|
128
|
-
|
|
129
|
-
await commandPoller.tick();
|
|
130
|
-
}, POLL_INTERVAL_MS);
|
|
127
|
+
commandPoller = createCommandPoller({ relay, control, intervalMs: POLL_INTERVAL_MS, errorBackoffMs: POLL_INTERVAL_MS });
|
|
128
|
+
commandPoller.start();
|
|
131
129
|
}
|
|
132
130
|
|
|
133
131
|
async function registerUntilConnected(): Promise<void> {
|
|
@@ -208,7 +206,7 @@ async function healthCheck(): Promise<void> {
|
|
|
208
206
|
|
|
209
207
|
async function shutdown(): Promise<void> {
|
|
210
208
|
console.error("[orchestrator] Shutting down...");
|
|
211
|
-
|
|
209
|
+
commandPoller?.stop();
|
|
212
210
|
if (healthCheckTimer) clearInterval(healthCheckTimer);
|
|
213
211
|
if (guestReaperTimer) clearInterval(guestReaperTimer);
|
|
214
212
|
if (apiServer) apiServer.stop();
|
package/src/process.ts
ADDED
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
export interface ExecResult {
|
|
2
|
+
ok: boolean;
|
|
3
|
+
exitCode: number | null;
|
|
4
|
+
stdout: string;
|
|
5
|
+
stderr: string;
|
|
6
|
+
}
|
|
7
|
+
|
|
8
|
+
interface ExecOptions {
|
|
9
|
+
cwd?: string;
|
|
10
|
+
env?: Record<string, string | undefined>;
|
|
11
|
+
stdout?: "pipe" | "ignore";
|
|
12
|
+
stderr?: "pipe" | "ignore";
|
|
13
|
+
trimStdout?: boolean;
|
|
14
|
+
trimStderr?: boolean;
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
async function readStream(stream: ReadableStream<Uint8Array> | undefined): Promise<string> {
|
|
18
|
+
if (!stream) return "";
|
|
19
|
+
return await new Response(stream).text();
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
export async function execProcess(cmd: string[], options: ExecOptions = {}): Promise<ExecResult> {
|
|
23
|
+
const proc = Bun.spawn(cmd, {
|
|
24
|
+
cwd: options.cwd,
|
|
25
|
+
env: options.env,
|
|
26
|
+
stdin: "ignore",
|
|
27
|
+
stdout: options.stdout ?? "pipe",
|
|
28
|
+
stderr: options.stderr ?? "pipe",
|
|
29
|
+
});
|
|
30
|
+
const [exitCode, stdout, stderr] = await Promise.all([
|
|
31
|
+
proc.exited,
|
|
32
|
+
options.stdout === "ignore" ? Promise.resolve("") : readStream(proc.stdout),
|
|
33
|
+
options.stderr === "ignore" ? Promise.resolve("") : readStream(proc.stderr),
|
|
34
|
+
]);
|
|
35
|
+
return {
|
|
36
|
+
ok: exitCode === 0,
|
|
37
|
+
exitCode,
|
|
38
|
+
stdout: options.trimStdout === false ? stdout : stdout.trim(),
|
|
39
|
+
stderr: options.trimStderr === false ? stderr : stderr.trim(),
|
|
40
|
+
};
|
|
41
|
+
}
|
package/src/relay.ts
CHANGED
|
@@ -68,6 +68,8 @@ export interface RelayCommand {
|
|
|
68
68
|
const RECONNECT_INITIAL_MS = 30_000;
|
|
69
69
|
const RECONNECT_MAX_MS = 3_600_000; // 1 hour
|
|
70
70
|
const RECONNECT_JITTER_MS = 1_000;
|
|
71
|
+
const TRANSIENT_ABORT_RECONNECT_MS = 250;
|
|
72
|
+
const ABORT_FAILURES_BEFORE_DISCONNECT = 3;
|
|
71
73
|
|
|
72
74
|
export function buildRegistrationMeta(
|
|
73
75
|
config: Pick<OrchestratorConfig, "tmuxPrefix">,
|
|
@@ -94,6 +96,7 @@ export function createRelayClient(config: OrchestratorConfig, probeCache: Provid
|
|
|
94
96
|
const agentId = `orchestrator-${config.id}`;
|
|
95
97
|
let heartbeatTimer: Timer | null = null;
|
|
96
98
|
let connected = false;
|
|
99
|
+
let abortFailures = 0;
|
|
97
100
|
const reconnectMgr = new ReconnectionManager({ initialMs: RECONNECT_INITIAL_MS, maxMs: RECONNECT_MAX_MS, jitterMs: RECONNECT_JITTER_MS });
|
|
98
101
|
let cursorFloor = 0;
|
|
99
102
|
let apiUrl: string | undefined;
|
|
@@ -134,6 +137,7 @@ export function createRelayClient(config: OrchestratorConfig, probeCache: Provid
|
|
|
134
137
|
const registered = await res.json().catch(() => null) as { runtimeToken?: { token?: string } } | null;
|
|
135
138
|
if (registered?.runtimeToken?.token) http.setToken(registered.runtimeToken.token);
|
|
136
139
|
connected = true;
|
|
140
|
+
abortFailures = 0;
|
|
137
141
|
reconnectMgr.reset();
|
|
138
142
|
|
|
139
143
|
// Bootstrap message cursor
|
|
@@ -167,7 +171,20 @@ export function createRelayClient(config: OrchestratorConfig, probeCache: Provid
|
|
|
167
171
|
connected = true;
|
|
168
172
|
reconnectMgr.reset();
|
|
169
173
|
}
|
|
174
|
+
abortFailures = 0;
|
|
170
175
|
} catch (err) {
|
|
176
|
+
if (isAbortError(err)) {
|
|
177
|
+
abortFailures += 1;
|
|
178
|
+
if (connected && abortFailures === 1) {
|
|
179
|
+
console.error(`[orchestrator] Relay heartbeat timed out: ${err}; retrying quickly`);
|
|
180
|
+
}
|
|
181
|
+
if (abortFailures < ABORT_FAILURES_BEFORE_DISCONNECT) {
|
|
182
|
+
await reconnect(TRANSIENT_ABORT_RECONNECT_MS);
|
|
183
|
+
return;
|
|
184
|
+
}
|
|
185
|
+
} else {
|
|
186
|
+
abortFailures = 0;
|
|
187
|
+
}
|
|
171
188
|
if (connected) {
|
|
172
189
|
console.error(`[orchestrator] Lost connection to relay: ${err}`);
|
|
173
190
|
connected = false;
|
|
@@ -176,9 +193,10 @@ export function createRelayClient(config: OrchestratorConfig, probeCache: Provid
|
|
|
176
193
|
}
|
|
177
194
|
}
|
|
178
195
|
|
|
179
|
-
async function reconnect(): Promise<void> {
|
|
180
|
-
const delayMs = reconnectMgr.nextDelay();
|
|
181
|
-
|
|
196
|
+
async function reconnect(delayOverrideMs?: number): Promise<void> {
|
|
197
|
+
const delayMs = delayOverrideMs ?? reconnectMgr.nextDelay();
|
|
198
|
+
const label = delayMs < 1000 ? `${delayMs}ms` : `${Math.round(delayMs / 1000)}s`;
|
|
199
|
+
console.error(`[orchestrator] Reconnecting in ${label}...`);
|
|
182
200
|
await new Promise((resolve) => setTimeout(resolve, delayMs));
|
|
183
201
|
try {
|
|
184
202
|
await register();
|
|
@@ -258,3 +276,8 @@ export function createRelayClient(config: OrchestratorConfig, probeCache: Provid
|
|
|
258
276
|
get connected() { return connected; },
|
|
259
277
|
};
|
|
260
278
|
}
|
|
279
|
+
|
|
280
|
+
function isAbortError(error: unknown): boolean {
|
|
281
|
+
return error instanceof DOMException && error.name === "AbortError"
|
|
282
|
+
|| typeof error === "object" && error !== null && (error as { name?: string }).name === "AbortError";
|
|
283
|
+
}
|
|
@@ -0,0 +1,193 @@
|
|
|
1
|
+
import { createHash } from "node:crypto";
|
|
2
|
+
import { existsSync, mkdirSync, renameSync, rmSync, statSync, writeFileSync } from "node:fs";
|
|
3
|
+
import { basename, dirname, join, resolve } from "node:path";
|
|
4
|
+
import { errMessage, isPathWithinBase } from "agent-relay-sdk";
|
|
5
|
+
import { git, requireGit } from "../git";
|
|
6
|
+
import { execProcess } from "../process";
|
|
7
|
+
import type { ProjectAcquisitionManifest } from "./types";
|
|
8
|
+
|
|
9
|
+
export interface ProjectAcquisitionResult {
|
|
10
|
+
applied: boolean;
|
|
11
|
+
action: "cloned" | "synced" | "noop";
|
|
12
|
+
rootPath: string;
|
|
13
|
+
remoteUrl: string;
|
|
14
|
+
ref?: string;
|
|
15
|
+
headSha?: string;
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
const inFlight = new Map<string, Promise<ProjectAcquisitionResult>>();
|
|
19
|
+
const LOCK_TIMEOUT_MS = 5 * 60_000;
|
|
20
|
+
const LOCK_POLL_MS = 100;
|
|
21
|
+
|
|
22
|
+
export async function applyProjectAcquisitionManifest(
|
|
23
|
+
manifest: ProjectAcquisitionManifest | undefined,
|
|
24
|
+
baseDir: string,
|
|
25
|
+
): Promise<ProjectAcquisitionResult | undefined> {
|
|
26
|
+
if (!manifest) return undefined;
|
|
27
|
+
const rootPath = resolve(manifest.rootPath);
|
|
28
|
+
const prior = inFlight.get(rootPath);
|
|
29
|
+
if (prior) return prior;
|
|
30
|
+
const run = withAcquisitionLock(rootPath, baseDir, () => applyManifestLocked(manifest, baseDir));
|
|
31
|
+
inFlight.set(rootPath, run);
|
|
32
|
+
try {
|
|
33
|
+
return await run;
|
|
34
|
+
} finally {
|
|
35
|
+
if (inFlight.get(rootPath) === run) inFlight.delete(rootPath);
|
|
36
|
+
}
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
async function withAcquisitionLock(rootPath: string, baseDir: string, fn: () => Promise<ProjectAcquisitionResult>): Promise<ProjectAcquisitionResult> {
|
|
40
|
+
const lockDir = join(resolve(baseDir), ".agent-relay", "locks", `acquire-${hash(rootPath)}.lock`);
|
|
41
|
+
mkdirSync(dirname(lockDir), { recursive: true });
|
|
42
|
+
const started = Date.now();
|
|
43
|
+
for (;;) {
|
|
44
|
+
try {
|
|
45
|
+
mkdirSync(lockDir);
|
|
46
|
+
writeFileSync(join(lockDir, "owner"), `${process.pid}\n${rootPath}\n`);
|
|
47
|
+
break;
|
|
48
|
+
} catch (error) {
|
|
49
|
+
if (Date.now() - started > LOCK_TIMEOUT_MS) {
|
|
50
|
+
throw new Error(`repo acquisition lock timed out for ${rootPath}: ${errMessage(error)}`);
|
|
51
|
+
}
|
|
52
|
+
await new Promise((resolve) => setTimeout(resolve, LOCK_POLL_MS));
|
|
53
|
+
}
|
|
54
|
+
}
|
|
55
|
+
try {
|
|
56
|
+
return await fn();
|
|
57
|
+
} finally {
|
|
58
|
+
rmSync(lockDir, { recursive: true, force: true });
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
async function applyManifestLocked(manifest: ProjectAcquisitionManifest, baseDir: string): Promise<ProjectAcquisitionResult> {
|
|
63
|
+
validateManifest(manifest, baseDir);
|
|
64
|
+
const rootPath = resolve(manifest.rootPath);
|
|
65
|
+
const remoteUrl = manifest.remoteUrl.trim();
|
|
66
|
+
let action: ProjectAcquisitionResult["action"] = "noop";
|
|
67
|
+
if (!existsSync(rootPath)) {
|
|
68
|
+
await cloneRoot(manifest, baseDir);
|
|
69
|
+
action = "cloned";
|
|
70
|
+
}
|
|
71
|
+
const syncAction = await syncRoot(manifest);
|
|
72
|
+
if (syncAction === "synced" && action !== "cloned") action = "synced";
|
|
73
|
+
return {
|
|
74
|
+
applied: true,
|
|
75
|
+
action,
|
|
76
|
+
rootPath,
|
|
77
|
+
remoteUrl,
|
|
78
|
+
...(manifest.ref ? { ref: manifest.ref } : {}),
|
|
79
|
+
headSha: (await git(["rev-parse", "HEAD"], rootPath)).stdout || undefined,
|
|
80
|
+
};
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
function validateManifest(manifest: ProjectAcquisitionManifest, baseDir: string): void {
|
|
84
|
+
const rootPath = resolve(manifest.rootPath);
|
|
85
|
+
if (!manifest.remoteUrl.trim()) throw new Error("project acquisition remoteUrl is required");
|
|
86
|
+
if (!isPathWithinBase(rootPath, baseDir) || rootPath === resolve(baseDir)) {
|
|
87
|
+
throw new Error(`project acquisition rootPath must be within orchestrator baseDir: ${baseDir}`);
|
|
88
|
+
}
|
|
89
|
+
if (!isPathWithinBase(resolve(manifest.cwd), rootPath)) {
|
|
90
|
+
throw new Error(`project acquisition cwd must be within rootPath: ${manifest.cwd}`);
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
async function cloneRoot(manifest: ProjectAcquisitionManifest, baseDir: string): Promise<void> {
|
|
95
|
+
const rootPath = resolve(manifest.rootPath);
|
|
96
|
+
const parent = dirname(rootPath);
|
|
97
|
+
if (!isPathWithinBase(parent, baseDir)) throw new Error(`project acquisition parent must be within orchestrator baseDir: ${parent}`);
|
|
98
|
+
mkdirSync(parent, { recursive: true });
|
|
99
|
+
const tmp = join(parent, `.${basename(rootPath)}.agent-relay-clone-${process.pid}-${Date.now()}`);
|
|
100
|
+
rmSync(tmp, { recursive: true, force: true });
|
|
101
|
+
const args = ["clone", "--origin", "origin"];
|
|
102
|
+
if (manifest.ref) args.push("--branch", manifest.ref);
|
|
103
|
+
args.push(manifest.remoteUrl.trim(), tmp);
|
|
104
|
+
const cloned = await runGit(args, parent);
|
|
105
|
+
if (!cloned.ok) {
|
|
106
|
+
rmSync(tmp, { recursive: true, force: true });
|
|
107
|
+
throw new Error(`git clone failed for ${rootPath}: ${cloned.stderr || cloned.stdout}`);
|
|
108
|
+
}
|
|
109
|
+
try {
|
|
110
|
+
renameSync(tmp, rootPath);
|
|
111
|
+
} catch (error) {
|
|
112
|
+
rmSync(tmp, { recursive: true, force: true });
|
|
113
|
+
throw new Error(`failed to install cloned repo at ${rootPath}: ${errMessage(error)}`);
|
|
114
|
+
}
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
async function syncRoot(manifest: ProjectAcquisitionManifest): Promise<"synced" | "noop"> {
|
|
118
|
+
const rootPath = resolve(manifest.rootPath);
|
|
119
|
+
await assertExistingGitRoot(rootPath);
|
|
120
|
+
await assertRemote(rootPath, manifest.remoteUrl.trim());
|
|
121
|
+
const status = await git(["status", "--porcelain"], rootPath);
|
|
122
|
+
if (!status.ok) throw new Error(`git status failed for ${rootPath}: ${status.stderr}`);
|
|
123
|
+
if (status.stdout.trim()) throw new Error(`project root ${rootPath} has local changes; refusing ff-only sync before spawn`);
|
|
124
|
+
const fetch = await git(["fetch", "--prune", "origin"], rootPath);
|
|
125
|
+
if (!fetch.ok) throw new Error(`git fetch failed for ${rootPath}: ${fetch.stderr || fetch.stdout}`);
|
|
126
|
+
const target = await checkoutSyncTarget(rootPath, manifest.ref);
|
|
127
|
+
if (!target) return "noop";
|
|
128
|
+
const head = await requireGit(["rev-parse", "HEAD"], rootPath);
|
|
129
|
+
const targetHead = await requireGit(["rev-parse", target], rootPath);
|
|
130
|
+
if (head === targetHead) return "noop";
|
|
131
|
+
if (!(await git(["merge-base", "--is-ancestor", "HEAD", target], rootPath)).ok) {
|
|
132
|
+
throw new Error(`project root ${rootPath} has diverged from ${target}; refusing non-fast-forward sync`);
|
|
133
|
+
}
|
|
134
|
+
const merged = await git(["merge", "--ff-only", target], rootPath);
|
|
135
|
+
if (!merged.ok) throw new Error(`git ff-only sync failed for ${rootPath}: ${merged.stderr || merged.stdout}`);
|
|
136
|
+
return "synced";
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
async function assertExistingGitRoot(rootPath: string): Promise<void> {
|
|
140
|
+
let stat;
|
|
141
|
+
try {
|
|
142
|
+
stat = statSync(rootPath);
|
|
143
|
+
} catch (error) {
|
|
144
|
+
throw new Error(`project root does not exist after acquisition: ${rootPath}: ${errMessage(error)}`);
|
|
145
|
+
}
|
|
146
|
+
if (!stat.isDirectory()) throw new Error(`project root exists but is not a directory: ${rootPath}`);
|
|
147
|
+
const top = await git(["rev-parse", "--show-toplevel"], rootPath);
|
|
148
|
+
if (!top.ok || resolve(top.stdout) !== rootPath) throw new Error(`project root exists but is not a git checkout root: ${rootPath}`);
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
async function assertRemote(rootPath: string, remoteUrl: string): Promise<void> {
|
|
152
|
+
const current = await git(["remote", "get-url", "origin"], rootPath);
|
|
153
|
+
if (!current.ok || !current.stdout) throw new Error(`project root ${rootPath} has no origin remote`);
|
|
154
|
+
if (current.stdout.trim() !== remoteUrl) {
|
|
155
|
+
throw new Error(`project root ${rootPath} origin mismatch: expected ${remoteUrl}, found ${current.stdout.trim()}`);
|
|
156
|
+
}
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
async function checkoutSyncTarget(rootPath: string, ref: string | undefined): Promise<string | undefined> {
|
|
160
|
+
if (ref?.trim()) {
|
|
161
|
+
const name = ref.trim();
|
|
162
|
+
const remoteBranch = `refs/remotes/origin/${name}`;
|
|
163
|
+
if ((await git(["show-ref", "--verify", "--quiet", remoteBranch], rootPath)).ok) {
|
|
164
|
+
if ((await git(["show-ref", "--verify", "--quiet", `refs/heads/${name}`], rootPath)).ok) {
|
|
165
|
+
const checked = await git(["checkout", name], rootPath);
|
|
166
|
+
if (!checked.ok) throw new Error(`git checkout ${name} failed for ${rootPath}: ${checked.stderr || checked.stdout}`);
|
|
167
|
+
} else {
|
|
168
|
+
const checked = await git(["checkout", "-B", name, `origin/${name}`], rootPath);
|
|
169
|
+
if (!checked.ok) throw new Error(`git checkout ${name} failed for ${rootPath}: ${checked.stderr || checked.stdout}`);
|
|
170
|
+
}
|
|
171
|
+
return `origin/${name}`;
|
|
172
|
+
}
|
|
173
|
+
if ((await git(["rev-parse", "--verify", name], rootPath)).ok) {
|
|
174
|
+
const checked = await git(["checkout", name], rootPath);
|
|
175
|
+
if (!checked.ok) throw new Error(`git checkout ${name} failed for ${rootPath}: ${checked.stderr || checked.stdout}`);
|
|
176
|
+
return undefined;
|
|
177
|
+
}
|
|
178
|
+
throw new Error(`project acquisition ref "${name}" not found on origin for ${rootPath}`);
|
|
179
|
+
}
|
|
180
|
+
const upstream = await git(["rev-parse", "--abbrev-ref", "--symbolic-full-name", "@{u}"], rootPath);
|
|
181
|
+
if (upstream.ok && upstream.stdout) return upstream.stdout;
|
|
182
|
+
const originHead = await git(["symbolic-ref", "--short", "refs/remotes/origin/HEAD"], rootPath);
|
|
183
|
+
if (originHead.ok && originHead.stdout) return originHead.stdout;
|
|
184
|
+
throw new Error(`project root ${rootPath} has no upstream or origin/HEAD for ff-only sync`);
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
async function runGit(args: string[], cwd: string): Promise<{ ok: boolean; stdout: string; stderr: string }> {
|
|
188
|
+
return await execProcess(["git", ...args], { cwd });
|
|
189
|
+
}
|
|
190
|
+
|
|
191
|
+
function hash(value: string): string {
|
|
192
|
+
return createHash("sha1").update(resolve(value)).digest("hex").slice(0, 16);
|
|
193
|
+
}
|