@love-moon/conductor-cli 0.12.0 → 0.13.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +24 -0
- package/bin/conductor-remote.js +6 -1
- package/bin/conductor.js +1 -1
- package/package.json +5 -5
- package/src/daemon.js +51 -1
- package/src/fire-session-registry.js +19 -0
- package/src/remote/client.js +9 -3
- package/src/remote/exec.js +132 -61
- package/src/remote/wait.js +133 -0
package/CHANGELOG.md
CHANGED
|
@@ -1,5 +1,29 @@
|
|
|
1
1
|
# @love-moon/conductor-cli
|
|
2
2
|
|
|
3
|
+
## 0.13.0
|
|
4
|
+
|
|
5
|
+
### Minor Changes
|
|
6
|
+
|
|
7
|
+
- 8edbed5: `conductor remote exec` is safer to drive from an agent, and gains a `wait` verb.
|
|
8
|
+
|
|
9
|
+
- New `conductor remote wait -t <daemon> <runId>` re-attaches to a command that
|
|
10
|
+
`exec` left running past its `--timeout`, polling and reporting it exactly as
|
|
11
|
+
`exec` would have. `exec` now prints that command in its "still running" hint
|
|
12
|
+
instead of a raw API path.
|
|
13
|
+
- `exec` retries the initial request on `429 too many concurrent remote exec
|
|
14
|
+
requests` with backoff (up to ~15 s), so a burst of parallel short commands no
|
|
15
|
+
longer fails outright. Other errors are still not retried: a 5xx may already
|
|
16
|
+
have started the command.
|
|
17
|
+
- `exec` and `wait` handle SIGINT/SIGTERM: the run id is printed before exiting,
|
|
18
|
+
and with `--kill-on-timeout` the remote command is cancelled as well. Before,
|
|
19
|
+
a tool timeout that killed the CLI silently orphaned the remote process.
|
|
20
|
+
|
|
21
|
+
### Patch Changes
|
|
22
|
+
|
|
23
|
+
- Updated dependencies [80ab13e]
|
|
24
|
+
- @love-moon/conductor-sdk@0.13.0
|
|
25
|
+
- @love-moon/ai-sdk@0.13.0
|
|
26
|
+
|
|
3
27
|
## 0.12.0
|
|
4
28
|
|
|
5
29
|
### Minor Changes
|
package/bin/conductor-remote.js
CHANGED
|
@@ -19,12 +19,14 @@ import { fileURLToPath } from "node:url";
|
|
|
19
19
|
import { EXIT } from "../src/remote/client.js";
|
|
20
20
|
import { runRemoteExec, showHelp as showExecHelp } from "../src/remote/exec.js";
|
|
21
21
|
import { runRemoteCp, showHelp as showCpHelp } from "../src/remote/cp.js";
|
|
22
|
+
import { runRemoteWait, showHelp as showWaitHelp } from "../src/remote/wait.js";
|
|
22
23
|
|
|
23
24
|
export { EXIT };
|
|
24
25
|
|
|
25
26
|
const VERBS = new Map([
|
|
26
27
|
["exec", runRemoteExec],
|
|
27
28
|
["cp", runRemoteCp],
|
|
29
|
+
["wait", runRemoteWait],
|
|
28
30
|
]);
|
|
29
31
|
|
|
30
32
|
const isMainModule = (() => {
|
|
@@ -42,6 +44,7 @@ Usage:
|
|
|
42
44
|
Verbs:
|
|
43
45
|
exec Run a command on another daemon's host
|
|
44
46
|
cp Copy a file to or from another daemon's host
|
|
47
|
+
wait Wait for a command that \`exec\` left running (by run id)
|
|
45
48
|
|
|
46
49
|
Options:
|
|
47
50
|
-h, --help Show this help
|
|
@@ -50,10 +53,12 @@ Examples:
|
|
|
50
53
|
conductor remote exec --target ubuntu --workspace /srv/app -- git log --oneline -5
|
|
51
54
|
conductor remote cp ./build.tar.gz ubuntu:/srv/app/build.tar.gz
|
|
52
55
|
conductor remote cp ubuntu:/var/log/conductor.log ./conductor.log
|
|
56
|
+
conductor remote wait --target ubuntu <runId>
|
|
53
57
|
|
|
54
58
|
For verb-specific help:
|
|
55
59
|
conductor remote exec --help
|
|
56
60
|
conductor remote cp --help
|
|
61
|
+
conductor remote wait --help
|
|
57
62
|
`);
|
|
58
63
|
}
|
|
59
64
|
|
|
@@ -84,7 +89,7 @@ export async function runRemote(argv, deps = {}) {
|
|
|
84
89
|
return handler(argv.slice(1), deps);
|
|
85
90
|
}
|
|
86
91
|
|
|
87
|
-
export { showExecHelp, showCpHelp };
|
|
92
|
+
export { showExecHelp, showCpHelp, showWaitHelp };
|
|
88
93
|
|
|
89
94
|
if (isMainModule) {
|
|
90
95
|
// `process.exitCode` rather than `process.exit()`: writes to a pipe are async,
|
package/bin/conductor.js
CHANGED
|
@@ -137,7 +137,7 @@ Subcommands:
|
|
|
137
137
|
project Manage Conductor projects (list/show/create/...)
|
|
138
138
|
issue Manage issues (list/show/create/update/start/done)
|
|
139
139
|
task Manage tasks (create/list/show/send/messages/schedule)
|
|
140
|
-
remote Run commands and copy files on another daemon's host (exec/cp)
|
|
140
|
+
remote Run commands and copy files on another daemon's host (exec/cp/wait)
|
|
141
141
|
|
|
142
142
|
Options:
|
|
143
143
|
-h, --help Show this help message
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@love-moon/conductor-cli",
|
|
3
|
-
"version": "0.
|
|
4
|
-
"gitCommitId": "
|
|
3
|
+
"version": "0.13.0",
|
|
4
|
+
"gitCommitId": "e1e507f",
|
|
5
5
|
"repository": {
|
|
6
6
|
"type": "git",
|
|
7
7
|
"url": "git+https://github.com/lovemoon-ai/conductor.git"
|
|
@@ -24,8 +24,8 @@
|
|
|
24
24
|
"test": "node --test test/*.test.js"
|
|
25
25
|
},
|
|
26
26
|
"dependencies": {
|
|
27
|
-
"@love-moon/ai-sdk": "0.
|
|
28
|
-
"@love-moon/conductor-sdk": "0.
|
|
27
|
+
"@love-moon/ai-sdk": "0.13.0",
|
|
28
|
+
"@love-moon/conductor-sdk": "0.13.0",
|
|
29
29
|
"@github/copilot-sdk": "^0.3.0",
|
|
30
30
|
"chrome-launcher": "^1.2.1",
|
|
31
31
|
"chrome-remote-interface": "^0.33.0",
|
|
@@ -38,7 +38,7 @@
|
|
|
38
38
|
},
|
|
39
39
|
"optionalDependencies": {
|
|
40
40
|
"@roamhq/wrtc": "^0.10.0",
|
|
41
|
-
"@love-moon/chat-web": "0.
|
|
41
|
+
"@love-moon/chat-web": "0.13.0"
|
|
42
42
|
},
|
|
43
43
|
"pnpm": {
|
|
44
44
|
"onlyBuiltDependencies": [
|
package/src/daemon.js
CHANGED
|
@@ -26,6 +26,7 @@ import {
|
|
|
26
26
|
} from "./conductor-paths.js";
|
|
27
27
|
import {
|
|
28
28
|
deleteFireSessionRecord,
|
|
29
|
+
listFireSessionTaskIds,
|
|
29
30
|
pruneFireSessionRecords,
|
|
30
31
|
readFireSessionRecord,
|
|
31
32
|
resolveFireSessionRegistryDir,
|
|
@@ -2306,6 +2307,30 @@ export function startDaemon(config = {}, deps = {}) {
|
|
|
2306
2307
|
return FIRE_TMUX_MODE_ENABLED && !FIRE_TMUX_MODE_ACTIVE && isTmuxAvailable();
|
|
2307
2308
|
}
|
|
2308
2309
|
|
|
2310
|
+
// The third door into the same mass kill: `tmux` genuinely absent from PATH.
|
|
2311
|
+
// A daemon restarted by launchd/systemd/cron — or during a `brew upgrade
|
|
2312
|
+
// tmux` — logs the direct-spawn fallback and then cannot enumerate sessions
|
|
2313
|
+
// by any means, so the guard above (which needs tmux to answer) declines and
|
|
2314
|
+
// every Fire the predecessor deliberately left running gets killed.
|
|
2315
|
+
//
|
|
2316
|
+
// The hand-off records on disk are the remaining evidence: they name exactly
|
|
2317
|
+
// the tasks whose liveness this process is unable to answer for. Scoped to
|
|
2318
|
+
// those rather than skipping the sweep wholesale, because a host that simply
|
|
2319
|
+
// never installs tmux spawns its Fires directly and those DO die with their
|
|
2320
|
+
// daemon — blanket-skipping would strand them at `running` with nobody left
|
|
2321
|
+
// to report their death, which is worse than killing them. Nothing prunes
|
|
2322
|
+
// the registry in this state (the prune needs a conclusive listing), so a
|
|
2323
|
+
// later start with a healthy tmux still adopts or reaps them properly.
|
|
2324
|
+
function taskIdsWithUnverifiableTmuxFire() {
|
|
2325
|
+
if (!FIRE_TMUX_MODE_ENABLED || FIRE_TMUX_MODE_ACTIVE) return new Set();
|
|
2326
|
+
try {
|
|
2327
|
+
return listFireSessionTaskIds(FIRE_SESSION_REGISTRY_DIR);
|
|
2328
|
+
} catch (error) {
|
|
2329
|
+
logError(`Failed to read tmux hand-off records: ${error?.message || error}`);
|
|
2330
|
+
return new Set();
|
|
2331
|
+
}
|
|
2332
|
+
}
|
|
2333
|
+
|
|
2309
2334
|
// Sessions already identified as orphaned shells and killed. `tmux
|
|
2310
2335
|
// kill-session` is asynchronous, so a sweep running moments later still
|
|
2311
2336
|
// sees the name in `list-sessions` — and by then the hand-off record is
|
|
@@ -4856,12 +4881,26 @@ export function startDaemon(config = {}, deps = {}) {
|
|
|
4856
4881
|
);
|
|
4857
4882
|
return;
|
|
4858
4883
|
}
|
|
4884
|
+
const unverifiableTmuxTaskIds = taskIdsWithUnverifiableTmuxFire();
|
|
4885
|
+
const skippedUnverifiable = [];
|
|
4859
4886
|
const deadTasks = staleTasks.filter((task) => {
|
|
4887
|
+
const taskId = String(task?.id || "");
|
|
4860
4888
|
// A spawn already in flight has no record yet either; killing it
|
|
4861
4889
|
// would shoot down a Fire this daemon is in the middle of starting.
|
|
4862
|
-
if (pendingTaskStarts.has(
|
|
4890
|
+
if (pendingTaskStarts.has(taskId)) return false;
|
|
4891
|
+
if (unverifiableTmuxTaskIds.has(taskId)) {
|
|
4892
|
+
skippedUnverifiable.push(taskId);
|
|
4893
|
+
return false;
|
|
4894
|
+
}
|
|
4863
4895
|
return !adoptLiveTmuxFireForTask(task, tmuxSessions?.sessions || []);
|
|
4864
4896
|
});
|
|
4897
|
+
if (skippedUnverifiable.length) {
|
|
4898
|
+
logError(
|
|
4899
|
+
`tmux is not on PATH, so the Fires a previous daemon left for ${skippedUnverifiable.length} ` +
|
|
4900
|
+
`task(s) cannot be checked; skipping stale recovery for them rather than killing what this ` +
|
|
4901
|
+
`daemon cannot see. Restart with tmux available to adopt or reap them.`,
|
|
4902
|
+
);
|
|
4903
|
+
}
|
|
4865
4904
|
|
|
4866
4905
|
if (deadTasks.length === 0) {
|
|
4867
4906
|
return;
|
|
@@ -4960,12 +4999,23 @@ export function startDaemon(config = {}, deps = {}) {
|
|
|
4960
4999
|
);
|
|
4961
5000
|
return;
|
|
4962
5001
|
}
|
|
5002
|
+
const unverifiableTmuxTaskIds = taskIdsWithUnverifiableTmuxFire();
|
|
4963
5003
|
for (const task of assigned) {
|
|
4964
5004
|
const taskId = String(task?.id || "");
|
|
4965
5005
|
if (!taskId) continue;
|
|
4966
5006
|
if (localTaskIds.has(taskId) || pendingTaskStarts.has(taskId)) {
|
|
4967
5007
|
continue;
|
|
4968
5008
|
}
|
|
5009
|
+
// No tmux binary to ask, but a predecessor filed a hand-off record for
|
|
5010
|
+
// this task: we cannot see its Fire, which is not the same as it being
|
|
5011
|
+
// dead. See taskIdsWithUnverifiableTmuxFire.
|
|
5012
|
+
if (unverifiableTmuxTaskIds.has(taskId)) {
|
|
5013
|
+
logError(
|
|
5014
|
+
`Task ${taskId} has a tmux hand-off record but tmux is not on PATH; skipping reconcile ` +
|
|
5015
|
+
`rather than killing a Fire this daemon cannot see`,
|
|
5016
|
+
);
|
|
5017
|
+
continue;
|
|
5018
|
+
}
|
|
4969
5019
|
// Adopts on the spot rather than merely skipping: a survivor that
|
|
4970
5020
|
// dodges the kill but gains no watcher would sit at `running`
|
|
4971
5021
|
// forever, because nothing else reports a tmux Fire's death.
|
|
@@ -134,3 +134,22 @@ export function pruneFireSessionRecords(dir, liveSessionNames, { fsImpl = fs } =
|
|
|
134
134
|
}
|
|
135
135
|
return pruned;
|
|
136
136
|
}
|
|
137
|
+
|
|
138
|
+
// Task ids a previous daemon left a tmux Fire running for. This is the only
|
|
139
|
+
// evidence available to a successor that has no `tmux` binary to enumerate
|
|
140
|
+
// sessions with, so it is what scopes that daemon's refusal to kill.
|
|
141
|
+
export function listFireSessionTaskIds(dir, { fsImpl = fs } = {}) {
|
|
142
|
+
let entries;
|
|
143
|
+
try {
|
|
144
|
+
entries = fsImpl.readdirSync(dir);
|
|
145
|
+
} catch {
|
|
146
|
+
return new Set();
|
|
147
|
+
}
|
|
148
|
+
const taskIds = new Set();
|
|
149
|
+
for (const entry of entries) {
|
|
150
|
+
if (!entry.endsWith(".json")) continue;
|
|
151
|
+
const record = readFireSessionRecord(dir, entry.slice(0, -".json".length), { fsImpl });
|
|
152
|
+
if (record?.taskId) taskIds.add(record.taskId);
|
|
153
|
+
}
|
|
154
|
+
return taskIds;
|
|
155
|
+
}
|
package/src/remote/client.js
CHANGED
|
@@ -97,8 +97,14 @@ export function isRetryable(error) {
|
|
|
97
97
|
return status >= 500 || status === 429;
|
|
98
98
|
}
|
|
99
99
|
|
|
100
|
-
/**
|
|
101
|
-
|
|
100
|
+
/**
|
|
101
|
+
* Retry `fn` on transient failures, handing it the attempt number. `retryable`
|
|
102
|
+
* narrows what counts as transient: a caller whose request is not idempotent
|
|
103
|
+
* (starting a command) must not retry a 5xx that may already have taken effect.
|
|
104
|
+
*/
|
|
105
|
+
export async function withRetry(fn, {
|
|
106
|
+
attempts = 4, baseDelayMs = 500, sleep = delay, retryable = isRetryable,
|
|
107
|
+
} = {}) {
|
|
102
108
|
let lastError;
|
|
103
109
|
for (let attempt = 0; attempt < attempts; attempt += 1) {
|
|
104
110
|
try {
|
|
@@ -106,7 +112,7 @@ export async function withRetry(fn, { attempts = 4, baseDelayMs = 500, sleep = d
|
|
|
106
112
|
} catch (error) {
|
|
107
113
|
lastError = error;
|
|
108
114
|
// An aborted transfer is the caller's own deadline firing, not a blip.
|
|
109
|
-
if (error?.name === "AbortError" || !
|
|
115
|
+
if (error?.name === "AbortError" || !retryable(error)) throw error;
|
|
110
116
|
if (attempt === attempts - 1) break;
|
|
111
117
|
await sleep(baseDelayMs * 2 ** attempt);
|
|
112
118
|
}
|
package/src/remote/exec.js
CHANGED
|
@@ -15,6 +15,7 @@ import {
|
|
|
15
15
|
delay,
|
|
16
16
|
loadCliConfig,
|
|
17
17
|
parseTimeoutMs as parseSharedTimeoutMs,
|
|
18
|
+
withRetry,
|
|
18
19
|
} from "./client.js";
|
|
19
20
|
|
|
20
21
|
export { EXIT, UsageError };
|
|
@@ -23,6 +24,14 @@ const DEFAULT_TIMEOUT_MS = 60_000;
|
|
|
23
24
|
const POLL_INTERVAL_MS = 1_000;
|
|
24
25
|
/** How long a single request may block server-side before we switch to polling. */
|
|
25
26
|
const POST_WAIT_MS = 10_000;
|
|
27
|
+
/**
|
|
28
|
+
* A 429 means the per-user in-flight cap (8) is full — nothing was started, so
|
|
29
|
+
* retrying is safe. Six attempts at 500ms doubling waits out ~15s, longer than
|
|
30
|
+
* one saturating POST can block (POST_WAIT_MS). Nothing else is retried here:
|
|
31
|
+
* a 5xx may have started the command already, and a second POST would run it twice.
|
|
32
|
+
*/
|
|
33
|
+
const POST_RETRY = { attempts: 6, retryable: (error) => error?.status === 429 };
|
|
34
|
+
const INTERRUPT_SIGNALS = ["SIGINT", "SIGTERM"];
|
|
26
35
|
|
|
27
36
|
const VALUE_FLAGS = new Map([
|
|
28
37
|
["--target", "target"],
|
|
@@ -120,7 +129,8 @@ Options:
|
|
|
120
129
|
--timeout <dur> Overall deadline, e.g. 30s, 2m, 500ms (default: 60s)
|
|
121
130
|
-e, --env KEY=VALUE Extra environment variable (repeatable)
|
|
122
131
|
--json Print the raw run result as JSON
|
|
123
|
-
--kill-on-timeout Stop the remote command when --timeout is reached
|
|
132
|
+
--kill-on-timeout Stop the remote command when --timeout is reached or
|
|
133
|
+
this CLI is interrupted (Ctrl-C, SIGTERM)
|
|
124
134
|
(default: it keeps running on the target)
|
|
125
135
|
--config-file <p> Conductor config file to authenticate with
|
|
126
136
|
-h, --help Show this help
|
|
@@ -132,6 +142,9 @@ Notes:
|
|
|
132
142
|
Exit codes follow ssh: the remote command's own code is passed through, and
|
|
133
143
|
255 means this CLI failed (bad usage, daemon offline, network error).
|
|
134
144
|
|
|
145
|
+
A command that outlives --timeout keeps running on the target; the run id is
|
|
146
|
+
printed and \`conductor remote wait -t <daemon> <runId>\` picks it up again.
|
|
147
|
+
|
|
135
148
|
Examples:
|
|
136
149
|
conductor remote exec --target ubuntu --workspace /home/duino/ws/holomotion ls .
|
|
137
150
|
conductor remote exec -t ubuntu -w /srv/app -- git log --oneline -5
|
|
@@ -139,40 +152,23 @@ Examples:
|
|
|
139
152
|
}
|
|
140
153
|
|
|
141
154
|
/**
|
|
142
|
-
*
|
|
143
|
-
*
|
|
144
|
-
*
|
|
145
|
-
* behaviour, no argv parsing and no printing. `remote cp -r` uses it to drive
|
|
146
|
-
* `tar` on the target.
|
|
155
|
+
* Poll `run` until it leaves `running`, the deadline passes, or `signal`
|
|
156
|
+
* aborts. With `killOnTimeout` a run that is still going at that point is
|
|
157
|
+
* cancelled on the target. Shared by `exec` (after its POST) and `wait`.
|
|
147
158
|
*/
|
|
148
|
-
export async function
|
|
159
|
+
export async function waitForRun(config, basePath, initialRun, options = {}) {
|
|
149
160
|
const {
|
|
150
|
-
|
|
151
|
-
workspace,
|
|
152
|
-
env,
|
|
153
|
-
timeoutMs = DEFAULT_TIMEOUT_MS,
|
|
161
|
+
deadline,
|
|
154
162
|
fetchImpl = globalThis.fetch,
|
|
155
163
|
sleep = delay,
|
|
156
164
|
now = () => Date.now(),
|
|
157
165
|
killOnTimeout = false,
|
|
166
|
+
signal,
|
|
158
167
|
} = options;
|
|
159
168
|
|
|
160
|
-
|
|
161
|
-
const deadline = now() + timeoutMs;
|
|
162
|
-
|
|
163
|
-
let run = await callApi(config, "POST", basePath, {
|
|
164
|
-
command,
|
|
165
|
-
args,
|
|
166
|
-
...(workspace ? { workspace } : {}),
|
|
167
|
-
...(env && Object.keys(env).length > 0 ? { env } : {}),
|
|
168
|
-
// Deliberately short, and independent of the overall deadline: that is
|
|
169
|
-
// owned by the poll loop below. Handing the daemon the full budget would
|
|
170
|
-
// make one HTTP request block for it and leave the loop unreachable.
|
|
171
|
-
timeoutMs: Math.min(timeoutMs, POST_WAIT_MS),
|
|
172
|
-
}, fetchImpl);
|
|
173
|
-
|
|
169
|
+
let run = initialRun;
|
|
174
170
|
let pollError = null;
|
|
175
|
-
while (run?.status === "running" && now() < deadline) {
|
|
171
|
+
while (run?.status === "running" && now() < deadline && !signal?.aborted) {
|
|
176
172
|
if (!run.runId) {
|
|
177
173
|
pollError = new Error("daemon reported a running command but returned no runId");
|
|
178
174
|
break;
|
|
@@ -211,7 +207,47 @@ export async function execRemote(config, target, command, options = {}) {
|
|
|
211
207
|
}
|
|
212
208
|
}
|
|
213
209
|
|
|
214
|
-
return { run, pollError, killError
|
|
210
|
+
return { run, pollError, killError };
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
/**
|
|
214
|
+
* Run one command on `target` and return the final run record.
|
|
215
|
+
*
|
|
216
|
+
* This is the programmatic half of `runRemoteExec`: same two-phase POST-then-poll
|
|
217
|
+
* behaviour, no argv parsing and no printing. `remote cp -r` uses it to drive
|
|
218
|
+
* `tar` on the target.
|
|
219
|
+
*/
|
|
220
|
+
export async function execRemote(config, target, command, options = {}) {
|
|
221
|
+
const {
|
|
222
|
+
args = [],
|
|
223
|
+
workspace,
|
|
224
|
+
env,
|
|
225
|
+
timeoutMs = DEFAULT_TIMEOUT_MS,
|
|
226
|
+
fetchImpl = globalThis.fetch,
|
|
227
|
+
sleep = delay,
|
|
228
|
+
now = () => Date.now(),
|
|
229
|
+
killOnTimeout = false,
|
|
230
|
+
signal,
|
|
231
|
+
} = options;
|
|
232
|
+
|
|
233
|
+
const basePath = `/api/agents/${encodeURIComponent(target)}/exec`;
|
|
234
|
+
const deadline = now() + timeoutMs;
|
|
235
|
+
|
|
236
|
+
const started = await withRetry(() => callApi(config, "POST", basePath, {
|
|
237
|
+
command,
|
|
238
|
+
args,
|
|
239
|
+
...(workspace ? { workspace } : {}),
|
|
240
|
+
...(env && Object.keys(env).length > 0 ? { env } : {}),
|
|
241
|
+
// Deliberately short, and independent of the overall deadline: that is
|
|
242
|
+
// owned by the poll loop below. Handing the daemon the full budget would
|
|
243
|
+
// make one HTTP request block for it and leave the loop unreachable.
|
|
244
|
+
timeoutMs: Math.min(timeoutMs, POST_WAIT_MS),
|
|
245
|
+
}, fetchImpl), { ...POST_RETRY, sleep });
|
|
246
|
+
|
|
247
|
+
const waited = await waitForRun(config, basePath, started, {
|
|
248
|
+
deadline, fetchImpl, sleep, now, killOnTimeout, signal,
|
|
249
|
+
});
|
|
250
|
+
return { ...waited, basePath };
|
|
215
251
|
}
|
|
216
252
|
|
|
217
253
|
/**
|
|
@@ -229,6 +265,60 @@ export async function execRemoteOrThrow(config, target, command, options = {}) {
|
|
|
229
265
|
throw new Error(detail ? `${what} on ${target}: ${detail}` : `${what} on ${target}`);
|
|
230
266
|
}
|
|
231
267
|
|
|
268
|
+
/**
|
|
269
|
+
* An AbortSignal that fires on Ctrl-C / SIGTERM. Installing a listener also
|
|
270
|
+
* stops Node from exiting on the signal, which is the point: the CLI gets to
|
|
271
|
+
* print the run id (and cancel the run with --kill-on-timeout) before it goes.
|
|
272
|
+
* Callers must `release()` so the default behaviour comes back.
|
|
273
|
+
*/
|
|
274
|
+
export function interruptSignal(proc = process) {
|
|
275
|
+
const controller = new AbortController();
|
|
276
|
+
const onSignal = () => controller.abort();
|
|
277
|
+
for (const name of INTERRUPT_SIGNALS) proc.on(name, onSignal);
|
|
278
|
+
return {
|
|
279
|
+
signal: controller.signal,
|
|
280
|
+
release: () => {
|
|
281
|
+
for (const name of INTERRUPT_SIGNALS) proc.off(name, onSignal);
|
|
282
|
+
},
|
|
283
|
+
};
|
|
284
|
+
}
|
|
285
|
+
|
|
286
|
+
/** Print a finished (or abandoned) run the way `exec` and `wait` both do, and pick the exit code. */
|
|
287
|
+
export function reportRun(run, { consoleImpl, json, target, pollError, waitedMs, interrupted }) {
|
|
288
|
+
if (json) {
|
|
289
|
+
consoleImpl.log(JSON.stringify(run, null, 2));
|
|
290
|
+
} else {
|
|
291
|
+
if (run?.stdoutTail) process.stdout.write(run.stdoutTail);
|
|
292
|
+
if (run?.stderrTail) process.stderr.write(run.stderrTail);
|
|
293
|
+
if (run?.truncated) {
|
|
294
|
+
consoleImpl.error(`[conductor] output truncated; showing the tail only`);
|
|
295
|
+
}
|
|
296
|
+
if (run?.error) {
|
|
297
|
+
consoleImpl.error(`Error: ${run.error}`);
|
|
298
|
+
}
|
|
299
|
+
}
|
|
300
|
+
|
|
301
|
+
if (run?.status === "running") {
|
|
302
|
+
if (pollError) {
|
|
303
|
+
consoleImpl.error(`[conductor] last status poll failed: ${pollError.message}`);
|
|
304
|
+
}
|
|
305
|
+
const why = interrupted ? "interrupted" : `still running on ${target} after ${waitedMs}ms`;
|
|
306
|
+
consoleImpl.error(
|
|
307
|
+
`[conductor] ${why}; it keeps going on ${target} — ` +
|
|
308
|
+
`resume with: conductor remote wait -t ${target} ${run.runId} ` +
|
|
309
|
+
`(add --kill-on-timeout to stop it instead)`,
|
|
310
|
+
);
|
|
311
|
+
return EXIT.CLI_ERROR;
|
|
312
|
+
}
|
|
313
|
+
if (run?.status === "cancelled") {
|
|
314
|
+
return EXIT.CLI_ERROR;
|
|
315
|
+
}
|
|
316
|
+
if (typeof run?.exitCode === "number") {
|
|
317
|
+
return run.exitCode;
|
|
318
|
+
}
|
|
319
|
+
return run?.status === "completed" ? EXIT.OK : EXIT.CLI_ERROR;
|
|
320
|
+
}
|
|
321
|
+
|
|
232
322
|
export async function runRemoteExec(argv, deps = {}) {
|
|
233
323
|
const consoleImpl = deps.console || console;
|
|
234
324
|
const fetchImpl = deps.fetch || globalThis.fetch;
|
|
@@ -278,12 +368,12 @@ export async function runRemoteExec(argv, deps = {}) {
|
|
|
278
368
|
return EXIT.CLI_ERROR;
|
|
279
369
|
}
|
|
280
370
|
|
|
371
|
+
const interrupt = interruptSignal(deps.process);
|
|
281
372
|
let run;
|
|
282
373
|
let pollError = null;
|
|
283
|
-
let basePath;
|
|
284
374
|
try {
|
|
285
375
|
let killError;
|
|
286
|
-
({ run, pollError, killError
|
|
376
|
+
({ run, pollError, killError } = await execRemote(config, target, command[0], {
|
|
287
377
|
args: command.slice(1),
|
|
288
378
|
workspace: options.workspace,
|
|
289
379
|
env: options.env,
|
|
@@ -292,46 +382,27 @@ export async function runRemoteExec(argv, deps = {}) {
|
|
|
292
382
|
sleep,
|
|
293
383
|
now,
|
|
294
384
|
killOnTimeout: options.killOnTimeout,
|
|
385
|
+
signal: interrupt.signal,
|
|
295
386
|
}));
|
|
296
387
|
if (killError) {
|
|
297
388
|
consoleImpl.error(`[conductor] failed to stop the run on ${target}: ${killError.message}`);
|
|
298
389
|
} else if (options.killOnTimeout && run?.status === "cancelled") {
|
|
299
|
-
|
|
390
|
+
const why = interrupt.signal.aborted ? "interrupted" : "deadline reached";
|
|
391
|
+
consoleImpl.error(`[conductor] ${why}; stopped the command on ${target}`);
|
|
300
392
|
}
|
|
301
393
|
} catch (error) {
|
|
302
394
|
consoleImpl.error(`Error: ${error.message}`);
|
|
303
395
|
return EXIT.CLI_ERROR;
|
|
396
|
+
} finally {
|
|
397
|
+
interrupt.release();
|
|
304
398
|
}
|
|
305
399
|
|
|
306
|
-
|
|
307
|
-
consoleImpl
|
|
308
|
-
|
|
309
|
-
|
|
310
|
-
|
|
311
|
-
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
if (run?.error) {
|
|
315
|
-
consoleImpl.error(`Error: ${run.error}`);
|
|
316
|
-
}
|
|
317
|
-
}
|
|
318
|
-
|
|
319
|
-
if (run?.status === "running") {
|
|
320
|
-
if (pollError) {
|
|
321
|
-
consoleImpl.error(`[conductor] last status poll failed: ${pollError.message}`);
|
|
322
|
-
}
|
|
323
|
-
consoleImpl.error(
|
|
324
|
-
`[conductor] still running on ${target} after ${timeoutMs}ms; ` +
|
|
325
|
-
`it keeps going there — poll GET ${basePath}/runs/${run.runId}, ` +
|
|
326
|
-
`stop it with DELETE on the same path, or use --kill-on-timeout`,
|
|
327
|
-
);
|
|
328
|
-
return EXIT.CLI_ERROR;
|
|
329
|
-
}
|
|
330
|
-
if (run?.status === "cancelled") {
|
|
331
|
-
return EXIT.CLI_ERROR;
|
|
332
|
-
}
|
|
333
|
-
if (typeof run?.exitCode === "number") {
|
|
334
|
-
return run.exitCode;
|
|
335
|
-
}
|
|
336
|
-
return run?.status === "completed" ? EXIT.OK : EXIT.CLI_ERROR;
|
|
400
|
+
return reportRun(run, {
|
|
401
|
+
consoleImpl,
|
|
402
|
+
json: options.json,
|
|
403
|
+
target,
|
|
404
|
+
pollError,
|
|
405
|
+
waitedMs: timeoutMs,
|
|
406
|
+
interrupted: interrupt.signal.aborted,
|
|
407
|
+
});
|
|
337
408
|
}
|
|
@@ -0,0 +1,133 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `conductor remote wait` — pick up a command started by `remote exec` that
|
|
3
|
+
* outlived its deadline (or the process that started it).
|
|
4
|
+
*
|
|
5
|
+
* `exec` prints the run id whenever it leaves a command behind; this verb
|
|
6
|
+
* polls that run the same way `exec` would have and reports it identically,
|
|
7
|
+
* so a caller can always finish what it started.
|
|
8
|
+
*/
|
|
9
|
+
|
|
10
|
+
import process from "node:process";
|
|
11
|
+
|
|
12
|
+
import { EXIT, callApi, delay, loadCliConfig } from "./client.js";
|
|
13
|
+
import {
|
|
14
|
+
interruptSignal,
|
|
15
|
+
parseArgs,
|
|
16
|
+
parseTimeoutMs,
|
|
17
|
+
reportRun,
|
|
18
|
+
waitForRun,
|
|
19
|
+
} from "./exec.js";
|
|
20
|
+
|
|
21
|
+
export function showHelp(consoleImpl = console) {
|
|
22
|
+
consoleImpl.log(`conductor remote wait - wait for a command left running by \`remote exec\`
|
|
23
|
+
|
|
24
|
+
Usage:
|
|
25
|
+
conductor remote wait --target <daemon> [options] <runId>
|
|
26
|
+
|
|
27
|
+
Options:
|
|
28
|
+
-t, --target <daemon> Daemon the command is running on (required)
|
|
29
|
+
--timeout <dur> How long to wait this time, e.g. 30s, 2m (default: 60s)
|
|
30
|
+
--json Print the raw run result as JSON
|
|
31
|
+
--kill-on-timeout Stop the remote command when --timeout is reached or
|
|
32
|
+
this CLI is interrupted (default: it keeps running)
|
|
33
|
+
--config-file <p> Conductor config file to authenticate with
|
|
34
|
+
-h, --help Show this help
|
|
35
|
+
|
|
36
|
+
Output and exit codes match \`conductor remote exec\`: the command's own exit
|
|
37
|
+
code is passed through, 255 means this CLI failed or the command is still running.
|
|
38
|
+
|
|
39
|
+
Example:
|
|
40
|
+
conductor remote exec -t ubuntu --timeout 30s -- make build
|
|
41
|
+
# ... [conductor] still running on ubuntu after 30000ms; resume with:
|
|
42
|
+
conductor remote wait -t ubuntu --timeout 20m 3f9c1b2e-...
|
|
43
|
+
`);
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
export async function runRemoteWait(argv, deps = {}) {
|
|
47
|
+
const consoleImpl = deps.console || console;
|
|
48
|
+
const fetchImpl = deps.fetch || globalThis.fetch;
|
|
49
|
+
const env = deps.env || process.env;
|
|
50
|
+
const sleep = deps.sleep || delay;
|
|
51
|
+
const now = deps.now || (() => Date.now());
|
|
52
|
+
|
|
53
|
+
let parsed;
|
|
54
|
+
try {
|
|
55
|
+
parsed = parseArgs(argv);
|
|
56
|
+
} catch (error) {
|
|
57
|
+
consoleImpl.error(`Error: ${error.message}`);
|
|
58
|
+
return EXIT.CLI_ERROR;
|
|
59
|
+
}
|
|
60
|
+
const { options, command } = parsed;
|
|
61
|
+
|
|
62
|
+
if (options.help) {
|
|
63
|
+
showHelp(consoleImpl);
|
|
64
|
+
return EXIT.OK;
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
const target = typeof options.target === "string" ? options.target.trim() : "";
|
|
68
|
+
if (!target) {
|
|
69
|
+
consoleImpl.error("Error: --target <daemon> is required");
|
|
70
|
+
showHelp(consoleImpl);
|
|
71
|
+
return EXIT.CLI_ERROR;
|
|
72
|
+
}
|
|
73
|
+
if (command.length !== 1) {
|
|
74
|
+
consoleImpl.error(command.length === 0 ? "Error: no runId given" : "Error: expected exactly one runId");
|
|
75
|
+
showHelp(consoleImpl);
|
|
76
|
+
return EXIT.CLI_ERROR;
|
|
77
|
+
}
|
|
78
|
+
const runId = command[0];
|
|
79
|
+
|
|
80
|
+
let timeoutMs;
|
|
81
|
+
try {
|
|
82
|
+
timeoutMs = parseTimeoutMs(options.timeout);
|
|
83
|
+
} catch (error) {
|
|
84
|
+
consoleImpl.error(`Error: ${error.message}`);
|
|
85
|
+
return EXIT.CLI_ERROR;
|
|
86
|
+
}
|
|
87
|
+
|
|
88
|
+
let config;
|
|
89
|
+
try {
|
|
90
|
+
config = deps.config || loadCliConfig(options.configFile, env);
|
|
91
|
+
} catch (error) {
|
|
92
|
+
consoleImpl.error(`Error: ${error.message}`);
|
|
93
|
+
return EXIT.CLI_ERROR;
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
const basePath = `/api/agents/${encodeURIComponent(target)}/exec`;
|
|
97
|
+
const interrupt = interruptSignal(deps.process);
|
|
98
|
+
let run;
|
|
99
|
+
let pollError = null;
|
|
100
|
+
try {
|
|
101
|
+
const deadline = now() + timeoutMs;
|
|
102
|
+
run = await callApi(config, "GET", `${basePath}/runs/${encodeURIComponent(runId)}`, null, fetchImpl);
|
|
103
|
+
let killError;
|
|
104
|
+
({ run, pollError, killError } = await waitForRun(config, basePath, run, {
|
|
105
|
+
deadline,
|
|
106
|
+
fetchImpl,
|
|
107
|
+
sleep,
|
|
108
|
+
now,
|
|
109
|
+
killOnTimeout: options.killOnTimeout,
|
|
110
|
+
signal: interrupt.signal,
|
|
111
|
+
}));
|
|
112
|
+
if (killError) {
|
|
113
|
+
consoleImpl.error(`[conductor] failed to stop the run on ${target}: ${killError.message}`);
|
|
114
|
+
} else if (options.killOnTimeout && run?.status === "cancelled") {
|
|
115
|
+
const why = interrupt.signal.aborted ? "interrupted" : "deadline reached";
|
|
116
|
+
consoleImpl.error(`[conductor] ${why}; stopped the command on ${target}`);
|
|
117
|
+
}
|
|
118
|
+
} catch (error) {
|
|
119
|
+
consoleImpl.error(`Error: ${error.message}`);
|
|
120
|
+
return EXIT.CLI_ERROR;
|
|
121
|
+
} finally {
|
|
122
|
+
interrupt.release();
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
return reportRun(run, {
|
|
126
|
+
consoleImpl,
|
|
127
|
+
json: options.json,
|
|
128
|
+
target,
|
|
129
|
+
pollError,
|
|
130
|
+
waitedMs: timeoutMs,
|
|
131
|
+
interrupted: interrupt.signal.aborted,
|
|
132
|
+
});
|
|
133
|
+
}
|