@nowcrew/daemon 0.5.29 → 0.5.31
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +17 -1
- package/dist/config.js +13 -1
- package/dist/execution-journal.js +96 -5
- package/dist/execution-protocol.js +1 -0
- package/dist/execution-runner.js +52 -29
- package/dist/host-execution-coordinator.js +241 -0
- package/dist/local-executor.js +78 -0
- package/dist/machine-info.js +16 -1
- package/dist/remote/claude-bridge.js +402 -0
- package/dist/remote/claude-channel.js +164 -0
- package/dist/remote/codex-client.js +408 -0
- package/dist/remote/codex-runtime.js +77 -0
- package/dist/remote/config.js +83 -0
- package/dist/remote/gateway.js +572 -0
- package/dist/remote/protocol.js +178 -0
- package/dist/remote/remote-cli.js +233 -0
- package/dist/remote/session-discovery.js +249 -0
- package/dist/remote/wrapper.js +40 -0
- package/dist/runner.js +4 -0
- package/dist/runtime-startup-gate.js +91 -0
- package/dist/runtimes/codex-app-server-runner.js +226 -29
- package/dist/serve.js +70 -8
- package/dist/shared-execution-slots.js +48 -33
- package/package.json +1 -1
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
import { randomUUID } from "node:crypto";
|
|
2
|
+
const UUID = /^[0-9a-f]{8}-[0-9a-f]{4}-[1-8][0-9a-f]{3}-[89ab][0-9a-f]{3}-[0-9a-f]{12}$/i;
|
|
3
|
+
function optionValue(args, long, short) {
|
|
4
|
+
for (let index = 0; index < args.length; index += 1) {
|
|
5
|
+
const value = args[index];
|
|
6
|
+
if (value.startsWith(`${long}=`))
|
|
7
|
+
return value.slice(long.length + 1);
|
|
8
|
+
if (value === long || (short && value === short)) {
|
|
9
|
+
const next = args[index + 1];
|
|
10
|
+
return next && !next.startsWith("-") ? next : null;
|
|
11
|
+
}
|
|
12
|
+
}
|
|
13
|
+
return null;
|
|
14
|
+
}
|
|
15
|
+
export function resolveClaudeWrapperSession(args, createId = randomUUID) {
|
|
16
|
+
if (args.includes("--continue") || args.includes("-c")) {
|
|
17
|
+
throw new Error("NowCrew remote requires an explicit session ID; use --resume <session-id> instead of --continue");
|
|
18
|
+
}
|
|
19
|
+
if (args.includes("--fork-session")) {
|
|
20
|
+
throw new Error("NowCrew remote cannot identify a fork chosen inside Claude; start a new session or resume without --fork-session");
|
|
21
|
+
}
|
|
22
|
+
const sessionId = optionValue(args, "--session-id") ?? optionValue(args, "--resume", "-r") ?? createId();
|
|
23
|
+
if (!UUID.test(sessionId)) {
|
|
24
|
+
throw new Error("Claude remote sessions require a UUID in --session-id or --resume");
|
|
25
|
+
}
|
|
26
|
+
const hasSessionSelector = args.some((value) => value === "--session-id" || value.startsWith("--session-id=")
|
|
27
|
+
|| value === "--resume" || value.startsWith("--resume=") || value === "-r");
|
|
28
|
+
return {
|
|
29
|
+
sessionId,
|
|
30
|
+
args: hasSessionSelector ? [...args] : ["--session-id", sessionId, ...args],
|
|
31
|
+
};
|
|
32
|
+
}
|
|
33
|
+
export function buildCodexWrappedInvocation(userArgs, socketPath = "") {
|
|
34
|
+
const hasRemote = userArgs.some((value) => value === "--remote" || value.startsWith("--remote="));
|
|
35
|
+
return {
|
|
36
|
+
bin: "codex",
|
|
37
|
+
args: hasRemote ? [...userArgs] : ["--remote", socketPath ? `unix://${socketPath}` : "unix://", ...userArgs],
|
|
38
|
+
env: process.env,
|
|
39
|
+
};
|
|
40
|
+
}
|
package/dist/runner.js
CHANGED
|
@@ -106,6 +106,10 @@ export async function runAgent(config, input, onActivity = defaultPrint, onConso
|
|
|
106
106
|
}, { onActivity, onConsole }, {
|
|
107
107
|
launchRuntime: dependencies.launchRuntime ?? ((request) => launchSupervisedRuntime(request, dependencies.startSupervisor, dependencies.cancellation, dependencies.platform)),
|
|
108
108
|
...(dependencies.cancellation === undefined ? {} : { cancellation: dependencies.cancellation }),
|
|
109
|
+
...(dependencies.startupGate === undefined ? {} : { startupGate: dependencies.startupGate }),
|
|
110
|
+
...(dependencies.startupTimeoutMs === undefined
|
|
111
|
+
? {}
|
|
112
|
+
: { startupTimeoutMs: dependencies.startupTimeoutMs }),
|
|
109
113
|
});
|
|
110
114
|
const activities = [...local.activities];
|
|
111
115
|
if (!input.scheduled && (runtime === "codex" || runtime === "kimi")
|
|
@@ -0,0 +1,91 @@
|
|
|
1
|
+
export function createRuntimeStartupGate(limits, now = Date.now) {
|
|
2
|
+
const activeByRuntime = new Map();
|
|
3
|
+
const queue = [];
|
|
4
|
+
let activeTotal = 0;
|
|
5
|
+
let lastLaunchAt = Number.NEGATIVE_INFINITY;
|
|
6
|
+
const promote = () => {
|
|
7
|
+
while (activeTotal < limits.maxStartingTotal) {
|
|
8
|
+
const index = queue.findIndex((entry) => !entry.released
|
|
9
|
+
&& (activeByRuntime.get(entry.runtime) ?? 0) < limits.maxStartingPerRuntime);
|
|
10
|
+
if (index < 0)
|
|
11
|
+
return;
|
|
12
|
+
const [next] = queue.splice(index, 1);
|
|
13
|
+
if (!next || next.released)
|
|
14
|
+
continue;
|
|
15
|
+
next.promoted = true;
|
|
16
|
+
activeTotal += 1;
|
|
17
|
+
activeByRuntime.set(next.runtime, (activeByRuntime.get(next.runtime) ?? 0) + 1);
|
|
18
|
+
const grant = () => {
|
|
19
|
+
next.timer = null;
|
|
20
|
+
if (next.released)
|
|
21
|
+
return;
|
|
22
|
+
next.launchGranted = true;
|
|
23
|
+
lastLaunchAt = now();
|
|
24
|
+
next.resolve();
|
|
25
|
+
};
|
|
26
|
+
const delay = Math.max(0, lastLaunchAt + limits.startupGapMs - now());
|
|
27
|
+
if (delay === 0)
|
|
28
|
+
grant();
|
|
29
|
+
else
|
|
30
|
+
next.timer = setTimeout(grant, delay);
|
|
31
|
+
}
|
|
32
|
+
};
|
|
33
|
+
return {
|
|
34
|
+
reserve: (runtime) => {
|
|
35
|
+
let resolveReady;
|
|
36
|
+
const ready = new Promise((resolve) => { resolveReady = resolve; });
|
|
37
|
+
const entry = {
|
|
38
|
+
runtime,
|
|
39
|
+
released: false,
|
|
40
|
+
promoted: false,
|
|
41
|
+
launchGranted: false,
|
|
42
|
+
timer: null,
|
|
43
|
+
resolve: resolveReady,
|
|
44
|
+
};
|
|
45
|
+
queue.push(entry);
|
|
46
|
+
promote();
|
|
47
|
+
return {
|
|
48
|
+
ready,
|
|
49
|
+
isQueued: () => !entry.launchGranted && !entry.released,
|
|
50
|
+
release: () => {
|
|
51
|
+
if (entry.released)
|
|
52
|
+
return;
|
|
53
|
+
entry.released = true;
|
|
54
|
+
if (entry.timer !== null)
|
|
55
|
+
clearTimeout(entry.timer);
|
|
56
|
+
if (entry.promoted) {
|
|
57
|
+
activeTotal = Math.max(0, activeTotal - 1);
|
|
58
|
+
const nextForRuntime = Math.max(0, (activeByRuntime.get(runtime) ?? 1) - 1);
|
|
59
|
+
if (nextForRuntime === 0)
|
|
60
|
+
activeByRuntime.delete(runtime);
|
|
61
|
+
else
|
|
62
|
+
activeByRuntime.set(runtime, nextForRuntime);
|
|
63
|
+
}
|
|
64
|
+
else {
|
|
65
|
+
const index = queue.indexOf(entry);
|
|
66
|
+
if (index >= 0)
|
|
67
|
+
queue.splice(index, 1);
|
|
68
|
+
}
|
|
69
|
+
promote();
|
|
70
|
+
},
|
|
71
|
+
};
|
|
72
|
+
},
|
|
73
|
+
snapshot: () => ({
|
|
74
|
+
startingTotal: activeTotal,
|
|
75
|
+
queuedTotal: queue.length,
|
|
76
|
+
startingByRuntime: {
|
|
77
|
+
claude: activeByRuntime.get("claude") ?? 0,
|
|
78
|
+
codex: activeByRuntime.get("codex") ?? 0,
|
|
79
|
+
kimi: activeByRuntime.get("kimi") ?? 0,
|
|
80
|
+
},
|
|
81
|
+
}),
|
|
82
|
+
};
|
|
83
|
+
}
|
|
84
|
+
export function isRuntimeReadyEvent(runtime, event) {
|
|
85
|
+
if (typeof event !== "object" || event === null)
|
|
86
|
+
return false;
|
|
87
|
+
const value = event;
|
|
88
|
+
if (runtime === "claude")
|
|
89
|
+
return value.type === "system" && value.subtype === "init";
|
|
90
|
+
return value.type === "thread.started";
|
|
91
|
+
}
|
|
@@ -4,6 +4,7 @@ import { parseArgs } from "node:util";
|
|
|
4
4
|
import { pathToFileURL } from "node:url";
|
|
5
5
|
import spawn from "cross-spawn";
|
|
6
6
|
import { z } from "zod";
|
|
7
|
+
import { signalSupervisorTree } from "../execution-supervisor.js";
|
|
7
8
|
import { startFirstProgressWatchdog } from "./progress-watchdog.js";
|
|
8
9
|
const RunnerInputSchema = z.object({
|
|
9
10
|
systemPrompt: z.string().min(1),
|
|
@@ -18,6 +19,22 @@ const RunnerInputSchema = z.object({
|
|
|
18
19
|
const RPC_TIMEOUT_MS = 30_000;
|
|
19
20
|
const INITIALIZE_RPC_TIMEOUT_MS = 60_000;
|
|
20
21
|
const ERROR_MESSAGE_CAP = 2_000;
|
|
22
|
+
// LocalExecutor retains the final 2,000 stderr characters; leave room for stage and error lines.
|
|
23
|
+
const STDERR_TAIL_CAP = 1_200;
|
|
24
|
+
const STDERR_LINE_CAPTURE_CAP = 1_200;
|
|
25
|
+
const STDERR_LINE_OMITTED = "[stderr line omitted: exceeded capture limit]\n";
|
|
26
|
+
const MAX_INITIALIZE_ATTEMPTS = 2;
|
|
27
|
+
const PROCESS_TREE_STOP_TIMEOUT_MS = 1_000;
|
|
28
|
+
class CodexRpcTimeoutError extends Error {
|
|
29
|
+
method;
|
|
30
|
+
timeoutMs;
|
|
31
|
+
constructor(method, timeoutMs) {
|
|
32
|
+
super(`Codex app-server ${method} timed out after ${timeoutMs}ms`);
|
|
33
|
+
this.method = method;
|
|
34
|
+
this.timeoutMs = timeoutMs;
|
|
35
|
+
this.name = "CodexRpcTimeoutError";
|
|
36
|
+
}
|
|
37
|
+
}
|
|
21
38
|
export function codexRpcTimeoutMs(method) {
|
|
22
39
|
return method === "initialize" ? INITIALIZE_RPC_TIMEOUT_MS : RPC_TIMEOUT_MS;
|
|
23
40
|
}
|
|
@@ -29,6 +46,87 @@ function safeErrorMessage(error, secrets) {
|
|
|
29
46
|
}
|
|
30
47
|
return message.slice(0, ERROR_MESSAGE_CAP);
|
|
31
48
|
}
|
|
49
|
+
function sensitiveEnvironmentValues(env) {
|
|
50
|
+
return Object.entries(env)
|
|
51
|
+
.filter(([name, value]) => value !== undefined
|
|
52
|
+
&& /(?:api.?key|token|secret|password|credential|authorization)/i.test(name))
|
|
53
|
+
.map(([, value]) => value);
|
|
54
|
+
}
|
|
55
|
+
function redactionSecrets(input, env) {
|
|
56
|
+
const values = [input.systemPrompt, input.wakePrompt, ...sensitiveEnvironmentValues(env)];
|
|
57
|
+
return [...new Set(values.flatMap((value) => [
|
|
58
|
+
value,
|
|
59
|
+
...value.split(/\r?\n/).map((line) => line.trim()),
|
|
60
|
+
]).filter(Boolean))];
|
|
61
|
+
}
|
|
62
|
+
export function redactCodexStderr(text, secrets) {
|
|
63
|
+
let redacted = text;
|
|
64
|
+
for (const secret of [...secrets].sort((left, right) => right.length - left.length)) {
|
|
65
|
+
if (secret)
|
|
66
|
+
redacted = redacted.replaceAll(secret, "[redacted]");
|
|
67
|
+
}
|
|
68
|
+
return redacted
|
|
69
|
+
.replace(/(Bearer\s+)[^\s"']+/gi, "$1[redacted]")
|
|
70
|
+
.replace(/\bsk-[A-Za-z0-9_-]{6,}\b/g, "[redacted]")
|
|
71
|
+
.replace(/((?:api[_-]?key|access[_-]?token|auth(?:orization)?|password|secret)\s*[=:]\s*)[^\s"']+/gi, "$1[redacted]");
|
|
72
|
+
}
|
|
73
|
+
class StderrCapture {
|
|
74
|
+
secrets;
|
|
75
|
+
tail = "";
|
|
76
|
+
pendingLine = "";
|
|
77
|
+
omittingLine = false;
|
|
78
|
+
constructor(child, secrets) {
|
|
79
|
+
this.secrets = secrets;
|
|
80
|
+
child.stderr?.setEncoding("utf8");
|
|
81
|
+
child.stderr?.on("data", (chunk) => this.append(chunk));
|
|
82
|
+
child.stderr?.once("end", () => this.flush());
|
|
83
|
+
}
|
|
84
|
+
redactedTail() {
|
|
85
|
+
const pending = this.omittingLine
|
|
86
|
+
? STDERR_LINE_OMITTED
|
|
87
|
+
: redactCodexStderr(this.pendingLine, this.secrets);
|
|
88
|
+
return `${this.tail}${pending}`.slice(-STDERR_TAIL_CAP).trim();
|
|
89
|
+
}
|
|
90
|
+
append(chunk) {
|
|
91
|
+
let remaining = chunk;
|
|
92
|
+
while (remaining.length > 0) {
|
|
93
|
+
const newline = remaining.indexOf("\n");
|
|
94
|
+
const segment = newline < 0 ? remaining : remaining.slice(0, newline);
|
|
95
|
+
if (!this.omittingLine) {
|
|
96
|
+
if (this.pendingLine.length + segment.length <= STDERR_LINE_CAPTURE_CAP) {
|
|
97
|
+
this.pendingLine += segment;
|
|
98
|
+
}
|
|
99
|
+
else {
|
|
100
|
+
this.pendingLine = "";
|
|
101
|
+
this.omittingLine = true;
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
if (newline < 0)
|
|
105
|
+
return;
|
|
106
|
+
this.emit(this.omittingLine ? STDERR_LINE_OMITTED : `${this.pendingLine}\n`);
|
|
107
|
+
this.pendingLine = "";
|
|
108
|
+
this.omittingLine = false;
|
|
109
|
+
remaining = remaining.slice(newline + 1);
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
flush() {
|
|
113
|
+
if (this.pendingLine.length === 0 && !this.omittingLine)
|
|
114
|
+
return;
|
|
115
|
+
this.emit(this.omittingLine ? STDERR_LINE_OMITTED : this.pendingLine);
|
|
116
|
+
this.pendingLine = "";
|
|
117
|
+
this.omittingLine = false;
|
|
118
|
+
}
|
|
119
|
+
emit(text) {
|
|
120
|
+
const redacted = redactCodexStderr(text, this.secrets);
|
|
121
|
+
process.stderr.write(redacted);
|
|
122
|
+
this.tail = `${this.tail}${redacted}`.slice(-STDERR_TAIL_CAP);
|
|
123
|
+
}
|
|
124
|
+
}
|
|
125
|
+
function logStage(stage, status, attempt, startedAt, detail) {
|
|
126
|
+
const suffix = detail === undefined ? "" : ` ${detail}`;
|
|
127
|
+
process.stderr.write(`[codex-app-server] stage=${stage} status=${status} attempt=${attempt}`
|
|
128
|
+
+ ` elapsed_ms=${Date.now() - startedAt}${suffix}\n`);
|
|
129
|
+
}
|
|
32
130
|
function jsonLine(event) {
|
|
33
131
|
if (process.stdout.write(`${JSON.stringify(event)}\n`))
|
|
34
132
|
return Promise.resolve();
|
|
@@ -117,7 +215,7 @@ class CodexRpcClient {
|
|
|
117
215
|
return new Promise((resolve, reject) => {
|
|
118
216
|
const timer = setTimeout(() => {
|
|
119
217
|
this.pending.delete(id);
|
|
120
|
-
reject(new
|
|
218
|
+
reject(new CodexRpcTimeoutError(method, timeoutMs));
|
|
121
219
|
}, timeoutMs);
|
|
122
220
|
this.pending.set(id, { resolve, reject, timer });
|
|
123
221
|
this.write({ jsonrpc: "2.0", id, method, params });
|
|
@@ -184,31 +282,70 @@ class CodexRpcClient {
|
|
|
184
282
|
this.pending.clear();
|
|
185
283
|
}
|
|
186
284
|
}
|
|
187
|
-
|
|
188
|
-
if (
|
|
285
|
+
function processTreeAlive(child) {
|
|
286
|
+
if (process.platform === "win32" || child.pid === undefined) {
|
|
287
|
+
return child.exitCode === null && child.signalCode === null;
|
|
288
|
+
}
|
|
289
|
+
try {
|
|
290
|
+
process.kill(-child.pid, 0);
|
|
291
|
+
return true;
|
|
292
|
+
}
|
|
293
|
+
catch (error) {
|
|
294
|
+
return error.code === "EPERM";
|
|
295
|
+
}
|
|
296
|
+
}
|
|
297
|
+
async function signalProcessTree(child, signal) {
|
|
298
|
+
if (child.pid !== undefined) {
|
|
299
|
+
try {
|
|
300
|
+
await signalSupervisorTree(child.pid, signal);
|
|
301
|
+
return;
|
|
302
|
+
}
|
|
303
|
+
catch (error) {
|
|
304
|
+
if (error.code !== "ESRCH")
|
|
305
|
+
throw error;
|
|
306
|
+
}
|
|
307
|
+
}
|
|
308
|
+
if (child.exitCode === null && child.signalCode === null)
|
|
309
|
+
child.kill(signal);
|
|
310
|
+
}
|
|
311
|
+
async function waitForProcessTreeExit(child, timeoutMs) {
|
|
312
|
+
const deadline = Date.now() + timeoutMs;
|
|
313
|
+
while (processTreeAlive(child) && Date.now() < deadline) {
|
|
314
|
+
await new Promise((resolve) => setTimeout(resolve, 25));
|
|
315
|
+
}
|
|
316
|
+
return !processTreeAlive(child);
|
|
317
|
+
}
|
|
318
|
+
async function stopChildTree(child) {
|
|
319
|
+
if (!processTreeAlive(child))
|
|
320
|
+
return;
|
|
321
|
+
await signalProcessTree(child, "SIGTERM");
|
|
322
|
+
if (await waitForProcessTreeExit(child, PROCESS_TREE_STOP_TIMEOUT_MS))
|
|
189
323
|
return;
|
|
190
|
-
|
|
191
|
-
child
|
|
192
|
-
|
|
193
|
-
const graceful = await Promise.race([
|
|
194
|
-
closed.then(() => true),
|
|
195
|
-
new Promise((resolve) => { timer = setTimeout(() => resolve(false), 1_000); }),
|
|
196
|
-
]);
|
|
197
|
-
if (timer !== undefined)
|
|
198
|
-
clearTimeout(timer);
|
|
199
|
-
if (!graceful && child.exitCode === null && child.signalCode === null) {
|
|
200
|
-
child.kill("SIGKILL");
|
|
201
|
-
await closed;
|
|
324
|
+
await signalProcessTree(child, "SIGKILL");
|
|
325
|
+
if (!await waitForProcessTreeExit(child, PROCESS_TREE_STOP_TIMEOUT_MS)) {
|
|
326
|
+
throw new Error(`Codex app-server process tree ${child.pid ?? "unknown"} did not exit`);
|
|
202
327
|
}
|
|
203
328
|
}
|
|
204
|
-
|
|
205
|
-
const
|
|
329
|
+
async function runCodexAppServerAttempt(bin, input, attempt, initializeTimeoutMs) {
|
|
330
|
+
const attemptStartedAt = Date.now();
|
|
206
331
|
const child = spawn(bin, ["app-server", "--listen", "stdio://"], {
|
|
207
332
|
cwd: process.cwd(),
|
|
208
333
|
env: process.env,
|
|
209
334
|
stdio: ["pipe", "pipe", "pipe"],
|
|
335
|
+
detached: process.platform !== "win32",
|
|
336
|
+
});
|
|
337
|
+
const secrets = redactionSecrets(input, process.env);
|
|
338
|
+
const stderr = new StderrCapture(child, secrets);
|
|
339
|
+
logStage("spawn", "start", attempt, attemptStartedAt, `pid=${child.pid ?? "unknown"}`);
|
|
340
|
+
let didSpawn = false;
|
|
341
|
+
child.once("spawn", () => {
|
|
342
|
+
didSpawn = true;
|
|
343
|
+
logStage("spawn", "ok", attempt, attemptStartedAt, `pid=${child.pid ?? "unknown"}`);
|
|
344
|
+
});
|
|
345
|
+
child.once("error", () => {
|
|
346
|
+
if (!didSpawn)
|
|
347
|
+
logStage("spawn", "error", attempt, attemptStartedAt);
|
|
210
348
|
});
|
|
211
|
-
child.stderr?.pipe(process.stderr, { end: false });
|
|
212
349
|
let threadId = null;
|
|
213
350
|
let announcedThreadId = null;
|
|
214
351
|
let turnId = null;
|
|
@@ -246,6 +383,11 @@ export async function runCodexAppServer(bin) {
|
|
|
246
383
|
process.stderr.write(`Codex turn error: ${detail.slice(0, ERROR_MESSAGE_CAP)}\n`);
|
|
247
384
|
}
|
|
248
385
|
}, completionReject);
|
|
386
|
+
let treeStopPromise = null;
|
|
387
|
+
const ensureChildTreeStopped = () => {
|
|
388
|
+
treeStopPromise ??= stopChildTree(child);
|
|
389
|
+
return treeStopPromise;
|
|
390
|
+
};
|
|
249
391
|
let cancelling = false;
|
|
250
392
|
const cancel = async () => {
|
|
251
393
|
if (cancelling)
|
|
@@ -254,17 +396,22 @@ export async function runCodexAppServer(bin) {
|
|
|
254
396
|
if (threadId !== null && turnId !== null) {
|
|
255
397
|
await rpc.request("turn/interrupt", { threadId, turnId }, 5_000).catch(() => undefined);
|
|
256
398
|
}
|
|
257
|
-
await
|
|
399
|
+
await ensureChildTreeStopped();
|
|
258
400
|
};
|
|
259
401
|
const onSignal = () => { void cancel().finally(() => process.exit(130)); };
|
|
260
402
|
process.once("SIGTERM", onSignal);
|
|
261
403
|
process.once("SIGINT", onSignal);
|
|
404
|
+
let activeStage = "initialize";
|
|
405
|
+
let stageStartedAt = Date.now();
|
|
262
406
|
try {
|
|
263
407
|
await rpc.request("initialize", {
|
|
264
408
|
clientInfo: { name: "nowcrew-daemon", version: "1" },
|
|
265
409
|
capabilities: { experimentalApi: true, requestAttestation: false },
|
|
266
|
-
});
|
|
410
|
+
}, initializeTimeoutMs);
|
|
411
|
+
logStage(activeStage, "ok", attempt, stageStartedAt);
|
|
267
412
|
rpc.notify("initialized");
|
|
413
|
+
activeStage = input.resume ? "thread_resume" : "thread_start";
|
|
414
|
+
stageStartedAt = Date.now();
|
|
268
415
|
const threadParams = {
|
|
269
416
|
cwd: process.cwd(),
|
|
270
417
|
approvalPolicy: "never",
|
|
@@ -275,6 +422,7 @@ export async function runCodexAppServer(bin) {
|
|
|
275
422
|
const thread = input.resume && input.sessionId !== undefined
|
|
276
423
|
? await rpc.request("thread/resume", { threadId: input.sessionId, ...threadParams })
|
|
277
424
|
: await rpc.request("thread/start", threadParams);
|
|
425
|
+
logStage(activeStage, "ok", attempt, stageStartedAt);
|
|
278
426
|
threadId = thread.thread.id;
|
|
279
427
|
if (announcedThreadId !== threadId) {
|
|
280
428
|
announcedThreadId = threadId;
|
|
@@ -284,6 +432,8 @@ export async function runCodexAppServer(bin) {
|
|
|
284
432
|
completionReject(new Error("Codex produced no semantic progress within the startup window"));
|
|
285
433
|
void cancel();
|
|
286
434
|
});
|
|
435
|
+
activeStage = "turn_start";
|
|
436
|
+
stageStartedAt = Date.now();
|
|
287
437
|
const started = await rpc.request("turn/start", {
|
|
288
438
|
threadId,
|
|
289
439
|
input: [
|
|
@@ -294,8 +444,12 @@ export async function runCodexAppServer(bin) {
|
|
|
294
444
|
? {}
|
|
295
445
|
: { effort: input.reasoning }),
|
|
296
446
|
});
|
|
447
|
+
logStage(activeStage, "ok", attempt, stageStartedAt);
|
|
297
448
|
turnId = started.turn.id;
|
|
449
|
+
activeStage = "turn_complete";
|
|
450
|
+
stageStartedAt = Date.now();
|
|
298
451
|
const completed = await completion;
|
|
452
|
+
logStage(activeStage, "ok", attempt, stageStartedAt);
|
|
299
453
|
firstProgress.stop();
|
|
300
454
|
const usage = lastUsage?.last;
|
|
301
455
|
await jsonLine({
|
|
@@ -309,33 +463,76 @@ export async function runCodexAppServer(bin) {
|
|
|
309
463
|
}),
|
|
310
464
|
});
|
|
311
465
|
if (completed.turn?.status === "completed")
|
|
312
|
-
return 0;
|
|
466
|
+
return { code: 0, initializeTimedOut: false };
|
|
313
467
|
const detail = completed.turn?.error?.message ?? `turn status ${completed.turn?.status ?? "unknown"}`;
|
|
314
468
|
process.stderr.write(`Codex turn failed: ${detail.slice(0, ERROR_MESSAGE_CAP)}\n`);
|
|
315
|
-
return
|
|
469
|
+
return {
|
|
470
|
+
code: completed.turn?.status === "interrupted" ? 130 : 1,
|
|
471
|
+
initializeTimedOut: false,
|
|
472
|
+
};
|
|
316
473
|
}
|
|
317
474
|
catch (error) {
|
|
318
|
-
|
|
319
|
-
|
|
475
|
+
const initializeTimedOut = error instanceof CodexRpcTimeoutError && error.method === "initialize";
|
|
476
|
+
if (initializeTimedOut) {
|
|
477
|
+
const tail = stderr.redactedTail();
|
|
478
|
+
logStage(activeStage, "timeout", attempt, stageStartedAt, `stderr_tail=${JSON.stringify(tail || "[empty]")}`);
|
|
479
|
+
}
|
|
480
|
+
else {
|
|
481
|
+
logStage(activeStage, "error", attempt, stageStartedAt);
|
|
482
|
+
}
|
|
483
|
+
process.stderr.write(`Codex app-server execution failed: ${safeErrorMessage(error, secrets)}\n`);
|
|
484
|
+
return { code: 1, initializeTimedOut };
|
|
320
485
|
}
|
|
321
486
|
finally {
|
|
322
487
|
firstProgress.stop();
|
|
323
488
|
process.off("SIGTERM", onSignal);
|
|
324
489
|
process.off("SIGINT", onSignal);
|
|
325
|
-
|
|
490
|
+
const cleanupStartedAt = Date.now();
|
|
491
|
+
try {
|
|
492
|
+
await ensureChildTreeStopped();
|
|
493
|
+
logStage("cleanup", "ok", attempt, cleanupStartedAt);
|
|
494
|
+
}
|
|
495
|
+
catch (error) {
|
|
496
|
+
logStage("cleanup", "error", attempt, cleanupStartedAt);
|
|
497
|
+
throw error;
|
|
498
|
+
}
|
|
499
|
+
}
|
|
500
|
+
}
|
|
501
|
+
export async function runCodexAppServer(bin, options = {}) {
|
|
502
|
+
const input = await readRunnerInput();
|
|
503
|
+
const initializeTimeoutMs = Math.min(options.initializeTimeoutMs ?? INITIALIZE_RPC_TIMEOUT_MS, INITIALIZE_RPC_TIMEOUT_MS);
|
|
504
|
+
for (let attempt = 1; attempt <= MAX_INITIALIZE_ATTEMPTS; attempt += 1) {
|
|
505
|
+
const result = await runCodexAppServerAttempt(bin, input, attempt, initializeTimeoutMs);
|
|
506
|
+
if (!result.initializeTimedOut || attempt === MAX_INITIALIZE_ATTEMPTS)
|
|
507
|
+
return result.code;
|
|
508
|
+
process.stderr.write(`[codex-app-server] stage=retry status=start attempt=${attempt + 1} elapsed_ms=0 reason=initialize_timeout\n`);
|
|
326
509
|
}
|
|
510
|
+
return 1;
|
|
327
511
|
}
|
|
328
|
-
function
|
|
512
|
+
function configFromArgv(argv) {
|
|
329
513
|
const { values } = parseArgs({
|
|
330
514
|
args: [...argv],
|
|
331
|
-
options: {
|
|
515
|
+
options: {
|
|
516
|
+
bin: { type: "string" },
|
|
517
|
+
"initialize-timeout-ms": { type: "string" },
|
|
518
|
+
},
|
|
332
519
|
});
|
|
333
520
|
if (!values.bin)
|
|
334
521
|
throw new Error("--bin is required");
|
|
335
|
-
|
|
522
|
+
const rawTimeout = values["initialize-timeout-ms"];
|
|
523
|
+
if (rawTimeout === undefined)
|
|
524
|
+
return { bin: values.bin };
|
|
525
|
+
const initializeTimeoutMs = Number(rawTimeout);
|
|
526
|
+
if (!Number.isInteger(initializeTimeoutMs) || initializeTimeoutMs <= 0) {
|
|
527
|
+
throw new Error("--initialize-timeout-ms must be a positive integer");
|
|
528
|
+
}
|
|
529
|
+
return { bin: values.bin, initializeTimeoutMs };
|
|
336
530
|
}
|
|
337
531
|
if (process.argv[1] !== undefined && import.meta.url === pathToFileURL(process.argv[1]).href) {
|
|
338
|
-
|
|
532
|
+
const config = configFromArgv(process.argv.slice(2));
|
|
533
|
+
runCodexAppServer(config.bin, config.initializeTimeoutMs === undefined
|
|
534
|
+
? {}
|
|
535
|
+
: { initializeTimeoutMs: config.initializeTimeoutMs })
|
|
339
536
|
.then((code) => { process.exitCode = code; })
|
|
340
537
|
.catch((error) => {
|
|
341
538
|
process.stderr.write(`Codex app-server runner failed: ${safeErrorMessage(error, [])}\n`);
|
package/dist/serve.js
CHANGED
|
@@ -28,6 +28,8 @@ import { closeWebSocketWithinDeadline } from "./websocket-shutdown.js";
|
|
|
28
28
|
import { reconcileExecutionJournal } from "./execution-recovery.js";
|
|
29
29
|
import { createSharedSlotManager } from "./shared-execution-slots.js";
|
|
30
30
|
import { createCompletionRetransmitter } from "./completion-retransmitter.js";
|
|
31
|
+
import { createRuntimeStartupGate } from "./runtime-startup-gate.js";
|
|
32
|
+
import { createHostExecutionCoordinator, hostCoordinatedSlotManager, hostCoordinatedStartupGate, } from "./host-execution-coordinator.js";
|
|
31
33
|
// normalize.ts 的活动种类 → activity 枚举
|
|
32
34
|
const ACTIVITY_MAP = {
|
|
33
35
|
init: "working", text: "thinking", reading: "reading", sending: "sending",
|
|
@@ -60,7 +62,9 @@ export function serve(config, opts = {}) {
|
|
|
60
62
|
const executeProtocol = opts.execution?.runExecution ?? runExecution;
|
|
61
63
|
let detectedExecutionRuntimes = [];
|
|
62
64
|
let runtimeFacts = null;
|
|
63
|
-
const
|
|
65
|
+
const hostCoordinator = opts.execution?.hostCoordinator ?? createHostExecutionCoordinator();
|
|
66
|
+
const sharedSlots = hostCoordinatedSlotManager(createSharedSlotManager(config.executionLimits), hostCoordinator);
|
|
67
|
+
const runtimeStartupGate = hostCoordinatedStartupGate(createRuntimeStartupGate(config.executionLimits), hostCoordinator);
|
|
64
68
|
const knownExecutionHashes = new Map();
|
|
65
69
|
const executionReservations = new Map();
|
|
66
70
|
const executionRuns = new Map();
|
|
@@ -115,8 +119,11 @@ export function serve(config, opts = {}) {
|
|
|
115
119
|
if ((entry.state === "completed" || entry.state === "interrupted") && entry.completion !== null) {
|
|
116
120
|
return { executionId: entry.executionId, state: entry.state, completion: entry.completion, updatedAt: entry.updatedAt };
|
|
117
121
|
}
|
|
122
|
+
if (entry.state === "running" && entry.runtimeReadyAt !== null) {
|
|
123
|
+
return { executionId: entry.executionId, state: "running", updatedAt: entry.runtimeReadyAt };
|
|
124
|
+
}
|
|
118
125
|
if (entry.state === "accepted" || entry.state === "running") {
|
|
119
|
-
return { executionId: entry.executionId, state:
|
|
126
|
+
return { executionId: entry.executionId, state: "accepted", updatedAt: entry.acceptedAt };
|
|
120
127
|
}
|
|
121
128
|
return null;
|
|
122
129
|
};
|
|
@@ -137,10 +144,10 @@ export function serve(config, opts = {}) {
|
|
|
137
144
|
state: acceptanceState, effectivePermission: entry.effectivePermission ?? "workspace_write",
|
|
138
145
|
at: entry.acceptedAt,
|
|
139
146
|
});
|
|
140
|
-
if (entry.state === "running" && entry.
|
|
147
|
+
if (entry.state === "running" && entry.runtimeReadyAt !== null) {
|
|
141
148
|
safeExecutionSend({
|
|
142
149
|
type: "execution:started", protocolVersion: 1,
|
|
143
|
-
executionId: entry.executionId, at: entry.
|
|
150
|
+
executionId: entry.executionId, at: entry.runtimeReadyAt,
|
|
144
151
|
});
|
|
145
152
|
}
|
|
146
153
|
}
|
|
@@ -335,8 +342,30 @@ export function serve(config, opts = {}) {
|
|
|
335
342
|
}
|
|
336
343
|
return;
|
|
337
344
|
}
|
|
338
|
-
knownExecutionHashes.set(spec.executionId, hash);
|
|
339
345
|
const reservation = sharedSlots.reserve(spec.agent.handle, "execution");
|
|
346
|
+
if (!reservation.accepted) {
|
|
347
|
+
dslog("execution.machine_queue_rejected", "机器执行队列已满", {
|
|
348
|
+
level: "WARN",
|
|
349
|
+
execution_id: spec.executionId,
|
|
350
|
+
agent_handle: spec.agent.handle,
|
|
351
|
+
...reservation.facts,
|
|
352
|
+
});
|
|
353
|
+
safeExecutionSend(ExecutionRejectedSchema.parse({
|
|
354
|
+
type: "execution:rejected", protocolVersion: 1, executionId: spec.executionId,
|
|
355
|
+
reason: "resource_limit", message: "Local machine execution queue is full",
|
|
356
|
+
at: new Date().toISOString(),
|
|
357
|
+
}));
|
|
358
|
+
return;
|
|
359
|
+
}
|
|
360
|
+
knownExecutionHashes.set(spec.executionId, hash);
|
|
361
|
+
const machineQueueEnteredAt = Date.now();
|
|
362
|
+
if (reservation.isQueued()) {
|
|
363
|
+
dslog("execution.machine_queued", "execution 已进入机器队列", {
|
|
364
|
+
execution_id: spec.executionId,
|
|
365
|
+
agent_handle: spec.agent.handle,
|
|
366
|
+
...reservation.facts,
|
|
367
|
+
});
|
|
368
|
+
}
|
|
340
369
|
executionReservations.set(spec.executionId, reservation);
|
|
341
370
|
const cancellation = cancellationFor(spec.executionId);
|
|
342
371
|
const cleanupExecutionReservation = () => {
|
|
@@ -371,8 +400,22 @@ export function serve(config, opts = {}) {
|
|
|
371
400
|
...reservation.facts,
|
|
372
401
|
},
|
|
373
402
|
report: reportExecutionFrame,
|
|
403
|
+
startupGate: runtimeStartupGate,
|
|
404
|
+
startupTimeoutMs: config.executionLimits.startupTimeoutMs,
|
|
374
405
|
...(reservation.state === undefined ? {} : {
|
|
375
|
-
slot: {
|
|
406
|
+
slot: {
|
|
407
|
+
state: reservation.state,
|
|
408
|
+
ready: reservation.ready.then(() => {
|
|
409
|
+
const snapshot = sharedSlots.snapshot();
|
|
410
|
+
dslog("execution.machine_slot_ready", "execution 获得机器执行名额", {
|
|
411
|
+
execution_id: spec.executionId,
|
|
412
|
+
agent_handle: spec.agent.handle,
|
|
413
|
+
queue_ms: Date.now() - machineQueueEnteredAt,
|
|
414
|
+
active_total: snapshot.activeTotal,
|
|
415
|
+
queued_total: snapshot.queuedTotal,
|
|
416
|
+
});
|
|
417
|
+
}),
|
|
418
|
+
},
|
|
376
419
|
}),
|
|
377
420
|
cancellation,
|
|
378
421
|
}).finally(() => {
|
|
@@ -539,12 +582,26 @@ export function serve(config, opts = {}) {
|
|
|
539
582
|
}
|
|
540
583
|
running.add(key);
|
|
541
584
|
legacyReservation = sharedSlots.reserve(msg.agentHandle, "legacy");
|
|
585
|
+
if (legacyReservation.isQueued()) {
|
|
586
|
+
const snapshot = sharedSlots.snapshot();
|
|
587
|
+
dslog("run.machine_queued", "legacy run 已进入机器队列", {
|
|
588
|
+
...runKeys,
|
|
589
|
+
active_total: snapshot.activeTotal,
|
|
590
|
+
queued_total: snapshot.queuedTotal,
|
|
591
|
+
});
|
|
592
|
+
}
|
|
542
593
|
await awaitWithCancellation(legacyReservation.ready, controller.cancellation);
|
|
543
594
|
const queueMs = Date.now() - queueWaitStart;
|
|
595
|
+
const machineSnapshot = sharedSlots.snapshot();
|
|
544
596
|
const threadLabel = threadId ?? null;
|
|
545
597
|
const from = msg.wake?.senderHandle ?? "?";
|
|
546
598
|
const incoming = msg.wake?.content ?? "";
|
|
547
|
-
dslog("run.start", `开始运行 ${msg.agentHandle}`, {
|
|
599
|
+
dslog("run.start", `开始运行 ${msg.agentHandle}`, {
|
|
600
|
+
...runKeys,
|
|
601
|
+
queue_ms: queueMs,
|
|
602
|
+
active_total: machineSnapshot.activeTotal,
|
|
603
|
+
queued_total: machineSnapshot.queuedTotal,
|
|
604
|
+
});
|
|
548
605
|
log(`\n${"─".repeat(56)}`);
|
|
549
606
|
log(`🔔 唤醒 agent=${msg.agentHandle} reason=${msg.reason ?? "?"}`);
|
|
550
607
|
log(` channel = ${msg.channelId}`);
|
|
@@ -650,7 +707,11 @@ export function serve(config, opts = {}) {
|
|
|
650
707
|
...(!scheduled && threadId ? { wakeMessageId: threadId } : {}),
|
|
651
708
|
...(!scheduled && msg.wake?.seq !== undefined ? { wakeContextUpToSeq: msg.wake.seq } : {}),
|
|
652
709
|
...(attemptWake ? { wake: attemptWake } : {}),
|
|
653
|
-
}, reportActivity, reportConsole, {
|
|
710
|
+
}, reportActivity, reportConsole, {
|
|
711
|
+
cancellation: controller.cancellation,
|
|
712
|
+
startupGate: runtimeStartupGate,
|
|
713
|
+
startupTimeoutMs: config.executionLimits.startupTimeoutMs,
|
|
714
|
+
});
|
|
654
715
|
});
|
|
655
716
|
const result = mergeRunAgentResults(guarded.results);
|
|
656
717
|
// 本轮 token 用量上报:runner 已从 result 事件提取(含缓存读/写细分),
|
|
@@ -848,6 +909,7 @@ export function serve(config, opts = {}) {
|
|
|
848
909
|
for (const run of legacyRuns.values())
|
|
849
910
|
run.controller.request();
|
|
850
911
|
await deadline.waitFor(Promise.all(pending));
|
|
912
|
+
await deadline.waitFor(hostCoordinator.drain());
|
|
851
913
|
await executionJournal.close({ signal: deadline.signal });
|
|
852
914
|
}
|
|
853
915
|
finally {
|