@triflux/remote 10.50.1 → 10.51.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/cto/brief.mjs +0 -12
- package/cto/collect.mjs +16 -247
- package/cto/current.schema.json +4 -20
- package/cto/events.mjs +13 -294
- package/cto/hygiene.mjs +26 -346
- package/cto/lake-root.mjs +114 -16
- package/cto/status.mjs +33 -128
- package/hub/index.mjs +0 -8
- package/hub/store-adapter.mjs +1 -1
- package/hub/team/agent-map.json +0 -1
- package/hub/team/ansi.mjs +1 -35
- package/hub/team/backend.mjs +9 -145
- package/hub/team/cli/commands/attach.mjs +1 -17
- package/hub/team/cli/commands/control.mjs +12 -38
- package/hub/team/cli/commands/debug.mjs +1 -37
- package/hub/team/cli/commands/focus.mjs +5 -31
- package/hub/team/cli/commands/interrupt.mjs +6 -25
- package/hub/team/cli/commands/kill.mjs +1 -30
- package/hub/team/cli/commands/list.mjs +0 -16
- package/hub/team/cli/commands/send.mjs +6 -26
- package/hub/team/cli/commands/start/index.mjs +41 -133
- package/hub/team/cli/commands/start/parse-args.mjs +15 -40
- package/hub/team/cli/commands/start/start-headless.mjs +138 -105
- package/hub/team/cli/commands/start/start-mux.mjs +0 -3
- package/hub/team/cli/commands/status.mjs +2 -57
- package/hub/team/cli/commands/stop.mjs +14 -19
- package/hub/team/cli/help.mjs +4 -5
- package/hub/team/cli/manifest.mjs +5 -1
- package/hub/team/cli/render.mjs +0 -12
- package/hub/team/cli/services/hub-client.mjs +2 -88
- package/hub/team/cli/services/runtime-mode.mjs +27 -98
- package/hub/team/handoff.mjs +3 -34
- package/hub/team/headless.mjs +14 -755
- package/hub/team/orchestrator.mjs +4 -30
- package/hub/team/pane.mjs +6 -28
- package/hub/team/psmux.mjs +16 -403
- package/hub/team/remote-session.mjs +1 -2
- package/hub/team/session.mjs +8 -119
- package/hub/team/swarm-intent.mjs +0 -198
- package/hub/team/terminal-opener.mjs +11 -35
- package/hub/team/tui-core.mjs +1 -1
- package/hub/team/tui-viewer.mjs +2 -2
- package/hub/team/tui.mjs +8 -269
- package/hub/team/wt-manager.mjs +2 -2
- package/hub/workers/factory.mjs +4 -37
- package/hub/workers/lib/jsonrpc-ws-uds.mjs +0 -2
- package/package.json +1 -1
- package/scripts/lib/agent-route-policy.mjs +162 -14
- package/scripts/lib/claudemd-scanner.mjs +2 -199
- package/scripts/lib/cli-claude.mjs +0 -2
- package/scripts/lib/context.mjs +3 -19
- package/scripts/lib/env-probe.mjs +5 -64
- package/scripts/lib/gemini-profiles.mjs +2 -69
- package/scripts/lib/legacy-mcp-cleanup.mjs +14 -3
- package/scripts/lib/machine-profile.mjs +3 -17
- package/scripts/lib/mcp-filter.mjs +109 -53
- package/scripts/lib/mcp-health.mjs +2 -2
- package/scripts/lib/mcp-server-catalog.mjs +2 -2
- package/scripts/lib/psmux-info.mjs +25 -104
- package/scripts/lib/skill-template.mjs +0 -271
- package/cto/hygiene-actions.mjs +0 -355
- package/cto/hygiene-notify.mjs +0 -170
- package/hub/team/build-worker-prompt.mjs +0 -92
- package/hub/team/check-mcp-hub.mjs +0 -49
- package/hub/team/claude-native-bridge.mjs +0 -987
- package/hub/team/cli/commands/start/start-in-process.mjs +0 -50
- package/hub/team/cli/commands/start/start-wt.mjs +0 -86
- package/hub/team/cli/services/native-control.mjs +0 -145
- package/hub/team/codex-review.mjs +0 -520
- package/hub/team/conductor-mesh-bridge.mjs +0 -122
- package/hub/team/conductor.mjs +0 -1529
- package/hub/team/consensus-meta.mjs +0 -176
- package/hub/team/cto-pull.mjs +0 -31
- package/hub/team/daemon-pty-tmux-bridge.mjs +0 -146
- package/hub/team/dashboard.mjs +0 -322
- package/hub/team/event-log.mjs +0 -84
- package/hub/team/execution-mode.mjs +0 -324
- package/hub/team/extract-completion-payload.mjs +0 -110
- package/hub/team/headless-bridge-session.mjs +0 -17
- package/hub/team/health-probe.mjs +0 -435
- package/hub/team/interactive-native-launcher.mjs +0 -119
- package/hub/team/interactive-native-worker.mjs +0 -80
- package/hub/team/interactive-tui-transport.mjs +0 -479
- package/hub/team/launcher-template.mjs +0 -144
- package/hub/team/mcp-selector.mjs +0 -145
- package/hub/team/native-supervisor.mjs +0 -460
- package/hub/team/native.mjs +0 -727
- package/hub/team/recovery-store.mjs +0 -149
- package/hub/team/remote-probe.mjs +0 -292
- package/hub/team/remote-watcher.mjs +0 -506
- package/hub/team/runtime-strategy.mjs +0 -200
- package/hub/team/schemas/worker-completion.json +0 -39
- package/hub/team/sentinel-capture.mjs +0 -127
- package/hub/team/swarm-cli.mjs +0 -390
- package/hub/team/swarm-hypervisor.mjs +0 -2206
- package/hub/team/swarm-locks.mjs +0 -335
- package/hub/team/swarm-planner.mjs +0 -371
- package/hub/team/swarm-preflight.mjs +0 -366
- package/hub/team/swarm-reconciler.mjs +0 -242
- package/hub/team/tui-remote-adapter.mjs +0 -377
- package/hub/team/worker-completion-validator.mjs +0 -79
- package/hub/team/worktree-lifecycle.mjs +0 -639
- package/hub/team/wt-templates.json +0 -43
- package/hub/workers/antigravity-route-worker.mjs +0 -179
- package/hub/workers/gemini-worker.mjs +0 -217
- package/scripts/lib/agent-json.mjs +0 -27
- package/scripts/lib/async.mjs +0 -174
- package/scripts/lib/cli-agy.mjs +0 -85
- package/scripts/lib/cli-codex.mjs +0 -62
- package/scripts/lib/cli-gemini.mjs +0 -37
- package/scripts/lib/dynamic-route-cli.mjs +0 -107
- package/scripts/lib/env.mjs +0 -14
- package/scripts/lib/hub.mjs +0 -48
- package/scripts/lib/pid.mjs +0 -63
- package/scripts/lib/quota.mjs +0 -47
- package/scripts/lib/skill-state.mjs +0 -220
- package/scripts/lib/team.mjs +0 -75
- package/scripts/lib/timeout.mjs +0 -66
- package/scripts/lib/tmp.mjs +0 -26
- package/scripts/lib/toml.mjs +0 -132
package/hub/team/conductor.mjs
DELETED
|
@@ -1,1529 +0,0 @@
|
|
|
1
|
-
// hub/team/conductor.mjs — 세션 오케스트레이션 Conductor
|
|
2
|
-
// native-supervisor.mjs의 spawn/kill을 래핑하되, 상태 머신 + health probe +
|
|
3
|
-
// auto-restart + event log를 추가하여 "조용한 실패"를 구조적으로 불가능하게 만든다.
|
|
4
|
-
//
|
|
5
|
-
// 기존 native-supervisor와의 차이:
|
|
6
|
-
// 1. 상태 머신 (alive/dead → 7 states + 2 terminal)
|
|
7
|
-
// 2. Health probe 4단계 (+ INPUT_WAIT 감지)
|
|
8
|
-
// 3. Auto-restart (maxRestarts=3)
|
|
9
|
-
// 4. JSONL event log (블랙박스 리코더)
|
|
10
|
-
|
|
11
|
-
import { spawnSync } from "node:child_process";
|
|
12
|
-
import { EventEmitter } from "node:events";
|
|
13
|
-
import {
|
|
14
|
-
copyFileSync,
|
|
15
|
-
createWriteStream,
|
|
16
|
-
mkdirSync,
|
|
17
|
-
readFileSync,
|
|
18
|
-
} from "node:fs";
|
|
19
|
-
import { homedir } from "node:os";
|
|
20
|
-
import { dirname, join } from "node:path";
|
|
21
|
-
import { createRegistry } from "@triflux/core/mesh/mesh-registry.mjs";
|
|
22
|
-
import { resolveNestedCodexAgentProfile } from "../../scripts/lib/cli-codex.mjs";
|
|
23
|
-
import { codexProfileConfigOverrides } from "../../scripts/lib/codex-profile-config.mjs";
|
|
24
|
-
import {
|
|
25
|
-
buildDisabledCliError,
|
|
26
|
-
normalizeCliName,
|
|
27
|
-
resolveCliPolicy,
|
|
28
|
-
} from "../../scripts/lib/machine-profile.mjs";
|
|
29
|
-
import { broker } from "@triflux/core/hub/account-broker.mjs";
|
|
30
|
-
import { runPreflight as runCodexPreflight } from "@triflux/core/hub/codex-preflight.mjs";
|
|
31
|
-
import {
|
|
32
|
-
__internal__ as dynamicRoutingInternal,
|
|
33
|
-
evaluateDynamicRoute,
|
|
34
|
-
isDynamicRoutingEnabled,
|
|
35
|
-
} from "@triflux/core/hub/dynamic-routing-engine.mjs";
|
|
36
|
-
import {
|
|
37
|
-
findProcessTree,
|
|
38
|
-
killProcessTree,
|
|
39
|
-
killProcessTreeSnapshot,
|
|
40
|
-
} from "@triflux/core/hub/lib/process-utils.mjs";
|
|
41
|
-
import { execFile, spawn } from "@triflux/core/hub/lib/spawn-trace.mjs";
|
|
42
|
-
import { killProcess } from "@triflux/core/hub/platform.mjs";
|
|
43
|
-
import { tryGetCachedSnapshot } from "@triflux/core/hub/routing-snapshot.mjs";
|
|
44
|
-
import { createHubHealthChecker } from "./check-mcp-hub.mjs";
|
|
45
|
-
import { createConductorMeshBridge } from "./conductor-mesh-bridge.mjs";
|
|
46
|
-
import {
|
|
47
|
-
ensureConductorRegistry,
|
|
48
|
-
getConductorRegistry,
|
|
49
|
-
} from "./conductor-registry.mjs";
|
|
50
|
-
import { readCtoSnapshot } from "./cto-pull.mjs";
|
|
51
|
-
import { createEventLog } from "./event-log.mjs";
|
|
52
|
-
import { buildSpawnSpecForMode, MODES } from "./execution-mode.mjs";
|
|
53
|
-
import { extractCompletionPayload } from "./extract-completion-payload.mjs";
|
|
54
|
-
import { createHealthProbe } from "./health-probe.mjs";
|
|
55
|
-
import { buildLauncher } from "./launcher-template.mjs";
|
|
56
|
-
import { createSentinelCapture } from "./sentinel-capture.mjs";
|
|
57
|
-
import {
|
|
58
|
-
buildSynapseTaskSummary,
|
|
59
|
-
heartbeatSynapseSession,
|
|
60
|
-
registerSynapseSession,
|
|
61
|
-
unregisterSynapseSession,
|
|
62
|
-
} from "./synapse-http.mjs";
|
|
63
|
-
import { buildWorkerSandboxEnv } from "./worker-sandbox.mjs";
|
|
64
|
-
|
|
65
|
-
/** 세션 상태 */
|
|
66
|
-
export const STATES = Object.freeze({
|
|
67
|
-
INIT: "init",
|
|
68
|
-
STARTING: "starting",
|
|
69
|
-
HEALTHY: "healthy",
|
|
70
|
-
STALLED: "stalled",
|
|
71
|
-
INPUT_WAIT: "input_wait",
|
|
72
|
-
FAILED: "failed",
|
|
73
|
-
RESTARTING: "restarting",
|
|
74
|
-
DEAD: "dead",
|
|
75
|
-
COMPLETED: "completed",
|
|
76
|
-
});
|
|
77
|
-
|
|
78
|
-
/** 유효한 상태 전이 테이블 */
|
|
79
|
-
const TRANSITIONS = Object.freeze({
|
|
80
|
-
[STATES.INIT]: [STATES.STARTING],
|
|
81
|
-
[STATES.STARTING]: [STATES.HEALTHY, STATES.FAILED, STATES.COMPLETED],
|
|
82
|
-
[STATES.HEALTHY]: [
|
|
83
|
-
STATES.STALLED,
|
|
84
|
-
STATES.INPUT_WAIT,
|
|
85
|
-
STATES.FAILED,
|
|
86
|
-
STATES.COMPLETED,
|
|
87
|
-
],
|
|
88
|
-
[STATES.STALLED]: [STATES.HEALTHY, STATES.FAILED],
|
|
89
|
-
[STATES.INPUT_WAIT]: [STATES.HEALTHY, STATES.FAILED],
|
|
90
|
-
[STATES.FAILED]: [STATES.RESTARTING, STATES.DEAD],
|
|
91
|
-
[STATES.RESTARTING]: [STATES.STARTING],
|
|
92
|
-
[STATES.DEAD]: [],
|
|
93
|
-
[STATES.COMPLETED]: [],
|
|
94
|
-
});
|
|
95
|
-
|
|
96
|
-
const TERMINAL_STATES = new Set([STATES.DEAD, STATES.COMPLETED]);
|
|
97
|
-
const DEFAULT_MAX_RESTARTS = 3;
|
|
98
|
-
const DEFAULT_GRACE_MS = 10_000;
|
|
99
|
-
|
|
100
|
-
/**
|
|
101
|
-
* Auth 파일을 lease 소스에서 CLI 대상 경로로 복사.
|
|
102
|
-
* @param {object} lease — broker lease 객체 (mode, authFile 필수)
|
|
103
|
-
* @param {'codex'|'gemini'} agent
|
|
104
|
-
* @param {string} sessionId — 로깅용
|
|
105
|
-
* @param {object} eventLog — createEventLog 인스턴스
|
|
106
|
-
* @param {object} brokerInstance — AccountBroker 인스턴스
|
|
107
|
-
*/
|
|
108
|
-
function swapAuthFile(
|
|
109
|
-
lease,
|
|
110
|
-
agent,
|
|
111
|
-
sessionId,
|
|
112
|
-
eventLog,
|
|
113
|
-
brokerInstance,
|
|
114
|
-
targetEnv = {},
|
|
115
|
-
) {
|
|
116
|
-
if (lease?.mode !== "auth" || !lease.authFile) return true;
|
|
117
|
-
if (
|
|
118
|
-
agent === "codex" &&
|
|
119
|
-
lease.id &&
|
|
120
|
-
brokerInstance &&
|
|
121
|
-
typeof brokerInstance.syncAuthToSource === "function"
|
|
122
|
-
) {
|
|
123
|
-
const result = brokerInstance.syncAuthToSource(lease.id);
|
|
124
|
-
if (result?.copied || result?.reason === "up_to_date") {
|
|
125
|
-
eventLog.append("auth_copy", {
|
|
126
|
-
session: sessionId,
|
|
127
|
-
agent,
|
|
128
|
-
dest: result.sourcePath,
|
|
129
|
-
reason: result.reason,
|
|
130
|
-
});
|
|
131
|
-
return true;
|
|
132
|
-
}
|
|
133
|
-
eventLog.append("auth_copy_error", {
|
|
134
|
-
session: sessionId,
|
|
135
|
-
dest: result?.sourcePath || join(homedir(), ".codex", "auth.json"),
|
|
136
|
-
error: result?.reason || "sync_failed",
|
|
137
|
-
});
|
|
138
|
-
return false;
|
|
139
|
-
}
|
|
140
|
-
const targetHome = targetEnv.HOME || homedir();
|
|
141
|
-
const codexHome = targetEnv.CODEX_HOME || join(targetHome, ".codex");
|
|
142
|
-
const dests =
|
|
143
|
-
agent === "codex"
|
|
144
|
-
? [join(codexHome, "auth.json")]
|
|
145
|
-
: [
|
|
146
|
-
join(targetHome, ".gemini", "oauth_creds.json"),
|
|
147
|
-
join(targetHome, ".gemini", "gemini-credentials.json"),
|
|
148
|
-
];
|
|
149
|
-
let allOk = true;
|
|
150
|
-
for (const dest of dests) {
|
|
151
|
-
try {
|
|
152
|
-
mkdirSync(dirname(dest), { recursive: true });
|
|
153
|
-
copyFileSync(lease.authFile, dest);
|
|
154
|
-
eventLog.append("auth_copy", { session: sessionId, agent, dest });
|
|
155
|
-
} catch (err) {
|
|
156
|
-
allOk = false;
|
|
157
|
-
eventLog.append("auth_copy_error", {
|
|
158
|
-
session: sessionId,
|
|
159
|
-
dest,
|
|
160
|
-
error: err.message,
|
|
161
|
-
});
|
|
162
|
-
}
|
|
163
|
-
}
|
|
164
|
-
return allOk;
|
|
165
|
-
}
|
|
166
|
-
|
|
167
|
-
/**
|
|
168
|
-
* Conductor 팩토리.
|
|
169
|
-
* @param {object} opts
|
|
170
|
-
* @param {string} opts.logsDir — 이벤트 로그 디렉토리
|
|
171
|
-
* @param {number} [opts.maxRestarts=3]
|
|
172
|
-
* @param {number} [opts.graceMs=10000] — shutdown grace period
|
|
173
|
-
* @param {object} [opts.probeOpts] — health-probe 옵션 오버라이드
|
|
174
|
-
* @param {object} [opts.broker] — AccountBroker 인스턴스 (tierFallback 구독용)
|
|
175
|
-
* @returns {Conductor}
|
|
176
|
-
*/
|
|
177
|
-
/**
|
|
178
|
-
* Conductor 생성.
|
|
179
|
-
* @param {object} [opts]
|
|
180
|
-
* @param {object} [opts.deps] — 의존성 주입 (테스트용 mock 지원)
|
|
181
|
-
* @param {Function} [opts.deps.execFile] — execFile 오버라이드
|
|
182
|
-
* @param {Function} [opts.deps.spawn] — spawn 오버라이드 (로컬/원격 child process 모두)
|
|
183
|
-
*/
|
|
184
|
-
export function createConductor(opts = {}) {
|
|
185
|
-
const spawnFn = opts.deps?.spawn || spawn;
|
|
186
|
-
const processUtils = {
|
|
187
|
-
killProcessTree: opts.deps?.killProcessTree || killProcessTree,
|
|
188
|
-
killProcessTreeSnapshot:
|
|
189
|
-
opts.deps?.killProcessTreeSnapshot || killProcessTreeSnapshot,
|
|
190
|
-
findProcessTree: opts.deps?.findProcessTree || findProcessTree,
|
|
191
|
-
};
|
|
192
|
-
const {
|
|
193
|
-
logsDir,
|
|
194
|
-
maxRestarts = DEFAULT_MAX_RESTARTS,
|
|
195
|
-
graceMs = DEFAULT_GRACE_MS,
|
|
196
|
-
probeOpts = {},
|
|
197
|
-
broker: optsBroker,
|
|
198
|
-
} = opts;
|
|
199
|
-
|
|
200
|
-
if (!logsDir) throw new Error("logsDir is required");
|
|
201
|
-
mkdirSync(logsDir, { recursive: true });
|
|
202
|
-
|
|
203
|
-
const emitter = new EventEmitter();
|
|
204
|
-
const sessions = new Map();
|
|
205
|
-
let shuttingDown = false;
|
|
206
|
-
const publicApi = null;
|
|
207
|
-
const synapseOpts = {
|
|
208
|
-
baseUrl: opts.synapseBaseUrl,
|
|
209
|
-
fetchImpl: opts.synapseFetch,
|
|
210
|
-
};
|
|
211
|
-
const getActiveBroker = () => optsBroker ?? broker;
|
|
212
|
-
|
|
213
|
-
function buildSynapseMeta(session, state = session.state, reason = "") {
|
|
214
|
-
return {
|
|
215
|
-
sessionId: session.id,
|
|
216
|
-
host:
|
|
217
|
-
typeof session.config.host === "string" &&
|
|
218
|
-
session.config.host.length > 0
|
|
219
|
-
? session.config.host
|
|
220
|
-
: "local",
|
|
221
|
-
taskSummary: buildSynapseTaskSummary(session.config.prompt),
|
|
222
|
-
status: state,
|
|
223
|
-
reason,
|
|
224
|
-
isRemote: Boolean(session.config.remote),
|
|
225
|
-
};
|
|
226
|
-
}
|
|
227
|
-
|
|
228
|
-
// 공유 event log (모든 세션 이벤트를 하나의 JSONL에)
|
|
229
|
-
const eventLog = createEventLog(join(logsDir, "conductor-events.jsonl"));
|
|
230
|
-
|
|
231
|
-
function logCtoContext(sessionId) {
|
|
232
|
-
const snapshot = readCtoSnapshot({ lakeRoot: opts.ctoLakeRoot });
|
|
233
|
-
if (!snapshot) return;
|
|
234
|
-
eventLog.append("cto_context", {
|
|
235
|
-
session: sessionId,
|
|
236
|
-
snapshot: {
|
|
237
|
-
schema_version: snapshot.schema_version,
|
|
238
|
-
repo: snapshot.repo,
|
|
239
|
-
sources: snapshot.sources,
|
|
240
|
-
ledger_tail: snapshot.ledger_tail,
|
|
241
|
-
live_sessions: snapshot.live_sessions,
|
|
242
|
-
active_shards: snapshot.active_shards,
|
|
243
|
-
},
|
|
244
|
-
});
|
|
245
|
-
}
|
|
246
|
-
|
|
247
|
-
/**
|
|
248
|
-
* 세션 상태 전이.
|
|
249
|
-
* @param {object} session
|
|
250
|
-
* @param {string} nextState
|
|
251
|
-
* @param {string} [reason]
|
|
252
|
-
*/
|
|
253
|
-
function transition(session, nextState, reason = "") {
|
|
254
|
-
const valid = TRANSITIONS[session.state] || [];
|
|
255
|
-
if (!valid.includes(nextState)) {
|
|
256
|
-
eventLog.append("invalid_transition", {
|
|
257
|
-
session: session.id,
|
|
258
|
-
from: session.state,
|
|
259
|
-
to: nextState,
|
|
260
|
-
reason,
|
|
261
|
-
});
|
|
262
|
-
return false;
|
|
263
|
-
}
|
|
264
|
-
|
|
265
|
-
const prev = session.state;
|
|
266
|
-
session.state = nextState;
|
|
267
|
-
|
|
268
|
-
eventLog.append("stateChange", {
|
|
269
|
-
session: session.id,
|
|
270
|
-
from: prev,
|
|
271
|
-
to: nextState,
|
|
272
|
-
reason,
|
|
273
|
-
restarts: session.restarts,
|
|
274
|
-
});
|
|
275
|
-
|
|
276
|
-
emitter.emit("stateChange", {
|
|
277
|
-
sessionId: session.id,
|
|
278
|
-
from: prev,
|
|
279
|
-
to: nextState,
|
|
280
|
-
reason,
|
|
281
|
-
});
|
|
282
|
-
|
|
283
|
-
if (nextState === STATES.HEALTHY) {
|
|
284
|
-
registerSynapseSession(
|
|
285
|
-
buildSynapseMeta(session, nextState, reason),
|
|
286
|
-
synapseOpts,
|
|
287
|
-
);
|
|
288
|
-
}
|
|
289
|
-
heartbeatSynapseSession(
|
|
290
|
-
session.id,
|
|
291
|
-
buildSynapseMeta(session, nextState, reason),
|
|
292
|
-
synapseOpts,
|
|
293
|
-
);
|
|
294
|
-
if (nextState === STATES.COMPLETED || nextState === STATES.DEAD) {
|
|
295
|
-
unregisterSynapseSession(session.id, synapseOpts);
|
|
296
|
-
}
|
|
297
|
-
|
|
298
|
-
// Terminal state cleanup
|
|
299
|
-
if (TERMINAL_STATES.has(nextState)) {
|
|
300
|
-
session.probe?.stop();
|
|
301
|
-
getConductorRegistry()?.unregister?.(session.id, publicApi);
|
|
302
|
-
}
|
|
303
|
-
|
|
304
|
-
return true;
|
|
305
|
-
}
|
|
306
|
-
|
|
307
|
-
/**
|
|
308
|
-
* 프로세스를 강제 종료.
|
|
309
|
-
* Windows: taskkill /T /F /PID (프로세스 트리). POSIX: SIGKILL.
|
|
310
|
-
*/
|
|
311
|
-
function forceKill(pid) {
|
|
312
|
-
if (!pid || pid <= 0) return;
|
|
313
|
-
killProcess(pid, {
|
|
314
|
-
signal: "SIGKILL",
|
|
315
|
-
tree: true,
|
|
316
|
-
force: true,
|
|
317
|
-
timeout: 5000,
|
|
318
|
-
});
|
|
319
|
-
}
|
|
320
|
-
|
|
321
|
-
/**
|
|
322
|
-
* 원격 세션의 psmux 세션을 SSH 경유로 kill.
|
|
323
|
-
* fire-and-forget: 실패해도 에러 전파 안 함.
|
|
324
|
-
*/
|
|
325
|
-
function killRemoteSession(session) {
|
|
326
|
-
const host = session.config.host;
|
|
327
|
-
if (!host) return;
|
|
328
|
-
let sshUser = session.config.sshUser;
|
|
329
|
-
let sshIp = host;
|
|
330
|
-
// hosts.json에서 ssh_user/IP 해결
|
|
331
|
-
try {
|
|
332
|
-
const hostsPath = join(
|
|
333
|
-
opts.repoRoot || process.cwd(),
|
|
334
|
-
"references",
|
|
335
|
-
"hosts.json",
|
|
336
|
-
);
|
|
337
|
-
const hosts = JSON.parse(readFileSync(hostsPath, "utf8"));
|
|
338
|
-
const hostCfg = hosts.hosts?.[host];
|
|
339
|
-
if (hostCfg) {
|
|
340
|
-
sshUser = sshUser || hostCfg.ssh_user;
|
|
341
|
-
sshIp = hostCfg.tailscale?.ip || host;
|
|
342
|
-
}
|
|
343
|
-
} catch {
|
|
344
|
-
/* hosts.json 없으면 fallback */
|
|
345
|
-
}
|
|
346
|
-
if (!sshUser) return;
|
|
347
|
-
const execFn = opts.deps?.execFile || execFile;
|
|
348
|
-
execFn(
|
|
349
|
-
"ssh",
|
|
350
|
-
[`${sshUser}@${sshIp}`, "psmux", "kill-session", "-t", session.id],
|
|
351
|
-
{ timeout: 10_000, reason: "conductor:killRemoteSession" },
|
|
352
|
-
() => {},
|
|
353
|
-
);
|
|
354
|
-
eventLog.append("remote_kill", {
|
|
355
|
-
session: session.id,
|
|
356
|
-
host,
|
|
357
|
-
sshUser,
|
|
358
|
-
sshIp,
|
|
359
|
-
});
|
|
360
|
-
}
|
|
361
|
-
|
|
362
|
-
/**
|
|
363
|
-
* 단일 세션의 child process를 정리.
|
|
364
|
-
* 원격 세션은 SSH 경유 psmux kill-session으로 정리.
|
|
365
|
-
*/
|
|
366
|
-
async function cleanupChild(session) {
|
|
367
|
-
session.probe?.stop();
|
|
368
|
-
|
|
369
|
-
// 원격 세션 — SSH 경유 psmux kill-session
|
|
370
|
-
if (session.config.remote) {
|
|
371
|
-
killRemoteSession(session);
|
|
372
|
-
return;
|
|
373
|
-
}
|
|
374
|
-
|
|
375
|
-
const child = session.child;
|
|
376
|
-
if (!child) return;
|
|
377
|
-
|
|
378
|
-
const pid = child.pid;
|
|
379
|
-
if (!pid) return;
|
|
380
|
-
|
|
381
|
-
try {
|
|
382
|
-
if (session.descendantSnapshot?.length > 0) {
|
|
383
|
-
processUtils.killProcessTreeSnapshot(session.descendantSnapshot);
|
|
384
|
-
}
|
|
385
|
-
} catch {
|
|
386
|
-
/* best-effort cleanup */
|
|
387
|
-
}
|
|
388
|
-
|
|
389
|
-
if (!session.alive) return;
|
|
390
|
-
|
|
391
|
-
// SIGTERM 먼저
|
|
392
|
-
try {
|
|
393
|
-
child.kill("SIGTERM");
|
|
394
|
-
} catch {
|
|
395
|
-
/* already dead */
|
|
396
|
-
}
|
|
397
|
-
|
|
398
|
-
// Grace period 대기
|
|
399
|
-
const exited = await new Promise((resolve) => {
|
|
400
|
-
const timer = setTimeout(() => {
|
|
401
|
-
resolve(false);
|
|
402
|
-
}, graceMs);
|
|
403
|
-
child.once("exit", () => {
|
|
404
|
-
clearTimeout(timer);
|
|
405
|
-
resolve(true);
|
|
406
|
-
});
|
|
407
|
-
});
|
|
408
|
-
|
|
409
|
-
if (!exited) {
|
|
410
|
-
try {
|
|
411
|
-
processUtils.killProcessTree(pid);
|
|
412
|
-
} catch {
|
|
413
|
-
forceKill(pid);
|
|
414
|
-
}
|
|
415
|
-
session.alive = false;
|
|
416
|
-
}
|
|
417
|
-
}
|
|
418
|
-
|
|
419
|
-
/**
|
|
420
|
-
* Health probe 콜백 — probe 결과에 따라 상태 전이 판단.
|
|
421
|
-
*/
|
|
422
|
-
function handleProbeResult(session, result) {
|
|
423
|
-
if (TERMINAL_STATES.has(session.state)) return;
|
|
424
|
-
if (session.state === STATES.INIT || session.state === STATES.RESTARTING)
|
|
425
|
-
return;
|
|
426
|
-
|
|
427
|
-
eventLog.append("health", {
|
|
428
|
-
session: session.id,
|
|
429
|
-
...result,
|
|
430
|
-
});
|
|
431
|
-
|
|
432
|
-
// L0 실패 — 로컬: exit handler에서 처리. 원격: probe가 유일한 감지 수단.
|
|
433
|
-
if (result.l0 === "fail") {
|
|
434
|
-
if (session.config.remote) {
|
|
435
|
-
handleFailure(session, "remote_L0_fail");
|
|
436
|
-
}
|
|
437
|
-
return;
|
|
438
|
-
}
|
|
439
|
-
|
|
440
|
-
// L3 completed (원격 완료 토큰 감지)
|
|
441
|
-
if (result.l3 === "completed" && session.config.remote) {
|
|
442
|
-
transition(session, STATES.COMPLETED, "remote_completion_token");
|
|
443
|
-
emitter.emit("completed", { sessionId: session.id });
|
|
444
|
-
if (typeof session.config.onCompleted === "function") {
|
|
445
|
-
session.config.onCompleted({ sessionId: session.id });
|
|
446
|
-
}
|
|
447
|
-
maybeAutoShutdown();
|
|
448
|
-
return;
|
|
449
|
-
}
|
|
450
|
-
|
|
451
|
-
// L1 INPUT_WAIT 감지
|
|
452
|
-
if (result.l1 === "input_wait" && session.state === STATES.HEALTHY) {
|
|
453
|
-
transition(
|
|
454
|
-
session,
|
|
455
|
-
STATES.INPUT_WAIT,
|
|
456
|
-
`input_wait:${result.inputWaitPattern}`,
|
|
457
|
-
);
|
|
458
|
-
emitter.emit("inputWait", {
|
|
459
|
-
sessionId: session.id,
|
|
460
|
-
pattern: result.inputWaitPattern,
|
|
461
|
-
});
|
|
462
|
-
return;
|
|
463
|
-
}
|
|
464
|
-
|
|
465
|
-
// INPUT_WAIT → output 재개 시 HEALTHY 복귀
|
|
466
|
-
if (session.state === STATES.INPUT_WAIT && result.l1 === "ok") {
|
|
467
|
-
transition(session, STATES.HEALTHY, "output_resumed");
|
|
468
|
-
return;
|
|
469
|
-
}
|
|
470
|
-
|
|
471
|
-
// L1 stall
|
|
472
|
-
if (result.l1 === "stall" && session.state === STATES.HEALTHY) {
|
|
473
|
-
transition(session, STATES.STALLED, "L1_stall");
|
|
474
|
-
return;
|
|
475
|
-
}
|
|
476
|
-
|
|
477
|
-
// STALLED → output 재개 시 HEALTHY 복귀
|
|
478
|
-
if (session.state === STATES.STALLED && result.l1 === "ok") {
|
|
479
|
-
transition(session, STATES.HEALTHY, "output_resumed");
|
|
480
|
-
return;
|
|
481
|
-
}
|
|
482
|
-
|
|
483
|
-
// L3 timeout (아직 STARTING 상태)
|
|
484
|
-
if (result.l3 === "timeout" && session.state === STATES.STARTING) {
|
|
485
|
-
handleFailure(session, "L3_timeout");
|
|
486
|
-
return;
|
|
487
|
-
}
|
|
488
|
-
|
|
489
|
-
// STARTING → L0 ok + L3 ok → HEALTHY
|
|
490
|
-
if (
|
|
491
|
-
session.state === STATES.STARTING &&
|
|
492
|
-
result.l0 === "ok" &&
|
|
493
|
-
result.l3 === "ok"
|
|
494
|
-
) {
|
|
495
|
-
transition(session, STATES.HEALTHY, "probe_healthy");
|
|
496
|
-
return;
|
|
497
|
-
}
|
|
498
|
-
|
|
499
|
-
// STARTING → L0 ok (L3 아직 미판정) → STARTING 유지 (대기)
|
|
500
|
-
}
|
|
501
|
-
|
|
502
|
-
/**
|
|
503
|
-
* 실패 처리 — restart 또는 DEAD.
|
|
504
|
-
*/
|
|
505
|
-
async function handleFailure(session, reason) {
|
|
506
|
-
if (TERMINAL_STATES.has(session.state)) return;
|
|
507
|
-
|
|
508
|
-
transition(session, STATES.FAILED, reason);
|
|
509
|
-
|
|
510
|
-
if (session.restarts < maxRestarts) {
|
|
511
|
-
transition(
|
|
512
|
-
session,
|
|
513
|
-
STATES.RESTARTING,
|
|
514
|
-
`restart_${session.restarts + 1}/${maxRestarts}`,
|
|
515
|
-
);
|
|
516
|
-
session.restarts += 1;
|
|
517
|
-
const backoffMs = Math.min(1000 * 2 ** (session.restarts - 1), 30_000);
|
|
518
|
-
session._respawnTimer = setTimeout(() => {
|
|
519
|
-
session._respawnTimer = null;
|
|
520
|
-
if (session.config.remote) {
|
|
521
|
-
startRemoteSession(session);
|
|
522
|
-
} else {
|
|
523
|
-
void respawnSession(session);
|
|
524
|
-
}
|
|
525
|
-
}, backoffMs);
|
|
526
|
-
} else {
|
|
527
|
-
await cleanupChild(session);
|
|
528
|
-
transition(session, STATES.DEAD, `maxRestarts(${maxRestarts})_exceeded`);
|
|
529
|
-
emitter.emit("dead", { sessionId: session.id, reason });
|
|
530
|
-
|
|
531
|
-
// broker release on final death
|
|
532
|
-
const activeBroker = getActiveBroker();
|
|
533
|
-
if (activeBroker && session.config.accountId) {
|
|
534
|
-
activeBroker.release(session.config.accountId, {
|
|
535
|
-
ok: false,
|
|
536
|
-
failureMode: session.lastFailureMode,
|
|
537
|
-
});
|
|
538
|
-
if (session.lastFailureMode === "rate_limited") {
|
|
539
|
-
activeBroker.markRateLimited(session.config.accountId, 5 * 60 * 1000);
|
|
540
|
-
}
|
|
541
|
-
}
|
|
542
|
-
}
|
|
543
|
-
}
|
|
544
|
-
|
|
545
|
-
/**
|
|
546
|
-
* 세션의 child process를 (재)시작.
|
|
547
|
-
*/
|
|
548
|
-
async function respawnSession(session) {
|
|
549
|
-
// 기존 child 정리
|
|
550
|
-
await cleanupChild(session);
|
|
551
|
-
|
|
552
|
-
transition(
|
|
553
|
-
session,
|
|
554
|
-
STATES.STARTING,
|
|
555
|
-
session.restarts > 0 ? "respawn" : "initial",
|
|
556
|
-
);
|
|
557
|
-
|
|
558
|
-
const launcher = session.launcher;
|
|
559
|
-
const outPath = join(logsDir, `${session.id}.out.log`);
|
|
560
|
-
const errPath = join(logsDir, `${session.id}.err.log`);
|
|
561
|
-
mkdirSync(logsDir, { recursive: true });
|
|
562
|
-
|
|
563
|
-
const outWs = createWriteStream(outPath, { flags: "a" });
|
|
564
|
-
const errWs = createWriteStream(errPath, { flags: "a" });
|
|
565
|
-
|
|
566
|
-
let outputBytes = 0;
|
|
567
|
-
let recentOutput = "";
|
|
568
|
-
// #115 F7 wiring: stdout-only tail for completion payload extraction.
|
|
569
|
-
let stdoutTail = "";
|
|
570
|
-
// #125: sentinel-framed capture (preferred over the 16 KiB tail; survives
|
|
571
|
-
// payloads larger than the sliding window).
|
|
572
|
-
const sentinelCapture = createSentinelCapture();
|
|
573
|
-
|
|
574
|
-
const spawnCwd = session.config.workdir || launcher.cwd || undefined;
|
|
575
|
-
let excludeMcpServers = [];
|
|
576
|
-
const requestedMcpServers = Array.isArray(session.config.mcpServers)
|
|
577
|
-
? session.config.mcpServers
|
|
578
|
-
: [];
|
|
579
|
-
if (session.config.agent === "codex" && requestedMcpServers.length > 0) {
|
|
580
|
-
const preflightFn = opts.deps?.codexPreflight || runCodexPreflight;
|
|
581
|
-
try {
|
|
582
|
-
const preflight = await preflightFn({
|
|
583
|
-
mcpServers: requestedMcpServers,
|
|
584
|
-
subcommand: "exec",
|
|
585
|
-
workdir: spawnCwd,
|
|
586
|
-
});
|
|
587
|
-
excludeMcpServers = Array.isArray(preflight?.excludeMcpServers)
|
|
588
|
-
? preflight.excludeMcpServers
|
|
589
|
-
: [];
|
|
590
|
-
for (const warning of preflight?.warnings || []) {
|
|
591
|
-
eventLog.append("codex_preflight_warning", {
|
|
592
|
-
session: session.id,
|
|
593
|
-
warning,
|
|
594
|
-
});
|
|
595
|
-
emitter.emit("warning", {
|
|
596
|
-
type: "codex_preflight_warning",
|
|
597
|
-
session: session.id,
|
|
598
|
-
warning,
|
|
599
|
-
});
|
|
600
|
-
}
|
|
601
|
-
} catch (err) {
|
|
602
|
-
eventLog.append("codex_preflight_failed", {
|
|
603
|
-
session: session.id,
|
|
604
|
-
error: err.message,
|
|
605
|
-
});
|
|
606
|
-
emitter.emit("warning", {
|
|
607
|
-
type: "codex_preflight_failed",
|
|
608
|
-
session: session.id,
|
|
609
|
-
error: err.message,
|
|
610
|
-
});
|
|
611
|
-
}
|
|
612
|
-
}
|
|
613
|
-
|
|
614
|
-
let spawnSpec;
|
|
615
|
-
try {
|
|
616
|
-
spawnSpec = buildSpawnSpecForMode(MODES.HEADLESS, {
|
|
617
|
-
cli: session.config.agent,
|
|
618
|
-
prompt: session.config.prompt,
|
|
619
|
-
profile: session.config.profile,
|
|
620
|
-
role: session.config.role,
|
|
621
|
-
model: session.config.model,
|
|
622
|
-
env: session.config.env,
|
|
623
|
-
mcpServers: session.config.mcpServers,
|
|
624
|
-
excludeMcpServers,
|
|
625
|
-
resolveCommand: opts.deps?.resolveCliExecutable,
|
|
626
|
-
onWarning: (message, details = {}) => {
|
|
627
|
-
eventLog.append("mcp_server_skipped", {
|
|
628
|
-
session: session.id,
|
|
629
|
-
message,
|
|
630
|
-
...details,
|
|
631
|
-
});
|
|
632
|
-
emitter.emit("warning", {
|
|
633
|
-
type: "mcp_server_skipped",
|
|
634
|
-
session: session.id,
|
|
635
|
-
message,
|
|
636
|
-
...details,
|
|
637
|
-
});
|
|
638
|
-
},
|
|
639
|
-
});
|
|
640
|
-
} catch (err) {
|
|
641
|
-
// respawnSession 은 `void` 로 호출되므로 여기서 던지면 처리되지 않은 거부가
|
|
642
|
-
// 된다. spec 생성 실패를 명시 실패로 접어 세션이 매달리지 않게 한다.
|
|
643
|
-
// 정책 차단과 그 밖의 spec 생성 실패는 원인이 다르므로 이벤트를 나눈다.
|
|
644
|
-
const message = err?.message || String(err);
|
|
645
|
-
const blockedByPolicy = err?.code === "TFX_CLI_DISABLED";
|
|
646
|
-
const eventName = blockedByPolicy
|
|
647
|
-
? "session_spawn_blocked"
|
|
648
|
-
: "session_spawn_failed";
|
|
649
|
-
const reason = blockedByPolicy ? "spawn_blocked" : "spawn_failed";
|
|
650
|
-
const detail = {
|
|
651
|
-
session: session.id,
|
|
652
|
-
agent: session.config.agent,
|
|
653
|
-
code: err?.code || null,
|
|
654
|
-
error: message,
|
|
655
|
-
};
|
|
656
|
-
eventLog.append(eventName, detail);
|
|
657
|
-
emitter.emit("warning", { type: eventName, ...detail });
|
|
658
|
-
handleFailure(session, `${reason}:${message}`);
|
|
659
|
-
return;
|
|
660
|
-
}
|
|
661
|
-
|
|
662
|
-
// #90 branch guard: shard spawn cwd가 main 브랜치면 즉시 abort.
|
|
663
|
-
// swarm shard는 반드시 shard 전용 worktree 브랜치에서 실행되어야 한다.
|
|
664
|
-
// cwd fallback으로 main working tree에 떨어진 경우 차단.
|
|
665
|
-
if (session.config.branchGuard && spawnCwd) {
|
|
666
|
-
try {
|
|
667
|
-
const r = spawnSync("git", ["rev-parse", "--abbrev-ref", "HEAD"], {
|
|
668
|
-
cwd: spawnCwd,
|
|
669
|
-
encoding: "utf8",
|
|
670
|
-
windowsHide: true,
|
|
671
|
-
timeout: 5_000,
|
|
672
|
-
});
|
|
673
|
-
const curBranch = String(r.stdout || "").trim();
|
|
674
|
-
if (curBranch === "main" || curBranch === "master") {
|
|
675
|
-
eventLog.append("branch_guard_refused", {
|
|
676
|
-
session: session.id,
|
|
677
|
-
cwd: spawnCwd,
|
|
678
|
-
branch: curBranch,
|
|
679
|
-
});
|
|
680
|
-
handleFailure(session, `branch_guard:${curBranch}_refused`);
|
|
681
|
-
return;
|
|
682
|
-
}
|
|
683
|
-
} catch (err) {
|
|
684
|
-
eventLog.append("branch_guard_probe_failed", {
|
|
685
|
-
session: session.id,
|
|
686
|
-
cwd: spawnCwd,
|
|
687
|
-
error: err.message,
|
|
688
|
-
});
|
|
689
|
-
// 프로브 실패는 비차단 — git 미설치/non-repo 환경 대응
|
|
690
|
-
}
|
|
691
|
-
}
|
|
692
|
-
|
|
693
|
-
const spawnEnv = {
|
|
694
|
-
...process.env,
|
|
695
|
-
...launcher.env,
|
|
696
|
-
...(session.config.env || {}),
|
|
697
|
-
...(session.config.branchGuard ? { TRIFLUX_GIT_BRANCH_GUARD: "1" } : {}),
|
|
698
|
-
};
|
|
699
|
-
if (session.config.workerSandbox?.root) {
|
|
700
|
-
eventLog.append("worker_sandbox_env", {
|
|
701
|
-
session: session.id,
|
|
702
|
-
root: session.config.workerSandbox.root,
|
|
703
|
-
codeHomePreserved: Boolean(spawnEnv.CODEX_HOME),
|
|
704
|
-
});
|
|
705
|
-
}
|
|
706
|
-
|
|
707
|
-
let child;
|
|
708
|
-
try {
|
|
709
|
-
child = spawnFn(spawnSpec.command, spawnSpec.args, {
|
|
710
|
-
shell: false,
|
|
711
|
-
cwd: spawnCwd,
|
|
712
|
-
env: spawnEnv,
|
|
713
|
-
reason: `conductor:respawnSession:${session.id}`,
|
|
714
|
-
stdio: ["pipe", "pipe", "pipe"],
|
|
715
|
-
windowsHide: true,
|
|
716
|
-
});
|
|
717
|
-
} catch (err) {
|
|
718
|
-
eventLog.append("spawn_error", {
|
|
719
|
-
session: session.id,
|
|
720
|
-
error: err.message,
|
|
721
|
-
cwd: spawnCwd || null,
|
|
722
|
-
command: spawnSpec.command,
|
|
723
|
-
});
|
|
724
|
-
handleFailure(session, `spawn_error:${err.message}`);
|
|
725
|
-
return;
|
|
726
|
-
}
|
|
727
|
-
|
|
728
|
-
session.child = child;
|
|
729
|
-
session.outPath = outPath;
|
|
730
|
-
session.errPath = errPath;
|
|
731
|
-
session.descendantSnapshot = [];
|
|
732
|
-
|
|
733
|
-
// #116-C: spawnSpec.stdinPrompt 가 true 면 codex 의 oh-my-codex hook
|
|
734
|
-
// (macOS) 가 매우 긴 argv-inline prompt 에서 SessionStart Failed → exit 1
|
|
735
|
-
// 회귀를 유발하므로 prompt 를 stdin write 로 분리해서 전달한다.
|
|
736
|
-
// stdinPrompt 없으면 기존 흐름 — codex "Reading additional input..." 대기
|
|
737
|
-
// 방지를 위해 stdin 즉시 닫기.
|
|
738
|
-
if (child.stdin) {
|
|
739
|
-
if (spawnSpec.stdinPrompt && typeof spawnSpec.prompt === "string") {
|
|
740
|
-
try {
|
|
741
|
-
child.stdin.write(spawnSpec.prompt);
|
|
742
|
-
} catch (err) {
|
|
743
|
-
eventLog.append("spawn_stdin_write_error", {
|
|
744
|
-
session: session.id,
|
|
745
|
-
error: err.message,
|
|
746
|
-
});
|
|
747
|
-
}
|
|
748
|
-
}
|
|
749
|
-
child.stdin.end();
|
|
750
|
-
}
|
|
751
|
-
|
|
752
|
-
eventLog.append("spawn", {
|
|
753
|
-
session: session.id,
|
|
754
|
-
agent: session.config.agent,
|
|
755
|
-
pid: child.pid,
|
|
756
|
-
command: spawnSpec.command,
|
|
757
|
-
args: spawnSpec.args,
|
|
758
|
-
legacyCommand: launcher.command,
|
|
759
|
-
restart: session.restarts,
|
|
760
|
-
});
|
|
761
|
-
|
|
762
|
-
if (session._descendantSnapshotTimer) {
|
|
763
|
-
clearTimeout(session._descendantSnapshotTimer);
|
|
764
|
-
}
|
|
765
|
-
session._descendantSnapshotTimer = setTimeout(() => {
|
|
766
|
-
session._descendantSnapshotTimer = null;
|
|
767
|
-
if (session.child !== child) return;
|
|
768
|
-
try {
|
|
769
|
-
session.descendantSnapshot = processUtils.findProcessTree(child.pid);
|
|
770
|
-
} catch {
|
|
771
|
-
/* best-effort snapshot */
|
|
772
|
-
}
|
|
773
|
-
}, 200);
|
|
774
|
-
|
|
775
|
-
// stdout+stderr 통합 추적 (F3 해결: stderr만 출력되는 경우도 advancing 판정)
|
|
776
|
-
const trackOutput = (buf) => {
|
|
777
|
-
outputBytes += buf.length;
|
|
778
|
-
const txt = String(buf);
|
|
779
|
-
// 최근 2KB만 유지 (INPUT_WAIT 패턴 감지용)
|
|
780
|
-
recentOutput += txt;
|
|
781
|
-
if (recentOutput.length > 2048) {
|
|
782
|
-
recentOutput = recentOutput.slice(-2048);
|
|
783
|
-
}
|
|
784
|
-
};
|
|
785
|
-
|
|
786
|
-
child.stdout?.on("data", (buf) => {
|
|
787
|
-
outWs.write(buf);
|
|
788
|
-
trackOutput(buf);
|
|
789
|
-
const text = buf.toString("utf8");
|
|
790
|
-
stdoutTail = (stdoutTail + text).slice(-16384);
|
|
791
|
-
sentinelCapture.push(text);
|
|
792
|
-
});
|
|
793
|
-
child.stderr?.on("data", (buf) => {
|
|
794
|
-
errWs.write(buf);
|
|
795
|
-
trackOutput(buf);
|
|
796
|
-
});
|
|
797
|
-
|
|
798
|
-
child.on("exit", (code, signal) => {
|
|
799
|
-
session.alive = false;
|
|
800
|
-
if (session._descendantSnapshotTimer) {
|
|
801
|
-
clearTimeout(session._descendantSnapshotTimer);
|
|
802
|
-
session._descendantSnapshotTimer = null;
|
|
803
|
-
}
|
|
804
|
-
try {
|
|
805
|
-
outWs.end();
|
|
806
|
-
} catch {
|
|
807
|
-
/* ignore */
|
|
808
|
-
}
|
|
809
|
-
try {
|
|
810
|
-
errWs.end();
|
|
811
|
-
} catch {
|
|
812
|
-
/* ignore */
|
|
813
|
-
}
|
|
814
|
-
|
|
815
|
-
eventLog.append("exit", {
|
|
816
|
-
session: session.id,
|
|
817
|
-
code,
|
|
818
|
-
signal,
|
|
819
|
-
restart: session.restarts,
|
|
820
|
-
});
|
|
821
|
-
|
|
822
|
-
if (TERMINAL_STATES.has(session.state)) return;
|
|
823
|
-
if (shuttingDown) return;
|
|
824
|
-
|
|
825
|
-
if (code === 0 && !signal) {
|
|
826
|
-
transition(session, STATES.COMPLETED, "exit_0");
|
|
827
|
-
// #125 Codex R1 MEDIUM: if sentinel capture overflowed past maxBytes
|
|
828
|
-
// and lost its BEGIN marker, do NOT fall back to stdoutTail/brace-scan
|
|
829
|
-
// — the worker tried sentinel protocol and the payload was clearly
|
|
830
|
-
// unbounded. Surface as "no payload" rather than silent partial.
|
|
831
|
-
let extracted;
|
|
832
|
-
if (sentinelCapture.isOverflow()) {
|
|
833
|
-
extracted = null;
|
|
834
|
-
} else {
|
|
835
|
-
const sentinelSnapshot = sentinelCapture.snapshot();
|
|
836
|
-
extracted = extractCompletionPayload(sentinelSnapshot || stdoutTail);
|
|
837
|
-
}
|
|
838
|
-
const completionPayload = extracted ? extracted.payload : undefined;
|
|
839
|
-
emitter.emit("completed", { sessionId: session.id, completionPayload });
|
|
840
|
-
if (typeof session.config.onCompleted === "function") {
|
|
841
|
-
session.config.onCompleted({
|
|
842
|
-
sessionId: session.id,
|
|
843
|
-
completionPayload,
|
|
844
|
-
});
|
|
845
|
-
}
|
|
846
|
-
const activeBroker = getActiveBroker();
|
|
847
|
-
if (activeBroker && session.config.accountId) {
|
|
848
|
-
activeBroker.release(session.config.accountId, { ok: true });
|
|
849
|
-
}
|
|
850
|
-
} else {
|
|
851
|
-
// detect rate_limited from recent output before handleFailure
|
|
852
|
-
if (
|
|
853
|
-
/(rate.?limit|quota|throttl|too.many.requests|429|usage.limit)/iu.test(
|
|
854
|
-
recentOutput,
|
|
855
|
-
)
|
|
856
|
-
) {
|
|
857
|
-
session.lastFailureMode = "rate_limited";
|
|
858
|
-
}
|
|
859
|
-
handleFailure(session, `exit_code:${code},signal:${signal}`);
|
|
860
|
-
}
|
|
861
|
-
|
|
862
|
-
maybeAutoShutdown();
|
|
863
|
-
});
|
|
864
|
-
|
|
865
|
-
child.on("error", (err) => {
|
|
866
|
-
session.alive = false;
|
|
867
|
-
if (session._descendantSnapshotTimer) {
|
|
868
|
-
clearTimeout(session._descendantSnapshotTimer);
|
|
869
|
-
session._descendantSnapshotTimer = null;
|
|
870
|
-
}
|
|
871
|
-
eventLog.append("child_error", {
|
|
872
|
-
session: session.id,
|
|
873
|
-
error: err.message,
|
|
874
|
-
});
|
|
875
|
-
if (!shuttingDown && !TERMINAL_STATES.has(session.state)) {
|
|
876
|
-
handleFailure(session, `child_error:${err.message}`);
|
|
877
|
-
}
|
|
878
|
-
});
|
|
879
|
-
|
|
880
|
-
session.alive = true;
|
|
881
|
-
|
|
882
|
-
// Health probe 설정
|
|
883
|
-
session.probe?.stop();
|
|
884
|
-
const probe = createHealthProbe(
|
|
885
|
-
{
|
|
886
|
-
get pid() {
|
|
887
|
-
return child.pid;
|
|
888
|
-
},
|
|
889
|
-
get alive() {
|
|
890
|
-
return session.alive;
|
|
891
|
-
},
|
|
892
|
-
getOutputBytes: () => outputBytes,
|
|
893
|
-
getRecentOutput: () => recentOutput,
|
|
894
|
-
},
|
|
895
|
-
{
|
|
896
|
-
...probeOpts,
|
|
897
|
-
// #165: default off → on. atomic write (#162) 로 race 제거됨.
|
|
898
|
-
// opt-out: TFX_PROBE_WRITE_STATE=0 명시.
|
|
899
|
-
writeStateFile:
|
|
900
|
-
probeOpts.writeStateFile ?? process.env.TFX_PROBE_WRITE_STATE !== "0",
|
|
901
|
-
// #168 P3: L2 (hub /health) checker wiring. probeOpts 가 명시 주입하면
|
|
902
|
-
// 그걸 우선. 아니면 TFX_PROBE_L2=0 로 opt-out. 나머지 경우 hub URL 기반
|
|
903
|
-
// default checker 주입 → deriveState 가 `mcp_initializing` 라벨을 생산 →
|
|
904
|
-
// heartbeat (read_probe_state) 가 probe-grace 분기로 감.
|
|
905
|
-
enableL2: probeOpts.enableL2 ?? process.env.TFX_PROBE_L2 !== "0",
|
|
906
|
-
checkMcp:
|
|
907
|
-
probeOpts.checkMcp ||
|
|
908
|
-
(process.env.TFX_PROBE_L2 === "0"
|
|
909
|
-
? undefined
|
|
910
|
-
: createHubHealthChecker({ hubUrl: process.env.TFX_HUB_URL })),
|
|
911
|
-
onProbe: (result) => handleProbeResult(session, result),
|
|
912
|
-
},
|
|
913
|
-
);
|
|
914
|
-
session.probe = probe;
|
|
915
|
-
probe.start();
|
|
916
|
-
}
|
|
917
|
-
|
|
918
|
-
/**
|
|
919
|
-
* 원격 세션 시작 — SSH child process로 직접 실행.
|
|
920
|
-
* psmux 불필요. 원격 호스트에서 claude -p 를 SSH 경유로 실행하고
|
|
921
|
-
* 로컬 child process로 관리한다.
|
|
922
|
-
*/
|
|
923
|
-
function startRemoteSession(session) {
|
|
924
|
-
transition(session, STATES.STARTING, "remote_initial");
|
|
925
|
-
|
|
926
|
-
const { host, prompt, agent, workdir: remoteWorkdir } = session.config;
|
|
927
|
-
|
|
928
|
-
// SSH args를 배열로 구성하여 셸 이스케이프 문제 회피
|
|
929
|
-
const cdPrefix = remoteWorkdir ? `cd ${remoteWorkdir} && ` : "";
|
|
930
|
-
|
|
931
|
-
let remoteBin;
|
|
932
|
-
if (agent === "claude") {
|
|
933
|
-
remoteBin = "claude --output-format text";
|
|
934
|
-
} else if (agent === "gemini") {
|
|
935
|
-
remoteBin = "gemini -y";
|
|
936
|
-
} else {
|
|
937
|
-
const codexHome = session.config.env?.CODEX_HOME;
|
|
938
|
-
const profile = resolveNestedCodexAgentProfile(
|
|
939
|
-
session.config.role || "executor",
|
|
940
|
-
{
|
|
941
|
-
profileOverride: session.config.profile ?? "auto",
|
|
942
|
-
globalProfile:
|
|
943
|
-
session.config.env?.TFX_CODEX_PROFILE ??
|
|
944
|
-
process.env.TFX_CODEX_PROFILE,
|
|
945
|
-
codexHome,
|
|
946
|
-
},
|
|
947
|
-
);
|
|
948
|
-
const configArgs = codexProfileConfigOverrides(profile, {
|
|
949
|
-
codexHome,
|
|
950
|
-
disallowUltra: true,
|
|
951
|
-
enforceCanonicalProfile: true,
|
|
952
|
-
})
|
|
953
|
-
.map((override) => `-c '${override.replace(/'/g, `'\\''`)}'`)
|
|
954
|
-
.join(" ");
|
|
955
|
-
remoteBin = `codex exec${configArgs ? ` ${configArgs}` : ""} -s danger-full-access --dangerously-bypass-approvals-and-sandbox`;
|
|
956
|
-
}
|
|
957
|
-
|
|
958
|
-
// prompt는 stdin으로 전달 — 셸 이스케이프 문제 완전 회피
|
|
959
|
-
const sshArgs = [
|
|
960
|
-
"-o",
|
|
961
|
-
"ConnectTimeout=30",
|
|
962
|
-
"-o",
|
|
963
|
-
"BatchMode=yes",
|
|
964
|
-
host,
|
|
965
|
-
`${cdPrefix}${remoteBin}`,
|
|
966
|
-
];
|
|
967
|
-
|
|
968
|
-
const sshCmd = `ssh ${sshArgs.join(" ")}`;
|
|
969
|
-
|
|
970
|
-
eventLog.append("remote_start", {
|
|
971
|
-
session: session.id,
|
|
972
|
-
host,
|
|
973
|
-
command: sshCmd,
|
|
974
|
-
});
|
|
975
|
-
|
|
976
|
-
const outPath = join(logsDir, `${session.id}.out.log`);
|
|
977
|
-
const errPath = join(logsDir, `${session.id}.err.log`);
|
|
978
|
-
const outWs = createWriteStream(outPath, { flags: "a" });
|
|
979
|
-
const errWs = createWriteStream(errPath, { flags: "a" });
|
|
980
|
-
|
|
981
|
-
let child;
|
|
982
|
-
try {
|
|
983
|
-
child = spawnFn("ssh", sshArgs, {
|
|
984
|
-
env: process.env,
|
|
985
|
-
reason: `conductor:remoteSession:${session.id}`,
|
|
986
|
-
stdio: ["pipe", "pipe", "pipe"],
|
|
987
|
-
windowsHide: true,
|
|
988
|
-
});
|
|
989
|
-
} catch (err) {
|
|
990
|
-
eventLog.append("spawn_error", {
|
|
991
|
-
session: session.id,
|
|
992
|
-
error: err.message,
|
|
993
|
-
});
|
|
994
|
-
handleFailure(session, `spawn_error:${err.message}`);
|
|
995
|
-
return;
|
|
996
|
-
}
|
|
997
|
-
|
|
998
|
-
session.child = child;
|
|
999
|
-
session.outPath = outPath;
|
|
1000
|
-
session.errPath = errPath;
|
|
1001
|
-
session.alive = true;
|
|
1002
|
-
|
|
1003
|
-
// stdin으로 프롬프트 전달 — 셸 이스케이프 완전 우회
|
|
1004
|
-
if (child.stdin) {
|
|
1005
|
-
child.stdin.write(prompt);
|
|
1006
|
-
child.stdin.end();
|
|
1007
|
-
}
|
|
1008
|
-
|
|
1009
|
-
eventLog.append("spawn", {
|
|
1010
|
-
session: session.id,
|
|
1011
|
-
agent,
|
|
1012
|
-
pid: child.pid,
|
|
1013
|
-
command: sshCmd,
|
|
1014
|
-
remote: true,
|
|
1015
|
-
host,
|
|
1016
|
-
});
|
|
1017
|
-
|
|
1018
|
-
// stdout/stderr 추적 (recentOutput으로 rate_limit 패턴 감지 가능)
|
|
1019
|
-
let outputBytes = 0;
|
|
1020
|
-
let recentOutput = "";
|
|
1021
|
-
// #115 F7 wiring: stdout-only tail for completion payload extraction.
|
|
1022
|
-
let stdoutTail = "";
|
|
1023
|
-
// #125: sentinel-framed capture (preferred over the 16 KiB tail).
|
|
1024
|
-
const sentinelCapture = createSentinelCapture();
|
|
1025
|
-
const trackOutput = (buf) => {
|
|
1026
|
-
outputBytes += buf.length;
|
|
1027
|
-
const text = buf.toString("utf8");
|
|
1028
|
-
recentOutput = (recentOutput + text).slice(-2000);
|
|
1029
|
-
};
|
|
1030
|
-
|
|
1031
|
-
child.stdout?.on("data", (buf) => {
|
|
1032
|
-
trackOutput(buf);
|
|
1033
|
-
outWs.write(buf);
|
|
1034
|
-
const text = buf.toString("utf8");
|
|
1035
|
-
stdoutTail = (stdoutTail + text).slice(-16384);
|
|
1036
|
-
sentinelCapture.push(text);
|
|
1037
|
-
});
|
|
1038
|
-
child.stderr?.on("data", (buf) => {
|
|
1039
|
-
trackOutput(buf);
|
|
1040
|
-
errWs.write(buf);
|
|
1041
|
-
});
|
|
1042
|
-
|
|
1043
|
-
// 상태를 healthy로 전이
|
|
1044
|
-
transition(session, STATES.HEALTHY, "remote_ssh_spawned");
|
|
1045
|
-
|
|
1046
|
-
// Health probe — 주기적으로 출력 변화 체크 (F3 stall 감지)
|
|
1047
|
-
let lastBytes = 0;
|
|
1048
|
-
let stallChecks = 0;
|
|
1049
|
-
const stallThreshold = probeOpts.stallChecks || 8; // 8 * intervalMs = stall
|
|
1050
|
-
const healthInterval = setInterval(() => {
|
|
1051
|
-
if (TERMINAL_STATES.has(session.state)) {
|
|
1052
|
-
clearInterval(healthInterval);
|
|
1053
|
-
return;
|
|
1054
|
-
}
|
|
1055
|
-
if (outputBytes > lastBytes) {
|
|
1056
|
-
lastBytes = outputBytes;
|
|
1057
|
-
stallChecks = 0;
|
|
1058
|
-
if (session.state !== STATES.HEALTHY) {
|
|
1059
|
-
transition(session, STATES.HEALTHY, "output_advancing");
|
|
1060
|
-
}
|
|
1061
|
-
} else {
|
|
1062
|
-
stallChecks++;
|
|
1063
|
-
if (stallChecks >= stallThreshold && session.state === STATES.HEALTHY) {
|
|
1064
|
-
transition(session, STATES.STALLED, "remote_no_output");
|
|
1065
|
-
}
|
|
1066
|
-
}
|
|
1067
|
-
}, probeOpts.intervalMs || 5000);
|
|
1068
|
-
|
|
1069
|
-
// exit 핸들러
|
|
1070
|
-
child.once("exit", (code) => {
|
|
1071
|
-
clearInterval(healthInterval);
|
|
1072
|
-
outWs.end();
|
|
1073
|
-
errWs.end();
|
|
1074
|
-
session.alive = false;
|
|
1075
|
-
|
|
1076
|
-
eventLog.append("remote_exit", {
|
|
1077
|
-
session: session.id,
|
|
1078
|
-
code,
|
|
1079
|
-
outputBytes,
|
|
1080
|
-
host,
|
|
1081
|
-
});
|
|
1082
|
-
|
|
1083
|
-
if (shuttingDown || TERMINAL_STATES.has(session.state)) return;
|
|
1084
|
-
|
|
1085
|
-
// 원격 세션은 broker lease를 사용하지 않으므로 release 불필요
|
|
1086
|
-
// (spawnSession에서 config.remote === true일 때 lease 건너뜀)
|
|
1087
|
-
if (code === 0) {
|
|
1088
|
-
transition(session, STATES.COMPLETED, `exit_${code}`);
|
|
1089
|
-
// #125 Codex R1 MEDIUM: same overflow guard as the local exit path.
|
|
1090
|
-
let extracted;
|
|
1091
|
-
if (sentinelCapture.isOverflow()) {
|
|
1092
|
-
extracted = null;
|
|
1093
|
-
} else {
|
|
1094
|
-
const sentinelSnapshot = sentinelCapture.snapshot();
|
|
1095
|
-
extracted = extractCompletionPayload(sentinelSnapshot || stdoutTail);
|
|
1096
|
-
}
|
|
1097
|
-
const completionPayload = extracted ? extracted.payload : undefined;
|
|
1098
|
-
emitter.emit("completed", { sessionId: session.id, completionPayload });
|
|
1099
|
-
if (typeof session.config.onCompleted === "function") {
|
|
1100
|
-
session.config.onCompleted({
|
|
1101
|
-
sessionId: session.id,
|
|
1102
|
-
completionPayload,
|
|
1103
|
-
});
|
|
1104
|
-
}
|
|
1105
|
-
maybeAutoShutdown();
|
|
1106
|
-
} else {
|
|
1107
|
-
handleFailure(session, `remote_exit_${code}`);
|
|
1108
|
-
}
|
|
1109
|
-
});
|
|
1110
|
-
|
|
1111
|
-
child.once("error", (err) => {
|
|
1112
|
-
clearInterval(healthInterval);
|
|
1113
|
-
outWs.end();
|
|
1114
|
-
errWs.end();
|
|
1115
|
-
eventLog.append("spawn_error", {
|
|
1116
|
-
session: session.id,
|
|
1117
|
-
error: err.message,
|
|
1118
|
-
});
|
|
1119
|
-
if (!shuttingDown && !TERMINAL_STATES.has(session.state)) {
|
|
1120
|
-
handleFailure(session, `spawn_error:${err.message}`);
|
|
1121
|
-
}
|
|
1122
|
-
});
|
|
1123
|
-
}
|
|
1124
|
-
|
|
1125
|
-
/**
|
|
1126
|
-
* 모든 세션이 terminal이면 auto-shutdown.
|
|
1127
|
-
*/
|
|
1128
|
-
function maybeAutoShutdown() {
|
|
1129
|
-
if (shuttingDown) return;
|
|
1130
|
-
const allTerminal = [...sessions.values()].every((s) =>
|
|
1131
|
-
TERMINAL_STATES.has(s.state),
|
|
1132
|
-
);
|
|
1133
|
-
if (allTerminal && sessions.size > 0) {
|
|
1134
|
-
emitter.emit("allCompleted");
|
|
1135
|
-
}
|
|
1136
|
-
}
|
|
1137
|
-
|
|
1138
|
-
// ── Public API ──────────────────────────────────────────────
|
|
1139
|
-
|
|
1140
|
-
/**
|
|
1141
|
-
* 새 세션 spawn.
|
|
1142
|
-
* @param {object} config
|
|
1143
|
-
* @param {string} config.id — 세션 ID (unique)
|
|
1144
|
-
* @param {'codex'|'gemini'|'claude'} config.agent
|
|
1145
|
-
* @param {string} config.prompt
|
|
1146
|
-
* @param {string} [config.profile]
|
|
1147
|
-
* @param {string} [config.workdir]
|
|
1148
|
-
* @param {string} [config.model]
|
|
1149
|
-
* @param {boolean} [config.remote=false] — 원격 세션 여부
|
|
1150
|
-
* @param {string} [config.host] — SSH 호스트 (remote=true 필수)
|
|
1151
|
-
* @param {string} [config.paneTarget] — psmux pane target (remote용)
|
|
1152
|
-
* @param {string} [config.sessionName] — psmux 세션 이름 (remote용)
|
|
1153
|
-
* @param {function} [config.onCompleted] — 세션 완료 시 콜백 ({sessionId}) => void
|
|
1154
|
-
* @returns {string} session ID
|
|
1155
|
-
*/
|
|
1156
|
-
function spawnSession(config) {
|
|
1157
|
-
if (shuttingDown) throw new Error("Conductor is shutting down");
|
|
1158
|
-
if (!config.id) throw new Error("session id is required");
|
|
1159
|
-
if (sessions.has(config.id))
|
|
1160
|
-
throw new Error(`Session "${config.id}" already exists`);
|
|
1161
|
-
if (config.remote && !config.host)
|
|
1162
|
-
throw new Error("host is required for remote sessions");
|
|
1163
|
-
|
|
1164
|
-
// Phase 1 dynamic routing wire-up (opt-in env TRIFLUX_DYNAMIC_ROUTING).
|
|
1165
|
-
// sync 로만 평가해서 broker.lease atomic 인변량 (commit 761fb7a8 — leased
|
|
1166
|
-
// account = executing account) 을 깨지 않는다. snapshot 은 D-2 cache 의
|
|
1167
|
-
// sync getter 만 사용 — cache miss 면 evaluateDynamicRoute 가 fail-closed
|
|
1168
|
-
// 로 codex-default 를 반환한다.
|
|
1169
|
-
if (
|
|
1170
|
-
isDynamicRoutingEnabled(process.env) &&
|
|
1171
|
-
!config.remote &&
|
|
1172
|
-
config.agent
|
|
1173
|
-
) {
|
|
1174
|
-
try {
|
|
1175
|
-
const policy = dynamicRoutingInternal.loadPolicy();
|
|
1176
|
-
const snapshot = tryGetCachedSnapshot();
|
|
1177
|
-
const decision = evaluateDynamicRoute(
|
|
1178
|
-
{
|
|
1179
|
-
task_id: config.id,
|
|
1180
|
-
team_size: 1,
|
|
1181
|
-
agent_hint: config.agent,
|
|
1182
|
-
},
|
|
1183
|
-
snapshot || {},
|
|
1184
|
-
policy,
|
|
1185
|
-
{ now: Date.now() },
|
|
1186
|
-
);
|
|
1187
|
-
const shard = decision?.shards?.[0];
|
|
1188
|
-
if (shard?.cli && shard.cli !== config.agent) {
|
|
1189
|
-
eventLog.append("dynamic_route_override", {
|
|
1190
|
-
session: config.id,
|
|
1191
|
-
original: config.agent,
|
|
1192
|
-
override: shard.cli,
|
|
1193
|
-
scenario: decision.scenario ?? "unknown",
|
|
1194
|
-
decisionId: decision.decisionId ?? null,
|
|
1195
|
-
snapshotAgeMs: decision.snapshotAgeMs ?? null,
|
|
1196
|
-
});
|
|
1197
|
-
config = { ...config, agent: shard.cli };
|
|
1198
|
-
}
|
|
1199
|
-
} catch (err) {
|
|
1200
|
-
eventLog.append("dynamic_route_error", {
|
|
1201
|
-
session: config.id,
|
|
1202
|
-
error: err?.message || String(err),
|
|
1203
|
-
});
|
|
1204
|
-
}
|
|
1205
|
-
}
|
|
1206
|
-
|
|
1207
|
-
// TFX_DISABLE_* SSOT 집행. 계정 임대와 worker sandbox 생성보다 앞에 둔다.
|
|
1208
|
-
// 뒤에 두면 정책으로 거부된 세션이 샌드박스 디렉터리와 auth.json 사본을
|
|
1209
|
-
// 남기고, 임대 반납이 회로 차단기 실패로 집계돼 멀쩡한 계정이 cooldown 에
|
|
1210
|
-
// 들어간다. 다른 CLI 로 자동 전환하지 않고 호출자에게 그대로 던진다.
|
|
1211
|
-
// claude 처럼 정책 대상이 아닌 agent 는 프로파일 파일을 읽지 않는다 (F4 와 동일).
|
|
1212
|
-
if (!config.remote && normalizeCliName(config.agent)) {
|
|
1213
|
-
const blockedCli = buildDisabledCliError(
|
|
1214
|
-
config.agent,
|
|
1215
|
-
resolveCliPolicy(process.env),
|
|
1216
|
-
);
|
|
1217
|
-
if (blockedCli) {
|
|
1218
|
-
eventLog.append("session_spawn_blocked", {
|
|
1219
|
-
session: config.id,
|
|
1220
|
-
agent: config.agent,
|
|
1221
|
-
code: blockedCli.code || null,
|
|
1222
|
-
error: blockedCli.message,
|
|
1223
|
-
});
|
|
1224
|
-
emitter.emit("warning", {
|
|
1225
|
-
type: "session_spawn_blocked",
|
|
1226
|
-
session: config.id,
|
|
1227
|
-
agent: config.agent,
|
|
1228
|
-
code: blockedCli.code || null,
|
|
1229
|
-
error: blockedCli.message,
|
|
1230
|
-
});
|
|
1231
|
-
throw blockedCli;
|
|
1232
|
-
}
|
|
1233
|
-
}
|
|
1234
|
-
|
|
1235
|
-
// broker lease (graceful — broker null if accounts.json absent)
|
|
1236
|
-
let lease = null;
|
|
1237
|
-
const activeBroker = getActiveBroker();
|
|
1238
|
-
if (activeBroker && config.agent && !config.remote) {
|
|
1239
|
-
lease = activeBroker.lease({ provider: config.agent });
|
|
1240
|
-
if (lease === null) {
|
|
1241
|
-
const eta = activeBroker.nextAvailableEta(config.agent);
|
|
1242
|
-
eventLog.append("broker_no_lease", {
|
|
1243
|
-
session: config.id,
|
|
1244
|
-
agent: config.agent,
|
|
1245
|
-
eta: eta ? new Date(eta).toISOString() : "unknown",
|
|
1246
|
-
});
|
|
1247
|
-
// 계정이 모두 cooldown이어도 세션 생성 자체는 유지한다.
|
|
1248
|
-
// 로컬 테스트/단일 계정 없는 환경에서도 상태 머신이 일관되게 동작해야 한다.
|
|
1249
|
-
}
|
|
1250
|
-
}
|
|
1251
|
-
|
|
1252
|
-
// apply lease profile/env/auth to config (immutable — new object)
|
|
1253
|
-
const leaseEnv =
|
|
1254
|
-
lease?.authFile && config.agent === "codex"
|
|
1255
|
-
? { ...(lease.env || {}), CODEX_HOME: dirname(lease.authFile) }
|
|
1256
|
-
: lease?.env || {};
|
|
1257
|
-
|
|
1258
|
-
let resolvedConfig = lease
|
|
1259
|
-
? {
|
|
1260
|
-
...config,
|
|
1261
|
-
profile: lease.profile ?? config.profile,
|
|
1262
|
-
env: { ...(config.env || {}), ...leaseEnv },
|
|
1263
|
-
accountId: lease.id,
|
|
1264
|
-
}
|
|
1265
|
-
: {
|
|
1266
|
-
...config,
|
|
1267
|
-
env: { ...(config.env || {}) },
|
|
1268
|
-
};
|
|
1269
|
-
|
|
1270
|
-
if (!resolvedConfig.remote) {
|
|
1271
|
-
const sandbox = buildWorkerSandboxEnv({
|
|
1272
|
-
cwd: resolvedConfig.workdir || process.cwd(),
|
|
1273
|
-
sessionId: resolvedConfig.id,
|
|
1274
|
-
agent: resolvedConfig.agent,
|
|
1275
|
-
env: { ...process.env, ...(resolvedConfig.env || {}) },
|
|
1276
|
-
});
|
|
1277
|
-
resolvedConfig = {
|
|
1278
|
-
...resolvedConfig,
|
|
1279
|
-
env: { ...(resolvedConfig.env || {}), ...sandbox.env },
|
|
1280
|
-
workerSandbox: sandbox.disabled
|
|
1281
|
-
? { disabled: true }
|
|
1282
|
-
: { root: sandbox.root, home: sandbox.home },
|
|
1283
|
-
};
|
|
1284
|
-
}
|
|
1285
|
-
|
|
1286
|
-
// auth file copy — broker resolved absolute path, conductor does the actual copy
|
|
1287
|
-
swapAuthFile(
|
|
1288
|
-
lease,
|
|
1289
|
-
config.agent,
|
|
1290
|
-
config.id,
|
|
1291
|
-
eventLog,
|
|
1292
|
-
activeBroker,
|
|
1293
|
-
resolvedConfig.env,
|
|
1294
|
-
);
|
|
1295
|
-
|
|
1296
|
-
// 원격 세션은 launcher 불필요 (이미 원격에서 실행 중)
|
|
1297
|
-
const launcher = resolvedConfig.remote
|
|
1298
|
-
? null
|
|
1299
|
-
: buildLauncher({
|
|
1300
|
-
agent: resolvedConfig.agent,
|
|
1301
|
-
profile: resolvedConfig.profile,
|
|
1302
|
-
prompt: resolvedConfig.prompt,
|
|
1303
|
-
workdir: resolvedConfig.workdir,
|
|
1304
|
-
model: resolvedConfig.model,
|
|
1305
|
-
});
|
|
1306
|
-
|
|
1307
|
-
const session = {
|
|
1308
|
-
id: resolvedConfig.id,
|
|
1309
|
-
config: resolvedConfig,
|
|
1310
|
-
launcher,
|
|
1311
|
-
state: STATES.INIT,
|
|
1312
|
-
child: null,
|
|
1313
|
-
probe: null,
|
|
1314
|
-
alive: false,
|
|
1315
|
-
restarts: 0,
|
|
1316
|
-
outPath: null,
|
|
1317
|
-
errPath: null,
|
|
1318
|
-
createdAt: Date.now(),
|
|
1319
|
-
descendantSnapshot: [],
|
|
1320
|
-
};
|
|
1321
|
-
|
|
1322
|
-
sessions.set(resolvedConfig.id, session);
|
|
1323
|
-
getConductorRegistry()?.register?.(resolvedConfig.id, publicApi);
|
|
1324
|
-
logCtoContext(resolvedConfig.id);
|
|
1325
|
-
|
|
1326
|
-
if (resolvedConfig.remote) {
|
|
1327
|
-
startRemoteSession(session);
|
|
1328
|
-
} else {
|
|
1329
|
-
void respawnSession(session);
|
|
1330
|
-
}
|
|
1331
|
-
return resolvedConfig.id;
|
|
1332
|
-
}
|
|
1333
|
-
|
|
1334
|
-
/**
|
|
1335
|
-
* 세션 kill.
|
|
1336
|
-
* @param {string} id
|
|
1337
|
-
* @param {string} [reason]
|
|
1338
|
-
*/
|
|
1339
|
-
async function killSession(id, reason = "user_kill") {
|
|
1340
|
-
const session = sessions.get(id);
|
|
1341
|
-
if (!session) return;
|
|
1342
|
-
if (TERMINAL_STATES.has(session.state)) return;
|
|
1343
|
-
|
|
1344
|
-
if (session._respawnTimer) {
|
|
1345
|
-
clearTimeout(session._respawnTimer);
|
|
1346
|
-
session._respawnTimer = null;
|
|
1347
|
-
}
|
|
1348
|
-
eventLog.append("kill", { session: id, reason });
|
|
1349
|
-
await cleanupChild(session);
|
|
1350
|
-
transition(session, STATES.FAILED, reason);
|
|
1351
|
-
transition(session, STATES.DEAD, reason);
|
|
1352
|
-
}
|
|
1353
|
-
|
|
1354
|
-
/**
|
|
1355
|
-
* 세션에 stdin 입력 전송 (INPUT_WAIT 해소용).
|
|
1356
|
-
* @param {string} id
|
|
1357
|
-
* @param {string} text
|
|
1358
|
-
*/
|
|
1359
|
-
function sendInput(id, text) {
|
|
1360
|
-
const session = sessions.get(id);
|
|
1361
|
-
if (!session) return false;
|
|
1362
|
-
|
|
1363
|
-
// 원격 세션 — stdin 미지원 (psmux send-keys는 별도 경로)
|
|
1364
|
-
if (session.config.remote) {
|
|
1365
|
-
eventLog.append("stdin_remote_unsupported", { session: id });
|
|
1366
|
-
return false;
|
|
1367
|
-
}
|
|
1368
|
-
|
|
1369
|
-
if (!session.child) return false;
|
|
1370
|
-
if (!session.child.stdin?.writable) return false;
|
|
1371
|
-
try {
|
|
1372
|
-
session.child.stdin.write(`${text}\n`);
|
|
1373
|
-
eventLog.append("stdin", { session: id, text: text.slice(0, 100) });
|
|
1374
|
-
return true;
|
|
1375
|
-
} catch {
|
|
1376
|
-
return false;
|
|
1377
|
-
}
|
|
1378
|
-
}
|
|
1379
|
-
|
|
1380
|
-
/**
|
|
1381
|
-
* 전체 세션 스냅샷.
|
|
1382
|
-
* @returns {object[]}
|
|
1383
|
-
*/
|
|
1384
|
-
function getSnapshot() {
|
|
1385
|
-
return [...sessions.values()].map((s) => ({
|
|
1386
|
-
id: s.id,
|
|
1387
|
-
agent: s.config.agent,
|
|
1388
|
-
state: s.state,
|
|
1389
|
-
pid: s.child?.pid || null,
|
|
1390
|
-
remote: s.config.remote || false,
|
|
1391
|
-
host: s.config.host || null,
|
|
1392
|
-
restarts: s.restarts,
|
|
1393
|
-
health: s.probe?.getStatus() || null,
|
|
1394
|
-
outPath: s.outPath,
|
|
1395
|
-
errPath: s.errPath,
|
|
1396
|
-
createdAt: s.createdAt,
|
|
1397
|
-
}));
|
|
1398
|
-
}
|
|
1399
|
-
|
|
1400
|
-
/**
|
|
1401
|
-
* Graceful shutdown — 전체 세션 종료.
|
|
1402
|
-
*/
|
|
1403
|
-
async function shutdown(reason = "shutdown") {
|
|
1404
|
-
if (shuttingDown) return;
|
|
1405
|
-
shuttingDown = true;
|
|
1406
|
-
|
|
1407
|
-
eventLog.append("shutdown", { reason, sessions: sessions.size });
|
|
1408
|
-
|
|
1409
|
-
const cleanedSessionIds = new Set();
|
|
1410
|
-
const cleanups = [...sessions.values()]
|
|
1411
|
-
.filter((s) => {
|
|
1412
|
-
if (cleanedSessionIds.has(s.id)) return false;
|
|
1413
|
-
cleanedSessionIds.add(s.id);
|
|
1414
|
-
return true;
|
|
1415
|
-
})
|
|
1416
|
-
.map(async (s) => {
|
|
1417
|
-
if (s._respawnTimer) {
|
|
1418
|
-
clearTimeout(s._respawnTimer);
|
|
1419
|
-
s._respawnTimer = null;
|
|
1420
|
-
}
|
|
1421
|
-
if (s._descendantSnapshotTimer) {
|
|
1422
|
-
clearTimeout(s._descendantSnapshotTimer);
|
|
1423
|
-
s._descendantSnapshotTimer = null;
|
|
1424
|
-
}
|
|
1425
|
-
s.probe?.stop();
|
|
1426
|
-
await cleanupChild(s);
|
|
1427
|
-
if (!TERMINAL_STATES.has(s.state)) {
|
|
1428
|
-
transition(s, STATES.FAILED, reason);
|
|
1429
|
-
transition(s, STATES.DEAD, reason);
|
|
1430
|
-
}
|
|
1431
|
-
});
|
|
1432
|
-
|
|
1433
|
-
await Promise.allSettled(cleanups);
|
|
1434
|
-
process.removeListener("SIGINT", onSignal);
|
|
1435
|
-
process.removeListener("SIGTERM", onSignal);
|
|
1436
|
-
if (brokerInstance && onTierFallback) {
|
|
1437
|
-
brokerInstance.removeListener("tierFallback", onTierFallback);
|
|
1438
|
-
}
|
|
1439
|
-
if (conductor._meshBridge) conductor._meshBridge.detach();
|
|
1440
|
-
await eventLog.flush();
|
|
1441
|
-
await eventLog.close();
|
|
1442
|
-
emitter.emit("shutdown");
|
|
1443
|
-
}
|
|
1444
|
-
|
|
1445
|
-
// ── broker tierFallback 구독 ──────────────────────────────────
|
|
1446
|
-
// 토큰 만료 등으로 상위 tier 계정이 불가능해지면, 활성 세션의 auth를 새 lease로 교체.
|
|
1447
|
-
const brokerInstance = optsBroker ?? null;
|
|
1448
|
-
const onTierFallback = brokerInstance
|
|
1449
|
-
? ({ provider }) => {
|
|
1450
|
-
for (const session of sessions.values()) {
|
|
1451
|
-
if (TERMINAL_STATES.has(session.state)) continue;
|
|
1452
|
-
if (session.config.agent !== provider) continue;
|
|
1453
|
-
if (session.config.remote) continue;
|
|
1454
|
-
|
|
1455
|
-
const newLease = brokerInstance.lease({ provider });
|
|
1456
|
-
if (!newLease) continue;
|
|
1457
|
-
if (newLease.authFile && provider === "codex") {
|
|
1458
|
-
session.config.env = {
|
|
1459
|
-
...(session.config.env || {}),
|
|
1460
|
-
CODEX_HOME: dirname(newLease.authFile),
|
|
1461
|
-
};
|
|
1462
|
-
}
|
|
1463
|
-
|
|
1464
|
-
swapAuthFile(
|
|
1465
|
-
newLease,
|
|
1466
|
-
provider,
|
|
1467
|
-
session.id,
|
|
1468
|
-
eventLog,
|
|
1469
|
-
brokerInstance,
|
|
1470
|
-
session.config.env,
|
|
1471
|
-
);
|
|
1472
|
-
eventLog.append("tier_fallback_swap", {
|
|
1473
|
-
session: session.id,
|
|
1474
|
-
agent: provider,
|
|
1475
|
-
newAccount: newLease.id,
|
|
1476
|
-
});
|
|
1477
|
-
}
|
|
1478
|
-
}
|
|
1479
|
-
: null;
|
|
1480
|
-
|
|
1481
|
-
if (brokerInstance && onTierFallback) {
|
|
1482
|
-
brokerInstance.on("tierFallback", onTierFallback);
|
|
1483
|
-
}
|
|
1484
|
-
|
|
1485
|
-
// Shutdown traps
|
|
1486
|
-
const onSignal = () => {
|
|
1487
|
-
void shutdown("signal");
|
|
1488
|
-
};
|
|
1489
|
-
process.on("SIGINT", onSignal);
|
|
1490
|
-
process.on("SIGTERM", onSignal);
|
|
1491
|
-
|
|
1492
|
-
const conductor = {
|
|
1493
|
-
spawnSession,
|
|
1494
|
-
killSession,
|
|
1495
|
-
sendInput,
|
|
1496
|
-
getSnapshot,
|
|
1497
|
-
getMeshRegistry() {
|
|
1498
|
-
return this._meshRegistry || null;
|
|
1499
|
-
},
|
|
1500
|
-
shutdown,
|
|
1501
|
-
on: emitter.on.bind(emitter),
|
|
1502
|
-
off: emitter.off.bind(emitter),
|
|
1503
|
-
get sessionCount() {
|
|
1504
|
-
return sessions.size;
|
|
1505
|
-
},
|
|
1506
|
-
get isShuttingDown() {
|
|
1507
|
-
return shuttingDown;
|
|
1508
|
-
},
|
|
1509
|
-
get eventLogPath() {
|
|
1510
|
-
return eventLog.filePath;
|
|
1511
|
-
},
|
|
1512
|
-
};
|
|
1513
|
-
|
|
1514
|
-
if (opts.enableMesh !== false) {
|
|
1515
|
-
try {
|
|
1516
|
-
const registry = opts.meshRegistry || createRegistry();
|
|
1517
|
-
const bridge = createConductorMeshBridge(conductor, registry);
|
|
1518
|
-
bridge.attach();
|
|
1519
|
-
conductor._meshBridge = bridge;
|
|
1520
|
-
conductor._meshRegistry = registry;
|
|
1521
|
-
} catch {
|
|
1522
|
-
// mesh 실패해도 conductor 정상 동작
|
|
1523
|
-
}
|
|
1524
|
-
}
|
|
1525
|
-
|
|
1526
|
-
const frozenApi = Object.freeze(conductor);
|
|
1527
|
-
ensureConductorRegistry();
|
|
1528
|
-
return frozenApi;
|
|
1529
|
-
}
|