@triflux/remote 10.50.1 → 10.51.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (120) hide show
  1. package/cto/brief.mjs +0 -12
  2. package/cto/collect.mjs +16 -247
  3. package/cto/current.schema.json +4 -20
  4. package/cto/events.mjs +13 -294
  5. package/cto/hygiene.mjs +26 -346
  6. package/cto/lake-root.mjs +114 -16
  7. package/cto/status.mjs +33 -128
  8. package/hub/index.mjs +0 -8
  9. package/hub/store-adapter.mjs +1 -1
  10. package/hub/team/agent-map.json +0 -1
  11. package/hub/team/ansi.mjs +1 -35
  12. package/hub/team/backend.mjs +9 -145
  13. package/hub/team/cli/commands/attach.mjs +1 -17
  14. package/hub/team/cli/commands/control.mjs +12 -38
  15. package/hub/team/cli/commands/debug.mjs +1 -37
  16. package/hub/team/cli/commands/focus.mjs +5 -31
  17. package/hub/team/cli/commands/interrupt.mjs +6 -25
  18. package/hub/team/cli/commands/kill.mjs +1 -30
  19. package/hub/team/cli/commands/list.mjs +0 -16
  20. package/hub/team/cli/commands/send.mjs +6 -26
  21. package/hub/team/cli/commands/start/index.mjs +41 -133
  22. package/hub/team/cli/commands/start/parse-args.mjs +15 -40
  23. package/hub/team/cli/commands/start/start-headless.mjs +138 -105
  24. package/hub/team/cli/commands/start/start-mux.mjs +0 -3
  25. package/hub/team/cli/commands/status.mjs +2 -57
  26. package/hub/team/cli/commands/stop.mjs +14 -19
  27. package/hub/team/cli/help.mjs +4 -5
  28. package/hub/team/cli/manifest.mjs +5 -1
  29. package/hub/team/cli/render.mjs +0 -12
  30. package/hub/team/cli/services/hub-client.mjs +2 -88
  31. package/hub/team/cli/services/runtime-mode.mjs +27 -98
  32. package/hub/team/handoff.mjs +3 -34
  33. package/hub/team/headless.mjs +14 -755
  34. package/hub/team/orchestrator.mjs +4 -30
  35. package/hub/team/pane.mjs +6 -28
  36. package/hub/team/psmux.mjs +16 -403
  37. package/hub/team/remote-session.mjs +1 -2
  38. package/hub/team/session.mjs +8 -119
  39. package/hub/team/swarm-intent.mjs +0 -198
  40. package/hub/team/terminal-opener.mjs +11 -35
  41. package/hub/team/tui-core.mjs +1 -1
  42. package/hub/team/tui-viewer.mjs +2 -2
  43. package/hub/team/tui.mjs +8 -269
  44. package/hub/team/wt-manager.mjs +2 -2
  45. package/hub/workers/factory.mjs +4 -37
  46. package/hub/workers/lib/jsonrpc-ws-uds.mjs +0 -2
  47. package/package.json +1 -1
  48. package/scripts/lib/agent-route-policy.mjs +162 -14
  49. package/scripts/lib/claudemd-scanner.mjs +2 -199
  50. package/scripts/lib/cli-claude.mjs +0 -2
  51. package/scripts/lib/context.mjs +3 -19
  52. package/scripts/lib/env-probe.mjs +5 -64
  53. package/scripts/lib/gemini-profiles.mjs +2 -69
  54. package/scripts/lib/legacy-mcp-cleanup.mjs +14 -3
  55. package/scripts/lib/machine-profile.mjs +3 -17
  56. package/scripts/lib/mcp-filter.mjs +109 -53
  57. package/scripts/lib/mcp-health.mjs +2 -2
  58. package/scripts/lib/mcp-server-catalog.mjs +2 -2
  59. package/scripts/lib/psmux-info.mjs +25 -104
  60. package/scripts/lib/skill-template.mjs +0 -271
  61. package/cto/hygiene-actions.mjs +0 -355
  62. package/cto/hygiene-notify.mjs +0 -170
  63. package/hub/team/build-worker-prompt.mjs +0 -92
  64. package/hub/team/check-mcp-hub.mjs +0 -49
  65. package/hub/team/claude-native-bridge.mjs +0 -987
  66. package/hub/team/cli/commands/start/start-in-process.mjs +0 -50
  67. package/hub/team/cli/commands/start/start-wt.mjs +0 -86
  68. package/hub/team/cli/services/native-control.mjs +0 -145
  69. package/hub/team/codex-review.mjs +0 -520
  70. package/hub/team/conductor-mesh-bridge.mjs +0 -122
  71. package/hub/team/conductor.mjs +0 -1529
  72. package/hub/team/consensus-meta.mjs +0 -176
  73. package/hub/team/cto-pull.mjs +0 -31
  74. package/hub/team/daemon-pty-tmux-bridge.mjs +0 -146
  75. package/hub/team/dashboard.mjs +0 -322
  76. package/hub/team/event-log.mjs +0 -84
  77. package/hub/team/execution-mode.mjs +0 -324
  78. package/hub/team/extract-completion-payload.mjs +0 -110
  79. package/hub/team/headless-bridge-session.mjs +0 -17
  80. package/hub/team/health-probe.mjs +0 -435
  81. package/hub/team/interactive-native-launcher.mjs +0 -119
  82. package/hub/team/interactive-native-worker.mjs +0 -80
  83. package/hub/team/interactive-tui-transport.mjs +0 -479
  84. package/hub/team/launcher-template.mjs +0 -144
  85. package/hub/team/mcp-selector.mjs +0 -145
  86. package/hub/team/native-supervisor.mjs +0 -460
  87. package/hub/team/native.mjs +0 -727
  88. package/hub/team/recovery-store.mjs +0 -149
  89. package/hub/team/remote-probe.mjs +0 -292
  90. package/hub/team/remote-watcher.mjs +0 -506
  91. package/hub/team/runtime-strategy.mjs +0 -200
  92. package/hub/team/schemas/worker-completion.json +0 -39
  93. package/hub/team/sentinel-capture.mjs +0 -127
  94. package/hub/team/swarm-cli.mjs +0 -390
  95. package/hub/team/swarm-hypervisor.mjs +0 -2206
  96. package/hub/team/swarm-locks.mjs +0 -335
  97. package/hub/team/swarm-planner.mjs +0 -371
  98. package/hub/team/swarm-preflight.mjs +0 -366
  99. package/hub/team/swarm-reconciler.mjs +0 -242
  100. package/hub/team/tui-remote-adapter.mjs +0 -377
  101. package/hub/team/worker-completion-validator.mjs +0 -79
  102. package/hub/team/worktree-lifecycle.mjs +0 -639
  103. package/hub/team/wt-templates.json +0 -43
  104. package/hub/workers/antigravity-route-worker.mjs +0 -179
  105. package/hub/workers/gemini-worker.mjs +0 -217
  106. package/scripts/lib/agent-json.mjs +0 -27
  107. package/scripts/lib/async.mjs +0 -174
  108. package/scripts/lib/cli-agy.mjs +0 -85
  109. package/scripts/lib/cli-codex.mjs +0 -62
  110. package/scripts/lib/cli-gemini.mjs +0 -37
  111. package/scripts/lib/dynamic-route-cli.mjs +0 -107
  112. package/scripts/lib/env.mjs +0 -14
  113. package/scripts/lib/hub.mjs +0 -48
  114. package/scripts/lib/pid.mjs +0 -63
  115. package/scripts/lib/quota.mjs +0 -47
  116. package/scripts/lib/skill-state.mjs +0 -220
  117. package/scripts/lib/team.mjs +0 -75
  118. package/scripts/lib/timeout.mjs +0 -66
  119. package/scripts/lib/tmp.mjs +0 -26
  120. package/scripts/lib/toml.mjs +0 -132
@@ -1,1529 +0,0 @@
1
- // hub/team/conductor.mjs — 세션 오케스트레이션 Conductor
2
- // native-supervisor.mjs의 spawn/kill을 래핑하되, 상태 머신 + health probe +
3
- // auto-restart + event log를 추가하여 "조용한 실패"를 구조적으로 불가능하게 만든다.
4
- //
5
- // 기존 native-supervisor와의 차이:
6
- // 1. 상태 머신 (alive/dead → 7 states + 2 terminal)
7
- // 2. Health probe 4단계 (+ INPUT_WAIT 감지)
8
- // 3. Auto-restart (maxRestarts=3)
9
- // 4. JSONL event log (블랙박스 리코더)
10
-
11
- import { spawnSync } from "node:child_process";
12
- import { EventEmitter } from "node:events";
13
- import {
14
- copyFileSync,
15
- createWriteStream,
16
- mkdirSync,
17
- readFileSync,
18
- } from "node:fs";
19
- import { homedir } from "node:os";
20
- import { dirname, join } from "node:path";
21
- import { createRegistry } from "@triflux/core/mesh/mesh-registry.mjs";
22
- import { resolveNestedCodexAgentProfile } from "../../scripts/lib/cli-codex.mjs";
23
- import { codexProfileConfigOverrides } from "../../scripts/lib/codex-profile-config.mjs";
24
- import {
25
- buildDisabledCliError,
26
- normalizeCliName,
27
- resolveCliPolicy,
28
- } from "../../scripts/lib/machine-profile.mjs";
29
- import { broker } from "@triflux/core/hub/account-broker.mjs";
30
- import { runPreflight as runCodexPreflight } from "@triflux/core/hub/codex-preflight.mjs";
31
- import {
32
- __internal__ as dynamicRoutingInternal,
33
- evaluateDynamicRoute,
34
- isDynamicRoutingEnabled,
35
- } from "@triflux/core/hub/dynamic-routing-engine.mjs";
36
- import {
37
- findProcessTree,
38
- killProcessTree,
39
- killProcessTreeSnapshot,
40
- } from "@triflux/core/hub/lib/process-utils.mjs";
41
- import { execFile, spawn } from "@triflux/core/hub/lib/spawn-trace.mjs";
42
- import { killProcess } from "@triflux/core/hub/platform.mjs";
43
- import { tryGetCachedSnapshot } from "@triflux/core/hub/routing-snapshot.mjs";
44
- import { createHubHealthChecker } from "./check-mcp-hub.mjs";
45
- import { createConductorMeshBridge } from "./conductor-mesh-bridge.mjs";
46
- import {
47
- ensureConductorRegistry,
48
- getConductorRegistry,
49
- } from "./conductor-registry.mjs";
50
- import { readCtoSnapshot } from "./cto-pull.mjs";
51
- import { createEventLog } from "./event-log.mjs";
52
- import { buildSpawnSpecForMode, MODES } from "./execution-mode.mjs";
53
- import { extractCompletionPayload } from "./extract-completion-payload.mjs";
54
- import { createHealthProbe } from "./health-probe.mjs";
55
- import { buildLauncher } from "./launcher-template.mjs";
56
- import { createSentinelCapture } from "./sentinel-capture.mjs";
57
- import {
58
- buildSynapseTaskSummary,
59
- heartbeatSynapseSession,
60
- registerSynapseSession,
61
- unregisterSynapseSession,
62
- } from "./synapse-http.mjs";
63
- import { buildWorkerSandboxEnv } from "./worker-sandbox.mjs";
64
-
65
- /** 세션 상태 */
66
- export const STATES = Object.freeze({
67
- INIT: "init",
68
- STARTING: "starting",
69
- HEALTHY: "healthy",
70
- STALLED: "stalled",
71
- INPUT_WAIT: "input_wait",
72
- FAILED: "failed",
73
- RESTARTING: "restarting",
74
- DEAD: "dead",
75
- COMPLETED: "completed",
76
- });
77
-
78
- /** 유효한 상태 전이 테이블 */
79
- const TRANSITIONS = Object.freeze({
80
- [STATES.INIT]: [STATES.STARTING],
81
- [STATES.STARTING]: [STATES.HEALTHY, STATES.FAILED, STATES.COMPLETED],
82
- [STATES.HEALTHY]: [
83
- STATES.STALLED,
84
- STATES.INPUT_WAIT,
85
- STATES.FAILED,
86
- STATES.COMPLETED,
87
- ],
88
- [STATES.STALLED]: [STATES.HEALTHY, STATES.FAILED],
89
- [STATES.INPUT_WAIT]: [STATES.HEALTHY, STATES.FAILED],
90
- [STATES.FAILED]: [STATES.RESTARTING, STATES.DEAD],
91
- [STATES.RESTARTING]: [STATES.STARTING],
92
- [STATES.DEAD]: [],
93
- [STATES.COMPLETED]: [],
94
- });
95
-
96
- const TERMINAL_STATES = new Set([STATES.DEAD, STATES.COMPLETED]);
97
- const DEFAULT_MAX_RESTARTS = 3;
98
- const DEFAULT_GRACE_MS = 10_000;
99
-
100
- /**
101
- * Auth 파일을 lease 소스에서 CLI 대상 경로로 복사.
102
- * @param {object} lease — broker lease 객체 (mode, authFile 필수)
103
- * @param {'codex'|'gemini'} agent
104
- * @param {string} sessionId — 로깅용
105
- * @param {object} eventLog — createEventLog 인스턴스
106
- * @param {object} brokerInstance — AccountBroker 인스턴스
107
- */
108
- function swapAuthFile(
109
- lease,
110
- agent,
111
- sessionId,
112
- eventLog,
113
- brokerInstance,
114
- targetEnv = {},
115
- ) {
116
- if (lease?.mode !== "auth" || !lease.authFile) return true;
117
- if (
118
- agent === "codex" &&
119
- lease.id &&
120
- brokerInstance &&
121
- typeof brokerInstance.syncAuthToSource === "function"
122
- ) {
123
- const result = brokerInstance.syncAuthToSource(lease.id);
124
- if (result?.copied || result?.reason === "up_to_date") {
125
- eventLog.append("auth_copy", {
126
- session: sessionId,
127
- agent,
128
- dest: result.sourcePath,
129
- reason: result.reason,
130
- });
131
- return true;
132
- }
133
- eventLog.append("auth_copy_error", {
134
- session: sessionId,
135
- dest: result?.sourcePath || join(homedir(), ".codex", "auth.json"),
136
- error: result?.reason || "sync_failed",
137
- });
138
- return false;
139
- }
140
- const targetHome = targetEnv.HOME || homedir();
141
- const codexHome = targetEnv.CODEX_HOME || join(targetHome, ".codex");
142
- const dests =
143
- agent === "codex"
144
- ? [join(codexHome, "auth.json")]
145
- : [
146
- join(targetHome, ".gemini", "oauth_creds.json"),
147
- join(targetHome, ".gemini", "gemini-credentials.json"),
148
- ];
149
- let allOk = true;
150
- for (const dest of dests) {
151
- try {
152
- mkdirSync(dirname(dest), { recursive: true });
153
- copyFileSync(lease.authFile, dest);
154
- eventLog.append("auth_copy", { session: sessionId, agent, dest });
155
- } catch (err) {
156
- allOk = false;
157
- eventLog.append("auth_copy_error", {
158
- session: sessionId,
159
- dest,
160
- error: err.message,
161
- });
162
- }
163
- }
164
- return allOk;
165
- }
166
-
167
- /**
168
- * Conductor 팩토리.
169
- * @param {object} opts
170
- * @param {string} opts.logsDir — 이벤트 로그 디렉토리
171
- * @param {number} [opts.maxRestarts=3]
172
- * @param {number} [opts.graceMs=10000] — shutdown grace period
173
- * @param {object} [opts.probeOpts] — health-probe 옵션 오버라이드
174
- * @param {object} [opts.broker] — AccountBroker 인스턴스 (tierFallback 구독용)
175
- * @returns {Conductor}
176
- */
177
- /**
178
- * Conductor 생성.
179
- * @param {object} [opts]
180
- * @param {object} [opts.deps] — 의존성 주입 (테스트용 mock 지원)
181
- * @param {Function} [opts.deps.execFile] — execFile 오버라이드
182
- * @param {Function} [opts.deps.spawn] — spawn 오버라이드 (로컬/원격 child process 모두)
183
- */
184
- export function createConductor(opts = {}) {
185
- const spawnFn = opts.deps?.spawn || spawn;
186
- const processUtils = {
187
- killProcessTree: opts.deps?.killProcessTree || killProcessTree,
188
- killProcessTreeSnapshot:
189
- opts.deps?.killProcessTreeSnapshot || killProcessTreeSnapshot,
190
- findProcessTree: opts.deps?.findProcessTree || findProcessTree,
191
- };
192
- const {
193
- logsDir,
194
- maxRestarts = DEFAULT_MAX_RESTARTS,
195
- graceMs = DEFAULT_GRACE_MS,
196
- probeOpts = {},
197
- broker: optsBroker,
198
- } = opts;
199
-
200
- if (!logsDir) throw new Error("logsDir is required");
201
- mkdirSync(logsDir, { recursive: true });
202
-
203
- const emitter = new EventEmitter();
204
- const sessions = new Map();
205
- let shuttingDown = false;
206
- const publicApi = null;
207
- const synapseOpts = {
208
- baseUrl: opts.synapseBaseUrl,
209
- fetchImpl: opts.synapseFetch,
210
- };
211
- const getActiveBroker = () => optsBroker ?? broker;
212
-
213
- function buildSynapseMeta(session, state = session.state, reason = "") {
214
- return {
215
- sessionId: session.id,
216
- host:
217
- typeof session.config.host === "string" &&
218
- session.config.host.length > 0
219
- ? session.config.host
220
- : "local",
221
- taskSummary: buildSynapseTaskSummary(session.config.prompt),
222
- status: state,
223
- reason,
224
- isRemote: Boolean(session.config.remote),
225
- };
226
- }
227
-
228
- // 공유 event log (모든 세션 이벤트를 하나의 JSONL에)
229
- const eventLog = createEventLog(join(logsDir, "conductor-events.jsonl"));
230
-
231
- function logCtoContext(sessionId) {
232
- const snapshot = readCtoSnapshot({ lakeRoot: opts.ctoLakeRoot });
233
- if (!snapshot) return;
234
- eventLog.append("cto_context", {
235
- session: sessionId,
236
- snapshot: {
237
- schema_version: snapshot.schema_version,
238
- repo: snapshot.repo,
239
- sources: snapshot.sources,
240
- ledger_tail: snapshot.ledger_tail,
241
- live_sessions: snapshot.live_sessions,
242
- active_shards: snapshot.active_shards,
243
- },
244
- });
245
- }
246
-
247
- /**
248
- * 세션 상태 전이.
249
- * @param {object} session
250
- * @param {string} nextState
251
- * @param {string} [reason]
252
- */
253
- function transition(session, nextState, reason = "") {
254
- const valid = TRANSITIONS[session.state] || [];
255
- if (!valid.includes(nextState)) {
256
- eventLog.append("invalid_transition", {
257
- session: session.id,
258
- from: session.state,
259
- to: nextState,
260
- reason,
261
- });
262
- return false;
263
- }
264
-
265
- const prev = session.state;
266
- session.state = nextState;
267
-
268
- eventLog.append("stateChange", {
269
- session: session.id,
270
- from: prev,
271
- to: nextState,
272
- reason,
273
- restarts: session.restarts,
274
- });
275
-
276
- emitter.emit("stateChange", {
277
- sessionId: session.id,
278
- from: prev,
279
- to: nextState,
280
- reason,
281
- });
282
-
283
- if (nextState === STATES.HEALTHY) {
284
- registerSynapseSession(
285
- buildSynapseMeta(session, nextState, reason),
286
- synapseOpts,
287
- );
288
- }
289
- heartbeatSynapseSession(
290
- session.id,
291
- buildSynapseMeta(session, nextState, reason),
292
- synapseOpts,
293
- );
294
- if (nextState === STATES.COMPLETED || nextState === STATES.DEAD) {
295
- unregisterSynapseSession(session.id, synapseOpts);
296
- }
297
-
298
- // Terminal state cleanup
299
- if (TERMINAL_STATES.has(nextState)) {
300
- session.probe?.stop();
301
- getConductorRegistry()?.unregister?.(session.id, publicApi);
302
- }
303
-
304
- return true;
305
- }
306
-
307
- /**
308
- * 프로세스를 강제 종료.
309
- * Windows: taskkill /T /F /PID (프로세스 트리). POSIX: SIGKILL.
310
- */
311
- function forceKill(pid) {
312
- if (!pid || pid <= 0) return;
313
- killProcess(pid, {
314
- signal: "SIGKILL",
315
- tree: true,
316
- force: true,
317
- timeout: 5000,
318
- });
319
- }
320
-
321
- /**
322
- * 원격 세션의 psmux 세션을 SSH 경유로 kill.
323
- * fire-and-forget: 실패해도 에러 전파 안 함.
324
- */
325
- function killRemoteSession(session) {
326
- const host = session.config.host;
327
- if (!host) return;
328
- let sshUser = session.config.sshUser;
329
- let sshIp = host;
330
- // hosts.json에서 ssh_user/IP 해결
331
- try {
332
- const hostsPath = join(
333
- opts.repoRoot || process.cwd(),
334
- "references",
335
- "hosts.json",
336
- );
337
- const hosts = JSON.parse(readFileSync(hostsPath, "utf8"));
338
- const hostCfg = hosts.hosts?.[host];
339
- if (hostCfg) {
340
- sshUser = sshUser || hostCfg.ssh_user;
341
- sshIp = hostCfg.tailscale?.ip || host;
342
- }
343
- } catch {
344
- /* hosts.json 없으면 fallback */
345
- }
346
- if (!sshUser) return;
347
- const execFn = opts.deps?.execFile || execFile;
348
- execFn(
349
- "ssh",
350
- [`${sshUser}@${sshIp}`, "psmux", "kill-session", "-t", session.id],
351
- { timeout: 10_000, reason: "conductor:killRemoteSession" },
352
- () => {},
353
- );
354
- eventLog.append("remote_kill", {
355
- session: session.id,
356
- host,
357
- sshUser,
358
- sshIp,
359
- });
360
- }
361
-
362
- /**
363
- * 단일 세션의 child process를 정리.
364
- * 원격 세션은 SSH 경유 psmux kill-session으로 정리.
365
- */
366
- async function cleanupChild(session) {
367
- session.probe?.stop();
368
-
369
- // 원격 세션 — SSH 경유 psmux kill-session
370
- if (session.config.remote) {
371
- killRemoteSession(session);
372
- return;
373
- }
374
-
375
- const child = session.child;
376
- if (!child) return;
377
-
378
- const pid = child.pid;
379
- if (!pid) return;
380
-
381
- try {
382
- if (session.descendantSnapshot?.length > 0) {
383
- processUtils.killProcessTreeSnapshot(session.descendantSnapshot);
384
- }
385
- } catch {
386
- /* best-effort cleanup */
387
- }
388
-
389
- if (!session.alive) return;
390
-
391
- // SIGTERM 먼저
392
- try {
393
- child.kill("SIGTERM");
394
- } catch {
395
- /* already dead */
396
- }
397
-
398
- // Grace period 대기
399
- const exited = await new Promise((resolve) => {
400
- const timer = setTimeout(() => {
401
- resolve(false);
402
- }, graceMs);
403
- child.once("exit", () => {
404
- clearTimeout(timer);
405
- resolve(true);
406
- });
407
- });
408
-
409
- if (!exited) {
410
- try {
411
- processUtils.killProcessTree(pid);
412
- } catch {
413
- forceKill(pid);
414
- }
415
- session.alive = false;
416
- }
417
- }
418
-
419
- /**
420
- * Health probe 콜백 — probe 결과에 따라 상태 전이 판단.
421
- */
422
- function handleProbeResult(session, result) {
423
- if (TERMINAL_STATES.has(session.state)) return;
424
- if (session.state === STATES.INIT || session.state === STATES.RESTARTING)
425
- return;
426
-
427
- eventLog.append("health", {
428
- session: session.id,
429
- ...result,
430
- });
431
-
432
- // L0 실패 — 로컬: exit handler에서 처리. 원격: probe가 유일한 감지 수단.
433
- if (result.l0 === "fail") {
434
- if (session.config.remote) {
435
- handleFailure(session, "remote_L0_fail");
436
- }
437
- return;
438
- }
439
-
440
- // L3 completed (원격 완료 토큰 감지)
441
- if (result.l3 === "completed" && session.config.remote) {
442
- transition(session, STATES.COMPLETED, "remote_completion_token");
443
- emitter.emit("completed", { sessionId: session.id });
444
- if (typeof session.config.onCompleted === "function") {
445
- session.config.onCompleted({ sessionId: session.id });
446
- }
447
- maybeAutoShutdown();
448
- return;
449
- }
450
-
451
- // L1 INPUT_WAIT 감지
452
- if (result.l1 === "input_wait" && session.state === STATES.HEALTHY) {
453
- transition(
454
- session,
455
- STATES.INPUT_WAIT,
456
- `input_wait:${result.inputWaitPattern}`,
457
- );
458
- emitter.emit("inputWait", {
459
- sessionId: session.id,
460
- pattern: result.inputWaitPattern,
461
- });
462
- return;
463
- }
464
-
465
- // INPUT_WAIT → output 재개 시 HEALTHY 복귀
466
- if (session.state === STATES.INPUT_WAIT && result.l1 === "ok") {
467
- transition(session, STATES.HEALTHY, "output_resumed");
468
- return;
469
- }
470
-
471
- // L1 stall
472
- if (result.l1 === "stall" && session.state === STATES.HEALTHY) {
473
- transition(session, STATES.STALLED, "L1_stall");
474
- return;
475
- }
476
-
477
- // STALLED → output 재개 시 HEALTHY 복귀
478
- if (session.state === STATES.STALLED && result.l1 === "ok") {
479
- transition(session, STATES.HEALTHY, "output_resumed");
480
- return;
481
- }
482
-
483
- // L3 timeout (아직 STARTING 상태)
484
- if (result.l3 === "timeout" && session.state === STATES.STARTING) {
485
- handleFailure(session, "L3_timeout");
486
- return;
487
- }
488
-
489
- // STARTING → L0 ok + L3 ok → HEALTHY
490
- if (
491
- session.state === STATES.STARTING &&
492
- result.l0 === "ok" &&
493
- result.l3 === "ok"
494
- ) {
495
- transition(session, STATES.HEALTHY, "probe_healthy");
496
- return;
497
- }
498
-
499
- // STARTING → L0 ok (L3 아직 미판정) → STARTING 유지 (대기)
500
- }
501
-
502
- /**
503
- * 실패 처리 — restart 또는 DEAD.
504
- */
505
- async function handleFailure(session, reason) {
506
- if (TERMINAL_STATES.has(session.state)) return;
507
-
508
- transition(session, STATES.FAILED, reason);
509
-
510
- if (session.restarts < maxRestarts) {
511
- transition(
512
- session,
513
- STATES.RESTARTING,
514
- `restart_${session.restarts + 1}/${maxRestarts}`,
515
- );
516
- session.restarts += 1;
517
- const backoffMs = Math.min(1000 * 2 ** (session.restarts - 1), 30_000);
518
- session._respawnTimer = setTimeout(() => {
519
- session._respawnTimer = null;
520
- if (session.config.remote) {
521
- startRemoteSession(session);
522
- } else {
523
- void respawnSession(session);
524
- }
525
- }, backoffMs);
526
- } else {
527
- await cleanupChild(session);
528
- transition(session, STATES.DEAD, `maxRestarts(${maxRestarts})_exceeded`);
529
- emitter.emit("dead", { sessionId: session.id, reason });
530
-
531
- // broker release on final death
532
- const activeBroker = getActiveBroker();
533
- if (activeBroker && session.config.accountId) {
534
- activeBroker.release(session.config.accountId, {
535
- ok: false,
536
- failureMode: session.lastFailureMode,
537
- });
538
- if (session.lastFailureMode === "rate_limited") {
539
- activeBroker.markRateLimited(session.config.accountId, 5 * 60 * 1000);
540
- }
541
- }
542
- }
543
- }
544
-
545
- /**
546
- * 세션의 child process를 (재)시작.
547
- */
548
- async function respawnSession(session) {
549
- // 기존 child 정리
550
- await cleanupChild(session);
551
-
552
- transition(
553
- session,
554
- STATES.STARTING,
555
- session.restarts > 0 ? "respawn" : "initial",
556
- );
557
-
558
- const launcher = session.launcher;
559
- const outPath = join(logsDir, `${session.id}.out.log`);
560
- const errPath = join(logsDir, `${session.id}.err.log`);
561
- mkdirSync(logsDir, { recursive: true });
562
-
563
- const outWs = createWriteStream(outPath, { flags: "a" });
564
- const errWs = createWriteStream(errPath, { flags: "a" });
565
-
566
- let outputBytes = 0;
567
- let recentOutput = "";
568
- // #115 F7 wiring: stdout-only tail for completion payload extraction.
569
- let stdoutTail = "";
570
- // #125: sentinel-framed capture (preferred over the 16 KiB tail; survives
571
- // payloads larger than the sliding window).
572
- const sentinelCapture = createSentinelCapture();
573
-
574
- const spawnCwd = session.config.workdir || launcher.cwd || undefined;
575
- let excludeMcpServers = [];
576
- const requestedMcpServers = Array.isArray(session.config.mcpServers)
577
- ? session.config.mcpServers
578
- : [];
579
- if (session.config.agent === "codex" && requestedMcpServers.length > 0) {
580
- const preflightFn = opts.deps?.codexPreflight || runCodexPreflight;
581
- try {
582
- const preflight = await preflightFn({
583
- mcpServers: requestedMcpServers,
584
- subcommand: "exec",
585
- workdir: spawnCwd,
586
- });
587
- excludeMcpServers = Array.isArray(preflight?.excludeMcpServers)
588
- ? preflight.excludeMcpServers
589
- : [];
590
- for (const warning of preflight?.warnings || []) {
591
- eventLog.append("codex_preflight_warning", {
592
- session: session.id,
593
- warning,
594
- });
595
- emitter.emit("warning", {
596
- type: "codex_preflight_warning",
597
- session: session.id,
598
- warning,
599
- });
600
- }
601
- } catch (err) {
602
- eventLog.append("codex_preflight_failed", {
603
- session: session.id,
604
- error: err.message,
605
- });
606
- emitter.emit("warning", {
607
- type: "codex_preflight_failed",
608
- session: session.id,
609
- error: err.message,
610
- });
611
- }
612
- }
613
-
614
- let spawnSpec;
615
- try {
616
- spawnSpec = buildSpawnSpecForMode(MODES.HEADLESS, {
617
- cli: session.config.agent,
618
- prompt: session.config.prompt,
619
- profile: session.config.profile,
620
- role: session.config.role,
621
- model: session.config.model,
622
- env: session.config.env,
623
- mcpServers: session.config.mcpServers,
624
- excludeMcpServers,
625
- resolveCommand: opts.deps?.resolveCliExecutable,
626
- onWarning: (message, details = {}) => {
627
- eventLog.append("mcp_server_skipped", {
628
- session: session.id,
629
- message,
630
- ...details,
631
- });
632
- emitter.emit("warning", {
633
- type: "mcp_server_skipped",
634
- session: session.id,
635
- message,
636
- ...details,
637
- });
638
- },
639
- });
640
- } catch (err) {
641
- // respawnSession 은 `void` 로 호출되므로 여기서 던지면 처리되지 않은 거부가
642
- // 된다. spec 생성 실패를 명시 실패로 접어 세션이 매달리지 않게 한다.
643
- // 정책 차단과 그 밖의 spec 생성 실패는 원인이 다르므로 이벤트를 나눈다.
644
- const message = err?.message || String(err);
645
- const blockedByPolicy = err?.code === "TFX_CLI_DISABLED";
646
- const eventName = blockedByPolicy
647
- ? "session_spawn_blocked"
648
- : "session_spawn_failed";
649
- const reason = blockedByPolicy ? "spawn_blocked" : "spawn_failed";
650
- const detail = {
651
- session: session.id,
652
- agent: session.config.agent,
653
- code: err?.code || null,
654
- error: message,
655
- };
656
- eventLog.append(eventName, detail);
657
- emitter.emit("warning", { type: eventName, ...detail });
658
- handleFailure(session, `${reason}:${message}`);
659
- return;
660
- }
661
-
662
- // #90 branch guard: shard spawn cwd가 main 브랜치면 즉시 abort.
663
- // swarm shard는 반드시 shard 전용 worktree 브랜치에서 실행되어야 한다.
664
- // cwd fallback으로 main working tree에 떨어진 경우 차단.
665
- if (session.config.branchGuard && spawnCwd) {
666
- try {
667
- const r = spawnSync("git", ["rev-parse", "--abbrev-ref", "HEAD"], {
668
- cwd: spawnCwd,
669
- encoding: "utf8",
670
- windowsHide: true,
671
- timeout: 5_000,
672
- });
673
- const curBranch = String(r.stdout || "").trim();
674
- if (curBranch === "main" || curBranch === "master") {
675
- eventLog.append("branch_guard_refused", {
676
- session: session.id,
677
- cwd: spawnCwd,
678
- branch: curBranch,
679
- });
680
- handleFailure(session, `branch_guard:${curBranch}_refused`);
681
- return;
682
- }
683
- } catch (err) {
684
- eventLog.append("branch_guard_probe_failed", {
685
- session: session.id,
686
- cwd: spawnCwd,
687
- error: err.message,
688
- });
689
- // 프로브 실패는 비차단 — git 미설치/non-repo 환경 대응
690
- }
691
- }
692
-
693
- const spawnEnv = {
694
- ...process.env,
695
- ...launcher.env,
696
- ...(session.config.env || {}),
697
- ...(session.config.branchGuard ? { TRIFLUX_GIT_BRANCH_GUARD: "1" } : {}),
698
- };
699
- if (session.config.workerSandbox?.root) {
700
- eventLog.append("worker_sandbox_env", {
701
- session: session.id,
702
- root: session.config.workerSandbox.root,
703
- codeHomePreserved: Boolean(spawnEnv.CODEX_HOME),
704
- });
705
- }
706
-
707
- let child;
708
- try {
709
- child = spawnFn(spawnSpec.command, spawnSpec.args, {
710
- shell: false,
711
- cwd: spawnCwd,
712
- env: spawnEnv,
713
- reason: `conductor:respawnSession:${session.id}`,
714
- stdio: ["pipe", "pipe", "pipe"],
715
- windowsHide: true,
716
- });
717
- } catch (err) {
718
- eventLog.append("spawn_error", {
719
- session: session.id,
720
- error: err.message,
721
- cwd: spawnCwd || null,
722
- command: spawnSpec.command,
723
- });
724
- handleFailure(session, `spawn_error:${err.message}`);
725
- return;
726
- }
727
-
728
- session.child = child;
729
- session.outPath = outPath;
730
- session.errPath = errPath;
731
- session.descendantSnapshot = [];
732
-
733
- // #116-C: spawnSpec.stdinPrompt 가 true 면 codex 의 oh-my-codex hook
734
- // (macOS) 가 매우 긴 argv-inline prompt 에서 SessionStart Failed → exit 1
735
- // 회귀를 유발하므로 prompt 를 stdin write 로 분리해서 전달한다.
736
- // stdinPrompt 없으면 기존 흐름 — codex "Reading additional input..." 대기
737
- // 방지를 위해 stdin 즉시 닫기.
738
- if (child.stdin) {
739
- if (spawnSpec.stdinPrompt && typeof spawnSpec.prompt === "string") {
740
- try {
741
- child.stdin.write(spawnSpec.prompt);
742
- } catch (err) {
743
- eventLog.append("spawn_stdin_write_error", {
744
- session: session.id,
745
- error: err.message,
746
- });
747
- }
748
- }
749
- child.stdin.end();
750
- }
751
-
752
- eventLog.append("spawn", {
753
- session: session.id,
754
- agent: session.config.agent,
755
- pid: child.pid,
756
- command: spawnSpec.command,
757
- args: spawnSpec.args,
758
- legacyCommand: launcher.command,
759
- restart: session.restarts,
760
- });
761
-
762
- if (session._descendantSnapshotTimer) {
763
- clearTimeout(session._descendantSnapshotTimer);
764
- }
765
- session._descendantSnapshotTimer = setTimeout(() => {
766
- session._descendantSnapshotTimer = null;
767
- if (session.child !== child) return;
768
- try {
769
- session.descendantSnapshot = processUtils.findProcessTree(child.pid);
770
- } catch {
771
- /* best-effort snapshot */
772
- }
773
- }, 200);
774
-
775
- // stdout+stderr 통합 추적 (F3 해결: stderr만 출력되는 경우도 advancing 판정)
776
- const trackOutput = (buf) => {
777
- outputBytes += buf.length;
778
- const txt = String(buf);
779
- // 최근 2KB만 유지 (INPUT_WAIT 패턴 감지용)
780
- recentOutput += txt;
781
- if (recentOutput.length > 2048) {
782
- recentOutput = recentOutput.slice(-2048);
783
- }
784
- };
785
-
786
- child.stdout?.on("data", (buf) => {
787
- outWs.write(buf);
788
- trackOutput(buf);
789
- const text = buf.toString("utf8");
790
- stdoutTail = (stdoutTail + text).slice(-16384);
791
- sentinelCapture.push(text);
792
- });
793
- child.stderr?.on("data", (buf) => {
794
- errWs.write(buf);
795
- trackOutput(buf);
796
- });
797
-
798
- child.on("exit", (code, signal) => {
799
- session.alive = false;
800
- if (session._descendantSnapshotTimer) {
801
- clearTimeout(session._descendantSnapshotTimer);
802
- session._descendantSnapshotTimer = null;
803
- }
804
- try {
805
- outWs.end();
806
- } catch {
807
- /* ignore */
808
- }
809
- try {
810
- errWs.end();
811
- } catch {
812
- /* ignore */
813
- }
814
-
815
- eventLog.append("exit", {
816
- session: session.id,
817
- code,
818
- signal,
819
- restart: session.restarts,
820
- });
821
-
822
- if (TERMINAL_STATES.has(session.state)) return;
823
- if (shuttingDown) return;
824
-
825
- if (code === 0 && !signal) {
826
- transition(session, STATES.COMPLETED, "exit_0");
827
- // #125 Codex R1 MEDIUM: if sentinel capture overflowed past maxBytes
828
- // and lost its BEGIN marker, do NOT fall back to stdoutTail/brace-scan
829
- // — the worker tried sentinel protocol and the payload was clearly
830
- // unbounded. Surface as "no payload" rather than silent partial.
831
- let extracted;
832
- if (sentinelCapture.isOverflow()) {
833
- extracted = null;
834
- } else {
835
- const sentinelSnapshot = sentinelCapture.snapshot();
836
- extracted = extractCompletionPayload(sentinelSnapshot || stdoutTail);
837
- }
838
- const completionPayload = extracted ? extracted.payload : undefined;
839
- emitter.emit("completed", { sessionId: session.id, completionPayload });
840
- if (typeof session.config.onCompleted === "function") {
841
- session.config.onCompleted({
842
- sessionId: session.id,
843
- completionPayload,
844
- });
845
- }
846
- const activeBroker = getActiveBroker();
847
- if (activeBroker && session.config.accountId) {
848
- activeBroker.release(session.config.accountId, { ok: true });
849
- }
850
- } else {
851
- // detect rate_limited from recent output before handleFailure
852
- if (
853
- /(rate.?limit|quota|throttl|too.many.requests|429|usage.limit)/iu.test(
854
- recentOutput,
855
- )
856
- ) {
857
- session.lastFailureMode = "rate_limited";
858
- }
859
- handleFailure(session, `exit_code:${code},signal:${signal}`);
860
- }
861
-
862
- maybeAutoShutdown();
863
- });
864
-
865
- child.on("error", (err) => {
866
- session.alive = false;
867
- if (session._descendantSnapshotTimer) {
868
- clearTimeout(session._descendantSnapshotTimer);
869
- session._descendantSnapshotTimer = null;
870
- }
871
- eventLog.append("child_error", {
872
- session: session.id,
873
- error: err.message,
874
- });
875
- if (!shuttingDown && !TERMINAL_STATES.has(session.state)) {
876
- handleFailure(session, `child_error:${err.message}`);
877
- }
878
- });
879
-
880
- session.alive = true;
881
-
882
- // Health probe 설정
883
- session.probe?.stop();
884
- const probe = createHealthProbe(
885
- {
886
- get pid() {
887
- return child.pid;
888
- },
889
- get alive() {
890
- return session.alive;
891
- },
892
- getOutputBytes: () => outputBytes,
893
- getRecentOutput: () => recentOutput,
894
- },
895
- {
896
- ...probeOpts,
897
- // #165: default off → on. atomic write (#162) 로 race 제거됨.
898
- // opt-out: TFX_PROBE_WRITE_STATE=0 명시.
899
- writeStateFile:
900
- probeOpts.writeStateFile ?? process.env.TFX_PROBE_WRITE_STATE !== "0",
901
- // #168 P3: L2 (hub /health) checker wiring. probeOpts 가 명시 주입하면
902
- // 그걸 우선. 아니면 TFX_PROBE_L2=0 로 opt-out. 나머지 경우 hub URL 기반
903
- // default checker 주입 → deriveState 가 `mcp_initializing` 라벨을 생산 →
904
- // heartbeat (read_probe_state) 가 probe-grace 분기로 감.
905
- enableL2: probeOpts.enableL2 ?? process.env.TFX_PROBE_L2 !== "0",
906
- checkMcp:
907
- probeOpts.checkMcp ||
908
- (process.env.TFX_PROBE_L2 === "0"
909
- ? undefined
910
- : createHubHealthChecker({ hubUrl: process.env.TFX_HUB_URL })),
911
- onProbe: (result) => handleProbeResult(session, result),
912
- },
913
- );
914
- session.probe = probe;
915
- probe.start();
916
- }
917
-
918
- /**
919
- * 원격 세션 시작 — SSH child process로 직접 실행.
920
- * psmux 불필요. 원격 호스트에서 claude -p 를 SSH 경유로 실행하고
921
- * 로컬 child process로 관리한다.
922
- */
923
- function startRemoteSession(session) {
924
- transition(session, STATES.STARTING, "remote_initial");
925
-
926
- const { host, prompt, agent, workdir: remoteWorkdir } = session.config;
927
-
928
- // SSH args를 배열로 구성하여 셸 이스케이프 문제 회피
929
- const cdPrefix = remoteWorkdir ? `cd ${remoteWorkdir} && ` : "";
930
-
931
- let remoteBin;
932
- if (agent === "claude") {
933
- remoteBin = "claude --output-format text";
934
- } else if (agent === "gemini") {
935
- remoteBin = "gemini -y";
936
- } else {
937
- const codexHome = session.config.env?.CODEX_HOME;
938
- const profile = resolveNestedCodexAgentProfile(
939
- session.config.role || "executor",
940
- {
941
- profileOverride: session.config.profile ?? "auto",
942
- globalProfile:
943
- session.config.env?.TFX_CODEX_PROFILE ??
944
- process.env.TFX_CODEX_PROFILE,
945
- codexHome,
946
- },
947
- );
948
- const configArgs = codexProfileConfigOverrides(profile, {
949
- codexHome,
950
- disallowUltra: true,
951
- enforceCanonicalProfile: true,
952
- })
953
- .map((override) => `-c '${override.replace(/'/g, `'\\''`)}'`)
954
- .join(" ");
955
- remoteBin = `codex exec${configArgs ? ` ${configArgs}` : ""} -s danger-full-access --dangerously-bypass-approvals-and-sandbox`;
956
- }
957
-
958
- // prompt는 stdin으로 전달 — 셸 이스케이프 문제 완전 회피
959
- const sshArgs = [
960
- "-o",
961
- "ConnectTimeout=30",
962
- "-o",
963
- "BatchMode=yes",
964
- host,
965
- `${cdPrefix}${remoteBin}`,
966
- ];
967
-
968
- const sshCmd = `ssh ${sshArgs.join(" ")}`;
969
-
970
- eventLog.append("remote_start", {
971
- session: session.id,
972
- host,
973
- command: sshCmd,
974
- });
975
-
976
- const outPath = join(logsDir, `${session.id}.out.log`);
977
- const errPath = join(logsDir, `${session.id}.err.log`);
978
- const outWs = createWriteStream(outPath, { flags: "a" });
979
- const errWs = createWriteStream(errPath, { flags: "a" });
980
-
981
- let child;
982
- try {
983
- child = spawnFn("ssh", sshArgs, {
984
- env: process.env,
985
- reason: `conductor:remoteSession:${session.id}`,
986
- stdio: ["pipe", "pipe", "pipe"],
987
- windowsHide: true,
988
- });
989
- } catch (err) {
990
- eventLog.append("spawn_error", {
991
- session: session.id,
992
- error: err.message,
993
- });
994
- handleFailure(session, `spawn_error:${err.message}`);
995
- return;
996
- }
997
-
998
- session.child = child;
999
- session.outPath = outPath;
1000
- session.errPath = errPath;
1001
- session.alive = true;
1002
-
1003
- // stdin으로 프롬프트 전달 — 셸 이스케이프 완전 우회
1004
- if (child.stdin) {
1005
- child.stdin.write(prompt);
1006
- child.stdin.end();
1007
- }
1008
-
1009
- eventLog.append("spawn", {
1010
- session: session.id,
1011
- agent,
1012
- pid: child.pid,
1013
- command: sshCmd,
1014
- remote: true,
1015
- host,
1016
- });
1017
-
1018
- // stdout/stderr 추적 (recentOutput으로 rate_limit 패턴 감지 가능)
1019
- let outputBytes = 0;
1020
- let recentOutput = "";
1021
- // #115 F7 wiring: stdout-only tail for completion payload extraction.
1022
- let stdoutTail = "";
1023
- // #125: sentinel-framed capture (preferred over the 16 KiB tail).
1024
- const sentinelCapture = createSentinelCapture();
1025
- const trackOutput = (buf) => {
1026
- outputBytes += buf.length;
1027
- const text = buf.toString("utf8");
1028
- recentOutput = (recentOutput + text).slice(-2000);
1029
- };
1030
-
1031
- child.stdout?.on("data", (buf) => {
1032
- trackOutput(buf);
1033
- outWs.write(buf);
1034
- const text = buf.toString("utf8");
1035
- stdoutTail = (stdoutTail + text).slice(-16384);
1036
- sentinelCapture.push(text);
1037
- });
1038
- child.stderr?.on("data", (buf) => {
1039
- trackOutput(buf);
1040
- errWs.write(buf);
1041
- });
1042
-
1043
- // 상태를 healthy로 전이
1044
- transition(session, STATES.HEALTHY, "remote_ssh_spawned");
1045
-
1046
- // Health probe — 주기적으로 출력 변화 체크 (F3 stall 감지)
1047
- let lastBytes = 0;
1048
- let stallChecks = 0;
1049
- const stallThreshold = probeOpts.stallChecks || 8; // 8 * intervalMs = stall
1050
- const healthInterval = setInterval(() => {
1051
- if (TERMINAL_STATES.has(session.state)) {
1052
- clearInterval(healthInterval);
1053
- return;
1054
- }
1055
- if (outputBytes > lastBytes) {
1056
- lastBytes = outputBytes;
1057
- stallChecks = 0;
1058
- if (session.state !== STATES.HEALTHY) {
1059
- transition(session, STATES.HEALTHY, "output_advancing");
1060
- }
1061
- } else {
1062
- stallChecks++;
1063
- if (stallChecks >= stallThreshold && session.state === STATES.HEALTHY) {
1064
- transition(session, STATES.STALLED, "remote_no_output");
1065
- }
1066
- }
1067
- }, probeOpts.intervalMs || 5000);
1068
-
1069
- // exit 핸들러
1070
- child.once("exit", (code) => {
1071
- clearInterval(healthInterval);
1072
- outWs.end();
1073
- errWs.end();
1074
- session.alive = false;
1075
-
1076
- eventLog.append("remote_exit", {
1077
- session: session.id,
1078
- code,
1079
- outputBytes,
1080
- host,
1081
- });
1082
-
1083
- if (shuttingDown || TERMINAL_STATES.has(session.state)) return;
1084
-
1085
- // 원격 세션은 broker lease를 사용하지 않으므로 release 불필요
1086
- // (spawnSession에서 config.remote === true일 때 lease 건너뜀)
1087
- if (code === 0) {
1088
- transition(session, STATES.COMPLETED, `exit_${code}`);
1089
- // #125 Codex R1 MEDIUM: same overflow guard as the local exit path.
1090
- let extracted;
1091
- if (sentinelCapture.isOverflow()) {
1092
- extracted = null;
1093
- } else {
1094
- const sentinelSnapshot = sentinelCapture.snapshot();
1095
- extracted = extractCompletionPayload(sentinelSnapshot || stdoutTail);
1096
- }
1097
- const completionPayload = extracted ? extracted.payload : undefined;
1098
- emitter.emit("completed", { sessionId: session.id, completionPayload });
1099
- if (typeof session.config.onCompleted === "function") {
1100
- session.config.onCompleted({
1101
- sessionId: session.id,
1102
- completionPayload,
1103
- });
1104
- }
1105
- maybeAutoShutdown();
1106
- } else {
1107
- handleFailure(session, `remote_exit_${code}`);
1108
- }
1109
- });
1110
-
1111
- child.once("error", (err) => {
1112
- clearInterval(healthInterval);
1113
- outWs.end();
1114
- errWs.end();
1115
- eventLog.append("spawn_error", {
1116
- session: session.id,
1117
- error: err.message,
1118
- });
1119
- if (!shuttingDown && !TERMINAL_STATES.has(session.state)) {
1120
- handleFailure(session, `spawn_error:${err.message}`);
1121
- }
1122
- });
1123
- }
1124
-
1125
- /**
1126
- * 모든 세션이 terminal이면 auto-shutdown.
1127
- */
1128
- function maybeAutoShutdown() {
1129
- if (shuttingDown) return;
1130
- const allTerminal = [...sessions.values()].every((s) =>
1131
- TERMINAL_STATES.has(s.state),
1132
- );
1133
- if (allTerminal && sessions.size > 0) {
1134
- emitter.emit("allCompleted");
1135
- }
1136
- }
1137
-
1138
- // ── Public API ──────────────────────────────────────────────
1139
-
1140
- /**
1141
- * 새 세션 spawn.
1142
- * @param {object} config
1143
- * @param {string} config.id — 세션 ID (unique)
1144
- * @param {'codex'|'gemini'|'claude'} config.agent
1145
- * @param {string} config.prompt
1146
- * @param {string} [config.profile]
1147
- * @param {string} [config.workdir]
1148
- * @param {string} [config.model]
1149
- * @param {boolean} [config.remote=false] — 원격 세션 여부
1150
- * @param {string} [config.host] — SSH 호스트 (remote=true 필수)
1151
- * @param {string} [config.paneTarget] — psmux pane target (remote용)
1152
- * @param {string} [config.sessionName] — psmux 세션 이름 (remote용)
1153
- * @param {function} [config.onCompleted] — 세션 완료 시 콜백 ({sessionId}) => void
1154
- * @returns {string} session ID
1155
- */
1156
- function spawnSession(config) {
1157
- if (shuttingDown) throw new Error("Conductor is shutting down");
1158
- if (!config.id) throw new Error("session id is required");
1159
- if (sessions.has(config.id))
1160
- throw new Error(`Session "${config.id}" already exists`);
1161
- if (config.remote && !config.host)
1162
- throw new Error("host is required for remote sessions");
1163
-
1164
- // Phase 1 dynamic routing wire-up (opt-in env TRIFLUX_DYNAMIC_ROUTING).
1165
- // sync 로만 평가해서 broker.lease atomic 인변량 (commit 761fb7a8 — leased
1166
- // account = executing account) 을 깨지 않는다. snapshot 은 D-2 cache 의
1167
- // sync getter 만 사용 — cache miss 면 evaluateDynamicRoute 가 fail-closed
1168
- // 로 codex-default 를 반환한다.
1169
- if (
1170
- isDynamicRoutingEnabled(process.env) &&
1171
- !config.remote &&
1172
- config.agent
1173
- ) {
1174
- try {
1175
- const policy = dynamicRoutingInternal.loadPolicy();
1176
- const snapshot = tryGetCachedSnapshot();
1177
- const decision = evaluateDynamicRoute(
1178
- {
1179
- task_id: config.id,
1180
- team_size: 1,
1181
- agent_hint: config.agent,
1182
- },
1183
- snapshot || {},
1184
- policy,
1185
- { now: Date.now() },
1186
- );
1187
- const shard = decision?.shards?.[0];
1188
- if (shard?.cli && shard.cli !== config.agent) {
1189
- eventLog.append("dynamic_route_override", {
1190
- session: config.id,
1191
- original: config.agent,
1192
- override: shard.cli,
1193
- scenario: decision.scenario ?? "unknown",
1194
- decisionId: decision.decisionId ?? null,
1195
- snapshotAgeMs: decision.snapshotAgeMs ?? null,
1196
- });
1197
- config = { ...config, agent: shard.cli };
1198
- }
1199
- } catch (err) {
1200
- eventLog.append("dynamic_route_error", {
1201
- session: config.id,
1202
- error: err?.message || String(err),
1203
- });
1204
- }
1205
- }
1206
-
1207
- // TFX_DISABLE_* SSOT 집행. 계정 임대와 worker sandbox 생성보다 앞에 둔다.
1208
- // 뒤에 두면 정책으로 거부된 세션이 샌드박스 디렉터리와 auth.json 사본을
1209
- // 남기고, 임대 반납이 회로 차단기 실패로 집계돼 멀쩡한 계정이 cooldown 에
1210
- // 들어간다. 다른 CLI 로 자동 전환하지 않고 호출자에게 그대로 던진다.
1211
- // claude 처럼 정책 대상이 아닌 agent 는 프로파일 파일을 읽지 않는다 (F4 와 동일).
1212
- if (!config.remote && normalizeCliName(config.agent)) {
1213
- const blockedCli = buildDisabledCliError(
1214
- config.agent,
1215
- resolveCliPolicy(process.env),
1216
- );
1217
- if (blockedCli) {
1218
- eventLog.append("session_spawn_blocked", {
1219
- session: config.id,
1220
- agent: config.agent,
1221
- code: blockedCli.code || null,
1222
- error: blockedCli.message,
1223
- });
1224
- emitter.emit("warning", {
1225
- type: "session_spawn_blocked",
1226
- session: config.id,
1227
- agent: config.agent,
1228
- code: blockedCli.code || null,
1229
- error: blockedCli.message,
1230
- });
1231
- throw blockedCli;
1232
- }
1233
- }
1234
-
1235
- // broker lease (graceful — broker null if accounts.json absent)
1236
- let lease = null;
1237
- const activeBroker = getActiveBroker();
1238
- if (activeBroker && config.agent && !config.remote) {
1239
- lease = activeBroker.lease({ provider: config.agent });
1240
- if (lease === null) {
1241
- const eta = activeBroker.nextAvailableEta(config.agent);
1242
- eventLog.append("broker_no_lease", {
1243
- session: config.id,
1244
- agent: config.agent,
1245
- eta: eta ? new Date(eta).toISOString() : "unknown",
1246
- });
1247
- // 계정이 모두 cooldown이어도 세션 생성 자체는 유지한다.
1248
- // 로컬 테스트/단일 계정 없는 환경에서도 상태 머신이 일관되게 동작해야 한다.
1249
- }
1250
- }
1251
-
1252
- // apply lease profile/env/auth to config (immutable — new object)
1253
- const leaseEnv =
1254
- lease?.authFile && config.agent === "codex"
1255
- ? { ...(lease.env || {}), CODEX_HOME: dirname(lease.authFile) }
1256
- : lease?.env || {};
1257
-
1258
- let resolvedConfig = lease
1259
- ? {
1260
- ...config,
1261
- profile: lease.profile ?? config.profile,
1262
- env: { ...(config.env || {}), ...leaseEnv },
1263
- accountId: lease.id,
1264
- }
1265
- : {
1266
- ...config,
1267
- env: { ...(config.env || {}) },
1268
- };
1269
-
1270
- if (!resolvedConfig.remote) {
1271
- const sandbox = buildWorkerSandboxEnv({
1272
- cwd: resolvedConfig.workdir || process.cwd(),
1273
- sessionId: resolvedConfig.id,
1274
- agent: resolvedConfig.agent,
1275
- env: { ...process.env, ...(resolvedConfig.env || {}) },
1276
- });
1277
- resolvedConfig = {
1278
- ...resolvedConfig,
1279
- env: { ...(resolvedConfig.env || {}), ...sandbox.env },
1280
- workerSandbox: sandbox.disabled
1281
- ? { disabled: true }
1282
- : { root: sandbox.root, home: sandbox.home },
1283
- };
1284
- }
1285
-
1286
- // auth file copy — broker resolved absolute path, conductor does the actual copy
1287
- swapAuthFile(
1288
- lease,
1289
- config.agent,
1290
- config.id,
1291
- eventLog,
1292
- activeBroker,
1293
- resolvedConfig.env,
1294
- );
1295
-
1296
- // 원격 세션은 launcher 불필요 (이미 원격에서 실행 중)
1297
- const launcher = resolvedConfig.remote
1298
- ? null
1299
- : buildLauncher({
1300
- agent: resolvedConfig.agent,
1301
- profile: resolvedConfig.profile,
1302
- prompt: resolvedConfig.prompt,
1303
- workdir: resolvedConfig.workdir,
1304
- model: resolvedConfig.model,
1305
- });
1306
-
1307
- const session = {
1308
- id: resolvedConfig.id,
1309
- config: resolvedConfig,
1310
- launcher,
1311
- state: STATES.INIT,
1312
- child: null,
1313
- probe: null,
1314
- alive: false,
1315
- restarts: 0,
1316
- outPath: null,
1317
- errPath: null,
1318
- createdAt: Date.now(),
1319
- descendantSnapshot: [],
1320
- };
1321
-
1322
- sessions.set(resolvedConfig.id, session);
1323
- getConductorRegistry()?.register?.(resolvedConfig.id, publicApi);
1324
- logCtoContext(resolvedConfig.id);
1325
-
1326
- if (resolvedConfig.remote) {
1327
- startRemoteSession(session);
1328
- } else {
1329
- void respawnSession(session);
1330
- }
1331
- return resolvedConfig.id;
1332
- }
1333
-
1334
- /**
1335
- * 세션 kill.
1336
- * @param {string} id
1337
- * @param {string} [reason]
1338
- */
1339
- async function killSession(id, reason = "user_kill") {
1340
- const session = sessions.get(id);
1341
- if (!session) return;
1342
- if (TERMINAL_STATES.has(session.state)) return;
1343
-
1344
- if (session._respawnTimer) {
1345
- clearTimeout(session._respawnTimer);
1346
- session._respawnTimer = null;
1347
- }
1348
- eventLog.append("kill", { session: id, reason });
1349
- await cleanupChild(session);
1350
- transition(session, STATES.FAILED, reason);
1351
- transition(session, STATES.DEAD, reason);
1352
- }
1353
-
1354
- /**
1355
- * 세션에 stdin 입력 전송 (INPUT_WAIT 해소용).
1356
- * @param {string} id
1357
- * @param {string} text
1358
- */
1359
- function sendInput(id, text) {
1360
- const session = sessions.get(id);
1361
- if (!session) return false;
1362
-
1363
- // 원격 세션 — stdin 미지원 (psmux send-keys는 별도 경로)
1364
- if (session.config.remote) {
1365
- eventLog.append("stdin_remote_unsupported", { session: id });
1366
- return false;
1367
- }
1368
-
1369
- if (!session.child) return false;
1370
- if (!session.child.stdin?.writable) return false;
1371
- try {
1372
- session.child.stdin.write(`${text}\n`);
1373
- eventLog.append("stdin", { session: id, text: text.slice(0, 100) });
1374
- return true;
1375
- } catch {
1376
- return false;
1377
- }
1378
- }
1379
-
1380
- /**
1381
- * 전체 세션 스냅샷.
1382
- * @returns {object[]}
1383
- */
1384
- function getSnapshot() {
1385
- return [...sessions.values()].map((s) => ({
1386
- id: s.id,
1387
- agent: s.config.agent,
1388
- state: s.state,
1389
- pid: s.child?.pid || null,
1390
- remote: s.config.remote || false,
1391
- host: s.config.host || null,
1392
- restarts: s.restarts,
1393
- health: s.probe?.getStatus() || null,
1394
- outPath: s.outPath,
1395
- errPath: s.errPath,
1396
- createdAt: s.createdAt,
1397
- }));
1398
- }
1399
-
1400
- /**
1401
- * Graceful shutdown — 전체 세션 종료.
1402
- */
1403
- async function shutdown(reason = "shutdown") {
1404
- if (shuttingDown) return;
1405
- shuttingDown = true;
1406
-
1407
- eventLog.append("shutdown", { reason, sessions: sessions.size });
1408
-
1409
- const cleanedSessionIds = new Set();
1410
- const cleanups = [...sessions.values()]
1411
- .filter((s) => {
1412
- if (cleanedSessionIds.has(s.id)) return false;
1413
- cleanedSessionIds.add(s.id);
1414
- return true;
1415
- })
1416
- .map(async (s) => {
1417
- if (s._respawnTimer) {
1418
- clearTimeout(s._respawnTimer);
1419
- s._respawnTimer = null;
1420
- }
1421
- if (s._descendantSnapshotTimer) {
1422
- clearTimeout(s._descendantSnapshotTimer);
1423
- s._descendantSnapshotTimer = null;
1424
- }
1425
- s.probe?.stop();
1426
- await cleanupChild(s);
1427
- if (!TERMINAL_STATES.has(s.state)) {
1428
- transition(s, STATES.FAILED, reason);
1429
- transition(s, STATES.DEAD, reason);
1430
- }
1431
- });
1432
-
1433
- await Promise.allSettled(cleanups);
1434
- process.removeListener("SIGINT", onSignal);
1435
- process.removeListener("SIGTERM", onSignal);
1436
- if (brokerInstance && onTierFallback) {
1437
- brokerInstance.removeListener("tierFallback", onTierFallback);
1438
- }
1439
- if (conductor._meshBridge) conductor._meshBridge.detach();
1440
- await eventLog.flush();
1441
- await eventLog.close();
1442
- emitter.emit("shutdown");
1443
- }
1444
-
1445
- // ── broker tierFallback 구독 ──────────────────────────────────
1446
- // 토큰 만료 등으로 상위 tier 계정이 불가능해지면, 활성 세션의 auth를 새 lease로 교체.
1447
- const brokerInstance = optsBroker ?? null;
1448
- const onTierFallback = brokerInstance
1449
- ? ({ provider }) => {
1450
- for (const session of sessions.values()) {
1451
- if (TERMINAL_STATES.has(session.state)) continue;
1452
- if (session.config.agent !== provider) continue;
1453
- if (session.config.remote) continue;
1454
-
1455
- const newLease = brokerInstance.lease({ provider });
1456
- if (!newLease) continue;
1457
- if (newLease.authFile && provider === "codex") {
1458
- session.config.env = {
1459
- ...(session.config.env || {}),
1460
- CODEX_HOME: dirname(newLease.authFile),
1461
- };
1462
- }
1463
-
1464
- swapAuthFile(
1465
- newLease,
1466
- provider,
1467
- session.id,
1468
- eventLog,
1469
- brokerInstance,
1470
- session.config.env,
1471
- );
1472
- eventLog.append("tier_fallback_swap", {
1473
- session: session.id,
1474
- agent: provider,
1475
- newAccount: newLease.id,
1476
- });
1477
- }
1478
- }
1479
- : null;
1480
-
1481
- if (brokerInstance && onTierFallback) {
1482
- brokerInstance.on("tierFallback", onTierFallback);
1483
- }
1484
-
1485
- // Shutdown traps
1486
- const onSignal = () => {
1487
- void shutdown("signal");
1488
- };
1489
- process.on("SIGINT", onSignal);
1490
- process.on("SIGTERM", onSignal);
1491
-
1492
- const conductor = {
1493
- spawnSession,
1494
- killSession,
1495
- sendInput,
1496
- getSnapshot,
1497
- getMeshRegistry() {
1498
- return this._meshRegistry || null;
1499
- },
1500
- shutdown,
1501
- on: emitter.on.bind(emitter),
1502
- off: emitter.off.bind(emitter),
1503
- get sessionCount() {
1504
- return sessions.size;
1505
- },
1506
- get isShuttingDown() {
1507
- return shuttingDown;
1508
- },
1509
- get eventLogPath() {
1510
- return eventLog.filePath;
1511
- },
1512
- };
1513
-
1514
- if (opts.enableMesh !== false) {
1515
- try {
1516
- const registry = opts.meshRegistry || createRegistry();
1517
- const bridge = createConductorMeshBridge(conductor, registry);
1518
- bridge.attach();
1519
- conductor._meshBridge = bridge;
1520
- conductor._meshRegistry = registry;
1521
- } catch {
1522
- // mesh 실패해도 conductor 정상 동작
1523
- }
1524
- }
1525
-
1526
- const frozenApi = Object.freeze(conductor);
1527
- ensureConductorRegistry();
1528
- return frozenApi;
1529
- }