@triflux/remote 10.41.1 → 10.43.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/cto/hygiene-actions.mjs +355 -0
- package/cto/hygiene.mjs +79 -9
- package/cto/steward.mjs +362 -0
- package/hub/log-retention.mjs +726 -0
- package/hub/pipe.mjs +5 -1
- package/hub/public/tray.html +50 -53
- package/hub/schema.sql +149 -0
- package/hub/server.mjs +44 -22
- package/hub/store.mjs +37 -19
- package/hub/team/claude-daemon-control.mjs +33 -12
- package/hub/team/cli/commands/start/parse-args.mjs +2 -2
- package/hub/team/headless.mjs +275 -24
- package/hub/team/intervention.mjs +530 -0
- package/hub/team/retry-state-machine.mjs +6 -2
- package/hub/team/swarm-hypervisor.mjs +19 -2
- package/hub/team/swarm-locks.mjs +17 -1
- package/hub/team/uds-orchestrator.mjs +62 -9
- package/hub/tools.mjs +5 -7
- package/hub/workers/codex-app-server-worker.mjs +54 -5
- package/hub/workers/codex-mcp.mjs +1 -1
- package/hub/workers/delegator-mcp.mjs +18 -18
- package/package.json +1 -1
- package/scripts/lib/cli-agy.mjs +1 -0
- package/scripts/lib/cli-claude.mjs +1 -0
- package/scripts/lib/cli-codex.mjs +22 -21
- package/scripts/lib/codex-profile-config.mjs +2 -2
|
@@ -0,0 +1,530 @@
|
|
|
1
|
+
// 무활동 감지는 호출자 소유다. 이 모듈은 개입 사다리만 집행한다.
|
|
2
|
+
// ① interrupt+재지시 → ② graceful+resume → ③ SIGTERM → ④ SIGKILL
|
|
3
|
+
import { execFile } from "node:child_process";
|
|
4
|
+
import { statSync } from "node:fs";
|
|
5
|
+
import { readdir, readlink } from "node:fs/promises";
|
|
6
|
+
import { promisify } from "node:util";
|
|
7
|
+
import { isPidAlive } from "@triflux/core/hub/lib/process-utils.mjs";
|
|
8
|
+
import { killProcess } from "@triflux/core/hub/platform.mjs";
|
|
9
|
+
import {
|
|
10
|
+
resolveHardCeilingMs,
|
|
11
|
+
resolveStallInterventionMs as resolveStallThresholdMs,
|
|
12
|
+
} from "@triflux/core/hub/lib/timeout-defaults.mjs";
|
|
13
|
+
|
|
14
|
+
export { resolveHardCeilingMs, resolveStallThresholdMs };
|
|
15
|
+
|
|
16
|
+
export const CHANNELS = Object.freeze({
|
|
17
|
+
CLAUDE_DAEMON: "claude-daemon",
|
|
18
|
+
TMUX_PANE: "tmux-pane",
|
|
19
|
+
CODEX_APP_SERVER: "codex-app-server",
|
|
20
|
+
PROCESS: "process",
|
|
21
|
+
});
|
|
22
|
+
|
|
23
|
+
export const STEPS = Object.freeze({
|
|
24
|
+
REINSTRUCT: "reinstruct",
|
|
25
|
+
RESUME: "resume",
|
|
26
|
+
SIGTERM: "sigterm",
|
|
27
|
+
SIGKILL: "sigkill",
|
|
28
|
+
});
|
|
29
|
+
|
|
30
|
+
export const INTERVENTION_DEFAULTS = Object.freeze({
|
|
31
|
+
interruptSettleMs: 3_000,
|
|
32
|
+
reinstructInjectTimeoutMs: 8_000,
|
|
33
|
+
reinstructWaitMs: 300_000,
|
|
34
|
+
resumeDispatchTimeoutMs: 60_000,
|
|
35
|
+
resumeWaitMs: 300_000,
|
|
36
|
+
activityPollMs: 5_000,
|
|
37
|
+
sigtermGraceMs: 5_000,
|
|
38
|
+
});
|
|
39
|
+
|
|
40
|
+
export const DEFAULT_REINSTRUCT_PROMPT =
|
|
41
|
+
"[triflux intervention] 무활동이 감지되어 인터럽트했다. 직전 작업을 이어가라: " +
|
|
42
|
+
"1) 마지막으로 완료한 단계를 한 줄로 요약하고, 2) 남은 단계를 계속 진행하라. " +
|
|
43
|
+
"장시간 침묵하는 단일 도구 호출 대신 진행 상황을 주기적으로 출력하라.";
|
|
44
|
+
|
|
45
|
+
const ROLLOUT_UUID_RE =
|
|
46
|
+
/rollout-.*-([0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12})\.jsonl$/u;
|
|
47
|
+
const execFileAsync = promisify(execFile);
|
|
48
|
+
const resumeCapabilityCache = new Map();
|
|
49
|
+
|
|
50
|
+
function positiveSeconds(env, key) {
|
|
51
|
+
const value = Number.parseInt(String(env?.[key] ?? ""), 10);
|
|
52
|
+
return Number.isFinite(value) && value > 0 ? value * 1_000 : undefined;
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
function sleepFor(ms) {
|
|
56
|
+
return new Promise((resolve) => setTimeout(resolve, ms));
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
export function detectChannel(target = {}) {
|
|
60
|
+
if (Object.values(CHANNELS).includes(target.channel)) return target.channel;
|
|
61
|
+
if (target.appServer?.client && target.appServer?.threadId)
|
|
62
|
+
return CHANNELS.CODEX_APP_SERVER;
|
|
63
|
+
if (
|
|
64
|
+
target.daemon?.controlSock &&
|
|
65
|
+
(target.daemon.short || target.daemon.sessionId)
|
|
66
|
+
)
|
|
67
|
+
return CHANNELS.CLAUDE_DAEMON;
|
|
68
|
+
if (typeof target.paneId === "string" && target.paneId)
|
|
69
|
+
return CHANNELS.TMUX_PANE;
|
|
70
|
+
if (Number.isInteger(target.pid) && target.pid > 0) return CHANNELS.PROCESS;
|
|
71
|
+
return null;
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
export function extractCodexSessionIdFromRolloutPath(rolloutPath) {
|
|
75
|
+
const match = ROLLOUT_UUID_RE.exec(String(rolloutPath || ""));
|
|
76
|
+
return match ? match[1].toLowerCase() : null;
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
function isRolloutPath(value, codexHome) {
|
|
80
|
+
const file = String(value || "").replaceAll("\\", "/");
|
|
81
|
+
const home = String(codexHome || "")
|
|
82
|
+
.replaceAll("\\", "/")
|
|
83
|
+
.replace(/\/$/u, "");
|
|
84
|
+
return (
|
|
85
|
+
/\/sessions\/.*\/rollout-[^/]+\.jsonl$/u.test(file) &&
|
|
86
|
+
(!home || file.startsWith(`${home}/sessions/`))
|
|
87
|
+
);
|
|
88
|
+
}
|
|
89
|
+
|
|
90
|
+
/** Best-effort only: neither failure nor a missing open rollout is exceptional. */
|
|
91
|
+
export async function resolveCodexRolloutFile({
|
|
92
|
+
pid,
|
|
93
|
+
codexHome,
|
|
94
|
+
deps = {},
|
|
95
|
+
} = {}) {
|
|
96
|
+
const numericPid = Number(pid);
|
|
97
|
+
if (!Number.isInteger(numericPid) || numericPid <= 0) return null;
|
|
98
|
+
const run = deps.execFileAsync || execFileAsync;
|
|
99
|
+
const candidates = [];
|
|
100
|
+
|
|
101
|
+
try {
|
|
102
|
+
const result = await run("lsof", ["-Fn", "-p", String(numericPid)], {
|
|
103
|
+
timeout: 5_000,
|
|
104
|
+
});
|
|
105
|
+
const output = typeof result === "string" ? result : result?.stdout;
|
|
106
|
+
for (const line of String(output || "").split(/\r?\n/u)) {
|
|
107
|
+
if (line.startsWith("n") && isRolloutPath(line.slice(1), codexHome))
|
|
108
|
+
candidates.push(line.slice(1));
|
|
109
|
+
}
|
|
110
|
+
} catch {
|
|
111
|
+
// Linux /proc fallback below; macOS simply returns null if lsof is unavailable.
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
if (candidates.length) return candidates[0];
|
|
115
|
+
const list = deps.readdir || readdir;
|
|
116
|
+
const link = deps.readlink || readlink;
|
|
117
|
+
try {
|
|
118
|
+
for (const fd of await list(`/proc/${numericPid}/fd`)) {
|
|
119
|
+
const candidate = await link(`/proc/${numericPid}/fd/${fd}`);
|
|
120
|
+
if (isRolloutPath(candidate, codexHome)) return candidate;
|
|
121
|
+
}
|
|
122
|
+
} catch {
|
|
123
|
+
// /proc is intentionally optional (macOS/Windows do not expose it).
|
|
124
|
+
}
|
|
125
|
+
return null;
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
export function buildResumeArgv({ cli, sessionId, prompt } = {}) {
|
|
129
|
+
if (!sessionId) return null;
|
|
130
|
+
if (cli === "claude") return ["claude", "-p", "--resume", sessionId, prompt];
|
|
131
|
+
if (cli === "codex") return ["codex", "exec", "resume", sessionId, prompt];
|
|
132
|
+
return null;
|
|
133
|
+
}
|
|
134
|
+
|
|
135
|
+
export async function probeResumeCapability(
|
|
136
|
+
cli,
|
|
137
|
+
{ execFileAsync: run = execFileAsync, cache = resumeCapabilityCache } = {},
|
|
138
|
+
) {
|
|
139
|
+
if (cache.has(cli)) return cache.get(cli);
|
|
140
|
+
let supported = false;
|
|
141
|
+
try {
|
|
142
|
+
if (cli === "claude") supported = true;
|
|
143
|
+
else if (cli === "codex") {
|
|
144
|
+
await run("codex", ["exec", "resume", "--help"], { timeout: 8_000 });
|
|
145
|
+
supported = true;
|
|
146
|
+
}
|
|
147
|
+
} catch {
|
|
148
|
+
supported = false;
|
|
149
|
+
}
|
|
150
|
+
cache.set(cli, supported);
|
|
151
|
+
return supported;
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
export function createFileActivitySource({
|
|
155
|
+
files = [],
|
|
156
|
+
rolloutFile = null,
|
|
157
|
+
deps = {},
|
|
158
|
+
} = {}) {
|
|
159
|
+
const stat = deps.statSync || statSync;
|
|
160
|
+
const targets = [...files, ...(rolloutFile ? [rolloutFile] : [])].filter(
|
|
161
|
+
Boolean,
|
|
162
|
+
);
|
|
163
|
+
return () =>
|
|
164
|
+
targets
|
|
165
|
+
.map((file) => {
|
|
166
|
+
try {
|
|
167
|
+
const current = stat(file);
|
|
168
|
+
return `${current.size}:${Math.floor(current.mtimeMs)}`;
|
|
169
|
+
} catch {
|
|
170
|
+
return "-";
|
|
171
|
+
}
|
|
172
|
+
})
|
|
173
|
+
.join("|");
|
|
174
|
+
}
|
|
175
|
+
|
|
176
|
+
function defaultCli(channel, target) {
|
|
177
|
+
if (target.cli) return target.cli;
|
|
178
|
+
if (channel === CHANNELS.CLAUDE_DAEMON) return "claude";
|
|
179
|
+
if (channel === CHANNELS.CODEX_APP_SERVER) return "codex";
|
|
180
|
+
return null;
|
|
181
|
+
}
|
|
182
|
+
|
|
183
|
+
function processPid(target) {
|
|
184
|
+
const pid = Number(target.pid || target.appServer?.pid);
|
|
185
|
+
return Number.isInteger(pid) && pid > 0 ? pid : null;
|
|
186
|
+
}
|
|
187
|
+
|
|
188
|
+
export function createInterventionLadder({
|
|
189
|
+
target = {},
|
|
190
|
+
reinstructPrompt = DEFAULT_REINSTRUCT_PROMPT,
|
|
191
|
+
readActivitySignature,
|
|
192
|
+
config = {},
|
|
193
|
+
log = () => {},
|
|
194
|
+
deps = {},
|
|
195
|
+
} = {}) {
|
|
196
|
+
if (typeof readActivitySignature !== "function")
|
|
197
|
+
throw new Error(
|
|
198
|
+
"createInterventionLadder requires readActivitySignature()",
|
|
199
|
+
);
|
|
200
|
+
|
|
201
|
+
const cfg = {
|
|
202
|
+
...INTERVENTION_DEFAULTS,
|
|
203
|
+
...(positiveSeconds(process.env, "TFX_INTERVENTION_REINSTRUCT_WAIT_SEC")
|
|
204
|
+
? {
|
|
205
|
+
reinstructWaitMs: positiveSeconds(
|
|
206
|
+
process.env,
|
|
207
|
+
"TFX_INTERVENTION_REINSTRUCT_WAIT_SEC",
|
|
208
|
+
),
|
|
209
|
+
}
|
|
210
|
+
: {}),
|
|
211
|
+
...(positiveSeconds(process.env, "TFX_INTERVENTION_RESUME_WAIT_SEC")
|
|
212
|
+
? {
|
|
213
|
+
resumeWaitMs: positiveSeconds(
|
|
214
|
+
process.env,
|
|
215
|
+
"TFX_INTERVENTION_RESUME_WAIT_SEC",
|
|
216
|
+
),
|
|
217
|
+
}
|
|
218
|
+
: {}),
|
|
219
|
+
...config,
|
|
220
|
+
};
|
|
221
|
+
const sleep = deps.sleep || sleepFor;
|
|
222
|
+
const alive = deps.isPidAlive || isPidAlive;
|
|
223
|
+
const kill = deps.killProcess || killProcess;
|
|
224
|
+
const state = { step: null, attempts: [] };
|
|
225
|
+
let emit = () => {};
|
|
226
|
+
|
|
227
|
+
const loadDaemonControl = async () =>
|
|
228
|
+
deps.daemonControl || (await import("./claude-daemon-control.mjs"));
|
|
229
|
+
const loadPane = async () => {
|
|
230
|
+
if (deps.psmuxExec && deps.injectPrompt)
|
|
231
|
+
return { psmuxExec: deps.psmuxExec, injectPrompt: deps.injectPrompt };
|
|
232
|
+
const [{ psmuxExec }, { injectPrompt }] = await Promise.all([
|
|
233
|
+
import("./psmux.mjs"),
|
|
234
|
+
import("./pane.mjs"),
|
|
235
|
+
]);
|
|
236
|
+
return { psmuxExec, injectPrompt };
|
|
237
|
+
};
|
|
238
|
+
const loadPsmux = async () => {
|
|
239
|
+
if (deps.psmuxExec) return deps.psmuxExec;
|
|
240
|
+
return (await import("./psmux.mjs")).psmuxExec;
|
|
241
|
+
};
|
|
242
|
+
|
|
243
|
+
async function signature() {
|
|
244
|
+
return await readActivitySignature();
|
|
245
|
+
}
|
|
246
|
+
|
|
247
|
+
async function waitForActivity(baseline, waitMs) {
|
|
248
|
+
const deadline = Date.now() + waitMs;
|
|
249
|
+
do {
|
|
250
|
+
if ((await signature()) !== baseline) return true;
|
|
251
|
+
if (Date.now() >= deadline) break;
|
|
252
|
+
await sleep(
|
|
253
|
+
Math.min(cfg.activityPollMs, Math.max(0, deadline - Date.now())),
|
|
254
|
+
);
|
|
255
|
+
} while (Date.now() < deadline);
|
|
256
|
+
return (await signature()) !== baseline;
|
|
257
|
+
}
|
|
258
|
+
|
|
259
|
+
async function record(step, action) {
|
|
260
|
+
const startedAt = Date.now();
|
|
261
|
+
emit({ step, phase: "start" });
|
|
262
|
+
let result;
|
|
263
|
+
try {
|
|
264
|
+
result = await action();
|
|
265
|
+
} catch {
|
|
266
|
+
result = { ok: false, detail: "operation_failed" };
|
|
267
|
+
}
|
|
268
|
+
const attempt = {
|
|
269
|
+
step,
|
|
270
|
+
ok: result?.ok === true,
|
|
271
|
+
detail: result?.detail || (result?.ok ? "ok" : "operation_failed"),
|
|
272
|
+
startedAt,
|
|
273
|
+
endedAt: Date.now(),
|
|
274
|
+
};
|
|
275
|
+
state.attempts.push(attempt);
|
|
276
|
+
emit({ step, phase: "end", ok: attempt.ok, detail: attempt.detail });
|
|
277
|
+
return attempt;
|
|
278
|
+
}
|
|
279
|
+
|
|
280
|
+
async function tryReinstruct(channel) {
|
|
281
|
+
return await record(STEPS.REINSTRUCT, async () => {
|
|
282
|
+
if (channel === CHANNELS.CLAUDE_DAEMON) {
|
|
283
|
+
const daemon = target.daemon || {};
|
|
284
|
+
let control;
|
|
285
|
+
try {
|
|
286
|
+
control = await loadDaemonControl();
|
|
287
|
+
} catch {
|
|
288
|
+
return { ok: false, detail: "channel_unavailable" };
|
|
289
|
+
}
|
|
290
|
+
const auth =
|
|
291
|
+
(await control.buildDaemonControlAuth?.(daemon.configDir)) || {};
|
|
292
|
+
const interrupted = await control.interruptClaudeDaemonSession({
|
|
293
|
+
controlSock: daemon.controlSock,
|
|
294
|
+
short: daemon.short,
|
|
295
|
+
...auth,
|
|
296
|
+
});
|
|
297
|
+
if (interrupted?.inputSent !== true)
|
|
298
|
+
return { ok: false, detail: "interrupt_not_delivered" };
|
|
299
|
+
await sleep(cfg.interruptSettleMs);
|
|
300
|
+
const attached = await control.attachClaudeDaemonSession({
|
|
301
|
+
controlSock: daemon.controlSock,
|
|
302
|
+
short: daemon.short,
|
|
303
|
+
input: reinstructPrompt,
|
|
304
|
+
...auth,
|
|
305
|
+
timeoutMs: cfg.reinstructInjectTimeoutMs,
|
|
306
|
+
completionQuiescenceMs: 0,
|
|
307
|
+
});
|
|
308
|
+
return attached?.inputSent === true
|
|
309
|
+
? { ok: true, detail: "injected" }
|
|
310
|
+
: { ok: false, detail: "inject_failed" };
|
|
311
|
+
}
|
|
312
|
+
if (channel === CHANNELS.TMUX_PANE) {
|
|
313
|
+
if (target.interactive !== true)
|
|
314
|
+
return { ok: false, detail: "headless_pane_no_input_channel" };
|
|
315
|
+
let pane;
|
|
316
|
+
try {
|
|
317
|
+
pane = await loadPane();
|
|
318
|
+
} catch {
|
|
319
|
+
return { ok: false, detail: "channel_unavailable" };
|
|
320
|
+
}
|
|
321
|
+
await pane.psmuxExec([
|
|
322
|
+
"send-keys",
|
|
323
|
+
"-t",
|
|
324
|
+
target.paneId,
|
|
325
|
+
target.cli === "claude" ? "Escape" : "C-c",
|
|
326
|
+
]);
|
|
327
|
+
await sleep(cfg.interruptSettleMs);
|
|
328
|
+
await pane.injectPrompt(target.paneId, reinstructPrompt, {
|
|
329
|
+
cli: target.cli || null,
|
|
330
|
+
});
|
|
331
|
+
return { ok: true, detail: "injected" };
|
|
332
|
+
}
|
|
333
|
+
if (channel === CHANNELS.CODEX_APP_SERVER) {
|
|
334
|
+
const { client, threadId } = target.appServer || {};
|
|
335
|
+
await client.request("turn/interrupt", { threadId }, 10_000);
|
|
336
|
+
await client.request(
|
|
337
|
+
"turn/start",
|
|
338
|
+
{ threadId, input: [{ type: "text", text: reinstructPrompt }] },
|
|
339
|
+
30_000,
|
|
340
|
+
);
|
|
341
|
+
return { ok: true, detail: "injected" };
|
|
342
|
+
}
|
|
343
|
+
return { ok: false, detail: "channel_unsupported" };
|
|
344
|
+
});
|
|
345
|
+
}
|
|
346
|
+
|
|
347
|
+
async function resolveSessionId(channel, cli) {
|
|
348
|
+
if (target.sessionId) return target.sessionId;
|
|
349
|
+
if (target.daemon?.sessionId) return target.daemon.sessionId;
|
|
350
|
+
if (channel === CHANNELS.CODEX_APP_SERVER)
|
|
351
|
+
return target.appServer?.threadId || null;
|
|
352
|
+
const direct = extractCodexSessionIdFromRolloutPath(target.rolloutFile);
|
|
353
|
+
if (direct) return direct;
|
|
354
|
+
if (cli !== "codex") return null;
|
|
355
|
+
const resolveRollout =
|
|
356
|
+
deps.resolveCodexRolloutFile || resolveCodexRolloutFile;
|
|
357
|
+
const rolloutFile = await resolveRollout({
|
|
358
|
+
pid: processPid(target),
|
|
359
|
+
codexHome: target.codexHome,
|
|
360
|
+
deps,
|
|
361
|
+
});
|
|
362
|
+
return extractCodexSessionIdFromRolloutPath(rolloutFile);
|
|
363
|
+
}
|
|
364
|
+
|
|
365
|
+
async function terminate(
|
|
366
|
+
channel,
|
|
367
|
+
{ force = false, recordAttempt = true } = {},
|
|
368
|
+
) {
|
|
369
|
+
const step = force ? STEPS.SIGKILL : STEPS.SIGTERM;
|
|
370
|
+
const signal = force ? "SIGKILL" : "SIGTERM";
|
|
371
|
+
const action = async () => {
|
|
372
|
+
if (channel === CHANNELS.CLAUDE_DAEMON) {
|
|
373
|
+
const daemon = target.daemon || {};
|
|
374
|
+
let control;
|
|
375
|
+
try {
|
|
376
|
+
control = await loadDaemonControl();
|
|
377
|
+
} catch {
|
|
378
|
+
return { ok: false, detail: "channel_unavailable" };
|
|
379
|
+
}
|
|
380
|
+
const auth =
|
|
381
|
+
(await control.buildDaemonControlAuth?.(daemon.configDir)) || {};
|
|
382
|
+
const sessionId = target.sessionId || daemon.sessionId;
|
|
383
|
+
const result = sessionId
|
|
384
|
+
? await control.sendKillBySessionId({
|
|
385
|
+
daemonPaths: {
|
|
386
|
+
controlSock: daemon.controlSock,
|
|
387
|
+
configDir: daemon.configDir,
|
|
388
|
+
},
|
|
389
|
+
sessionId,
|
|
390
|
+
...auth,
|
|
391
|
+
})
|
|
392
|
+
: await control.killDaemonJob?.(
|
|
393
|
+
daemon.controlSock,
|
|
394
|
+
daemon.short,
|
|
395
|
+
auth,
|
|
396
|
+
);
|
|
397
|
+
return result?.ok === true
|
|
398
|
+
? { ok: true, detail: "daemon_killed" }
|
|
399
|
+
: { ok: false, detail: "daemon_kill_failed" };
|
|
400
|
+
}
|
|
401
|
+
|
|
402
|
+
if (channel === CHANNELS.TMUX_PANE) {
|
|
403
|
+
let psmuxExec;
|
|
404
|
+
try {
|
|
405
|
+
psmuxExec = await loadPsmux();
|
|
406
|
+
} catch {
|
|
407
|
+
return { ok: false, detail: "channel_unavailable" };
|
|
408
|
+
}
|
|
409
|
+
const output = await psmuxExec([
|
|
410
|
+
"list-panes",
|
|
411
|
+
"-t",
|
|
412
|
+
target.paneId,
|
|
413
|
+
"-F",
|
|
414
|
+
"#{pane_pid}",
|
|
415
|
+
]);
|
|
416
|
+
const panePid = Number(
|
|
417
|
+
String(output || "")
|
|
418
|
+
.trim()
|
|
419
|
+
.split(/\s+/u)[0],
|
|
420
|
+
);
|
|
421
|
+
if (!Number.isInteger(panePid) || panePid <= 0) {
|
|
422
|
+
if (!force) return { ok: false, detail: "pane_pid_unavailable" };
|
|
423
|
+
await psmuxExec(["kill-pane", "-t", target.paneId]);
|
|
424
|
+
return { ok: true, detail: "pane_killed" };
|
|
425
|
+
}
|
|
426
|
+
const sent = kill(panePid, {
|
|
427
|
+
signal,
|
|
428
|
+
tree: true,
|
|
429
|
+
...(force ? { force: true } : {}),
|
|
430
|
+
});
|
|
431
|
+
if (!sent) return { ok: false, detail: "signal_failed" };
|
|
432
|
+
await sleep(
|
|
433
|
+
force ? Math.min(cfg.sigtermGraceMs, 250) : cfg.sigtermGraceMs,
|
|
434
|
+
);
|
|
435
|
+
if (!alive(panePid)) return { ok: true, detail: "terminated" };
|
|
436
|
+
if (force) await psmuxExec(["kill-pane", "-t", target.paneId]);
|
|
437
|
+
return { ok: false, detail: "still_alive" };
|
|
438
|
+
}
|
|
439
|
+
|
|
440
|
+
const pid = processPid(target);
|
|
441
|
+
if (!pid) return { ok: false, detail: "pid_unavailable" };
|
|
442
|
+
const sent = kill(pid, {
|
|
443
|
+
signal,
|
|
444
|
+
tree: true,
|
|
445
|
+
...(force ? { force: true } : {}),
|
|
446
|
+
});
|
|
447
|
+
if (!sent) return { ok: false, detail: "signal_failed" };
|
|
448
|
+
await sleep(
|
|
449
|
+
force ? Math.min(cfg.sigtermGraceMs, 250) : cfg.sigtermGraceMs,
|
|
450
|
+
);
|
|
451
|
+
return !alive(pid)
|
|
452
|
+
? { ok: true, detail: "terminated" }
|
|
453
|
+
: { ok: false, detail: "still_alive" };
|
|
454
|
+
};
|
|
455
|
+
return recordAttempt ? await record(step, action) : await action();
|
|
456
|
+
}
|
|
457
|
+
|
|
458
|
+
async function tryResume(channel) {
|
|
459
|
+
return await record(STEPS.RESUME, async () => {
|
|
460
|
+
const cli = defaultCli(channel, target);
|
|
461
|
+
const probe = deps.probeResumeCapability || probeResumeCapability;
|
|
462
|
+
const supported = await probe(cli, { execFileAsync: deps.execFileAsync });
|
|
463
|
+
if (!supported) return { ok: false, detail: "capability_unsupported" };
|
|
464
|
+
const sessionId = await resolveSessionId(channel, cli);
|
|
465
|
+
if (!sessionId) return { ok: false, detail: "session_id_unavailable" };
|
|
466
|
+
if (typeof deps.resumeHandler !== "function")
|
|
467
|
+
return { ok: false, detail: "no_resume_handler" };
|
|
468
|
+
|
|
469
|
+
// A resume handler owns relaunching; only stop after all prerequisites exist.
|
|
470
|
+
const stopped = await terminate(channel, { recordAttempt: false });
|
|
471
|
+
if (!stopped.ok) return { ok: false, detail: "graceful_stop_failed" };
|
|
472
|
+
const resumed = await deps.resumeHandler({
|
|
473
|
+
cli,
|
|
474
|
+
sessionId,
|
|
475
|
+
prompt: reinstructPrompt,
|
|
476
|
+
target,
|
|
477
|
+
argv: buildResumeArgv({ cli, sessionId, prompt: reinstructPrompt }),
|
|
478
|
+
});
|
|
479
|
+
return resumed?.ok === false
|
|
480
|
+
? { ok: false, detail: "resume_dispatch_failed" }
|
|
481
|
+
: { ok: true, detail: "resume_dispatched" };
|
|
482
|
+
});
|
|
483
|
+
}
|
|
484
|
+
|
|
485
|
+
async function intervene() {
|
|
486
|
+
state.step = null;
|
|
487
|
+
state.attempts.length = 0;
|
|
488
|
+
const channel = detectChannel(target);
|
|
489
|
+
emit = (event) => {
|
|
490
|
+
try {
|
|
491
|
+
log({ ts: Date.now(), channel, ...event });
|
|
492
|
+
} catch {
|
|
493
|
+
// Logging must never change recovery behaviour.
|
|
494
|
+
}
|
|
495
|
+
};
|
|
496
|
+
|
|
497
|
+
const finish = (outcome, step) => {
|
|
498
|
+
state.step = step;
|
|
499
|
+
return {
|
|
500
|
+
ok: outcome !== "failed",
|
|
501
|
+
outcome,
|
|
502
|
+
step,
|
|
503
|
+
channel,
|
|
504
|
+
attempts: [...state.attempts],
|
|
505
|
+
};
|
|
506
|
+
};
|
|
507
|
+
|
|
508
|
+
if (channel && channel !== CHANNELS.PROCESS) {
|
|
509
|
+
const baseline = await signature();
|
|
510
|
+
const attempt = await tryReinstruct(channel);
|
|
511
|
+
if (attempt.ok && (await waitForActivity(baseline, cfg.reinstructWaitMs)))
|
|
512
|
+
return finish("reactivated", STEPS.REINSTRUCT);
|
|
513
|
+
}
|
|
514
|
+
|
|
515
|
+
const resumeBaseline = await signature();
|
|
516
|
+
const resumed = await tryResume(channel);
|
|
517
|
+
if (resumed.ok && (await waitForActivity(resumeBaseline, cfg.resumeWaitMs)))
|
|
518
|
+
return finish("resumed", STEPS.RESUME);
|
|
519
|
+
|
|
520
|
+
const terminated = await terminate(channel);
|
|
521
|
+
if (terminated.ok) return finish("terminated", STEPS.SIGTERM);
|
|
522
|
+
const killed = await terminate(channel, { force: true });
|
|
523
|
+
return finish(killed.ok ? "killed" : "failed", STEPS.SIGKILL);
|
|
524
|
+
}
|
|
525
|
+
|
|
526
|
+
return Object.freeze({
|
|
527
|
+
intervene,
|
|
528
|
+
getState: () => ({ step: state.step, attempts: [...state.attempts] }),
|
|
529
|
+
});
|
|
530
|
+
}
|
|
@@ -41,10 +41,14 @@ export const MODES = Object.freeze({
|
|
|
41
41
|
const ESCALATION_CHAIN_CONFIG_PATH = ".triflux/config/escalation-chain.json";
|
|
42
42
|
|
|
43
43
|
// Escalation chain (2026-05-27 정책):
|
|
44
|
-
// 1. codex gpt-5.
|
|
44
|
+
// 1. codex gpt-5.6-sol — 기존 최상위 Codex escalation 단계
|
|
45
45
|
// 2. claude opus-4-8 — 최종 수단
|
|
46
46
|
const DEFAULT_ESCALATION_CHAIN = Object.freeze([
|
|
47
|
-
Object.freeze({
|
|
47
|
+
Object.freeze({
|
|
48
|
+
cli: "codex",
|
|
49
|
+
model: "gpt-5.6-sol",
|
|
50
|
+
profile: "gpt56_sol_xhigh",
|
|
51
|
+
}),
|
|
48
52
|
Object.freeze({ cli: "claude", model: "opus-4-8" }),
|
|
49
53
|
]);
|
|
50
54
|
|
|
@@ -20,6 +20,7 @@ import {
|
|
|
20
20
|
isDynamicRoutingEnabled,
|
|
21
21
|
} from "@triflux/core/hub/dynamic-routing-engine.mjs";
|
|
22
22
|
import { cleanupShardProcesses } from "@triflux/core/hub/lib/process-utils.mjs";
|
|
23
|
+
import { DEFAULT_SWARM_LOCK_TTL_MS } from "@triflux/core/hub/lib/timeout-defaults.mjs";
|
|
23
24
|
import { getHostConfig } from "@triflux/core/hub/lib/ssh-command.mjs";
|
|
24
25
|
import { buildWorkerPrompt } from "./build-worker-prompt.mjs";
|
|
25
26
|
import { registerSwarmShard } from "./claude-native-bridge.mjs";
|
|
@@ -189,7 +190,6 @@ function isInteractiveNativeBridgeShard(shard) {
|
|
|
189
190
|
* @param {number} [opts.maxRestarts=2] — per-shard max restarts
|
|
190
191
|
* @param {boolean} [opts.nativeBridge=false] — expose local shards in Claude agents
|
|
191
192
|
* @param {number} [opts.graceMs=10000] — conductor shutdown grace period
|
|
192
|
-
* @param {number} [opts.integrationTimeoutMs=60000] — max time for integration phase
|
|
193
193
|
* @param {boolean} [opts.keepFailedWorktrees=false] — preserve failed shard worktrees for debugging
|
|
194
194
|
* @param {object} [opts.probeOpts] — health probe overrides
|
|
195
195
|
* @param {object} [opts.deps] — dependency injection for testing
|
|
@@ -205,7 +205,6 @@ export function createSwarmHypervisor(opts) {
|
|
|
205
205
|
nativeBridge = false,
|
|
206
206
|
graceMs = 10_000,
|
|
207
207
|
keepFailedWorktrees = false,
|
|
208
|
-
_integrationTimeoutMs = 60_000,
|
|
209
208
|
probeOpts = {},
|
|
210
209
|
_deps = {},
|
|
211
210
|
} = opts;
|
|
@@ -1911,6 +1910,7 @@ export function createSwarmHypervisor(opts) {
|
|
|
1911
1910
|
*/
|
|
1912
1911
|
/** Hub keepalive — Hub crash recovery (idle timeout은 기본 비활성이므로 crash 복구 용도) */
|
|
1913
1912
|
let hubKeepaliveTimer = null;
|
|
1913
|
+
let lockRenewTimer = null;
|
|
1914
1914
|
function startHubKeepalive() {
|
|
1915
1915
|
// 5분마다 Hub /status 핑 — crash 감지 시 ensureHubAlive로 재시작
|
|
1916
1916
|
hubKeepaliveTimer = setInterval(
|
|
@@ -1960,6 +1960,21 @@ export function createSwarmHypervisor(opts) {
|
|
|
1960
1960
|
}
|
|
1961
1961
|
}
|
|
1962
1962
|
|
|
1963
|
+
function startLockRenewal() {
|
|
1964
|
+
lockRenewTimer = setInterval(() => {
|
|
1965
|
+
if (!lockManager) return;
|
|
1966
|
+
for (const shardName of workers.keys()) {
|
|
1967
|
+
if (!completedShards.has(shardName) && !failures.has(shardName)) lockManager.renew(shardName);
|
|
1968
|
+
}
|
|
1969
|
+
}, DEFAULT_SWARM_LOCK_TTL_MS / 2);
|
|
1970
|
+
lockRenewTimer.unref?.();
|
|
1971
|
+
}
|
|
1972
|
+
|
|
1973
|
+
function stopLockRenewal() {
|
|
1974
|
+
if (lockRenewTimer) clearInterval(lockRenewTimer);
|
|
1975
|
+
lockRenewTimer = null;
|
|
1976
|
+
}
|
|
1977
|
+
|
|
1963
1978
|
async function launch(swarmPlan) {
|
|
1964
1979
|
if (state !== SWARM_STATES.PLANNING) {
|
|
1965
1980
|
throw new Error(`Cannot launch in state "${state}"`);
|
|
@@ -2053,6 +2068,7 @@ export function createSwarmHypervisor(opts) {
|
|
|
2053
2068
|
persistPath: join(workdir, ".triflux", "swarm-locks.json"),
|
|
2054
2069
|
});
|
|
2055
2070
|
|
|
2071
|
+
startLockRenewal();
|
|
2056
2072
|
// Hub keepalive 시작
|
|
2057
2073
|
startHubKeepalive();
|
|
2058
2074
|
|
|
@@ -2108,6 +2124,7 @@ export function createSwarmHypervisor(opts) {
|
|
|
2108
2124
|
*/
|
|
2109
2125
|
async function shutdown(reason = "shutdown") {
|
|
2110
2126
|
stopHubKeepalive();
|
|
2127
|
+
stopLockRenewal();
|
|
2111
2128
|
eventLog.append("swarm_shutdown", { reason, state });
|
|
2112
2129
|
|
|
2113
2130
|
const shutdowns = [];
|
package/hub/team/swarm-locks.mjs
CHANGED
|
@@ -5,8 +5,9 @@
|
|
|
5
5
|
|
|
6
6
|
import { existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
|
|
7
7
|
import { dirname, relative, resolve } from "node:path";
|
|
8
|
+
import { DEFAULT_SWARM_LOCK_TTL_MS } from "@triflux/core/hub/lib/timeout-defaults.mjs";
|
|
8
9
|
|
|
9
|
-
const LOCK_TTL_MS =
|
|
10
|
+
const LOCK_TTL_MS = DEFAULT_SWARM_LOCK_TTL_MS;
|
|
10
11
|
|
|
11
12
|
// Distribution-critical paths. A swarm worker that modifies these without
|
|
12
13
|
// having them explicitly listed in its shard lease is almost certainly a
|
|
@@ -208,6 +209,20 @@ export function createSwarmLocks(opts = {}) {
|
|
|
208
209
|
return count;
|
|
209
210
|
}
|
|
210
211
|
|
|
212
|
+
function renew(workerId) {
|
|
213
|
+
pruneExpired();
|
|
214
|
+
const ts = now();
|
|
215
|
+
let count = 0;
|
|
216
|
+
for (const entry of locks.values()) {
|
|
217
|
+
if (entry.workerId === workerId) {
|
|
218
|
+
entry.acquiredAt = ts;
|
|
219
|
+
count += 1;
|
|
220
|
+
}
|
|
221
|
+
}
|
|
222
|
+
if (count > 0) persist();
|
|
223
|
+
return count;
|
|
224
|
+
}
|
|
225
|
+
|
|
211
226
|
/**
|
|
212
227
|
* Check if a file write would violate any lease.
|
|
213
228
|
* @param {string} workerId — the worker attempting the write
|
|
@@ -308,6 +323,7 @@ export function createSwarmLocks(opts = {}) {
|
|
|
308
323
|
return Object.freeze({
|
|
309
324
|
acquire,
|
|
310
325
|
release,
|
|
326
|
+
renew,
|
|
311
327
|
check,
|
|
312
328
|
validateChanges,
|
|
313
329
|
snapshot,
|