@triflux/remote 10.30.1 → 10.32.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/hub/server.mjs +34 -1
- package/hub/team/cto-auto-collect.mjs +109 -0
- package/hub/team/synapse-registry.mjs +83 -1
- package/package.json +1 -1
package/hub/server.mjs
CHANGED
|
@@ -56,6 +56,7 @@ import { createStoreAdapter } from "./store-adapter.mjs";
|
|
|
56
56
|
import { createGitPreflight } from "./team/git-preflight.mjs";
|
|
57
57
|
import { nativeProxy } from "./team/nativeProxy.mjs";
|
|
58
58
|
import { createSwarmLocks } from "./team/swarm-locks.mjs";
|
|
59
|
+
import { createCtoAutoCollector } from "./team/cto-auto-collect.mjs";
|
|
59
60
|
import {
|
|
60
61
|
createSynapseRegistry,
|
|
61
62
|
projectPeer,
|
|
@@ -69,6 +70,21 @@ registerTeamBridge(nativeProxy);
|
|
|
69
70
|
const hubLog = createModuleLogger("hub");
|
|
70
71
|
const brokerDiagnosticsAttached = new WeakSet();
|
|
71
72
|
|
|
73
|
+
async function runCtoCollect(args, opts) {
|
|
74
|
+
try {
|
|
75
|
+
const mod = await import("triflux/cto/collect.mjs");
|
|
76
|
+
if (typeof mod?.runCollect === "function") {
|
|
77
|
+
return await mod.runCollect(args, opts);
|
|
78
|
+
}
|
|
79
|
+
} catch (error) {
|
|
80
|
+
hubLog.warn(
|
|
81
|
+
{ err: String(error?.message || error) },
|
|
82
|
+
"cto.auto_collect.remote_collect_unavailable",
|
|
83
|
+
);
|
|
84
|
+
}
|
|
85
|
+
return null;
|
|
86
|
+
}
|
|
87
|
+
|
|
72
88
|
const MAX_BODY_SIZE = 1024 * 1024;
|
|
73
89
|
const PUBLIC_PATHS = new Set(["/", "/status", "/health", "/healthz"]);
|
|
74
90
|
const RATE_LIMIT_MAX = 100; // requests per window
|
|
@@ -1009,10 +1025,16 @@ export async function startHub({
|
|
|
1009
1025
|
registry: synapseRegistry,
|
|
1010
1026
|
locks: swarmLocks,
|
|
1011
1027
|
});
|
|
1028
|
+
const ctoAutoCollector = createCtoAutoCollector({
|
|
1029
|
+
registry: synapseRegistry,
|
|
1030
|
+
runCollect: runCtoCollect,
|
|
1031
|
+
logger: hubLog,
|
|
1032
|
+
});
|
|
1012
1033
|
|
|
1013
1034
|
// Synapse Layer 5: emitter subscribers — bridge events to hub logging
|
|
1014
|
-
synapseEmitter.on("synapse.session.started", ({ sessionId }) => {
|
|
1035
|
+
synapseEmitter.on("synapse.session.started", ({ sessionId, session }) => {
|
|
1015
1036
|
hubLog.info({ sessionId }, "synapse.session.started");
|
|
1037
|
+
ctoAutoCollector.handleSessionStarted({ sessionId, session });
|
|
1016
1038
|
});
|
|
1017
1039
|
synapseEmitter.on("synapse.session.heartbeat", ({ sessionId }) => {
|
|
1018
1040
|
hubLog.debug({ sessionId }, "synapse.session.heartbeat");
|
|
@@ -2054,6 +2076,16 @@ export async function startHub({
|
|
|
2054
2076
|
}, 60000);
|
|
2055
2077
|
sessionTimer.unref();
|
|
2056
2078
|
|
|
2079
|
+
const synapsePruneTimer = setInterval(() => {
|
|
2080
|
+
try {
|
|
2081
|
+
const { count } = synapseRegistry.pruneExpired();
|
|
2082
|
+
if (count > 0) hubLog.info({ count }, "synapse.prune");
|
|
2083
|
+
} catch (err) {
|
|
2084
|
+
hubLog.warn({ err }, "synapse.prune_failed");
|
|
2085
|
+
}
|
|
2086
|
+
}, 60000);
|
|
2087
|
+
synapsePruneTimer.unref();
|
|
2088
|
+
|
|
2057
2089
|
// 고아 node.exe 프로세스 + stale spawn 세션 주기적 정리 (5분마다)
|
|
2058
2090
|
// TFX_DISABLE_ORPHAN_CLEANUP=1 로 비활성 (active SSH/swarm 세션 보호 회피용 hotfix gate)
|
|
2059
2091
|
const orphanCleanupTimer = setInterval(
|
|
@@ -2288,6 +2320,7 @@ export async function startHub({
|
|
|
2288
2320
|
router.stopSweeper();
|
|
2289
2321
|
clearInterval(hitlTimer);
|
|
2290
2322
|
clearInterval(sessionTimer);
|
|
2323
|
+
clearInterval(synapsePruneTimer);
|
|
2291
2324
|
clearInterval(rateLimitTimer);
|
|
2292
2325
|
clearInterval(orphanCleanupTimer);
|
|
2293
2326
|
if (idleTimer) {
|
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
import { existsSync, statSync } from "node:fs";
|
|
2
|
+
import { join } from "node:path";
|
|
3
|
+
|
|
4
|
+
const DEFAULT_DEBOUNCE_MS = 120_000;
|
|
5
|
+
const OFF_VALUES = new Set(["0", "false", "off", "no"]);
|
|
6
|
+
|
|
7
|
+
export function isCtoAutoCollectDisabled(env = process.env) {
|
|
8
|
+
return OFF_VALUES.has(
|
|
9
|
+
String(env?.TFX_CTO_AUTO_COLLECT ?? "")
|
|
10
|
+
.trim()
|
|
11
|
+
.toLowerCase(),
|
|
12
|
+
);
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
function freshCurrentMd(projectRoot, now, debounceMs) {
|
|
16
|
+
try {
|
|
17
|
+
const currentMd = join(projectRoot, ".triflux", "lake", "current.md");
|
|
18
|
+
if (!existsSync(currentMd)) return false;
|
|
19
|
+
return now - statSync(currentMd).mtimeMs < debounceMs;
|
|
20
|
+
} catch {
|
|
21
|
+
return false;
|
|
22
|
+
}
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
export function createCtoAutoCollector(opts = {}) {
|
|
26
|
+
const registry = opts.registry;
|
|
27
|
+
const runCollect = opts.runCollect;
|
|
28
|
+
const logger = opts.logger || null;
|
|
29
|
+
const debounceMs = Number(opts.debounceMs || DEFAULT_DEBOUNCE_MS);
|
|
30
|
+
const now = opts.now || (() => Date.now());
|
|
31
|
+
const env = opts.env || process.env;
|
|
32
|
+
const lastCollectMs = opts.lastCollectMs || new Map();
|
|
33
|
+
const inFlight = new Set();
|
|
34
|
+
|
|
35
|
+
function handleSessionStarted({ sessionId, session } = {}) {
|
|
36
|
+
if (isCtoAutoCollectDisabled(env))
|
|
37
|
+
return { triggered: false, reason: "disabled" };
|
|
38
|
+
if (!registry || typeof registry.querySessions !== "function") {
|
|
39
|
+
return { triggered: false, reason: "missing-registry" };
|
|
40
|
+
}
|
|
41
|
+
if (typeof runCollect !== "function") {
|
|
42
|
+
return { triggered: false, reason: "missing-runCollect" };
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
const cwd = typeof session?.cwd === "string" ? session.cwd : "";
|
|
46
|
+
const worktree =
|
|
47
|
+
typeof session?.worktreePath === "string" ? session.worktreePath : "";
|
|
48
|
+
const projectRoot = worktree || cwd;
|
|
49
|
+
if (!projectRoot)
|
|
50
|
+
return { triggered: false, reason: "missing-project-root" };
|
|
51
|
+
|
|
52
|
+
let peers = [];
|
|
53
|
+
try {
|
|
54
|
+
peers = registry.querySessions({
|
|
55
|
+
cwd,
|
|
56
|
+
worktree,
|
|
57
|
+
excludeSessionId: sessionId,
|
|
58
|
+
});
|
|
59
|
+
} catch (error) {
|
|
60
|
+
logger?.warn?.(
|
|
61
|
+
{ sessionId, err: String(error?.message || error) },
|
|
62
|
+
"cto.auto_collect.query_failed",
|
|
63
|
+
);
|
|
64
|
+
return { triggered: false, reason: "query-failed" };
|
|
65
|
+
}
|
|
66
|
+
if (!Array.isArray(peers) || peers.length < 1) {
|
|
67
|
+
return { triggered: false, reason: "no-peers" };
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
const currentTime = now();
|
|
71
|
+
const last = lastCollectMs.get(projectRoot) || 0;
|
|
72
|
+
if (currentTime - last < debounceMs) {
|
|
73
|
+
return { triggered: false, reason: "debounced" };
|
|
74
|
+
}
|
|
75
|
+
if (freshCurrentMd(projectRoot, currentTime, debounceMs)) {
|
|
76
|
+
lastCollectMs.set(projectRoot, currentTime);
|
|
77
|
+
return { triggered: false, reason: "fresh-lake" };
|
|
78
|
+
}
|
|
79
|
+
|
|
80
|
+
lastCollectMs.set(projectRoot, currentTime);
|
|
81
|
+
const promise = Promise.resolve()
|
|
82
|
+
.then(() => runCollect([], { rootDir: projectRoot }))
|
|
83
|
+
.catch((error) => {
|
|
84
|
+
logger?.warn?.(
|
|
85
|
+
{ projectRoot, err: String(error?.message || error) },
|
|
86
|
+
"cto.auto_collect.failed",
|
|
87
|
+
);
|
|
88
|
+
})
|
|
89
|
+
.finally(() => {
|
|
90
|
+
inFlight.delete(promise);
|
|
91
|
+
});
|
|
92
|
+
inFlight.add(promise);
|
|
93
|
+
return {
|
|
94
|
+
triggered: true,
|
|
95
|
+
reason: "triggered",
|
|
96
|
+
projectRoot,
|
|
97
|
+
peerCount: peers.length,
|
|
98
|
+
};
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
async function drain() {
|
|
102
|
+
await Promise.allSettled([...inFlight]);
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
return Object.freeze({
|
|
106
|
+
handleSessionStarted,
|
|
107
|
+
drain,
|
|
108
|
+
});
|
|
109
|
+
}
|
|
@@ -5,6 +5,7 @@ const DEFAULT_LOCAL_HEARTBEAT_INTERVAL_MS = 5_000;
|
|
|
5
5
|
const DEFAULT_LOCAL_TIMEOUT_MS = 30_000;
|
|
6
6
|
const DEFAULT_REMOTE_HEARTBEAT_INTERVAL_MS = 15_000;
|
|
7
7
|
const DEFAULT_REMOTE_TIMEOUT_MS = 90_000;
|
|
8
|
+
const DEFAULT_EXPIRE_TIMEOUT_MS = 24 * 60 * 60 * 1000;
|
|
8
9
|
// Interactive (Claude/Codex) sessions idle for long stretches; a 30s local
|
|
9
10
|
// timeout produces stale false-positives. 5-minute TTL + an `idle` state
|
|
10
11
|
// distinguishes "alive but inactive" from "presumed dead".
|
|
@@ -133,6 +134,7 @@ export function createSynapseRegistry(opts = {}) {
|
|
|
133
134
|
remoteTimeoutMs = DEFAULT_REMOTE_TIMEOUT_MS,
|
|
134
135
|
interactiveHeartbeatIntervalMs = DEFAULT_INTERACTIVE_HEARTBEAT_INTERVAL_MS,
|
|
135
136
|
interactiveTimeoutMs = DEFAULT_INTERACTIVE_TIMEOUT_MS,
|
|
137
|
+
expireTimeoutMs = DEFAULT_EXPIRE_TIMEOUT_MS,
|
|
136
138
|
} = opts;
|
|
137
139
|
|
|
138
140
|
const sessions = new Map();
|
|
@@ -355,10 +357,89 @@ export function createSynapseRegistry(opts = {}) {
|
|
|
355
357
|
return true;
|
|
356
358
|
}
|
|
357
359
|
|
|
360
|
+
// stale/expired 세션이 expireTimeoutMs 넘게 누적되면 Map에서 제거.
|
|
361
|
+
// live(active/idle)는 lastHeartbeat가 오래돼도 보존 — git-preflight dirty-file 가드가 의존.
|
|
362
|
+
function pruneExpired(opts2 = {}) {
|
|
363
|
+
if (destroyed) return { removed: [], count: 0 };
|
|
364
|
+
|
|
365
|
+
const cutoff =
|
|
366
|
+
typeof opts2.olderThanMs === "number"
|
|
367
|
+
? opts2.olderThanMs
|
|
368
|
+
: expireTimeoutMs;
|
|
369
|
+
const removed = [];
|
|
370
|
+
const currentTime = now();
|
|
371
|
+
|
|
372
|
+
for (const [sessionId, session] of sessions) {
|
|
373
|
+
if (
|
|
374
|
+
isLiveStatus(session.status) ||
|
|
375
|
+
currentTime - session.lastHeartbeat <= cutoff
|
|
376
|
+
) {
|
|
377
|
+
continue;
|
|
378
|
+
}
|
|
379
|
+
|
|
380
|
+
stopMonitor(sessionId);
|
|
381
|
+
sessions.delete(sessionId);
|
|
382
|
+
removed.push(sessionId);
|
|
383
|
+
notifyRemoved(session);
|
|
384
|
+
}
|
|
385
|
+
|
|
386
|
+
if (removed.length > 0) persist();
|
|
387
|
+
return { removed, count: removed.length };
|
|
388
|
+
}
|
|
389
|
+
|
|
358
390
|
function heartbeat(sessionId, partialMeta = null) {
|
|
359
391
|
const normalized = normalizeSessionId(sessionId);
|
|
360
392
|
const session = sessions.get(normalized);
|
|
361
|
-
if (!session)
|
|
393
|
+
if (!session) {
|
|
394
|
+
// self-heal은 위치 정보(worktreePath 또는 cwd)가 있는 heartbeat에서만
|
|
395
|
+
// 발동한다. UserPromptSubmit heartbeat는 항상 worktreePath를 보내므로
|
|
396
|
+
// SessionStart register POST drop 복구 경로는 커버되고, taskSummary만 담은
|
|
397
|
+
// 빈약한 heartbeat로 위치 없는 가비지 세션이 register 되는 것은 막는다
|
|
398
|
+
// (그 경우는 기존대로 false → 400/heartbeat failed).
|
|
399
|
+
const hasLocator =
|
|
400
|
+
partialMeta &&
|
|
401
|
+
typeof partialMeta === "object" &&
|
|
402
|
+
!Array.isArray(partialMeta) &&
|
|
403
|
+
((typeof partialMeta.worktreePath === "string" &&
|
|
404
|
+
partialMeta.worktreePath.length > 0) ||
|
|
405
|
+
(typeof partialMeta.cwd === "string" && partialMeta.cwd.length > 0));
|
|
406
|
+
const canSelfHeal = Boolean(normalized) && hasLocator;
|
|
407
|
+
if (!canSelfHeal) return false;
|
|
408
|
+
|
|
409
|
+
// 미등록 세션 heartbeat = SessionStart register POST가 drop된 경우.
|
|
410
|
+
// self-heal로 register 복구(영구 미등록 방지).
|
|
411
|
+
//
|
|
412
|
+
// sessionKind는 명시값이 없으면 "interactive"로 본다. 실제 복구 경로인
|
|
413
|
+
// heartbeatInteractiveSession은 worktreePath/host/taskSummary만 보내고
|
|
414
|
+
// sessionKind는 생략하는데, sanitizeSession 기본값("headless")으로 떨어지면
|
|
415
|
+
// 복구된 Claude 세션이 30초 local timeout을 받아 idle/5분 TTL을 잃고 곧장
|
|
416
|
+
// stale로 전이돼 getActive()/peer discovery에서 사라진다(이 PRD가 고치려던
|
|
417
|
+
// 원래 증상의 재현). heartbeat 기반 liveness로 살아나는 세션은 interactive다.
|
|
418
|
+
const sanitizedKind =
|
|
419
|
+
typeof partialMeta.sessionKind === "string"
|
|
420
|
+
? partialMeta.sessionKind
|
|
421
|
+
: "interactive";
|
|
422
|
+
const healed = sanitizeSession(
|
|
423
|
+
{
|
|
424
|
+
...partialMeta,
|
|
425
|
+
sessionId: normalized,
|
|
426
|
+
sessionKind: sanitizedKind,
|
|
427
|
+
status: "active",
|
|
428
|
+
lastHeartbeat: now(),
|
|
429
|
+
},
|
|
430
|
+
normalized,
|
|
431
|
+
);
|
|
432
|
+
if (!healed) return false;
|
|
433
|
+
|
|
434
|
+
sessions.set(normalized, healed);
|
|
435
|
+
startMonitor(normalized);
|
|
436
|
+
persist();
|
|
437
|
+
emitter?.emit("synapse.session.started", {
|
|
438
|
+
sessionId: normalized,
|
|
439
|
+
session: cloneSession(healed),
|
|
440
|
+
});
|
|
441
|
+
return true;
|
|
442
|
+
}
|
|
362
443
|
|
|
363
444
|
const wasRemote = session.isRemote;
|
|
364
445
|
const updated = { ...session, lastHeartbeat: now(), status: "active" };
|
|
@@ -492,6 +573,7 @@ export function createSynapseRegistry(opts = {}) {
|
|
|
492
573
|
return Object.freeze({
|
|
493
574
|
register,
|
|
494
575
|
unregister,
|
|
576
|
+
pruneExpired,
|
|
495
577
|
heartbeat,
|
|
496
578
|
getActive,
|
|
497
579
|
getAll,
|