@triflux/remote 10.43.0 → 10.45.0

package/hub/server.mjs

@@ -32,13 +32,6 @@ import {
 import { broker as brokerInstance, reloadBroker } from "@triflux/core/hub/account-broker.mjs";
 import { createAdaptiveEngine } from "@triflux/core/hub/adaptive.mjs";
 import { createAssignCallbackServer } from "@triflux/core/hub/assign-callbacks.mjs";
-import {
-  clampDurationMs,
-  DEFAULT_ASSIGN_TIMEOUT_MS,
-  DEFAULT_ASSIGN_TTL_MS,
-  DEFAULT_REGISTER_TIMEOUT_SEC,
-  resolveWorkerLeaseTtlMs,
-} from "@triflux/core/hub/lib/timeout-defaults.mjs";
 import { DelegatorService } from "@triflux/core/hub/delegator/index.mjs";
 import { createHitlManager } from "@triflux/core/hub/hitl.mjs";
 import {
@@ -51,6 +44,13 @@ import {
   cleanupStaleFsmonitorDaemons,
 } from "@triflux/core/hub/lib/process-utils.mjs";
 import * as spawnTrace from "@triflux/core/hub/lib/spawn-trace.mjs";
+import {
+  clampDurationMs,
+  DEFAULT_ASSIGN_TIMEOUT_MS,
+  DEFAULT_ASSIGN_TTL_MS,
+  DEFAULT_REGISTER_TIMEOUT_SEC,
+  resolveWorkerLeaseTtlMs,
+} from "@triflux/core/hub/lib/timeout-defaults.mjs";
 import {
   recordRequest,
   recordWorker,
@@ -961,6 +961,7 @@ function servePublicFile(res, path) {
  * @param {string} [opts.host]
  * @param {string|number} [opts.sessionId]
  * @param {(options: { cwd: string }) => object} [opts.createDelegatorWorker]
+ * @param {(dbPath: string) => object|Promise<object>} [opts.createStoreAdapterFn] test seam
  */
 export async function startHub({
   port: portOpt,
@@ -968,6 +969,7 @@ export async function startHub({
   host = "127.0.0.1",
   sessionId = process.pid,
   createDelegatorWorker = createDelegatorMcpWorker,
+  createStoreAdapterFn = createStoreAdapter,
 } = {}) {
   const resolvedPort = parseHubPort(portOpt);
   const portSpecified = Number.isFinite(resolvedPort);
@@ -980,7 +982,13 @@ export async function startHub({
         detectPeer: () => livePeer,
       });
 
-  if (portSpecified && !ephemeralPort && livePeer.alive && livePidPort && port !== livePidPort) {
+  if (
+    portSpecified &&
+    !ephemeralPort &&
+    livePeer.alive &&
+    livePidPort &&
+    port !== livePidPort
+  ) {
     hubLog.warn(
       {
         requestedPort: port,
@@ -991,9 +999,14 @@ export async function startHub({
     );
   }
 
-  const existingHub = ephemeralPort ? null : await tryReuseExistingHub({
-    port, portSpecified, host, detectPeer: () => livePeer,
-  });
+  const existingHub = ephemeralPort
+    ? null
+    : await tryReuseExistingHub({
+        port,
+        portSpecified,
+        host,
+        detectPeer: () => livePeer,
+      });
   if (existingHub) return existingHub;
 
   attachBrokerDiagnostics(brokerInstance);
@@ -1031,12 +1044,14 @@ export async function startHub({
     lockHeld = false;
   };
 
-  const lockedExistingHub = ephemeralPort ? null : await tryReuseExistingHub({
-    port,
-    portSpecified,
-    host,
-    detectPeer: () => detectLivePeer(),
-  });
+  const lockedExistingHub = ephemeralPort
+    ? null
+    : await tryReuseExistingHub({
+        port,
+        portSpecified,
+        host,
+        detectPeer: () => detectLivePeer(),
+      });
   if (lockedExistingHub) {
     releaseStartupLock();
     return lockedExistingHub;
@@ -1049,7 +1064,7 @@ export async function startHub({
     cleanupOwnedTokenFile({ token: HUB_TOKEN, port });
   }
 
-  const store = await createStoreAdapter(dbPath);
+  const store = await createStoreAdapterFn(dbPath);
   const router = createRouter(store);
   const fingerprintService = createAdaptiveFingerprintService({ store });
 
@@ -1545,6 +1560,38 @@ export async function startHub({
         try {
           const body = await parseBody(req);
           const { sessionId } = body || {};
+          if (body?.agent_id && body?.cli) {
+            const registered = router.registerAgent({
+              agent_id: body.agent_id,
+              cli: body.cli,
+              capabilities: Array.isArray(body.capabilities)
+                ? body.capabilities
+                : [],
+              topics: Array.isArray(body.topics) ? body.topics : [],
+              heartbeat_ttl_ms: body.heartbeat_ttl_ms ?? 300000,
+              metadata:
+                body.metadata && typeof body.metadata === "object"
+                  ? body.metadata
+                  : { source: "session_start" },
+              ...(Object.hasOwn(body, "project_id")
+                ? { project_id: body.project_id }
+                : {}),
+              ...(Object.hasOwn(body, "session_id")
+                ? { session_id: body.session_id }
+                : {}),
+              ...(Object.hasOwn(body, "host_id")
+                ? { host_id: body.host_id }
+                : {}),
+              ...(Object.hasOwn(body, "transport_locators_json")
+                ? { transport_locators_json: body.transport_locators_json }
+                : {}),
+            });
+            if (!registered?.ok) {
+              throw new Error(
+                registered?.error?.message || "hub agent register failed",
+              );
+            }
+          }
           const result = synapseRegistry.register(sessionId, body);
           if (!result?.ok) {
             throw new Error(result?.reason || "register failed");
@@ -1675,6 +1722,8 @@ export async function startHub({
               agent_id,
               reason = "manual",
               requested_by = "http",
+              hub_instance_id,
+              store_fingerprint,
             } = body;
             if (!role || !agent_id) {
               return writeJson(res, 400, {
@@ -1687,6 +1736,8 @@ export async function startHub({
               agent_id,
               reason,
               requested_by,
+              hub_instance_id,
+              store_fingerprint,
             });
             return writeJson(res, result.ok ? 200 : 409, result);
           }
@@ -1699,6 +1750,12 @@ export async function startHub({
               topics = [],
               capabilities = [],
               metadata = {},
+              project_id,
+              session_id,
+              host_id,
+              transport_locators_json,
+              hub_instance_id,
+              store_fingerprint,
             } = body;
             if (!agent_id || !cli) {
               return writeJson(res, 400, {
@@ -1714,7 +1771,9 @@ export async function startHub({
               metadata?.task ||
               cli;
             const workerStartedAt = performance.now();
-            const hasExplicitTimeout = Number.isFinite(Number(body.timeout_sec));
+            const hasExplicitTimeout = Number.isFinite(
+              Number(body.timeout_sec),
+            );
             const heartbeat_ttl_ms = hasExplicitTimeout
               ? (timeout_sec + 120) * 1000
               : Math.max((timeout_sec + 120) * 1000, resolveWorkerLeaseTtlMs());
@@ -1725,6 +1784,14 @@ export async function startHub({
               topics,
               heartbeat_ttl_ms,
               metadata,
+              ...(project_id === undefined ? {} : { project_id }),
+              ...(session_id === undefined ? {} : { session_id }),
+              ...(host_id === undefined ? {} : { host_id }),
+              ...(transport_locators_json === undefined
+                ? {}
+                : { transport_locators_json }),
+              ...(hub_instance_id === undefined ? {} : { hub_instance_id }),
+              ...(store_fingerprint === undefined ? {} : { store_fingerprint }),
             });
             if (result.ok) {
               workerSpans.set(agent_id, {
@@ -1759,11 +1826,24 @@ export async function startHub({
           }
 
           if (path === "/bridge/heartbeat" && req.method === "POST") {
-            const { agent_id, ttl_ms } = body;
-            if (!agent_id) return writeJson(res, 400, { ok: false, error: "agent_id 필수" });
+            const {
+              agent_id,
+              ttl_ms,
+              presence_generation,
+              hub_instance_id,
+              store_fingerprint,
+            } = body;
+            if (!agent_id)
+              return writeJson(res, 400, { ok: false, error: "agent_id 필수" });
             const result = await pipe.executeCommand("heartbeat", {
               agent_id,
-              heartbeat_ttl_ms: clampDurationMs(ttl_ms, resolveWorkerLeaseTtlMs()),
+              heartbeat_ttl_ms: clampDurationMs(
+                ttl_ms,
+                resolveWorkerLeaseTtlMs(),
+              ),
+              presence_generation,
+              hub_instance_id,
+              store_fingerprint,
             });
             return writeJson(res, result.ok ? 200 : 404, result);
           }
@@ -1844,6 +1924,9 @@ export async function startHub({
               status,
               include_metrics = true,
               trace_id,
+              presence_generation,
+              hub_instance_id,
+              store_fingerprint,
             } = body;
 
             if (agent_id && status) {
@@ -1864,7 +1947,15 @@ export async function startHub({
               ]).has(normalizedStatus)
                 ? normalizedStatus
                 : "online";
-              router.updateAgentStatus(normalizedAgentId, statusForStore);
+              const transition = router.transitionAgentStatus(
+                normalizedAgentId,
+                statusForStore,
+                presence_generation,
+                { hub_instance_id, store_fingerprint },
+              );
+              if (!transition.ok) {
+                return writeJson(res, 409, transition);
+              }
               const snapshot = await pipe.executeQuery("status", {
                 scope: "agent",
                 agent_id: normalizedAgentId,
@@ -1878,6 +1969,8 @@ export async function startHub({
                   reported_status: normalizedStatus,
                   reported_at_ms: Date.now(),
                   snapshot: snapshot?.data?.agent || null,
+                  hub_instance_id: transition.data.hub_instance_id,
+                  store_fingerprint: transition.data.store_fingerprint,
                 },
               });
             }
@@ -1887,6 +1980,8 @@ export async function startHub({
               agent_id,
               include_metrics,
               trace_id,
+              hub_instance_id,
+              store_fingerprint,
             });
             return writeJson(res, 200, result);
           }
@@ -2106,12 +2201,20 @@ export async function startHub({
           }
 
           if (path === "/bridge/deregister" && req.method === "POST") {
-            const { agent_id } = body;
+            const {
+              agent_id,
+              presence_generation,
+              hub_instance_id,
+              store_fingerprint,
+            } = body;
             if (!agent_id) {
               return writeJson(res, 400, { ok: false, error: "agent_id 필수" });
             }
             const result = await pipe.executeCommand("deregister", {
               agent_id,
+              presence_generation,
+              hub_instance_id,
+              store_fingerprint,
             });
             if (result.ok) {
               finishWorkerTrace(agent_id, "deregister");
@@ -2255,102 +2358,107 @@ export async function startHub({
   httpServer.requestTimeout = 30000;
   httpServer.headersTimeout = 10000;
 
-  router.startSweeper();
+  let hitlTimer = null;
+  let sessionTimer = null;
+  let synapsePruneTimer = null;
+  let orphanCleanupTimer = null;
+  let rateLimitTimer = null;
 
-  const hitlTimer = setInterval(() => {
-    try {
-      hitl.checkTimeouts();
-    } catch (err) {
-      hubLog.warn({ err }, "hitl.timeout_check_failed");
-    }
-  }, 10000);
-  hitlTimer.unref();
-
-  // MCP session TTL: sessions idle for SESSION_TTL_MS are closed automatically.
-  // Configurable via SESSION_TTL_MS (default 30 minutes). The sweep runs every 60 s.
-  const SESSION_TTL_MS =
-    parseInt(process.env.TFX_SESSION_TTL_MS || "", 10) || 30 * 60 * 1000;
-  const sessionTimer = setInterval(() => {
-    const now = Date.now();
-    for (const [sid, session] of transports) {
-      if (now - (session.transport._lastActivity || 0) <= SESSION_TTL_MS)
-        continue;
-      void closeMcpTransportSession(sid, session, "session_ttl");
-    }
-  }, 60000);
-  sessionTimer.unref();
+  const startBackgroundTimers = () => {
+    hitlTimer = setInterval(() => {
+      try {
+        hitl.checkTimeouts();
+      } catch (err) {
+        hubLog.warn({ err }, "hitl.timeout_check_failed");
+      }
+    }, 10000);
+    hitlTimer.unref();
+
+    // MCP session TTL: sessions idle for SESSION_TTL_MS are closed automatically.
+    // Configurable via SESSION_TTL_MS (default 30 minutes). The sweep runs every 60 s.
+    const SESSION_TTL_MS =
+      parseInt(process.env.TFX_SESSION_TTL_MS || "", 10) || 30 * 60 * 1000;
+    sessionTimer = setInterval(() => {
+      const now = Date.now();
+      for (const [sid, session] of transports) {
+        if (now - (session.transport._lastActivity || 0) <= SESSION_TTL_MS)
+          continue;
+        void closeMcpTransportSession(sid, session, "session_ttl");
+      }
+    }, 60000);
+    sessionTimer.unref();
 
-  const synapsePruneTimer = setInterval(() => {
-    try {
-      const { count } = synapseRegistry.pruneExpired();
-      if (count > 0) hubLog.info({ count }, "synapse.prune");
-    } catch (err) {
-      hubLog.warn({ err }, "synapse.prune_failed");
-    }
-  }, 60000);
-  synapsePruneTimer.unref();
-
-  // 고아 node.exe 프로세스 + stale spawn 세션 주기적 정리 (5분마다)
-  // TFX_DISABLE_ORPHAN_CLEANUP=1 로 비활성 (active SSH/swarm 세션 보호 회피용 hotfix gate)
-  const orphanCleanupTimer = setInterval(
-    () => {
-      if (process.env.TFX_DISABLE_ORPHAN_CLEANUP === "1") return;
+    synapsePruneTimer = setInterval(() => {
       try {
-        const { killed, killedProcesses = [] } = cleanupOrphanNodeProcesses();
-        const {
-          killed: runtimeKilled,
-          killedProcesses: runtimeKilledProcesses = [],
-        } = cleanupOrphanRuntimeProcesses();
-        const totalKilled = killed + runtimeKilled;
-        if (totalKilled > 0) {
-          hubLog.info(
-            {
-              killed: totalKilled,
-              processes: [...killedProcesses, ...runtimeKilledProcesses],
-              caller: "timer",
-            },
-            "hub.orphan_cleanup",
-          );
-        }
+        const { count } = synapseRegistry.pruneExpired();
+        if (count > 0) hubLog.info({ count }, "synapse.prune");
+      } catch (err) {
+        hubLog.warn({ err }, "synapse.prune_failed");
+      }
+    }, 60000);
+    synapsePruneTimer.unref();
+
+    // 고아 node.exe 프로세스 + stale spawn 세션 주기적 정리 (5분마다)
+    // TFX_DISABLE_ORPHAN_CLEANUP=1 로 비활성 (active SSH/swarm 세션 보호 회피용 hotfix gate)
+    orphanCleanupTimer = setInterval(
+      () => {
+        if (process.env.TFX_DISABLE_ORPHAN_CLEANUP === "1") return;
+        try {
+          const { killed, killedProcesses = [] } = cleanupOrphanNodeProcesses();
+          const {
+            killed: runtimeKilled,
+            killedProcesses: runtimeKilledProcesses = [],
+          } = cleanupOrphanRuntimeProcesses();
+          const totalKilled = killed + runtimeKilled;
+          if (totalKilled > 0) {
+            hubLog.info(
+              {
+                killed: totalKilled,
+                processes: [...killedProcesses, ...runtimeKilledProcesses],
+                caller: "timer",
+              },
+              "hub.orphan_cleanup",
+            );
+          }
 
-        const { killed: fsmonitorKilled, stale } = cleanupStaleFsmonitorDaemons(
-          {
-            minAgeMs: 24 * 60 * 60 * 1000,
-          },
-        );
-        if (fsmonitorKilled > 0) {
-          hubLog.info(
-            { killed: fsmonitorKilled, stale: stale.length },
-            "hub.fsmonitor_cleanup",
-          );
-        }
-      } catch {}
+          const { killed: fsmonitorKilled, stale } =
+            cleanupStaleFsmonitorDaemons({
+              minAgeMs: 24 * 60 * 60 * 1000,
+            });
+          if (fsmonitorKilled > 0) {
+            hubLog.info(
+              { killed: fsmonitorKilled, stale: stale.length },
+              "hub.fsmonitor_cleanup",
+            );
+          }
+        } catch {}
 
-      // stale tfx-spawn-* psmux 세션 정리 (30분 이상 idle)
-      try {
-        const staleKilled = cleanupStaleSpawnSessions(hubLog);
-        if (staleKilled > 0) {
-          hubLog.info({ killed: staleKilled }, "hub.stale_spawn_cleanup");
+        // stale tfx-spawn-* psmux 세션 정리 (30분 이상 idle)
+        try {
+          const staleKilled = cleanupStaleSpawnSessions(hubLog);
+          if (staleKilled > 0) {
+            hubLog.info({ killed: staleKilled }, "hub.stale_spawn_cleanup");
+          }
+        } catch {}
+      },
+      5 * 60 * 1000,
+    );
+    orphanCleanupTimer.unref();
+
+    // Evict stale rate-limit buckets once per minute to bound memory usage.
+    rateLimitTimer = setInterval(() => {
+      const cutoff = Date.now() - RATE_LIMIT_WINDOW_MS;
+      for (const [ip, timestamps] of rateLimitMap) {
+        const fresh = timestamps.filter((t) => t >= cutoff);
+        if (fresh.length === 0) {
+          rateLimitMap.delete(ip);
+        } else {
+          rateLimitMap.set(ip, fresh);
         }
-      } catch {}
-    },
-    5 * 60 * 1000,
-  );
-  orphanCleanupTimer.unref();
-
-  // Evict stale rate-limit buckets once per minute to bound memory usage.
-  const rateLimitTimer = setInterval(() => {
-    const cutoff = Date.now() - RATE_LIMIT_WINDOW_MS;
-    for (const [ip, timestamps] of rateLimitMap) {
-      const fresh = timestamps.filter((t) => t >= cutoff);
-      if (fresh.length === 0) {
-        rateLimitMap.delete(ip);
-      } else {
-        rateLimitMap.set(ip, fresh);
       }
-    }
-  }, RATE_LIMIT_WINDOW_MS);
-  rateLimitTimer.unref();
+    }, RATE_LIMIT_WINDOW_MS);
+    rateLimitTimer.unref();
+  };
 
   mkdirSync(PID_DIR, { recursive: true });
 
@@ -2358,6 +2466,19 @@ export async function startHub({
     try {
       router.stopSweeper();
     } catch {}
+    for (const timer of [
+      hitlTimer,
+      sessionTimer,
+      synapsePruneTimer,
+      orphanCleanupTimer,
+      rateLimitTimer,
+    ]) {
+      if (timer) clearInterval(timer);
+    }
+    for (const [sid, session] of Array.from(transports)) {
+      await closeMcpTransportSession(sid, session, "hub.startup_failure");
+    }
+    transports.clear();
     try {
       await pipe.stop();
     } catch {}
@@ -2367,6 +2488,9 @@ export async function startHub({
     try {
       await delegatorWorker.stop();
     } catch {}
+    try {
+      synapseRegistry.destroy();
+    } catch {}
     try {
       store.close();
     } catch {}
@@ -2377,6 +2501,22 @@ export async function startHub({
   };
 
   try {
+    router.sweepExpired({ now_ms: Date.now(), source: "startup" });
+    router.startSweeper({
+      run_immediately: false,
+      interval_ms: 10000,
+      onSweepError: ({ source, error, consecutive_failures }) => {
+        hubLog.warn(
+          {
+            source,
+            code: error?.code || "UNKNOWN",
+            consecutive_failures,
+          },
+          "hub.sweep_failed",
+        );
+      },
+    });
+    startBackgroundTimers();
     await pipe.start();
     await assignCallbacks.start();
   } catch (error) {
@@ -2445,7 +2585,8 @@ export async function startHub({
         httpServer.off("error", onError);
         try {
           const boundAddress = httpServer.address();
-          if (boundAddress && typeof boundAddress === "object") port = boundAddress.port;
+          if (boundAddress && typeof boundAddress === "object")
+            port = boundAddress.port;
           let idleTimer = null;
           let stopPromise = null;
 
@@ -2466,25 +2607,28 @@ export async function startHub({
             idleTimeoutMs: hubIdleTimeoutMs,
           };
 
-          if (!ephemeralPort) writeState({
-            pid: process.pid,
-            port,
-            host,
-            auth_mode: HUB_TOKEN ? "token-required" : "localhost-only",
-            url: info.url,
-            pipe_path: pipe.path,
-            pipePath: pipe.path,
-            assign_callback_pipe_path: assignCallbacks.path,
-            assignCallbackPipePath: assignCallbacks.path,
-            authMode: HUB_TOKEN ? "token-required" : "localhost-only",
-            startedAt,
-            started: startedAtMs,
-            version,
-            sessionId,
-            session_id: sessionId,
-          });
+          if (!ephemeralPort) {
+            writeState({
+              pid: process.pid,
+              port,
+              host,
+              auth_mode: HUB_TOKEN ? "token-required" : "localhost-only",
+              url: info.url,
+              pipe_path: pipe.path,
+              pipePath: pipe.path,
+              assign_callback_pipe_path: assignCallbacks.path,
+              assignCallbackPipePath: assignCallbacks.path,
+              authMode: HUB_TOKEN ? "token-required" : "localhost-only",
+              startedAt,
+              started: startedAtMs,
+              version,
+              sessionId,
+              session_id: sessionId,
+            });
+          }
           releaseStartupLock();
-          if (!ephemeralPort) void syncHubMcpSettingsIfAvailable({ hubUrl: info.url });
+          if (!ephemeralPort)
+            void syncHubMcpSettingsIfAvailable({ hubUrl: info.url });
 
           hubLog.info(
             {