@triflux/remote 10.44.0 → 10.46.0

package/hub/pipe.mjs

@@ -267,18 +267,11 @@ export function createPipeServer({
         const result = router.refreshAgentLease(
           agentId,
           payload.heartbeat_ttl_ms || heartbeatTtlMs,
+          payload.presence_generation,
+          payload,
         );
         if (client) touchClient(client);
-        if (result?.effective?.updated === false) {
-          return {
-            ok: false,
-            error: {
-              code: "AGENT_NOT_FOUND",
-              message: `heartbeat target not registered: ${agentId}`,
-            },
-          };
-        }
-        return { ok: true, data: result };
+        return result;
       }
 
       case "publish": {
@@ -357,12 +350,14 @@ export function createPipeServer({
 
       case "deregister": {
         const agentId = resolveAgentId(client, payload);
-        router.updateAgentStatus(agentId, "offline");
+        const result = router.transitionAgentStatus(
+          agentId,
+          "offline",
+          payload.presence_generation,
+          payload,
+        );
         if (client) touchClient(client);
-        return {
-          ok: true,
-          data: { agent_id: agentId, status: "offline" },
-        };
+        return result;
       }
 
       case "send_input": {

package/hub/schema.sql

@@ -10,6 +10,7 @@ CREATE TABLE IF NOT EXISTS agents (
   topics_json TEXT NOT NULL DEFAULT '[]',
   last_seen_ms INTEGER NOT NULL,
   lease_expires_ms INTEGER NOT NULL,
+  presence_generation INTEGER NOT NULL DEFAULT 0,
   status TEXT NOT NULL CHECK (status IN ('online','stale','offline')),
   metadata_json TEXT NOT NULL DEFAULT '{}',
   project_id TEXT,

package/hub/server.mjs

@@ -961,6 +961,7 @@ function servePublicFile(res, path) {
  * @param {string} [opts.host]
  * @param {string|number} [opts.sessionId]
  * @param {(options: { cwd: string }) => object} [opts.createDelegatorWorker]
+ * @param {(dbPath: string) => object|Promise<object>} [opts.createStoreAdapterFn] test seam
  */
 export async function startHub({
   port: portOpt,
@@ -968,6 +969,7 @@ export async function startHub({
   host = "127.0.0.1",
   sessionId = process.pid,
   createDelegatorWorker = createDelegatorMcpWorker,
+  createStoreAdapterFn = createStoreAdapter,
 } = {}) {
   const resolvedPort = parseHubPort(portOpt);
   const portSpecified = Number.isFinite(resolvedPort);
@@ -1062,7 +1064,7 @@ export async function startHub({
     cleanupOwnedTokenFile({ token: HUB_TOKEN, port });
   }
 
-  const store = await createStoreAdapter(dbPath);
+  const store = await createStoreAdapterFn(dbPath);
   const router = createRouter(store);
   const fingerprintService = createAdaptiveFingerprintService({ store });
 
@@ -1720,6 +1722,8 @@ export async function startHub({
               agent_id,
               reason = "manual",
               requested_by = "http",
+              hub_instance_id,
+              store_fingerprint,
             } = body;
             if (!role || !agent_id) {
               return writeJson(res, 400, {
@@ -1732,6 +1736,8 @@ export async function startHub({
               agent_id,
               reason,
               requested_by,
+              hub_instance_id,
+              store_fingerprint,
             });
             return writeJson(res, result.ok ? 200 : 409, result);
           }
@@ -1748,6 +1754,8 @@ export async function startHub({
               session_id,
               host_id,
               transport_locators_json,
+              hub_instance_id,
+              store_fingerprint,
             } = body;
             if (!agent_id || !cli) {
               return writeJson(res, 400, {
@@ -1782,6 +1790,8 @@ export async function startHub({
               ...(transport_locators_json === undefined
                 ? {}
                 : { transport_locators_json }),
+              ...(hub_instance_id === undefined ? {} : { hub_instance_id }),
+              ...(store_fingerprint === undefined ? {} : { store_fingerprint }),
             });
             if (result.ok) {
               workerSpans.set(agent_id, {
@@ -1816,7 +1826,13 @@ export async function startHub({
           }
 
           if (path === "/bridge/heartbeat" && req.method === "POST") {
-            const { agent_id, ttl_ms } = body;
+            const {
+              agent_id,
+              ttl_ms,
+              presence_generation,
+              hub_instance_id,
+              store_fingerprint,
+            } = body;
             if (!agent_id)
               return writeJson(res, 400, { ok: false, error: "agent_id 필수" });
             const result = await pipe.executeCommand("heartbeat", {
@@ -1825,6 +1841,9 @@ export async function startHub({
                 ttl_ms,
                 resolveWorkerLeaseTtlMs(),
               ),
+              presence_generation,
+              hub_instance_id,
+              store_fingerprint,
             });
             return writeJson(res, result.ok ? 200 : 404, result);
           }
@@ -1905,6 +1924,9 @@ export async function startHub({
               status,
               include_metrics = true,
               trace_id,
+              presence_generation,
+              hub_instance_id,
+              store_fingerprint,
             } = body;
 
             if (agent_id && status) {
@@ -1925,7 +1947,15 @@ export async function startHub({
               ]).has(normalizedStatus)
                 ? normalizedStatus
                 : "online";
-              router.updateAgentStatus(normalizedAgentId, statusForStore);
+              const transition = router.transitionAgentStatus(
+                normalizedAgentId,
+                statusForStore,
+                presence_generation,
+                { hub_instance_id, store_fingerprint },
+              );
+              if (!transition.ok) {
+                return writeJson(res, 409, transition);
+              }
               const snapshot = await pipe.executeQuery("status", {
                 scope: "agent",
                 agent_id: normalizedAgentId,
@@ -1939,6 +1969,8 @@ export async function startHub({
                   reported_status: normalizedStatus,
                   reported_at_ms: Date.now(),
                   snapshot: snapshot?.data?.agent || null,
+                  hub_instance_id: transition.data.hub_instance_id,
+                  store_fingerprint: transition.data.store_fingerprint,
                 },
               });
             }
@@ -1948,6 +1980,8 @@ export async function startHub({
               agent_id,
               include_metrics,
               trace_id,
+              hub_instance_id,
+              store_fingerprint,
             });
             return writeJson(res, 200, result);
           }
@@ -2167,12 +2201,20 @@ export async function startHub({
           }
 
           if (path === "/bridge/deregister" && req.method === "POST") {
-            const { agent_id } = body;
+            const {
+              agent_id,
+              presence_generation,
+              hub_instance_id,
+              store_fingerprint,
+            } = body;
             if (!agent_id) {
               return writeJson(res, 400, { ok: false, error: "agent_id 필수" });
             }
             const result = await pipe.executeCommand("deregister", {
               agent_id,
+              presence_generation,
+              hub_instance_id,
+              store_fingerprint,
             });
             if (result.ok) {
               finishWorkerTrace(agent_id, "deregister");
@@ -2316,102 +2358,107 @@ export async function startHub({
   httpServer.requestTimeout = 30000;
   httpServer.headersTimeout = 10000;
 
-  router.startSweeper();
+  let hitlTimer = null;
+  let sessionTimer = null;
+  let synapsePruneTimer = null;
+  let orphanCleanupTimer = null;
+  let rateLimitTimer = null;
 
-  const hitlTimer = setInterval(() => {
-    try {
-      hitl.checkTimeouts();
-    } catch (err) {
-      hubLog.warn({ err }, "hitl.timeout_check_failed");
-    }
-  }, 10000);
-  hitlTimer.unref();
-
-  // MCP session TTL: sessions idle for SESSION_TTL_MS are closed automatically.
-  // Configurable via SESSION_TTL_MS (default 30 minutes). The sweep runs every 60 s.
-  const SESSION_TTL_MS =
-    parseInt(process.env.TFX_SESSION_TTL_MS || "", 10) || 30 * 60 * 1000;
-  const sessionTimer = setInterval(() => {
-    const now = Date.now();
-    for (const [sid, session] of transports) {
-      if (now - (session.transport._lastActivity || 0) <= SESSION_TTL_MS)
-        continue;
-      void closeMcpTransportSession(sid, session, "session_ttl");
-    }
-  }, 60000);
-  sessionTimer.unref();
+  const startBackgroundTimers = () => {
+    hitlTimer = setInterval(() => {
+      try {
+        hitl.checkTimeouts();
+      } catch (err) {
+        hubLog.warn({ err }, "hitl.timeout_check_failed");
+      }
+    }, 10000);
+    hitlTimer.unref();
+
+    // MCP session TTL: sessions idle for SESSION_TTL_MS are closed automatically.
+    // Configurable via SESSION_TTL_MS (default 30 minutes). The sweep runs every 60 s.
+    const SESSION_TTL_MS =
+      parseInt(process.env.TFX_SESSION_TTL_MS || "", 10) || 30 * 60 * 1000;
+    sessionTimer = setInterval(() => {
+      const now = Date.now();
+      for (const [sid, session] of transports) {
+        if (now - (session.transport._lastActivity || 0) <= SESSION_TTL_MS)
+          continue;
+        void closeMcpTransportSession(sid, session, "session_ttl");
+      }
+    }, 60000);
+    sessionTimer.unref();
 
-  const synapsePruneTimer = setInterval(() => {
-    try {
-      const { count } = synapseRegistry.pruneExpired();
-      if (count > 0) hubLog.info({ count }, "synapse.prune");
-    } catch (err) {
-      hubLog.warn({ err }, "synapse.prune_failed");
-    }
-  }, 60000);
-  synapsePruneTimer.unref();
-
-  // 고아 node.exe 프로세스 + stale spawn 세션 주기적 정리 (5분마다)
-  // TFX_DISABLE_ORPHAN_CLEANUP=1 로 비활성 (active SSH/swarm 세션 보호 회피용 hotfix gate)
-  const orphanCleanupTimer = setInterval(
-    () => {
-      if (process.env.TFX_DISABLE_ORPHAN_CLEANUP === "1") return;
+    synapsePruneTimer = setInterval(() => {
       try {
-        const { killed, killedProcesses = [] } = cleanupOrphanNodeProcesses();
-        const {
-          killed: runtimeKilled,
-          killedProcesses: runtimeKilledProcesses = [],
-        } = cleanupOrphanRuntimeProcesses();
-        const totalKilled = killed + runtimeKilled;
-        if (totalKilled > 0) {
-          hubLog.info(
-            {
-              killed: totalKilled,
-              processes: [...killedProcesses, ...runtimeKilledProcesses],
-              caller: "timer",
-            },
-            "hub.orphan_cleanup",
-          );
-        }
+        const { count } = synapseRegistry.pruneExpired();
+        if (count > 0) hubLog.info({ count }, "synapse.prune");
+      } catch (err) {
+        hubLog.warn({ err }, "synapse.prune_failed");
+      }
+    }, 60000);
+    synapsePruneTimer.unref();
+
+    // 고아 node.exe 프로세스 + stale spawn 세션 주기적 정리 (5분마다)
+    // TFX_DISABLE_ORPHAN_CLEANUP=1 로 비활성 (active SSH/swarm 세션 보호 회피용 hotfix gate)
+    orphanCleanupTimer = setInterval(
+      () => {
+        if (process.env.TFX_DISABLE_ORPHAN_CLEANUP === "1") return;
+        try {
+          const { killed, killedProcesses = [] } = cleanupOrphanNodeProcesses();
+          const {
+            killed: runtimeKilled,
+            killedProcesses: runtimeKilledProcesses = [],
+          } = cleanupOrphanRuntimeProcesses();
+          const totalKilled = killed + runtimeKilled;
+          if (totalKilled > 0) {
+            hubLog.info(
+              {
+                killed: totalKilled,
+                processes: [...killedProcesses, ...runtimeKilledProcesses],
+                caller: "timer",
+              },
+              "hub.orphan_cleanup",
+            );
+          }
 
-        const { killed: fsmonitorKilled, stale } = cleanupStaleFsmonitorDaemons(
-          {
-            minAgeMs: 24 * 60 * 60 * 1000,
-          },
-        );
-        if (fsmonitorKilled > 0) {
-          hubLog.info(
-            { killed: fsmonitorKilled, stale: stale.length },
-            "hub.fsmonitor_cleanup",
-          );
-        }
-      } catch {}
+          const { killed: fsmonitorKilled, stale } =
+            cleanupStaleFsmonitorDaemons({
+              minAgeMs: 24 * 60 * 60 * 1000,
+            });
+          if (fsmonitorKilled > 0) {
+            hubLog.info(
+              { killed: fsmonitorKilled, stale: stale.length },
+              "hub.fsmonitor_cleanup",
+            );
+          }
+        } catch {}
 
-      // stale tfx-spawn-* psmux 세션 정리 (30분 이상 idle)
-      try {
-        const staleKilled = cleanupStaleSpawnSessions(hubLog);
-        if (staleKilled > 0) {
-          hubLog.info({ killed: staleKilled }, "hub.stale_spawn_cleanup");
+        // stale tfx-spawn-* psmux 세션 정리 (30분 이상 idle)
+        try {
+          const staleKilled = cleanupStaleSpawnSessions(hubLog);
+          if (staleKilled > 0) {
+            hubLog.info({ killed: staleKilled }, "hub.stale_spawn_cleanup");
+          }
+        } catch {}
+      },
+      5 * 60 * 1000,
+    );
+    orphanCleanupTimer.unref();
+
+    // Evict stale rate-limit buckets once per minute to bound memory usage.
+    rateLimitTimer = setInterval(() => {
+      const cutoff = Date.now() - RATE_LIMIT_WINDOW_MS;
+      for (const [ip, timestamps] of rateLimitMap) {
+        const fresh = timestamps.filter((t) => t >= cutoff);
+        if (fresh.length === 0) {
+          rateLimitMap.delete(ip);
+        } else {
+          rateLimitMap.set(ip, fresh);
         }
-      } catch {}
-    },
-    5 * 60 * 1000,
-  );
-  orphanCleanupTimer.unref();
-
-  // Evict stale rate-limit buckets once per minute to bound memory usage.
-  const rateLimitTimer = setInterval(() => {
-    const cutoff = Date.now() - RATE_LIMIT_WINDOW_MS;
-    for (const [ip, timestamps] of rateLimitMap) {
-      const fresh = timestamps.filter((t) => t >= cutoff);
-      if (fresh.length === 0) {
-        rateLimitMap.delete(ip);
-      } else {
-        rateLimitMap.set(ip, fresh);
       }
-    }
-  }, RATE_LIMIT_WINDOW_MS);
-  rateLimitTimer.unref();
+    }, RATE_LIMIT_WINDOW_MS);
+    rateLimitTimer.unref();
+  };
 
   mkdirSync(PID_DIR, { recursive: true });
 
@@ -2419,6 +2466,19 @@ export async function startHub({
     try {
       router.stopSweeper();
     } catch {}
+    for (const timer of [
+      hitlTimer,
+      sessionTimer,
+      synapsePruneTimer,
+      orphanCleanupTimer,
+      rateLimitTimer,
+    ]) {
+      if (timer) clearInterval(timer);
+    }
+    for (const [sid, session] of Array.from(transports)) {
+      await closeMcpTransportSession(sid, session, "hub.startup_failure");
+    }
+    transports.clear();
     try {
       await pipe.stop();
     } catch {}
@@ -2428,6 +2488,9 @@ export async function startHub({
     try {
       await delegatorWorker.stop();
     } catch {}
+    try {
+      synapseRegistry.destroy();
+    } catch {}
     try {
       store.close();
     } catch {}
@@ -2438,6 +2501,22 @@ export async function startHub({
   };
 
   try {
+    router.sweepExpired({ now_ms: Date.now(), source: "startup" });
+    router.startSweeper({
+      run_immediately: false,
+      interval_ms: 10000,
+      onSweepError: ({ source, error, consecutive_failures }) => {
+        hubLog.warn(
+          {
+            source,
+            code: error?.code || "UNKNOWN",
+            consecutive_failures,
+          },
+          "hub.sweep_failed",
+        );
+      },
+    });
+    startBackgroundTimers();
     await pipe.start();
     await assignCallbacks.start();
   } catch (error) {