@sleep2agi/commhub-server 0.9.0-preview.95 → 0.9.0-preview.96

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@sleep2agi/commhub-server",
-  "version": "0.9.0-preview.95",
+  "version": "0.9.0-preview.96",
   "description": "CommHub Server — AI Agent communication hub with MCP protocol, multi-network isolation, user auth, and MCP tools (17 collaboration-core + node/provider ops tools; authoritative list: docs-site/docs/api/mcp-tools.md).",
   "type": "module",
   "main": "src/index.ts",

package/src/node-identity-conflict-http.test.ts

@@ -0,0 +1,269 @@
+// #507 — two live copies of one node (directory copied verbatim: same node token,
+// same alias) used to hold two SSE subscriptions on `${net}:${alias}`. Every new
+// task was pushed to BOTH (each copy ran it), and either copy's shutdown
+// report_status(offline) took the node offline under the other → alias_offline.
+//
+// Pinned here, through the real HTTP entrances (GET /events/:alias with the node
+// token, POST /mcp report_status, POST /api/task, GET /api/stats/sse):
+//   1. newest connection wins: the older node stream gets node_connection_superseded
+//      (reason superseded_by_new_connection) and is closed; /api/stats/sse shows 1.
+//   2. a new task reaches exactly ONE node subscriber; a dashboard monitor on the
+//      same channel is untouched and still sees it.
+//   3. node_identity_conflict reaches the network observer stream and
+//      /api/stats/sse.identity_conflicts, with both remotes and no token.
+//   4. offline only when the last node connection leaves; tasks in between are
+//      delivered, not alias_offline.
+//   5. same instance id = a reconnect (replaced_by_reconnect, no conflict);
+//      different ids = instance_match "different"; another node's token watching
+//      this alias never supersedes it; a plain single-node stop is offline at once.
+//
+// 跑法:cd server && COMMHUB_DB=/tmp/x.db bun test src/node-identity-conflict-http.test.ts
+//       PG:COMMHUB_TEST_PG_URL=… COMMHUB_PG_EXPERIMENTAL=1(tests/test2123-hub-postgres-ladder 里注册)
+import { afterAll, beforeAll, describe, expect, test } from "bun:test";
+import { mkdtempSync, rmSync } from "node:fs";
+import { join } from "node:path";
+import { tmpdir } from "node:os";
+import { createNetworkTokenForNode, register } from "./auth.js";
+import { db } from "./db.js";
+
+const dir = mkdtempSync(join(tmpdir(), "anet-node-conflict-"));
+const activeDbPath = process.env.COMMHUB_DB ?? (process.env.COMMHUB_TEST_PG_URL ? "postgres" : undefined);
+if (!activeDbPath) throw new Error("node-identity-conflict requires COMMHUB_DB (or COMMHUB_TEST_PG_URL) before module import");
+
+const stamp = Date.now();
+let server: any;
+let base = "";
+let ownerToken = "", ownerId = "", NET = "";
+const nodes: Record<string, { alias: string; nodeId: string; token: string }> = {};
+
+type Sub = { events: any[]; done: boolean; abort: () => void; ready: Promise<void> };
+
+function subscribe(path: string, token: string, headers: Record<string, string> = {}): Sub {
+  const ctrl = new AbortController();
+  const sub: Sub = { events: [], done: false, abort: () => ctrl.abort(), ready: Promise.resolve() };
+  let markReady!: () => void;
+  sub.ready = new Promise<void>((r) => { markReady = r; });
+  (async () => {
+    try {
+      const res = await fetch(`${base}${path}`, { headers: { Authorization: `Bearer ${token}`, Accept: "text/event-stream", ...headers }, signal: ctrl.signal });
+      if (!res.ok || !res.body) throw new Error(`subscribe ${path} → ${res.status}`);
+      const reader = res.body.getReader();
+      const dec = new TextDecoder();
+      let buf = "";
+      while (true) {
+        const { done, value } = await reader.read();
+        if (done) break;
+        buf += dec.decode(value, { stream: true });
+        const lines = buf.split("\n"); buf = lines.pop() || "";
+        for (const line of lines) {
+          if (!line.startsWith("data: ")) continue;
+          const ev = JSON.parse(line.slice(6));
+          sub.events.push(ev);
+          if (ev.type === "connected") markReady();
+        }
+      }
+    } catch { /* aborted */ }
+    sub.done = true;
+    markReady();
+  })();
+  return sub;
+}
+
+async function until(cond: () => boolean, what: string, ms = 3000) {
+  const t0 = Date.now();
+  while (!cond()) {
+    if (Date.now() - t0 > ms) throw new Error(`timed out waiting for: ${what}`);
+    await new Promise((r) => setTimeout(r, 20));
+  }
+}
+const settle = () => new Promise((r) => setTimeout(r, 150));
+
+async function api(token: string, method: string, path: string, body?: unknown) {
+  const res = await fetch(`${base}${path}`, {
+    method,
+    headers: { Authorization: `Bearer ${token}`, "Content-Type": "application/json" },
+    body: body === undefined ? undefined : JSON.stringify(body),
+  });
+  return { status: res.status, body: await res.json().catch(() => null) as any };
+}
+
+async function tool(token: string, name: string, args: Record<string, unknown>) {
+  const res = await fetch(`${base}/mcp`, {
+    method: "POST",
+    headers: { Authorization: `Bearer ${token}`, "Content-Type": "application/json", Accept: "application/json, text/event-stream", "MCP-Protocol-Version": "2025-03-26" },
+    body: JSON.stringify({ jsonrpc: "2.0", id: 1, method: "tools/call", params: { name, arguments: args } }),
+  });
+  const raw = await res.text();
+  const data = raw.split("\n").filter((x) => x.startsWith("data:"));
+  const payload = data.length ? JSON.parse(data.at(-1)!.slice(5).trim()) : JSON.parse(raw);
+  const text = payload.result.content[0].text;
+  if (!text.startsWith("{")) throw new Error(`${name}: ${text}`);
+  return JSON.parse(text);
+}
+
+const reportStatus = (n: { alias: string; nodeId: string; token: string }, status: string) =>
+  tool(n.token, "report_status", { resume_id: `sdk-${n.nodeId}`, alias: n.alias, status, node_id: n.nodeId, network_id: NET });
+const storedStatus = (alias: string) =>
+  db.get<{ status: string }>("SELECT status FROM sessions WHERE alias = ?1 AND network_id = ?2", alias, NET)?.status;
+let seq = 0;
+const sendTask = (alias: string) => api(ownerToken, "POST", "/api/task", { alias, task: `conflict task ${stamp} #${++seq}`, network_id: NET });
+const nodeStream = (n: { alias: string; token: string }, headers: Record<string, string> = {}) =>
+  subscribe(`/events/${encodeURIComponent(n.alias)}`, n.token, headers);
+const newTasks = (s: Sub) => s.events.filter((e) => e.type === "new_task");
+const sseCount = async (alias: string) => (await api(nodes.a.token, "GET", "/api/stats/sse")).body.sessions[`${NET}:${alias}`] ?? 0;
+
+beforeAll(async () => {
+  process.env.COMMHUB_UPLOADS_DIR = join(dir, "uploads");
+  process.env.HOST = "127.0.0.1";
+  const owner = register(`nic_owner_${stamp}`, "NodeConflictOwner123!", undefined, "seed");
+  ownerToken = owner.token!; ownerId = owner.user!.user_id; NET = owner.network_id!;
+  for (const key of ["a", "b", "c", "d", "e"]) {
+    const alias = `nic-${key}-${stamp}`;
+    const nodeId = `n_nic_${key}_${stamp}`;
+    const minted = createNetworkTokenForNode(ownerId, NET, alias, nodeId);
+    expect(minted.ok).toBe(true);
+    nodes[key] = { alias, nodeId, token: minted.token! };
+  }
+  const mod: any = await import("./server.js");
+  server = mod.bootServer({ port: 0, hostname: "127.0.0.1" });
+  base = `http://127.0.0.1:${server.port}`;
+  for (const n of Object.values(nodes)) expect((await reportStatus(n, "idle")).ok).toBe(true);
+}, 30_000);
+
+afterAll(() => {
+  try { server?.stop?.(true); } catch {}
+  try { rmSync(dir, { recursive: true, force: true }); } catch {}
+});
+
+describe("#507 two copies of one node (same token, same alias)", () => {
+  const subs: Record<string, Sub> = {};
+
+  test("newest connection wins: the older node stream is told why and closed", async () => {
+    const n = nodes.a;
+    subs.observer = subscribe(`/events/network/${NET}`, ownerToken);
+    subs.monitor = subscribe(`/events/${encodeURIComponent(n.alias)}?network_id=${NET}`, ownerToken);
+    await subs.observer.ready; await subs.monitor.ready;
+    subs.copyA = nodeStream(n, { "User-Agent": "copy-A" });
+    await subs.copyA.ready;
+    expect(await sseCount(n.alias)).toBe(2); // copy A + dashboard monitor
+
+    subs.copyB = nodeStream(n, { "User-Agent": "copy-B" });
+    await subs.copyB.ready;
+    await until(() => subs.copyA.done, "copy A's stream to be closed");
+    const notice = subs.copyA.events.find((e) => e.type === "node_connection_superseded");
+    expect(notice).toBeTruthy();
+    expect(notice.reason).toBe("superseded_by_new_connection");
+    expect(notice.instance_match).toBe("unknown");
+    expect(subs.copyB.done).toBe(false);
+    expect(subs.monitor.done).toBe(false); // a monitor is never superseded
+    expect(await sseCount(n.alias)).toBe(2); // copy B + monitor
+  });
+
+  test("a new task reaches exactly one node subscriber (the monitor still sees it)", async () => {
+    const r = await sendTask(nodes.a.alias);
+    expect(r.status).toBe(200);
+    expect(r.body.ok).toBe(true);
+    await until(() => newTasks(subs.copyB).length >= 1 && newTasks(subs.monitor).length >= 1, "new_task on copy B + monitor");
+    await settle();
+    expect(newTasks(subs.copyB).length).toBe(1);
+    expect(newTasks(subs.copyA).length).toBe(0);
+    expect(newTasks(subs.monitor).length).toBe(1);
+  });
+
+  test("node_identity_conflict reaches the observer stream and /api/stats/sse, with remotes and no token", async () => {
+    await until(() => subs.observer.events.some((e) => e.type === "node_identity_conflict"), "observer conflict event");
+    const ev = subs.observer.events.find((e) => e.type === "node_identity_conflict");
+    expect(ev.alias).toBe(nodes.a.alias);
+    expect(ev.node_id).toBe(nodes.a.nodeId);
+    expect(ev.policy).toBe("newest_connection_wins");
+    expect(ev.superseded.user_agent).toBe("copy-A");
+    expect(ev.current.user_agent).toBe("copy-B");
+    expect(ev.superseded.remote).toMatch(/^127\.0\.0\.1:\d+$/);
+    expect(ev.current.remote).toMatch(/^127\.0\.0\.1:\d+$/);
+    expect(ev.superseded.remote).not.toBe(ev.current.remote);
+    const stats = await api(nodes.a.token, "GET", "/api/stats/sse");
+    const mine = stats.body.identity_conflicts.filter((c: any) => c.alias === nodes.a.alias);
+    expect(mine.length).toBe(1);
+    expect(mine[0].network_id).toBe(NET);
+    const raw = JSON.stringify(stats.body) + JSON.stringify(ev);
+    expect(raw).not.toContain("ntok_");
+    expect(raw).not.toContain(nodes.a.token);
+    const audit = db.get<{ n: number }>("SELECT COUNT(*) AS n FROM audit_log WHERE action = 'node_identity_conflict' AND target_id = ?1", nodes.a.nodeId);
+    expect(audit?.n).toBe(1);
+  });
+
+  test("offline only when the last node connection leaves; tasks in between are delivered", async () => {
+    const n = nodes.a;
+    // Copy A (the superseded one) shuts down and says so.
+    const fromA = await reportStatus(n, "offline");
+    expect(fromA.ok).toBe(true);
+    expect(fromA.offline_deferred).toBe(true);
+    expect(storedStatus(n.alias)).not.toBe("offline");
+    const r = await sendTask(n.alias);
+    expect(r.status).toBe(200);
+    expect(r.body.ok).toBe(true);
+    expect(r.body.error).toBeUndefined();
+    await until(() => newTasks(subs.copyB).length >= 2, "second new_task on copy B");
+
+    // Copy B stops too: it reports offline while its stream is still open, then exits.
+    const fromB = await reportStatus(n, "offline");
+    expect(fromB.offline_deferred).toBe(true);
+    expect(storedStatus(n.alias)).not.toBe("offline");
+    subs.copyB.abort();
+    await until(() => storedStatus(n.alias) === "offline", "deferred offline applied once the last node stream closed");
+    expect((await sendTask(n.alias)).body.error).toBe("alias_offline");
+    subs.monitor.abort(); subs.observer.abort();
+  });
+});
+
+describe("#507 what is and is not a conflict", () => {
+  test("same instance id = a reconnect of the same process: replaced, no conflict recorded", async () => {
+    const n = nodes.b;
+    const first = nodeStream(n, { "X-Anet-Instance-Id": "inst-b-1" });
+    await first.ready;
+    const second = nodeStream(n, { "X-Anet-Instance-Id": "inst-b-1" });
+    await second.ready;
+    await until(() => first.done, "old stream replaced");
+    expect(first.events.find((e) => e.type === "node_connection_superseded")?.reason).toBe("replaced_by_reconnect");
+    const stats = await api(n.token, "GET", "/api/stats/sse");
+    expect(stats.body.identity_conflicts.filter((c: any) => c.alias === n.alias)).toEqual([]);
+    // No conflict → a stop is exactly as before: offline at once.
+    const r = await reportStatus(n, "offline");
+    expect(r.offline_deferred).toBeUndefined();
+    expect(storedStatus(n.alias)).toBe("offline");
+    second.abort();
+  });
+
+  test("different instance ids are reported as instance_match=different", async () => {
+    const n = nodes.c;
+    const first = nodeStream(n, { "X-Anet-Instance-Id": "inst-c-host1" });
+    await first.ready;
+    const second = subscribe(`/events/${encodeURIComponent(n.alias)}?instance_id=inst-c-host2`, n.token);
+    await second.ready;
+    await until(() => first.done, "old stream superseded");
+    const stats = await api(n.token, "GET", "/api/stats/sse");
+    const mine = stats.body.identity_conflicts.filter((c: any) => c.alias === n.alias);
+    expect(mine.length).toBe(1);
+    expect(mine[0].instance_match).toBe("different");
+    expect(mine[0].superseded.instance_id).toBe("inst-c-host1");
+    expect(mine[0].current.instance_id).toBe("inst-c-host2");
+    second.abort();
+  });
+
+  test("another node's token watching this alias never supersedes the node", async () => {
+    const n = nodes.d;
+    const own = nodeStream(n);
+    await own.ready;
+    const watcher = subscribe(`/events/${encodeURIComponent(n.alias)}`, nodes.e.token);
+    await watcher.ready;
+    await settle();
+    expect(own.done).toBe(false);
+    expect(own.events.some((e) => e.type === "node_connection_superseded")).toBe(false);
+    watcher.abort();
+    // And a plain single-node stop (no conflict) is offline immediately, as before.
+    const r = await reportStatus(n, "offline");
+    expect(r.offline_deferred).toBeUndefined();
+    expect(storedStatus(n.alias)).toBe("offline");
+    own.abort();
+  });
+});

package/src/node-identity-conflict.ts

@@ -0,0 +1,142 @@
+// #507 — two live processes holding the same node identity (a node directory
+// copied verbatim and started elsewhere: same node token, same alias).
+//
+// push.ts enforces "at most one node-scoped SSE subscriber per (network, alias)",
+// newest connection wins (policy + rationale next to enforceSingleNodeSubscriber).
+// This module is the Hub-side bookkeeping around that:
+//
+//   1. node_identity_conflict — every supersede between connections that are not
+//      provably the same process is logged, written to audit_log, pushed to the
+//      network observer stream (dashboard) and kept in a small ring that
+//      GET /api/stats/sse returns. Carries both connections' remote info; never
+//      a token.
+//
+//   2. offline while another copy is still connected — a copy's shutdown
+//      report_status(offline) used to mark the node offline even though the other
+//      copy was connected, so new tasks got alias_offline. While a conflict is
+//      recent for that alias and a node-scoped subscriber is still connected, the
+//      offline is DEFERRED: stored status stays as it was, and the offline is
+//      applied only when the last node-scoped subscriber leaves (or dropped when the
+//      surviving copy reports a fresh status). Outside a conflict window nothing
+//      changes: a normal single-node stop marks offline immediately, as before.
+
+import { db, logAudit } from "./db.js";
+import {
+  hasNodeSubscriber,
+  onNodeIdentityConflict,
+  onNodeSubscribersDrained,
+  pushEvent,
+  pushNetworkObserverEvent,
+  type NodeIdentityConflict,
+} from "./push.js";
+
+/** How long after a conflict an offline report is treated as "maybe the other copy". */
+export const CONFLICT_WINDOW_MS = 10 * 60 * 1000;
+const RING_MAX = 50;
+/** Observer/audit emission is rate-limited per alias (logs are not). A flapping
+ *  pair supersedes about once a second; one event a minute is enough to see it. */
+const EMIT_EVERY_MS = 60 * 1000;
+
+const ring: NodeIdentityConflict[] = [];
+const lastConflictAt = new Map<string, number>();
+const lastEmittedAt = new Map<string, number>();
+const pendingOffline = new Map<string, { resumeId: string; at: number }>();
+
+const k = (alias: string, networkId: string | null | undefined) => `${networkId || "global"}:${alias}`;
+
+function publicInfo(i: NodeIdentityConflict["current"]) {
+  return {
+    instance_id: i.instanceId,
+    remote: i.remote,
+    user_agent: i.userAgent,
+    connected_at: new Date(i.connectedAt).toISOString(),
+  };
+}
+
+export function conflictPayload(c: NodeIdentityConflict) {
+  return {
+    type: "node_identity_conflict",
+    alias: c.alias,
+    node_id: c.nodeId,
+    instance_match: c.instanceMatch,
+    policy: "newest_connection_wins",
+    superseded: publicInfo(c.superseded),
+    current: publicInfo(c.current),
+    at: new Date(c.at).toISOString(),
+  };
+}
+
+function handleConflict(c: NodeIdentityConflict): void {
+  const key = k(c.alias, c.networkId);
+  lastConflictAt.set(key, c.at);
+  ring.push(c);
+  if (ring.length > RING_MAX) ring.splice(0, ring.length - RING_MAX);
+  const payload = conflictPayload(c);
+  console.warn(`[node_identity_conflict] ${c.key} node_id=${c.nodeId ?? "?"} instance_match=${c.instanceMatch} ` +
+    `superseded=${c.superseded.remote ?? "?"}(${c.superseded.instanceId ?? "no-instance-id"}) ` +
+    `by=${c.current.remote ?? "?"}(${c.current.instanceId ?? "no-instance-id"}) — ` +
+    `two processes are using this node's identity (copied node directory?); newest connection kept`);
+  const last = lastEmittedAt.get(key) ?? 0;
+  if (c.at - last < EMIT_EVERY_MS) return;
+  lastEmittedAt.set(key, c.at);
+  pushNetworkObserverEvent(c.networkId, payload);
+  logAudit(null, null, "node_identity_conflict", "node", c.nodeId ?? c.alias, JSON.stringify(payload), c.current.remote ?? undefined, c.networkId ?? undefined);
+}
+
+function handleDrained(alias: string, networkId: string | null): void {
+  const key = k(alias, networkId);
+  const pending = pendingOffline.get(key);
+  if (!pending) return;
+  pendingOffline.delete(key);
+  try {
+    db.run(
+      "UPDATE sessions SET status = 'offline', updated_at = datetime('now') WHERE resume_id = ?1 AND alias = ?2 AND network_id = ?3",
+      [pending.resumeId, alias, networkId ?? "default"],
+    );
+    console.log(`[node_identity_conflict] ${key} last node connection gone → deferred offline applied`);
+    pushEvent(alias, { type: "status_update", alias, status: "offline", progress: null, host: null, process_telemetry: null }, networkId);
+  } catch (e: any) {
+    console.log(`[node_identity_conflict] deferred offline failed for ${key}: ${e?.message || e}`);
+  }
+}
+
+onNodeIdentityConflict(handleConflict);
+onNodeSubscribersDrained(handleDrained);
+
+/** report_status hook. Returns true when an offline report must NOT be stored now
+ *  (another copy of this node is still connected); it is then applied when the last
+ *  node-scoped subscriber leaves. Any non-offline report clears a pending offline. */
+export function deferOfflineIfAnotherCopyConnected(
+  alias: string,
+  networkId: string | null | undefined,
+  status: string,
+  resumeId: string,
+  now = Date.now(),
+): boolean {
+  const key = k(alias, networkId);
+  if (status !== "offline") {
+    pendingOffline.delete(key);
+    return false;
+  }
+  const conflictAt = lastConflictAt.get(key);
+  if (conflictAt === undefined || now - conflictAt > CONFLICT_WINDOW_MS) return false;
+  if (!hasNodeSubscriber(alias, networkId)) return false;
+  pendingOffline.set(key, { resumeId, at: now });
+  console.warn(`[node_identity_conflict] ${key} report_status(offline) deferred: a node connection is still open ` +
+    `and a second copy was seen ${Math.round((now - conflictAt) / 1000)}s ago`);
+  return true;
+}
+
+/** Recent conflicts, newest last, optionally limited to one network. */
+export function recentNodeIdentityConflicts(networkId?: string | null) {
+  return ring
+    .filter((c) => networkId === undefined || c.networkId === networkId)
+    .map((c) => ({ network_id: c.networkId, ...conflictPayload(c) }));
+}
+
+export function __resetNodeIdentityConflictsForTest(): void {
+  ring.length = 0;
+  lastConflictAt.clear();
+  lastEmittedAt.clear();
+  pendingOffline.clear();
+}

package/src/push.ts

@@ -88,6 +88,36 @@ type SSEClient = {
   subscriberUserId?: string;
   /** 只推满足条件的事件(受限成员的网络观察流用)。缺省 = 全推。 */
   eventFilter?: (event: Record<string, unknown>) => boolean;
+  /** #507 — set only on a node's OWN task stream (/events/<alias> opened with the
+   *  node token bound to that alias). Dashboard monitors / master-token streams on
+   *  the same key leave it unset and are never superseded or counted here. */
+  node?: NodeSubscriberInfo;
+};
+
+/** #507 — what the Hub knows about one node-scoped SSE connection. No tokens. */
+export type NodeSubscriberInfo = {
+  nodeId: string | null;
+  /** Optional, client-supplied (`X-Anet-Instance-Id` header or `?instance_id=`).
+   *  null = the client did not say (every agent-node released so far). */
+  instanceId: string | null;
+  remote: string | null;
+  userAgent: string | null;
+  connectedAt: number;
+};
+
+/** #507 — one supersede of a node-scoped connection by a newer one. */
+export type NodeIdentityConflict = {
+  key: string;
+  alias: string;
+  networkId: string | null;
+  nodeId: string | null;
+  /** "same" = both connections carried the same instance id (a reconnect of the
+   *  same process; not reported as a conflict). "different" = both carried ids and
+   *  they differ (two live copies). "unknown" = at least one carried none. */
+  instanceMatch: "same" | "different" | "unknown";
+  superseded: NodeSubscriberInfo;
+  current: NodeSubscriberInfo;
+  at: number;
 };
 
 // 一个 session 可能有多个 SSE 连接（重连时短暂并存）
@@ -158,6 +188,9 @@ function closeClient(client: SSEClient, reason: string): void {
     // we're done with it.
   }
   console.log(`[${ts()}] SSE ✕ ${printableKey(client.key)} closed (reason=${reason})`);
+  if (client.node && reason !== SUPERSEDED_REASON && !hasLiveNodeClient(client.key)) {
+    notifyNodeDrained(client.key);
+  }
   if (isUserStreamClient(client)) {
     touchUserSeen(client.subscriberUserId!);
     if (!isUserOnline(client.subscriberUserId!)) notifyPresence(client.subscriberUserId!, false);
@@ -260,14 +293,114 @@ function ensureLivenessSweep(): void {
 // Public API
 // ─────────────────────────────────────────────────────────────────────
 
-/** 创建 SSE Response 并注册到 clients map */
-export function createSSEStream(sessionName: string, networkId?: string | null): Response {
+/** 创建 SSE Response 并注册到 clients map。
+ *  #507 — pass `opts.node` only for a node's own stream (node token bound to this
+ *  alias); such a stream supersedes any older node-scoped stream on the same key. */
+export function createSSEStream(
+  sessionName: string,
+  networkId?: string | null,
+  opts: { node?: Omit<NodeSubscriberInfo, "connectedAt"> } = {},
+): Response {
   return createStreamForKey(
     clientKey(sessionName, networkId),
     { type: "connected", session: sessionName, network_id: networkId ?? null },
+    opts.node ? { node: { ...opts.node, connectedAt: Date.now() } } : {},
   );
 }
 
+// ── #507 node identity conflict: at most ONE node-scoped subscriber per key ──
+//
+// Before #507 every connection on `${networkId}:${alias}` received every push. A
+// node directory copied verbatim (same node token, same alias) and started
+// elsewhere therefore got every new_task twice — both copies ran it — and either
+// copy's shutdown report_status(offline) marked the node offline while the other
+// was still connected.
+//
+// Policy: NEWEST CONNECTION WINS. Measured reconnect path (agent-node connectSSE →
+// superviseChild): a restarted process's old socket is closed by the OS, so a
+// legit restart never overlaps its own previous stream; an overlap only happens
+// with (a) a half-open ghost the Hub has not reaped yet (up to STUCK_CLOSE_MS) or
+// (b) a second live copy. In (a) the new connection is the real one — keeping the
+// old would park tasks on a dead socket — so the newest must win. In (b) the
+// loser reconnects ~1 s later and wins back: the copies flap, which is loud
+// (log + node_identity_conflict event + /api/stats/sse) instead of silently
+// double-executing. An "only if both alive > N s" rule cannot tell (a) from (b):
+// a ghost looks alive to the Hub until it is reaped.
+export const SUPERSEDED_REASON = "superseded_by_new_connection";
+const REPLACED_REASON = "replaced_by_reconnect";
+
+let nodeConflictListener: ((c: NodeIdentityConflict) => void) | null = null;
+let nodeDrainedListener: ((alias: string, networkId: string | null) => void) | null = null;
+
+/** Called for every supersede whose instances are not provably the same process. */
+export function onNodeIdentityConflict(cb: ((c: NodeIdentityConflict) => void) | null): void {
+  nodeConflictListener = cb;
+}
+
+/** Called when the LAST node-scoped subscriber of a key goes away (not when it is
+ *  superseded — the newer connection is still there). */
+export function onNodeSubscribersDrained(cb: ((alias: string, networkId: string | null) => void) | null): void {
+  nodeDrainedListener = cb;
+}
+
+function splitClientKey(key: string): { alias: string; networkId: string | null } {
+  const i = key.indexOf(":");
+  const net = key.slice(0, i);
+  return { alias: key.slice(i + 1), networkId: net === "global" ? null : net };
+}
+
+function hasLiveNodeClient(key: string): boolean {
+  return (clients.get(key) ?? []).some((c) => !c.closed && !!c.node);
+}
+
+function notifyNodeDrained(key: string): void {
+  if (!nodeDrainedListener) return;
+  const { alias, networkId } = splitClientKey(key);
+  try { nodeDrainedListener(alias, networkId); } catch (e: any) {
+    console.log(`[${ts()}] node drained listener failed: ${e?.message || e}`);
+  }
+}
+
+/** Is a node-scoped (node-token) subscriber connected on this alias right now? */
+export function hasNodeSubscriber(sessionName: string, networkId?: string | null): boolean {
+  return hasLiveNodeClient(clientKey(sessionName, networkId));
+}
+
+/** Keep only `keep` among the node-scoped clients of `key`; close the others with
+ *  a final `node_connection_superseded` frame. Dashboard monitors are untouched. */
+function enforceSingleNodeSubscriber(key: string, keep: SSEClient): void {
+  const arr = clients.get(key);
+  if (!arr || !keep.node) return;
+  for (const old of arr) {
+    if (old === keep || old.closed || !old.node) continue;
+    const a = old.node.instanceId, b = keep.node.instanceId;
+    const instanceMatch: NodeIdentityConflict["instanceMatch"] = a && b ? (a === b ? "same" : "different") : "unknown";
+    const reason = instanceMatch === "same" ? REPLACED_REASON : SUPERSEDED_REASON;
+    tryEnqueueBytes(old, old.encoder.encode(`data: ${JSON.stringify({
+      type: "node_connection_superseded",
+      reason,
+      instance_match: instanceMatch,
+      // The newer connection's network origin, so the loser's log says WHERE the
+      // other copy runs. Never a token.
+      by: { instance_id: keep.node.instanceId, remote: keep.node.remote },
+    })}\n\n`));
+    closeClient(old, reason);
+    if (instanceMatch !== "same" && nodeConflictListener) {
+      const { alias, networkId } = splitClientKey(key);
+      try {
+        nodeConflictListener({
+          key: printableKey(key), alias, networkId,
+          nodeId: keep.node.nodeId ?? old.node.nodeId,
+          instanceMatch, superseded: old.node, current: keep.node, at: Date.now(),
+        });
+      } catch (e: any) {
+        console.log(`[${ts()}] node conflict listener failed: ${e?.message || e}`);
+      }
+    }
+  }
+  pruneClosed(key);
+}
+
 /** #461 — 创建网络级观察者 SSE Response（dashboard 观察第三方流量）。
  *  Shares the exact same registration / backpressure / liveness
  *  machinery as session streams; only the key scheme differs. */
@@ -295,7 +428,7 @@ export function createUserEventStream(networkId: string, userId: string): Respon
 function createStreamForKey(
   key: string,
   initialEvent: Record<string, unknown>,
-  opts: { subscriberUserId?: string; eventFilter?: (event: Record<string, unknown>) => boolean } = {},
+  opts: { subscriberUserId?: string; eventFilter?: (event: Record<string, unknown>) => boolean; node?: NodeSubscriberInfo } = {},
 ): Response {
   const encoder = new TextEncoder();
   let client: SSEClient;
@@ -310,6 +443,7 @@ function createStreamForKey(
         key,
         ...(opts.subscriberUserId ? { subscriberUserId: opts.subscriberUserId } : {}),
         ...(opts.eventFilter ? { eventFilter: opts.eventFilter } : {}),
+        ...(opts.node ? { node: opts.node } : {}),
       };
 
       const presenceUser = isUserStreamClient(client) ? client.subscriberUserId! : null;
@@ -318,6 +452,7 @@ function createStreamForKey(
       clients.get(key)!.push(client);
       if (presenceUser) touchUserSeen(presenceUser);
       console.log(`[${ts()}] SSE ← ${printableKey(key)} connected (${clients.get(key)!.length} clients)`);
+      if (client.node) enforceSingleNodeSubscriber(key, client);
 
       // Send initial connected frame through the backpressure guard so
       // even the first byte respects the contract.
@@ -406,9 +541,16 @@ function rekeyClient(oldSessionName: string, newSessionName: string, networkId?:
   for (const c of oldClients) c.key = newKey;
 
   const existing = clients.get(newKey) || [];
-  clients.set(newKey, existing.concat(oldClients));
+  const merged = existing.concat(oldClients);
+  clients.set(newKey, merged);
   clients.delete(oldKey);
   console.log(`[${ts()}] SSE ↔ rekey ${oldKey} → ${newKey} (${oldClients.length} clients)`);
+  // #507 — a rename can merge two node-scoped streams onto one key; newest wins.
+  const nodeClients = merged.filter((c) => !c.closed && c.node);
+  if (nodeClients.length > 1) {
+    const newest = nodeClients.reduce((a, b) => (b.node!.connectedAt >= a.node!.connectedAt ? b : a));
+    enforceSingleNodeSubscriber(newKey, newest);
+  }
   return oldClients.length;
 }
 

package/src/server.ts

@@ -18,6 +18,7 @@ import { WebStandardStreamableHTTPServerTransport } from "@modelcontextprotocol/
 import { z } from "zod/v4";
 import { registerTools } from "./tools.js";
 import { db, logTaskEvent, logAudit, syncScheduledRunForTask } from "./db.js";
+import { recentNodeIdentityConflicts } from "./node-identity-conflict.js";
 import { createSSEStream, createNetworkObserverStream, createUserEventStream, pushEvent, pushNetworkObserverEvent, getSSEStats, PRINTABLE_OBSERVER_KEY_PREFIX, closeUserStreamsInNetwork, getUserPresence, onUserPresenceChange, pushUserEventToNetwork } from "./push.js";
 import { assertNodeActive } from "./lifecycle-guard.js";
 import { pendingInboxCount } from "./inbox-count.js";
@@ -245,6 +246,24 @@ function getClientIP(req: Request, server?: any): string {
   return fwd ? fwd.split(",")[0].trim() : (req.headers.get("x-real-ip") ?? "unknown");
 }
 
+// #507 — what identifies one node-scoped SSE connection (no tokens). instance_id is
+// optional and client-supplied (`X-Anet-Instance-Id` header or `?instance_id=`); no
+// released agent-node sends it yet, so absence means "unknown instance".
+const INSTANCE_ID_RE = /^[A-Za-z0-9][A-Za-z0-9._:-]{0,127}$/;
+function nodeSubscriberInfo(req: Request, server: any, tokenId: string | null, alias: string, networkId: string) {
+  const rawInstance = req.headers.get("x-anet-instance-id") ?? new URL(req.url).searchParams.get("instance_id");
+  const instanceId = rawInstance && INSTANCE_ID_RE.test(rawInstance) ? rawInstance : null;
+  const direct = server?.requestIP?.(req);
+  const fwd = req.headers.get("x-forwarded-for")?.split(",")[0]?.trim();
+  const hop = direct?.address ? `${direct.address}:${direct.port}` : null;
+  const remote = fwd ? `${fwd}${hop ? ` via ${hop}` : ""}` : hop;
+  const bound = tokenId
+    ? db.get<{ bound_node_id: string | null }>("SELECT bound_node_id FROM api_tokens WHERE token_id = ?1", tokenId)?.bound_node_id ?? null
+    : null;
+  const nodeId = bound ?? db.get<{ node_id: string | null }>("SELECT node_id FROM sessions WHERE alias = ?1 AND network_id = ?2", alias, networkId)?.node_id ?? null;
+  return { nodeId, instanceId, remote, userAgent: req.headers.get("user-agent")?.slice(0, 200) ?? null };
+}
+
 function isLocalhostIP(ip: string): boolean {
   return ip === "127.0.0.1" || ip === "::1" || ip === "::ffff:127.0.0.1" || ip === "localhost";
 }
@@ -1134,6 +1153,12 @@ return Bun.serve({
           nodeId: db.get<{ node_id: string | null }>("SELECT node_id FROM sessions WHERE alias = ?1 AND network_id = ?2", sessionName, scopedNetId)?.node_id ?? null,
         }));
         if (streamDenied) return streamDenied;
+        // #507 — the node's OWN stream (token bound to this alias) is node-scoped:
+        // at most one per (network, alias), newest wins. A node token watching some
+        // other alias stays a plain monitor and can never supersede that node.
+        if (authCtx.tokenName === `node:${sessionName}` && authCtx.networkId === scopedNetId) {
+          return createSSEStream(sessionName, scopedNetId, { node: nodeSubscriberInfo(req, server, authCtx.tokenId, sessionName, scopedNetId) });
+        }
         return createSSEStream(sessionName, scopedNetId);
       }
 
@@ -2573,13 +2598,13 @@ return Bun.serve({
         const bound = restAuth.networkId;
         const sse = getSSEStats();
         const sessions = Object.fromEntries(Object.entries(sse.sessions).filter(([key]) => key.split(":").includes(bound)));
-        return withCors(req, Response.json({ ok: true, total: Object.values(sessions).reduce((a, b) => a + b, 0), sessions, scope: "network", network_id: bound }));
+        return withCors(req, Response.json({ ok: true, total: Object.values(sessions).reduce((a, b) => a + b, 0), sessions, scope: "network", network_id: bound, identity_conflicts: recentNodeIdentityConflicts(bound) }));
       }
       if (restAuth && !isAdmin) {
         return withCors(req, Response.json({ ok: false, error: "admin or master token required" }, { status: 403 }));
       }
       const sse = getSSEStats();
-      return withCors(req, Response.json({ ok: true, total: sse.total, sessions: sse.sessions }));
+      return withCors(req, Response.json({ ok: true, total: sse.total, sessions: sse.sessions, identity_conflicts: recentNodeIdentityConflicts() }));
     }
 
     // ── REST: all sessions status ──

package/src/tools.ts

@@ -8,6 +8,7 @@ import { parseAliasFilter } from "./alias-filter.js";
 import { createHash } from "node:crypto";
 import { db, uuidv4, logTaskEvent, chainReplyToParent, hashToken, generateId, generateNetworkToken, syncScheduledRunForTask } from "./db.js";
 import { getSSEStats, hasSubscribers, hasUserSubscribers, pushEvent, pushNetworkObserverEvent, pushUserEvent } from "./push.js";
+import { deferOfflineIfAnotherCopyConnected } from "./node-identity-conflict.js";
 import { assertNodeActive } from "./lifecycle-guard.js";
 import { pendingInboxCount } from "./inbox-count.js";
 import { getUserNetworkRole, createNetworkTokenForNode } from "./auth.js";
@@ -963,6 +964,13 @@ export function registerTools(server: McpServer, clientIP?: string, enforceNetwo
           db.run("UPDATE api_tokens SET name = ?1 WHERE token_id = ?2", [`node:${effectiveAlias}`, callerTokenId]);
         } catch {}
       }
+      // #507 — a second copy of this node (same token + alias) may still be connected;
+      // its sibling's shutdown offline must not take the node offline under it.
+      const offlineDeferred = deferOfflineIfAnotherCopyConnected(effectiveAlias, sessionNetId, status, resume_id);
+      const priorStatus = offlineDeferred
+        ? db.get<{ status: string | null }>("SELECT status FROM sessions WHERE resume_id = ?1", resume_id)?.status
+        : null;
+      const storedStatus = offlineDeferred ? (priorStatus && priorStatus !== "offline" ? priorStatus : "idle") : status;
       const trimmedOutput = output?.slice(0, 4000);
       const hostHostname = host?.hostname || hn || null;
       const hostIp = host?.ip || clientIP || null;
@@ -1063,7 +1071,7 @@ export function registerTools(server: McpServer, clientIP?: string, enforceNetwo
              external_schedules = COALESCE(?34, sessions.external_schedules),
              peer_reply_inbox_capable = ?35,
              last_seen_at = datetime('now'), updated_at = datetime('now')`,
-          [resume_id, effectiveAlias, tmux ?? null, srv ?? null, hostIp, hostHostname, ag ?? null, pd ?? null, ver ?? null, status, task ?? null, trimmedOutput ?? null, progress ?? null, score ?? null, node_id ?? null, session_id ?? null, config_path ?? null, channels ?? null, sessionNetId, mdl ?? null, cpuLoad1m, cpuCores, memTotalGb, memUsedGb, memAvailGb, diskTotalGb, diskUsedGb, diskAvailGb, processRssBytes, processRssMb, processCpuPct, processUptimeSeconds, processInFlightCount, externalSchedulesJson, peerReplyInboxCapable ? 1 : 0]
+          [resume_id, effectiveAlias, tmux ?? null, srv ?? null, hostIp, hostHostname, ag ?? null, pd ?? null, ver ?? null, storedStatus, task ?? null, trimmedOutput ?? null, progress ?? null, score ?? null, node_id ?? null, session_id ?? null, config_path ?? null, channels ?? null, sessionNetId, mdl ?? null, cpuLoad1m, cpuCores, memTotalGb, memUsedGb, memAvailGb, diskTotalGb, diskUsedGb, diskAvailGb, processRssBytes, processRssMb, processCpuPct, processUptimeSeconds, processInFlightCount, externalSchedulesJson, peerReplyInboxCapable ? 1 : 0]
         );
         if (handover) {
           // 换 resume_id 那条路径上 ON CONFLICT 不会触发;INSERT 完再把没上报(仍为 NULL)的
@@ -1133,7 +1141,7 @@ export function registerTools(server: McpServer, clientIP?: string, enforceNetwo
         type: "status_update",
         alias: effectiveAlias,
         ...(canonical.renamed ? { renamed_from: alias } : {}),
-        status,
+        status: storedStatus,
         progress: progress ?? null,
         host: statusHostTelemetry,
         process_telemetry: statusProcessTelemetry,
@@ -1205,6 +1213,7 @@ export function registerTools(server: McpServer, clientIP?: string, enforceNetwo
               resume_id,
               alias: effectiveAlias,
               ...(canonical.renamed ? { renamed_from: alias } : {}),
+              ...(offlineDeferred ? { offline_deferred: true, reason: "node_identity_conflict" } : {}),
               inbox_count: row?.cnt ?? 0,
             }),
           },