@sleep2agi/commhub-server 0.9.0-preview.85 → 0.9.0-preview.87
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/model-auth-notify-http.test.ts +181 -0
- package/src/model-auth-notify.ts +132 -0
- package/src/node-health-dispatch-http.test.ts +279 -0
- package/src/node-health-guard.ts +105 -0
- package/src/scheduled-tasks.ts +14 -0
- package/src/server.ts +22 -0
- package/src/tools.ts +28 -4
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@sleep2agi/commhub-server",
|
|
3
|
-
"version": "0.9.0-preview.
|
|
3
|
+
"version": "0.9.0-preview.87",
|
|
4
4
|
"description": "CommHub Server — AI Agent communication hub with MCP protocol, multi-network isolation, user auth, and MCP tools (17 collaboration-core + node/provider ops tools; authoritative list: docs-site/docs/api/mcp-tools.md).",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "src/index.ts",
|
|
@@ -0,0 +1,181 @@
|
|
|
1
|
+
// #462 —— 节点模型登录失效 → 通知节点主人一次。
|
|
2
|
+
//
|
|
3
|
+
// 钉住:
|
|
4
|
+
// 1. 翻转判定(纯函数):进入 revoked/expired 发一次;坏→坏不再发(含 revoked⇄expired);回到 ok 才重新上膛;
|
|
5
|
+
// unknown / 没报既不发也不上膛。
|
|
6
|
+
// 2. 真 report_status(节点令牌):ok→expired→expired→ok→expired = 恰好 2 条,只给节点主人;
|
|
7
|
+
// 网络 owner 和别的成员 0 条。正文带别名、原因、`codex login` 修法,不教人拷别人的 auth.json。
|
|
8
|
+
// 3. 别的节点 / 用户令牌替它报坏状态 → 不发(与 #448 同一条「只有本节点令牌能替自己说健康」)。
|
|
9
|
+
// 4. Hub 重启(内存标记清空)后第一眼看到坏状态:主人还有没读的同类消息 → 不再发;读过了 → 发。
|
|
10
|
+
//
|
|
11
|
+
// 跑法:cd server && COMMHUB_DB=/tmp/x.db bun test src/model-auth-notify-http.test.ts
|
|
12
|
+
// PG:COMMHUB_TEST_PG_URL=… COMMHUB_PG_EXPERIMENTAL=1(tests/test2123-hub-postgres-ladder 里注册)
|
|
13
|
+
import { afterAll, beforeAll, beforeEach, describe, expect, test } from "bun:test";
|
|
14
|
+
import { mkdtempSync } from "node:fs";
|
|
15
|
+
import { join } from "node:path";
|
|
16
|
+
import { tmpdir } from "node:os";
|
|
17
|
+
import { Client } from "@modelcontextprotocol/sdk/client/index.js";
|
|
18
|
+
import { InMemoryTransport } from "@modelcontextprotocol/sdk/inMemory.js";
|
|
19
|
+
import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
|
|
20
|
+
import { addNetworkMember, createNetworkTokenForNode, register } from "./auth.js";
|
|
21
|
+
import { db } from "./db.js";
|
|
22
|
+
import { registerTools } from "./tools.js";
|
|
23
|
+
import { clearNodeHealthStore } from "./node-health-store.js";
|
|
24
|
+
import { __resetModelAuthNoticesForTest, decideModelAuthNotice, MODEL_AUTH_NOTICE_KIND, modelAuthNoticeText } from "./model-auth-notify.js";
|
|
25
|
+
|
|
26
|
+
const dir = mkdtempSync(join(tmpdir(), "anet-model-auth-notify-"));
|
|
27
|
+
const activeDbPath = process.env.COMMHUB_DB ?? (process.env.COMMHUB_TEST_PG_URL ? "postgres" : undefined);
|
|
28
|
+
if (!activeDbPath) throw new Error("model-auth-notify requires COMMHUB_DB (or COMMHUB_TEST_PG_URL) before module import");
|
|
29
|
+
|
|
30
|
+
const PW = "ModelAuthNotify123!";
|
|
31
|
+
const stamp = Date.now();
|
|
32
|
+
const NODE = `ma-node-${stamp}`;
|
|
33
|
+
const NODE_ID = `n_ma_node_${stamp}`;
|
|
34
|
+
const PEER = `ma-peer-${stamp}`;
|
|
35
|
+
const PEER_ID = `n_ma_peer_${stamp}`;
|
|
36
|
+
let netId = "";
|
|
37
|
+
let bossId = "", noraId = "", ottoId = "";
|
|
38
|
+
let nodeTokenId = "", peerTokenId = "";
|
|
39
|
+
|
|
40
|
+
async function connectAs(alias: string, userId: string, tokenId: string, isNetworkToken = true) {
|
|
41
|
+
const s = new McpServer({ name: "model-auth-notify", version: "1" });
|
|
42
|
+
registerTools(s, undefined, netId, userId, alias, isNetworkToken, tokenId);
|
|
43
|
+
const client = new Client({ name: "model-auth-notify-client", version: "1" });
|
|
44
|
+
const [ct, st] = InMemoryTransport.createLinkedPair();
|
|
45
|
+
await s.connect(st);
|
|
46
|
+
await client.connect(ct);
|
|
47
|
+
const report = async (model_auth: string | undefined, asAlias = NODE, nodeId = NODE_ID) => {
|
|
48
|
+
const r: any = await client.callTool({
|
|
49
|
+
name: "report_status",
|
|
50
|
+
arguments: { resume_id: `sdk-${nodeId}`, alias: asAlias, status: "idle", node_id: nodeId, network_id: netId, health: { bridge: "ok", ...(model_auth ? { model_auth } : {}) } },
|
|
51
|
+
});
|
|
52
|
+
expect(r.isError === true).toBe(false);
|
|
53
|
+
};
|
|
54
|
+
return { report, close: async () => { await client.close(); await s.close(); } };
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
const notices = (userId: string) => db.all<{ message_id: string; from_session: string; title: string; content: string; severity: string; meta_json: string; acked: number }>(
|
|
58
|
+
"SELECT message_id, from_session, title, content, severity, meta_json, acked FROM user_inbox WHERE user_id = ?1 AND network_id = ?2 AND kind = ?3 ORDER BY created_at, message_id",
|
|
59
|
+
userId, netId, MODEL_AUTH_NOTICE_KIND,
|
|
60
|
+
);
|
|
61
|
+
const clearNotices = () => db.run("DELETE FROM user_inbox WHERE network_id = ?1 AND kind = ?2", [netId, MODEL_AUTH_NOTICE_KIND]);
|
|
62
|
+
|
|
63
|
+
beforeAll(() => {
|
|
64
|
+
process.env.COMMHUB_UPLOADS_DIR = join(dir, "uploads");
|
|
65
|
+
const boss = register(`ma_boss_${stamp}`, PW);
|
|
66
|
+
expect(boss.ok).toBe(true);
|
|
67
|
+
bossId = boss.user!.user_id; netId = boss.network_id!;
|
|
68
|
+
const nora = register(`ma_nora_${stamp}`, PW);
|
|
69
|
+
const otto = register(`ma_otto_${stamp}`, PW);
|
|
70
|
+
noraId = nora.user!.user_id; ottoId = otto.user!.user_id;
|
|
71
|
+
addNetworkMember(netId, noraId, "member", bossId, { agentAccess: "all" });
|
|
72
|
+
addNetworkMember(netId, ottoId, "member", bossId, { agentAccess: "all" });
|
|
73
|
+
// 节点主人是 nora(她铸的节点令牌 → nodes.owner_user_id),不是网络 owner boss。
|
|
74
|
+
const minted = createNetworkTokenForNode(noraId, netId, NODE, NODE_ID);
|
|
75
|
+
expect(minted.ok).toBe(true);
|
|
76
|
+
nodeTokenId = minted.token_id!;
|
|
77
|
+
const peer = createNetworkTokenForNode(ottoId, netId, PEER, PEER_ID);
|
|
78
|
+
expect(peer.ok).toBe(true);
|
|
79
|
+
peerTokenId = peer.token_id!;
|
|
80
|
+
});
|
|
81
|
+
beforeEach(() => { __resetModelAuthNoticesForTest(); clearNodeHealthStore(); clearNotices(); });
|
|
82
|
+
afterAll(() => { clearNotices(); __resetModelAuthNoticesForTest(); clearNodeHealthStore(); });
|
|
83
|
+
|
|
84
|
+
describe("#462 transition logic", () => {
|
|
85
|
+
test("bad once per entry, re-armed only by ok", () => {
|
|
86
|
+
const seq: Array<Parameters<typeof decideModelAuthNotice>[1]> = ["ok", "expired", "expired", "revoked", "unknown", undefined, "expired", "ok", "unknown", "revoked", "revoked"];
|
|
87
|
+
let mark: ReturnType<typeof decideModelAuthNotice>["mark"];
|
|
88
|
+
const fired: boolean[] = [];
|
|
89
|
+
for (const s of seq) { const d = decideModelAuthNotice(mark, s); fired.push(d.notify); mark = d.mark; }
|
|
90
|
+
expect(fired).toEqual([false, true, false, false, false, false, false, false, false, true, false]);
|
|
91
|
+
});
|
|
92
|
+
|
|
93
|
+
test("first sight is bad → notify; unknown/missing never arms", () => {
|
|
94
|
+
expect(decideModelAuthNotice(undefined, "revoked")).toEqual({ notify: true, mark: "notified" });
|
|
95
|
+
expect(decideModelAuthNotice(undefined, "unknown")).toEqual({ notify: false, mark: undefined });
|
|
96
|
+
expect(decideModelAuthNotice("notified", "unknown")).toEqual({ notify: false, mark: "notified" });
|
|
97
|
+
expect(decideModelAuthNotice("notified", undefined)).toEqual({ notify: false, mark: "notified" });
|
|
98
|
+
expect(decideModelAuthNotice("notified", "ok")).toEqual({ notify: false, mark: "armed" });
|
|
99
|
+
});
|
|
100
|
+
|
|
101
|
+
test("text names the node, the cause and the per-node re-login; never says to copy credentials", () => {
|
|
102
|
+
for (const s of ["revoked", "expired"] as const) {
|
|
103
|
+
const { title, text } = modelAuthNoticeText("node-a", s);
|
|
104
|
+
expect(title).toBe("节点登录失效");
|
|
105
|
+
expect(text).toContain("节点 node-a");
|
|
106
|
+
expect(text).toContain("codex login");
|
|
107
|
+
expect(text).toContain("CODEX_HOME=");
|
|
108
|
+
expect(text).toContain("不要拷别的节点的 auth.json");
|
|
109
|
+
}
|
|
110
|
+
expect(modelAuthNoticeText("node-a", "revoked").text).toContain("作废");
|
|
111
|
+
expect(modelAuthNoticeText("node-a", "expired").text).toContain("过期");
|
|
112
|
+
});
|
|
113
|
+
});
|
|
114
|
+
|
|
115
|
+
describe("#462 report_status → owner notice", () => {
|
|
116
|
+
test("ok → expired → expired → ok → expired = exactly 2 notices, owner only", async () => {
|
|
117
|
+
const n = await connectAs(NODE, noraId, nodeTokenId);
|
|
118
|
+
try {
|
|
119
|
+
for (const s of ["ok", "expired", "expired", "ok", "expired"]) await n.report(s);
|
|
120
|
+
} finally { await n.close(); }
|
|
121
|
+
const mine = notices(noraId);
|
|
122
|
+
expect(mine.length).toBe(2);
|
|
123
|
+
for (const m of mine) {
|
|
124
|
+
expect(m.from_session).toBe(NODE);
|
|
125
|
+
expect(m.title).toBe("节点登录失效");
|
|
126
|
+
expect(m.severity).toBe("warning");
|
|
127
|
+
expect(m.content).toContain(`节点 ${NODE}`);
|
|
128
|
+
expect(m.content).toContain("codex login");
|
|
129
|
+
expect(JSON.parse(m.meta_json)).toEqual({ model_auth_notice: { alias: NODE, state: "expired" } });
|
|
130
|
+
}
|
|
131
|
+
expect(notices(bossId).length).toBe(0);
|
|
132
|
+
expect(notices(ottoId).length).toBe(0);
|
|
133
|
+
});
|
|
134
|
+
|
|
135
|
+
test("a node that only ever reports ok / unknown / nothing gets no notice", async () => {
|
|
136
|
+
const n = await connectAs(NODE, noraId, nodeTokenId);
|
|
137
|
+
try {
|
|
138
|
+
for (const s of ["unknown", "ok", undefined, "unknown", "ok"]) await n.report(s);
|
|
139
|
+
} finally { await n.close(); }
|
|
140
|
+
expect(notices(noraId).length).toBe(0);
|
|
141
|
+
});
|
|
142
|
+
|
|
143
|
+
test("another node or a user login cannot trigger a notice for this node", async () => {
|
|
144
|
+
const p = await connectAs(PEER, ottoId, peerTokenId);
|
|
145
|
+
try { await p.report("revoked", NODE, NODE_ID); } finally { await p.close(); }
|
|
146
|
+
const u = await connectAs("", bossId, "", false);
|
|
147
|
+
try { await u.report("revoked", NODE, NODE_ID); } finally { await u.close(); }
|
|
148
|
+
expect(notices(noraId).length).toBe(0);
|
|
149
|
+
expect(notices(bossId).length).toBe(0);
|
|
150
|
+
expect(notices(ottoId).length).toBe(0);
|
|
151
|
+
});
|
|
152
|
+
|
|
153
|
+
test("hub restart: an unread notice suppresses the repeat; once read, the next sight notifies", async () => {
|
|
154
|
+
const n = await connectAs(NODE, noraId, nodeTokenId);
|
|
155
|
+
try {
|
|
156
|
+
await n.report("revoked");
|
|
157
|
+
expect(notices(noraId).length).toBe(1);
|
|
158
|
+
__resetModelAuthNoticesForTest(); // = Hub 重启,内存标记没了
|
|
159
|
+
await n.report("revoked");
|
|
160
|
+
expect(notices(noraId).length).toBe(1);
|
|
161
|
+
db.run("UPDATE user_inbox SET acked = 1 WHERE user_id = ?1 AND network_id = ?2 AND kind = ?3", [noraId, netId, MODEL_AUTH_NOTICE_KIND]);
|
|
162
|
+
__resetModelAuthNoticesForTest();
|
|
163
|
+
await n.report("revoked");
|
|
164
|
+
expect(notices(noraId).length).toBe(2);
|
|
165
|
+
await n.report("revoked");
|
|
166
|
+
expect(notices(noraId).length).toBe(2);
|
|
167
|
+
} finally { await n.close(); }
|
|
168
|
+
});
|
|
169
|
+
|
|
170
|
+
test("owner no longer in the network → nobody is notified", async () => {
|
|
171
|
+
db.run("DELETE FROM network_members WHERE network_id = ?1 AND user_id = ?2", [netId, noraId]);
|
|
172
|
+
try {
|
|
173
|
+
const n = await connectAs(NODE, noraId, nodeTokenId);
|
|
174
|
+
try { await n.report("expired"); } finally { await n.close(); }
|
|
175
|
+
expect(notices(noraId).length).toBe(0);
|
|
176
|
+
expect(notices(bossId).length).toBe(0);
|
|
177
|
+
} finally {
|
|
178
|
+
addNetworkMember(netId, noraId, "member", bossId, { agentAccess: "all" });
|
|
179
|
+
}
|
|
180
|
+
});
|
|
181
|
+
});
|
|
@@ -0,0 +1,132 @@
|
|
|
1
|
+
// #462(#448 的子项)—— 节点模型登录失效时通知节点主人。
|
|
2
|
+
//
|
|
3
|
+
// 现场:codex 节点的 refresh token 被作废 / 过期后,节点自己知道(report_status.health.model_auth),
|
|
4
|
+
// Hub 也会拒绝给它派活(#460 node-health-guard.ts),但**没有人被告诉** —— 主人要等到某次派活被 409
|
|
5
|
+
// 才发现。这里在 report_status 收下健康报告的那一刻看 model_auth 的翻转:进入 revoked / expired 时
|
|
6
|
+
// 给节点主人发**一条**消息,说是哪个节点、出了什么事、怎么重新登录。
|
|
7
|
+
//
|
|
8
|
+
// 🔴 只检测 + 通知。不自动换号、不拷别的节点的 auth.json、不重新 stage 凭据(owner 定的边界)。
|
|
9
|
+
//
|
|
10
|
+
// 去重:每次「进入坏状态」只发一条;回到 ok 才重新上膛。unknown / 没报 model_auth 既不发也不上膛
|
|
11
|
+
// (一次模型调用都还没跑的节点报 unknown,不能把它当「恢复了」,否则 expired→unknown→expired 会刷屏)。
|
|
12
|
+
// revoked ⇄ expired 互相切换不算新的翻转 —— 修法是同一条命令。
|
|
13
|
+
// 状态在内存里(同 node-health-store.ts);Hub 重启后第一次看到坏状态时,先查收件人有没有**还没读**的同类
|
|
14
|
+
// 消息,有就不再发 —— 否则每次升级 Hub 都会给还没修好的节点主人再发一遍。
|
|
15
|
+
//
|
|
16
|
+
// 收件人 = 节点主人:上报令牌绑定的 nodes.owner_user_id(RFC-036,建节点时定死),没有绑定(老令牌)就取
|
|
17
|
+
// 铸这个节点令牌的用户(api_tokens.user_id —— 新令牌里两者相同)。只发给这一个人;不在本网络了就不发。
|
|
18
|
+
// 怎么送:user_inbox(与 send_desktop_message 同一张表、同一条 /events/users/me 推送),from_session = 节点别名,
|
|
19
|
+
// 所以 App 把它显示在**与该节点的会话**里(主动消息)+ 未读角标 + 通知。不新增渠道。
|
|
20
|
+
|
|
21
|
+
import { db, uuidv4 } from "./db.js";
|
|
22
|
+
import { getUserNetworkRole } from "./auth.js";
|
|
23
|
+
import { pushUserEvent } from "./push.js";
|
|
24
|
+
import { degradedLayers } from "./node-health-guard.js";
|
|
25
|
+
import type { NodeHealth } from "./node-health-store.js";
|
|
26
|
+
|
|
27
|
+
export const MODEL_AUTH_NOTICE_KIND = "node_model_auth";
|
|
28
|
+
const MAX_TRACKED = 5_000;
|
|
29
|
+
|
|
30
|
+
type ModelAuth = NodeHealth["model_auth"];
|
|
31
|
+
/** armed = 下一次进入坏状态要发;notified = 这一段坏状态已经发过。 */
|
|
32
|
+
export type NoticeMark = "armed" | "notified";
|
|
33
|
+
|
|
34
|
+
export const isBadModelAuth = (s: ModelAuth): s is "revoked" | "expired" => s === "revoked" || s === "expired";
|
|
35
|
+
|
|
36
|
+
/**
|
|
37
|
+
* 纯函数:上一份标记 + 这一份 model_auth → 发不发、新标记。
|
|
38
|
+
* prev = undefined 表示这个 Hub 进程还没见过这个节点(调用方另查库里有没有未读的旧通知)。
|
|
39
|
+
*/
|
|
40
|
+
export function decideModelAuthNotice(prev: NoticeMark | undefined, state: ModelAuth): { notify: boolean; mark: NoticeMark | undefined } {
|
|
41
|
+
if (state === "ok") return { notify: false, mark: "armed" };
|
|
42
|
+
if (isBadModelAuth(state)) {
|
|
43
|
+
if (prev === "notified") return { notify: false, mark: "notified" };
|
|
44
|
+
return { notify: true, mark: "notified" };
|
|
45
|
+
}
|
|
46
|
+
return { notify: false, mark: prev }; // unknown / 没报:保持原样
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
export function modelAuthNoticeText(alias: string, state: "revoked" | "expired"): { title: string; text: string } {
|
|
50
|
+
const what = state === "revoked"
|
|
51
|
+
? "模型登录凭据已被作废(refresh token revoked),需要重新登录"
|
|
52
|
+
: "模型登录已过期,且没能自动刷新";
|
|
53
|
+
// 修法与派发被拒时(#460)给的同一句,别出现两套说法。
|
|
54
|
+
const hint = degradedLayers({ model_auth: state })[0]?.hint ?? "";
|
|
55
|
+
return {
|
|
56
|
+
title: "节点登录失效",
|
|
57
|
+
text: `节点 ${alias} 的${what}。在它恢复之前,派给它的任务会被拒绝。\n\n怎么修:${hint}。\n登录好之后节点会自动报告恢复,不用再做别的。`,
|
|
58
|
+
};
|
|
59
|
+
}
|
|
60
|
+
|
|
61
|
+
const marks = new Map<string, NoticeMark>();
|
|
62
|
+
const keyOf = (networkId: string, alias: string) => `${networkId}\0${alias}`;
|
|
63
|
+
|
|
64
|
+
export function __resetModelAuthNoticesForTest(): void { marks.clear(); }
|
|
65
|
+
|
|
66
|
+
/** 上报令牌对应的节点主人。 */
|
|
67
|
+
function ownerOfNodeToken(tokenId: string, networkId: string): string | null {
|
|
68
|
+
const row = db.get<{ user_id: string | null; owner_user_id: string | null }>(
|
|
69
|
+
`SELECT t.user_id, n.owner_user_id FROM api_tokens t LEFT JOIN nodes n ON n.node_id = t.bound_node_id
|
|
70
|
+
WHERE t.token_id = ?1 AND t.network_id = ?2`,
|
|
71
|
+
tokenId, networkId,
|
|
72
|
+
);
|
|
73
|
+
return row?.owner_user_id || row?.user_id || null;
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
function hasUnreadNotice(userId: string, networkId: string, alias: string): boolean {
|
|
77
|
+
return !!db.get(
|
|
78
|
+
"SELECT 1 AS hit FROM user_inbox WHERE user_id = ?1 AND network_id = ?2 AND from_session = ?3 AND kind = ?4 AND acked = 0 LIMIT 1",
|
|
79
|
+
userId, networkId, alias, MODEL_AUTH_NOTICE_KIND,
|
|
80
|
+
);
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
/**
|
|
84
|
+
* report_status 收下一份(已验过是本节点令牌报的)健康报告之后调用。返回这次通知到的 user_id(测试用;没发 = [])。
|
|
85
|
+
* 任何异常都吞掉:这是附带的通知,不能让一份正常的 report_status 失败。
|
|
86
|
+
*/
|
|
87
|
+
export function noteModelAuthHealth(input: { networkId: string; alias: string; tokenId: string; health: NodeHealth }): string[] {
|
|
88
|
+
try {
|
|
89
|
+
const { networkId, alias, tokenId, health } = input;
|
|
90
|
+
const state = health.model_auth;
|
|
91
|
+
const key = keyOf(networkId, alias);
|
|
92
|
+
const prev = marks.get(key);
|
|
93
|
+
const { notify, mark } = decideModelAuthNotice(prev, state);
|
|
94
|
+
if (mark) {
|
|
95
|
+
marks.delete(key);
|
|
96
|
+
marks.set(key, mark);
|
|
97
|
+
if (marks.size > MAX_TRACKED) marks.delete(marks.keys().next().value!);
|
|
98
|
+
}
|
|
99
|
+
if (!notify || !isBadModelAuth(state)) return [];
|
|
100
|
+
|
|
101
|
+
const owner = ownerOfNodeToken(tokenId, networkId);
|
|
102
|
+
if (!owner || !getUserNetworkRole(owner, networkId)) return [];
|
|
103
|
+
// Hub 重启后的第一眼:主人还有一条没读的同类消息 → 这是同一段坏状态,不再发。
|
|
104
|
+
if (prev === undefined && hasUnreadNotice(owner, networkId, alias)) return [];
|
|
105
|
+
|
|
106
|
+
const { title, text } = modelAuthNoticeText(alias, state);
|
|
107
|
+
const messageId = `dm_auth_${uuidv4()}`;
|
|
108
|
+
const meta = { model_auth_notice: { alias, state } };
|
|
109
|
+
db.run(
|
|
110
|
+
`INSERT INTO user_inbox (message_id, network_id, user_id, from_session, kind, title, content, severity, meta_json)
|
|
111
|
+
VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, 'warning', ?8)
|
|
112
|
+
ON CONFLICT(message_id) DO NOTHING`,
|
|
113
|
+
[messageId, networkId, owner, alias, MODEL_AUTH_NOTICE_KIND, title, text, JSON.stringify(meta)],
|
|
114
|
+
);
|
|
115
|
+
// 推送在写库之后:收到推送的 App 立刻回读得到这一行。
|
|
116
|
+
pushUserEvent(networkId, owner, {
|
|
117
|
+
type: "desktop_message",
|
|
118
|
+
message_id: messageId,
|
|
119
|
+
kind: MODEL_AUTH_NOTICE_KIND,
|
|
120
|
+
from: alias,
|
|
121
|
+
title,
|
|
122
|
+
message: text,
|
|
123
|
+
severity: "warning",
|
|
124
|
+
created_at: new Date().toISOString(),
|
|
125
|
+
meta,
|
|
126
|
+
});
|
|
127
|
+
return [owner];
|
|
128
|
+
} catch (e: any) {
|
|
129
|
+
console.error(`[model-auth-notify] ${input.alias}: ${e?.message ?? e}`);
|
|
130
|
+
return [];
|
|
131
|
+
}
|
|
132
|
+
}
|
|
@@ -0,0 +1,279 @@
|
|
|
1
|
+
// #460(#448 子项)—— 降级节点拒收新任务。
|
|
2
|
+
//
|
|
3
|
+
// 钉住:
|
|
4
|
+
// 1. 新鲜健康报告说某层坏了(app_server.ok=false / tui.ok=false / model_auth revoked|expired)
|
|
5
|
+
// → REST /api/task 409、MCP send_task / retry_task / reassign_task 返回 node_degraded,带层、原因、修法;
|
|
6
|
+
// 定时任务的这一次 run 记 failed + error_code=node_degraded,不建任务行。
|
|
7
|
+
// 2. 不知道 = 放行:没有报告(老 agent-node)、报告过期(> TTL)、各层都 ok、某层没报 —— 与改动前逐字相同的成功路径。
|
|
8
|
+
// 3. 逃生口 force=true 只对用户令牌生效;节点令牌带 force 照样被拒(不让 agent 绕过对另一个 agent 的健康判断)。
|
|
9
|
+
// 4. 老客户端请求形状(不带 force)在健康 / 未知节点上完全照旧 —— 兼容回放。
|
|
10
|
+
//
|
|
11
|
+
// 跑法:cd server && COMMHUB_DB=/tmp/x.db bun test src/node-health-dispatch-http.test.ts
|
|
12
|
+
// PG:COMMHUB_TEST_PG_URL=… COMMHUB_PG_EXPERIMENTAL=1(tests/test2123-hub-postgres-ladder 里注册)
|
|
13
|
+
import { afterAll, beforeAll, beforeEach, describe, expect, test } from "bun:test";
|
|
14
|
+
import { mkdtempSync } from "node:fs";
|
|
15
|
+
import { join } from "node:path";
|
|
16
|
+
import { tmpdir } from "node:os";
|
|
17
|
+
import { Client } from "@modelcontextprotocol/sdk/client/index.js";
|
|
18
|
+
import { InMemoryTransport } from "@modelcontextprotocol/sdk/inMemory.js";
|
|
19
|
+
import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
|
|
20
|
+
import { createNetworkTokenForNode, register } from "./auth.js";
|
|
21
|
+
import { db } from "./db.js";
|
|
22
|
+
import { registerTools } from "./tools.js";
|
|
23
|
+
import { clearNodeHealthStore, NODE_HEALTH_TTL_MS, recordNodeHealth } from "./node-health-store.js";
|
|
24
|
+
import { degradedLayers } from "./node-health-guard.js";
|
|
25
|
+
import { runDueScheduledTasks } from "./scheduled-tasks.js";
|
|
26
|
+
|
|
27
|
+
const dir = mkdtempSync(join(tmpdir(), "anet-health-dispatch-"));
|
|
28
|
+
const activeDbPath = process.env.COMMHUB_DB ?? (process.env.COMMHUB_TEST_PG_URL ? "postgres" : undefined);
|
|
29
|
+
if (!activeDbPath) throw new Error("node-health-dispatch requires COMMHUB_DB (or COMMHUB_TEST_PG_URL) before module import");
|
|
30
|
+
|
|
31
|
+
let server: any;
|
|
32
|
+
let base = "";
|
|
33
|
+
let ownerToken = "", ownerId = "", networkId = "";
|
|
34
|
+
let peerNodeToken = "", peerNodeTokenId = "";
|
|
35
|
+
const stamp = Date.now();
|
|
36
|
+
const TARGET = `hd-target-${stamp}`;
|
|
37
|
+
const TARGET_NODE = `n_hd_target_${stamp}`;
|
|
38
|
+
const PEER = `hd-peer-${stamp}`;
|
|
39
|
+
const PEER_NODE = `n_hd_peer_${stamp}`;
|
|
40
|
+
let seq = 0;
|
|
41
|
+
const text = (what: string) => `${what} #${++seq}`; // distinct content: the 5-minute dedup must not interfere
|
|
42
|
+
|
|
43
|
+
async function api(token: string, path: string, init?: RequestInit) {
|
|
44
|
+
const res = await fetch(`${base}${path}`, {
|
|
45
|
+
...init,
|
|
46
|
+
headers: { Authorization: `Bearer ${token}`, "Content-Type": "application/json", ...(init?.headers || {}) },
|
|
47
|
+
});
|
|
48
|
+
return { status: res.status, body: await res.json() as any };
|
|
49
|
+
}
|
|
50
|
+
const postTask = (token: string, extra: Record<string, unknown> = {}) =>
|
|
51
|
+
api(token, "/api/task", { method: "POST", body: JSON.stringify({ alias: TARGET, task: text("rest"), network_id: networkId, ...extra }) });
|
|
52
|
+
|
|
53
|
+
async function mcpAs(identity: { net: string | null; user: string; alias: string; isNetworkToken: boolean; tokenId: string }) {
|
|
54
|
+
const s = new McpServer({ name: "health-dispatch", version: "1" });
|
|
55
|
+
registerTools(s, undefined, identity.net, identity.user, identity.alias, identity.isNetworkToken, identity.tokenId);
|
|
56
|
+
const client = new Client({ name: "health-dispatch-client", version: "1" });
|
|
57
|
+
const [ct, st] = InMemoryTransport.createLinkedPair();
|
|
58
|
+
await s.connect(st);
|
|
59
|
+
await client.connect(ct);
|
|
60
|
+
const call = async (name: string, args: Record<string, unknown>) => {
|
|
61
|
+
const r: any = await client.callTool({ name, arguments: args });
|
|
62
|
+
const raw = r.content?.[0]?.text ?? "null";
|
|
63
|
+
try { return JSON.parse(raw); } catch { return { raw }; }
|
|
64
|
+
};
|
|
65
|
+
return { call, close: async () => { await client.close(); await s.close(); } };
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
const DOWN_APP = { bridge: "ok" as const, app_server: { ok: false, rtt_ms: null, last_error: "connect ECONNREFUSED 127.0.0.1:4500" }, model_auth: "ok" as const };
|
|
69
|
+
const HEALTHY = { bridge: "ok" as const, app_server: { ok: true, rtt_ms: 3, last_error: null }, tui: { ok: true, reason: "running" }, model_auth: "ok" as const };
|
|
70
|
+
const setHealth = (h: any, at?: number) => recordNodeHealth(networkId, TARGET, h, at);
|
|
71
|
+
const taskCount = () => db.get<{ n: number }>("SELECT COUNT(*) AS n FROM tasks WHERE to_name = ?1 AND network_id = ?2", TARGET, networkId)!.n;
|
|
72
|
+
|
|
73
|
+
beforeAll(async () => {
|
|
74
|
+
process.env.COMMHUB_UPLOADS_DIR = join(dir, "uploads");
|
|
75
|
+
process.env.HOST = "127.0.0.1";
|
|
76
|
+
const owner = register(`hd_owner_${stamp}`, "HealthDispatchOwner123!", undefined, "seed");
|
|
77
|
+
ownerToken = owner.token!; ownerId = owner.user!.user_id; networkId = owner.network_id!;
|
|
78
|
+
for (const [nodeId, alias] of [[TARGET_NODE, TARGET], [PEER_NODE, PEER]]) {
|
|
79
|
+
db.run(`INSERT INTO nodes (node_id, node_name, alias, runtime, network_id, owner_user_id) VALUES (?1, ?2, ?2, 'codex-app-server', ?3, ?4)`, [nodeId, alias, networkId, ownerId]);
|
|
80
|
+
db.run(`INSERT INTO sessions (resume_id, alias, status, node_id, network_id) VALUES (?1, ?2, 'idle', ?3, ?4)`, [`r_${nodeId}`, alias, nodeId, networkId]);
|
|
81
|
+
}
|
|
82
|
+
const minted = createNetworkTokenForNode(ownerId, networkId, PEER, PEER_NODE);
|
|
83
|
+
peerNodeToken = minted.token!; peerNodeTokenId = minted.token_id!;
|
|
84
|
+
const mod: any = await import("./server.js");
|
|
85
|
+
server = mod.bootServer({ port: 0, hostname: "127.0.0.1" });
|
|
86
|
+
base = `http://127.0.0.1:${server.port}`;
|
|
87
|
+
}, 30_000);
|
|
88
|
+
afterAll(() => { server?.stop?.(true); clearNodeHealthStore(); });
|
|
89
|
+
beforeEach(() => clearNodeHealthStore());
|
|
90
|
+
|
|
91
|
+
describe("#460 which layers count as degraded", () => {
|
|
92
|
+
test("each failing layer is named; ok / missing layers are not", () => {
|
|
93
|
+
expect(degradedLayers(null)).toEqual([]);
|
|
94
|
+
expect(degradedLayers(HEALTHY)).toEqual([]);
|
|
95
|
+
expect(degradedLayers({ bridge: "ok", model_auth: "unknown" })).toEqual([]);
|
|
96
|
+
expect(degradedLayers(DOWN_APP).map(l => [l.layer, l.label])).toEqual([["app_server", "App Server 断开"]]);
|
|
97
|
+
expect(degradedLayers({ tui: { ok: false, reason: "sleep-placeholder" } }).map(l => [l.layer, l.reason, l.label])).toEqual([["tui", "sleep-placeholder", "TUI 被占位(sleep)"]]);
|
|
98
|
+
expect(degradedLayers({ model_auth: "revoked" }).map(l => l.label)).toEqual(["需要重新登录"]);
|
|
99
|
+
expect(degradedLayers({ model_auth: "expired" }).map(l => l.label)).toEqual(["登录已过期"]);
|
|
100
|
+
// 修法只指向本节点重新登录,不教人拷别人的凭据
|
|
101
|
+
expect(degradedLayers({ model_auth: "revoked" })[0].hint).toContain("不要拷别的节点的 auth.json");
|
|
102
|
+
});
|
|
103
|
+
});
|
|
104
|
+
|
|
105
|
+
describe("#460 REST /api/task", () => {
|
|
106
|
+
test("unknown health (old agent-node) → dispatched exactly as before", async () => {
|
|
107
|
+
const before = taskCount();
|
|
108
|
+
const r = await postTask(ownerToken);
|
|
109
|
+
expect(r.status).toBe(200);
|
|
110
|
+
expect(r.body.ok).toBe(true);
|
|
111
|
+
expect(taskCount()).toBe(before + 1);
|
|
112
|
+
});
|
|
113
|
+
|
|
114
|
+
test("healthy report → dispatched", async () => {
|
|
115
|
+
setHealth(HEALTHY);
|
|
116
|
+
expect((await postTask(ownerToken)).status).toBe(200);
|
|
117
|
+
});
|
|
118
|
+
|
|
119
|
+
test("App Server down → 409 node_degraded with layer, reason and hint; no task row", async () => {
|
|
120
|
+
setHealth(DOWN_APP);
|
|
121
|
+
const before = taskCount();
|
|
122
|
+
const r = await postTask(ownerToken);
|
|
123
|
+
expect(r.status).toBe(409);
|
|
124
|
+
expect(r.body.ok).toBe(false);
|
|
125
|
+
expect(r.body.error).toBe("node_degraded");
|
|
126
|
+
expect(r.body.alias).toBe(TARGET);
|
|
127
|
+
expect(r.body.layers.map((l: any) => l.layer)).toEqual(["app_server"]);
|
|
128
|
+
expect(r.body.layers[0].reason).toContain("ECONNREFUSED");
|
|
129
|
+
expect(r.body.message).toContain("App Server 断开");
|
|
130
|
+
expect(r.body.hint).toContain("force=true");
|
|
131
|
+
expect(r.body.force_allowed).toBe(true);
|
|
132
|
+
expect(typeof r.body.health_observed_ms_ago).toBe("number");
|
|
133
|
+
expect(taskCount()).toBe(before);
|
|
134
|
+
});
|
|
135
|
+
|
|
136
|
+
test("TUI gone and login revoked are refused too, all layers listed", async () => {
|
|
137
|
+
setHealth({ bridge: "ok", app_server: { ok: true, rtt_ms: 2, last_error: null }, tui: { ok: false, reason: "session-missing" }, model_auth: "revoked" });
|
|
138
|
+
const r = await postTask(ownerToken);
|
|
139
|
+
expect(r.status).toBe(409);
|
|
140
|
+
expect(r.body.layers.map((l: any) => l.layer)).toEqual(["tui", "model_auth"]);
|
|
141
|
+
});
|
|
142
|
+
|
|
143
|
+
test("expired login alone is refused", async () => {
|
|
144
|
+
setHealth({ bridge: "ok", model_auth: "expired" });
|
|
145
|
+
expect((await postTask(ownerToken)).body.error).toBe("node_degraded");
|
|
146
|
+
});
|
|
147
|
+
|
|
148
|
+
test("stale report (older than the TTL) = unknown → dispatched", async () => {
|
|
149
|
+
setHealth(DOWN_APP, Date.now() - NODE_HEALTH_TTL_MS - 1_000);
|
|
150
|
+
expect((await postTask(ownerToken)).status).toBe(200);
|
|
151
|
+
});
|
|
152
|
+
|
|
153
|
+
test("owner escape hatch: force=true with a user token dispatches", async () => {
|
|
154
|
+
setHealth(DOWN_APP);
|
|
155
|
+
const before = taskCount();
|
|
156
|
+
const r = await postTask(ownerToken, { force: true });
|
|
157
|
+
expect(r.status).toBe(200);
|
|
158
|
+
expect(taskCount()).toBe(before + 1);
|
|
159
|
+
});
|
|
160
|
+
|
|
161
|
+
test("a node token cannot force past another node's health", async () => {
|
|
162
|
+
setHealth(DOWN_APP);
|
|
163
|
+
const r = await api(peerNodeToken, "/api/task", { method: "POST", body: JSON.stringify({ alias: TARGET, task: text("node-force"), force: true }) });
|
|
164
|
+
expect(r.status).toBe(409);
|
|
165
|
+
expect(r.body.error).toBe("node_degraded");
|
|
166
|
+
expect(r.body.force_allowed).toBe(false);
|
|
167
|
+
expect(r.body.hint).not.toContain("force=true");
|
|
168
|
+
});
|
|
169
|
+
|
|
170
|
+
test("old-client request shapes are untouched on a healthy node (compat replay)", async () => {
|
|
171
|
+
setHealth(HEALTHY);
|
|
172
|
+
// the app's send (meta + client_request_id), a plain anet send, and a parent-linked dispatch
|
|
173
|
+
const shapes = [
|
|
174
|
+
{ alias: TARGET, task: text("app"), network_id: networkId, priority: "normal", meta: { client_request_id: `cr_${stamp}_${seq}` } },
|
|
175
|
+
{ alias: TARGET, task: text("anet"), network_id: networkId },
|
|
176
|
+
{ alias: TARGET, task: text("ttl"), network_id: networkId, priority: "high", ttl_seconds: 600 },
|
|
177
|
+
];
|
|
178
|
+
for (const shape of shapes) {
|
|
179
|
+
const r = await api(ownerToken, "/api/task", { method: "POST", body: JSON.stringify(shape) });
|
|
180
|
+
expect(r.status).toBe(200);
|
|
181
|
+
expect(r.body.ok).toBe(true);
|
|
182
|
+
}
|
|
183
|
+
});
|
|
184
|
+
});
|
|
185
|
+
|
|
186
|
+
describe("#460 MCP send_task / retry_task / reassign_task", () => {
|
|
187
|
+
const asOwner = () => ({ net: null, user: ownerId, alias: `hd_owner_${stamp}`, isNetworkToken: false, tokenId: "tok_hd_owner_login" });
|
|
188
|
+
const asPeerNode = () => ({ net: networkId, user: ownerId, alias: PEER, isNetworkToken: true, tokenId: peerNodeTokenId });
|
|
189
|
+
|
|
190
|
+
test("send_task from an agent to a degraded node → node_degraded (no queueing); force ignored for node tokens", async () => {
|
|
191
|
+
setHealth(DOWN_APP);
|
|
192
|
+
const peer = await mcpAs(asPeerNode());
|
|
193
|
+
try {
|
|
194
|
+
const before = taskCount();
|
|
195
|
+
const r = await peer.call("send_task", { alias: TARGET, task: text("mcp"), force: true });
|
|
196
|
+
expect(r.ok).toBe(false);
|
|
197
|
+
expect(r.error).toBe("node_degraded");
|
|
198
|
+
expect(r.force_allowed).toBe(false);
|
|
199
|
+
expect(taskCount()).toBe(before);
|
|
200
|
+
} finally { await peer.close(); }
|
|
201
|
+
});
|
|
202
|
+
|
|
203
|
+
test("send_task: unknown health → ok; owner force → ok", async () => {
|
|
204
|
+
const owner = await mcpAs(asOwner());
|
|
205
|
+
try {
|
|
206
|
+
expect((await owner.call("send_task", { alias: TARGET, task: text("mcp-unknown"), network_id: networkId })).ok).toBe(true);
|
|
207
|
+
setHealth({ bridge: "ok", model_auth: "revoked" });
|
|
208
|
+
expect((await owner.call("send_task", { alias: TARGET, task: text("mcp-refused"), network_id: networkId })).error).toBe("node_degraded");
|
|
209
|
+
expect((await owner.call("send_task", { alias: TARGET, task: text("mcp-forced"), network_id: networkId, force: true })).ok).toBe(true);
|
|
210
|
+
} finally { await owner.close(); }
|
|
211
|
+
});
|
|
212
|
+
|
|
213
|
+
test("retry_task and reassign_task onto a degraded node are refused", async () => {
|
|
214
|
+
const owner = await mcpAs(asOwner());
|
|
215
|
+
try {
|
|
216
|
+
const sent = await owner.call("send_task", { alias: PEER, task: text("to-peer"), network_id: networkId });
|
|
217
|
+
expect(sent.ok).toBe(true);
|
|
218
|
+
setHealth(DOWN_APP);
|
|
219
|
+
const re = await owner.call("reassign_task", { task_id: sent.task_id ?? sent.message_id, new_alias: TARGET, network_id: networkId });
|
|
220
|
+
expect(re.error).toBe("node_degraded");
|
|
221
|
+
const failed = await owner.call("send_task", { alias: TARGET, task: text("to-fail"), network_id: networkId, force: true });
|
|
222
|
+
const failedId = failed.task_id ?? failed.message_id;
|
|
223
|
+
db.run("UPDATE tasks SET status = 'failed' WHERE task_id = ?1", [failedId]);
|
|
224
|
+
const retry = await owner.call("retry_task", { task_id: failedId, network_id: networkId });
|
|
225
|
+
expect(retry.error).toBe("node_degraded");
|
|
226
|
+
clearNodeHealthStore();
|
|
227
|
+
expect((await owner.call("retry_task", { task_id: failedId, network_id: networkId })).ok).toBe(true);
|
|
228
|
+
} finally { await owner.close(); }
|
|
229
|
+
});
|
|
230
|
+
});
|
|
231
|
+
|
|
232
|
+
describe("#460 scheduled tasks", () => {
|
|
233
|
+
test("a due run on a degraded node fails with node_degraded and creates no task", async () => {
|
|
234
|
+
const created = await api(ownerToken, "/api/scheduled-tasks", {
|
|
235
|
+
method: "POST",
|
|
236
|
+
body: JSON.stringify({ network_id: networkId, name: `hd-schedule-${stamp}`, target_node_id: TARGET_NODE, task: text("scheduled"), timezone: "UTC", schedule: { type: "interval", every_seconds: 60 } }),
|
|
237
|
+
});
|
|
238
|
+
expect(created.status).toBe(201);
|
|
239
|
+
const scheduleId = created.body.schedule.schedule_id;
|
|
240
|
+
setHealth({ bridge: "ok", tui: { ok: false, reason: "pane-dead" }, model_auth: "ok" });
|
|
241
|
+
const before = taskCount();
|
|
242
|
+
db.run("UPDATE scheduled_tasks SET next_run_at = ?1 WHERE schedule_id = ?2", [new Date(Date.now() - 60_000).toISOString(), scheduleId]);
|
|
243
|
+
runDueScheduledTasks();
|
|
244
|
+
const run = db.get<{ status: string; error_code: string | null; error_message: string | null; task_id: string | null }>(
|
|
245
|
+
"SELECT status, error_code, error_message, task_id FROM scheduled_task_runs WHERE schedule_id = ?1 ORDER BY created_at DESC LIMIT 1", scheduleId);
|
|
246
|
+
expect(run?.status).toBe("failed");
|
|
247
|
+
expect(run?.error_code).toBe("node_degraded");
|
|
248
|
+
expect(run?.error_message).toContain("TUI 已退出");
|
|
249
|
+
expect(run?.task_id ?? null).toBeNull();
|
|
250
|
+
expect(taskCount()).toBe(before);
|
|
251
|
+
});
|
|
252
|
+
});
|
|
253
|
+
|
|
254
|
+
describe("#460 /api/status carries `degraded` only for degraded nodes", () => {
|
|
255
|
+
const row = async (light: boolean) => {
|
|
256
|
+
const r = await api(ownerToken, `/api/status?network_id=${encodeURIComponent(networkId)}${light ? "&light=1" : ""}`);
|
|
257
|
+
expect(r.status).toBe(200);
|
|
258
|
+
return r.body.sessions.find((x: any) => x.alias === TARGET);
|
|
259
|
+
};
|
|
260
|
+
test("unknown / healthy → no `degraded` key at all (light response bytes unchanged)", async () => {
|
|
261
|
+
expect("degraded" in (await row(true))).toBe(false);
|
|
262
|
+
setHealth(HEALTHY);
|
|
263
|
+
expect("degraded" in (await row(true))).toBe(false);
|
|
264
|
+
expect("degraded" in (await row(false))).toBe(false);
|
|
265
|
+
});
|
|
266
|
+
test("degraded → light and full rows list the failing layers with the app's labels", async () => {
|
|
267
|
+
setHealth({ bridge: "ok", app_server: { ok: false, rtt_ms: null, last_error: "refused" }, model_auth: "revoked" });
|
|
268
|
+
const want = [
|
|
269
|
+
{ layer: "app_server", label: "App Server 断开", reason: "refused" },
|
|
270
|
+
{ layer: "model_auth", label: "需要重新登录", reason: "revoked" },
|
|
271
|
+
];
|
|
272
|
+
expect((await row(true)).degraded).toEqual(want);
|
|
273
|
+
expect((await row(false)).degraded).toEqual(want);
|
|
274
|
+
});
|
|
275
|
+
test("stale → no `degraded`", async () => {
|
|
276
|
+
setHealth(DOWN_APP, Date.now() - NODE_HEALTH_TTL_MS - 1_000);
|
|
277
|
+
expect("degraded" in (await row(true))).toBe(false);
|
|
278
|
+
});
|
|
279
|
+
});
|
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
// #460(#448 的子项)—— 降级节点拒收新任务。
|
|
2
|
+
//
|
|
3
|
+
// 现场:节点在 Hub 上 idle / in_flight=0,实际 app-server 端口没人听、TUI 变成 `sleep infinity`、
|
|
4
|
+
// 或登录态 revoked —— 派过去的任务**静默排队**,谁都不知道它永远不会被执行。agent-node .94 起
|
|
5
|
+
// 把这几层报上来(report_status.health,node-health-store.ts 只存内存、10 分钟 TTL);这里在**派发**
|
|
6
|
+
// 那一刻读它:新鲜的健康报告说某一层坏了,就直接拒,带上是哪一层、怎么修。
|
|
7
|
+
//
|
|
8
|
+
// 🔴 只拒「确知坏了」的:没有报告(老 agent-node / claude-code 节点)、报告已过期(>TTL)、
|
|
9
|
+
// 或某层没报(比如非共存节点没有 tui)= 不知道 = 放行。宁可像以前一样排队,
|
|
10
|
+
// 也不能因为缺数据把一台正常的老节点挡在外面。
|
|
11
|
+
// 🔴 只管**新任务派发**(send_task / REST /api/task / 定时任务 / retry / reassign / broadcast)。
|
|
12
|
+
// 回复、send_message、ack 不经过这里 —— 降级节点发出的回复照常送达,发给它的普通消息也照常入库。
|
|
13
|
+
// 逃生口:用户(不是节点令牌)可以带 force=true 强制派发 —— 比如明知 TUI 不在、但 bridge 能干活。
|
|
14
|
+
// 节点令牌不能 force:不让 agent 绕过对另一个 agent 的健康判断。
|
|
15
|
+
|
|
16
|
+
import { readNodeHealth, type NodeHealth } from "./node-health-store.js";
|
|
17
|
+
|
|
18
|
+
export type DegradedLayer = {
|
|
19
|
+
layer: "app_server" | "tui" | "model_auth";
|
|
20
|
+
/** 机器可读的原因:app_server 的错误摘要 / tui 的 reason / model_auth 的状态。 */
|
|
21
|
+
reason: string;
|
|
22
|
+
/** 给人看的一句(中文,与 app 的「降级」徽标同一套措辞)。 */
|
|
23
|
+
label: string;
|
|
24
|
+
hint: string;
|
|
25
|
+
};
|
|
26
|
+
|
|
27
|
+
const TUI_LABEL: Record<string, string> = {
|
|
28
|
+
"session-missing": "TUI 会话不在",
|
|
29
|
+
"pane-dead": "TUI 已退出",
|
|
30
|
+
"sleep-placeholder": "TUI 被占位(sleep)",
|
|
31
|
+
"tmux-unavailable": "tmux 不可用",
|
|
32
|
+
};
|
|
33
|
+
|
|
34
|
+
/** 一份健康报告里确知坏掉的层;没报的层不算。 */
|
|
35
|
+
export function degradedLayers(health: NodeHealth | null | undefined): DegradedLayer[] {
|
|
36
|
+
if (!health) return [];
|
|
37
|
+
const out: DegradedLayer[] = [];
|
|
38
|
+
if (health.app_server && health.app_server.ok === false) {
|
|
39
|
+
out.push({
|
|
40
|
+
layer: "app_server",
|
|
41
|
+
reason: (health.app_server.last_error || "unreachable").slice(0, 200),
|
|
42
|
+
label: "App Server 断开",
|
|
43
|
+
hint: "节点的 codex app-server 端口没有响应:在节点所在机器上重启该节点(anet node restart <别名>)",
|
|
44
|
+
});
|
|
45
|
+
}
|
|
46
|
+
if (health.tui && health.tui.ok === false) {
|
|
47
|
+
out.push({
|
|
48
|
+
layer: "tui",
|
|
49
|
+
reason: health.tui.reason,
|
|
50
|
+
label: TUI_LABEL[health.tui.reason] ?? "TUI 不可用",
|
|
51
|
+
hint: "共存 TUI 不在运行:在节点所在机器上重新拉起该节点的 TUI(anet node restart <别名>)",
|
|
52
|
+
});
|
|
53
|
+
}
|
|
54
|
+
if (health.model_auth === "revoked" || health.model_auth === "expired") {
|
|
55
|
+
out.push({
|
|
56
|
+
layer: "model_auth",
|
|
57
|
+
reason: health.model_auth,
|
|
58
|
+
label: health.model_auth === "revoked" ? "需要重新登录" : "登录已过期",
|
|
59
|
+
hint: "该节点自己的 CODEX_HOME 需要重新登录(在那台机器上:CODEX_HOME=<节点目录>/codex-home codex login);不要拷别的节点的 auth.json",
|
|
60
|
+
});
|
|
61
|
+
}
|
|
62
|
+
return out;
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
export type HealthGuardResult =
|
|
66
|
+
| { ok: true; forced?: true }
|
|
67
|
+
| {
|
|
68
|
+
ok: false;
|
|
69
|
+
error: "node_degraded";
|
|
70
|
+
alias: string;
|
|
71
|
+
layers: DegradedLayer[];
|
|
72
|
+
message: string;
|
|
73
|
+
hint: string;
|
|
74
|
+
health_observed_ms_ago: number;
|
|
75
|
+
/** 能不能用 force=true 绕过(只有用户令牌能)。 */
|
|
76
|
+
force_allowed: boolean;
|
|
77
|
+
};
|
|
78
|
+
|
|
79
|
+
/**
|
|
80
|
+
* 派发前的健康门。networkId 用与 report_status 写入时同一个键(sessionNetId = 网络 id ?? "default")。
|
|
81
|
+
* force 只在 forceAllowed(用户令牌)时生效。
|
|
82
|
+
*/
|
|
83
|
+
export function assertNodeHealthy(
|
|
84
|
+
alias: string,
|
|
85
|
+
networkId: string | null | undefined,
|
|
86
|
+
opts: { force?: boolean; forceAllowed: boolean; now?: number },
|
|
87
|
+
): HealthGuardResult {
|
|
88
|
+
if (!alias) return { ok: true };
|
|
89
|
+
const hit = readNodeHealth(networkId ?? "default", alias, opts.now);
|
|
90
|
+
if (!hit) return { ok: true };
|
|
91
|
+
const layers = degradedLayers(hit.health);
|
|
92
|
+
if (!layers.length) return { ok: true };
|
|
93
|
+
if (opts.force && opts.forceAllowed) return { ok: true, forced: true };
|
|
94
|
+
const labels = layers.map(l => l.label).join("、");
|
|
95
|
+
return {
|
|
96
|
+
ok: false,
|
|
97
|
+
error: "node_degraded",
|
|
98
|
+
alias,
|
|
99
|
+
layers,
|
|
100
|
+
message: `节点 ${alias} 处于降级状态(${labels}),任务不会被执行,已拒绝派发。`,
|
|
101
|
+
hint: layers[0].hint + (opts.forceAllowed ? "。确需强制派发可带 force=true。" : ""),
|
|
102
|
+
health_observed_ms_ago: hit.observed_ms_ago,
|
|
103
|
+
force_allowed: opts.forceAllowed,
|
|
104
|
+
};
|
|
105
|
+
}
|
package/src/scheduled-tasks.ts
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { db, logTaskEvent } from "./db.js";
|
|
2
2
|
import { assertNodeActive } from "./lifecycle-guard.js";
|
|
3
|
+
import { assertNodeHealthy } from "./node-health-guard.js";
|
|
3
4
|
import { addNetworkScope, canRestWriteNetwork, singleNetworkId, type RestNetworkScope } from "./network-scope.js";
|
|
4
5
|
import { canMessageAgent } from "./agent-access.js";
|
|
5
6
|
import { hasSubscribers, pushEvent, pushNetworkObserverEvent } from "./push.js";
|
|
@@ -290,6 +291,19 @@ export function dispatchScheduledOccurrence(row: ScheduledRow, scheduledFor: str
|
|
|
290
291
|
return;
|
|
291
292
|
}
|
|
292
293
|
|
|
294
|
+
// #460 — the target reports a degraded layer (App Server down / TUI gone / login revoked): the run fails
|
|
295
|
+
// with that reason instead of queueing a task nobody will execute. Unknown / stale health runs as before.
|
|
296
|
+
const health = assertNodeHealthy(node.alias, row.network_id, { forceAllowed: false });
|
|
297
|
+
if (!health.ok) {
|
|
298
|
+
finalStatus = "failed";
|
|
299
|
+
db.run(
|
|
300
|
+
"UPDATE scheduled_task_runs SET status = 'failed', error_code = 'node_degraded', error_message = ?1, completed_at = datetime('now') WHERE run_id = ?2",
|
|
301
|
+
[health.message.slice(0, 500), runId],
|
|
302
|
+
);
|
|
303
|
+
if (advanceSchedule) advance(row, scheduledFor, advanceAfter);
|
|
304
|
+
return;
|
|
305
|
+
}
|
|
306
|
+
|
|
293
307
|
const taskId = crypto.randomUUID();
|
|
294
308
|
// claude-code 裸节点的 node-server 上报 report_status 不带 node_id,它的 sessions 行
|
|
295
309
|
// node_id 永远是 NULL —— 只按 node_id 找会话,这类节点每一次都判成 queued、门铃不响,
|
package/src/server.ts
CHANGED
|
@@ -2,6 +2,7 @@ import { buildServeErrorResponse } from "./serve-error.js";
|
|
|
2
2
|
import { maybeGzipResponse, trimLightTask } from "./http-gzip";
|
|
3
3
|
import { statusAliasResolverRead } from "./status-alias-resolver.js";
|
|
4
4
|
import { readNodeHealth } from "./node-health-store.js";
|
|
5
|
+
import { assertNodeHealthy, degradedLayers } from "./node-health-guard.js";
|
|
5
6
|
import { auditDepartment, createDepartment, deleteDepartment, listDepartments, setMemberDepartment, updateDepartment } from "./departments.js";
|
|
6
7
|
import { redactMessageRow } from "./redact-tokens.js";
|
|
7
8
|
import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
|
|
@@ -750,8 +751,18 @@ const TaskSchema = z.object({
|
|
|
750
751
|
// stays unified regardless of transport (REST or MCP).
|
|
751
752
|
attachments: z.any().optional(),
|
|
752
753
|
meta: z.any().optional(),
|
|
754
|
+
// #460 — dispatch even when the target reports a degraded health layer. Honoured for user
|
|
755
|
+
// tokens only (a node token cannot override another node's health). Old clients never send it.
|
|
756
|
+
force: z.boolean().optional(),
|
|
753
757
|
});
|
|
754
758
|
|
|
759
|
+
/** #460 — `{ degraded: [{ layer, label, reason }] }` when the node's fresh health says a layer is down, else `{}`. */
|
|
760
|
+
function degradedField(networkId: string | null | undefined, alias: string): { degraded?: Array<{ layer: string; label: string; reason: string }> } {
|
|
761
|
+
const h = readNodeHealth(networkId ?? "default", alias);
|
|
762
|
+
const layers = degradedLayers(h?.health);
|
|
763
|
+
return layers.length ? { degraded: layers.map(l => ({ layer: l.layer, label: l.label, reason: l.reason })) } : {};
|
|
764
|
+
}
|
|
765
|
+
|
|
755
766
|
const BroadcastSchema = z.object({
|
|
756
767
|
message: z.string().min(1).max(10000),
|
|
757
768
|
filter_server: z.string().max(200).optional(),
|
|
@@ -2317,6 +2328,10 @@ return Bun.serve({
|
|
|
2317
2328
|
// Only when filtering by node_id (or for the old alias resolver), so the unfiltered light response
|
|
2318
2329
|
// keeps its exact bytes.
|
|
2319
2330
|
...(withNodeId ? { node_id: s.node_id ?? null } : {}),
|
|
2331
|
+
// #460 — the app's list reads the light projection; a node whose fresh health says a layer is down
|
|
2332
|
+
// carries the failing layers here (the same ones dispatch refuses with node_degraded). Absent for every
|
|
2333
|
+
// healthy / unknown node, so the response is byte-identical unless something is actually degraded.
|
|
2334
|
+
...degradedField(s.network_id, s.alias),
|
|
2320
2335
|
};
|
|
2321
2336
|
}
|
|
2322
2337
|
const externalSchedules = (() => {
|
|
@@ -2359,6 +2374,7 @@ return Bun.serve({
|
|
|
2359
2374
|
const h = readNodeHealth(s.network_id, s.alias);
|
|
2360
2375
|
return { health: h?.health ?? null, health_observed_ms_ago: h?.observed_ms_ago ?? null };
|
|
2361
2376
|
})(),
|
|
2377
|
+
...degradedField(s.network_id, s.alias),
|
|
2362
2378
|
};
|
|
2363
2379
|
});
|
|
2364
2380
|
const summary = sessions.reduce((acc: any, session: any) => {
|
|
@@ -3211,6 +3227,12 @@ return Bun.serve({
|
|
|
3211
3227
|
return withCors(req, Response.json(lc, { status: 409 }));
|
|
3212
3228
|
}
|
|
3213
3229
|
}
|
|
3230
|
+
// #460 — refuse dispatch to a node whose fresh health says a layer is down (409, same as the
|
|
3231
|
+
// lifecycle guard above). Unknown / stale health = allowed: old nodes keep working.
|
|
3232
|
+
{
|
|
3233
|
+
const hg = assertNodeHealthy(targetAlias, taskNetId ?? null, { force: body.force === true, forceAllowed: !restAuth?.networkId });
|
|
3234
|
+
if (!hg.ok) return withCors(req, Response.json(hg, { status: 409 }));
|
|
3235
|
+
}
|
|
3214
3236
|
// Mirror send_task MCP: write inbox + tasks rows in a single
|
|
3215
3237
|
// transaction so the dispatch is visible to dashboard's Tasks page
|
|
3216
3238
|
// and the parent_task_id lineage chain. Previously this endpoint
|
package/src/tools.ts
CHANGED
|
@@ -2,6 +2,8 @@ import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
|
|
|
2
2
|
import { parseDbTimestampMs } from "./db-timestamp.js";
|
|
3
3
|
import { z } from "zod/v4";
|
|
4
4
|
import { nodeHealthSchema, recordNodeHealth } from "./node-health-store.js";
|
|
5
|
+
import { assertNodeHealthy } from "./node-health-guard.js";
|
|
6
|
+
import { noteModelAuthHealth } from "./model-auth-notify.js";
|
|
5
7
|
import { parseAliasFilter } from "./alias-filter.js";
|
|
6
8
|
import { createHash } from "node:crypto";
|
|
7
9
|
import { db, uuidv4, logTaskEvent, chainReplyToParent, hashToken, generateId, generateNetworkToken, syncScheduledRunForTask } from "./db.js";
|
|
@@ -1061,7 +1063,11 @@ export function registerTools(server: McpServer, clientIP?: string, enforceNetwo
|
|
|
1061
1063
|
const acceptedHealth = nodeHealth && callerTokenIsNetwork && callerAlias && callerAlias === effectiveAlias
|
|
1062
1064
|
? nodeHealth
|
|
1063
1065
|
: undefined;
|
|
1064
|
-
if (acceptedHealth)
|
|
1066
|
+
if (acceptedHealth) {
|
|
1067
|
+
recordNodeHealth(sessionNetId, effectiveAlias, acceptedHealth);
|
|
1068
|
+
// #462 — model login went bad → tell the node's owner once (detect + notify only).
|
|
1069
|
+
if (callerTokenId) noteModelAuthHealth({ networkId: sessionNetId, alias: effectiveAlias, tokenId: callerTokenId, health: acceptedHealth });
|
|
1070
|
+
}
|
|
1065
1071
|
pushEvent(effectiveAlias, {
|
|
1066
1072
|
type: "status_update",
|
|
1067
1073
|
alias: effectiveAlias,
|
|
@@ -1632,8 +1638,9 @@ export function registerTools(server: McpServer, clientIP?: string, enforceNetwo
|
|
|
1632
1638
|
network_id: z.string().max(200).optional().describe("Network scope"),
|
|
1633
1639
|
parent_task_id: z.string().max(200).optional().describe("Parent task this dispatch is on behalf of. When the child task replies the hub will auto-chain the answer to the parent task's originator, so the user sees the final result even if the intermediate session ends."),
|
|
1634
1640
|
meta: z.any().optional().describe("Optional structured task metadata, e.g. { attachments: [{ type, path, url, mime, name, size }] }."),
|
|
1641
|
+
force: z.boolean().optional().describe("#460 — dispatch even when the target reports a degraded health layer (user tokens only; ignored for node tokens)"),
|
|
1635
1642
|
},
|
|
1636
|
-
async ({ alias, task, priority, context, from_session: _fromIn, ttl_seconds, network_id: netId, parent_task_id: parentIn, meta }) => { const fromMismatch = fromIdentityMismatchReply(_fromIn); if (fromMismatch) return fromMismatch; const from_session = defaultFrom(_fromIn);
|
|
1643
|
+
async ({ alias, task, priority, context, from_session: _fromIn, ttl_seconds, network_id: netId, parent_task_id: parentIn, meta, force }) => { const fromMismatch = fromIdentityMismatchReply(_fromIn); if (fromMismatch) return fromMismatch; const from_session = defaultFrom(_fromIn);
|
|
1637
1644
|
const effectiveNetId = getNetworkId(netId);
|
|
1638
1645
|
const authOrigin: TaskAuthOrigin = callerTokenIsNetwork
|
|
1639
1646
|
? "node"
|
|
@@ -1787,6 +1794,11 @@ export function registerTools(server: McpServer, clientIP?: string, enforceNetwo
|
|
|
1787
1794
|
const lc = assertNodeActive(targetAlias, effectiveNetId ?? null);
|
|
1788
1795
|
if (!lc.ok) return { content: [{ type: "text" as const, text: JSON.stringify(lc) }] };
|
|
1789
1796
|
}
|
|
1797
|
+
// #460 — a node whose fresh health says a layer is down will not run the task: refuse instead of queueing silently.
|
|
1798
|
+
{
|
|
1799
|
+
const hg = assertNodeHealthy(targetAlias, effectiveNetId ?? null, { force, forceAllowed: !callerTokenIsNetwork });
|
|
1800
|
+
if (!hg.ok) return { content: [{ type: "text" as const, text: JSON.stringify(hg) }] };
|
|
1801
|
+
}
|
|
1790
1802
|
|
|
1791
1803
|
console.log(`[${ts()}] ${from_session} → send_task → ${targetAlias}: ${task.slice(0, 60)}${priority === "high" ? " [HIGH]" : ""}${canonical.renamed ? ` [renamed from ${alias}]` : ""}`);
|
|
1792
1804
|
const fromNodeId = resolveNodeIdForAlias(from_session, effectiveNetId);
|
|
@@ -2346,8 +2358,9 @@ export function registerTools(server: McpServer, clientIP?: string, enforceNetwo
|
|
|
2346
2358
|
task_id: z.string().min(1).max(200).describe("Task ID to retry"),
|
|
2347
2359
|
from_session: z.string().max(200).optional(),
|
|
2348
2360
|
network_id: z.string().max(200).optional().describe("Network scope (auto-resolved for single-network user tokens)"),
|
|
2361
|
+
force: z.boolean().optional().describe("#460 — dispatch even when the target reports a degraded health layer (user tokens only; ignored for node tokens)"),
|
|
2349
2362
|
},
|
|
2350
|
-
async ({ task_id, from_session: _fromIn, network_id: netId }) => { const fromMismatch = fromIdentityMismatchReply(_fromIn); if (fromMismatch) return fromMismatch; const from_session = defaultFrom(_fromIn);
|
|
2363
|
+
async ({ task_id, from_session: _fromIn, network_id: netId, force }) => { const fromMismatch = fromIdentityMismatchReply(_fromIn); if (fromMismatch) return fromMismatch; const from_session = defaultFrom(_fromIn);
|
|
2351
2364
|
const effectiveNetId = getNetworkId(netId);
|
|
2352
2365
|
if (!canWrite(effectiveNetId)) return writeDeniedReply(effectiveNetId);
|
|
2353
2366
|
console.log(`[${ts()}] ${from_session} → retry_task → ${task_id.slice(0, 8)}`);
|
|
@@ -2367,6 +2380,11 @@ export function registerTools(server: McpServer, clientIP?: string, enforceNetwo
|
|
|
2367
2380
|
const lc = assertNodeActive(task.to_name, effectiveNetId ?? task.network_id ?? null);
|
|
2368
2381
|
if (!lc.ok) return { content: [{ type: "text" as const, text: JSON.stringify(lc) }] };
|
|
2369
2382
|
}
|
|
2383
|
+
// #460 — a node whose fresh health says a layer is down will not run the task: refuse instead of queueing silently.
|
|
2384
|
+
{
|
|
2385
|
+
const hg = assertNodeHealthy(task.to_name, effectiveNetId ?? task.network_id ?? null, { force, forceAllowed: !callerTokenIsNetwork });
|
|
2386
|
+
if (!hg.ok) return { content: [{ type: "text" as const, text: JSON.stringify(hg) }] };
|
|
2387
|
+
}
|
|
2370
2388
|
db.transaction(() => {
|
|
2371
2389
|
// Reset task status
|
|
2372
2390
|
const updateParams: any[] = [task_id];
|
|
@@ -2559,8 +2577,9 @@ export function registerTools(server: McpServer, clientIP?: string, enforceNetwo
|
|
|
2559
2577
|
new_alias: z.string().min(1).max(200).describe("Target agent alias"),
|
|
2560
2578
|
from_session: z.string().max(200).optional(),
|
|
2561
2579
|
network_id: z.string().max(200).optional().describe("Network scope (auto-resolved for single-network user tokens)"),
|
|
2580
|
+
force: z.boolean().optional().describe("#460 — dispatch even when the target reports a degraded health layer (user tokens only; ignored for node tokens)"),
|
|
2562
2581
|
},
|
|
2563
|
-
async ({ task_id, new_alias, from_session: _fromIn, network_id: netId }) => { const fromMismatch = fromIdentityMismatchReply(_fromIn); if (fromMismatch) return fromMismatch; const from_session = defaultFrom(_fromIn);
|
|
2582
|
+
async ({ task_id, new_alias, from_session: _fromIn, network_id: netId, force }) => { const fromMismatch = fromIdentityMismatchReply(_fromIn); if (fromMismatch) return fromMismatch; const from_session = defaultFrom(_fromIn);
|
|
2564
2583
|
const effectiveNetId = getNetworkId(netId);
|
|
2565
2584
|
if (!canWrite(effectiveNetId)) return writeDeniedReply(effectiveNetId);
|
|
2566
2585
|
console.log(`[${ts()}] ${from_session} → reassign_task → ${task_id.slice(0, 8)} → ${new_alias}`);
|
|
@@ -2582,6 +2601,11 @@ export function registerTools(server: McpServer, clientIP?: string, enforceNetwo
|
|
|
2582
2601
|
const lc = assertNodeActive(reassignedAlias, effectiveNetId ?? task.network_id ?? null);
|
|
2583
2602
|
if (!lc.ok) return { content: [{ type: "text" as const, text: JSON.stringify(lc) }] };
|
|
2584
2603
|
}
|
|
2604
|
+
// #460 — a node whose fresh health says a layer is down will not run the task: refuse instead of queueing silently.
|
|
2605
|
+
{
|
|
2606
|
+
const hg = assertNodeHealthy(reassignedAlias, effectiveNetId ?? task.network_id ?? null, { force, forceAllowed: !callerTokenIsNetwork });
|
|
2607
|
+
if (!hg.ok) return { content: [{ type: "text" as const, text: JSON.stringify(hg) }] };
|
|
2608
|
+
}
|
|
2585
2609
|
db.transaction(() => {
|
|
2586
2610
|
// Ack old inbox to prevent original agent from picking it up
|
|
2587
2611
|
const inboxParams: any[] = [task_id];
|