@sleep2agi/commhub-server 0.9.0-preview.87 → 0.9.0-preview.88
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/agent-notice.ts +48 -0
- package/src/db.ts +2 -2
- package/src/http-gzip.ts +3 -1
- package/src/model-auth-notify.ts +7 -23
- package/src/node-health-store.ts +16 -1
- package/src/scheduled-stuck-http.test.ts +238 -0
- package/src/scheduled-stuck.ts +153 -0
- package/src/scheduled-tasks.ts +32 -2
- package/src/server.ts +19 -3
- package/src/status-read-cache-http.test.ts +229 -0
- package/src/status-read-cache.ts +108 -0
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@sleep2agi/commhub-server",
|
|
3
|
-
"version": "0.9.0-preview.
|
|
3
|
+
"version": "0.9.0-preview.88",
|
|
4
4
|
"description": "CommHub Server — AI Agent communication hub with MCP protocol, multi-network isolation, user auth, and MCP tools (17 collaboration-core + node/provider ops tools; authoritative list: docs-site/docs/api/mcp-tools.md).",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "src/index.ts",
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
// Hub 以某个节点的名义发给一个用户的一条通知(#462 登录失效、#464 定时任务卡住)。
|
|
2
|
+
//
|
|
3
|
+
// 走 user_inbox + /events/users/me 的 desktop_message —— 与 send_desktop_message 同一张表、同一条推送,
|
|
4
|
+
// from_session = 节点别名,所以 App 把它显示在**与该节点的会话**里(主动消息)+ 未读角标 + 通知。不新增渠道。
|
|
5
|
+
// 只发给一个人;他不在本网络了就不发。推送在写库之后:收到推送的 App 立刻回读得到这一行。
|
|
6
|
+
|
|
7
|
+
import { db, uuidv4 } from "./db.js";
|
|
8
|
+
import { getUserNetworkRole } from "./auth.js";
|
|
9
|
+
import { pushUserEvent } from "./push.js";
|
|
10
|
+
|
|
11
|
+
export type AgentNotice = {
|
|
12
|
+
networkId: string;
|
|
13
|
+
userId: string;
|
|
14
|
+
/** 节点别名:通知出现在与它的会话里。 */
|
|
15
|
+
fromAlias: string;
|
|
16
|
+
kind: string;
|
|
17
|
+
title: string;
|
|
18
|
+
text: string;
|
|
19
|
+
severity?: "info" | "warning" | "error";
|
|
20
|
+
meta: Record<string, unknown>;
|
|
21
|
+
/** 消息 id 前缀(`dm_` 开头,与 App 的去重键同一套)。 */
|
|
22
|
+
idPrefix: string;
|
|
23
|
+
};
|
|
24
|
+
|
|
25
|
+
/** 返回 message_id;没发(收件人不在本网络)→ null。 */
|
|
26
|
+
export function sendAgentNotice(n: AgentNotice): string | null {
|
|
27
|
+
if (!getUserNetworkRole(n.userId, n.networkId)) return null;
|
|
28
|
+
const messageId = `${n.idPrefix}${uuidv4()}`;
|
|
29
|
+
const severity = n.severity ?? "warning";
|
|
30
|
+
db.run(
|
|
31
|
+
`INSERT INTO user_inbox (message_id, network_id, user_id, from_session, kind, title, content, severity, meta_json)
|
|
32
|
+
VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9)
|
|
33
|
+
ON CONFLICT(message_id) DO NOTHING`,
|
|
34
|
+
[messageId, n.networkId, n.userId, n.fromAlias, n.kind, n.title, n.text, severity, JSON.stringify(n.meta)],
|
|
35
|
+
);
|
|
36
|
+
pushUserEvent(n.networkId, n.userId, {
|
|
37
|
+
type: "desktop_message",
|
|
38
|
+
message_id: messageId,
|
|
39
|
+
kind: n.kind,
|
|
40
|
+
from: n.fromAlias,
|
|
41
|
+
title: n.title,
|
|
42
|
+
message: n.text,
|
|
43
|
+
severity,
|
|
44
|
+
created_at: new Date().toISOString(),
|
|
45
|
+
meta: n.meta,
|
|
46
|
+
});
|
|
47
|
+
return messageId;
|
|
48
|
+
}
|
package/src/db.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
|
-
import { createAdapter, type DbAdapter } from "./db-adapter";
|
|
1
|
+
import { createAdapter, type DbAdapter } from "./db-adapter"; import { withStatementHook } from "./status-read-cache"; // #431 写 sessions 的语句让 /api/status 缓存失效
|
|
2
2
|
|
|
3
|
-
export const db: DbAdapter = createAdapter();
|
|
3
|
+
export const db: DbAdapter = withStatementHook(createAdapter());
|
|
4
4
|
|
|
5
5
|
// Schema
|
|
6
6
|
db.exec(`
|
package/src/http-gzip.ts
CHANGED
|
@@ -56,7 +56,9 @@ function gzipCached(key: string | undefined, raw: () => Promise<Uint8Array>): Pr
|
|
|
56
56
|
}
|
|
57
57
|
return raw().then(bytes => {
|
|
58
58
|
if (bytes.byteLength < GZIP_MIN_BYTES) return { raw: bytes, gz: null };
|
|
59
|
-
|
|
59
|
+
// 可复用的正文(带键)压一次、存起来、反复发:压缩级别拉到 9 —— 多花的那点 CPU 只花一次,
|
|
60
|
+
// 而线上每次都少发 ~3%(#431 实测:旧节点别名解析读 29.6 KB → 28.6 KB)。不带键的照旧默认级别。
|
|
61
|
+
const gz = key ? Bun.gzipSync(bytes, { level: 9 }) : Bun.gzipSync(bytes);
|
|
60
62
|
if (key) gzipCacheStats.misses++;
|
|
61
63
|
if (key && gz.byteLength <= GZIP_CACHE_MAX_BYTES / 4) {
|
|
62
64
|
gzipCache.set(key, gz);
|
package/src/model-auth-notify.ts
CHANGED
|
@@ -15,12 +15,11 @@
|
|
|
15
15
|
//
|
|
16
16
|
// 收件人 = 节点主人:上报令牌绑定的 nodes.owner_user_id(RFC-036,建节点时定死),没有绑定(老令牌)就取
|
|
17
17
|
// 铸这个节点令牌的用户(api_tokens.user_id —— 新令牌里两者相同)。只发给这一个人;不在本网络了就不发。
|
|
18
|
-
// 怎么送:user_inbox
|
|
19
|
-
// 所以 App 把它显示在**与该节点的会话**里(主动消息)+ 未读角标 + 通知。不新增渠道。
|
|
18
|
+
// 怎么送:agent-notice.ts(user_inbox + /events/users/me,from_session = 节点别名)。
|
|
20
19
|
|
|
21
|
-
import { db
|
|
20
|
+
import { db } from "./db.js";
|
|
22
21
|
import { getUserNetworkRole } from "./auth.js";
|
|
23
|
-
import {
|
|
22
|
+
import { sendAgentNotice } from "./agent-notice.js";
|
|
24
23
|
import { degradedLayers } from "./node-health-guard.js";
|
|
25
24
|
import type { NodeHealth } from "./node-health-store.js";
|
|
26
25
|
|
|
@@ -104,26 +103,11 @@ export function noteModelAuthHealth(input: { networkId: string; alias: string; t
|
|
|
104
103
|
if (prev === undefined && hasUnreadNotice(owner, networkId, alias)) return [];
|
|
105
104
|
|
|
106
105
|
const { title, text } = modelAuthNoticeText(alias, state);
|
|
107
|
-
const
|
|
108
|
-
|
|
109
|
-
|
|
110
|
-
`INSERT INTO user_inbox (message_id, network_id, user_id, from_session, kind, title, content, severity, meta_json)
|
|
111
|
-
VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, 'warning', ?8)
|
|
112
|
-
ON CONFLICT(message_id) DO NOTHING`,
|
|
113
|
-
[messageId, networkId, owner, alias, MODEL_AUTH_NOTICE_KIND, title, text, JSON.stringify(meta)],
|
|
114
|
-
);
|
|
115
|
-
// 推送在写库之后:收到推送的 App 立刻回读得到这一行。
|
|
116
|
-
pushUserEvent(networkId, owner, {
|
|
117
|
-
type: "desktop_message",
|
|
118
|
-
message_id: messageId,
|
|
119
|
-
kind: MODEL_AUTH_NOTICE_KIND,
|
|
120
|
-
from: alias,
|
|
121
|
-
title,
|
|
122
|
-
message: text,
|
|
123
|
-
severity: "warning",
|
|
124
|
-
created_at: new Date().toISOString(),
|
|
125
|
-
meta,
|
|
106
|
+
const sent = sendAgentNotice({
|
|
107
|
+
networkId, userId: owner, fromAlias: alias, kind: MODEL_AUTH_NOTICE_KIND, title, text,
|
|
108
|
+
meta: { model_auth_notice: { alias, state } }, idPrefix: "dm_auth_",
|
|
126
109
|
});
|
|
110
|
+
if (!sent) return [];
|
|
127
111
|
return [owner];
|
|
128
112
|
} catch (e: any) {
|
|
129
113
|
console.error(`[model-auth-notify] ${input.alias}: ${e?.message ?? e}`);
|
package/src/node-health-store.ts
CHANGED
|
@@ -34,7 +34,22 @@ export type NodeHealth = NonNullable<z.infer<typeof nodeHealthSchema>>;
|
|
|
34
34
|
const store = new Map<string, { health: NodeHealth; at: number }>();
|
|
35
35
|
const keyOf = (networkId: string | null | undefined, alias: string) => `${networkId ?? "default"}\0${alias}`;
|
|
36
36
|
|
|
37
|
+
// #431 —— /api/status 的记忆化(status-read-cache.ts)按这两样判断正文还对不对:任何一次上报 → 版本 +1;
|
|
38
|
+
// 任何一份报告过 TTL 的那一刻(degraded / health 字段会消失)→ 由 nodeHealthNextExpiryAt 给出的时刻失效。
|
|
39
|
+
let version = 0;
|
|
40
|
+
export function nodeHealthVersion(): number { return version; }
|
|
41
|
+
/** 最早一份还没过期的报告的过期时刻(ms);一份都没有 → Infinity。 */
|
|
42
|
+
export function nodeHealthNextExpiryAt(now = Date.now()): number {
|
|
43
|
+
let next = Infinity;
|
|
44
|
+
for (const v of store.values()) {
|
|
45
|
+
const at = v.at + NODE_HEALTH_TTL_MS;
|
|
46
|
+
if (at >= now && at < next) next = at;
|
|
47
|
+
}
|
|
48
|
+
return next;
|
|
49
|
+
}
|
|
50
|
+
|
|
37
51
|
export function recordNodeHealth(networkId: string | null | undefined, alias: string, health: NodeHealth, now = Date.now()): void {
|
|
52
|
+
version++;
|
|
38
53
|
store.set(keyOf(networkId, alias), { health, at: now });
|
|
39
54
|
// 有界:按 TTL 顺手清掉过期的,别让改过名/删掉的节点在内存里攒着。
|
|
40
55
|
if (store.size > 256) {
|
|
@@ -54,4 +69,4 @@ export function readNodeHealth(
|
|
|
54
69
|
}
|
|
55
70
|
|
|
56
71
|
/** 测试用。 */
|
|
57
|
-
export function clearNodeHealthStore(): void { store.clear(); }
|
|
72
|
+
export function clearNodeHealthStore(): void { store.clear(); version++; }
|
|
@@ -0,0 +1,238 @@
|
|
|
1
|
+
// #464 —— 定时任务被一次没结束的执行卡住:告警 + 超时放行。
|
|
2
|
+
//
|
|
3
|
+
// 钉住:
|
|
4
|
+
// 1. 默认值:超时 = clamp(6 × 间隔, 1h, 24h);环境变量能改。
|
|
5
|
+
// 2. 同一个挡路任务连续挡掉 3 次 → 给排程创建者发恰好 1 条通知(别的成员 0 条);第 4、5 次不再发;
|
|
6
|
+
// 挡路任务一换(下一次真派出去了)重新上膛。
|
|
7
|
+
// 3. 挡路任务开着超过超时 → 它变成 expired(行还在、收件箱行 acked、run 记 expired/task_expired、
|
|
8
|
+
// error_message 说明是调度器超时),这一次照常派发。超时前已发过「卡住」→ 不再发;没发过 → 发一条「已超时」。
|
|
9
|
+
// 4. 晚到的回复:send_reply 以 reply_task_terminal 拒掉,旧 run / 旧任务 / 新任务 / 新 run 都不变,创建者不多收东西。
|
|
10
|
+
// 5. GET /runs 带 blocked_by_task_id / blocked_by_state。
|
|
11
|
+
//
|
|
12
|
+
// 跑法:cd server && COMMHUB_DB=/tmp/x.db bun test src/scheduled-stuck-http.test.ts
|
|
13
|
+
// PG:COMMHUB_TEST_PG_URL=… COMMHUB_PG_EXPERIMENTAL=1(tests/test2123-hub-postgres-ladder 里注册)
|
|
14
|
+
import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, test } from "bun:test";
|
|
15
|
+
import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
|
|
16
|
+
import { addNetworkMember, createNetworkTokenForNode, register } from "./auth.js";
|
|
17
|
+
import { db } from "./db.js";
|
|
18
|
+
import { dispatchScheduledOccurrence, handleScheduledTaskRequest } from "./scheduled-tasks.js";
|
|
19
|
+
import { SCHEDULE_STUCK_NOTICE_KIND, stuckTimeoutMs } from "./scheduled-stuck.js";
|
|
20
|
+
import { registerTools } from "./tools.js";
|
|
21
|
+
|
|
22
|
+
const activeDbPath = process.env.COMMHUB_DB ?? (process.env.COMMHUB_TEST_PG_URL ? "postgres" : undefined);
|
|
23
|
+
if (!activeDbPath) throw new Error("scheduled-stuck requires COMMHUB_DB (or COMMHUB_TEST_PG_URL) before module import");
|
|
24
|
+
|
|
25
|
+
const stamp = Date.now();
|
|
26
|
+
const ALIAS = `stuck-node-${stamp}`;
|
|
27
|
+
const NODE_ID = `n_stuck_${stamp}`;
|
|
28
|
+
let NET = "", creatorId = "", otherId = "";
|
|
29
|
+
const ENV_KEYS = ["COMMHUB_SCHEDULE_STUCK_NOTICE_SKIPS", "COMMHUB_SCHEDULE_STUCK_TIMEOUT_FACTOR", "COMMHUB_SCHEDULE_STUCK_TIMEOUT_FLOOR_SEC", "COMMHUB_SCHEDULE_STUCK_TIMEOUT_CAP_SEC"];
|
|
30
|
+
|
|
31
|
+
function seedSchedule(name = "日报汇总"): any {
|
|
32
|
+
const scheduleId = `sched_${crypto.randomUUID()}`;
|
|
33
|
+
db.run(
|
|
34
|
+
`INSERT INTO scheduled_tasks
|
|
35
|
+
(schedule_id, network_id, created_by, name, target_node_id, target_alias, task_content, priority,
|
|
36
|
+
schedule_type, schedule_json, timezone, status, next_run_at)
|
|
37
|
+
VALUES (?1, ?2, ?3, ?4, ?5, ?6, 'scheduled task', 'normal', 'interval',
|
|
38
|
+
'{"type":"interval","every_seconds":60}', 'Asia/Shanghai', 'active', datetime('now', '+1 minute'))`,
|
|
39
|
+
[scheduleId, NET, creatorId, name, NODE_ID, ALIAS],
|
|
40
|
+
);
|
|
41
|
+
return db.get("SELECT * FROM scheduled_tasks WHERE schedule_id = ?1", scheduleId);
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
let slot = 0;
|
|
45
|
+
const nextSlot = () => new Date(Date.UTC(2026, 9, 2, 0, 0, 0) + (++slot) * 60_000).toISOString();
|
|
46
|
+
const fire = (schedule: any, now = new Date()) => dispatchScheduledOccurrence(schedule, nextSlot(), false, now);
|
|
47
|
+
|
|
48
|
+
const notices = (userId: string) => db.all<{ title: string; content: string; meta_json: string; from_session: string }>(
|
|
49
|
+
"SELECT title, content, meta_json, from_session FROM user_inbox WHERE user_id = ?1 AND network_id = ?2 AND kind = ?3 ORDER BY created_at, message_id",
|
|
50
|
+
userId, NET, SCHEDULE_STUCK_NOTICE_KIND,
|
|
51
|
+
);
|
|
52
|
+
const runRow = (runId: string) => db.get<any>("SELECT * FROM scheduled_task_runs WHERE run_id = ?1", runId);
|
|
53
|
+
const taskRow = (taskId: string) => db.get<any>("SELECT * FROM tasks WHERE task_id = ?1", taskId);
|
|
54
|
+
/** 把任务的开始时间往前拨(数据库里的 UTC 无时区串)。 */
|
|
55
|
+
const backdate = (taskId: string, ms: number) => db.run("UPDATE tasks SET created_at = ?1 WHERE task_id = ?2", [new Date(Date.now() - ms).toISOString().slice(0, 19).replace("T", " "), taskId]);
|
|
56
|
+
|
|
57
|
+
async function nodeReply(args: Record<string, unknown>): Promise<any> {
|
|
58
|
+
const mcp = new McpServer({ name: "sched-stuck", version: "0" }) as any;
|
|
59
|
+
const handlers: Record<string, (a: any) => Promise<any>> = {};
|
|
60
|
+
const original = mcp.tool.bind(mcp);
|
|
61
|
+
mcp.tool = (name: string, ...rest: any[]) => {
|
|
62
|
+
const handler = rest.at(-1);
|
|
63
|
+
if (typeof handler === "function") handlers[name] = handler;
|
|
64
|
+
return original(name, ...rest);
|
|
65
|
+
};
|
|
66
|
+
registerTools(mcp, undefined, NET, creatorId, ALIAS, true, null);
|
|
67
|
+
const out = await handlers.send_reply(args);
|
|
68
|
+
return JSON.parse(out.content[0].text);
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
beforeAll(() => {
|
|
72
|
+
const creator = register(`stuck_creator_${stamp}`, "StuckCreator123!", undefined, "seed");
|
|
73
|
+
expect(creator.ok).toBe(true);
|
|
74
|
+
NET = creator.network_id!; creatorId = creator.user!.user_id;
|
|
75
|
+
const other = register(`stuck_other_${stamp}`, "StuckOther123!", undefined, "seed");
|
|
76
|
+
otherId = other.user!.user_id;
|
|
77
|
+
expect(addNetworkMember(NET, otherId, "member", creatorId, { agentAccess: "all" }).ok).toBe(true);
|
|
78
|
+
expect(createNetworkTokenForNode(creatorId, NET, ALIAS).ok).toBe(true);
|
|
79
|
+
db.run("INSERT INTO nodes (node_id, node_name, alias, runtime, network_id) VALUES (?1, ?2, ?2, 'claude-code', ?3)", [NODE_ID, ALIAS, NET]);
|
|
80
|
+
db.run("INSERT INTO sessions (resume_id, alias, status, node_id, network_id) VALUES (?1, ?2, 'idle', ?3, ?4)", [`r_stuck_${stamp}`, ALIAS, NODE_ID, NET]);
|
|
81
|
+
});
|
|
82
|
+
beforeEach(() => {
|
|
83
|
+
for (const k of ENV_KEYS) delete process.env[k];
|
|
84
|
+
db.run("DELETE FROM user_inbox WHERE network_id = ?1 AND kind = ?2", [NET, SCHEDULE_STUCK_NOTICE_KIND]);
|
|
85
|
+
// 上一个用例留下的开着的任务不许挡这一个(每个用例自己的排程,只按 schedule 查,保险起见也关掉)
|
|
86
|
+
db.run("UPDATE tasks SET status = 'replied' WHERE network_id = ?1 AND status IN ('created', 'delivered', 'acked', 'running')", [NET]);
|
|
87
|
+
});
|
|
88
|
+
afterEach(() => { for (const k of ENV_KEYS) delete process.env[k]; });
|
|
89
|
+
afterAll(() => { db.run("DELETE FROM user_inbox WHERE network_id = ?1 AND kind = ?2", [NET, SCHEDULE_STUCK_NOTICE_KIND]); });
|
|
90
|
+
|
|
91
|
+
describe("#464 defaults", () => {
|
|
92
|
+
test("timeout = clamp(6 × interval, 1h, 24h)", () => {
|
|
93
|
+
const iv = (s: number) => JSON.stringify({ type: "interval", every_seconds: s });
|
|
94
|
+
expect(stuckTimeoutMs(iv(60))).toBe(3_600_000);
|
|
95
|
+
expect(stuckTimeoutMs(iv(600))).toBe(3_600_000);
|
|
96
|
+
expect(stuckTimeoutMs(iv(1800))).toBe(3 * 3_600_000);
|
|
97
|
+
expect(stuckTimeoutMs(iv(3600))).toBe(6 * 3_600_000);
|
|
98
|
+
expect(stuckTimeoutMs(iv(86_400))).toBe(24 * 3_600_000);
|
|
99
|
+
expect(stuckTimeoutMs(JSON.stringify({ type: "daily", time: "09:00" }))).toBe(24 * 3_600_000);
|
|
100
|
+
expect(stuckTimeoutMs(JSON.stringify({ type: "once", run_at: "2026-10-02T00:00:00Z" }))).toBe(3_600_000);
|
|
101
|
+
process.env.COMMHUB_SCHEDULE_STUCK_TIMEOUT_FACTOR = "0";
|
|
102
|
+
process.env.COMMHUB_SCHEDULE_STUCK_TIMEOUT_FLOOR_SEC = "20";
|
|
103
|
+
expect(stuckTimeoutMs(iv(60))).toBe(20_000);
|
|
104
|
+
process.env.COMMHUB_SCHEDULE_STUCK_TIMEOUT_FLOOR_SEC = "junk";
|
|
105
|
+
expect(stuckTimeoutMs(iv(60))).toBe(3_600_000);
|
|
106
|
+
});
|
|
107
|
+
});
|
|
108
|
+
|
|
109
|
+
describe("#464 stuck notice", () => {
|
|
110
|
+
test("3rd consecutive skip behind the same task → exactly one notice to the creator, none to others", () => {
|
|
111
|
+
const s = seedSchedule();
|
|
112
|
+
const first = fire(s);
|
|
113
|
+
expect(first.status).toBe("delivered");
|
|
114
|
+
const skips = [fire(s), fire(s)];
|
|
115
|
+
expect(notices(creatorId).length).toBe(0);
|
|
116
|
+
skips.push(fire(s), fire(s), fire(s));
|
|
117
|
+
expect(skips.map((r) => r.status)).toEqual(["skipped", "skipped", "skipped", "skipped", "skipped"]);
|
|
118
|
+
const mine = notices(creatorId);
|
|
119
|
+
expect(mine.length).toBe(1);
|
|
120
|
+
expect(notices(otherId).length).toBe(0);
|
|
121
|
+
const n = mine[0];
|
|
122
|
+
expect(n.from_session).toBe(ALIAS);
|
|
123
|
+
expect(n.title).toBe("定时任务被卡住");
|
|
124
|
+
for (const frag of ["日报汇总", first.taskId!, ALIAS, "(Asia/Shanghai)", "3 次", "1 小时"]) expect(n.content).toContain(frag);
|
|
125
|
+
expect(JSON.parse(n.meta_json).schedule_stuck).toEqual({ reason: "stuck", schedule_id: s.schedule_id, blocked_by_task_id: first.taskId, skips: 3 });
|
|
126
|
+
});
|
|
127
|
+
|
|
128
|
+
test("re-armed once a run is dispatched again: a second episode notifies again", () => {
|
|
129
|
+
const s = seedSchedule();
|
|
130
|
+
const first = fire(s);
|
|
131
|
+
for (let i = 0; i < 4; i++) fire(s);
|
|
132
|
+
db.run("UPDATE tasks SET status = 'replied', completed_at = datetime('now') WHERE task_id = ?1", [first.taskId]);
|
|
133
|
+
const second = fire(s);
|
|
134
|
+
expect(second.status).toBe("delivered");
|
|
135
|
+
for (let i = 0; i < 3; i++) fire(s);
|
|
136
|
+
const mine = notices(creatorId);
|
|
137
|
+
expect(mine.length).toBe(2);
|
|
138
|
+
// 两条通知同一秒写入,created_at 排不出先后:按挡路任务比集合。
|
|
139
|
+
expect(mine.map((m) => JSON.parse(m.meta_json).schedule_stuck.blocked_by_task_id).sort()).toEqual([first.taskId, second.taskId].sort());
|
|
140
|
+
});
|
|
141
|
+
|
|
142
|
+
test("COMMHUB_SCHEDULE_STUCK_NOTICE_SKIPS changes N", () => {
|
|
143
|
+
process.env.COMMHUB_SCHEDULE_STUCK_NOTICE_SKIPS = "1";
|
|
144
|
+
const s = seedSchedule();
|
|
145
|
+
fire(s); fire(s);
|
|
146
|
+
expect(notices(creatorId).length).toBe(1);
|
|
147
|
+
});
|
|
148
|
+
|
|
149
|
+
test("schedule without a creator → no notice, skips still recorded", () => {
|
|
150
|
+
const s = seedSchedule();
|
|
151
|
+
db.run("UPDATE scheduled_tasks SET created_by = NULL WHERE schedule_id = ?1", [s.schedule_id]);
|
|
152
|
+
const row = db.get("SELECT * FROM scheduled_tasks WHERE schedule_id = ?1", s.schedule_id);
|
|
153
|
+
fire(row); for (let i = 0; i < 4; i++) expect(fire(row).status).toBe("skipped");
|
|
154
|
+
expect(notices(creatorId).length).toBe(0);
|
|
155
|
+
});
|
|
156
|
+
});
|
|
157
|
+
|
|
158
|
+
describe("#464 stuck timeout", () => {
|
|
159
|
+
test("blocker past the timeout is expired (row kept) and this occurrence dispatches; no second notice", () => {
|
|
160
|
+
const s = seedSchedule();
|
|
161
|
+
const first = fire(s);
|
|
162
|
+
db.run("UPDATE tasks SET status = 'running' WHERE task_id = ?1", [first.taskId]);
|
|
163
|
+
for (let i = 0; i < 3; i++) fire(s);
|
|
164
|
+
expect(notices(creatorId).length).toBe(1);
|
|
165
|
+
|
|
166
|
+
backdate(first.taskId!, 59 * 60_000);
|
|
167
|
+
expect(fire(s).status).toBe("skipped"); // 59 分钟 < 1 小时
|
|
168
|
+
backdate(first.taskId!, 61 * 60_000);
|
|
169
|
+
const next = fire(s);
|
|
170
|
+
expect(next.status).toBe("delivered");
|
|
171
|
+
expect(next.taskId).not.toBe(first.taskId);
|
|
172
|
+
|
|
173
|
+
const old = taskRow(first.taskId!);
|
|
174
|
+
expect(old.status).toBe("expired");
|
|
175
|
+
expect(old.content).toBe("scheduled task");
|
|
176
|
+
expect(db.get<any>("SELECT acked FROM inbox WHERE task_id = ?1", first.taskId).acked).toBeTruthy();
|
|
177
|
+
const oldRun = runRow(first.runId);
|
|
178
|
+
expect(oldRun.status).toBe("expired");
|
|
179
|
+
expect(oldRun.error_code).toBe("task_expired");
|
|
180
|
+
expect(oldRun.error_message).toContain("timed out by the scheduler");
|
|
181
|
+
expect(db.get<any>("SELECT COUNT(*) AS n FROM task_events WHERE task_id = ?1 AND to_status = 'expired' AND actor = 'hub-scheduler'", first.taskId).n).toBe(1);
|
|
182
|
+
expect(notices(creatorId).length).toBe(1); // 每段卡住恰好一条
|
|
183
|
+
});
|
|
184
|
+
|
|
185
|
+
test("timed out before N skips (long interval) → one 'timed out' notice instead", () => {
|
|
186
|
+
const s = seedSchedule("每日巡检");
|
|
187
|
+
const first = fire(s);
|
|
188
|
+
expect(fire(s).status).toBe("skipped");
|
|
189
|
+
backdate(first.taskId!, 2 * 3_600_000);
|
|
190
|
+
expect(fire(s).status).toBe("delivered");
|
|
191
|
+
const mine = notices(creatorId);
|
|
192
|
+
expect(mine.length).toBe(1);
|
|
193
|
+
expect(mine[0].title).toBe("定时任务已超时放行");
|
|
194
|
+
expect(mine[0].content).toContain(first.taskId!);
|
|
195
|
+
expect(JSON.parse(mine[0].meta_json).schedule_stuck.reason).toBe("timed_out");
|
|
196
|
+
});
|
|
197
|
+
|
|
198
|
+
test("a late reply to the expired task is refused and changes nothing", async () => {
|
|
199
|
+
const s = seedSchedule();
|
|
200
|
+
const first = fire(s);
|
|
201
|
+
backdate(first.taskId!, 2 * 3_600_000);
|
|
202
|
+
const next = fire(s);
|
|
203
|
+
expect(next.status).toBe("delivered");
|
|
204
|
+
const inboxBefore = db.get<any>("SELECT COUNT(*) AS n FROM inbox WHERE network_id = ?1", NET).n;
|
|
205
|
+
const out = await nodeReply({ alias: "scheduler", text: "late answer", in_reply_to: first.taskId, status: "replied", from_session: ALIAS });
|
|
206
|
+
expect(out.ok).toBe(false);
|
|
207
|
+
expect(out.error).toBe("reply_task_terminal");
|
|
208
|
+
expect(out.reply_queued).toBe(false);
|
|
209
|
+
expect(taskRow(first.taskId!).status).toBe("expired");
|
|
210
|
+
expect(runRow(first.runId).status).toBe("expired");
|
|
211
|
+
expect(taskRow(next.taskId!).status).toBe("delivered");
|
|
212
|
+
expect(runRow(next.runId).status).toBe("delivered");
|
|
213
|
+
expect(db.get<any>("SELECT COUNT(*) AS n FROM inbox WHERE network_id = ?1", NET).n).toBe(inboxBefore);
|
|
214
|
+
// 新那条照常能被回复,run 照常收尾
|
|
215
|
+
const ok = await nodeReply({ alias: "scheduler", text: "on time", in_reply_to: next.taskId, status: "replied", from_session: ALIAS });
|
|
216
|
+
expect(ok.ok).toBe(true);
|
|
217
|
+
expect(runRow(next.runId).status).toBe("replied");
|
|
218
|
+
});
|
|
219
|
+
});
|
|
220
|
+
|
|
221
|
+
describe("#464 history clarity", () => {
|
|
222
|
+
test("GET /runs exposes blocked_by_task_id and blocked_by_state on skipped rows", async () => {
|
|
223
|
+
const s = seedSchedule();
|
|
224
|
+
const first = fire(s);
|
|
225
|
+
const second = fire(s);
|
|
226
|
+
const url = new URL(`http://hub.test/api/scheduled-tasks/${s.schedule_id}/runs`);
|
|
227
|
+
const res = await handleScheduledTaskRequest({
|
|
228
|
+
req: new Request(url, { method: "GET" }), url,
|
|
229
|
+
auth: { userId: creatorId, networkId: NET, username: "creator" },
|
|
230
|
+
isAdmin: false, isNodeToken: false, scope: { networkId: NET, networkIds: [NET] } as any,
|
|
231
|
+
});
|
|
232
|
+
const body = await res!.json() as any;
|
|
233
|
+
const skipped = body.runs.find((r: any) => r.run_id === second.runId);
|
|
234
|
+
expect(skipped.status).toBe("skipped");
|
|
235
|
+
expect(skipped.blocked_by_task_id).toBe(first.taskId);
|
|
236
|
+
expect(skipped.blocked_by_state).toBe("not_received");
|
|
237
|
+
});
|
|
238
|
+
});
|
|
@@ -0,0 +1,153 @@
|
|
|
1
|
+
// #464 —— 定时任务被一次没结束的执行卡住:告警 + 超时放行。
|
|
2
|
+
//
|
|
3
|
+
// 现场:overlap=skip 的排程,上一次派出的任务节点一直没有终态回复 → 之后每一次都记成
|
|
4
|
+
// skipped / previous_run_active,没有超时、也没人被告诉(当天 ~25 分钟,10-01 ~7 小时)。
|
|
5
|
+
//
|
|
6
|
+
// 1. 告警:同一个「卡住的任务」(blocked_by_task_id)连续挡掉 N 次 → 给排程的创建者发**一条**通知
|
|
7
|
+
// (agent-notice.ts,与 #462 同一条路,出现在与目标节点的会话里)。按挡路的任务计数,所以
|
|
8
|
+
// 一段卡住只发一次;任务一换(下一次真派出去了)自然重新上膛。不依赖内存,Hub 重启不会重发。
|
|
9
|
+
// 2. 超时:挡路的任务开着超过 stuckTimeoutMs(间隔)就按**已有的** expired 状态结束它(同 TTL 巡检:
|
|
10
|
+
// tasks.status='expired' + 收件箱行置 acked + db.ts 的镜像把这次 run 记成 expired / task_expired),
|
|
11
|
+
// 然后这一次照常派发。任务行不删,照样能查;晚到的回复被 send_reply 以 reply_task_terminal 拒掉,
|
|
12
|
+
// 什么都不改(不重开 run、不碰新派出去的那条)。
|
|
13
|
+
// 如果超时发生在 N 次之前(长间隔的排程,比如每天一次),那一刻补发一条「已超时结束」—— 每段卡住
|
|
14
|
+
// 恰好一条通知。
|
|
15
|
+
//
|
|
16
|
+
// 默认值(PR 里有推导):N=3;超时 = clamp(6 × 间隔, 下限 1 小时, 上限 24 小时)。
|
|
17
|
+
// 下限 1 小时:一轮 agent 任务正常要跑几十分钟,不能因为排程间隔短就把还在干活的任务掐掉。
|
|
18
|
+
// 上限 24 小时:Hub 本来就给定时任务 24 小时的 expires_at(没被取走时由巡检过期),开着超过一天的
|
|
19
|
+
// 定时任务不是「还在做」。6 × 间隔:间隔长的排程(每小时一次)给它 6 小时,而不是 1 小时。
|
|
20
|
+
// 环境变量可改(运维 / 测试):COMMHUB_SCHEDULE_STUCK_NOTICE_SKIPS、COMMHUB_SCHEDULE_STUCK_TIMEOUT_FACTOR、
|
|
21
|
+
// COMMHUB_SCHEDULE_STUCK_TIMEOUT_FLOOR_SEC、COMMHUB_SCHEDULE_STUCK_TIMEOUT_CAP_SEC。
|
|
22
|
+
|
|
23
|
+
import { db, syncScheduledRunForTask } from "./db.js";
|
|
24
|
+
import { parseDbTimestampMs } from "./db-timestamp.js";
|
|
25
|
+
import { sendAgentNotice } from "./agent-notice.js";
|
|
26
|
+
|
|
27
|
+
export const SCHEDULE_STUCK_NOTICE_KIND = "schedule_stuck";
|
|
28
|
+
|
|
29
|
+
function envNumber(name: string, fallback: number, min: number): number {
|
|
30
|
+
const raw = process.env[name];
|
|
31
|
+
if (raw === undefined || raw.trim() === "") return fallback;
|
|
32
|
+
const n = Number(raw);
|
|
33
|
+
return Number.isFinite(n) && n >= min ? n : fallback;
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
export function stuckNoticeSkips(): number {
|
|
37
|
+
return Math.floor(envNumber("COMMHUB_SCHEDULE_STUCK_NOTICE_SKIPS", 3, 1));
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
/** 排程的名义间隔(秒):interval 取 every_seconds;daily / weekly 取一天(两次之间最短一天);once 取 0。 */
|
|
41
|
+
export function nominalIntervalSeconds(scheduleJson: string): number {
|
|
42
|
+
try {
|
|
43
|
+
const spec = JSON.parse(scheduleJson) as { type?: string; every_seconds?: number };
|
|
44
|
+
if (spec.type === "interval" && Number.isFinite(spec.every_seconds)) return Number(spec.every_seconds);
|
|
45
|
+
if (spec.type === "daily" || spec.type === "weekly") return 86_400;
|
|
46
|
+
} catch {}
|
|
47
|
+
return 0;
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
export function stuckTimeoutMs(scheduleJson: string): number {
|
|
51
|
+
const factor = envNumber("COMMHUB_SCHEDULE_STUCK_TIMEOUT_FACTOR", 6, 0);
|
|
52
|
+
const floor = envNumber("COMMHUB_SCHEDULE_STUCK_TIMEOUT_FLOOR_SEC", 3_600, 1);
|
|
53
|
+
const cap = Math.max(floor, envNumber("COMMHUB_SCHEDULE_STUCK_TIMEOUT_CAP_SEC", 86_400, 1));
|
|
54
|
+
return Math.min(cap, Math.max(floor, factor * nominalIntervalSeconds(scheduleJson))) * 1000;
|
|
55
|
+
}
|
|
56
|
+
|
|
57
|
+
type Blocker = { task_id: string; created_at: string; status: string };
|
|
58
|
+
|
|
59
|
+
/** 挡路的任务开了多久已经到超时 → true。读不出时间 = 不超时(宁可多挡一次,不错杀)。 */
|
|
60
|
+
export function blockerTimedOut(blocker: Pick<Blocker, "created_at">, scheduleJson: string, now: Date): boolean {
|
|
61
|
+
const started = parseDbTimestampMs(blocker.created_at);
|
|
62
|
+
if (!Number.isFinite(started)) return false;
|
|
63
|
+
return now.getTime() - started >= stuckTimeoutMs(scheduleJson);
|
|
64
|
+
}
|
|
65
|
+
|
|
66
|
+
/**
|
|
67
|
+
* 在调用方的事务里按超时结束挡路的任务。只动还开着的任务(条件写),返回是否真的结束了它。
|
|
68
|
+
* 与 TTL 巡检(server.ts patrolExpiredTasks)同一套写法:status=expired、收件箱行置 acked、镜像到 run。
|
|
69
|
+
*/
|
|
70
|
+
export function expireStuckBlocker(taskId: string, networkId: string, ageMs: number, limitMs: number): boolean {
|
|
71
|
+
const changed = db.run(
|
|
72
|
+
`UPDATE tasks SET status = 'expired', completed_at = datetime('now')
|
|
73
|
+
WHERE task_id = ?1 AND network_id = ?2 AND status IN ('created', 'delivered', 'acked', 'running')`,
|
|
74
|
+
[taskId, networkId],
|
|
75
|
+
);
|
|
76
|
+
if (changed.changes < 1) return false;
|
|
77
|
+
db.run("UPDATE inbox SET acked = 1 WHERE COALESCE(task_id, id) = ?1 AND acked = 0", [taskId]);
|
|
78
|
+
syncScheduledRunForTask(taskId, networkId);
|
|
79
|
+
db.run(
|
|
80
|
+
"UPDATE scheduled_task_runs SET error_message = ?1 WHERE task_id = ?2 AND network_id = ?3",
|
|
81
|
+
[`timed out by the scheduler: open for ${Math.round(ageMs / 60_000)} min (limit ${Math.round(limitMs / 60_000)} min); the next occurrence was dispatched`, taskId, networkId],
|
|
82
|
+
);
|
|
83
|
+
return true;
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
/** 这个任务已经挡掉了几次(只算 previous_run_active 的跳过)。 */
|
|
87
|
+
export function skipsBlockedBy(scheduleId: string, taskId: string): number {
|
|
88
|
+
return db.get<{ n: number }>(
|
|
89
|
+
"SELECT COUNT(*) AS n FROM scheduled_task_runs WHERE schedule_id = ?1 AND blocked_by_task_id = ?2 AND error_code = 'previous_run_active'",
|
|
90
|
+
scheduleId, taskId,
|
|
91
|
+
)?.n ?? 0;
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
export type StuckNotice = {
|
|
95
|
+
reason: "stuck" | "timed_out";
|
|
96
|
+
scheduleId: string;
|
|
97
|
+
scheduleName: string;
|
|
98
|
+
networkId: string;
|
|
99
|
+
createdBy: string | null;
|
|
100
|
+
alias: string;
|
|
101
|
+
timezone: string;
|
|
102
|
+
taskId: string;
|
|
103
|
+
taskCreatedAt: string;
|
|
104
|
+
skips: number;
|
|
105
|
+
timeoutMs: number;
|
|
106
|
+
};
|
|
107
|
+
|
|
108
|
+
function fmtTime(dbTs: string, timezone: string): string {
|
|
109
|
+
const ms = parseDbTimestampMs(dbTs);
|
|
110
|
+
if (!Number.isFinite(ms)) return dbTs;
|
|
111
|
+
try {
|
|
112
|
+
const s = new Intl.DateTimeFormat("sv-SE", { timeZone: timezone, year: "numeric", month: "2-digit", day: "2-digit", hour: "2-digit", minute: "2-digit", hourCycle: "h23" }).format(new Date(ms));
|
|
113
|
+
return `${s} (${timezone})`;
|
|
114
|
+
} catch { return dbTs; }
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
function fmtDuration(ms: number): string {
|
|
118
|
+
const min = Math.round(ms / 60_000);
|
|
119
|
+
if (min < 60) return `${min} 分钟`;
|
|
120
|
+
const h = Math.floor(min / 60), m = min % 60;
|
|
121
|
+
return m ? `${h} 小时 ${m} 分钟` : `${h} 小时`;
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
export function stuckNoticeText(n: StuckNotice): { title: string; text: string } {
|
|
125
|
+
const started = fmtTime(n.taskCreatedAt, n.timezone);
|
|
126
|
+
if (n.reason === "stuck") {
|
|
127
|
+
return {
|
|
128
|
+
title: "定时任务被卡住",
|
|
129
|
+
text: `定时任务「${n.scheduleName}」被上一次的执行卡住了:任务 ${n.taskId} 于 ${started} 派给节点 ${n.alias},到现在还没有完成,之后的 ${n.skips} 次执行都被跳过。\n\n`
|
|
130
|
+
+ `它开着超过 ${fmtDuration(n.timeoutMs)} 后会被自动按超时结束,下一次照常派发。等不及可以在任务列表里取消这条任务。`,
|
|
131
|
+
};
|
|
132
|
+
}
|
|
133
|
+
return {
|
|
134
|
+
title: "定时任务已超时放行",
|
|
135
|
+
text: `定时任务「${n.scheduleName}」上一次的任务 ${n.taskId}(${started} 派给节点 ${n.alias})开着超过 ${fmtDuration(n.timeoutMs)} 还没有完成,已按超时结束,不再挡后面的执行。`,
|
|
136
|
+
};
|
|
137
|
+
}
|
|
138
|
+
|
|
139
|
+
/** 提交之后调用。没有创建者 / 创建者不在网络里 → 不发。 */
|
|
140
|
+
export function sendStuckNotice(n: StuckNotice): string | null {
|
|
141
|
+
if (!n.createdBy) return null;
|
|
142
|
+
try {
|
|
143
|
+
const { title, text } = stuckNoticeText(n);
|
|
144
|
+
return sendAgentNotice({
|
|
145
|
+
networkId: n.networkId, userId: n.createdBy, fromAlias: n.alias, kind: SCHEDULE_STUCK_NOTICE_KIND, title, text,
|
|
146
|
+
meta: { schedule_stuck: { reason: n.reason, schedule_id: n.scheduleId, blocked_by_task_id: n.taskId, skips: n.skips } },
|
|
147
|
+
idPrefix: "dm_sched_",
|
|
148
|
+
});
|
|
149
|
+
} catch (e: any) {
|
|
150
|
+
console.error(`[scheduled-tasks] stuck notice failed schedule=${n.scheduleId}: ${e?.message || e}`);
|
|
151
|
+
return null;
|
|
152
|
+
}
|
|
153
|
+
}
|
package/src/scheduled-tasks.ts
CHANGED
|
@@ -5,6 +5,8 @@ import { addNetworkScope, canRestWriteNetwork, singleNetworkId, type RestNetwork
|
|
|
5
5
|
import { canMessageAgent } from "./agent-access.js";
|
|
6
6
|
import { hasSubscribers, pushEvent, pushNetworkObserverEvent } from "./push.js";
|
|
7
7
|
import { SCHEDULED_TASK_STORAGE_SELECT } from "./rest-projections.js";
|
|
8
|
+
import { parseDbTimestampMs } from "./db-timestamp.js";
|
|
9
|
+
import { blockerTimedOut, expireStuckBlocker, sendStuckNotice, skipsBlockedBy, stuckNoticeSkips, stuckTimeoutMs, type StuckNotice } from "./scheduled-stuck.js";
|
|
8
10
|
|
|
9
11
|
export type ScheduleSpec =
|
|
10
12
|
| { type: "once"; run_at: string }
|
|
@@ -203,6 +205,13 @@ function validateTarget(networkId: string, nodeId: unknown): { node_id: string;
|
|
|
203
205
|
return { node_id: node.node_id, alias: node.alias };
|
|
204
206
|
}
|
|
205
207
|
|
|
208
|
+
function stuckNoticeFor(row: ScheduledRow, reason: StuckNotice["reason"], blocker: { task_id: string; created_at: string }, skips: number, timeoutMs: number): StuckNotice {
|
|
209
|
+
return {
|
|
210
|
+
reason, scheduleId: row.schedule_id, scheduleName: row.name, networkId: row.network_id, createdBy: row.created_by,
|
|
211
|
+
alias: row.target_alias, timezone: row.timezone, taskId: blocker.task_id, taskCreatedAt: blocker.created_at, skips, timeoutMs,
|
|
212
|
+
};
|
|
213
|
+
}
|
|
214
|
+
|
|
206
215
|
type DispatchEvent = { alias: string; networkId: string; taskId: string; priority: string; state: "delivered" | "queued" };
|
|
207
216
|
|
|
208
217
|
export function dispatchScheduledOccurrence(row: ScheduledRow, scheduledFor: string, advanceSchedule: boolean, advanceAfter = new Date()): { runId: string; taskId?: string; status: string; event?: DispatchEvent } {
|
|
@@ -210,6 +219,8 @@ export function dispatchScheduledOccurrence(row: ScheduledRow, scheduledFor: str
|
|
|
210
219
|
let event: DispatchEvent | undefined;
|
|
211
220
|
let finalStatus = "failed";
|
|
212
221
|
let createdTaskId: string | undefined;
|
|
222
|
+
const notices: StuckNotice[] = [];
|
|
223
|
+
const timedOut: Array<{ taskId: string; fromStatus: string }> = [];
|
|
213
224
|
|
|
214
225
|
db.transaction(() => {
|
|
215
226
|
// UNIQUE(schedule_id, scheduled_for) is the cross-process claim. If two
|
|
@@ -225,8 +236,8 @@ export function dispatchScheduledOccurrence(row: ScheduledRow, scheduledFor: str
|
|
|
225
236
|
// received:节点那边有没有把这条任务取走过(inbox 行被 ack,或任务已进 acked/running)。
|
|
226
237
|
// 跳过记录带上它,客户端才分得清「节点还没收到」和「节点在处理」—— 前者是投递故障,
|
|
227
238
|
// 后者才是真的上一次还没做完。
|
|
228
|
-
const
|
|
229
|
-
`SELECT r.task_id,
|
|
239
|
+
const findOpen = () => db.get<{ task_id: string; received: number; created_at: string; status: string }>(
|
|
240
|
+
`SELECT r.task_id, t.created_at, t.status,
|
|
230
241
|
CASE WHEN t.status IN ('acked', 'running')
|
|
231
242
|
OR EXISTS (SELECT 1 FROM inbox i WHERE (i.task_id = r.task_id OR i.id = r.task_id) AND i.acked = 1)
|
|
232
243
|
THEN 1 ELSE 0 END AS received
|
|
@@ -236,6 +247,20 @@ export function dispatchScheduledOccurrence(row: ScheduledRow, scheduledFor: str
|
|
|
236
247
|
ORDER BY r.created_at DESC LIMIT 1`,
|
|
237
248
|
row.schedule_id, ...OPEN_TASK_STATUSES,
|
|
238
249
|
);
|
|
250
|
+
let open = findOpen();
|
|
251
|
+
// #464 — a blocker open past the stuck timeout is ended as `expired` (the existing terminal state) and
|
|
252
|
+
// this occurrence dispatches normally. Bounded: each pass ends one exact open task or stops.
|
|
253
|
+
for (let guard = 0; open && guard < 20 && blockerTimedOut(open, row.schedule_json, advanceAfter); guard++) {
|
|
254
|
+
const limitMs = stuckTimeoutMs(row.schedule_json);
|
|
255
|
+
const ageMs = advanceAfter.getTime() - parseDbTimestampMs(open.created_at);
|
|
256
|
+
if (!expireStuckBlocker(open.task_id, row.network_id, ageMs, limitMs)) break;
|
|
257
|
+
timedOut.push({ taskId: open.task_id, fromStatus: open.status });
|
|
258
|
+
// Each stuck episode gets exactly one notice: if the stuck notice already went out (N skips), stay quiet.
|
|
259
|
+
if (skipsBlockedBy(row.schedule_id, open.task_id) < stuckNoticeSkips()) {
|
|
260
|
+
notices.push(stuckNoticeFor(row, "timed_out", open, skipsBlockedBy(row.schedule_id, open.task_id), limitMs));
|
|
261
|
+
}
|
|
262
|
+
open = findOpen();
|
|
263
|
+
}
|
|
239
264
|
if (open) {
|
|
240
265
|
finalStatus = "skipped";
|
|
241
266
|
const blockedState = open.received ? "in_progress" : "not_received";
|
|
@@ -251,6 +276,9 @@ export function dispatchScheduledOccurrence(row: ScheduledRow, scheduledFor: str
|
|
|
251
276
|
blockedState,
|
|
252
277
|
],
|
|
253
278
|
);
|
|
279
|
+
// #464 — the Nth consecutive skip behind the same task tells the schedule's creator, once.
|
|
280
|
+
const skips = skipsBlockedBy(row.schedule_id, open.task_id);
|
|
281
|
+
if (skips === stuckNoticeSkips()) notices.push(stuckNoticeFor(row, "stuck", open, skips, stuckTimeoutMs(row.schedule_json)));
|
|
254
282
|
if (advanceSchedule) advance(row, scheduledFor, advanceAfter);
|
|
255
283
|
return;
|
|
256
284
|
}
|
|
@@ -349,6 +377,8 @@ export function dispatchScheduledOccurrence(row: ScheduledRow, scheduledFor: str
|
|
|
349
377
|
// never turn a durably-created task into an API-level failure: callers may
|
|
350
378
|
// retry an apparent failure and create a second manual occurrence.
|
|
351
379
|
try {
|
|
380
|
+
for (const t of timedOut) logTaskEvent(t.taskId, t.fromStatus, "expired", "hub-scheduler", `stuck timeout; schedule=${row.schedule_id} run=${runId}`);
|
|
381
|
+
for (const n of notices) sendStuckNotice(n);
|
|
352
382
|
if (createdTaskId) logTaskEvent(createdTaskId, null, "delivered", "hub-scheduler", `schedule=${row.schedule_id} run=${runId}`);
|
|
353
383
|
if (event) {
|
|
354
384
|
const pending = db.get<{ cnt: number }>("SELECT COUNT(*) AS cnt FROM inbox WHERE session_name = ?1 AND network_id = ?2 AND acked = 0", event.alias, event.networkId);
|
package/src/server.ts
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import { buildServeErrorResponse } from "./serve-error.js";
|
|
2
|
-
import { maybeGzipResponse, trimLightTask } from "./http-gzip";
|
|
2
|
+
import { markGzipReusable, maybeGzipResponse, trimLightTask } from "./http-gzip";
|
|
3
|
+
import { ifNoneMatchHits, memoStatusBody } from "./status-read-cache";
|
|
3
4
|
import { statusAliasResolverRead } from "./status-alias-resolver.js";
|
|
4
5
|
import { readNodeHealth } from "./node-health-store.js";
|
|
5
6
|
import { assertNodeHealthy, degradedLayers } from "./node-health-guard.js";
|
|
@@ -2310,6 +2311,11 @@ return Bun.serve({
|
|
|
2310
2311
|
sql += ` AND node_id = ?${params.length}`;
|
|
2311
2312
|
}
|
|
2312
2313
|
sql += " ORDER BY updated_at DESC";
|
|
2314
|
+
// #431 —— 同一份正文不重复算(status-read-cache.ts):键 = 最终 SQL + 参数(含调用者能看到谁)+ 投影;
|
|
2315
|
+
// 写 sessions / 健康上报 / 健康过期就失效。正文逐字节等于当场重算(status-read-cache-http.test.ts)。
|
|
2316
|
+
const memoKey = JSON.stringify([sql, params, isLight, withNodeId]);
|
|
2317
|
+
const { body: statusBody, etag: statusEtag } = memoStatusBody(memoKey, () => {
|
|
2318
|
+
let timeDependent = false;
|
|
2313
2319
|
// `model` comes straight from the explicit sessions projection; `runtime` is
|
|
2314
2320
|
// derived from the raw `agent` field. Both default to null for old nodes
|
|
2315
2321
|
// that never reported a model — the dashboard falls back to a placeholder.
|
|
@@ -2372,6 +2378,8 @@ return Bun.serve({
|
|
|
2372
2378
|
// null = not reported / stale / older agent-node — never "healthy".
|
|
2373
2379
|
...(() => {
|
|
2374
2380
|
const h = readNodeHealth(s.network_id, s.alias);
|
|
2381
|
+
// health_observed_ms_ago 随时间走:带上它的正文不能缓存。
|
|
2382
|
+
if (h) timeDependent = true;
|
|
2375
2383
|
return { health: h?.health ?? null, health_observed_ms_ago: h?.observed_ms_ago ?? null };
|
|
2376
2384
|
})(),
|
|
2377
2385
|
...degradedField(s.network_id, s.alias),
|
|
@@ -2384,8 +2392,16 @@ return Bun.serve({
|
|
|
2384
2392
|
else acc.idle++;
|
|
2385
2393
|
return acc;
|
|
2386
2394
|
}, { idle: 0, working: 0, offline: 0, total: sessions.length });
|
|
2387
|
-
|
|
2388
|
-
|
|
2395
|
+
return { body: JSON.stringify({ ok: true, sessions, summary }), cacheable: !timeDependent };
|
|
2396
|
+
});
|
|
2397
|
+
// 强 ETag = 正文哈希;If-None-Match 命中回 304 空体(旧 agent-node 不发,app / dashboard 接上后受益)。
|
|
2398
|
+
const statusHeaders: Record<string, string> = { ETag: statusEtag, "Cache-Control": "private, no-cache" };
|
|
2399
|
+
if (aliasResolverRead) statusHeaders["X-Status-Projection"] = "alias-resolver";
|
|
2400
|
+
if (ifNoneMatchHits(req.headers.get("if-none-match"), statusEtag)) {
|
|
2401
|
+
return withCors(req, new Response(null, { status: 304, headers: statusHeaders }));
|
|
2402
|
+
}
|
|
2403
|
+
// Content-Type 与 Response.json 的一致;同一份正文的 gzip 结果按 ETag 复用(http-gzip.ts)。
|
|
2404
|
+
const res = markGzipReusable(new Response(statusBody, { headers: { ...statusHeaders, "Content-Type": "application/json;charset=utf-8" } }), statusEtag);
|
|
2389
2405
|
return withCors(req, res);
|
|
2390
2406
|
}
|
|
2391
2407
|
|
|
@@ -0,0 +1,229 @@
|
|
|
1
|
+
// #431 —— GET /api/status 记忆化 + 强 ETag(status-read-cache.ts)。钉住:
|
|
2
|
+
// - 正文逐字节不变:随机写序列(改 sessions、健康上报、健康过期、改名清理)之后,每种读法「可能命中缓存的那一份」
|
|
3
|
+
// 都等于「清空缓存当场重算的那一份」;而且确实命中过(不是全靠重算凑出来的绿);
|
|
4
|
+
// - JSON.stringify 与原来的 Response.json 对这些正文逐字节一致;Content-Type 不变;
|
|
5
|
+
// - 不同调用者(另一个网络的用户、节点令牌的别名解析读法)键不串;
|
|
6
|
+
// - 全量投影里有健康报告(health_observed_ms_ago 随时间走)就不缓存;
|
|
7
|
+
// - ETag / If-None-Match → 304;gzip 解开等于原文。
|
|
8
|
+
import { describe, expect, test, beforeAll, afterAll } from "bun:test";
|
|
9
|
+
import { mkdtempSync, rmSync } from "fs";
|
|
10
|
+
import { join } from "path";
|
|
11
|
+
import { tmpdir } from "os";
|
|
12
|
+
import { gunzipSync } from "zlib";
|
|
13
|
+
import { register, createNetworkTokenForNode } from "./auth.js";
|
|
14
|
+
import { db } from "./db.js";
|
|
15
|
+
import { clearNodeHealthStore, recordNodeHealth, NODE_HEALTH_TTL_MS } from "./node-health-store.js";
|
|
16
|
+
import { __resetStatusCacheForTest, __setStatusCacheBypassForTest, noteStatement, statusCacheStats, statusWriteGeneration } from "./status-read-cache.js";
|
|
17
|
+
|
|
18
|
+
const DIR = mkdtempSync(join(tmpdir(), "anet-status-read-cache-"));
|
|
19
|
+
let BASE = "";
|
|
20
|
+
let hub: any = null;
|
|
21
|
+
const PW = "StatusReadCachePassw0rd!";
|
|
22
|
+
let userToken = "", userId = "", NET = "", nodeToken = "", otherToken = "", OTHER_NET = "";
|
|
23
|
+
const ROWS = 60;
|
|
24
|
+
|
|
25
|
+
// 时间戳在 JS 里算好再绑定(SQLite 的 datetime('now', …) 拼接在 PostgreSQL 上没有,这份测试两种库都跑)。
|
|
26
|
+
const ts = (offsetSec: number) => new Date(Date.now() + offsetSec * 1000).toISOString().replace("T", " ").slice(0, 19);
|
|
27
|
+
function seed(i: number) {
|
|
28
|
+
const node = `node_src_${i}`, alias = i % 2 ? `示例-${i}` : `src-${i}`;
|
|
29
|
+
db.run(`INSERT INTO nodes (node_id, node_name, alias, network_id, owner_user_id) VALUES (?1, ?2, ?2, ?3, ?4)`, [node, alias, NET, userId]);
|
|
30
|
+
db.run(
|
|
31
|
+
`INSERT INTO sessions (resume_id, alias, node_id, status, network_id, task, output, project_dir, hostname, server, agent, version,
|
|
32
|
+
cpu_load_1min, mem_total_gb, process_rss_bytes, external_schedules, updated_at, last_seen_at, registered_at)
|
|
33
|
+
VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?9, ?10, '2.5.0-preview.88', '1.5', '61.4', 121868288, ?11,
|
|
34
|
+
?12, ?13, '2026-09-01 00:00:00')`,
|
|
35
|
+
[`r_${node}`, alias, node, i % 3 ? "idle" : "offline", NET, `示例任务 ${i}:\n "引号" \\ 反斜杠
`.repeat(1 + (i % 5)),
|
|
36
|
+
i % 4 ? null : `示例输出 ${i}`, `/home/user/project-${i}`, `sample-host-${i % 4}`,
|
|
37
|
+
i % 2 ? "agent-node:codex-app-server" : "claude-code", i % 3 ? null : JSON.stringify([{ id: `s${i}`, every: 60 }]), ts(-i), ts(0)],
|
|
38
|
+
);
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
async function get(path: string, headers: Record<string, string>) {
|
|
42
|
+
const res = await fetch(`${BASE}${path}`, { headers });
|
|
43
|
+
const buf = new Uint8Array(await res.arrayBuffer());
|
|
44
|
+
return { status: res.status, buf, headers: res.headers };
|
|
45
|
+
}
|
|
46
|
+
const text = (b: Uint8Array) => new TextDecoder().decode(b);
|
|
47
|
+
|
|
48
|
+
type Variant = { name: string; path: () => string; headers: () => Record<string, string> };
|
|
49
|
+
const VARIANTS: Variant[] = [
|
|
50
|
+
{ name: "user full", path: () => `/api/status?network_id=${NET}`, headers: () => ({ Authorization: `Bearer ${userToken}` }) },
|
|
51
|
+
{ name: "user light", path: () => `/api/status?network_id=${NET}&light=1`, headers: () => ({ Authorization: `Bearer ${userToken}` }) },
|
|
52
|
+
{ name: "user alias", path: () => `/api/status?network_id=${NET}&alias=${encodeURIComponent("示例-7")}`, headers: () => ({ Authorization: `Bearer ${userToken}` }) },
|
|
53
|
+
{ name: "node light node_id", path: () => `/api/status?network_id=${NET}&light=1&node_id=node_src_9`, headers: () => ({ Authorization: `Bearer ${nodeToken}` }) },
|
|
54
|
+
{ name: "old alias resolver", path: () => `/api/status?network_id=${NET}`, headers: () => ({ Authorization: `Bearer ${nodeToken}`, Accept: "application/json" }) },
|
|
55
|
+
{ name: "node full=1", path: () => `/api/status?network_id=${NET}&full=1`, headers: () => ({ Authorization: `Bearer ${nodeToken}`, Accept: "application/json" }) },
|
|
56
|
+
{ name: "other network user", path: () => `/api/status?network_id=${OTHER_NET}`, headers: () => ({ Authorization: `Bearer ${otherToken}` }) },
|
|
57
|
+
];
|
|
58
|
+
|
|
59
|
+
beforeAll(async () => {
|
|
60
|
+
process.env.COMMHUB_DB ||= join(DIR, "hub.db");
|
|
61
|
+
process.env.COMMHUB_UPLOADS_DIR = join(DIR, "uploads");
|
|
62
|
+
process.env.HOST = "127.0.0.1";
|
|
63
|
+
const a = register(`src_owner_${Date.now()}`, PW);
|
|
64
|
+
userToken = a.token!; userId = a.user!.user_id; NET = a.network_id!;
|
|
65
|
+
for (let i = 0; i < ROWS; i++) seed(i);
|
|
66
|
+
nodeToken = createNetworkTokenForNode(userId, NET, "src-8", "node_src_8").token!;
|
|
67
|
+
const b = register(`src_other_${Date.now()}`, PW);
|
|
68
|
+
otherToken = b.token!; OTHER_NET = b.network_id!;
|
|
69
|
+
const mod: any = await import("./server.js");
|
|
70
|
+
hub = mod.bootServer({ port: 0, hostname: "127.0.0.1" });
|
|
71
|
+
BASE = `http://127.0.0.1:${hub.port}`;
|
|
72
|
+
}, 30_000);
|
|
73
|
+
|
|
74
|
+
afterAll(() => {
|
|
75
|
+
try { hub?.stop?.(true); } catch {}
|
|
76
|
+
try { rmSync(DIR, { recursive: true, force: true }); } catch {}
|
|
77
|
+
});
|
|
78
|
+
|
|
79
|
+
describe("write detection", () => {
|
|
80
|
+
test("writes that name sessions bump the generation; reads and other tables do not", () => {
|
|
81
|
+
const g = statusWriteGeneration();
|
|
82
|
+
noteStatement("SELECT * FROM sessions");
|
|
83
|
+
noteStatement("UPDATE tasks SET status = 'x'");
|
|
84
|
+
noteStatement("INSERT INTO inbox (id) VALUES (1)");
|
|
85
|
+
expect(statusWriteGeneration()).toBe(g);
|
|
86
|
+
noteStatement(" update sessions set status='idle'");
|
|
87
|
+
noteStatement("INSERT INTO sessions (resume_id) VALUES ('x') RETURNING resume_id");
|
|
88
|
+
noteStatement("DELETE FROM sessions WHERE alias = 'x'");
|
|
89
|
+
noteStatement("WITH t AS (SELECT 1) UPDATE sessions SET task = NULL");
|
|
90
|
+
expect(statusWriteGeneration()).toBe(g + 4);
|
|
91
|
+
});
|
|
92
|
+
|
|
93
|
+
test("the hook sits on the shared db adapter (every module's writes go through it)", () => {
|
|
94
|
+
const g = statusWriteGeneration();
|
|
95
|
+
db.run("UPDATE sessions SET progress = progress WHERE 1=0");
|
|
96
|
+
expect(statusWriteGeneration()).toBe(g + 1);
|
|
97
|
+
});
|
|
98
|
+
|
|
99
|
+
test("no foreign key / trigger can change sessions without naming it", () => {
|
|
100
|
+
if (db.dialect === "postgres") {
|
|
101
|
+
const fks = db.all<{ c: number }>(`SELECT COUNT(*) AS c FROM information_schema.referential_constraints rc
|
|
102
|
+
JOIN information_schema.table_constraints tc ON tc.constraint_name = rc.unique_constraint_name WHERE tc.table_name = 'sessions'`);
|
|
103
|
+
expect(Number(fks[0]?.c ?? 0)).toBe(0);
|
|
104
|
+
const trg = db.all<{ c: number }>(`SELECT COUNT(*) AS c FROM information_schema.triggers WHERE action_statement ILIKE '%sessions%' OR event_object_table = 'sessions'`);
|
|
105
|
+
expect(Number(trg[0]?.c ?? 0)).toBe(0);
|
|
106
|
+
return;
|
|
107
|
+
}
|
|
108
|
+
const fks = db.all<{ sql: string }>("SELECT sql FROM sqlite_master WHERE sql IS NOT NULL").map(r => r.sql);
|
|
109
|
+
expect(fks.filter(s => /references\s+sessions\b/i.test(s))).toEqual([]);
|
|
110
|
+
expect(fks.filter(s => /create\s+trigger/i.test(s) && /\bsessions\b/i.test(s))).toEqual([]);
|
|
111
|
+
});
|
|
112
|
+
});
|
|
113
|
+
|
|
114
|
+
describe("GET /api/status bodies are byte-identical with the cache", () => {
|
|
115
|
+
test("random writes / health reports / expiry: cached == recomputed, and the cache is really used", async () => {
|
|
116
|
+
__resetStatusCacheForTest();
|
|
117
|
+
let rnd = 461;
|
|
118
|
+
const r = () => (rnd = (rnd * 1103515245 + 12345) % 2147483648) / 2147483648;
|
|
119
|
+
let compared = 0, hits = 0;
|
|
120
|
+
for (let step = 0; step < 60; step++) {
|
|
121
|
+
const op = Math.floor(r() * 6);
|
|
122
|
+
const i = Math.floor(r() * ROWS);
|
|
123
|
+
const alias = i % 2 ? `示例-${i}` : `src-${i}`;
|
|
124
|
+
if (op === 0) db.run(`UPDATE sessions SET status = ?1, updated_at = ?2 WHERE node_id = ?3`, [r() < 0.5 ? "idle" : "working", ts(step), `node_src_${i}`]);
|
|
125
|
+
if (op === 1) db.run(`UPDATE sessions SET task = ?1 WHERE node_id = ?2`, [`新任务 ${step}`, `node_src_${i}`]);
|
|
126
|
+
if (op === 2) recordNodeHealth(NET, alias, { bridge: "ok", app_server: { ok: r() < 0.5, rtt_ms: 3, last_error: null }, model_auth: "ok" });
|
|
127
|
+
if (op === 3) {
|
|
128
|
+
// A report that expires a few ms from now: the degraded field must disappear on time.
|
|
129
|
+
recordNodeHealth(NET, alias, { bridge: "ok", app_server: { ok: false, rtt_ms: null, last_error: "closed (code=1006)" }, model_auth: "ok" }, Date.now() - NODE_HEALTH_TTL_MS + 30);
|
|
130
|
+
await Bun.sleep(40);
|
|
131
|
+
}
|
|
132
|
+
if (op === 4) db.run(`UPDATE sessions SET alias = ?1 WHERE node_id = ?2`, [`${alias}-改`, `node_src_${i}`]);
|
|
133
|
+
// op 5: no write at all — the next reads must be cache hits for the cacheable variants.
|
|
134
|
+
for (const v of VARIANTS) {
|
|
135
|
+
const first = await get(v.path(), v.headers());
|
|
136
|
+
const before = statusCacheStats.hits;
|
|
137
|
+
const again = await get(v.path(), v.headers());
|
|
138
|
+
hits += statusCacheStats.hits - before;
|
|
139
|
+
// Fresh = recomputed right now, without dropping what the cache holds (stale entries must stay in play
|
|
140
|
+
// across steps — clearing here would hide a missed invalidation).
|
|
141
|
+
__setStatusCacheBypassForTest(true);
|
|
142
|
+
const fresh = await get(v.path(), v.headers()).finally(() => __setStatusCacheBypassForTest(false));
|
|
143
|
+
expect(first.status).toBe(200);
|
|
144
|
+
const norm = (b: Uint8Array) => text(b).replace(/"health_observed_ms_ago":\d+/g, '"health_observed_ms_ago":0');
|
|
145
|
+
expect(norm(first.buf)).toBe(norm(fresh.buf));
|
|
146
|
+
expect(norm(again.buf)).toBe(norm(fresh.buf));
|
|
147
|
+
if (!text(fresh.buf).includes('"health_observed_ms_ago":')) expect(text(again.buf)).toBe(text(fresh.buf));
|
|
148
|
+
compared++;
|
|
149
|
+
}
|
|
150
|
+
}
|
|
151
|
+
expect(compared).toBe(60 * VARIANTS.length);
|
|
152
|
+
// Cacheable variants (everything except a full body carrying a health report) read twice in a row must hit.
|
|
153
|
+
expect(hits).toBeGreaterThan(60 * 4);
|
|
154
|
+
}, 120_000);
|
|
155
|
+
|
|
156
|
+
test("JSON.stringify == Response.json for these bodies; headers keep Content-Type, add ETag", async () => {
|
|
157
|
+
for (const v of VARIANTS) {
|
|
158
|
+
const res = await get(v.path(), v.headers());
|
|
159
|
+
const body = text(res.buf);
|
|
160
|
+
expect(await Response.json(JSON.parse(body)).text()).toBe(body);
|
|
161
|
+
expect(res.headers.get("content-type")).toBe("application/json;charset=utf-8");
|
|
162
|
+
expect(res.headers.get("etag")).toMatch(/^"s-[0-9a-z]+-[0-9a-z]+"$/);
|
|
163
|
+
}
|
|
164
|
+
});
|
|
165
|
+
|
|
166
|
+
test("keys do not leak across callers", async () => {
|
|
167
|
+
const owner = JSON.parse(text((await get(`/api/status?network_id=${NET}`, { Authorization: `Bearer ${userToken}` })).buf));
|
|
168
|
+
const other = JSON.parse(text((await get(`/api/status?network_id=${OTHER_NET}`, { Authorization: `Bearer ${otherToken}` })).buf));
|
|
169
|
+
expect(owner.sessions.length).toBe(ROWS);
|
|
170
|
+
expect(other.sessions.length).toBe(0);
|
|
171
|
+
const resolver = await get(`/api/status?network_id=${NET}`, { Authorization: `Bearer ${nodeToken}`, Accept: "application/json" });
|
|
172
|
+
expect(resolver.headers.get("x-status-projection")).toBe("alias-resolver");
|
|
173
|
+
expect(Object.keys(JSON.parse(text(resolver.buf)).sessions[0])).not.toContain("host");
|
|
174
|
+
});
|
|
175
|
+
|
|
176
|
+
test("a health report expiring (no write at all) invalidates the cached body on time", async () => {
|
|
177
|
+
__resetStatusCacheForTest();
|
|
178
|
+
clearNodeHealthStore();
|
|
179
|
+
const v = VARIANTS[1]; // user light: carries `degraded` for a fresh down app_server
|
|
180
|
+
const [{ alias: target }] = db.all<{ alias: string }>("SELECT alias FROM sessions WHERE node_id = 'node_src_2'");
|
|
181
|
+
recordNodeHealth(NET, target, { bridge: "ok", app_server: { ok: false, rtt_ms: null, last_error: "closed (code=1006)" }, model_auth: "ok" }, Date.now() - NODE_HEALTH_TTL_MS + 1_500);
|
|
182
|
+
const during = text((await get(v.path(), v.headers())).buf);
|
|
183
|
+
expect(during).toContain('"degraded":[{"layer":"app_server"');
|
|
184
|
+
const hitsBefore = statusCacheStats.hits;
|
|
185
|
+
expect(text((await get(v.path(), v.headers())).buf)).toBe(during);
|
|
186
|
+
expect(statusCacheStats.hits).toBe(hitsBefore + 1); // it was cached
|
|
187
|
+
await Bun.sleep(1_600);
|
|
188
|
+
const after = text((await get(v.path(), v.headers())).buf);
|
|
189
|
+
expect(after).not.toContain('"degraded"');
|
|
190
|
+
__resetStatusCacheForTest();
|
|
191
|
+
expect(text((await get(v.path(), v.headers())).buf)).toBe(after);
|
|
192
|
+
});
|
|
193
|
+
|
|
194
|
+
test("full projection with a health report is not cached (its body moves with time)", async () => {
|
|
195
|
+
__resetStatusCacheForTest();
|
|
196
|
+
recordNodeHealth(NET, "src-0", { bridge: "ok", model_auth: "ok" });
|
|
197
|
+
const v = VARIANTS[0];
|
|
198
|
+
const a = text((await get(v.path(), v.headers())).buf);
|
|
199
|
+
await Bun.sleep(15);
|
|
200
|
+
const b = text((await get(v.path(), v.headers())).buf);
|
|
201
|
+
expect(statusCacheStats.uncacheable).toBeGreaterThanOrEqual(2);
|
|
202
|
+
expect(a).not.toBe(b); // health_observed_ms_ago moved — a cached copy would have frozen it
|
|
203
|
+
});
|
|
204
|
+
});
|
|
205
|
+
|
|
206
|
+
describe("ETag / 304 / gzip", () => {
|
|
207
|
+
test("If-None-Match with the current ETag → 304 empty; after a write → 200 with a new ETag", async () => {
|
|
208
|
+
const h = { Authorization: `Bearer ${nodeToken}`, Accept: "application/json" };
|
|
209
|
+
const first = await get(`/api/status?network_id=${NET}`, h);
|
|
210
|
+
const etag = first.headers.get("etag")!;
|
|
211
|
+
const not = await get(`/api/status?network_id=${NET}`, { ...h, "If-None-Match": etag });
|
|
212
|
+
expect(not.status).toBe(304);
|
|
213
|
+
expect(not.buf.byteLength).toBe(0);
|
|
214
|
+
expect(not.headers.get("x-status-projection")).toBe("alias-resolver");
|
|
215
|
+
db.run(`UPDATE sessions SET status = 'working', updated_at = ?1 WHERE node_id = 'node_src_3'`, [ts(120)]);
|
|
216
|
+
const changed = await get(`/api/status?network_id=${NET}`, { ...h, "If-None-Match": etag });
|
|
217
|
+
expect(changed.status).toBe(200);
|
|
218
|
+
expect(changed.headers.get("etag")).not.toBe(etag);
|
|
219
|
+
});
|
|
220
|
+
|
|
221
|
+
test("gzip (the old node's Accept-Encoding: gzip, deflate) decodes to the same bytes", async () => {
|
|
222
|
+
const h = { Authorization: `Bearer ${nodeToken}`, Accept: "application/json" };
|
|
223
|
+
const plain = await get(`/api/status?network_id=${NET}`, h);
|
|
224
|
+
const res = await fetch(`${BASE}/api/status?network_id=${NET}`, { headers: { ...h, "Accept-Encoding": "gzip, deflate" }, decompress: false } as any);
|
|
225
|
+
expect(res.headers.get("content-encoding")).toBe("gzip");
|
|
226
|
+
const raw = new Uint8Array(await res.arrayBuffer());
|
|
227
|
+
expect(text(gunzipSync(raw))).toBe(text(plain.buf));
|
|
228
|
+
});
|
|
229
|
+
});
|
|
@@ -0,0 +1,108 @@
|
|
|
1
|
+
// #431(app「连接较慢」)—— GET /api/status 的序列化结果记忆化 + 强 ETag。
|
|
2
|
+
//
|
|
3
|
+
// 现场(2026-10-02 生产 /api/stats/routes,15 分钟窗口):GET /api/status 2000 次 ≈ 2.2 次/秒,全部来自 UA「node」
|
|
4
|
+
// (旧版 agent-node 的别名解析器,.82 起给它 light+node_id 投影),线上平均 30 KB/次(已经是 gzip 后的字节:
|
|
5
|
+
// node 的 fetch 发 `Accept-Encoding: gzip, deflate`)。两次读之间表多半没变(report_status 约 0.75 次/秒),
|
|
6
|
+
// 但每次都重新 SELECT 300 行、拼对象、JSON.stringify、再 gzip 一遍。
|
|
7
|
+
//
|
|
8
|
+
// 这里只做「同一份正文不重复算」,**不改任何正文**:
|
|
9
|
+
// 键 = 最终 SQL + 参数(里面已经编码了调用者能看到哪些节点)+ 投影开关;
|
|
10
|
+
// 失效 = 任何写 sessions 表的语句(db.ts 里统一挂钩,SQLite / PG 同一处)、任何健康上报、
|
|
11
|
+
// 任何一份健康报告过 TTL(那一刻 degraded / health 字段会变)。
|
|
12
|
+
// 全量投影里 `health_observed_ms_ago` 随时间走:只要有一行带健康报告,这份正文就是时间的函数,不缓存。
|
|
13
|
+
// 命中时正文和当场重算逐字节相同(status-read-cache-http.test.ts 用随机写序列钉住)。
|
|
14
|
+
//
|
|
15
|
+
// ETag:正文的哈希,`If-None-Match` 命中回 304 空体。🔴 旧 agent-node 不发 If-None-Match(fetch 没有 HTTP 缓存),
|
|
16
|
+
// 所以 ETag 只帮得上会带它的客户端(app / dashboard 以后接上);隧道上省字节靠的是同一份 gzip 只压一次
|
|
17
|
+
// 且用更高的压缩级别(http-gzip.ts:可复用的正文压一次、存起来,级别 9)。
|
|
18
|
+
|
|
19
|
+
import type { DbAdapter } from "./db-adapter";
|
|
20
|
+
import { nodeHealthNextExpiryAt, nodeHealthVersion } from "./node-health-store.js";
|
|
21
|
+
|
|
22
|
+
let sessionsWriteGen = 0;
|
|
23
|
+
|
|
24
|
+
const WRITE_START = /^\s*(?:insert|update|delete|replace|upsert|merge|alter|drop|create|truncate|with)\b/i;
|
|
25
|
+
const SESSIONS_WORD = /\bsessions\b/i;
|
|
26
|
+
|
|
27
|
+
/**
|
|
28
|
+
* 每条经过 db 的语句都过一遍。写语句(含带 RETURNING 的、DDL)里点名了 sessions → 代数 +1。读语句不动。
|
|
29
|
+
* 不会有「没点名 sessions 却改了它」的写:两种方言的 schema 里都没有指向 sessions 的外键 / 级联删除
|
|
30
|
+
* (status-read-cache.test.ts 钉住这一点)。
|
|
31
|
+
*/
|
|
32
|
+
export function noteStatement(sql: string): void {
|
|
33
|
+
if (WRITE_START.test(sql) && SESSIONS_WORD.test(sql)) sessionsWriteGen++;
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
/** 原生 exec(迁移 / DDL / 多语句):不逐句判断,一律失效。 */
|
|
37
|
+
export function noteExec(): void { sessionsWriteGen++; }
|
|
38
|
+
|
|
39
|
+
export function statusWriteGeneration(): number { return sessionsWriteGen; }
|
|
40
|
+
|
|
41
|
+
/**
|
|
42
|
+
* db.ts 用它包住唯一的那个适配器:每条语句先报给上面两个函数。挂在这一个出口上,SQLite / PG 一样,
|
|
43
|
+
* 不靠每个写 sessions 的地方各自记得。直接换掉实例上的四个方法(不用 Proxy):有测试按 `db.all = …`
|
|
44
|
+
* 临时打桩数查询次数,方法得是普通可写属性。
|
|
45
|
+
*/
|
|
46
|
+
export function withStatementHook(inner: DbAdapter): DbAdapter {
|
|
47
|
+
const run = inner.run.bind(inner), get = inner.get.bind(inner), all = inner.all.bind(inner), exec = inner.exec.bind(inner);
|
|
48
|
+
inner.run = (sql: string, params?: any[]) => { noteStatement(sql); return run(sql, params); };
|
|
49
|
+
inner.get = ((sql: string, ...params: any[]) => { noteStatement(sql); return get(sql, ...params); }) as DbAdapter["get"];
|
|
50
|
+
inner.all = ((sql: string, ...params: any[]) => { noteStatement(sql); return all(sql, ...params); }) as DbAdapter["all"];
|
|
51
|
+
inner.exec = (sql: string) => { noteExec(); return exec(sql); };
|
|
52
|
+
return inner;
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
type Entry = { gen: number; healthVer: number; validUntil: number; body: string; etag: string };
|
|
56
|
+
const MAX_ENTRIES = 64;
|
|
57
|
+
const memo = new Map<string, Entry>();
|
|
58
|
+
export const statusCacheStats = { hits: 0, misses: 0, uncacheable: 0 };
|
|
59
|
+
|
|
60
|
+
function etagOf(body: string): string {
|
|
61
|
+
return `"s-${Bun.hash(body).toString(36)}-${body.length.toString(36)}"`;
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
/**
|
|
65
|
+
* 取或算。compute 返回正文和它能不能缓存(全量投影里有健康报告 = 正文随时间变 = 不能)。
|
|
66
|
+
* 返回的 etag 总是这份正文的哈希(不缓存的也给,便于 304)。
|
|
67
|
+
*/
|
|
68
|
+
let bypass = false;
|
|
69
|
+
/** 测试用:临时绕过缓存(既不读也不写),拿「当场重算」的那一份来对比,而不清掉缓存里已有的条目。 */
|
|
70
|
+
export function __setStatusCacheBypassForTest(on: boolean): void { bypass = on; }
|
|
71
|
+
|
|
72
|
+
export function memoStatusBody(key: string, compute: () => { body: string; cacheable: boolean }, now = Date.now()): { body: string; etag: string; hit: boolean } {
|
|
73
|
+
if (bypass) { const { body } = compute(); return { body, etag: etagOf(body), hit: false }; }
|
|
74
|
+
const gen = sessionsWriteGen;
|
|
75
|
+
const healthVer = nodeHealthVersion();
|
|
76
|
+
const hit = memo.get(key);
|
|
77
|
+
if (hit && hit.gen === gen && hit.healthVer === healthVer && now < hit.validUntil) {
|
|
78
|
+
statusCacheStats.hits++;
|
|
79
|
+
memo.delete(key);
|
|
80
|
+
memo.set(key, hit); // LRU
|
|
81
|
+
return { body: hit.body, etag: hit.etag, hit: true };
|
|
82
|
+
}
|
|
83
|
+
const { body, cacheable } = compute();
|
|
84
|
+
const etag = etagOf(body);
|
|
85
|
+
// 以算之前的代数入账:算的过程中若有写(改名清理之类),下一次比较就对不上、重算 —— 宁可多算一次也不发旧正文。
|
|
86
|
+
if (!cacheable) {
|
|
87
|
+
statusCacheStats.uncacheable++;
|
|
88
|
+
memo.delete(key);
|
|
89
|
+
return { body, etag, hit: false };
|
|
90
|
+
}
|
|
91
|
+
statusCacheStats.misses++;
|
|
92
|
+
memo.set(key, { gen, healthVer, validUntil: nodeHealthNextExpiryAt(now), body, etag });
|
|
93
|
+
while (memo.size > MAX_ENTRIES) memo.delete(memo.keys().next().value as string);
|
|
94
|
+
return { body, etag, hit: false };
|
|
95
|
+
}
|
|
96
|
+
|
|
97
|
+
/** If-None-Match 里有这个 ETag(或 `*`)。弱比较:去掉 W/ 前缀。 */
|
|
98
|
+
export function ifNoneMatchHits(header: string | null, etag: string): boolean {
|
|
99
|
+
if (!header) return false;
|
|
100
|
+
const bare = (t: string) => t.trim().replace(/^W\//, "");
|
|
101
|
+
return header.split(",").some((t) => t.trim() === "*" || bare(t) === bare(etag));
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
/** 测试用。 */
|
|
105
|
+
export function __resetStatusCacheForTest(): void {
|
|
106
|
+
memo.clear();
|
|
107
|
+
statusCacheStats.hits = statusCacheStats.misses = statusCacheStats.uncacheable = 0;
|
|
108
|
+
}
|