@sleep2agi/commhub-server 0.9.0-preview.13 → 0.9.0-preview.131

package/src/scheduled-doorbell-nodeless-session.test.ts

@@ -0,0 +1,134 @@
+import { afterAll, beforeEach, describe, expect, test } from "bun:test";
+import { db } from "./db.js";
+import { createSSEStream } from "./push.js";
+import { dispatchScheduledOccurrence } from "./scheduled-tasks.js";
+
+// A claude-code node's channel server reports status without node_id, so its
+// sessions row has node_id NULL. The scheduler used to look the session up by
+// node_id only, judged every occurrence "queued" and never rang the doorbell:
+// the task sat in the inbox until an unrelated push drained it (20–36 min in
+// production).
+
+const NET = "net_sched_doorbell_nodeless";
+const NODE = "n_sched_doorbell_cc";
+const ALIAS = "sched-doorbell-cc";
+
+function cleanup(): void {
+  for (const table of ["task_events", "inbox", "scheduled_task_runs", "scheduled_tasks", "tasks", "sessions", "nodes"]) {
+    try { db.run(`DELETE FROM ${table} WHERE network_id = ?1`, [NET]); } catch {}
+  }
+}
+
+function seedNode(): void {
+  db.run(
+    `INSERT INTO nodes (node_id, node_name, alias, runtime, network_id, lifecycle_state)
+     VALUES (?1, ?2, ?2, 'claude-code-cli', ?3, 'active')`,
+    [NODE, ALIAS, NET],
+  );
+}
+
+function seedSession(status: string, nodeId: string | null): void {
+  db.run(
+    `INSERT INTO sessions (resume_id, alias, status, node_id, network_id, agent, updated_at, last_seen_at)
+     VALUES (?1, ?2, ?3, ?4, ?5, 'claude-code', datetime('now'), datetime('now'))`,
+    [`cc-${crypto.randomUUID()}`, ALIAS, status, nodeId, NET],
+  );
+}
+
+function seedSchedule(): any {
+  const scheduleId = `sched_${crypto.randomUUID()}`;
+  db.run(
+    `INSERT INTO scheduled_tasks
+       (schedule_id, network_id, name, target_node_id, target_alias, task_content, priority,
+        schedule_type, schedule_json, timezone, status, next_run_at)
+     VALUES (?1, ?2, 'doorbell', ?3, ?4, 'scheduled doorbell task', 'normal', 'interval',
+             '{"type":"interval","every_seconds":3600}', 'UTC', 'active', datetime('now', '+1 hour'))`,
+    [scheduleId, NET, NODE, ALIAS],
+  );
+  return db.get("SELECT * FROM scheduled_tasks WHERE schedule_id = ?1", scheduleId);
+}
+
+/** Subscribe like the channel server does and collect decoded SSE frames. */
+function subscribe(): { frames: any[]; close: () => Promise<void> } {
+  const res = createSSEStream(ALIAS, NET);
+  const reader = res.body!.getReader();
+  const decoder = new TextDecoder();
+  const frames: any[] = [];
+  let buffer = "";
+  let open = true;
+  (async () => {
+    while (open) {
+      const { done, value } = await reader.read();
+      if (done) break;
+      buffer += decoder.decode(value, { stream: true });
+      const blocks = buffer.split("\n\n");
+      buffer = blocks.pop() || "";
+      for (const block of blocks) {
+        const line = block.split("\n").find((l) => l.startsWith("data: "));
+        if (line) frames.push(JSON.parse(line.slice(6)));
+      }
+    }
+  })().catch(() => {});
+  return {
+    frames,
+    close: async () => {
+      open = false;
+      await reader.cancel().catch(() => {});
+    },
+  };
+}
+
+async function settle(): Promise<void> {
+  await new Promise((resolve) => setTimeout(resolve, 20));
+}
+
+beforeEach(() => {
+  cleanup();
+  seedNode();
+});
+
+afterAll(cleanup);
+
+describe("scheduler doorbell for sessions reported without node_id", () => {
+  test("idle node_id-less session is delivered and its subscriber receives new_task", async () => {
+    seedSession("idle", null);
+    const sub = subscribe();
+    try {
+      await settle();
+      const result = dispatchScheduledOccurrence(seedSchedule(), new Date().toISOString(), false);
+      await settle();
+      expect(result.status).toBe("delivered");
+      expect(db.get<{ status: string }>("SELECT status FROM scheduled_task_runs WHERE run_id = ?1", result.runId)?.status).toBe("delivered");
+      const wake = sub.frames.filter((f) => f.type === "new_task");
+      expect(wake).toHaveLength(1);
+      expect(wake[0].from).toBe("scheduler");
+      expect(wake[0].inbox_count).toBe(1);
+    } finally {
+      await sub.close();
+    }
+  });
+
+  test("an offline session is skipped even if a stale subscriber is still connected", async () => {
+    seedSession("offline", null);
+    const sub = subscribe();
+    try {
+      await settle();
+      const result = dispatchScheduledOccurrence(seedSchedule(), new Date().toISOString(), false);
+      await settle();
+      expect(result.status).toBe("skipped");
+      expect(db.get<{ error_code: string; task_id: string | null }>("SELECT error_code, task_id FROM scheduled_task_runs WHERE run_id = ?1", result.runId))
+        .toEqual({ error_code: "target_offline", task_id: null });
+      expect(sub.frames.filter((f) => f.type === "new_task")).toHaveLength(0);
+    } finally {
+      await sub.close();
+    }
+  });
+
+  test("offline session with nobody listening is skipped without a task", () => {
+    seedSession("offline", null);
+    const result = dispatchScheduledOccurrence(seedSchedule(), new Date().toISOString(), false);
+    expect(result.status).toBe("skipped");
+    expect(db.get<{ error_code: string; task_id: string | null }>("SELECT error_code, task_id FROM scheduled_task_runs WHERE run_id = ?1", result.runId))
+      .toEqual({ error_code: "target_offline", task_id: null });
+  });
+});

package/src/scheduled-failures-http.test.ts

@@ -0,0 +1,233 @@
+// #523 —— 定时任务连续失败:告诉创建者(可选自动暂停)。
+//
+// 钉住:
+//   1. 连续 5 次失败(派出去的任务以 failed 结束)→ 给创建者恰好 1 条通知(别的成员 0 条),第 6–10 次不再发;
+//      通知里有排程名、目标节点、失败次数、最近一次的错误(截断)和怎么暂停。
+//   2. 派发时就失败(目标节点不存在)同样计数。
+//   3. 中间成功一次 → 重新上膛:再连续 5 次失败才发第二条。
+//   4. 一直失败:24 小时内不重发;上次告警早于 24 小时 → 再发一条。
+//   5. 默认不自动暂停;COMMHUB_SCHEDULE_FAILURE_AUTO_PAUSE_RUNS=3 → 第 3 次失败后排程 paused、next_run_at 清空、
+//      revision+1,并通知一次。
+//   6. GET /runs 返回 consecutive_failures / failure_alert_threshold / last_failure_alert_at,
+//      任务以 failed 结束的 run 的 error_message 取自任务的回复正文。
+//
+// 跑法:cd server && COMMHUB_DB=/tmp/x.db bun test src/scheduled-failures-http.test.ts
+//       PG:COMMHUB_TEST_PG_URL=… COMMHUB_PG_EXPERIMENTAL=1(tests/test2123-hub-postgres-ladder 里注册)
+import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, test } from "bun:test";
+import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
+import { addNetworkMember, createNetworkTokenForNode, register } from "./auth.js";
+import { db } from "./db.js";
+import { dispatchScheduledOccurrence, handleScheduledTaskRequest } from "./scheduled-tasks.js";
+import { SCHEDULE_FAILING_NOTICE_KIND, failureStreak } from "./scheduled-failures.js";
+import { registerTools } from "./tools.js";
+
+const activeDbPath = process.env.COMMHUB_DB ?? (process.env.COMMHUB_TEST_PG_URL ? "postgres" : undefined);
+if (!activeDbPath) throw new Error("scheduled-failures requires COMMHUB_DB (or COMMHUB_TEST_PG_URL) before module import");
+
+const stamp = Date.now();
+const ALIAS = `fail-node-${stamp}`;
+const NODE_ID = `n_fail_${stamp}`;
+let NET = "", creatorId = "", otherId = "";
+const ENV_KEYS = ["COMMHUB_SCHEDULE_FAILURE_NOTICE_RUNS", "COMMHUB_SCHEDULE_FAILURE_RENOTICE_SEC", "COMMHUB_SCHEDULE_FAILURE_AUTO_PAUSE_RUNS"];
+const SAFETY_ERROR = "执行出错: upstream model refused the request (safety system). " + "x".repeat(600);
+
+function seedSchedule(name = "前后端分离巡检", targetNodeId = NODE_ID): any {
+  const scheduleId = `sched_${crypto.randomUUID()}`;
+  db.run(
+    `INSERT INTO scheduled_tasks
+       (schedule_id, network_id, created_by, name, target_node_id, target_alias, task_content, priority,
+        schedule_type, schedule_json, timezone, status, next_run_at)
+     VALUES (?1, ?2, ?3, ?4, ?5, ?6, 'scheduled task', 'normal', 'interval',
+             '{"type":"interval","every_seconds":120}', 'Asia/Shanghai', 'active', datetime('now', '+2 minutes'))`,
+    [scheduleId, NET, creatorId, name, targetNodeId, ALIAS],
+  );
+  return reload(scheduleId);
+}
+const reload = (scheduleId: string) => db.get<any>("SELECT * FROM scheduled_tasks WHERE schedule_id = ?1", scheduleId);
+
+let slot = 0;
+const nextSlot = () => new Date(Date.UTC(2026, 9, 4, 0, 0, 0) + (++slot) * 120_000).toISOString();
+const fire = (schedule: any, now = new Date()) => dispatchScheduledOccurrence(schedule, nextSlot(), false, now);
+
+const notices = (userId: string) => db.all<{ title: string; content: string; meta_json: string; from_session: string; severity: string }>(
+  "SELECT title, content, meta_json, from_session, severity FROM user_inbox WHERE user_id = ?1 AND network_id = ?2 AND kind = ?3 ORDER BY created_at, message_id",
+  userId, NET, SCHEDULE_FAILING_NOTICE_KIND,
+);
+const noticesFor = (userId: string, scheduleId: string) => notices(userId).filter((n) => JSON.parse(n.meta_json).schedule_failing.schedule_id === scheduleId);
+
+async function nodeReply(args: Record<string, unknown>): Promise<any> {
+  const mcp = new McpServer({ name: "sched-fail", version: "0" }) as any;
+  const handlers: Record<string, (a: any) => Promise<any>> = {};
+  const original = mcp.tool.bind(mcp);
+  mcp.tool = (name: string, ...rest: any[]) => {
+    const handler = rest.at(-1);
+    if (typeof handler === "function") handlers[name] = handler;
+    return original(name, ...rest);
+  };
+  registerTools(mcp, undefined, NET, creatorId, ALIAS, true, null);
+  const out = await handlers.send_reply(args);
+  return JSON.parse(out.content[0].text);
+}
+
+/** 派一次,节点以 failed(或 replied)回复。 */
+async function runOnce(schedule: any, outcome: "failed" | "replied" = "failed"): Promise<{ runId: string; taskId: string }> {
+  const r = fire(schedule);
+  expect(r.status).toBe("delivered");
+  const out = await nodeReply({ alias: "scheduler", text: outcome === "failed" ? SAFETY_ERROR : "done", in_reply_to: r.taskId, status: outcome, from_session: ALIAS });
+  expect(out.ok).toBe(true);
+  return { runId: r.runId, taskId: r.taskId! };
+}
+
+async function getRuns(scheduleId: string): Promise<any> {
+  const url = new URL(`http://hub.test/api/scheduled-tasks/${scheduleId}/runs`);
+  const res = await handleScheduledTaskRequest({
+    req: new Request(url, { method: "GET" }), url,
+    auth: { userId: creatorId, networkId: NET, username: "creator" },
+    isAdmin: false, isNodeToken: false, scope: { networkId: NET, networkIds: [NET] } as any,
+  });
+  expect(res!.status).toBe(200);
+  return res!.json();
+}
+
+beforeAll(() => {
+  const creator = register(`fail_creator_${stamp}`, "FailCreator123!", undefined, "seed");
+  expect(creator.ok).toBe(true);
+  NET = creator.network_id!; creatorId = creator.user!.user_id;
+  const other = register(`fail_other_${stamp}`, "FailOther123!", undefined, "seed");
+  otherId = other.user!.user_id;
+  expect(addNetworkMember(NET, otherId, "member", creatorId, { agentAccess: "all" }).ok).toBe(true);
+  expect(createNetworkTokenForNode(creatorId, NET, ALIAS).ok).toBe(true);
+  db.run("INSERT INTO nodes (node_id, node_name, alias, runtime, network_id) VALUES (?1, ?2, ?2, 'claude-code', ?3)", [NODE_ID, ALIAS, NET]);
+  db.run("INSERT INTO sessions (resume_id, alias, status, node_id, network_id) VALUES (?1, ?2, 'idle', ?3, ?4)", [`r_fail_${stamp}`, ALIAS, NODE_ID, NET]);
+});
+beforeEach(() => { for (const k of ENV_KEYS) delete process.env[k]; });
+afterEach(() => { for (const k of ENV_KEYS) delete process.env[k]; });
+afterAll(() => { db.run("DELETE FROM user_inbox WHERE network_id = ?1 AND kind = ?2", [NET, SCHEDULE_FAILING_NOTICE_KIND]); });
+
+describe("#523 consecutive-failure alert", () => {
+  test("5 failed runs → exactly one notice to the creator with name/target/count/error/how to pause; runs 6–10 stay quiet", async () => {
+    const s = seedSchedule();
+    for (let i = 0; i < 4; i++) await runOnce(s);
+    expect(noticesFor(creatorId, s.schedule_id).length).toBe(0);
+    await runOnce(s);
+    const mine = noticesFor(creatorId, s.schedule_id);
+    expect(mine.length).toBe(1);
+    expect(noticesFor(otherId, s.schedule_id).length).toBe(0);
+    const n = mine[0];
+    expect(n.from_session).toBe(ALIAS);
+    expect(n.title).toBe("定时任务连续失败");
+    expect(n.severity).toBe("error");
+    for (const frag of ["前后端分离巡检", ALIAS, "连续 5 次", "safety system", "task_failed", "暂停", s.schedule_id]) expect(n.content).toContain(frag);
+    expect(n.content).not.toContain("x".repeat(400)); // 错误被截断
+    expect(JSON.parse(n.meta_json).schedule_failing).toEqual({ reason: "failing", schedule_id: s.schedule_id, consecutive_failures: 5, error_code: "task_failed" });
+    for (let i = 0; i < 5; i++) await runOnce(s);
+    expect(noticesFor(creatorId, s.schedule_id).length).toBe(1);
+    expect(reload(s.schedule_id).status).toBe("active"); // 默认不自动暂停
+  });
+
+  test("dispatch-time failures (target node gone) count too", () => {
+    const s = seedSchedule("孤儿排程", `n_missing_${stamp}`);
+    for (let i = 0; i < 5; i++) expect(fire(s).status).toBe("failed");
+    const mine = noticesFor(creatorId, s.schedule_id);
+    expect(mine.length).toBe(1);
+    expect(mine[0].content).toContain("target_node_not_found");
+    expect(mine[0].content).toContain("The bound node no longer exists");
+  });
+
+  test("a success in between re-arms: the next 5 failures notify again", async () => {
+    const s = seedSchedule();
+    for (let i = 0; i < 5; i++) await runOnce(s);
+    expect(noticesFor(creatorId, s.schedule_id).length).toBe(1);
+    await runOnce(s, "replied");
+    expect(failureStreak(s.schedule_id).consecutiveFailures).toBe(0);
+    for (let i = 0; i < 4; i++) await runOnce(s);
+    expect(noticesFor(creatorId, s.schedule_id).length).toBe(1);
+    await runOnce(s);
+    expect(noticesFor(creatorId, s.schedule_id).length).toBe(2);
+  });
+
+  test("still failing: no repeat within 24h; one more once the last alert is older than 24h", async () => {
+    const s = seedSchedule();
+    for (let i = 0; i < 5; i++) await runOnce(s);
+    expect(noticesFor(creatorId, s.schedule_id).length).toBe(1);
+    db.run("UPDATE scheduled_tasks SET failure_alert_at = ?1 WHERE schedule_id = ?2", [new Date(Date.now() - 23 * 3_600_000).toISOString(), s.schedule_id]);
+    await runOnce(s);
+    expect(noticesFor(creatorId, s.schedule_id).length).toBe(1);
+    db.run("UPDATE scheduled_tasks SET failure_alert_at = ?1 WHERE schedule_id = ?2", [new Date(Date.now() - 25 * 3_600_000).toISOString(), s.schedule_id]);
+    await runOnce(s);
+    expect(noticesFor(creatorId, s.schedule_id).length).toBe(2);
+    await runOnce(s);
+    expect(noticesFor(creatorId, s.schedule_id).length).toBe(2);
+  });
+
+  test("COMMHUB_SCHEDULE_FAILURE_NOTICE_RUNS changes N; no creator → no notice", async () => {
+    process.env.COMMHUB_SCHEDULE_FAILURE_NOTICE_RUNS = "2";
+    const s = seedSchedule();
+    await runOnce(s);
+    expect(noticesFor(creatorId, s.schedule_id).length).toBe(0);
+    await runOnce(s);
+    expect(noticesFor(creatorId, s.schedule_id).length).toBe(1);
+
+    const orphan = seedSchedule();
+    db.run("UPDATE scheduled_tasks SET created_by = NULL WHERE schedule_id = ?1", [orphan.schedule_id]);
+    const row = reload(orphan.schedule_id);
+    for (let i = 0; i < 3; i++) await runOnce(row);
+    expect(noticesFor(creatorId, orphan.schedule_id).length).toBe(0);
+  });
+});
+
+describe("#523 auto-pause (opt-in)", () => {
+  test("off by default: 10 failures leave the schedule active", async () => {
+    const s = seedSchedule();
+    for (let i = 0; i < 10; i++) await runOnce(s);
+    const after = reload(s.schedule_id);
+    expect(after.status).toBe("active");
+    expect(after.next_run_at).not.toBeNull();
+  });
+
+  test("COMMHUB_SCHEDULE_FAILURE_AUTO_PAUSE_RUNS=3 pauses after the 3rd failure and tells the creator once", async () => {
+    process.env.COMMHUB_SCHEDULE_FAILURE_AUTO_PAUSE_RUNS = "3";
+    const s = seedSchedule();
+    await runOnce(s); await runOnce(s);
+    expect(reload(s.schedule_id).status).toBe("active");
+    await runOnce(s);
+    const after = reload(s.schedule_id);
+    expect(after.status).toBe("paused");
+    expect(after.next_run_at).toBeNull();
+    expect(after.revision).toBe(s.revision + 1);
+    const mine = noticesFor(creatorId, s.schedule_id);
+    expect(mine.length).toBe(1);
+    expect(mine[0].title).toBe("定时任务连续失败,已自动暂停");
+    expect(JSON.parse(mine[0].meta_json).schedule_failing.reason).toBe("paused");
+    // 手动 run-now 再失败两次到 N=5:同一段失败,已经通知过 → 不再发
+    await runOnce(after); await runOnce(after);
+    expect(noticesFor(creatorId, s.schedule_id).length).toBe(1);
+  });
+});
+
+describe("#523 GET /runs", () => {
+  test("exposes consecutive_failures and the failed task's reason as error_message", async () => {
+    const s = seedSchedule();
+    await runOnce(s, "replied");
+    const fails = [await runOnce(s), await runOnce(s)];
+    const body = await getRuns(s.schedule_id);
+    expect(body.ok).toBe(true);
+    expect(body.consecutive_failures).toBe(2);
+    expect(body.failure_alert_threshold).toBe(5);
+    expect(body.last_failure_alert_at).toBeNull();
+    expect(body.runs.length).toBe(3);
+    const failed = body.runs.find((r: any) => r.run_id === fails[1].runId);
+    expect(failed.status).toBe("failed");
+    expect(failed.error_code).toBe("task_failed");
+    expect(failed.error_message).toContain("safety system");
+    expect(failed.error_message.length).toBeLessThanOrEqual(500);
+    for (const k of ["scheduled_for", "created_at", "completed_at", "task_id"]) expect(k in failed).toBe(true);
+    const ok = body.runs.find((r: any) => r.status === "replied");
+    expect(ok.error_message).toBeNull();
+
+    for (let i = 0; i < 3; i++) await runOnce(s);
+    const again = await getRuns(s.schedule_id);
+    expect(again.consecutive_failures).toBe(5);
+    expect(typeof again.last_failure_alert_at).toBe("string");
+  });
+});

package/src/scheduled-failures.ts

@@ -0,0 +1,179 @@
+// #523 —— 定时任务连续失败:告诉创建者(可选自动暂停)。
+//
+// 现场:一个每 2 分钟一次的排程,每一次派出去的任务都被节点以 failed 结束(上游模型拒绝),一天 720 次,
+//   排程照样 active,创建者什么都不知道。#464 只管「被上一次卡住」(skipped),不管「每次都失败」。
+//
+// 1. 计数:consecutive_failures = 最近一次成功(run.status='replied')之后,status='failed' 的 run 数。
+//    skipped / cancelled / expired / 还没结束的 run 不算失败,也不打断连续(expired 由 #464 的超时告警负责)。
+//    失败有两种来源,都会走到这里:派发时就失败(节点不在 / 被停用 / 降级 / 创建者无权限,
+//    scheduled-tasks.ts 提交后调用),以及派出去的任务以 failed 结束(db.ts 的 run 镜像触发钩子)。
+// 2. 告警:连续失败 ≥ N(默认 5)→ 给排程创建者发**一条**通知(agent-notice.ts,与 #462/#464 同一条路,
+//    出现在与目标节点的会话里)。去重落在 scheduled_tasks 两列上(条件写,一次只有一个调用方抢得到):
+//    failure_alert_key = 告警那一刻「最近一次成功」的 run_id —— 之后又成功过,key 就变了,下一段失败重新上膛;
+//    failure_alert_at  = 告警时间 —— 一直没成功过,24 小时后再提醒一次。Hub 重启不会重发。
+// 3. 自动暂停:默认关。COMMHUB_SCHEDULE_FAILURE_AUTO_PAUSE_RUNS=M(≥1)时,连续失败 ≥ M 的 active 排程
+//    被置为 paused(next_run_at 清空,revision+1,与用户手动暂停同形),并通知创建者。
+//
+// 环境变量:COMMHUB_SCHEDULE_FAILURE_NOTICE_RUNS(N,默认 5)、COMMHUB_SCHEDULE_FAILURE_RENOTICE_SEC
+//   (默认 86400)、COMMHUB_SCHEDULE_FAILURE_AUTO_PAUSE_RUNS(M,默认 0 = 关)。
+
+import { db, setScheduledRunTerminalHook } from "./db.js";
+import { sendAgentNotice } from "./agent-notice.js";
+
+export const SCHEDULE_FAILING_NOTICE_KIND = "schedule_failing";
+const ERROR_PREVIEW_CHARS = 300;
+
+function envInt(name: string, fallback: number, min: number): number {
+  const raw = process.env[name];
+  if (raw === undefined || raw.trim() === "") return fallback;
+  const n = Number(raw);
+  return Number.isFinite(n) && n >= min ? Math.floor(n) : fallback;
+}
+
+export function failureNoticeRuns(): number { return envInt("COMMHUB_SCHEDULE_FAILURE_NOTICE_RUNS", 5, 1); }
+export function failureRenoticeMs(): number { return envInt("COMMHUB_SCHEDULE_FAILURE_RENOTICE_SEC", 86_400, 1) * 1000; }
+/** 0 = 不自动暂停(默认)。 */
+export function failureAutoPauseRuns(): number { return envInt("COMMHUB_SCHEDULE_FAILURE_AUTO_PAUSE_RUNS", 0, 0); }
+
+export type FailureStreak = { consecutiveFailures: number; lastSuccessRunId: string | null };
+
+/** 最近一次成功之后的失败 run 数。scheduled_for 每个排程唯一(UNIQUE),按它排序是确定的。 */
+export function failureStreak(scheduleId: string): FailureStreak {
+  const lastSuccess = db.get<{ run_id: string; scheduled_for: string }>(
+    "SELECT run_id, scheduled_for FROM scheduled_task_runs WHERE schedule_id = ?1 AND status = 'replied' ORDER BY scheduled_for DESC LIMIT 1",
+    scheduleId,
+  );
+  const n = db.get<{ n: number }>(
+    "SELECT COUNT(*) AS n FROM scheduled_task_runs WHERE schedule_id = ?1 AND status = 'failed' AND scheduled_for > ?2",
+    scheduleId, lastSuccess?.scheduled_for ?? "",
+  )?.n ?? 0;
+  return { consecutiveFailures: Number(n), lastSuccessRunId: lastSuccess?.run_id ?? null };
+}
+
+type ScheduleForAlert = {
+  schedule_id: string;
+  network_id: string;
+  created_by: string | null;
+  name: string;
+  target_alias: string;
+  status: string;
+};
+
+export type FailureNotice = {
+  reason: "failing" | "paused";
+  scheduleId: string;
+  scheduleName: string;
+  networkId: string;
+  createdBy: string;
+  alias: string;
+  failures: number;
+  errorCode: string | null;
+  errorMessage: string | null;
+};
+
+function latestFailure(scheduleId: string): { error_code: string | null; error_message: string | null } {
+  const row = db.get<{ error_code: string | null; error_message: string | null; result: string | null }>(
+    `SELECT r.error_code, r.error_message, t.result
+       FROM scheduled_task_runs r
+       LEFT JOIN tasks t ON t.task_id = r.task_id AND t.network_id = r.network_id
+      WHERE r.schedule_id = ?1 AND r.status = 'failed'
+      ORDER BY r.scheduled_for DESC LIMIT 1`,
+    scheduleId,
+  );
+  // 任务以 failed 结束时 run.error_message 为空,原因在任务的回复正文里。
+  return { error_code: row?.error_code ?? null, error_message: row?.error_message || row?.result || null };
+}
+
+function truncate(s: string, max: number): string {
+  const flat = s.replace(/\s+/g, " ").trim();
+  return flat.length > max ? `${flat.slice(0, max)}…` : flat;
+}
+
+export function failureNoticeText(n: FailureNotice): { title: string; text: string } {
+  const err = n.errorMessage ? truncate(n.errorMessage, ERROR_PREVIEW_CHARS) : "(没有错误详情)";
+  const code = n.errorCode ? ` [${n.errorCode}]` : "";
+  const head = `定时任务「${n.scheduleName}」(派给节点 ${n.alias})已经连续 ${n.failures} 次执行失败。\n\n最近一次的错误${code}:${err}\n\n`;
+  if (n.reason === "paused") {
+    return {
+      title: "定时任务连续失败,已自动暂停",
+      text: `${head}按 Hub 的设置,它已被自动暂停,不会再派发。处理好原因后,在定时任务里把它恢复为「启用」即可。`,
+    };
+  }
+  return {
+    title: "定时任务连续失败",
+    text: `${head}它仍在按计划运行。如需停止,在定时任务里暂停它(或 PATCH /api/scheduled-tasks/${n.scheduleId},body 带 {"status":"paused","revision":<当前 revision>})。`
+      + `在它重新成功之前,24 小时内不会再提醒。`,
+  };
+}
+
+export function sendFailureNotice(n: FailureNotice): string | null {
+  try {
+    const { title, text } = failureNoticeText(n);
+    return sendAgentNotice({
+      networkId: n.networkId, userId: n.createdBy, fromAlias: n.alias, kind: SCHEDULE_FAILING_NOTICE_KIND, title, text, severity: "error",
+      meta: { schedule_failing: { reason: n.reason, schedule_id: n.scheduleId, consecutive_failures: n.failures, error_code: n.errorCode } },
+      idPrefix: "dm_sched_",
+    });
+  } catch (e: any) {
+    console.error(`[scheduled-tasks] failure notice failed schedule=${n.scheduleId}: ${e?.message || e}`);
+    return null;
+  }
+}
+
+/**
+ * 一个 run 以 failed 结束之后调用(派发时失败 / 任务 failed 都算)。返回发出的通知(测试用),没有则 null。
+ * 告警与暂停都是条件写:并发的两个调用方只有一个写得进去,所以每段失败恰好一条通知。
+ */
+export function evaluateScheduleFailures(scheduleId: string, now = new Date()): FailureNotice | null {
+  const row = db.get<ScheduleForAlert>(
+    "SELECT schedule_id, network_id, created_by, name, target_alias, status FROM scheduled_tasks WHERE schedule_id = ?1",
+    scheduleId,
+  );
+  if (!row) return null;
+  const streak = failureStreak(scheduleId);
+  const noticeAt = failureNoticeRuns();
+  const pauseAt = failureAutoPauseRuns();
+  const wantsPause = pauseAt > 0 && streak.consecutiveFailures >= pauseAt && row.status === "active";
+  if (streak.consecutiveFailures < noticeAt && !wantsPause) return null;
+
+  let paused = false;
+  if (wantsPause) {
+    paused = db.run(
+      "UPDATE scheduled_tasks SET status = 'paused', next_run_at = NULL, revision = revision + 1, updated_at = datetime('now') WHERE schedule_id = ?1 AND status = 'active'",
+      [scheduleId],
+    ).changes === 1;
+  }
+  const key = streak.lastSuccessRunId ?? "";
+  const cutoff = new Date(now.getTime() - failureRenoticeMs()).toISOString();
+  // 去重:同一段失败(key 没变)且上次告警不到 24 小时 → 不写、不发。暂停是另一件事,照发。
+  const claimed = db.run(
+    `UPDATE scheduled_tasks SET failure_alert_at = ?1, failure_alert_key = ?2
+      WHERE schedule_id = ?3
+        AND (failure_alert_at IS NULL OR COALESCE(failure_alert_key, '') <> ?2 OR failure_alert_at <= ?4)`,
+    [now.toISOString(), key, scheduleId, cutoff],
+  ).changes === 1;
+  if (paused && !claimed) {
+    db.run("UPDATE scheduled_tasks SET failure_alert_at = ?1, failure_alert_key = ?2 WHERE schedule_id = ?3", [now.toISOString(), key, scheduleId]);
+  }
+  if (!paused && !claimed) return null;
+  if (!row.created_by) return null;
+  const latest = latestFailure(scheduleId);
+  const notice: FailureNotice = {
+    reason: paused ? "paused" : "failing",
+    scheduleId, scheduleName: row.name, networkId: row.network_id, createdBy: row.created_by, alias: row.target_alias,
+    failures: streak.consecutiveFailures, errorCode: latest.error_code, errorMessage: latest.error_message,
+  };
+  sendFailureNotice(notice);
+  return notice;
+}
+
+/** GET /runs 的附加字段。 */
+export function failureSummary(scheduleId: string): { consecutive_failures: number; failure_alert_threshold: number; last_failure_alert_at: string | null } {
+  const streak = failureStreak(scheduleId);
+  const alertAt = db.get<{ failure_alert_at: string | null }>("SELECT failure_alert_at FROM scheduled_tasks WHERE schedule_id = ?1", scheduleId)?.failure_alert_at ?? null;
+  return { consecutive_failures: streak.consecutiveFailures, failure_alert_threshold: failureNoticeRuns(), last_failure_alert_at: alertAt };
+}
+
+setScheduledRunTerminalHook((scheduleId, status) => {
+  if (status === "failed") evaluateScheduleFailures(scheduleId);
+});