@sleep2agi/commhub-server 0.9.0-preview.100 → 0.9.0-preview.102

package/src/scheduled-failures-http.test.ts

@@ -0,0 +1,233 @@
+// #523 —— 定时任务连续失败:告诉创建者(可选自动暂停)。
+//
+// 钉住:
+//   1. 连续 5 次失败(派出去的任务以 failed 结束)→ 给创建者恰好 1 条通知(别的成员 0 条),第 6–10 次不再发;
+//      通知里有排程名、目标节点、失败次数、最近一次的错误(截断)和怎么暂停。
+//   2. 派发时就失败(目标节点不存在)同样计数。
+//   3. 中间成功一次 → 重新上膛:再连续 5 次失败才发第二条。
+//   4. 一直失败:24 小时内不重发;上次告警早于 24 小时 → 再发一条。
+//   5. 默认不自动暂停;COMMHUB_SCHEDULE_FAILURE_AUTO_PAUSE_RUNS=3 → 第 3 次失败后排程 paused、next_run_at 清空、
+//      revision+1,并通知一次。
+//   6. GET /runs 返回 consecutive_failures / failure_alert_threshold / last_failure_alert_at,
+//      任务以 failed 结束的 run 的 error_message 取自任务的回复正文。
+//
+// 跑法:cd server && COMMHUB_DB=/tmp/x.db bun test src/scheduled-failures-http.test.ts
+//       PG:COMMHUB_TEST_PG_URL=… COMMHUB_PG_EXPERIMENTAL=1(tests/test2123-hub-postgres-ladder 里注册)
+import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, test } from "bun:test";
+import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
+import { addNetworkMember, createNetworkTokenForNode, register } from "./auth.js";
+import { db } from "./db.js";
+import { dispatchScheduledOccurrence, handleScheduledTaskRequest } from "./scheduled-tasks.js";
+import { SCHEDULE_FAILING_NOTICE_KIND, failureStreak } from "./scheduled-failures.js";
+import { registerTools } from "./tools.js";
+
+const activeDbPath = process.env.COMMHUB_DB ?? (process.env.COMMHUB_TEST_PG_URL ? "postgres" : undefined);
+if (!activeDbPath) throw new Error("scheduled-failures requires COMMHUB_DB (or COMMHUB_TEST_PG_URL) before module import");
+
+const stamp = Date.now();
+const ALIAS = `fail-node-${stamp}`;
+const NODE_ID = `n_fail_${stamp}`;
+let NET = "", creatorId = "", otherId = "";
+const ENV_KEYS = ["COMMHUB_SCHEDULE_FAILURE_NOTICE_RUNS", "COMMHUB_SCHEDULE_FAILURE_RENOTICE_SEC", "COMMHUB_SCHEDULE_FAILURE_AUTO_PAUSE_RUNS"];
+const SAFETY_ERROR = "执行出错: upstream model refused the request (safety system). " + "x".repeat(600);
+
+function seedSchedule(name = "前后端分离巡检", targetNodeId = NODE_ID): any {
+  const scheduleId = `sched_${crypto.randomUUID()}`;
+  db.run(
+    `INSERT INTO scheduled_tasks
+       (schedule_id, network_id, created_by, name, target_node_id, target_alias, task_content, priority,
+        schedule_type, schedule_json, timezone, status, next_run_at)
+     VALUES (?1, ?2, ?3, ?4, ?5, ?6, 'scheduled task', 'normal', 'interval',
+             '{"type":"interval","every_seconds":120}', 'Asia/Shanghai', 'active', datetime('now', '+2 minutes'))`,
+    [scheduleId, NET, creatorId, name, targetNodeId, ALIAS],
+  );
+  return reload(scheduleId);
+}
+const reload = (scheduleId: string) => db.get<any>("SELECT * FROM scheduled_tasks WHERE schedule_id = ?1", scheduleId);
+
+let slot = 0;
+const nextSlot = () => new Date(Date.UTC(2026, 9, 4, 0, 0, 0) + (++slot) * 120_000).toISOString();
+const fire = (schedule: any, now = new Date()) => dispatchScheduledOccurrence(schedule, nextSlot(), false, now);
+
+const notices = (userId: string) => db.all<{ title: string; content: string; meta_json: string; from_session: string; severity: string }>(
+  "SELECT title, content, meta_json, from_session, severity FROM user_inbox WHERE user_id = ?1 AND network_id = ?2 AND kind = ?3 ORDER BY created_at, message_id",
+  userId, NET, SCHEDULE_FAILING_NOTICE_KIND,
+);
+const noticesFor = (userId: string, scheduleId: string) => notices(userId).filter((n) => JSON.parse(n.meta_json).schedule_failing.schedule_id === scheduleId);
+
+async function nodeReply(args: Record<string, unknown>): Promise<any> {
+  const mcp = new McpServer({ name: "sched-fail", version: "0" }) as any;
+  const handlers: Record<string, (a: any) => Promise<any>> = {};
+  const original = mcp.tool.bind(mcp);
+  mcp.tool = (name: string, ...rest: any[]) => {
+    const handler = rest.at(-1);
+    if (typeof handler === "function") handlers[name] = handler;
+    return original(name, ...rest);
+  };
+  registerTools(mcp, undefined, NET, creatorId, ALIAS, true, null);
+  const out = await handlers.send_reply(args);
+  return JSON.parse(out.content[0].text);
+}
+
+/** 派一次,节点以 failed(或 replied)回复。 */
+async function runOnce(schedule: any, outcome: "failed" | "replied" = "failed"): Promise<{ runId: string; taskId: string }> {
+  const r = fire(schedule);
+  expect(r.status).toBe("delivered");
+  const out = await nodeReply({ alias: "scheduler", text: outcome === "failed" ? SAFETY_ERROR : "done", in_reply_to: r.taskId, status: outcome, from_session: ALIAS });
+  expect(out.ok).toBe(true);
+  return { runId: r.runId, taskId: r.taskId! };
+}
+
+async function getRuns(scheduleId: string): Promise<any> {
+  const url = new URL(`http://hub.test/api/scheduled-tasks/${scheduleId}/runs`);
+  const res = await handleScheduledTaskRequest({
+    req: new Request(url, { method: "GET" }), url,
+    auth: { userId: creatorId, networkId: NET, username: "creator" },
+    isAdmin: false, isNodeToken: false, scope: { networkId: NET, networkIds: [NET] } as any,
+  });
+  expect(res!.status).toBe(200);
+  return res!.json();
+}
+
+beforeAll(() => {
+  const creator = register(`fail_creator_${stamp}`, "FailCreator123!", undefined, "seed");
+  expect(creator.ok).toBe(true);
+  NET = creator.network_id!; creatorId = creator.user!.user_id;
+  const other = register(`fail_other_${stamp}`, "FailOther123!", undefined, "seed");
+  otherId = other.user!.user_id;
+  expect(addNetworkMember(NET, otherId, "member", creatorId, { agentAccess: "all" }).ok).toBe(true);
+  expect(createNetworkTokenForNode(creatorId, NET, ALIAS).ok).toBe(true);
+  db.run("INSERT INTO nodes (node_id, node_name, alias, runtime, network_id) VALUES (?1, ?2, ?2, 'claude-code', ?3)", [NODE_ID, ALIAS, NET]);
+  db.run("INSERT INTO sessions (resume_id, alias, status, node_id, network_id) VALUES (?1, ?2, 'idle', ?3, ?4)", [`r_fail_${stamp}`, ALIAS, NODE_ID, NET]);
+});
+beforeEach(() => { for (const k of ENV_KEYS) delete process.env[k]; });
+afterEach(() => { for (const k of ENV_KEYS) delete process.env[k]; });
+afterAll(() => { db.run("DELETE FROM user_inbox WHERE network_id = ?1 AND kind = ?2", [NET, SCHEDULE_FAILING_NOTICE_KIND]); });
+
+describe("#523 consecutive-failure alert", () => {
+  test("5 failed runs → exactly one notice to the creator with name/target/count/error/how to pause; runs 6–10 stay quiet", async () => {
+    const s = seedSchedule();
+    for (let i = 0; i < 4; i++) await runOnce(s);
+    expect(noticesFor(creatorId, s.schedule_id).length).toBe(0);
+    await runOnce(s);
+    const mine = noticesFor(creatorId, s.schedule_id);
+    expect(mine.length).toBe(1);
+    expect(noticesFor(otherId, s.schedule_id).length).toBe(0);
+    const n = mine[0];
+    expect(n.from_session).toBe(ALIAS);
+    expect(n.title).toBe("定时任务连续失败");
+    expect(n.severity).toBe("error");
+    for (const frag of ["前后端分离巡检", ALIAS, "连续 5 次", "safety system", "task_failed", "暂停", s.schedule_id]) expect(n.content).toContain(frag);
+    expect(n.content).not.toContain("x".repeat(400)); // 错误被截断
+    expect(JSON.parse(n.meta_json).schedule_failing).toEqual({ reason: "failing", schedule_id: s.schedule_id, consecutive_failures: 5, error_code: "task_failed" });
+    for (let i = 0; i < 5; i++) await runOnce(s);
+    expect(noticesFor(creatorId, s.schedule_id).length).toBe(1);
+    expect(reload(s.schedule_id).status).toBe("active"); // 默认不自动暂停
+  });
+
+  test("dispatch-time failures (target node gone) count too", () => {
+    const s = seedSchedule("孤儿排程", `n_missing_${stamp}`);
+    for (let i = 0; i < 5; i++) expect(fire(s).status).toBe("failed");
+    const mine = noticesFor(creatorId, s.schedule_id);
+    expect(mine.length).toBe(1);
+    expect(mine[0].content).toContain("target_node_not_found");
+    expect(mine[0].content).toContain("The bound node no longer exists");
+  });
+
+  test("a success in between re-arms: the next 5 failures notify again", async () => {
+    const s = seedSchedule();
+    for (let i = 0; i < 5; i++) await runOnce(s);
+    expect(noticesFor(creatorId, s.schedule_id).length).toBe(1);
+    await runOnce(s, "replied");
+    expect(failureStreak(s.schedule_id).consecutiveFailures).toBe(0);
+    for (let i = 0; i < 4; i++) await runOnce(s);
+    expect(noticesFor(creatorId, s.schedule_id).length).toBe(1);
+    await runOnce(s);
+    expect(noticesFor(creatorId, s.schedule_id).length).toBe(2);
+  });
+
+  test("still failing: no repeat within 24h; one more once the last alert is older than 24h", async () => {
+    const s = seedSchedule();
+    for (let i = 0; i < 5; i++) await runOnce(s);
+    expect(noticesFor(creatorId, s.schedule_id).length).toBe(1);
+    db.run("UPDATE scheduled_tasks SET failure_alert_at = ?1 WHERE schedule_id = ?2", [new Date(Date.now() - 23 * 3_600_000).toISOString(), s.schedule_id]);
+    await runOnce(s);
+    expect(noticesFor(creatorId, s.schedule_id).length).toBe(1);
+    db.run("UPDATE scheduled_tasks SET failure_alert_at = ?1 WHERE schedule_id = ?2", [new Date(Date.now() - 25 * 3_600_000).toISOString(), s.schedule_id]);
+    await runOnce(s);
+    expect(noticesFor(creatorId, s.schedule_id).length).toBe(2);
+    await runOnce(s);
+    expect(noticesFor(creatorId, s.schedule_id).length).toBe(2);
+  });
+
+  test("COMMHUB_SCHEDULE_FAILURE_NOTICE_RUNS changes N; no creator → no notice", async () => {
+    process.env.COMMHUB_SCHEDULE_FAILURE_NOTICE_RUNS = "2";
+    const s = seedSchedule();
+    await runOnce(s);
+    expect(noticesFor(creatorId, s.schedule_id).length).toBe(0);
+    await runOnce(s);
+    expect(noticesFor(creatorId, s.schedule_id).length).toBe(1);
+
+    const orphan = seedSchedule();
+    db.run("UPDATE scheduled_tasks SET created_by = NULL WHERE schedule_id = ?1", [orphan.schedule_id]);
+    const row = reload(orphan.schedule_id);
+    for (let i = 0; i < 3; i++) await runOnce(row);
+    expect(noticesFor(creatorId, orphan.schedule_id).length).toBe(0);
+  });
+});
+
+describe("#523 auto-pause (opt-in)", () => {
+  test("off by default: 10 failures leave the schedule active", async () => {
+    const s = seedSchedule();
+    for (let i = 0; i < 10; i++) await runOnce(s);
+    const after = reload(s.schedule_id);
+    expect(after.status).toBe("active");
+    expect(after.next_run_at).not.toBeNull();
+  });
+
+  test("COMMHUB_SCHEDULE_FAILURE_AUTO_PAUSE_RUNS=3 pauses after the 3rd failure and tells the creator once", async () => {
+    process.env.COMMHUB_SCHEDULE_FAILURE_AUTO_PAUSE_RUNS = "3";
+    const s = seedSchedule();
+    await runOnce(s); await runOnce(s);
+    expect(reload(s.schedule_id).status).toBe("active");
+    await runOnce(s);
+    const after = reload(s.schedule_id);
+    expect(after.status).toBe("paused");
+    expect(after.next_run_at).toBeNull();
+    expect(after.revision).toBe(s.revision + 1);
+    const mine = noticesFor(creatorId, s.schedule_id);
+    expect(mine.length).toBe(1);
+    expect(mine[0].title).toBe("定时任务连续失败,已自动暂停");
+    expect(JSON.parse(mine[0].meta_json).schedule_failing.reason).toBe("paused");
+    // 手动 run-now 再失败两次到 N=5:同一段失败,已经通知过 → 不再发
+    await runOnce(after); await runOnce(after);
+    expect(noticesFor(creatorId, s.schedule_id).length).toBe(1);
+  });
+});
+
+describe("#523 GET /runs", () => {
+  test("exposes consecutive_failures and the failed task's reason as error_message", async () => {
+    const s = seedSchedule();
+    await runOnce(s, "replied");
+    const fails = [await runOnce(s), await runOnce(s)];
+    const body = await getRuns(s.schedule_id);
+    expect(body.ok).toBe(true);
+    expect(body.consecutive_failures).toBe(2);
+    expect(body.failure_alert_threshold).toBe(5);
+    expect(body.last_failure_alert_at).toBeNull();
+    expect(body.runs.length).toBe(3);
+    const failed = body.runs.find((r: any) => r.run_id === fails[1].runId);
+    expect(failed.status).toBe("failed");
+    expect(failed.error_code).toBe("task_failed");
+    expect(failed.error_message).toContain("safety system");
+    expect(failed.error_message.length).toBeLessThanOrEqual(500);
+    for (const k of ["scheduled_for", "created_at", "completed_at", "task_id"]) expect(k in failed).toBe(true);
+    const ok = body.runs.find((r: any) => r.status === "replied");
+    expect(ok.error_message).toBeNull();
+
+    for (let i = 0; i < 3; i++) await runOnce(s);
+    const again = await getRuns(s.schedule_id);
+    expect(again.consecutive_failures).toBe(5);
+    expect(typeof again.last_failure_alert_at).toBe("string");
+  });
+});

package/src/scheduled-failures.ts

@@ -0,0 +1,179 @@
+// #523 —— 定时任务连续失败:告诉创建者(可选自动暂停)。
+//
+// 现场:一个每 2 分钟一次的排程,每一次派出去的任务都被节点以 failed 结束(上游模型拒绝),一天 720 次,
+//   排程照样 active,创建者什么都不知道。#464 只管「被上一次卡住」(skipped),不管「每次都失败」。
+//
+// 1. 计数:consecutive_failures = 最近一次成功(run.status='replied')之后,status='failed' 的 run 数。
+//    skipped / cancelled / expired / 还没结束的 run 不算失败,也不打断连续(expired 由 #464 的超时告警负责)。
+//    失败有两种来源,都会走到这里:派发时就失败(节点不在 / 被停用 / 降级 / 创建者无权限,
+//    scheduled-tasks.ts 提交后调用),以及派出去的任务以 failed 结束(db.ts 的 run 镜像触发钩子)。
+// 2. 告警:连续失败 ≥ N(默认 5)→ 给排程创建者发**一条**通知(agent-notice.ts,与 #462/#464 同一条路,
+//    出现在与目标节点的会话里)。去重落在 scheduled_tasks 两列上(条件写,一次只有一个调用方抢得到):
+//    failure_alert_key = 告警那一刻「最近一次成功」的 run_id —— 之后又成功过,key 就变了,下一段失败重新上膛;
+//    failure_alert_at  = 告警时间 —— 一直没成功过,24 小时后再提醒一次。Hub 重启不会重发。
+// 3. 自动暂停:默认关。COMMHUB_SCHEDULE_FAILURE_AUTO_PAUSE_RUNS=M(≥1)时,连续失败 ≥ M 的 active 排程
+//    被置为 paused(next_run_at 清空,revision+1,与用户手动暂停同形),并通知创建者。
+//
+// 环境变量:COMMHUB_SCHEDULE_FAILURE_NOTICE_RUNS(N,默认 5)、COMMHUB_SCHEDULE_FAILURE_RENOTICE_SEC
+//   (默认 86400)、COMMHUB_SCHEDULE_FAILURE_AUTO_PAUSE_RUNS(M,默认 0 = 关)。
+
+import { db, setScheduledRunTerminalHook } from "./db.js";
+import { sendAgentNotice } from "./agent-notice.js";
+
+export const SCHEDULE_FAILING_NOTICE_KIND = "schedule_failing";
+const ERROR_PREVIEW_CHARS = 300;
+
+function envInt(name: string, fallback: number, min: number): number {
+  const raw = process.env[name];
+  if (raw === undefined || raw.trim() === "") return fallback;
+  const n = Number(raw);
+  return Number.isFinite(n) && n >= min ? Math.floor(n) : fallback;
+}
+
+export function failureNoticeRuns(): number { return envInt("COMMHUB_SCHEDULE_FAILURE_NOTICE_RUNS", 5, 1); }
+export function failureRenoticeMs(): number { return envInt("COMMHUB_SCHEDULE_FAILURE_RENOTICE_SEC", 86_400, 1) * 1000; }
+/** 0 = 不自动暂停(默认)。 */
+export function failureAutoPauseRuns(): number { return envInt("COMMHUB_SCHEDULE_FAILURE_AUTO_PAUSE_RUNS", 0, 0); }
+
+export type FailureStreak = { consecutiveFailures: number; lastSuccessRunId: string | null };
+
+/** 最近一次成功之后的失败 run 数。scheduled_for 每个排程唯一(UNIQUE),按它排序是确定的。 */
+export function failureStreak(scheduleId: string): FailureStreak {
+  const lastSuccess = db.get<{ run_id: string; scheduled_for: string }>(
+    "SELECT run_id, scheduled_for FROM scheduled_task_runs WHERE schedule_id = ?1 AND status = 'replied' ORDER BY scheduled_for DESC LIMIT 1",
+    scheduleId,
+  );
+  const n = db.get<{ n: number }>(
+    "SELECT COUNT(*) AS n FROM scheduled_task_runs WHERE schedule_id = ?1 AND status = 'failed' AND scheduled_for > ?2",
+    scheduleId, lastSuccess?.scheduled_for ?? "",
+  )?.n ?? 0;
+  return { consecutiveFailures: Number(n), lastSuccessRunId: lastSuccess?.run_id ?? null };
+}
+
+type ScheduleForAlert = {
+  schedule_id: string;
+  network_id: string;
+  created_by: string | null;
+  name: string;
+  target_alias: string;
+  status: string;
+};
+
+export type FailureNotice = {
+  reason: "failing" | "paused";
+  scheduleId: string;
+  scheduleName: string;
+  networkId: string;
+  createdBy: string;
+  alias: string;
+  failures: number;
+  errorCode: string | null;
+  errorMessage: string | null;
+};
+
+function latestFailure(scheduleId: string): { error_code: string | null; error_message: string | null } {
+  const row = db.get<{ error_code: string | null; error_message: string | null; result: string | null }>(
+    `SELECT r.error_code, r.error_message, t.result
+       FROM scheduled_task_runs r
+       LEFT JOIN tasks t ON t.task_id = r.task_id AND t.network_id = r.network_id
+      WHERE r.schedule_id = ?1 AND r.status = 'failed'
+      ORDER BY r.scheduled_for DESC LIMIT 1`,
+    scheduleId,
+  );
+  // 任务以 failed 结束时 run.error_message 为空,原因在任务的回复正文里。
+  return { error_code: row?.error_code ?? null, error_message: row?.error_message || row?.result || null };
+}
+
+function truncate(s: string, max: number): string {
+  const flat = s.replace(/\s+/g, " ").trim();
+  return flat.length > max ? `${flat.slice(0, max)}…` : flat;
+}
+
+export function failureNoticeText(n: FailureNotice): { title: string; text: string } {
+  const err = n.errorMessage ? truncate(n.errorMessage, ERROR_PREVIEW_CHARS) : "(没有错误详情)";
+  const code = n.errorCode ? ` [${n.errorCode}]` : "";
+  const head = `定时任务「${n.scheduleName}」(派给节点 ${n.alias})已经连续 ${n.failures} 次执行失败。\n\n最近一次的错误${code}:${err}\n\n`;
+  if (n.reason === "paused") {
+    return {
+      title: "定时任务连续失败,已自动暂停",
+      text: `${head}按 Hub 的设置,它已被自动暂停,不会再派发。处理好原因后,在定时任务里把它恢复为「启用」即可。`,
+    };
+  }
+  return {
+    title: "定时任务连续失败",
+    text: `${head}它仍在按计划运行。如需停止,在定时任务里暂停它(或 PATCH /api/scheduled-tasks/${n.scheduleId},body 带 {"status":"paused","revision":<当前 revision>})。`
+      + `在它重新成功之前,24 小时内不会再提醒。`,
+  };
+}
+
+export function sendFailureNotice(n: FailureNotice): string | null {
+  try {
+    const { title, text } = failureNoticeText(n);
+    return sendAgentNotice({
+      networkId: n.networkId, userId: n.createdBy, fromAlias: n.alias, kind: SCHEDULE_FAILING_NOTICE_KIND, title, text, severity: "error",
+      meta: { schedule_failing: { reason: n.reason, schedule_id: n.scheduleId, consecutive_failures: n.failures, error_code: n.errorCode } },
+      idPrefix: "dm_sched_",
+    });
+  } catch (e: any) {
+    console.error(`[scheduled-tasks] failure notice failed schedule=${n.scheduleId}: ${e?.message || e}`);
+    return null;
+  }
+}
+
+/**
+ * 一个 run 以 failed 结束之后调用(派发时失败 / 任务 failed 都算)。返回发出的通知(测试用),没有则 null。
+ * 告警与暂停都是条件写:并发的两个调用方只有一个写得进去,所以每段失败恰好一条通知。
+ */
+export function evaluateScheduleFailures(scheduleId: string, now = new Date()): FailureNotice | null {
+  const row = db.get<ScheduleForAlert>(
+    "SELECT schedule_id, network_id, created_by, name, target_alias, status FROM scheduled_tasks WHERE schedule_id = ?1",
+    scheduleId,
+  );
+  if (!row) return null;
+  const streak = failureStreak(scheduleId);
+  const noticeAt = failureNoticeRuns();
+  const pauseAt = failureAutoPauseRuns();
+  const wantsPause = pauseAt > 0 && streak.consecutiveFailures >= pauseAt && row.status === "active";
+  if (streak.consecutiveFailures < noticeAt && !wantsPause) return null;
+
+  let paused = false;
+  if (wantsPause) {
+    paused = db.run(
+      "UPDATE scheduled_tasks SET status = 'paused', next_run_at = NULL, revision = revision + 1, updated_at = datetime('now') WHERE schedule_id = ?1 AND status = 'active'",
+      [scheduleId],
+    ).changes === 1;
+  }
+  const key = streak.lastSuccessRunId ?? "";
+  const cutoff = new Date(now.getTime() - failureRenoticeMs()).toISOString();
+  // 去重:同一段失败(key 没变)且上次告警不到 24 小时 → 不写、不发。暂停是另一件事,照发。
+  const claimed = db.run(
+    `UPDATE scheduled_tasks SET failure_alert_at = ?1, failure_alert_key = ?2
+      WHERE schedule_id = ?3
+        AND (failure_alert_at IS NULL OR COALESCE(failure_alert_key, '') <> ?2 OR failure_alert_at <= ?4)`,
+    [now.toISOString(), key, scheduleId, cutoff],
+  ).changes === 1;
+  if (paused && !claimed) {
+    db.run("UPDATE scheduled_tasks SET failure_alert_at = ?1, failure_alert_key = ?2 WHERE schedule_id = ?3", [now.toISOString(), key, scheduleId]);
+  }
+  if (!paused && !claimed) return null;
+  if (!row.created_by) return null;
+  const latest = latestFailure(scheduleId);
+  const notice: FailureNotice = {
+    reason: paused ? "paused" : "failing",
+    scheduleId, scheduleName: row.name, networkId: row.network_id, createdBy: row.created_by, alias: row.target_alias,
+    failures: streak.consecutiveFailures, errorCode: latest.error_code, errorMessage: latest.error_message,
+  };
+  sendFailureNotice(notice);
+  return notice;
+}
+
+/** GET /runs 的附加字段。 */
+export function failureSummary(scheduleId: string): { consecutive_failures: number; failure_alert_threshold: number; last_failure_alert_at: string | null } {
+  const streak = failureStreak(scheduleId);
+  const alertAt = db.get<{ failure_alert_at: string | null }>("SELECT failure_alert_at FROM scheduled_tasks WHERE schedule_id = ?1", scheduleId)?.failure_alert_at ?? null;
+  return { consecutive_failures: streak.consecutiveFailures, failure_alert_threshold: failureNoticeRuns(), last_failure_alert_at: alertAt };
+}
+
+setScheduledRunTerminalHook((scheduleId, status) => {
+  if (status === "failed") evaluateScheduleFailures(scheduleId);
+});

package/src/scheduled-tasks.ts

@@ -7,6 +7,7 @@ import { hasSubscribers, pushEvent, pushNetworkObserverEvent } from "./push.js";
 import { SCHEDULED_TASK_STORAGE_SELECT } from "./rest-projections.js";
 import { parseDbTimestampMs } from "./db-timestamp.js";
 import { blockerTimedOut, expireStuckBlocker, sendStuckNotice, skipsBlockedBy, stuckNoticeSkips, stuckTimeoutMs, type StuckNotice } from "./scheduled-stuck.js";
+import { evaluateScheduleFailures, failureSummary } from "./scheduled-failures.js";
 
 export type ScheduleSpec =
   | { type: "once"; run_at: string }
@@ -379,6 +380,8 @@ export function dispatchScheduledOccurrence(row: ScheduledRow, scheduledFor: str
   try {
     for (const t of timedOut) logTaskEvent(t.taskId, t.fromStatus, "expired", "hub-scheduler", `stuck timeout; schedule=${row.schedule_id} run=${runId}`);
     for (const n of notices) sendStuckNotice(n);
+    // #523 — a run that failed at dispatch counts toward the consecutive-failure alert (task failures arrive via db.ts).
+    if (finalStatus === "failed") evaluateScheduleFailures(row.schedule_id, advanceAfter);
     if (createdTaskId) logTaskEvent(createdTaskId, null, "delivered", "hub-scheduler", `schedule=${row.schedule_id} run=${runId}`);
     if (event) {
       const pending = db.get<{ cnt: number }>("SELECT COUNT(*) AS cnt FROM inbox WHERE session_name = ?1 AND network_id = ?2 AND acked = 0", event.alias, event.networkId);
@@ -548,10 +551,16 @@ export async function handleScheduledTaskRequest(ctx: ScheduledRequestContext):
   if (sub === "runs" && req.method === "GET") {
     const limit = Math.max(1, Math.min(200, Number(url.searchParams.get("limit")) || 50));
     const runs = db.all(
-      "SELECT run_id, schedule_id, scheduled_for, task_id, status, error_code, error_message, blocked_by_task_id, blocked_by_state, created_at, completed_at FROM scheduled_task_runs WHERE schedule_id = ?1 AND network_id = ?2 ORDER BY created_at DESC LIMIT ?3",
+      // #523 — a task that ended `failed` leaves run.error_message empty; its reason is the task's reply text.
+      `SELECT r.run_id, r.schedule_id, r.scheduled_for, r.task_id, r.status, r.error_code,
+              CASE WHEN r.error_message IS NULL AND r.status = 'failed' THEN SUBSTR(t.result, 1, 500) ELSE r.error_message END AS error_message,
+              r.blocked_by_task_id, r.blocked_by_state, r.created_at, r.completed_at
+         FROM scheduled_task_runs r
+         LEFT JOIN tasks t ON t.task_id = r.task_id AND t.network_id = r.network_id
+        WHERE r.schedule_id = ?1 AND r.network_id = ?2 ORDER BY r.created_at DESC, r.scheduled_for DESC LIMIT ?3`,
       row.schedule_id, row.network_id, limit,
     );
-    return Response.json({ ok: true, runs });
+    return Response.json({ ok: true, runs, ...failureSummary(row.schedule_id) });
   }
   if (!sub && req.method === "GET") return Response.json({ ok: true, schedule: decodeRow(row) });
   if (!writeAllowed(ctx, row.network_id)) return jsonError("permission_denied", 403);

package/src/server.ts

@@ -83,6 +83,7 @@ import { SIDE_THREAD_FEATURE_FLAG, SideThreadCoordinator, SideThreadPortRegistry
 import { handleSideThreadHttpRequest } from "./side-thread-http.js";
 import { createProductionSideThreadTransport } from "./side-thread-production.js";
 import { parseHubTimestamp } from "./hub-timestamp";
+import { logTaskRejection } from "./task-rejection-log.js";
 
 const PORT = resolvePort(process.env.PORT);
 const HOST = process.env.HOST || "127.0.0.1";
@@ -3411,15 +3412,18 @@ return Bun.serve({
 
     // ── REST: send task ──
     if (url.pathname === "/api/task" && req.method === "POST") {
+      // #552 — every non-2xx below goes through reject() so it leaves one log line
+      // (status, error, user, requested alias/network/client_request_id; never token/text).
       let raw: unknown;
+      const reject = (res: Response) => logTaskRejection(res, restAuth?.username, raw);
       try {
         raw = await req.json();
       } catch {
-        return withCors(req, Response.json({ error: "invalid JSON" }, { status: 400 }));
+        return reject(withCors(req, Response.json({ error: "invalid JSON" }, { status: 400 })));
       }
       const parsed = TaskSchema.safeParse(raw);
       if (!parsed.success) {
-        return withCors(req, Response.json({ error: "invalid input", details: parsed.error.format() }, { status: 400 }));
+        return reject(withCors(req, Response.json({ error: "invalid input", details: parsed.error.format() }, { status: 400 })));
       }
       const body = parsed.data;
       let taskNetId: string | null = null;
@@ -3427,7 +3431,7 @@ return Bun.serve({
         taskNetId = restAuth.networkId;
       } else if (body.network_id) {
         if (restAuth && !isAdmin && !getUserNetworkRole(restAuth.userId, body.network_id)) {
-          return withCors(req, Response.json({ ok: false, error: "access denied to requested network" }, { status: 403 }));
+          return reject(withCors(req, Response.json({ ok: false, error: "access denied to requested network" }, { status: 403 })));
         }
         taskNetId = body.network_id;
       } else {
@@ -3436,34 +3440,34 @@ return Bun.serve({
           : null;
       }
       if (restAuth && !taskNetId) {
-        return withCors(req, Response.json({
+        return reject(withCors(req, Response.json({
           ok: false,
           error: "network_id_required",
           message: "network_id is required when the user token has zero or multiple network memberships",
-        }, { status: 400 }));
+        }, { status: 400 })));
       }
       // 多用户 Agent 权限:受限成员只能给授权且 can_message 的 Agent 发;
       // 目标不存在 / 没授权 / 只读授权 → 同一个 403,不留 alias 探测差异。发件人固定为自己的用户名。
       const restrictedSender = !!restAuth && !isAdmin && !restAuth.networkId && isAgentRestricted(restAuth.userId, taskNetId!);
       if (restrictedSender) {
         if (!canRestWriteNetworkAsHuman(restAuth, taskNetId, isAdmin)) {
-          return withCors(req, Response.json({ ok: false, error: "permission_denied" }, { status: 403 }));
+          return reject(withCors(req, Response.json({ ok: false, error: "permission_denied" }, { status: 403 })));
         }
         const restrictedTarget = resolveCanonicalAlias(taskNetId, body.alias).alias;
         const restrictedSession = db.get<{ node_id: string | null }>("SELECT node_id FROM sessions WHERE alias = ?1 AND network_id = ?2", restrictedTarget, taskNetId);
         if (!restrictedSession || !canMessageAgent(restAuth!.userId, taskNetId, { alias: restrictedTarget, nodeId: restrictedSession.node_id })) {
-          return withCors(req, Response.json({ ok: false, error: "agent_not_granted", message: "this agent has not been granted to you; ask a network admin" }, { status: 403 }));
+          return reject(withCors(req, Response.json({ ok: false, error: "agent_not_granted", message: "this agent has not been granted to you; ask a network admin" }, { status: 403 })));
         }
         if (typeof body.from === "string" && body.from.trim() && body.from.trim() !== restAuth!.username) {
-          return withCors(req, Response.json({ ok: false, error: "from_session_identity_mismatch", message: "restricted members always send as their own username" }, { status: 403 }));
+          return reject(withCors(req, Response.json({ ok: false, error: "from_session_identity_mismatch", message: "restricted members always send as their own username" }, { status: 403 })));
         }
         for (const raw of [(body as any).attachments, (body as any).meta?.attachments]) {
           if (restrictedMemberAttachmentsDenied(restAuth!.userId, restAuth!.username, taskNetId!, raw)) {
-            return withCors(req, Response.json({ ok: false, error: "attachment_not_accessible" }, { status: 403 }));
+            return reject(withCors(req, Response.json({ ok: false, error: "attachment_not_accessible" }, { status: 403 })));
           }
         }
       } else if (!canRestWriteNetwork(restAuth, taskNetId, isAdmin)) {
-        return withCors(req, Response.json({ ok: false, error: "permission_denied" }, { status: 403 }));
+        return reject(withCors(req, Response.json({ ok: false, error: "permission_denied" }, { status: 403 })));
       }
       const canonical = resolveCanonicalAlias(taskNetId, body.alias);
       const targetAlias = canonical.alias;
@@ -3471,17 +3475,17 @@ return Bun.serve({
         alias: targetAlias,
         nodeId: db.get<{ node_id: string | null }>("SELECT node_id FROM sessions WHERE alias = ?1 AND network_id = ?2", targetAlias, taskNetId)?.node_id ?? null,
       }));
-      if (nodeDenied) return nodeDenied;
+      if (nodeDenied) return reject(nodeDenied);
       const target = resolveRestDeliveryTarget(targetAlias, taskNetId);
       if (target.state === "not_found") {
-        return withCors(req, Response.json({
+        return reject(withCors(req, Response.json({
           ok: false,
           error: "alias_not_found",
           message: target.message,
           alias: targetAlias,
           queued: false,
           ...(canonical.renamed ? { renamed_from: body.alias, renamed_to: targetAlias } : {}),
-        }, { status: 404 }));
+        }, { status: 404 })));
       }
       // Identity binding, mirroring the MCP transport (tools.ts defaultFrom /
       // fromIdentityMismatchReply). Both transports resolve the same
@@ -3507,13 +3511,13 @@ return Bun.serve({
         requestedFrom: body.from,
       });
       if (!identity.ok) {
-        return withCors(req, Response.json({
+        return reject(withCors(req, Response.json({
           ok: false,
           error: identity.error,
           message: identity.message,
           token_alias: identity.tokenAlias,
           requested_from_session: identity.requestedFromSession,
-        }, { status: 403 }));
+        }, { status: 403 })));
       }
       const fromSession = identity.fromSession;
       const ttlSeconds = (body as any).ttl_seconds || 3600;
@@ -3528,7 +3532,7 @@ return Bun.serve({
       // size cap, etc.).
       const attachmentsResult = validateAttachments((body as any).attachments ?? (body as any).meta?.attachments);
       if (!attachmentsResult.ok) {
-        return withCors(req, Response.json({ ok: false, error: "bad_attachments", message: attachmentsResult.error }, { status: 400 }));
+        return reject(withCors(req, Response.json({ ok: false, error: "bad_attachments", message: attachmentsResult.error }, { status: 400 })));
       }
       const mergedMeta = attachmentsResult.attachments.length
         ? { ...((body as any).meta && typeof (body as any).meta === "object" ? (body as any).meta : {}), attachments: attachmentsResult.attachments }
@@ -3616,14 +3620,14 @@ return Bun.serve({
       {
         const lc = assertNodeActive(targetAlias, taskNetId ?? null);
         if (!lc.ok) {
-          return withCors(req, Response.json(lc, { status: 409 }));
+          return reject(withCors(req, Response.json(lc, { status: 409 })));
         }
       }
       // #460 — refuse dispatch to a node whose fresh health says a layer is down (409, same as the
       // lifecycle guard above). Unknown / stale health = allowed: old nodes keep working.
       {
         const hg = assertNodeHealthy(targetAlias, taskNetId ?? null, { force: body.force === true, forceAllowed: !restAuth?.networkId });
-        if (!hg.ok) return withCors(req, Response.json(hg, { status: 409 }));
+        if (!hg.ok) return reject(withCors(req, Response.json(hg, { status: 409 })));
       }
       // #500 step 2 — what is already queued on the target, read BEFORE this task is written (advice only, never blocks).
       const queueInfo = dispatchQueueInfo(targetAlias, taskNetId, target.session?.status, ttlSeconds);