@sleep2agi/commhub-server 0.9.0-preview.1 → 0.9.0-preview.101

package/src/scheduled-failures.ts

@@ -0,0 +1,179 @@
+// #523 —— 定时任务连续失败:告诉创建者(可选自动暂停)。
+//
+// 现场:一个每 2 分钟一次的排程,每一次派出去的任务都被节点以 failed 结束(上游模型拒绝),一天 720 次,
+//   排程照样 active,创建者什么都不知道。#464 只管「被上一次卡住」(skipped),不管「每次都失败」。
+//
+// 1. 计数:consecutive_failures = 最近一次成功(run.status='replied')之后,status='failed' 的 run 数。
+//    skipped / cancelled / expired / 还没结束的 run 不算失败,也不打断连续(expired 由 #464 的超时告警负责)。
+//    失败有两种来源,都会走到这里:派发时就失败(节点不在 / 被停用 / 降级 / 创建者无权限,
+//    scheduled-tasks.ts 提交后调用),以及派出去的任务以 failed 结束(db.ts 的 run 镜像触发钩子)。
+// 2. 告警:连续失败 ≥ N(默认 5)→ 给排程创建者发**一条**通知(agent-notice.ts,与 #462/#464 同一条路,
+//    出现在与目标节点的会话里)。去重落在 scheduled_tasks 两列上(条件写,一次只有一个调用方抢得到):
+//    failure_alert_key = 告警那一刻「最近一次成功」的 run_id —— 之后又成功过,key 就变了,下一段失败重新上膛;
+//    failure_alert_at  = 告警时间 —— 一直没成功过,24 小时后再提醒一次。Hub 重启不会重发。
+// 3. 自动暂停:默认关。COMMHUB_SCHEDULE_FAILURE_AUTO_PAUSE_RUNS=M(≥1)时,连续失败 ≥ M 的 active 排程
+//    被置为 paused(next_run_at 清空,revision+1,与用户手动暂停同形),并通知创建者。
+//
+// 环境变量:COMMHUB_SCHEDULE_FAILURE_NOTICE_RUNS(N,默认 5)、COMMHUB_SCHEDULE_FAILURE_RENOTICE_SEC
+//   (默认 86400)、COMMHUB_SCHEDULE_FAILURE_AUTO_PAUSE_RUNS(M,默认 0 = 关)。
+
+import { db, setScheduledRunTerminalHook } from "./db.js";
+import { sendAgentNotice } from "./agent-notice.js";
+
+export const SCHEDULE_FAILING_NOTICE_KIND = "schedule_failing";
+const ERROR_PREVIEW_CHARS = 300;
+
+function envInt(name: string, fallback: number, min: number): number {
+  const raw = process.env[name];
+  if (raw === undefined || raw.trim() === "") return fallback;
+  const n = Number(raw);
+  return Number.isFinite(n) && n >= min ? Math.floor(n) : fallback;
+}
+
+export function failureNoticeRuns(): number { return envInt("COMMHUB_SCHEDULE_FAILURE_NOTICE_RUNS", 5, 1); }
+export function failureRenoticeMs(): number { return envInt("COMMHUB_SCHEDULE_FAILURE_RENOTICE_SEC", 86_400, 1) * 1000; }
+/** 0 = 不自动暂停(默认)。 */
+export function failureAutoPauseRuns(): number { return envInt("COMMHUB_SCHEDULE_FAILURE_AUTO_PAUSE_RUNS", 0, 0); }
+
+export type FailureStreak = { consecutiveFailures: number; lastSuccessRunId: string | null };
+
+/** 最近一次成功之后的失败 run 数。scheduled_for 每个排程唯一(UNIQUE),按它排序是确定的。 */
+export function failureStreak(scheduleId: string): FailureStreak {
+  const lastSuccess = db.get<{ run_id: string; scheduled_for: string }>(
+    "SELECT run_id, scheduled_for FROM scheduled_task_runs WHERE schedule_id = ?1 AND status = 'replied' ORDER BY scheduled_for DESC LIMIT 1",
+    scheduleId,
+  );
+  const n = db.get<{ n: number }>(
+    "SELECT COUNT(*) AS n FROM scheduled_task_runs WHERE schedule_id = ?1 AND status = 'failed' AND scheduled_for > ?2",
+    scheduleId, lastSuccess?.scheduled_for ?? "",
+  )?.n ?? 0;
+  return { consecutiveFailures: Number(n), lastSuccessRunId: lastSuccess?.run_id ?? null };
+}
+
+type ScheduleForAlert = {
+  schedule_id: string;
+  network_id: string;
+  created_by: string | null;
+  name: string;
+  target_alias: string;
+  status: string;
+};
+
+export type FailureNotice = {
+  reason: "failing" | "paused";
+  scheduleId: string;
+  scheduleName: string;
+  networkId: string;
+  createdBy: string;
+  alias: string;
+  failures: number;
+  errorCode: string | null;
+  errorMessage: string | null;
+};
+
+function latestFailure(scheduleId: string): { error_code: string | null; error_message: string | null } {
+  const row = db.get<{ error_code: string | null; error_message: string | null; result: string | null }>(
+    `SELECT r.error_code, r.error_message, t.result
+       FROM scheduled_task_runs r
+       LEFT JOIN tasks t ON t.task_id = r.task_id AND t.network_id = r.network_id
+      WHERE r.schedule_id = ?1 AND r.status = 'failed'
+      ORDER BY r.scheduled_for DESC LIMIT 1`,
+    scheduleId,
+  );
+  // 任务以 failed 结束时 run.error_message 为空,原因在任务的回复正文里。
+  return { error_code: row?.error_code ?? null, error_message: row?.error_message || row?.result || null };
+}
+
+function truncate(s: string, max: number): string {
+  const flat = s.replace(/\s+/g, " ").trim();
+  return flat.length > max ? `${flat.slice(0, max)}…` : flat;
+}
+
+export function failureNoticeText(n: FailureNotice): { title: string; text: string } {
+  const err = n.errorMessage ? truncate(n.errorMessage, ERROR_PREVIEW_CHARS) : "(没有错误详情)";
+  const code = n.errorCode ? ` [${n.errorCode}]` : "";
+  const head = `定时任务「${n.scheduleName}」(派给节点 ${n.alias})已经连续 ${n.failures} 次执行失败。\n\n最近一次的错误${code}:${err}\n\n`;
+  if (n.reason === "paused") {
+    return {
+      title: "定时任务连续失败,已自动暂停",
+      text: `${head}按 Hub 的设置,它已被自动暂停,不会再派发。处理好原因后,在定时任务里把它恢复为「启用」即可。`,
+    };
+  }
+  return {
+    title: "定时任务连续失败",
+    text: `${head}它仍在按计划运行。如需停止,在定时任务里暂停它(或 PATCH /api/scheduled-tasks/${n.scheduleId},body 带 {"status":"paused","revision":<当前 revision>})。`
+      + `在它重新成功之前,24 小时内不会再提醒。`,
+  };
+}
+
+export function sendFailureNotice(n: FailureNotice): string | null {
+  try {
+    const { title, text } = failureNoticeText(n);
+    return sendAgentNotice({
+      networkId: n.networkId, userId: n.createdBy, fromAlias: n.alias, kind: SCHEDULE_FAILING_NOTICE_KIND, title, text, severity: "error",
+      meta: { schedule_failing: { reason: n.reason, schedule_id: n.scheduleId, consecutive_failures: n.failures, error_code: n.errorCode } },
+      idPrefix: "dm_sched_",
+    });
+  } catch (e: any) {
+    console.error(`[scheduled-tasks] failure notice failed schedule=${n.scheduleId}: ${e?.message || e}`);
+    return null;
+  }
+}
+
+/**
+ * 一个 run 以 failed 结束之后调用(派发时失败 / 任务 failed 都算)。返回发出的通知(测试用),没有则 null。
+ * 告警与暂停都是条件写:并发的两个调用方只有一个写得进去,所以每段失败恰好一条通知。
+ */
+export function evaluateScheduleFailures(scheduleId: string, now = new Date()): FailureNotice | null {
+  const row = db.get<ScheduleForAlert>(
+    "SELECT schedule_id, network_id, created_by, name, target_alias, status FROM scheduled_tasks WHERE schedule_id = ?1",
+    scheduleId,
+  );
+  if (!row) return null;
+  const streak = failureStreak(scheduleId);
+  const noticeAt = failureNoticeRuns();
+  const pauseAt = failureAutoPauseRuns();
+  const wantsPause = pauseAt > 0 && streak.consecutiveFailures >= pauseAt && row.status === "active";
+  if (streak.consecutiveFailures < noticeAt && !wantsPause) return null;
+
+  let paused = false;
+  if (wantsPause) {
+    paused = db.run(
+      "UPDATE scheduled_tasks SET status = 'paused', next_run_at = NULL, revision = revision + 1, updated_at = datetime('now') WHERE schedule_id = ?1 AND status = 'active'",
+      [scheduleId],
+    ).changes === 1;
+  }
+  const key = streak.lastSuccessRunId ?? "";
+  const cutoff = new Date(now.getTime() - failureRenoticeMs()).toISOString();
+  // 去重:同一段失败(key 没变)且上次告警不到 24 小时 → 不写、不发。暂停是另一件事,照发。
+  const claimed = db.run(
+    `UPDATE scheduled_tasks SET failure_alert_at = ?1, failure_alert_key = ?2
+      WHERE schedule_id = ?3
+        AND (failure_alert_at IS NULL OR COALESCE(failure_alert_key, '') <> ?2 OR failure_alert_at <= ?4)`,
+    [now.toISOString(), key, scheduleId, cutoff],
+  ).changes === 1;
+  if (paused && !claimed) {
+    db.run("UPDATE scheduled_tasks SET failure_alert_at = ?1, failure_alert_key = ?2 WHERE schedule_id = ?3", [now.toISOString(), key, scheduleId]);
+  }
+  if (!paused && !claimed) return null;
+  if (!row.created_by) return null;
+  const latest = latestFailure(scheduleId);
+  const notice: FailureNotice = {
+    reason: paused ? "paused" : "failing",
+    scheduleId, scheduleName: row.name, networkId: row.network_id, createdBy: row.created_by, alias: row.target_alias,
+    failures: streak.consecutiveFailures, errorCode: latest.error_code, errorMessage: latest.error_message,
+  };
+  sendFailureNotice(notice);
+  return notice;
+}
+
+/** GET /runs 的附加字段。 */
+export function failureSummary(scheduleId: string): { consecutive_failures: number; failure_alert_threshold: number; last_failure_alert_at: string | null } {
+  const streak = failureStreak(scheduleId);
+  const alertAt = db.get<{ failure_alert_at: string | null }>("SELECT failure_alert_at FROM scheduled_tasks WHERE schedule_id = ?1", scheduleId)?.failure_alert_at ?? null;
+  return { consecutive_failures: streak.consecutiveFailures, failure_alert_threshold: failureNoticeRuns(), last_failure_alert_at: alertAt };
+}
+
+setScheduledRunTerminalHook((scheduleId, status) => {
+  if (status === "failed") evaluateScheduleFailures(scheduleId);
+});

package/src/scheduled-reply-unread-http.test.ts

@@ -0,0 +1,212 @@
+// 定时任务的回复要进建排程那个人的未读(Vincent 2026-09-30「定时任务触发的…那个红点它是不显示的」)。
+//
+// scheduler 派的任务 from_name='scheduler',回复以前按 from_name 落进 inbox(session_name='scheduler'):
+// app 会话里看得到(会话取自 tasks 表),但 unread_by_agent 只数 session_name=用户名 的行 ⇒ 永远没红点。
+// 这里走真 Hub(bootServer)+ 真节点令牌回复路径(send_reply,callerTokenIsNetwork=true)。
+import { afterAll, beforeAll, describe, expect, test } from "bun:test";
+import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
+import { addNetworkMember, createNetworkTokenForNode, register } from "./auth.js";
+import { db } from "./db.js";
+import { runDueScheduledTasks } from "./scheduled-tasks.js";
+import { registerTools } from "./tools.js";
+
+const activeDbPath = process.env.COMMHUB_DB ?? (process.env.COMMHUB_TEST_PG_URL ? "postgres" : undefined);
+if (!activeDbPath) throw new Error("scheduled-reply-unread requires COMMHUB_DB (or COMMHUB_TEST_PG_URL) before module import");
+
+let server: any;
+let base = "";
+let ownerToken = "";
+let ownerName = "";
+let memberToken = "";
+let memberName = "";
+let networkId = "";
+let ownerId = "";
+const stamp = Date.now();
+const ALIAS = `sched-reply-node-${stamp}`;
+const nodeId = `n_sched_reply_${stamp}`;
+
+async function api(token: string, path: string, init?: RequestInit) {
+  const res = await fetch(`${base}${path}`, {
+    ...init,
+    headers: { Authorization: `Bearer ${token}`, "Content-Type": "application/json", ...(init?.headers || {}) },
+  });
+  return { status: res.status, body: await res.json() as any };
+}
+
+/** 节点令牌身份调 send_reply(与真节点的 commhub_reply 同一个 handler、同一组归属校验)。 */
+async function nodeReply(args: Record<string, unknown>): Promise<any> {
+  const mcp = new McpServer({ name: "sched-reply", version: "0" }) as any;
+  const handlers: Record<string, (a: any) => Promise<any>> = {};
+  const original = mcp.tool.bind(mcp);
+  mcp.tool = (name: string, ...rest: any[]) => {
+    const handler = rest.at(-1);
+    if (typeof handler === "function") handlers[name] = handler;
+    return original(name, ...rest);
+  };
+  registerTools(mcp, undefined, networkId, ownerId, ALIAS, true, null);
+  const out = await handlers.send_reply(args);
+  return JSON.parse(out.content[0].text);
+}
+
+async function unreadFor(token: string): Promise<Record<string, number>> {
+  const res = await api(token, "/api/messages?scope=user&limit=50");
+  expect(res.status).toBe(200);
+  return res.body.unread_by_agent ?? {};
+}
+
+async function ackAgent(token: string) {
+  const res = await api(token, "/api/messages/ack", { method: "POST", body: JSON.stringify({ agent: ALIAS }) });
+  expect(res.status).toBe(200);
+  return res.body;
+}
+
+/** 建一个排程、让它立刻到期、跑一次,返回它派出的 task_id。 */
+async function fireSchedule(token: string, name: string): Promise<{ scheduleId: string; taskId: string }> {
+  const created = await api(token, "/api/scheduled-tasks", {
+    method: "POST",
+    body: JSON.stringify({
+      network_id: networkId, name, target_node_id: nodeId, task: `${name} tick`,
+      timezone: "UTC", schedule: { type: "interval", every_seconds: 120 },
+    }),
+  });
+  expect(created.status).toBe(201);
+  const scheduleId = created.body.schedule.schedule_id as string;
+  // 别的排程不许在这次 sweep 里一起跑
+  db.run("UPDATE scheduled_tasks SET next_run_at = ?1 WHERE schedule_id != ?2", [new Date(Date.now() + 24 * 3600_000).toISOString(), scheduleId]);
+  db.run("UPDATE scheduled_tasks SET next_run_at = ?1 WHERE schedule_id = ?2", [new Date(Date.now() - 60_000).toISOString(), scheduleId]);
+  await runDueScheduledTasks();
+  const run = db.get<{ task_id: string | null }>(
+    "SELECT task_id FROM scheduled_task_runs WHERE schedule_id = ?1 ORDER BY scheduled_for DESC LIMIT 1", scheduleId,
+  );
+  expect(run?.task_id).toBeTruthy();
+  const task = db.get<{ from_name: string }>("SELECT from_name FROM tasks WHERE task_id = ?1", run!.task_id!);
+  expect(task?.from_name).toBe("scheduler");
+  return { scheduleId, taskId: run!.task_id! };
+}
+
+function replyRow(taskId: string) {
+  return db.get<{ session_name: string; from_session: string; acked: number }>(
+    "SELECT session_name, from_session, acked FROM inbox WHERE type = 'reply' AND in_reply_to = ?1", taskId,
+  );
+}
+
+beforeAll(async () => {
+  ownerName = `sched_reply_owner_${stamp}`;
+  const owner = register(ownerName, "SchedReplyOwner123!", undefined, "seed");
+  expect(owner.ok).toBe(true);
+  ownerToken = owner.token!;
+  networkId = owner.network_id!;
+  ownerId = db.get<{ owner_id: string }>("SELECT owner_id FROM networks WHERE network_id = ?1", networkId)!.owner_id;
+
+  memberName = `sched_reply_member_${stamp}`;
+  const member = register(memberName, "SchedReplyMember123!", undefined, "seed");
+  expect(member.ok).toBe(true);
+  memberToken = member.token!;
+  const memberId = db.get<{ user_id: string }>("SELECT user_id FROM users WHERE username = ?1", memberName)!.user_id;
+  expect(addNetworkMember(networkId, memberId, "member", ownerId, { agentAccess: "all" }).ok).toBe(true);
+
+  expect(createNetworkTokenForNode(ownerId, networkId, ALIAS).ok).toBe(true);
+  db.run(
+    "INSERT INTO nodes (node_id, node_name, alias, runtime, network_id) VALUES (?1, ?2, ?2, 'claude-code', ?3)",
+    [nodeId, ALIAS, networkId],
+  );
+  db.run(
+    "INSERT INTO sessions (resume_id, alias, status, node_id, network_id) VALUES (?1, ?2, 'idle', ?3, ?4)",
+    [`r_sched_reply_${stamp}`, ALIAS, nodeId, networkId],
+  );
+  const mod: any = await import("./server.js");
+  server = mod.bootServer({ port: 0, hostname: "127.0.0.1" });
+  base = `http://127.0.0.1:${server.port}`;
+}, 30_000);
+
+afterAll(() => {
+  try { server?.stop?.(true); } catch {}
+});
+
+describe("scheduled-task replies reach the schedule creator's unread", () => {
+  test("a direct reply still lands on the sender and counts once", async () => {
+    await ackAgent(ownerToken);
+    const sent = await api(ownerToken, "/api/task", { method: "POST", body: JSON.stringify({ network_id: networkId, alias: ALIAS, task: "direct hello" }) });
+    expect(sent.status).toBe(200);
+    const taskId = sent.body.task_id as string;
+    const reply = await nodeReply({ in_reply_to: taskId, text: "direct answer", status: "replied" });
+    expect(reply.ok).toBe(true);
+    expect(replyRow(taskId)?.session_name).toBe(ownerName);
+    expect((await unreadFor(ownerToken))[ALIAS]).toBe(1);
+    await ackAgent(ownerToken);
+    expect((await unreadFor(ownerToken))[ALIAS]).toBeUndefined();
+  });
+
+  test("a scheduled-task reply counts for the creator, and ack-by-agent clears it", async () => {
+    await ackAgent(ownerToken);
+    const { taskId } = await fireSchedule(ownerToken, "owner-every-2m");
+    // 节点不带 alias(SDK 运行时)——收件人由 in_reply_to 推出
+    const reply = await nodeReply({ in_reply_to: taskId, text: "tick answer", status: "replied" });
+    expect(reply.ok).toBe(true);
+    expect(replyRow(taskId)).toEqual({ session_name: ownerName, from_session: ALIAS, acked: 0 });
+    expect(db.get<{ status: string; result: string }>("SELECT status, result FROM tasks WHERE task_id = ?1", taskId))
+      .toEqual({ status: "replied", result: "tick answer" });
+    expect((await unreadFor(ownerToken))[ALIAS]).toBe(1);
+
+    const acked = await ackAgent(ownerToken);
+    expect(acked.acked_inbox).toBe(1);
+    expect((await unreadFor(ownerToken))[ALIAS]).toBeUndefined();
+    expect(replyRow(taskId)?.acked).toBe(1);
+  });
+
+  test("a node that names alias='scheduler' explicitly is routed the same way", async () => {
+    await ackAgent(ownerToken);
+    const { taskId } = await fireSchedule(ownerToken, "owner-explicit-alias");
+    const reply = await nodeReply({ alias: "scheduler", in_reply_to: taskId, text: "explicit answer", status: "replied" });
+    expect(reply.ok).toBe(true);
+    expect(replyRow(taskId)?.session_name).toBe(ownerName);
+    expect((await unreadFor(ownerToken))[ALIAS]).toBe(1);
+    await ackAgent(ownerToken);
+  });
+
+  test("another member of the network does not get the owner's scheduled reply, and gets their own", async () => {
+    await ackAgent(ownerToken);
+    await ackAgent(memberToken);
+    const ownerRun = await fireSchedule(ownerToken, "owner-only");
+    expect((await nodeReply({ in_reply_to: ownerRun.taskId, text: "for owner", status: "replied" })).ok).toBe(true);
+    expect((await unreadFor(memberToken))[ALIAS]).toBeUndefined();
+    expect((await unreadFor(ownerToken))[ALIAS]).toBe(1);
+
+    const memberRun = await fireSchedule(memberToken, "member-only");
+    expect((await nodeReply({ in_reply_to: memberRun.taskId, text: "for member", status: "replied" })).ok).toBe(true);
+    expect(replyRow(memberRun.taskId)?.session_name).toBe(memberName);
+    expect((await unreadFor(memberToken))[ALIAS]).toBe(1);
+    expect((await unreadFor(ownerToken))[ALIAS]).toBe(1);
+    await ackAgent(ownerToken);
+    await ackAgent(memberToken);
+  });
+
+  test("a schedule with no creator falls back to 'scheduler'", async () => {
+    await ackAgent(ownerToken);
+    const { scheduleId, taskId } = await fireSchedule(ownerToken, "creatorless");
+    db.run("UPDATE scheduled_tasks SET created_by = NULL WHERE schedule_id = ?1", [scheduleId]);
+    expect((await nodeReply({ in_reply_to: taskId, text: "nobody's", status: "replied" })).ok).toBe(true);
+    expect(replyRow(taskId)?.session_name).toBe("scheduler");
+    expect((await unreadFor(ownerToken))[ALIAS]).toBeUndefined();
+  });
+
+  test("a creator whose username is also a node alias falls back to 'scheduler' (the collision guard)", async () => {
+    await ackAgent(ownerToken);
+    const { taskId } = await fireSchedule(ownerToken, "colliding-creator");
+    const clashNodeId = `n_clash_${stamp}`;
+    db.run(
+      "INSERT INTO nodes (node_id, node_name, alias, runtime, network_id) VALUES (?1, ?2, ?2, 'claude-code', ?3)",
+      [clashNodeId, ownerName, networkId],
+    );
+    try {
+      expect((await nodeReply({ in_reply_to: taskId, text: "not a todo", status: "replied" })).ok).toBe(true);
+      // 那个同名节点收不到这条(它不是节点的待办),用户名也拿不到(inbox 半边整体跳过)
+      expect(replyRow(taskId)?.session_name).toBe("scheduler");
+      expect(db.get<{ n: number }>(
+        "SELECT COUNT(*) AS n FROM inbox WHERE session_name = ?1 AND in_reply_to = ?2", ownerName, taskId,
+      )?.n).toBe(0);
+    } finally {
+      db.run("DELETE FROM nodes WHERE node_id = ?1", [clashNodeId]);
+    }
+  });
+});

package/src/scheduled-run-terminal.test.ts

@@ -0,0 +1,268 @@
+import { afterAll, beforeEach, describe, expect, test } from "bun:test";
+import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
+import { db, chainReplyToParent } from "./db.js";
+import { dispatchScheduledOccurrence } from "./scheduled-tasks.js";
+import { registerTools } from "./tools.js";
+import { patrolExpiredTasks } from "./server.js";
+
+const NET = "net_690_scheduler_terminal";
+const USER = "u_690_owner";
+const NODE = "n_690_grok";
+const ALIAS = "A站GrokTUI-690";
+
+type Handler = (args: any, extra?: any) => Promise<{ content: Array<{ type: "text"; text: string }> }>;
+
+function cleanup(): void {
+  for (const table of ["task_events", "inbox", "scheduled_task_runs", "scheduled_tasks", "tasks", "sessions", "nodes", "network_members", "networks", "users"]) {
+    try { db.run(`DELETE FROM ${table} WHERE network_id = ?1`, [NET]); } catch {}
+  }
+  try { db.run("DELETE FROM users WHERE user_id = ?1", [USER]); } catch {}
+}
+
+function seedIdentity(): void {
+  db.run("INSERT INTO users (user_id, username, password_hash, role, created_at) VALUES (?1, ?2, 'x', 'user', datetime('now'))", [USER, USER]);
+  db.run("INSERT INTO networks (network_id, network_name, owner_id, created_at) VALUES (?1, ?1, ?2, datetime('now'))", [NET, USER]);
+  db.run("INSERT INTO network_members (user_id, network_id, role, joined_at) VALUES (?1, ?2, 'owner', datetime('now'))", [USER, NET]);
+  db.run(
+    `INSERT INTO nodes (node_id, node_name, alias, runtime, network_id, lifecycle_state)
+     VALUES (?1, ?2, ?2, 'grok-build-acp', ?3, 'active')`,
+    [NODE, ALIAS, NET],
+  );
+  db.run(
+    `INSERT INTO sessions (resume_id, alias, status, node_id, network_id, updated_at, last_seen_at)
+     VALUES ('r_690', ?1, 'idle', ?2, ?3, datetime('now'), datetime('now'))`,
+    [ALIAS, NODE, NET],
+  );
+}
+
+function handlers(): Record<string, Handler> {
+  const server = new McpServer({ name: "test690", version: "0" }) as any;
+  const result: Record<string, Handler> = {};
+  const original = server.tool.bind(server);
+  server.tool = (name: string, ...args: any[]) => {
+    const handler = args.at(-1);
+    if (typeof handler === "function") result[name] = handler;
+    return original(name, ...args);
+  };
+  registerTools(server, undefined, NET, USER, USER, false, null);
+  return result;
+}
+
+async function call(handler: Handler, args: any): Promise<any> {
+  const reply = await handler(args);
+  return JSON.parse(reply.content[0].text);
+}
+
+function seedScheduledTask(status = "delivered", suffix = crypto.randomUUID()): { taskId: string; runId: string; scheduleId: string } {
+  const taskId = `task_${suffix}`;
+  const runId = `srun_${suffix}`;
+  const scheduleId = `sched_${suffix}`;
+  const meta = JSON.stringify({ scheduled_task_id: scheduleId, scheduled_run_id: runId, auth_origin: "hub_scheduler" });
+  db.run(
+    `INSERT INTO scheduled_tasks
+       (schedule_id, network_id, name, target_node_id, target_alias, task_content, priority,
+        schedule_type, schedule_json, timezone, status, next_run_at)
+     VALUES (?1, ?2, 'test690', ?3, ?4, 'test690 task', 'normal', 'interval',
+             '{"type":"interval","every_seconds":3600}', 'UTC', 'active', datetime('now', '+1 hour'))`,
+    [scheduleId, NET, NODE, ALIAS],
+  );
+  db.run(
+    `INSERT INTO tasks
+       (task_id, from_name, to_node_id, to_name, priority, status, content, requires_response,
+        created_at, delivered_at, expires_at, network_id, meta_json)
+     VALUES (?1, 'scheduler', ?2, ?3, 'normal', ?4, 'test690 task', 'reply',
+             datetime('now'), datetime('now'), datetime('now', '+1 hour'), ?5, ?6)`,
+    [taskId, NODE, ALIAS, status, NET, meta],
+  );
+  db.run(
+    `INSERT INTO inbox (id, task_id, session_name, node_id, type, priority, content, from_session, requires_response, network_id, meta_json)
+     VALUES (?1, ?1, ?2, ?3, 'task', 'normal', 'test690 task', 'scheduler', 'reply', ?4, ?5)`,
+    [taskId, ALIAS, NODE, NET, meta],
+  );
+  db.run(
+    `INSERT INTO scheduled_task_runs (run_id, schedule_id, network_id, scheduled_for, task_id, status, completed_at)
+     VALUES (?1, ?2, ?3, datetime('now'), ?4, ?5, datetime('now'))`,
+    [runId, scheduleId, NET, taskId, status],
+  );
+  return { taskId, runId, scheduleId };
+}
+
+const readRun = (runId: string) => db.get<any>(
+  "SELECT run_id, task_id, network_id, status, error_code, completed_at FROM scheduled_task_runs WHERE run_id = ?1",
+  runId,
+);
+
+beforeEach(() => { cleanup(); seedIdentity(); });
+afterAll(cleanup);
+
+describe("#690 scheduled run follows the real task lifecycle", () => {
+  test("dispatch is open until the task reaches a terminal state", () => {
+    const scheduleId = `sched_dispatch_${crypto.randomUUID()}`;
+    const row: any = {
+      schedule_id: scheduleId,
+      network_id: NET,
+      name: "dispatch open",
+      target_node_id: NODE,
+      target_alias: ALIAS,
+      task_content: "dispatch open",
+      priority: "normal",
+      schedule_type: "interval",
+      schedule_json: JSON.stringify({ type: "interval", every_seconds: 3600 }),
+      timezone: "UTC",
+      overlap_policy: "skip",
+      misfire_policy: "catch_up_once",
+      status: "active",
+      next_run_at: new Date(Date.now() + 3600_000).toISOString(),
+      last_run_at: null,
+      revision: 1,
+    };
+    db.run(
+      `INSERT INTO scheduled_tasks
+         (schedule_id, network_id, name, target_node_id, target_alias, task_content, priority,
+          schedule_type, schedule_json, timezone, overlap_policy, misfire_policy, status, next_run_at)
+       VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14)`,
+      [scheduleId, NET, row.name, NODE, ALIAS, row.task_content, row.priority, row.schedule_type,
+       row.schedule_json, row.timezone, row.overlap_policy, row.misfire_policy, row.status, row.next_run_at],
+    );
+    const dispatched = dispatchScheduledOccurrence(row, new Date().toISOString(), false);
+    const run = readRun(dispatched.runId);
+    expect(["delivered", "queued"]).toContain(run.status);
+    expect(run.completed_at).toBeNull();
+  });
+
+  test("send_reply terminalizes the exact bound run", async () => {
+    const { taskId, runId } = seedScheduledTask();
+    const reply = await call(handlers().send_reply, {
+      alias: ALIAS,
+      text: "verified",
+      in_reply_to: taskId,
+      status: "replied",
+      network_id: NET,
+      attachments: [{ type: "file", file_id: "file_690_reply", name: "evidence.json", size: 12, mime: "application/json" }],
+    });
+    expect(reply.ok).toBe(true);
+    expect(readRun(runId).status).toBe("replied");
+    expect(readRun(runId).completed_at).toBeTruthy();
+  });
+
+  test("task and run terminal transition roll back together on an injected run-write failure", async () => {
+    const { taskId, runId } = seedScheduledTask();
+    db.exec(
+      `CREATE TRIGGER test690_fail_run_terminal
+         BEFORE UPDATE OF status ON scheduled_task_runs
+         WHEN NEW.run_id = '${runId}' AND NEW.status = 'replied'
+       BEGIN
+         SELECT RAISE(ABORT, 'test690_injected_run_failure');
+       END`,
+    );
+    let threw = false;
+    try {
+      await call(handlers().send_reply, {
+        alias: ALIAS,
+        text: "must roll back",
+        in_reply_to: taskId,
+        status: "replied",
+        network_id: NET,
+      });
+    } catch (error: any) {
+      threw = /test690_injected_run_failure/.test(String(error?.message || error));
+    } finally {
+      db.exec("DROP TRIGGER IF EXISTS test690_fail_run_terminal");
+    }
+    expect(threw).toBe(true);
+    expect(db.get<{ status: string }>("SELECT status FROM tasks WHERE task_id = ?1", taskId)?.status).toBe("delivered");
+    expect(readRun(runId).status).toBe("delivered");
+    expect(db.get<{ count: number }>("SELECT COUNT(*) AS count FROM inbox WHERE in_reply_to = ?1", taskId)?.count).toBe(0);
+  });
+
+  test("cancel terminalizes, retry reopens, and the retry result closes the same run", async () => {
+    const { taskId, runId } = seedScheduledTask();
+    const tools = handlers();
+    expect((await call(tools.cancel_task, { task_id: taskId, reason: "operator", network_id: NET })).ok).toBe(true);
+    expect(readRun(runId).status).toBe("cancelled");
+    expect(readRun(runId).completed_at).toBeTruthy();
+
+    expect((await call(tools.retry_task, { task_id: taskId, network_id: NET })).ok).toBe(true);
+    expect(readRun(runId).status).toBe("delivered");
+    expect(readRun(runId).completed_at).toBeNull();
+
+    expect((await call(tools.send_reply, {
+      alias: ALIAS,
+      text: "runtime failed",
+      in_reply_to: taskId,
+      status: "failed",
+      network_id: NET,
+    })).ok).toBe(true);
+    expect(readRun(runId).status).toBe("failed");
+    expect(readRun(runId).error_code).toBe("task_failed");
+  });
+
+  test("report_completion terminalizes a scheduler run", async () => {
+    const { taskId, runId } = seedScheduledTask();
+    const reply = await call(handlers().report_completion, {
+      alias: ALIAS,
+      task: taskId,
+      result: "done",
+      network_id: NET,
+    });
+    expect(reply.ok).toBe(true);
+    expect(readRun(runId).status).toBe("replied");
+  });
+
+  test("expiration patrol closes the exact run in the same lifecycle pass", () => {
+    const { taskId, runId } = seedScheduledTask();
+    db.run("UPDATE tasks SET expires_at = datetime('now', '-1 minute') WHERE task_id = ?1", [taskId]);
+    patrolExpiredTasks();
+    expect(db.get<{ status: string }>("SELECT status FROM tasks WHERE task_id = ?1", taskId)?.status).toBe("expired");
+    expect(readRun(runId).status).toBe("expired");
+    expect(readRun(runId).error_code).toBe("task_expired");
+    expect(readRun(runId).completed_at).toBeTruthy();
+  });
+
+  test("child auto-chain terminalizes a scheduled parent", () => {
+    const { taskId: parentId, runId } = seedScheduledTask();
+    const childId = `child_${crypto.randomUUID()}`;
+    db.run(
+      `INSERT INTO tasks
+         (task_id, parent_task_id, from_name, to_name, priority, status, content, requires_response, created_at, network_id)
+       VALUES (?1, ?2, ?3, 'A站负责人', 'normal', 'replied', 'verify candidate', 'reply', datetime('now'), ?4)`,
+      [childId, parentId, ALIAS, NET],
+    );
+    const result = chainReplyToParent(childId, "news_id=14519", "replied", 5, NET);
+    expect(result.chained).toBe(true);
+    expect(readRun(runId).status).toBe("replied");
+  });
+
+  test("forged metadata cannot cross-bind another task or network run", async () => {
+    const victim = seedScheduledTask("delivered", "victim");
+    const attackerTaskId = "task_attacker";
+    const forgedMeta = JSON.stringify({ scheduled_task_id: victim.scheduleId, scheduled_run_id: victim.runId, auth_origin: "hub_scheduler" });
+    db.run(
+      `INSERT INTO tasks
+         (task_id, from_name, to_node_id, to_name, priority, status, content, requires_response, created_at, network_id, meta_json)
+       VALUES (?1, 'scheduler', ?2, ?3, 'normal', 'delivered', 'forged', 'reply', datetime('now'), ?4, ?5)`,
+      [attackerTaskId, NODE, ALIAS, NET, forgedMeta],
+    );
+    await call(handlers().send_reply, {
+      alias: ALIAS,
+      text: "forged close",
+      in_reply_to: attackerTaskId,
+      status: "replied",
+      network_id: NET,
+    });
+    expect(readRun(victim.runId).status).toBe("delivered");
+  });
+
+  test("a task cannot close a run whose persisted network binding differs", async () => {
+    const victim = seedScheduledTask("delivered", "foreign_network");
+    db.run("UPDATE scheduled_task_runs SET network_id = 'net_690_foreign' WHERE run_id = ?1", [victim.runId]);
+    await call(handlers().send_reply, {
+      alias: ALIAS,
+      text: "must not cross network",
+      in_reply_to: victim.taskId,
+      status: "replied",
+      network_id: NET,
+    });
+    expect(readRun(victim.runId).status).toBe("delivered");
+  });
+});