@sleep2agi/commhub-server 0.9.0-preview.14 → 0.9.0-preview.140

package/src/scheduled-stuck-http.test.ts

@@ -0,0 +1,312 @@
+// #464 —— 定时任务被一次没结束的执行卡住:告警 + 超时放行。
+//
+// 钉住:
+//   1. 默认值:超时 = clamp(6 × 间隔, 1h, 24h);环境变量能改。
+//   2. 同一个挡路任务连续挡掉 3 次 → 给排程创建者发恰好 1 条通知(别的成员 0 条);第 4、5 次不再发;
+//      挡路任务一换(下一次真派出去了)重新上膛。
+//   3. 挡路任务开着超过超时 → 它变成 expired(行还在、收件箱行 acked、run 记 expired/task_expired、
+//      error_message 说明是调度器超时),这一次照常派发。超时前已发过「卡住」→ 不再发;没发过 → 发一条「已超时」。
+//   4. 晚到的回复:send_reply 以 reply_task_terminal 拒掉,旧 run / 旧任务 / 新任务 / 新 run 都不变,创建者不多收东西。
+//   5. GET /runs 带 blocked_by_task_id / blocked_by_state。
+//
+// 跑法:cd server && COMMHUB_DB=/tmp/x.db bun test src/scheduled-stuck-http.test.ts
+//       PG:COMMHUB_TEST_PG_URL=… COMMHUB_PG_EXPERIMENTAL=1(tests/test2123-hub-postgres-ladder 里注册)
+import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, test } from "bun:test";
+import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
+import { addNetworkMember, createNetworkTokenForNode, register } from "./auth.js";
+import { db } from "./db.js";
+import { dispatchScheduledOccurrence, handleScheduledTaskRequest } from "./scheduled-tasks.js";
+import { blockerTimedOut, SCHEDULE_STUCK_NOTICE_KIND, stuckTimeoutMs } from "./scheduled-stuck.js";
+import { registerTools } from "./tools.js";
+
+const activeDbPath = process.env.COMMHUB_DB ?? (process.env.COMMHUB_TEST_PG_URL ? "postgres" : undefined);
+if (!activeDbPath) throw new Error("scheduled-stuck requires COMMHUB_DB (or COMMHUB_TEST_PG_URL) before module import");
+
+const stamp = Date.now();
+const ALIAS = `stuck-node-${stamp}`;
+const NODE_ID = `n_stuck_${stamp}`;
+let NET = "", creatorId = "", otherId = "";
+const ENV_KEYS = ["COMMHUB_SCHEDULE_STUCK_NOTICE_SKIPS", "COMMHUB_SCHEDULE_STUCK_TIMEOUT_FACTOR", "COMMHUB_SCHEDULE_STUCK_TIMEOUT_FLOOR_SEC", "COMMHUB_SCHEDULE_STUCK_TIMEOUT_CAP_SEC"];
+
+function seedSchedule(name = "日报汇总"): any {
+  const scheduleId = `sched_${crypto.randomUUID()}`;
+  db.run(
+    `INSERT INTO scheduled_tasks
+       (schedule_id, network_id, created_by, name, target_node_id, target_alias, task_content, priority,
+        schedule_type, schedule_json, timezone, status, next_run_at)
+     VALUES (?1, ?2, ?3, ?4, ?5, ?6, 'scheduled task', 'normal', 'interval',
+             '{"type":"interval","every_seconds":60}', 'Asia/Shanghai', 'active', datetime('now', '+1 minute'))`,
+    [scheduleId, NET, creatorId, name, NODE_ID, ALIAS],
+  );
+  return db.get("SELECT * FROM scheduled_tasks WHERE schedule_id = ?1", scheduleId);
+}
+
+let slot = 0;
+const nextSlot = () => new Date(Date.UTC(2026, 9, 2, 0, 0, 0) + (++slot) * 60_000).toISOString();
+const fire = (schedule: any, now = new Date()) => dispatchScheduledOccurrence(schedule, nextSlot(), false, now);
+
+const notices = (userId: string) => db.all<{ title: string; content: string; meta_json: string; from_session: string }>(
+  "SELECT title, content, meta_json, from_session FROM user_inbox WHERE user_id = ?1 AND network_id = ?2 AND kind = ?3 ORDER BY created_at, message_id",
+  userId, NET, SCHEDULE_STUCK_NOTICE_KIND,
+);
+const runRow = (runId: string) => db.get<any>("SELECT * FROM scheduled_task_runs WHERE run_id = ?1", runId);
+const taskRow = (taskId: string) => db.get<any>("SELECT * FROM tasks WHERE task_id = ?1", taskId);
+/** 把任务的开始时间往前拨(数据库里的 UTC 无时区串)。 */
+const backdate = (taskId: string, ms: number) => db.run("UPDATE tasks SET created_at = ?1 WHERE task_id = ?2", [new Date(Date.now() - ms).toISOString().slice(0, 19).replace("T", " "), taskId]);
+
+async function nodeReply(args: Record<string, unknown>): Promise<any> {
+  const mcp = new McpServer({ name: "sched-stuck", version: "0" }) as any;
+  const handlers: Record<string, (a: any) => Promise<any>> = {};
+  const original = mcp.tool.bind(mcp);
+  mcp.tool = (name: string, ...rest: any[]) => {
+    const handler = rest.at(-1);
+    if (typeof handler === "function") handlers[name] = handler;
+    return original(name, ...rest);
+  };
+  registerTools(mcp, undefined, NET, creatorId, ALIAS, true, null);
+  const out = await handlers.send_reply(args);
+  return JSON.parse(out.content[0].text);
+}
+
+beforeAll(() => {
+  const creator = register(`stuck_creator_${stamp}`, "StuckCreator123!", undefined, "seed");
+  expect(creator.ok).toBe(true);
+  NET = creator.network_id!; creatorId = creator.user!.user_id;
+  const other = register(`stuck_other_${stamp}`, "StuckOther123!", undefined, "seed");
+  otherId = other.user!.user_id;
+  expect(addNetworkMember(NET, otherId, "member", creatorId, { agentAccess: "all" }).ok).toBe(true);
+  expect(createNetworkTokenForNode(creatorId, NET, ALIAS).ok).toBe(true);
+  db.run("INSERT INTO nodes (node_id, node_name, alias, runtime, network_id) VALUES (?1, ?2, ?2, 'claude-code', ?3)", [NODE_ID, ALIAS, NET]);
+  db.run("INSERT INTO sessions (resume_id, alias, status, node_id, network_id) VALUES (?1, ?2, 'idle', ?3, ?4)", [`r_stuck_${stamp}`, ALIAS, NODE_ID, NET]);
+});
+beforeEach(() => {
+  for (const k of ENV_KEYS) delete process.env[k];
+  db.run("DELETE FROM user_inbox WHERE network_id = ?1 AND kind = ?2", [NET, SCHEDULE_STUCK_NOTICE_KIND]);
+  // 上一个用例留下的开着的任务不许挡这一个(每个用例自己的排程,只按 schedule 查,保险起见也关掉)
+  db.run("UPDATE tasks SET status = 'replied' WHERE network_id = ?1 AND status IN ('created', 'delivered', 'acked', 'running')", [NET]);
+});
+afterEach(() => { for (const k of ENV_KEYS) delete process.env[k]; });
+afterAll(() => { db.run("DELETE FROM user_inbox WHERE network_id = ?1 AND kind = ?2", [NET, SCHEDULE_STUCK_NOTICE_KIND]); });
+
+describe("#464 defaults", () => {
+  test("timeout = clamp(6 × interval, 1h, 24h)", () => {
+    const iv = (s: number) => JSON.stringify({ type: "interval", every_seconds: s });
+    expect(stuckTimeoutMs(iv(60))).toBe(3_600_000);
+    expect(stuckTimeoutMs(iv(600))).toBe(3_600_000);
+    expect(stuckTimeoutMs(iv(1800))).toBe(3 * 3_600_000);
+    expect(stuckTimeoutMs(iv(3600))).toBe(6 * 3_600_000);
+    expect(stuckTimeoutMs(iv(86_400))).toBe(24 * 3_600_000);
+    expect(stuckTimeoutMs(JSON.stringify({ type: "daily", time: "09:00" }))).toBe(24 * 3_600_000);
+    expect(stuckTimeoutMs(JSON.stringify({ type: "once", run_at: "2026-10-02T00:00:00Z" }))).toBe(3_600_000);
+    process.env.COMMHUB_SCHEDULE_STUCK_TIMEOUT_FACTOR = "0";
+    process.env.COMMHUB_SCHEDULE_STUCK_TIMEOUT_FLOOR_SEC = "20";
+    expect(stuckTimeoutMs(iv(60))).toBe(20_000);
+    process.env.COMMHUB_SCHEDULE_STUCK_TIMEOUT_FLOOR_SEC = "junk";
+    expect(stuckTimeoutMs(iv(60))).toBe(3_600_000);
+  });
+});
+
+describe("#464 stuck notice", () => {
+  test("3rd consecutive skip behind the same task → exactly one notice to the creator, none to others", () => {
+    const s = seedSchedule();
+    const first = fire(s);
+    expect(first.status).toBe("delivered");
+    const skips = [fire(s), fire(s)];
+    expect(notices(creatorId).length).toBe(0);
+    skips.push(fire(s), fire(s), fire(s));
+    expect(skips.map((r) => r.status)).toEqual(["skipped", "skipped", "skipped", "skipped", "skipped"]);
+    const mine = notices(creatorId);
+    expect(mine.length).toBe(1);
+    expect(notices(otherId).length).toBe(0);
+    const n = mine[0];
+    expect(n.from_session).toBe(ALIAS);
+    expect(n.title).toBe("定时任务被卡住");
+    for (const frag of ["日报汇总", first.taskId!, ALIAS, "(Asia/Shanghai)", "3 次", "1 小时"]) expect(n.content).toContain(frag);
+    expect(JSON.parse(n.meta_json).schedule_stuck).toEqual({ reason: "stuck", schedule_id: s.schedule_id, blocked_by_task_id: first.taskId, skips: 3 });
+  });
+
+  test("re-armed once a run is dispatched again: a second episode notifies again", () => {
+    const s = seedSchedule();
+    const first = fire(s);
+    for (let i = 0; i < 4; i++) fire(s);
+    db.run("UPDATE tasks SET status = 'replied', completed_at = datetime('now') WHERE task_id = ?1", [first.taskId]);
+    const second = fire(s);
+    expect(second.status).toBe("delivered");
+    for (let i = 0; i < 3; i++) fire(s);
+    const mine = notices(creatorId);
+    expect(mine.length).toBe(2);
+    // 两条通知同一秒写入,created_at 排不出先后:按挡路任务比集合。
+    expect(mine.map((m) => JSON.parse(m.meta_json).schedule_stuck.blocked_by_task_id).sort()).toEqual([first.taskId, second.taskId].sort());
+  });
+
+  test("COMMHUB_SCHEDULE_STUCK_NOTICE_SKIPS changes N", () => {
+    process.env.COMMHUB_SCHEDULE_STUCK_NOTICE_SKIPS = "1";
+    const s = seedSchedule();
+    fire(s); fire(s);
+    expect(notices(creatorId).length).toBe(1);
+  });
+
+  test("schedule without a creator → no notice, skips still recorded", () => {
+    const s = seedSchedule();
+    db.run("UPDATE scheduled_tasks SET created_by = NULL WHERE schedule_id = ?1", [s.schedule_id]);
+    const row = db.get("SELECT * FROM scheduled_tasks WHERE schedule_id = ?1", s.schedule_id);
+    fire(row); for (let i = 0; i < 4; i++) expect(fire(row).status).toBe("skipped");
+    expect(notices(creatorId).length).toBe(0);
+  });
+});
+
+describe("#464 stuck timeout", () => {
+  test("blocker past the timeout is expired (row kept) and this occurrence dispatches; no second notice", () => {
+    const s = seedSchedule();
+    const first = fire(s);
+    db.run("UPDATE tasks SET status = 'running' WHERE task_id = ?1", [first.taskId]);
+    for (let i = 0; i < 3; i++) fire(s);
+    expect(notices(creatorId).length).toBe(1);
+
+    backdate(first.taskId!, 59 * 60_000);
+    expect(fire(s).status).toBe("skipped"); // 59 分钟 < 1 小时
+    backdate(first.taskId!, 61 * 60_000);
+    const next = fire(s);
+    expect(next.status).toBe("delivered");
+    expect(next.taskId).not.toBe(first.taskId);
+
+    const old = taskRow(first.taskId!);
+    expect(old.status).toBe("expired");
+    expect(old.content).toBe("scheduled task");
+    expect(db.get<any>("SELECT acked FROM inbox WHERE task_id = ?1", first.taskId).acked).toBeTruthy();
+    const oldRun = runRow(first.runId);
+    expect(oldRun.status).toBe("expired");
+    expect(oldRun.error_code).toBe("task_expired");
+    expect(oldRun.error_message).toContain("timed out by the scheduler");
+    expect(db.get<any>("SELECT COUNT(*) AS n FROM task_events WHERE task_id = ?1 AND to_status = 'expired' AND actor = 'hub-scheduler'", first.taskId).n).toBe(1);
+    expect(notices(creatorId).length).toBe(1); // 每段卡住恰好一条
+  });
+
+  test("timed out before N skips (long interval) → one 'timed out' notice instead", () => {
+    const s = seedSchedule("每日巡检");
+    const first = fire(s);
+    expect(fire(s).status).toBe("skipped");
+    backdate(first.taskId!, 2 * 3_600_000);
+    expect(fire(s).status).toBe("delivered");
+    const mine = notices(creatorId);
+    expect(mine.length).toBe(1);
+    expect(mine[0].title).toBe("定时任务已超时放行");
+    expect(mine[0].content).toContain(first.taskId!);
+    expect(JSON.parse(mine[0].meta_json).schedule_stuck.reason).toBe("timed_out");
+  });
+
+  test("a late reply to the expired task is refused and changes nothing", async () => {
+    const s = seedSchedule();
+    const first = fire(s);
+    backdate(first.taskId!, 2 * 3_600_000);
+    const next = fire(s);
+    expect(next.status).toBe("delivered");
+    const inboxBefore = db.get<any>("SELECT COUNT(*) AS n FROM inbox WHERE network_id = ?1", NET).n;
+    const out = await nodeReply({ alias: "scheduler", text: "late answer", in_reply_to: first.taskId, status: "replied", from_session: ALIAS });
+    expect(out.ok).toBe(false);
+    expect(out.error).toBe("reply_task_terminal");
+    expect(out.reply_queued).toBe(false);
+    expect(taskRow(first.taskId!).status).toBe("expired");
+    expect(runRow(first.runId).status).toBe("expired");
+    expect(taskRow(next.taskId!).status).toBe("delivered");
+    expect(runRow(next.runId).status).toBe("delivered");
+    expect(db.get<any>("SELECT COUNT(*) AS n FROM inbox WHERE network_id = ?1", NET).n).toBe(inboxBefore);
+    // 新那条照常能被回复,run 照常收尾
+    const ok = await nodeReply({ alias: "scheduler", text: "on time", in_reply_to: next.taskId, status: "replied", from_session: ALIAS });
+    expect(ok.ok).toBe(true);
+    expect(runRow(next.runId).status).toBe("replied");
+  });
+});
+
+describe("#464 history clarity", () => {
+  test("GET /runs exposes blocked_by_task_id and blocked_by_state on skipped rows", async () => {
+    const s = seedSchedule();
+    const first = fire(s);
+    const second = fire(s);
+    const url = new URL(`http://hub.test/api/scheduled-tasks/${s.schedule_id}/runs`);
+    const res = await handleScheduledTaskRequest({
+      req: new Request(url, { method: "GET" }), url,
+      auth: { userId: creatorId, networkId: NET, username: "creator" },
+      isAdmin: false, isNodeToken: false, scope: { networkId: NET, networkIds: [NET] } as any,
+    });
+    const body = await res!.json() as any;
+    const skipped = body.runs.find((r: any) => r.run_id === second.runId);
+    expect(skipped.status).toBe("skipped");
+    expect(skipped.blocked_by_task_id).toBe(first.taskId);
+    expect(skipped.blocked_by_state).toBe("not_received");
+  });
+});
+
+
+describe("#1112 execution clock", () => {
+  const ts = (now: Date, minutes: number) => new Date(now.getTime() - minutes * 60_000).toISOString();
+  test("queue budget: creation over one hour but consumption under one hour keeps the exact run open", () => {
+    const s = seedSchedule();
+    const first = fire(s);
+    const now = new Date();
+    db.run("UPDATE tasks SET status = 'running', created_at = ?1, consumed_at = ?2 WHERE task_id = ?3", [ts(now, 61), ts(now, 49), first.taskId]);
+    expect(fire(s, now).status).toBe("skipped");
+    expect(taskRow(first.taskId!).status).toBe("running");
+    expect(runRow(first.runId).status).toBe("delivered");
+    expect(db.get<any>("SELECT COUNT(*) AS n FROM task_events WHERE task_id = ?1 AND to_status = 'expired'", first.taskId).n).toBe(0);
+    db.run("UPDATE tasks SET created_at = ?1, consumed_at = ?2 WHERE task_id = ?3", [ts(now, 73), ts(now, 61), first.taskId]);
+    const later = fire(s, now);
+    expect(later.status).toBe("delivered");
+    expect(taskRow(first.taskId!).status).toBe("expired");
+    expect(runRow(first.runId).error_message).toContain("since consumed_at");
+  });
+  test("queue budget: legacy execution start protects the queued task, consumption takes precedence", () => {
+    const s = seedSchedule(); const first = fire(s); const now = new Date();
+    db.run("UPDATE tasks SET status = 'running', created_at = ?1, started_at = ?2 WHERE task_id = ?3", [ts(now, 75), ts(now, 40), first.taskId]);
+    expect(fire(s, now).status).toBe("skipped");
+    db.run("UPDATE tasks SET started_at = ?1, consumed_at = ?2 WHERE task_id = ?3", [ts(now, 70), ts(now, 40), first.taskId]);
+    expect(fire(s, now).status).toBe("skipped");
+  });
+  test("submission and ACK alone do not renew queue time", () => {
+    const s = seedSchedule(); const first = fire(s); const now = new Date();
+    db.run("UPDATE tasks SET status = 'acked', created_at = ?1, runtime_submitted_at = ?2 WHERE task_id = ?3", [ts(now, 61), ts(now, 1), first.taskId]);
+    expect(fire(s, now).status).toBe("delivered");
+    expect(runRow(first.runId).error_message).toContain("since created_at");
+  });
+  test("total cap: a very late consumption cannot keep an occurrence open beyond 24 hours", () => {
+    const s = seedSchedule(); const first = fire(s); const now = new Date();
+    db.run("UPDATE tasks SET status = 'running', created_at = ?1, consumed_at = ?2 WHERE task_id = ?3", [ts(now, 1441), ts(now, 1), first.taskId]);
+    expect(fire(s, now).status).toBe("delivered");
+    expect(runRow(first.runId).status).toBe("expired");
+    expect(runRow(first.runId).error_message).toContain("created_at (total cap)");
+    expect(runRow(first.runId).error_message).toContain("limit 1440 min");
+  });
+  test("invalid, future and pre-creation evidence falls back; UTC and exact execution boundary remain stable", () => {
+    const now = new Date("2026-10-13T03:00:00Z");
+    const schedule = JSON.stringify({ type: "interval", every_seconds: 60 });
+    for (const consumed_at of [null, "invalid", "2026-10-14T00:00:00Z", "2026-10-12T00:00:00Z"]) {
+      expect(blockerTimedOut({ created_at: "2026-10-13 01:00:00", consumed_at }, schedule, now)).toBe(true);
+    }
+    expect(blockerTimedOut({ created_at: "2026-10-13 01:00:00", consumed_at: "2026-10-13 02:00:01" }, schedule, now)).toBe(false);
+    expect(blockerTimedOut({ created_at: "2026-10-13 01:00:00", consumed_at: "2026-10-13 02:00:00" }, schedule, now)).toBe(true);
+  });
+});
+
+
+describe("#1113 activity idle clock",()=>{
+ test("fresh exact-task activity protects a long consumed turn, silent activity still expires",()=>{
+  const s=seedSchedule(),first=fire(s),now=new Date(),at=(m:number)=>new Date(now.getTime()-m*60_000).toISOString();
+  db.run("UPDATE tasks SET status='running',created_at=?1,consumed_at=?2,last_activity_at=?3 WHERE task_id=?4",[at(180),at(120),at(1),first.taskId]);
+  expect(fire(s,now).status).toBe("skipped");expect(taskRow(first.taskId!).status).toBe("running");
+  db.run("UPDATE tasks SET last_activity_at=?1 WHERE task_id=?2",[at(61),first.taskId]);expect(fire(s,now).status).toBe("delivered");
+  expect(runRow(first.runId).error_message).toContain("since last_activity_at");
+ });
+ test("fresh activity cannot evade the absolute total cap",()=>{
+  const s=seedSchedule(),first=fire(s),now=new Date(),at=(m:number)=>new Date(now.getTime()-m*60_000).toISOString();
+  db.run("UPDATE tasks SET status='running',created_at=?1,consumed_at=?2,last_activity_at=?3 WHERE task_id=?4",[at(1441),at(1440),at(1),first.taskId]);
+  expect(fire(s,now).status).toBe("delivered");expect(runRow(first.runId).error_message).toContain("total cap");
+ });
+ test("activity without consumption, before consumption or in the future never renews",()=>{
+  const now=new Date("2026-10-13T04:00:00Z"),schedule=JSON.stringify({type:"interval",every_seconds:60});
+  for(const row of [
+   {created_at:"2026-10-13 01:00:00",last_activity_at:"2026-10-13 03:59:00"},
+   {created_at:"2026-10-13 01:00:00",consumed_at:"2026-10-13 02:00:00",last_activity_at:"2026-10-13 01:59:00"},
+   {created_at:"2026-10-13 01:00:00",consumed_at:"2026-10-13 02:00:00",last_activity_at:"2026-10-13 04:01:00"},
+  ])expect(blockerTimedOut(row,schedule,now)).toBe(true);
+ });
+});

package/src/scheduled-stuck.ts

@@ -0,0 +1,186 @@
+// #464 —— 定时任务被一次没结束的执行卡住:告警 + 超时放行。
+//
+// 现场:overlap=skip 的排程,上一次派出的任务节点一直没有终态回复 → 之后每一次都记成
+//   skipped / previous_run_active,没有超时、也没人被告诉(当天 ~25 分钟,10-01 ~7 小时)。
+//
+// 1. 告警:同一个「卡住的任务」(blocked_by_task_id)连续挡掉 N 次 → 给排程的创建者发**一条**通知
+//    (agent-notice.ts,与 #462 同一条路,出现在与目标节点的会话里)。按挡路的任务计数,所以
+//    一段卡住只发一次;任务一换(下一次真派出去了)自然重新上膛。不依赖内存,Hub 重启不会重发。
+// 2. 超时:挡路的任务开着超过 stuckTimeoutMs(间隔)就按**已有的** expired 状态结束它(同 TTL 巡检:
+//    tasks.status='expired' + 收件箱行置 acked + db.ts 的镜像把这次 run 记成 expired / task_expired),
+//    然后这一次照常派发。任务行不删,照样能查;晚到的回复被 send_reply 以 reply_task_terminal 拒掉,
+//    什么都不改(不重开 run、不碰新派出去的那条)。
+//    如果超时发生在 N 次之前(长间隔的排程,比如每天一次),那一刻补发一条「已超时结束」—— 每段卡住
+//    恰好一条通知。
+//
+// 默认值(PR 里有推导):N=3;超时 = clamp(6 × 间隔, 下限 1 小时, 上限 24 小时)。
+// #1112:从 consumed_at 起算,旧节点回落 started_at,未执行回落 created_at;
+//       自创建时间的总保护上限仍为 cap,节点心跳/提交/ACK 不刷新执行时钟。
+//   下限 1 小时:一轮 agent 任务正常要跑几十分钟,不能因为排程间隔短就把还在干活的任务掐掉。
+//   上限 24 小时:Hub 本来就给定时任务 24 小时的 expires_at(没被取走时由巡检过期),开着超过一天的
+//   定时任务不是「还在做」。6 × 间隔:间隔长的排程(每小时一次)给它 6 小时,而不是 1 小时。
+// 环境变量可改(运维 / 测试):COMMHUB_SCHEDULE_STUCK_NOTICE_SKIPS、COMMHUB_SCHEDULE_STUCK_TIMEOUT_FACTOR、
+//   COMMHUB_SCHEDULE_STUCK_TIMEOUT_FLOOR_SEC、COMMHUB_SCHEDULE_STUCK_TIMEOUT_CAP_SEC。
+
+import { db, syncScheduledRunForTask } from "./db.js";
+import { parseDbTimestampMs } from "./db-timestamp.js";
+import { sendAgentNotice } from "./agent-notice.js";
+
+export const SCHEDULE_STUCK_NOTICE_KIND = "schedule_stuck";
+
+function envNumber(name: string, fallback: number, min: number): number {
+  const raw = process.env[name];
+  if (raw === undefined || raw.trim() === "") return fallback;
+  const n = Number(raw);
+  return Number.isFinite(n) && n >= min ? n : fallback;
+}
+
+export function stuckNoticeSkips(): number {
+  return Math.floor(envNumber("COMMHUB_SCHEDULE_STUCK_NOTICE_SKIPS", 3, 1));
+}
+
+/** 排程的名义间隔(秒):interval 取 every_seconds;daily / weekly 取一天(两次之间最短一天);once 取 0。 */
+export function nominalIntervalSeconds(scheduleJson: string): number {
+  try {
+    const spec = JSON.parse(scheduleJson) as { type?: string; every_seconds?: number };
+    if (spec.type === "interval" && Number.isFinite(spec.every_seconds)) return Number(spec.every_seconds);
+    if (spec.type === "daily" || spec.type === "weekly") return 86_400;
+  } catch {}
+  return 0;
+}
+
+export function stuckTimeoutMs(scheduleJson: string): number {
+  const factor = envNumber("COMMHUB_SCHEDULE_STUCK_TIMEOUT_FACTOR", 6, 0);
+  const floor = envNumber("COMMHUB_SCHEDULE_STUCK_TIMEOUT_FLOOR_SEC", 3_600, 1);
+  const cap = Math.max(floor, envNumber("COMMHUB_SCHEDULE_STUCK_TIMEOUT_CAP_SEC", 86_400, 1));
+  return Math.min(cap, Math.max(floor, factor * nominalIntervalSeconds(scheduleJson))) * 1000;
+}
+
+export type StuckBlockerClock = {
+  created_at: string;
+  consumed_at?: string | null;
+  last_activity_at?: string | null;
+  started_at?: string | null;
+};
+
+/** #1105: queue time is not execution time. Submission/ACK/node heartbeat are not consumption. */
+export function blockerTimeoutClock(blocker: StuckBlockerClock, scheduleJson: string, now: Date): { ageMs: number; limitMs: number; source: string } {
+  const created = parseDbTimestampMs(blocker.created_at);
+  const nowMs = now.getTime();
+  const floor = envNumber("COMMHUB_SCHEDULE_STUCK_TIMEOUT_FLOOR_SEC", 3_600, 1);
+  const capMs = Math.max(floor, envNumber("COMMHUB_SCHEDULE_STUCK_TIMEOUT_CAP_SEC", 86_400, 1)) * 1000;
+  // Even a very late consumption cannot hold this occurrence beyond the total cap.
+  if (Number.isFinite(created) && nowMs - created >= capMs) {
+    return { ageMs: nowMs - created, limitMs: capMs, source: "created_at (total cap)" };
+  }
+  for (const source of ["consumed_at", "started_at"] as const) {
+    const value = blocker[source];
+    if (!value) continue;
+    const at = parseDbTimestampMs(value);
+    if (Number.isFinite(at) && at <= nowMs && (!Number.isFinite(created) || at >= created)) {
+      if (source === "consumed_at" && blocker.last_activity_at) {
+        const activity = parseDbTimestampMs(blocker.last_activity_at);
+        if (Number.isFinite(activity) && activity >= at && activity <= nowMs) {
+          return { ageMs: nowMs - activity, limitMs: stuckTimeoutMs(scheduleJson), source: "last_activity_at" };
+        }
+      }
+      return { ageMs: nowMs - at, limitMs: stuckTimeoutMs(scheduleJson), source };
+    }
+  }
+  return { ageMs: nowMs - created, limitMs: stuckTimeoutMs(scheduleJson), source: "created_at" };
+}
+
+/** 挡路的任务开了多久已经到超时 → true。读不出时间 = 不超时(宁可多挡一次,不错杀)。 */
+export function blockerTimedOut(blocker: StuckBlockerClock, scheduleJson: string, now: Date): boolean {
+  const clock = blockerTimeoutClock(blocker, scheduleJson, now);
+  return Number.isFinite(clock.ageMs) && clock.ageMs >= clock.limitMs;
+}
+
+/**
+ * 在调用方的事务里按超时结束挡路的任务。只动还开着的任务(条件写),返回是否真的结束了它。
+ * 与 TTL 巡检(server.ts patrolExpiredTasks)同一套写法:status=expired、收件箱行置 acked、镜像到 run。
+ */
+export function expireStuckBlocker(taskId: string, networkId: string, ageMs: number, limitMs: number, source = "created_at"): boolean {
+  const changed = db.run(
+    `UPDATE tasks SET status = 'expired', completed_at = datetime('now')
+      WHERE task_id = ?1 AND network_id = ?2 AND status IN ('created', 'delivered', 'acked', 'running')`,
+    [taskId, networkId],
+  );
+  if (changed.changes < 1) return false;
+  db.run("UPDATE inbox SET acked = 1 WHERE COALESCE(task_id, id) = ?1 AND acked = 0", [taskId]);
+  syncScheduledRunForTask(taskId, networkId);
+  db.run(
+    "UPDATE scheduled_task_runs SET error_message = ?1 WHERE task_id = ?2 AND network_id = ?3",
+    [`timed out by the scheduler: ${Math.round(ageMs / 60_000)} min since ${source} (limit ${Math.round(limitMs / 60_000)} min); the next occurrence was dispatched`, taskId, networkId],
+  );
+  return true;
+}
+
+/** 这个任务已经挡掉了几次(只算 previous_run_active 的跳过)。 */
+export function skipsBlockedBy(scheduleId: string, taskId: string): number {
+  return db.get<{ n: number }>(
+    "SELECT COUNT(*) AS n FROM scheduled_task_runs WHERE schedule_id = ?1 AND blocked_by_task_id = ?2 AND error_code = 'previous_run_active'",
+    scheduleId, taskId,
+  )?.n ?? 0;
+}
+
+export type StuckNotice = {
+  reason: "stuck" | "timed_out";
+  scheduleId: string;
+  scheduleName: string;
+  networkId: string;
+  createdBy: string | null;
+  alias: string;
+  timezone: string;
+  taskId: string;
+  taskCreatedAt: string;
+  skips: number;
+  timeoutMs: number;
+};
+
+function fmtTime(dbTs: string, timezone: string): string {
+  const ms = parseDbTimestampMs(dbTs);
+  if (!Number.isFinite(ms)) return dbTs;
+  try {
+    const s = new Intl.DateTimeFormat("sv-SE", { timeZone: timezone, year: "numeric", month: "2-digit", day: "2-digit", hour: "2-digit", minute: "2-digit", hourCycle: "h23" }).format(new Date(ms));
+    return `${s} (${timezone})`;
+  } catch { return dbTs; }
+}
+
+function fmtDuration(ms: number): string {
+  const min = Math.round(ms / 60_000);
+  if (min < 60) return `${min} 分钟`;
+  const h = Math.floor(min / 60), m = min % 60;
+  return m ? `${h} 小时 ${m} 分钟` : `${h} 小时`;
+}
+
+export function stuckNoticeText(n: StuckNotice): { title: string; text: string } {
+  const started = fmtTime(n.taskCreatedAt, n.timezone);
+  if (n.reason === "stuck") {
+    return {
+      title: "定时任务被卡住",
+      text: `定时任务「${n.scheduleName}」被上一次的执行卡住了:任务 ${n.taskId} 于 ${started} 派给节点 ${n.alias},到现在还没有完成,之后的 ${n.skips} 次执行都被跳过。\n\n`
+        + `距上次任务活动或实际开始执行超过 ${fmtDuration(n.timeoutMs)} 后会被自动按超时结束;尚未开始的任务从派发起计时,所有执行仍受总时限保护。下一次照常派发。等不及可以在任务列表里取消这条任务。`,
+    };
+  }
+  return {
+    title: "定时任务已超时放行",
+    text: `定时任务「${n.scheduleName}」上一次的任务 ${n.taskId}(${started} 派给节点 ${n.alias})超过 ${fmtDuration(n.timeoutMs)} 的执行或总保护时限还没有完成,已按超时结束,不再挡后面的执行。`,
+  };
+}
+
+/** 提交之后调用。没有创建者 / 创建者不在网络里 → 不发。 */
+export function sendStuckNotice(n: StuckNotice): string | null {
+  if (!n.createdBy) return null;
+  try {
+    const { title, text } = stuckNoticeText(n);
+    return sendAgentNotice({
+      networkId: n.networkId, userId: n.createdBy, fromAlias: n.alias, kind: SCHEDULE_STUCK_NOTICE_KIND, title, text,
+      meta: { schedule_stuck: { reason: n.reason, schedule_id: n.scheduleId, blocked_by_task_id: n.taskId, skips: n.skips } },
+      idPrefix: "dm_sched_",
+    });
+  } catch (e: any) {
+    console.error(`[scheduled-tasks] stuck notice failed schedule=${n.scheduleId}: ${e?.message || e}`);
+    return null;
+  }
+}