@sleep2agi/commhub-server 0.9.0-preview.14 → 0.9.0-preview.140

package/src/scheduled-backoff-http.test.ts

@@ -0,0 +1,193 @@
+import { afterAll, beforeAll, describe, expect, test } from "bun:test";
+import { createNetworkTokenForNode, register } from "./auth.js";
+import { db, syncScheduledRunForTask } from "./db.js";
+import { refreshScheduledBackoff } from "./scheduled-backoff.js";
+import { getScheduleRow, dispatchScheduledOccurrence, runDueScheduledTasks } from "./scheduled-tasks.js";
+
+if (!process.env.COMMHUB_DB) throw new Error("test1278 requires an explicit disposable SQLite COMMHUB_DB");
+let server: any, base = "", network = "", ownerToken = "", ownerId = "";
+const stamp = Date.now(), nodes: Record<string, { id: string; alias: string; token: string; tokenId: string }> = {};
+async function rest(token: string, method: string, path: string, body?: unknown) {
+  const r = await fetch(base + path, { method, headers: { Authorization: `Bearer ${token}`,
+    "Content-Type": "application/json" }, body: body === undefined ? undefined : JSON.stringify(body) });
+  return { status: r.status, body: await r.json() as any };
+}
+async function mcp(token: string, name: string, args: Record<string, unknown>) {
+  const r = await fetch(base + "/mcp", { method: "POST", headers: { Authorization: `Bearer ${token}`,
+    "Content-Type": "application/json", Accept: "application/json, text/event-stream" },
+    body: JSON.stringify({ jsonrpc: "2.0", id: 1, method: "tools/call", params: { name, arguments: args } }) });
+  const raw = await r.text(), line = raw.split("\n").find(x => x.startsWith("data: "));
+  const envelope = JSON.parse(line ? line.slice(6) : raw);
+  try { return JSON.parse(envelope.result.content[0].text); }
+  catch { return { ok: false, transportError: true }; }
+}
+const readRun = (taskId: string) => db.get<any>(
+  "SELECT run_id, schedule_id, status, work_outcome, work_outcome_node_id, schedule_revision FROM scheduled_task_runs WHERE task_id = ?1 AND network_id = ?2", taskId, network)!;
+const readTask = (taskId: string) => db.get<any>("SELECT status, result FROM tasks WHERE task_id = ?1", taskId)!;
+const countReplies = (taskId: string) => db.get<{ n: number }>("SELECT COUNT(*) AS n FROM inbox WHERE in_reply_to = ?1 AND type = 'reply'", taskId)!.n;
+async function fire(enabled = true, interval = 120) {
+  const created = await rest(ownerToken, "POST", "/api/scheduled-tasks", { network_id: network,
+    name: `test1278-${crypto.randomUUID()}`, target_node_id: nodes.a.id, task: "fixture1278 work",
+    schedule: { type: "interval", every_seconds: interval }, timezone: "UTC", no_change_backoff: enabled });
+  expect(created.status).toBe(201);
+  const scheduleId = created.body.schedule.schedule_id;
+  db.run("UPDATE scheduled_tasks SET next_run_at = ?1 WHERE schedule_id != ?2", [new Date(Date.now() + 86400_000).toISOString(), scheduleId]);
+  db.run("UPDATE scheduled_tasks SET next_run_at = ?1 WHERE schedule_id = ?2", [new Date(Date.now() - 1000).toISOString(), scheduleId]);
+  await runDueScheduledTasks();
+  const run = db.get<any>("SELECT task_id FROM scheduled_task_runs WHERE schedule_id = ?1", scheduleId)!;
+  expect(run.task_id).toBeTruthy();
+  return { taskId: run.task_id as string, scheduleId };
+}
+const reply = (taskId: string, meta?: unknown, status = "replied", token = nodes.a.token, extra = {}) =>
+  mcp(token, "send_reply", { in_reply_to: taskId, text: "本轮无新增", status, ...(meta === undefined ? {} : { meta }), ...extra });
+const cadence = (id: string) => db.get<any>("SELECT next_run_at, last_run_at, revision, schedule_json, target_node_id FROM scheduled_tasks WHERE schedule_id = ?1", id)!;
+
+beforeAll(async () => {
+  const owner = register(`work_owner_${stamp}`, "Fixture1274Only!x"); ownerToken = owner.token!;
+  network = owner.network_id!; ownerId = owner.user!.user_id;
+  for (const key of ["a", "b"]) {
+    const id = `n_work_${key}_${stamp}`, alias = `work-${key}-${stamp}`;
+    const token = createNetworkTokenForNode(ownerId, network, alias, id); expect(token.ok).toBe(true);
+    nodes[key] = { id, alias, token: token.token!, tokenId: token.token_id! };
+    db.run("INSERT INTO sessions (resume_id, alias, status, node_id, network_id) VALUES (?1, ?2, 'idle', ?3, ?4)", [`r_${id}`, alias, id, network]);
+  }
+  const mod = await import("./server.js"); server = mod.bootServer({ port: 0, hostname: "127.0.0.1" });
+  base = `http://127.0.0.1:${server.port}`;
+}, 30_000);
+afterAll(() => server?.stop?.(true));
+
+
+const state = (id: string) => db.get<any>("SELECT no_change_backoff, backoff_streak, backoff_effective_seconds, backoff_last_run_id, next_run_at, revision, schedule_json, status FROM scheduled_tasks WHERE schedule_id = ?1", id)!;
+let occurrence = 0;
+async function nextRun(id: string) {
+  // Isolated fixture accelerates the clock, without touching live schedules.
+  const date = new Date(Date.now() + (++occurrence) * 1000).toISOString();
+  const out = dispatchScheduledOccurrence(getScheduleRow(id)!, date, true);
+  expect(out.taskId).toBeTruthy(); return out.taskId!;
+}
+async function empties(id: string, n: number) {
+  for (let i = 0; i < n; i++) expect((await reply(await nextRun(id), { scheduled_work_outcome: "no_changes" })).ok).toBe(true);
+}
+async function atThree() {
+  const x = await fire(); expect((await reply(x.taskId, { scheduled_work_outcome: "no_changes" })).ok).toBe(true);
+  await empties(x.scheduleId, 2); return x;
+}
+describe("#1278 opt-in durable no-change backoff through real HTTP/MCP", () => {
+  test("3 and 6 exact empty successes change effective cadence, preserving base and revision", async () => {
+    const { taskId, scheduleId } = await fire(), initial = state(scheduleId);
+    expect((await reply(taskId, { scheduled_work_outcome: "no_changes" })).ok).toBe(true);
+    await empties(scheduleId, 1); expect(state(scheduleId).backoff_effective_seconds).toBe(120);
+    await empties(scheduleId, 1); const three = state(scheduleId);
+    expect(three).toMatchObject({ backoff_streak: 3, backoff_effective_seconds: 600, revision: initial.revision, schedule_json: initial.schedule_json });
+    expect(Date.parse(three.next_run_at) - Date.now()).toBeGreaterThan(590_000);
+    await empties(scheduleId, 3); expect(state(scheduleId)).toMatchObject({ backoff_streak: 6, backoff_effective_seconds: 1800 });
+    const wire = await rest(ownerToken, "GET", `/api/scheduled-tasks/${scheduleId}`);
+    expect(wire.body.schedule).toMatchObject({ no_change_backoff: true, schedule: { every_seconds: 120 }, backoff: { streak: 6, effective_seconds: 1800 } });
+    expect(wire.body.schedule.backoff_last_run_id).toBeUndefined();
+  });
+  test("changed restores base and nearer next occurrence; no natural-language inference", async () => {
+    const { scheduleId } = await atThree();
+    expect((await reply(await nextRun(scheduleId), { scheduled_work_outcome: "changed" })).ok).toBe(true);
+    expect(state(scheduleId)).toMatchObject({ backoff_streak: 0, backoff_effective_seconds: 120 });
+    expect(Date.parse(state(scheduleId).next_run_at) - Date.now()).toBeLessThanOrEqual(120_000);
+  });
+  test("unknown success and ordinary failure break the sequence", async () => {
+    for (const status of ["replied", "failed"]) {
+      const { scheduleId } = await atThree();
+      expect((await reply(await nextRun(scheduleId), undefined, status)).ok).toBe(true);
+      expect(state(scheduleId)).toMatchObject({ backoff_streak: 0, backoff_effective_seconds: 120 });
+      await empties(scheduleId, 2); expect(state(scheduleId).backoff_effective_seconds).toBe(120);
+    }
+  });
+  test("legacy disabled schedules preserve exact cadence and wire shape", async () => {
+    const { taskId, scheduleId } = await fire(false), before = state(scheduleId);
+    expect((await reply(taskId, { scheduled_work_outcome: "no_changes" })).ok).toBe(true);
+    expect(state(scheduleId)).toEqual(before);
+    const wire = await rest(ownerToken, "GET", `/api/scheduled-tasks/${scheduleId}`);
+    expect(wire.body.schedule.no_change_backoff).toBeUndefined(); expect(wire.body.schedule.backoff).toBeUndefined();
+  });
+  test("a slower base interval is never accelerated", async () => {
+    const { taskId, scheduleId } = await fire(true, 3600);
+    await reply(taskId, { scheduled_work_outcome: "no_changes" }); await empties(scheduleId, 5);
+    expect(state(scheduleId)).toMatchObject({ backoff_streak: 6, backoff_effective_seconds: 3600 });
+  });
+  test("duplicate callbacks and another DB process cannot increment twice", async () => {
+    const { scheduleId } = await atThree(), before = state(scheduleId);
+    const latest = db.get<any>("SELECT task_id FROM scheduled_task_runs WHERE run_id = ?1", before.backoff_last_run_id)!;
+    expect((await reply(latest.task_id, { scheduled_work_outcome: "no_changes" })).ok).toBe(false);
+    syncScheduledRunForTask(latest.task_id, network); expect(state(scheduleId)).toEqual(before);
+    const child = Bun.spawn([process.execPath, "-e", `import {db} from './server/src/db.ts'; import {refreshScheduledBackoff} from './server/src/scheduled-backoff.ts'; db.transaction(()=>refreshScheduledBackoff(${JSON.stringify(scheduleId)},${JSON.stringify(network)})); console.log(JSON.stringify(db.get('SELECT backoff_streak,backoff_effective_seconds FROM scheduled_tasks WHERE schedule_id=?1',${JSON.stringify(scheduleId)})));`], { cwd: process.cwd(), stdout: "pipe", stderr: "pipe", env: process.env });
+    const output = await new Response(child.stdout).text(); expect(await child.exited).toBe(0);
+    expect(JSON.parse(output.trim().split("\n").at(-1)!)).toEqual({ backoff_streak: 3, backoff_effective_seconds: 600 });
+    expect(state(scheduleId)).toEqual(before);
+  });
+  test("old generation completion after edit cannot rearm", async () => {
+    const { scheduleId } = await atThree(), old = await nextRun(scheduleId), revision = state(scheduleId).revision;
+    const edit = await rest(ownerToken, "PATCH", `/api/scheduled-tasks/${scheduleId}`, { revision, name: "fresh generation" }); expect(edit.status).toBe(200);
+    const before = state(scheduleId); expect(before.backoff_streak).toBe(0);
+    expect((await reply(old, { scheduled_work_outcome: "no_changes" })).ok).toBe(true); expect(state(scheduleId)).toEqual(before);
+  });
+  test("manual wake resets durable state and fences an in-flight old generation", async () => {
+    const { scheduleId } = await atThree(), old = await nextRun(scheduleId), revision = state(scheduleId).revision;
+    // Existing overlap skip remains effective. Wake still resets cadence.
+    const wake = await rest(ownerToken, "POST", `/api/scheduled-tasks/${scheduleId}/run-now`); expect(wake.status).toBe(202);
+    expect(state(scheduleId)).toMatchObject({ backoff_streak: 0, backoff_effective_seconds: null, revision: revision + 1 });
+    const before = state(scheduleId); await reply(old, { scheduled_work_outcome: "no_changes" }); expect(state(scheduleId)).toEqual(before);
+  });
+  test("pause/cancel never resurrect cadence; resumed policy starts a new generation", async () => {
+    const { scheduleId } = await atThree(), old = await nextRun(scheduleId);
+    expect((await rest(ownerToken, "PATCH", `/api/scheduled-tasks/${scheduleId}`, { revision: state(scheduleId).revision, status: "paused" })).status).toBe(200);
+    await reply(old, { scheduled_work_outcome: "changed" }); expect(state(scheduleId)).toMatchObject({ status: "paused", next_run_at: null, backoff_streak: 0 });
+    expect((await rest(ownerToken, "POST", `/api/scheduled-tasks/${scheduleId}/run-now`)).status).toBe(202);
+    expect(state(scheduleId).next_run_at).toBeNull();
+    expect((await rest(ownerToken, "DELETE", `/api/scheduled-tasks/${scheduleId}`)).status).toBe(200);
+    expect((await rest(ownerToken, "POST", `/api/scheduled-tasks/${scheduleId}/run-now`)).status).toBe(409);
+    expect(state(scheduleId)).toMatchObject({ status: "cancelled", next_run_at: null });
+  });
+  test("retry success is excluded from empty streaks and failure breaks prior successes", async () => {
+    const { scheduleId } = await atThree(), task = await nextRun(scheduleId);
+    await reply(task, undefined, "failed"); expect(state(scheduleId).backoff_streak).toBe(0);
+    expect((await mcp(ownerToken, "retry_task", { task_id: task, network_id: network })).ok).toBe(true);
+    expect((await reply(task, { scheduled_work_outcome: "no_changes" })).ok).toBe(true);
+    expect(state(scheduleId).backoff_streak).toBe(0); await empties(scheduleId, 2);
+    expect(state(scheduleId).backoff_streak).toBe(2);
+  });
+  test("overlap skip breaks the sequence and an older completion cannot bypass the skip", async () => {
+    const { scheduleId } = await atThree(), active = await nextRun(scheduleId);
+    const skipped = dispatchScheduledOccurrence(getScheduleRow(scheduleId)!, new Date(Date.now() + (++occurrence) * 1000).toISOString(), true);
+    expect(skipped.status).toBe("skipped"); expect(state(scheduleId).backoff_streak).toBe(0);
+    await reply(active, { scheduled_work_outcome: "no_changes" }); expect(state(scheduleId).backoff_streak).toBe(0);
+  });
+  test("current reporter target mismatch cannot count as empty", async () => {
+    const { scheduleId } = await atThree(), task = await nextRun(scheduleId);
+    // Fixture-only reassignment emulates an authorized assignment change.
+    db.run("UPDATE tasks SET to_node_id = ?1, to_name = ?2 WHERE task_id = ?3", [nodes.b.id, nodes.b.alias, task]);
+    expect((await reply(task, { scheduled_work_outcome: "no_changes" }, "replied", nodes.b.token)).ok).toBe(true);
+    expect(state(scheduleId).backoff_streak).toBe(0);
+  });
+  test("injected cadence-write fault rolls back task, inbox, outcome, and state together", async () => {
+    const { taskId, scheduleId } = await fire(), before = state(scheduleId);
+    db.exec(`CREATE TRIGGER test1278_fault BEFORE UPDATE OF backoff_streak ON scheduled_tasks WHEN NEW.schedule_id='${scheduleId}' BEGIN SELECT RAISE(ABORT,'test1278_fault'); END`);
+    try { expect((await reply(taskId, { scheduled_work_outcome: "no_changes" })).ok).not.toBe(true); }
+    finally { db.exec("DROP TRIGGER test1278_fault"); }
+    expect(state(scheduleId)).toEqual(before); expect(readTask(taskId).status).toBe("delivered");
+    expect(readRun(taskId).work_outcome).toBeNull(); expect(countReplies(taskId)).toBe(0);
+    expect((await reply(taskId, { scheduled_work_outcome: "no_changes" })).ok).toBe(true);
+  });
+  test("REST rejects malformed and non-interval policy; MCP respects creator ownership", async () => {
+    for (const policy of ["true", 1, null]) {
+      const bad = await rest(ownerToken, "POST", "/api/scheduled-tasks", { network_id: network, name: "invalid1278", target_node_id: nodes.a.id, task: "fixture", schedule: { type: "interval", every_seconds: 120 }, no_change_backoff: policy }); expect(bad.status).toBe(400);
+    }
+    const bad = await rest(ownerToken, "POST", "/api/scheduled-tasks", { network_id: network, name: "once1278", target_node_id: nodes.a.id, task: "fixture", schedule: { type: "once", run_at: new Date(Date.now()+86400000).toISOString() }, no_change_backoff: true }); expect(bad.status).toBe(400);
+    const created = await mcp(nodes.a.token, "schedule_create", { name: "agent1278", task: "fixture", schedule: { type: "interval", every_seconds: 120 }, no_change_backoff: true }); expect(created.ok).toBe(true);
+    expect(created.schedule.no_change_backoff).toBe(true);
+    const id = created.schedule.schedule_id;
+    expect((await mcp(nodes.b.token, "schedule_update", { schedule_id: id, no_change_backoff: false })).ok).toBe(false);
+    expect((await mcp(nodes.a.token, "schedule_update", { schedule_id: id, no_change_backoff: false })).ok).toBe(true);
+    expect(state(id).no_change_backoff).toBe(0);
+    const human = await fire(); expect((await mcp(nodes.a.token, "schedule_update", { schedule_id: human.scheduleId, no_change_backoff: false })).ok).toBe(false);
+    db.run("UPDATE scheduled_tasks SET overlap_policy = 'allow' WHERE schedule_id = ?1", human.scheduleId);
+    const allow = await rest(ownerToken, "PATCH", `/api/scheduled-tasks/${human.scheduleId}`, { revision: state(human.scheduleId).revision, no_change_backoff: true });
+    expect(allow.status).toBe(400); expect(allow.body.error).toBe("no_change_backoff_requires_skip_overlap");
+  });
+});

package/src/scheduled-backoff.ts

@@ -0,0 +1,64 @@
+import { db } from "./db.js";
+
+type WakeHook = (id: string, network: string, now: Date) => boolean;
+let requirementWakeHook: WakeHook | null = null;
+export function setRequirementWakeHook(hook: WakeHook): void { requirementWakeHook = hook; }
+
+export function parseBackoffPolicy(raw: unknown, type: string, fallback = false): boolean {
+  if (raw !== undefined && typeof raw !== "boolean") throw new Error("invalid_no_change_backoff");
+  const enabled = raw === undefined ? fallback : raw as boolean;
+  if (enabled && type !== "interval") throw new Error("no_change_backoff_requires_interval");
+  return enabled;
+}
+
+/** SQLite only. Called in the reply/dispatch transaction; no catch-and-continue
+ * on state-write failure. Deriving a capped streak from authoritative runs
+ * makes duplicate or out-of-order callbacks unable to increment it twice. */
+export function refreshScheduledBackoff(scheduleId: string, networkId: string, now = new Date()): void {
+  if (db.transactionalFeaturesRefusal !== null) return;
+  if (requirementWakeHook?.(scheduleId, networkId, now)) return;
+  const row = db.get<any>(`SELECT status, revision, target_node_id, schedule_json,
+    no_change_backoff, overlap_policy, requirement_wake_json, backoff_streak, backoff_effective_seconds,
+    backoff_last_run_id, next_run_at FROM scheduled_tasks
+    WHERE schedule_id = ?1 AND network_id = ?2`, scheduleId, networkId);
+  if (!row || row.status !== "active" || row.no_change_backoff !== 1 || row.overlap_policy !== "skip") return;
+  // A DB-only process with no watcher loaded may not bypass its fence.
+  if (row.requirement_wake_json && !requirementWakeHook) return;
+  const spec = JSON.parse(row.schedule_json);
+  if (spec.type !== "interval") return;
+  // Include failures/skips/unknowns: excluding them would turn separated
+  // successes into a fictitious consecutive empty sequence. Six is a cap.
+  const runs = db.all<any>(`SELECT r.run_id, r.status, r.work_outcome,
+    r.schedule_revision, r.work_outcome_node_id, r.backoff_excluded,
+    t.status AS task_status, t.to_node_id, t.from_name
+    FROM scheduled_task_runs r LEFT JOIN tasks t
+      ON t.task_id = r.task_id AND t.network_id = r.network_id
+    WHERE r.schedule_id = ?1 AND r.network_id = ?2
+    ORDER BY r.scheduled_for DESC, r.created_at DESC, r.run_id DESC LIMIT 6`, scheduleId, networkId);
+  const latest = runs[0];
+  if (!latest || latest.schedule_revision !== row.revision
+    || latest.run_id === row.backoff_last_run_id
+    || ["claiming", "queued", "delivered"].includes(latest.status)) return;
+  let streak = 0;
+  for (const run of runs) {
+    if (run.schedule_revision !== row.revision || run.status !== "replied"
+      || run.task_status !== "replied" || run.work_outcome !== "no_changes"
+      || run.work_outcome_node_id !== row.target_node_id || run.to_node_id !== row.target_node_id
+      || run.from_name !== "scheduler" || run.backoff_excluded !== 0) break;
+    streak++;
+  }
+  const effective = Math.max(spec.every_seconds, streak >= 6 ? 1800 : streak >= 3 ? 600 : 0);
+  const previous = row.backoff_effective_seconds ?? spec.every_seconds;
+  let next = row.next_run_at;
+  if (effective !== previous && next) {
+    const candidate = now.getTime() + effective * 1000;
+    next = new Date(effective > previous ? Math.max(Date.parse(next), candidate)
+      : Math.min(Date.parse(next), candidate)).toISOString();
+  }
+  const result = db.run(`UPDATE scheduled_tasks SET backoff_streak = ?1,
+    backoff_effective_seconds = ?2, backoff_last_run_id = ?3, next_run_at = ?4,
+    updated_at = datetime('now') WHERE schedule_id = ?5 AND network_id = ?6
+    AND revision = ?7 AND status = 'active' AND no_change_backoff = 1`,
+    [streak, effective, latest.run_id, next, scheduleId, networkId, row.revision]);
+  if (result.changes !== 1) throw new Error("scheduled_backoff_atomic_write_failed");
+}

package/src/scheduled-doorbell-nodeless-session.test.ts

@@ -0,0 +1,134 @@
+import { afterAll, beforeEach, describe, expect, test } from "bun:test";
+import { db } from "./db.js";
+import { createSSEStream } from "./push.js";
+import { dispatchScheduledOccurrence } from "./scheduled-tasks.js";
+
+// A claude-code node's channel server reports status without node_id, so its
+// sessions row has node_id NULL. The scheduler used to look the session up by
+// node_id only, judged every occurrence "queued" and never rang the doorbell:
+// the task sat in the inbox until an unrelated push drained it (20–36 min in
+// production).
+
+const NET = "net_sched_doorbell_nodeless";
+const NODE = "n_sched_doorbell_cc";
+const ALIAS = "sched-doorbell-cc";
+
+function cleanup(): void {
+  for (const table of ["task_events", "inbox", "scheduled_task_runs", "scheduled_tasks", "tasks", "sessions", "nodes"]) {
+    try { db.run(`DELETE FROM ${table} WHERE network_id = ?1`, [NET]); } catch {}
+  }
+}
+
+function seedNode(): void {
+  db.run(
+    `INSERT INTO nodes (node_id, node_name, alias, runtime, network_id, lifecycle_state)
+     VALUES (?1, ?2, ?2, 'claude-code-cli', ?3, 'active')`,
+    [NODE, ALIAS, NET],
+  );
+}
+
+function seedSession(status: string, nodeId: string | null): void {
+  db.run(
+    `INSERT INTO sessions (resume_id, alias, status, node_id, network_id, agent, updated_at, last_seen_at)
+     VALUES (?1, ?2, ?3, ?4, ?5, 'claude-code', datetime('now'), datetime('now'))`,
+    [`cc-${crypto.randomUUID()}`, ALIAS, status, nodeId, NET],
+  );
+}
+
+function seedSchedule(): any {
+  const scheduleId = `sched_${crypto.randomUUID()}`;
+  db.run(
+    `INSERT INTO scheduled_tasks
+       (schedule_id, network_id, name, target_node_id, target_alias, task_content, priority,
+        schedule_type, schedule_json, timezone, status, next_run_at)
+     VALUES (?1, ?2, 'doorbell', ?3, ?4, 'scheduled doorbell task', 'normal', 'interval',
+             '{"type":"interval","every_seconds":3600}', 'UTC', 'active', datetime('now', '+1 hour'))`,
+    [scheduleId, NET, NODE, ALIAS],
+  );
+  return db.get("SELECT * FROM scheduled_tasks WHERE schedule_id = ?1", scheduleId);
+}
+
+/** Subscribe like the channel server does and collect decoded SSE frames. */
+function subscribe(): { frames: any[]; close: () => Promise<void> } {
+  const res = createSSEStream(ALIAS, NET);
+  const reader = res.body!.getReader();
+  const decoder = new TextDecoder();
+  const frames: any[] = [];
+  let buffer = "";
+  let open = true;
+  (async () => {
+    while (open) {
+      const { done, value } = await reader.read();
+      if (done) break;
+      buffer += decoder.decode(value, { stream: true });
+      const blocks = buffer.split("\n\n");
+      buffer = blocks.pop() || "";
+      for (const block of blocks) {
+        const line = block.split("\n").find((l) => l.startsWith("data: "));
+        if (line) frames.push(JSON.parse(line.slice(6)));
+      }
+    }
+  })().catch(() => {});
+  return {
+    frames,
+    close: async () => {
+      open = false;
+      await reader.cancel().catch(() => {});
+    },
+  };
+}
+
+async function settle(): Promise<void> {
+  await new Promise((resolve) => setTimeout(resolve, 20));
+}
+
+beforeEach(() => {
+  cleanup();
+  seedNode();
+});
+
+afterAll(cleanup);
+
+describe("scheduler doorbell for sessions reported without node_id", () => {
+  test("idle node_id-less session is delivered and its subscriber receives new_task", async () => {
+    seedSession("idle", null);
+    const sub = subscribe();
+    try {
+      await settle();
+      const result = dispatchScheduledOccurrence(seedSchedule(), new Date().toISOString(), false);
+      await settle();
+      expect(result.status).toBe("delivered");
+      expect(db.get<{ status: string }>("SELECT status FROM scheduled_task_runs WHERE run_id = ?1", result.runId)?.status).toBe("delivered");
+      const wake = sub.frames.filter((f) => f.type === "new_task");
+      expect(wake).toHaveLength(1);
+      expect(wake[0].from).toBe("scheduler");
+      expect(wake[0].inbox_count).toBe(1);
+    } finally {
+      await sub.close();
+    }
+  });
+
+  test("an offline session is skipped even if a stale subscriber is still connected", async () => {
+    seedSession("offline", null);
+    const sub = subscribe();
+    try {
+      await settle();
+      const result = dispatchScheduledOccurrence(seedSchedule(), new Date().toISOString(), false);
+      await settle();
+      expect(result.status).toBe("skipped");
+      expect(db.get<{ error_code: string; task_id: string | null }>("SELECT error_code, task_id FROM scheduled_task_runs WHERE run_id = ?1", result.runId))
+        .toEqual({ error_code: "target_offline", task_id: null });
+      expect(sub.frames.filter((f) => f.type === "new_task")).toHaveLength(0);
+    } finally {
+      await sub.close();
+    }
+  });
+
+  test("offline session with nobody listening is skipped without a task", () => {
+    seedSession("offline", null);
+    const result = dispatchScheduledOccurrence(seedSchedule(), new Date().toISOString(), false);
+    expect(result.status).toBe("skipped");
+    expect(db.get<{ error_code: string; task_id: string | null }>("SELECT error_code, task_id FROM scheduled_task_runs WHERE run_id = ?1", result.runId))
+      .toEqual({ error_code: "target_offline", task_id: null });
+  });
+});

package/src/scheduled-failures-http.test.ts

@@ -0,0 +1,233 @@
+// #523 —— 定时任务连续失败:告诉创建者(可选自动暂停)。
+//
+// 钉住:
+//   1. 连续 5 次失败(派出去的任务以 failed 结束)→ 给创建者恰好 1 条通知(别的成员 0 条),第 6–10 次不再发;
+//      通知里有排程名、目标节点、失败次数、最近一次的错误(截断)和怎么暂停。
+//   2. 派发时就失败(目标节点不存在)同样计数。
+//   3. 中间成功一次 → 重新上膛:再连续 5 次失败才发第二条。
+//   4. 一直失败:24 小时内不重发;上次告警早于 24 小时 → 再发一条。
+//   5. 默认不自动暂停;COMMHUB_SCHEDULE_FAILURE_AUTO_PAUSE_RUNS=3 → 第 3 次失败后排程 paused、next_run_at 清空、
+//      revision+1,并通知一次。
+//   6. GET /runs 返回 consecutive_failures / failure_alert_threshold / last_failure_alert_at,
+//      任务以 failed 结束的 run 的 error_message 取自任务的回复正文。
+//
+// 跑法:cd server && COMMHUB_DB=/tmp/x.db bun test src/scheduled-failures-http.test.ts
+//       PG:COMMHUB_TEST_PG_URL=… COMMHUB_PG_EXPERIMENTAL=1(tests/test2123-hub-postgres-ladder 里注册)
+import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, test } from "bun:test";
+import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
+import { addNetworkMember, createNetworkTokenForNode, register } from "./auth.js";
+import { db } from "./db.js";
+import { dispatchScheduledOccurrence, handleScheduledTaskRequest } from "./scheduled-tasks.js";
+import { SCHEDULE_FAILING_NOTICE_KIND, failureStreak } from "./scheduled-failures.js";
+import { registerTools } from "./tools.js";
+
+const activeDbPath = process.env.COMMHUB_DB ?? (process.env.COMMHUB_TEST_PG_URL ? "postgres" : undefined);
+if (!activeDbPath) throw new Error("scheduled-failures requires COMMHUB_DB (or COMMHUB_TEST_PG_URL) before module import");
+
+const stamp = Date.now();
+const ALIAS = `fail-node-${stamp}`;
+const NODE_ID = `n_fail_${stamp}`;
+let NET = "", creatorId = "", otherId = "";
+const ENV_KEYS = ["COMMHUB_SCHEDULE_FAILURE_NOTICE_RUNS", "COMMHUB_SCHEDULE_FAILURE_RENOTICE_SEC", "COMMHUB_SCHEDULE_FAILURE_AUTO_PAUSE_RUNS"];
+const SAFETY_ERROR = "执行出错: upstream model refused the request (safety system). " + "x".repeat(600);
+
+function seedSchedule(name = "前后端分离巡检", targetNodeId = NODE_ID): any {
+  const scheduleId = `sched_${crypto.randomUUID()}`;
+  db.run(
+    `INSERT INTO scheduled_tasks
+       (schedule_id, network_id, created_by, name, target_node_id, target_alias, task_content, priority,
+        schedule_type, schedule_json, timezone, status, next_run_at)
+     VALUES (?1, ?2, ?3, ?4, ?5, ?6, 'scheduled task', 'normal', 'interval',
+             '{"type":"interval","every_seconds":120}', 'Asia/Shanghai', 'active', datetime('now', '+2 minutes'))`,
+    [scheduleId, NET, creatorId, name, targetNodeId, ALIAS],
+  );
+  return reload(scheduleId);
+}
+const reload = (scheduleId: string) => db.get<any>("SELECT * FROM scheduled_tasks WHERE schedule_id = ?1", scheduleId);
+
+let slot = 0;
+const nextSlot = () => new Date(Date.UTC(2026, 9, 4, 0, 0, 0) + (++slot) * 120_000).toISOString();
+const fire = (schedule: any, now = new Date()) => dispatchScheduledOccurrence(schedule, nextSlot(), false, now);
+
+const notices = (userId: string) => db.all<{ title: string; content: string; meta_json: string; from_session: string; severity: string }>(
+  "SELECT title, content, meta_json, from_session, severity FROM user_inbox WHERE user_id = ?1 AND network_id = ?2 AND kind = ?3 ORDER BY created_at, message_id",
+  userId, NET, SCHEDULE_FAILING_NOTICE_KIND,
+);
+const noticesFor = (userId: string, scheduleId: string) => notices(userId).filter((n) => JSON.parse(n.meta_json).schedule_failing.schedule_id === scheduleId);
+
+async function nodeReply(args: Record<string, unknown>): Promise<any> {
+  const mcp = new McpServer({ name: "sched-fail", version: "0" }) as any;
+  const handlers: Record<string, (a: any) => Promise<any>> = {};
+  const original = mcp.tool.bind(mcp);
+  mcp.tool = (name: string, ...rest: any[]) => {
+    const handler = rest.at(-1);
+    if (typeof handler === "function") handlers[name] = handler;
+    return original(name, ...rest);
+  };
+  registerTools(mcp, undefined, NET, creatorId, ALIAS, true, null);
+  const out = await handlers.send_reply(args);
+  return JSON.parse(out.content[0].text);
+}
+
+/** 派一次,节点以 failed(或 replied)回复。 */
+async function runOnce(schedule: any, outcome: "failed" | "replied" = "failed"): Promise<{ runId: string; taskId: string }> {
+  const r = fire(schedule);
+  expect(r.status).toBe("delivered");
+  const out = await nodeReply({ alias: "scheduler", text: outcome === "failed" ? SAFETY_ERROR : "done", in_reply_to: r.taskId, status: outcome, from_session: ALIAS });
+  expect(out.ok).toBe(true);
+  return { runId: r.runId, taskId: r.taskId! };
+}
+
+async function getRuns(scheduleId: string): Promise<any> {
+  const url = new URL(`http://hub.test/api/scheduled-tasks/${scheduleId}/runs`);
+  const res = await handleScheduledTaskRequest({
+    req: new Request(url, { method: "GET" }), url,
+    auth: { userId: creatorId, networkId: NET, username: "creator" },
+    isAdmin: false, isNodeToken: false, scope: { networkId: NET, networkIds: [NET] } as any,
+  });
+  expect(res!.status).toBe(200);
+  return res!.json();
+}
+
+beforeAll(() => {
+  const creator = register(`fail_creator_${stamp}`, "FailCreator123!", undefined, "seed");
+  expect(creator.ok).toBe(true);
+  NET = creator.network_id!; creatorId = creator.user!.user_id;
+  const other = register(`fail_other_${stamp}`, "FailOther123!", undefined, "seed");
+  otherId = other.user!.user_id;
+  expect(addNetworkMember(NET, otherId, "member", creatorId, { agentAccess: "all" }).ok).toBe(true);
+  expect(createNetworkTokenForNode(creatorId, NET, ALIAS).ok).toBe(true);
+  db.run("INSERT INTO nodes (node_id, node_name, alias, runtime, network_id) VALUES (?1, ?2, ?2, 'claude-code', ?3)", [NODE_ID, ALIAS, NET]);
+  db.run("INSERT INTO sessions (resume_id, alias, status, node_id, network_id) VALUES (?1, ?2, 'idle', ?3, ?4)", [`r_fail_${stamp}`, ALIAS, NODE_ID, NET]);
+});
+beforeEach(() => { for (const k of ENV_KEYS) delete process.env[k]; });
+afterEach(() => { for (const k of ENV_KEYS) delete process.env[k]; });
+afterAll(() => { db.run("DELETE FROM user_inbox WHERE network_id = ?1 AND kind = ?2", [NET, SCHEDULE_FAILING_NOTICE_KIND]); });
+
+describe("#523 consecutive-failure alert", () => {
+  test("5 failed runs → exactly one notice to the creator with name/target/count/error/how to pause; runs 6–10 stay quiet", async () => {
+    const s = seedSchedule();
+    for (let i = 0; i < 4; i++) await runOnce(s);
+    expect(noticesFor(creatorId, s.schedule_id).length).toBe(0);
+    await runOnce(s);
+    const mine = noticesFor(creatorId, s.schedule_id);
+    expect(mine.length).toBe(1);
+    expect(noticesFor(otherId, s.schedule_id).length).toBe(0);
+    const n = mine[0];
+    expect(n.from_session).toBe(ALIAS);
+    expect(n.title).toBe("定时任务连续失败");
+    expect(n.severity).toBe("error");
+    for (const frag of ["前后端分离巡检", ALIAS, "连续 5 次", "safety system", "task_failed", "暂停", s.schedule_id]) expect(n.content).toContain(frag);
+    expect(n.content).not.toContain("x".repeat(400)); // 错误被截断
+    expect(JSON.parse(n.meta_json).schedule_failing).toEqual({ reason: "failing", schedule_id: s.schedule_id, consecutive_failures: 5, error_code: "task_failed" });
+    for (let i = 0; i < 5; i++) await runOnce(s);
+    expect(noticesFor(creatorId, s.schedule_id).length).toBe(1);
+    expect(reload(s.schedule_id).status).toBe("active"); // 默认不自动暂停
+  });
+
+  test("dispatch-time failures (target node gone) count too", () => {
+    const s = seedSchedule("孤儿排程", `n_missing_${stamp}`);
+    for (let i = 0; i < 5; i++) expect(fire(s).status).toBe("failed");
+    const mine = noticesFor(creatorId, s.schedule_id);
+    expect(mine.length).toBe(1);
+    expect(mine[0].content).toContain("target_node_not_found");
+    expect(mine[0].content).toContain("The bound node no longer exists");
+  });
+
+  test("a success in between re-arms: the next 5 failures notify again", async () => {
+    const s = seedSchedule();
+    for (let i = 0; i < 5; i++) await runOnce(s);
+    expect(noticesFor(creatorId, s.schedule_id).length).toBe(1);
+    await runOnce(s, "replied");
+    expect(failureStreak(s.schedule_id).consecutiveFailures).toBe(0);
+    for (let i = 0; i < 4; i++) await runOnce(s);
+    expect(noticesFor(creatorId, s.schedule_id).length).toBe(1);
+    await runOnce(s);
+    expect(noticesFor(creatorId, s.schedule_id).length).toBe(2);
+  });
+
+  test("still failing: no repeat within 24h; one more once the last alert is older than 24h", async () => {
+    const s = seedSchedule();
+    for (let i = 0; i < 5; i++) await runOnce(s);
+    expect(noticesFor(creatorId, s.schedule_id).length).toBe(1);
+    db.run("UPDATE scheduled_tasks SET failure_alert_at = ?1 WHERE schedule_id = ?2", [new Date(Date.now() - 23 * 3_600_000).toISOString(), s.schedule_id]);
+    await runOnce(s);
+    expect(noticesFor(creatorId, s.schedule_id).length).toBe(1);
+    db.run("UPDATE scheduled_tasks SET failure_alert_at = ?1 WHERE schedule_id = ?2", [new Date(Date.now() - 25 * 3_600_000).toISOString(), s.schedule_id]);
+    await runOnce(s);
+    expect(noticesFor(creatorId, s.schedule_id).length).toBe(2);
+    await runOnce(s);
+    expect(noticesFor(creatorId, s.schedule_id).length).toBe(2);
+  });
+
+  test("COMMHUB_SCHEDULE_FAILURE_NOTICE_RUNS changes N; no creator → no notice", async () => {
+    process.env.COMMHUB_SCHEDULE_FAILURE_NOTICE_RUNS = "2";
+    const s = seedSchedule();
+    await runOnce(s);
+    expect(noticesFor(creatorId, s.schedule_id).length).toBe(0);
+    await runOnce(s);
+    expect(noticesFor(creatorId, s.schedule_id).length).toBe(1);
+
+    const orphan = seedSchedule();
+    db.run("UPDATE scheduled_tasks SET created_by = NULL WHERE schedule_id = ?1", [orphan.schedule_id]);
+    const row = reload(orphan.schedule_id);
+    for (let i = 0; i < 3; i++) await runOnce(row);
+    expect(noticesFor(creatorId, orphan.schedule_id).length).toBe(0);
+  });
+});
+
+describe("#523 auto-pause (opt-in)", () => {
+  test("off by default: 10 failures leave the schedule active", async () => {
+    const s = seedSchedule();
+    for (let i = 0; i < 10; i++) await runOnce(s);
+    const after = reload(s.schedule_id);
+    expect(after.status).toBe("active");
+    expect(after.next_run_at).not.toBeNull();
+  });
+
+  test("COMMHUB_SCHEDULE_FAILURE_AUTO_PAUSE_RUNS=3 pauses after the 3rd failure and tells the creator once", async () => {
+    process.env.COMMHUB_SCHEDULE_FAILURE_AUTO_PAUSE_RUNS = "3";
+    const s = seedSchedule();
+    await runOnce(s); await runOnce(s);
+    expect(reload(s.schedule_id).status).toBe("active");
+    await runOnce(s);
+    const after = reload(s.schedule_id);
+    expect(after.status).toBe("paused");
+    expect(after.next_run_at).toBeNull();
+    expect(after.revision).toBe(s.revision + 1);
+    const mine = noticesFor(creatorId, s.schedule_id);
+    expect(mine.length).toBe(1);
+    expect(mine[0].title).toBe("定时任务连续失败,已自动暂停");
+    expect(JSON.parse(mine[0].meta_json).schedule_failing.reason).toBe("paused");
+    // 手动 run-now 再失败两次到 N=5:同一段失败,已经通知过 → 不再发
+    await runOnce(after); await runOnce(after);
+    expect(noticesFor(creatorId, s.schedule_id).length).toBe(1);
+  });
+});
+
+describe("#523 GET /runs", () => {
+  test("exposes consecutive_failures and the failed task's reason as error_message", async () => {
+    const s = seedSchedule();
+    await runOnce(s, "replied");
+    const fails = [await runOnce(s), await runOnce(s)];
+    const body = await getRuns(s.schedule_id);
+    expect(body.ok).toBe(true);
+    expect(body.consecutive_failures).toBe(2);
+    expect(body.failure_alert_threshold).toBe(5);
+    expect(body.last_failure_alert_at).toBeNull();
+    expect(body.runs.length).toBe(3);
+    const failed = body.runs.find((r: any) => r.run_id === fails[1].runId);
+    expect(failed.status).toBe("failed");
+    expect(failed.error_code).toBe("task_failed");
+    expect(failed.error_message).toContain("safety system");
+    expect(failed.error_message.length).toBeLessThanOrEqual(500);
+    for (const k of ["scheduled_for", "created_at", "completed_at", "task_id"]) expect(k in failed).toBe(true);
+    const ok = body.runs.find((r: any) => r.status === "replied");
+    expect(ok.error_message).toBeNull();
+
+    for (let i = 0; i < 3; i++) await runOnce(s);
+    const again = await getRuns(s.schedule_id);
+    expect(again.consecutive_failures).toBe(5);
+    expect(typeof again.last_failure_alert_at).toBe("string");
+  });
+});