@sleep2agi/commhub-server 0.9.0-preview.133 → 0.9.0-preview.134

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@sleep2agi/commhub-server",
-  "version": "0.9.0-preview.133",
+  "version": "0.9.0-preview.134",
   "description": "CommHub Server \u2014 AI Agent communication hub with MCP protocol, multi-network isolation, user auth, and MCP tools (17 collaboration-core + node/provider ops tools; authoritative list: docs-site/docs/api/mcp-tools.md).",
   "type": "module",
   "main": "src/index.ts",

package/src/scheduled-stuck-http.test.ts

@@ -16,7 +16,7 @@ import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
 import { addNetworkMember, createNetworkTokenForNode, register } from "./auth.js";
 import { db } from "./db.js";
 import { dispatchScheduledOccurrence, handleScheduledTaskRequest } from "./scheduled-tasks.js";
-import { SCHEDULE_STUCK_NOTICE_KIND, stuckTimeoutMs } from "./scheduled-stuck.js";
+import { blockerTimedOut, SCHEDULE_STUCK_NOTICE_KIND, stuckTimeoutMs } from "./scheduled-stuck.js";
 import { registerTools } from "./tools.js";
 
 const activeDbPath = process.env.COMMHUB_DB ?? (process.env.COMMHUB_TEST_PG_URL ? "postgres" : undefined);
@@ -236,3 +236,53 @@ describe("#464 history clarity", () => {
     expect(skipped.blocked_by_state).toBe("not_received");
   });
 });
+
+
+describe("#1112 execution clock", () => {
+  const ts = (now: Date, minutes: number) => new Date(now.getTime() - minutes * 60_000).toISOString();
+  test("queue budget: creation over one hour but consumption under one hour keeps the exact run open", () => {
+    const s = seedSchedule();
+    const first = fire(s);
+    const now = new Date();
+    db.run("UPDATE tasks SET status = 'running', created_at = ?1, consumed_at = ?2 WHERE task_id = ?3", [ts(now, 61), ts(now, 49), first.taskId]);
+    expect(fire(s, now).status).toBe("skipped");
+    expect(taskRow(first.taskId!).status).toBe("running");
+    expect(runRow(first.runId).status).toBe("delivered");
+    expect(db.get<any>("SELECT COUNT(*) AS n FROM task_events WHERE task_id = ?1 AND to_status = 'expired'", first.taskId).n).toBe(0);
+    db.run("UPDATE tasks SET created_at = ?1, consumed_at = ?2 WHERE task_id = ?3", [ts(now, 73), ts(now, 61), first.taskId]);
+    const later = fire(s, now);
+    expect(later.status).toBe("delivered");
+    expect(taskRow(first.taskId!).status).toBe("expired");
+    expect(runRow(first.runId).error_message).toContain("since consumed_at");
+  });
+  test("queue budget: legacy execution start protects the queued task, consumption takes precedence", () => {
+    const s = seedSchedule(); const first = fire(s); const now = new Date();
+    db.run("UPDATE tasks SET status = 'running', created_at = ?1, started_at = ?2 WHERE task_id = ?3", [ts(now, 75), ts(now, 40), first.taskId]);
+    expect(fire(s, now).status).toBe("skipped");
+    db.run("UPDATE tasks SET started_at = ?1, consumed_at = ?2 WHERE task_id = ?3", [ts(now, 70), ts(now, 40), first.taskId]);
+    expect(fire(s, now).status).toBe("skipped");
+  });
+  test("submission and ACK alone do not renew queue time", () => {
+    const s = seedSchedule(); const first = fire(s); const now = new Date();
+    db.run("UPDATE tasks SET status = 'acked', created_at = ?1, runtime_submitted_at = ?2 WHERE task_id = ?3", [ts(now, 61), ts(now, 1), first.taskId]);
+    expect(fire(s, now).status).toBe("delivered");
+    expect(runRow(first.runId).error_message).toContain("since created_at");
+  });
+  test("total cap: a very late consumption cannot keep an occurrence open beyond 24 hours", () => {
+    const s = seedSchedule(); const first = fire(s); const now = new Date();
+    db.run("UPDATE tasks SET status = 'running', created_at = ?1, consumed_at = ?2 WHERE task_id = ?3", [ts(now, 1441), ts(now, 1), first.taskId]);
+    expect(fire(s, now).status).toBe("delivered");
+    expect(runRow(first.runId).status).toBe("expired");
+    expect(runRow(first.runId).error_message).toContain("created_at (total cap)");
+    expect(runRow(first.runId).error_message).toContain("limit 1440 min");
+  });
+  test("invalid, future and pre-creation evidence falls back; UTC and exact execution boundary remain stable", () => {
+    const now = new Date("2026-10-13T03:00:00Z");
+    const schedule = JSON.stringify({ type: "interval", every_seconds: 60 });
+    for (const consumed_at of [null, "invalid", "2026-10-14T00:00:00Z", "2026-10-12T00:00:00Z"]) {
+      expect(blockerTimedOut({ created_at: "2026-10-13 01:00:00", consumed_at }, schedule, now)).toBe(true);
+    }
+    expect(blockerTimedOut({ created_at: "2026-10-13 01:00:00", consumed_at: "2026-10-13 02:00:01" }, schedule, now)).toBe(false);
+    expect(blockerTimedOut({ created_at: "2026-10-13 01:00:00", consumed_at: "2026-10-13 02:00:00" }, schedule, now)).toBe(true);
+  });
+});

package/src/scheduled-stuck.ts

@@ -14,6 +14,8 @@
 //    恰好一条通知。
 //
 // 默认值(PR 里有推导):N=3;超时 = clamp(6 × 间隔, 下限 1 小时, 上限 24 小时)。
+// #1112:从 consumed_at 起算,旧节点回落 started_at,未执行回落 created_at;
+//       自创建时间的总保护上限仍为 cap,节点心跳/提交/ACK 不刷新执行时钟。
 //   下限 1 小时:一轮 agent 任务正常要跑几十分钟,不能因为排程间隔短就把还在干活的任务掐掉。
 //   上限 24 小时:Hub 本来就给定时任务 24 小时的 expires_at(没被取走时由巡检过期),开着超过一天的
 //   定时任务不是「还在做」。6 × 间隔:间隔长的排程(每小时一次)给它 6 小时,而不是 1 小时。
@@ -54,20 +56,44 @@ export function stuckTimeoutMs(scheduleJson: string): number {
   return Math.min(cap, Math.max(floor, factor * nominalIntervalSeconds(scheduleJson))) * 1000;
 }
 
-type Blocker = { task_id: string; created_at: string; status: string };
+export type StuckBlockerClock = {
+  created_at: string;
+  consumed_at?: string | null;
+  started_at?: string | null;
+};
+
+/** #1105: queue time is not execution time. Submission/ACK/node heartbeat are not consumption. */
+export function blockerTimeoutClock(blocker: StuckBlockerClock, scheduleJson: string, now: Date): { ageMs: number; limitMs: number; source: string } {
+  const created = parseDbTimestampMs(blocker.created_at);
+  const nowMs = now.getTime();
+  const floor = envNumber("COMMHUB_SCHEDULE_STUCK_TIMEOUT_FLOOR_SEC", 3_600, 1);
+  const capMs = Math.max(floor, envNumber("COMMHUB_SCHEDULE_STUCK_TIMEOUT_CAP_SEC", 86_400, 1)) * 1000;
+  // Even a very late consumption cannot hold this occurrence beyond the total cap.
+  if (Number.isFinite(created) && nowMs - created >= capMs) {
+    return { ageMs: nowMs - created, limitMs: capMs, source: "created_at (total cap)" };
+  }
+  for (const source of ["consumed_at", "started_at"] as const) {
+    const value = blocker[source];
+    if (!value) continue;
+    const at = parseDbTimestampMs(value);
+    if (Number.isFinite(at) && at <= nowMs && (!Number.isFinite(created) || at >= created)) {
+      return { ageMs: nowMs - at, limitMs: stuckTimeoutMs(scheduleJson), source };
+    }
+  }
+  return { ageMs: nowMs - created, limitMs: stuckTimeoutMs(scheduleJson), source: "created_at" };
+}
 
 /** 挡路的任务开了多久已经到超时 → true。读不出时间 = 不超时(宁可多挡一次,不错杀)。 */
-export function blockerTimedOut(blocker: Pick<Blocker, "created_at">, scheduleJson: string, now: Date): boolean {
-  const started = parseDbTimestampMs(blocker.created_at);
-  if (!Number.isFinite(started)) return false;
-  return now.getTime() - started >= stuckTimeoutMs(scheduleJson);
+export function blockerTimedOut(blocker: StuckBlockerClock, scheduleJson: string, now: Date): boolean {
+  const clock = blockerTimeoutClock(blocker, scheduleJson, now);
+  return Number.isFinite(clock.ageMs) && clock.ageMs >= clock.limitMs;
 }
 
 /**
  * 在调用方的事务里按超时结束挡路的任务。只动还开着的任务(条件写),返回是否真的结束了它。
  * 与 TTL 巡检(server.ts patrolExpiredTasks)同一套写法:status=expired、收件箱行置 acked、镜像到 run。
  */
-export function expireStuckBlocker(taskId: string, networkId: string, ageMs: number, limitMs: number): boolean {
+export function expireStuckBlocker(taskId: string, networkId: string, ageMs: number, limitMs: number, source = "created_at"): boolean {
   const changed = db.run(
     `UPDATE tasks SET status = 'expired', completed_at = datetime('now')
       WHERE task_id = ?1 AND network_id = ?2 AND status IN ('created', 'delivered', 'acked', 'running')`,
@@ -78,7 +104,7 @@ export function expireStuckBlocker(taskId: string, networkId: string, ageMs: num
   syncScheduledRunForTask(taskId, networkId);
   db.run(
     "UPDATE scheduled_task_runs SET error_message = ?1 WHERE task_id = ?2 AND network_id = ?3",
-    [`timed out by the scheduler: open for ${Math.round(ageMs / 60_000)} min (limit ${Math.round(limitMs / 60_000)} min); the next occurrence was dispatched`, taskId, networkId],
+    [`timed out by the scheduler: ${Math.round(ageMs / 60_000)} min since ${source} (limit ${Math.round(limitMs / 60_000)} min); the next occurrence was dispatched`, taskId, networkId],
   );
   return true;
 }
@@ -127,12 +153,12 @@ export function stuckNoticeText(n: StuckNotice): { title: string; text: string }
     return {
       title: "定时任务被卡住",
       text: `定时任务「${n.scheduleName}」被上一次的执行卡住了:任务 ${n.taskId} 于 ${started} 派给节点 ${n.alias},到现在还没有完成,之后的 ${n.skips} 次执行都被跳过。\n\n`
-        + `它开着超过 ${fmtDuration(n.timeoutMs)} 后会被自动按超时结束,下一次照常派发。等不及可以在任务列表里取消这条任务。`,
+        + `从实际开始执行起超过 ${fmtDuration(n.timeoutMs)} 后会被自动按超时结束;尚未开始的任务从派发起计时,所有执行仍受总时限保护。下一次照常派发。等不及可以在任务列表里取消这条任务。`,
     };
   }
   return {
     title: "定时任务已超时放行",
-    text: `定时任务「${n.scheduleName}」上一次的任务 ${n.taskId}(${started} 派给节点 ${n.alias})开着超过 ${fmtDuration(n.timeoutMs)} 还没有完成,已按超时结束,不再挡后面的执行。`,
+    text: `定时任务「${n.scheduleName}」上一次的任务 ${n.taskId}(${started} 派给节点 ${n.alias})超过 ${fmtDuration(n.timeoutMs)} 的执行或总保护时限还没有完成,已按超时结束,不再挡后面的执行。`,
   };
 }
 

package/src/scheduled-tasks.ts

@@ -8,8 +8,7 @@ import { canMessageAgent, type AgentRef } from "./agent-access.js";
 import { dispatchVerdict } from "./node-permissions.js";
 import { hasSubscribers, pushEvent, pushNetworkObserverEvent } from "./push.js";
 import { SCHEDULED_TASK_STORAGE_SELECT } from "./rest-projections.js";
-import { parseDbTimestampMs } from "./db-timestamp.js";
-import { blockerTimedOut, expireStuckBlocker, sendStuckNotice, skipsBlockedBy, stuckNoticeSkips, stuckTimeoutMs, type StuckNotice } from "./scheduled-stuck.js";
+import { blockerTimedOut, blockerTimeoutClock, expireStuckBlocker, sendStuckNotice, skipsBlockedBy, stuckNoticeSkips, stuckTimeoutMs, type StuckNotice } from "./scheduled-stuck.js";
 import { evaluateScheduleFailures, failureSummary } from "./scheduled-failures.js";
 import { scheduledTargetOffline, sendOfflineNotice, type OfflineNotice } from "./scheduled-offline.js";
 
@@ -413,8 +412,8 @@ export function dispatchScheduledOccurrence(row: ScheduledRow, scheduledFor: str
       // received:节点那边有没有把这条任务取走过(inbox 行被 ack,或任务已进 acked/running)。
       // 跳过记录带上它,客户端才分得清「节点还没收到」和「节点在处理」—— 前者是投递故障,
       // 后者才是真的上一次还没做完。
-      const findOpen = () => db.get<{ task_id: string; received: number; created_at: string; status: string }>(
-        `SELECT r.task_id, t.created_at, t.status,
+      const findOpen = () => db.get<{ task_id: string; received: number; created_at: string; consumed_at: string | null; started_at: string | null; status: string }>(
+        `SELECT r.task_id, t.created_at, t.consumed_at, t.started_at, t.status,
                 CASE WHEN t.status IN ('acked', 'running')
                        OR EXISTS (SELECT 1 FROM inbox i WHERE (i.task_id = r.task_id OR i.id = r.task_id) AND i.acked = 1)
                      THEN 1 ELSE 0 END AS received
@@ -428,9 +427,8 @@ export function dispatchScheduledOccurrence(row: ScheduledRow, scheduledFor: str
       // #464 — a blocker open past the stuck timeout is ended as `expired` (the existing terminal state) and
       // this occurrence dispatches normally. Bounded: each pass ends one exact open task or stops.
       for (let guard = 0; open && guard < 20 && blockerTimedOut(open, row.schedule_json, advanceAfter); guard++) {
-        const limitMs = stuckTimeoutMs(row.schedule_json);
-        const ageMs = advanceAfter.getTime() - parseDbTimestampMs(open.created_at);
-        if (!expireStuckBlocker(open.task_id, row.network_id, ageMs, limitMs)) break;
+        const { ageMs, limitMs, source } = blockerTimeoutClock(open, row.schedule_json, advanceAfter);
+        if (!expireStuckBlocker(open.task_id, row.network_id, ageMs, limitMs, source)) break;
         timedOut.push({ taskId: open.task_id, fromStatus: open.status });
         // Each stuck episode gets exactly one notice: if the stuck notice already went out (N skips), stay quiet.
         if (skipsBlockedBy(row.schedule_id, open.task_id) < stuckNoticeSkips()) {