@sleep2agi/commhub-server 0.9.0-preview.132 → 0.9.0-preview.134
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/package.json +1 -1
- package/src/node-caller-identity-http.test.ts +109 -0
- package/src/node-daemon-bindings.ts +8 -3
- package/src/node-token-identity.ts +35 -0
- package/src/rename.ts +11 -3
- package/src/scheduled-stuck-http.test.ts +51 -1
- package/src/scheduled-stuck.ts +35 -9
- package/src/scheduled-tasks.ts +5 -7
- package/src/stop-delete-node.test.ts +0 -1
- package/src/tools.ts +6 -24
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@sleep2agi/commhub-server",
|
|
3
|
-
"version": "0.9.0-preview.
|
|
3
|
+
"version": "0.9.0-preview.134",
|
|
4
4
|
"description": "CommHub Server \u2014 AI Agent communication hub with MCP protocol, multi-network isolation, user auth, and MCP tools (17 collaboration-core + node/provider ops tools; authoritative list: docs-site/docs/api/mcp-tools.md).",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "src/index.ts",
|
|
@@ -288,6 +288,115 @@ const report = (tok: Tok, alias: string, nodeId: string, networkId: string, extr
|
|
|
288
288
|
...extra,
|
|
289
289
|
});
|
|
290
290
|
|
|
291
|
+
function lifecycleFixture(key: string) {
|
|
292
|
+
const daemonId = `node_life_daemon_${key}_${stamp}`, daemonAlias = `life-daemon-${key}-${stamp}`;
|
|
293
|
+
const childId = `node_life_${key}_${stamp}`, childAlias = `life-child-${key}-${stamp}`;
|
|
294
|
+
seedNode(daemonId, daemonAlias, NET, ownerId); seedNode(childId, childAlias, NET, ownerId);
|
|
295
|
+
seedSession(daemonId, daemonAlias, NET);
|
|
296
|
+
db.run("UPDATE nodes SET hostname='lifecycle-host',config_snapshot=?1 WHERE node_id=?2",
|
|
297
|
+
[JSON.stringify({ role: "host_supervisor", daemon_capabilities: { adopt_capable: true } }), daemonId]);
|
|
298
|
+
db.run("UPDATE nodes SET hostname='lifecycle-host',config_snapshot='{}' WHERE node_id=?1", [childId]);
|
|
299
|
+
db.run("UPDATE sessions SET last_seen_at=datetime('now') WHERE node_id=?1", [daemonId]);
|
|
300
|
+
return { daemonId, daemonAlias, childId, childAlias };
|
|
301
|
+
}
|
|
302
|
+
|
|
303
|
+
test("token lifecycle: adoption rejects a different node's same-name credential", async () => {
|
|
304
|
+
const f = lifecycleFixture("adopt-denied");
|
|
305
|
+
const otherNode = `n_life_other_${stamp}`;
|
|
306
|
+
seedNode(otherNode, `life-other-${stamp}`, NET, outsiderId);
|
|
307
|
+
mint(outsiderId, NET, f.daemonAlias, 2, otherNode);
|
|
308
|
+
mint(outsiderId, NET, f.daemonAlias, 0, null);
|
|
309
|
+
const result = await tool(ownerToken, "request_adopt_node", { node_id: f.childId, daemon_node_id: f.daemonId, workdir: "/workspace/fixture" });
|
|
310
|
+
expect(result.error).toBe("daemon_offline");
|
|
311
|
+
expect(db.get("SELECT request_id FROM node_daemon_bindings WHERE node_id=?1", f.childId)).toBeNull();
|
|
312
|
+
});
|
|
313
|
+
|
|
314
|
+
for (const shape of ["bound", "unbound"] as const) {
|
|
315
|
+
test(`token lifecycle: adoption accepts verified ${shape} credentials`, async () => {
|
|
316
|
+
const f = lifecycleFixture(`adopt-${shape}`);
|
|
317
|
+
mint(ownerId, NET, shape === "bound" ? victimAlias : f.daemonAlias, shape === "bound" ? 2 : 0, shape === "bound" ? f.daemonId : null);
|
|
318
|
+
const result = await tool(ownerToken, "request_adopt_node", { node_id: f.childId, daemon_node_id: f.daemonId, workdir: "/workspace/fixture" });
|
|
319
|
+
expect(result.ok).toBe(true);
|
|
320
|
+
});
|
|
321
|
+
}
|
|
322
|
+
|
|
323
|
+
test("token lifecycle: another node's heartbeat cannot make the daemon ready", async () => {
|
|
324
|
+
const f=lifecycleFixture("heartbeat-denied");
|
|
325
|
+
mint(ownerId,NET,f.daemonAlias,2,f.daemonId);
|
|
326
|
+
const other=`node_life_heartbeat_other_${stamp}`;
|
|
327
|
+
seedNode(other,f.daemonAlias,NET,outsiderId);
|
|
328
|
+
db.run("DELETE FROM sessions WHERE node_id=?1",[f.daemonId]);
|
|
329
|
+
seedSession(other,f.daemonAlias,NET);
|
|
330
|
+
db.run("UPDATE sessions SET last_seen_at=datetime('now') WHERE node_id=?1",[other]);
|
|
331
|
+
const result=await tool(ownerToken,"request_adopt_node",{node_id:f.childId,daemon_node_id:f.daemonId,workdir:"/workspace/fixture"});
|
|
332
|
+
expect(result.error).toBe("daemon_offline");
|
|
333
|
+
});
|
|
334
|
+
|
|
335
|
+
test("token lifecycle: a unique legacy null-ID heartbeat remains supported", async () => {
|
|
336
|
+
const f=lifecycleFixture("heartbeat-legacy");
|
|
337
|
+
mint(ownerId,NET,f.daemonAlias,0,null);
|
|
338
|
+
db.run("UPDATE sessions SET node_id=NULL WHERE node_id=?1",[f.daemonId]);
|
|
339
|
+
expect((await tool(ownerToken,"request_adopt_node",{node_id:f.childId,daemon_node_id:f.daemonId,workdir:"/workspace/fixture"})).ok).toBe(true);
|
|
340
|
+
});
|
|
341
|
+
|
|
342
|
+
for (const shape of ["frozen", "legacy null", "wrong frozen", "same alias rebuild"] as const) {
|
|
343
|
+
test(`token lifecycle: delete ${shape} revokes only the target credential`, async () => {
|
|
344
|
+
const key = shape.replaceAll(" ", "-"); const f = lifecycleFixture(key);
|
|
345
|
+
const daemonTok = mint(ownerId, NET, f.daemonAlias, 2, f.daemonId);
|
|
346
|
+
const wrong = mint(outsiderId, NET, f.childAlias, 0, null);
|
|
347
|
+
const childTok = mint(ownerId, NET, shape === "frozen" ? victimAlias : f.childAlias, 2, f.childId);
|
|
348
|
+
db.run(`INSERT INTO node_create_requests(request_id,daemon_node_id,child_name,child_node_id,network_id,runtime,flags_json,env_keys,status,child_token_id,created_at,created_by_token)
|
|
349
|
+
VALUES(?1,?2,?3,?4,?5,'fixture','{}','[]','started',?6,?7,'fixture')`,
|
|
350
|
+
[`cr_${f.childId.slice(5)}`, f.daemonId, f.childAlias, f.childId, NET, childTok.tokenId, Date.now()]);
|
|
351
|
+
const dispatched = await tool(ownerToken, "delete_node", { child_node_id: f.childId, daemon_node_id: f.daemonId, network_id: NET, confirm_alias: f.childAlias });
|
|
352
|
+
expect(dispatched.ok).toBe(true);
|
|
353
|
+
expect(db.get<{ child_token_id: string }>("SELECT child_token_id FROM node_stop_requests WHERE request_id=?1", dispatched.request_id)?.child_token_id).toBe(childTok.tokenId);
|
|
354
|
+
if (shape === "legacy null" || shape === "wrong frozen") db.run("UPDATE node_stop_requests SET child_token_id=?1 WHERE request_id=?2", [shape === "wrong frozen" ? wrong.tokenId : null, dispatched.request_id]);
|
|
355
|
+
let rebuilt: Tok | null = null;
|
|
356
|
+
if (shape === "same alias rebuild") {
|
|
357
|
+
db.run("DELETE FROM nodes WHERE node_id=?1", [f.childId]);
|
|
358
|
+
const newId = `n_life_rebuilt_${stamp}`;seedNode(newId, f.childAlias, NET, ownerId);
|
|
359
|
+
rebuilt = mint(ownerId, NET, f.childAlias, 2, newId);
|
|
360
|
+
}
|
|
361
|
+
const ack = await tool(daemonTok.token, "ack_stop_request", { request_id: dispatched.request_id, status: "stopped" });
|
|
362
|
+
expect(ack.ok).toBe(true);
|
|
363
|
+
const revoked = (id: string) => db.get<{ revoked_at: string | null }>("SELECT revoked_at FROM api_tokens WHERE token_id=?1", id)?.revoked_at;
|
|
364
|
+
expect(revoked(childTok.tokenId)).not.toBeNull();
|
|
365
|
+
expect(revoked(wrong.tokenId)).toBeNull();
|
|
366
|
+
if (rebuilt) expect(revoked(rebuilt.tokenId)).toBeNull();
|
|
367
|
+
expect(db.get("SELECT node_id FROM nodes WHERE node_id=?1", f.childId)).toBeNull();
|
|
368
|
+
});
|
|
369
|
+
}
|
|
370
|
+
|
|
371
|
+
test("token lifecycle: rename updates verified labels and preserves unrelated credentials", async () => {
|
|
372
|
+
const alias = `life-rename-${stamp}`, nodeId = `n_life_rename_${stamp}`, target = `life-renamed-${stamp}`;
|
|
373
|
+
seedNode(nodeId, alias, NET, ownerId);seedSession(nodeId, alias, NET);
|
|
374
|
+
const unrelated = mint(outsiderId, NET, alias, 0, null);
|
|
375
|
+
const bound = mint(ownerId, NET, victimAlias, 2, nodeId);
|
|
376
|
+
const legacy = mint(ownerId, NET, alias, 0, null);
|
|
377
|
+
const foreign = mint(outsiderId, NET2, alias, 0, null);
|
|
378
|
+
const before = [unrelated,foreign].map(t => tokenName(t.tokenId));
|
|
379
|
+
const prepare = await api(ownerToken, "POST", "/api/node-rename/prepare", { network_id: NET, old_alias: alias, new_alias: target });
|
|
380
|
+
expect(prepare.body.ok).toBe(true);
|
|
381
|
+
expect((await api(ownerToken, "POST", "/api/node-rename/commit", { txn_id: prepare.body.txn_id })).body.ok).toBe(true);
|
|
382
|
+
expect(tokenName(bound.tokenId)).toBe(`node:${target}`);
|
|
383
|
+
expect(tokenName(legacy.tokenId)).toBe(`node:${target}`);
|
|
384
|
+
expect([unrelated,foreign].map(t => tokenName(t.tokenId))).toEqual(before);
|
|
385
|
+
});
|
|
386
|
+
|
|
387
|
+
test("token lifecycle: rename refuses ambiguous node rows before and after prepare", async () => {
|
|
388
|
+
const alias=`life-dup-rename-${stamp}`, nodeId=`n_life_dupr_${stamp}`, target=`life-dup-target-${stamp}`;
|
|
389
|
+
seedNode(nodeId,alias,NET,ownerId);seedSession(nodeId,alias,NET);
|
|
390
|
+
const mine=mint(ownerId,NET,alias,2,nodeId);
|
|
391
|
+
const prepared=await api(ownerToken,"POST","/api/node-rename/prepare",{network_id:NET,old_alias:alias,new_alias:target});expect(prepared.body.ok).toBe(true);
|
|
392
|
+
seedNode(`n_life_dupo_${stamp}`,alias,NET,outsiderId);
|
|
393
|
+
expect((await api(ownerToken,"POST","/api/node-rename/commit",{txn_id:prepared.body.txn_id})).body.error).toBe("node_alias_ambiguous");
|
|
394
|
+
expect((await api(ownerToken,"POST","/api/node-rename/prepare",{network_id:NET,old_alias:alias,new_alias:target})).body.error).toBe("node_alias_ambiguous");
|
|
395
|
+
expect(tokenName(mine.tokenId)).toBe(`node:${alias}`);
|
|
396
|
+
expect(db.get("SELECT status FROM rename_txn WHERE txn_id=?1",prepared.body.txn_id)).toEqual({status:"prepared"});
|
|
397
|
+
await api(ownerToken,"POST","/api/node-rename/abort",{txn_id:prepared.body.txn_id});
|
|
398
|
+
});
|
|
399
|
+
|
|
291
400
|
beforeAll(async () => {
|
|
292
401
|
process.env.COMMHUB_UPLOADS_DIR = join(dir, "uploads");
|
|
293
402
|
process.env.HOST = "127.0.0.1";
|
|
@@ -3,6 +3,7 @@ import { getUserNetworkRole } from "./auth.js";
|
|
|
3
3
|
import { pushEvent } from "./push.js";
|
|
4
4
|
import { canRestWriteNetwork } from "./network-scope.js";
|
|
5
5
|
import { parseDbTimestampMs } from "./db-timestamp.js";
|
|
6
|
+
import { activeNodeTokenIds } from "./node-token-identity.js";
|
|
6
7
|
|
|
7
8
|
type Node = { node_id: string; alias: string; network_id: string; owner_user_id: string | null; hostname: string | null; config_snapshot: string | null };
|
|
8
9
|
type Binding = { request_id: string; network_id: string; node_id: string; daemon_node_id: string; workdir: string; requested_by: string; status: string; error: string | null };
|
|
@@ -41,11 +42,15 @@ function daemonReady(daemon: Node | null, node: Node): string | null {
|
|
|
41
42
|
const snap = snapshot(daemon);
|
|
42
43
|
if (snap.role !== "host_supervisor" || snap.daemon_capabilities?.adopt_capable !== true) return "daemon_not_adopt_capable";
|
|
43
44
|
if (!node.hostname || daemon.hostname !== node.hostname) return "hostname_mismatch";
|
|
44
|
-
const session = db.get<{ last_seen_at: string }>(
|
|
45
|
+
const session = db.get<{ last_seen_at: string }>(
|
|
46
|
+
`SELECT last_seen_at FROM sessions WHERE alias=?1 AND network_id=?2
|
|
47
|
+
AND (node_id=?3 OR (node_id IS NULL AND NOT EXISTS (
|
|
48
|
+
SELECT 1 FROM nodes n WHERE n.network_id=?2 AND n.alias=?1 AND n.node_id!=?3)))`,
|
|
49
|
+
daemon.alias, daemon.network_id, daemon.node_id,
|
|
50
|
+
);
|
|
45
51
|
const seen = parseDbTimestampMs(session?.last_seen_at ?? "");
|
|
46
52
|
if (!Number.isFinite(seen) || Date.now() - seen > 300_000) return "daemon_offline";
|
|
47
|
-
|
|
48
|
-
return token ? null : "daemon_offline";
|
|
53
|
+
return activeNodeTokenIds(daemon.node_id, daemon.network_id).length ? null : "daemon_offline";
|
|
49
54
|
}
|
|
50
55
|
|
|
51
56
|
export function requestAdopt(ctx: Human, args: { node_id: string; daemon_node_id: string; workdir: string }) {
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
import { db } from "./db.js";
|
|
2
|
+
import { resolveNodeCaller } from "./create-node.js";
|
|
3
|
+
|
|
4
|
+
/** Active credentials belonging to an immutable node, including verified legacy unbound tokens. */
|
|
5
|
+
export function activeNodeTokenIds(nodeId: string, networkId: string): string[] {
|
|
6
|
+
const node = db.get<{ alias: string }>("SELECT alias FROM nodes WHERE node_id=?1 AND network_id=?2", nodeId, networkId);
|
|
7
|
+
if (!node) return [];
|
|
8
|
+
const tokens = db.all<{ token_id: string }>(
|
|
9
|
+
`SELECT token_id FROM api_tokens WHERE network_id=?1 AND scope='network' AND revoked_at IS NULL
|
|
10
|
+
AND (bound_node_id=?2 OR (bound_node_id IS NULL AND name=?3)) ORDER BY token_id`,
|
|
11
|
+
networkId, nodeId, `node:${node.alias}`,
|
|
12
|
+
);
|
|
13
|
+
return tokens.filter(token => {
|
|
14
|
+
const caller = resolveNodeCaller(token.token_id);
|
|
15
|
+
return caller.ok && caller.kind === "node" && caller.nodeId === nodeId && caller.networkId === networkId;
|
|
16
|
+
}).map(token => token.token_id);
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
/** Preserve the dispatch-time freeze; old requests without it use verified node identity. */
|
|
20
|
+
export function revokeNodeTokens(nodeId: string, networkId: string, frozenTokenId: string | null): void {
|
|
21
|
+
let ids: string[];
|
|
22
|
+
if (frozenTokenId) {
|
|
23
|
+
const token = db.get<{ bound_node_id: string | null }>(
|
|
24
|
+
"SELECT bound_node_id FROM api_tokens WHERE token_id=?1 AND network_id=?2 AND scope='network' AND revoked_at IS NULL",
|
|
25
|
+
frozenTokenId, networkId,
|
|
26
|
+
);
|
|
27
|
+
// An immutable binding remains sufficient for revocation even after its node row was removed.
|
|
28
|
+
const caller = token && !token.bound_node_id ? resolveNodeCaller(frozenTokenId) : null;
|
|
29
|
+
ids = token?.bound_node_id === nodeId || (caller?.ok && caller.kind === "node" && caller.nodeId === nodeId && caller.networkId === networkId)
|
|
30
|
+
? [frozenTokenId] : activeNodeTokenIds(nodeId, networkId);
|
|
31
|
+
} else {
|
|
32
|
+
ids = activeNodeTokenIds(nodeId, networkId);
|
|
33
|
+
}
|
|
34
|
+
for (const id of ids) db.run("UPDATE api_tokens SET revoked_at=datetime('now') WHERE token_id=?1 AND network_id=?2 AND revoked_at IS NULL", [id, networkId]);
|
|
35
|
+
}
|
package/src/rename.ts
CHANGED
|
@@ -19,6 +19,7 @@ import { canSeeAgent, isAgentRestricted } from "./agent-access";
|
|
|
19
19
|
import { pushEvent } from "./push";
|
|
20
20
|
import { eventBus } from "./event_bus";
|
|
21
21
|
import { aliasControlCharacter, describeAliasControlCharacter } from "./node-alias-control";
|
|
22
|
+
import { activeNodeTokenIds } from "./node-token-identity.js";
|
|
22
23
|
|
|
23
24
|
export interface RenameResult {
|
|
24
25
|
ok: boolean;
|
|
@@ -127,6 +128,9 @@ export function prepareRename(
|
|
|
127
128
|
};
|
|
128
129
|
}
|
|
129
130
|
if (oldAlias === newAlias) return { ok: false, error: "old and new alias are identical" };
|
|
131
|
+
if (db.all("SELECT node_id FROM nodes WHERE network_id=?1 AND alias=?2", networkId, oldAlias).length > 1) {
|
|
132
|
+
return { ok: false, error: "node_alias_ambiguous" };
|
|
133
|
+
}
|
|
130
134
|
|
|
131
135
|
// old-alias must exist as a session in this network
|
|
132
136
|
const oldSession = db.get<any>(
|
|
@@ -167,6 +171,10 @@ export function commitRename(userId: string, txnId: string): RenameResult {
|
|
|
167
171
|
if (txn.status === "committed") return { ok: true, txn_id: txnId }; // idempotent
|
|
168
172
|
if (txn.status === "aborted") return { ok: false, error: "rename transaction was aborted" };
|
|
169
173
|
if (!hasWriteAccess(userId, txn.network_id)) return { ok: false, error: "no write access to this network" };
|
|
174
|
+
const nodes = db.all<{ node_id: string }>("SELECT node_id FROM nodes WHERE network_id=?1 AND alias=?2", txn.network_id, txn.old_alias);
|
|
175
|
+
if (nodes.length > 1) return { ok: false, error: "node_alias_ambiguous" };
|
|
176
|
+
// Resolve labels before changing the node alias, so legacy unbound ownership remains verifiable.
|
|
177
|
+
const tokenIds = nodes[0] ? activeNodeTokenIds(nodes[0].node_id, txn.network_id) : [];
|
|
170
178
|
|
|
171
179
|
// Defense-in-depth: new-alias still free in both registries (prepare already CAS'd,
|
|
172
180
|
// but a parallel direct registration could have raced in).
|
|
@@ -182,9 +190,9 @@ export function commitRename(userId: string, txnId: string): RenameResult {
|
|
|
182
190
|
db.run(
|
|
183
191
|
"UPDATE nodes SET alias = ?1, updated_at = datetime('now') WHERE network_id = ?2 AND alias = ?3",
|
|
184
192
|
[txn.new_alias, txn.network_id, txn.old_alias]);
|
|
185
|
-
db.run(
|
|
186
|
-
"UPDATE api_tokens SET name = ?1 WHERE
|
|
187
|
-
[`node:${txn.new_alias}`,
|
|
193
|
+
for (const tokenId of tokenIds) db.run(
|
|
194
|
+
"UPDATE api_tokens SET name = ?1 WHERE token_id = ?2 AND network_id = ?3",
|
|
195
|
+
[`node:${txn.new_alias}`, tokenId, txn.network_id]);
|
|
188
196
|
db.run(
|
|
189
197
|
"UPDATE rename_txn SET status = 'committed', committed_at = datetime('now') WHERE txn_id = ?1",
|
|
190
198
|
[txnId]);
|
|
@@ -16,7 +16,7 @@ import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
|
|
|
16
16
|
import { addNetworkMember, createNetworkTokenForNode, register } from "./auth.js";
|
|
17
17
|
import { db } from "./db.js";
|
|
18
18
|
import { dispatchScheduledOccurrence, handleScheduledTaskRequest } from "./scheduled-tasks.js";
|
|
19
|
-
import { SCHEDULE_STUCK_NOTICE_KIND, stuckTimeoutMs } from "./scheduled-stuck.js";
|
|
19
|
+
import { blockerTimedOut, SCHEDULE_STUCK_NOTICE_KIND, stuckTimeoutMs } from "./scheduled-stuck.js";
|
|
20
20
|
import { registerTools } from "./tools.js";
|
|
21
21
|
|
|
22
22
|
const activeDbPath = process.env.COMMHUB_DB ?? (process.env.COMMHUB_TEST_PG_URL ? "postgres" : undefined);
|
|
@@ -236,3 +236,53 @@ describe("#464 history clarity", () => {
|
|
|
236
236
|
expect(skipped.blocked_by_state).toBe("not_received");
|
|
237
237
|
});
|
|
238
238
|
});
|
|
239
|
+
|
|
240
|
+
|
|
241
|
+
describe("#1112 execution clock", () => {
|
|
242
|
+
const ts = (now: Date, minutes: number) => new Date(now.getTime() - minutes * 60_000).toISOString();
|
|
243
|
+
test("queue budget: creation over one hour but consumption under one hour keeps the exact run open", () => {
|
|
244
|
+
const s = seedSchedule();
|
|
245
|
+
const first = fire(s);
|
|
246
|
+
const now = new Date();
|
|
247
|
+
db.run("UPDATE tasks SET status = 'running', created_at = ?1, consumed_at = ?2 WHERE task_id = ?3", [ts(now, 61), ts(now, 49), first.taskId]);
|
|
248
|
+
expect(fire(s, now).status).toBe("skipped");
|
|
249
|
+
expect(taskRow(first.taskId!).status).toBe("running");
|
|
250
|
+
expect(runRow(first.runId).status).toBe("delivered");
|
|
251
|
+
expect(db.get<any>("SELECT COUNT(*) AS n FROM task_events WHERE task_id = ?1 AND to_status = 'expired'", first.taskId).n).toBe(0);
|
|
252
|
+
db.run("UPDATE tasks SET created_at = ?1, consumed_at = ?2 WHERE task_id = ?3", [ts(now, 73), ts(now, 61), first.taskId]);
|
|
253
|
+
const later = fire(s, now);
|
|
254
|
+
expect(later.status).toBe("delivered");
|
|
255
|
+
expect(taskRow(first.taskId!).status).toBe("expired");
|
|
256
|
+
expect(runRow(first.runId).error_message).toContain("since consumed_at");
|
|
257
|
+
});
|
|
258
|
+
test("queue budget: legacy execution start protects the queued task, consumption takes precedence", () => {
|
|
259
|
+
const s = seedSchedule(); const first = fire(s); const now = new Date();
|
|
260
|
+
db.run("UPDATE tasks SET status = 'running', created_at = ?1, started_at = ?2 WHERE task_id = ?3", [ts(now, 75), ts(now, 40), first.taskId]);
|
|
261
|
+
expect(fire(s, now).status).toBe("skipped");
|
|
262
|
+
db.run("UPDATE tasks SET started_at = ?1, consumed_at = ?2 WHERE task_id = ?3", [ts(now, 70), ts(now, 40), first.taskId]);
|
|
263
|
+
expect(fire(s, now).status).toBe("skipped");
|
|
264
|
+
});
|
|
265
|
+
test("submission and ACK alone do not renew queue time", () => {
|
|
266
|
+
const s = seedSchedule(); const first = fire(s); const now = new Date();
|
|
267
|
+
db.run("UPDATE tasks SET status = 'acked', created_at = ?1, runtime_submitted_at = ?2 WHERE task_id = ?3", [ts(now, 61), ts(now, 1), first.taskId]);
|
|
268
|
+
expect(fire(s, now).status).toBe("delivered");
|
|
269
|
+
expect(runRow(first.runId).error_message).toContain("since created_at");
|
|
270
|
+
});
|
|
271
|
+
test("total cap: a very late consumption cannot keep an occurrence open beyond 24 hours", () => {
|
|
272
|
+
const s = seedSchedule(); const first = fire(s); const now = new Date();
|
|
273
|
+
db.run("UPDATE tasks SET status = 'running', created_at = ?1, consumed_at = ?2 WHERE task_id = ?3", [ts(now, 1441), ts(now, 1), first.taskId]);
|
|
274
|
+
expect(fire(s, now).status).toBe("delivered");
|
|
275
|
+
expect(runRow(first.runId).status).toBe("expired");
|
|
276
|
+
expect(runRow(first.runId).error_message).toContain("created_at (total cap)");
|
|
277
|
+
expect(runRow(first.runId).error_message).toContain("limit 1440 min");
|
|
278
|
+
});
|
|
279
|
+
test("invalid, future and pre-creation evidence falls back; UTC and exact execution boundary remain stable", () => {
|
|
280
|
+
const now = new Date("2026-10-13T03:00:00Z");
|
|
281
|
+
const schedule = JSON.stringify({ type: "interval", every_seconds: 60 });
|
|
282
|
+
for (const consumed_at of [null, "invalid", "2026-10-14T00:00:00Z", "2026-10-12T00:00:00Z"]) {
|
|
283
|
+
expect(blockerTimedOut({ created_at: "2026-10-13 01:00:00", consumed_at }, schedule, now)).toBe(true);
|
|
284
|
+
}
|
|
285
|
+
expect(blockerTimedOut({ created_at: "2026-10-13 01:00:00", consumed_at: "2026-10-13 02:00:01" }, schedule, now)).toBe(false);
|
|
286
|
+
expect(blockerTimedOut({ created_at: "2026-10-13 01:00:00", consumed_at: "2026-10-13 02:00:00" }, schedule, now)).toBe(true);
|
|
287
|
+
});
|
|
288
|
+
});
|
package/src/scheduled-stuck.ts
CHANGED
|
@@ -14,6 +14,8 @@
|
|
|
14
14
|
// 恰好一条通知。
|
|
15
15
|
//
|
|
16
16
|
// 默认值(PR 里有推导):N=3;超时 = clamp(6 × 间隔, 下限 1 小时, 上限 24 小时)。
|
|
17
|
+
// #1112:从 consumed_at 起算,旧节点回落 started_at,未执行回落 created_at;
|
|
18
|
+
// 自创建时间的总保护上限仍为 cap,节点心跳/提交/ACK 不刷新执行时钟。
|
|
17
19
|
// 下限 1 小时:一轮 agent 任务正常要跑几十分钟,不能因为排程间隔短就把还在干活的任务掐掉。
|
|
18
20
|
// 上限 24 小时:Hub 本来就给定时任务 24 小时的 expires_at(没被取走时由巡检过期),开着超过一天的
|
|
19
21
|
// 定时任务不是「还在做」。6 × 间隔:间隔长的排程(每小时一次)给它 6 小时,而不是 1 小时。
|
|
@@ -54,20 +56,44 @@ export function stuckTimeoutMs(scheduleJson: string): number {
|
|
|
54
56
|
return Math.min(cap, Math.max(floor, factor * nominalIntervalSeconds(scheduleJson))) * 1000;
|
|
55
57
|
}
|
|
56
58
|
|
|
57
|
-
type
|
|
59
|
+
export type StuckBlockerClock = {
|
|
60
|
+
created_at: string;
|
|
61
|
+
consumed_at?: string | null;
|
|
62
|
+
started_at?: string | null;
|
|
63
|
+
};
|
|
64
|
+
|
|
65
|
+
/** #1105: queue time is not execution time. Submission/ACK/node heartbeat are not consumption. */
|
|
66
|
+
export function blockerTimeoutClock(blocker: StuckBlockerClock, scheduleJson: string, now: Date): { ageMs: number; limitMs: number; source: string } {
|
|
67
|
+
const created = parseDbTimestampMs(blocker.created_at);
|
|
68
|
+
const nowMs = now.getTime();
|
|
69
|
+
const floor = envNumber("COMMHUB_SCHEDULE_STUCK_TIMEOUT_FLOOR_SEC", 3_600, 1);
|
|
70
|
+
const capMs = Math.max(floor, envNumber("COMMHUB_SCHEDULE_STUCK_TIMEOUT_CAP_SEC", 86_400, 1)) * 1000;
|
|
71
|
+
// Even a very late consumption cannot hold this occurrence beyond the total cap.
|
|
72
|
+
if (Number.isFinite(created) && nowMs - created >= capMs) {
|
|
73
|
+
return { ageMs: nowMs - created, limitMs: capMs, source: "created_at (total cap)" };
|
|
74
|
+
}
|
|
75
|
+
for (const source of ["consumed_at", "started_at"] as const) {
|
|
76
|
+
const value = blocker[source];
|
|
77
|
+
if (!value) continue;
|
|
78
|
+
const at = parseDbTimestampMs(value);
|
|
79
|
+
if (Number.isFinite(at) && at <= nowMs && (!Number.isFinite(created) || at >= created)) {
|
|
80
|
+
return { ageMs: nowMs - at, limitMs: stuckTimeoutMs(scheduleJson), source };
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
return { ageMs: nowMs - created, limitMs: stuckTimeoutMs(scheduleJson), source: "created_at" };
|
|
84
|
+
}
|
|
58
85
|
|
|
59
86
|
/** 挡路的任务开了多久已经到超时 → true。读不出时间 = 不超时(宁可多挡一次,不错杀)。 */
|
|
60
|
-
export function blockerTimedOut(blocker:
|
|
61
|
-
const
|
|
62
|
-
|
|
63
|
-
return now.getTime() - started >= stuckTimeoutMs(scheduleJson);
|
|
87
|
+
export function blockerTimedOut(blocker: StuckBlockerClock, scheduleJson: string, now: Date): boolean {
|
|
88
|
+
const clock = blockerTimeoutClock(blocker, scheduleJson, now);
|
|
89
|
+
return Number.isFinite(clock.ageMs) && clock.ageMs >= clock.limitMs;
|
|
64
90
|
}
|
|
65
91
|
|
|
66
92
|
/**
|
|
67
93
|
* 在调用方的事务里按超时结束挡路的任务。只动还开着的任务(条件写),返回是否真的结束了它。
|
|
68
94
|
* 与 TTL 巡检(server.ts patrolExpiredTasks)同一套写法:status=expired、收件箱行置 acked、镜像到 run。
|
|
69
95
|
*/
|
|
70
|
-
export function expireStuckBlocker(taskId: string, networkId: string, ageMs: number, limitMs: number): boolean {
|
|
96
|
+
export function expireStuckBlocker(taskId: string, networkId: string, ageMs: number, limitMs: number, source = "created_at"): boolean {
|
|
71
97
|
const changed = db.run(
|
|
72
98
|
`UPDATE tasks SET status = 'expired', completed_at = datetime('now')
|
|
73
99
|
WHERE task_id = ?1 AND network_id = ?2 AND status IN ('created', 'delivered', 'acked', 'running')`,
|
|
@@ -78,7 +104,7 @@ export function expireStuckBlocker(taskId: string, networkId: string, ageMs: num
|
|
|
78
104
|
syncScheduledRunForTask(taskId, networkId);
|
|
79
105
|
db.run(
|
|
80
106
|
"UPDATE scheduled_task_runs SET error_message = ?1 WHERE task_id = ?2 AND network_id = ?3",
|
|
81
|
-
[`timed out by the scheduler:
|
|
107
|
+
[`timed out by the scheduler: ${Math.round(ageMs / 60_000)} min since ${source} (limit ${Math.round(limitMs / 60_000)} min); the next occurrence was dispatched`, taskId, networkId],
|
|
82
108
|
);
|
|
83
109
|
return true;
|
|
84
110
|
}
|
|
@@ -127,12 +153,12 @@ export function stuckNoticeText(n: StuckNotice): { title: string; text: string }
|
|
|
127
153
|
return {
|
|
128
154
|
title: "定时任务被卡住",
|
|
129
155
|
text: `定时任务「${n.scheduleName}」被上一次的执行卡住了:任务 ${n.taskId} 于 ${started} 派给节点 ${n.alias},到现在还没有完成,之后的 ${n.skips} 次执行都被跳过。\n\n`
|
|
130
|
-
+
|
|
156
|
+
+ `从实际开始执行起超过 ${fmtDuration(n.timeoutMs)} 后会被自动按超时结束;尚未开始的任务从派发起计时,所有执行仍受总时限保护。下一次照常派发。等不及可以在任务列表里取消这条任务。`,
|
|
131
157
|
};
|
|
132
158
|
}
|
|
133
159
|
return {
|
|
134
160
|
title: "定时任务已超时放行",
|
|
135
|
-
text: `定时任务「${n.scheduleName}」上一次的任务 ${n.taskId}(${started} 派给节点 ${n.alias})
|
|
161
|
+
text: `定时任务「${n.scheduleName}」上一次的任务 ${n.taskId}(${started} 派给节点 ${n.alias})超过 ${fmtDuration(n.timeoutMs)} 的执行或总保护时限还没有完成,已按超时结束,不再挡后面的执行。`,
|
|
136
162
|
};
|
|
137
163
|
}
|
|
138
164
|
|
package/src/scheduled-tasks.ts
CHANGED
|
@@ -8,8 +8,7 @@ import { canMessageAgent, type AgentRef } from "./agent-access.js";
|
|
|
8
8
|
import { dispatchVerdict } from "./node-permissions.js";
|
|
9
9
|
import { hasSubscribers, pushEvent, pushNetworkObserverEvent } from "./push.js";
|
|
10
10
|
import { SCHEDULED_TASK_STORAGE_SELECT } from "./rest-projections.js";
|
|
11
|
-
import {
|
|
12
|
-
import { blockerTimedOut, expireStuckBlocker, sendStuckNotice, skipsBlockedBy, stuckNoticeSkips, stuckTimeoutMs, type StuckNotice } from "./scheduled-stuck.js";
|
|
11
|
+
import { blockerTimedOut, blockerTimeoutClock, expireStuckBlocker, sendStuckNotice, skipsBlockedBy, stuckNoticeSkips, stuckTimeoutMs, type StuckNotice } from "./scheduled-stuck.js";
|
|
13
12
|
import { evaluateScheduleFailures, failureSummary } from "./scheduled-failures.js";
|
|
14
13
|
import { scheduledTargetOffline, sendOfflineNotice, type OfflineNotice } from "./scheduled-offline.js";
|
|
15
14
|
|
|
@@ -413,8 +412,8 @@ export function dispatchScheduledOccurrence(row: ScheduledRow, scheduledFor: str
|
|
|
413
412
|
// received:节点那边有没有把这条任务取走过(inbox 行被 ack,或任务已进 acked/running)。
|
|
414
413
|
// 跳过记录带上它,客户端才分得清「节点还没收到」和「节点在处理」—— 前者是投递故障,
|
|
415
414
|
// 后者才是真的上一次还没做完。
|
|
416
|
-
const findOpen = () => db.get<{ task_id: string; received: number; created_at: string; status: string }>(
|
|
417
|
-
`SELECT r.task_id, t.created_at, t.status,
|
|
415
|
+
const findOpen = () => db.get<{ task_id: string; received: number; created_at: string; consumed_at: string | null; started_at: string | null; status: string }>(
|
|
416
|
+
`SELECT r.task_id, t.created_at, t.consumed_at, t.started_at, t.status,
|
|
418
417
|
CASE WHEN t.status IN ('acked', 'running')
|
|
419
418
|
OR EXISTS (SELECT 1 FROM inbox i WHERE (i.task_id = r.task_id OR i.id = r.task_id) AND i.acked = 1)
|
|
420
419
|
THEN 1 ELSE 0 END AS received
|
|
@@ -428,9 +427,8 @@ export function dispatchScheduledOccurrence(row: ScheduledRow, scheduledFor: str
|
|
|
428
427
|
// #464 — a blocker open past the stuck timeout is ended as `expired` (the existing terminal state) and
|
|
429
428
|
// this occurrence dispatches normally. Bounded: each pass ends one exact open task or stops.
|
|
430
429
|
for (let guard = 0; open && guard < 20 && blockerTimedOut(open, row.schedule_json, advanceAfter); guard++) {
|
|
431
|
-
const limitMs =
|
|
432
|
-
|
|
433
|
-
if (!expireStuckBlocker(open.task_id, row.network_id, ageMs, limitMs)) break;
|
|
430
|
+
const { ageMs, limitMs, source } = blockerTimeoutClock(open, row.schedule_json, advanceAfter);
|
|
431
|
+
if (!expireStuckBlocker(open.task_id, row.network_id, ageMs, limitMs, source)) break;
|
|
434
432
|
timedOut.push({ taskId: open.task_id, fromStatus: open.status });
|
|
435
433
|
// Each stuck episode gets exactly one notice: if the stuck notice already went out (N skips), stay quiet.
|
|
436
434
|
if (skipsBlockedBy(row.schedule_id, open.task_id) < stuckNoticeSkips()) {
|
|
@@ -742,7 +742,6 @@ describe("inbox-enqueue lifecycle_state guard (RFC-027 §2.3 race-free)", () =>
|
|
|
742
742
|
const r = await call(tools.send_task, {
|
|
743
743
|
alias: CHILD_A_ALIAS,
|
|
744
744
|
task: "racey post-SIGTERM task",
|
|
745
|
-
from_session: "sender-x",
|
|
746
745
|
network_id: NET_A,
|
|
747
746
|
});
|
|
748
747
|
expect(r.ok).toBe(false);
|
package/src/tools.ts
CHANGED
|
@@ -15,6 +15,7 @@ import { assertNodeHealthy } from "./node-health-guard.js";
|
|
|
15
15
|
import { noteModelAuthHealth } from "./model-auth-notify.js";
|
|
16
16
|
import { numberedAliasInClause, parseAliasFilter } from "./alias-filter.js";
|
|
17
17
|
import { createHash } from "node:crypto";
|
|
18
|
+
import { activeNodeTokenIds, revokeNodeTokens } from "./node-token-identity.js";
|
|
18
19
|
import { db, uuidv4, logTaskEvent, chainReplyToParent, hashToken, generateId, generateNetworkToken, syncScheduledRunForTask } from "./db.js";
|
|
19
20
|
import { sessionTaskOnDispatch } from "./session-task-on-dispatch.js";
|
|
20
21
|
import { TASK_CONTENT_MAX, sessionTaskPreview } from "./shared/task-content-limit.js";
|
|
@@ -5400,12 +5401,8 @@ export function registerTools(server: McpServer, clientIP?: string, enforceNetwo
|
|
|
5400
5401
|
// #1448 finding-5 — 记下该 child **当前活着的** ntok 的 token_id,让 ack 精确按
|
|
5401
5402
|
// token_id 撤销,而非按 name='node:<alias>' 广撤。后者会在「同 alias 节点在这次
|
|
5402
5403
|
// delete 未收敛的窗口内被重建(拿到同名新 token)」时,把新 token 一并误撤。这里
|
|
5403
|
-
//
|
|
5404
|
-
const
|
|
5405
|
-
`SELECT token_id FROM api_tokens WHERE network_id = ?1 AND name = ?2 AND revoked_at IS NULL`,
|
|
5406
|
-
node.network_id, `node:${node.alias}`,
|
|
5407
|
-
);
|
|
5408
|
-
const childTokenId = childTokenRow?.token_id ?? null;
|
|
5404
|
+
// 在派发时刻定住经解析器验证的节点身份;旧请求也不按名字广撤。
|
|
5405
|
+
const childTokenId = activeNodeTokenIds(node.node_id, node.network_id)[0] ?? null;
|
|
5409
5406
|
|
|
5410
5407
|
// §4.5 D8 — lifecycle UPDATE + audit INSERTs + request INSERT atomic.
|
|
5411
5408
|
// PR1.1: use db.transaction() (SQLiteAdapter wraps better-sqlite3's
|
|
@@ -5643,24 +5640,9 @@ export function registerTools(server: McpServer, clientIP?: string, enforceNetwo
|
|
|
5643
5640
|
});
|
|
5644
5641
|
} else if (status === "stopped" && row.action === "delete") {
|
|
5645
5642
|
// Revoke the child's ntok, then DELETE the nodes row.
|
|
5646
|
-
//
|
|
5647
|
-
//
|
|
5648
|
-
|
|
5649
|
-
// 按 token_id 撤避免了「同 alias 在本次 delete 窗口内被重建、拿到同名新
|
|
5650
|
-
// token」时把新 token 一并误撤。
|
|
5651
|
-
if (row.child_token_id) {
|
|
5652
|
-
db.run(
|
|
5653
|
-
`UPDATE api_tokens SET revoked_at = datetime('now')
|
|
5654
|
-
WHERE token_id = ?1 AND revoked_at IS NULL`,
|
|
5655
|
-
[row.child_token_id],
|
|
5656
|
-
);
|
|
5657
|
-
} else {
|
|
5658
|
-
db.run(
|
|
5659
|
-
`UPDATE api_tokens SET revoked_at = datetime('now')
|
|
5660
|
-
WHERE network_id = ?1 AND name = ?2 AND revoked_at IS NULL`,
|
|
5661
|
-
[row.network_id, `node:${row.child_alias}`],
|
|
5662
|
-
);
|
|
5663
|
-
}
|
|
5643
|
+
// Validate the frozen token's immutable identity; legacy requests
|
|
5644
|
+
// resolve only credentials belonging to this node before deleting it.
|
|
5645
|
+
revokeNodeTokens(row.child_node_id, row.network_id, row.child_token_id);
|
|
5664
5646
|
db.run(`DELETE FROM nodes WHERE node_id = ?1`, [row.child_node_id]);
|
|
5665
5647
|
// Same as REST DELETE /api/nodes/:id: without this the sessions row
|
|
5666
5648
|
// stays "offline" and /api/status lists the deleted node forever.
|