@sleep2agi/commhub-server 0.9.0-preview.21 → 0.9.0-preview.210

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (397) hide show
  1. package/README.md +130 -10
  2. package/bin/commhub.ts +99 -11
  3. package/package.json +7 -7
  4. package/src/ack-create-request-transport.test.ts +182 -0
  5. package/src/ack-create-request.test.ts +50 -7
  6. package/src/admin-delete-network-http.test.ts +228 -0
  7. package/src/admin-networks-http.test.ts +88 -0
  8. package/src/adoption-candidates-http.test.ts +137 -0
  9. package/src/adoption-candidates.ts +158 -0
  10. package/src/agent-access-batch.test.ts +210 -0
  11. package/src/agent-access.ts +487 -0
  12. package/src/agent-acl-groups-http.test.ts +247 -0
  13. package/src/agent-acl-http.test.ts +659 -0
  14. package/src/agent-acl-slice3-http.test.ts +267 -0
  15. package/src/agent-department-http.test.ts +215 -0
  16. package/src/agent-groups-perf.test.ts +223 -0
  17. package/src/agent-notice.ts +48 -0
  18. package/src/agent-team-whoami-http.test.ts +98 -0
  19. package/src/agent-teams-http.test.ts +330 -0
  20. package/src/agent-teams.ts +244 -0
  21. package/src/alias-baseline-2571.test.ts +162 -0
  22. package/src/alias-baseline.ts +113 -0
  23. package/src/alias-filter.test.ts +72 -0
  24. package/src/alias-filter.ts +42 -0
  25. package/src/api-host-supervisors-can-create-nodes.test.ts +350 -0
  26. package/src/api-host-supervisors-fallback.test.ts +21 -12
  27. package/src/api-host-supervisors-runtime-readiness.test.ts +195 -0
  28. package/src/api-nodes-shape.test.ts +120 -13
  29. package/src/auth-password-change-cli-tokens-http.test.ts +423 -0
  30. package/src/auth-sessions-http.test.ts +250 -0
  31. package/src/auth-validate.test.ts +5 -1
  32. package/src/auth-weak-password-login.test.ts +176 -0
  33. package/src/auth.ts +609 -51
  34. package/src/avatar-validate.test.ts +163 -0
  35. package/src/avatar-validate.ts +103 -0
  36. package/src/backfill-node-ids-2664.test.ts +128 -0
  37. package/src/binding-generation-http.test.ts +121 -0
  38. package/src/blocked-exit-on-reply.test.ts +122 -0
  39. package/src/boot-side-effect.test.ts +124 -0
  40. package/src/broadcast-fanout-2721.test.ts +118 -0
  41. package/src/broadcast-fanout.ts +98 -0
  42. package/src/chain-attempt-dedup.test.ts +447 -0
  43. package/src/clone-node.test.ts +248 -0
  44. package/src/clone-node.ts +289 -0
  45. package/src/codex-config-flags.ts +77 -0
  46. package/src/codex-fork-contract.ts +23 -0
  47. package/src/codex-fork-migration.test.ts +38 -0
  48. package/src/commhub-bin-args.test.ts +85 -0
  49. package/src/config-apply-validate.test.ts +42 -1
  50. package/src/config-apply-validate.ts +30 -2
  51. package/src/create-node-blocked-gate.test.ts +356 -0
  52. package/src/create-node-network-id.test.ts +79 -0
  53. package/src/create-node-tool-schema.test.ts +49 -0
  54. package/src/create-node-unicode-name.test.ts +120 -0
  55. package/src/create-node-validate.test.ts +199 -7
  56. package/src/create-node-validate.ts +232 -15
  57. package/src/create-node-workdir.test.ts +289 -0
  58. package/src/create-node.ts +88 -23
  59. package/src/daemon-provider-snapshots.test.ts +421 -0
  60. package/src/daemon-provider-snapshots.ts +327 -0
  61. package/src/daemon-token-owner-http.test.ts +236 -0
  62. package/src/dashboard-slash-routing-http.test.ts +109 -0
  63. package/src/db-adapter-guard.test.ts +148 -0
  64. package/src/db-adapter-translate.test.ts +128 -0
  65. package/src/db-adapter.ts +377 -91
  66. package/src/db-timestamp.test.ts +38 -0
  67. package/src/db-timestamp.ts +15 -0
  68. package/src/db.ts +1306 -153
  69. package/src/delivered-without-push.test.ts +140 -0
  70. package/src/department-groups-http.test.ts +209 -0
  71. package/src/department-groups-sync-http.test.ts +328 -0
  72. package/src/department-groups.ts +434 -0
  73. package/src/department-heads-http.test.ts +330 -0
  74. package/src/department-heads.ts +226 -0
  75. package/src/departments-agent-count.test.ts +129 -0
  76. package/src/departments-http.test.ts +185 -0
  77. package/src/departments-mcp.test.ts +128 -0
  78. package/src/departments.ts +318 -0
  79. package/src/desktop-message-seam.test.ts +150 -0
  80. package/src/desktop-message.test.ts +368 -0
  81. package/src/dm-file-scope-http.test.ts +178 -0
  82. package/src/empty-node-sweeper.test.ts +104 -0
  83. package/src/empty-node-sweeper.ts +88 -0
  84. package/src/env-transport-gate-http.test.ts +240 -0
  85. package/src/env-transport-gate.ts +80 -0
  86. package/src/env-transport-policy.ts +56 -0
  87. package/src/event-loop-lag.test.ts +180 -0
  88. package/src/event-loop-lag.ts +181 -0
  89. package/src/ext-token-shared.test.ts +60 -0
  90. package/src/ext-validation-regression.test.ts +131 -0
  91. package/src/external-schedule-edits-http.test.ts +321 -0
  92. package/src/external-schedule-edits-timestamps.test.ts +64 -0
  93. package/src/external-schedule-edits.ts +327 -0
  94. package/src/file-download-authz-dev-open.test.ts +135 -0
  95. package/src/file-download-authz.test.ts +404 -0
  96. package/src/file-download-date-bucket.test.ts +436 -0
  97. package/src/file-network-scope-dev-open.test.ts +97 -0
  98. package/src/file-network-scope.test.ts +732 -0
  99. package/src/group-membership-events-http.test.ts +264 -0
  100. package/src/group-messages-http.test.ts +482 -0
  101. package/src/group-messages.ts +297 -0
  102. package/src/health-host-disk-2756.test.ts +49 -0
  103. package/src/health-redaction.test.ts +685 -0
  104. package/src/host-supervisor-identity-http.test.ts +152 -0
  105. package/src/host-supervisor-identity.ts +92 -0
  106. package/src/host-supervisors-daemon-version.test.ts +137 -0
  107. package/src/http-gzip.test.ts +94 -0
  108. package/src/http-gzip.ts +112 -0
  109. package/src/hub-catalog-read-http.test.ts +187 -0
  110. package/src/hub-provider-config.test.ts +139 -0
  111. package/src/hub-provider-config.ts +105 -0
  112. package/src/hub-timestamp.test.ts +38 -0
  113. package/src/hub-timestamp.ts +27 -0
  114. package/src/human-dm.ts +215 -0
  115. package/src/human-inbox.ts +42 -0
  116. package/src/inbox-caller.ts +49 -0
  117. package/src/inbox-count-sse.test.ts +310 -0
  118. package/src/inbox-count.ts +29 -0
  119. package/src/index.ts +15 -2385
  120. package/src/list-host-supervisors.test.ts +5 -4
  121. package/src/mcp-context-cost-http.test.ts +127 -0
  122. package/src/mcp-inflight.test.ts +92 -0
  123. package/src/mcp-inflight.ts +38 -0
  124. package/src/mcp-projects-events-http.test.ts +162 -0
  125. package/src/mcp-route-label-http.test.ts +120 -0
  126. package/src/mcp-write-network-resolution.test.ts +432 -0
  127. package/src/member-audit-network-id-http.test.ts +93 -0
  128. package/src/member-presence-http.test.ts +210 -0
  129. package/src/messages-ack-by-agent-http.test.ts +142 -0
  130. package/src/migrate-sqlite-to-pg.test.ts +47 -0
  131. package/src/migrate-sqlite-to-pg.ts +263 -0
  132. package/src/model-auth-notify-http.test.ts +181 -0
  133. package/src/model-auth-notify.ts +116 -0
  134. package/src/network-name-http.test.ts +75 -0
  135. package/src/network-rules-http.test.ts +169 -0
  136. package/src/network-rules.ts +129 -0
  137. package/src/network-scope.ts +276 -0
  138. package/src/network-token-membership-http.test.ts +139 -0
  139. package/src/node-alias-control.ts +12 -0
  140. package/src/node-attrs-validate.ts +88 -0
  141. package/src/node-attrs.test.ts +258 -0
  142. package/src/node-caller-identity-http.test.ts +1083 -0
  143. package/src/node-create-request-status-http.test.ts +105 -0
  144. package/src/node-daemon-bindings.test.ts +132 -0
  145. package/src/node-daemon-bindings.ts +135 -0
  146. package/src/node-failure-alert.test.ts +150 -0
  147. package/src/node-failure-alert.ts +177 -0
  148. package/src/node-files-transport.test.ts +344 -0
  149. package/src/node-health-codex-login.test.ts +120 -0
  150. package/src/node-health-dispatch-http.test.ts +279 -0
  151. package/src/node-health-guard.ts +105 -0
  152. package/src/node-health-passthrough.test.ts +115 -0
  153. package/src/node-health-store.ts +101 -0
  154. package/src/node-id-alias.test.ts +190 -0
  155. package/src/node-identity-conflict-http.test.ts +269 -0
  156. package/src/node-identity-conflict.ts +142 -0
  157. package/src/node-identity-mismatch.test.ts +74 -0
  158. package/src/node-in-flight-tasks.ts +31 -0
  159. package/src/node-lifecycle-controllable.test.ts +64 -0
  160. package/src/node-lifecycle-controllable.ts +53 -0
  161. package/src/node-lifecycle-read-http.test.ts +250 -0
  162. package/src/node-lifecycle-read.ts +124 -0
  163. package/src/node-logs-transport.test.ts +263 -0
  164. package/src/node-peer-allowlist.test.ts +114 -0
  165. package/src/node-permissions-http.test.ts +648 -0
  166. package/src/node-permissions.ts +494 -0
  167. package/src/node-request-retention.test.ts +281 -0
  168. package/src/node-request-retention.ts +120 -0
  169. package/src/node-secret-rest-http.test.ts +190 -0
  170. package/src/node-serviceability.test.ts +86 -0
  171. package/src/node-serviceability.ts +124 -0
  172. package/src/node-session-control-http.test.ts +310 -0
  173. package/src/node-session-control.ts +258 -0
  174. package/src/node-skills-transport.test.ts +209 -0
  175. package/src/node-token-identity.ts +35 -0
  176. package/src/node-token-ownership.ts +51 -0
  177. package/src/node-token-privilege-http.test.ts +158 -0
  178. package/src/not-found-api-http.test.ts +67 -0
  179. package/src/observer-avatar-http.test.ts +442 -0
  180. package/src/observer-push.test.ts +252 -0
  181. package/src/opencode-catalog-report-http.test.ts +139 -0
  182. package/src/opencode-v2-catalog.test.ts +268 -0
  183. package/src/opencode-v2-catalog.ts +352 -0
  184. package/src/package-version-consistency.test.ts +25 -0
  185. package/src/parent-source-marker.test.ts +50 -0
  186. package/src/password-network-tokens.ts +25 -0
  187. package/src/peer-reply-atomic.test.ts +455 -0
  188. package/src/peer-reply-capability.test.ts +70 -0
  189. package/src/pg-worker.ts +87 -0
  190. package/src/project-dir-warn-clear-http.test.ts +132 -0
  191. package/src/project-viewer-can-http.test.ts +116 -0
  192. package/src/push.test.ts +20 -10
  193. package/src/push.ts +466 -13
  194. package/src/redact-tokens.test.ts +65 -0
  195. package/src/redact-tokens.ts +40 -0
  196. package/src/rename.ts +38 -10
  197. package/src/reply-target-derivation.test.ts +96 -0
  198. package/src/report-status-handover-capabilities-http.test.ts +124 -0
  199. package/src/report-status-host-schema.test.ts +235 -0
  200. package/src/report-status-resume-id-handover.test.ts +184 -0
  201. package/src/report-status-secret-keys.test.ts +140 -0
  202. package/src/require-explicit-test-db.ts +9 -0
  203. package/src/requirement-due-reminders-http.test.ts +620 -0
  204. package/src/requirement-due-reminders.ts +416 -0
  205. package/src/requirement-events-http.test.ts +268 -0
  206. package/src/requirement-events.test.ts +45 -0
  207. package/src/requirement-events.ts +144 -0
  208. package/src/requirement-notify.ts +165 -0
  209. package/src/requirement-participant-notify-http.test.ts +309 -0
  210. package/src/requirement-tag-ops-http.test.ts +425 -0
  211. package/src/requirement-tag-tree-http.test.ts +275 -0
  212. package/src/requirement-tag-tree.ts +127 -0
  213. package/src/requirement-tags.ts +102 -0
  214. package/src/requirements-abandoned-http.test.ts +236 -0
  215. package/src/requirements-comment-http.test.ts +167 -0
  216. package/src/requirements-completed-at-migration.test.ts +70 -0
  217. package/src/requirements-errors-mcp-http.test.ts +164 -0
  218. package/src/requirements-errors.ts +158 -0
  219. package/src/requirements-etag-http.test.ts +91 -0
  220. package/src/requirements-http.test.ts +709 -0
  221. package/src/requirements-last-event-http.test.ts +349 -0
  222. package/src/requirements-list-mcp-slim-http.test.ts +155 -0
  223. package/src/requirements-list-slim-http.test.ts +324 -0
  224. package/src/requirements-mcp.test.ts +163 -0
  225. package/src/requirements-migrate.ts +283 -0
  226. package/src/requirements-owner-strict-mcp-http.test.ts +145 -0
  227. package/src/requirements-people-http.test.ts +114 -0
  228. package/src/requirements-people-mcp-http.test.ts +231 -0
  229. package/src/requirements-people.ts +196 -0
  230. package/src/requirements-priority-lowest.test.ts +114 -0
  231. package/src/requirements-projects-parent-http.test.ts +123 -0
  232. package/src/requirements-search-http.test.ts +182 -0
  233. package/src/requirements-search.test.ts +32 -0
  234. package/src/requirements-search.ts +145 -0
  235. package/src/requirements-seq-http.test.ts +193 -0
  236. package/src/requirements-seq-migration.test.ts +63 -0
  237. package/src/requirements-start-date.test.ts +125 -0
  238. package/src/requirements-stats-http.test.ts +357 -0
  239. package/src/requirements-stats.ts +195 -0
  240. package/src/requirements-status-participants-http.test.ts +222 -0
  241. package/src/requirements-upgrade-e2e.test.ts +126 -0
  242. package/src/requirements-upgrade-migration.test.ts +98 -0
  243. package/src/requirements.ts +2027 -0
  244. package/src/resolve-port.test.ts +57 -0
  245. package/src/resolve-port.ts +44 -0
  246. package/src/response-charset.test.ts +130 -0
  247. package/src/rest-alias-network-fallback-http.test.ts +123 -0
  248. package/src/rest-alias-network-fallback.test.ts +60 -0
  249. package/src/rest-alias-network-fallback.ts +65 -0
  250. package/src/rest-explicit-columns-http.test.ts +266 -0
  251. package/src/rest-identity.test.ts +130 -0
  252. package/src/rest-identity.ts +141 -0
  253. package/src/rest-projections.ts +72 -0
  254. package/src/rest-read-network-real-chain.test.ts +97 -0
  255. package/src/rest-write-network-resolution.test.ts +133 -0
  256. package/src/restart-readiness-http.test.ts +121 -0
  257. package/src/restricted-files.ts +76 -0
  258. package/src/retention-first-sweep-quiet.test.ts +31 -0
  259. package/src/retention.test.ts +100 -40
  260. package/src/retention.ts +423 -104
  261. package/src/route-callers-http.test.ts +120 -0
  262. package/src/route-timing.test.ts +211 -0
  263. package/src/route-timing.ts +262 -0
  264. package/src/rules-file-duplicate-alias.test.ts +234 -0
  265. package/src/rules-file-session-target.test.ts +207 -0
  266. package/src/rules-file-transport.test.ts +237 -0
  267. package/src/runtime-label.test.ts +22 -0
  268. package/src/runtime-label.ts +21 -0
  269. package/src/runtime-readiness.ts +90 -0
  270. package/src/schedule-agent-mcp-http.test.ts +391 -0
  271. package/src/schedule-agent.ts +192 -0
  272. package/src/schedule-starved.test.ts +92 -0
  273. package/src/schedule-starved.ts +114 -0
  274. package/src/scheduled-backoff-http.test.ts +193 -0
  275. package/src/scheduled-backoff.ts +64 -0
  276. package/src/scheduled-doorbell-nodeless-session.test.ts +134 -0
  277. package/src/scheduled-failure-backoff-2622.test.ts +145 -0
  278. package/src/scheduled-failure-code.test.ts +73 -0
  279. package/src/scheduled-failure-code.ts +105 -0
  280. package/src/scheduled-failures-http.test.ts +279 -0
  281. package/src/scheduled-failures.ts +377 -0
  282. package/src/scheduled-noprogress-backoff-2622.test.ts +133 -0
  283. package/src/scheduled-offline-backoff-probe.test.ts +56 -0
  284. package/src/scheduled-offline.test.ts +131 -0
  285. package/src/scheduled-offline.ts +66 -0
  286. package/src/scheduled-outcome-guidance-http.test.ts +139 -0
  287. package/src/scheduled-outcome-guidance.ts +9 -0
  288. package/src/scheduled-reply-unread-http.test.ts +212 -0
  289. package/src/scheduled-requirement-wake-http.test.ts +279 -0
  290. package/src/scheduled-requirement-wake.ts +125 -0
  291. package/src/scheduled-run-terminal.test.ts +268 -0
  292. package/src/scheduled-skip-blocking-task.test.ts +120 -0
  293. package/src/scheduled-skip-path-backoff-2622.test.ts +98 -0
  294. package/src/scheduled-stuck-http.test.ts +312 -0
  295. package/src/scheduled-stuck-notice-cap.test.ts +80 -0
  296. package/src/scheduled-stuck.ts +219 -0
  297. package/src/scheduled-tasks-http.test.ts +679 -0
  298. package/src/scheduled-tasks.ts +959 -0
  299. package/src/scheduled-work-outcome-http.test.ts +154 -0
  300. package/src/scheduled-work-outcome.ts +50 -0
  301. package/src/scoped-secrets.ts +141 -0
  302. package/src/send-dedup-request-id.test.ts +229 -0
  303. package/src/send-reply-agent-warning.test.ts +202 -0
  304. package/src/send-reply-alias-optional.test.ts +131 -0
  305. package/src/send-reply-attachments.test.ts +458 -0
  306. package/src/send_dedup.test.ts +54 -0
  307. package/src/send_dedup.ts +52 -6
  308. package/src/serve-error.test.ts +38 -0
  309. package/src/serve-error.ts +27 -0
  310. package/src/server.ts +6220 -0
  311. package/src/session-task-on-dispatch.ts +14 -0
  312. package/src/session-token-ownership-http.test.ts +122 -0
  313. package/src/shared/codex-inventory.ts +41 -0
  314. package/src/shared/daemon-provider-login.ts +31 -0
  315. package/src/shared/daemon-provider-snapshot.ts +52 -0
  316. package/src/shared/external-schedule-contract.ts +60 -0
  317. package/src/shared/node-name.test.ts +62 -0
  318. package/src/shared/node-name.ts +161 -0
  319. package/src/shared/provider-application.ts +29 -0
  320. package/src/shared/provider-collection.ts +14 -0
  321. package/src/shared/provider-initialization.ts +8 -0
  322. package/src/shared/reserved-env-drift.test.ts +10 -0
  323. package/src/shared/scoped-secret-contract.ts +5 -0
  324. package/src/shared/task-content-limit-drift.test.ts +28 -0
  325. package/src/shared/task-content-limit.ts +25 -0
  326. package/src/side-thread-command-transport.test.ts +312 -0
  327. package/src/side-thread-command-transport.ts +318 -0
  328. package/src/side-thread-http-integration.test.ts +233 -0
  329. package/src/side-thread-http.ts +504 -0
  330. package/src/side-thread-production.test.ts +72 -0
  331. package/src/side-thread-production.ts +118 -0
  332. package/src/side-thread.test.ts +1067 -0
  333. package/src/side-thread.ts +2499 -0
  334. package/src/skillhub-http.test.ts +108 -0
  335. package/src/sqlite-close-checkpoint.test.ts +78 -0
  336. package/src/start-node.test.ts +264 -0
  337. package/src/status-alias-filter-http.test.ts +106 -0
  338. package/src/status-alias-resolver-http.test.ts +163 -0
  339. package/src/status-alias-resolver.ts +19 -0
  340. package/src/status-node-id-filter-http.test.ts +161 -0
  341. package/src/status-read-cache-http.test.ts +229 -0
  342. package/src/status-read-cache.ts +122 -0
  343. package/src/stop-delete-node.test.ts +567 -1
  344. package/src/task-access-http.test.ts +402 -0
  345. package/src/task-access.ts +420 -0
  346. package/src/task-activity.ts +60 -0
  347. package/src/task-auth-origin-http.test.ts +93 -0
  348. package/src/task-auth-origin.test.ts +24 -0
  349. package/src/task-auth-origin.ts +7 -0
  350. package/src/task-consumption.test.ts +482 -0
  351. package/src/task-diagnostic-http.test.ts +149 -0
  352. package/src/task-diagnostic.test.ts +115 -0
  353. package/src/task-diagnostic.ts +118 -0
  354. package/src/task-expiry-notice-http.test.ts +287 -0
  355. package/src/task-expiry-notice.ts +315 -0
  356. package/src/task-idempotency-mcp.test.ts +142 -0
  357. package/src/task-idempotency.test.ts +76 -0
  358. package/src/task-idempotency.ts +81 -0
  359. package/src/task-late-replies.ts +54 -0
  360. package/src/task-late-reply-http.test.ts +152 -0
  361. package/src/task-lifecycle-watcher-busy.test.ts +114 -0
  362. package/src/task-lifecycle-watcher.test.ts +204 -0
  363. package/src/task-lifecycle-watcher.ts +109 -0
  364. package/src/task-network-resolution-http.test.ts +140 -0
  365. package/src/task-offline-delivery-2668.test.ts +119 -0
  366. package/src/task-offline-delivery.ts +81 -0
  367. package/src/task-orphan-http.test.ts +287 -0
  368. package/src/task-orphan.ts +119 -0
  369. package/src/task-queue-ahead-http.test.ts +309 -0
  370. package/src/task-queue-ahead-scale.test.ts +236 -0
  371. package/src/task-queue-ahead.ts +186 -0
  372. package/src/task-rejection-log-http.test.ts +160 -0
  373. package/src/task-rejection-log.ts +49 -0
  374. package/src/task-sender-http.test.ts +143 -0
  375. package/src/task-stale-open-http.test.ts +290 -0
  376. package/src/task-stale-open.ts +103 -0
  377. package/src/tasks-pagination-http.test.ts +141 -0
  378. package/src/team-rules-status-http.test.ts +90 -0
  379. package/src/team-rules-status.test.ts +59 -0
  380. package/src/team-rules-status.ts +72 -0
  381. package/src/tool-audience-http.test.ts +230 -0
  382. package/src/tool-audience.ts +120 -0
  383. package/src/tools-list-json-schema.test.ts +94 -0
  384. package/src/tools.ts +4119 -391
  385. package/src/upload-early-reject-guard.test.ts +155 -0
  386. package/src/uploads-http.test.ts +97 -15
  387. package/src/uploads.ts +103 -6
  388. package/src/user-avatar-http.test.ts +74 -0
  389. package/src/user-avatar-image.ts +160 -0
  390. package/src/user-avatar-migration.test.ts +25 -0
  391. package/src/user-avatar-upload-http.test.ts +348 -0
  392. package/src/user-inbox-by-agent-http.test.ts +161 -0
  393. package/src/user-inbox-netid-notnull.test.ts +122 -0
  394. package/src/user-inbox-read-http.test.ts +223 -0
  395. package/src/user-inbox-schema.test.ts +54 -0
  396. package/src/version-stats.test.ts +66 -0
  397. package/src/version-stats.ts +109 -0
@@ -0,0 +1,145 @@
1
+ // #2622 —— 定时任务「连续失败 → 间隔翻倍 → 成功即恢复」。
2
+ //
3
+ // 为什么需要它(生产 24h,2026-10-27):
4
+ // 一个 `every_seconds=180` 的排程 351 次失败 / 24 次成功 —— 失败与成功交替,
5
+ // 所以「连续失败」永远攒不到 #523 的自动暂停阈值,而**每一次失败都按 3 分钟原速重试**。
6
+ // `#523` 的机制(连续 5 次通知、可选自动暂停)都在,缺的只是「放慢」这一档。
7
+ //
8
+ // 这个文件钉住四件事:
9
+ // 1. 连续失败达到阈值 ⇒ `backoff_effective_seconds` 按 2 倍递增,且有上限;
10
+ // 2. `next_run_at` 只往后推,不会被“放大后又缩短”的抖动来回拉;
11
+ // 3. 成功一次 ⇒ 立刻清空退避,`next_run_at` 拉回正常节奏;
12
+ // 4. 与 `no_change_backoff` 互斥(那套逻辑自己拥有这一列),关掉开关时完全不介入。
13
+
14
+ import { afterAll, afterEach, beforeAll, describe, expect, test } from "bun:test";
15
+ import { register } from "./auth.js";
16
+ import { db } from "./db.js";
17
+ import { applyFailureBackoff, failureBackoffRuns } from "./scheduled-failures.js";
18
+
19
+ const stamp = Date.now();
20
+ let NET = "";
21
+ let creatorId = "";
22
+ const ENV_KEY = "COMMHUB_SCHEDULE_FAILURE_BACKOFF_RUNS";
23
+ const savedEnv = process.env[ENV_KEY];
24
+ let slot = 0;
25
+
26
+ function seedSchedule(everySeconds = 180, patch: Record<string, unknown> = {}) {
27
+ const scheduleId = `sched_bo_${crypto.randomUUID()}`;
28
+ db.run(
29
+ `INSERT INTO scheduled_tasks
30
+ (schedule_id, network_id, created_by, name, target_node_id, target_alias, task_content, priority,
31
+ schedule_type, schedule_json, timezone, status, next_run_at, overlap_policy)
32
+ VALUES (?1, ?2, ?3, 'backoff fixture', 'n_bo', 'bo-node', 'x', 'normal', 'interval', ?4,
33
+ 'Asia/Shanghai', 'active', ?5, 'skip')`,
34
+ [scheduleId, NET, creatorId, JSON.stringify({ type: "interval", every_seconds: everySeconds }),
35
+ new Date(Date.now() + everySeconds * 1000).toISOString()],
36
+ );
37
+ for (const [k, v] of Object.entries(patch)) db.run(`UPDATE scheduled_tasks SET ${k} = ?1 WHERE schedule_id = ?2`, [v as any, scheduleId]);
38
+ return scheduleId;
39
+ }
40
+
41
+ function addRun(scheduleId: string, status: string, minutesFromNow: number) {
42
+ const runId = `srun_bo_${crypto.randomUUID()}`;
43
+ db.run(
44
+ `INSERT INTO scheduled_task_runs (run_id, schedule_id, network_id, scheduled_for, status, created_at)
45
+ VALUES (?1, ?2, ?3, ?4, ?5, ?4)`,
46
+ [runId, scheduleId, NET, new Date(Date.now() + minutesFromNow * 60_000).toISOString(), status],
47
+ );
48
+ return runId;
49
+ }
50
+
51
+ const reload = (id: string) => db.get<any>("SELECT * FROM scheduled_tasks WHERE schedule_id = ?1", id)!;
52
+
53
+ beforeAll(async () => {
54
+ const auth = register(`bo_owner_${stamp}`, "Bo-Owner-Strong-1!", undefined, "tester");
55
+ expect(auth.ok).toBe(true);
56
+ NET = auth.network_id!;
57
+ creatorId = auth.user_id!;
58
+ process.env[ENV_KEY] = "6";
59
+ });
60
+
61
+ afterEach(() => {
62
+ process.env[ENV_KEY] = "6";
63
+ });
64
+
65
+ afterAll(() => {
66
+ try { db.run(`DELETE FROM scheduled_task_runs WHERE network_id = ?1`, [NET]); } catch {}
67
+ try { db.run(`DELETE FROM scheduled_tasks WHERE network_id = ?1`, [NET]); } catch {}
68
+ if (savedEnv === undefined) delete process.env[ENV_KEY]; else process.env[ENV_KEY] = savedEnv;
69
+ });
70
+
71
+ describe("#2622 failure backoff", () => {
72
+ test("默认阈值是 6,且可以关掉", () => {
73
+ delete process.env[ENV_KEY];
74
+ expect(failureBackoffRuns()).toBe(6);
75
+ process.env[ENV_KEY] = "0";
76
+ expect(failureBackoffRuns()).toBe(0);
77
+ });
78
+
79
+ test("失败次数未到阈值:不写退避", () => {
80
+ const id = seedSchedule();
81
+ for (let i = 0; i < 5; i++) addRun(id, "failed", -60 + i);
82
+ const out = applyFailureBackoff(id);
83
+ expect(out).toEqual({ streak: 5, effective_seconds: null });
84
+ expect(reload(id).backoff_effective_seconds ?? null).toBeNull();
85
+ });
86
+
87
+ test("达到阈值后间隔翻倍,且 next_run_at 只会往后推", () => {
88
+ const id = seedSchedule(180);
89
+ for (let i = 0; i < 6; i++) addRun(id, "failed", -60 + i);
90
+ const before = reload(id).next_run_at;
91
+ const out = applyFailureBackoff(id);
92
+ expect(out!.effective_seconds).toBe(360); // 180 × 2
93
+ const after = reload(id);
94
+ expect(after.backoff_effective_seconds).toBe(360);
95
+ expect(Date.parse(after.next_run_at)).toBeGreaterThanOrEqual(Date.parse(before));
96
+ });
97
+
98
+ test("失败继续累积:指数放大且有 1 小时上限", () => {
99
+ const id = seedSchedule(180);
100
+ for (let i = 0; i < 12; i++) addRun(id, "failed", -120 + i);
101
+ const out = applyFailureBackoff(id);
102
+ // 180 × 2^k 会被 3600 截断
103
+ expect(out!.effective_seconds).toBeLessThanOrEqual(3600);
104
+ expect(out!.streak).toBe(12);
105
+ });
106
+
107
+ test("成功一次 ⇒ 立刻清空退避并把 next_run_at 拉回正常节奏", () => {
108
+ const id = seedSchedule(180);
109
+ for (let i = 0; i < 8; i++) addRun(id, "failed", -120 + i);
110
+ applyFailureBackoff(id);
111
+ expect(reload(id).backoff_effective_seconds).toBeGreaterThan(180);
112
+ addRun(id, "replied", -1);
113
+ const out = applyFailureBackoff(id);
114
+ expect(out!.streak).toBe(0);
115
+ expect(out!.effective_seconds).toBeNull();
116
+ expect(reload(id).backoff_effective_seconds ?? null).toBeNull();
117
+ });
118
+
119
+ test("no_change_backoff 的排程不被这条逻辑改写", () => {
120
+ const id = seedSchedule(180, { no_change_backoff: 1 });
121
+ for (let i = 0; i < 9; i++) addRun(id, "failed", -120 + i);
122
+ expect(applyFailureBackoff(id)).toBeNull();
123
+ expect(reload(id).backoff_effective_seconds ?? null).toBeNull();
124
+ });
125
+
126
+ test("非 interval 排程不参与", () => {
127
+ const id = seedSchedule(180, { schedule_type: "daily", schedule_json: JSON.stringify({ type: "daily", time: "09:00" }) });
128
+ for (let i = 0; i < 9; i++) addRun(id, "failed", -120 + i);
129
+ expect(applyFailureBackoff(id)).toBeNull();
130
+ });
131
+
132
+ test("paused 的排程不参与(暂停已经把它停住了)", () => {
133
+ const id = seedSchedule(180, { status: "paused", next_run_at: null });
134
+ for (let i = 0; i < 9; i++) addRun(id, "failed", -120 + i);
135
+ expect(applyFailureBackoff(id)).toBeNull();
136
+ });
137
+
138
+ test("开关设为 0 时完全不介入", () => {
139
+ const id = seedSchedule(180);
140
+ for (let i = 0; i < 9; i++) addRun(id, "failed", -120 + i);
141
+ process.env[ENV_KEY] = "0";
142
+ expect(applyFailureBackoff(id)).toBeNull();
143
+ expect(reload(id).backoff_effective_seconds ?? null).toBeNull();
144
+ });
145
+ });
@@ -0,0 +1,73 @@
1
+ // #1879 —— 失败分类:共享会话忙与真失败分开。
2
+ import { describe, expect, test } from "bun:test";
3
+ import { scheduledFailureCode, busyElapsedNote } from "./scheduled-failure-code.js";
4
+
5
+ describe("scheduledFailureCode", () => {
6
+ test("共享会话忙 → task_failed_busy(两种情况都覆盖)", () => {
7
+ const notSubmitted = "[TMA云盘欧] opencode 任务未提交:共享会话在 30 分钟 内一直处于忙碌状态(可能有人正在 TUI 里跑一轮),本任务没有发出";
8
+ expect(scheduledFailureCode("failed", notSubmitted)).toBe("task_failed_busy");
9
+ const stillRunning = "[通信客户端欧] ⏳ opencode 任务仍在节点的 TUI 会话里运行,没有被中止;bridge 等待回复已达 30 分钟";
10
+ expect(scheduledFailureCode("failed", stillRunning)).toBe("task_failed_busy");
11
+ });
12
+
13
+ test("真失败仍然是 task_failed(不能被分类吞掉)", () => {
14
+ expect(scheduledFailureCode("failed", "runtime failed")).toBe("task_failed");
15
+ expect(scheduledFailureCode("failed", null)).toBe("task_failed");
16
+ // 提到「共享会话」但没有任何忙碌措辞 ⇒ 仍然算真失败,不能被分类吞掉
17
+ expect(scheduledFailureCode("failed", "共享会话已释放,任务失败原因不明")).toBe("task_failed");
18
+ });
19
+
20
+ test("非 failed 状态不受影响", () => {
21
+ expect(scheduledFailureCode("replied", "共享会话忙")).toBeNull();
22
+ expect(scheduledFailureCode("cancelled", "共享会话一直处于忙碌")).toBe("task_cancelled");
23
+ expect(scheduledFailureCode("expired", null)).toBe("task_expired");
24
+ });
25
+ });
26
+
27
+ // #2884 —— 两个量必须分开报:排队(创建→开始处理)与会话等待(开始处理→失败)。
28
+ describe("busyElapsedNote", () => {
29
+ test("生产里那条 412 分钟:拆成排队 382 + 会话等待 30.1", () => {
30
+ // 原始数据取自 tasks 表(2026-10-27,TMA需求欧)
31
+ const note = busyElapsedNote("2026-10-27 13:54:48", "2026-10-27 20:16:52", "2026-10-27 20:46:57");
32
+ expect(note).toBeTruthy();
33
+ expect(note!).toContain("实测总耗时 412 分钟");
34
+ expect(note!).toContain("排队 382 分钟");
35
+ expect(note!).toContain("会话等待 30.1 分钟");
36
+ // 关键:不能让人以为 30 分钟就是全部
37
+ expect(note!).toContain("不代表总共等了多久");
38
+ });
39
+
40
+ test("没有被排队的(排队 0):照样把两段都说出来,不省略", () => {
41
+ const note = busyElapsedNote("2026-10-28 12:11:07", "2026-10-28 12:11:08", "2026-10-28 12:41:10");
42
+ expect(note!).toContain("排队 0 分钟");
43
+ expect(note!).toContain("会话等待 30 分钟");
44
+ });
45
+
46
+ test("节点没上报开始处理时刻:只说总计,不硬凑分子", () => {
47
+ const note = busyElapsedNote("2026-10-27 13:54:48", null, "2026-10-27 20:46:57");
48
+ expect(note!).toContain("实测总耗时 412 分钟");
49
+ expect(note!).toContain("无法拆出排队与会话等待");
50
+ expect(note!).not.toContain("会话等待 30");
51
+ });
52
+
53
+ test("时刻缺失或倒挂 ⇒ null(宁可不写,也不编一个负数)", () => {
54
+ expect(busyElapsedNote(null, null, "2026-10-27 20:46:57")).toBeNull();
55
+ expect(busyElapsedNote("2026-10-27 13:54:48", null, null)).toBeNull();
56
+ expect(busyElapsedNote("2026-10-27 20:00:00", null, "2026-10-27 13:00:00")).toBeNull();
57
+ });
58
+
59
+ test("跨库:PG 可能给 Date / 毫秒数,不能静默失效", () => {
60
+ const created = new Date(Date.now() - 412 * 60_000);
61
+ const started = new Date(Date.now() - 30 * 60_000);
62
+ const completed = new Date();
63
+ const fromDates = busyElapsedNote(created, started, completed);
64
+ expect(fromDates).toContain("排队 382 分钟");
65
+ expect(fromDates).toContain("会话等待 30 分钟");
66
+
67
+ const fromNumbers = busyElapsedNote(created.getTime(), started.getTime(), completed.getTime());
68
+ expect(fromNumbers).toContain("排队 382 分钟");
69
+
70
+ // 空串也要当「没有」,不能 parse 成 1970
71
+ expect(busyElapsedNote("", "", "")).toBeNull();
72
+ });
73
+ });
@@ -0,0 +1,105 @@
1
+ // #1879 —— 定时任务失败分类(纯函数,不 import db,便于单测)。
2
+ //
3
+ // 生产 24h(2026-10-28):144 条 failed run 的 error_code 全是 `task_failed`,
4
+ // 但其中 125 条(66%)的真实原因是「共享会话被人的 TUI 占着,任务 30 分钟内没提交上去」。
5
+ // 混在一个桶里,报表分不出来,也没法给它们配不同策略。
6
+ //
7
+ // 只做分类,**不改等待语义**:近 24h 有 108 个任务是等到 >25 分钟才成功的,
8
+ // 「忙就立刻失败」会打掉它们。
9
+ //
10
+ // `failureStreak()` 按 status='failed' 计数(不是按 error_code),所以新增一个 code
11
+ // 不影响「连续 N 次失败告警 / 自动暂停」的阈值。
12
+ export function scheduledFailureCode(status: string, result: string | null): string | null {
13
+ if (status === "replied") return null;
14
+ const base = `task_${status}`;
15
+ if (status !== "failed") return base;
16
+ const text = result ?? "";
17
+ if (/共享会话/.test(text) && /忙碌|一直处于|未获提交/.test(text)) return "task_failed_busy";
18
+ if (/仍在节点的 TUI 会话里运行|等待回复已达\s*30\s*分钟/.test(text)) return "task_failed_busy";
19
+ return base;
20
+ }
21
+
22
+ // #2884 —— 「排了多久」和「会话等了多久」是两个量,别用同一个数字代表。
23
+ //
24
+ // 现场(2026-10-27 实测,tasks 表 created_at → started_at → completed_at):
25
+ // 一条 412 分钟才失败的任务,拆开是 **排队 382 分钟 + 会话等待 30.1 分钟**。
26
+ // 失败文案只说了「30 分钟」,因为 30 分钟是「轮到它之后等共享会话空出来」的预算;
27
+ // 在轮到自己**之前**的排队既没有上限、也没有被写进文案。
28
+ //
29
+ // 后果不是文案不好看,是报表失真:按 error_code 聚合时这 180 条(94%)都被算成
30
+ // 「共享会话忙」,于是「排队几小时」这一层被彻底掩盖 —— 而它才是能操作的那一层
31
+ // (加节点 / 减排程 / 拉间隔),会话忙只能等人离开 TUI。
32
+ //
33
+ // 这里只做**如实测量与上报**,不改等待语义:fail fast 会改变派发行为,
34
+ // 要单独定阈值(排队上限独立成卡)。阈值是不变量,得先有数再选。
35
+
36
+ function parseDbTime(v: string | Date | number | null | undefined): number | null {
37
+ if (v === null || v === undefined || v === "") return null;
38
+ // 跨库:SQLite 给字符串(`YYYY-MM-DD HH:MM:SS`,UTC 无 Z),
39
+ // PG 的 timestamp 列可能直接给 Date 或毫秒数。两种都认,
40
+ // 否则这条投影在 PG 上会静默失效(返回 null ⇒ 不显示拆解),
41
+ // 而「静默失效」比「报错」难发现得多。
42
+ if (v instanceof Date) return Number.isFinite(v.getTime()) ? v.getTime() : null;
43
+ if (typeof v === "number") return Number.isFinite(v) ? v : null;
44
+ // 字符串里的空格要换掉:Date.parse 对 `... 20:16:52` 会按本地时区解释。
45
+ const ms = Date.parse(`${String(v).trim().replace(" ", "T")}Z`);
46
+ return Number.isFinite(ms) ? ms : null;
47
+ }
48
+
49
+ function fmtMinutes(ms: number): string {
50
+ const min = ms / 60_000;
51
+ if (min >= 100) return `${Math.round(min)} 分钟`;
52
+ return `${(Math.round(min * 10) / 10).toString()} 分钟`;
53
+ }
54
+
55
+ /**
56
+ * 把一条失败任务拆成「排队 / 会话等待」两段,供 run 记录与任务详情展示。
57
+ *
58
+ * 读不出时刻就不编:宁可只说总计,也不把两段硬凑成 30 分钟。
59
+ *
60
+ * #2884 —— 为什么排程与非排程任务都要覆盖:
61
+ * 现场那条 412 分钟的任务**不是排程任务**(scheduled_task_runs 里查不到),是直连任务。
62
+ * 只改排程 run 的投影,最该被修的那一类反而看不到拆解。
63
+ */
64
+ export function busyElapsedNote(
65
+ createdAt: string | Date | number | null | undefined,
66
+ startedAt: string | Date | number | null | undefined,
67
+ completedAt: string | Date | number | null | undefined,
68
+ ): string | null {
69
+ const created = parseDbTime(createdAt);
70
+ const completed = parseDbTime(completedAt);
71
+ if (created === null || completed === null) return null;
72
+ const total = completed - created;
73
+ if (!(total >= 0)) return null;
74
+ const started = parseDbTime(startedAt);
75
+ if (started === null) {
76
+ return `实测总耗时 ${fmtMinutes(total)}(节点未上报开始处理的时刻,无法拆出排队与会话等待)`;
77
+ }
78
+ const queued = Math.max(0, started - created);
79
+ const waited = Math.max(0, completed - started);
80
+ return `实测总耗时 ${fmtMinutes(total)}:排队 ${fmtMinutes(queued)}(任务创建→节点开始处理)`
81
+ + ` + 会话等待 ${fmtMinutes(waited)}(节点开始处理→失败)。`
82
+ + `失败文案里的等待上限只代表后一段,不代表总共等了多久。`;
83
+ }
84
+
85
+ /** 幂等标记:已经加过拆解就不再重复追加。 */
86
+ const BUSY_NOTE_MARK = "实测总耗时";
87
+ export const BUSY_NOTE_MARKER = BUSY_NOTE_MARK;
88
+
89
+ /**
90
+ * #2884 —— 在读取投影上补「排队 / 会话等待」拆解(幂等,不改写入路径)。
91
+ *
92
+ * 只对 `task_failed_busy` 生效:其它失败原因的耗时与「排队」无关,
93
+ * 追加会让失败的真正原因更难看清。
94
+ */
95
+ export function withBusyElapsed<T extends {
96
+ status?: string | null; result?: string | null;
97
+ created_at?: string | null; started_at?: string | null; completed_at?: string | null;
98
+ }>(task: T): T {
99
+ if (!task || scheduledFailureCode(task.status ?? null, task.result ?? null) !== "task_failed_busy") return task;
100
+ const current = typeof task.result === "string" ? task.result : null;
101
+ if (current && current.includes(BUSY_NOTE_MARK)) return task; // 已加过
102
+ const note = busyElapsedNote(task.created_at, task.started_at, task.completed_at);
103
+ if (!note) return task;
104
+ return { ...task, result: [current, note].filter(Boolean).join(" ") };
105
+ }
@@ -0,0 +1,279 @@
1
+ // #523 —— 定时任务连续失败:告诉创建者(可选自动暂停)。
2
+ //
3
+ // 钉住:
4
+ // 1. 连续 5 次失败(派出去的任务以 failed 结束)→ 给创建者恰好 1 条通知(别的成员 0 条),第 6–10 次不再发;
5
+ // 通知里有排程名、目标节点、失败次数、最近一次的错误(截断)和怎么暂停。
6
+ // 2. 派发时就失败(目标节点不存在)同样计数。
7
+ // 3. 中间成功一次 → 重新上膛:再连续 5 次失败才发第二条。
8
+ // 4. 一直失败:24 小时内不重发;上次告警早于 24 小时 → 再发一条。
9
+ // 5. 默认不自动暂停;COMMHUB_SCHEDULE_FAILURE_AUTO_PAUSE_RUNS=3 → 第 3 次失败后排程 paused、next_run_at 清空、
10
+ // revision+1,并通知一次。
11
+ // 6. GET /runs 返回 consecutive_failures / failure_alert_threshold / last_failure_alert_at,
12
+ // 任务以 failed 结束的 run 的 error_message 取自任务的回复正文。
13
+ //
14
+ // 跑法:cd server && COMMHUB_DB=/tmp/x.db bun test src/scheduled-failures-http.test.ts
15
+ // PG:COMMHUB_TEST_PG_URL=… COMMHUB_PG_EXPERIMENTAL=1(tests/test2123-hub-postgres-ladder 里注册)
16
+ import { afterAll, afterEach, beforeAll, beforeEach, describe, expect, test } from "bun:test";
17
+ import { McpServer } from "@modelcontextprotocol/sdk/server/mcp.js";
18
+ import { addNetworkMember, createNetworkTokenForNode, register } from "./auth.js";
19
+ import { db } from "./db.js";
20
+ import { dispatchScheduledOccurrence, handleScheduledTaskRequest } from "./scheduled-tasks.js";
21
+ import { SCHEDULE_FAILING_NOTICE_KIND, failureStreak } from "./scheduled-failures.js";
22
+ import { registerTools } from "./tools.js";
23
+
24
+ const activeDbPath = process.env.COMMHUB_DB ?? (process.env.COMMHUB_TEST_PG_URL ? "postgres" : undefined);
25
+ if (!activeDbPath) throw new Error("scheduled-failures requires COMMHUB_DB (or COMMHUB_TEST_PG_URL) before module import");
26
+
27
+ const stamp = Date.now();
28
+ const ALIAS = `fail-node-${stamp}`;
29
+ const NODE_ID = `n_fail_${stamp}`;
30
+ let NET = "", creatorId = "", otherId = "";
31
+ const ENV_KEYS = ["COMMHUB_SCHEDULE_FAILURE_NOTICE_RUNS", "COMMHUB_SCHEDULE_FAILURE_RENOTICE_SEC", "COMMHUB_SCHEDULE_FAILURE_AUTO_PAUSE_RUNS"];
32
+ const SAFETY_ERROR = "执行出错: upstream model refused the request (safety system). " + "x".repeat(600);
33
+
34
+ function seedSchedule(name = "前后端分离巡检", targetNodeId = NODE_ID): any {
35
+ const scheduleId = `sched_${crypto.randomUUID()}`;
36
+ db.run(
37
+ `INSERT INTO scheduled_tasks
38
+ (schedule_id, network_id, created_by, name, target_node_id, target_alias, task_content, priority,
39
+ schedule_type, schedule_json, timezone, status, next_run_at)
40
+ VALUES (?1, ?2, ?3, ?4, ?5, ?6, 'scheduled task', 'normal', 'interval',
41
+ '{"type":"interval","every_seconds":120}', 'Asia/Shanghai', 'active', datetime('now', '+2 minutes'))`,
42
+ [scheduleId, NET, creatorId, name, targetNodeId, ALIAS],
43
+ );
44
+ return reload(scheduleId);
45
+ }
46
+ const reload = (scheduleId: string) => db.get<any>("SELECT * FROM scheduled_tasks WHERE schedule_id = ?1", scheduleId);
47
+
48
+ let slot = 0;
49
+ const nextSlot = () => new Date(Date.UTC(2026, 9, 4, 0, 0, 0) + (++slot) * 120_000).toISOString();
50
+ const fire = (schedule: any, now = new Date()) => dispatchScheduledOccurrence(schedule, nextSlot(), false, now);
51
+
52
+ const notices = (userId: string) => db.all<{ title: string; content: string; meta_json: string; from_session: string; severity: string }>(
53
+ "SELECT title, content, meta_json, from_session, severity FROM user_inbox WHERE user_id = ?1 AND network_id = ?2 AND kind = ?3 ORDER BY created_at, message_id",
54
+ userId, NET, SCHEDULE_FAILING_NOTICE_KIND,
55
+ );
56
+ const noticesFor = (userId: string, scheduleId: string) => notices(userId).filter((n) => JSON.parse(n.meta_json).schedule_failing.schedule_id === scheduleId);
57
+
58
+ async function nodeReply(args: Record<string, unknown>): Promise<any> {
59
+ const mcp = new McpServer({ name: "sched-fail", version: "0" }) as any;
60
+ const handlers: Record<string, (a: any) => Promise<any>> = {};
61
+ const original = mcp.tool.bind(mcp);
62
+ mcp.tool = (name: string, ...rest: any[]) => {
63
+ const handler = rest.at(-1);
64
+ if (typeof handler === "function") handlers[name] = handler;
65
+ return original(name, ...rest);
66
+ };
67
+ registerTools(mcp, undefined, NET, creatorId, ALIAS, true, null);
68
+ const out = await handlers.send_reply(args);
69
+ return JSON.parse(out.content[0].text);
70
+ }
71
+
72
+ /** 派一次,节点以 failed(或 replied)回复。 */
73
+ async function runOnce(schedule: any, outcome: "failed" | "replied" = "failed"): Promise<{ runId: string; taskId: string }> {
74
+ const r = fire(schedule);
75
+ expect(r.status).toBe("delivered");
76
+ const out = await nodeReply({ alias: "scheduler", text: outcome === "failed" ? SAFETY_ERROR : "done", in_reply_to: r.taskId, status: outcome, from_session: ALIAS });
77
+ expect(out.ok).toBe(true);
78
+ return { runId: r.runId, taskId: r.taskId! };
79
+ }
80
+
81
+ async function getRuns(scheduleId: string): Promise<any> {
82
+ const url = new URL(`http://hub.test/api/scheduled-tasks/${scheduleId}/runs`);
83
+ const res = await handleScheduledTaskRequest({
84
+ req: new Request(url, { method: "GET" }), url,
85
+ auth: { userId: creatorId, networkId: NET, username: "creator" },
86
+ isAdmin: false, isNodeToken: false, scope: { networkId: NET, networkIds: [NET] } as any,
87
+ });
88
+ expect(res!.status).toBe(200);
89
+ return res!.json();
90
+ }
91
+
92
+ beforeAll(() => {
93
+ const creator = register(`fail_creator_${stamp}`, "FailCreator123!", undefined, "seed");
94
+ expect(creator.ok).toBe(true);
95
+ NET = creator.network_id!; creatorId = creator.user!.user_id;
96
+ const other = register(`fail_other_${stamp}`, "FailOther123!", undefined, "seed");
97
+ otherId = other.user!.user_id;
98
+ expect(addNetworkMember(NET, otherId, "member", creatorId, { agentAccess: "all" }).ok).toBe(true);
99
+ expect(createNetworkTokenForNode(creatorId, NET, ALIAS).ok).toBe(true);
100
+ db.run("INSERT INTO nodes (node_id, node_name, alias, runtime, network_id) VALUES (?1, ?2, ?2, 'claude-code', ?3)", [NODE_ID, ALIAS, NET]);
101
+ db.run("INSERT INTO sessions (resume_id, alias, status, node_id, network_id) VALUES (?1, ?2, 'idle', ?3, ?4)", [`r_fail_${stamp}`, ALIAS, NODE_ID, NET]);
102
+ });
103
+ beforeEach(() => { for (const k of ENV_KEYS) delete process.env[k]; });
104
+ afterEach(() => { for (const k of ENV_KEYS) delete process.env[k]; });
105
+ afterAll(() => { db.run("DELETE FROM user_inbox WHERE network_id = ?1 AND kind = ?2", [NET, SCHEDULE_FAILING_NOTICE_KIND]); });
106
+
107
+ describe("#523 consecutive-failure alert", () => {
108
+ test("5 failed runs → exactly one notice to the creator with name/target/count/error/how to pause; runs 6–10 stay quiet", async () => {
109
+ const s = seedSchedule();
110
+ for (let i = 0; i < 4; i++) await runOnce(s);
111
+ expect(noticesFor(creatorId, s.schedule_id).length).toBe(0);
112
+ await runOnce(s);
113
+ const mine = noticesFor(creatorId, s.schedule_id);
114
+ expect(mine.length).toBe(1);
115
+ expect(noticesFor(otherId, s.schedule_id).length).toBe(0);
116
+ const n = mine[0];
117
+ expect(n.from_session).toBe(ALIAS);
118
+ expect(n.title).toBe("定时任务连续失败");
119
+ expect(n.severity).toBe("error");
120
+ for (const frag of ["前后端分离巡检", ALIAS, "连续 5 次", "safety system", "task_failed", "暂停", s.schedule_id]) expect(n.content).toContain(frag);
121
+ expect(n.content).not.toContain("x".repeat(400)); // 错误被截断
122
+ expect(JSON.parse(n.meta_json).schedule_failing).toEqual({ reason: "failing", schedule_id: s.schedule_id, consecutive_failures: 5, error_code: "task_failed" });
123
+ for (let i = 0; i < 5; i++) await runOnce(s);
124
+ expect(noticesFor(creatorId, s.schedule_id).length).toBe(1);
125
+ expect(reload(s.schedule_id).status).toBe("active"); // 默认不自动暂停
126
+ });
127
+
128
+ test("dispatch-time failures (target node gone) count too", () => {
129
+ const s = seedSchedule("孤儿排程", `n_missing_${stamp}`);
130
+ for (let i = 0; i < 5; i++) expect(fire(s).status).toBe("failed");
131
+ const mine = noticesFor(creatorId, s.schedule_id);
132
+ expect(mine.length).toBe(1);
133
+ expect(mine[0].content).toContain("target_node_not_found");
134
+ expect(mine[0].content).toContain("The bound node no longer exists");
135
+ });
136
+
137
+ test("a success in between re-arms: the next 5 failures notify again", async () => {
138
+ const s = seedSchedule();
139
+ for (let i = 0; i < 5; i++) await runOnce(s);
140
+ expect(noticesFor(creatorId, s.schedule_id).length).toBe(1);
141
+ await runOnce(s, "replied");
142
+ expect(failureStreak(s.schedule_id).consecutiveFailures).toBe(0);
143
+ for (let i = 0; i < 4; i++) await runOnce(s);
144
+ expect(noticesFor(creatorId, s.schedule_id).length).toBe(1);
145
+ await runOnce(s);
146
+ expect(noticesFor(creatorId, s.schedule_id).length).toBe(2);
147
+ });
148
+
149
+ test("still failing: no repeat within 24h; one more once the last alert is older than 24h", async () => {
150
+ const s = seedSchedule();
151
+ for (let i = 0; i < 5; i++) await runOnce(s);
152
+ expect(noticesFor(creatorId, s.schedule_id).length).toBe(1);
153
+ db.run("UPDATE scheduled_tasks SET failure_alert_at = ?1 WHERE schedule_id = ?2", [new Date(Date.now() - 23 * 3_600_000).toISOString(), s.schedule_id]);
154
+ await runOnce(s);
155
+ expect(noticesFor(creatorId, s.schedule_id).length).toBe(1);
156
+ db.run("UPDATE scheduled_tasks SET failure_alert_at = ?1 WHERE schedule_id = ?2", [new Date(Date.now() - 25 * 3_600_000).toISOString(), s.schedule_id]);
157
+ await runOnce(s);
158
+ expect(noticesFor(creatorId, s.schedule_id).length).toBe(2);
159
+ await runOnce(s);
160
+ expect(noticesFor(creatorId, s.schedule_id).length).toBe(2);
161
+ });
162
+
163
+ test("COMMHUB_SCHEDULE_FAILURE_NOTICE_RUNS changes N; no creator → no notice", async () => {
164
+ process.env.COMMHUB_SCHEDULE_FAILURE_NOTICE_RUNS = "2";
165
+ const s = seedSchedule();
166
+ await runOnce(s);
167
+ expect(noticesFor(creatorId, s.schedule_id).length).toBe(0);
168
+ await runOnce(s);
169
+ expect(noticesFor(creatorId, s.schedule_id).length).toBe(1);
170
+
171
+ const orphan = seedSchedule();
172
+ db.run("UPDATE scheduled_tasks SET created_by = NULL WHERE schedule_id = ?1", [orphan.schedule_id]);
173
+ const row = reload(orphan.schedule_id);
174
+ for (let i = 0; i < 3; i++) await runOnce(row);
175
+ expect(noticesFor(creatorId, orphan.schedule_id).length).toBe(0);
176
+ });
177
+ });
178
+
179
+ describe("#523 auto-pause (opt-in)", () => {
180
+ test("off by default: 10 failures leave the schedule active", async () => {
181
+ const s = seedSchedule();
182
+ for (let i = 0; i < 10; i++) await runOnce(s);
183
+ const after = reload(s.schedule_id);
184
+ expect(after.status).toBe("active");
185
+ expect(after.next_run_at).not.toBeNull();
186
+ });
187
+
188
+ test("COMMHUB_SCHEDULE_FAILURE_AUTO_PAUSE_RUNS=3 pauses after the 3rd failure and tells the creator once", async () => {
189
+ process.env.COMMHUB_SCHEDULE_FAILURE_AUTO_PAUSE_RUNS = "3";
190
+ const s = seedSchedule();
191
+ await runOnce(s); await runOnce(s);
192
+ expect(reload(s.schedule_id).status).toBe("active");
193
+ await runOnce(s);
194
+ const after = reload(s.schedule_id);
195
+ expect(after.status).toBe("paused");
196
+ expect(after.next_run_at).toBeNull();
197
+ expect(after.revision).toBe(s.revision + 1);
198
+ const mine = noticesFor(creatorId, s.schedule_id);
199
+ expect(mine.length).toBe(1);
200
+ expect(mine[0].title).toBe("定时任务连续失败,已自动暂停");
201
+ expect(JSON.parse(mine[0].meta_json).schedule_failing.reason).toBe("paused");
202
+ // 手动 run-now 再失败两次到 N=5:同一段失败,已经通知过 → 不再发
203
+ await runOnce(after); await runOnce(after);
204
+ expect(noticesFor(creatorId, s.schedule_id).length).toBe(1);
205
+ });
206
+ });
207
+
208
+ describe("#523 GET /runs", () => {
209
+ test("exposes consecutive_failures and the failed task's reason as error_message", async () => {
210
+ const s = seedSchedule();
211
+ await runOnce(s, "replied");
212
+ const fails = [await runOnce(s), await runOnce(s)];
213
+ const body = await getRuns(s.schedule_id);
214
+ expect(body.ok).toBe(true);
215
+ expect(body.consecutive_failures).toBe(2);
216
+ expect(body.failure_alert_threshold).toBe(5);
217
+ expect(body.last_failure_alert_at).toBeNull();
218
+ expect(body.runs.length).toBe(3);
219
+ const failed = body.runs.find((r: any) => r.run_id === fails[1].runId);
220
+ expect(failed.status).toBe("failed");
221
+ expect(failed.error_code).toBe("task_failed");
222
+ expect(failed.error_message).toContain("safety system");
223
+ expect(failed.error_message.length).toBeLessThanOrEqual(500);
224
+ for (const k of ["scheduled_for", "created_at", "completed_at", "task_id"]) expect(k in failed).toBe(true);
225
+ const ok = body.runs.find((r: any) => r.status === "replied");
226
+ expect(ok.error_message).toBeNull();
227
+
228
+ for (let i = 0; i < 3; i++) await runOnce(s);
229
+ const again = await getRuns(s.schedule_id);
230
+ expect(again.consecutive_failures).toBe(5);
231
+ expect(typeof again.last_failure_alert_at).toBe("string");
232
+ });
233
+ });
234
+
235
+ // #2884 —— 共享会话忙:run 记录要把「排队」和「会话等待」两个量分开报。
236
+ // 生产实测(2026-10-27, TMA需求欧):一条 412 分钟才失败的任务,拆开是排队 382 + 会话等待 30。
237
+ // 文案只说过「30 分钟」,于是报表把所有长等待都算成「会话忙」,掩盖了「排队几小时」这一层。
238
+ describe("#2884 busy runs report queue and session wait separately", () => {
239
+ const BUSY = "[TMA需求欧] opencode 任务未提交:共享会话在 30 分钟 内一直处于忙碌状态(可能有人正在 TUI 里跑一轮),本任务没有发出";
240
+
241
+ test("error_message carries both numbers; the node's own wording is kept", async () => {
242
+ const s = seedSchedule("忙的情况");
243
+ const r = fire(s);
244
+ expect(r.status).toBe("delivered");
245
+ // 造出「排了 382 分钟、轮到后等了 30 分钟」:把创建时刻前移 412 分钟、开始处理前移 30 分钟,
246
+ // 回复会把 completed_at 设成现在。
247
+ // 时间用 JS 算再传参:`datetime('now','-N minutes')` 是 SQLite 方言,
248
+ // 这套用例在 PG ladder 里也跑(test2123),方言会让它直接报错 (#2884)。
249
+ const minutesAgo = (m: number) => new Date(Date.now() - m * 60_000).toISOString().replace("T", " ").replace(/\..+/, "");
250
+ db.run(
251
+ "UPDATE tasks SET created_at = ?2, started_at = ?3 WHERE task_id = ?1",
252
+ [r.taskId, minutesAgo(412), minutesAgo(30)],
253
+ );
254
+ const out = await nodeReply({ alias: "scheduler", text: BUSY, in_reply_to: r.taskId, status: "failed", from_session: ALIAS });
255
+ expect(out.ok).toBe(true);
256
+
257
+ const body = await getRuns(s.schedule_id);
258
+ const run = body.runs.find((x: any) => x.run_id === r.runId);
259
+ expect(run.status).toBe("failed");
260
+ expect(run.error_code).toBe("task_failed_busy");
261
+ // 两个量分开
262
+ expect(run.error_message).toContain("排队 382 分钟");
263
+ expect(run.error_message).toContain("会话等待 30 分钟");
264
+ // 节点原文没有被读数覆盖
265
+ expect(run.error_message).toContain("共享会话");
266
+ // 不能让人把 30 分钟读成总时长
267
+ expect(run.error_message).toContain("不代表总共等了多久");
268
+ });
269
+
270
+ test("not-busy failures never get the breakdown appended", async () => {
271
+ const s = seedSchedule("真失败");
272
+ const { runId } = await runOnce(s);
273
+ const body = await getRuns(s.schedule_id);
274
+ const run = body.runs.find((x: any) => x.run_id === runId);
275
+ expect(run.error_code).toBe("task_failed");
276
+ expect(run.error_message).toContain("safety system");
277
+ expect(run.error_message).not.toContain("排队");
278
+ });
279
+ });