@sleep2agi/commhub-server 0.9.0-preview.28 → 0.9.0-preview.281

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (482) hide show
  1. package/README.md +120 -3
  2. package/bin/commhub.ts +93 -9
  3. package/build-info.json +7 -0
  4. package/package.json +10 -8
  5. package/src/ack-create-request-transport.test.ts +182 -0
  6. package/src/ack-create-request.test.ts +52 -8
  7. package/src/admin-delete-network-http.test.ts +229 -0
  8. package/src/admin-networks-http.test.ts +88 -0
  9. package/src/adoption-candidates-http.test.ts +140 -0
  10. package/src/adoption-candidates.ts +158 -0
  11. package/src/agent-access-batch.test.ts +215 -0
  12. package/src/agent-access.ts +585 -0
  13. package/src/agent-acl-groups-http.test.ts +247 -0
  14. package/src/agent-acl-http.test.ts +691 -0
  15. package/src/agent-acl-slice3-http.test.ts +888 -0
  16. package/src/agent-department-grants.test.ts +129 -0
  17. package/src/agent-department-http.test.ts +215 -0
  18. package/src/agent-groups-perf.test.ts +228 -0
  19. package/src/agent-notice.ts +48 -0
  20. package/src/agent-team-whoami-http.test.ts +98 -0
  21. package/src/agent-teams-http.test.ts +330 -0
  22. package/src/agent-teams.ts +244 -0
  23. package/src/alias-baseline-2571.test.ts +162 -0
  24. package/src/alias-baseline.ts +113 -0
  25. package/src/alias-filter.test.ts +72 -0
  26. package/src/alias-filter.ts +42 -0
  27. package/src/all-status-cache.test.ts +31 -0
  28. package/src/all-status-cache.ts +37 -0
  29. package/src/anet-variant-real-daemon-http.test.ts +185 -0
  30. package/src/api-host-supervisors-can-create-nodes.test.ts +350 -0
  31. package/src/api-host-supervisors-fallback.test.ts +6 -7
  32. package/src/api-host-supervisors-runtime-readiness.test.ts +195 -0
  33. package/src/api-nodes-shape.test.ts +131 -4
  34. package/src/auth-abuse-counters.test.ts +144 -0
  35. package/src/auth-abuse-counters.ts +122 -0
  36. package/src/auth-layer-shape.test.ts +105 -0
  37. package/src/auth-password-change-cli-tokens-http.test.ts +423 -0
  38. package/src/auth-sessions-http.test.ts +260 -0
  39. package/src/auth-validate.test.ts +5 -1
  40. package/src/auth-weak-password-login.test.ts +176 -0
  41. package/src/auth.ts +641 -68
  42. package/src/backfill-node-ids-2664.test.ts +128 -0
  43. package/src/binding-generation-http.test.ts +141 -0
  44. package/src/blocked-exit-on-reply.test.ts +122 -0
  45. package/src/broadcast-fanout-2721.test.ts +149 -0
  46. package/src/broadcast-fanout.ts +117 -0
  47. package/src/build-commit-report-http.test.ts +219 -0
  48. package/src/build-info.test.ts +60 -0
  49. package/src/build-info.ts +85 -0
  50. package/src/chain-attempt-dedup.test.ts +447 -0
  51. package/src/clip.test.ts +36 -0
  52. package/src/clip.ts +48 -0
  53. package/src/clone-node.test.ts +269 -0
  54. package/src/clone-node.ts +289 -0
  55. package/src/codex-config-flags.ts +77 -0
  56. package/src/codex-fork-contract.ts +23 -0
  57. package/src/codex-fork-migration.test.ts +38 -0
  58. package/src/commhub-bin-args.test.ts +85 -0
  59. package/src/config-apply-sec1.test.ts +37 -0
  60. package/src/config-apply-validate.test.ts +188 -1
  61. package/src/config-apply-validate.ts +226 -3
  62. package/src/contracts/session-project-create.ts +14 -0
  63. package/src/contracts/session-source-delivery.ts +16 -0
  64. package/src/contracts/session-source-directory.ts +54 -0
  65. package/src/contracts/session-source-evidence.ts +72 -0
  66. package/src/contracts/session-source-execution.ts +24 -0
  67. package/src/contracts/session-source-transport.ts +18 -0
  68. package/src/cors-origins.test.ts +73 -0
  69. package/src/create-node-blocked-gate.test.ts +365 -0
  70. package/src/create-node-error-domain.test.ts +127 -0
  71. package/src/create-node-network-id.test.ts +91 -0
  72. package/src/create-node-tool-schema.test.ts +59 -0
  73. package/src/create-node-unicode-name.test.ts +126 -0
  74. package/src/create-node-validate.test.ts +199 -7
  75. package/src/create-node-validate.ts +232 -15
  76. package/src/create-node-workdir.test.ts +298 -0
  77. package/src/create-node.ts +88 -23
  78. package/src/daemon-provider-snapshots.test.ts +453 -0
  79. package/src/daemon-provider-snapshots.ts +327 -0
  80. package/src/daemon-token-owner-http.test.ts +346 -0
  81. package/src/dashboard-slash-routing-http.test.ts +109 -0
  82. package/src/db-adapter-guard.test.ts +148 -0
  83. package/src/db-adapter-translate.test.ts +128 -0
  84. package/src/db-adapter.ts +377 -91
  85. package/src/db-timestamp.test.ts +38 -0
  86. package/src/db-timestamp.ts +15 -0
  87. package/src/db.ts +1334 -154
  88. package/src/delivered-without-push.test.ts +140 -0
  89. package/src/department-groups-http.test.ts +209 -0
  90. package/src/department-groups-sync-http.test.ts +328 -0
  91. package/src/department-groups.ts +434 -0
  92. package/src/department-heads-http.test.ts +350 -0
  93. package/src/department-heads.ts +226 -0
  94. package/src/departments-agent-count.test.ts +139 -0
  95. package/src/departments-http.test.ts +185 -0
  96. package/src/departments-mcp-error-domain.test.ts +135 -0
  97. package/src/departments-mcp.test.ts +128 -0
  98. package/src/departments.ts +318 -0
  99. package/src/desktop-message-seam.test.ts +150 -0
  100. package/src/desktop-message.test.ts +368 -0
  101. package/src/dm-file-scope-http.test.ts +227 -0
  102. package/src/empty-node-sweeper.test.ts +104 -0
  103. package/src/empty-node-sweeper.ts +88 -0
  104. package/src/env-transport-gate-http.test.ts +240 -0
  105. package/src/env-transport-gate.ts +80 -0
  106. package/src/env-transport-policy.ts +56 -0
  107. package/src/etag-of.test.ts +57 -0
  108. package/src/event-loop-lag.test.ts +180 -0
  109. package/src/event-loop-lag.ts +201 -0
  110. package/src/expiry-counters.test.ts +187 -0
  111. package/src/expiry-counters.ts +227 -0
  112. package/src/expiry-evidence-db.test.ts +103 -0
  113. package/src/ext-token-shared.test.ts +60 -0
  114. package/src/ext-validation-regression.test.ts +10 -17
  115. package/src/external-schedule-edits-http.test.ts +321 -0
  116. package/src/external-schedule-edits-timestamps.test.ts +64 -0
  117. package/src/external-schedule-edits.ts +327 -0
  118. package/src/file-download-authz-dev-open.test.ts +1 -1
  119. package/src/file-download-authz.test.ts +25 -4
  120. package/src/file-download-date-bucket.test.ts +1 -1
  121. package/src/file-network-scope-dev-open.test.ts +1 -1
  122. package/src/file-network-scope.test.ts +7 -6
  123. package/src/group-membership-events-http.test.ts +264 -0
  124. package/src/group-messages-http.test.ts +483 -0
  125. package/src/group-messages.ts +297 -0
  126. package/src/health-anon-readers.ts +67 -0
  127. package/src/health-host-disk-2756.test.ts +71 -0
  128. package/src/health-identity-root.test.ts +98 -0
  129. package/src/health-redaction.test.ts +1603 -7
  130. package/src/health-status-reasons.test.ts +84 -0
  131. package/src/host-supervisor-identity-http.test.ts +152 -0
  132. package/src/host-supervisor-identity.ts +92 -0
  133. package/src/host-supervisors-daemon-version.test.ts +137 -0
  134. package/src/http-gzip.test.ts +94 -0
  135. package/src/http-gzip.ts +112 -0
  136. package/src/hub-catalog-read-http.test.ts +664 -0
  137. package/src/hub-provider-config.test.ts +166 -0
  138. package/src/hub-provider-config.ts +202 -0
  139. package/src/hub-timestamp.test.ts +38 -0
  140. package/src/hub-timestamp.ts +27 -0
  141. package/src/human-dm.ts +215 -0
  142. package/src/human-inbox.ts +42 -0
  143. package/src/inbox-caller.ts +49 -0
  144. package/src/inbox-count-sse.test.ts +318 -0
  145. package/src/inbox-count.ts +29 -0
  146. package/src/index.ts +6 -2
  147. package/src/launched-by-cli-report-http.test.ts +155 -0
  148. package/src/license-state.test.ts +105 -0
  149. package/src/license-state.ts +48 -0
  150. package/src/lifecycle-error-domain.test.ts +149 -0
  151. package/src/list-ack-error-domain.test.ts +108 -0
  152. package/src/list-host-supervisors.test.ts +15 -4
  153. package/src/list-tasks-omit-default.test.ts +84 -0
  154. package/src/list-tasks-since-last.test.ts +100 -0
  155. package/src/list-tasks-since.test.ts +104 -0
  156. package/src/mcp-context-cost-http.test.ts +127 -0
  157. package/src/mcp-inflight.test.ts +92 -0
  158. package/src/mcp-inflight.ts +38 -0
  159. package/src/mcp-projects-events-http.test.ts +168 -0
  160. package/src/mcp-route-label-http.test.ts +120 -0
  161. package/src/mcp-write-network-resolution.test.ts +21 -0
  162. package/src/member-audit-network-id-http.test.ts +93 -0
  163. package/src/member-presence-http.test.ts +210 -0
  164. package/src/messages-ack-by-agent-http.test.ts +142 -0
  165. package/src/migrate-sqlite-to-pg.test.ts +47 -0
  166. package/src/migrate-sqlite-to-pg.ts +263 -0
  167. package/src/model-auth-notify-http.test.ts +181 -0
  168. package/src/model-auth-notify.ts +116 -0
  169. package/src/network-name-http.test.ts +75 -0
  170. package/src/network-quota-config.test.ts +100 -0
  171. package/src/network-rules-http.test.ts +169 -0
  172. package/src/network-rules.ts +129 -0
  173. package/src/network-scope.ts +222 -3
  174. package/src/network-token-membership-http.test.ts +186 -0
  175. package/src/node-alias-control.ts +12 -0
  176. package/src/node-attrs.test.ts +47 -2
  177. package/src/node-caller-identity-http.test.ts +1108 -0
  178. package/src/node-config-last-update.test.ts +126 -0
  179. package/src/node-config-last-update.ts +107 -0
  180. package/src/node-create-request-status-http.test.ts +105 -0
  181. package/src/node-daemon-bindings.test.ts +142 -0
  182. package/src/node-daemon-bindings.ts +135 -0
  183. package/src/node-failure-alert.test.ts +150 -0
  184. package/src/node-failure-alert.ts +177 -0
  185. package/src/node-files-transport.test.ts +350 -0
  186. package/src/node-health-codex-login.test.ts +120 -0
  187. package/src/node-health-dispatch-http.test.ts +373 -0
  188. package/src/node-health-guard-origin.test.ts +98 -0
  189. package/src/node-health-guard.ts +121 -0
  190. package/src/node-health-passthrough.test.ts +115 -0
  191. package/src/node-health-store.ts +101 -0
  192. package/src/node-id-alias.test.ts +190 -0
  193. package/src/node-identity-conflict-http.test.ts +269 -0
  194. package/src/node-identity-conflict.ts +142 -0
  195. package/src/node-identity-mismatch.test.ts +74 -0
  196. package/src/node-in-flight-tasks.ts +31 -0
  197. package/src/node-inbox-allowlist-http.test.ts +320 -0
  198. package/src/node-lifecycle-controllable.test.ts +64 -0
  199. package/src/node-lifecycle-controllable.ts +53 -0
  200. package/src/node-lifecycle-read-http.test.ts +257 -0
  201. package/src/node-lifecycle-read.ts +126 -0
  202. package/src/node-logs-transport.test.ts +266 -0
  203. package/src/node-peer-allowlist.test.ts +114 -0
  204. package/src/node-permissions-http.test.ts +901 -0
  205. package/src/node-permissions.ts +627 -0
  206. package/src/node-request-retention.test.ts +281 -0
  207. package/src/node-request-retention.ts +120 -0
  208. package/src/node-secret-rest-http.test.ts +190 -0
  209. package/src/node-secret-vault-inject.test.ts +132 -0
  210. package/src/node-security-write-gate.test.ts +72 -0
  211. package/src/node-serviceability.test.ts +86 -0
  212. package/src/node-serviceability.ts +124 -0
  213. package/src/node-session-control-http.test.ts +398 -0
  214. package/src/node-session-control.ts +316 -0
  215. package/src/node-skills-transport.test.ts +212 -0
  216. package/src/node-token-identity.ts +35 -0
  217. package/src/node-token-ownership.ts +51 -0
  218. package/src/node-token-privilege-http.test.ts +224 -0
  219. package/src/node-token-usage.test.ts +128 -0
  220. package/src/node-token-usage.ts +228 -0
  221. package/src/not-found-api-http.test.ts +67 -0
  222. package/src/observer-avatar-http.test.ts +103 -10
  223. package/src/observer-push.test.ts +76 -0
  224. package/src/opencode-catalog-report-http.test.ts +139 -0
  225. package/src/opencode-v2-catalog.test.ts +282 -0
  226. package/src/opencode-v2-catalog.ts +357 -0
  227. package/src/package-self-contained.test.ts +32 -0
  228. package/src/package-version-consistency.test.ts +25 -0
  229. package/src/parent-infer.test.ts +82 -0
  230. package/src/parent-infer.ts +110 -0
  231. package/src/parent-source-marker.test.ts +70 -0
  232. package/src/parent-source-orphan.test.ts +121 -0
  233. package/src/password-network-tokens.ts +25 -0
  234. package/src/peer-reply-atomic.test.ts +462 -0
  235. package/src/peer-reply-capability.test.ts +70 -0
  236. package/src/pg-worker.ts +87 -0
  237. package/src/poll-pacing.test.ts +74 -0
  238. package/src/poll-pacing.ts +123 -0
  239. package/src/probe-provider-error-domain.test.ts +145 -0
  240. package/src/project-dir-warn-clear-http.test.ts +132 -0
  241. package/src/project-viewer-can-http.test.ts +116 -0
  242. package/src/provider-skill-error-domain.test.ts +130 -0
  243. package/src/provider-tools-shape.test.ts +165 -0
  244. package/src/push.test.ts +20 -10
  245. package/src/push.ts +373 -9
  246. package/src/redact-tokens.test.ts +86 -0
  247. package/src/redact-tokens.ts +43 -0
  248. package/src/rejection-counters.test.ts +72 -0
  249. package/src/rejection-counters.ts +173 -0
  250. package/src/rename.ts +38 -10
  251. package/src/reply-target-derivation.test.ts +96 -0
  252. package/src/report-status-handover-capabilities-http.test.ts +124 -0
  253. package/src/report-status-host-schema.test.ts +272 -0
  254. package/src/report-status-resume-id-handover.test.ts +184 -0
  255. package/src/report-status-secret-keys.test.ts +140 -0
  256. package/src/require-explicit-test-db.ts +9 -0
  257. package/src/requirement-due-reminders-http.test.ts +620 -0
  258. package/src/requirement-due-reminders.ts +416 -0
  259. package/src/requirement-events-http.test.ts +318 -0
  260. package/src/requirement-events.test.ts +45 -0
  261. package/src/requirement-events.ts +144 -0
  262. package/src/requirement-notify.ts +165 -0
  263. package/src/requirement-participant-notify-http.test.ts +313 -0
  264. package/src/requirement-tag-ops-http.test.ts +454 -0
  265. package/src/requirement-tag-tree-http.test.ts +275 -0
  266. package/src/requirement-tag-tree.ts +127 -0
  267. package/src/requirement-tags.ts +102 -0
  268. package/src/requirements-abandoned-http.test.ts +236 -0
  269. package/src/requirements-assignee-drift.test.ts +154 -0
  270. package/src/requirements-comment-http.test.ts +174 -0
  271. package/src/requirements-completed-at-migration.test.ts +70 -0
  272. package/src/requirements-errors-mcp-http.test.ts +164 -0
  273. package/src/requirements-errors.ts +158 -0
  274. package/src/requirements-etag-http.test.ts +91 -0
  275. package/src/requirements-http.test.ts +709 -0
  276. package/src/requirements-last-event-http.test.ts +349 -0
  277. package/src/requirements-list-mcp-slim-http.test.ts +155 -0
  278. package/src/requirements-list-slim-http.test.ts +324 -0
  279. package/src/requirements-mcp.test.ts +180 -0
  280. package/src/requirements-migrate.ts +283 -0
  281. package/src/requirements-owner-strict-mcp-http.test.ts +145 -0
  282. package/src/requirements-people-http.test.ts +114 -0
  283. package/src/requirements-people-mcp-http.test.ts +231 -0
  284. package/src/requirements-people.ts +196 -0
  285. package/src/requirements-priority-lowest.test.ts +114 -0
  286. package/src/requirements-projects-network-echo.test.ts +57 -0
  287. package/src/requirements-projects-parent-http.test.ts +123 -0
  288. package/src/requirements-search-http.test.ts +182 -0
  289. package/src/requirements-search.test.ts +32 -0
  290. package/src/requirements-search.ts +145 -0
  291. package/src/requirements-seq-http.test.ts +193 -0
  292. package/src/requirements-seq-migration.test.ts +63 -0
  293. package/src/requirements-start-date.test.ts +125 -0
  294. package/src/requirements-stats-http.test.ts +357 -0
  295. package/src/requirements-stats.ts +195 -0
  296. package/src/requirements-status-participants-http.test.ts +222 -0
  297. package/src/requirements-upgrade-e2e.test.ts +126 -0
  298. package/src/requirements-upgrade-migration.test.ts +98 -0
  299. package/src/requirements.ts +2205 -0
  300. package/src/resolve-port.test.ts +57 -0
  301. package/src/resolve-port.ts +44 -0
  302. package/src/rest-alias-network-fallback-http.test.ts +123 -0
  303. package/src/rest-alias-network-fallback.test.ts +60 -0
  304. package/src/rest-alias-network-fallback.ts +65 -0
  305. package/src/rest-explicit-columns-http.test.ts +266 -0
  306. package/src/rest-identity.test.ts +130 -0
  307. package/src/rest-identity.ts +141 -0
  308. package/src/rest-projections.ts +72 -0
  309. package/src/rest-read-network-real-chain.test.ts +97 -0
  310. package/src/rest-write-network-resolution.test.ts +133 -0
  311. package/src/restart-readiness-http.test.ts +121 -0
  312. package/src/restricted-files.ts +76 -0
  313. package/src/retention-first-sweep-quiet.test.ts +31 -0
  314. package/src/retention.test.ts +100 -40
  315. package/src/retention.ts +423 -104
  316. package/src/route-callers-http.test.ts +120 -0
  317. package/src/route-timing.test.ts +218 -0
  318. package/src/route-timing.ts +262 -0
  319. package/src/rules-file-duplicate-alias.test.ts +234 -0
  320. package/src/rules-file-session-target.test.ts +207 -0
  321. package/src/rules-file-transport.test.ts +280 -0
  322. package/src/runtime-label.test.ts +22 -0
  323. package/src/runtime-label.ts +21 -0
  324. package/src/runtime-readiness.ts +90 -0
  325. package/src/schedule-agent-mcp-http.test.ts +486 -0
  326. package/src/schedule-agent.ts +252 -0
  327. package/src/schedule-starved.test.ts +92 -0
  328. package/src/schedule-starved.ts +114 -0
  329. package/src/scheduled-backoff-http.test.ts +203 -0
  330. package/src/scheduled-backoff.ts +64 -0
  331. package/src/scheduled-doorbell-nodeless-session.test.ts +134 -0
  332. package/src/scheduled-failure-backoff-2622.test.ts +145 -0
  333. package/src/scheduled-failure-code.test.ts +73 -0
  334. package/src/scheduled-failure-code.ts +105 -0
  335. package/src/scheduled-failures-http.test.ts +279 -0
  336. package/src/scheduled-failures.ts +425 -0
  337. package/src/scheduled-noprogress-backoff-2622.test.ts +163 -0
  338. package/src/scheduled-offline-backoff-probe.test.ts +56 -0
  339. package/src/scheduled-offline.test.ts +131 -0
  340. package/src/scheduled-offline.ts +66 -0
  341. package/src/scheduled-outcome-guidance-http.test.ts +139 -0
  342. package/src/scheduled-outcome-guidance.ts +9 -0
  343. package/src/scheduled-reply-unread-http.test.ts +212 -0
  344. package/src/scheduled-requirement-wake-http.test.ts +289 -0
  345. package/src/scheduled-requirement-wake.ts +125 -0
  346. package/src/scheduled-run-terminal.test.ts +290 -0
  347. package/src/scheduled-skip-blocking-task.test.ts +120 -0
  348. package/src/scheduled-skip-path-backoff-2622.test.ts +98 -0
  349. package/src/scheduled-stuck-http.test.ts +316 -0
  350. package/src/scheduled-stuck-notice-cap.test.ts +80 -0
  351. package/src/scheduled-stuck.ts +219 -0
  352. package/src/scheduled-tasks-http.test.ts +679 -0
  353. package/src/scheduled-tasks.ts +980 -0
  354. package/src/scheduled-work-outcome-http.test.ts +154 -0
  355. package/src/scheduled-work-outcome.ts +50 -0
  356. package/src/scoped-secrets.ts +167 -0
  357. package/src/secret-echo-negative-control.test.ts +118 -0
  358. package/src/send-dedup-request-id.test.ts +229 -0
  359. package/src/send-reply-agent-warning.test.ts +38 -38
  360. package/src/send-reply-alias-optional.test.ts +131 -0
  361. package/src/send-reply-attachments.test.ts +28 -7
  362. package/src/send_dedup.test.ts +54 -0
  363. package/src/send_dedup.ts +52 -6
  364. package/src/serve-error.test.ts +73 -0
  365. package/src/serve-error.ts +56 -0
  366. package/src/server.ts +5103 -377
  367. package/src/session-fork-store.test.ts +43 -0
  368. package/src/session-fork-store.ts +51 -0
  369. package/src/session-project-attach-migration.test.ts +23 -0
  370. package/src/session-project-attach-native.test.ts +123 -0
  371. package/src/session-project-create-http.test.ts +408 -0
  372. package/src/session-project-create-native.test.ts +292 -0
  373. package/src/session-project-create.ts +159 -0
  374. package/src/session-source-admission-store.ts +58 -0
  375. package/src/session-source-create.test.ts +155 -0
  376. package/src/session-source-create.ts +97 -0
  377. package/src/session-source-delivery.ts +114 -0
  378. package/src/session-source-directory-http.test.ts +196 -0
  379. package/src/session-source-evidence.test.ts +905 -0
  380. package/src/session-source-evidence.ts +191 -0
  381. package/src/session-source-execution.ts +87 -0
  382. package/src/session-source-grant.test.ts +117 -0
  383. package/src/session-source-grant.ts +78 -0
  384. package/src/session-source-issuer.test.ts +250 -0
  385. package/src/session-source-issuer.ts +164 -0
  386. package/src/session-source-transport.ts +381 -0
  387. package/src/session-task-on-dispatch.ts +14 -0
  388. package/src/session-token-ownership-http.test.ts +122 -0
  389. package/src/shared/codex-inventory.ts +41 -0
  390. package/src/shared/daemon-provider-login.ts +31 -0
  391. package/src/shared/daemon-provider-snapshot.ts +52 -0
  392. package/src/shared/external-schedule-contract.ts +60 -0
  393. package/src/shared/node-name.test.ts +62 -0
  394. package/src/shared/node-name.ts +161 -0
  395. package/src/shared/probe-host-allowlist-drift.test.ts +45 -0
  396. package/src/shared/provider-application.ts +29 -0
  397. package/src/shared/provider-collection.ts +14 -0
  398. package/src/shared/provider-initialization.ts +8 -0
  399. package/src/shared/reserved-env-drift.test.ts +10 -0
  400. package/src/shared/scoped-secret-contract.ts +5 -0
  401. package/src/shared/task-content-limit-drift.test.ts +28 -0
  402. package/src/shared/task-content-limit.ts +25 -0
  403. package/src/side-thread-command-transport.test.ts +312 -0
  404. package/src/side-thread-command-transport.ts +318 -0
  405. package/src/side-thread-http-integration.test.ts +233 -0
  406. package/src/side-thread-http.ts +504 -0
  407. package/src/side-thread-production.test.ts +72 -0
  408. package/src/side-thread-production.ts +118 -0
  409. package/src/side-thread.test.ts +1067 -0
  410. package/src/side-thread.ts +2499 -0
  411. package/src/skillhub-http.test.ts +144 -0
  412. package/src/sqlite-close-checkpoint.test.ts +78 -0
  413. package/src/start-node.test.ts +265 -0
  414. package/src/start-request-error-domain.test.ts +182 -0
  415. package/src/start-skill-error-domain.test.ts +122 -0
  416. package/src/stats-versions-http.test.ts +65 -0
  417. package/src/status-alias-filter-http.test.ts +106 -0
  418. package/src/status-alias-resolver-http.test.ts +163 -0
  419. package/src/status-alias-resolver.ts +19 -0
  420. package/src/status-node-id-filter-http.test.ts +161 -0
  421. package/src/status-node-secrets-capable-http.test.ts +44 -0
  422. package/src/status-read-cache-http.test.ts +229 -0
  423. package/src/status-read-cache.ts +129 -0
  424. package/src/stop-delete-node.test.ts +567 -1
  425. package/src/task-access-http.test.ts +407 -0
  426. package/src/task-access.ts +420 -0
  427. package/src/task-activity.ts +60 -0
  428. package/src/task-auth-origin-http.test.ts +93 -0
  429. package/src/task-auth-origin.test.ts +24 -0
  430. package/src/task-auth-origin.ts +7 -0
  431. package/src/task-consumption.test.ts +512 -0
  432. package/src/task-diagnostic-http.test.ts +149 -0
  433. package/src/task-diagnostic.test.ts +115 -0
  434. package/src/task-diagnostic.ts +118 -0
  435. package/src/task-expiry-notice-http.test.ts +298 -0
  436. package/src/task-expiry-notice.ts +318 -0
  437. package/src/task-idempotency-mcp.test.ts +148 -0
  438. package/src/task-idempotency.test.ts +76 -0
  439. package/src/task-idempotency.ts +81 -0
  440. package/src/task-late-replies.ts +54 -0
  441. package/src/task-late-reply-http.test.ts +227 -0
  442. package/src/task-lifecycle-watcher-busy.test.ts +114 -0
  443. package/src/task-lifecycle-watcher.test.ts +209 -0
  444. package/src/task-lifecycle-watcher.ts +109 -0
  445. package/src/task-network-resolution-http.test.ts +140 -0
  446. package/src/task-not-found-error-domain.test.ts +117 -0
  447. package/src/task-offline-delivery-2668.test.ts +119 -0
  448. package/src/task-offline-delivery.ts +81 -0
  449. package/src/task-orphan-http.test.ts +287 -0
  450. package/src/task-orphan.ts +119 -0
  451. package/src/task-queue-ahead-http.test.ts +309 -0
  452. package/src/task-queue-ahead-scale.test.ts +236 -0
  453. package/src/task-queue-ahead.ts +186 -0
  454. package/src/task-rejection-log-http.test.ts +160 -0
  455. package/src/task-rejection-log.ts +61 -0
  456. package/src/task-sender-http.test.ts +143 -0
  457. package/src/task-stale-open-http.test.ts +292 -0
  458. package/src/task-stale-open.ts +103 -0
  459. package/src/tasks-pagination-http.test.ts +141 -0
  460. package/src/team-rules-status-http.test.ts +90 -0
  461. package/src/team-rules-status.test.ts +59 -0
  462. package/src/team-rules-status.ts +72 -0
  463. package/src/test-work-root.ts +18 -0
  464. package/src/tmux-allowlist.test.ts +48 -0
  465. package/src/token-usage-http.test.ts +156 -0
  466. package/src/tool-audience-http.test.ts +266 -0
  467. package/src/tool-audience.ts +130 -0
  468. package/src/tools-list-json-schema.test.ts +94 -0
  469. package/src/tools.ts +4557 -472
  470. package/src/update-node-config-error-domain.test.ts +107 -0
  471. package/src/uploads-http.test.ts +6 -4
  472. package/src/uploads.ts +23 -4
  473. package/src/user-avatar-http.test.ts +74 -0
  474. package/src/user-avatar-image.ts +160 -0
  475. package/src/user-avatar-migration.test.ts +25 -0
  476. package/src/user-avatar-upload-http.test.ts +353 -0
  477. package/src/user-inbox-by-agent-http.test.ts +189 -0
  478. package/src/user-inbox-netid-notnull.test.ts +122 -0
  479. package/src/user-inbox-read-http.test.ts +223 -0
  480. package/src/user-inbox-schema.test.ts +54 -0
  481. package/src/version-stats.test.ts +66 -0
  482. package/src/version-stats.ts +109 -0
@@ -0,0 +1,180 @@
1
+ // #2614 —— 事件循环停顿监测 + 慢 SQL 日志的单元测试。
2
+ //
3
+ // 这两个东西的价值不在“能测出停顿”,而在**停顿发生时留下可对齐的证据**:
4
+ // 2026-10-26 生产两次 RSS 3.6/4.5 GB 的尖峰,事后只能靠“日志静默几十秒”
5
+ // 手工推断,`/api/stats/routes` 只给出受害者请求(含排队时间)。
6
+ // 所以测试要钉住的是语义:怎么算一次停顿、留什么档、日志阈值怎么生效、关闭后还计数。
7
+
8
+ import { describe, expect, test } from "bun:test";
9
+ import {
10
+ eventLoopStats,
11
+ noteRequestServed,
12
+ recordGap,
13
+ resetEventLoopLagForTests,
14
+ startEventLoopLagMonitor,
15
+ } from "./event-loop-lag.js";
16
+ import { formatSlowSql, slowSqlThresholdMs } from "./db-adapter.js";
17
+
18
+ describe("event-loop-lag: recordGap", () => {
19
+ test("正常 tick(≈interval)不算停顿", () => {
20
+ resetEventLoopLagForTests();
21
+ expect(recordGap(260, { intervalMs: 250, thresholdMs: 2000 })).toBe(10);
22
+ const s = eventLoopStats();
23
+ expect(s.stall_count).toBe(0);
24
+ expect(s.max_gap_ms).toBe(260);
25
+ expect(s.recent_stalls).toEqual([]);
26
+ });
27
+
28
+ test("超过阈值的 tick 记一条停顿,带 stall_ms / rss_mb / at", () => {
29
+ resetEventLoopLagForTests();
30
+ const now = Date.parse("2026-10-26T22:10:41Z");
31
+ const stall = recordGap(28_000, { intervalMs: 250, thresholdMs: 2000, rssMb: 4502, now });
32
+ expect(stall).toBe(27_750);
33
+ const s = eventLoopStats();
34
+ expect(s.stall_count).toBe(1);
35
+ expect(s.last_stall_ms).toBe(27_750);
36
+ expect(s.last_stall_at).toBe("2026-10-26T22:10:41.000Z");
37
+ expect(s.max_gap_ms).toBe(28_000);
38
+ expect(s.recent_stalls[0]).toMatchObject({ gap_ms: 28_000, stall_ms: 27_750, rss_mb: 4502 });
39
+ });
40
+
41
+ test("recent_stalls 最新在前,且只保留 20 条", () => {
42
+ resetEventLoopLagForTests();
43
+ for (let i = 0; i < 25; i++) {
44
+ recordGap(5_000 + i, { intervalMs: 250, thresholdMs: 2_000, rssMb: 300, now: 1_700_000_000_000 + i * 1_000 });
45
+ }
46
+ const s = eventLoopStats();
47
+ expect(s.stall_count).toBe(25);
48
+ expect(s.recent_stalls.length).toBe(20);
49
+ // 最新一条是 i=24
50
+ expect(s.recent_stalls[0].gap_ms).toBe(5_024);
51
+ expect(s.recent_stalls[19].gap_ms).toBe(5_005);
52
+ });
53
+
54
+ test("记录这段间隔里服务了多少请求(区分“阻塞”与“timer 饥饿”)", () => {
55
+ resetEventLoopLagForTests();
56
+ recordGap(85_000, { intervalMs: 250, thresholdMs: 2_000, rssMb: 2076, served: 137 });
57
+ const s = eventLoopStats();
58
+ expect(s.recent_stalls[0].requests_since_last_tick).toBe(137);
59
+ // 0 服务数 = 这段时间真的什么都没推进
60
+ recordGap(30_000, { intervalMs: 250, thresholdMs: 2_000, rssMb: 500, served: 0 });
61
+ expect(eventLoopStats().recent_stalls[0].requests_since_last_tick).toBe(0);
62
+ });
63
+
64
+ test("max_gap_ms 保留历史最大值(回落后不被覆盖)", () => {
65
+ resetEventLoopLagForTests();
66
+ recordGap(47_000, { intervalMs: 250, thresholdMs: 2_000 });
67
+ recordGap(260, { intervalMs: 250, thresholdMs: 2_000 });
68
+ const s = eventLoopStats();
69
+ expect(s.max_gap_ms).toBe(47_000);
70
+ expect(s.last_gap_ms).toBe(260);
71
+ });
72
+ });
73
+
74
+ describe("event-loop-lag: startEventLoopLagMonitor", () => {
75
+ test("阈值 0 时不打日志,但仍然计数", async () => {
76
+ resetEventLoopLagForTests();
77
+ const logs: string[] = [];
78
+ const timer = startEventLoopLagMonitor({
79
+ intervalMs: 20,
80
+ thresholdMs: 0,
81
+ log: (m) => logs.push(m),
82
+ rssMb: () => 123,
83
+ });
84
+ // 同步阻塞 80ms,让下一次 tick 一定超阈值
85
+ const until = performance.now() + 80;
86
+ while (performance.now() < until) { /* spin */ }
87
+ await Bun.sleep(60);
88
+ clearInterval(timer);
89
+ expect(logs).toEqual([]);
90
+ expect(eventLoopStats().max_gap_ms).toBeGreaterThan(0);
91
+ });
92
+
93
+ test("真实停顿会打一行 [commhub stall] 且带 rss", async () => {
94
+ resetEventLoopLagForTests();
95
+ const logs: string[] = [];
96
+ const timer = startEventLoopLagMonitor({
97
+ intervalMs: 20,
98
+ thresholdMs: 40,
99
+ log: (m) => logs.push(m),
100
+ rssMb: () => 4502,
101
+ // 这条测的是「停顿怎么打日志」,不是启动尖峰。测试进程刚起来几秒,
102
+ // 不关掉启动宽限的话第一个 tick 会被当成启动尖峰跳过(见 #2721 那组用例)。
103
+ bootGraceMs: 0,
104
+ });
105
+ const until = performance.now() + 120;
106
+ while (performance.now() < until) { /* spin 阻塞事件循环 */ }
107
+ await Bun.sleep(60);
108
+ clearInterval(timer);
109
+ expect(logs.length).toBeGreaterThan(0);
110
+ expect(logs[0]).toStartWith("[commhub stall] loop idle ~");
111
+ expect(logs[0]).toContain("rss 4502 MB");
112
+ expect(logs[0]).toContain("served 0 req");
113
+ });
114
+ });
115
+
116
+ describe("db-adapter: 慢 SQL 日志", () => {
117
+ test("默认阈值 1000ms;环境变量可调,0 表示关闭", () => {
118
+ const prev = process.env.COMMHUB_SLOW_SQL_MS;
119
+ try {
120
+ delete process.env.COMMHUB_SLOW_SQL_MS;
121
+ expect(slowSqlThresholdMs()).toBe(1000);
122
+ process.env.COMMHUB_SLOW_SQL_MS = "250";
123
+ expect(slowSqlThresholdMs()).toBe(250);
124
+ process.env.COMMHUB_SLOW_SQL_MS = "0";
125
+ expect(slowSqlThresholdMs()).toBe(0);
126
+ process.env.COMMHUB_SLOW_SQL_MS = "not-a-number";
127
+ expect(slowSqlThresholdMs()).toBe(1000);
128
+ } finally {
129
+ if (prev === undefined) delete process.env.COMMHUB_SLOW_SQL_MS;
130
+ else process.env.COMMHUB_SLOW_SQL_MS = prev;
131
+ }
132
+ });
133
+
134
+ test("日志压平换行并截断,且不含绑定参数", () => {
135
+ const sql = "DELETE FROM agent_telemetry\n WHERE created_at < datetime('now', ?1)";
136
+ const line = formatSlowSql(10_588.4, sql);
137
+ expect(line).toStartWith("[commhub slow-sql] 10588ms DELETE FROM agent_telemetry WHERE created_at < datetime('now', ?1)");
138
+ expect(line).not.toContain("\n");
139
+ const long = formatSlowSql(1_200, "SELECT " + "x".repeat(500));
140
+ expect(long.length).toBeLessThanOrEqual("[commhub slow-sql] 1200ms ".length + 160);
141
+ });
142
+ });
143
+
144
+ // #2721 —— **只跳过第一个 tick**。
145
+ //
146
+ // 事故形状:进程启动后 3 秒记到 gap 2972ms —— 那是「监控开始 → 第一次 tick」被
147
+ // 模块加载/开库/迁移占满的时间,度量的是启动耗时,不是线上停顿。
148
+ //
149
+ // 第一版实现用的是 10 秒**时间窗**,等于监控在前 10 秒什么都不记,
150
+ // 把下面那条「阈值 40ms、spin 120ms 应打停顿日志」的既有测试整体吞掉了
151
+ // —— 修完 qa.yml 事故、QA 恢复后立刻红。窗口越宽越像「把监控关一会儿」。
152
+ describe("#2721 启动尖峰不算停顿(判据:进程 uptime)", () => {
153
+ test("进程刚启动时的第一个 gap 被跳过", async () => {
154
+ resetEventLoopLagForTests();
155
+ const logs: string[] = [];
156
+ const timer = startEventLoopLagMonitor({
157
+ intervalMs: 20, thresholdMs: 40, log: (m) => logs.push(m), rssMb: () => 123,
158
+ bootGraceMs: 15_000, uptimeMs: () => 3_000, // 模拟「启动后 3 秒」
159
+ });
160
+ const until = performance.now() + 120;
161
+ while (performance.now() < until) { /* spin 落在第一个 tick */ }
162
+ await Bun.sleep(80);
163
+ clearInterval(timer);
164
+ expect(logs).toEqual([]);
165
+ });
166
+
167
+ test("进程已经跑了一会儿 ⇒ 第一个 gap 照常记录(不能把监控关掉)", async () => {
168
+ resetEventLoopLagForTests();
169
+ const logs: string[] = [];
170
+ const timer = startEventLoopLagMonitor({
171
+ intervalMs: 20, thresholdMs: 40, log: (m) => logs.push(m), rssMb: () => 123,
172
+ bootGraceMs: 15_000, uptimeMs: () => 600_000, // 进程跑了 10 分钟
173
+ });
174
+ const until = performance.now() + 120;
175
+ while (performance.now() < until) { /* spin */ }
176
+ await Bun.sleep(80);
177
+ clearInterval(timer);
178
+ expect(logs.length).toBeGreaterThan(0);
179
+ });
180
+ });
@@ -0,0 +1,201 @@
1
+ // #2614 —— 事件循环停顿监测(只观测,不改任何行为)。
2
+ //
3
+ // 背景:2026-10-26 21:55–22:12 生产 Hub 出现两次内存尖峰(RSS 3.6 → 4.5 GB),
4
+ // 伴随机 `mcp_inflight` 387 → 807。事后看日志才发现**同一时刻日志静默 14–47 秒**
5
+ // ——Hub 是单线程(Bun + 同步 SQLite),日志静默就等于事件循环被阻塞。
6
+ //
7
+ // 现有 `/api/stats/routes` 只能告诉我们“哪个请求**受害**了”(它的耗时包含了排队时间),
8
+ // 不能告诉我们“当时**谁**在阻塞”。缺的就是停顿本身的时间序列。
9
+ //
10
+ // 这个模块补上这一格:每 `intervalMs` 醒一次,量 `now - lastTick`;
11
+ // 正常情况 ≈ interval,超出 `thresholdMs` 的部分就是一次真实停顿。
12
+ // 停顿会:① 计数/留档(`/health` 的 `event_loop` 字段可读);
13
+ // ② 打一行日志(带当时的 RSS,方便和内存曲线对齐)。
14
+ //
15
+ // 注意:`setInterval` 自己也会被阻塞 —— 这正是我们要测量的东西。
16
+ // 停顿结束后第一次 tick 的 gap 就是停顿长度,不需要外部进程配合。
17
+
18
+ export type EventLoopStall = {
19
+ /** 这个间隔里处理完的请求数:0 = 真的没推进;>0 = 循环在干活,只是 timer 被饿住。 */
20
+ /** ISO 时间(停顿**结束**的时刻,即这次 tick 的时间)。 */
21
+ at: string;
22
+ /** 距上次 tick 的实际间隔(毫秒)。等于 interval + 停顿。 */
23
+ gap_ms: number;
24
+ /** 停顿量 = gap - interval。 */
25
+ stall_ms: number;
26
+ /** 停顿结束时的进程 RSS(MB),用于和内存尖峰对齐。 */
27
+ rss_mb: number;
28
+ requests_since_last_tick?: number;
29
+ };
30
+
31
+ const KEEP_STALLS = 20;
32
+
33
+ /**
34
+ * #2614(第二轮)—— 2026-10-27 00:39 线上出现 gap 84887ms,但**同一时段日志在正常写**
35
+ * (00:38:52 / 00:38:58 / 00:39:08 …),而且 `/api/stats/routes` 里最慢的请求只有 2.4s。
36
+ * 也就是说:事件循环**没有被阻塞 84 秒**,是 **timer 回调被 I/O 队列饿住**了。
37
+ * 只看 tick 间隔会把这两种情况混成一条“卡了 84 秒”的假结论。
38
+ *
39
+ * 区分办法:数一下这个间隔里**服务了多少请求**。
40
+ * requests_since_last_tick > 0 ⇒ 循环在干活 ⇒ 这是 timer 饥饿(高负载/大批回调)
41
+ * requests_since_last_tick = 0 ⇒ 真的没有推进任何请求 ⇒ 才可能是同步阻塞
42
+ */
43
+ let servedSinceTick = 0;
44
+
45
+ /** 请求处理完成时调用(route-timing 里)。只计数,不参与任何决策。 */
46
+ export function noteRequestServed(): void {
47
+ servedSinceTick++;
48
+ }
49
+
50
+ let maxGapMs = 0;
51
+ let lastGapMs = 0;
52
+ let stallCount = 0;
53
+ let lastStallAt: string | null = null;
54
+ let lastStallMs = 0;
55
+ const stalls: EventLoopStall[] = [];
56
+
57
+ /** 记录一次 tick 间隔。返回这次 tick 的停顿量(毫秒,可为负→归零)。 */
58
+ export function recordGap(
59
+ gapMs: number,
60
+ opts: { intervalMs?: number; thresholdMs?: number; rssMb?: number; now?: number; served?: number } = {},
61
+ ): number {
62
+ const interval = opts.intervalMs ?? 250;
63
+ const threshold = opts.thresholdMs ?? 2000;
64
+ lastGapMs = gapMs;
65
+ if (gapMs > maxGapMs) maxGapMs = gapMs;
66
+ const stallMs = Math.max(0, gapMs - interval);
67
+ if (stallMs < threshold) return stallMs;
68
+ const at = new Date(opts.now ?? Date.now()).toISOString();
69
+ stallCount++;
70
+ lastStallAt = at;
71
+ lastStallMs = stallMs;
72
+ stalls.push({
73
+ at,
74
+ gap_ms: Math.round(gapMs),
75
+ stall_ms: Math.round(stallMs),
76
+ rss_mb: opts.rssMb ?? 0,
77
+ requests_since_last_tick: opts.served ?? 0,
78
+ });
79
+ while (stalls.length > KEEP_STALLS) stalls.shift();
80
+ return stallMs;
81
+ }
82
+
83
+ /**
84
+ * 2026-10-31 —— 窗口两端必须跟着读数一起走(#3174)。
85
+ *
86
+ * 原来 event_loop 给了 stall_count / last_stall_at,却没有窗口起点:
87
+ * 计数器随进程重启归零,而「归零」与「从未发生」在读数上同值 ⇒
88
+ * 单独引用 stall_count 的读者会把它读成「从上线以来没卡过」。
89
+ *
90
+ * ⇒ 现在两个时刻戳都在这对象里:`since`(起点,模块初始化时冻结)与 `observed_at`(读的时刻)。
91
+ * 引用者不必再去 /health 顶层猜窗口,窗口与读数同域。
92
+ * 判据:分子分母各要自己的时刻戳;**被声明的那个永远不算**。
93
+ */
94
+ let sinceIso = new Date().toISOString();
95
+
96
+ export function eventLoopStats(): {
97
+ max_gap_ms: number;
98
+ last_gap_ms: number;
99
+ stall_count: number;
100
+ last_stall_ms: number;
101
+ last_stall_at: string | null;
102
+ recent_stalls: EventLoopStall[];
103
+ /** 窗口起点(本进程 event-loop 观察开始时刻)。 */
104
+ since: string;
105
+ /** 这次读的时刻。与 `since` 一起才构成窗口;单独任一个都不构成。 */
106
+ observed_at: string;
107
+ } {
108
+ return {
109
+ max_gap_ms: Math.round(maxGapMs),
110
+ last_gap_ms: Math.round(lastGapMs),
111
+ stall_count: stallCount,
112
+ last_stall_ms: Math.round(lastStallMs),
113
+ last_stall_at: lastStallAt,
114
+ recent_stalls: [...stalls].reverse(),
115
+ since: sinceIso,
116
+ observed_at: new Date().toISOString(),
117
+ };
118
+ }
119
+
120
+ /** 只给测试用:清空累计状态。 */
121
+ export function resetEventLoopLagForTests(): void {
122
+ maxGapMs = 0;
123
+ lastGapMs = 0;
124
+ stallCount = 0;
125
+ lastStallAt = null;
126
+ lastStallMs = 0;
127
+ stalls.length = 0;
128
+ sinceIso = new Date().toISOString(); // 重置窗口起点 —— 否则测试里「归零」与「没归零」同形
129
+ }
130
+
131
+ function envMs(name: string, fallback: number): number {
132
+ const raw = Number(process.env[name]);
133
+ return Number.isFinite(raw) && raw >= 0 ? raw : fallback;
134
+ }
135
+
136
+ /**
137
+ * 启动监测。返回 timer(调用方负责 clearInterval)。
138
+ * - `COMMHUB_STALL_LOG_MS`:停顿日志阈值,默认 2000 ms,设 0 关闭日志(仍计数);
139
+ * - `COMMHUB_STALL_TICK_MS`:采样间隔,默认 250 ms。
140
+ */
141
+ export function startEventLoopLagMonitor(opts: {
142
+ intervalMs?: number;
143
+ thresholdMs?: number;
144
+ log?: (msg: string) => void;
145
+ rssMb?: () => number;
146
+ /**
147
+ * #2721 —— 启动期的 tick gap 不算停顿。判据是**进程启动多久了**,不是第几个 tick。
148
+ *
149
+ * 事故形状:进程 22:21:06 启动,14:21:09.835Z 记到 gap 2972ms —— 启动后 3 秒。
150
+ * 那段 gap 度量的是「监控开始 → 第一次 tick」被模块加载/开库/迁移占满的时间。
151
+ *
152
+ * 两版错误实现(都试过,都被既有测试抓住):
153
+ * ① 10 秒**时间窗** → 监控在前 10 秒什么都不记,「阈值 40ms + spin 120ms 应打日志」直接吞掉
154
+ * ② 只跳**第一个 tick** → 测试里的 spin 恰好落在第一个 tick,同样吞掉
155
+ *
156
+ * 真正的区别不是「第几个 tick」,而是**这段时间里进程有没有可能还在启动**。
157
+ * 所以:只有当 tick 到来时进程 uptime 仍小于 bootGraceMs,才把 gap 当作启动尖峰。
158
+ * 启动 20 秒之后的第一条 gap 依旧照常记录 —— 那时候它不可能是启动耗时。
159
+ */
160
+ bootGraceMs?: number;
161
+ /** 测试注入:返回进程已运行的毫秒数(默认 process.uptime())。 */
162
+ uptimeMs?: () => number;
163
+ } = {}): ReturnType<typeof setInterval> {
164
+ const intervalMs = opts.intervalMs ?? Math.max(50, envMs("COMMHUB_STALL_TICK_MS", 250));
165
+ const thresholdMs = opts.thresholdMs ?? envMs("COMMHUB_STALL_LOG_MS", 2000);
166
+ const log = opts.log ?? ((m: string) => console.log(m));
167
+ const rssMb = opts.rssMb ?? (() => Math.round(process.memoryUsage().rss / 1048576));
168
+ let last = performance.now();
169
+ const startedAt = last;
170
+ // #2721 —— 启动预热不算停顿。
171
+ // 实测:2026-10-29 这次进程 22:21:06 起来,14:21:09.835Z 记到一次 gap 2972ms,
172
+ // 也就是**启动后 3 秒**。那段时间在做模块加载 / 打开 DB / 迁移,第一个 tick 必然偏大。
173
+ // 不排除它的后果是:每次部署都会在 `/health` 里留一条「停顿」,而它和线上真实卡顿长得一模一样。
174
+ const bootGraceMs = opts.bootGraceMs ?? envMs("COMMHUB_STALL_BOOT_GRACE_MS", 15_000);
175
+ const uptimeMs = opts.uptimeMs ?? (() => process.uptime() * 1000);
176
+ let firstTick = true;
177
+ const timer = setInterval(() => {
178
+ const now = performance.now();
179
+ const gap = now - last;
180
+ last = now;
181
+ const rss = rssMb();
182
+ const served = servedSinceTick;
183
+ servedSinceTick = 0;
184
+ // 启动尖峰:只可能是**第一个** tick,且到来时进程才刚起来。
185
+ // 两个条件缺一不可 —— 只看「第一个 tick」会吞掉运行中立刻发生的真实停顿。
186
+ if (firstTick) {
187
+ firstTick = false;
188
+ if (uptimeMs() < bootGraceMs) return;
189
+ }
190
+ const stall = recordGap(gap, { intervalMs, thresholdMs, rssMb: rss, served });
191
+ if (thresholdMs > 0 && stall >= thresholdMs) {
192
+ // 和 retention/slow-sql 的日志同一前缀,方便 grep。
193
+ // `served` 是判据的关键:它非零就说明循环在干活,这段“gap”是 timer 被饿住,不是阻塞。
194
+ log(`[commhub stall] loop idle ~${Math.round(stall)}ms (tick gap ${Math.round(gap)}ms, rss ${rss} MB, served ${served} req)`);
195
+ }
196
+ }, intervalMs);
197
+ // 监测不该拖住进程退出。
198
+ (timer as unknown as { unref?: () => void }).unref?.();
199
+ last = performance.now();
200
+ return timer;
201
+ }
@@ -0,0 +1,187 @@
1
+ // #3271 —— 「到死都没被取走」的计数:**正控 + 负数用例**。
2
+ //
3
+ // 现场:一台节点上 4 条任务,一次又一次「等满 60 分钟 ⇒ expired」,而积压数**两次读数都是 4**。
4
+ // 「又过期了一条」是可见的(Hub 会发通知),但「这台节点连续 N 次一次都没取走」没有读数。
5
+ //
6
+ // 这个计数器属于「只有异常路径能让它 +1」那一类 ⇒ 必须有 must-fire:
7
+ // ① 注入一条**未消费**就过期的任务 ⇒ 计数 +1(正控)
8
+ // ② 注入一条**已消费**后过期的任务 ⇒ 计数**不动**(负数用例)
9
+ // 少了 ②,这个计数就退化成「过期了多少条」——而那是另一个更没用的数。
10
+ import { afterAll, beforeEach, describe, expect, test } from "bun:test";
11
+ import { db } from "./db.js";
12
+ import { neverConsumedExpiryCounters, noteNeverConsumedExpiry, resetNeverConsumedExpiryCountersForTest } from "./expiry-counters.js";
13
+ import { patrolExpiredTasks } from "./server.js";
14
+
15
+ const NET = "net_expiry_cnt";
16
+
17
+ function seedTask(id: string, opts: { toName: string; consumed: boolean; expiredSecondsAgo?: number }) {
18
+ const ago = opts.expiredSecondsAgo ?? 120;
19
+ db.run(
20
+ `INSERT INTO tasks (task_id, network_id, from_name, to_name, content, status, created_at, expires_at, consumed_at)
21
+ VALUES (?1, ?2, 'sender', ?3, 'fixture', 'delivered', datetime('now', '-1 hour'), datetime('now', ?4), ?5)`,
22
+ [id, NET, opts.toName, `-${ago} seconds`, opts.consumed ? new Date(Date.now() - 3600_000).toISOString().replace("T", " ").slice(0, 19) : null],
23
+ );
24
+ }
25
+
26
+ function cleanup() {
27
+ try { db.run("DELETE FROM tasks WHERE network_id = ?1", [NET]); } catch {}
28
+ }
29
+
30
+ beforeEach(() => { cleanup(); resetNeverConsumedExpiryCountersForTest(); });
31
+ afterAll(() => { cleanup(); });
32
+
33
+ describe("#3271 从未取走过的过期计数", () => {
34
+ test("🔴 must-fire:未消费就过期的任务 ⇒ 该节点计数 +1", () => {
35
+ seedTask("t_never_1", { toName: "stuck-node", consumed: false });
36
+ patrolExpiredTasks();
37
+ const c = neverConsumedExpiryCounters();
38
+ expect(c.by_node["stuck-node"]).toBe(1);
39
+ expect(c.total).toBe(1);
40
+ expect(typeof c.since).toBe("string"); // 窗口左端必须在,否则「归零」与「没发生」同值
41
+ });
42
+
43
+ test("🔴 负数用例:**已消费**后过期的任务 ⇒ 计数不动(那不是「没人取」)", () => {
44
+ seedTask("t_consumed_1", { toName: "busy-but-alive", consumed: true });
45
+ patrolExpiredTasks();
46
+ expect(neverConsumedExpiryCounters().total).toBe(0);
47
+ });
48
+
49
+ test("按节点分桶:两个节点各一条 ⇒ 各自 1,总数 2", () => {
50
+ seedTask("t_a", { toName: "node-a", consumed: false });
51
+ seedTask("t_b", { toName: "node-b", consumed: false });
52
+ patrolExpiredTasks();
53
+ const c = neverConsumedExpiryCounters();
54
+ expect(c.by_node).toEqual({ "node-a": 1, "node-b": 1 });
55
+ expect(c.total).toBe(2);
56
+ });
57
+
58
+ test("同一个节点连续 N 次 ⇒ 累加(这是「这一台从来不吃活」的证据形态)", () => {
59
+ for (let i = 0; i < 4; i += 1) noteNeverConsumedExpiry("ghost-node");
60
+ expect(neverConsumedExpiryCounters().by_node["ghost-node"]).toBe(4);
61
+ });
62
+
63
+ test("桶按名字排序且空桶不出现 —— 读数要能直接对比两次快照", () => {
64
+ noteNeverConsumedExpiry("z-node");
65
+ noteNeverConsumedExpiry("a-node");
66
+ const c = neverConsumedExpiryCounters();
67
+ expect(Object.keys(c.by_node)).toEqual(["a-node", "z-node"]);
68
+ });
69
+ });
70
+
71
+ // ---- 派单侧硬闸(#3271)----
72
+ import { neverConsumeGate, NEVER_CONSUME_LIMIT, NEVER_CONSUME_STALE_MS } from "./expiry-counters.js";
73
+
74
+ describe("#3271 派单侧硬闸:从未取走过的目标", () => {
75
+ const NOW = Date.parse("2026-10-31T10:00:00.000Z");
76
+ const counters = (n: number, alias = "ghost", lastAt = new Date(NOW - 60_000).toISOString()) => ({
77
+ since: "2026-10-31T00:00:00.000Z",
78
+ total: n,
79
+ by_node: n > 0 ? { [alias]: n } : {},
80
+ last_at: n > 0 ? { [alias]: lastAt } : {},
81
+ });
82
+
83
+ test("窗口内 0–2 次 ⇒ 放行(一次两次可能是宿主重启,三次才是形态)", () => {
84
+ for (const n of [0, 1, 2]) {
85
+ expect(neverConsumeGate({ alias: "ghost", counters: counters(n), force: false, forceAllowed: true, now: Date.parse("2026-10-31T10:00:00.000Z") }).ok).toBe(true);
86
+ }
87
+ });
88
+
89
+ test("🔴 达到阈值 ⇒ 拒绝,且拒绝体给全(alias / expiries / window_since / 下一步)", () => {
90
+ const r = neverConsumeGate({ alias: "ghost", counters: counters(NEVER_CONSUME_LIMIT), force: false, forceAllowed: true, now: Date.parse("2026-10-31T10:00:00.000Z") });
91
+ expect(r.ok).toBe(false);
92
+ if (r.ok) throw new Error("unreachable");
93
+ expect(r.error).toBe("target_never_consumes");
94
+ expect(r.alias).toBe("ghost");
95
+ expect(r.expiries).toBe(NEVER_CONSUME_LIMIT);
96
+ expect(r.window_since).toBe("2026-10-31T00:00:00.000Z");
97
+ expect(r.message).toContain("force:true");
98
+ });
99
+
100
+ test("人令牌 + force ⇒ 放行(出口存在,但必须显式)", () => {
101
+ expect(neverConsumeGate({ alias: "ghost", counters: counters(9), force: true, forceAllowed: true, now: Date.parse("2026-10-31T10:00:00.000Z") }).ok).toBe(true);
102
+ });
103
+
104
+ test("🔴 节点令牌不能自己放行:force 传了也不算(forceAllowed=false)", () => {
105
+ const r = neverConsumeGate({ alias: "ghost", counters: counters(9), force: true, forceAllowed: false, now: Date.parse("2026-10-31T10:00:00.000Z") });
106
+ expect(r.ok).toBe(false);
107
+ });
108
+
109
+ test("别的节点的计数不影响本目标(分桶不是总数)", () => {
110
+ const c = { since: "s", total: 9, by_node: { other: 9 }, last_at: { other: new Date().toISOString() } };
111
+ expect(neverConsumeGate({ alias: "ghost", counters: c, force: false, forceAllowed: true, now: Date.parse("2026-10-31T10:00:00.000Z") }).ok).toBe(true);
112
+ });
113
+ });
114
+
115
+ describe("#3271 证据的生命期必须 ≥ 故障的生命期(进程内计数器不够)", () => {
116
+ const NOW = Date.parse("2026-10-31T10:00:00.000Z");
117
+ const empty = { since: "2026-10-31T10:00:00.000Z", total: 0, by_node: {}, last_at: {} };
118
+
119
+ // 🔴 这条是 2026-10-31 生产实测的 must-fire:
120
+ // `.254` 部署(22:39)后,我给一台**已经 10 次到死没被取走**的节点派单,
121
+ // 闸返回 ok:true —— 因为计数器随进程重启归零,而故障是以「周」计的。
122
+ test("🔴 内存计数为空,但 tasks 表里有 3 条历史 ⇒ 必须拒绝", () => {
123
+ const r = neverConsumeGate({
124
+ alias: "ghost", counters: empty, force: false, forceAllowed: true, now: NOW,
125
+ persistent: { expiries: 3, last_at: new Date(NOW - 60_000).toISOString().replace("T", " ").slice(0, 19), last_reply_at: null },
126
+ });
127
+ expect(r.ok).toBe(false);
128
+ if (r.ok) throw new Error("unreachable");
129
+ expect(r.expiries).toBe(3);
130
+ });
131
+
132
+ test("两个来源取**并集**(内存 2 + 表里 3 ⇒ 按 3 判,不是相加也不是取小)", () => {
133
+ const counters = { since: "s", total: 2, by_node: { ghost: 2 }, last_at: { ghost: new Date(NOW - 60_000).toISOString() } };
134
+ const r = neverConsumeGate({
135
+ alias: "ghost", counters, force: false, forceAllowed: true, now: NOW,
136
+ persistent: { expiries: 3, last_at: new Date(NOW - 60_000).toISOString().replace("T", " ").slice(0, 19), last_reply_at: null },
137
+ });
138
+ expect(r.ok).toBe(false);
139
+ if (r.ok) throw new Error("unreachable");
140
+ expect(r.expiries).toBe(3);
141
+ });
142
+
143
+ // 🔴 语义修正(2026-10-31 第二次实测):DB 证据**不用固定 TTL**。
144
+ // 固定 2 小时是我拍的,而那台节点的故障节律是「天」——于是闸在 22 小时前的那次过期之后
145
+ // 判「已恢复」并放行,而它这 22 小时一条都没回。
146
+ // 现在 DB 证据自带边界(「最后一次回复之后」),陈旧不陈旧由回复本身回答。
147
+ test("🔴 表里的历史很旧、但之后没有回复 ⇒ 仍然拒绝(固定 TTL 只留给纯内存计数)", () => {
148
+ const r = neverConsumeGate({
149
+ alias: "ghost", counters: empty, force: false, forceAllowed: true, now: NOW,
150
+ persistent: { expiries: 9, last_at: "2026-10-20T00:00:00", last_reply_at: "2026-10-18T00:00:00" },
151
+ });
152
+ expect(r.ok).toBe(false);
153
+ });
154
+
155
+ test("纯内存计数仍然有 2 小时时效(没有回复时间可比,只能按时长兜底)", () => {
156
+ const counters = { since: "s", total: 9, by_node: { ghost: 9 }, last_at: { ghost: new Date(NOW - 3 * 3600_000).toISOString() } };
157
+ expect(neverConsumeGate({ alias: "ghost", counters, force: false, forceAllowed: true, now: NOW }).ok).toBe(true);
158
+ });
159
+ });
160
+
161
+ describe("#3271 硬闸的时效:只数不衰减会把恢复了的节点永久拉黑", () => {
162
+ const NOW = Date.parse("2026-10-31T10:00:00.000Z");
163
+ const mk = (lastAtIso: string) => ({
164
+ since: "2026-10-31T00:00:00.000Z",
165
+ total: 5,
166
+ by_node: { ghost: 5 },
167
+ last_at: { ghost: lastAtIso },
168
+ });
169
+
170
+ test("最近一次是 1 分钟前 ⇒ 仍然拒(形态是新鲜的)", () => {
171
+ const r = neverConsumeGate({ alias: "ghost", counters: mk(new Date(NOW - 60_000).toISOString()), force: false, forceAllowed: true, now: NOW });
172
+ expect(r.ok).toBe(false);
173
+ });
174
+
175
+ test("🔴 最近一次超过 TTL ⇒ 放行(节点可能已恢复;计数仍留在读数里)", () => {
176
+ const r = neverConsumeGate({ alias: "ghost", counters: mk(new Date(NOW - NEVER_CONSUME_STALE_MS - 1).toISOString()), force: false, forceAllowed: true, now: NOW });
177
+ expect(r.ok).toBe(true);
178
+ });
179
+
180
+ test("读数里带 last_at —— 光有计数看不出「多久没犯了」", () => {
181
+ resetNeverConsumedExpiryCountersForTest(new Date(NOW));
182
+ noteNeverConsumedExpiry("ghost", NOW);
183
+ const c = neverConsumedExpiryCounters();
184
+ expect(c.last_at["ghost"]).toBe(new Date(NOW).toISOString());
185
+ expect(c.by_node["ghost"]).toBe(1);
186
+ });
187
+ });