@sleep2agi/commhub-server 0.9.0-preview.2 → 0.9.0-preview.200

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (412) hide show
  1. package/README.md +130 -10
  2. package/bin/commhub.ts +99 -11
  3. package/package.json +7 -7
  4. package/src/ack-create-request-transport.test.ts +182 -0
  5. package/src/ack-create-request.test.ts +374 -0
  6. package/src/admin-delete-network-http.test.ts +228 -0
  7. package/src/admin-networks-http.test.ts +88 -0
  8. package/src/adoption-candidates-http.test.ts +137 -0
  9. package/src/adoption-candidates.ts +158 -0
  10. package/src/agent-access-batch.test.ts +210 -0
  11. package/src/agent-access.ts +487 -0
  12. package/src/agent-acl-groups-http.test.ts +247 -0
  13. package/src/agent-acl-http.test.ts +659 -0
  14. package/src/agent-acl-slice3-http.test.ts +267 -0
  15. package/src/agent-department-http.test.ts +215 -0
  16. package/src/agent-groups-perf.test.ts +223 -0
  17. package/src/agent-notice.ts +48 -0
  18. package/src/agent-team-whoami-http.test.ts +98 -0
  19. package/src/agent-teams-http.test.ts +330 -0
  20. package/src/agent-teams.ts +244 -0
  21. package/src/alias-baseline-2571.test.ts +162 -0
  22. package/src/alias-baseline.ts +113 -0
  23. package/src/alias-filter.test.ts +72 -0
  24. package/src/alias-filter.ts +42 -0
  25. package/src/api-host-supervisors-can-create-nodes.test.ts +350 -0
  26. package/src/api-host-supervisors-fallback.test.ts +271 -0
  27. package/src/api-host-supervisors-runtime-readiness.test.ts +195 -0
  28. package/src/api-nodes-shape.test.ts +279 -0
  29. package/src/auth-password-change-cli-tokens-http.test.ts +423 -0
  30. package/src/auth-sessions-http.test.ts +250 -0
  31. package/src/auth-validate.test.ts +5 -1
  32. package/src/auth-weak-password-login.test.ts +176 -0
  33. package/src/auth.ts +611 -51
  34. package/src/avatar-validate.test.ts +163 -0
  35. package/src/avatar-validate.ts +103 -0
  36. package/src/backfill-node-ids-2664.test.ts +128 -0
  37. package/src/binding-generation-http.test.ts +121 -0
  38. package/src/blocked-exit-on-reply.test.ts +122 -0
  39. package/src/boot-side-effect.test.ts +124 -0
  40. package/src/broadcast-fanout-2721.test.ts +118 -0
  41. package/src/broadcast-fanout.ts +98 -0
  42. package/src/chain-attempt-dedup.test.ts +447 -0
  43. package/src/clone-node.test.ts +248 -0
  44. package/src/clone-node.ts +289 -0
  45. package/src/codex-config-flags.ts +77 -0
  46. package/src/codex-fork-contract.ts +23 -0
  47. package/src/codex-fork-migration.test.ts +38 -0
  48. package/src/commhub-bin-args.test.ts +85 -0
  49. package/src/config-apply-sec1.test.ts +107 -0
  50. package/src/config-apply-validate.test.ts +134 -1
  51. package/src/config-apply-validate.ts +111 -5
  52. package/src/create-node-blocked-gate.test.ts +356 -0
  53. package/src/create-node-network-id.test.ts +79 -0
  54. package/src/create-node-tool-schema.test.ts +49 -0
  55. package/src/create-node-unicode-name.test.ts +120 -0
  56. package/src/create-node-validate.test.ts +379 -0
  57. package/src/create-node-validate.ts +423 -0
  58. package/src/create-node-workdir.test.ts +289 -0
  59. package/src/create-node.test.ts +354 -0
  60. package/src/create-node.ts +411 -0
  61. package/src/daemon-provider-snapshots.test.ts +421 -0
  62. package/src/daemon-provider-snapshots.ts +327 -0
  63. package/src/daemon-token-owner-http.test.ts +236 -0
  64. package/src/dashboard-slash-routing-http.test.ts +109 -0
  65. package/src/db-adapter-guard.test.ts +148 -0
  66. package/src/db-adapter-translate.test.ts +128 -0
  67. package/src/db-adapter.ts +377 -91
  68. package/src/db-timestamp.test.ts +38 -0
  69. package/src/db-timestamp.ts +15 -0
  70. package/src/db.ts +1501 -152
  71. package/src/delivered-without-push.test.ts +140 -0
  72. package/src/department-groups-http.test.ts +209 -0
  73. package/src/department-groups-sync-http.test.ts +328 -0
  74. package/src/department-groups.ts +434 -0
  75. package/src/department-heads-http.test.ts +330 -0
  76. package/src/department-heads.ts +226 -0
  77. package/src/departments-agent-count.test.ts +129 -0
  78. package/src/departments-http.test.ts +185 -0
  79. package/src/departments-mcp.test.ts +128 -0
  80. package/src/departments.ts +318 -0
  81. package/src/desktop-message-seam.test.ts +150 -0
  82. package/src/desktop-message.test.ts +368 -0
  83. package/src/dm-file-scope-http.test.ts +178 -0
  84. package/src/empty-node-sweeper.test.ts +104 -0
  85. package/src/empty-node-sweeper.ts +88 -0
  86. package/src/env-transport-gate-http.test.ts +177 -0
  87. package/src/env-transport-gate.ts +74 -0
  88. package/src/env-transport-policy.ts +21 -0
  89. package/src/event-loop-lag.test.ts +139 -0
  90. package/src/event-loop-lag.ts +150 -0
  91. package/src/ext-token-shared.test.ts +60 -0
  92. package/src/ext-validation-regression.test.ts +131 -0
  93. package/src/external-schedule-edits-http.test.ts +321 -0
  94. package/src/external-schedule-edits-timestamps.test.ts +64 -0
  95. package/src/external-schedule-edits.ts +327 -0
  96. package/src/file-download-authz-dev-open.test.ts +135 -0
  97. package/src/file-download-authz.test.ts +404 -0
  98. package/src/file-download-date-bucket.test.ts +436 -0
  99. package/src/file-network-scope-dev-open.test.ts +97 -0
  100. package/src/file-network-scope.test.ts +732 -0
  101. package/src/group-membership-events-http.test.ts +264 -0
  102. package/src/group-messages-http.test.ts +482 -0
  103. package/src/group-messages.ts +297 -0
  104. package/src/health-host-disk-2756.test.ts +49 -0
  105. package/src/health-redaction.test.ts +685 -0
  106. package/src/host-supervisor-identity-http.test.ts +152 -0
  107. package/src/host-supervisor-identity.ts +92 -0
  108. package/src/host-supervisors-daemon-version.test.ts +137 -0
  109. package/src/http-gzip.test.ts +94 -0
  110. package/src/http-gzip.ts +112 -0
  111. package/src/hub-catalog-read-http.test.ts +187 -0
  112. package/src/hub-provider-config.test.ts +139 -0
  113. package/src/hub-provider-config.ts +105 -0
  114. package/src/hub-timestamp.test.ts +38 -0
  115. package/src/hub-timestamp.ts +27 -0
  116. package/src/human-dm.ts +215 -0
  117. package/src/human-inbox.ts +42 -0
  118. package/src/inbox-caller.ts +49 -0
  119. package/src/inbox-count-sse.test.ts +310 -0
  120. package/src/inbox-count.ts +29 -0
  121. package/src/index.ts +15 -2172
  122. package/src/lifecycle-guard.ts +50 -0
  123. package/src/list-host-supervisors.test.ts +315 -0
  124. package/src/mcp-context-cost-http.test.ts +127 -0
  125. package/src/mcp-inflight.test.ts +92 -0
  126. package/src/mcp-inflight.ts +38 -0
  127. package/src/mcp-projects-events-http.test.ts +162 -0
  128. package/src/mcp-route-label-http.test.ts +120 -0
  129. package/src/mcp-write-network-resolution.test.ts +432 -0
  130. package/src/member-audit-network-id-http.test.ts +93 -0
  131. package/src/member-presence-http.test.ts +210 -0
  132. package/src/messages-ack-by-agent-http.test.ts +142 -0
  133. package/src/migrate-sqlite-to-pg.test.ts +47 -0
  134. package/src/migrate-sqlite-to-pg.ts +263 -0
  135. package/src/model-auth-notify-http.test.ts +181 -0
  136. package/src/model-auth-notify.ts +116 -0
  137. package/src/network-name-http.test.ts +75 -0
  138. package/src/network-rules-http.test.ts +169 -0
  139. package/src/network-rules.ts +129 -0
  140. package/src/network-scope.ts +276 -0
  141. package/src/network-token-membership-http.test.ts +139 -0
  142. package/src/node-alias-control.ts +12 -0
  143. package/src/node-attrs-validate.ts +88 -0
  144. package/src/node-attrs.test.ts +258 -0
  145. package/src/node-caller-identity-http.test.ts +1083 -0
  146. package/src/node-create-request-status-http.test.ts +105 -0
  147. package/src/node-daemon-bindings.test.ts +132 -0
  148. package/src/node-daemon-bindings.ts +135 -0
  149. package/src/node-failure-alert.test.ts +150 -0
  150. package/src/node-failure-alert.ts +177 -0
  151. package/src/node-files-transport.test.ts +344 -0
  152. package/src/node-health-codex-login.test.ts +120 -0
  153. package/src/node-health-dispatch-http.test.ts +279 -0
  154. package/src/node-health-guard.ts +105 -0
  155. package/src/node-health-passthrough.test.ts +115 -0
  156. package/src/node-health-store.ts +101 -0
  157. package/src/node-id-alias.test.ts +190 -0
  158. package/src/node-identity-conflict-http.test.ts +269 -0
  159. package/src/node-identity-conflict.ts +142 -0
  160. package/src/node-identity-mismatch.test.ts +74 -0
  161. package/src/node-in-flight-tasks.ts +31 -0
  162. package/src/node-lifecycle-controllable.test.ts +64 -0
  163. package/src/node-lifecycle-controllable.ts +53 -0
  164. package/src/node-lifecycle-read-http.test.ts +250 -0
  165. package/src/node-lifecycle-read.ts +124 -0
  166. package/src/node-logs-transport.test.ts +263 -0
  167. package/src/node-peer-allowlist.test.ts +79 -0
  168. package/src/node-permissions-http.test.ts +345 -0
  169. package/src/node-permissions.ts +393 -0
  170. package/src/node-request-retention.test.ts +281 -0
  171. package/src/node-request-retention.ts +120 -0
  172. package/src/node-secret-rest-http.test.ts +190 -0
  173. package/src/node-serviceability.test.ts +86 -0
  174. package/src/node-serviceability.ts +124 -0
  175. package/src/node-session-control-http.test.ts +310 -0
  176. package/src/node-session-control.ts +258 -0
  177. package/src/node-skills-transport.test.ts +209 -0
  178. package/src/node-token-identity.ts +35 -0
  179. package/src/node-token-ownership.ts +51 -0
  180. package/src/node-token-privilege-http.test.ts +158 -0
  181. package/src/not-found-api-http.test.ts +67 -0
  182. package/src/observer-avatar-http.test.ts +442 -0
  183. package/src/observer-push.test.ts +252 -0
  184. package/src/opencode-catalog-report-http.test.ts +139 -0
  185. package/src/opencode-v2-catalog.test.ts +268 -0
  186. package/src/opencode-v2-catalog.ts +352 -0
  187. package/src/package-version-consistency.test.ts +25 -0
  188. package/src/parent-source-marker.test.ts +50 -0
  189. package/src/password-network-tokens.ts +25 -0
  190. package/src/peer-reply-atomic.test.ts +455 -0
  191. package/src/peer-reply-capability.test.ts +70 -0
  192. package/src/pg-worker.ts +87 -0
  193. package/src/probe-validate.test.ts +221 -0
  194. package/src/probe-validate.ts +104 -0
  195. package/src/probe.test.ts +80 -0
  196. package/src/probe.ts +200 -0
  197. package/src/project-dir-warn-clear-http.test.ts +132 -0
  198. package/src/project-viewer-can-http.test.ts +116 -0
  199. package/src/push.test.ts +20 -10
  200. package/src/push.ts +466 -13
  201. package/src/redact-tokens.test.ts +65 -0
  202. package/src/redact-tokens.ts +40 -0
  203. package/src/rename.ts +38 -10
  204. package/src/reply-target-derivation.test.ts +96 -0
  205. package/src/report-status-handover-capabilities-http.test.ts +124 -0
  206. package/src/report-status-host-schema.test.ts +235 -0
  207. package/src/report-status-resume-id-handover.test.ts +184 -0
  208. package/src/report-status-secret-keys.test.ts +140 -0
  209. package/src/require-explicit-test-db.ts +9 -0
  210. package/src/requirement-due-reminders-http.test.ts +620 -0
  211. package/src/requirement-due-reminders.ts +416 -0
  212. package/src/requirement-events-http.test.ts +268 -0
  213. package/src/requirement-events.test.ts +45 -0
  214. package/src/requirement-events.ts +144 -0
  215. package/src/requirement-notify.ts +165 -0
  216. package/src/requirement-participant-notify-http.test.ts +309 -0
  217. package/src/requirement-tag-ops-http.test.ts +425 -0
  218. package/src/requirement-tag-tree-http.test.ts +275 -0
  219. package/src/requirement-tag-tree.ts +127 -0
  220. package/src/requirement-tags.ts +102 -0
  221. package/src/requirements-abandoned-http.test.ts +236 -0
  222. package/src/requirements-comment-http.test.ts +167 -0
  223. package/src/requirements-completed-at-migration.test.ts +70 -0
  224. package/src/requirements-errors-mcp-http.test.ts +164 -0
  225. package/src/requirements-errors.ts +158 -0
  226. package/src/requirements-etag-http.test.ts +91 -0
  227. package/src/requirements-http.test.ts +709 -0
  228. package/src/requirements-last-event-http.test.ts +349 -0
  229. package/src/requirements-list-mcp-slim-http.test.ts +155 -0
  230. package/src/requirements-list-slim-http.test.ts +324 -0
  231. package/src/requirements-mcp.test.ts +163 -0
  232. package/src/requirements-migrate.ts +283 -0
  233. package/src/requirements-owner-strict-mcp-http.test.ts +145 -0
  234. package/src/requirements-people-http.test.ts +114 -0
  235. package/src/requirements-people-mcp-http.test.ts +231 -0
  236. package/src/requirements-people.ts +196 -0
  237. package/src/requirements-priority-lowest.test.ts +114 -0
  238. package/src/requirements-projects-parent-http.test.ts +123 -0
  239. package/src/requirements-search-http.test.ts +182 -0
  240. package/src/requirements-search.test.ts +32 -0
  241. package/src/requirements-search.ts +145 -0
  242. package/src/requirements-seq-http.test.ts +193 -0
  243. package/src/requirements-seq-migration.test.ts +63 -0
  244. package/src/requirements-start-date.test.ts +125 -0
  245. package/src/requirements-stats-http.test.ts +357 -0
  246. package/src/requirements-stats.ts +195 -0
  247. package/src/requirements-status-participants-http.test.ts +222 -0
  248. package/src/requirements-upgrade-e2e.test.ts +126 -0
  249. package/src/requirements-upgrade-migration.test.ts +98 -0
  250. package/src/requirements.ts +2027 -0
  251. package/src/resolve-port.test.ts +57 -0
  252. package/src/resolve-port.ts +44 -0
  253. package/src/response-charset.test.ts +130 -0
  254. package/src/rest-alias-network-fallback-http.test.ts +123 -0
  255. package/src/rest-alias-network-fallback.test.ts +60 -0
  256. package/src/rest-alias-network-fallback.ts +65 -0
  257. package/src/rest-explicit-columns-http.test.ts +266 -0
  258. package/src/rest-identity.test.ts +130 -0
  259. package/src/rest-identity.ts +141 -0
  260. package/src/rest-projections.ts +72 -0
  261. package/src/rest-read-network-real-chain.test.ts +97 -0
  262. package/src/rest-write-network-resolution.test.ts +133 -0
  263. package/src/restart-readiness-http.test.ts +121 -0
  264. package/src/restricted-files.ts +76 -0
  265. package/src/retention-first-sweep-quiet.test.ts +31 -0
  266. package/src/retention.test.ts +100 -40
  267. package/src/retention.ts +423 -104
  268. package/src/route-callers-http.test.ts +120 -0
  269. package/src/route-timing.test.ts +211 -0
  270. package/src/route-timing.ts +262 -0
  271. package/src/rules-file-duplicate-alias.test.ts +234 -0
  272. package/src/rules-file-session-target.test.ts +207 -0
  273. package/src/rules-file-transport.test.ts +237 -0
  274. package/src/runtime-label.test.ts +22 -0
  275. package/src/runtime-label.ts +21 -0
  276. package/src/runtime-readiness.ts +90 -0
  277. package/src/schedule-agent-mcp-http.test.ts +391 -0
  278. package/src/schedule-agent.ts +192 -0
  279. package/src/schedule-starved.test.ts +92 -0
  280. package/src/schedule-starved.ts +114 -0
  281. package/src/scheduled-backoff-http.test.ts +193 -0
  282. package/src/scheduled-backoff.ts +64 -0
  283. package/src/scheduled-doorbell-nodeless-session.test.ts +134 -0
  284. package/src/scheduled-failure-backoff-2622.test.ts +145 -0
  285. package/src/scheduled-failure-code.test.ts +73 -0
  286. package/src/scheduled-failure-code.ts +105 -0
  287. package/src/scheduled-failures-http.test.ts +279 -0
  288. package/src/scheduled-failures.ts +377 -0
  289. package/src/scheduled-noprogress-backoff-2622.test.ts +133 -0
  290. package/src/scheduled-offline-backoff-probe.test.ts +56 -0
  291. package/src/scheduled-offline.test.ts +131 -0
  292. package/src/scheduled-offline.ts +66 -0
  293. package/src/scheduled-outcome-guidance-http.test.ts +139 -0
  294. package/src/scheduled-outcome-guidance.ts +9 -0
  295. package/src/scheduled-reply-unread-http.test.ts +212 -0
  296. package/src/scheduled-requirement-wake-http.test.ts +279 -0
  297. package/src/scheduled-requirement-wake.ts +125 -0
  298. package/src/scheduled-run-terminal.test.ts +268 -0
  299. package/src/scheduled-skip-blocking-task.test.ts +120 -0
  300. package/src/scheduled-skip-path-backoff-2622.test.ts +98 -0
  301. package/src/scheduled-stuck-http.test.ts +312 -0
  302. package/src/scheduled-stuck-notice-cap.test.ts +80 -0
  303. package/src/scheduled-stuck.ts +219 -0
  304. package/src/scheduled-tasks-http.test.ts +679 -0
  305. package/src/scheduled-tasks.ts +959 -0
  306. package/src/scheduled-work-outcome-http.test.ts +154 -0
  307. package/src/scheduled-work-outcome.ts +50 -0
  308. package/src/scoped-secrets.ts +139 -0
  309. package/src/send-dedup-request-id.test.ts +229 -0
  310. package/src/send-reply-agent-warning.test.ts +202 -0
  311. package/src/send-reply-alias-optional.test.ts +131 -0
  312. package/src/send-reply-attachments.test.ts +458 -0
  313. package/src/send_dedup.test.ts +54 -0
  314. package/src/send_dedup.ts +52 -6
  315. package/src/serve-error.test.ts +38 -0
  316. package/src/serve-error.ts +27 -0
  317. package/src/server.ts +5997 -0
  318. package/src/session-task-on-dispatch.ts +14 -0
  319. package/src/session-token-ownership-http.test.ts +122 -0
  320. package/src/shared/codex-inventory.ts +41 -0
  321. package/src/shared/daemon-provider-login.ts +31 -0
  322. package/src/shared/daemon-provider-snapshot.ts +52 -0
  323. package/src/shared/external-schedule-contract.ts +60 -0
  324. package/src/shared/node-name.test.ts +62 -0
  325. package/src/shared/node-name.ts +161 -0
  326. package/src/shared/probe-host-allowlist-drift.test.ts +54 -0
  327. package/src/shared/probe-host-allowlist.ts +128 -0
  328. package/src/shared/provider-application.ts +29 -0
  329. package/src/shared/provider-collection.ts +14 -0
  330. package/src/shared/provider-initialization.ts +8 -0
  331. package/src/shared/reserved-env-drift.test.ts +53 -0
  332. package/src/shared/reserved-env.test.ts +35 -0
  333. package/src/shared/reserved-env.ts +31 -0
  334. package/src/shared/scoped-secret-contract.ts +5 -0
  335. package/src/shared/task-content-limit-drift.test.ts +28 -0
  336. package/src/shared/task-content-limit.ts +25 -0
  337. package/src/side-thread-command-transport.test.ts +312 -0
  338. package/src/side-thread-command-transport.ts +318 -0
  339. package/src/side-thread-http-integration.test.ts +233 -0
  340. package/src/side-thread-http.ts +504 -0
  341. package/src/side-thread-production.test.ts +72 -0
  342. package/src/side-thread-production.ts +118 -0
  343. package/src/side-thread.test.ts +1067 -0
  344. package/src/side-thread.ts +2499 -0
  345. package/src/skillhub-http.test.ts +108 -0
  346. package/src/sqlite-close-checkpoint.test.ts +78 -0
  347. package/src/start-node.test.ts +264 -0
  348. package/src/status-alias-filter-http.test.ts +106 -0
  349. package/src/status-alias-resolver-http.test.ts +163 -0
  350. package/src/status-alias-resolver.ts +19 -0
  351. package/src/status-node-id-filter-http.test.ts +161 -0
  352. package/src/status-read-cache-http.test.ts +229 -0
  353. package/src/status-read-cache.ts +122 -0
  354. package/src/stop-delete-node.test.ts +1286 -0
  355. package/src/task-access-http.test.ts +402 -0
  356. package/src/task-access.ts +420 -0
  357. package/src/task-activity.ts +60 -0
  358. package/src/task-auth-origin-http.test.ts +93 -0
  359. package/src/task-auth-origin.test.ts +24 -0
  360. package/src/task-auth-origin.ts +7 -0
  361. package/src/task-consumption.test.ts +482 -0
  362. package/src/task-diagnostic-http.test.ts +149 -0
  363. package/src/task-diagnostic.test.ts +115 -0
  364. package/src/task-diagnostic.ts +118 -0
  365. package/src/task-expiry-notice-http.test.ts +287 -0
  366. package/src/task-expiry-notice.ts +315 -0
  367. package/src/task-idempotency-mcp.test.ts +142 -0
  368. package/src/task-idempotency.test.ts +76 -0
  369. package/src/task-idempotency.ts +81 -0
  370. package/src/task-late-replies.ts +54 -0
  371. package/src/task-late-reply-http.test.ts +152 -0
  372. package/src/task-lifecycle-watcher-busy.test.ts +114 -0
  373. package/src/task-lifecycle-watcher.test.ts +204 -0
  374. package/src/task-lifecycle-watcher.ts +109 -0
  375. package/src/task-network-resolution-http.test.ts +140 -0
  376. package/src/task-offline-delivery-2668.test.ts +119 -0
  377. package/src/task-offline-delivery.ts +81 -0
  378. package/src/task-orphan-http.test.ts +287 -0
  379. package/src/task-orphan.ts +119 -0
  380. package/src/task-queue-ahead-http.test.ts +309 -0
  381. package/src/task-queue-ahead-scale.test.ts +236 -0
  382. package/src/task-queue-ahead.ts +186 -0
  383. package/src/task-rejection-log-http.test.ts +160 -0
  384. package/src/task-rejection-log.ts +49 -0
  385. package/src/task-sender-http.test.ts +143 -0
  386. package/src/task-stale-open-http.test.ts +290 -0
  387. package/src/task-stale-open.ts +103 -0
  388. package/src/tasks-pagination-http.test.ts +141 -0
  389. package/src/team-rules-status-http.test.ts +90 -0
  390. package/src/team-rules-status.test.ts +59 -0
  391. package/src/team-rules-status.ts +72 -0
  392. package/src/tool-audience-http.test.ts +230 -0
  393. package/src/tool-audience.ts +120 -0
  394. package/src/tools-list-json-schema.test.ts +94 -0
  395. package/src/tools.ts +5815 -348
  396. package/src/update-provider.test.ts +234 -0
  397. package/src/upload-early-reject-guard.test.ts +155 -0
  398. package/src/uploads-http.test.ts +97 -15
  399. package/src/uploads.test.ts +66 -0
  400. package/src/uploads.ts +138 -5
  401. package/src/user-avatar-http.test.ts +74 -0
  402. package/src/user-avatar-image.ts +160 -0
  403. package/src/user-avatar-migration.test.ts +25 -0
  404. package/src/user-avatar-upload-http.test.ts +348 -0
  405. package/src/user-inbox-by-agent-http.test.ts +161 -0
  406. package/src/user-inbox-netid-notnull.test.ts +122 -0
  407. package/src/user-inbox-read-http.test.ts +223 -0
  408. package/src/user-inbox-schema.test.ts +54 -0
  409. package/src/vault.test.ts +200 -0
  410. package/src/vault.ts +156 -0
  411. package/src/version-stats.test.ts +66 -0
  412. package/src/version-stats.ts +109 -0
@@ -0,0 +1,377 @@
1
+ // #523 —— 定时任务连续失败:告诉创建者(可选自动暂停)。
2
+ //
3
+ // 现场:一个每 2 分钟一次的排程,每一次派出去的任务都被节点以 failed 结束(上游模型拒绝),一天 720 次,
4
+ // 排程照样 active,创建者什么都不知道。#464 只管「被上一次卡住」(skipped),不管「每次都失败」。
5
+ //
6
+ // 1. 计数:consecutive_failures = 最近一次成功(run.status='replied')之后,status='failed' 的 run 数。
7
+ // skipped / cancelled / expired / 还没结束的 run 不算失败,也不打断连续(expired 由 #464 的超时告警负责)。
8
+ // 失败有两种来源,都会走到这里:派发时就失败(节点不在 / 被停用 / 降级 / 创建者无权限,
9
+ // scheduled-tasks.ts 提交后调用),以及派出去的任务以 failed 结束(db.ts 的 run 镜像触发钩子)。
10
+ // 2. 告警:连续失败 ≥ N(默认 5)→ 给排程创建者发**一条**通知(agent-notice.ts,与 #462/#464 同一条路,
11
+ // 出现在与目标节点的会话里)。去重落在 scheduled_tasks 两列上(条件写,一次只有一个调用方抢得到):
12
+ // failure_alert_key = 告警那一刻「最近一次成功」的 run_id —— 之后又成功过,key 就变了,下一段失败重新上膛;
13
+ // failure_alert_at = 告警时间 —— 一直没成功过,24 小时后再提醒一次。Hub 重启不会重发。
14
+ // 3. 自动暂停:默认关。COMMHUB_SCHEDULE_FAILURE_AUTO_PAUSE_RUNS=M(≥1)时,连续失败 ≥ M 的 active 排程
15
+ // 被置为 paused(next_run_at 清空,revision+1,与用户手动暂停同形),并通知创建者。
16
+ //
17
+ // 环境变量:COMMHUB_SCHEDULE_FAILURE_NOTICE_RUNS(N,默认 5)、COMMHUB_SCHEDULE_FAILURE_RENOTICE_SEC
18
+ // (默认 86400)、COMMHUB_SCHEDULE_FAILURE_AUTO_PAUSE_RUNS(M,默认 0 = 关)。
19
+
20
+ import { db, setScheduledRunTerminalHook } from "./db.js";
21
+ import { sendAgentNotice } from "./agent-notice.js";
22
+ import { parseHubTimestamp } from "./hub-timestamp.js";
23
+ import { busyElapsedNote } from "./scheduled-failure-code.js";
24
+
25
+ export const SCHEDULE_FAILING_NOTICE_KIND = "schedule_failing";
26
+ const ERROR_PREVIEW_CHARS = 300;
27
+
28
+ function envInt(name: string, fallback: number, min: number): number {
29
+ const raw = process.env[name];
30
+ if (raw === undefined || raw.trim() === "") return fallback;
31
+ const n = Number(raw);
32
+ return Number.isFinite(n) && n >= min ? Math.floor(n) : fallback;
33
+ }
34
+
35
+ export function failureNoticeRuns(): number { return envInt("COMMHUB_SCHEDULE_FAILURE_NOTICE_RUNS", 5, 1); }
36
+ export function failureRenoticeMs(): number { return envInt("COMMHUB_SCHEDULE_FAILURE_RENOTICE_SEC", 86_400, 1) * 1000; }
37
+ /** 0 = 不自动暂停(默认)。 */
38
+ export function failureAutoPauseRuns(): number { return envInt("COMMHUB_SCHEDULE_FAILURE_AUTO_PAUSE_RUNS", 0, 0); }
39
+
40
+ export type FailureStreak = { consecutiveFailures: number; lastSuccessRunId: string | null };
41
+
42
+ /** 最近一次成功之后的失败 run 数。scheduled_for 每个排程唯一(UNIQUE),按它排序是确定的。 */
43
+ export function failureStreak(scheduleId: string): FailureStreak {
44
+ const lastSuccess = db.get<{ run_id: string; scheduled_for: string }>(
45
+ "SELECT run_id, scheduled_for FROM scheduled_task_runs WHERE schedule_id = ?1 AND status = 'replied' ORDER BY scheduled_for DESC LIMIT 1",
46
+ scheduleId,
47
+ );
48
+ const n = db.get<{ n: number }>(
49
+ "SELECT COUNT(*) AS n FROM scheduled_task_runs WHERE schedule_id = ?1 AND status = 'failed' AND scheduled_for > ?2",
50
+ scheduleId, lastSuccess?.scheduled_for ?? "",
51
+ )?.n ?? 0;
52
+ return { consecutiveFailures: Number(n), lastSuccessRunId: lastSuccess?.run_id ?? null };
53
+ }
54
+
55
+ type ScheduleForAlert = {
56
+ schedule_id: string;
57
+ network_id: string;
58
+ created_by: string | null;
59
+ name: string;
60
+ target_alias: string;
61
+ status: string;
62
+ };
63
+
64
+ export type FailureNotice = {
65
+ reason: "failing" | "paused";
66
+ scheduleId: string;
67
+ scheduleName: string;
68
+ networkId: string;
69
+ createdBy: string;
70
+ alias: string;
71
+ failures: number;
72
+ errorCode: string | null;
73
+ errorMessage: string | null;
74
+ };
75
+
76
+ function latestFailure(scheduleId: string): { error_code: string | null; error_message: string | null } {
77
+ const row = db.get<{
78
+ error_code: string | null; error_message: string | null; result: string | null;
79
+ task_created_at: string | null; task_started_at: string | null; task_completed_at: string | null;
80
+ }>(
81
+ `SELECT r.error_code, r.error_message, t.result,
82
+ t.created_at AS task_created_at, t.started_at AS task_started_at, t.completed_at AS task_completed_at
83
+ FROM scheduled_task_runs r
84
+ LEFT JOIN tasks t ON t.task_id = r.task_id AND t.network_id = r.network_id
85
+ WHERE r.schedule_id = ?1 AND r.status = 'failed'
86
+ ORDER BY r.scheduled_for DESC LIMIT 1`,
87
+ scheduleId,
88
+ );
89
+ // 任务以 failed 结束时 run.error_message 为空,原因在任务的回复正文里。
90
+ // #2884 —— 共享会话忙的那一类,再把「排队 / 会话等待」两段实测补在后面:
91
+ // 只读拼接,不改写入路径,所以节点原文不会被覆盖。
92
+ const base = row?.error_message || row?.result || null;
93
+ if (row?.error_code !== "task_failed_busy") return { error_code: row?.error_code ?? null, error_message: base };
94
+ const note = busyElapsedNote(row?.task_created_at, row?.task_started_at, row?.task_completed_at);
95
+ return {
96
+ error_code: row?.error_code ?? null,
97
+ error_message: note ? [base, note].filter(Boolean).join(" ") : base,
98
+ };
99
+ }
100
+
101
+ function truncate(s: string, max: number): string {
102
+ const flat = s.replace(/\s+/g, " ").trim();
103
+ return flat.length > max ? `${flat.slice(0, max)}…` : flat;
104
+ }
105
+
106
+ export function failureNoticeText(n: FailureNotice): { title: string; text: string } {
107
+ const err = n.errorMessage ? truncate(n.errorMessage, ERROR_PREVIEW_CHARS) : "(没有错误详情)";
108
+ const code = n.errorCode ? ` [${n.errorCode}]` : "";
109
+ const head = `定时任务「${n.scheduleName}」(派给节点 ${n.alias})已经连续 ${n.failures} 次执行失败。\n\n最近一次的错误${code}:${err}\n\n`;
110
+ if (n.reason === "paused") {
111
+ return {
112
+ title: "定时任务连续失败,已自动暂停",
113
+ text: `${head}按 Hub 的设置,它已被自动暂停,不会再派发。处理好原因后,在定时任务里把它恢复为「启用」即可。`,
114
+ };
115
+ }
116
+ return {
117
+ title: "定时任务连续失败",
118
+ text: `${head}它仍在按计划运行。如需停止,在定时任务里暂停它(或 PATCH /api/scheduled-tasks/${n.scheduleId},body 带 {"status":"paused","revision":<当前 revision>})。`
119
+ + `在它重新成功之前,24 小时内不会再提醒。`,
120
+ };
121
+ }
122
+
123
+ export function sendFailureNotice(n: FailureNotice): string | null {
124
+ try {
125
+ const { title, text } = failureNoticeText(n);
126
+ return sendAgentNotice({
127
+ networkId: n.networkId, userId: n.createdBy, fromAlias: n.alias, kind: SCHEDULE_FAILING_NOTICE_KIND, title, text, severity: "error",
128
+ meta: { schedule_failing: { reason: n.reason, schedule_id: n.scheduleId, consecutive_failures: n.failures, error_code: n.errorCode } },
129
+ idPrefix: "dm_sched_",
130
+ });
131
+ } catch (e: any) {
132
+ console.error(`[scheduled-tasks] failure notice failed schedule=${n.scheduleId}: ${e?.message || e}`);
133
+ return null;
134
+ }
135
+ }
136
+
137
+ /**
138
+ * 一个 run 以 failed 结束之后调用(派发时失败 / 任务 failed 都算)。返回发出的通知(测试用),没有则 null。
139
+ * 告警与暂停都是条件写:并发的两个调用方只有一个写得进去,所以每段失败恰好一条通知。
140
+ */
141
+ export function evaluateScheduleFailures(scheduleId: string, now = new Date()): FailureNotice | null {
142
+ const row = db.get<ScheduleForAlert>(
143
+ "SELECT schedule_id, network_id, created_by, name, target_alias, status FROM scheduled_tasks WHERE schedule_id = ?1",
144
+ scheduleId,
145
+ );
146
+ if (!row) return null;
147
+ const streak = failureStreak(scheduleId);
148
+ const noticeAt = failureNoticeRuns();
149
+ const pauseAt = failureAutoPauseRuns();
150
+ const wantsPause = pauseAt > 0 && streak.consecutiveFailures >= pauseAt && row.status === "active";
151
+ if (streak.consecutiveFailures < noticeAt && !wantsPause) return null;
152
+
153
+ let paused = false;
154
+ if (wantsPause) {
155
+ paused = db.run(
156
+ "UPDATE scheduled_tasks SET status = 'paused', next_run_at = NULL, revision = revision + 1, updated_at = datetime('now') WHERE schedule_id = ?1 AND status = 'active'",
157
+ [scheduleId],
158
+ ).changes === 1;
159
+ }
160
+ const key = streak.lastSuccessRunId ?? "";
161
+ const cutoff = new Date(now.getTime() - failureRenoticeMs()).toISOString();
162
+ // 去重:同一段失败(key 没变)且上次告警不到 24 小时 → 不写、不发。暂停是另一件事,照发。
163
+ const claimed = db.run(
164
+ `UPDATE scheduled_tasks SET failure_alert_at = ?1, failure_alert_key = ?2
165
+ WHERE schedule_id = ?3
166
+ AND (failure_alert_at IS NULL OR COALESCE(failure_alert_key, '') <> ?2 OR failure_alert_at <= ?4)`,
167
+ [now.toISOString(), key, scheduleId, cutoff],
168
+ ).changes === 1;
169
+ if (paused && !claimed) {
170
+ db.run("UPDATE scheduled_tasks SET failure_alert_at = ?1, failure_alert_key = ?2 WHERE schedule_id = ?3", [now.toISOString(), key, scheduleId]);
171
+ }
172
+ if (!paused && !claimed) return null;
173
+ if (!row.created_by) return null;
174
+ const latest = latestFailure(scheduleId);
175
+ const notice: FailureNotice = {
176
+ reason: paused ? "paused" : "failing",
177
+ scheduleId, scheduleName: row.name, networkId: row.network_id, createdBy: row.created_by, alias: row.target_alias,
178
+ failures: streak.consecutiveFailures, errorCode: latest.error_code, errorMessage: latest.error_message,
179
+ };
180
+ sendFailureNotice(notice);
181
+ return notice;
182
+ }
183
+
184
+ /**
185
+ * #2622 —— 失败退避:连续失败达到 N 次后把**间隔翻倍**(上限 1 小时),成功一次立刻恢复原节奏。
186
+ *
187
+ * 为什么不是「暂停」:`#523` 已经有 `COMMHUB_SCHEDULE_FAILURE_AUTO_PAUSE_RUNS`(默认关),
188
+ * 但它会把排程**停下来**,需要人来恢复。对“上游额度暂时打满”这类外部原因,暂停太久同样是损失;
189
+ * 退避只是**放慢重试**,外部原因一恢复就会自己跑回原节奏。
190
+ *
191
+ * 现场(2026-10-27 生产 24h):一个 `every_seconds=180` 的排程失败 351 次、成功 24 次 ——
192
+ * 失败与成功交替出现,所以“连续失败”很少攒到暂停阈值,而**每次失败都在按 3 分钟原速重试**。
193
+ *
194
+ * 存储复用 `backoff_effective_seconds`(派发时按 `max(every_seconds, backoff_effective_seconds)` 生效),
195
+ * 与 `no_change_backoff` 互斥:那套逻辑自己会写这一列,这里只在 `no_change_backoff = 0` 时介入。
196
+ *
197
+ * 环境变量:`COMMHUB_SCHEDULE_FAILURE_BACKOFF_RUNS`(默认 6;0 = 关闭)。
198
+ */
199
+ export function failureBackoffRuns(): number { return envInt("COMMHUB_SCHEDULE_FAILURE_BACKOFF_RUNS", 6, 0); }
200
+
201
+ const FAILURE_BACKOFF_CAP_SECONDS = 3600;
202
+
203
+ export function applyFailureBackoff(scheduleId: string, now = new Date()): { streak: number; effective_seconds: number | null } | null {
204
+ const from = failureBackoffRuns();
205
+ if (from <= 0) return null;
206
+ const row = db.get<any>(
207
+ `SELECT schedule_id, revision, status, schedule_type, schedule_json, no_change_backoff,
208
+ backoff_effective_seconds, next_run_at
209
+ FROM scheduled_tasks WHERE schedule_id = ?1`,
210
+ scheduleId,
211
+ );
212
+ if (!row || row.status !== "active" || row.no_change_backoff === 1) return null;
213
+ if (row.schedule_type !== "interval") return null;
214
+ let spec: any = null;
215
+ try { spec = JSON.parse(row.schedule_json || "{}"); } catch { spec = null; }
216
+ const base = spec && spec.type === "interval" ? Number(spec.every_seconds) : NaN;
217
+ if (!Number.isFinite(base) || base <= 0) return null;
218
+
219
+ const streak = failureStreak(scheduleId).consecutiveFailures;
220
+ const effective = streak >= from
221
+ ? Math.min(FAILURE_BACKOFF_CAP_SECONDS, base * 2 ** Math.min(streak - from + 1, 6))
222
+ : null;
223
+ const target = effective ?? base;
224
+ const previous = row.backoff_effective_seconds ?? base;
225
+ let next = row.next_run_at;
226
+ if (target !== previous && next) {
227
+ // 放大:至少推到 now + target(不把已经更远的 next 拉近);
228
+ // 恢复:至少拉回 now + target(不把已经更近的 next 推远)。
229
+ const candidate = new Date(now.getTime() + target * 1000).toISOString();
230
+ const existing = Date.parse(next);
231
+ if (Number.isFinite(existing)) {
232
+ next = new Date(target > previous ? Math.max(existing, Date.parse(candidate)) : Math.min(existing, Date.parse(candidate))).toISOString();
233
+ }
234
+ }
235
+ db.run(
236
+ `UPDATE scheduled_tasks SET backoff_streak = ?1, backoff_effective_seconds = ?2, next_run_at = ?3,
237
+ updated_at = datetime('now') WHERE schedule_id = ?4 AND revision = ?5 AND status = 'active'`,
238
+ [streak, effective, next, scheduleId, row.revision],
239
+ );
240
+ return { streak, effective_seconds: effective };
241
+ }
242
+
243
+ /**
244
+ * #2622(第二轮)—— 「跑不起来」的两类跳过也要退避。
245
+ *
246
+ * 生产 24h(2026-10-27):`previous_run_active` 2286 条、`target_offline` 1951 条,
247
+ * 而 `failureStreak()` 只数 `status='failed'`,**skipped 完全不计**,
248
+ * 于是一个 `every_seconds=180` 的排程可以在目标永久离线的状态下每 3 分钟写一行 skip,
249
+ * 一天 470 条,永不减速。
250
+ *
251
+ * 这里只做「放慢」,不改变派发语义:
252
+ * · 同一 error_code 连续出现 ≥ N(默认 3)⇒ 间隔按 2 倍放大,上限 1800 秒;
253
+ * · `target_offline` 的链用更长的上限(7200 秒 = 2 小时)——目标不在时没有理由每 3 分钟试一次;
254
+ * · 换链(原因变了)或出现成功 ⇒ 立刻回到原间隔;
255
+ * · 与 `no_change_backoff=1` 的排程互斥(那套逻辑自己拥有 backoff_* 三列)。
256
+ */
257
+ const NO_PROGRESS_CODES = ["previous_run_active", "target_offline"] as const;
258
+ /** 统计窗口:看最近多少次 run 里同因跳过出现了几次(默认 10)。 */
259
+ const NO_PROGRESS_WINDOW = 10;
260
+ export type NoProgressCause = (typeof NO_PROGRESS_CODES)[number];
261
+
262
+ function noProgressFromRuns(runs: Array<{ status: string; error_code: string | null }>): { cause: NoProgressCause | null; streak: number } {
263
+ const first = runs[0];
264
+ if (!first) return { cause: null, streak: 0 };
265
+ let cause: NoProgressCause | null = null;
266
+ if (first.status === "failed") cause = null; // 由 applyFailureBackoff 负责,避免两套逻辑抢同一列
267
+ else if (first.status === "skipped" && (NO_PROGRESS_CODES as readonly string[]).includes(first.error_code ?? "")) {
268
+ cause = first.error_code as NoProgressCause;
269
+ }
270
+ if (!cause) return { cause: null, streak: 0 };
271
+ // #2622(实测修正)—— **按窗口计数,不按连续链**。
272
+ // 生产 4.3 小时实测:一批排程的模式是「跳过 → 跳过 → 成功 → 跳过…」,
273
+ // 成功会把连续链打断,于是 streak 永远攒不到阈值,退避一次都没触发
274
+ // (previous_run_active 24h 从 2397/天 只降到 2258/天)。
275
+ // 改成「最近 N 次里同因跳过出现几次」:仍然要求最新一次是同因跳过(不干扰成功后的恢复),
276
+ // 但如果它在这 N 次里反复出现,就按窗口值放慢。
277
+ const window = runs.slice(0, NO_PROGRESS_WINDOW);
278
+ let streak = 0;
279
+ for (const r of window) {
280
+ if (r.status === "skipped" && r.error_code === cause) streak++;
281
+ }
282
+ return { cause, streak };
283
+ }
284
+
285
+ export function noProgressBackoffRuns(): number {
286
+ return envInt("COMMHUB_SCHEDULE_NO_PROGRESS_BACKOFF_RUNS", 3, 1);
287
+ }
288
+
289
+ export function applyNoProgressBackoff(scheduleId: string, now = new Date()): { streak: number; effective_seconds: number | null; cause: NoProgressCause } | null {
290
+ const from = noProgressBackoffRuns();
291
+ if (from <= 0) return null;
292
+ const row = db.get<any>(
293
+ `SELECT schedule_id, revision, status, schedule_type, schedule_json, no_change_backoff,
294
+ backoff_effective_seconds, next_run_at
295
+ FROM scheduled_tasks WHERE schedule_id = ?1`,
296
+ scheduleId,
297
+ );
298
+ if (!row || row.status !== "active" || row.no_change_backoff === 1) return null;
299
+ if (row.schedule_type !== "interval") return null;
300
+ let spec: any = null;
301
+ try { spec = JSON.parse(row.schedule_json || "{}"); } catch { spec = null; }
302
+ const base = spec && spec.type === "interval" ? Number(spec.every_seconds) : NaN;
303
+ if (!Number.isFinite(base) || base <= 0) return null;
304
+
305
+ const runs = db.all<{ status: string; error_code: string | null }>(
306
+ `SELECT status, error_code FROM scheduled_task_runs
307
+ WHERE schedule_id = ?1 ORDER BY scheduled_for DESC, created_at DESC LIMIT 12`,
308
+ scheduleId,
309
+ );
310
+ const { cause, streak } = noProgressFromRuns(runs);
311
+
312
+ // 回到原间隔(成功 / 换链 / 未达阈值都走这里;只在真的退避过时才写)
313
+ const restore = () => {
314
+ if ((row.backoff_streak ?? 0) === 0 && row.backoff_effective_seconds == null) return;
315
+ // next_run_at 是 hub 的 TEXT UTC(无时区标记),必须走 parseHubTimestamp
316
+ const cur = parseHubTimestamp(row.next_run_at);
317
+ const next = cur === null
318
+ ? row.next_run_at
319
+ : new Date(Math.min(cur, now.getTime() + base * 1000)).toISOString();
320
+ db.run(
321
+ `UPDATE scheduled_tasks SET backoff_streak = 0, backoff_effective_seconds = NULL, next_run_at = ?1,
322
+ updated_at = datetime('now') WHERE schedule_id = ?2 AND revision = ?3 AND status = 'active'`,
323
+ [next, scheduleId, row.revision],
324
+ );
325
+ };
326
+
327
+ if (!cause) { restore(); return null; }
328
+
329
+ // 上限按原因分开:
330
+ // previous_run_active —— 上一次还在跑,30 分钟足够等到它结束;
331
+ // target_offline —— 目标连续 6 次不在(按 180s 基底已是 18 分钟),
332
+ // 再用 3 分钟一次的节奏去敲没有意义;2 小时一次仍能及时接住它回来。
333
+ // 判据:#2622「同类原因日增量 ≤20」——180s 基底配 2 小时上限 ⇒ 12 次/天。
334
+ const cap = cause === "target_offline" ? 7200 : 1800;
335
+ if (streak < from) { restore(); return null; }
336
+
337
+ const effective = Math.min(cap, base * 2 ** Math.min(streak - from + 1, 5));
338
+ const previous = row.backoff_effective_seconds ?? base;
339
+ let next = row.next_run_at;
340
+ // 🔴 生产实测(2026-10-28,`.181` 的两行日志):
341
+ // offline-backoff effective=5760s next_before=+180s ← 退避值没推进去
342
+ // offline-backoff-final next_after=+360s ← 二次 advance 又加了一个 base
343
+ // 根因就在这里:原来用 `effective !== previous` 做守卫,退避一旦进入**稳态**(例如已是 5760),
344
+ // 这个条件恒为 false ⇒ 不再推 next_run_at;而调用方在此之前刚 `advance()` 把 next 重置成 +base。
345
+ // 于是「退避值写着 5760,实际每 360s 跳一次」,且永远不会自愈。
346
+ // 正确语义:只要这一步是「同因跳过」,next_run_at 至少要 ≥ now + effective(只能往后推,不往前拉)。
347
+ if (next) {
348
+ const candidate = now.getTime() + effective * 1000;
349
+ const existing = parseHubTimestamp(next);
350
+ if (existing !== null) {
351
+ next = new Date(Math.max(existing, candidate)).toISOString();
352
+ }
353
+ }
354
+ db.run(
355
+ `UPDATE scheduled_tasks SET backoff_streak = ?1, backoff_effective_seconds = ?2, next_run_at = ?3,
356
+ updated_at = datetime('now') WHERE schedule_id = ?4 AND revision = ?5 AND status = 'active'`,
357
+ [streak, effective, next, scheduleId, row.revision],
358
+ );
359
+ return { streak, effective_seconds: effective, cause };
360
+ }
361
+
362
+ /** GET /runs 的附加字段。 */
363
+ export function failureSummary(scheduleId: string): { consecutive_failures: number; failure_alert_threshold: number; last_failure_alert_at: string | null } {
364
+ const streak = failureStreak(scheduleId);
365
+ const alertAt = db.get<{ failure_alert_at: string | null }>("SELECT failure_alert_at FROM scheduled_tasks WHERE schedule_id = ?1", scheduleId)?.failure_alert_at ?? null;
366
+ return { consecutive_failures: streak.consecutiveFailures, failure_alert_threshold: failureNoticeRuns(), last_failure_alert_at: alertAt };
367
+ }
368
+
369
+ setScheduledRunTerminalHook((scheduleId, status) => {
370
+ if (status === "failed") evaluateScheduleFailures(scheduleId);
371
+ // #2622 —— 成功一次就把间隔恢复原样;失败则按连续失败次数放大间隔。
372
+ try {
373
+ applyFailureBackoff(scheduleId);
374
+ } catch (e: any) {
375
+ console.error(`[scheduled-tasks] failure backoff failed schedule=${scheduleId}: ${e?.message || e}`);
376
+ }
377
+ });
@@ -0,0 +1,133 @@
1
+ // #2622(第二轮)—— 「跑不起来」的两类跳过也要退避。
2
+ //
3
+ // 生产 24h(2026-10-27):previous_run_active 2286 条、target_offline 1951 条,
4
+ // 而 failureStreak 只数 status='failed',skipped 完全不计 ⇒ 永远按原速重试。
5
+ //
6
+ // 这个文件钉住四件事:
7
+ // 1. 同一 error_code 连续 ≥3 次 ⇒ 间隔放大(有上限:previous_run_active 1800s / target_offline 3600s);
8
+ // 2. 换链(原因变了)或出现成功 ⇒ 立刻回到原间隔;
9
+ // 3. no_change_backoff=1 的排程由另一套逻辑拥有 backoff 列,这里完全不介入;
10
+ // 4. 非 interval 排程不介入。
11
+
12
+ import { afterAll, beforeAll, describe, expect, test } from "bun:test";
13
+ import { register } from "./auth.js";
14
+ import { db } from "./db.js";
15
+ import { applyNoProgressBackoff, noProgressBackoffRuns } from "./scheduled-failures.js";
16
+
17
+ const stamp = Date.now();
18
+ let NET = "";
19
+ let creatorId = "";
20
+
21
+ function seedSchedule(everySeconds = 180, extra: Record<string, unknown> = {}) {
22
+ const id = `sched_np_${crypto.randomUUID()}`;
23
+ db.run(
24
+ `INSERT INTO scheduled_tasks
25
+ (schedule_id, network_id, created_by, name, target_node_id, target_alias, task_content, priority,
26
+ schedule_type, schedule_json, timezone, status, next_run_at, overlap_policy)
27
+ VALUES (?1, ?2, ?3, 'np fixture', 'n_np', 'np-node', 'x', 'normal', 'interval', ?4,
28
+ 'Asia/Shanghai', 'active', ?5, 'skip')`,
29
+ [id, NET, creatorId, JSON.stringify({ type: "interval", every_seconds: everySeconds }),
30
+ new Date(Date.now() + everySeconds * 1000).toISOString()],
31
+ );
32
+ for (const [k, v] of Object.entries(extra)) db.run(`UPDATE scheduled_tasks SET ${k} = ?1 WHERE schedule_id = ?2`, [v as any, id]);
33
+ return id;
34
+ }
35
+
36
+ let seq = 0;
37
+ function addRun(scheduleId: string, status: string, code: string | null) {
38
+ seq += 1;
39
+ const when = new Date(Date.now() + seq * 1000).toISOString();
40
+ db.run(
41
+ `INSERT INTO scheduled_task_runs (run_id, schedule_id, network_id, scheduled_for, status, error_code, created_at)
42
+ VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?4)`,
43
+ [`srun_np_${crypto.randomUUID()}`, scheduleId, NET, when, status, code],
44
+ );
45
+ }
46
+
47
+ const reload = (id: string) => db.get<any>("SELECT * FROM scheduled_tasks WHERE schedule_id = ?1", id)!;
48
+
49
+ beforeAll(async () => {
50
+ const auth = register(`np_owner_${stamp}`, "Np-Owner-Strong-1!", undefined, "tester");
51
+ expect(auth.ok).toBe(true);
52
+ NET = auth.network_id!;
53
+ creatorId = auth.user_id!;
54
+ delete process.env.COMMHUB_SCHEDULE_NO_PROGRESS_BACKOFF_RUNS;
55
+ });
56
+
57
+ afterAll(() => { delete process.env.COMMHUB_SCHEDULE_NO_PROGRESS_BACKOFF_RUNS; });
58
+
59
+ describe("applyNoProgressBackoff (#2622)", () => {
60
+ test("defaults to 3 consecutive same-cause skips", () => {
61
+ expect(noProgressBackoffRuns()).toBe(3);
62
+ });
63
+
64
+ test("target_offline streak widens the interval (capped at 2h)", () => {
65
+ const id = seedSchedule(180);
66
+ for (let i = 0; i < 4; i++) addRun(id, "skipped", "target_offline");
67
+ const r = applyNoProgressBackoff(id)!;
68
+ expect(r.cause).toBe("target_offline");
69
+ expect(r.streak).toBe(4);
70
+ expect(r.effective_seconds).toBeGreaterThan(180);
71
+ expect(r.effective_seconds!).toBeLessThanOrEqual(7200);
72
+ expect(reload(id).backoff_effective_seconds).toBe(r.effective_seconds);
73
+ });
74
+
75
+ test("previous_run_active uses a shorter cap (30min)", () => {
76
+ const id = seedSchedule(180);
77
+ for (let i = 0; i < 10; i++) addRun(id, "skipped", "previous_run_active");
78
+ const r = applyNoProgressBackoff(id)!;
79
+ expect(r.cause).toBe("previous_run_active");
80
+ expect(r.effective_seconds).toBe(1800);
81
+ });
82
+
83
+ test("below the threshold it leaves the interval alone", () => {
84
+ const id = seedSchedule(180);
85
+ addRun(id, "skipped", "target_offline");
86
+ addRun(id, "skipped", "target_offline");
87
+ expect(applyNoProgressBackoff(id)).toBeNull();
88
+ expect(reload(id).backoff_effective_seconds).toBeNull();
89
+ });
90
+
91
+ test("a success resets the streak and restores the base interval", () => {
92
+ const id = seedSchedule(180);
93
+ for (let i = 0; i < 5; i++) addRun(id, "skipped", "target_offline");
94
+ const widened = applyNoProgressBackoff(id)!;
95
+ expect(widened.effective_seconds!).toBeGreaterThan(180);
96
+ addRun(id, "replied", null); // 成功一次
97
+ expect(applyNoProgressBackoff(id)).toBeNull();
98
+ const row = reload(id);
99
+ expect(row.backoff_streak).toBe(0);
100
+ expect(row.backoff_effective_seconds).toBeNull();
101
+ });
102
+
103
+ test("switching cause (different error_code) does not keep the streak", () => {
104
+ const id = seedSchedule(180);
105
+ addRun(id, "skipped", "target_offline");
106
+ addRun(id, "skipped", "target_offline");
107
+ addRun(id, "skipped", "previous_run_active"); // 最新一条换链 ⇒ streak=1
108
+ expect(applyNoProgressBackoff(id)).toBeNull();
109
+ });
110
+
111
+
112
+ test("alternating skip/success pattern still backs off (window, not strict chain)", () => {
113
+ // 生产实测:一批排程是「跳过 → 跳过 → 成功 → 跳过…」,成功会把连续链打断,
114
+ // 按严格链计数永远攒不到阈值。窗口计数必须能抓住它。
115
+ const id = seedSchedule(180);
116
+ addRun(id, "skipped", "previous_run_active");
117
+ addRun(id, "skipped", "previous_run_active");
118
+ addRun(id, "replied", null); // 成功打断链
119
+ addRun(id, "skipped", "previous_run_active");
120
+ const r = applyNoProgressBackoff(id);
121
+ expect(r).not.toBeNull();
122
+ expect(r!.cause).toBe("previous_run_active");
123
+ expect(r!.streak).toBe(3);
124
+ expect(r!.effective_seconds).toBeGreaterThan(180);
125
+ });
126
+
127
+ test("schedules that opt into no_change_backoff are left to the other mechanism", () => {
128
+ const id = seedSchedule(180, { no_change_backoff: 1 });
129
+ for (let i = 0; i < 5; i++) addRun(id, "skipped", "target_offline");
130
+ expect(applyNoProgressBackoff(id)).toBeNull();
131
+ expect(reload(id).backoff_effective_seconds).toBeNull();
132
+ });
133
+ });
@@ -0,0 +1,56 @@
1
+ // 探针:target_offline 连续跳过后,next_run_at 到底被推到哪里?
2
+ // 生产现象(2026-10-28 08:45):backoff_effective_seconds=5760,但实际间隔 360s。
3
+ // 这里用真实调度入口 dispatchScheduledOccurrence 复现,看 next_run_at 与实际 cadence。
4
+ // Run: COMMHUB_DB=/tmp/target-offline-backoff.db bun test src/scheduled-offline-backoff-probe.test.ts
5
+ import { describe, expect, test, beforeAll } from 'bun:test';
6
+ import { register } from './auth.js';
7
+ import { db } from './db.js';
8
+ import { dispatchScheduledOccurrence, getScheduleRow } from './scheduled-tasks.js';
9
+
10
+ const stamp = Date.now();
11
+ const TARGET_ID = `n_offb_${stamp}`, TARGET = `offb-${stamp}`;
12
+ let NET = '', USER = '';
13
+
14
+ function seed() {
15
+ const id = `sched_offb_${crypto.randomUUID()}`;
16
+ db.run(`INSERT INTO scheduled_tasks (schedule_id, network_id, created_by, name, target_node_id, target_alias,
17
+ task_content, priority, schedule_type, schedule_json, timezone, status, next_run_at, overlap_policy)
18
+ VALUES (?1,?2,?3,'offb',?4,?5,'x','normal','interval','{"type":"interval","every_seconds":180}','Asia/Shanghai','active',datetime('now','+3 minutes'),'skip')`,
19
+ [id, NET, USER, TARGET_ID, TARGET]);
20
+ return getScheduleRow(id)!;
21
+ }
22
+
23
+ beforeAll(() => {
24
+ const owner = register(`offb_owner_${stamp}`, 'OffbOwnerPassw0rd!', undefined, 'seed');
25
+ expect(owner.ok).toBe(true);
26
+ NET = owner.network_id!; USER = owner.user!.user_id;
27
+ db.run("INSERT INTO nodes (node_id, node_name, alias, runtime, network_id, owner_user_id) VALUES (?1,?2,?2,'codex',?3,?4)", [TARGET_ID, TARGET, NET, USER]);
28
+ // 目标会话:10 分钟没心跳 ⇒ offline
29
+ db.run("INSERT INTO sessions (resume_id, alias, status, node_id, network_id, last_seen_at) VALUES (?1,?2,'idle',?3,?4,datetime('now','-10 minutes'))",
30
+ [`r_offb_${stamp}`, TARGET, TARGET_ID, NET]);
31
+ });
32
+
33
+ describe('target_offline backoff 是否真的推后 next_run_at', () => {
34
+ // 稳态回归(2026-10-28 生产实测):退避值已经等于 effective 时,原来 `effective !== previous`
35
+ // 的守卫恒假 ⇒ 不再推 next_run_at ⇒ 实际每 360s 跳一次。这里连打 6 次(eff 已进稳态)验证仍推。
36
+ test('连续 4 次离线跳过后,next_run_at 应 ≈ now + effective(而不是 base 的倍数)', () => {
37
+ const s = seed();
38
+ const rows: any[] = [];
39
+ for (let i = 0; i < 9; i++) { // 9 次 ⇒ streak 到 10 ⇒ eff 进入稳态 5760,覆盖「第二次起不推」的坑
40
+ const at = new Date(Date.now() + i * 10).toISOString();
41
+ const r = dispatchScheduledOccurrence(getScheduleRow(s.schedule_id)!, at, true, new Date());
42
+ const after = db.get<any>("SELECT backoff_effective_seconds, backoff_streak, next_run_at FROM scheduled_tasks WHERE schedule_id = ?1", s.schedule_id);
43
+ const nextMs = after.next_run_at ? Date.parse(String(after.next_run_at).replace(' ', 'T') + (String(after.next_run_at).endsWith('Z') ? '' : 'Z')) : NaN;
44
+ rows.push({ i, status: r.status, eff: after.backoff_effective_seconds, streak: after.backoff_streak, next_raw: after.next_run_at, gap_s: Number.isFinite(nextMs) ? Math.round((nextMs - Date.parse(at))/1000) : 'NaN' });
45
+ }
46
+ console.log('每次 fire 后:', JSON.stringify(rows));
47
+ const last = rows[rows.length - 1];
48
+ // 断言:最后一次的 next_run_at 与 effective 同量级(允许 ±2 个 base 的容差)
49
+ if (!last.eff) throw new Error('backoff_effective_seconds 始终为空:退避没有生效');
50
+ if (last.gap_s < last.eff) throw new Error(`next_run_at 只推后 ${last.gap_s}s,但 effective=${last.eff}s`);
51
+ // 稳态:所有 fire 的 gap 都必须 ≥ 当次的 effective(没有「第二次起就不推」)
52
+ for (const row of rows) {
53
+ if (row.eff && row.gap_s < row.eff) throw new Error(`第 ${row.i} 次:gap=${row.gap_s}s < effective=${row.eff}s(稳态没推)`);
54
+ }
55
+ });
56
+ });