@sleep2agi/commhub-server 0.9.0-preview.27 → 0.9.0-preview.271
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +120 -3
- package/bin/commhub.ts +93 -9
- package/build-info.json +7 -0
- package/package.json +10 -8
- package/src/ack-create-request-transport.test.ts +182 -0
- package/src/ack-create-request.test.ts +52 -8
- package/src/admin-delete-network-http.test.ts +229 -0
- package/src/admin-networks-http.test.ts +88 -0
- package/src/adoption-candidates-http.test.ts +140 -0
- package/src/adoption-candidates.ts +158 -0
- package/src/agent-access-batch.test.ts +210 -0
- package/src/agent-access.ts +585 -0
- package/src/agent-acl-groups-http.test.ts +247 -0
- package/src/agent-acl-http.test.ts +691 -0
- package/src/agent-acl-slice3-http.test.ts +885 -0
- package/src/agent-department-grants.test.ts +129 -0
- package/src/agent-department-http.test.ts +215 -0
- package/src/agent-groups-perf.test.ts +223 -0
- package/src/agent-notice.ts +48 -0
- package/src/agent-team-whoami-http.test.ts +98 -0
- package/src/agent-teams-http.test.ts +330 -0
- package/src/agent-teams.ts +244 -0
- package/src/alias-baseline-2571.test.ts +162 -0
- package/src/alias-baseline.ts +113 -0
- package/src/alias-filter.test.ts +72 -0
- package/src/alias-filter.ts +42 -0
- package/src/api-host-supervisors-can-create-nodes.test.ts +350 -0
- package/src/api-host-supervisors-fallback.test.ts +6 -7
- package/src/api-host-supervisors-runtime-readiness.test.ts +195 -0
- package/src/api-nodes-shape.test.ts +131 -4
- package/src/auth-abuse-counters.test.ts +144 -0
- package/src/auth-abuse-counters.ts +122 -0
- package/src/auth-layer-shape.test.ts +105 -0
- package/src/auth-password-change-cli-tokens-http.test.ts +423 -0
- package/src/auth-sessions-http.test.ts +260 -0
- package/src/auth-validate.test.ts +5 -1
- package/src/auth-weak-password-login.test.ts +176 -0
- package/src/auth.ts +641 -68
- package/src/backfill-node-ids-2664.test.ts +128 -0
- package/src/binding-generation-http.test.ts +141 -0
- package/src/blocked-exit-on-reply.test.ts +122 -0
- package/src/broadcast-fanout-2721.test.ts +149 -0
- package/src/broadcast-fanout.ts +117 -0
- package/src/build-commit-report-http.test.ts +128 -0
- package/src/build-info.test.ts +60 -0
- package/src/build-info.ts +85 -0
- package/src/chain-attempt-dedup.test.ts +447 -0
- package/src/clip.test.ts +36 -0
- package/src/clip.ts +48 -0
- package/src/clone-node.test.ts +269 -0
- package/src/clone-node.ts +289 -0
- package/src/codex-config-flags.ts +77 -0
- package/src/codex-fork-contract.ts +23 -0
- package/src/codex-fork-migration.test.ts +38 -0
- package/src/commhub-bin-args.test.ts +85 -0
- package/src/config-apply-validate.test.ts +188 -1
- package/src/config-apply-validate.ts +222 -3
- package/src/cors-origins.test.ts +73 -0
- package/src/create-node-blocked-gate.test.ts +365 -0
- package/src/create-node-error-domain.test.ts +127 -0
- package/src/create-node-network-id.test.ts +91 -0
- package/src/create-node-tool-schema.test.ts +49 -0
- package/src/create-node-unicode-name.test.ts +126 -0
- package/src/create-node-validate.test.ts +199 -7
- package/src/create-node-validate.ts +232 -15
- package/src/create-node-workdir.test.ts +298 -0
- package/src/create-node.ts +88 -23
- package/src/daemon-provider-snapshots.test.ts +453 -0
- package/src/daemon-provider-snapshots.ts +327 -0
- package/src/daemon-token-owner-http.test.ts +346 -0
- package/src/dashboard-slash-routing-http.test.ts +109 -0
- package/src/db-adapter-guard.test.ts +148 -0
- package/src/db-adapter-translate.test.ts +128 -0
- package/src/db-adapter.ts +377 -91
- package/src/db-timestamp.test.ts +38 -0
- package/src/db-timestamp.ts +15 -0
- package/src/db.ts +1298 -153
- package/src/delivered-without-push.test.ts +140 -0
- package/src/department-groups-http.test.ts +209 -0
- package/src/department-groups-sync-http.test.ts +328 -0
- package/src/department-groups.ts +434 -0
- package/src/department-heads-http.test.ts +350 -0
- package/src/department-heads.ts +226 -0
- package/src/departments-agent-count.test.ts +139 -0
- package/src/departments-http.test.ts +185 -0
- package/src/departments-mcp-error-domain.test.ts +135 -0
- package/src/departments-mcp.test.ts +128 -0
- package/src/departments.ts +318 -0
- package/src/desktop-message-seam.test.ts +150 -0
- package/src/desktop-message.test.ts +368 -0
- package/src/dm-file-scope-http.test.ts +227 -0
- package/src/empty-node-sweeper.test.ts +104 -0
- package/src/empty-node-sweeper.ts +88 -0
- package/src/env-transport-gate-http.test.ts +240 -0
- package/src/env-transport-gate.ts +80 -0
- package/src/env-transport-policy.ts +56 -0
- package/src/etag-of.test.ts +57 -0
- package/src/event-loop-lag.test.ts +180 -0
- package/src/event-loop-lag.ts +201 -0
- package/src/expiry-counters.test.ts +187 -0
- package/src/expiry-counters.ts +227 -0
- package/src/expiry-evidence-db.test.ts +103 -0
- package/src/ext-token-shared.test.ts +60 -0
- package/src/ext-validation-regression.test.ts +10 -17
- package/src/external-schedule-edits-http.test.ts +321 -0
- package/src/external-schedule-edits-timestamps.test.ts +64 -0
- package/src/external-schedule-edits.ts +327 -0
- package/src/file-download-authz-dev-open.test.ts +1 -1
- package/src/file-download-authz.test.ts +25 -4
- package/src/file-download-date-bucket.test.ts +1 -1
- package/src/file-network-scope-dev-open.test.ts +1 -1
- package/src/file-network-scope.test.ts +7 -6
- package/src/group-membership-events-http.test.ts +264 -0
- package/src/group-messages-http.test.ts +483 -0
- package/src/group-messages.ts +297 -0
- package/src/health-anon-readers.ts +67 -0
- package/src/health-host-disk-2756.test.ts +71 -0
- package/src/health-identity-root.test.ts +98 -0
- package/src/health-redaction.test.ts +1599 -7
- package/src/health-status-reasons.test.ts +84 -0
- package/src/host-supervisor-identity-http.test.ts +152 -0
- package/src/host-supervisor-identity.ts +92 -0
- package/src/host-supervisors-daemon-version.test.ts +137 -0
- package/src/http-gzip.test.ts +94 -0
- package/src/http-gzip.ts +112 -0
- package/src/hub-catalog-read-http.test.ts +664 -0
- package/src/hub-provider-config.test.ts +166 -0
- package/src/hub-provider-config.ts +202 -0
- package/src/hub-timestamp.test.ts +38 -0
- package/src/hub-timestamp.ts +27 -0
- package/src/human-dm.ts +215 -0
- package/src/human-inbox.ts +42 -0
- package/src/inbox-caller.ts +49 -0
- package/src/inbox-count-sse.test.ts +318 -0
- package/src/inbox-count.ts +29 -0
- package/src/index.ts +6 -2
- package/src/license-state.test.ts +105 -0
- package/src/license-state.ts +48 -0
- package/src/lifecycle-error-domain.test.ts +149 -0
- package/src/list-ack-error-domain.test.ts +108 -0
- package/src/list-host-supervisors.test.ts +15 -4
- package/src/list-tasks-omit-default.test.ts +84 -0
- package/src/list-tasks-since-last.test.ts +100 -0
- package/src/list-tasks-since.test.ts +104 -0
- package/src/mcp-context-cost-http.test.ts +127 -0
- package/src/mcp-inflight.test.ts +92 -0
- package/src/mcp-inflight.ts +38 -0
- package/src/mcp-projects-events-http.test.ts +168 -0
- package/src/mcp-route-label-http.test.ts +120 -0
- package/src/mcp-write-network-resolution.test.ts +21 -0
- package/src/member-audit-network-id-http.test.ts +93 -0
- package/src/member-presence-http.test.ts +210 -0
- package/src/messages-ack-by-agent-http.test.ts +142 -0
- package/src/migrate-sqlite-to-pg.test.ts +47 -0
- package/src/migrate-sqlite-to-pg.ts +263 -0
- package/src/model-auth-notify-http.test.ts +181 -0
- package/src/model-auth-notify.ts +116 -0
- package/src/network-name-http.test.ts +75 -0
- package/src/network-quota-config.test.ts +100 -0
- package/src/network-rules-http.test.ts +169 -0
- package/src/network-rules.ts +129 -0
- package/src/network-scope.ts +222 -3
- package/src/network-token-membership-http.test.ts +186 -0
- package/src/node-alias-control.ts +12 -0
- package/src/node-attrs.test.ts +47 -2
- package/src/node-caller-identity-http.test.ts +1108 -0
- package/src/node-config-last-update.test.ts +126 -0
- package/src/node-config-last-update.ts +107 -0
- package/src/node-create-request-status-http.test.ts +105 -0
- package/src/node-daemon-bindings.test.ts +142 -0
- package/src/node-daemon-bindings.ts +135 -0
- package/src/node-failure-alert.test.ts +150 -0
- package/src/node-failure-alert.ts +177 -0
- package/src/node-files-transport.test.ts +350 -0
- package/src/node-health-codex-login.test.ts +120 -0
- package/src/node-health-dispatch-http.test.ts +373 -0
- package/src/node-health-guard-origin.test.ts +98 -0
- package/src/node-health-guard.ts +121 -0
- package/src/node-health-passthrough.test.ts +115 -0
- package/src/node-health-store.ts +101 -0
- package/src/node-id-alias.test.ts +190 -0
- package/src/node-identity-conflict-http.test.ts +269 -0
- package/src/node-identity-conflict.ts +142 -0
- package/src/node-identity-mismatch.test.ts +74 -0
- package/src/node-in-flight-tasks.ts +31 -0
- package/src/node-lifecycle-controllable.test.ts +64 -0
- package/src/node-lifecycle-controllable.ts +53 -0
- package/src/node-lifecycle-read-http.test.ts +257 -0
- package/src/node-lifecycle-read.ts +124 -0
- package/src/node-logs-transport.test.ts +266 -0
- package/src/node-peer-allowlist.test.ts +114 -0
- package/src/node-permissions-http.test.ts +901 -0
- package/src/node-permissions.ts +496 -0
- package/src/node-request-retention.test.ts +281 -0
- package/src/node-request-retention.ts +120 -0
- package/src/node-secret-rest-http.test.ts +190 -0
- package/src/node-secret-vault-inject.test.ts +132 -0
- package/src/node-security-write-gate.test.ts +72 -0
- package/src/node-serviceability.test.ts +86 -0
- package/src/node-serviceability.ts +124 -0
- package/src/node-session-control-http.test.ts +310 -0
- package/src/node-session-control.ts +258 -0
- package/src/node-skills-transport.test.ts +212 -0
- package/src/node-token-identity.ts +35 -0
- package/src/node-token-ownership.ts +51 -0
- package/src/node-token-privilege-http.test.ts +224 -0
- package/src/node-token-usage.test.ts +90 -0
- package/src/node-token-usage.ts +164 -0
- package/src/not-found-api-http.test.ts +67 -0
- package/src/observer-avatar-http.test.ts +103 -10
- package/src/observer-push.test.ts +76 -0
- package/src/opencode-catalog-report-http.test.ts +139 -0
- package/src/opencode-v2-catalog.test.ts +268 -0
- package/src/opencode-v2-catalog.ts +356 -0
- package/src/package-version-consistency.test.ts +25 -0
- package/src/parent-infer.test.ts +82 -0
- package/src/parent-infer.ts +110 -0
- package/src/parent-source-marker.test.ts +70 -0
- package/src/parent-source-orphan.test.ts +121 -0
- package/src/password-network-tokens.ts +25 -0
- package/src/peer-reply-atomic.test.ts +462 -0
- package/src/peer-reply-capability.test.ts +70 -0
- package/src/pg-worker.ts +87 -0
- package/src/poll-pacing.test.ts +74 -0
- package/src/poll-pacing.ts +123 -0
- package/src/probe-provider-error-domain.test.ts +145 -0
- package/src/project-dir-warn-clear-http.test.ts +132 -0
- package/src/project-viewer-can-http.test.ts +116 -0
- package/src/provider-skill-error-domain.test.ts +130 -0
- package/src/provider-tools-shape.test.ts +165 -0
- package/src/push.test.ts +20 -10
- package/src/push.ts +373 -9
- package/src/redact-tokens.test.ts +86 -0
- package/src/redact-tokens.ts +43 -0
- package/src/rejection-counters.test.ts +72 -0
- package/src/rejection-counters.ts +173 -0
- package/src/rename.ts +38 -10
- package/src/reply-target-derivation.test.ts +96 -0
- package/src/report-status-handover-capabilities-http.test.ts +124 -0
- package/src/report-status-host-schema.test.ts +235 -0
- package/src/report-status-resume-id-handover.test.ts +184 -0
- package/src/report-status-secret-keys.test.ts +140 -0
- package/src/require-explicit-test-db.ts +9 -0
- package/src/requirement-due-reminders-http.test.ts +620 -0
- package/src/requirement-due-reminders.ts +416 -0
- package/src/requirement-events-http.test.ts +318 -0
- package/src/requirement-events.test.ts +45 -0
- package/src/requirement-events.ts +144 -0
- package/src/requirement-notify.ts +165 -0
- package/src/requirement-participant-notify-http.test.ts +313 -0
- package/src/requirement-tag-ops-http.test.ts +454 -0
- package/src/requirement-tag-tree-http.test.ts +275 -0
- package/src/requirement-tag-tree.ts +127 -0
- package/src/requirement-tags.ts +102 -0
- package/src/requirements-abandoned-http.test.ts +236 -0
- package/src/requirements-assignee-drift.test.ts +154 -0
- package/src/requirements-comment-http.test.ts +174 -0
- package/src/requirements-completed-at-migration.test.ts +70 -0
- package/src/requirements-errors-mcp-http.test.ts +164 -0
- package/src/requirements-errors.ts +158 -0
- package/src/requirements-etag-http.test.ts +91 -0
- package/src/requirements-http.test.ts +709 -0
- package/src/requirements-last-event-http.test.ts +349 -0
- package/src/requirements-list-mcp-slim-http.test.ts +155 -0
- package/src/requirements-list-slim-http.test.ts +324 -0
- package/src/requirements-mcp.test.ts +180 -0
- package/src/requirements-migrate.ts +283 -0
- package/src/requirements-owner-strict-mcp-http.test.ts +145 -0
- package/src/requirements-people-http.test.ts +114 -0
- package/src/requirements-people-mcp-http.test.ts +231 -0
- package/src/requirements-people.ts +196 -0
- package/src/requirements-priority-lowest.test.ts +114 -0
- package/src/requirements-projects-network-echo.test.ts +57 -0
- package/src/requirements-projects-parent-http.test.ts +123 -0
- package/src/requirements-search-http.test.ts +182 -0
- package/src/requirements-search.test.ts +32 -0
- package/src/requirements-search.ts +145 -0
- package/src/requirements-seq-http.test.ts +193 -0
- package/src/requirements-seq-migration.test.ts +63 -0
- package/src/requirements-start-date.test.ts +125 -0
- package/src/requirements-stats-http.test.ts +357 -0
- package/src/requirements-stats.ts +195 -0
- package/src/requirements-status-participants-http.test.ts +222 -0
- package/src/requirements-upgrade-e2e.test.ts +126 -0
- package/src/requirements-upgrade-migration.test.ts +98 -0
- package/src/requirements.ts +2205 -0
- package/src/resolve-port.test.ts +57 -0
- package/src/resolve-port.ts +44 -0
- package/src/rest-alias-network-fallback-http.test.ts +123 -0
- package/src/rest-alias-network-fallback.test.ts +60 -0
- package/src/rest-alias-network-fallback.ts +65 -0
- package/src/rest-explicit-columns-http.test.ts +266 -0
- package/src/rest-identity.test.ts +130 -0
- package/src/rest-identity.ts +141 -0
- package/src/rest-projections.ts +72 -0
- package/src/rest-read-network-real-chain.test.ts +97 -0
- package/src/rest-write-network-resolution.test.ts +133 -0
- package/src/restart-readiness-http.test.ts +121 -0
- package/src/restricted-files.ts +76 -0
- package/src/retention-first-sweep-quiet.test.ts +31 -0
- package/src/retention.test.ts +100 -40
- package/src/retention.ts +423 -104
- package/src/route-callers-http.test.ts +120 -0
- package/src/route-timing.test.ts +218 -0
- package/src/route-timing.ts +262 -0
- package/src/rules-file-duplicate-alias.test.ts +234 -0
- package/src/rules-file-session-target.test.ts +207 -0
- package/src/rules-file-transport.test.ts +271 -0
- package/src/runtime-label.test.ts +22 -0
- package/src/runtime-label.ts +21 -0
- package/src/runtime-readiness.ts +90 -0
- package/src/schedule-agent-mcp-http.test.ts +486 -0
- package/src/schedule-agent.ts +252 -0
- package/src/schedule-starved.test.ts +92 -0
- package/src/schedule-starved.ts +114 -0
- package/src/scheduled-backoff-http.test.ts +203 -0
- package/src/scheduled-backoff.ts +64 -0
- package/src/scheduled-doorbell-nodeless-session.test.ts +134 -0
- package/src/scheduled-failure-backoff-2622.test.ts +145 -0
- package/src/scheduled-failure-code.test.ts +73 -0
- package/src/scheduled-failure-code.ts +105 -0
- package/src/scheduled-failures-http.test.ts +279 -0
- package/src/scheduled-failures.ts +425 -0
- package/src/scheduled-noprogress-backoff-2622.test.ts +163 -0
- package/src/scheduled-offline-backoff-probe.test.ts +56 -0
- package/src/scheduled-offline.test.ts +131 -0
- package/src/scheduled-offline.ts +66 -0
- package/src/scheduled-outcome-guidance-http.test.ts +139 -0
- package/src/scheduled-outcome-guidance.ts +9 -0
- package/src/scheduled-reply-unread-http.test.ts +212 -0
- package/src/scheduled-requirement-wake-http.test.ts +289 -0
- package/src/scheduled-requirement-wake.ts +125 -0
- package/src/scheduled-run-terminal.test.ts +290 -0
- package/src/scheduled-skip-blocking-task.test.ts +120 -0
- package/src/scheduled-skip-path-backoff-2622.test.ts +98 -0
- package/src/scheduled-stuck-http.test.ts +316 -0
- package/src/scheduled-stuck-notice-cap.test.ts +80 -0
- package/src/scheduled-stuck.ts +219 -0
- package/src/scheduled-tasks-http.test.ts +679 -0
- package/src/scheduled-tasks.ts +959 -0
- package/src/scheduled-work-outcome-http.test.ts +154 -0
- package/src/scheduled-work-outcome.ts +50 -0
- package/src/scoped-secrets.ts +167 -0
- package/src/secret-echo-negative-control.test.ts +118 -0
- package/src/send-dedup-request-id.test.ts +229 -0
- package/src/send-reply-agent-warning.test.ts +38 -38
- package/src/send-reply-alias-optional.test.ts +131 -0
- package/src/send-reply-attachments.test.ts +28 -7
- package/src/send_dedup.test.ts +54 -0
- package/src/send_dedup.ts +52 -6
- package/src/serve-error.test.ts +73 -0
- package/src/serve-error.ts +56 -0
- package/src/server.ts +4963 -377
- package/src/session-task-on-dispatch.ts +14 -0
- package/src/session-token-ownership-http.test.ts +122 -0
- package/src/shared/codex-inventory.ts +41 -0
- package/src/shared/daemon-provider-login.ts +31 -0
- package/src/shared/daemon-provider-snapshot.ts +52 -0
- package/src/shared/external-schedule-contract.ts +60 -0
- package/src/shared/node-name.test.ts +62 -0
- package/src/shared/node-name.ts +161 -0
- package/src/shared/probe-host-allowlist-drift.test.ts +45 -0
- package/src/shared/provider-application.ts +29 -0
- package/src/shared/provider-collection.ts +14 -0
- package/src/shared/provider-initialization.ts +8 -0
- package/src/shared/reserved-env-drift.test.ts +10 -0
- package/src/shared/scoped-secret-contract.ts +5 -0
- package/src/shared/task-content-limit-drift.test.ts +28 -0
- package/src/shared/task-content-limit.ts +25 -0
- package/src/side-thread-command-transport.test.ts +312 -0
- package/src/side-thread-command-transport.ts +318 -0
- package/src/side-thread-http-integration.test.ts +233 -0
- package/src/side-thread-http.ts +504 -0
- package/src/side-thread-production.test.ts +72 -0
- package/src/side-thread-production.ts +118 -0
- package/src/side-thread.test.ts +1067 -0
- package/src/side-thread.ts +2499 -0
- package/src/skillhub-http.test.ts +144 -0
- package/src/sqlite-close-checkpoint.test.ts +78 -0
- package/src/start-node.test.ts +265 -0
- package/src/start-request-error-domain.test.ts +182 -0
- package/src/start-skill-error-domain.test.ts +122 -0
- package/src/stats-versions-http.test.ts +65 -0
- package/src/status-alias-filter-http.test.ts +106 -0
- package/src/status-alias-resolver-http.test.ts +163 -0
- package/src/status-alias-resolver.ts +19 -0
- package/src/status-node-id-filter-http.test.ts +161 -0
- package/src/status-read-cache-http.test.ts +229 -0
- package/src/status-read-cache.ts +129 -0
- package/src/stop-delete-node.test.ts +567 -1
- package/src/task-access-http.test.ts +407 -0
- package/src/task-access.ts +420 -0
- package/src/task-activity.ts +60 -0
- package/src/task-auth-origin-http.test.ts +93 -0
- package/src/task-auth-origin.test.ts +24 -0
- package/src/task-auth-origin.ts +7 -0
- package/src/task-consumption.test.ts +508 -0
- package/src/task-diagnostic-http.test.ts +149 -0
- package/src/task-diagnostic.test.ts +115 -0
- package/src/task-diagnostic.ts +118 -0
- package/src/task-expiry-notice-http.test.ts +298 -0
- package/src/task-expiry-notice.ts +318 -0
- package/src/task-idempotency-mcp.test.ts +148 -0
- package/src/task-idempotency.test.ts +76 -0
- package/src/task-idempotency.ts +81 -0
- package/src/task-late-replies.ts +54 -0
- package/src/task-late-reply-http.test.ts +227 -0
- package/src/task-lifecycle-watcher-busy.test.ts +114 -0
- package/src/task-lifecycle-watcher.test.ts +209 -0
- package/src/task-lifecycle-watcher.ts +109 -0
- package/src/task-network-resolution-http.test.ts +140 -0
- package/src/task-not-found-error-domain.test.ts +114 -0
- package/src/task-offline-delivery-2668.test.ts +119 -0
- package/src/task-offline-delivery.ts +81 -0
- package/src/task-orphan-http.test.ts +287 -0
- package/src/task-orphan.ts +119 -0
- package/src/task-queue-ahead-http.test.ts +309 -0
- package/src/task-queue-ahead-scale.test.ts +236 -0
- package/src/task-queue-ahead.ts +186 -0
- package/src/task-rejection-log-http.test.ts +160 -0
- package/src/task-rejection-log.ts +61 -0
- package/src/task-sender-http.test.ts +143 -0
- package/src/task-stale-open-http.test.ts +292 -0
- package/src/task-stale-open.ts +103 -0
- package/src/tasks-pagination-http.test.ts +141 -0
- package/src/team-rules-status-http.test.ts +90 -0
- package/src/team-rules-status.test.ts +59 -0
- package/src/team-rules-status.ts +72 -0
- package/src/tmux-allowlist.test.ts +48 -0
- package/src/tool-audience-http.test.ts +248 -0
- package/src/tool-audience.ts +130 -0
- package/src/tools-list-json-schema.test.ts +94 -0
- package/src/tools.ts +4340 -386
- package/src/update-node-config-error-domain.test.ts +107 -0
- package/src/uploads-http.test.ts +6 -4
- package/src/uploads.ts +23 -4
- package/src/user-avatar-http.test.ts +74 -0
- package/src/user-avatar-image.ts +160 -0
- package/src/user-avatar-migration.test.ts +25 -0
- package/src/user-avatar-upload-http.test.ts +353 -0
- package/src/user-inbox-by-agent-http.test.ts +189 -0
- package/src/user-inbox-netid-notnull.test.ts +122 -0
- package/src/user-inbox-read-http.test.ts +223 -0
- package/src/user-inbox-schema.test.ts +54 -0
- package/src/version-stats.test.ts +66 -0
- package/src/version-stats.ts +109 -0
|
@@ -0,0 +1,180 @@
|
|
|
1
|
+
// #2614 —— 事件循环停顿监测 + 慢 SQL 日志的单元测试。
|
|
2
|
+
//
|
|
3
|
+
// 这两个东西的价值不在“能测出停顿”,而在**停顿发生时留下可对齐的证据**:
|
|
4
|
+
// 2026-10-26 生产两次 RSS 3.6/4.5 GB 的尖峰,事后只能靠“日志静默几十秒”
|
|
5
|
+
// 手工推断,`/api/stats/routes` 只给出受害者请求(含排队时间)。
|
|
6
|
+
// 所以测试要钉住的是语义:怎么算一次停顿、留什么档、日志阈值怎么生效、关闭后还计数。
|
|
7
|
+
|
|
8
|
+
import { describe, expect, test } from "bun:test";
|
|
9
|
+
import {
|
|
10
|
+
eventLoopStats,
|
|
11
|
+
noteRequestServed,
|
|
12
|
+
recordGap,
|
|
13
|
+
resetEventLoopLagForTests,
|
|
14
|
+
startEventLoopLagMonitor,
|
|
15
|
+
} from "./event-loop-lag.js";
|
|
16
|
+
import { formatSlowSql, slowSqlThresholdMs } from "./db-adapter.js";
|
|
17
|
+
|
|
18
|
+
describe("event-loop-lag: recordGap", () => {
|
|
19
|
+
test("正常 tick(≈interval)不算停顿", () => {
|
|
20
|
+
resetEventLoopLagForTests();
|
|
21
|
+
expect(recordGap(260, { intervalMs: 250, thresholdMs: 2000 })).toBe(10);
|
|
22
|
+
const s = eventLoopStats();
|
|
23
|
+
expect(s.stall_count).toBe(0);
|
|
24
|
+
expect(s.max_gap_ms).toBe(260);
|
|
25
|
+
expect(s.recent_stalls).toEqual([]);
|
|
26
|
+
});
|
|
27
|
+
|
|
28
|
+
test("超过阈值的 tick 记一条停顿,带 stall_ms / rss_mb / at", () => {
|
|
29
|
+
resetEventLoopLagForTests();
|
|
30
|
+
const now = Date.parse("2026-10-26T22:10:41Z");
|
|
31
|
+
const stall = recordGap(28_000, { intervalMs: 250, thresholdMs: 2000, rssMb: 4502, now });
|
|
32
|
+
expect(stall).toBe(27_750);
|
|
33
|
+
const s = eventLoopStats();
|
|
34
|
+
expect(s.stall_count).toBe(1);
|
|
35
|
+
expect(s.last_stall_ms).toBe(27_750);
|
|
36
|
+
expect(s.last_stall_at).toBe("2026-10-26T22:10:41.000Z");
|
|
37
|
+
expect(s.max_gap_ms).toBe(28_000);
|
|
38
|
+
expect(s.recent_stalls[0]).toMatchObject({ gap_ms: 28_000, stall_ms: 27_750, rss_mb: 4502 });
|
|
39
|
+
});
|
|
40
|
+
|
|
41
|
+
test("recent_stalls 最新在前,且只保留 20 条", () => {
|
|
42
|
+
resetEventLoopLagForTests();
|
|
43
|
+
for (let i = 0; i < 25; i++) {
|
|
44
|
+
recordGap(5_000 + i, { intervalMs: 250, thresholdMs: 2_000, rssMb: 300, now: 1_700_000_000_000 + i * 1_000 });
|
|
45
|
+
}
|
|
46
|
+
const s = eventLoopStats();
|
|
47
|
+
expect(s.stall_count).toBe(25);
|
|
48
|
+
expect(s.recent_stalls.length).toBe(20);
|
|
49
|
+
// 最新一条是 i=24
|
|
50
|
+
expect(s.recent_stalls[0].gap_ms).toBe(5_024);
|
|
51
|
+
expect(s.recent_stalls[19].gap_ms).toBe(5_005);
|
|
52
|
+
});
|
|
53
|
+
|
|
54
|
+
test("记录这段间隔里服务了多少请求(区分“阻塞”与“timer 饥饿”)", () => {
|
|
55
|
+
resetEventLoopLagForTests();
|
|
56
|
+
recordGap(85_000, { intervalMs: 250, thresholdMs: 2_000, rssMb: 2076, served: 137 });
|
|
57
|
+
const s = eventLoopStats();
|
|
58
|
+
expect(s.recent_stalls[0].requests_since_last_tick).toBe(137);
|
|
59
|
+
// 0 服务数 = 这段时间真的什么都没推进
|
|
60
|
+
recordGap(30_000, { intervalMs: 250, thresholdMs: 2_000, rssMb: 500, served: 0 });
|
|
61
|
+
expect(eventLoopStats().recent_stalls[0].requests_since_last_tick).toBe(0);
|
|
62
|
+
});
|
|
63
|
+
|
|
64
|
+
test("max_gap_ms 保留历史最大值(回落后不被覆盖)", () => {
|
|
65
|
+
resetEventLoopLagForTests();
|
|
66
|
+
recordGap(47_000, { intervalMs: 250, thresholdMs: 2_000 });
|
|
67
|
+
recordGap(260, { intervalMs: 250, thresholdMs: 2_000 });
|
|
68
|
+
const s = eventLoopStats();
|
|
69
|
+
expect(s.max_gap_ms).toBe(47_000);
|
|
70
|
+
expect(s.last_gap_ms).toBe(260);
|
|
71
|
+
});
|
|
72
|
+
});
|
|
73
|
+
|
|
74
|
+
describe("event-loop-lag: startEventLoopLagMonitor", () => {
|
|
75
|
+
test("阈值 0 时不打日志,但仍然计数", async () => {
|
|
76
|
+
resetEventLoopLagForTests();
|
|
77
|
+
const logs: string[] = [];
|
|
78
|
+
const timer = startEventLoopLagMonitor({
|
|
79
|
+
intervalMs: 20,
|
|
80
|
+
thresholdMs: 0,
|
|
81
|
+
log: (m) => logs.push(m),
|
|
82
|
+
rssMb: () => 123,
|
|
83
|
+
});
|
|
84
|
+
// 同步阻塞 80ms,让下一次 tick 一定超阈值
|
|
85
|
+
const until = performance.now() + 80;
|
|
86
|
+
while (performance.now() < until) { /* spin */ }
|
|
87
|
+
await Bun.sleep(60);
|
|
88
|
+
clearInterval(timer);
|
|
89
|
+
expect(logs).toEqual([]);
|
|
90
|
+
expect(eventLoopStats().max_gap_ms).toBeGreaterThan(0);
|
|
91
|
+
});
|
|
92
|
+
|
|
93
|
+
test("真实停顿会打一行 [commhub stall] 且带 rss", async () => {
|
|
94
|
+
resetEventLoopLagForTests();
|
|
95
|
+
const logs: string[] = [];
|
|
96
|
+
const timer = startEventLoopLagMonitor({
|
|
97
|
+
intervalMs: 20,
|
|
98
|
+
thresholdMs: 40,
|
|
99
|
+
log: (m) => logs.push(m),
|
|
100
|
+
rssMb: () => 4502,
|
|
101
|
+
// 这条测的是「停顿怎么打日志」,不是启动尖峰。测试进程刚起来几秒,
|
|
102
|
+
// 不关掉启动宽限的话第一个 tick 会被当成启动尖峰跳过(见 #2721 那组用例)。
|
|
103
|
+
bootGraceMs: 0,
|
|
104
|
+
});
|
|
105
|
+
const until = performance.now() + 120;
|
|
106
|
+
while (performance.now() < until) { /* spin 阻塞事件循环 */ }
|
|
107
|
+
await Bun.sleep(60);
|
|
108
|
+
clearInterval(timer);
|
|
109
|
+
expect(logs.length).toBeGreaterThan(0);
|
|
110
|
+
expect(logs[0]).toStartWith("[commhub stall] loop idle ~");
|
|
111
|
+
expect(logs[0]).toContain("rss 4502 MB");
|
|
112
|
+
expect(logs[0]).toContain("served 0 req");
|
|
113
|
+
});
|
|
114
|
+
});
|
|
115
|
+
|
|
116
|
+
describe("db-adapter: 慢 SQL 日志", () => {
|
|
117
|
+
test("默认阈值 1000ms;环境变量可调,0 表示关闭", () => {
|
|
118
|
+
const prev = process.env.COMMHUB_SLOW_SQL_MS;
|
|
119
|
+
try {
|
|
120
|
+
delete process.env.COMMHUB_SLOW_SQL_MS;
|
|
121
|
+
expect(slowSqlThresholdMs()).toBe(1000);
|
|
122
|
+
process.env.COMMHUB_SLOW_SQL_MS = "250";
|
|
123
|
+
expect(slowSqlThresholdMs()).toBe(250);
|
|
124
|
+
process.env.COMMHUB_SLOW_SQL_MS = "0";
|
|
125
|
+
expect(slowSqlThresholdMs()).toBe(0);
|
|
126
|
+
process.env.COMMHUB_SLOW_SQL_MS = "not-a-number";
|
|
127
|
+
expect(slowSqlThresholdMs()).toBe(1000);
|
|
128
|
+
} finally {
|
|
129
|
+
if (prev === undefined) delete process.env.COMMHUB_SLOW_SQL_MS;
|
|
130
|
+
else process.env.COMMHUB_SLOW_SQL_MS = prev;
|
|
131
|
+
}
|
|
132
|
+
});
|
|
133
|
+
|
|
134
|
+
test("日志压平换行并截断,且不含绑定参数", () => {
|
|
135
|
+
const sql = "DELETE FROM agent_telemetry\n WHERE created_at < datetime('now', ?1)";
|
|
136
|
+
const line = formatSlowSql(10_588.4, sql);
|
|
137
|
+
expect(line).toStartWith("[commhub slow-sql] 10588ms DELETE FROM agent_telemetry WHERE created_at < datetime('now', ?1)");
|
|
138
|
+
expect(line).not.toContain("\n");
|
|
139
|
+
const long = formatSlowSql(1_200, "SELECT " + "x".repeat(500));
|
|
140
|
+
expect(long.length).toBeLessThanOrEqual("[commhub slow-sql] 1200ms ".length + 160);
|
|
141
|
+
});
|
|
142
|
+
});
|
|
143
|
+
|
|
144
|
+
// #2721 —— **只跳过第一个 tick**。
|
|
145
|
+
//
|
|
146
|
+
// 事故形状:进程启动后 3 秒记到 gap 2972ms —— 那是「监控开始 → 第一次 tick」被
|
|
147
|
+
// 模块加载/开库/迁移占满的时间,度量的是启动耗时,不是线上停顿。
|
|
148
|
+
//
|
|
149
|
+
// 第一版实现用的是 10 秒**时间窗**,等于监控在前 10 秒什么都不记,
|
|
150
|
+
// 把下面那条「阈值 40ms、spin 120ms 应打停顿日志」的既有测试整体吞掉了
|
|
151
|
+
// —— 修完 qa.yml 事故、QA 恢复后立刻红。窗口越宽越像「把监控关一会儿」。
|
|
152
|
+
describe("#2721 启动尖峰不算停顿(判据:进程 uptime)", () => {
|
|
153
|
+
test("进程刚启动时的第一个 gap 被跳过", async () => {
|
|
154
|
+
resetEventLoopLagForTests();
|
|
155
|
+
const logs: string[] = [];
|
|
156
|
+
const timer = startEventLoopLagMonitor({
|
|
157
|
+
intervalMs: 20, thresholdMs: 40, log: (m) => logs.push(m), rssMb: () => 123,
|
|
158
|
+
bootGraceMs: 15_000, uptimeMs: () => 3_000, // 模拟「启动后 3 秒」
|
|
159
|
+
});
|
|
160
|
+
const until = performance.now() + 120;
|
|
161
|
+
while (performance.now() < until) { /* spin 落在第一个 tick */ }
|
|
162
|
+
await Bun.sleep(80);
|
|
163
|
+
clearInterval(timer);
|
|
164
|
+
expect(logs).toEqual([]);
|
|
165
|
+
});
|
|
166
|
+
|
|
167
|
+
test("进程已经跑了一会儿 ⇒ 第一个 gap 照常记录(不能把监控关掉)", async () => {
|
|
168
|
+
resetEventLoopLagForTests();
|
|
169
|
+
const logs: string[] = [];
|
|
170
|
+
const timer = startEventLoopLagMonitor({
|
|
171
|
+
intervalMs: 20, thresholdMs: 40, log: (m) => logs.push(m), rssMb: () => 123,
|
|
172
|
+
bootGraceMs: 15_000, uptimeMs: () => 600_000, // 进程跑了 10 分钟
|
|
173
|
+
});
|
|
174
|
+
const until = performance.now() + 120;
|
|
175
|
+
while (performance.now() < until) { /* spin */ }
|
|
176
|
+
await Bun.sleep(80);
|
|
177
|
+
clearInterval(timer);
|
|
178
|
+
expect(logs.length).toBeGreaterThan(0);
|
|
179
|
+
});
|
|
180
|
+
});
|
|
@@ -0,0 +1,201 @@
|
|
|
1
|
+
// #2614 —— 事件循环停顿监测(只观测,不改任何行为)。
|
|
2
|
+
//
|
|
3
|
+
// 背景:2026-10-26 21:55–22:12 生产 Hub 出现两次内存尖峰(RSS 3.6 → 4.5 GB),
|
|
4
|
+
// 伴随机 `mcp_inflight` 387 → 807。事后看日志才发现**同一时刻日志静默 14–47 秒**
|
|
5
|
+
// ——Hub 是单线程(Bun + 同步 SQLite),日志静默就等于事件循环被阻塞。
|
|
6
|
+
//
|
|
7
|
+
// 现有 `/api/stats/routes` 只能告诉我们“哪个请求**受害**了”(它的耗时包含了排队时间),
|
|
8
|
+
// 不能告诉我们“当时**谁**在阻塞”。缺的就是停顿本身的时间序列。
|
|
9
|
+
//
|
|
10
|
+
// 这个模块补上这一格:每 `intervalMs` 醒一次,量 `now - lastTick`;
|
|
11
|
+
// 正常情况 ≈ interval,超出 `thresholdMs` 的部分就是一次真实停顿。
|
|
12
|
+
// 停顿会:① 计数/留档(`/health` 的 `event_loop` 字段可读);
|
|
13
|
+
// ② 打一行日志(带当时的 RSS,方便和内存曲线对齐)。
|
|
14
|
+
//
|
|
15
|
+
// 注意:`setInterval` 自己也会被阻塞 —— 这正是我们要测量的东西。
|
|
16
|
+
// 停顿结束后第一次 tick 的 gap 就是停顿长度,不需要外部进程配合。
|
|
17
|
+
|
|
18
|
+
export type EventLoopStall = {
|
|
19
|
+
/** 这个间隔里处理完的请求数:0 = 真的没推进;>0 = 循环在干活,只是 timer 被饿住。 */
|
|
20
|
+
/** ISO 时间(停顿**结束**的时刻,即这次 tick 的时间)。 */
|
|
21
|
+
at: string;
|
|
22
|
+
/** 距上次 tick 的实际间隔(毫秒)。等于 interval + 停顿。 */
|
|
23
|
+
gap_ms: number;
|
|
24
|
+
/** 停顿量 = gap - interval。 */
|
|
25
|
+
stall_ms: number;
|
|
26
|
+
/** 停顿结束时的进程 RSS(MB),用于和内存尖峰对齐。 */
|
|
27
|
+
rss_mb: number;
|
|
28
|
+
requests_since_last_tick?: number;
|
|
29
|
+
};
|
|
30
|
+
|
|
31
|
+
const KEEP_STALLS = 20;
|
|
32
|
+
|
|
33
|
+
/**
|
|
34
|
+
* #2614(第二轮)—— 2026-10-27 00:39 线上出现 gap 84887ms,但**同一时段日志在正常写**
|
|
35
|
+
* (00:38:52 / 00:38:58 / 00:39:08 …),而且 `/api/stats/routes` 里最慢的请求只有 2.4s。
|
|
36
|
+
* 也就是说:事件循环**没有被阻塞 84 秒**,是 **timer 回调被 I/O 队列饿住**了。
|
|
37
|
+
* 只看 tick 间隔会把这两种情况混成一条“卡了 84 秒”的假结论。
|
|
38
|
+
*
|
|
39
|
+
* 区分办法:数一下这个间隔里**服务了多少请求**。
|
|
40
|
+
* requests_since_last_tick > 0 ⇒ 循环在干活 ⇒ 这是 timer 饥饿(高负载/大批回调)
|
|
41
|
+
* requests_since_last_tick = 0 ⇒ 真的没有推进任何请求 ⇒ 才可能是同步阻塞
|
|
42
|
+
*/
|
|
43
|
+
let servedSinceTick = 0;
|
|
44
|
+
|
|
45
|
+
/** 请求处理完成时调用(route-timing 里)。只计数,不参与任何决策。 */
|
|
46
|
+
export function noteRequestServed(): void {
|
|
47
|
+
servedSinceTick++;
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
let maxGapMs = 0;
|
|
51
|
+
let lastGapMs = 0;
|
|
52
|
+
let stallCount = 0;
|
|
53
|
+
let lastStallAt: string | null = null;
|
|
54
|
+
let lastStallMs = 0;
|
|
55
|
+
const stalls: EventLoopStall[] = [];
|
|
56
|
+
|
|
57
|
+
/** 记录一次 tick 间隔。返回这次 tick 的停顿量(毫秒,可为负→归零)。 */
|
|
58
|
+
export function recordGap(
|
|
59
|
+
gapMs: number,
|
|
60
|
+
opts: { intervalMs?: number; thresholdMs?: number; rssMb?: number; now?: number; served?: number } = {},
|
|
61
|
+
): number {
|
|
62
|
+
const interval = opts.intervalMs ?? 250;
|
|
63
|
+
const threshold = opts.thresholdMs ?? 2000;
|
|
64
|
+
lastGapMs = gapMs;
|
|
65
|
+
if (gapMs > maxGapMs) maxGapMs = gapMs;
|
|
66
|
+
const stallMs = Math.max(0, gapMs - interval);
|
|
67
|
+
if (stallMs < threshold) return stallMs;
|
|
68
|
+
const at = new Date(opts.now ?? Date.now()).toISOString();
|
|
69
|
+
stallCount++;
|
|
70
|
+
lastStallAt = at;
|
|
71
|
+
lastStallMs = stallMs;
|
|
72
|
+
stalls.push({
|
|
73
|
+
at,
|
|
74
|
+
gap_ms: Math.round(gapMs),
|
|
75
|
+
stall_ms: Math.round(stallMs),
|
|
76
|
+
rss_mb: opts.rssMb ?? 0,
|
|
77
|
+
requests_since_last_tick: opts.served ?? 0,
|
|
78
|
+
});
|
|
79
|
+
while (stalls.length > KEEP_STALLS) stalls.shift();
|
|
80
|
+
return stallMs;
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
/**
|
|
84
|
+
* 2026-10-31 —— 窗口两端必须跟着读数一起走(#3174)。
|
|
85
|
+
*
|
|
86
|
+
* 原来 event_loop 给了 stall_count / last_stall_at,却没有窗口起点:
|
|
87
|
+
* 计数器随进程重启归零,而「归零」与「从未发生」在读数上同值 ⇒
|
|
88
|
+
* 单独引用 stall_count 的读者会把它读成「从上线以来没卡过」。
|
|
89
|
+
*
|
|
90
|
+
* ⇒ 现在两个时刻戳都在这对象里:`since`(起点,模块初始化时冻结)与 `observed_at`(读的时刻)。
|
|
91
|
+
* 引用者不必再去 /health 顶层猜窗口,窗口与读数同域。
|
|
92
|
+
* 判据:分子分母各要自己的时刻戳;**被声明的那个永远不算**。
|
|
93
|
+
*/
|
|
94
|
+
let sinceIso = new Date().toISOString();
|
|
95
|
+
|
|
96
|
+
export function eventLoopStats(): {
|
|
97
|
+
max_gap_ms: number;
|
|
98
|
+
last_gap_ms: number;
|
|
99
|
+
stall_count: number;
|
|
100
|
+
last_stall_ms: number;
|
|
101
|
+
last_stall_at: string | null;
|
|
102
|
+
recent_stalls: EventLoopStall[];
|
|
103
|
+
/** 窗口起点(本进程 event-loop 观察开始时刻)。 */
|
|
104
|
+
since: string;
|
|
105
|
+
/** 这次读的时刻。与 `since` 一起才构成窗口;单独任一个都不构成。 */
|
|
106
|
+
observed_at: string;
|
|
107
|
+
} {
|
|
108
|
+
return {
|
|
109
|
+
max_gap_ms: Math.round(maxGapMs),
|
|
110
|
+
last_gap_ms: Math.round(lastGapMs),
|
|
111
|
+
stall_count: stallCount,
|
|
112
|
+
last_stall_ms: Math.round(lastStallMs),
|
|
113
|
+
last_stall_at: lastStallAt,
|
|
114
|
+
recent_stalls: [...stalls].reverse(),
|
|
115
|
+
since: sinceIso,
|
|
116
|
+
observed_at: new Date().toISOString(),
|
|
117
|
+
};
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
/** 只给测试用:清空累计状态。 */
|
|
121
|
+
export function resetEventLoopLagForTests(): void {
|
|
122
|
+
maxGapMs = 0;
|
|
123
|
+
lastGapMs = 0;
|
|
124
|
+
stallCount = 0;
|
|
125
|
+
lastStallAt = null;
|
|
126
|
+
lastStallMs = 0;
|
|
127
|
+
stalls.length = 0;
|
|
128
|
+
sinceIso = new Date().toISOString(); // 重置窗口起点 —— 否则测试里「归零」与「没归零」同形
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
function envMs(name: string, fallback: number): number {
|
|
132
|
+
const raw = Number(process.env[name]);
|
|
133
|
+
return Number.isFinite(raw) && raw >= 0 ? raw : fallback;
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
/**
|
|
137
|
+
* 启动监测。返回 timer(调用方负责 clearInterval)。
|
|
138
|
+
* - `COMMHUB_STALL_LOG_MS`:停顿日志阈值,默认 2000 ms,设 0 关闭日志(仍计数);
|
|
139
|
+
* - `COMMHUB_STALL_TICK_MS`:采样间隔,默认 250 ms。
|
|
140
|
+
*/
|
|
141
|
+
export function startEventLoopLagMonitor(opts: {
|
|
142
|
+
intervalMs?: number;
|
|
143
|
+
thresholdMs?: number;
|
|
144
|
+
log?: (msg: string) => void;
|
|
145
|
+
rssMb?: () => number;
|
|
146
|
+
/**
|
|
147
|
+
* #2721 —— 启动期的 tick gap 不算停顿。判据是**进程启动多久了**,不是第几个 tick。
|
|
148
|
+
*
|
|
149
|
+
* 事故形状:进程 22:21:06 启动,14:21:09.835Z 记到 gap 2972ms —— 启动后 3 秒。
|
|
150
|
+
* 那段 gap 度量的是「监控开始 → 第一次 tick」被模块加载/开库/迁移占满的时间。
|
|
151
|
+
*
|
|
152
|
+
* 两版错误实现(都试过,都被既有测试抓住):
|
|
153
|
+
* ① 10 秒**时间窗** → 监控在前 10 秒什么都不记,「阈值 40ms + spin 120ms 应打日志」直接吞掉
|
|
154
|
+
* ② 只跳**第一个 tick** → 测试里的 spin 恰好落在第一个 tick,同样吞掉
|
|
155
|
+
*
|
|
156
|
+
* 真正的区别不是「第几个 tick」,而是**这段时间里进程有没有可能还在启动**。
|
|
157
|
+
* 所以:只有当 tick 到来时进程 uptime 仍小于 bootGraceMs,才把 gap 当作启动尖峰。
|
|
158
|
+
* 启动 20 秒之后的第一条 gap 依旧照常记录 —— 那时候它不可能是启动耗时。
|
|
159
|
+
*/
|
|
160
|
+
bootGraceMs?: number;
|
|
161
|
+
/** 测试注入:返回进程已运行的毫秒数(默认 process.uptime())。 */
|
|
162
|
+
uptimeMs?: () => number;
|
|
163
|
+
} = {}): ReturnType<typeof setInterval> {
|
|
164
|
+
const intervalMs = opts.intervalMs ?? Math.max(50, envMs("COMMHUB_STALL_TICK_MS", 250));
|
|
165
|
+
const thresholdMs = opts.thresholdMs ?? envMs("COMMHUB_STALL_LOG_MS", 2000);
|
|
166
|
+
const log = opts.log ?? ((m: string) => console.log(m));
|
|
167
|
+
const rssMb = opts.rssMb ?? (() => Math.round(process.memoryUsage().rss / 1048576));
|
|
168
|
+
let last = performance.now();
|
|
169
|
+
const startedAt = last;
|
|
170
|
+
// #2721 —— 启动预热不算停顿。
|
|
171
|
+
// 实测:2026-10-29 这次进程 22:21:06 起来,14:21:09.835Z 记到一次 gap 2972ms,
|
|
172
|
+
// 也就是**启动后 3 秒**。那段时间在做模块加载 / 打开 DB / 迁移,第一个 tick 必然偏大。
|
|
173
|
+
// 不排除它的后果是:每次部署都会在 `/health` 里留一条「停顿」,而它和线上真实卡顿长得一模一样。
|
|
174
|
+
const bootGraceMs = opts.bootGraceMs ?? envMs("COMMHUB_STALL_BOOT_GRACE_MS", 15_000);
|
|
175
|
+
const uptimeMs = opts.uptimeMs ?? (() => process.uptime() * 1000);
|
|
176
|
+
let firstTick = true;
|
|
177
|
+
const timer = setInterval(() => {
|
|
178
|
+
const now = performance.now();
|
|
179
|
+
const gap = now - last;
|
|
180
|
+
last = now;
|
|
181
|
+
const rss = rssMb();
|
|
182
|
+
const served = servedSinceTick;
|
|
183
|
+
servedSinceTick = 0;
|
|
184
|
+
// 启动尖峰:只可能是**第一个** tick,且到来时进程才刚起来。
|
|
185
|
+
// 两个条件缺一不可 —— 只看「第一个 tick」会吞掉运行中立刻发生的真实停顿。
|
|
186
|
+
if (firstTick) {
|
|
187
|
+
firstTick = false;
|
|
188
|
+
if (uptimeMs() < bootGraceMs) return;
|
|
189
|
+
}
|
|
190
|
+
const stall = recordGap(gap, { intervalMs, thresholdMs, rssMb: rss, served });
|
|
191
|
+
if (thresholdMs > 0 && stall >= thresholdMs) {
|
|
192
|
+
// 和 retention/slow-sql 的日志同一前缀,方便 grep。
|
|
193
|
+
// `served` 是判据的关键:它非零就说明循环在干活,这段“gap”是 timer 被饿住,不是阻塞。
|
|
194
|
+
log(`[commhub stall] loop idle ~${Math.round(stall)}ms (tick gap ${Math.round(gap)}ms, rss ${rss} MB, served ${served} req)`);
|
|
195
|
+
}
|
|
196
|
+
}, intervalMs);
|
|
197
|
+
// 监测不该拖住进程退出。
|
|
198
|
+
(timer as unknown as { unref?: () => void }).unref?.();
|
|
199
|
+
last = performance.now();
|
|
200
|
+
return timer;
|
|
201
|
+
}
|
|
@@ -0,0 +1,187 @@
|
|
|
1
|
+
// #3271 —— 「到死都没被取走」的计数:**正控 + 负数用例**。
|
|
2
|
+
//
|
|
3
|
+
// 现场:一台节点上 4 条任务,一次又一次「等满 60 分钟 ⇒ expired」,而积压数**两次读数都是 4**。
|
|
4
|
+
// 「又过期了一条」是可见的(Hub 会发通知),但「这台节点连续 N 次一次都没取走」没有读数。
|
|
5
|
+
//
|
|
6
|
+
// 这个计数器属于「只有异常路径能让它 +1」那一类 ⇒ 必须有 must-fire:
|
|
7
|
+
// ① 注入一条**未消费**就过期的任务 ⇒ 计数 +1(正控)
|
|
8
|
+
// ② 注入一条**已消费**后过期的任务 ⇒ 计数**不动**(负数用例)
|
|
9
|
+
// 少了 ②,这个计数就退化成「过期了多少条」——而那是另一个更没用的数。
|
|
10
|
+
import { afterAll, beforeEach, describe, expect, test } from "bun:test";
|
|
11
|
+
import { db } from "./db.js";
|
|
12
|
+
import { neverConsumedExpiryCounters, noteNeverConsumedExpiry, resetNeverConsumedExpiryCountersForTest } from "./expiry-counters.js";
|
|
13
|
+
import { patrolExpiredTasks } from "./server.js";
|
|
14
|
+
|
|
15
|
+
const NET = "net_expiry_cnt";
|
|
16
|
+
|
|
17
|
+
function seedTask(id: string, opts: { toName: string; consumed: boolean; expiredSecondsAgo?: number }) {
|
|
18
|
+
const ago = opts.expiredSecondsAgo ?? 120;
|
|
19
|
+
db.run(
|
|
20
|
+
`INSERT INTO tasks (task_id, network_id, from_name, to_name, content, status, created_at, expires_at, consumed_at)
|
|
21
|
+
VALUES (?1, ?2, 'sender', ?3, 'fixture', 'delivered', datetime('now', '-1 hour'), datetime('now', ?4), ?5)`,
|
|
22
|
+
[id, NET, opts.toName, `-${ago} seconds`, opts.consumed ? new Date(Date.now() - 3600_000).toISOString().replace("T", " ").slice(0, 19) : null],
|
|
23
|
+
);
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
function cleanup() {
|
|
27
|
+
try { db.run("DELETE FROM tasks WHERE network_id = ?1", [NET]); } catch {}
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
beforeEach(() => { cleanup(); resetNeverConsumedExpiryCountersForTest(); });
|
|
31
|
+
afterAll(() => { cleanup(); });
|
|
32
|
+
|
|
33
|
+
describe("#3271 从未取走过的过期计数", () => {
|
|
34
|
+
test("🔴 must-fire:未消费就过期的任务 ⇒ 该节点计数 +1", () => {
|
|
35
|
+
seedTask("t_never_1", { toName: "stuck-node", consumed: false });
|
|
36
|
+
patrolExpiredTasks();
|
|
37
|
+
const c = neverConsumedExpiryCounters();
|
|
38
|
+
expect(c.by_node["stuck-node"]).toBe(1);
|
|
39
|
+
expect(c.total).toBe(1);
|
|
40
|
+
expect(typeof c.since).toBe("string"); // 窗口左端必须在,否则「归零」与「没发生」同值
|
|
41
|
+
});
|
|
42
|
+
|
|
43
|
+
test("🔴 负数用例:**已消费**后过期的任务 ⇒ 计数不动(那不是「没人取」)", () => {
|
|
44
|
+
seedTask("t_consumed_1", { toName: "busy-but-alive", consumed: true });
|
|
45
|
+
patrolExpiredTasks();
|
|
46
|
+
expect(neverConsumedExpiryCounters().total).toBe(0);
|
|
47
|
+
});
|
|
48
|
+
|
|
49
|
+
test("按节点分桶:两个节点各一条 ⇒ 各自 1,总数 2", () => {
|
|
50
|
+
seedTask("t_a", { toName: "node-a", consumed: false });
|
|
51
|
+
seedTask("t_b", { toName: "node-b", consumed: false });
|
|
52
|
+
patrolExpiredTasks();
|
|
53
|
+
const c = neverConsumedExpiryCounters();
|
|
54
|
+
expect(c.by_node).toEqual({ "node-a": 1, "node-b": 1 });
|
|
55
|
+
expect(c.total).toBe(2);
|
|
56
|
+
});
|
|
57
|
+
|
|
58
|
+
test("同一个节点连续 N 次 ⇒ 累加(这是「这一台从来不吃活」的证据形态)", () => {
|
|
59
|
+
for (let i = 0; i < 4; i += 1) noteNeverConsumedExpiry("ghost-node");
|
|
60
|
+
expect(neverConsumedExpiryCounters().by_node["ghost-node"]).toBe(4);
|
|
61
|
+
});
|
|
62
|
+
|
|
63
|
+
test("桶按名字排序且空桶不出现 —— 读数要能直接对比两次快照", () => {
|
|
64
|
+
noteNeverConsumedExpiry("z-node");
|
|
65
|
+
noteNeverConsumedExpiry("a-node");
|
|
66
|
+
const c = neverConsumedExpiryCounters();
|
|
67
|
+
expect(Object.keys(c.by_node)).toEqual(["a-node", "z-node"]);
|
|
68
|
+
});
|
|
69
|
+
});
|
|
70
|
+
|
|
71
|
+
// ---- 派单侧硬闸(#3271)----
|
|
72
|
+
import { neverConsumeGate, NEVER_CONSUME_LIMIT, NEVER_CONSUME_STALE_MS } from "./expiry-counters.js";
|
|
73
|
+
|
|
74
|
+
describe("#3271 派单侧硬闸:从未取走过的目标", () => {
|
|
75
|
+
const NOW = Date.parse("2026-10-31T10:00:00.000Z");
|
|
76
|
+
const counters = (n: number, alias = "ghost", lastAt = new Date(NOW - 60_000).toISOString()) => ({
|
|
77
|
+
since: "2026-10-31T00:00:00.000Z",
|
|
78
|
+
total: n,
|
|
79
|
+
by_node: n > 0 ? { [alias]: n } : {},
|
|
80
|
+
last_at: n > 0 ? { [alias]: lastAt } : {},
|
|
81
|
+
});
|
|
82
|
+
|
|
83
|
+
test("窗口内 0–2 次 ⇒ 放行(一次两次可能是宿主重启,三次才是形态)", () => {
|
|
84
|
+
for (const n of [0, 1, 2]) {
|
|
85
|
+
expect(neverConsumeGate({ alias: "ghost", counters: counters(n), force: false, forceAllowed: true, now: Date.parse("2026-10-31T10:00:00.000Z") }).ok).toBe(true);
|
|
86
|
+
}
|
|
87
|
+
});
|
|
88
|
+
|
|
89
|
+
test("🔴 达到阈值 ⇒ 拒绝,且拒绝体给全(alias / expiries / window_since / 下一步)", () => {
|
|
90
|
+
const r = neverConsumeGate({ alias: "ghost", counters: counters(NEVER_CONSUME_LIMIT), force: false, forceAllowed: true, now: Date.parse("2026-10-31T10:00:00.000Z") });
|
|
91
|
+
expect(r.ok).toBe(false);
|
|
92
|
+
if (r.ok) throw new Error("unreachable");
|
|
93
|
+
expect(r.error).toBe("target_never_consumes");
|
|
94
|
+
expect(r.alias).toBe("ghost");
|
|
95
|
+
expect(r.expiries).toBe(NEVER_CONSUME_LIMIT);
|
|
96
|
+
expect(r.window_since).toBe("2026-10-31T00:00:00.000Z");
|
|
97
|
+
expect(r.message).toContain("force:true");
|
|
98
|
+
});
|
|
99
|
+
|
|
100
|
+
test("人令牌 + force ⇒ 放行(出口存在,但必须显式)", () => {
|
|
101
|
+
expect(neverConsumeGate({ alias: "ghost", counters: counters(9), force: true, forceAllowed: true, now: Date.parse("2026-10-31T10:00:00.000Z") }).ok).toBe(true);
|
|
102
|
+
});
|
|
103
|
+
|
|
104
|
+
test("🔴 节点令牌不能自己放行:force 传了也不算(forceAllowed=false)", () => {
|
|
105
|
+
const r = neverConsumeGate({ alias: "ghost", counters: counters(9), force: true, forceAllowed: false, now: Date.parse("2026-10-31T10:00:00.000Z") });
|
|
106
|
+
expect(r.ok).toBe(false);
|
|
107
|
+
});
|
|
108
|
+
|
|
109
|
+
test("别的节点的计数不影响本目标(分桶不是总数)", () => {
|
|
110
|
+
const c = { since: "s", total: 9, by_node: { other: 9 }, last_at: { other: new Date().toISOString() } };
|
|
111
|
+
expect(neverConsumeGate({ alias: "ghost", counters: c, force: false, forceAllowed: true, now: Date.parse("2026-10-31T10:00:00.000Z") }).ok).toBe(true);
|
|
112
|
+
});
|
|
113
|
+
});
|
|
114
|
+
|
|
115
|
+
describe("#3271 证据的生命期必须 ≥ 故障的生命期(进程内计数器不够)", () => {
|
|
116
|
+
const NOW = Date.parse("2026-10-31T10:00:00.000Z");
|
|
117
|
+
const empty = { since: "2026-10-31T10:00:00.000Z", total: 0, by_node: {}, last_at: {} };
|
|
118
|
+
|
|
119
|
+
// 🔴 这条是 2026-10-31 生产实测的 must-fire:
|
|
120
|
+
// `.254` 部署(22:39)后,我给一台**已经 10 次到死没被取走**的节点派单,
|
|
121
|
+
// 闸返回 ok:true —— 因为计数器随进程重启归零,而故障是以「周」计的。
|
|
122
|
+
test("🔴 内存计数为空,但 tasks 表里有 3 条历史 ⇒ 必须拒绝", () => {
|
|
123
|
+
const r = neverConsumeGate({
|
|
124
|
+
alias: "ghost", counters: empty, force: false, forceAllowed: true, now: NOW,
|
|
125
|
+
persistent: { expiries: 3, last_at: new Date(NOW - 60_000).toISOString().replace("T", " ").slice(0, 19), last_reply_at: null },
|
|
126
|
+
});
|
|
127
|
+
expect(r.ok).toBe(false);
|
|
128
|
+
if (r.ok) throw new Error("unreachable");
|
|
129
|
+
expect(r.expiries).toBe(3);
|
|
130
|
+
});
|
|
131
|
+
|
|
132
|
+
test("两个来源取**并集**(内存 2 + 表里 3 ⇒ 按 3 判,不是相加也不是取小)", () => {
|
|
133
|
+
const counters = { since: "s", total: 2, by_node: { ghost: 2 }, last_at: { ghost: new Date(NOW - 60_000).toISOString() } };
|
|
134
|
+
const r = neverConsumeGate({
|
|
135
|
+
alias: "ghost", counters, force: false, forceAllowed: true, now: NOW,
|
|
136
|
+
persistent: { expiries: 3, last_at: new Date(NOW - 60_000).toISOString().replace("T", " ").slice(0, 19), last_reply_at: null },
|
|
137
|
+
});
|
|
138
|
+
expect(r.ok).toBe(false);
|
|
139
|
+
if (r.ok) throw new Error("unreachable");
|
|
140
|
+
expect(r.expiries).toBe(3);
|
|
141
|
+
});
|
|
142
|
+
|
|
143
|
+
// 🔴 语义修正(2026-10-31 第二次实测):DB 证据**不用固定 TTL**。
|
|
144
|
+
// 固定 2 小时是我拍的,而那台节点的故障节律是「天」——于是闸在 22 小时前的那次过期之后
|
|
145
|
+
// 判「已恢复」并放行,而它这 22 小时一条都没回。
|
|
146
|
+
// 现在 DB 证据自带边界(「最后一次回复之后」),陈旧不陈旧由回复本身回答。
|
|
147
|
+
test("🔴 表里的历史很旧、但之后没有回复 ⇒ 仍然拒绝(固定 TTL 只留给纯内存计数)", () => {
|
|
148
|
+
const r = neverConsumeGate({
|
|
149
|
+
alias: "ghost", counters: empty, force: false, forceAllowed: true, now: NOW,
|
|
150
|
+
persistent: { expiries: 9, last_at: "2026-10-20T00:00:00", last_reply_at: "2026-10-18T00:00:00" },
|
|
151
|
+
});
|
|
152
|
+
expect(r.ok).toBe(false);
|
|
153
|
+
});
|
|
154
|
+
|
|
155
|
+
test("纯内存计数仍然有 2 小时时效(没有回复时间可比,只能按时长兜底)", () => {
|
|
156
|
+
const counters = { since: "s", total: 9, by_node: { ghost: 9 }, last_at: { ghost: new Date(NOW - 3 * 3600_000).toISOString() } };
|
|
157
|
+
expect(neverConsumeGate({ alias: "ghost", counters, force: false, forceAllowed: true, now: NOW }).ok).toBe(true);
|
|
158
|
+
});
|
|
159
|
+
});
|
|
160
|
+
|
|
161
|
+
describe("#3271 硬闸的时效:只数不衰减会把恢复了的节点永久拉黑", () => {
|
|
162
|
+
const NOW = Date.parse("2026-10-31T10:00:00.000Z");
|
|
163
|
+
const mk = (lastAtIso: string) => ({
|
|
164
|
+
since: "2026-10-31T00:00:00.000Z",
|
|
165
|
+
total: 5,
|
|
166
|
+
by_node: { ghost: 5 },
|
|
167
|
+
last_at: { ghost: lastAtIso },
|
|
168
|
+
});
|
|
169
|
+
|
|
170
|
+
test("最近一次是 1 分钟前 ⇒ 仍然拒(形态是新鲜的)", () => {
|
|
171
|
+
const r = neverConsumeGate({ alias: "ghost", counters: mk(new Date(NOW - 60_000).toISOString()), force: false, forceAllowed: true, now: NOW });
|
|
172
|
+
expect(r.ok).toBe(false);
|
|
173
|
+
});
|
|
174
|
+
|
|
175
|
+
test("🔴 最近一次超过 TTL ⇒ 放行(节点可能已恢复;计数仍留在读数里)", () => {
|
|
176
|
+
const r = neverConsumeGate({ alias: "ghost", counters: mk(new Date(NOW - NEVER_CONSUME_STALE_MS - 1).toISOString()), force: false, forceAllowed: true, now: NOW });
|
|
177
|
+
expect(r.ok).toBe(true);
|
|
178
|
+
});
|
|
179
|
+
|
|
180
|
+
test("读数里带 last_at —— 光有计数看不出「多久没犯了」", () => {
|
|
181
|
+
resetNeverConsumedExpiryCountersForTest(new Date(NOW));
|
|
182
|
+
noteNeverConsumedExpiry("ghost", NOW);
|
|
183
|
+
const c = neverConsumedExpiryCounters();
|
|
184
|
+
expect(c.last_at["ghost"]).toBe(new Date(NOW).toISOString());
|
|
185
|
+
expect(c.by_node["ghost"]).toBe(1);
|
|
186
|
+
});
|
|
187
|
+
});
|