@sleep2agi/commhub-server 0.9.0-preview.26 → 0.9.0-preview.260
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +120 -3
- package/bin/commhub.ts +93 -9
- package/build-info.json +7 -0
- package/package.json +10 -8
- package/src/ack-create-request-transport.test.ts +182 -0
- package/src/ack-create-request.test.ts +52 -8
- package/src/admin-delete-network-http.test.ts +228 -0
- package/src/admin-networks-http.test.ts +88 -0
- package/src/adoption-candidates-http.test.ts +140 -0
- package/src/adoption-candidates.ts +158 -0
- package/src/agent-access-batch.test.ts +210 -0
- package/src/agent-access.ts +487 -0
- package/src/agent-acl-groups-http.test.ts +247 -0
- package/src/agent-acl-http.test.ts +680 -0
- package/src/agent-acl-slice3-http.test.ts +885 -0
- package/src/agent-department-http.test.ts +215 -0
- package/src/agent-groups-perf.test.ts +223 -0
- package/src/agent-notice.ts +48 -0
- package/src/agent-team-whoami-http.test.ts +98 -0
- package/src/agent-teams-http.test.ts +330 -0
- package/src/agent-teams.ts +244 -0
- package/src/alias-baseline-2571.test.ts +162 -0
- package/src/alias-baseline.ts +113 -0
- package/src/alias-filter.test.ts +72 -0
- package/src/alias-filter.ts +42 -0
- package/src/api-host-supervisors-can-create-nodes.test.ts +350 -0
- package/src/api-host-supervisors-fallback.test.ts +6 -7
- package/src/api-host-supervisors-runtime-readiness.test.ts +195 -0
- package/src/api-nodes-shape.test.ts +131 -4
- package/src/auth-abuse-counters.test.ts +125 -0
- package/src/auth-abuse-counters.ts +122 -0
- package/src/auth-password-change-cli-tokens-http.test.ts +423 -0
- package/src/auth-sessions-http.test.ts +260 -0
- package/src/auth-validate.test.ts +5 -1
- package/src/auth-weak-password-login.test.ts +176 -0
- package/src/auth.ts +609 -56
- package/src/backfill-node-ids-2664.test.ts +128 -0
- package/src/binding-generation-http.test.ts +141 -0
- package/src/blocked-exit-on-reply.test.ts +122 -0
- package/src/broadcast-fanout-2721.test.ts +149 -0
- package/src/broadcast-fanout.ts +117 -0
- package/src/build-commit-report-http.test.ts +128 -0
- package/src/build-info.test.ts +60 -0
- package/src/build-info.ts +85 -0
- package/src/chain-attempt-dedup.test.ts +447 -0
- package/src/clip.test.ts +36 -0
- package/src/clip.ts +48 -0
- package/src/clone-node.test.ts +269 -0
- package/src/clone-node.ts +289 -0
- package/src/codex-config-flags.ts +77 -0
- package/src/codex-fork-contract.ts +23 -0
- package/src/codex-fork-migration.test.ts +38 -0
- package/src/commhub-bin-args.test.ts +85 -0
- package/src/config-apply-validate.test.ts +188 -1
- package/src/config-apply-validate.ts +222 -3
- package/src/cors-origins.test.ts +73 -0
- package/src/create-node-blocked-gate.test.ts +365 -0
- package/src/create-node-error-domain.test.ts +124 -0
- package/src/create-node-network-id.test.ts +91 -0
- package/src/create-node-tool-schema.test.ts +49 -0
- package/src/create-node-unicode-name.test.ts +126 -0
- package/src/create-node-validate.test.ts +199 -7
- package/src/create-node-validate.ts +232 -15
- package/src/create-node-workdir.test.ts +298 -0
- package/src/create-node.ts +88 -23
- package/src/daemon-provider-snapshots.test.ts +453 -0
- package/src/daemon-provider-snapshots.ts +327 -0
- package/src/daemon-token-owner-http.test.ts +346 -0
- package/src/dashboard-slash-routing-http.test.ts +109 -0
- package/src/db-adapter-guard.test.ts +148 -0
- package/src/db-adapter-translate.test.ts +128 -0
- package/src/db-adapter.ts +377 -91
- package/src/db-timestamp.test.ts +38 -0
- package/src/db-timestamp.ts +15 -0
- package/src/db.ts +1269 -153
- package/src/delivered-without-push.test.ts +140 -0
- package/src/department-groups-http.test.ts +209 -0
- package/src/department-groups-sync-http.test.ts +328 -0
- package/src/department-groups.ts +434 -0
- package/src/department-heads-http.test.ts +350 -0
- package/src/department-heads.ts +226 -0
- package/src/departments-agent-count.test.ts +139 -0
- package/src/departments-http.test.ts +185 -0
- package/src/departments-mcp-error-domain.test.ts +135 -0
- package/src/departments-mcp.test.ts +128 -0
- package/src/departments.ts +318 -0
- package/src/desktop-message-seam.test.ts +150 -0
- package/src/desktop-message.test.ts +368 -0
- package/src/dm-file-scope-http.test.ts +178 -0
- package/src/empty-node-sweeper.test.ts +104 -0
- package/src/empty-node-sweeper.ts +88 -0
- package/src/env-transport-gate-http.test.ts +240 -0
- package/src/env-transport-gate.ts +80 -0
- package/src/env-transport-policy.ts +56 -0
- package/src/etag-of.test.ts +57 -0
- package/src/event-loop-lag.test.ts +180 -0
- package/src/event-loop-lag.ts +201 -0
- package/src/expiry-counters.test.ts +187 -0
- package/src/expiry-counters.ts +227 -0
- package/src/expiry-evidence-db.test.ts +103 -0
- package/src/ext-token-shared.test.ts +60 -0
- package/src/ext-validation-regression.test.ts +10 -17
- package/src/external-schedule-edits-http.test.ts +321 -0
- package/src/external-schedule-edits-timestamps.test.ts +64 -0
- package/src/external-schedule-edits.ts +327 -0
- package/src/file-download-authz-dev-open.test.ts +1 -1
- package/src/file-download-authz.test.ts +25 -4
- package/src/file-download-date-bucket.test.ts +1 -1
- package/src/file-network-scope-dev-open.test.ts +1 -1
- package/src/file-network-scope.test.ts +7 -6
- package/src/group-membership-events-http.test.ts +264 -0
- package/src/group-messages-http.test.ts +483 -0
- package/src/group-messages.ts +297 -0
- package/src/health-anon-readers.ts +67 -0
- package/src/health-host-disk-2756.test.ts +71 -0
- package/src/health-redaction.test.ts +1576 -7
- package/src/host-supervisor-identity-http.test.ts +152 -0
- package/src/host-supervisor-identity.ts +92 -0
- package/src/host-supervisors-daemon-version.test.ts +137 -0
- package/src/http-gzip.test.ts +94 -0
- package/src/http-gzip.ts +112 -0
- package/src/hub-catalog-read-http.test.ts +641 -0
- package/src/hub-provider-config.test.ts +166 -0
- package/src/hub-provider-config.ts +202 -0
- package/src/hub-timestamp.test.ts +38 -0
- package/src/hub-timestamp.ts +27 -0
- package/src/human-dm.ts +215 -0
- package/src/human-inbox.ts +42 -0
- package/src/inbox-caller.ts +49 -0
- package/src/inbox-count-sse.test.ts +318 -0
- package/src/inbox-count.ts +29 -0
- package/src/index.ts +6 -2
- package/src/license-state.test.ts +56 -0
- package/src/license-state.ts +48 -0
- package/src/lifecycle-error-domain.test.ts +109 -0
- package/src/list-ack-error-domain.test.ts +108 -0
- package/src/list-host-supervisors.test.ts +15 -4
- package/src/mcp-context-cost-http.test.ts +127 -0
- package/src/mcp-inflight.test.ts +92 -0
- package/src/mcp-inflight.ts +38 -0
- package/src/mcp-projects-events-http.test.ts +168 -0
- package/src/mcp-route-label-http.test.ts +120 -0
- package/src/mcp-write-network-resolution.test.ts +21 -0
- package/src/member-audit-network-id-http.test.ts +93 -0
- package/src/member-presence-http.test.ts +210 -0
- package/src/messages-ack-by-agent-http.test.ts +142 -0
- package/src/migrate-sqlite-to-pg.test.ts +47 -0
- package/src/migrate-sqlite-to-pg.ts +263 -0
- package/src/model-auth-notify-http.test.ts +181 -0
- package/src/model-auth-notify.ts +116 -0
- package/src/network-name-http.test.ts +75 -0
- package/src/network-rules-http.test.ts +169 -0
- package/src/network-rules.ts +129 -0
- package/src/network-scope.ts +222 -3
- package/src/network-token-membership-http.test.ts +186 -0
- package/src/node-alias-control.ts +12 -0
- package/src/node-attrs.test.ts +47 -2
- package/src/node-caller-identity-http.test.ts +1108 -0
- package/src/node-create-request-status-http.test.ts +105 -0
- package/src/node-daemon-bindings.test.ts +142 -0
- package/src/node-daemon-bindings.ts +135 -0
- package/src/node-failure-alert.test.ts +150 -0
- package/src/node-failure-alert.ts +177 -0
- package/src/node-files-transport.test.ts +350 -0
- package/src/node-health-codex-login.test.ts +120 -0
- package/src/node-health-dispatch-http.test.ts +373 -0
- package/src/node-health-guard.ts +121 -0
- package/src/node-health-passthrough.test.ts +115 -0
- package/src/node-health-store.ts +101 -0
- package/src/node-id-alias.test.ts +190 -0
- package/src/node-identity-conflict-http.test.ts +269 -0
- package/src/node-identity-conflict.ts +142 -0
- package/src/node-identity-mismatch.test.ts +74 -0
- package/src/node-in-flight-tasks.ts +31 -0
- package/src/node-lifecycle-controllable.test.ts +64 -0
- package/src/node-lifecycle-controllable.ts +53 -0
- package/src/node-lifecycle-read-http.test.ts +257 -0
- package/src/node-lifecycle-read.ts +124 -0
- package/src/node-logs-transport.test.ts +266 -0
- package/src/node-peer-allowlist.test.ts +114 -0
- package/src/node-permissions-http.test.ts +728 -0
- package/src/node-permissions.ts +494 -0
- package/src/node-request-retention.test.ts +281 -0
- package/src/node-request-retention.ts +120 -0
- package/src/node-secret-rest-http.test.ts +190 -0
- package/src/node-security-write-gate.test.ts +72 -0
- package/src/node-serviceability.test.ts +86 -0
- package/src/node-serviceability.ts +124 -0
- package/src/node-session-control-http.test.ts +310 -0
- package/src/node-session-control.ts +258 -0
- package/src/node-skills-transport.test.ts +212 -0
- package/src/node-token-identity.ts +35 -0
- package/src/node-token-ownership.ts +51 -0
- package/src/node-token-privilege-http.test.ts +158 -0
- package/src/not-found-api-http.test.ts +67 -0
- package/src/observer-avatar-http.test.ts +103 -10
- package/src/observer-push.test.ts +76 -0
- package/src/opencode-catalog-report-http.test.ts +139 -0
- package/src/opencode-v2-catalog.test.ts +268 -0
- package/src/opencode-v2-catalog.ts +356 -0
- package/src/package-version-consistency.test.ts +25 -0
- package/src/parent-source-marker.test.ts +50 -0
- package/src/password-network-tokens.ts +25 -0
- package/src/peer-reply-atomic.test.ts +462 -0
- package/src/peer-reply-capability.test.ts +70 -0
- package/src/pg-worker.ts +87 -0
- package/src/poll-pacing.test.ts +74 -0
- package/src/poll-pacing.ts +123 -0
- package/src/probe-provider-error-domain.test.ts +141 -0
- package/src/project-dir-warn-clear-http.test.ts +132 -0
- package/src/project-viewer-can-http.test.ts +116 -0
- package/src/provider-skill-error-domain.test.ts +130 -0
- package/src/provider-tools-shape.test.ts +165 -0
- package/src/push.test.ts +20 -10
- package/src/push.ts +373 -9
- package/src/redact-tokens.test.ts +86 -0
- package/src/redact-tokens.ts +43 -0
- package/src/rejection-counters.test.ts +72 -0
- package/src/rejection-counters.ts +173 -0
- package/src/rename.ts +38 -10
- package/src/reply-target-derivation.test.ts +96 -0
- package/src/report-status-handover-capabilities-http.test.ts +124 -0
- package/src/report-status-host-schema.test.ts +235 -0
- package/src/report-status-resume-id-handover.test.ts +184 -0
- package/src/report-status-secret-keys.test.ts +140 -0
- package/src/require-explicit-test-db.ts +9 -0
- package/src/requirement-due-reminders-http.test.ts +620 -0
- package/src/requirement-due-reminders.ts +416 -0
- package/src/requirement-events-http.test.ts +318 -0
- package/src/requirement-events.test.ts +45 -0
- package/src/requirement-events.ts +144 -0
- package/src/requirement-notify.ts +165 -0
- package/src/requirement-participant-notify-http.test.ts +313 -0
- package/src/requirement-tag-ops-http.test.ts +454 -0
- package/src/requirement-tag-tree-http.test.ts +275 -0
- package/src/requirement-tag-tree.ts +127 -0
- package/src/requirement-tags.ts +102 -0
- package/src/requirements-abandoned-http.test.ts +236 -0
- package/src/requirements-assignee-drift.test.ts +154 -0
- package/src/requirements-comment-http.test.ts +174 -0
- package/src/requirements-completed-at-migration.test.ts +70 -0
- package/src/requirements-errors-mcp-http.test.ts +164 -0
- package/src/requirements-errors.ts +158 -0
- package/src/requirements-etag-http.test.ts +91 -0
- package/src/requirements-http.test.ts +709 -0
- package/src/requirements-last-event-http.test.ts +349 -0
- package/src/requirements-list-mcp-slim-http.test.ts +155 -0
- package/src/requirements-list-slim-http.test.ts +324 -0
- package/src/requirements-mcp.test.ts +180 -0
- package/src/requirements-migrate.ts +283 -0
- package/src/requirements-owner-strict-mcp-http.test.ts +145 -0
- package/src/requirements-people-http.test.ts +114 -0
- package/src/requirements-people-mcp-http.test.ts +231 -0
- package/src/requirements-people.ts +196 -0
- package/src/requirements-priority-lowest.test.ts +114 -0
- package/src/requirements-projects-network-echo.test.ts +57 -0
- package/src/requirements-projects-parent-http.test.ts +123 -0
- package/src/requirements-search-http.test.ts +182 -0
- package/src/requirements-search.test.ts +32 -0
- package/src/requirements-search.ts +145 -0
- package/src/requirements-seq-http.test.ts +193 -0
- package/src/requirements-seq-migration.test.ts +63 -0
- package/src/requirements-start-date.test.ts +125 -0
- package/src/requirements-stats-http.test.ts +357 -0
- package/src/requirements-stats.ts +195 -0
- package/src/requirements-status-participants-http.test.ts +222 -0
- package/src/requirements-upgrade-e2e.test.ts +126 -0
- package/src/requirements-upgrade-migration.test.ts +98 -0
- package/src/requirements.ts +2205 -0
- package/src/resolve-port.test.ts +57 -0
- package/src/resolve-port.ts +44 -0
- package/src/rest-alias-network-fallback-http.test.ts +123 -0
- package/src/rest-alias-network-fallback.test.ts +60 -0
- package/src/rest-alias-network-fallback.ts +65 -0
- package/src/rest-explicit-columns-http.test.ts +266 -0
- package/src/rest-identity.test.ts +130 -0
- package/src/rest-identity.ts +141 -0
- package/src/rest-projections.ts +72 -0
- package/src/rest-read-network-real-chain.test.ts +97 -0
- package/src/rest-write-network-resolution.test.ts +133 -0
- package/src/restart-readiness-http.test.ts +121 -0
- package/src/restricted-files.ts +76 -0
- package/src/retention-first-sweep-quiet.test.ts +31 -0
- package/src/retention.test.ts +100 -40
- package/src/retention.ts +423 -104
- package/src/route-callers-http.test.ts +120 -0
- package/src/route-timing.test.ts +218 -0
- package/src/route-timing.ts +262 -0
- package/src/rules-file-duplicate-alias.test.ts +234 -0
- package/src/rules-file-session-target.test.ts +207 -0
- package/src/rules-file-transport.test.ts +271 -0
- package/src/runtime-label.test.ts +22 -0
- package/src/runtime-label.ts +21 -0
- package/src/runtime-readiness.ts +90 -0
- package/src/schedule-agent-mcp-http.test.ts +486 -0
- package/src/schedule-agent.ts +252 -0
- package/src/schedule-starved.test.ts +92 -0
- package/src/schedule-starved.ts +114 -0
- package/src/scheduled-backoff-http.test.ts +203 -0
- package/src/scheduled-backoff.ts +64 -0
- package/src/scheduled-doorbell-nodeless-session.test.ts +134 -0
- package/src/scheduled-failure-backoff-2622.test.ts +145 -0
- package/src/scheduled-failure-code.test.ts +73 -0
- package/src/scheduled-failure-code.ts +105 -0
- package/src/scheduled-failures-http.test.ts +279 -0
- package/src/scheduled-failures.ts +425 -0
- package/src/scheduled-noprogress-backoff-2622.test.ts +163 -0
- package/src/scheduled-offline-backoff-probe.test.ts +56 -0
- package/src/scheduled-offline.test.ts +131 -0
- package/src/scheduled-offline.ts +66 -0
- package/src/scheduled-outcome-guidance-http.test.ts +139 -0
- package/src/scheduled-outcome-guidance.ts +9 -0
- package/src/scheduled-reply-unread-http.test.ts +212 -0
- package/src/scheduled-requirement-wake-http.test.ts +289 -0
- package/src/scheduled-requirement-wake.ts +125 -0
- package/src/scheduled-run-terminal.test.ts +290 -0
- package/src/scheduled-skip-blocking-task.test.ts +120 -0
- package/src/scheduled-skip-path-backoff-2622.test.ts +98 -0
- package/src/scheduled-stuck-http.test.ts +316 -0
- package/src/scheduled-stuck-notice-cap.test.ts +80 -0
- package/src/scheduled-stuck.ts +219 -0
- package/src/scheduled-tasks-http.test.ts +679 -0
- package/src/scheduled-tasks.ts +959 -0
- package/src/scheduled-work-outcome-http.test.ts +154 -0
- package/src/scheduled-work-outcome.ts +50 -0
- package/src/scoped-secrets.ts +141 -0
- package/src/secret-echo-negative-control.test.ts +118 -0
- package/src/send-dedup-request-id.test.ts +229 -0
- package/src/send-reply-agent-warning.test.ts +38 -38
- package/src/send-reply-alias-optional.test.ts +131 -0
- package/src/send-reply-attachments.test.ts +28 -7
- package/src/send_dedup.test.ts +54 -0
- package/src/send_dedup.ts +52 -6
- package/src/serve-error.test.ts +73 -0
- package/src/serve-error.ts +56 -0
- package/src/server.ts +4842 -325
- package/src/session-task-on-dispatch.ts +14 -0
- package/src/session-token-ownership-http.test.ts +122 -0
- package/src/shared/codex-inventory.ts +41 -0
- package/src/shared/daemon-provider-login.ts +31 -0
- package/src/shared/daemon-provider-snapshot.ts +52 -0
- package/src/shared/external-schedule-contract.ts +60 -0
- package/src/shared/node-name.test.ts +62 -0
- package/src/shared/node-name.ts +161 -0
- package/src/shared/probe-host-allowlist-drift.test.ts +45 -0
- package/src/shared/provider-application.ts +29 -0
- package/src/shared/provider-collection.ts +14 -0
- package/src/shared/provider-initialization.ts +8 -0
- package/src/shared/reserved-env-drift.test.ts +10 -0
- package/src/shared/scoped-secret-contract.ts +5 -0
- package/src/shared/task-content-limit-drift.test.ts +28 -0
- package/src/shared/task-content-limit.ts +25 -0
- package/src/side-thread-command-transport.test.ts +312 -0
- package/src/side-thread-command-transport.ts +318 -0
- package/src/side-thread-http-integration.test.ts +233 -0
- package/src/side-thread-http.ts +504 -0
- package/src/side-thread-production.test.ts +72 -0
- package/src/side-thread-production.ts +118 -0
- package/src/side-thread.test.ts +1067 -0
- package/src/side-thread.ts +2499 -0
- package/src/skillhub-http.test.ts +144 -0
- package/src/sqlite-close-checkpoint.test.ts +78 -0
- package/src/start-node.test.ts +265 -0
- package/src/start-request-error-domain.test.ts +182 -0
- package/src/start-skill-error-domain.test.ts +122 -0
- package/src/stats-versions-http.test.ts +65 -0
- package/src/status-alias-filter-http.test.ts +106 -0
- package/src/status-alias-resolver-http.test.ts +163 -0
- package/src/status-alias-resolver.ts +19 -0
- package/src/status-node-id-filter-http.test.ts +161 -0
- package/src/status-read-cache-http.test.ts +229 -0
- package/src/status-read-cache.ts +129 -0
- package/src/stop-delete-node.test.ts +567 -1
- package/src/task-access-http.test.ts +402 -0
- package/src/task-access.ts +420 -0
- package/src/task-activity.ts +60 -0
- package/src/task-auth-origin-http.test.ts +93 -0
- package/src/task-auth-origin.test.ts +24 -0
- package/src/task-auth-origin.ts +7 -0
- package/src/task-consumption.test.ts +508 -0
- package/src/task-diagnostic-http.test.ts +149 -0
- package/src/task-diagnostic.test.ts +115 -0
- package/src/task-diagnostic.ts +118 -0
- package/src/task-expiry-notice-http.test.ts +298 -0
- package/src/task-expiry-notice.ts +318 -0
- package/src/task-idempotency-mcp.test.ts +148 -0
- package/src/task-idempotency.test.ts +76 -0
- package/src/task-idempotency.ts +81 -0
- package/src/task-late-replies.ts +54 -0
- package/src/task-late-reply-http.test.ts +227 -0
- package/src/task-lifecycle-watcher-busy.test.ts +114 -0
- package/src/task-lifecycle-watcher.test.ts +209 -0
- package/src/task-lifecycle-watcher.ts +109 -0
- package/src/task-network-resolution-http.test.ts +140 -0
- package/src/task-not-found-error-domain.test.ts +76 -0
- package/src/task-offline-delivery-2668.test.ts +119 -0
- package/src/task-offline-delivery.ts +81 -0
- package/src/task-orphan-http.test.ts +287 -0
- package/src/task-orphan.ts +119 -0
- package/src/task-queue-ahead-http.test.ts +309 -0
- package/src/task-queue-ahead-scale.test.ts +236 -0
- package/src/task-queue-ahead.ts +186 -0
- package/src/task-rejection-log-http.test.ts +160 -0
- package/src/task-rejection-log.ts +61 -0
- package/src/task-sender-http.test.ts +143 -0
- package/src/task-stale-open-http.test.ts +292 -0
- package/src/task-stale-open.ts +103 -0
- package/src/tasks-pagination-http.test.ts +141 -0
- package/src/team-rules-status-http.test.ts +90 -0
- package/src/team-rules-status.test.ts +59 -0
- package/src/team-rules-status.ts +72 -0
- package/src/tmux-allowlist.test.ts +48 -0
- package/src/tool-audience-http.test.ts +248 -0
- package/src/tool-audience.ts +127 -0
- package/src/tools-list-json-schema.test.ts +94 -0
- package/src/tools.ts +4215 -397
- package/src/update-node-config-error-domain.test.ts +107 -0
- package/src/uploads-http.test.ts +6 -4
- package/src/uploads.ts +23 -4
- package/src/user-avatar-http.test.ts +74 -0
- package/src/user-avatar-image.ts +160 -0
- package/src/user-avatar-migration.test.ts +25 -0
- package/src/user-avatar-upload-http.test.ts +348 -0
- package/src/user-inbox-by-agent-http.test.ts +165 -0
- package/src/user-inbox-netid-notnull.test.ts +122 -0
- package/src/user-inbox-read-http.test.ts +223 -0
- package/src/user-inbox-schema.test.ts +54 -0
- package/src/version-stats.test.ts +66 -0
- package/src/version-stats.ts +109 -0
|
@@ -0,0 +1,180 @@
|
|
|
1
|
+
// #2614 —— 事件循环停顿监测 + 慢 SQL 日志的单元测试。
|
|
2
|
+
//
|
|
3
|
+
// 这两个东西的价值不在“能测出停顿”,而在**停顿发生时留下可对齐的证据**:
|
|
4
|
+
// 2026-10-26 生产两次 RSS 3.6/4.5 GB 的尖峰,事后只能靠“日志静默几十秒”
|
|
5
|
+
// 手工推断,`/api/stats/routes` 只给出受害者请求(含排队时间)。
|
|
6
|
+
// 所以测试要钉住的是语义:怎么算一次停顿、留什么档、日志阈值怎么生效、关闭后还计数。
|
|
7
|
+
|
|
8
|
+
import { describe, expect, test } from "bun:test";
|
|
9
|
+
import {
|
|
10
|
+
eventLoopStats,
|
|
11
|
+
noteRequestServed,
|
|
12
|
+
recordGap,
|
|
13
|
+
resetEventLoopLagForTests,
|
|
14
|
+
startEventLoopLagMonitor,
|
|
15
|
+
} from "./event-loop-lag.js";
|
|
16
|
+
import { formatSlowSql, slowSqlThresholdMs } from "./db-adapter.js";
|
|
17
|
+
|
|
18
|
+
describe("event-loop-lag: recordGap", () => {
|
|
19
|
+
test("正常 tick(≈interval)不算停顿", () => {
|
|
20
|
+
resetEventLoopLagForTests();
|
|
21
|
+
expect(recordGap(260, { intervalMs: 250, thresholdMs: 2000 })).toBe(10);
|
|
22
|
+
const s = eventLoopStats();
|
|
23
|
+
expect(s.stall_count).toBe(0);
|
|
24
|
+
expect(s.max_gap_ms).toBe(260);
|
|
25
|
+
expect(s.recent_stalls).toEqual([]);
|
|
26
|
+
});
|
|
27
|
+
|
|
28
|
+
test("超过阈值的 tick 记一条停顿,带 stall_ms / rss_mb / at", () => {
|
|
29
|
+
resetEventLoopLagForTests();
|
|
30
|
+
const now = Date.parse("2026-10-26T22:10:41Z");
|
|
31
|
+
const stall = recordGap(28_000, { intervalMs: 250, thresholdMs: 2000, rssMb: 4502, now });
|
|
32
|
+
expect(stall).toBe(27_750);
|
|
33
|
+
const s = eventLoopStats();
|
|
34
|
+
expect(s.stall_count).toBe(1);
|
|
35
|
+
expect(s.last_stall_ms).toBe(27_750);
|
|
36
|
+
expect(s.last_stall_at).toBe("2026-10-26T22:10:41.000Z");
|
|
37
|
+
expect(s.max_gap_ms).toBe(28_000);
|
|
38
|
+
expect(s.recent_stalls[0]).toMatchObject({ gap_ms: 28_000, stall_ms: 27_750, rss_mb: 4502 });
|
|
39
|
+
});
|
|
40
|
+
|
|
41
|
+
test("recent_stalls 最新在前,且只保留 20 条", () => {
|
|
42
|
+
resetEventLoopLagForTests();
|
|
43
|
+
for (let i = 0; i < 25; i++) {
|
|
44
|
+
recordGap(5_000 + i, { intervalMs: 250, thresholdMs: 2_000, rssMb: 300, now: 1_700_000_000_000 + i * 1_000 });
|
|
45
|
+
}
|
|
46
|
+
const s = eventLoopStats();
|
|
47
|
+
expect(s.stall_count).toBe(25);
|
|
48
|
+
expect(s.recent_stalls.length).toBe(20);
|
|
49
|
+
// 最新一条是 i=24
|
|
50
|
+
expect(s.recent_stalls[0].gap_ms).toBe(5_024);
|
|
51
|
+
expect(s.recent_stalls[19].gap_ms).toBe(5_005);
|
|
52
|
+
});
|
|
53
|
+
|
|
54
|
+
test("记录这段间隔里服务了多少请求(区分“阻塞”与“timer 饥饿”)", () => {
|
|
55
|
+
resetEventLoopLagForTests();
|
|
56
|
+
recordGap(85_000, { intervalMs: 250, thresholdMs: 2_000, rssMb: 2076, served: 137 });
|
|
57
|
+
const s = eventLoopStats();
|
|
58
|
+
expect(s.recent_stalls[0].requests_since_last_tick).toBe(137);
|
|
59
|
+
// 0 服务数 = 这段时间真的什么都没推进
|
|
60
|
+
recordGap(30_000, { intervalMs: 250, thresholdMs: 2_000, rssMb: 500, served: 0 });
|
|
61
|
+
expect(eventLoopStats().recent_stalls[0].requests_since_last_tick).toBe(0);
|
|
62
|
+
});
|
|
63
|
+
|
|
64
|
+
test("max_gap_ms 保留历史最大值(回落后不被覆盖)", () => {
|
|
65
|
+
resetEventLoopLagForTests();
|
|
66
|
+
recordGap(47_000, { intervalMs: 250, thresholdMs: 2_000 });
|
|
67
|
+
recordGap(260, { intervalMs: 250, thresholdMs: 2_000 });
|
|
68
|
+
const s = eventLoopStats();
|
|
69
|
+
expect(s.max_gap_ms).toBe(47_000);
|
|
70
|
+
expect(s.last_gap_ms).toBe(260);
|
|
71
|
+
});
|
|
72
|
+
});
|
|
73
|
+
|
|
74
|
+
describe("event-loop-lag: startEventLoopLagMonitor", () => {
|
|
75
|
+
test("阈值 0 时不打日志,但仍然计数", async () => {
|
|
76
|
+
resetEventLoopLagForTests();
|
|
77
|
+
const logs: string[] = [];
|
|
78
|
+
const timer = startEventLoopLagMonitor({
|
|
79
|
+
intervalMs: 20,
|
|
80
|
+
thresholdMs: 0,
|
|
81
|
+
log: (m) => logs.push(m),
|
|
82
|
+
rssMb: () => 123,
|
|
83
|
+
});
|
|
84
|
+
// 同步阻塞 80ms,让下一次 tick 一定超阈值
|
|
85
|
+
const until = performance.now() + 80;
|
|
86
|
+
while (performance.now() < until) { /* spin */ }
|
|
87
|
+
await Bun.sleep(60);
|
|
88
|
+
clearInterval(timer);
|
|
89
|
+
expect(logs).toEqual([]);
|
|
90
|
+
expect(eventLoopStats().max_gap_ms).toBeGreaterThan(0);
|
|
91
|
+
});
|
|
92
|
+
|
|
93
|
+
test("真实停顿会打一行 [commhub stall] 且带 rss", async () => {
|
|
94
|
+
resetEventLoopLagForTests();
|
|
95
|
+
const logs: string[] = [];
|
|
96
|
+
const timer = startEventLoopLagMonitor({
|
|
97
|
+
intervalMs: 20,
|
|
98
|
+
thresholdMs: 40,
|
|
99
|
+
log: (m) => logs.push(m),
|
|
100
|
+
rssMb: () => 4502,
|
|
101
|
+
// 这条测的是「停顿怎么打日志」,不是启动尖峰。测试进程刚起来几秒,
|
|
102
|
+
// 不关掉启动宽限的话第一个 tick 会被当成启动尖峰跳过(见 #2721 那组用例)。
|
|
103
|
+
bootGraceMs: 0,
|
|
104
|
+
});
|
|
105
|
+
const until = performance.now() + 120;
|
|
106
|
+
while (performance.now() < until) { /* spin 阻塞事件循环 */ }
|
|
107
|
+
await Bun.sleep(60);
|
|
108
|
+
clearInterval(timer);
|
|
109
|
+
expect(logs.length).toBeGreaterThan(0);
|
|
110
|
+
expect(logs[0]).toStartWith("[commhub stall] loop idle ~");
|
|
111
|
+
expect(logs[0]).toContain("rss 4502 MB");
|
|
112
|
+
expect(logs[0]).toContain("served 0 req");
|
|
113
|
+
});
|
|
114
|
+
});
|
|
115
|
+
|
|
116
|
+
describe("db-adapter: 慢 SQL 日志", () => {
|
|
117
|
+
test("默认阈值 1000ms;环境变量可调,0 表示关闭", () => {
|
|
118
|
+
const prev = process.env.COMMHUB_SLOW_SQL_MS;
|
|
119
|
+
try {
|
|
120
|
+
delete process.env.COMMHUB_SLOW_SQL_MS;
|
|
121
|
+
expect(slowSqlThresholdMs()).toBe(1000);
|
|
122
|
+
process.env.COMMHUB_SLOW_SQL_MS = "250";
|
|
123
|
+
expect(slowSqlThresholdMs()).toBe(250);
|
|
124
|
+
process.env.COMMHUB_SLOW_SQL_MS = "0";
|
|
125
|
+
expect(slowSqlThresholdMs()).toBe(0);
|
|
126
|
+
process.env.COMMHUB_SLOW_SQL_MS = "not-a-number";
|
|
127
|
+
expect(slowSqlThresholdMs()).toBe(1000);
|
|
128
|
+
} finally {
|
|
129
|
+
if (prev === undefined) delete process.env.COMMHUB_SLOW_SQL_MS;
|
|
130
|
+
else process.env.COMMHUB_SLOW_SQL_MS = prev;
|
|
131
|
+
}
|
|
132
|
+
});
|
|
133
|
+
|
|
134
|
+
test("日志压平换行并截断,且不含绑定参数", () => {
|
|
135
|
+
const sql = "DELETE FROM agent_telemetry\n WHERE created_at < datetime('now', ?1)";
|
|
136
|
+
const line = formatSlowSql(10_588.4, sql);
|
|
137
|
+
expect(line).toStartWith("[commhub slow-sql] 10588ms DELETE FROM agent_telemetry WHERE created_at < datetime('now', ?1)");
|
|
138
|
+
expect(line).not.toContain("\n");
|
|
139
|
+
const long = formatSlowSql(1_200, "SELECT " + "x".repeat(500));
|
|
140
|
+
expect(long.length).toBeLessThanOrEqual("[commhub slow-sql] 1200ms ".length + 160);
|
|
141
|
+
});
|
|
142
|
+
});
|
|
143
|
+
|
|
144
|
+
// #2721 —— **只跳过第一个 tick**。
|
|
145
|
+
//
|
|
146
|
+
// 事故形状:进程启动后 3 秒记到 gap 2972ms —— 那是「监控开始 → 第一次 tick」被
|
|
147
|
+
// 模块加载/开库/迁移占满的时间,度量的是启动耗时,不是线上停顿。
|
|
148
|
+
//
|
|
149
|
+
// 第一版实现用的是 10 秒**时间窗**,等于监控在前 10 秒什么都不记,
|
|
150
|
+
// 把下面那条「阈值 40ms、spin 120ms 应打停顿日志」的既有测试整体吞掉了
|
|
151
|
+
// —— 修完 qa.yml 事故、QA 恢复后立刻红。窗口越宽越像「把监控关一会儿」。
|
|
152
|
+
describe("#2721 启动尖峰不算停顿(判据:进程 uptime)", () => {
|
|
153
|
+
test("进程刚启动时的第一个 gap 被跳过", async () => {
|
|
154
|
+
resetEventLoopLagForTests();
|
|
155
|
+
const logs: string[] = [];
|
|
156
|
+
const timer = startEventLoopLagMonitor({
|
|
157
|
+
intervalMs: 20, thresholdMs: 40, log: (m) => logs.push(m), rssMb: () => 123,
|
|
158
|
+
bootGraceMs: 15_000, uptimeMs: () => 3_000, // 模拟「启动后 3 秒」
|
|
159
|
+
});
|
|
160
|
+
const until = performance.now() + 120;
|
|
161
|
+
while (performance.now() < until) { /* spin 落在第一个 tick */ }
|
|
162
|
+
await Bun.sleep(80);
|
|
163
|
+
clearInterval(timer);
|
|
164
|
+
expect(logs).toEqual([]);
|
|
165
|
+
});
|
|
166
|
+
|
|
167
|
+
test("进程已经跑了一会儿 ⇒ 第一个 gap 照常记录(不能把监控关掉)", async () => {
|
|
168
|
+
resetEventLoopLagForTests();
|
|
169
|
+
const logs: string[] = [];
|
|
170
|
+
const timer = startEventLoopLagMonitor({
|
|
171
|
+
intervalMs: 20, thresholdMs: 40, log: (m) => logs.push(m), rssMb: () => 123,
|
|
172
|
+
bootGraceMs: 15_000, uptimeMs: () => 600_000, // 进程跑了 10 分钟
|
|
173
|
+
});
|
|
174
|
+
const until = performance.now() + 120;
|
|
175
|
+
while (performance.now() < until) { /* spin */ }
|
|
176
|
+
await Bun.sleep(80);
|
|
177
|
+
clearInterval(timer);
|
|
178
|
+
expect(logs.length).toBeGreaterThan(0);
|
|
179
|
+
});
|
|
180
|
+
});
|
|
@@ -0,0 +1,201 @@
|
|
|
1
|
+
// #2614 —— 事件循环停顿监测(只观测,不改任何行为)。
|
|
2
|
+
//
|
|
3
|
+
// 背景:2026-10-26 21:55–22:12 生产 Hub 出现两次内存尖峰(RSS 3.6 → 4.5 GB),
|
|
4
|
+
// 伴随机 `mcp_inflight` 387 → 807。事后看日志才发现**同一时刻日志静默 14–47 秒**
|
|
5
|
+
// ——Hub 是单线程(Bun + 同步 SQLite),日志静默就等于事件循环被阻塞。
|
|
6
|
+
//
|
|
7
|
+
// 现有 `/api/stats/routes` 只能告诉我们“哪个请求**受害**了”(它的耗时包含了排队时间),
|
|
8
|
+
// 不能告诉我们“当时**谁**在阻塞”。缺的就是停顿本身的时间序列。
|
|
9
|
+
//
|
|
10
|
+
// 这个模块补上这一格:每 `intervalMs` 醒一次,量 `now - lastTick`;
|
|
11
|
+
// 正常情况 ≈ interval,超出 `thresholdMs` 的部分就是一次真实停顿。
|
|
12
|
+
// 停顿会:① 计数/留档(`/health` 的 `event_loop` 字段可读);
|
|
13
|
+
// ② 打一行日志(带当时的 RSS,方便和内存曲线对齐)。
|
|
14
|
+
//
|
|
15
|
+
// 注意:`setInterval` 自己也会被阻塞 —— 这正是我们要测量的东西。
|
|
16
|
+
// 停顿结束后第一次 tick 的 gap 就是停顿长度,不需要外部进程配合。
|
|
17
|
+
|
|
18
|
+
export type EventLoopStall = {
|
|
19
|
+
/** 这个间隔里处理完的请求数:0 = 真的没推进;>0 = 循环在干活,只是 timer 被饿住。 */
|
|
20
|
+
/** ISO 时间(停顿**结束**的时刻,即这次 tick 的时间)。 */
|
|
21
|
+
at: string;
|
|
22
|
+
/** 距上次 tick 的实际间隔(毫秒)。等于 interval + 停顿。 */
|
|
23
|
+
gap_ms: number;
|
|
24
|
+
/** 停顿量 = gap - interval。 */
|
|
25
|
+
stall_ms: number;
|
|
26
|
+
/** 停顿结束时的进程 RSS(MB),用于和内存尖峰对齐。 */
|
|
27
|
+
rss_mb: number;
|
|
28
|
+
requests_since_last_tick?: number;
|
|
29
|
+
};
|
|
30
|
+
|
|
31
|
+
const KEEP_STALLS = 20;
|
|
32
|
+
|
|
33
|
+
/**
|
|
34
|
+
* #2614(第二轮)—— 2026-10-27 00:39 线上出现 gap 84887ms,但**同一时段日志在正常写**
|
|
35
|
+
* (00:38:52 / 00:38:58 / 00:39:08 …),而且 `/api/stats/routes` 里最慢的请求只有 2.4s。
|
|
36
|
+
* 也就是说:事件循环**没有被阻塞 84 秒**,是 **timer 回调被 I/O 队列饿住**了。
|
|
37
|
+
* 只看 tick 间隔会把这两种情况混成一条“卡了 84 秒”的假结论。
|
|
38
|
+
*
|
|
39
|
+
* 区分办法:数一下这个间隔里**服务了多少请求**。
|
|
40
|
+
* requests_since_last_tick > 0 ⇒ 循环在干活 ⇒ 这是 timer 饥饿(高负载/大批回调)
|
|
41
|
+
* requests_since_last_tick = 0 ⇒ 真的没有推进任何请求 ⇒ 才可能是同步阻塞
|
|
42
|
+
*/
|
|
43
|
+
let servedSinceTick = 0;
|
|
44
|
+
|
|
45
|
+
/** 请求处理完成时调用(route-timing 里)。只计数,不参与任何决策。 */
|
|
46
|
+
export function noteRequestServed(): void {
|
|
47
|
+
servedSinceTick++;
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
let maxGapMs = 0;
|
|
51
|
+
let lastGapMs = 0;
|
|
52
|
+
let stallCount = 0;
|
|
53
|
+
let lastStallAt: string | null = null;
|
|
54
|
+
let lastStallMs = 0;
|
|
55
|
+
const stalls: EventLoopStall[] = [];
|
|
56
|
+
|
|
57
|
+
/** 记录一次 tick 间隔。返回这次 tick 的停顿量(毫秒,可为负→归零)。 */
|
|
58
|
+
export function recordGap(
|
|
59
|
+
gapMs: number,
|
|
60
|
+
opts: { intervalMs?: number; thresholdMs?: number; rssMb?: number; now?: number; served?: number } = {},
|
|
61
|
+
): number {
|
|
62
|
+
const interval = opts.intervalMs ?? 250;
|
|
63
|
+
const threshold = opts.thresholdMs ?? 2000;
|
|
64
|
+
lastGapMs = gapMs;
|
|
65
|
+
if (gapMs > maxGapMs) maxGapMs = gapMs;
|
|
66
|
+
const stallMs = Math.max(0, gapMs - interval);
|
|
67
|
+
if (stallMs < threshold) return stallMs;
|
|
68
|
+
const at = new Date(opts.now ?? Date.now()).toISOString();
|
|
69
|
+
stallCount++;
|
|
70
|
+
lastStallAt = at;
|
|
71
|
+
lastStallMs = stallMs;
|
|
72
|
+
stalls.push({
|
|
73
|
+
at,
|
|
74
|
+
gap_ms: Math.round(gapMs),
|
|
75
|
+
stall_ms: Math.round(stallMs),
|
|
76
|
+
rss_mb: opts.rssMb ?? 0,
|
|
77
|
+
requests_since_last_tick: opts.served ?? 0,
|
|
78
|
+
});
|
|
79
|
+
while (stalls.length > KEEP_STALLS) stalls.shift();
|
|
80
|
+
return stallMs;
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
/**
|
|
84
|
+
* 2026-10-31 —— 窗口两端必须跟着读数一起走(#3174)。
|
|
85
|
+
*
|
|
86
|
+
* 原来 event_loop 给了 stall_count / last_stall_at,却没有窗口起点:
|
|
87
|
+
* 计数器随进程重启归零,而「归零」与「从未发生」在读数上同值 ⇒
|
|
88
|
+
* 单独引用 stall_count 的读者会把它读成「从上线以来没卡过」。
|
|
89
|
+
*
|
|
90
|
+
* ⇒ 现在两个时刻戳都在这对象里:`since`(起点,模块初始化时冻结)与 `observed_at`(读的时刻)。
|
|
91
|
+
* 引用者不必再去 /health 顶层猜窗口,窗口与读数同域。
|
|
92
|
+
* 判据:分子分母各要自己的时刻戳;**被声明的那个永远不算**。
|
|
93
|
+
*/
|
|
94
|
+
let sinceIso = new Date().toISOString();
|
|
95
|
+
|
|
96
|
+
export function eventLoopStats(): {
|
|
97
|
+
max_gap_ms: number;
|
|
98
|
+
last_gap_ms: number;
|
|
99
|
+
stall_count: number;
|
|
100
|
+
last_stall_ms: number;
|
|
101
|
+
last_stall_at: string | null;
|
|
102
|
+
recent_stalls: EventLoopStall[];
|
|
103
|
+
/** 窗口起点(本进程 event-loop 观察开始时刻)。 */
|
|
104
|
+
since: string;
|
|
105
|
+
/** 这次读的时刻。与 `since` 一起才构成窗口;单独任一个都不构成。 */
|
|
106
|
+
observed_at: string;
|
|
107
|
+
} {
|
|
108
|
+
return {
|
|
109
|
+
max_gap_ms: Math.round(maxGapMs),
|
|
110
|
+
last_gap_ms: Math.round(lastGapMs),
|
|
111
|
+
stall_count: stallCount,
|
|
112
|
+
last_stall_ms: Math.round(lastStallMs),
|
|
113
|
+
last_stall_at: lastStallAt,
|
|
114
|
+
recent_stalls: [...stalls].reverse(),
|
|
115
|
+
since: sinceIso,
|
|
116
|
+
observed_at: new Date().toISOString(),
|
|
117
|
+
};
|
|
118
|
+
}
|
|
119
|
+
|
|
120
|
+
/** 只给测试用:清空累计状态。 */
|
|
121
|
+
export function resetEventLoopLagForTests(): void {
|
|
122
|
+
maxGapMs = 0;
|
|
123
|
+
lastGapMs = 0;
|
|
124
|
+
stallCount = 0;
|
|
125
|
+
lastStallAt = null;
|
|
126
|
+
lastStallMs = 0;
|
|
127
|
+
stalls.length = 0;
|
|
128
|
+
sinceIso = new Date().toISOString(); // 重置窗口起点 —— 否则测试里「归零」与「没归零」同形
|
|
129
|
+
}
|
|
130
|
+
|
|
131
|
+
function envMs(name: string, fallback: number): number {
|
|
132
|
+
const raw = Number(process.env[name]);
|
|
133
|
+
return Number.isFinite(raw) && raw >= 0 ? raw : fallback;
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
/**
|
|
137
|
+
* 启动监测。返回 timer(调用方负责 clearInterval)。
|
|
138
|
+
* - `COMMHUB_STALL_LOG_MS`:停顿日志阈值,默认 2000 ms,设 0 关闭日志(仍计数);
|
|
139
|
+
* - `COMMHUB_STALL_TICK_MS`:采样间隔,默认 250 ms。
|
|
140
|
+
*/
|
|
141
|
+
export function startEventLoopLagMonitor(opts: {
|
|
142
|
+
intervalMs?: number;
|
|
143
|
+
thresholdMs?: number;
|
|
144
|
+
log?: (msg: string) => void;
|
|
145
|
+
rssMb?: () => number;
|
|
146
|
+
/**
|
|
147
|
+
* #2721 —— 启动期的 tick gap 不算停顿。判据是**进程启动多久了**,不是第几个 tick。
|
|
148
|
+
*
|
|
149
|
+
* 事故形状:进程 22:21:06 启动,14:21:09.835Z 记到 gap 2972ms —— 启动后 3 秒。
|
|
150
|
+
* 那段 gap 度量的是「监控开始 → 第一次 tick」被模块加载/开库/迁移占满的时间。
|
|
151
|
+
*
|
|
152
|
+
* 两版错误实现(都试过,都被既有测试抓住):
|
|
153
|
+
* ① 10 秒**时间窗** → 监控在前 10 秒什么都不记,「阈值 40ms + spin 120ms 应打日志」直接吞掉
|
|
154
|
+
* ② 只跳**第一个 tick** → 测试里的 spin 恰好落在第一个 tick,同样吞掉
|
|
155
|
+
*
|
|
156
|
+
* 真正的区别不是「第几个 tick」,而是**这段时间里进程有没有可能还在启动**。
|
|
157
|
+
* 所以:只有当 tick 到来时进程 uptime 仍小于 bootGraceMs,才把 gap 当作启动尖峰。
|
|
158
|
+
* 启动 20 秒之后的第一条 gap 依旧照常记录 —— 那时候它不可能是启动耗时。
|
|
159
|
+
*/
|
|
160
|
+
bootGraceMs?: number;
|
|
161
|
+
/** 测试注入:返回进程已运行的毫秒数(默认 process.uptime())。 */
|
|
162
|
+
uptimeMs?: () => number;
|
|
163
|
+
} = {}): ReturnType<typeof setInterval> {
|
|
164
|
+
const intervalMs = opts.intervalMs ?? Math.max(50, envMs("COMMHUB_STALL_TICK_MS", 250));
|
|
165
|
+
const thresholdMs = opts.thresholdMs ?? envMs("COMMHUB_STALL_LOG_MS", 2000);
|
|
166
|
+
const log = opts.log ?? ((m: string) => console.log(m));
|
|
167
|
+
const rssMb = opts.rssMb ?? (() => Math.round(process.memoryUsage().rss / 1048576));
|
|
168
|
+
let last = performance.now();
|
|
169
|
+
const startedAt = last;
|
|
170
|
+
// #2721 —— 启动预热不算停顿。
|
|
171
|
+
// 实测:2026-10-29 这次进程 22:21:06 起来,14:21:09.835Z 记到一次 gap 2972ms,
|
|
172
|
+
// 也就是**启动后 3 秒**。那段时间在做模块加载 / 打开 DB / 迁移,第一个 tick 必然偏大。
|
|
173
|
+
// 不排除它的后果是:每次部署都会在 `/health` 里留一条「停顿」,而它和线上真实卡顿长得一模一样。
|
|
174
|
+
const bootGraceMs = opts.bootGraceMs ?? envMs("COMMHUB_STALL_BOOT_GRACE_MS", 15_000);
|
|
175
|
+
const uptimeMs = opts.uptimeMs ?? (() => process.uptime() * 1000);
|
|
176
|
+
let firstTick = true;
|
|
177
|
+
const timer = setInterval(() => {
|
|
178
|
+
const now = performance.now();
|
|
179
|
+
const gap = now - last;
|
|
180
|
+
last = now;
|
|
181
|
+
const rss = rssMb();
|
|
182
|
+
const served = servedSinceTick;
|
|
183
|
+
servedSinceTick = 0;
|
|
184
|
+
// 启动尖峰:只可能是**第一个** tick,且到来时进程才刚起来。
|
|
185
|
+
// 两个条件缺一不可 —— 只看「第一个 tick」会吞掉运行中立刻发生的真实停顿。
|
|
186
|
+
if (firstTick) {
|
|
187
|
+
firstTick = false;
|
|
188
|
+
if (uptimeMs() < bootGraceMs) return;
|
|
189
|
+
}
|
|
190
|
+
const stall = recordGap(gap, { intervalMs, thresholdMs, rssMb: rss, served });
|
|
191
|
+
if (thresholdMs > 0 && stall >= thresholdMs) {
|
|
192
|
+
// 和 retention/slow-sql 的日志同一前缀,方便 grep。
|
|
193
|
+
// `served` 是判据的关键:它非零就说明循环在干活,这段“gap”是 timer 被饿住,不是阻塞。
|
|
194
|
+
log(`[commhub stall] loop idle ~${Math.round(stall)}ms (tick gap ${Math.round(gap)}ms, rss ${rss} MB, served ${served} req)`);
|
|
195
|
+
}
|
|
196
|
+
}, intervalMs);
|
|
197
|
+
// 监测不该拖住进程退出。
|
|
198
|
+
(timer as unknown as { unref?: () => void }).unref?.();
|
|
199
|
+
last = performance.now();
|
|
200
|
+
return timer;
|
|
201
|
+
}
|
|
@@ -0,0 +1,187 @@
|
|
|
1
|
+
// #3271 —— 「到死都没被取走」的计数:**正控 + 负数用例**。
|
|
2
|
+
//
|
|
3
|
+
// 现场:一台节点上 4 条任务,一次又一次「等满 60 分钟 ⇒ expired」,而积压数**两次读数都是 4**。
|
|
4
|
+
// 「又过期了一条」是可见的(Hub 会发通知),但「这台节点连续 N 次一次都没取走」没有读数。
|
|
5
|
+
//
|
|
6
|
+
// 这个计数器属于「只有异常路径能让它 +1」那一类 ⇒ 必须有 must-fire:
|
|
7
|
+
// ① 注入一条**未消费**就过期的任务 ⇒ 计数 +1(正控)
|
|
8
|
+
// ② 注入一条**已消费**后过期的任务 ⇒ 计数**不动**(负数用例)
|
|
9
|
+
// 少了 ②,这个计数就退化成「过期了多少条」——而那是另一个更没用的数。
|
|
10
|
+
import { afterAll, beforeEach, describe, expect, test } from "bun:test";
|
|
11
|
+
import { db } from "./db.js";
|
|
12
|
+
import { neverConsumedExpiryCounters, noteNeverConsumedExpiry, resetNeverConsumedExpiryCountersForTest } from "./expiry-counters.js";
|
|
13
|
+
import { patrolExpiredTasks } from "./server.js";
|
|
14
|
+
|
|
15
|
+
const NET = "net_expiry_cnt";
|
|
16
|
+
|
|
17
|
+
function seedTask(id: string, opts: { toName: string; consumed: boolean; expiredSecondsAgo?: number }) {
|
|
18
|
+
const ago = opts.expiredSecondsAgo ?? 120;
|
|
19
|
+
db.run(
|
|
20
|
+
`INSERT INTO tasks (task_id, network_id, from_name, to_name, content, status, created_at, expires_at, consumed_at)
|
|
21
|
+
VALUES (?1, ?2, 'sender', ?3, 'fixture', 'delivered', datetime('now', '-1 hour'), datetime('now', ?4), ?5)`,
|
|
22
|
+
[id, NET, opts.toName, `-${ago} seconds`, opts.consumed ? new Date(Date.now() - 3600_000).toISOString().replace("T", " ").slice(0, 19) : null],
|
|
23
|
+
);
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
function cleanup() {
|
|
27
|
+
try { db.run("DELETE FROM tasks WHERE network_id = ?1", [NET]); } catch {}
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
beforeEach(() => { cleanup(); resetNeverConsumedExpiryCountersForTest(); });
|
|
31
|
+
afterAll(() => { cleanup(); });
|
|
32
|
+
|
|
33
|
+
describe("#3271 从未取走过的过期计数", () => {
|
|
34
|
+
test("🔴 must-fire:未消费就过期的任务 ⇒ 该节点计数 +1", () => {
|
|
35
|
+
seedTask("t_never_1", { toName: "stuck-node", consumed: false });
|
|
36
|
+
patrolExpiredTasks();
|
|
37
|
+
const c = neverConsumedExpiryCounters();
|
|
38
|
+
expect(c.by_node["stuck-node"]).toBe(1);
|
|
39
|
+
expect(c.total).toBe(1);
|
|
40
|
+
expect(typeof c.since).toBe("string"); // 窗口左端必须在,否则「归零」与「没发生」同值
|
|
41
|
+
});
|
|
42
|
+
|
|
43
|
+
test("🔴 负数用例:**已消费**后过期的任务 ⇒ 计数不动(那不是「没人取」)", () => {
|
|
44
|
+
seedTask("t_consumed_1", { toName: "busy-but-alive", consumed: true });
|
|
45
|
+
patrolExpiredTasks();
|
|
46
|
+
expect(neverConsumedExpiryCounters().total).toBe(0);
|
|
47
|
+
});
|
|
48
|
+
|
|
49
|
+
test("按节点分桶:两个节点各一条 ⇒ 各自 1,总数 2", () => {
|
|
50
|
+
seedTask("t_a", { toName: "node-a", consumed: false });
|
|
51
|
+
seedTask("t_b", { toName: "node-b", consumed: false });
|
|
52
|
+
patrolExpiredTasks();
|
|
53
|
+
const c = neverConsumedExpiryCounters();
|
|
54
|
+
expect(c.by_node).toEqual({ "node-a": 1, "node-b": 1 });
|
|
55
|
+
expect(c.total).toBe(2);
|
|
56
|
+
});
|
|
57
|
+
|
|
58
|
+
test("同一个节点连续 N 次 ⇒ 累加(这是「这一台从来不吃活」的证据形态)", () => {
|
|
59
|
+
for (let i = 0; i < 4; i += 1) noteNeverConsumedExpiry("ghost-node");
|
|
60
|
+
expect(neverConsumedExpiryCounters().by_node["ghost-node"]).toBe(4);
|
|
61
|
+
});
|
|
62
|
+
|
|
63
|
+
test("桶按名字排序且空桶不出现 —— 读数要能直接对比两次快照", () => {
|
|
64
|
+
noteNeverConsumedExpiry("z-node");
|
|
65
|
+
noteNeverConsumedExpiry("a-node");
|
|
66
|
+
const c = neverConsumedExpiryCounters();
|
|
67
|
+
expect(Object.keys(c.by_node)).toEqual(["a-node", "z-node"]);
|
|
68
|
+
});
|
|
69
|
+
});
|
|
70
|
+
|
|
71
|
+
// ---- 派单侧硬闸(#3271)----
|
|
72
|
+
import { neverConsumeGate, NEVER_CONSUME_LIMIT, NEVER_CONSUME_STALE_MS } from "./expiry-counters.js";
|
|
73
|
+
|
|
74
|
+
describe("#3271 派单侧硬闸:从未取走过的目标", () => {
|
|
75
|
+
const NOW = Date.parse("2026-10-31T10:00:00.000Z");
|
|
76
|
+
const counters = (n: number, alias = "ghost", lastAt = new Date(NOW - 60_000).toISOString()) => ({
|
|
77
|
+
since: "2026-10-31T00:00:00.000Z",
|
|
78
|
+
total: n,
|
|
79
|
+
by_node: n > 0 ? { [alias]: n } : {},
|
|
80
|
+
last_at: n > 0 ? { [alias]: lastAt } : {},
|
|
81
|
+
});
|
|
82
|
+
|
|
83
|
+
test("窗口内 0–2 次 ⇒ 放行(一次两次可能是宿主重启,三次才是形态)", () => {
|
|
84
|
+
for (const n of [0, 1, 2]) {
|
|
85
|
+
expect(neverConsumeGate({ alias: "ghost", counters: counters(n), force: false, forceAllowed: true, now: Date.parse("2026-10-31T10:00:00.000Z") }).ok).toBe(true);
|
|
86
|
+
}
|
|
87
|
+
});
|
|
88
|
+
|
|
89
|
+
test("🔴 达到阈值 ⇒ 拒绝,且拒绝体给全(alias / expiries / window_since / 下一步)", () => {
|
|
90
|
+
const r = neverConsumeGate({ alias: "ghost", counters: counters(NEVER_CONSUME_LIMIT), force: false, forceAllowed: true, now: Date.parse("2026-10-31T10:00:00.000Z") });
|
|
91
|
+
expect(r.ok).toBe(false);
|
|
92
|
+
if (r.ok) throw new Error("unreachable");
|
|
93
|
+
expect(r.error).toBe("target_never_consumes");
|
|
94
|
+
expect(r.alias).toBe("ghost");
|
|
95
|
+
expect(r.expiries).toBe(NEVER_CONSUME_LIMIT);
|
|
96
|
+
expect(r.window_since).toBe("2026-10-31T00:00:00.000Z");
|
|
97
|
+
expect(r.message).toContain("force:true");
|
|
98
|
+
});
|
|
99
|
+
|
|
100
|
+
test("人令牌 + force ⇒ 放行(出口存在,但必须显式)", () => {
|
|
101
|
+
expect(neverConsumeGate({ alias: "ghost", counters: counters(9), force: true, forceAllowed: true, now: Date.parse("2026-10-31T10:00:00.000Z") }).ok).toBe(true);
|
|
102
|
+
});
|
|
103
|
+
|
|
104
|
+
test("🔴 节点令牌不能自己放行:force 传了也不算(forceAllowed=false)", () => {
|
|
105
|
+
const r = neverConsumeGate({ alias: "ghost", counters: counters(9), force: true, forceAllowed: false, now: Date.parse("2026-10-31T10:00:00.000Z") });
|
|
106
|
+
expect(r.ok).toBe(false);
|
|
107
|
+
});
|
|
108
|
+
|
|
109
|
+
test("别的节点的计数不影响本目标(分桶不是总数)", () => {
|
|
110
|
+
const c = { since: "s", total: 9, by_node: { other: 9 }, last_at: { other: new Date().toISOString() } };
|
|
111
|
+
expect(neverConsumeGate({ alias: "ghost", counters: c, force: false, forceAllowed: true, now: Date.parse("2026-10-31T10:00:00.000Z") }).ok).toBe(true);
|
|
112
|
+
});
|
|
113
|
+
});
|
|
114
|
+
|
|
115
|
+
describe("#3271 证据的生命期必须 ≥ 故障的生命期(进程内计数器不够)", () => {
|
|
116
|
+
const NOW = Date.parse("2026-10-31T10:00:00.000Z");
|
|
117
|
+
const empty = { since: "2026-10-31T10:00:00.000Z", total: 0, by_node: {}, last_at: {} };
|
|
118
|
+
|
|
119
|
+
// 🔴 这条是 2026-10-31 生产实测的 must-fire:
|
|
120
|
+
// `.254` 部署(22:39)后,我给一台**已经 10 次到死没被取走**的节点派单,
|
|
121
|
+
// 闸返回 ok:true —— 因为计数器随进程重启归零,而故障是以「周」计的。
|
|
122
|
+
test("🔴 内存计数为空,但 tasks 表里有 3 条历史 ⇒ 必须拒绝", () => {
|
|
123
|
+
const r = neverConsumeGate({
|
|
124
|
+
alias: "ghost", counters: empty, force: false, forceAllowed: true, now: NOW,
|
|
125
|
+
persistent: { expiries: 3, last_at: new Date(NOW - 60_000).toISOString().replace("T", " ").slice(0, 19), last_reply_at: null },
|
|
126
|
+
});
|
|
127
|
+
expect(r.ok).toBe(false);
|
|
128
|
+
if (r.ok) throw new Error("unreachable");
|
|
129
|
+
expect(r.expiries).toBe(3);
|
|
130
|
+
});
|
|
131
|
+
|
|
132
|
+
test("两个来源取**并集**(内存 2 + 表里 3 ⇒ 按 3 判,不是相加也不是取小)", () => {
|
|
133
|
+
const counters = { since: "s", total: 2, by_node: { ghost: 2 }, last_at: { ghost: new Date(NOW - 60_000).toISOString() } };
|
|
134
|
+
const r = neverConsumeGate({
|
|
135
|
+
alias: "ghost", counters, force: false, forceAllowed: true, now: NOW,
|
|
136
|
+
persistent: { expiries: 3, last_at: new Date(NOW - 60_000).toISOString().replace("T", " ").slice(0, 19), last_reply_at: null },
|
|
137
|
+
});
|
|
138
|
+
expect(r.ok).toBe(false);
|
|
139
|
+
if (r.ok) throw new Error("unreachable");
|
|
140
|
+
expect(r.expiries).toBe(3);
|
|
141
|
+
});
|
|
142
|
+
|
|
143
|
+
// 🔴 语义修正(2026-10-31 第二次实测):DB 证据**不用固定 TTL**。
|
|
144
|
+
// 固定 2 小时是我拍的,而那台节点的故障节律是「天」——于是闸在 22 小时前的那次过期之后
|
|
145
|
+
// 判「已恢复」并放行,而它这 22 小时一条都没回。
|
|
146
|
+
// 现在 DB 证据自带边界(「最后一次回复之后」),陈旧不陈旧由回复本身回答。
|
|
147
|
+
test("🔴 表里的历史很旧、但之后没有回复 ⇒ 仍然拒绝(固定 TTL 只留给纯内存计数)", () => {
|
|
148
|
+
const r = neverConsumeGate({
|
|
149
|
+
alias: "ghost", counters: empty, force: false, forceAllowed: true, now: NOW,
|
|
150
|
+
persistent: { expiries: 9, last_at: "2026-10-20T00:00:00", last_reply_at: "2026-10-18T00:00:00" },
|
|
151
|
+
});
|
|
152
|
+
expect(r.ok).toBe(false);
|
|
153
|
+
});
|
|
154
|
+
|
|
155
|
+
test("纯内存计数仍然有 2 小时时效(没有回复时间可比,只能按时长兜底)", () => {
|
|
156
|
+
const counters = { since: "s", total: 9, by_node: { ghost: 9 }, last_at: { ghost: new Date(NOW - 3 * 3600_000).toISOString() } };
|
|
157
|
+
expect(neverConsumeGate({ alias: "ghost", counters, force: false, forceAllowed: true, now: NOW }).ok).toBe(true);
|
|
158
|
+
});
|
|
159
|
+
});
|
|
160
|
+
|
|
161
|
+
describe("#3271 硬闸的时效:只数不衰减会把恢复了的节点永久拉黑", () => {
|
|
162
|
+
const NOW = Date.parse("2026-10-31T10:00:00.000Z");
|
|
163
|
+
const mk = (lastAtIso: string) => ({
|
|
164
|
+
since: "2026-10-31T00:00:00.000Z",
|
|
165
|
+
total: 5,
|
|
166
|
+
by_node: { ghost: 5 },
|
|
167
|
+
last_at: { ghost: lastAtIso },
|
|
168
|
+
});
|
|
169
|
+
|
|
170
|
+
test("最近一次是 1 分钟前 ⇒ 仍然拒(形态是新鲜的)", () => {
|
|
171
|
+
const r = neverConsumeGate({ alias: "ghost", counters: mk(new Date(NOW - 60_000).toISOString()), force: false, forceAllowed: true, now: NOW });
|
|
172
|
+
expect(r.ok).toBe(false);
|
|
173
|
+
});
|
|
174
|
+
|
|
175
|
+
test("🔴 最近一次超过 TTL ⇒ 放行(节点可能已恢复;计数仍留在读数里)", () => {
|
|
176
|
+
const r = neverConsumeGate({ alias: "ghost", counters: mk(new Date(NOW - NEVER_CONSUME_STALE_MS - 1).toISOString()), force: false, forceAllowed: true, now: NOW });
|
|
177
|
+
expect(r.ok).toBe(true);
|
|
178
|
+
});
|
|
179
|
+
|
|
180
|
+
test("读数里带 last_at —— 光有计数看不出「多久没犯了」", () => {
|
|
181
|
+
resetNeverConsumedExpiryCountersForTest(new Date(NOW));
|
|
182
|
+
noteNeverConsumedExpiry("ghost", NOW);
|
|
183
|
+
const c = neverConsumedExpiryCounters();
|
|
184
|
+
expect(c.last_at["ghost"]).toBe(new Date(NOW).toISOString());
|
|
185
|
+
expect(c.by_node["ghost"]).toBe(1);
|
|
186
|
+
});
|
|
187
|
+
});
|