@sleep2agi/commhub-server 0.9.0-preview.22 → 0.9.0-preview.221
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +129 -9
- package/bin/commhub.ts +93 -9
- package/package.json +7 -7
- package/src/ack-create-request-transport.test.ts +182 -0
- package/src/ack-create-request.test.ts +50 -7
- package/src/admin-delete-network-http.test.ts +228 -0
- package/src/admin-networks-http.test.ts +88 -0
- package/src/adoption-candidates-http.test.ts +137 -0
- package/src/adoption-candidates.ts +158 -0
- package/src/agent-access-batch.test.ts +210 -0
- package/src/agent-access.ts +487 -0
- package/src/agent-acl-groups-http.test.ts +247 -0
- package/src/agent-acl-http.test.ts +659 -0
- package/src/agent-acl-slice3-http.test.ts +654 -0
- package/src/agent-department-http.test.ts +215 -0
- package/src/agent-groups-perf.test.ts +223 -0
- package/src/agent-notice.ts +48 -0
- package/src/agent-team-whoami-http.test.ts +98 -0
- package/src/agent-teams-http.test.ts +330 -0
- package/src/agent-teams.ts +244 -0
- package/src/alias-baseline-2571.test.ts +162 -0
- package/src/alias-baseline.ts +113 -0
- package/src/alias-filter.test.ts +72 -0
- package/src/alias-filter.ts +42 -0
- package/src/api-host-supervisors-can-create-nodes.test.ts +350 -0
- package/src/api-host-supervisors-fallback.test.ts +6 -7
- package/src/api-host-supervisors-runtime-readiness.test.ts +195 -0
- package/src/api-nodes-shape.test.ts +120 -13
- package/src/auth-password-change-cli-tokens-http.test.ts +423 -0
- package/src/auth-sessions-http.test.ts +250 -0
- package/src/auth-validate.test.ts +5 -1
- package/src/auth-weak-password-login.test.ts +176 -0
- package/src/auth.ts +609 -51
- package/src/avatar-validate.test.ts +52 -2
- package/src/avatar-validate.ts +35 -1
- package/src/backfill-node-ids-2664.test.ts +128 -0
- package/src/binding-generation-http.test.ts +121 -0
- package/src/blocked-exit-on-reply.test.ts +122 -0
- package/src/broadcast-fanout-2721.test.ts +149 -0
- package/src/broadcast-fanout.ts +117 -0
- package/src/chain-attempt-dedup.test.ts +447 -0
- package/src/clone-node.test.ts +248 -0
- package/src/clone-node.ts +289 -0
- package/src/codex-config-flags.ts +77 -0
- package/src/codex-fork-contract.ts +23 -0
- package/src/codex-fork-migration.test.ts +38 -0
- package/src/commhub-bin-args.test.ts +85 -0
- package/src/config-apply-validate.test.ts +188 -1
- package/src/config-apply-validate.ts +222 -3
- package/src/cors-origins.test.ts +56 -0
- package/src/create-node-blocked-gate.test.ts +356 -0
- package/src/create-node-network-id.test.ts +91 -0
- package/src/create-node-tool-schema.test.ts +49 -0
- package/src/create-node-unicode-name.test.ts +120 -0
- package/src/create-node-validate.test.ts +199 -7
- package/src/create-node-validate.ts +232 -15
- package/src/create-node-workdir.test.ts +289 -0
- package/src/create-node.ts +88 -23
- package/src/daemon-provider-snapshots.test.ts +421 -0
- package/src/daemon-provider-snapshots.ts +327 -0
- package/src/daemon-token-owner-http.test.ts +236 -0
- package/src/dashboard-slash-routing-http.test.ts +109 -0
- package/src/db-adapter-guard.test.ts +148 -0
- package/src/db-adapter-translate.test.ts +128 -0
- package/src/db-adapter.ts +377 -91
- package/src/db-timestamp.test.ts +38 -0
- package/src/db-timestamp.ts +15 -0
- package/src/db.ts +1296 -153
- package/src/delivered-without-push.test.ts +140 -0
- package/src/department-groups-http.test.ts +209 -0
- package/src/department-groups-sync-http.test.ts +328 -0
- package/src/department-groups.ts +434 -0
- package/src/department-heads-http.test.ts +350 -0
- package/src/department-heads.ts +226 -0
- package/src/departments-agent-count.test.ts +132 -0
- package/src/departments-http.test.ts +185 -0
- package/src/departments-mcp.test.ts +128 -0
- package/src/departments.ts +318 -0
- package/src/desktop-message-seam.test.ts +150 -0
- package/src/desktop-message.test.ts +368 -0
- package/src/dm-file-scope-http.test.ts +178 -0
- package/src/empty-node-sweeper.test.ts +104 -0
- package/src/empty-node-sweeper.ts +88 -0
- package/src/env-transport-gate-http.test.ts +240 -0
- package/src/env-transport-gate.ts +80 -0
- package/src/env-transport-policy.ts +56 -0
- package/src/etag-of.test.ts +57 -0
- package/src/event-loop-lag.test.ts +180 -0
- package/src/event-loop-lag.ts +201 -0
- package/src/ext-token-shared.test.ts +60 -0
- package/src/ext-validation-regression.test.ts +131 -0
- package/src/external-schedule-edits-http.test.ts +321 -0
- package/src/external-schedule-edits-timestamps.test.ts +64 -0
- package/src/external-schedule-edits.ts +327 -0
- package/src/file-download-authz-dev-open.test.ts +135 -0
- package/src/file-download-authz.test.ts +404 -0
- package/src/file-download-date-bucket.test.ts +436 -0
- package/src/file-network-scope-dev-open.test.ts +97 -0
- package/src/file-network-scope.test.ts +732 -0
- package/src/group-membership-events-http.test.ts +264 -0
- package/src/group-messages-http.test.ts +482 -0
- package/src/group-messages.ts +297 -0
- package/src/health-anon-readers.ts +59 -0
- package/src/health-host-disk-2756.test.ts +71 -0
- package/src/health-redaction.test.ts +1583 -2
- package/src/host-supervisor-identity-http.test.ts +152 -0
- package/src/host-supervisor-identity.ts +92 -0
- package/src/host-supervisors-daemon-version.test.ts +137 -0
- package/src/http-gzip.test.ts +94 -0
- package/src/http-gzip.ts +112 -0
- package/src/hub-catalog-read-http.test.ts +266 -0
- package/src/hub-provider-config.test.ts +156 -0
- package/src/hub-provider-config.ts +124 -0
- package/src/hub-timestamp.test.ts +38 -0
- package/src/hub-timestamp.ts +27 -0
- package/src/human-dm.ts +215 -0
- package/src/human-inbox.ts +42 -0
- package/src/inbox-caller.ts +49 -0
- package/src/inbox-count-sse.test.ts +310 -0
- package/src/inbox-count.ts +29 -0
- package/src/index.ts +6 -2
- package/src/license-state.test.ts +56 -0
- package/src/license-state.ts +48 -0
- package/src/list-host-supervisors.test.ts +15 -4
- package/src/mcp-context-cost-http.test.ts +127 -0
- package/src/mcp-inflight.test.ts +92 -0
- package/src/mcp-inflight.ts +38 -0
- package/src/mcp-projects-events-http.test.ts +165 -0
- package/src/mcp-route-label-http.test.ts +120 -0
- package/src/mcp-write-network-resolution.test.ts +435 -0
- package/src/member-audit-network-id-http.test.ts +93 -0
- package/src/member-presence-http.test.ts +210 -0
- package/src/messages-ack-by-agent-http.test.ts +142 -0
- package/src/migrate-sqlite-to-pg.test.ts +47 -0
- package/src/migrate-sqlite-to-pg.ts +263 -0
- package/src/model-auth-notify-http.test.ts +181 -0
- package/src/model-auth-notify.ts +116 -0
- package/src/network-name-http.test.ts +75 -0
- package/src/network-rules-http.test.ts +169 -0
- package/src/network-rules.ts +129 -0
- package/src/network-scope.ts +276 -0
- package/src/network-token-membership-http.test.ts +139 -0
- package/src/node-alias-control.ts +12 -0
- package/src/node-attrs-validate.ts +88 -0
- package/src/node-attrs.test.ts +258 -0
- package/src/node-caller-identity-http.test.ts +1091 -0
- package/src/node-create-request-status-http.test.ts +105 -0
- package/src/node-daemon-bindings.test.ts +132 -0
- package/src/node-daemon-bindings.ts +135 -0
- package/src/node-failure-alert.test.ts +150 -0
- package/src/node-failure-alert.ts +177 -0
- package/src/node-files-transport.test.ts +344 -0
- package/src/node-health-codex-login.test.ts +120 -0
- package/src/node-health-dispatch-http.test.ts +335 -0
- package/src/node-health-guard.ts +121 -0
- package/src/node-health-passthrough.test.ts +115 -0
- package/src/node-health-store.ts +101 -0
- package/src/node-id-alias.test.ts +190 -0
- package/src/node-identity-conflict-http.test.ts +269 -0
- package/src/node-identity-conflict.ts +142 -0
- package/src/node-identity-mismatch.test.ts +74 -0
- package/src/node-in-flight-tasks.ts +31 -0
- package/src/node-lifecycle-controllable.test.ts +64 -0
- package/src/node-lifecycle-controllable.ts +53 -0
- package/src/node-lifecycle-read-http.test.ts +250 -0
- package/src/node-lifecycle-read.ts +124 -0
- package/src/node-logs-transport.test.ts +263 -0
- package/src/node-peer-allowlist.test.ts +114 -0
- package/src/node-permissions-http.test.ts +728 -0
- package/src/node-permissions.ts +494 -0
- package/src/node-request-retention.test.ts +281 -0
- package/src/node-request-retention.ts +120 -0
- package/src/node-secret-rest-http.test.ts +190 -0
- package/src/node-serviceability.test.ts +86 -0
- package/src/node-serviceability.ts +124 -0
- package/src/node-session-control-http.test.ts +310 -0
- package/src/node-session-control.ts +258 -0
- package/src/node-skills-transport.test.ts +212 -0
- package/src/node-token-identity.ts +35 -0
- package/src/node-token-ownership.ts +51 -0
- package/src/node-token-privilege-http.test.ts +158 -0
- package/src/not-found-api-http.test.ts +67 -0
- package/src/observer-avatar-http.test.ts +102 -9
- package/src/observer-push.test.ts +76 -0
- package/src/opencode-catalog-report-http.test.ts +139 -0
- package/src/opencode-v2-catalog.test.ts +268 -0
- package/src/opencode-v2-catalog.ts +356 -0
- package/src/package-version-consistency.test.ts +25 -0
- package/src/parent-source-marker.test.ts +50 -0
- package/src/password-network-tokens.ts +25 -0
- package/src/peer-reply-atomic.test.ts +457 -0
- package/src/peer-reply-capability.test.ts +70 -0
- package/src/pg-worker.ts +87 -0
- package/src/poll-pacing.test.ts +74 -0
- package/src/poll-pacing.ts +123 -0
- package/src/project-dir-warn-clear-http.test.ts +132 -0
- package/src/project-viewer-can-http.test.ts +116 -0
- package/src/push.test.ts +20 -10
- package/src/push.ts +382 -9
- package/src/redact-tokens.test.ts +86 -0
- package/src/redact-tokens.ts +43 -0
- package/src/rejection-counters.test.ts +72 -0
- package/src/rejection-counters.ts +173 -0
- package/src/rename.ts +38 -10
- package/src/reply-target-derivation.test.ts +96 -0
- package/src/report-status-handover-capabilities-http.test.ts +124 -0
- package/src/report-status-host-schema.test.ts +235 -0
- package/src/report-status-resume-id-handover.test.ts +184 -0
- package/src/report-status-secret-keys.test.ts +140 -0
- package/src/require-explicit-test-db.ts +9 -0
- package/src/requirement-due-reminders-http.test.ts +620 -0
- package/src/requirement-due-reminders.ts +416 -0
- package/src/requirement-events-http.test.ts +295 -0
- package/src/requirement-events.test.ts +45 -0
- package/src/requirement-events.ts +144 -0
- package/src/requirement-notify.ts +165 -0
- package/src/requirement-participant-notify-http.test.ts +309 -0
- package/src/requirement-tag-ops-http.test.ts +425 -0
- package/src/requirement-tag-tree-http.test.ts +275 -0
- package/src/requirement-tag-tree.ts +127 -0
- package/src/requirement-tags.ts +102 -0
- package/src/requirements-abandoned-http.test.ts +236 -0
- package/src/requirements-comment-http.test.ts +167 -0
- package/src/requirements-completed-at-migration.test.ts +70 -0
- package/src/requirements-errors-mcp-http.test.ts +164 -0
- package/src/requirements-errors.ts +158 -0
- package/src/requirements-etag-http.test.ts +91 -0
- package/src/requirements-http.test.ts +709 -0
- package/src/requirements-last-event-http.test.ts +349 -0
- package/src/requirements-list-mcp-slim-http.test.ts +155 -0
- package/src/requirements-list-slim-http.test.ts +324 -0
- package/src/requirements-mcp.test.ts +163 -0
- package/src/requirements-migrate.ts +283 -0
- package/src/requirements-owner-strict-mcp-http.test.ts +145 -0
- package/src/requirements-people-http.test.ts +114 -0
- package/src/requirements-people-mcp-http.test.ts +231 -0
- package/src/requirements-people.ts +196 -0
- package/src/requirements-priority-lowest.test.ts +114 -0
- package/src/requirements-projects-network-echo.test.ts +57 -0
- package/src/requirements-projects-parent-http.test.ts +123 -0
- package/src/requirements-search-http.test.ts +182 -0
- package/src/requirements-search.test.ts +32 -0
- package/src/requirements-search.ts +145 -0
- package/src/requirements-seq-http.test.ts +193 -0
- package/src/requirements-seq-migration.test.ts +63 -0
- package/src/requirements-start-date.test.ts +125 -0
- package/src/requirements-stats-http.test.ts +357 -0
- package/src/requirements-stats.ts +195 -0
- package/src/requirements-status-participants-http.test.ts +222 -0
- package/src/requirements-upgrade-e2e.test.ts +126 -0
- package/src/requirements-upgrade-migration.test.ts +98 -0
- package/src/requirements.ts +2057 -0
- package/src/resolve-port.test.ts +57 -0
- package/src/resolve-port.ts +44 -0
- package/src/rest-alias-network-fallback-http.test.ts +123 -0
- package/src/rest-alias-network-fallback.test.ts +60 -0
- package/src/rest-alias-network-fallback.ts +65 -0
- package/src/rest-explicit-columns-http.test.ts +266 -0
- package/src/rest-identity.test.ts +130 -0
- package/src/rest-identity.ts +141 -0
- package/src/rest-projections.ts +72 -0
- package/src/rest-read-network-real-chain.test.ts +97 -0
- package/src/rest-write-network-resolution.test.ts +133 -0
- package/src/restart-readiness-http.test.ts +121 -0
- package/src/restricted-files.ts +76 -0
- package/src/retention-first-sweep-quiet.test.ts +31 -0
- package/src/retention.test.ts +100 -40
- package/src/retention.ts +423 -104
- package/src/route-callers-http.test.ts +120 -0
- package/src/route-timing.test.ts +218 -0
- package/src/route-timing.ts +262 -0
- package/src/rules-file-duplicate-alias.test.ts +234 -0
- package/src/rules-file-session-target.test.ts +207 -0
- package/src/rules-file-transport.test.ts +237 -0
- package/src/runtime-label.test.ts +22 -0
- package/src/runtime-label.ts +21 -0
- package/src/runtime-readiness.ts +90 -0
- package/src/schedule-agent-mcp-http.test.ts +432 -0
- package/src/schedule-agent.ts +252 -0
- package/src/schedule-starved.test.ts +92 -0
- package/src/schedule-starved.ts +114 -0
- package/src/scheduled-backoff-http.test.ts +193 -0
- package/src/scheduled-backoff.ts +64 -0
- package/src/scheduled-doorbell-nodeless-session.test.ts +134 -0
- package/src/scheduled-failure-backoff-2622.test.ts +145 -0
- package/src/scheduled-failure-code.test.ts +73 -0
- package/src/scheduled-failure-code.ts +105 -0
- package/src/scheduled-failures-http.test.ts +279 -0
- package/src/scheduled-failures.ts +425 -0
- package/src/scheduled-noprogress-backoff-2622.test.ts +163 -0
- package/src/scheduled-offline-backoff-probe.test.ts +56 -0
- package/src/scheduled-offline.test.ts +131 -0
- package/src/scheduled-offline.ts +66 -0
- package/src/scheduled-outcome-guidance-http.test.ts +139 -0
- package/src/scheduled-outcome-guidance.ts +9 -0
- package/src/scheduled-reply-unread-http.test.ts +212 -0
- package/src/scheduled-requirement-wake-http.test.ts +279 -0
- package/src/scheduled-requirement-wake.ts +125 -0
- package/src/scheduled-run-terminal.test.ts +268 -0
- package/src/scheduled-skip-blocking-task.test.ts +120 -0
- package/src/scheduled-skip-path-backoff-2622.test.ts +98 -0
- package/src/scheduled-stuck-http.test.ts +316 -0
- package/src/scheduled-stuck-notice-cap.test.ts +80 -0
- package/src/scheduled-stuck.ts +219 -0
- package/src/scheduled-tasks-http.test.ts +679 -0
- package/src/scheduled-tasks.ts +959 -0
- package/src/scheduled-work-outcome-http.test.ts +154 -0
- package/src/scheduled-work-outcome.ts +50 -0
- package/src/scoped-secrets.ts +141 -0
- package/src/send-dedup-request-id.test.ts +229 -0
- package/src/send-reply-agent-warning.test.ts +202 -0
- package/src/send-reply-alias-optional.test.ts +131 -0
- package/src/send-reply-attachments.test.ts +458 -0
- package/src/send_dedup.test.ts +54 -0
- package/src/send_dedup.ts +52 -6
- package/src/serve-error.test.ts +73 -0
- package/src/serve-error.ts +56 -0
- package/src/server.ts +4945 -338
- package/src/session-task-on-dispatch.ts +14 -0
- package/src/session-token-ownership-http.test.ts +122 -0
- package/src/shared/codex-inventory.ts +41 -0
- package/src/shared/daemon-provider-login.ts +31 -0
- package/src/shared/daemon-provider-snapshot.ts +52 -0
- package/src/shared/external-schedule-contract.ts +60 -0
- package/src/shared/node-name.test.ts +62 -0
- package/src/shared/node-name.ts +161 -0
- package/src/shared/probe-host-allowlist-drift.test.ts +45 -0
- package/src/shared/provider-application.ts +29 -0
- package/src/shared/provider-collection.ts +14 -0
- package/src/shared/provider-initialization.ts +8 -0
- package/src/shared/reserved-env-drift.test.ts +10 -0
- package/src/shared/scoped-secret-contract.ts +5 -0
- package/src/shared/task-content-limit-drift.test.ts +28 -0
- package/src/shared/task-content-limit.ts +25 -0
- package/src/side-thread-command-transport.test.ts +312 -0
- package/src/side-thread-command-transport.ts +318 -0
- package/src/side-thread-http-integration.test.ts +233 -0
- package/src/side-thread-http.ts +504 -0
- package/src/side-thread-production.test.ts +72 -0
- package/src/side-thread-production.ts +118 -0
- package/src/side-thread.test.ts +1067 -0
- package/src/side-thread.ts +2499 -0
- package/src/skillhub-http.test.ts +142 -0
- package/src/sqlite-close-checkpoint.test.ts +78 -0
- package/src/start-node.test.ts +264 -0
- package/src/status-alias-filter-http.test.ts +106 -0
- package/src/status-alias-resolver-http.test.ts +163 -0
- package/src/status-alias-resolver.ts +19 -0
- package/src/status-node-id-filter-http.test.ts +161 -0
- package/src/status-read-cache-http.test.ts +229 -0
- package/src/status-read-cache.ts +129 -0
- package/src/stop-delete-node.test.ts +567 -1
- package/src/task-access-http.test.ts +402 -0
- package/src/task-access.ts +420 -0
- package/src/task-activity.ts +60 -0
- package/src/task-auth-origin-http.test.ts +93 -0
- package/src/task-auth-origin.test.ts +24 -0
- package/src/task-auth-origin.ts +7 -0
- package/src/task-consumption.test.ts +490 -0
- package/src/task-diagnostic-http.test.ts +149 -0
- package/src/task-diagnostic.test.ts +115 -0
- package/src/task-diagnostic.ts +118 -0
- package/src/task-expiry-notice-http.test.ts +287 -0
- package/src/task-expiry-notice.ts +315 -0
- package/src/task-idempotency-mcp.test.ts +142 -0
- package/src/task-idempotency.test.ts +76 -0
- package/src/task-idempotency.ts +81 -0
- package/src/task-late-replies.ts +54 -0
- package/src/task-late-reply-http.test.ts +227 -0
- package/src/task-lifecycle-watcher-busy.test.ts +114 -0
- package/src/task-lifecycle-watcher.test.ts +204 -0
- package/src/task-lifecycle-watcher.ts +109 -0
- package/src/task-network-resolution-http.test.ts +140 -0
- package/src/task-offline-delivery-2668.test.ts +119 -0
- package/src/task-offline-delivery.ts +81 -0
- package/src/task-orphan-http.test.ts +287 -0
- package/src/task-orphan.ts +119 -0
- package/src/task-queue-ahead-http.test.ts +309 -0
- package/src/task-queue-ahead-scale.test.ts +236 -0
- package/src/task-queue-ahead.ts +186 -0
- package/src/task-rejection-log-http.test.ts +160 -0
- package/src/task-rejection-log.ts +58 -0
- package/src/task-sender-http.test.ts +143 -0
- package/src/task-stale-open-http.test.ts +292 -0
- package/src/task-stale-open.ts +103 -0
- package/src/tasks-pagination-http.test.ts +141 -0
- package/src/team-rules-status-http.test.ts +90 -0
- package/src/team-rules-status.test.ts +59 -0
- package/src/team-rules-status.ts +72 -0
- package/src/tmux-allowlist.test.ts +48 -0
- package/src/tool-audience-http.test.ts +230 -0
- package/src/tool-audience.ts +127 -0
- package/src/tools-list-json-schema.test.ts +94 -0
- package/src/tools.ts +4320 -413
- package/src/upload-early-reject-guard.test.ts +155 -0
- package/src/uploads-http.test.ts +86 -7
- package/src/uploads.ts +103 -6
- package/src/user-avatar-http.test.ts +74 -0
- package/src/user-avatar-image.ts +160 -0
- package/src/user-avatar-migration.test.ts +25 -0
- package/src/user-avatar-upload-http.test.ts +348 -0
- package/src/user-inbox-by-agent-http.test.ts +161 -0
- package/src/user-inbox-netid-notnull.test.ts +122 -0
- package/src/user-inbox-read-http.test.ts +223 -0
- package/src/user-inbox-schema.test.ts +54 -0
- package/src/version-stats.test.ts +66 -0
- package/src/version-stats.ts +109 -0
package/src/retention.ts
CHANGED
|
@@ -8,9 +8,13 @@
|
|
|
8
8
|
// so a row delete by itself doesn't shrink the file.
|
|
9
9
|
//
|
|
10
10
|
// This module exposes:
|
|
11
|
-
// - sweepRetention(cfg): one-shot sweep across high-growth tables
|
|
11
|
+
// - sweepRetention(cfg): one-shot sweep across high-growth tables.
|
|
12
|
+
// **Async** since #2614: deletes run in bounded chunks with an event-loop
|
|
13
|
+
// yield between batches, so a big backlog no longer freezes the hub for
|
|
14
|
+
// ~10 seconds once an hour.
|
|
12
15
|
// - startRetentionSweeper(): wires a 1-hour setInterval, fires once
|
|
13
|
-
// at startup with an immediate (non-blocking) sweep
|
|
16
|
+
// at startup with an immediate (non-blocking) sweep, and refuses to
|
|
17
|
+
// start a second pass while one is still running.
|
|
14
18
|
//
|
|
15
19
|
// Defaults are conservative — long-tail debugging traces survive a
|
|
16
20
|
// week. Operators can override per env (COMMHUB_RETENTION_*_DAYS) for
|
|
@@ -20,6 +24,9 @@
|
|
|
20
24
|
// COMMHUB_DB=/tmp/test-retention.db bun test src/retention.test.ts
|
|
21
25
|
|
|
22
26
|
import { db } from "./db.js";
|
|
27
|
+
import { getMcpInflightStats } from "./mcp-inflight.js";
|
|
28
|
+
import { eventLoopStats } from "./event-loop-lag.js";
|
|
29
|
+
import { sweepNodeRequestContent, type NodeRequestSweepResult } from "./node-request-retention.js";
|
|
23
30
|
|
|
24
31
|
export type RetentionConfig = {
|
|
25
32
|
// High-frequency telemetry; default = 7 days.
|
|
@@ -52,6 +59,16 @@ export type SweepResult = {
|
|
|
52
59
|
tasks: number;
|
|
53
60
|
audit_log: number;
|
|
54
61
|
};
|
|
62
|
+
// node_rules_requests: file bytes purged by age / after read, rows deleted
|
|
63
|
+
// after 30 days. Fixed policy (privacy, not disk) — no env opt-out.
|
|
64
|
+
// #2614 —— 分批统计:总共跑了多少批 DELETE,哪张表因为 maxTableMs 被截断。
|
|
65
|
+
batches: number;
|
|
66
|
+
/** 整轮里单批最长耗时(毫秒)。生产上这条数字 = 「一次 sweep 最长阻塞多久」。 */
|
|
67
|
+
maxBatchMs: number;
|
|
68
|
+
/** 各阶段耗时(毫秒):多张表 + vacuum。生产上「15 秒里谁花的」靠它,不靠猜。 */
|
|
69
|
+
phases: Record<string, number>;
|
|
70
|
+
truncated: string[];
|
|
71
|
+
nodeRequests: NodeRequestSweepResult;
|
|
55
72
|
vacuum: {
|
|
56
73
|
walCheckpointPagesMoved: number | null;
|
|
57
74
|
incrementalFreedPages: number | null;
|
|
@@ -87,96 +104,312 @@ export function readRetentionConfig(): RetentionConfig {
|
|
|
87
104
|
};
|
|
88
105
|
}
|
|
89
106
|
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
107
|
+
// #2614 —— 分批扫(chunked sweep)。
|
|
108
|
+
//
|
|
109
|
+
// 事故现场(2026-10-26 生产 `.161`):每小时一次 `swept in 10588ms`,那 10 秒里
|
|
110
|
+
// 事件循环完全静默 —— `/health` 超时、在飞从 387 堆到 807、RSS 冲到 4.5 GB。
|
|
111
|
+
// 根因不是「删得多」,而是**一次 DELETE 把整批行放在一个语句里,中间不让出事件循环**。
|
|
112
|
+
//
|
|
113
|
+
// 修法(与 `.167` 的 terminal_events 回填同形):
|
|
114
|
+
// 1. 每批只删 chunkRows 行,用 `DELETE … WHERE rowid IN (SELECT rowid FROM t WHERE <谓词> LIMIT n)`
|
|
115
|
+
// —— 谓词有索引时这是索引范围扫描(O(n));没有索引时它退化成一次全扫,
|
|
116
|
+
// 所以「谓词必须有索引」是这条路径的前提(见下面的三条实测);
|
|
117
|
+
// 2. 每批之间 `await` 一次 setTimeout(0),把控制权交还事件循环;
|
|
118
|
+
// 3. 删的这段时间关掉 WAL 自动 checkpoint,本轮的 checkpoint 收敛到末尾那一次
|
|
119
|
+
// TRUNCATE(否则 checkpoint 会落进任意一批,把 90ms 的批次顶到 4 秒);
|
|
120
|
+
// 4. 单表最多跑 MAX_TABLE_MS,超了就停下并记日志(下一轮继续)。
|
|
121
|
+
//
|
|
122
|
+
// 三条实测(都在 1.09 GB 生产副本上):
|
|
123
|
+
// · 一次删完 3.1s / 单块 2686ms
|
|
124
|
+
// · 分批 + rowid 游标(副本里谓词**没有**索引) 5.3s / 单块 258ms
|
|
125
|
+
// · 分批 + rowid 游标(生产,谓词**有**索引) → 单块 4716ms 🔴 打回
|
|
126
|
+
// `ORDER BY rowid` 让优化器放弃 created_at 索引,为了凑够一批从 rowid=0 扫全表;
|
|
127
|
+
// 只删 266 行花了 4.7 秒(`.168` 上线第一拍的真实日志)。
|
|
128
|
+
// · 分批 + 索引驱动 LIMIT(现状,生产副本) 1.8s / 单块 249ms
|
|
129
|
+
// ⇒ 教训:**副本的 schema 也必须与生产一致**,否则「按副本优化」会变成新的全表扫。
|
|
130
|
+
// 总耗时略增是有意的取舍:**可被打断优先于最快跑完**。
|
|
131
|
+
const DEFAULT_CHUNK_ROWS = 2000;
|
|
132
|
+
/** 每次 sweep 最多回收多少空闲页(≈4 KB/页)。见下面的 incremental_vacuum 注释。 */
|
|
133
|
+
const INCREMENTAL_VACUUM_PAGES = 200;
|
|
134
|
+
const MAX_TABLE_MS = 10 * 60 * 1000;
|
|
135
|
+
|
|
136
|
+
/**
|
|
137
|
+
* #2614(第三轮)—— 首次 sweep 要等**循环真的安静下来**,而不是固定等 N 秒。
|
|
138
|
+
*
|
|
139
|
+
* 2026-10-27 两次部署实测:
|
|
140
|
+
* ```
|
|
141
|
+
* 18:55 部署:60 秒后 sweep → 3894ms / max 1238ms(风暴 2 分钟后仍在)
|
|
142
|
+
* 19:51 部署:180 秒后 sweep → 9153ms / max 3919ms(风暴持续 5 分钟,RSS 3.6 GB)
|
|
143
|
+
* ```
|
|
144
|
+
* 固定延迟只是把「撞上风暴」变成「可能撞上风暴」——风暴长短取决于同时重连多少节点。
|
|
145
|
+
*
|
|
146
|
+
* 判据(都来自 `/health` 的同名指标):
|
|
147
|
+
* · `mcp_inflight < 20`:没有成批请求在排队;
|
|
148
|
+
* · 最近一次 stall 至少在 60 秒前:循环刚刚不抖。
|
|
149
|
+
* 每 15 秒看一次,最多等 15 分钟;等不到就照常跑(宁可慢,不可不删)。
|
|
150
|
+
*/
|
|
151
|
+
export const FIRST_SWEEP_QUIET_INFLIGHT = 20;
|
|
152
|
+
export const FIRST_SWEEP_QUIET_STALL_AGE_MS = 60_000;
|
|
153
|
+
export const FIRST_SWEEP_MAX_WAIT_MS = 15 * 60_000;
|
|
154
|
+
export const FIRST_SWEEP_POLL_MS = 15_000;
|
|
155
|
+
|
|
156
|
+
/** 纯函数,便于单测:把「此刻该不该继续等」与时钟解耦。 */
|
|
157
|
+
export function firstSweepShouldWait(input: {
|
|
158
|
+
now: number;
|
|
159
|
+
inflight: number;
|
|
160
|
+
lastStallAtIso: string | null;
|
|
161
|
+
}): boolean {
|
|
162
|
+
if (input.inflight >= FIRST_SWEEP_QUIET_INFLIGHT) return true;
|
|
163
|
+
if (!input.lastStallAtIso) return false;
|
|
164
|
+
const last = Date.parse(input.lastStallAtIso);
|
|
165
|
+
if (!Number.isFinite(last)) return false;
|
|
166
|
+
return input.now - last < FIRST_SWEEP_QUIET_STALL_AGE_MS;
|
|
167
|
+
}
|
|
168
|
+
|
|
169
|
+
async function waitForQuietLoop(): Promise<void> {
|
|
170
|
+
const startedMs = Date.now();
|
|
171
|
+
while (Date.now() - startedMs < FIRST_SWEEP_MAX_WAIT_MS) {
|
|
172
|
+
const stats = eventLoopStats();
|
|
173
|
+
if (!firstSweepShouldWait({
|
|
174
|
+
now: Date.now(),
|
|
175
|
+
inflight: getMcpInflightStats().inflight,
|
|
176
|
+
lastStallAtIso: stats.last_stall_at,
|
|
177
|
+
})) {
|
|
178
|
+
return;
|
|
179
|
+
}
|
|
180
|
+
await Bun.sleep(FIRST_SWEEP_POLL_MS);
|
|
181
|
+
}
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
/** 启动后第一次 sweep 的延迟;默认 180 秒(见 startRetentionSweeper 里的实测)。 */
|
|
185
|
+
function readFirstSweepDelayMs(): number {
|
|
186
|
+
const raw = process.env.COMMHUB_RETENTION_FIRST_DELAY_MS;
|
|
187
|
+
if (raw === undefined || raw.trim() === "") return 180_000;
|
|
188
|
+
const n = Number(raw);
|
|
189
|
+
return Number.isFinite(n) && n >= 0 ? Math.floor(n) : 180_000;
|
|
190
|
+
}
|
|
191
|
+
|
|
192
|
+
function readChunkRows(): number {
|
|
193
|
+
const raw = process.env.COMMHUB_RETENTION_CHUNK_ROWS;
|
|
194
|
+
const n = Number(raw);
|
|
195
|
+
if (!Number.isFinite(n) || n < 1) return DEFAULT_CHUNK_ROWS;
|
|
196
|
+
// 上限只是防手滑(把「分批」配成一次删完就没有意义了),
|
|
197
|
+
// 真要大吞吐应该调小而不是调大 —— 这条盯的是最长阻塞,不是总时长。
|
|
198
|
+
return Math.min(50_000, Math.floor(n));
|
|
199
|
+
}
|
|
200
|
+
|
|
201
|
+
/** 当前连接的 WAL 自动 checkpoint 页数;非 SQLite 或读不到 → null(不改)。 */
|
|
202
|
+
function readWalAutoCheckpoint(): number | null {
|
|
203
|
+
if (db.dialect !== "sqlite") return null;
|
|
204
|
+
try {
|
|
205
|
+
const row = db.get<{ wal_autocheckpoint: number }>("PRAGMA wal_autocheckpoint");
|
|
206
|
+
return typeof row?.wal_autocheckpoint === "number" ? row.wal_autocheckpoint : null;
|
|
207
|
+
} catch {
|
|
208
|
+
return null;
|
|
209
|
+
}
|
|
210
|
+
}
|
|
211
|
+
|
|
212
|
+
function setWalAutoCheckpoint(pages: number): void {
|
|
213
|
+
if (db.dialect !== "sqlite") return;
|
|
214
|
+
try {
|
|
215
|
+
db.exec(`PRAGMA wal_autocheckpoint = ${Math.max(0, Math.floor(pages))}`);
|
|
216
|
+
} catch (e: any) {
|
|
217
|
+
console.log(`[commhub retention] wal_autocheckpoint=${pages} failed: ${e?.message ?? e}`);
|
|
218
|
+
}
|
|
219
|
+
}
|
|
220
|
+
|
|
221
|
+
function yieldToLoop(): Promise<void> {
|
|
222
|
+
return new Promise<void>((resolve) => setTimeout(resolve, 0));
|
|
223
|
+
}
|
|
224
|
+
|
|
225
|
+
export type TableSweepResult = {
|
|
226
|
+
deleted: number;
|
|
227
|
+
batches: number;
|
|
228
|
+
/** 这张表这轮花了多久(毫秒)—— 生产上排查「15 秒里到底谁花的」靠它。 */
|
|
229
|
+
ms: number;
|
|
230
|
+
/** 单批最长耗时(毫秒)—— 生产上「一次 sweep 最长阻塞多久」就是这个数。 */
|
|
231
|
+
maxBatchMs: number;
|
|
232
|
+
/** 单表跑满 maxTableMs 被截断;剩余行下一轮继续(不吞掉、不假装完成)。 */
|
|
233
|
+
truncated: boolean;
|
|
234
|
+
};
|
|
235
|
+
|
|
236
|
+
async function sweepTable(
|
|
237
|
+
table: string,
|
|
238
|
+
where: string,
|
|
239
|
+
param: string,
|
|
93
240
|
days: number,
|
|
94
|
-
|
|
241
|
+
chunkRows: number,
|
|
242
|
+
): Promise<TableSweepResult> {
|
|
95
243
|
// days < 0 → operator opt-out for this table; skip the DELETE.
|
|
96
244
|
// days === 0 is a legitimate "sweep everything older than now"
|
|
97
245
|
// (effectively a TRUNCATE); we honour it.
|
|
98
|
-
if (days < 0) return 0;
|
|
246
|
+
if (days < 0) return { deleted: 0, batches: 0, maxBatchMs: 0, ms: 0, truncated: false };
|
|
247
|
+
// PostgreSQL: 单条 DELETE + autovacuum 本来就按批回收,且报文里报的 10 秒停顿
|
|
248
|
+
// 出在 SQLite 侧(`#2614` 的生产现场是 SQLite)。这里不引入 rowid(pg 没有),
|
|
249
|
+
// 保持原样,避免在没观测到问题的方言上改语义。
|
|
250
|
+
if (db.dialect !== "sqlite") {
|
|
251
|
+
try {
|
|
252
|
+
const res = db.run(`DELETE FROM ${table} WHERE ${where}`, [param]);
|
|
253
|
+
return { deleted: res.changes ?? 0, batches: 1, maxBatchMs: 0, ms: 0, truncated: false };
|
|
254
|
+
} catch (e: any) {
|
|
255
|
+
console.log(`[commhub retention] sweep failed (${table}): ${e?.message ?? e}`);
|
|
256
|
+
return { deleted: 0, batches: 0, maxBatchMs: 0, ms: 0, truncated: false };
|
|
257
|
+
}
|
|
258
|
+
}
|
|
259
|
+
const startedMs = Date.now();
|
|
260
|
+
const tTable0 = performance.now();
|
|
261
|
+
let deleted = 0;
|
|
262
|
+
let batches = 0;
|
|
263
|
+
let maxBatchMs = 0;
|
|
99
264
|
try {
|
|
100
|
-
|
|
101
|
-
|
|
265
|
+
for (;;) {
|
|
266
|
+
const batchT0 = performance.now();
|
|
267
|
+
// 用**单调前进的 rowid 游标**取一批候选行,再按 rowid 删除。
|
|
268
|
+
//
|
|
269
|
+
// 为什么不是 `DELETE ... WHERE rowid IN (SELECT rowid FROM t WHERE <pred> LIMIT n)`:
|
|
270
|
+
// `created_at < ?` 没有覆盖全表的索引,每批的子查询都退化成一次覆盖索引扫描
|
|
271
|
+
// (EXPLAIN: `SCAN agent_telemetry USING COVERING INDEX idx_agent_telemetry_ip_time`)。
|
|
272
|
+
// 于是「每批都从表头扫一遍」,总代价 = 批数 × 表大小 —— 在 1.09 GB 生产副本上
|
|
273
|
+
// 实测:68k 行积压从一次删完的 2.7 秒劣化到 17 秒,单批仍能到 4.6 秒。
|
|
274
|
+
// 加上 `rowid > 游标 ORDER BY rowid LIMIT n` 后计划变成
|
|
275
|
+
// `SEARCH ... USING INTEGER PRIMARY KEY (rowid>?)`:每批只从游标往后读,
|
|
276
|
+
// 整轮总代价 ≈ 一次全表扫描 + 删除本身。
|
|
277
|
+
const rows = db.all<{ r: number }>(
|
|
278
|
+
`SELECT rowid AS r FROM ${table} WHERE ${where} LIMIT ?2`,
|
|
279
|
+
param,
|
|
280
|
+
chunkRows,
|
|
281
|
+
);
|
|
282
|
+
if (rows.length === 0) return { deleted, batches, maxBatchMs, ms: Math.round(performance.now() - tTable0), truncated: false };
|
|
283
|
+
const ids = rows.map((r) => r.r);
|
|
284
|
+
const placeholders = ids.map((_, i) => `?${i + 1}`).join(",");
|
|
285
|
+
const res = db.run(`DELETE FROM ${table} WHERE rowid IN (${placeholders})`, ids);
|
|
286
|
+
deleted += res.changes ?? 0;
|
|
287
|
+
batches += 1;
|
|
288
|
+
maxBatchMs = Math.max(maxBatchMs, performance.now() - batchT0);
|
|
289
|
+
// 这一批没取满 ⇒ 已经没有更多符合条件的行了,收工。
|
|
290
|
+
if (rows.length < chunkRows) return { deleted, batches, maxBatchMs, ms: Math.round(performance.now() - tTable0), truncated: false };
|
|
291
|
+
if (Date.now() - startedMs > MAX_TABLE_MS) {
|
|
292
|
+
console.log(
|
|
293
|
+
`[commhub retention] ${table}: stopped after ${batches} batches / ${deleted} rows ` +
|
|
294
|
+
`(>${Math.round(MAX_TABLE_MS / 60000)}min); remaining rows continue next sweep`,
|
|
295
|
+
);
|
|
296
|
+
return { deleted, batches, maxBatchMs, ms: Math.round(performance.now() - tTable0), truncated: true };
|
|
297
|
+
}
|
|
298
|
+
// 让出事件循环:这是 #2614 的全部要点 —— 停顿的可感知单位是「单批」。
|
|
299
|
+
await yieldToLoop();
|
|
300
|
+
}
|
|
102
301
|
} catch (e: any) {
|
|
103
|
-
console.log(`[commhub retention] sweep failed: ${e?.message ?? e}`);
|
|
104
|
-
return
|
|
302
|
+
console.log(`[commhub retention] sweep failed (${table}): ${e?.message ?? e}`);
|
|
303
|
+
return { deleted, batches, maxBatchMs, ms: Math.round(performance.now() - tTable0), truncated: false };
|
|
105
304
|
}
|
|
106
305
|
}
|
|
107
306
|
|
|
108
|
-
export function sweepRetention(cfg: RetentionConfig = readRetentionConfig()): SweepResult {
|
|
307
|
+
export async function sweepRetention(cfg: RetentionConfig = readRetentionConfig()): Promise<SweepResult> {
|
|
109
308
|
const startedAt = new Date().toISOString();
|
|
110
309
|
const t0 = Date.now();
|
|
310
|
+
const chunkRows = readChunkRows();
|
|
111
311
|
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
[`-${cfg.telemetryDays} days`],
|
|
115
|
-
cfg.telemetryDays,
|
|
116
|
-
);
|
|
117
|
-
const taskEvents = sweepOne(
|
|
118
|
-
`DELETE FROM task_events WHERE created_at < datetime('now', ?1)`,
|
|
119
|
-
[`-${cfg.taskEventsDays} days`],
|
|
120
|
-
cfg.taskEventsDays,
|
|
121
|
-
);
|
|
122
|
-
// Only sweep ACKED inbox rows — unacked may be in-flight delivery
|
|
123
|
-
// for an agent that's offline but coming back. Acked rows have
|
|
124
|
-
// served their purpose and exist only for debugging.
|
|
125
|
-
const inbox = sweepOne(
|
|
126
|
-
`DELETE FROM inbox WHERE acked = 1 AND created_at < datetime('now', ?1)`,
|
|
127
|
-
[`-${cfg.ackedInboxDays} days`],
|
|
128
|
-
cfg.ackedInboxDays,
|
|
129
|
-
);
|
|
130
|
-
// Terminal-task sweep — split into two DELETEs:
|
|
131
|
-
//
|
|
132
|
-
// (a) "safe terminals": cancelled / failed / expired / (theoretical)
|
|
133
|
-
// completed have no chain-ancestor risk and are safe to reap
|
|
134
|
-
// unconditionally past the horizon. NOTE: as of this writing
|
|
135
|
-
// 'completed' is dead code — no code path sets it (every reply
|
|
136
|
-
// uses 'replied'/'failed'/'cancelled' via send_reply or
|
|
137
|
-
// chainReplyToParent). Kept in the IN-list as a forward guard:
|
|
138
|
-
// if someone introduces a 'completed' write later, the sweep
|
|
139
|
-
// picks it up without code changes.
|
|
140
|
-
//
|
|
141
|
-
// (b) "replied" — was excluded entirely in the original PR, which
|
|
142
|
-
// missed the goal: every fulfilled task lives at 'replied'
|
|
143
|
-
// status, so the tasks table's MAIN GROWTH SOURCE would never
|
|
144
|
-
// be reclaimed. Reviewer caught this. Fix: child-ref-aware
|
|
145
|
-
// delete — reap an old replied row ONLY IF no other task
|
|
146
|
-
// references it as parent_task_id. A row with no children is
|
|
147
|
-
// not a chain ancestor and is safe to drop.
|
|
312
|
+
// #2614 —— 分批删期间关掉 WAL 自动 checkpoint,把 checkpoint 收敛到本轮末尾那一次
|
|
313
|
+
// 显式 wal_checkpoint(TRUNCATE)(本来就有的)。
|
|
148
314
|
//
|
|
149
|
-
//
|
|
150
|
-
//
|
|
151
|
-
//
|
|
152
|
-
//
|
|
153
|
-
//
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
315
|
+
// 为什么:自动 checkpoint 是**连接级、每 1000 页触发**的,它会落进任意一批里。
|
|
316
|
+
// 在 1.09 GB 的生产副本上实测(68k 行积压、批大小 2000):
|
|
317
|
+
// 自动 checkpoint 开 → 批次中位数 301ms,**最慢一批 4365ms**(分批白做)
|
|
318
|
+
// 自动 checkpoint 关 → 批次中位数 89ms,最慢一批 242ms
|
|
319
|
+
// 也就是说「分批」必须配「自己决定何时 checkpoint,而不是让 SQLite 在你删到一半时决定」。
|
|
320
|
+
const autoCheckpointBefore = readWalAutoCheckpoint();
|
|
321
|
+
if (autoCheckpointBefore !== null) setWalAutoCheckpoint(0);
|
|
322
|
+
let telemetry: TableSweepResult;
|
|
323
|
+
let taskEvents: TableSweepResult;
|
|
324
|
+
let inbox: TableSweepResult;
|
|
325
|
+
let tasksSafeTerminal: TableSweepResult;
|
|
326
|
+
let tasksRepliedChildSafe: TableSweepResult;
|
|
327
|
+
let auditLog: TableSweepResult;
|
|
328
|
+
try {
|
|
329
|
+
|
|
330
|
+
telemetry = await sweepTable(
|
|
331
|
+
"agent_telemetry",
|
|
332
|
+
`created_at < datetime('now', ?1)`,
|
|
333
|
+
`-${cfg.telemetryDays} days`,
|
|
334
|
+
cfg.telemetryDays,
|
|
335
|
+
chunkRows,
|
|
336
|
+
);
|
|
337
|
+
taskEvents = await sweepTable(
|
|
338
|
+
"task_events",
|
|
339
|
+
`created_at < datetime('now', ?1)`,
|
|
340
|
+
`-${cfg.taskEventsDays} days`,
|
|
341
|
+
cfg.taskEventsDays,
|
|
342
|
+
chunkRows,
|
|
343
|
+
);
|
|
344
|
+
// Only sweep ACKED inbox rows — unacked may be in-flight delivery
|
|
345
|
+
// for an agent that's offline but coming back. Acked rows have
|
|
346
|
+
// served their purpose and exist only for debugging.
|
|
347
|
+
inbox = await sweepTable(
|
|
348
|
+
"inbox",
|
|
349
|
+
`acked = 1 AND created_at < datetime('now', ?1)`,
|
|
350
|
+
`-${cfg.ackedInboxDays} days`,
|
|
351
|
+
cfg.ackedInboxDays,
|
|
352
|
+
chunkRows,
|
|
353
|
+
);
|
|
354
|
+
// Terminal-task sweep — split into two DELETEs:
|
|
355
|
+
//
|
|
356
|
+
// (a) "safe terminals": cancelled / failed / expired / (theoretical)
|
|
357
|
+
// completed have no chain-ancestor risk and are safe to reap
|
|
358
|
+
// unconditionally past the horizon. NOTE: as of this writing
|
|
359
|
+
// 'completed' is dead code — no code path sets it (every reply
|
|
360
|
+
// uses 'replied'/'failed'/'cancelled' via send_reply or
|
|
361
|
+
// chainReplyToParent). Kept in the IN-list as a forward guard:
|
|
362
|
+
// if someone introduces a 'completed' write later, the sweep
|
|
363
|
+
// picks it up without code changes.
|
|
364
|
+
//
|
|
365
|
+
// (b) "replied" — was excluded entirely in the original PR, which
|
|
366
|
+
// missed the goal: every fulfilled task lives at 'replied'
|
|
367
|
+
// status, so the tasks table's MAIN GROWTH SOURCE would never
|
|
368
|
+
// be reclaimed. Reviewer caught this. Fix: child-ref-aware
|
|
369
|
+
// delete — reap an old replied row ONLY IF no other task
|
|
370
|
+
// references it as parent_task_id. A row with no children is
|
|
371
|
+
// not a chain ancestor and is safe to drop.
|
|
372
|
+
//
|
|
373
|
+
// **Time field** (both DELETEs): use COALESCE(completed_at, created_at).
|
|
374
|
+
// Terminal status means the server set `completed_at` at the
|
|
375
|
+
// reply/fail/cancel moment. Sweeping on `created_at` alone would
|
|
376
|
+
// reap a task created 60d ago but completed today the instant it
|
|
377
|
+
// entered terminal state — zero post-terminal retention. The
|
|
378
|
+
// COALESCE falls back to created_at for legacy rows that pre-date
|
|
379
|
+
// the completed_at column (back-compat).
|
|
380
|
+
tasksSafeTerminal = await sweepTable(
|
|
381
|
+
"tasks",
|
|
382
|
+
`status IN ('completed', 'cancelled', 'failed', 'expired')
|
|
383
|
+
AND COALESCE(completed_at, created_at) < datetime('now', ?1)`,
|
|
384
|
+
`-${cfg.terminalTasksDays} days`,
|
|
385
|
+
cfg.terminalTasksDays,
|
|
386
|
+
chunkRows,
|
|
387
|
+
);
|
|
388
|
+
tasksRepliedChildSafe = await sweepTable(
|
|
389
|
+
"tasks",
|
|
390
|
+
`status = 'replied'
|
|
391
|
+
AND COALESCE(completed_at, created_at) < datetime('now', ?1)
|
|
392
|
+
AND NOT EXISTS (
|
|
393
|
+
SELECT 1 FROM tasks AS child
|
|
394
|
+
WHERE child.parent_task_id = tasks.task_id
|
|
395
|
+
)`,
|
|
396
|
+
`-${cfg.terminalTasksDays} days`,
|
|
397
|
+
cfg.terminalTasksDays,
|
|
398
|
+
chunkRows,
|
|
399
|
+
);
|
|
400
|
+
auditLog = await sweepTable(
|
|
401
|
+
"audit_log",
|
|
402
|
+
`created_at < datetime('now', ?1)`,
|
|
403
|
+
`-${cfg.auditLogDays} days`,
|
|
404
|
+
cfg.auditLogDays,
|
|
405
|
+
chunkRows,
|
|
406
|
+
);
|
|
407
|
+
} finally {
|
|
408
|
+
if (autoCheckpointBefore !== null) setWalAutoCheckpoint(autoCheckpointBefore);
|
|
409
|
+
}
|
|
410
|
+
const tasksDeleted = tasksSafeTerminal.deleted + tasksRepliedChildSafe.deleted;
|
|
411
|
+
|
|
412
|
+
const nodeRequests = sweepNodeRequestContent();
|
|
180
413
|
|
|
181
414
|
// VACUUM:
|
|
182
415
|
// - wal_checkpoint(TRUNCATE): trims the WAL file. Works on every
|
|
@@ -191,27 +424,50 @@ export function sweepRetention(cfg: RetentionConfig = readRetentionConfig()): Sw
|
|
|
191
424
|
// DBs must run a one-time blocking `VACUUM` themselves (out of
|
|
192
425
|
// scope for the sweeper, which deliberately stays non-
|
|
193
426
|
// blocking).
|
|
427
|
+
// Both are SQLite PRAGMAs; on PostgreSQL autovacuum does this job and
|
|
428
|
+
// the two counters stay null (RFC-039 S3).
|
|
194
429
|
let walCheckpointPagesMoved: number | null = null;
|
|
195
430
|
let incrementalFreedPages: number | null = null;
|
|
196
431
|
let errored = false;
|
|
197
|
-
|
|
432
|
+
const sqliteVacuum = db.dialect === "sqlite";
|
|
433
|
+
if (sqliteVacuum) {
|
|
198
434
|
// wal_checkpoint(TRUNCATE) returns a row { busy, log, checkpointed }.
|
|
199
435
|
// We expose `log` (page count moved into the main DB) for ops visibility.
|
|
200
|
-
|
|
201
|
-
|
|
202
|
-
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
436
|
+
//
|
|
437
|
+
// #2614 —— 重试 3 次:本轮刚跑完一批 `db.all` 时,同连接上可能还有未重置的
|
|
438
|
+
// 读语句,`wal_checkpoint` 会直接报 `database table is locked`(SQLITE_LOCKED,
|
|
439
|
+
// 不是 BUSY)。这是**瞬时**的:让出一次事件循环再试就好。别把它记成 errored
|
|
440
|
+
// —— 那会让「本轮 vacuum 是否出错」这个字段说谎。
|
|
441
|
+
let lastErr: any = null;
|
|
442
|
+
for (let attempt = 0; attempt < 3; attempt++) {
|
|
443
|
+
try {
|
|
444
|
+
const checkpoint = db.get<{ busy: number; log: number; checkpointed: number }>(
|
|
445
|
+
"PRAGMA wal_checkpoint(TRUNCATE)",
|
|
446
|
+
);
|
|
447
|
+
walCheckpointPagesMoved = checkpoint?.log ?? null;
|
|
448
|
+
lastErr = null;
|
|
449
|
+
break;
|
|
450
|
+
} catch (e: any) {
|
|
451
|
+
lastErr = e;
|
|
452
|
+
await yieldToLoop();
|
|
453
|
+
}
|
|
454
|
+
}
|
|
455
|
+
if (lastErr) {
|
|
456
|
+
console.log(`[commhub retention] wal_checkpoint failed after 3 attempts: ${lastErr?.message ?? lastErr}`);
|
|
457
|
+
errored = true;
|
|
458
|
+
}
|
|
207
459
|
}
|
|
208
|
-
try {
|
|
460
|
+
if (sqliteVacuum) try {
|
|
209
461
|
// incremental_vacuum is a no-op on databases not created with
|
|
210
462
|
// PRAGMA auto_vacuum = INCREMENTAL. For existing prod DBs that
|
|
211
463
|
// weren't, the operator would have to run a full one-time VACUUM
|
|
212
464
|
// to migrate — out of scope for the sweeper.
|
|
213
465
|
const freelistBefore = db.get<{ freelist_count: number }>("PRAGMA freelist_count");
|
|
214
|
-
|
|
466
|
+
// #2614 —— 必须带页数上限。`PRAGMA incremental_vacuum`(不带参数)会**一次回收
|
|
467
|
+
// 全部空闲页**:生产 `.169` 上线时那一拍整轮 15.3s,其中大部分是这个语句
|
|
468
|
+
// (它走 db.exec,不进 slow-sql 日志,所以第一眼看不见)。
|
|
469
|
+
// 每小时回 200 页(≈800 KB)足够跟上增长,又不会变成一次长阻塞;剩下的下次继续。
|
|
470
|
+
db.exec(`PRAGMA incremental_vacuum(${INCREMENTAL_VACUUM_PAGES})`);
|
|
215
471
|
const freelistAfter = db.get<{ freelist_count: number }>("PRAGMA freelist_count");
|
|
216
472
|
incrementalFreedPages =
|
|
217
473
|
(freelistBefore?.freelist_count ?? 0) - (freelistAfter?.freelist_count ?? 0);
|
|
@@ -224,12 +480,37 @@ export function sweepRetention(cfg: RetentionConfig = readRetentionConfig()): Sw
|
|
|
224
480
|
startedAt,
|
|
225
481
|
durationMs: Date.now() - t0,
|
|
226
482
|
deletes: {
|
|
227
|
-
agent_telemetry: telemetry,
|
|
228
|
-
task_events: taskEvents,
|
|
229
|
-
inbox,
|
|
230
|
-
tasks,
|
|
231
|
-
audit_log: auditLog,
|
|
483
|
+
agent_telemetry: telemetry.deleted,
|
|
484
|
+
task_events: taskEvents.deleted,
|
|
485
|
+
inbox: inbox.deleted,
|
|
486
|
+
tasks: tasksDeleted,
|
|
487
|
+
audit_log: auditLog.deleted,
|
|
488
|
+
},
|
|
489
|
+
// #2614 —— 分批可见性:批次数与「被 maxTableMs 截断的表」
|
|
490
|
+
batches:
|
|
491
|
+
telemetry.batches + taskEvents.batches + inbox.batches +
|
|
492
|
+
tasksSafeTerminal.batches + tasksRepliedChildSafe.batches + auditLog.batches,
|
|
493
|
+
// 生产上「一次 sweep 最长阻塞多久」= 这个数字(#2614 的验收看它)
|
|
494
|
+
phases: {
|
|
495
|
+
telemetry: telemetry.ms,
|
|
496
|
+
events: taskEvents.ms,
|
|
497
|
+
inbox: inbox.ms,
|
|
498
|
+
tasks: tasksSafeTerminal.ms + tasksRepliedChildSafe.ms,
|
|
499
|
+
audit: auditLog.ms,
|
|
232
500
|
},
|
|
501
|
+
maxBatchMs: Math.round(Math.max(
|
|
502
|
+
telemetry.maxBatchMs, taskEvents.maxBatchMs, inbox.maxBatchMs,
|
|
503
|
+
tasksSafeTerminal.maxBatchMs, tasksRepliedChildSafe.maxBatchMs, auditLog.maxBatchMs,
|
|
504
|
+
)),
|
|
505
|
+
truncated: [
|
|
506
|
+
["agent_telemetry", telemetry],
|
|
507
|
+
["task_events", taskEvents],
|
|
508
|
+
["inbox", inbox],
|
|
509
|
+
["tasks(safe)", tasksSafeTerminal],
|
|
510
|
+
["tasks(replied)", tasksRepliedChildSafe],
|
|
511
|
+
["audit_log", auditLog],
|
|
512
|
+
].filter(([, r]) => (r as TableSweepResult).truncated).map(([n]) => n as string),
|
|
513
|
+
nodeRequests,
|
|
233
514
|
vacuum: { walCheckpointPagesMoved, incrementalFreedPages, errored },
|
|
234
515
|
};
|
|
235
516
|
}
|
|
@@ -239,31 +520,69 @@ export function sweepRetention(cfg: RetentionConfig = readRetentionConfig()): Sw
|
|
|
239
520
|
export function startRetentionSweeper(
|
|
240
521
|
intervalMs = 60 * 60 * 1000, // 1 hour
|
|
241
522
|
cfg?: RetentionConfig,
|
|
523
|
+
firstDelayMs = readFirstSweepDelayMs(),
|
|
242
524
|
): ReturnType<typeof setInterval> {
|
|
243
|
-
|
|
525
|
+
// 防重入:一轮 sweep 现在会跨多个 tick(每批之间让出事件循环)。若上一轮还没
|
|
526
|
+
// 跑完就再来一轮,两轮会同时删同一批行 —— 白做功,还会把写锁争用放大。
|
|
527
|
+
let running = false;
|
|
528
|
+
const sweep = async () => {
|
|
529
|
+
if (running) {
|
|
530
|
+
console.log("[commhub retention] previous sweep still running — skipping this tick");
|
|
531
|
+
return;
|
|
532
|
+
}
|
|
533
|
+
running = true;
|
|
244
534
|
try {
|
|
245
|
-
const r = sweepRetention(cfg);
|
|
535
|
+
const r = await sweepRetention(cfg);
|
|
246
536
|
const totalDeletes =
|
|
247
537
|
r.deletes.agent_telemetry +
|
|
248
538
|
r.deletes.task_events +
|
|
249
539
|
r.deletes.inbox +
|
|
250
540
|
r.deletes.tasks +
|
|
251
541
|
r.deletes.audit_log;
|
|
252
|
-
|
|
542
|
+
const nr = r.nodeRequests;
|
|
543
|
+
if (totalDeletes > 0 || nr.purged > 0 || nr.deleted > 0 || nr.timedOut > 0 || r.vacuum.errored) {
|
|
253
544
|
console.log(
|
|
254
|
-
`[commhub retention] swept in ${r.durationMs}ms — ` +
|
|
545
|
+
`[commhub retention] swept in ${r.durationMs}ms (${r.batches} batches, max ${r.maxBatchMs}ms) — ` +
|
|
546
|
+
`phases ${Object.entries(r.phases).map(([k, v]) => `${k}=${v}ms`).join(" ")} · ` +
|
|
255
547
|
`telemetry=${r.deletes.agent_telemetry} events=${r.deletes.task_events} ` +
|
|
256
548
|
`inbox=${r.deletes.inbox} tasks=${r.deletes.tasks} audit=${r.deletes.audit_log} ` +
|
|
257
|
-
`
|
|
549
|
+
`nodeReqPurged=${nr.purged} nodeReqDeleted=${nr.deleted} nodeReqTimedOut=${nr.timedOut} ` +
|
|
550
|
+
`walPages=${r.vacuum.walCheckpointPagesMoved} freedPages=${r.vacuum.incrementalFreedPages}` +
|
|
551
|
+
(r.truncated.length ? ` truncated=${r.truncated.join(",")}` : ""),
|
|
258
552
|
);
|
|
259
553
|
}
|
|
260
554
|
} catch (e: any) {
|
|
261
555
|
console.log(`[commhub retention] sweep top-level failed: ${e?.message ?? e}`);
|
|
556
|
+
} finally {
|
|
557
|
+
running = false;
|
|
262
558
|
}
|
|
263
559
|
};
|
|
264
|
-
// Fire once
|
|
265
|
-
//
|
|
266
|
-
//
|
|
267
|
-
|
|
268
|
-
|
|
560
|
+
// Fire once after startup so a long-running hub eventually catches up.
|
|
561
|
+
//
|
|
562
|
+
// #2614(第二轮)—— 不再 setImmediate。2026-10-27 18:11 的启动实测:
|
|
563
|
+
// ```
|
|
564
|
+
// [commhub stall] loop idle ~14085ms (served 614) ← 重连风暴 + 启动工作
|
|
565
|
+
// [commhub retention] swept in 15693ms (max 957ms) — telemetry=3320 行/13928ms
|
|
566
|
+
// ```
|
|
567
|
+
// 同一批删除在稳态是 0.24ms/行,在启动那一分钟是 4.2ms/行(慢 17 倍)——
|
|
568
|
+
// 不是删除变慢了,是它在和重连风暴抢 CPU/WAL。而这一分钟的阻塞最容易被误记成
|
|
569
|
+
// “retention 又卡了”。默认延后 180 秒(可用 COMMHUB_RETENTION_FIRST_DELAY_MS 覆盖,0=立即)。
|
|
570
|
+
//
|
|
571
|
+
// 为什么是 180 而不是 60(.173 上线后实测修正):
|
|
572
|
+
// ```
|
|
573
|
+
// 18:55:24 boot
|
|
574
|
+
// 18:55:38 stall 20.6s (served 812) ← 重连风暴
|
|
575
|
+
// 18:56:24 retention(+60s) swept 3894ms, max 1238ms, telemetry=2964 行/2935ms = 0.99ms/行
|
|
576
|
+
// 18:56:26 stall 2.7s · 18:57:28 stall 3.4s · 18:57:31 stall 2.6s ← 风暴 2 分钟还没排空
|
|
577
|
+
// ```
|
|
578
|
+
// 稳态同一批删除是 0.24ms/行 ⇒ 60 秒时风暴仍在,max 单批被推到 1.2 秒(越过 1 秒慢 SQL 阈值)。
|
|
579
|
+
const firstSweep = async () => {
|
|
580
|
+
// 先等固定延迟越过重连风暴的**起点**,再等循环真的安静(见 waitForQuietLoop)。
|
|
581
|
+
if (firstDelayMs > 0) await Bun.sleep(firstDelayMs);
|
|
582
|
+
await waitForQuietLoop();
|
|
583
|
+
await sweep();
|
|
584
|
+
};
|
|
585
|
+
if (firstDelayMs > 0) setTimeout(() => { void firstSweep(); }, 0).unref?.();
|
|
586
|
+
else setImmediate(() => { void firstSweep(); });
|
|
587
|
+
return setInterval(() => { void sweep(); }, intervalMs);
|
|
269
588
|
}
|