@sleep2agi/commhub-server 0.9.0-preview.27 → 0.9.0-preview.271
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +120 -3
- package/bin/commhub.ts +93 -9
- package/build-info.json +7 -0
- package/package.json +10 -8
- package/src/ack-create-request-transport.test.ts +182 -0
- package/src/ack-create-request.test.ts +52 -8
- package/src/admin-delete-network-http.test.ts +229 -0
- package/src/admin-networks-http.test.ts +88 -0
- package/src/adoption-candidates-http.test.ts +140 -0
- package/src/adoption-candidates.ts +158 -0
- package/src/agent-access-batch.test.ts +210 -0
- package/src/agent-access.ts +585 -0
- package/src/agent-acl-groups-http.test.ts +247 -0
- package/src/agent-acl-http.test.ts +691 -0
- package/src/agent-acl-slice3-http.test.ts +885 -0
- package/src/agent-department-grants.test.ts +129 -0
- package/src/agent-department-http.test.ts +215 -0
- package/src/agent-groups-perf.test.ts +223 -0
- package/src/agent-notice.ts +48 -0
- package/src/agent-team-whoami-http.test.ts +98 -0
- package/src/agent-teams-http.test.ts +330 -0
- package/src/agent-teams.ts +244 -0
- package/src/alias-baseline-2571.test.ts +162 -0
- package/src/alias-baseline.ts +113 -0
- package/src/alias-filter.test.ts +72 -0
- package/src/alias-filter.ts +42 -0
- package/src/api-host-supervisors-can-create-nodes.test.ts +350 -0
- package/src/api-host-supervisors-fallback.test.ts +6 -7
- package/src/api-host-supervisors-runtime-readiness.test.ts +195 -0
- package/src/api-nodes-shape.test.ts +131 -4
- package/src/auth-abuse-counters.test.ts +144 -0
- package/src/auth-abuse-counters.ts +122 -0
- package/src/auth-layer-shape.test.ts +105 -0
- package/src/auth-password-change-cli-tokens-http.test.ts +423 -0
- package/src/auth-sessions-http.test.ts +260 -0
- package/src/auth-validate.test.ts +5 -1
- package/src/auth-weak-password-login.test.ts +176 -0
- package/src/auth.ts +641 -68
- package/src/backfill-node-ids-2664.test.ts +128 -0
- package/src/binding-generation-http.test.ts +141 -0
- package/src/blocked-exit-on-reply.test.ts +122 -0
- package/src/broadcast-fanout-2721.test.ts +149 -0
- package/src/broadcast-fanout.ts +117 -0
- package/src/build-commit-report-http.test.ts +128 -0
- package/src/build-info.test.ts +60 -0
- package/src/build-info.ts +85 -0
- package/src/chain-attempt-dedup.test.ts +447 -0
- package/src/clip.test.ts +36 -0
- package/src/clip.ts +48 -0
- package/src/clone-node.test.ts +269 -0
- package/src/clone-node.ts +289 -0
- package/src/codex-config-flags.ts +77 -0
- package/src/codex-fork-contract.ts +23 -0
- package/src/codex-fork-migration.test.ts +38 -0
- package/src/commhub-bin-args.test.ts +85 -0
- package/src/config-apply-validate.test.ts +188 -1
- package/src/config-apply-validate.ts +222 -3
- package/src/cors-origins.test.ts +73 -0
- package/src/create-node-blocked-gate.test.ts +365 -0
- package/src/create-node-error-domain.test.ts +127 -0
- package/src/create-node-network-id.test.ts +91 -0
- package/src/create-node-tool-schema.test.ts +49 -0
- package/src/create-node-unicode-name.test.ts +126 -0
- package/src/create-node-validate.test.ts +199 -7
- package/src/create-node-validate.ts +232 -15
- package/src/create-node-workdir.test.ts +298 -0
- package/src/create-node.ts +88 -23
- package/src/daemon-provider-snapshots.test.ts +453 -0
- package/src/daemon-provider-snapshots.ts +327 -0
- package/src/daemon-token-owner-http.test.ts +346 -0
- package/src/dashboard-slash-routing-http.test.ts +109 -0
- package/src/db-adapter-guard.test.ts +148 -0
- package/src/db-adapter-translate.test.ts +128 -0
- package/src/db-adapter.ts +377 -91
- package/src/db-timestamp.test.ts +38 -0
- package/src/db-timestamp.ts +15 -0
- package/src/db.ts +1298 -153
- package/src/delivered-without-push.test.ts +140 -0
- package/src/department-groups-http.test.ts +209 -0
- package/src/department-groups-sync-http.test.ts +328 -0
- package/src/department-groups.ts +434 -0
- package/src/department-heads-http.test.ts +350 -0
- package/src/department-heads.ts +226 -0
- package/src/departments-agent-count.test.ts +139 -0
- package/src/departments-http.test.ts +185 -0
- package/src/departments-mcp-error-domain.test.ts +135 -0
- package/src/departments-mcp.test.ts +128 -0
- package/src/departments.ts +318 -0
- package/src/desktop-message-seam.test.ts +150 -0
- package/src/desktop-message.test.ts +368 -0
- package/src/dm-file-scope-http.test.ts +227 -0
- package/src/empty-node-sweeper.test.ts +104 -0
- package/src/empty-node-sweeper.ts +88 -0
- package/src/env-transport-gate-http.test.ts +240 -0
- package/src/env-transport-gate.ts +80 -0
- package/src/env-transport-policy.ts +56 -0
- package/src/etag-of.test.ts +57 -0
- package/src/event-loop-lag.test.ts +180 -0
- package/src/event-loop-lag.ts +201 -0
- package/src/expiry-counters.test.ts +187 -0
- package/src/expiry-counters.ts +227 -0
- package/src/expiry-evidence-db.test.ts +103 -0
- package/src/ext-token-shared.test.ts +60 -0
- package/src/ext-validation-regression.test.ts +10 -17
- package/src/external-schedule-edits-http.test.ts +321 -0
- package/src/external-schedule-edits-timestamps.test.ts +64 -0
- package/src/external-schedule-edits.ts +327 -0
- package/src/file-download-authz-dev-open.test.ts +1 -1
- package/src/file-download-authz.test.ts +25 -4
- package/src/file-download-date-bucket.test.ts +1 -1
- package/src/file-network-scope-dev-open.test.ts +1 -1
- package/src/file-network-scope.test.ts +7 -6
- package/src/group-membership-events-http.test.ts +264 -0
- package/src/group-messages-http.test.ts +483 -0
- package/src/group-messages.ts +297 -0
- package/src/health-anon-readers.ts +67 -0
- package/src/health-host-disk-2756.test.ts +71 -0
- package/src/health-identity-root.test.ts +98 -0
- package/src/health-redaction.test.ts +1599 -7
- package/src/health-status-reasons.test.ts +84 -0
- package/src/host-supervisor-identity-http.test.ts +152 -0
- package/src/host-supervisor-identity.ts +92 -0
- package/src/host-supervisors-daemon-version.test.ts +137 -0
- package/src/http-gzip.test.ts +94 -0
- package/src/http-gzip.ts +112 -0
- package/src/hub-catalog-read-http.test.ts +664 -0
- package/src/hub-provider-config.test.ts +166 -0
- package/src/hub-provider-config.ts +202 -0
- package/src/hub-timestamp.test.ts +38 -0
- package/src/hub-timestamp.ts +27 -0
- package/src/human-dm.ts +215 -0
- package/src/human-inbox.ts +42 -0
- package/src/inbox-caller.ts +49 -0
- package/src/inbox-count-sse.test.ts +318 -0
- package/src/inbox-count.ts +29 -0
- package/src/index.ts +6 -2
- package/src/license-state.test.ts +105 -0
- package/src/license-state.ts +48 -0
- package/src/lifecycle-error-domain.test.ts +149 -0
- package/src/list-ack-error-domain.test.ts +108 -0
- package/src/list-host-supervisors.test.ts +15 -4
- package/src/list-tasks-omit-default.test.ts +84 -0
- package/src/list-tasks-since-last.test.ts +100 -0
- package/src/list-tasks-since.test.ts +104 -0
- package/src/mcp-context-cost-http.test.ts +127 -0
- package/src/mcp-inflight.test.ts +92 -0
- package/src/mcp-inflight.ts +38 -0
- package/src/mcp-projects-events-http.test.ts +168 -0
- package/src/mcp-route-label-http.test.ts +120 -0
- package/src/mcp-write-network-resolution.test.ts +21 -0
- package/src/member-audit-network-id-http.test.ts +93 -0
- package/src/member-presence-http.test.ts +210 -0
- package/src/messages-ack-by-agent-http.test.ts +142 -0
- package/src/migrate-sqlite-to-pg.test.ts +47 -0
- package/src/migrate-sqlite-to-pg.ts +263 -0
- package/src/model-auth-notify-http.test.ts +181 -0
- package/src/model-auth-notify.ts +116 -0
- package/src/network-name-http.test.ts +75 -0
- package/src/network-quota-config.test.ts +100 -0
- package/src/network-rules-http.test.ts +169 -0
- package/src/network-rules.ts +129 -0
- package/src/network-scope.ts +222 -3
- package/src/network-token-membership-http.test.ts +186 -0
- package/src/node-alias-control.ts +12 -0
- package/src/node-attrs.test.ts +47 -2
- package/src/node-caller-identity-http.test.ts +1108 -0
- package/src/node-config-last-update.test.ts +126 -0
- package/src/node-config-last-update.ts +107 -0
- package/src/node-create-request-status-http.test.ts +105 -0
- package/src/node-daemon-bindings.test.ts +142 -0
- package/src/node-daemon-bindings.ts +135 -0
- package/src/node-failure-alert.test.ts +150 -0
- package/src/node-failure-alert.ts +177 -0
- package/src/node-files-transport.test.ts +350 -0
- package/src/node-health-codex-login.test.ts +120 -0
- package/src/node-health-dispatch-http.test.ts +373 -0
- package/src/node-health-guard-origin.test.ts +98 -0
- package/src/node-health-guard.ts +121 -0
- package/src/node-health-passthrough.test.ts +115 -0
- package/src/node-health-store.ts +101 -0
- package/src/node-id-alias.test.ts +190 -0
- package/src/node-identity-conflict-http.test.ts +269 -0
- package/src/node-identity-conflict.ts +142 -0
- package/src/node-identity-mismatch.test.ts +74 -0
- package/src/node-in-flight-tasks.ts +31 -0
- package/src/node-lifecycle-controllable.test.ts +64 -0
- package/src/node-lifecycle-controllable.ts +53 -0
- package/src/node-lifecycle-read-http.test.ts +257 -0
- package/src/node-lifecycle-read.ts +124 -0
- package/src/node-logs-transport.test.ts +266 -0
- package/src/node-peer-allowlist.test.ts +114 -0
- package/src/node-permissions-http.test.ts +901 -0
- package/src/node-permissions.ts +496 -0
- package/src/node-request-retention.test.ts +281 -0
- package/src/node-request-retention.ts +120 -0
- package/src/node-secret-rest-http.test.ts +190 -0
- package/src/node-secret-vault-inject.test.ts +132 -0
- package/src/node-security-write-gate.test.ts +72 -0
- package/src/node-serviceability.test.ts +86 -0
- package/src/node-serviceability.ts +124 -0
- package/src/node-session-control-http.test.ts +310 -0
- package/src/node-session-control.ts +258 -0
- package/src/node-skills-transport.test.ts +212 -0
- package/src/node-token-identity.ts +35 -0
- package/src/node-token-ownership.ts +51 -0
- package/src/node-token-privilege-http.test.ts +224 -0
- package/src/node-token-usage.test.ts +90 -0
- package/src/node-token-usage.ts +164 -0
- package/src/not-found-api-http.test.ts +67 -0
- package/src/observer-avatar-http.test.ts +103 -10
- package/src/observer-push.test.ts +76 -0
- package/src/opencode-catalog-report-http.test.ts +139 -0
- package/src/opencode-v2-catalog.test.ts +268 -0
- package/src/opencode-v2-catalog.ts +356 -0
- package/src/package-version-consistency.test.ts +25 -0
- package/src/parent-infer.test.ts +82 -0
- package/src/parent-infer.ts +110 -0
- package/src/parent-source-marker.test.ts +70 -0
- package/src/parent-source-orphan.test.ts +121 -0
- package/src/password-network-tokens.ts +25 -0
- package/src/peer-reply-atomic.test.ts +462 -0
- package/src/peer-reply-capability.test.ts +70 -0
- package/src/pg-worker.ts +87 -0
- package/src/poll-pacing.test.ts +74 -0
- package/src/poll-pacing.ts +123 -0
- package/src/probe-provider-error-domain.test.ts +145 -0
- package/src/project-dir-warn-clear-http.test.ts +132 -0
- package/src/project-viewer-can-http.test.ts +116 -0
- package/src/provider-skill-error-domain.test.ts +130 -0
- package/src/provider-tools-shape.test.ts +165 -0
- package/src/push.test.ts +20 -10
- package/src/push.ts +373 -9
- package/src/redact-tokens.test.ts +86 -0
- package/src/redact-tokens.ts +43 -0
- package/src/rejection-counters.test.ts +72 -0
- package/src/rejection-counters.ts +173 -0
- package/src/rename.ts +38 -10
- package/src/reply-target-derivation.test.ts +96 -0
- package/src/report-status-handover-capabilities-http.test.ts +124 -0
- package/src/report-status-host-schema.test.ts +235 -0
- package/src/report-status-resume-id-handover.test.ts +184 -0
- package/src/report-status-secret-keys.test.ts +140 -0
- package/src/require-explicit-test-db.ts +9 -0
- package/src/requirement-due-reminders-http.test.ts +620 -0
- package/src/requirement-due-reminders.ts +416 -0
- package/src/requirement-events-http.test.ts +318 -0
- package/src/requirement-events.test.ts +45 -0
- package/src/requirement-events.ts +144 -0
- package/src/requirement-notify.ts +165 -0
- package/src/requirement-participant-notify-http.test.ts +313 -0
- package/src/requirement-tag-ops-http.test.ts +454 -0
- package/src/requirement-tag-tree-http.test.ts +275 -0
- package/src/requirement-tag-tree.ts +127 -0
- package/src/requirement-tags.ts +102 -0
- package/src/requirements-abandoned-http.test.ts +236 -0
- package/src/requirements-assignee-drift.test.ts +154 -0
- package/src/requirements-comment-http.test.ts +174 -0
- package/src/requirements-completed-at-migration.test.ts +70 -0
- package/src/requirements-errors-mcp-http.test.ts +164 -0
- package/src/requirements-errors.ts +158 -0
- package/src/requirements-etag-http.test.ts +91 -0
- package/src/requirements-http.test.ts +709 -0
- package/src/requirements-last-event-http.test.ts +349 -0
- package/src/requirements-list-mcp-slim-http.test.ts +155 -0
- package/src/requirements-list-slim-http.test.ts +324 -0
- package/src/requirements-mcp.test.ts +180 -0
- package/src/requirements-migrate.ts +283 -0
- package/src/requirements-owner-strict-mcp-http.test.ts +145 -0
- package/src/requirements-people-http.test.ts +114 -0
- package/src/requirements-people-mcp-http.test.ts +231 -0
- package/src/requirements-people.ts +196 -0
- package/src/requirements-priority-lowest.test.ts +114 -0
- package/src/requirements-projects-network-echo.test.ts +57 -0
- package/src/requirements-projects-parent-http.test.ts +123 -0
- package/src/requirements-search-http.test.ts +182 -0
- package/src/requirements-search.test.ts +32 -0
- package/src/requirements-search.ts +145 -0
- package/src/requirements-seq-http.test.ts +193 -0
- package/src/requirements-seq-migration.test.ts +63 -0
- package/src/requirements-start-date.test.ts +125 -0
- package/src/requirements-stats-http.test.ts +357 -0
- package/src/requirements-stats.ts +195 -0
- package/src/requirements-status-participants-http.test.ts +222 -0
- package/src/requirements-upgrade-e2e.test.ts +126 -0
- package/src/requirements-upgrade-migration.test.ts +98 -0
- package/src/requirements.ts +2205 -0
- package/src/resolve-port.test.ts +57 -0
- package/src/resolve-port.ts +44 -0
- package/src/rest-alias-network-fallback-http.test.ts +123 -0
- package/src/rest-alias-network-fallback.test.ts +60 -0
- package/src/rest-alias-network-fallback.ts +65 -0
- package/src/rest-explicit-columns-http.test.ts +266 -0
- package/src/rest-identity.test.ts +130 -0
- package/src/rest-identity.ts +141 -0
- package/src/rest-projections.ts +72 -0
- package/src/rest-read-network-real-chain.test.ts +97 -0
- package/src/rest-write-network-resolution.test.ts +133 -0
- package/src/restart-readiness-http.test.ts +121 -0
- package/src/restricted-files.ts +76 -0
- package/src/retention-first-sweep-quiet.test.ts +31 -0
- package/src/retention.test.ts +100 -40
- package/src/retention.ts +423 -104
- package/src/route-callers-http.test.ts +120 -0
- package/src/route-timing.test.ts +218 -0
- package/src/route-timing.ts +262 -0
- package/src/rules-file-duplicate-alias.test.ts +234 -0
- package/src/rules-file-session-target.test.ts +207 -0
- package/src/rules-file-transport.test.ts +271 -0
- package/src/runtime-label.test.ts +22 -0
- package/src/runtime-label.ts +21 -0
- package/src/runtime-readiness.ts +90 -0
- package/src/schedule-agent-mcp-http.test.ts +486 -0
- package/src/schedule-agent.ts +252 -0
- package/src/schedule-starved.test.ts +92 -0
- package/src/schedule-starved.ts +114 -0
- package/src/scheduled-backoff-http.test.ts +203 -0
- package/src/scheduled-backoff.ts +64 -0
- package/src/scheduled-doorbell-nodeless-session.test.ts +134 -0
- package/src/scheduled-failure-backoff-2622.test.ts +145 -0
- package/src/scheduled-failure-code.test.ts +73 -0
- package/src/scheduled-failure-code.ts +105 -0
- package/src/scheduled-failures-http.test.ts +279 -0
- package/src/scheduled-failures.ts +425 -0
- package/src/scheduled-noprogress-backoff-2622.test.ts +163 -0
- package/src/scheduled-offline-backoff-probe.test.ts +56 -0
- package/src/scheduled-offline.test.ts +131 -0
- package/src/scheduled-offline.ts +66 -0
- package/src/scheduled-outcome-guidance-http.test.ts +139 -0
- package/src/scheduled-outcome-guidance.ts +9 -0
- package/src/scheduled-reply-unread-http.test.ts +212 -0
- package/src/scheduled-requirement-wake-http.test.ts +289 -0
- package/src/scheduled-requirement-wake.ts +125 -0
- package/src/scheduled-run-terminal.test.ts +290 -0
- package/src/scheduled-skip-blocking-task.test.ts +120 -0
- package/src/scheduled-skip-path-backoff-2622.test.ts +98 -0
- package/src/scheduled-stuck-http.test.ts +316 -0
- package/src/scheduled-stuck-notice-cap.test.ts +80 -0
- package/src/scheduled-stuck.ts +219 -0
- package/src/scheduled-tasks-http.test.ts +679 -0
- package/src/scheduled-tasks.ts +959 -0
- package/src/scheduled-work-outcome-http.test.ts +154 -0
- package/src/scheduled-work-outcome.ts +50 -0
- package/src/scoped-secrets.ts +167 -0
- package/src/secret-echo-negative-control.test.ts +118 -0
- package/src/send-dedup-request-id.test.ts +229 -0
- package/src/send-reply-agent-warning.test.ts +38 -38
- package/src/send-reply-alias-optional.test.ts +131 -0
- package/src/send-reply-attachments.test.ts +28 -7
- package/src/send_dedup.test.ts +54 -0
- package/src/send_dedup.ts +52 -6
- package/src/serve-error.test.ts +73 -0
- package/src/serve-error.ts +56 -0
- package/src/server.ts +4963 -377
- package/src/session-task-on-dispatch.ts +14 -0
- package/src/session-token-ownership-http.test.ts +122 -0
- package/src/shared/codex-inventory.ts +41 -0
- package/src/shared/daemon-provider-login.ts +31 -0
- package/src/shared/daemon-provider-snapshot.ts +52 -0
- package/src/shared/external-schedule-contract.ts +60 -0
- package/src/shared/node-name.test.ts +62 -0
- package/src/shared/node-name.ts +161 -0
- package/src/shared/probe-host-allowlist-drift.test.ts +45 -0
- package/src/shared/provider-application.ts +29 -0
- package/src/shared/provider-collection.ts +14 -0
- package/src/shared/provider-initialization.ts +8 -0
- package/src/shared/reserved-env-drift.test.ts +10 -0
- package/src/shared/scoped-secret-contract.ts +5 -0
- package/src/shared/task-content-limit-drift.test.ts +28 -0
- package/src/shared/task-content-limit.ts +25 -0
- package/src/side-thread-command-transport.test.ts +312 -0
- package/src/side-thread-command-transport.ts +318 -0
- package/src/side-thread-http-integration.test.ts +233 -0
- package/src/side-thread-http.ts +504 -0
- package/src/side-thread-production.test.ts +72 -0
- package/src/side-thread-production.ts +118 -0
- package/src/side-thread.test.ts +1067 -0
- package/src/side-thread.ts +2499 -0
- package/src/skillhub-http.test.ts +144 -0
- package/src/sqlite-close-checkpoint.test.ts +78 -0
- package/src/start-node.test.ts +265 -0
- package/src/start-request-error-domain.test.ts +182 -0
- package/src/start-skill-error-domain.test.ts +122 -0
- package/src/stats-versions-http.test.ts +65 -0
- package/src/status-alias-filter-http.test.ts +106 -0
- package/src/status-alias-resolver-http.test.ts +163 -0
- package/src/status-alias-resolver.ts +19 -0
- package/src/status-node-id-filter-http.test.ts +161 -0
- package/src/status-read-cache-http.test.ts +229 -0
- package/src/status-read-cache.ts +129 -0
- package/src/stop-delete-node.test.ts +567 -1
- package/src/task-access-http.test.ts +407 -0
- package/src/task-access.ts +420 -0
- package/src/task-activity.ts +60 -0
- package/src/task-auth-origin-http.test.ts +93 -0
- package/src/task-auth-origin.test.ts +24 -0
- package/src/task-auth-origin.ts +7 -0
- package/src/task-consumption.test.ts +508 -0
- package/src/task-diagnostic-http.test.ts +149 -0
- package/src/task-diagnostic.test.ts +115 -0
- package/src/task-diagnostic.ts +118 -0
- package/src/task-expiry-notice-http.test.ts +298 -0
- package/src/task-expiry-notice.ts +318 -0
- package/src/task-idempotency-mcp.test.ts +148 -0
- package/src/task-idempotency.test.ts +76 -0
- package/src/task-idempotency.ts +81 -0
- package/src/task-late-replies.ts +54 -0
- package/src/task-late-reply-http.test.ts +227 -0
- package/src/task-lifecycle-watcher-busy.test.ts +114 -0
- package/src/task-lifecycle-watcher.test.ts +209 -0
- package/src/task-lifecycle-watcher.ts +109 -0
- package/src/task-network-resolution-http.test.ts +140 -0
- package/src/task-not-found-error-domain.test.ts +114 -0
- package/src/task-offline-delivery-2668.test.ts +119 -0
- package/src/task-offline-delivery.ts +81 -0
- package/src/task-orphan-http.test.ts +287 -0
- package/src/task-orphan.ts +119 -0
- package/src/task-queue-ahead-http.test.ts +309 -0
- package/src/task-queue-ahead-scale.test.ts +236 -0
- package/src/task-queue-ahead.ts +186 -0
- package/src/task-rejection-log-http.test.ts +160 -0
- package/src/task-rejection-log.ts +61 -0
- package/src/task-sender-http.test.ts +143 -0
- package/src/task-stale-open-http.test.ts +292 -0
- package/src/task-stale-open.ts +103 -0
- package/src/tasks-pagination-http.test.ts +141 -0
- package/src/team-rules-status-http.test.ts +90 -0
- package/src/team-rules-status.test.ts +59 -0
- package/src/team-rules-status.ts +72 -0
- package/src/tmux-allowlist.test.ts +48 -0
- package/src/tool-audience-http.test.ts +248 -0
- package/src/tool-audience.ts +130 -0
- package/src/tools-list-json-schema.test.ts +94 -0
- package/src/tools.ts +4340 -386
- package/src/update-node-config-error-domain.test.ts +107 -0
- package/src/uploads-http.test.ts +6 -4
- package/src/uploads.ts +23 -4
- package/src/user-avatar-http.test.ts +74 -0
- package/src/user-avatar-image.ts +160 -0
- package/src/user-avatar-migration.test.ts +25 -0
- package/src/user-avatar-upload-http.test.ts +353 -0
- package/src/user-inbox-by-agent-http.test.ts +189 -0
- package/src/user-inbox-netid-notnull.test.ts +122 -0
- package/src/user-inbox-read-http.test.ts +223 -0
- package/src/user-inbox-schema.test.ts +54 -0
- package/src/version-stats.test.ts +66 -0
- package/src/version-stats.ts +109 -0
|
@@ -0,0 +1,425 @@
|
|
|
1
|
+
// #523 —— 定时任务连续失败:告诉创建者(可选自动暂停)。
|
|
2
|
+
//
|
|
3
|
+
// 现场:一个每 2 分钟一次的排程,每一次派出去的任务都被节点以 failed 结束(上游模型拒绝),一天 720 次,
|
|
4
|
+
// 排程照样 active,创建者什么都不知道。#464 只管「被上一次卡住」(skipped),不管「每次都失败」。
|
|
5
|
+
//
|
|
6
|
+
// 1. 计数:consecutive_failures = 最近一次成功(run.status='replied')之后,status='failed' 的 run 数。
|
|
7
|
+
// skipped / cancelled / expired / 还没结束的 run 不算失败,也不打断连续(expired 由 #464 的超时告警负责)。
|
|
8
|
+
// 失败有两种来源,都会走到这里:派发时就失败(节点不在 / 被停用 / 降级 / 创建者无权限,
|
|
9
|
+
// scheduled-tasks.ts 提交后调用),以及派出去的任务以 failed 结束(db.ts 的 run 镜像触发钩子)。
|
|
10
|
+
// 2. 告警:连续失败 ≥ N(默认 5)→ 给排程创建者发**一条**通知(agent-notice.ts,与 #462/#464 同一条路,
|
|
11
|
+
// 出现在与目标节点的会话里)。去重落在 scheduled_tasks 两列上(条件写,一次只有一个调用方抢得到):
|
|
12
|
+
// failure_alert_key = 告警那一刻「最近一次成功」的 run_id —— 之后又成功过,key 就变了,下一段失败重新上膛;
|
|
13
|
+
// failure_alert_at = 告警时间 —— 一直没成功过,24 小时后再提醒一次。Hub 重启不会重发。
|
|
14
|
+
// 3. 自动暂停:默认关。COMMHUB_SCHEDULE_FAILURE_AUTO_PAUSE_RUNS=M(≥1)时,连续失败 ≥ M 的 active 排程
|
|
15
|
+
// 被置为 paused(next_run_at 清空,revision+1,与用户手动暂停同形),并通知创建者。
|
|
16
|
+
//
|
|
17
|
+
// 环境变量:COMMHUB_SCHEDULE_FAILURE_NOTICE_RUNS(N,默认 5)、COMMHUB_SCHEDULE_FAILURE_RENOTICE_SEC
|
|
18
|
+
// (默认 86400)、COMMHUB_SCHEDULE_FAILURE_AUTO_PAUSE_RUNS(M,默认 0 = 关)。
|
|
19
|
+
|
|
20
|
+
import { db, setScheduledRunTerminalHook } from "./db.js";
|
|
21
|
+
import { sendAgentNotice } from "./agent-notice.js";
|
|
22
|
+
import { parseHubTimestamp } from "./hub-timestamp.js";
|
|
23
|
+
import { busyElapsedNote } from "./scheduled-failure-code.js";
|
|
24
|
+
|
|
25
|
+
export const SCHEDULE_FAILING_NOTICE_KIND = "schedule_failing";
|
|
26
|
+
const ERROR_PREVIEW_CHARS = 300;
|
|
27
|
+
|
|
28
|
+
function envInt(name: string, fallback: number, min: number): number {
|
|
29
|
+
const raw = process.env[name];
|
|
30
|
+
if (raw === undefined || raw.trim() === "") return fallback;
|
|
31
|
+
const n = Number(raw);
|
|
32
|
+
return Number.isFinite(n) && n >= min ? Math.floor(n) : fallback;
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
export function failureNoticeRuns(): number { return envInt("COMMHUB_SCHEDULE_FAILURE_NOTICE_RUNS", 5, 1); }
|
|
36
|
+
export function failureRenoticeMs(): number { return envInt("COMMHUB_SCHEDULE_FAILURE_RENOTICE_SEC", 86_400, 1) * 1000; }
|
|
37
|
+
/** 0 = 不自动暂停(默认)。 */
|
|
38
|
+
export function failureAutoPauseRuns(): number { return envInt("COMMHUB_SCHEDULE_FAILURE_AUTO_PAUSE_RUNS", 0, 0); }
|
|
39
|
+
|
|
40
|
+
export type FailureStreak = { consecutiveFailures: number; lastSuccessRunId: string | null };
|
|
41
|
+
|
|
42
|
+
/** 最近一次成功之后的失败 run 数。scheduled_for 每个排程唯一(UNIQUE),按它排序是确定的。 */
|
|
43
|
+
export function failureStreak(scheduleId: string): FailureStreak {
|
|
44
|
+
const lastSuccess = db.get<{ run_id: string; scheduled_for: string }>(
|
|
45
|
+
"SELECT run_id, scheduled_for FROM scheduled_task_runs WHERE schedule_id = ?1 AND status = 'replied' ORDER BY scheduled_for DESC LIMIT 1",
|
|
46
|
+
scheduleId,
|
|
47
|
+
);
|
|
48
|
+
const n = db.get<{ n: number }>(
|
|
49
|
+
"SELECT COUNT(*) AS n FROM scheduled_task_runs WHERE schedule_id = ?1 AND status = 'failed' AND scheduled_for > ?2",
|
|
50
|
+
scheduleId, lastSuccess?.scheduled_for ?? "",
|
|
51
|
+
)?.n ?? 0;
|
|
52
|
+
return { consecutiveFailures: Number(n), lastSuccessRunId: lastSuccess?.run_id ?? null };
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
type ScheduleForAlert = {
|
|
56
|
+
schedule_id: string;
|
|
57
|
+
network_id: string;
|
|
58
|
+
created_by: string | null;
|
|
59
|
+
name: string;
|
|
60
|
+
target_alias: string;
|
|
61
|
+
status: string;
|
|
62
|
+
};
|
|
63
|
+
|
|
64
|
+
export type FailureNotice = {
|
|
65
|
+
reason: "failing" | "paused";
|
|
66
|
+
scheduleId: string;
|
|
67
|
+
scheduleName: string;
|
|
68
|
+
networkId: string;
|
|
69
|
+
createdBy: string;
|
|
70
|
+
alias: string;
|
|
71
|
+
failures: number;
|
|
72
|
+
errorCode: string | null;
|
|
73
|
+
errorMessage: string | null;
|
|
74
|
+
};
|
|
75
|
+
|
|
76
|
+
function latestFailure(scheduleId: string): { error_code: string | null; error_message: string | null } {
|
|
77
|
+
const row = db.get<{
|
|
78
|
+
error_code: string | null; error_message: string | null; result: string | null;
|
|
79
|
+
task_created_at: string | null; task_started_at: string | null; task_completed_at: string | null;
|
|
80
|
+
}>(
|
|
81
|
+
`SELECT r.error_code, r.error_message, t.result,
|
|
82
|
+
t.created_at AS task_created_at, t.started_at AS task_started_at, t.completed_at AS task_completed_at
|
|
83
|
+
FROM scheduled_task_runs r
|
|
84
|
+
LEFT JOIN tasks t ON t.task_id = r.task_id AND t.network_id = r.network_id
|
|
85
|
+
WHERE r.schedule_id = ?1 AND r.status = 'failed'
|
|
86
|
+
ORDER BY r.scheduled_for DESC LIMIT 1`,
|
|
87
|
+
scheduleId,
|
|
88
|
+
);
|
|
89
|
+
// 任务以 failed 结束时 run.error_message 为空,原因在任务的回复正文里。
|
|
90
|
+
// #2884 —— 共享会话忙的那一类,再把「排队 / 会话等待」两段实测补在后面:
|
|
91
|
+
// 只读拼接,不改写入路径,所以节点原文不会被覆盖。
|
|
92
|
+
const base = row?.error_message || row?.result || null;
|
|
93
|
+
if (row?.error_code !== "task_failed_busy") return { error_code: row?.error_code ?? null, error_message: base };
|
|
94
|
+
const note = busyElapsedNote(row?.task_created_at, row?.task_started_at, row?.task_completed_at);
|
|
95
|
+
return {
|
|
96
|
+
error_code: row?.error_code ?? null,
|
|
97
|
+
error_message: note ? [base, note].filter(Boolean).join(" ") : base,
|
|
98
|
+
};
|
|
99
|
+
}
|
|
100
|
+
|
|
101
|
+
function truncate(s: string, max: number): string {
|
|
102
|
+
const flat = s.replace(/\s+/g, " ").trim();
|
|
103
|
+
return flat.length > max ? `${flat.slice(0, max)}…` : flat;
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
export function failureNoticeText(n: FailureNotice): { title: string; text: string } {
|
|
107
|
+
const err = n.errorMessage ? truncate(n.errorMessage, ERROR_PREVIEW_CHARS) : "(没有错误详情)";
|
|
108
|
+
const code = n.errorCode ? ` [${n.errorCode}]` : "";
|
|
109
|
+
const head = `定时任务「${n.scheduleName}」(派给节点 ${n.alias})已经连续 ${n.failures} 次执行失败。\n\n最近一次的错误${code}:${err}\n\n`;
|
|
110
|
+
if (n.reason === "paused") {
|
|
111
|
+
return {
|
|
112
|
+
title: "定时任务连续失败,已自动暂停",
|
|
113
|
+
text: `${head}按 Hub 的设置,它已被自动暂停,不会再派发。处理好原因后,在定时任务里把它恢复为「启用」即可。`,
|
|
114
|
+
};
|
|
115
|
+
}
|
|
116
|
+
return {
|
|
117
|
+
title: "定时任务连续失败",
|
|
118
|
+
text: `${head}它仍在按计划运行。如需停止,在定时任务里暂停它(或 PATCH /api/scheduled-tasks/${n.scheduleId},body 带 {"status":"paused","revision":<当前 revision>})。`
|
|
119
|
+
+ `在它重新成功之前,24 小时内不会再提醒。`,
|
|
120
|
+
};
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
export function sendFailureNotice(n: FailureNotice): string | null {
|
|
124
|
+
try {
|
|
125
|
+
const { title, text } = failureNoticeText(n);
|
|
126
|
+
return sendAgentNotice({
|
|
127
|
+
networkId: n.networkId, userId: n.createdBy, fromAlias: n.alias, kind: SCHEDULE_FAILING_NOTICE_KIND, title, text, severity: "error",
|
|
128
|
+
meta: { schedule_failing: { reason: n.reason, schedule_id: n.scheduleId, consecutive_failures: n.failures, error_code: n.errorCode } },
|
|
129
|
+
idPrefix: "dm_sched_",
|
|
130
|
+
});
|
|
131
|
+
} catch (e: any) {
|
|
132
|
+
console.error(`[scheduled-tasks] failure notice failed schedule=${n.scheduleId}: ${e?.message || e}`);
|
|
133
|
+
return null;
|
|
134
|
+
}
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
/**
|
|
138
|
+
* 一个 run 以 failed 结束之后调用(派发时失败 / 任务 failed 都算)。返回发出的通知(测试用),没有则 null。
|
|
139
|
+
* 告警与暂停都是条件写:并发的两个调用方只有一个写得进去,所以每段失败恰好一条通知。
|
|
140
|
+
*/
|
|
141
|
+
export function evaluateScheduleFailures(scheduleId: string, now = new Date()): FailureNotice | null {
|
|
142
|
+
const row = db.get<ScheduleForAlert>(
|
|
143
|
+
"SELECT schedule_id, network_id, created_by, name, target_alias, status FROM scheduled_tasks WHERE schedule_id = ?1",
|
|
144
|
+
scheduleId,
|
|
145
|
+
);
|
|
146
|
+
if (!row) return null;
|
|
147
|
+
const streak = failureStreak(scheduleId);
|
|
148
|
+
const noticeAt = failureNoticeRuns();
|
|
149
|
+
const pauseAt = failureAutoPauseRuns();
|
|
150
|
+
const wantsPause = pauseAt > 0 && streak.consecutiveFailures >= pauseAt && row.status === "active";
|
|
151
|
+
if (streak.consecutiveFailures < noticeAt && !wantsPause) return null;
|
|
152
|
+
|
|
153
|
+
let paused = false;
|
|
154
|
+
if (wantsPause) {
|
|
155
|
+
paused = db.run(
|
|
156
|
+
"UPDATE scheduled_tasks SET status = 'paused', next_run_at = NULL, revision = revision + 1, updated_at = datetime('now') WHERE schedule_id = ?1 AND status = 'active'",
|
|
157
|
+
[scheduleId],
|
|
158
|
+
).changes === 1;
|
|
159
|
+
}
|
|
160
|
+
const key = streak.lastSuccessRunId ?? "";
|
|
161
|
+
const cutoff = new Date(now.getTime() - failureRenoticeMs()).toISOString();
|
|
162
|
+
// 去重:同一段失败(key 没变)且上次告警不到 24 小时 → 不写、不发。暂停是另一件事,照发。
|
|
163
|
+
const claimed = db.run(
|
|
164
|
+
`UPDATE scheduled_tasks SET failure_alert_at = ?1, failure_alert_key = ?2
|
|
165
|
+
WHERE schedule_id = ?3
|
|
166
|
+
AND (failure_alert_at IS NULL OR COALESCE(failure_alert_key, '') <> ?2 OR failure_alert_at <= ?4)`,
|
|
167
|
+
[now.toISOString(), key, scheduleId, cutoff],
|
|
168
|
+
).changes === 1;
|
|
169
|
+
if (paused && !claimed) {
|
|
170
|
+
db.run("UPDATE scheduled_tasks SET failure_alert_at = ?1, failure_alert_key = ?2 WHERE schedule_id = ?3", [now.toISOString(), key, scheduleId]);
|
|
171
|
+
}
|
|
172
|
+
if (!paused && !claimed) return null;
|
|
173
|
+
if (!row.created_by) return null;
|
|
174
|
+
const latest = latestFailure(scheduleId);
|
|
175
|
+
const notice: FailureNotice = {
|
|
176
|
+
reason: paused ? "paused" : "failing",
|
|
177
|
+
scheduleId, scheduleName: row.name, networkId: row.network_id, createdBy: row.created_by, alias: row.target_alias,
|
|
178
|
+
failures: streak.consecutiveFailures, errorCode: latest.error_code, errorMessage: latest.error_message,
|
|
179
|
+
};
|
|
180
|
+
sendFailureNotice(notice);
|
|
181
|
+
return notice;
|
|
182
|
+
}
|
|
183
|
+
|
|
184
|
+
/**
|
|
185
|
+
* #2622 —— 失败退避:连续失败达到 N 次后把**间隔翻倍**(上限 1 小时),成功一次立刻恢复原节奏。
|
|
186
|
+
*
|
|
187
|
+
* 为什么不是「暂停」:`#523` 已经有 `COMMHUB_SCHEDULE_FAILURE_AUTO_PAUSE_RUNS`(默认关),
|
|
188
|
+
* 但它会把排程**停下来**,需要人来恢复。对“上游额度暂时打满”这类外部原因,暂停太久同样是损失;
|
|
189
|
+
* 退避只是**放慢重试**,外部原因一恢复就会自己跑回原节奏。
|
|
190
|
+
*
|
|
191
|
+
* 现场(2026-10-27 生产 24h):一个 `every_seconds=180` 的排程失败 351 次、成功 24 次 ——
|
|
192
|
+
* 失败与成功交替出现,所以“连续失败”很少攒到暂停阈值,而**每次失败都在按 3 分钟原速重试**。
|
|
193
|
+
*
|
|
194
|
+
* 存储复用 `backoff_effective_seconds`(派发时按 `max(every_seconds, backoff_effective_seconds)` 生效),
|
|
195
|
+
* 与 `no_change_backoff` 互斥:那套逻辑自己会写这一列,这里只在 `no_change_backoff = 0` 时介入。
|
|
196
|
+
*
|
|
197
|
+
* 环境变量:`COMMHUB_SCHEDULE_FAILURE_BACKOFF_RUNS`(默认 6;0 = 关闭)。
|
|
198
|
+
*/
|
|
199
|
+
export function failureBackoffRuns(): number { return envInt("COMMHUB_SCHEDULE_FAILURE_BACKOFF_RUNS", 6, 0); }
|
|
200
|
+
|
|
201
|
+
const FAILURE_BACKOFF_CAP_SECONDS = 3600;
|
|
202
|
+
|
|
203
|
+
export function applyFailureBackoff(scheduleId: string, now = new Date()): { streak: number; effective_seconds: number | null } | null {
|
|
204
|
+
const from = failureBackoffRuns();
|
|
205
|
+
if (from <= 0) return null;
|
|
206
|
+
const row = db.get<any>(
|
|
207
|
+
`SELECT schedule_id, revision, status, schedule_type, schedule_json, no_change_backoff,
|
|
208
|
+
backoff_effective_seconds, next_run_at
|
|
209
|
+
FROM scheduled_tasks WHERE schedule_id = ?1`,
|
|
210
|
+
scheduleId,
|
|
211
|
+
);
|
|
212
|
+
if (!row || row.status !== "active" || row.no_change_backoff === 1) return null;
|
|
213
|
+
if (row.schedule_type !== "interval") return null;
|
|
214
|
+
let spec: any = null;
|
|
215
|
+
try { spec = JSON.parse(row.schedule_json || "{}"); } catch { spec = null; }
|
|
216
|
+
const base = spec && spec.type === "interval" ? Number(spec.every_seconds) : NaN;
|
|
217
|
+
if (!Number.isFinite(base) || base <= 0) return null;
|
|
218
|
+
|
|
219
|
+
const streak = failureStreak(scheduleId).consecutiveFailures;
|
|
220
|
+
const effective = streak >= from
|
|
221
|
+
? Math.min(FAILURE_BACKOFF_CAP_SECONDS, base * 2 ** Math.min(streak - from + 1, 6))
|
|
222
|
+
: null;
|
|
223
|
+
const target = effective ?? base;
|
|
224
|
+
const previous = row.backoff_effective_seconds ?? base;
|
|
225
|
+
let next = row.next_run_at;
|
|
226
|
+
if (target !== previous && next) {
|
|
227
|
+
// 放大:至少推到 now + target(不把已经更远的 next 拉近);
|
|
228
|
+
// 恢复:至少拉回 now + target(不把已经更近的 next 推远)。
|
|
229
|
+
const candidate = new Date(now.getTime() + target * 1000).toISOString();
|
|
230
|
+
const existing = Date.parse(next);
|
|
231
|
+
if (Number.isFinite(existing)) {
|
|
232
|
+
next = new Date(target > previous ? Math.max(existing, Date.parse(candidate)) : Math.min(existing, Date.parse(candidate))).toISOString();
|
|
233
|
+
}
|
|
234
|
+
}
|
|
235
|
+
db.run(
|
|
236
|
+
`UPDATE scheduled_tasks SET backoff_streak = ?1, backoff_effective_seconds = ?2, next_run_at = ?3,
|
|
237
|
+
updated_at = datetime('now') WHERE schedule_id = ?4 AND revision = ?5 AND status = 'active'`,
|
|
238
|
+
[streak, effective, next, scheduleId, row.revision],
|
|
239
|
+
);
|
|
240
|
+
return { streak, effective_seconds: effective };
|
|
241
|
+
}
|
|
242
|
+
|
|
243
|
+
/**
|
|
244
|
+
* #2622(第二轮)—— 「跑不起来」的两类跳过也要退避。
|
|
245
|
+
*
|
|
246
|
+
* 生产 24h(2026-10-27):`previous_run_active` 2286 条、`target_offline` 1951 条,
|
|
247
|
+
* 而 `failureStreak()` 只数 `status='failed'`,**skipped 完全不计**,
|
|
248
|
+
* 于是一个 `every_seconds=180` 的排程可以在目标永久离线的状态下每 3 分钟写一行 skip,
|
|
249
|
+
* 一天 470 条,永不减速。
|
|
250
|
+
*
|
|
251
|
+
* 这里只做「放慢」,不改变派发语义:
|
|
252
|
+
* · 同一 error_code 连续出现 ≥ N(默认 3)⇒ 间隔按 2 倍放大,上限 1800 秒;
|
|
253
|
+
* · `target_offline` 的链用更长的上限(7200 秒 = 2 小时)——目标不在时没有理由每 3 分钟试一次;
|
|
254
|
+
* · 换链(原因变了)或出现成功 ⇒ 立刻回到原间隔;
|
|
255
|
+
* · 与 `no_change_backoff=1` 的排程互斥(那套逻辑自己拥有 backoff_* 三列)。
|
|
256
|
+
*/
|
|
257
|
+
const NO_PROGRESS_CODES = ["previous_run_active", "target_offline"] as const;
|
|
258
|
+
|
|
259
|
+
/**
|
|
260
|
+
* #2935 —— 「目标根本不会接活」的**失败**码,也要按 no-progress 退避,而不是普通失败退避。
|
|
261
|
+
*
|
|
262
|
+
* 生产实测(2026-10-30,近 7 天):
|
|
263
|
+
*
|
|
264
|
+
* failed node_degraded 2391 其中 TMA云盘鲸 1388 · TMA云鲸 981
|
|
265
|
+
* 「节点 X 处于降级状态(TUI 会话不在),任务不会被执行,已拒绝派发」
|
|
266
|
+
* failed target_node_not_found 1166 「The bound node no longer exists in this network」
|
|
267
|
+
*
|
|
268
|
+
* 这两类不是「上游额度打满」那种会自愈的失败:
|
|
269
|
+
* · `node_degraded` 的节点已经离线好几天(TMA云盘鲸/云鲸 自 10-27 起没心跳),
|
|
270
|
+
* 它的恢复时间是「小时/天」级,不是「下一次 180 秒」级;
|
|
271
|
+
* · `target_node_not_found` 更彻底 —— 目标行已经不存在,**永远不可能成功**。
|
|
272
|
+
*
|
|
273
|
+
* 而它们走的是 `applyFailureBackoff`(上限 1 小时、且要求「连续」失败,被任何一次成功打断就清零),
|
|
274
|
+
* 于是 `every=180` 的排程在目标永久不可用的情况下依然每小时烧掉 20 次派发。
|
|
275
|
+
*
|
|
276
|
+
* 这里只把它们并进「无进展」链(上限 2 小时),**不暂停、不改谁来决定暂停**。
|
|
277
|
+
*/
|
|
278
|
+
const NO_PROGRESS_FAILED_CODES = ["node_degraded", "target_node_not_found"] as const;
|
|
279
|
+
/** 统计窗口:看最近多少次 run 里同因跳过出现了几次(默认 10)。 */
|
|
280
|
+
const NO_PROGRESS_WINDOW = 10;
|
|
281
|
+
export type NoProgressCause = (typeof NO_PROGRESS_CODES)[number] | (typeof NO_PROGRESS_FAILED_CODES)[number];
|
|
282
|
+
|
|
283
|
+
function noProgressFromRuns(runs: Array<{ status: string; error_code: string | null }>): { cause: NoProgressCause | null; streak: number } {
|
|
284
|
+
const first = runs[0];
|
|
285
|
+
if (!first) return { cause: null, streak: 0 };
|
|
286
|
+
let cause: NoProgressCause | null = null;
|
|
287
|
+
if (first.status === "failed") {
|
|
288
|
+
// 默认交给 applyFailureBackoff;但「目标根本不会接活」的两类并进来(见上面注释)。
|
|
289
|
+
const code = first.error_code ?? "";
|
|
290
|
+
cause = (NO_PROGRESS_FAILED_CODES as readonly string[]).includes(code) ? (code as unknown as NoProgressCause) : null;
|
|
291
|
+
}
|
|
292
|
+
else if (first.status === "skipped" && (NO_PROGRESS_CODES as readonly string[]).includes(first.error_code ?? "")) {
|
|
293
|
+
cause = first.error_code as NoProgressCause;
|
|
294
|
+
}
|
|
295
|
+
if (!cause) return { cause: null, streak: 0 };
|
|
296
|
+
// #2622(实测修正)—— **按窗口计数,不按连续链**。
|
|
297
|
+
// 生产 4.3 小时实测:一批排程的模式是「跳过 → 跳过 → 成功 → 跳过…」,
|
|
298
|
+
// 成功会把连续链打断,于是 streak 永远攒不到阈值,退避一次都没触发
|
|
299
|
+
// (previous_run_active 24h 从 2397/天 只降到 2258/天)。
|
|
300
|
+
// 改成「最近 N 次里同因跳过出现几次」:仍然要求最新一次是同因跳过(不干扰成功后的恢复),
|
|
301
|
+
// 但如果它在这 N 次里反复出现,就按窗口值放慢。
|
|
302
|
+
const window = runs.slice(0, NO_PROGRESS_WINDOW);
|
|
303
|
+
let streak = 0;
|
|
304
|
+
for (const r of window) {
|
|
305
|
+
// #2935 —— failed 也计入:node_degraded / target_node_not_found 的 run 状态是 `failed`,
|
|
306
|
+
// 只数 `skipped` 会让它们永远攒不到阈值(改了分类却没改计数口径 = 等于没改)。
|
|
307
|
+
if ((r.status === "skipped" || r.status === "failed") && r.error_code === cause) streak++;
|
|
308
|
+
}
|
|
309
|
+
return { cause, streak };
|
|
310
|
+
}
|
|
311
|
+
|
|
312
|
+
export function noProgressBackoffRuns(): number {
|
|
313
|
+
return envInt("COMMHUB_SCHEDULE_NO_PROGRESS_BACKOFF_RUNS", 3, 1);
|
|
314
|
+
}
|
|
315
|
+
|
|
316
|
+
export function applyNoProgressBackoff(scheduleId: string, now = new Date()): { streak: number; effective_seconds: number | null; cause: NoProgressCause } | null {
|
|
317
|
+
const from = noProgressBackoffRuns();
|
|
318
|
+
if (from <= 0) return null;
|
|
319
|
+
const row = db.get<any>(
|
|
320
|
+
`SELECT schedule_id, revision, status, schedule_type, schedule_json, no_change_backoff,
|
|
321
|
+
backoff_effective_seconds, next_run_at
|
|
322
|
+
FROM scheduled_tasks WHERE schedule_id = ?1`,
|
|
323
|
+
scheduleId,
|
|
324
|
+
);
|
|
325
|
+
if (!row || row.status !== "active" || row.no_change_backoff === 1) return null;
|
|
326
|
+
if (row.schedule_type !== "interval") return null;
|
|
327
|
+
let spec: any = null;
|
|
328
|
+
try { spec = JSON.parse(row.schedule_json || "{}"); } catch { spec = null; }
|
|
329
|
+
const base = spec && spec.type === "interval" ? Number(spec.every_seconds) : NaN;
|
|
330
|
+
if (!Number.isFinite(base) || base <= 0) return null;
|
|
331
|
+
|
|
332
|
+
const runs = db.all<{ status: string; error_code: string | null }>(
|
|
333
|
+
`SELECT status, error_code FROM scheduled_task_runs
|
|
334
|
+
WHERE schedule_id = ?1 ORDER BY scheduled_for DESC, created_at DESC LIMIT 12`,
|
|
335
|
+
scheduleId,
|
|
336
|
+
);
|
|
337
|
+
const { cause, streak } = noProgressFromRuns(runs);
|
|
338
|
+
|
|
339
|
+
// 回到原间隔(成功 / 换链 / 未达阈值都走这里;只在真的退避过时才写)
|
|
340
|
+
const restore = () => {
|
|
341
|
+
if ((row.backoff_streak ?? 0) === 0 && row.backoff_effective_seconds == null) return;
|
|
342
|
+
// next_run_at 是 hub 的 TEXT UTC(无时区标记),必须走 parseHubTimestamp
|
|
343
|
+
const cur = parseHubTimestamp(row.next_run_at);
|
|
344
|
+
const next = cur === null
|
|
345
|
+
? row.next_run_at
|
|
346
|
+
: new Date(Math.min(cur, now.getTime() + base * 1000)).toISOString();
|
|
347
|
+
db.run(
|
|
348
|
+
`UPDATE scheduled_tasks SET backoff_streak = 0, backoff_effective_seconds = NULL, next_run_at = ?1,
|
|
349
|
+
updated_at = datetime('now') WHERE schedule_id = ?2 AND revision = ?3 AND status = 'active'`,
|
|
350
|
+
[next, scheduleId, row.revision],
|
|
351
|
+
);
|
|
352
|
+
};
|
|
353
|
+
|
|
354
|
+
if (!cause) { restore(); return null; }
|
|
355
|
+
|
|
356
|
+
// 上限按原因分开:
|
|
357
|
+
// previous_run_active —— 上一次还在跑,30 分钟足够等到它结束;
|
|
358
|
+
// target_offline —— 目标连续 6 次不在(按 180s 基底已是 18 分钟),
|
|
359
|
+
// 再用 3 分钟一次的节奏去敲没有意义;2 小时一次仍能及时接住它回来。
|
|
360
|
+
// 判据:#2622「同类原因日增量 ≤20」——180s 基底配 2 小时上限 ⇒ 12 次/天。
|
|
361
|
+
// target_offline / node_degraded / target_node_not_found 都是「目标不接活」的形状,
|
|
362
|
+
// 恢复是小时/天级 ⇒ 统一用 2 小时上限。previous_run_active 仍用 30 分钟。
|
|
363
|
+
const cap = cause === "previous_run_active" ? 1800 : 7200;
|
|
364
|
+
if (streak < from) { restore(); return null; }
|
|
365
|
+
|
|
366
|
+
const effective = Math.min(cap, base * 2 ** Math.min(streak - from + 1, 5));
|
|
367
|
+
const previous = row.backoff_effective_seconds ?? base;
|
|
368
|
+
let next = row.next_run_at;
|
|
369
|
+
// 🔴 生产实测(2026-10-28,`.181` 的两行日志):
|
|
370
|
+
// offline-backoff effective=5760s next_before=+180s ← 退避值没推进去
|
|
371
|
+
// offline-backoff-final next_after=+360s ← 二次 advance 又加了一个 base
|
|
372
|
+
// 根因就在这里:原来用 `effective !== previous` 做守卫,退避一旦进入**稳态**(例如已是 5760),
|
|
373
|
+
// 这个条件恒为 false ⇒ 不再推 next_run_at;而调用方在此之前刚 `advance()` 把 next 重置成 +base。
|
|
374
|
+
// 于是「退避值写着 5760,实际每 360s 跳一次」,且永远不会自愈。
|
|
375
|
+
// 正确语义:只要这一步是「同因跳过」,next_run_at 至少要 ≥ now + effective(只能往后推,不往前拉)。
|
|
376
|
+
if (next) {
|
|
377
|
+
const candidate = now.getTime() + effective * 1000;
|
|
378
|
+
const existing = parseHubTimestamp(next);
|
|
379
|
+
if (existing !== null) {
|
|
380
|
+
next = new Date(Math.max(existing, candidate)).toISOString();
|
|
381
|
+
}
|
|
382
|
+
}
|
|
383
|
+
db.run(
|
|
384
|
+
`UPDATE scheduled_tasks SET backoff_streak = ?1, backoff_effective_seconds = ?2, next_run_at = ?3,
|
|
385
|
+
updated_at = datetime('now') WHERE schedule_id = ?4 AND revision = ?5 AND status = 'active'`,
|
|
386
|
+
[streak, effective, next, scheduleId, row.revision],
|
|
387
|
+
);
|
|
388
|
+
return { streak, effective_seconds: effective, cause };
|
|
389
|
+
}
|
|
390
|
+
|
|
391
|
+
/** GET /runs 的附加字段。 */
|
|
392
|
+
export function failureSummary(scheduleId: string): { consecutive_failures: number; failure_alert_threshold: number; last_failure_alert_at: string | null } {
|
|
393
|
+
const streak = failureStreak(scheduleId);
|
|
394
|
+
const alertAt = db.get<{ failure_alert_at: string | null }>("SELECT failure_alert_at FROM scheduled_tasks WHERE schedule_id = ?1", scheduleId)?.failure_alert_at ?? null;
|
|
395
|
+
return { consecutive_failures: streak.consecutiveFailures, failure_alert_threshold: failureNoticeRuns(), last_failure_alert_at: alertAt };
|
|
396
|
+
}
|
|
397
|
+
|
|
398
|
+
setScheduledRunTerminalHook((scheduleId, status) => {
|
|
399
|
+
if (status === "failed") evaluateScheduleFailures(scheduleId);
|
|
400
|
+
// #2622 —— 成功一次就把间隔恢复原样;失败则按连续失败次数放大间隔。
|
|
401
|
+
//
|
|
402
|
+
// #2935 —— 但「目标根本不会接活」的失败码(node_degraded / target_node_not_found)
|
|
403
|
+
// 不走失败退避:它们的恢复是小时/天级,用 1 小时上限 + 「连续失败」判据等于每 3 分钟继续烧。
|
|
404
|
+
// 这两类改走 no-progress 链(2 小时上限),与 skipped 的 target_offline 同一形状。
|
|
405
|
+
try {
|
|
406
|
+
if (isNoProgressFailedCode(lastRunErrorCode(scheduleId))) applyNoProgressBackoff(scheduleId);
|
|
407
|
+
else applyFailureBackoff(scheduleId);
|
|
408
|
+
} catch (e: any) {
|
|
409
|
+
console.error(`[scheduled-tasks] failure backoff failed schedule=${scheduleId}: ${e?.message || e}`);
|
|
410
|
+
}
|
|
411
|
+
});
|
|
412
|
+
|
|
413
|
+
function isNoProgressFailedCode(code: string | null): boolean {
|
|
414
|
+
return !!code && (NO_PROGRESS_FAILED_CODES as readonly string[]).includes(code);
|
|
415
|
+
}
|
|
416
|
+
|
|
417
|
+
/** 最近一条 run 的 error_code(终态 hook 里用它判断走哪条退避链)。 */
|
|
418
|
+
function lastRunErrorCode(scheduleId: string): string | null {
|
|
419
|
+
const row = db.get<{ error_code: string | null }>(
|
|
420
|
+
`SELECT error_code FROM scheduled_task_runs WHERE schedule_id = ?1
|
|
421
|
+
ORDER BY created_at DESC, run_id DESC LIMIT 1`,
|
|
422
|
+
scheduleId,
|
|
423
|
+
);
|
|
424
|
+
return row?.error_code ?? null;
|
|
425
|
+
}
|
|
@@ -0,0 +1,163 @@
|
|
|
1
|
+
// #2622(第二轮)—— 「跑不起来」的两类跳过也要退避。
|
|
2
|
+
//
|
|
3
|
+
// 生产 24h(2026-10-27):previous_run_active 2286 条、target_offline 1951 条,
|
|
4
|
+
// 而 failureStreak 只数 status='failed',skipped 完全不计 ⇒ 永远按原速重试。
|
|
5
|
+
//
|
|
6
|
+
// 这个文件钉住四件事:
|
|
7
|
+
// 1. 同一 error_code 连续 ≥3 次 ⇒ 间隔放大(有上限:previous_run_active 1800s / target_offline 3600s);
|
|
8
|
+
// 2. 换链(原因变了)或出现成功 ⇒ 立刻回到原间隔;
|
|
9
|
+
// 3. no_change_backoff=1 的排程由另一套逻辑拥有 backoff 列,这里完全不介入;
|
|
10
|
+
// 4. 非 interval 排程不介入。
|
|
11
|
+
|
|
12
|
+
import { afterAll, beforeAll, describe, expect, test } from "bun:test";
|
|
13
|
+
import { register } from "./auth.js";
|
|
14
|
+
import { db } from "./db.js";
|
|
15
|
+
import { applyNoProgressBackoff, noProgressBackoffRuns } from "./scheduled-failures.js";
|
|
16
|
+
|
|
17
|
+
const stamp = Date.now();
|
|
18
|
+
let NET = "";
|
|
19
|
+
let creatorId = "";
|
|
20
|
+
|
|
21
|
+
function seedSchedule(everySeconds = 180, extra: Record<string, unknown> = {}) {
|
|
22
|
+
const id = `sched_np_${crypto.randomUUID()}`;
|
|
23
|
+
db.run(
|
|
24
|
+
`INSERT INTO scheduled_tasks
|
|
25
|
+
(schedule_id, network_id, created_by, name, target_node_id, target_alias, task_content, priority,
|
|
26
|
+
schedule_type, schedule_json, timezone, status, next_run_at, overlap_policy)
|
|
27
|
+
VALUES (?1, ?2, ?3, 'np fixture', 'n_np', 'np-node', 'x', 'normal', 'interval', ?4,
|
|
28
|
+
'Asia/Shanghai', 'active', ?5, 'skip')`,
|
|
29
|
+
[id, NET, creatorId, JSON.stringify({ type: "interval", every_seconds: everySeconds }),
|
|
30
|
+
new Date(Date.now() + everySeconds * 1000).toISOString()],
|
|
31
|
+
);
|
|
32
|
+
for (const [k, v] of Object.entries(extra)) db.run(`UPDATE scheduled_tasks SET ${k} = ?1 WHERE schedule_id = ?2`, [v as any, id]);
|
|
33
|
+
return id;
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
let seq = 0;
|
|
37
|
+
function addRun(scheduleId: string, status: string, code: string | null) {
|
|
38
|
+
seq += 1;
|
|
39
|
+
const when = new Date(Date.now() + seq * 1000).toISOString();
|
|
40
|
+
db.run(
|
|
41
|
+
`INSERT INTO scheduled_task_runs (run_id, schedule_id, network_id, scheduled_for, status, error_code, created_at)
|
|
42
|
+
VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?4)`,
|
|
43
|
+
[`srun_np_${crypto.randomUUID()}`, scheduleId, NET, when, status, code],
|
|
44
|
+
);
|
|
45
|
+
}
|
|
46
|
+
|
|
47
|
+
const reload = (id: string) => db.get<any>("SELECT * FROM scheduled_tasks WHERE schedule_id = ?1", id)!;
|
|
48
|
+
|
|
49
|
+
beforeAll(async () => {
|
|
50
|
+
const auth = register(`np_owner_${stamp}`, "Np-Owner-Strong-1!", undefined, "tester");
|
|
51
|
+
expect(auth.ok).toBe(true);
|
|
52
|
+
NET = auth.network_id!;
|
|
53
|
+
creatorId = auth.user_id!;
|
|
54
|
+
delete process.env.COMMHUB_SCHEDULE_NO_PROGRESS_BACKOFF_RUNS;
|
|
55
|
+
});
|
|
56
|
+
|
|
57
|
+
afterAll(() => { delete process.env.COMMHUB_SCHEDULE_NO_PROGRESS_BACKOFF_RUNS; });
|
|
58
|
+
|
|
59
|
+
describe("applyNoProgressBackoff (#2622)", () => {
|
|
60
|
+
test("defaults to 3 consecutive same-cause skips", () => {
|
|
61
|
+
expect(noProgressBackoffRuns()).toBe(3);
|
|
62
|
+
});
|
|
63
|
+
|
|
64
|
+
test("target_offline streak widens the interval (capped at 2h)", () => {
|
|
65
|
+
const id = seedSchedule(180);
|
|
66
|
+
for (let i = 0; i < 4; i++) addRun(id, "skipped", "target_offline");
|
|
67
|
+
const r = applyNoProgressBackoff(id)!;
|
|
68
|
+
expect(r.cause).toBe("target_offline");
|
|
69
|
+
expect(r.streak).toBe(4);
|
|
70
|
+
expect(r.effective_seconds).toBeGreaterThan(180);
|
|
71
|
+
expect(r.effective_seconds!).toBeLessThanOrEqual(7200);
|
|
72
|
+
expect(reload(id).backoff_effective_seconds).toBe(r.effective_seconds);
|
|
73
|
+
});
|
|
74
|
+
|
|
75
|
+
test("previous_run_active uses a shorter cap (30min)", () => {
|
|
76
|
+
const id = seedSchedule(180);
|
|
77
|
+
for (let i = 0; i < 10; i++) addRun(id, "skipped", "previous_run_active");
|
|
78
|
+
const r = applyNoProgressBackoff(id)!;
|
|
79
|
+
expect(r.cause).toBe("previous_run_active");
|
|
80
|
+
expect(r.effective_seconds).toBe(1800);
|
|
81
|
+
});
|
|
82
|
+
|
|
83
|
+
test("below the threshold it leaves the interval alone", () => {
|
|
84
|
+
const id = seedSchedule(180);
|
|
85
|
+
addRun(id, "skipped", "target_offline");
|
|
86
|
+
addRun(id, "skipped", "target_offline");
|
|
87
|
+
expect(applyNoProgressBackoff(id)).toBeNull();
|
|
88
|
+
expect(reload(id).backoff_effective_seconds).toBeNull();
|
|
89
|
+
});
|
|
90
|
+
|
|
91
|
+
test("a success resets the streak and restores the base interval", () => {
|
|
92
|
+
const id = seedSchedule(180);
|
|
93
|
+
for (let i = 0; i < 5; i++) addRun(id, "skipped", "target_offline");
|
|
94
|
+
const widened = applyNoProgressBackoff(id)!;
|
|
95
|
+
expect(widened.effective_seconds!).toBeGreaterThan(180);
|
|
96
|
+
addRun(id, "replied", null); // 成功一次
|
|
97
|
+
expect(applyNoProgressBackoff(id)).toBeNull();
|
|
98
|
+
const row = reload(id);
|
|
99
|
+
expect(row.backoff_streak).toBe(0);
|
|
100
|
+
expect(row.backoff_effective_seconds).toBeNull();
|
|
101
|
+
});
|
|
102
|
+
|
|
103
|
+
test("switching cause (different error_code) does not keep the streak", () => {
|
|
104
|
+
const id = seedSchedule(180);
|
|
105
|
+
addRun(id, "skipped", "target_offline");
|
|
106
|
+
addRun(id, "skipped", "target_offline");
|
|
107
|
+
addRun(id, "skipped", "previous_run_active"); // 最新一条换链 ⇒ streak=1
|
|
108
|
+
expect(applyNoProgressBackoff(id)).toBeNull();
|
|
109
|
+
});
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
test("alternating skip/success pattern still backs off (window, not strict chain)", () => {
|
|
113
|
+
// 生产实测:一批排程是「跳过 → 跳过 → 成功 → 跳过…」,成功会把连续链打断,
|
|
114
|
+
// 按严格链计数永远攒不到阈值。窗口计数必须能抓住它。
|
|
115
|
+
const id = seedSchedule(180);
|
|
116
|
+
addRun(id, "skipped", "previous_run_active");
|
|
117
|
+
addRun(id, "skipped", "previous_run_active");
|
|
118
|
+
addRun(id, "replied", null); // 成功打断链
|
|
119
|
+
addRun(id, "skipped", "previous_run_active");
|
|
120
|
+
const r = applyNoProgressBackoff(id);
|
|
121
|
+
expect(r).not.toBeNull();
|
|
122
|
+
expect(r!.cause).toBe("previous_run_active");
|
|
123
|
+
expect(r!.streak).toBe(3);
|
|
124
|
+
expect(r!.effective_seconds).toBeGreaterThan(180);
|
|
125
|
+
});
|
|
126
|
+
|
|
127
|
+
test("schedules that opt into no_change_backoff are left to the other mechanism", () => {
|
|
128
|
+
const id = seedSchedule(180, { no_change_backoff: 1 });
|
|
129
|
+
for (let i = 0; i < 5; i++) addRun(id, "skipped", "target_offline");
|
|
130
|
+
expect(applyNoProgressBackoff(id)).toBeNull();
|
|
131
|
+
expect(reload(id).backoff_effective_seconds).toBeNull();
|
|
132
|
+
});
|
|
133
|
+
});
|
|
134
|
+
|
|
135
|
+
// #2935 —— 「目标根本不会接活」的**失败**码(node_degraded / target_node_not_found)
|
|
136
|
+
// 也必须走这条链。生产近 7 天:node_degraded 2391(TMA云盘鲸 1388 / TMA云鲸 981)、
|
|
137
|
+
// target_node_not_found 1166 —— 全部是 `every=180` 的排程在目标永久不可用时按原速烧。
|
|
138
|
+
describe("#2935 目标不会接活的失败码也退避", () => {
|
|
139
|
+
test("连续 node_degraded 失败 ⇒ 间隔放大到 2 小时上限(而不是失败退避的 1 小时)", () => {
|
|
140
|
+
const id = seedSchedule(180);
|
|
141
|
+
// 窗口是 10 条;指数封顶 5 ⇒ 180s 基底最多到 180*32 = 5760s。
|
|
142
|
+
// 注意:「cap 7200」对 180s 基底其实够不到 —— 真正生效的是指数封顶那一条。
|
|
143
|
+
for (let i = 0; i < 10; i++) addRun(id, "failed", "node_degraded");
|
|
144
|
+
const r = applyNoProgressBackoff(id)!;
|
|
145
|
+
expect(r.cause).toBe("node_degraded");
|
|
146
|
+
expect(r.effective_seconds).toBe(5760);
|
|
147
|
+
expect(r.effective_seconds!).toBeGreaterThan(3600); // 严格大于失败退避的 1 小时上限
|
|
148
|
+
});
|
|
149
|
+
|
|
150
|
+
test("连续 target_node_not_found 同样走 2 小时上限", () => {
|
|
151
|
+
const id = seedSchedule(180);
|
|
152
|
+
for (let i = 0; i < 10; i++) addRun(id, "failed", "target_node_not_found");
|
|
153
|
+
const r = applyNoProgressBackoff(id)!;
|
|
154
|
+
expect(r.cause).toBe("target_node_not_found");
|
|
155
|
+
expect(r.effective_seconds).toBe(5760);
|
|
156
|
+
});
|
|
157
|
+
|
|
158
|
+
test("普通 task_failed 不被本链接管(仍由失败退避负责)", () => {
|
|
159
|
+
const id = seedSchedule(180);
|
|
160
|
+
for (let i = 0; i < 6; i++) addRun(id, "failed", "task_failed");
|
|
161
|
+
expect(applyNoProgressBackoff(id)).toBeNull();
|
|
162
|
+
});
|
|
163
|
+
});
|
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
// 探针:target_offline 连续跳过后,next_run_at 到底被推到哪里?
|
|
2
|
+
// 生产现象(2026-10-28 08:45):backoff_effective_seconds=5760,但实际间隔 360s。
|
|
3
|
+
// 这里用真实调度入口 dispatchScheduledOccurrence 复现,看 next_run_at 与实际 cadence。
|
|
4
|
+
// Run: COMMHUB_DB=/tmp/target-offline-backoff.db bun test src/scheduled-offline-backoff-probe.test.ts
|
|
5
|
+
import { describe, expect, test, beforeAll } from 'bun:test';
|
|
6
|
+
import { register } from './auth.js';
|
|
7
|
+
import { db } from './db.js';
|
|
8
|
+
import { dispatchScheduledOccurrence, getScheduleRow } from './scheduled-tasks.js';
|
|
9
|
+
|
|
10
|
+
const stamp = Date.now();
|
|
11
|
+
const TARGET_ID = `n_offb_${stamp}`, TARGET = `offb-${stamp}`;
|
|
12
|
+
let NET = '', USER = '';
|
|
13
|
+
|
|
14
|
+
function seed() {
|
|
15
|
+
const id = `sched_offb_${crypto.randomUUID()}`;
|
|
16
|
+
db.run(`INSERT INTO scheduled_tasks (schedule_id, network_id, created_by, name, target_node_id, target_alias,
|
|
17
|
+
task_content, priority, schedule_type, schedule_json, timezone, status, next_run_at, overlap_policy)
|
|
18
|
+
VALUES (?1,?2,?3,'offb',?4,?5,'x','normal','interval','{"type":"interval","every_seconds":180}','Asia/Shanghai','active',datetime('now','+3 minutes'),'skip')`,
|
|
19
|
+
[id, NET, USER, TARGET_ID, TARGET]);
|
|
20
|
+
return getScheduleRow(id)!;
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
beforeAll(() => {
|
|
24
|
+
const owner = register(`offb_owner_${stamp}`, 'OffbOwnerPassw0rd!', undefined, 'seed');
|
|
25
|
+
expect(owner.ok).toBe(true);
|
|
26
|
+
NET = owner.network_id!; USER = owner.user!.user_id;
|
|
27
|
+
db.run("INSERT INTO nodes (node_id, node_name, alias, runtime, network_id, owner_user_id) VALUES (?1,?2,?2,'codex',?3,?4)", [TARGET_ID, TARGET, NET, USER]);
|
|
28
|
+
// 目标会话:10 分钟没心跳 ⇒ offline
|
|
29
|
+
db.run("INSERT INTO sessions (resume_id, alias, status, node_id, network_id, last_seen_at) VALUES (?1,?2,'idle',?3,?4,datetime('now','-10 minutes'))",
|
|
30
|
+
[`r_offb_${stamp}`, TARGET, TARGET_ID, NET]);
|
|
31
|
+
});
|
|
32
|
+
|
|
33
|
+
describe('target_offline backoff 是否真的推后 next_run_at', () => {
|
|
34
|
+
// 稳态回归(2026-10-28 生产实测):退避值已经等于 effective 时,原来 `effective !== previous`
|
|
35
|
+
// 的守卫恒假 ⇒ 不再推 next_run_at ⇒ 实际每 360s 跳一次。这里连打 6 次(eff 已进稳态)验证仍推。
|
|
36
|
+
test('连续 4 次离线跳过后,next_run_at 应 ≈ now + effective(而不是 base 的倍数)', () => {
|
|
37
|
+
const s = seed();
|
|
38
|
+
const rows: any[] = [];
|
|
39
|
+
for (let i = 0; i < 9; i++) { // 9 次 ⇒ streak 到 10 ⇒ eff 进入稳态 5760,覆盖「第二次起不推」的坑
|
|
40
|
+
const at = new Date(Date.now() + i * 10).toISOString();
|
|
41
|
+
const r = dispatchScheduledOccurrence(getScheduleRow(s.schedule_id)!, at, true, new Date());
|
|
42
|
+
const after = db.get<any>("SELECT backoff_effective_seconds, backoff_streak, next_run_at FROM scheduled_tasks WHERE schedule_id = ?1", s.schedule_id);
|
|
43
|
+
const nextMs = after.next_run_at ? Date.parse(String(after.next_run_at).replace(' ', 'T') + (String(after.next_run_at).endsWith('Z') ? '' : 'Z')) : NaN;
|
|
44
|
+
rows.push({ i, status: r.status, eff: after.backoff_effective_seconds, streak: after.backoff_streak, next_raw: after.next_run_at, gap_s: Number.isFinite(nextMs) ? Math.round((nextMs - Date.parse(at))/1000) : 'NaN' });
|
|
45
|
+
}
|
|
46
|
+
console.log('每次 fire 后:', JSON.stringify(rows));
|
|
47
|
+
const last = rows[rows.length - 1];
|
|
48
|
+
// 断言:最后一次的 next_run_at 与 effective 同量级(允许 ±2 个 base 的容差)
|
|
49
|
+
if (!last.eff) throw new Error('backoff_effective_seconds 始终为空:退避没有生效');
|
|
50
|
+
if (last.gap_s < last.eff) throw new Error(`next_run_at 只推后 ${last.gap_s}s,但 effective=${last.eff}s`);
|
|
51
|
+
// 稳态:所有 fire 的 gap 都必须 ≥ 当次的 effective(没有「第二次起就不推」)
|
|
52
|
+
for (const row of rows) {
|
|
53
|
+
if (row.eff && row.gap_s < row.eff) throw new Error(`第 ${row.i} 次:gap=${row.gap_s}s < effective=${row.eff}s(稳态没推)`);
|
|
54
|
+
}
|
|
55
|
+
});
|
|
56
|
+
});
|