@sleep2agi/commhub-server 0.9.0-preview.28 → 0.9.0-preview.280
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +120 -3
- package/bin/commhub.ts +93 -9
- package/build-info.json +7 -0
- package/package.json +10 -8
- package/src/ack-create-request-transport.test.ts +182 -0
- package/src/ack-create-request.test.ts +52 -8
- package/src/admin-delete-network-http.test.ts +229 -0
- package/src/admin-networks-http.test.ts +88 -0
- package/src/adoption-candidates-http.test.ts +140 -0
- package/src/adoption-candidates.ts +158 -0
- package/src/agent-access-batch.test.ts +215 -0
- package/src/agent-access.ts +585 -0
- package/src/agent-acl-groups-http.test.ts +247 -0
- package/src/agent-acl-http.test.ts +691 -0
- package/src/agent-acl-slice3-http.test.ts +888 -0
- package/src/agent-department-grants.test.ts +129 -0
- package/src/agent-department-http.test.ts +215 -0
- package/src/agent-groups-perf.test.ts +228 -0
- package/src/agent-notice.ts +48 -0
- package/src/agent-team-whoami-http.test.ts +98 -0
- package/src/agent-teams-http.test.ts +330 -0
- package/src/agent-teams.ts +244 -0
- package/src/alias-baseline-2571.test.ts +162 -0
- package/src/alias-baseline.ts +113 -0
- package/src/alias-filter.test.ts +72 -0
- package/src/alias-filter.ts +42 -0
- package/src/all-status-cache.test.ts +31 -0
- package/src/all-status-cache.ts +37 -0
- package/src/anet-variant-real-daemon-http.test.ts +184 -0
- package/src/api-host-supervisors-can-create-nodes.test.ts +350 -0
- package/src/api-host-supervisors-fallback.test.ts +6 -7
- package/src/api-host-supervisors-runtime-readiness.test.ts +195 -0
- package/src/api-nodes-shape.test.ts +131 -4
- package/src/auth-abuse-counters.test.ts +144 -0
- package/src/auth-abuse-counters.ts +122 -0
- package/src/auth-layer-shape.test.ts +105 -0
- package/src/auth-password-change-cli-tokens-http.test.ts +423 -0
- package/src/auth-sessions-http.test.ts +260 -0
- package/src/auth-validate.test.ts +5 -1
- package/src/auth-weak-password-login.test.ts +176 -0
- package/src/auth.ts +641 -68
- package/src/backfill-node-ids-2664.test.ts +128 -0
- package/src/binding-generation-http.test.ts +141 -0
- package/src/blocked-exit-on-reply.test.ts +122 -0
- package/src/broadcast-fanout-2721.test.ts +149 -0
- package/src/broadcast-fanout.ts +117 -0
- package/src/build-commit-report-http.test.ts +218 -0
- package/src/build-info.test.ts +60 -0
- package/src/build-info.ts +85 -0
- package/src/chain-attempt-dedup.test.ts +447 -0
- package/src/clip.test.ts +36 -0
- package/src/clip.ts +48 -0
- package/src/clone-node.test.ts +269 -0
- package/src/clone-node.ts +289 -0
- package/src/codex-config-flags.ts +77 -0
- package/src/codex-fork-contract.ts +23 -0
- package/src/codex-fork-migration.test.ts +38 -0
- package/src/commhub-bin-args.test.ts +85 -0
- package/src/config-apply-sec1.test.ts +37 -0
- package/src/config-apply-validate.test.ts +188 -1
- package/src/config-apply-validate.ts +226 -3
- package/src/contracts/session-project-create.ts +14 -0
- package/src/contracts/session-source-delivery.ts +16 -0
- package/src/contracts/session-source-directory.ts +54 -0
- package/src/contracts/session-source-evidence.ts +72 -0
- package/src/contracts/session-source-execution.ts +24 -0
- package/src/contracts/session-source-transport.ts +18 -0
- package/src/cors-origins.test.ts +73 -0
- package/src/create-node-blocked-gate.test.ts +365 -0
- package/src/create-node-error-domain.test.ts +127 -0
- package/src/create-node-network-id.test.ts +91 -0
- package/src/create-node-tool-schema.test.ts +59 -0
- package/src/create-node-unicode-name.test.ts +126 -0
- package/src/create-node-validate.test.ts +199 -7
- package/src/create-node-validate.ts +232 -15
- package/src/create-node-workdir.test.ts +298 -0
- package/src/create-node.ts +88 -23
- package/src/daemon-provider-snapshots.test.ts +453 -0
- package/src/daemon-provider-snapshots.ts +327 -0
- package/src/daemon-token-owner-http.test.ts +346 -0
- package/src/dashboard-slash-routing-http.test.ts +109 -0
- package/src/db-adapter-guard.test.ts +148 -0
- package/src/db-adapter-translate.test.ts +128 -0
- package/src/db-adapter.ts +377 -91
- package/src/db-timestamp.test.ts +38 -0
- package/src/db-timestamp.ts +15 -0
- package/src/db.ts +1334 -154
- package/src/delivered-without-push.test.ts +140 -0
- package/src/department-groups-http.test.ts +209 -0
- package/src/department-groups-sync-http.test.ts +328 -0
- package/src/department-groups.ts +434 -0
- package/src/department-heads-http.test.ts +350 -0
- package/src/department-heads.ts +226 -0
- package/src/departments-agent-count.test.ts +139 -0
- package/src/departments-http.test.ts +185 -0
- package/src/departments-mcp-error-domain.test.ts +135 -0
- package/src/departments-mcp.test.ts +128 -0
- package/src/departments.ts +318 -0
- package/src/desktop-message-seam.test.ts +150 -0
- package/src/desktop-message.test.ts +368 -0
- package/src/dm-file-scope-http.test.ts +227 -0
- package/src/empty-node-sweeper.test.ts +104 -0
- package/src/empty-node-sweeper.ts +88 -0
- package/src/env-transport-gate-http.test.ts +240 -0
- package/src/env-transport-gate.ts +80 -0
- package/src/env-transport-policy.ts +56 -0
- package/src/etag-of.test.ts +57 -0
- package/src/event-loop-lag.test.ts +180 -0
- package/src/event-loop-lag.ts +201 -0
- package/src/expiry-counters.test.ts +187 -0
- package/src/expiry-counters.ts +227 -0
- package/src/expiry-evidence-db.test.ts +103 -0
- package/src/ext-token-shared.test.ts +60 -0
- package/src/ext-validation-regression.test.ts +10 -17
- package/src/external-schedule-edits-http.test.ts +321 -0
- package/src/external-schedule-edits-timestamps.test.ts +64 -0
- package/src/external-schedule-edits.ts +327 -0
- package/src/file-download-authz-dev-open.test.ts +1 -1
- package/src/file-download-authz.test.ts +25 -4
- package/src/file-download-date-bucket.test.ts +1 -1
- package/src/file-network-scope-dev-open.test.ts +1 -1
- package/src/file-network-scope.test.ts +7 -6
- package/src/group-membership-events-http.test.ts +264 -0
- package/src/group-messages-http.test.ts +483 -0
- package/src/group-messages.ts +297 -0
- package/src/health-anon-readers.ts +67 -0
- package/src/health-host-disk-2756.test.ts +71 -0
- package/src/health-identity-root.test.ts +98 -0
- package/src/health-redaction.test.ts +1599 -7
- package/src/health-status-reasons.test.ts +84 -0
- package/src/host-supervisor-identity-http.test.ts +152 -0
- package/src/host-supervisor-identity.ts +92 -0
- package/src/host-supervisors-daemon-version.test.ts +137 -0
- package/src/http-gzip.test.ts +94 -0
- package/src/http-gzip.ts +112 -0
- package/src/hub-catalog-read-http.test.ts +664 -0
- package/src/hub-provider-config.test.ts +166 -0
- package/src/hub-provider-config.ts +202 -0
- package/src/hub-timestamp.test.ts +38 -0
- package/src/hub-timestamp.ts +27 -0
- package/src/human-dm.ts +215 -0
- package/src/human-inbox.ts +42 -0
- package/src/inbox-caller.ts +49 -0
- package/src/inbox-count-sse.test.ts +318 -0
- package/src/inbox-count.ts +29 -0
- package/src/index.ts +6 -2
- package/src/license-state.test.ts +105 -0
- package/src/license-state.ts +48 -0
- package/src/lifecycle-error-domain.test.ts +149 -0
- package/src/list-ack-error-domain.test.ts +108 -0
- package/src/list-host-supervisors.test.ts +15 -4
- package/src/list-tasks-omit-default.test.ts +84 -0
- package/src/list-tasks-since-last.test.ts +100 -0
- package/src/list-tasks-since.test.ts +104 -0
- package/src/mcp-context-cost-http.test.ts +127 -0
- package/src/mcp-inflight.test.ts +92 -0
- package/src/mcp-inflight.ts +38 -0
- package/src/mcp-projects-events-http.test.ts +168 -0
- package/src/mcp-route-label-http.test.ts +120 -0
- package/src/mcp-write-network-resolution.test.ts +21 -0
- package/src/member-audit-network-id-http.test.ts +93 -0
- package/src/member-presence-http.test.ts +210 -0
- package/src/messages-ack-by-agent-http.test.ts +142 -0
- package/src/migrate-sqlite-to-pg.test.ts +47 -0
- package/src/migrate-sqlite-to-pg.ts +263 -0
- package/src/model-auth-notify-http.test.ts +181 -0
- package/src/model-auth-notify.ts +116 -0
- package/src/network-name-http.test.ts +75 -0
- package/src/network-quota-config.test.ts +100 -0
- package/src/network-rules-http.test.ts +169 -0
- package/src/network-rules.ts +129 -0
- package/src/network-scope.ts +222 -3
- package/src/network-token-membership-http.test.ts +186 -0
- package/src/node-alias-control.ts +12 -0
- package/src/node-attrs.test.ts +47 -2
- package/src/node-caller-identity-http.test.ts +1108 -0
- package/src/node-config-last-update.test.ts +126 -0
- package/src/node-config-last-update.ts +107 -0
- package/src/node-create-request-status-http.test.ts +105 -0
- package/src/node-daemon-bindings.test.ts +142 -0
- package/src/node-daemon-bindings.ts +135 -0
- package/src/node-failure-alert.test.ts +150 -0
- package/src/node-failure-alert.ts +177 -0
- package/src/node-files-transport.test.ts +350 -0
- package/src/node-health-codex-login.test.ts +120 -0
- package/src/node-health-dispatch-http.test.ts +373 -0
- package/src/node-health-guard-origin.test.ts +98 -0
- package/src/node-health-guard.ts +121 -0
- package/src/node-health-passthrough.test.ts +115 -0
- package/src/node-health-store.ts +101 -0
- package/src/node-id-alias.test.ts +190 -0
- package/src/node-identity-conflict-http.test.ts +269 -0
- package/src/node-identity-conflict.ts +142 -0
- package/src/node-identity-mismatch.test.ts +74 -0
- package/src/node-in-flight-tasks.ts +31 -0
- package/src/node-lifecycle-controllable.test.ts +64 -0
- package/src/node-lifecycle-controllable.ts +53 -0
- package/src/node-lifecycle-read-http.test.ts +257 -0
- package/src/node-lifecycle-read.ts +126 -0
- package/src/node-logs-transport.test.ts +266 -0
- package/src/node-peer-allowlist.test.ts +114 -0
- package/src/node-permissions-http.test.ts +901 -0
- package/src/node-permissions.ts +496 -0
- package/src/node-request-retention.test.ts +281 -0
- package/src/node-request-retention.ts +120 -0
- package/src/node-secret-rest-http.test.ts +190 -0
- package/src/node-secret-vault-inject.test.ts +132 -0
- package/src/node-security-write-gate.test.ts +72 -0
- package/src/node-serviceability.test.ts +86 -0
- package/src/node-serviceability.ts +124 -0
- package/src/node-session-control-http.test.ts +398 -0
- package/src/node-session-control.ts +316 -0
- package/src/node-skills-transport.test.ts +212 -0
- package/src/node-token-identity.ts +35 -0
- package/src/node-token-ownership.ts +51 -0
- package/src/node-token-privilege-http.test.ts +224 -0
- package/src/node-token-usage.test.ts +128 -0
- package/src/node-token-usage.ts +228 -0
- package/src/not-found-api-http.test.ts +67 -0
- package/src/observer-avatar-http.test.ts +103 -10
- package/src/observer-push.test.ts +76 -0
- package/src/opencode-catalog-report-http.test.ts +139 -0
- package/src/opencode-v2-catalog.test.ts +282 -0
- package/src/opencode-v2-catalog.ts +357 -0
- package/src/package-self-contained.test.ts +32 -0
- package/src/package-version-consistency.test.ts +25 -0
- package/src/parent-infer.test.ts +82 -0
- package/src/parent-infer.ts +110 -0
- package/src/parent-source-marker.test.ts +70 -0
- package/src/parent-source-orphan.test.ts +121 -0
- package/src/password-network-tokens.ts +25 -0
- package/src/peer-reply-atomic.test.ts +462 -0
- package/src/peer-reply-capability.test.ts +70 -0
- package/src/pg-worker.ts +87 -0
- package/src/poll-pacing.test.ts +74 -0
- package/src/poll-pacing.ts +123 -0
- package/src/probe-provider-error-domain.test.ts +145 -0
- package/src/project-dir-warn-clear-http.test.ts +132 -0
- package/src/project-viewer-can-http.test.ts +116 -0
- package/src/provider-skill-error-domain.test.ts +130 -0
- package/src/provider-tools-shape.test.ts +165 -0
- package/src/push.test.ts +20 -10
- package/src/push.ts +373 -9
- package/src/redact-tokens.test.ts +86 -0
- package/src/redact-tokens.ts +43 -0
- package/src/rejection-counters.test.ts +72 -0
- package/src/rejection-counters.ts +173 -0
- package/src/rename.ts +38 -10
- package/src/reply-target-derivation.test.ts +96 -0
- package/src/report-status-handover-capabilities-http.test.ts +124 -0
- package/src/report-status-host-schema.test.ts +272 -0
- package/src/report-status-resume-id-handover.test.ts +184 -0
- package/src/report-status-secret-keys.test.ts +140 -0
- package/src/require-explicit-test-db.ts +9 -0
- package/src/requirement-due-reminders-http.test.ts +620 -0
- package/src/requirement-due-reminders.ts +416 -0
- package/src/requirement-events-http.test.ts +318 -0
- package/src/requirement-events.test.ts +45 -0
- package/src/requirement-events.ts +144 -0
- package/src/requirement-notify.ts +165 -0
- package/src/requirement-participant-notify-http.test.ts +313 -0
- package/src/requirement-tag-ops-http.test.ts +454 -0
- package/src/requirement-tag-tree-http.test.ts +275 -0
- package/src/requirement-tag-tree.ts +127 -0
- package/src/requirement-tags.ts +102 -0
- package/src/requirements-abandoned-http.test.ts +236 -0
- package/src/requirements-assignee-drift.test.ts +154 -0
- package/src/requirements-comment-http.test.ts +174 -0
- package/src/requirements-completed-at-migration.test.ts +70 -0
- package/src/requirements-errors-mcp-http.test.ts +164 -0
- package/src/requirements-errors.ts +158 -0
- package/src/requirements-etag-http.test.ts +91 -0
- package/src/requirements-http.test.ts +709 -0
- package/src/requirements-last-event-http.test.ts +349 -0
- package/src/requirements-list-mcp-slim-http.test.ts +155 -0
- package/src/requirements-list-slim-http.test.ts +324 -0
- package/src/requirements-mcp.test.ts +180 -0
- package/src/requirements-migrate.ts +283 -0
- package/src/requirements-owner-strict-mcp-http.test.ts +145 -0
- package/src/requirements-people-http.test.ts +114 -0
- package/src/requirements-people-mcp-http.test.ts +231 -0
- package/src/requirements-people.ts +196 -0
- package/src/requirements-priority-lowest.test.ts +114 -0
- package/src/requirements-projects-network-echo.test.ts +57 -0
- package/src/requirements-projects-parent-http.test.ts +123 -0
- package/src/requirements-search-http.test.ts +182 -0
- package/src/requirements-search.test.ts +32 -0
- package/src/requirements-search.ts +145 -0
- package/src/requirements-seq-http.test.ts +193 -0
- package/src/requirements-seq-migration.test.ts +63 -0
- package/src/requirements-start-date.test.ts +125 -0
- package/src/requirements-stats-http.test.ts +357 -0
- package/src/requirements-stats.ts +195 -0
- package/src/requirements-status-participants-http.test.ts +222 -0
- package/src/requirements-upgrade-e2e.test.ts +126 -0
- package/src/requirements-upgrade-migration.test.ts +98 -0
- package/src/requirements.ts +2205 -0
- package/src/resolve-port.test.ts +57 -0
- package/src/resolve-port.ts +44 -0
- package/src/rest-alias-network-fallback-http.test.ts +123 -0
- package/src/rest-alias-network-fallback.test.ts +60 -0
- package/src/rest-alias-network-fallback.ts +65 -0
- package/src/rest-explicit-columns-http.test.ts +266 -0
- package/src/rest-identity.test.ts +130 -0
- package/src/rest-identity.ts +141 -0
- package/src/rest-projections.ts +72 -0
- package/src/rest-read-network-real-chain.test.ts +97 -0
- package/src/rest-write-network-resolution.test.ts +133 -0
- package/src/restart-readiness-http.test.ts +121 -0
- package/src/restricted-files.ts +76 -0
- package/src/retention-first-sweep-quiet.test.ts +31 -0
- package/src/retention.test.ts +100 -40
- package/src/retention.ts +423 -104
- package/src/route-callers-http.test.ts +120 -0
- package/src/route-timing.test.ts +218 -0
- package/src/route-timing.ts +262 -0
- package/src/rules-file-duplicate-alias.test.ts +234 -0
- package/src/rules-file-session-target.test.ts +207 -0
- package/src/rules-file-transport.test.ts +280 -0
- package/src/runtime-label.test.ts +22 -0
- package/src/runtime-label.ts +21 -0
- package/src/runtime-readiness.ts +90 -0
- package/src/schedule-agent-mcp-http.test.ts +486 -0
- package/src/schedule-agent.ts +252 -0
- package/src/schedule-starved.test.ts +92 -0
- package/src/schedule-starved.ts +114 -0
- package/src/scheduled-backoff-http.test.ts +203 -0
- package/src/scheduled-backoff.ts +64 -0
- package/src/scheduled-doorbell-nodeless-session.test.ts +134 -0
- package/src/scheduled-failure-backoff-2622.test.ts +145 -0
- package/src/scheduled-failure-code.test.ts +73 -0
- package/src/scheduled-failure-code.ts +105 -0
- package/src/scheduled-failures-http.test.ts +279 -0
- package/src/scheduled-failures.ts +425 -0
- package/src/scheduled-noprogress-backoff-2622.test.ts +163 -0
- package/src/scheduled-offline-backoff-probe.test.ts +56 -0
- package/src/scheduled-offline.test.ts +131 -0
- package/src/scheduled-offline.ts +66 -0
- package/src/scheduled-outcome-guidance-http.test.ts +139 -0
- package/src/scheduled-outcome-guidance.ts +9 -0
- package/src/scheduled-reply-unread-http.test.ts +212 -0
- package/src/scheduled-requirement-wake-http.test.ts +289 -0
- package/src/scheduled-requirement-wake.ts +125 -0
- package/src/scheduled-run-terminal.test.ts +290 -0
- package/src/scheduled-skip-blocking-task.test.ts +120 -0
- package/src/scheduled-skip-path-backoff-2622.test.ts +98 -0
- package/src/scheduled-stuck-http.test.ts +316 -0
- package/src/scheduled-stuck-notice-cap.test.ts +80 -0
- package/src/scheduled-stuck.ts +219 -0
- package/src/scheduled-tasks-http.test.ts +679 -0
- package/src/scheduled-tasks.ts +964 -0
- package/src/scheduled-work-outcome-http.test.ts +154 -0
- package/src/scheduled-work-outcome.ts +50 -0
- package/src/scoped-secrets.ts +167 -0
- package/src/secret-echo-negative-control.test.ts +118 -0
- package/src/send-dedup-request-id.test.ts +229 -0
- package/src/send-reply-agent-warning.test.ts +38 -38
- package/src/send-reply-alias-optional.test.ts +131 -0
- package/src/send-reply-attachments.test.ts +28 -7
- package/src/send_dedup.test.ts +54 -0
- package/src/send_dedup.ts +52 -6
- package/src/serve-error.test.ts +73 -0
- package/src/serve-error.ts +56 -0
- package/src/server.ts +5033 -377
- package/src/session-fork-store.test.ts +43 -0
- package/src/session-fork-store.ts +51 -0
- package/src/session-project-attach-migration.test.ts +22 -0
- package/src/session-project-attach-native.test.ts +121 -0
- package/src/session-project-create-http.test.ts +407 -0
- package/src/session-project-create-native.test.ts +290 -0
- package/src/session-project-create.ts +159 -0
- package/src/session-source-admission-store.ts +58 -0
- package/src/session-source-create.test.ts +155 -0
- package/src/session-source-create.ts +97 -0
- package/src/session-source-delivery.ts +109 -0
- package/src/session-source-directory-http.test.ts +192 -0
- package/src/session-source-evidence.test.ts +898 -0
- package/src/session-source-evidence.ts +191 -0
- package/src/session-source-execution.ts +87 -0
- package/src/session-source-grant.test.ts +117 -0
- package/src/session-source-grant.ts +78 -0
- package/src/session-source-issuer.test.ts +250 -0
- package/src/session-source-issuer.ts +164 -0
- package/src/session-source-transport.ts +375 -0
- package/src/session-task-on-dispatch.ts +14 -0
- package/src/session-token-ownership-http.test.ts +122 -0
- package/src/shared/codex-inventory.ts +41 -0
- package/src/shared/daemon-provider-login.ts +31 -0
- package/src/shared/daemon-provider-snapshot.ts +52 -0
- package/src/shared/external-schedule-contract.ts +60 -0
- package/src/shared/node-name.test.ts +62 -0
- package/src/shared/node-name.ts +161 -0
- package/src/shared/probe-host-allowlist-drift.test.ts +45 -0
- package/src/shared/provider-application.ts +29 -0
- package/src/shared/provider-collection.ts +14 -0
- package/src/shared/provider-initialization.ts +8 -0
- package/src/shared/reserved-env-drift.test.ts +10 -0
- package/src/shared/scoped-secret-contract.ts +5 -0
- package/src/shared/task-content-limit-drift.test.ts +28 -0
- package/src/shared/task-content-limit.ts +25 -0
- package/src/side-thread-command-transport.test.ts +312 -0
- package/src/side-thread-command-transport.ts +318 -0
- package/src/side-thread-http-integration.test.ts +233 -0
- package/src/side-thread-http.ts +504 -0
- package/src/side-thread-production.test.ts +72 -0
- package/src/side-thread-production.ts +118 -0
- package/src/side-thread.test.ts +1067 -0
- package/src/side-thread.ts +2499 -0
- package/src/skillhub-http.test.ts +144 -0
- package/src/sqlite-close-checkpoint.test.ts +78 -0
- package/src/start-node.test.ts +265 -0
- package/src/start-request-error-domain.test.ts +182 -0
- package/src/start-skill-error-domain.test.ts +122 -0
- package/src/stats-versions-http.test.ts +65 -0
- package/src/status-alias-filter-http.test.ts +106 -0
- package/src/status-alias-resolver-http.test.ts +163 -0
- package/src/status-alias-resolver.ts +19 -0
- package/src/status-node-id-filter-http.test.ts +161 -0
- package/src/status-node-secrets-capable-http.test.ts +44 -0
- package/src/status-read-cache-http.test.ts +229 -0
- package/src/status-read-cache.ts +129 -0
- package/src/stop-delete-node.test.ts +567 -1
- package/src/task-access-http.test.ts +407 -0
- package/src/task-access.ts +420 -0
- package/src/task-activity.ts +60 -0
- package/src/task-auth-origin-http.test.ts +93 -0
- package/src/task-auth-origin.test.ts +24 -0
- package/src/task-auth-origin.ts +7 -0
- package/src/task-consumption.test.ts +512 -0
- package/src/task-diagnostic-http.test.ts +149 -0
- package/src/task-diagnostic.test.ts +115 -0
- package/src/task-diagnostic.ts +118 -0
- package/src/task-expiry-notice-http.test.ts +298 -0
- package/src/task-expiry-notice.ts +318 -0
- package/src/task-idempotency-mcp.test.ts +148 -0
- package/src/task-idempotency.test.ts +76 -0
- package/src/task-idempotency.ts +81 -0
- package/src/task-late-replies.ts +54 -0
- package/src/task-late-reply-http.test.ts +227 -0
- package/src/task-lifecycle-watcher-busy.test.ts +114 -0
- package/src/task-lifecycle-watcher.test.ts +209 -0
- package/src/task-lifecycle-watcher.ts +109 -0
- package/src/task-network-resolution-http.test.ts +140 -0
- package/src/task-not-found-error-domain.test.ts +117 -0
- package/src/task-offline-delivery-2668.test.ts +119 -0
- package/src/task-offline-delivery.ts +81 -0
- package/src/task-orphan-http.test.ts +287 -0
- package/src/task-orphan.ts +119 -0
- package/src/task-queue-ahead-http.test.ts +309 -0
- package/src/task-queue-ahead-scale.test.ts +236 -0
- package/src/task-queue-ahead.ts +186 -0
- package/src/task-rejection-log-http.test.ts +160 -0
- package/src/task-rejection-log.ts +61 -0
- package/src/task-sender-http.test.ts +143 -0
- package/src/task-stale-open-http.test.ts +292 -0
- package/src/task-stale-open.ts +103 -0
- package/src/tasks-pagination-http.test.ts +141 -0
- package/src/team-rules-status-http.test.ts +90 -0
- package/src/team-rules-status.test.ts +59 -0
- package/src/team-rules-status.ts +72 -0
- package/src/tmux-allowlist.test.ts +48 -0
- package/src/token-usage-http.test.ts +156 -0
- package/src/tool-audience-http.test.ts +266 -0
- package/src/tool-audience.ts +130 -0
- package/src/tools-list-json-schema.test.ts +94 -0
- package/src/tools.ts +4530 -471
- package/src/update-node-config-error-domain.test.ts +107 -0
- package/src/uploads-http.test.ts +6 -4
- package/src/uploads.ts +23 -4
- package/src/user-avatar-http.test.ts +74 -0
- package/src/user-avatar-image.ts +160 -0
- package/src/user-avatar-migration.test.ts +25 -0
- package/src/user-avatar-upload-http.test.ts +353 -0
- package/src/user-inbox-by-agent-http.test.ts +189 -0
- package/src/user-inbox-netid-notnull.test.ts +122 -0
- package/src/user-inbox-read-http.test.ts +223 -0
- package/src/user-inbox-schema.test.ts +54 -0
- package/src/version-stats.test.ts +66 -0
- package/src/version-stats.ts +109 -0
|
@@ -0,0 +1,150 @@
|
|
|
1
|
+
// #2840 —— 连续确定性失败 → 一条告警;且不因 timeout 或一次性失败而告警。
|
|
2
|
+
// 校准依据见模块头注释:只算确定性错误时 7 天 6 条(0.9/天);含 timeout 则是 35 条。
|
|
3
|
+
import { describe, expect, test, beforeAll, afterAll } from "bun:test";
|
|
4
|
+
import { mkdtempSync, rmSync } from "fs";
|
|
5
|
+
import { join } from "path";
|
|
6
|
+
import { tmpdir } from "os";
|
|
7
|
+
import { register } from "./auth.js";
|
|
8
|
+
import { db } from "./db.js";
|
|
9
|
+
import { scanNodeFailureAlerts, scanModelAuthBlockedNodes, classifyDeterministicFailure, NODE_FAILURE_ALERT_KIND, MODEL_AUTH_BLOCKED_CATEGORY } from "./node-failure-alert.js";
|
|
10
|
+
|
|
11
|
+
const DIR = mkdtempSync(join(tmpdir(), "anet-nfa-"));
|
|
12
|
+
let NET = "", UID = "", NODE = "";
|
|
13
|
+
|
|
14
|
+
function task(to: string, status: string, result: string, minutesAgo: number) {
|
|
15
|
+
const id = `t_nfa_${crypto.randomUUID()}`;
|
|
16
|
+
db.run(
|
|
17
|
+
`INSERT INTO tasks (task_id, from_name, to_name, priority, status, content, result, created_at, completed_at, network_id)
|
|
18
|
+
VALUES (?1,'scheduler',?2,'normal',?3,'c',?4, datetime('now', ?5), datetime('now', ?5), ?6)`,
|
|
19
|
+
[id, to, status, result, `-${minutesAgo} minutes`, NET],
|
|
20
|
+
);
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
const alerts = () => db.all<{ id: string }>(
|
|
24
|
+
"SELECT message_id AS id FROM user_inbox WHERE kind = ?1", NODE_FAILURE_ALERT_KIND,
|
|
25
|
+
);
|
|
26
|
+
|
|
27
|
+
beforeAll(() => {
|
|
28
|
+
process.env.COMMHUB_DB ||= join(DIR, "hub.db");
|
|
29
|
+
const a = register(`nfa_owner_${Date.now()}`, "NfaPassw0rd!x", undefined, "Owner");
|
|
30
|
+
expect(a.ok).toBe(true);
|
|
31
|
+
NET = a.network_id!; UID = a.user!.user_id;
|
|
32
|
+
NODE = `nfa-node-${Date.now()}`;
|
|
33
|
+
db.run("INSERT INTO nodes (node_id, node_name, alias, runtime, network_id, owner_user_id) VALUES (?1,?2,?2,'codex',?3,?4)",
|
|
34
|
+
[`n_${NODE}`, NODE, NET, UID]);
|
|
35
|
+
});
|
|
36
|
+
afterAll(() => { try { rmSync(DIR, { recursive: true, force: true }); } catch {} });
|
|
37
|
+
|
|
38
|
+
describe("#2840 节点连续确定性失败告警", () => {
|
|
39
|
+
test("分类:只认确定性错误,timeout 与任务内容错误都不算", () => {
|
|
40
|
+
expect(classifyDeterministicFailure("codex-app-server 错误: Missing environment variable: `DEEPSEEK_KEY`")).toBe("missing_env");
|
|
41
|
+
expect(classifyDeterministicFailure("spawn ENOENT")).toBe("binary_missing");
|
|
42
|
+
expect(classifyDeterministicFailure("401 Unauthorized")).toBe("login_required");
|
|
43
|
+
// #2964 —— 这是生产里真实出现的那一句(agent-node 的 codex-health 生成),
|
|
44
|
+
// 原来的正则在它上面返回 null ⇒ 凭据过期的节点一条告警都发不出去。
|
|
45
|
+
expect(classifyDeterministicFailure(
|
|
46
|
+
"codex-app-server 错误: model login expired: this node's CODEX_HOME (/home/user/.anet/nodes/n/codex-home) needs a fresh login — log in again there: CODEX_HOME=/x codex login. Not taking work until then (do not copy another node's auth.json).",
|
|
47
|
+
)).toBe("login_required");
|
|
48
|
+
expect(classifyDeterministicFailure("model login revoked: this node's CODEX_HOME needs a fresh login")).toBe("login_required");
|
|
49
|
+
// 反例:非登录类的失败不能被吸进来
|
|
50
|
+
expect(classifyDeterministicFailure("model login ok")).toBeNull();
|
|
51
|
+
expect(classifyDeterministicFailure("codex request 'turn/start' timed out")).toBeNull();
|
|
52
|
+
// #2891 —— 额度耗尽单列一类;限流(会自愈)**不能**混进来
|
|
53
|
+
expect(classifyDeterministicFailure("opencode 错误: 模型调用失败(provider.quota: The usage limit has been reached)")).toBe("quota_exhausted");
|
|
54
|
+
expect(classifyDeterministicFailure("provider.quota: Free usage limit reached.")).toBe("quota_exhausted");
|
|
55
|
+
// 限流不是额度:等几秒就好,不该惊动人
|
|
56
|
+
expect(classifyDeterministicFailure("429 Too Many Requests (rate limit)")).toBeNull();
|
|
57
|
+
// 任务正文里**提到**额度(不是错误原文)→ 不能命中
|
|
58
|
+
expect(classifyDeterministicFailure("本轮汇报:我把 provider.quota 的含义写进了 runbook")).toBeNull();
|
|
59
|
+
// 已知限制:正文**逐字引用**错误原文(含冒号)时仍会命中。
|
|
60
|
+
// 生产 7 天语料里这样的假阳性 1/507,且告警要「连续 5 次」才发,故不改判据;
|
|
61
|
+
// 这里把它钉成测试,避免以后有人误以为这是完备的语义判断。
|
|
62
|
+
expect(classifyDeterministicFailure("引用:provider.quota: The usage limit has been reached")).toBe("quota_exhausted");
|
|
63
|
+
expect(classifyDeterministicFailure("任务内容写错了")).toBeNull();
|
|
64
|
+
expect(classifyDeterministicFailure(null)).toBeNull();
|
|
65
|
+
});
|
|
66
|
+
|
|
67
|
+
test("连续 5 次缺环境变量 → 恰好一条告警;再次扫描不重复发", () => {
|
|
68
|
+
for (let i = 5; i >= 1; i--) task(NODE, "failed", "Missing environment variable: `DEEPSEEK_KEY`", i * 3);
|
|
69
|
+
const first = scanNodeFailureAlerts();
|
|
70
|
+
expect(first.nodes).toContain(NODE);
|
|
71
|
+
expect(alerts().length).toBe(1);
|
|
72
|
+
const again = scanNodeFailureAlerts();
|
|
73
|
+
expect(again.nodes).not.toContain(NODE);
|
|
74
|
+
expect(alerts().length).toBe(1);
|
|
75
|
+
});
|
|
76
|
+
|
|
77
|
+
test("timeout 连续 10 次不告警(这是校准结论的回归钉)", () => {
|
|
78
|
+
const other = `nfa-timeout-${Date.now()}`;
|
|
79
|
+
db.run("INSERT INTO nodes (node_id, node_name, alias, runtime, network_id, owner_user_id) VALUES (?1,?2,?2,'codex',?3,?4)",
|
|
80
|
+
[`n_${other}`, other, NET, UID]);
|
|
81
|
+
for (let i = 10; i >= 1; i--) task(other, "failed", "turn/start timed out", i * 2);
|
|
82
|
+
const r = scanNodeFailureAlerts();
|
|
83
|
+
expect(r.nodes).not.toContain(other);
|
|
84
|
+
expect(alerts().length).toBe(1); // 仍然只有前面那一条
|
|
85
|
+
});
|
|
86
|
+
|
|
87
|
+
test("中间出现一条成功就断开连续,不告警", () => {
|
|
88
|
+
const reset = `nfa-reset-${Date.now()}`;
|
|
89
|
+
db.run("INSERT INTO nodes (node_id, node_name, alias, runtime, network_id, owner_user_id) VALUES (?1,?2,?2,'codex',?3,?4)",
|
|
90
|
+
[`n_${reset}`, reset, NET, UID]);
|
|
91
|
+
task(reset, "failed", "Missing environment variable: `X`", 30);
|
|
92
|
+
task(reset, "failed", "Missing environment variable: `X`", 29);
|
|
93
|
+
task(reset, "replied", "ok 正常完成", 28);
|
|
94
|
+
task(reset, "failed", "Missing environment variable: `X`", 27);
|
|
95
|
+
task(reset, "failed", "Missing environment variable: `X`", 26);
|
|
96
|
+
const r = scanNodeFailureAlerts();
|
|
97
|
+
expect(r.nodes).not.toContain(reset);
|
|
98
|
+
});
|
|
99
|
+
});
|
|
100
|
+
|
|
101
|
+
// #2964 后半 —— 节点「在线但干不了活」:状态是 error、原因是登录失效,
|
|
102
|
+
// 但它一条任务都不接,所以「连续失败」那条告警路径永远不会触发。
|
|
103
|
+
describe("#2964 登录失效节点(不接活)", () => {
|
|
104
|
+
const session = (alias: string, status: string, task: string | null) => {
|
|
105
|
+
db.run(
|
|
106
|
+
`INSERT OR REPLACE INTO sessions (resume_id, alias, status, task, network_id, updated_at)
|
|
107
|
+
VALUES (?1, ?2, ?3, ?4, ?5, datetime('now'))`,
|
|
108
|
+
[`s_${alias}`, alias, status, task, NET],
|
|
109
|
+
);
|
|
110
|
+
db.run(
|
|
111
|
+
`INSERT OR IGNORE INTO nodes (node_id, node_name, alias, runtime, network_id, owner_user_id)
|
|
112
|
+
VALUES (?1, ?2, ?2, 'codex', ?3, ?4)`,
|
|
113
|
+
[`n_${alias}`, alias, NET, UID],
|
|
114
|
+
);
|
|
115
|
+
};
|
|
116
|
+
|
|
117
|
+
test("status=error 且原因是登录失效 → 告警一次;同一天再扫不重复", () => {
|
|
118
|
+
const alias = `nfa-auth-${Date.now()}`;
|
|
119
|
+
session(alias, "error", "model login expired: this node's CODEX_HOME needs a fresh login — log in again there");
|
|
120
|
+
const before = alerts().length;
|
|
121
|
+
expect(scanModelAuthBlockedNodes().nodes).toContain(alias);
|
|
122
|
+
expect(alerts().length).toBe(before + 1);
|
|
123
|
+
// 每节点每天一条:再扫一次不应新增
|
|
124
|
+
expect(scanModelAuthBlockedNodes().alerts).toBe(0);
|
|
125
|
+
});
|
|
126
|
+
|
|
127
|
+
test("idle 的节点不告警(没坏就别吵)", () => {
|
|
128
|
+
const alias = `nfa-auth-idle-${Date.now()}`;
|
|
129
|
+
session(alias, "idle", "model login expired: needs a fresh login");
|
|
130
|
+
expect(scanModelAuthBlockedNodes().nodes).not.toContain(alias);
|
|
131
|
+
});
|
|
132
|
+
|
|
133
|
+
test("error 但原因不是登录类(例如代码异常)不告警", () => {
|
|
134
|
+
const alias = `nfa-auth-other-${Date.now()}`;
|
|
135
|
+
session(alias, "error", "TypeError: cannot read properties of undefined (reading 'x')");
|
|
136
|
+
expect(scanModelAuthBlockedNodes().nodes).not.toContain(alias);
|
|
137
|
+
});
|
|
138
|
+
|
|
139
|
+
test("告警里带类别与当天日期(去重键)", () => {
|
|
140
|
+
const row = db.get<{ cat: string; day: string }>(
|
|
141
|
+
`SELECT json_extract(meta_json,'$.node_failure_alert.category') AS cat,
|
|
142
|
+
json_extract(meta_json,'$.node_failure_alert.first_seen') AS day
|
|
143
|
+
FROM user_inbox WHERE kind = ?1
|
|
144
|
+
AND json_extract(meta_json,'$.node_failure_alert.category') = ?2 LIMIT 1`,
|
|
145
|
+
NODE_FAILURE_ALERT_KIND, MODEL_AUTH_BLOCKED_CATEGORY,
|
|
146
|
+
);
|
|
147
|
+
expect(row?.cat).toBe(MODEL_AUTH_BLOCKED_CATEGORY);
|
|
148
|
+
expect(row?.day).toMatch(/^\d{4}-\d{2}-\d{2}$/);
|
|
149
|
+
});
|
|
150
|
+
});
|
|
@@ -0,0 +1,177 @@
|
|
|
1
|
+
// #2840 —— 节点「假活」告警:连续确定性失败 → 一条人类可见的通知。
|
|
2
|
+
//
|
|
3
|
+
// 现场(2026-10-28 只读统计):TMA客户端鲸 因为缺 `DEEPSEEK_KEY`,在 **9 天里失败 1433 次**,
|
|
4
|
+
// 期间没有任何告警。节点在线、进程在、版本正常、`get_session_status` 返回 idle,
|
|
5
|
+
// 失败只写在任务回执的 result 文本里 —— 巡检若只看状态就会判「没回复 = 没在做」。
|
|
6
|
+
//
|
|
7
|
+
// 设计约束(用近 7 天生产数据回放校准过,见 #2840):
|
|
8
|
+
// · **只对确定性错误告警**:缺环境变量 / 二进制缺失 / 登录失效 —— 这类不会自愈,必须人处理。
|
|
9
|
+
// 把 `timeout` 也算进去会让 7 天出现 35 条告警(其中 29 条是 timeout),等于制造新噪音源。
|
|
10
|
+
// · 阈值 5 次连续(同节点 + 同类别)。校准结果:只算确定性错误时 7 天 6 条(0.9/天),
|
|
11
|
+
// 而 TMA客户端鲸 那 1433 次会被压缩成 5 条 —— 压缩比 287:1。
|
|
12
|
+
// · 同一次故障只告一次:按 (节点, 类别, 首现时刻) 去重;彻底恢复(出现一条成功)后才重新武装。
|
|
13
|
+
// · 走已有 user_inbox 通道(与 schedule_stuck 同一套),不新造通知面。
|
|
14
|
+
import { db } from "./db.js";
|
|
15
|
+
import { sendAgentNotice } from "./agent-notice.js";
|
|
16
|
+
|
|
17
|
+
export const NODE_FAILURE_ALERT_KIND = "node_failure_alert";
|
|
18
|
+
/** 连续多少次确定性失败才告警。 */
|
|
19
|
+
export const ALERT_STREAK_THRESHOLD = 5;
|
|
20
|
+
/** 只看这么久以内的任务(更早的失败已经不可操作)。 */
|
|
21
|
+
export const SCAN_WINDOW_HOURS = 24;
|
|
22
|
+
|
|
23
|
+
type Category = "missing_env" | "binary_missing" | "login_required" | "quota_exhausted";
|
|
24
|
+
|
|
25
|
+
const PATTERNS: Array<{ category: Category; re: RegExp; label: string; hint: string }> = [
|
|
26
|
+
{ category: "missing_env", re: /Missing environment variable/i, label: "缺环境变量",
|
|
27
|
+
hint: "在节点所在机器补上该变量后重启节点(anet node start <alias>)" },
|
|
28
|
+
{ category: "binary_missing", re: /ENOENT|No such file or directory/i, label: "二进制/文件缺失",
|
|
29
|
+
hint: "检查该节点的运行时是否安装完整,必要时重装后重启节点" },
|
|
30
|
+
// #2964 —— 真实的登录失效文案长这样(agent-node/src/runtime/codex-health.ts 生成):
|
|
31
|
+
// "model login expired: this node's CODEX_HOME (/…/codex-home) needs a fresh login — log in again there: …"
|
|
32
|
+
// 原来的 /login required|Unauthorized|\b401\b/ **匹配不到这一句**:它写的是
|
|
33
|
+
// "needs a fresh login",不是 "login required"。实测 2026-10-29 一个凭据过期的节点
|
|
34
|
+
// 启动后每轮都以这句话失败,而 #2840 的告警一条都不会发 —— 分类器看不见它。
|
|
35
|
+
// 补两种状态词(expired / revoked)与那句固定结尾,仍然只认登录态,不扩大到泛化的 auth 字样。
|
|
36
|
+
{ category: "login_required", re: /login required|Unauthorized|\b401\b|model login (?:expired|revoked)|needs a fresh login/i, label: "凭据/登录失效",
|
|
37
|
+
hint: "重新登录该节点(anet node codex login-status 可先自检)" },
|
|
38
|
+
// #2891 —— 额度耗尽:与前三类同性质(不会自愈、必须人处理),但**必须与「限流」分开**。
|
|
39
|
+
//
|
|
40
|
+
// 校准(2026-10-28,近 7 天 2825 条失败):
|
|
41
|
+
// provider.quota / usage limit has been reached → 507 条(占全部失败 18%)
|
|
42
|
+
// 限流类文案(rate limit / 429 / too many requests) → 2 条
|
|
43
|
+
// ⇒ 宽泛匹配「额度|limit|quota」会到 513,多出来的 6 条是**任务正文在讨论额度**(假阳性),
|
|
44
|
+
// 所以这里只认 provider.quota / usage limit has been reached 这两句原文。
|
|
45
|
+
//
|
|
46
|
+
// 为什么不能把限流并进来:限流会自愈(等几秒重试即可),额度不会。
|
|
47
|
+
// 混成一类会让告警失去「必须人处理」这个唯一有用的性质。
|
|
48
|
+
{ category: "quota_exhausted", re: /provider\.quota:\s|usage limit has been reached/i, label: "模型额度耗尽",
|
|
49
|
+
hint: "该节点的上游额度已用完:换 key / 充值 / 换 provider 后重启节点;限流(rate limit)不属此类,会自愈" },
|
|
50
|
+
];
|
|
51
|
+
|
|
52
|
+
/** #2964 后半 —— 节点完全不接活时的告警类别(与「连续失败」区分开)。 */
|
|
53
|
+
export const MODEL_AUTH_BLOCKED_CATEGORY = "login_expired";
|
|
54
|
+
|
|
55
|
+
/**
|
|
56
|
+
* #2964 后半 —— 扫「在线但干不了活」的节点。
|
|
57
|
+
*
|
|
58
|
+
* 前一半(分类器认得出登录文案)只覆盖「节点产生了失败」的情况。
|
|
59
|
+
* 还有一种更安静的状态:节点启动后登录已失效,**一条任务都不接**,
|
|
60
|
+
* 于是 `#2840` 的「连续失败」永远不会触发 —— 但它同样干不了活。
|
|
61
|
+
*
|
|
62
|
+
* hub 侧的事实来源:agent-node 的 `gateStatusOnModelAuth` 会把这种节点的状态报成
|
|
63
|
+
* `error`,并把原因写进 `sessions.task`。这里直接读它,不做任何推断。
|
|
64
|
+
*
|
|
65
|
+
* 去重粒度:每节点每天最多一条(`first_seen` 用日期),因为这不是「某一次失败」,
|
|
66
|
+
* 而是一个需要人处理的状态;每隔 5 分钟提醒一次只会变成噪音。
|
|
67
|
+
*/
|
|
68
|
+
export function scanModelAuthBlockedNodes(now = new Date()): AlertScanResult {
|
|
69
|
+
const day = now.toISOString().slice(0, 10);
|
|
70
|
+
const rows = db.all<{ alias: string | null; network_id: string | null; task: string | null }>(
|
|
71
|
+
`SELECT alias, network_id, task FROM sessions WHERE status = 'error' AND task IS NOT NULL`,
|
|
72
|
+
);
|
|
73
|
+
const out: string[] = [];
|
|
74
|
+
for (const r of rows) {
|
|
75
|
+
const node = r.alias;
|
|
76
|
+
if (!node || !r.network_id) continue;
|
|
77
|
+
if (classifyDeterministicFailure(r.task) !== "login_required") continue;
|
|
78
|
+
const dup = db.get<{ n: number }>(
|
|
79
|
+
`SELECT COUNT(*) AS n FROM user_inbox WHERE kind = ?1 AND from_session = ?2
|
|
80
|
+
AND json_extract(meta_json, '$.node_failure_alert.category') = ?3
|
|
81
|
+
AND json_extract(meta_json, '$.node_failure_alert.first_seen') = ?4`,
|
|
82
|
+
NODE_FAILURE_ALERT_KIND, node, MODEL_AUTH_BLOCKED_CATEGORY, day,
|
|
83
|
+
)?.n ?? 0;
|
|
84
|
+
if (dup > 0) continue;
|
|
85
|
+
const owner = db.get<{ owner_user_id: string | null }>(
|
|
86
|
+
"SELECT owner_user_id FROM nodes WHERE alias = ?1 AND network_id = ?2", node, r.network_id,
|
|
87
|
+
)?.owner_user_id;
|
|
88
|
+
if (!owner) continue;
|
|
89
|
+
const sent = sendAgentNotice({
|
|
90
|
+
networkId: r.network_id,
|
|
91
|
+
userId: owner,
|
|
92
|
+
fromAlias: node,
|
|
93
|
+
kind: NODE_FAILURE_ALERT_KIND,
|
|
94
|
+
title: "节点在线但不会接活:登录失效",
|
|
95
|
+
text: `${node} 当前状态是 error,原因是登录失效。它不会接任何任务(不是慢,是干不了)。\n\n`
|
|
96
|
+
+ `节点自述:\n${String(r.task).slice(0, 300)}\n\n`
|
|
97
|
+
+ `处理建议:在该节点自己的 CODEX_HOME 里重新登录(不要拷贝别的节点的 auth.json),然后重启节点。`,
|
|
98
|
+
severity: "error",
|
|
99
|
+
meta: { node_failure_alert: { node, category: MODEL_AUTH_BLOCKED_CATEGORY, first_seen: day } },
|
|
100
|
+
idPrefix: "dm_nfa_",
|
|
101
|
+
});
|
|
102
|
+
if (sent) out.push(node);
|
|
103
|
+
}
|
|
104
|
+
return { alerts: out.length, nodes: out };
|
|
105
|
+
}
|
|
106
|
+
|
|
107
|
+
/** 只认确定性错误;timeout / 任务内容错误一律返回 null(那类会自愈或与节点无关)。 */
|
|
108
|
+
export function classifyDeterministicFailure(result: string | null | undefined): Category | null {
|
|
109
|
+
if (!result) return null;
|
|
110
|
+
for (const p of PATTERNS) if (p.re.test(result)) return p.category;
|
|
111
|
+
return null;
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
export type AlertScanResult = { alerts: number; nodes: string[] };
|
|
115
|
+
|
|
116
|
+
/**
|
|
117
|
+
* 扫一遍最近的失败,给「连续确定性失败」的节点发一次性告警。
|
|
118
|
+
* 纯读 + 发通知;不改任何任务/排程状态。
|
|
119
|
+
*/
|
|
120
|
+
export function scanNodeFailureAlerts(now = new Date()): AlertScanResult {
|
|
121
|
+
const since = new Date(now.getTime() - SCAN_WINDOW_HOURS * 3_600_000).toISOString().slice(0, 19).replace("T", " ");
|
|
122
|
+
const rows = db.all<{ to_name: string; network_id: string | null; created_at: string; status: string; result: string | null }>(
|
|
123
|
+
`SELECT to_name, network_id, created_at, status, result FROM tasks
|
|
124
|
+
WHERE created_at >= ?1 AND to_name IS NOT NULL
|
|
125
|
+
ORDER BY to_name, created_at`,
|
|
126
|
+
since,
|
|
127
|
+
);
|
|
128
|
+
// 按节点切段:streak 只数「连续」——中间出现任何非该类失败的成功/失败都要断开。
|
|
129
|
+
const streaks = new Map<string, { category: Category; count: number; firstAt: string; networkId: string | null; sample: string }>();
|
|
130
|
+
const cur = new Map<string, { category: Category; count: number; firstAt: string }>();
|
|
131
|
+
for (const r of rows) {
|
|
132
|
+
const cat = r.status === "failed" ? classifyDeterministicFailure(r.result) : null;
|
|
133
|
+
const node = r.to_name;
|
|
134
|
+
const prev = cur.get(node);
|
|
135
|
+
if (!cat) {
|
|
136
|
+
// 序列断了:连续计数和「已完成段落」都要清,否则一条成功之后旧段落仍会被告警。
|
|
137
|
+
cur.delete(node);
|
|
138
|
+
streaks.delete(node);
|
|
139
|
+
continue;
|
|
140
|
+
}
|
|
141
|
+
if (prev && prev.category === cat) {
|
|
142
|
+
prev.count += 1;
|
|
143
|
+
streaks.set(node, { category: cat, count: prev.count, firstAt: prev.firstAt, networkId: r.network_id, sample: String(r.result ?? "").slice(0, 200) });
|
|
144
|
+
} else {
|
|
145
|
+
cur.set(node, { category: cat, count: 1, firstAt: r.created_at });
|
|
146
|
+
streaks.delete(node);
|
|
147
|
+
}
|
|
148
|
+
}
|
|
149
|
+
const out: string[] = [];
|
|
150
|
+
for (const [node, s] of streaks) {
|
|
151
|
+
if (s.count < ALERT_STREAK_THRESHOLD) continue;
|
|
152
|
+
const meta = { node, category: s.category, first_seen: s.firstAt };
|
|
153
|
+
const dup = db.get<{ n: number }>(
|
|
154
|
+
`SELECT COUNT(*) AS n FROM user_inbox WHERE kind = ?1 AND from_session = ?2
|
|
155
|
+
AND json_extract(meta_json, '$.node_failure_alert.category') = ?3
|
|
156
|
+
AND json_extract(meta_json, '$.node_failure_alert.first_seen') = ?4`,
|
|
157
|
+
NODE_FAILURE_ALERT_KIND, node, s.category, s.firstAt,
|
|
158
|
+
)?.n ?? 0;
|
|
159
|
+
if (dup > 0) continue;
|
|
160
|
+
const owner = db.get<{ owner_user_id: string | null }>("SELECT owner_user_id FROM nodes WHERE alias = ?1 AND network_id = ?2", node, s.networkId)?.owner_user_id;
|
|
161
|
+
if (!owner || !s.networkId) continue;
|
|
162
|
+
const label = PATTERNS.find((p) => p.category === s.category)!;
|
|
163
|
+
const sent = sendAgentNotice({
|
|
164
|
+
networkId: s.networkId,
|
|
165
|
+
userId: owner,
|
|
166
|
+
fromAlias: node,
|
|
167
|
+
kind: NODE_FAILURE_ALERT_KIND,
|
|
168
|
+
title: `节点连续失败:${label.label}`,
|
|
169
|
+
text: `${node} 连续 ${s.count} 次以「${label.label}」失败(自 ${s.firstAt} 起)。\n\n最近一次回执节选:\n${s.sample}\n\n处理建议:${label.hint}`,
|
|
170
|
+
severity: "error",
|
|
171
|
+
meta: { node_failure_alert: meta },
|
|
172
|
+
idPrefix: "dm_nfa_",
|
|
173
|
+
});
|
|
174
|
+
if (sent) out.push(node);
|
|
175
|
+
}
|
|
176
|
+
return { alerts: out.length, nodes: out };
|
|
177
|
+
}
|