@sleep2agi/commhub-server 0.9.0-preview.22 → 0.9.0-preview.220
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +129 -9
- package/bin/commhub.ts +93 -9
- package/package.json +7 -7
- package/src/ack-create-request-transport.test.ts +182 -0
- package/src/ack-create-request.test.ts +50 -7
- package/src/admin-delete-network-http.test.ts +228 -0
- package/src/admin-networks-http.test.ts +88 -0
- package/src/adoption-candidates-http.test.ts +137 -0
- package/src/adoption-candidates.ts +158 -0
- package/src/agent-access-batch.test.ts +210 -0
- package/src/agent-access.ts +487 -0
- package/src/agent-acl-groups-http.test.ts +247 -0
- package/src/agent-acl-http.test.ts +659 -0
- package/src/agent-acl-slice3-http.test.ts +654 -0
- package/src/agent-department-http.test.ts +215 -0
- package/src/agent-groups-perf.test.ts +223 -0
- package/src/agent-notice.ts +48 -0
- package/src/agent-team-whoami-http.test.ts +98 -0
- package/src/agent-teams-http.test.ts +330 -0
- package/src/agent-teams.ts +244 -0
- package/src/alias-baseline-2571.test.ts +162 -0
- package/src/alias-baseline.ts +113 -0
- package/src/alias-filter.test.ts +72 -0
- package/src/alias-filter.ts +42 -0
- package/src/api-host-supervisors-can-create-nodes.test.ts +350 -0
- package/src/api-host-supervisors-fallback.test.ts +6 -7
- package/src/api-host-supervisors-runtime-readiness.test.ts +195 -0
- package/src/api-nodes-shape.test.ts +120 -13
- package/src/auth-password-change-cli-tokens-http.test.ts +423 -0
- package/src/auth-sessions-http.test.ts +250 -0
- package/src/auth-validate.test.ts +5 -1
- package/src/auth-weak-password-login.test.ts +176 -0
- package/src/auth.ts +609 -51
- package/src/avatar-validate.test.ts +52 -2
- package/src/avatar-validate.ts +35 -1
- package/src/backfill-node-ids-2664.test.ts +128 -0
- package/src/binding-generation-http.test.ts +121 -0
- package/src/blocked-exit-on-reply.test.ts +122 -0
- package/src/broadcast-fanout-2721.test.ts +149 -0
- package/src/broadcast-fanout.ts +117 -0
- package/src/chain-attempt-dedup.test.ts +447 -0
- package/src/clone-node.test.ts +248 -0
- package/src/clone-node.ts +289 -0
- package/src/codex-config-flags.ts +77 -0
- package/src/codex-fork-contract.ts +23 -0
- package/src/codex-fork-migration.test.ts +38 -0
- package/src/commhub-bin-args.test.ts +85 -0
- package/src/config-apply-validate.test.ts +188 -1
- package/src/config-apply-validate.ts +222 -3
- package/src/cors-origins.test.ts +40 -0
- package/src/create-node-blocked-gate.test.ts +356 -0
- package/src/create-node-network-id.test.ts +91 -0
- package/src/create-node-tool-schema.test.ts +49 -0
- package/src/create-node-unicode-name.test.ts +120 -0
- package/src/create-node-validate.test.ts +199 -7
- package/src/create-node-validate.ts +232 -15
- package/src/create-node-workdir.test.ts +289 -0
- package/src/create-node.ts +88 -23
- package/src/daemon-provider-snapshots.test.ts +421 -0
- package/src/daemon-provider-snapshots.ts +327 -0
- package/src/daemon-token-owner-http.test.ts +236 -0
- package/src/dashboard-slash-routing-http.test.ts +109 -0
- package/src/db-adapter-guard.test.ts +148 -0
- package/src/db-adapter-translate.test.ts +128 -0
- package/src/db-adapter.ts +377 -91
- package/src/db-timestamp.test.ts +38 -0
- package/src/db-timestamp.ts +15 -0
- package/src/db.ts +1296 -153
- package/src/delivered-without-push.test.ts +140 -0
- package/src/department-groups-http.test.ts +209 -0
- package/src/department-groups-sync-http.test.ts +328 -0
- package/src/department-groups.ts +434 -0
- package/src/department-heads-http.test.ts +350 -0
- package/src/department-heads.ts +226 -0
- package/src/departments-agent-count.test.ts +132 -0
- package/src/departments-http.test.ts +185 -0
- package/src/departments-mcp.test.ts +128 -0
- package/src/departments.ts +318 -0
- package/src/desktop-message-seam.test.ts +150 -0
- package/src/desktop-message.test.ts +368 -0
- package/src/dm-file-scope-http.test.ts +178 -0
- package/src/empty-node-sweeper.test.ts +104 -0
- package/src/empty-node-sweeper.ts +88 -0
- package/src/env-transport-gate-http.test.ts +240 -0
- package/src/env-transport-gate.ts +80 -0
- package/src/env-transport-policy.ts +56 -0
- package/src/etag-of.test.ts +57 -0
- package/src/event-loop-lag.test.ts +180 -0
- package/src/event-loop-lag.ts +201 -0
- package/src/ext-token-shared.test.ts +60 -0
- package/src/ext-validation-regression.test.ts +131 -0
- package/src/external-schedule-edits-http.test.ts +321 -0
- package/src/external-schedule-edits-timestamps.test.ts +64 -0
- package/src/external-schedule-edits.ts +327 -0
- package/src/file-download-authz-dev-open.test.ts +135 -0
- package/src/file-download-authz.test.ts +404 -0
- package/src/file-download-date-bucket.test.ts +436 -0
- package/src/file-network-scope-dev-open.test.ts +97 -0
- package/src/file-network-scope.test.ts +732 -0
- package/src/group-membership-events-http.test.ts +264 -0
- package/src/group-messages-http.test.ts +482 -0
- package/src/group-messages.ts +297 -0
- package/src/health-anon-readers.ts +59 -0
- package/src/health-host-disk-2756.test.ts +71 -0
- package/src/health-redaction.test.ts +1574 -2
- package/src/host-supervisor-identity-http.test.ts +152 -0
- package/src/host-supervisor-identity.ts +92 -0
- package/src/host-supervisors-daemon-version.test.ts +137 -0
- package/src/http-gzip.test.ts +94 -0
- package/src/http-gzip.ts +112 -0
- package/src/hub-catalog-read-http.test.ts +266 -0
- package/src/hub-provider-config.test.ts +156 -0
- package/src/hub-provider-config.ts +124 -0
- package/src/hub-timestamp.test.ts +38 -0
- package/src/hub-timestamp.ts +27 -0
- package/src/human-dm.ts +215 -0
- package/src/human-inbox.ts +42 -0
- package/src/inbox-caller.ts +49 -0
- package/src/inbox-count-sse.test.ts +310 -0
- package/src/inbox-count.ts +29 -0
- package/src/index.ts +6 -2
- package/src/license-state.test.ts +56 -0
- package/src/license-state.ts +48 -0
- package/src/list-host-supervisors.test.ts +15 -4
- package/src/mcp-context-cost-http.test.ts +127 -0
- package/src/mcp-inflight.test.ts +92 -0
- package/src/mcp-inflight.ts +38 -0
- package/src/mcp-projects-events-http.test.ts +165 -0
- package/src/mcp-route-label-http.test.ts +120 -0
- package/src/mcp-write-network-resolution.test.ts +435 -0
- package/src/member-audit-network-id-http.test.ts +93 -0
- package/src/member-presence-http.test.ts +210 -0
- package/src/messages-ack-by-agent-http.test.ts +142 -0
- package/src/migrate-sqlite-to-pg.test.ts +47 -0
- package/src/migrate-sqlite-to-pg.ts +263 -0
- package/src/model-auth-notify-http.test.ts +181 -0
- package/src/model-auth-notify.ts +116 -0
- package/src/network-name-http.test.ts +75 -0
- package/src/network-rules-http.test.ts +169 -0
- package/src/network-rules.ts +129 -0
- package/src/network-scope.ts +276 -0
- package/src/network-token-membership-http.test.ts +139 -0
- package/src/node-alias-control.ts +12 -0
- package/src/node-attrs-validate.ts +88 -0
- package/src/node-attrs.test.ts +258 -0
- package/src/node-caller-identity-http.test.ts +1091 -0
- package/src/node-create-request-status-http.test.ts +105 -0
- package/src/node-daemon-bindings.test.ts +132 -0
- package/src/node-daemon-bindings.ts +135 -0
- package/src/node-failure-alert.test.ts +150 -0
- package/src/node-failure-alert.ts +177 -0
- package/src/node-files-transport.test.ts +344 -0
- package/src/node-health-codex-login.test.ts +120 -0
- package/src/node-health-dispatch-http.test.ts +335 -0
- package/src/node-health-guard.ts +121 -0
- package/src/node-health-passthrough.test.ts +115 -0
- package/src/node-health-store.ts +101 -0
- package/src/node-id-alias.test.ts +190 -0
- package/src/node-identity-conflict-http.test.ts +269 -0
- package/src/node-identity-conflict.ts +142 -0
- package/src/node-identity-mismatch.test.ts +74 -0
- package/src/node-in-flight-tasks.ts +31 -0
- package/src/node-lifecycle-controllable.test.ts +64 -0
- package/src/node-lifecycle-controllable.ts +53 -0
- package/src/node-lifecycle-read-http.test.ts +250 -0
- package/src/node-lifecycle-read.ts +124 -0
- package/src/node-logs-transport.test.ts +263 -0
- package/src/node-peer-allowlist.test.ts +114 -0
- package/src/node-permissions-http.test.ts +728 -0
- package/src/node-permissions.ts +494 -0
- package/src/node-request-retention.test.ts +281 -0
- package/src/node-request-retention.ts +120 -0
- package/src/node-secret-rest-http.test.ts +190 -0
- package/src/node-serviceability.test.ts +86 -0
- package/src/node-serviceability.ts +124 -0
- package/src/node-session-control-http.test.ts +310 -0
- package/src/node-session-control.ts +258 -0
- package/src/node-skills-transport.test.ts +212 -0
- package/src/node-token-identity.ts +35 -0
- package/src/node-token-ownership.ts +51 -0
- package/src/node-token-privilege-http.test.ts +158 -0
- package/src/not-found-api-http.test.ts +67 -0
- package/src/observer-avatar-http.test.ts +102 -9
- package/src/observer-push.test.ts +76 -0
- package/src/opencode-catalog-report-http.test.ts +139 -0
- package/src/opencode-v2-catalog.test.ts +268 -0
- package/src/opencode-v2-catalog.ts +356 -0
- package/src/package-version-consistency.test.ts +25 -0
- package/src/parent-source-marker.test.ts +50 -0
- package/src/password-network-tokens.ts +25 -0
- package/src/peer-reply-atomic.test.ts +457 -0
- package/src/peer-reply-capability.test.ts +70 -0
- package/src/pg-worker.ts +87 -0
- package/src/poll-pacing.test.ts +74 -0
- package/src/poll-pacing.ts +123 -0
- package/src/project-dir-warn-clear-http.test.ts +132 -0
- package/src/project-viewer-can-http.test.ts +116 -0
- package/src/push.test.ts +20 -10
- package/src/push.ts +382 -9
- package/src/redact-tokens.test.ts +86 -0
- package/src/redact-tokens.ts +43 -0
- package/src/rejection-counters.test.ts +72 -0
- package/src/rejection-counters.ts +173 -0
- package/src/rename.ts +38 -10
- package/src/reply-target-derivation.test.ts +96 -0
- package/src/report-status-handover-capabilities-http.test.ts +124 -0
- package/src/report-status-host-schema.test.ts +235 -0
- package/src/report-status-resume-id-handover.test.ts +184 -0
- package/src/report-status-secret-keys.test.ts +140 -0
- package/src/require-explicit-test-db.ts +9 -0
- package/src/requirement-due-reminders-http.test.ts +620 -0
- package/src/requirement-due-reminders.ts +416 -0
- package/src/requirement-events-http.test.ts +295 -0
- package/src/requirement-events.test.ts +45 -0
- package/src/requirement-events.ts +144 -0
- package/src/requirement-notify.ts +165 -0
- package/src/requirement-participant-notify-http.test.ts +309 -0
- package/src/requirement-tag-ops-http.test.ts +425 -0
- package/src/requirement-tag-tree-http.test.ts +275 -0
- package/src/requirement-tag-tree.ts +127 -0
- package/src/requirement-tags.ts +102 -0
- package/src/requirements-abandoned-http.test.ts +236 -0
- package/src/requirements-comment-http.test.ts +167 -0
- package/src/requirements-completed-at-migration.test.ts +70 -0
- package/src/requirements-errors-mcp-http.test.ts +164 -0
- package/src/requirements-errors.ts +158 -0
- package/src/requirements-etag-http.test.ts +91 -0
- package/src/requirements-http.test.ts +709 -0
- package/src/requirements-last-event-http.test.ts +349 -0
- package/src/requirements-list-mcp-slim-http.test.ts +155 -0
- package/src/requirements-list-slim-http.test.ts +324 -0
- package/src/requirements-mcp.test.ts +163 -0
- package/src/requirements-migrate.ts +283 -0
- package/src/requirements-owner-strict-mcp-http.test.ts +145 -0
- package/src/requirements-people-http.test.ts +114 -0
- package/src/requirements-people-mcp-http.test.ts +231 -0
- package/src/requirements-people.ts +196 -0
- package/src/requirements-priority-lowest.test.ts +114 -0
- package/src/requirements-projects-network-echo.test.ts +57 -0
- package/src/requirements-projects-parent-http.test.ts +123 -0
- package/src/requirements-search-http.test.ts +182 -0
- package/src/requirements-search.test.ts +32 -0
- package/src/requirements-search.ts +145 -0
- package/src/requirements-seq-http.test.ts +193 -0
- package/src/requirements-seq-migration.test.ts +63 -0
- package/src/requirements-start-date.test.ts +125 -0
- package/src/requirements-stats-http.test.ts +357 -0
- package/src/requirements-stats.ts +195 -0
- package/src/requirements-status-participants-http.test.ts +222 -0
- package/src/requirements-upgrade-e2e.test.ts +126 -0
- package/src/requirements-upgrade-migration.test.ts +98 -0
- package/src/requirements.ts +2042 -0
- package/src/resolve-port.test.ts +57 -0
- package/src/resolve-port.ts +44 -0
- package/src/rest-alias-network-fallback-http.test.ts +123 -0
- package/src/rest-alias-network-fallback.test.ts +60 -0
- package/src/rest-alias-network-fallback.ts +65 -0
- package/src/rest-explicit-columns-http.test.ts +266 -0
- package/src/rest-identity.test.ts +130 -0
- package/src/rest-identity.ts +141 -0
- package/src/rest-projections.ts +72 -0
- package/src/rest-read-network-real-chain.test.ts +97 -0
- package/src/rest-write-network-resolution.test.ts +133 -0
- package/src/restart-readiness-http.test.ts +121 -0
- package/src/restricted-files.ts +76 -0
- package/src/retention-first-sweep-quiet.test.ts +31 -0
- package/src/retention.test.ts +100 -40
- package/src/retention.ts +423 -104
- package/src/route-callers-http.test.ts +120 -0
- package/src/route-timing.test.ts +218 -0
- package/src/route-timing.ts +262 -0
- package/src/rules-file-duplicate-alias.test.ts +234 -0
- package/src/rules-file-session-target.test.ts +207 -0
- package/src/rules-file-transport.test.ts +237 -0
- package/src/runtime-label.test.ts +22 -0
- package/src/runtime-label.ts +21 -0
- package/src/runtime-readiness.ts +90 -0
- package/src/schedule-agent-mcp-http.test.ts +432 -0
- package/src/schedule-agent.ts +252 -0
- package/src/schedule-starved.test.ts +92 -0
- package/src/schedule-starved.ts +114 -0
- package/src/scheduled-backoff-http.test.ts +193 -0
- package/src/scheduled-backoff.ts +64 -0
- package/src/scheduled-doorbell-nodeless-session.test.ts +134 -0
- package/src/scheduled-failure-backoff-2622.test.ts +145 -0
- package/src/scheduled-failure-code.test.ts +73 -0
- package/src/scheduled-failure-code.ts +105 -0
- package/src/scheduled-failures-http.test.ts +279 -0
- package/src/scheduled-failures.ts +425 -0
- package/src/scheduled-noprogress-backoff-2622.test.ts +163 -0
- package/src/scheduled-offline-backoff-probe.test.ts +56 -0
- package/src/scheduled-offline.test.ts +131 -0
- package/src/scheduled-offline.ts +66 -0
- package/src/scheduled-outcome-guidance-http.test.ts +139 -0
- package/src/scheduled-outcome-guidance.ts +9 -0
- package/src/scheduled-reply-unread-http.test.ts +212 -0
- package/src/scheduled-requirement-wake-http.test.ts +279 -0
- package/src/scheduled-requirement-wake.ts +125 -0
- package/src/scheduled-run-terminal.test.ts +268 -0
- package/src/scheduled-skip-blocking-task.test.ts +120 -0
- package/src/scheduled-skip-path-backoff-2622.test.ts +98 -0
- package/src/scheduled-stuck-http.test.ts +316 -0
- package/src/scheduled-stuck-notice-cap.test.ts +80 -0
- package/src/scheduled-stuck.ts +219 -0
- package/src/scheduled-tasks-http.test.ts +679 -0
- package/src/scheduled-tasks.ts +959 -0
- package/src/scheduled-work-outcome-http.test.ts +154 -0
- package/src/scheduled-work-outcome.ts +50 -0
- package/src/scoped-secrets.ts +141 -0
- package/src/send-dedup-request-id.test.ts +229 -0
- package/src/send-reply-agent-warning.test.ts +202 -0
- package/src/send-reply-alias-optional.test.ts +131 -0
- package/src/send-reply-attachments.test.ts +458 -0
- package/src/send_dedup.test.ts +54 -0
- package/src/send_dedup.ts +52 -6
- package/src/serve-error.test.ts +73 -0
- package/src/serve-error.ts +56 -0
- package/src/server.ts +4921 -344
- package/src/session-task-on-dispatch.ts +14 -0
- package/src/session-token-ownership-http.test.ts +122 -0
- package/src/shared/codex-inventory.ts +41 -0
- package/src/shared/daemon-provider-login.ts +31 -0
- package/src/shared/daemon-provider-snapshot.ts +52 -0
- package/src/shared/external-schedule-contract.ts +60 -0
- package/src/shared/node-name.test.ts +62 -0
- package/src/shared/node-name.ts +161 -0
- package/src/shared/probe-host-allowlist-drift.test.ts +45 -0
- package/src/shared/provider-application.ts +29 -0
- package/src/shared/provider-collection.ts +14 -0
- package/src/shared/provider-initialization.ts +8 -0
- package/src/shared/reserved-env-drift.test.ts +10 -0
- package/src/shared/scoped-secret-contract.ts +5 -0
- package/src/shared/task-content-limit-drift.test.ts +28 -0
- package/src/shared/task-content-limit.ts +25 -0
- package/src/side-thread-command-transport.test.ts +312 -0
- package/src/side-thread-command-transport.ts +318 -0
- package/src/side-thread-http-integration.test.ts +233 -0
- package/src/side-thread-http.ts +504 -0
- package/src/side-thread-production.test.ts +72 -0
- package/src/side-thread-production.ts +118 -0
- package/src/side-thread.test.ts +1067 -0
- package/src/side-thread.ts +2499 -0
- package/src/skillhub-http.test.ts +142 -0
- package/src/sqlite-close-checkpoint.test.ts +78 -0
- package/src/start-node.test.ts +264 -0
- package/src/status-alias-filter-http.test.ts +106 -0
- package/src/status-alias-resolver-http.test.ts +163 -0
- package/src/status-alias-resolver.ts +19 -0
- package/src/status-node-id-filter-http.test.ts +161 -0
- package/src/status-read-cache-http.test.ts +229 -0
- package/src/status-read-cache.ts +129 -0
- package/src/stop-delete-node.test.ts +567 -1
- package/src/task-access-http.test.ts +402 -0
- package/src/task-access.ts +420 -0
- package/src/task-activity.ts +60 -0
- package/src/task-auth-origin-http.test.ts +93 -0
- package/src/task-auth-origin.test.ts +24 -0
- package/src/task-auth-origin.ts +7 -0
- package/src/task-consumption.test.ts +490 -0
- package/src/task-diagnostic-http.test.ts +149 -0
- package/src/task-diagnostic.test.ts +115 -0
- package/src/task-diagnostic.ts +118 -0
- package/src/task-expiry-notice-http.test.ts +287 -0
- package/src/task-expiry-notice.ts +315 -0
- package/src/task-idempotency-mcp.test.ts +142 -0
- package/src/task-idempotency.test.ts +76 -0
- package/src/task-idempotency.ts +81 -0
- package/src/task-late-replies.ts +54 -0
- package/src/task-late-reply-http.test.ts +227 -0
- package/src/task-lifecycle-watcher-busy.test.ts +114 -0
- package/src/task-lifecycle-watcher.test.ts +204 -0
- package/src/task-lifecycle-watcher.ts +109 -0
- package/src/task-network-resolution-http.test.ts +140 -0
- package/src/task-offline-delivery-2668.test.ts +119 -0
- package/src/task-offline-delivery.ts +81 -0
- package/src/task-orphan-http.test.ts +287 -0
- package/src/task-orphan.ts +119 -0
- package/src/task-queue-ahead-http.test.ts +309 -0
- package/src/task-queue-ahead-scale.test.ts +236 -0
- package/src/task-queue-ahead.ts +186 -0
- package/src/task-rejection-log-http.test.ts +160 -0
- package/src/task-rejection-log.ts +58 -0
- package/src/task-sender-http.test.ts +143 -0
- package/src/task-stale-open-http.test.ts +292 -0
- package/src/task-stale-open.ts +103 -0
- package/src/tasks-pagination-http.test.ts +141 -0
- package/src/team-rules-status-http.test.ts +90 -0
- package/src/team-rules-status.test.ts +59 -0
- package/src/team-rules-status.ts +72 -0
- package/src/tmux-allowlist.test.ts +48 -0
- package/src/tool-audience-http.test.ts +230 -0
- package/src/tool-audience.ts +127 -0
- package/src/tools-list-json-schema.test.ts +94 -0
- package/src/tools.ts +4320 -413
- package/src/upload-early-reject-guard.test.ts +155 -0
- package/src/uploads-http.test.ts +86 -7
- package/src/uploads.ts +103 -6
- package/src/user-avatar-http.test.ts +74 -0
- package/src/user-avatar-image.ts +160 -0
- package/src/user-avatar-migration.test.ts +25 -0
- package/src/user-avatar-upload-http.test.ts +348 -0
- package/src/user-inbox-by-agent-http.test.ts +161 -0
- package/src/user-inbox-netid-notnull.test.ts +122 -0
- package/src/user-inbox-read-http.test.ts +223 -0
- package/src/user-inbox-schema.test.ts +54 -0
- package/src/version-stats.test.ts +66 -0
- package/src/version-stats.ts +109 -0
|
@@ -0,0 +1,227 @@
|
|
|
1
|
+
import { afterAll, beforeAll, expect, test } from "bun:test";
|
|
2
|
+
import { mkdtempSync } from "node:fs";
|
|
3
|
+
|
|
4
|
+
process.env.COMMHUB_DB ||= `${mkdtempSync("/tmp/late710-")}/hub.db`;
|
|
5
|
+
|
|
6
|
+
let db: any;
|
|
7
|
+
let hub: ReturnType<typeof Bun.serve>;
|
|
8
|
+
let base = "";
|
|
9
|
+
let owner: any;
|
|
10
|
+
let outsider: any;
|
|
11
|
+
let workerToken = "";
|
|
12
|
+
let siblingToken = "";
|
|
13
|
+
const workerId = "n_late710_worker";
|
|
14
|
+
const siblingId = "n_late710_sibling";
|
|
15
|
+
const taskId = "task_late710_terminal";
|
|
16
|
+
const threadId = "thread_late710_exact";
|
|
17
|
+
const turnId = "turn_late710_exact";
|
|
18
|
+
|
|
19
|
+
async function mcp(token: string, name: string, args: Record<string, unknown>) {
|
|
20
|
+
const response = await fetch(`${base}/mcp`, {
|
|
21
|
+
method: "POST",
|
|
22
|
+
headers: {
|
|
23
|
+
Authorization: `Bearer ${token}`,
|
|
24
|
+
"Content-Type": "application/json",
|
|
25
|
+
Accept: "application/json, text/event-stream",
|
|
26
|
+
"MCP-Protocol-Version": "2025-03-26",
|
|
27
|
+
},
|
|
28
|
+
body: JSON.stringify({ jsonrpc: "2.0", id: 1, method: "tools/call", params: { name, arguments: args } }),
|
|
29
|
+
});
|
|
30
|
+
expect(response.status).toBe(200);
|
|
31
|
+
const raw = await response.text();
|
|
32
|
+
const events = raw.split("\n").filter((line) => line.startsWith("data:"));
|
|
33
|
+
const envelope = JSON.parse(events.length ? events.at(-1)!.slice(5) : raw);
|
|
34
|
+
expect(envelope.error).toBeUndefined();
|
|
35
|
+
return JSON.parse(envelope.result.content[0].text);
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
async function get(token: string, path: string) {
|
|
39
|
+
const response = await fetch(`${base}${path}`, { headers: { Authorization: `Bearer ${token}` } });
|
|
40
|
+
return { status: response.status, body: await response.json() as any };
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
beforeAll(async () => {
|
|
44
|
+
({ db } = await import("./db.js"));
|
|
45
|
+
const { register } = await import("./auth.js");
|
|
46
|
+
const { generateNetworkToken, hashToken } = await import("./db.js");
|
|
47
|
+
owner = register("late710_owner", "Fixture-Strong1!");
|
|
48
|
+
outsider = register("late710_outside", "Fixture-Strong1!");
|
|
49
|
+
expect(owner.ok && outsider.ok).toBe(true);
|
|
50
|
+
|
|
51
|
+
const addNode = (nodeId: string, alias: string) => {
|
|
52
|
+
db.run(`INSERT INTO nodes(node_id,node_name,alias,network_id,owner_user_id,lifecycle_state)
|
|
53
|
+
VALUES(?1,?2,?2,?3,?4,'active')`, [nodeId, alias, owner.network_id, owner.user.user_id]);
|
|
54
|
+
db.run(`INSERT INTO sessions(resume_id,alias,node_id,network_id,status,last_seen_at)
|
|
55
|
+
VALUES(?1,?2,?1,?3,'idle',datetime('now'))`, [nodeId, alias, owner.network_id]);
|
|
56
|
+
const raw = generateNetworkToken();
|
|
57
|
+
db.run(`INSERT INTO api_tokens(token_id,user_id,network_id,scope,name,token_hash,bound_node_id,node_identity_epoch)
|
|
58
|
+
VALUES(?1,?2,?3,'network',?4,?5,?1,2)`, [nodeId, owner.user.user_id, owner.network_id, `node:${alias}`, hashToken(raw)]);
|
|
59
|
+
return raw;
|
|
60
|
+
};
|
|
61
|
+
workerToken = addNode(workerId, "late710-worker");
|
|
62
|
+
siblingToken = addNode(siblingId, "late710-sibling");
|
|
63
|
+
db.run(`INSERT INTO tasks
|
|
64
|
+
(task_id,from_name,to_name,to_node_id,status,content,result,network_id,thread_id,turn_id,completed_at)
|
|
65
|
+
VALUES(?1,?2,'late710-worker',?3,'failed','original task','watchdog interrupted',?4,?5,?6,datetime('now'))`,
|
|
66
|
+
[taskId, owner.user.username, workerId, owner.network_id, threadId, turnId]);
|
|
67
|
+
|
|
68
|
+
const { bootServer } = await import("./server.js");
|
|
69
|
+
hub = bootServer({ port: 0, hostname: "127.0.0.1" });
|
|
70
|
+
base = `http://127.0.0.1:${hub.port}`;
|
|
71
|
+
}, 60_000);
|
|
72
|
+
|
|
73
|
+
afterAll(() => hub?.stop(true));
|
|
74
|
+
|
|
75
|
+
test("exact executor and exact turn append one late receipt without rewriting terminal task", async () => {
|
|
76
|
+
const before = db.get("SELECT status,result,completed_at FROM tasks WHERE task_id=?1", taskId);
|
|
77
|
+
const accepted = await mcp(workerToken, "send_reply", {
|
|
78
|
+
in_reply_to: taskId,
|
|
79
|
+
text: "recovered final result",
|
|
80
|
+
status: "replied",
|
|
81
|
+
thread_id: threadId,
|
|
82
|
+
turn_id: turnId,
|
|
83
|
+
});
|
|
84
|
+
expect(accepted).toMatchObject({ ok: true, late: true, duplicate: false });
|
|
85
|
+
expect(db.get("SELECT status,result,completed_at FROM tasks WHERE task_id=?1", taskId)).toEqual(before);
|
|
86
|
+
expect(db.get("SELECT COUNT(*) AS n FROM task_events WHERE task_id=?1", taskId).n).toBe(0);
|
|
87
|
+
|
|
88
|
+
const detail = await get(owner.token, `/api/tasks/${taskId}`);
|
|
89
|
+
expect(detail.status).toBe(200);
|
|
90
|
+
expect(detail.body.task.late_replies).toEqual([expect.objectContaining({
|
|
91
|
+
late: true, task_id: taskId, from_node_id: workerId,
|
|
92
|
+
thread_id: threadId, turn_id: turnId, status: "replied", result: "recovered final result",
|
|
93
|
+
})]);
|
|
94
|
+
const legacyDetail = await get(owner.token, `/api/tasks?task_id=${taskId}&limit=1&network_id=${owner.network_id}`);
|
|
95
|
+
expect(legacyDetail.body.tasks[0].late_replies).toEqual(detail.body.task.late_replies);
|
|
96
|
+
const mcpDetail = await mcp(owner.token, "get_task", { task_id: taskId });
|
|
97
|
+
// 2026-10-31 —— **条件键要显式登记**(#3171):get_task 的 `pacing` 只在有 pacing 时出现,
|
|
98
|
+
// 所以这里不能写「键集恒等于三键」(那也是外部复核说的「条件键」)。
|
|
99
|
+
// 断言写成:两个必需键必在,且**除 pacing 外不许有别的键**。
|
|
100
|
+
const ks = Object.keys(mcpDetail);
|
|
101
|
+
expect(ks).toContain("ok");
|
|
102
|
+
expect(ks).toContain("task");
|
|
103
|
+
expect(ks.filter((k) => !["ok", "task", "pacing"].includes(k))).toEqual([]);
|
|
104
|
+
expect(mcpDetail.task.late_replies).toEqual(detail.body.task.late_replies);
|
|
105
|
+
|
|
106
|
+
const messages = await get(owner.token, `/api/messages?alias=${encodeURIComponent(owner.user.username)}&since=2000-01-01%2000:00:00`);
|
|
107
|
+
expect(messages.body.messages).toContainEqual(expect.objectContaining({ id: accepted.message_id, late: true, content: "recovered final result" }));
|
|
108
|
+
const inbox = await mcp(owner.token, "get_inbox", { alias: owner.user.username, limit: 100 });
|
|
109
|
+
expect(inbox.messages).toContainEqual(expect.objectContaining({ id: accepted.message_id, late: true, content: "recovered final result" }));
|
|
110
|
+
});
|
|
111
|
+
|
|
112
|
+
test("exact retry is idempotent and a changed payload conflicts", async () => {
|
|
113
|
+
const retry = await mcp(workerToken, "send_reply", {
|
|
114
|
+
in_reply_to: taskId, text: "recovered final result", status: "replied", thread_id: threadId, turn_id: turnId,
|
|
115
|
+
});
|
|
116
|
+
expect(retry).toMatchObject({ ok: true, late: true, duplicate: true });
|
|
117
|
+
expect(db.get("SELECT COUNT(*) AS n FROM task_late_replies WHERE task_id=?1", taskId).n).toBe(1);
|
|
118
|
+
expect(db.get("SELECT COUNT(*) AS n FROM inbox WHERE in_reply_to=?1 AND content='recovered final result'", taskId).n).toBe(1);
|
|
119
|
+
expect(await mcp(workerToken, "send_reply", {
|
|
120
|
+
in_reply_to: taskId, text: "different result", status: "replied", thread_id: threadId, turn_id: turnId,
|
|
121
|
+
})).toMatchObject({ ok: false, error: "late_reply_conflict" });
|
|
122
|
+
});
|
|
123
|
+
|
|
124
|
+
test("wrong executor, sibling turn, cross-network reader and legacy shape cannot append", async () => {
|
|
125
|
+
const count = () => db.get("SELECT COUNT(*) AS n FROM task_late_replies").n;
|
|
126
|
+
const before = count();
|
|
127
|
+
expect(await mcp(siblingToken, "send_reply", {
|
|
128
|
+
in_reply_to: taskId, text: "impostor", thread_id: threadId, turn_id: turnId,
|
|
129
|
+
})).toMatchObject({ ok: false, error: "late_reply_task_not_owned" });
|
|
130
|
+
expect(await mcp(workerToken, "send_reply", {
|
|
131
|
+
in_reply_to: taskId, text: "wrong turn", thread_id: threadId, turn_id: "turn_late710_sibling",
|
|
132
|
+
})).toMatchObject({ ok: false, error: "late_reply_context_mismatch" });
|
|
133
|
+
expect(await mcp(workerToken, "send_reply", {
|
|
134
|
+
in_reply_to: taskId, text: "legacy terminal reply",
|
|
135
|
+
})).toMatchObject({ ok: false, error: "reply_task_terminal" });
|
|
136
|
+
expect(await mcp(owner.token, "send_reply", {
|
|
137
|
+
in_reply_to: taskId, text: "human cannot forge runtime evidence", thread_id: threadId, turn_id: turnId,
|
|
138
|
+
})).toMatchObject({ ok: false, error: "late_reply_node_token_required" });
|
|
139
|
+
// Length bound is enforced in the handler (kept out of tools/list for #476).
|
|
140
|
+
for (const bad of ["", "x".repeat(201)]) {
|
|
141
|
+
expect(await mcp(workerToken, "send_reply", {
|
|
142
|
+
in_reply_to: taskId, text: "bad identity", thread_id: bad, turn_id: turnId,
|
|
143
|
+
})).toMatchObject({ ok: false, error: "late_reply_context_invalid" });
|
|
144
|
+
}
|
|
145
|
+
expect(count()).toBe(before);
|
|
146
|
+
expect((await get(outsider.token, `/api/tasks/${taskId}`)).status).toBe(404);
|
|
147
|
+
expect((await mcp(outsider.token, "get_task", { task_id: taskId })).ok).toBe(false);
|
|
148
|
+
});
|
|
149
|
+
|
|
150
|
+
test("nonterminal legacy reply retains its established behavior", async () => {
|
|
151
|
+
const openId = "task_late710_open";
|
|
152
|
+
db.run(`INSERT INTO tasks(task_id,from_name,to_name,to_node_id,status,content,network_id)
|
|
153
|
+
VALUES(?1,?2,'late710-worker',?3,'running','legacy open',?4)`, [openId, owner.user.username, workerId, owner.network_id]);
|
|
154
|
+
const result = await mcp(workerToken, "send_reply", { in_reply_to: openId, text: "ordinary final" });
|
|
155
|
+
expect(result).toMatchObject({ ok: true });
|
|
156
|
+
expect(result.late).toBeUndefined();
|
|
157
|
+
expect(db.get("SELECT status,result FROM tasks WHERE task_id=?1", openId)).toEqual({ status: "replied", result: "ordinary final" });
|
|
158
|
+
expect(db.get("SELECT COUNT(*) AS n FROM task_late_replies WHERE task_id=?1", openId).n).toBe(0);
|
|
159
|
+
});
|
|
160
|
+
|
|
161
|
+
// #3061 —— 任务**从未记下 turn 上下文**时,迟到结果也必须能收。
|
|
162
|
+
//
|
|
163
|
+
// 现场(2026-10-30 生产实测):
|
|
164
|
+
// 「仍在 TUI 会话里运行,等待回复已达 30 分钟上限」的失败任务 289 条,
|
|
165
|
+
// 其中 thread_id/turn_id 非空的 **0 条**;「任务未提交」781 条里只有 1 条有。
|
|
166
|
+
// ⇒ 这类任务的迟到结果永远匹配不上上下文,等于「活干完了但结果必定丢弃」。
|
|
167
|
+
// 全库 task_late_replies 只有 4 行,正是这条路走不通的证据。
|
|
168
|
+
test("既有任务没有 turn 上下文时,迟到结果仍可落账并投递给发送方", async () => {
|
|
169
|
+
const contextlessId = "task_late710_noctx";
|
|
170
|
+
db.run(`INSERT INTO tasks
|
|
171
|
+
(task_id,from_name,to_name,to_node_id,status,content,result,network_id,thread_id,turn_id,completed_at)
|
|
172
|
+
VALUES(?1,?2,'late710-worker',?3,'failed','never started a turn','共享会话忙,30 分钟内未提交',?4,NULL,NULL,datetime('now'))`,
|
|
173
|
+
[contextlessId, owner.user.username, workerId, owner.network_id]);
|
|
174
|
+
|
|
175
|
+
const accepted = await mcp(workerToken, "send_reply", {
|
|
176
|
+
in_reply_to: contextlessId,
|
|
177
|
+
text: "结果其实跑完了",
|
|
178
|
+
status: "replied",
|
|
179
|
+
thread_id: "thread_from_runtime_any",
|
|
180
|
+
turn_id: "turn_from_runtime_any",
|
|
181
|
+
});
|
|
182
|
+
expect(accepted).toMatchObject({ ok: true, late: true, duplicate: false });
|
|
183
|
+
// 任务行不被改写
|
|
184
|
+
expect(db.get("SELECT status,result FROM tasks WHERE task_id=?1", contextlessId))
|
|
185
|
+
.toEqual({ status: "failed", result: "共享会话忙,30 分钟内未提交" });
|
|
186
|
+
// 落进 append-only 账,并带上运行时给的上下文
|
|
187
|
+
const row = db.get("SELECT thread_id,turn_id,status,result FROM task_late_replies WHERE task_id=?1", contextlessId);
|
|
188
|
+
expect(row).toMatchObject({ thread_id: "thread_from_runtime_any", turn_id: "turn_from_runtime_any", status: "replied", result: "结果其实跑完了" });
|
|
189
|
+
// 发送方能在 inbox 里看到它(否则「活干完了没人知道」)
|
|
190
|
+
const inbox = await mcp(owner.token, "get_inbox", { alias: owner.user.username, limit: 100 });
|
|
191
|
+
expect(inbox.messages).toContainEqual(expect.objectContaining({ id: accepted.message_id, late: true, content: "结果其实跑完了" }));
|
|
192
|
+
// 不是这个任务的负责人,仍然拒(放宽只针对上下文,不放宽归属)
|
|
193
|
+
expect(await mcp(siblingToken, "send_reply", {
|
|
194
|
+
in_reply_to: contextlessId, text: "impostor", thread_id: "t", turn_id: "u",
|
|
195
|
+
})).toMatchObject({ ok: false, error: "late_reply_task_not_owned" });
|
|
196
|
+
});
|
|
197
|
+
|
|
198
|
+
// #3061 续 —— 任务**从未记下 turn 上下文**时,不该再要求调用方凭空给出 thread/turn。
|
|
199
|
+
//
|
|
200
|
+
// 现场(2026-10-30 通信Dash欧):它对一条 failed 任务调 send_reply 补结果,
|
|
201
|
+
// 拿到 `reply_task_terminal`(因为没带 thread_id/turn_id),只好先 retry_task 把任务
|
|
202
|
+
// 重置成 delivered 再回复 —— 账本里那条从 failed 变成 retried→replied,
|
|
203
|
+
// **「当初失败过」这个事实被抹掉了**。而 #3244 只放宽了「id 对不上」,没放宽「必须给 id」。
|
|
204
|
+
test("既有任务没有 turn 上下文时,不带 thread/turn 也能补迟到结果", async () => {
|
|
205
|
+
const noIdTask = "task_late710_noid";
|
|
206
|
+
db.run(`INSERT INTO tasks
|
|
207
|
+
(task_id,from_name,to_name,to_node_id,status,content,result,network_id,thread_id,turn_id,completed_at)
|
|
208
|
+
VALUES(?1,?2,'late710-worker',?3,'failed','timed out','[超时] 已达 30 分钟上限',?4,NULL,NULL,datetime('now'))`,
|
|
209
|
+
[noIdTask, owner.user.username, workerId, owner.network_id]);
|
|
210
|
+
|
|
211
|
+
const accepted = await mcp(workerToken, "send_reply", {
|
|
212
|
+
in_reply_to: noIdTask,
|
|
213
|
+
text: "没带 id 也应该收下",
|
|
214
|
+
status: "replied",
|
|
215
|
+
});
|
|
216
|
+
expect(accepted).toMatchObject({ ok: true, late: true, duplicate: false });
|
|
217
|
+
const row = db.get("SELECT status,result FROM task_late_replies WHERE task_id=?1", noIdTask);
|
|
218
|
+
expect(row).toMatchObject({ status: "replied", result: "没带 id 也应该收下" });
|
|
219
|
+
// 任务行仍不改写
|
|
220
|
+
expect(db.get("SELECT status FROM tasks WHERE task_id=?1", noIdTask)).toEqual({ status: "failed" });
|
|
221
|
+
|
|
222
|
+
// 反例:任务**有**上下文时,仍然必须给匹配的 id(不能被这次放宽带偏)
|
|
223
|
+
const mismatch = await mcp(siblingToken, "send_reply", {
|
|
224
|
+
in_reply_to: taskId, text: "impostor", thread_id: threadId, turn_id: turnId,
|
|
225
|
+
});
|
|
226
|
+
expect(mismatch).toMatchObject({ ok: false, error: "late_reply_task_not_owned" });
|
|
227
|
+
});
|
|
@@ -0,0 +1,114 @@
|
|
|
1
|
+
// #2571 —— delivered_stale 的**分流**(阈值 10m/30m,见 task-lifecycle-watcher.ts 顶部说明):目标在跑别的任务时不再报“可能离线”。
|
|
2
|
+
//
|
|
3
|
+
// 生产数据(24h):出现过 delivered_stale 的任务里 83% 最终成功,等待 p50 2.9 分钟、
|
|
4
|
+
// p90 27 分钟 ⇒ 事件事实成立,但把“目标正忙”写成“可能离线”会制造告警疲劳。
|
|
5
|
+
// 这里钉住三件事:
|
|
6
|
+
// 1. 目标 `working` ⇒ 写 `queued_behind_busy_<N>s`,**不写** `delivered_stale_<N>s`;
|
|
7
|
+
// 2. 目标 `idle`/`offline`/无 session ⇒ 写 `delivered_stale_<N>s`,**不写** busy 类;
|
|
8
|
+
// 3. 两类都仍然是 write-once(同一 task+阈值重复巡逻不重复写)。
|
|
9
|
+
|
|
10
|
+
import { afterAll, beforeAll, describe, expect, test } from "bun:test";
|
|
11
|
+
import { db } from "./db.js";
|
|
12
|
+
import { recordDeliveredStaleEvents } from "./task-lifecycle-watcher.js";
|
|
13
|
+
|
|
14
|
+
const NOW = new Date("2026-08-10T01:00:01.000Z");
|
|
15
|
+
let networkId = "";
|
|
16
|
+
|
|
17
|
+
function sqliteUtcTimestamp(date: Date): string {
|
|
18
|
+
return date.toISOString().slice(0, 19).replace("T", " ");
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
function insertTask(taskId: string, toName: string, deliveredAt: string) {
|
|
22
|
+
db.run(
|
|
23
|
+
`INSERT INTO tasks (task_id, from_name, to_name, status, content, delivered_at, network_id)
|
|
24
|
+
VALUES (?1, 'sender', ?2, 'delivered', 'busy-vs-stale fixture', ?3, ?4)`,
|
|
25
|
+
[taskId, toName, deliveredAt, networkId],
|
|
26
|
+
);
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
function insertSession(alias: string, status: string) {
|
|
30
|
+
db.run(
|
|
31
|
+
`INSERT INTO sessions (resume_id, alias, status, network_id, registered_at)
|
|
32
|
+
VALUES (?1, ?2, ?3, ?4, '2026-08-10 00:00:00')`,
|
|
33
|
+
[`res_${alias}`, alias, status, networkId],
|
|
34
|
+
);
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
function eventsFor(taskId: string): string[] {
|
|
38
|
+
return db
|
|
39
|
+
.all<{ event_type: string }>(`SELECT event_type FROM task_events WHERE task_id = ?1 ORDER BY event_type`, [taskId])
|
|
40
|
+
.map((row) => row.event_type);
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
beforeAll(async () => {
|
|
44
|
+
const { register } = await import("./auth.js");
|
|
45
|
+
const auth = register(`busy_watcher_${Date.now()}`, "BusyWatcher-Strong-1!", undefined, "watcher");
|
|
46
|
+
expect(auth.ok).toBe(true);
|
|
47
|
+
networkId = auth.network_id!;
|
|
48
|
+
|
|
49
|
+
// 目标在跑别的任务
|
|
50
|
+
insertSession("target-busy", "working");
|
|
51
|
+
insertTask("busy-600", "target-busy", "2026-08-10 00:50:01");
|
|
52
|
+
// 目标空闲却没开始(真异常)
|
|
53
|
+
insertSession("target-idle", "idle");
|
|
54
|
+
insertTask("idle-600", "target-idle", "2026-08-10 00:50:01");
|
|
55
|
+
// 目标掉线
|
|
56
|
+
insertSession("target-offline", "offline");
|
|
57
|
+
insertTask("offline-600", "target-offline", "2026-08-10 00:50:01");
|
|
58
|
+
// 完全没有 session 行(未知)
|
|
59
|
+
insertTask("unknown-600", "target-without-session", "2026-08-10 00:50:01");
|
|
60
|
+
}, 30_000);
|
|
61
|
+
|
|
62
|
+
afterAll(() => {
|
|
63
|
+
try {
|
|
64
|
+
db.run(`DELETE FROM task_events WHERE network_id = ?1`, [networkId]);
|
|
65
|
+
db.run(`DELETE FROM tasks WHERE network_id = ?1`, [networkId]);
|
|
66
|
+
db.run(`DELETE FROM sessions WHERE network_id = ?1`, [networkId]);
|
|
67
|
+
} catch {}
|
|
68
|
+
});
|
|
69
|
+
|
|
70
|
+
describe("#2571 stale vs queued-behind-busy", () => {
|
|
71
|
+
test("目标 working ⇒ 只写 queued_behind_busy_*,不写 delivered_stale_*", () => {
|
|
72
|
+
recordDeliveredStaleEvents(NOW);
|
|
73
|
+
const events = eventsFor("busy-600");
|
|
74
|
+
expect(events).toContain("task.warning.queued_behind_busy_600s");
|
|
75
|
+
expect(events).not.toContain("task.warning.delivered_stale_600s");
|
|
76
|
+
});
|
|
77
|
+
|
|
78
|
+
test("目标 idle / offline / 无 session ⇒ 只写 delivered_stale_*", () => {
|
|
79
|
+
recordDeliveredStaleEvents(NOW);
|
|
80
|
+
for (const taskId of ["idle-600", "offline-600", "unknown-600"]) {
|
|
81
|
+
const events = eventsFor(taskId);
|
|
82
|
+
expect(events).toContain("task.warning.delivered_stale_600s");
|
|
83
|
+
expect(events).not.toContain("task.warning.queued_behind_busy_600s");
|
|
84
|
+
}
|
|
85
|
+
});
|
|
86
|
+
|
|
87
|
+
test("两类都仍然是 write-once(重复巡逻不重复写)", () => {
|
|
88
|
+
const before = db.get<{ n: number }>(
|
|
89
|
+
`SELECT COUNT(*) AS n FROM task_events WHERE network_id = ?1 AND event_type LIKE 'task.warning.%'`,
|
|
90
|
+
[networkId],
|
|
91
|
+
)!.n;
|
|
92
|
+
const second = recordDeliveredStaleEvents(NOW);
|
|
93
|
+
const after = db.get<{ n: number }>(
|
|
94
|
+
`SELECT COUNT(*) AS n FROM task_events WHERE network_id = ?1 AND event_type LIKE 'task.warning.%'`,
|
|
95
|
+
[networkId],
|
|
96
|
+
)!.n;
|
|
97
|
+
expect(second.inserted).toBe(0);
|
|
98
|
+
expect(after).toBe(before);
|
|
99
|
+
});
|
|
100
|
+
|
|
101
|
+
test("detail 文案能区分“排队”和“没在跑”", () => {
|
|
102
|
+
const busy = db.get<{ detail: string }>(
|
|
103
|
+
`SELECT detail FROM task_events WHERE task_id='busy-600' AND event_type LIKE 'task.warning.queued_behind_busy_%'`,
|
|
104
|
+
);
|
|
105
|
+
const stale = db.get<{ detail: string }>(
|
|
106
|
+
`SELECT detail FROM task_events WHERE task_id='idle-600' AND event_type LIKE 'task.warning.delivered_stale_%'`,
|
|
107
|
+
);
|
|
108
|
+
expect(busy!.detail).toContain("queued after");
|
|
109
|
+
expect(stale!.detail).toContain("still delivered");
|
|
110
|
+
// 两类文案必须一眼可分:忙的那条不能说成“没在跑”,反之亦然
|
|
111
|
+
expect(busy!.detail).not.toContain("still delivered");
|
|
112
|
+
expect(stale!.detail).not.toContain("queued after");
|
|
113
|
+
});
|
|
114
|
+
});
|
|
@@ -0,0 +1,204 @@
|
|
|
1
|
+
import { afterAll, beforeAll, describe, expect, test } from "bun:test";
|
|
2
|
+
import { Database } from "bun:sqlite";
|
|
3
|
+
import { db, logTaskEvent } from "./db.js";
|
|
4
|
+
import { TASK_EVENT_REST_COLUMNS } from "./rest-projections.js";
|
|
5
|
+
import { recordDeliveredStaleEvents } from "./task-lifecycle-watcher.js";
|
|
6
|
+
|
|
7
|
+
const NOW = new Date("2026-08-10T01:00:01.000Z");
|
|
8
|
+
let server: ReturnType<typeof Bun.serve>;
|
|
9
|
+
let base = "";
|
|
10
|
+
let token = "";
|
|
11
|
+
let networkId = "";
|
|
12
|
+
|
|
13
|
+
function insertTask(taskId: string, status: string, deliveredAt: string | null, network = networkId) {
|
|
14
|
+
db.run(
|
|
15
|
+
`INSERT INTO tasks
|
|
16
|
+
(task_id, from_name, to_name, status, content, delivered_at, network_id)
|
|
17
|
+
VALUES (?1, 'sender', 'target', ?2, 'watcher fixture', ?3, ?4)`,
|
|
18
|
+
[taskId, status, deliveredAt, network],
|
|
19
|
+
);
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
function sqliteUtcTimestamp(date: Date): string {
|
|
23
|
+
return date.toISOString().slice(0, 19).replace("T", " ");
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
// 30s 而不是 bun 默认的 5s:这个 hook 跑 register()(KDF)+bootServer(),
|
|
27
|
+
// 本机空载基线 ~620ms,而 2026-08-17 与 2026-08-31 两次 CI 上它被 5s 上限打死
|
|
28
|
+
// (0 pass / 1 fail / Ran 1 test —— 见 #928、#1627)。5247ms 是**被截断的下界**,
|
|
29
|
+
// 不是测量值,所以余量按基线的 ~48x 取,而不是按那个数取。
|
|
30
|
+
beforeAll(async () => {
|
|
31
|
+
const { register } = await import("./auth.js");
|
|
32
|
+
const auth = register(`watcher_owner_${Date.now()}`, "WatcherOwner-Strong-1!", undefined, "watcher");
|
|
33
|
+
expect(auth.ok).toBe(true);
|
|
34
|
+
token = auth.token!;
|
|
35
|
+
networkId = auth.network_id!;
|
|
36
|
+
|
|
37
|
+
insertTask("stale-599", "delivered", "2026-08-10 00:50:02");
|
|
38
|
+
insertTask("stale-600", "delivered", "2026-08-10 00:50:01");
|
|
39
|
+
insertTask("stale-1800", "delivered", "2026-08-10 00:30:01");
|
|
40
|
+
insertTask("stale-acked", "acked", "2026-08-10 00:00:00");
|
|
41
|
+
insertTask("stale-no-delivery", "delivered", null);
|
|
42
|
+
|
|
43
|
+
const mod = await import("./server.js");
|
|
44
|
+
server = mod.bootServer({ port: 0, hostname: "127.0.0.1" });
|
|
45
|
+
base = `http://127.0.0.1:${server.port}`;
|
|
46
|
+
}, 30_000);
|
|
47
|
+
|
|
48
|
+
afterAll(() => {
|
|
49
|
+
try { server?.stop(true); } catch {}
|
|
50
|
+
});
|
|
51
|
+
|
|
52
|
+
/**
|
|
53
|
+
* 轮询到 `ready()` 为真,或到期抛错。
|
|
54
|
+
*
|
|
55
|
+
* 定长 sleep 的问题不是「慢」,是**报错报在错的层**:超时之后测试红在
|
|
56
|
+
* 「事件没写」这条断言上,读的人会去查 watcher,而真实原因可能是子进程
|
|
57
|
+
* 还没起来。这里到期时把那句话直接说出来。
|
|
58
|
+
*/
|
|
59
|
+
async function waitUntil(ready: () => boolean, timeoutMs: number, what: string | null): Promise<void> {
|
|
60
|
+
const deadline = Date.now() + timeoutMs;
|
|
61
|
+
for (;;) {
|
|
62
|
+
if (ready()) return;
|
|
63
|
+
if (Date.now() >= deadline) {
|
|
64
|
+
// what === null:到期本身就是期望结果(调用方随后自己断言),不抛。
|
|
65
|
+
if (what === null) return;
|
|
66
|
+
throw new Error(`timed out after ${timeoutMs}ms: ${what}`);
|
|
67
|
+
}
|
|
68
|
+
await Bun.sleep(25);
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
describe("#167 Hub delivered-stale lifecycle watcher", () => {
|
|
73
|
+
test("10m/30m thresholds are exact and non-delivered tasks stay silent", () => {
|
|
74
|
+
const first = recordDeliveredStaleEvents(NOW);
|
|
75
|
+
expect(first.inserted).toBe(3);
|
|
76
|
+
expect(first.by_threshold_seconds).toEqual({ 600: 2, 1800: 1 });
|
|
77
|
+
|
|
78
|
+
expect(db.get<{ count: number }>("SELECT COUNT(*) AS count FROM task_events WHERE task_id = 'stale-599'")!.count).toBe(0);
|
|
79
|
+
expect(db.get<{ count: number }>("SELECT COUNT(*) AS count FROM task_events WHERE task_id = 'stale-600'")!.count).toBe(1);
|
|
80
|
+
expect(db.get<{ count: number }>("SELECT COUNT(*) AS count FROM task_events WHERE task_id = 'stale-1800'")!.count).toBe(2);
|
|
81
|
+
expect(db.get<{ count: number }>("SELECT COUNT(*) AS count FROM task_events WHERE task_id = 'stale-acked'")!.count).toBe(0);
|
|
82
|
+
expect(db.get<{ count: number }>("SELECT COUNT(*) AS count FROM task_events WHERE task_id = 'stale-no-delivery'")!.count).toBe(0);
|
|
83
|
+
|
|
84
|
+
const sixty = db.all<any>("SELECT * FROM task_events WHERE task_id = 'stale-1800' ORDER BY event_type");
|
|
85
|
+
expect(sixty.map((row) => row.event_type)).toEqual([
|
|
86
|
+
"task.warning.delivered_stale_1800s",
|
|
87
|
+
"task.warning.delivered_stale_600s",
|
|
88
|
+
]);
|
|
89
|
+
expect(sixty.map((row) => row.event_key)).toEqual(sixty.map((row) => row.event_type));
|
|
90
|
+
expect(sixty.every((row) => row.from_status === "delivered" && row.to_status === "delivered")).toBe(true);
|
|
91
|
+
expect(sixty.every((row) => row.actor === "patrol")).toBe(true);
|
|
92
|
+
expect(sixty.every((row) => row.network_id === networkId)).toBe(true);
|
|
93
|
+
});
|
|
94
|
+
|
|
95
|
+
test("a second patrol is write-once for each task and threshold", () => {
|
|
96
|
+
const second = recordDeliveredStaleEvents(NOW);
|
|
97
|
+
expect(second.inserted).toBe(0);
|
|
98
|
+
expect(second.by_threshold_seconds).toEqual({ 600: 0, 1800: 0 });
|
|
99
|
+
expect(db.get<{ count: number }>(
|
|
100
|
+
"SELECT COUNT(*) AS count FROM task_events WHERE event_type LIKE 'task.warning.delivered_stale_%'",
|
|
101
|
+
)!.count).toBe(3);
|
|
102
|
+
});
|
|
103
|
+
|
|
104
|
+
test("ordinary lifecycle events get standard event names without a dedupe key", () => {
|
|
105
|
+
logTaskEvent("stale-600", null, "delivered", "target");
|
|
106
|
+
logTaskEvent("stale-600", "delivered", "acked", "target");
|
|
107
|
+
logTaskEvent("stale-600", "acked", "running", "target");
|
|
108
|
+
logTaskEvent("stale-600", "running", "replied", "target");
|
|
109
|
+
logTaskEvent("stale-600", "running", "failed", "target");
|
|
110
|
+
logTaskEvent("stale-600", "delivered", "expired", "target");
|
|
111
|
+
const rows = db.all<any>(
|
|
112
|
+
"SELECT event_type, event_key FROM task_events WHERE task_id = 'stale-600' AND actor = 'target' ORDER BY id",
|
|
113
|
+
);
|
|
114
|
+
expect(rows).toEqual([
|
|
115
|
+
{ event_type: "task.send.delivered", event_key: null },
|
|
116
|
+
{ event_type: "task.ack", event_key: null },
|
|
117
|
+
{ event_type: "task.started", event_key: null },
|
|
118
|
+
{ event_type: "task.replied", event_key: null },
|
|
119
|
+
{ event_type: "task.failed", event_key: null },
|
|
120
|
+
{ event_type: "task.expired", event_key: null },
|
|
121
|
+
]);
|
|
122
|
+
});
|
|
123
|
+
|
|
124
|
+
test("REST exposes event_type but keeps the internal idempotency key private", async () => {
|
|
125
|
+
expect(TASK_EVENT_REST_COLUMNS).toContain("event_type");
|
|
126
|
+
expect(TASK_EVENT_REST_COLUMNS).not.toContain("event_key" as any);
|
|
127
|
+
const response = await fetch(
|
|
128
|
+
`${base}/api/task_events?network_id=${encodeURIComponent(networkId)}&task_id=stale-1800`,
|
|
129
|
+
{ headers: { Authorization: `Bearer ${token}` } },
|
|
130
|
+
);
|
|
131
|
+
expect(response.status).toBe(200);
|
|
132
|
+
const body = await response.json() as any;
|
|
133
|
+
expect(body.events).toHaveLength(2);
|
|
134
|
+
expect(body.events.every((event: any) => typeof event.event_type === "string")).toBe(true);
|
|
135
|
+
expect(body.events.every((event: any) => event.event_key === undefined)).toBe(true);
|
|
136
|
+
});
|
|
137
|
+
|
|
138
|
+
test("startHub owns a live watcher timer instead of relying on import side effects", async () => {
|
|
139
|
+
const childDbPath = `/tmp/test683-live-${process.pid}-${Date.now()}.db`;
|
|
140
|
+
const init = Bun.spawnSync(["bun", "-e", "await import('./server/src/db.js')"], {
|
|
141
|
+
cwd: process.cwd(),
|
|
142
|
+
env: { ...process.env, COMMHUB_DB: childDbPath },
|
|
143
|
+
stdout: "pipe",
|
|
144
|
+
stderr: "pipe",
|
|
145
|
+
});
|
|
146
|
+
expect(init.exitCode).toBe(0);
|
|
147
|
+
|
|
148
|
+
const child = Bun.spawn(["bun", "run", "server/src/index.ts"], {
|
|
149
|
+
cwd: process.cwd(),
|
|
150
|
+
env: {
|
|
151
|
+
...process.env,
|
|
152
|
+
COMMHUB_DB: childDbPath,
|
|
153
|
+
COMMHUB_DELIVERED_STALE_PATROL_MS: "25",
|
|
154
|
+
PORT: "0",
|
|
155
|
+
},
|
|
156
|
+
stdout: "pipe",
|
|
157
|
+
stderr: "pipe",
|
|
158
|
+
});
|
|
159
|
+
// 这里**不**等「hub 就绪」——因为没有一个只有 hub 起来了才成立的廉价判据:
|
|
160
|
+
// db 文件在上面那步 init 里就已经存在了,拿它当条件的话这个等待恒真,等于没等。
|
|
161
|
+
// 真正需要「hub 起来了」的是下面那条断言,而它已经改成轮询到目标状态,
|
|
162
|
+
// hub 起得慢只是让它多等几轮。
|
|
163
|
+
//
|
|
164
|
+
// 这一步保留的是原来那条断言的原意:**子进程没有立刻崩**。所以只给它一个
|
|
165
|
+
// 短窗口,并且如果它在窗口内退出就立刻停下来报错,不用把 800ms 睡满。
|
|
166
|
+
await waitUntil(() => child.exitCode !== null, 800, null);
|
|
167
|
+
expect(child.exitCode).toBeNull();
|
|
168
|
+
|
|
169
|
+
const taskId = "stale-live-wiring";
|
|
170
|
+
const childDb = new Database(childDbPath);
|
|
171
|
+
try {
|
|
172
|
+
// startHub may still be finishing its startup migration transaction even
|
|
173
|
+
// though the process has stayed alive. Wait for that short-lived writer
|
|
174
|
+
// instead of turning scheduler timing into an immediate SQLITE_BUSY.
|
|
175
|
+
childDb.exec("PRAGMA busy_timeout = 5000");
|
|
176
|
+
childDb.run(
|
|
177
|
+
`INSERT INTO tasks
|
|
178
|
+
(task_id, from_name, to_name, status, content, delivered_at, network_id)
|
|
179
|
+
VALUES (?1, 'sender', 'target', 'delivered', 'live watcher fixture', ?2, ?3)`,
|
|
180
|
+
[taskId, sqliteUtcTimestamp(new Date(Date.now() - 601_000)), networkId],
|
|
181
|
+
);
|
|
182
|
+
expect(childDb.query<{ count: number }, [string]>(
|
|
183
|
+
"SELECT COUNT(*) AS count FROM task_events WHERE task_id = ?1",
|
|
184
|
+
).get(taskId)!.count).toBe(0);
|
|
185
|
+
// 巡检周期是 COMMHUB_DELIVERED_STALE_PATROL_MS=25ms —— 事件在插入后
|
|
186
|
+
// 几十毫秒内就该出现。原来这里是定长 sleep(3_200),纯粹是余量:
|
|
187
|
+
// 4.0s 的 sleep 装在 bun 默认的 5.0s 单测预算里,只剩 1s 给两次进程启动。
|
|
188
|
+
// 实测在 CI 上被这一条打红过(#798 让这个文件第一次进 CI 才暴露)。
|
|
189
|
+
// 改成「轮询到目标状态,或到期报错」:常态下快 ~60 倍,慢的时候等得起。
|
|
190
|
+
const countStale = () => childDb.query<{ count: number }, [string]>(
|
|
191
|
+
"SELECT COUNT(*) AS count FROM task_events WHERE task_id = ?1 AND event_type = 'task.warning.delivered_stale_600s'",
|
|
192
|
+
).get(taskId)!.count;
|
|
193
|
+
await waitUntil(() => countStale() === 1, 20_000,
|
|
194
|
+
"watcher did not write the delivered_stale_600s event");
|
|
195
|
+
expect(countStale()).toBe(1);
|
|
196
|
+
} finally {
|
|
197
|
+
childDb.close();
|
|
198
|
+
try { child.kill("SIGTERM"); } catch {}
|
|
199
|
+
await child.exited;
|
|
200
|
+
}
|
|
201
|
+
// 🔴 显式超时:这一条要起两个真 bun 进程,bun 默认的 5s 对它不成立。
|
|
202
|
+
// 上面两处已改成轮询,常态用不到这个上限;它只保证「慢」不会被报成「坏」。
|
|
203
|
+
}, 30_000);
|
|
204
|
+
});
|
|
@@ -0,0 +1,109 @@
|
|
|
1
|
+
import { db } from "./db.js";
|
|
2
|
+
import { ALIAS_BASELINE_TABLE, BASELINE_MAX_SECONDS, BASELINE_P99_FACTOR, ensureAliasBaselineTable } from "./alias-baseline.js";
|
|
3
|
+
|
|
4
|
+
export const DELIVERED_STALE_THRESHOLDS = [600, 1800] as const;
|
|
5
|
+
|
|
6
|
+
export type DeliveredStalePatrolResult = {
|
|
7
|
+
inserted: number;
|
|
8
|
+
by_threshold_seconds: Record<(typeof DELIVERED_STALE_THRESHOLDS)[number], number>;
|
|
9
|
+
};
|
|
10
|
+
|
|
11
|
+
function sqliteUtcTimestamp(date: Date): string {
|
|
12
|
+
return date.toISOString().slice(0, 19).replace("T", " ");
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
// #2571(第二轮)—— 阈值从 30s/60s 提到 10min/30min。
|
|
16
|
+
//
|
|
17
|
+
// 第一轮只做了“按目标忙/闲分流”,上线后(19:45–22:30)实测**告警量反而翻倍**
|
|
18
|
+
// (33/h → 73/h,33/h → 104/h),因为同一个任务会先在忙时写 busy、目标转闲后
|
|
19
|
+
// 再写一次 stale(46 个任务两类都出现)。而被标 stale 的任务里仍有 73.5%
|
|
20
|
+
// 最终成功。真正的病因不是“忙 vs 闲”,是**阈值本身离真实分布太远**:
|
|
21
|
+
// 7 天 11877 个成功任务,delivered → 首次活动 p50 = 1 秒、p90 = 6 秒、
|
|
22
|
+
// p95 = 57 秒、p99 = 12 分钟。
|
|
23
|
+
// ⇒ 30 秒的判据会命中 6.4% 的**正常**任务,60 秒命中 4.9%,等于把 p95–p99
|
|
24
|
+
// 的长尾全报警。改成 10 分钟(p99 之下)才有意义:超过它的比例 1.2%,
|
|
25
|
+
// 30 分钟是 0.2%,只留真异常。
|
|
26
|
+
//
|
|
27
|
+
// event_key 里带秒数 ⇒ 改阈值会产生新的 key,历史事件不受影响、可并存对比。
|
|
28
|
+
/**
|
|
29
|
+
* Persist Hub-authoritative warnings for tasks which are still delivered.
|
|
30
|
+
*
|
|
31
|
+
* The task row, not an agent-local timer, is the source of truth. event_key is
|
|
32
|
+
* equal to the stable event name and protected by a unique index, so every
|
|
33
|
+
* threshold is write-once across restarts, patrols, and multiple Hub workers.
|
|
34
|
+
*/
|
|
35
|
+
/**
|
|
36
|
+
* #2571 —— 把「慢」和「坏」分开:目标正在跑别的任务时,事件写成 `queued_behind_busy_*`;
|
|
37
|
+
* 目标空闲/离线/未知时,才写成 `delivered_stale_*`。
|
|
38
|
+
*
|
|
39
|
+
* 为什么是一条 SQL(而不是两条 INSERT):`test683` 的 mutation 门用**字符串锚点**钉住
|
|
40
|
+
* “write-once 冲突门”“只认 delivered”“网络字段透传”三件事;拆成两条 INSERT 会让锚点
|
|
41
|
+
* 命中次数变成 0 或让 mutation 只命中一半 —— 那就把“可被变异测试覆盖”这条性质弄丢了。
|
|
42
|
+
* 因此这里仍然只保留**一条** INSERT,用 CASE 选择事件名与文案。
|
|
43
|
+
*/
|
|
44
|
+
function thresholdLabel(thresholdSeconds: 600 | 1800): string {
|
|
45
|
+
return thresholdSeconds === 600 ? "10m" : "30m";
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
function recordStale(thresholdSeconds: 600 | 1800, cutoff: string, referenceIso: string): number {
|
|
49
|
+
const busyType = `task.warning.queued_behind_busy_${thresholdSeconds}s`;
|
|
50
|
+
const staleType = `task.warning.delivered_stale_${thresholdSeconds}s`;
|
|
51
|
+
const label = thresholdLabel(thresholdSeconds);
|
|
52
|
+
const busyDetail = `queued after ${label}; target is working on another task`;
|
|
53
|
+
const staleDetail = thresholdSeconds === 600
|
|
54
|
+
? `still delivered after ${label}; target is not working on another task and has not started`
|
|
55
|
+
: `still delivered after ${label}; target is idle or offline and not consuming`;
|
|
56
|
+
const result = db.run(
|
|
57
|
+
`INSERT INTO task_events
|
|
58
|
+
(task_id, from_status, to_status, event_type, event_key, actor, detail, network_id)
|
|
59
|
+
SELECT task_id, 'delivered', 'delivered',
|
|
60
|
+
CASE WHEN COALESCE((SELECT s.status FROM sessions s
|
|
61
|
+
WHERE s.alias = tasks.to_name AND s.network_id = tasks.network_id), 'unknown') = 'working'
|
|
62
|
+
THEN ?4 ELSE ?5 END,
|
|
63
|
+
CASE WHEN COALESCE((SELECT s.status FROM sessions s
|
|
64
|
+
WHERE s.alias = tasks.to_name AND s.network_id = tasks.network_id), 'unknown') = 'working'
|
|
65
|
+
THEN ?4 ELSE ?5 END,
|
|
66
|
+
'patrol',
|
|
67
|
+
CASE WHEN COALESCE((SELECT s.status FROM sessions s
|
|
68
|
+
WHERE s.alias = tasks.to_name AND s.network_id = tasks.network_id), 'unknown') = 'working'
|
|
69
|
+
THEN ?6 ELSE ?7 END,
|
|
70
|
+
network_id
|
|
71
|
+
FROM tasks
|
|
72
|
+
-- 这个子查询把列名改成 b_* 前缀:SELECT 列表里的 network_id 必须继续是 tasks 的
|
|
73
|
+
-- (test683 的 L9 变异门用 network_id + FROM tasks 相邻两行当锚点,裸列名一改锚点就失效)。
|
|
74
|
+
LEFT JOIN (SELECT alias AS b_alias, network_id AS b_network, p99_seconds AS b_p99
|
|
75
|
+
FROM alias_latency_baseline) b
|
|
76
|
+
ON b.b_alias = tasks.to_name AND b.b_network = tasks.network_id
|
|
77
|
+
WHERE status = 'delivered'
|
|
78
|
+
AND delivered_at IS NOT NULL
|
|
79
|
+
AND delivered_at <= ?3
|
|
80
|
+
-- #2571 第三轮:目标自己的 p99 基线比全局下限更宽时,用它(全队变慢不炸屏,
|
|
81
|
+
-- 单节点卡住仍会显形)。没有基线行 ⇒ 与 .163 行为完全一致。
|
|
82
|
+
AND (
|
|
83
|
+
b.b_p99 IS NULL
|
|
84
|
+
OR delivered_at <= datetime(?8, '-' || CAST(b.b_p99 * 1.5 AS INTEGER) || ' seconds')
|
|
85
|
+
-- 上限:再慢的节点,等满 30 分钟也必须报(否则 p99=50min 的节点永远不响)。
|
|
86
|
+
OR delivered_at <= datetime(?8, '-' || CAST(?9 AS INTEGER) || ' seconds')
|
|
87
|
+
)
|
|
88
|
+
ON CONFLICT(task_id, event_key) DO NOTHING`,
|
|
89
|
+
[staleType, staleDetail, cutoff, busyType, staleType, busyDetail, staleDetail, referenceIso, BASELINE_MAX_SECONDS],
|
|
90
|
+
);
|
|
91
|
+
return result.changes;
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
export function recordDeliveredStaleEvents(now = new Date()): DeliveredStalePatrolResult {
|
|
95
|
+
if (!Number.isFinite(now.getTime())) throw new Error("invalid lifecycle patrol time");
|
|
96
|
+
|
|
97
|
+
ensureAliasBaselineTable();
|
|
98
|
+
const byThreshold = { 600: 0, 1800: 0 };
|
|
99
|
+
let inserted = 0;
|
|
100
|
+
|
|
101
|
+
for (const thresholdSeconds of DELIVERED_STALE_THRESHOLDS) {
|
|
102
|
+
const cutoff = sqliteUtcTimestamp(new Date(now.getTime() - thresholdSeconds * 1000));
|
|
103
|
+
const changes = recordStale(thresholdSeconds, cutoff, now.toISOString());
|
|
104
|
+
byThreshold[thresholdSeconds] = changes;
|
|
105
|
+
inserted += changes;
|
|
106
|
+
}
|
|
107
|
+
|
|
108
|
+
return { inserted, by_threshold_seconds: byThreshold };
|
|
109
|
+
}
|