@sleep2agi/commhub-server 0.9.0-preview.20 → 0.9.0-preview.201
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +130 -10
- package/bin/commhub.ts +99 -11
- package/package.json +7 -7
- package/src/ack-create-request-transport.test.ts +182 -0
- package/src/ack-create-request.test.ts +50 -7
- package/src/admin-delete-network-http.test.ts +228 -0
- package/src/admin-networks-http.test.ts +88 -0
- package/src/adoption-candidates-http.test.ts +137 -0
- package/src/adoption-candidates.ts +158 -0
- package/src/agent-access-batch.test.ts +210 -0
- package/src/agent-access.ts +487 -0
- package/src/agent-acl-groups-http.test.ts +247 -0
- package/src/agent-acl-http.test.ts +659 -0
- package/src/agent-acl-slice3-http.test.ts +267 -0
- package/src/agent-department-http.test.ts +215 -0
- package/src/agent-groups-perf.test.ts +223 -0
- package/src/agent-notice.ts +48 -0
- package/src/agent-team-whoami-http.test.ts +98 -0
- package/src/agent-teams-http.test.ts +330 -0
- package/src/agent-teams.ts +244 -0
- package/src/alias-baseline-2571.test.ts +162 -0
- package/src/alias-baseline.ts +113 -0
- package/src/alias-filter.test.ts +72 -0
- package/src/alias-filter.ts +42 -0
- package/src/api-host-supervisors-can-create-nodes.test.ts +350 -0
- package/src/api-host-supervisors-fallback.test.ts +21 -12
- package/src/api-host-supervisors-runtime-readiness.test.ts +195 -0
- package/src/api-nodes-shape.test.ts +120 -13
- package/src/auth-password-change-cli-tokens-http.test.ts +423 -0
- package/src/auth-sessions-http.test.ts +250 -0
- package/src/auth-validate.test.ts +5 -1
- package/src/auth-weak-password-login.test.ts +176 -0
- package/src/auth.ts +609 -51
- package/src/avatar-validate.test.ts +163 -0
- package/src/avatar-validate.ts +103 -0
- package/src/backfill-node-ids-2664.test.ts +128 -0
- package/src/binding-generation-http.test.ts +121 -0
- package/src/blocked-exit-on-reply.test.ts +122 -0
- package/src/boot-side-effect.test.ts +124 -0
- package/src/broadcast-fanout-2721.test.ts +118 -0
- package/src/broadcast-fanout.ts +98 -0
- package/src/chain-attempt-dedup.test.ts +447 -0
- package/src/clone-node.test.ts +248 -0
- package/src/clone-node.ts +289 -0
- package/src/codex-config-flags.ts +77 -0
- package/src/codex-fork-contract.ts +23 -0
- package/src/codex-fork-migration.test.ts +38 -0
- package/src/commhub-bin-args.test.ts +85 -0
- package/src/config-apply-sec1.test.ts +107 -0
- package/src/config-apply-validate.test.ts +134 -1
- package/src/config-apply-validate.ts +111 -5
- package/src/create-node-blocked-gate.test.ts +356 -0
- package/src/create-node-network-id.test.ts +79 -0
- package/src/create-node-tool-schema.test.ts +49 -0
- package/src/create-node-unicode-name.test.ts +120 -0
- package/src/create-node-validate.test.ts +199 -7
- package/src/create-node-validate.ts +232 -15
- package/src/create-node-workdir.test.ts +289 -0
- package/src/create-node.ts +88 -23
- package/src/daemon-provider-snapshots.test.ts +421 -0
- package/src/daemon-provider-snapshots.ts +327 -0
- package/src/daemon-token-owner-http.test.ts +236 -0
- package/src/dashboard-slash-routing-http.test.ts +109 -0
- package/src/db-adapter-guard.test.ts +148 -0
- package/src/db-adapter-translate.test.ts +128 -0
- package/src/db-adapter.ts +377 -91
- package/src/db-timestamp.test.ts +38 -0
- package/src/db-timestamp.ts +15 -0
- package/src/db.ts +1306 -153
- package/src/delivered-without-push.test.ts +140 -0
- package/src/department-groups-http.test.ts +209 -0
- package/src/department-groups-sync-http.test.ts +328 -0
- package/src/department-groups.ts +434 -0
- package/src/department-heads-http.test.ts +330 -0
- package/src/department-heads.ts +226 -0
- package/src/departments-agent-count.test.ts +129 -0
- package/src/departments-http.test.ts +185 -0
- package/src/departments-mcp.test.ts +128 -0
- package/src/departments.ts +318 -0
- package/src/desktop-message-seam.test.ts +150 -0
- package/src/desktop-message.test.ts +368 -0
- package/src/dm-file-scope-http.test.ts +178 -0
- package/src/empty-node-sweeper.test.ts +104 -0
- package/src/empty-node-sweeper.ts +88 -0
- package/src/env-transport-gate-http.test.ts +177 -0
- package/src/env-transport-gate.ts +74 -0
- package/src/env-transport-policy.ts +21 -0
- package/src/event-loop-lag.test.ts +139 -0
- package/src/event-loop-lag.ts +150 -0
- package/src/ext-token-shared.test.ts +60 -0
- package/src/ext-validation-regression.test.ts +131 -0
- package/src/external-schedule-edits-http.test.ts +321 -0
- package/src/external-schedule-edits-timestamps.test.ts +64 -0
- package/src/external-schedule-edits.ts +327 -0
- package/src/file-download-authz-dev-open.test.ts +135 -0
- package/src/file-download-authz.test.ts +404 -0
- package/src/file-download-date-bucket.test.ts +436 -0
- package/src/file-network-scope-dev-open.test.ts +97 -0
- package/src/file-network-scope.test.ts +732 -0
- package/src/group-membership-events-http.test.ts +264 -0
- package/src/group-messages-http.test.ts +482 -0
- package/src/group-messages.ts +297 -0
- package/src/health-host-disk-2756.test.ts +49 -0
- package/src/health-redaction.test.ts +685 -0
- package/src/host-supervisor-identity-http.test.ts +152 -0
- package/src/host-supervisor-identity.ts +92 -0
- package/src/host-supervisors-daemon-version.test.ts +137 -0
- package/src/http-gzip.test.ts +94 -0
- package/src/http-gzip.ts +112 -0
- package/src/hub-catalog-read-http.test.ts +187 -0
- package/src/hub-provider-config.test.ts +139 -0
- package/src/hub-provider-config.ts +105 -0
- package/src/hub-timestamp.test.ts +38 -0
- package/src/hub-timestamp.ts +27 -0
- package/src/human-dm.ts +215 -0
- package/src/human-inbox.ts +42 -0
- package/src/inbox-caller.ts +49 -0
- package/src/inbox-count-sse.test.ts +310 -0
- package/src/inbox-count.ts +29 -0
- package/src/index.ts +15 -2380
- package/src/list-host-supervisors.test.ts +5 -4
- package/src/mcp-context-cost-http.test.ts +127 -0
- package/src/mcp-inflight.test.ts +92 -0
- package/src/mcp-inflight.ts +38 -0
- package/src/mcp-projects-events-http.test.ts +162 -0
- package/src/mcp-route-label-http.test.ts +120 -0
- package/src/mcp-write-network-resolution.test.ts +432 -0
- package/src/member-audit-network-id-http.test.ts +93 -0
- package/src/member-presence-http.test.ts +210 -0
- package/src/messages-ack-by-agent-http.test.ts +142 -0
- package/src/migrate-sqlite-to-pg.test.ts +47 -0
- package/src/migrate-sqlite-to-pg.ts +263 -0
- package/src/model-auth-notify-http.test.ts +181 -0
- package/src/model-auth-notify.ts +116 -0
- package/src/network-name-http.test.ts +75 -0
- package/src/network-rules-http.test.ts +169 -0
- package/src/network-rules.ts +129 -0
- package/src/network-scope.ts +276 -0
- package/src/network-token-membership-http.test.ts +139 -0
- package/src/node-alias-control.ts +12 -0
- package/src/node-attrs-validate.ts +88 -0
- package/src/node-attrs.test.ts +258 -0
- package/src/node-caller-identity-http.test.ts +1083 -0
- package/src/node-create-request-status-http.test.ts +105 -0
- package/src/node-daemon-bindings.test.ts +132 -0
- package/src/node-daemon-bindings.ts +135 -0
- package/src/node-failure-alert.test.ts +150 -0
- package/src/node-failure-alert.ts +177 -0
- package/src/node-files-transport.test.ts +344 -0
- package/src/node-health-codex-login.test.ts +120 -0
- package/src/node-health-dispatch-http.test.ts +279 -0
- package/src/node-health-guard.ts +105 -0
- package/src/node-health-passthrough.test.ts +115 -0
- package/src/node-health-store.ts +101 -0
- package/src/node-id-alias.test.ts +190 -0
- package/src/node-identity-conflict-http.test.ts +269 -0
- package/src/node-identity-conflict.ts +142 -0
- package/src/node-identity-mismatch.test.ts +74 -0
- package/src/node-in-flight-tasks.ts +31 -0
- package/src/node-lifecycle-controllable.test.ts +64 -0
- package/src/node-lifecycle-controllable.ts +53 -0
- package/src/node-lifecycle-read-http.test.ts +250 -0
- package/src/node-lifecycle-read.ts +124 -0
- package/src/node-logs-transport.test.ts +263 -0
- package/src/node-peer-allowlist.test.ts +114 -0
- package/src/node-permissions-http.test.ts +345 -0
- package/src/node-permissions.ts +420 -0
- package/src/node-request-retention.test.ts +281 -0
- package/src/node-request-retention.ts +120 -0
- package/src/node-secret-rest-http.test.ts +190 -0
- package/src/node-serviceability.test.ts +86 -0
- package/src/node-serviceability.ts +124 -0
- package/src/node-session-control-http.test.ts +310 -0
- package/src/node-session-control.ts +258 -0
- package/src/node-skills-transport.test.ts +209 -0
- package/src/node-token-identity.ts +35 -0
- package/src/node-token-ownership.ts +51 -0
- package/src/node-token-privilege-http.test.ts +158 -0
- package/src/not-found-api-http.test.ts +67 -0
- package/src/observer-avatar-http.test.ts +442 -0
- package/src/observer-push.test.ts +252 -0
- package/src/opencode-catalog-report-http.test.ts +139 -0
- package/src/opencode-v2-catalog.test.ts +268 -0
- package/src/opencode-v2-catalog.ts +352 -0
- package/src/package-version-consistency.test.ts +25 -0
- package/src/parent-source-marker.test.ts +50 -0
- package/src/password-network-tokens.ts +25 -0
- package/src/peer-reply-atomic.test.ts +455 -0
- package/src/peer-reply-capability.test.ts +70 -0
- package/src/pg-worker.ts +87 -0
- package/src/project-dir-warn-clear-http.test.ts +132 -0
- package/src/project-viewer-can-http.test.ts +116 -0
- package/src/push.test.ts +20 -10
- package/src/push.ts +466 -13
- package/src/redact-tokens.test.ts +65 -0
- package/src/redact-tokens.ts +40 -0
- package/src/rename.ts +38 -10
- package/src/reply-target-derivation.test.ts +96 -0
- package/src/report-status-handover-capabilities-http.test.ts +124 -0
- package/src/report-status-host-schema.test.ts +235 -0
- package/src/report-status-resume-id-handover.test.ts +184 -0
- package/src/report-status-secret-keys.test.ts +140 -0
- package/src/require-explicit-test-db.ts +9 -0
- package/src/requirement-due-reminders-http.test.ts +620 -0
- package/src/requirement-due-reminders.ts +416 -0
- package/src/requirement-events-http.test.ts +268 -0
- package/src/requirement-events.test.ts +45 -0
- package/src/requirement-events.ts +144 -0
- package/src/requirement-notify.ts +165 -0
- package/src/requirement-participant-notify-http.test.ts +309 -0
- package/src/requirement-tag-ops-http.test.ts +425 -0
- package/src/requirement-tag-tree-http.test.ts +275 -0
- package/src/requirement-tag-tree.ts +127 -0
- package/src/requirement-tags.ts +102 -0
- package/src/requirements-abandoned-http.test.ts +236 -0
- package/src/requirements-comment-http.test.ts +167 -0
- package/src/requirements-completed-at-migration.test.ts +70 -0
- package/src/requirements-errors-mcp-http.test.ts +164 -0
- package/src/requirements-errors.ts +158 -0
- package/src/requirements-etag-http.test.ts +91 -0
- package/src/requirements-http.test.ts +709 -0
- package/src/requirements-last-event-http.test.ts +349 -0
- package/src/requirements-list-mcp-slim-http.test.ts +155 -0
- package/src/requirements-list-slim-http.test.ts +324 -0
- package/src/requirements-mcp.test.ts +163 -0
- package/src/requirements-migrate.ts +283 -0
- package/src/requirements-owner-strict-mcp-http.test.ts +145 -0
- package/src/requirements-people-http.test.ts +114 -0
- package/src/requirements-people-mcp-http.test.ts +231 -0
- package/src/requirements-people.ts +196 -0
- package/src/requirements-priority-lowest.test.ts +114 -0
- package/src/requirements-projects-parent-http.test.ts +123 -0
- package/src/requirements-search-http.test.ts +182 -0
- package/src/requirements-search.test.ts +32 -0
- package/src/requirements-search.ts +145 -0
- package/src/requirements-seq-http.test.ts +193 -0
- package/src/requirements-seq-migration.test.ts +63 -0
- package/src/requirements-start-date.test.ts +125 -0
- package/src/requirements-stats-http.test.ts +357 -0
- package/src/requirements-stats.ts +195 -0
- package/src/requirements-status-participants-http.test.ts +222 -0
- package/src/requirements-upgrade-e2e.test.ts +126 -0
- package/src/requirements-upgrade-migration.test.ts +98 -0
- package/src/requirements.ts +2027 -0
- package/src/resolve-port.test.ts +57 -0
- package/src/resolve-port.ts +44 -0
- package/src/response-charset.test.ts +130 -0
- package/src/rest-alias-network-fallback-http.test.ts +123 -0
- package/src/rest-alias-network-fallback.test.ts +60 -0
- package/src/rest-alias-network-fallback.ts +65 -0
- package/src/rest-explicit-columns-http.test.ts +266 -0
- package/src/rest-identity.test.ts +130 -0
- package/src/rest-identity.ts +141 -0
- package/src/rest-projections.ts +72 -0
- package/src/rest-read-network-real-chain.test.ts +97 -0
- package/src/rest-write-network-resolution.test.ts +133 -0
- package/src/restart-readiness-http.test.ts +121 -0
- package/src/restricted-files.ts +76 -0
- package/src/retention-first-sweep-quiet.test.ts +31 -0
- package/src/retention.test.ts +100 -40
- package/src/retention.ts +423 -104
- package/src/route-callers-http.test.ts +120 -0
- package/src/route-timing.test.ts +211 -0
- package/src/route-timing.ts +262 -0
- package/src/rules-file-duplicate-alias.test.ts +234 -0
- package/src/rules-file-session-target.test.ts +207 -0
- package/src/rules-file-transport.test.ts +237 -0
- package/src/runtime-label.test.ts +22 -0
- package/src/runtime-label.ts +21 -0
- package/src/runtime-readiness.ts +90 -0
- package/src/schedule-agent-mcp-http.test.ts +391 -0
- package/src/schedule-agent.ts +192 -0
- package/src/schedule-starved.test.ts +92 -0
- package/src/schedule-starved.ts +114 -0
- package/src/scheduled-backoff-http.test.ts +193 -0
- package/src/scheduled-backoff.ts +64 -0
- package/src/scheduled-doorbell-nodeless-session.test.ts +134 -0
- package/src/scheduled-failure-backoff-2622.test.ts +145 -0
- package/src/scheduled-failure-code.test.ts +73 -0
- package/src/scheduled-failure-code.ts +105 -0
- package/src/scheduled-failures-http.test.ts +279 -0
- package/src/scheduled-failures.ts +377 -0
- package/src/scheduled-noprogress-backoff-2622.test.ts +133 -0
- package/src/scheduled-offline-backoff-probe.test.ts +56 -0
- package/src/scheduled-offline.test.ts +131 -0
- package/src/scheduled-offline.ts +66 -0
- package/src/scheduled-outcome-guidance-http.test.ts +139 -0
- package/src/scheduled-outcome-guidance.ts +9 -0
- package/src/scheduled-reply-unread-http.test.ts +212 -0
- package/src/scheduled-requirement-wake-http.test.ts +279 -0
- package/src/scheduled-requirement-wake.ts +125 -0
- package/src/scheduled-run-terminal.test.ts +268 -0
- package/src/scheduled-skip-blocking-task.test.ts +120 -0
- package/src/scheduled-skip-path-backoff-2622.test.ts +98 -0
- package/src/scheduled-stuck-http.test.ts +312 -0
- package/src/scheduled-stuck-notice-cap.test.ts +80 -0
- package/src/scheduled-stuck.ts +219 -0
- package/src/scheduled-tasks-http.test.ts +679 -0
- package/src/scheduled-tasks.ts +959 -0
- package/src/scheduled-work-outcome-http.test.ts +154 -0
- package/src/scheduled-work-outcome.ts +50 -0
- package/src/scoped-secrets.ts +139 -0
- package/src/send-dedup-request-id.test.ts +229 -0
- package/src/send-reply-agent-warning.test.ts +202 -0
- package/src/send-reply-alias-optional.test.ts +131 -0
- package/src/send-reply-attachments.test.ts +458 -0
- package/src/send_dedup.test.ts +54 -0
- package/src/send_dedup.ts +52 -6
- package/src/serve-error.test.ts +38 -0
- package/src/serve-error.ts +27 -0
- package/src/server.ts +6028 -0
- package/src/session-task-on-dispatch.ts +14 -0
- package/src/session-token-ownership-http.test.ts +122 -0
- package/src/shared/codex-inventory.ts +41 -0
- package/src/shared/daemon-provider-login.ts +31 -0
- package/src/shared/daemon-provider-snapshot.ts +52 -0
- package/src/shared/external-schedule-contract.ts +60 -0
- package/src/shared/node-name.test.ts +62 -0
- package/src/shared/node-name.ts +161 -0
- package/src/shared/provider-application.ts +29 -0
- package/src/shared/provider-collection.ts +14 -0
- package/src/shared/provider-initialization.ts +8 -0
- package/src/shared/reserved-env-drift.test.ts +10 -0
- package/src/shared/scoped-secret-contract.ts +5 -0
- package/src/shared/task-content-limit-drift.test.ts +28 -0
- package/src/shared/task-content-limit.ts +25 -0
- package/src/side-thread-command-transport.test.ts +312 -0
- package/src/side-thread-command-transport.ts +318 -0
- package/src/side-thread-http-integration.test.ts +233 -0
- package/src/side-thread-http.ts +504 -0
- package/src/side-thread-production.test.ts +72 -0
- package/src/side-thread-production.ts +118 -0
- package/src/side-thread.test.ts +1067 -0
- package/src/side-thread.ts +2499 -0
- package/src/skillhub-http.test.ts +108 -0
- package/src/sqlite-close-checkpoint.test.ts +78 -0
- package/src/start-node.test.ts +264 -0
- package/src/status-alias-filter-http.test.ts +106 -0
- package/src/status-alias-resolver-http.test.ts +163 -0
- package/src/status-alias-resolver.ts +19 -0
- package/src/status-node-id-filter-http.test.ts +161 -0
- package/src/status-read-cache-http.test.ts +229 -0
- package/src/status-read-cache.ts +122 -0
- package/src/stop-delete-node.test.ts +567 -1
- package/src/task-access-http.test.ts +402 -0
- package/src/task-access.ts +420 -0
- package/src/task-activity.ts +60 -0
- package/src/task-auth-origin-http.test.ts +93 -0
- package/src/task-auth-origin.test.ts +24 -0
- package/src/task-auth-origin.ts +7 -0
- package/src/task-consumption.test.ts +482 -0
- package/src/task-diagnostic-http.test.ts +149 -0
- package/src/task-diagnostic.test.ts +115 -0
- package/src/task-diagnostic.ts +118 -0
- package/src/task-expiry-notice-http.test.ts +287 -0
- package/src/task-expiry-notice.ts +315 -0
- package/src/task-idempotency-mcp.test.ts +142 -0
- package/src/task-idempotency.test.ts +76 -0
- package/src/task-idempotency.ts +81 -0
- package/src/task-late-replies.ts +54 -0
- package/src/task-late-reply-http.test.ts +152 -0
- package/src/task-lifecycle-watcher-busy.test.ts +114 -0
- package/src/task-lifecycle-watcher.test.ts +204 -0
- package/src/task-lifecycle-watcher.ts +109 -0
- package/src/task-network-resolution-http.test.ts +140 -0
- package/src/task-offline-delivery-2668.test.ts +119 -0
- package/src/task-offline-delivery.ts +81 -0
- package/src/task-orphan-http.test.ts +287 -0
- package/src/task-orphan.ts +119 -0
- package/src/task-queue-ahead-http.test.ts +309 -0
- package/src/task-queue-ahead-scale.test.ts +236 -0
- package/src/task-queue-ahead.ts +186 -0
- package/src/task-rejection-log-http.test.ts +160 -0
- package/src/task-rejection-log.ts +49 -0
- package/src/task-sender-http.test.ts +143 -0
- package/src/task-stale-open-http.test.ts +290 -0
- package/src/task-stale-open.ts +103 -0
- package/src/tasks-pagination-http.test.ts +141 -0
- package/src/team-rules-status-http.test.ts +90 -0
- package/src/team-rules-status.test.ts +59 -0
- package/src/team-rules-status.ts +72 -0
- package/src/tool-audience-http.test.ts +230 -0
- package/src/tool-audience.ts +120 -0
- package/src/tools-list-json-schema.test.ts +94 -0
- package/src/tools.ts +4226 -405
- package/src/upload-early-reject-guard.test.ts +155 -0
- package/src/uploads-http.test.ts +97 -15
- package/src/uploads.ts +103 -6
- package/src/user-avatar-http.test.ts +74 -0
- package/src/user-avatar-image.ts +160 -0
- package/src/user-avatar-migration.test.ts +25 -0
- package/src/user-avatar-upload-http.test.ts +348 -0
- package/src/user-inbox-by-agent-http.test.ts +161 -0
- package/src/user-inbox-netid-notnull.test.ts +122 -0
- package/src/user-inbox-read-http.test.ts +223 -0
- package/src/user-inbox-schema.test.ts +54 -0
- package/src/version-stats.test.ts +66 -0
- package/src/version-stats.ts +109 -0
|
@@ -0,0 +1,236 @@
|
|
|
1
|
+
// #500 follow-up to #2325 —— queue_depth at production scale. SQLite only (the plan assertions are SQLite's
|
|
2
|
+
// EXPLAIN QUERY PLAN; the result-equality side runs on PostgreSQL in task-queue-ahead-http.test.ts via test2123).
|
|
3
|
+
//
|
|
4
|
+
// The .100 release drill on a read-only copy of a production-shaped DB (306 sessions, 57k tasks, ~31k stuck in
|
|
5
|
+
// `acked` for weeks, ~900 `running`, only ~1.9k created in the last 24 h) measured full /api/status at p50 66–77 ms
|
|
6
|
+
// against 8–9 ms on .99: the queue_depth GROUP BY went through idx_tasks_status and walked every open-status row
|
|
7
|
+
// ever written to keep a 24 h window. This file seeds that shape and pins:
|
|
8
|
+
// 1. the plans: the whole-table queue_depth query and the per-send open-task count never use idx_tasks_status;
|
|
9
|
+
// the whole-table one is driven by the covering idx_tasks_created_queue (no task-row lookups), the alias list
|
|
10
|
+
// and the per-send count by idx_tasks_to_created;
|
|
11
|
+
// 2. the cost, relative only (no absolute ms, so a slow runner can't fail it): the queue_depth statement is at most
|
|
12
|
+
// half the #2325 statement measured on the same DB in the same loop; and, on the low percentile of alternated
|
|
13
|
+
// samples (#517), the statement costs at most half a full /api/status read (cache bypassed) without it, and that
|
|
14
|
+
// read with queue_depth is at most 50 % slower than with it switched off (= the pre-#2325 read);
|
|
15
|
+
// 3. the counts are still right at that scale (stale `acked` rows are outside the 24 h window).
|
|
16
|
+
//
|
|
17
|
+
// 跑法:cd server && COMMHUB_DB=/tmp/x.db bun test src/task-queue-ahead-scale.test.ts
|
|
18
|
+
|
|
19
|
+
import { afterAll, beforeAll, describe, expect, test } from "bun:test";
|
|
20
|
+
import { mkdtempSync, rmSync } from "node:fs";
|
|
21
|
+
import { tmpdir } from "node:os";
|
|
22
|
+
import { join } from "node:path";
|
|
23
|
+
|
|
24
|
+
const PG = !!process.env.COMMHUB_TEST_PG_URL;
|
|
25
|
+
const DIR = mkdtempSync(join(tmpdir(), "anet-queue-scale-"));
|
|
26
|
+
if (!PG) process.env.COMMHUB_DB ||= join(DIR, "hub.db");
|
|
27
|
+
process.env.COMMHUB_UPLOADS_DIR = join(DIR, "uploads");
|
|
28
|
+
process.env.HOST = "127.0.0.1";
|
|
29
|
+
|
|
30
|
+
const NODES = 306;
|
|
31
|
+
const STUCK_ACKED = 31_000; // older than 24 h, never terminal
|
|
32
|
+
const STUCK_RUNNING = 900; // older than 24 h
|
|
33
|
+
const OLD_DONE = 23_400; // replied / failed / expired, older than 24 h
|
|
34
|
+
const RECENT = 1_900; // created in the last 24 h
|
|
35
|
+
const OPEN_EVERY = 30; // 3 in every OPEN_EVERY recent tasks are still open (~190 of 1.9k; see the cost test)
|
|
36
|
+
const HOT = "scale-node-7"; // most stuck rows land on this one (the per-send plan must not care)
|
|
37
|
+
const N = 25;
|
|
38
|
+
/** #517: A/B pairs per pass, the percentile compared, and how many extra passes a failing pass gets. */
|
|
39
|
+
const AB_PAIRS = 60, LOW_P = 0.2, RETRIES = 2;
|
|
40
|
+
/** The #2325 statement (bare columns): the in-test baseline the fixed query is measured against. */
|
|
41
|
+
const SQL_2325 = `SELECT network_id, to_name, COUNT(*) AS n FROM tasks
|
|
42
|
+
WHERE status IN ('created', 'delivered', 'acked', 'running') AND created_at >= datetime('now', '-86400 seconds')
|
|
43
|
+
GROUP BY network_id, to_name`;
|
|
44
|
+
|
|
45
|
+
let db: any;
|
|
46
|
+
let hub: any = null;
|
|
47
|
+
let BASE = "", NET = "", token = "";
|
|
48
|
+
let mod: any;
|
|
49
|
+
let expectedOpen = new Map<string, number>();
|
|
50
|
+
let seedMs = 0, regMs = 0;
|
|
51
|
+
|
|
52
|
+
const pct = (xs: number[], p: number) => { const s = [...xs].sort((a, b) => a - b); return s[Math.min(s.length - 1, Math.floor(p * s.length))]; };
|
|
53
|
+
const plan = (sql: string, params: any[]) => (db.all("EXPLAIN QUERY PLAN " + sql, ...params) as Array<{ detail: string }>).map((r) => r.detail).join(" | ");
|
|
54
|
+
|
|
55
|
+
beforeAll(async () => {
|
|
56
|
+
if (PG) return;
|
|
57
|
+
({ db } = await import("./db.js"));
|
|
58
|
+
mod = await import("./task-queue-ahead.js");
|
|
59
|
+
const { register } = await import("./auth.js");
|
|
60
|
+
const seedStart = performance.now();
|
|
61
|
+
const u = register(`scale_${Date.now()}`, "QueueScale123!x");
|
|
62
|
+
token = u.token!; NET = u.network_id!;
|
|
63
|
+
// Seed with a handful of set-based INSERT … SELECT over a recursive CTE (no per-row JS round trips): the whole
|
|
64
|
+
// 57k-row seed is ~0.2 s, so the file stays far inside test638's per-file budget even under strace and with two
|
|
65
|
+
// aggregates running at once. Rows are deterministic (functions of the row number), so the expected counts below
|
|
66
|
+
// are computed from the same formulas, independently of the code under test.
|
|
67
|
+
const SEQ = `WITH RECURSIVE seq(i) AS (SELECT 0 UNION ALL SELECT i + 1 FROM seq WHERE i + 1 < ?1)`;
|
|
68
|
+
const OLD_AGE = `(90000 + (i * 7919) % 2592000)`; // 25 h … 25 h + 30 d, in seconds
|
|
69
|
+
const insertTasks = (prefix: string, count: number, toExpr: string, statusExpr: string, ageExpr: string, doneExpr: string) => db.run(
|
|
70
|
+
`${SEQ}
|
|
71
|
+
INSERT INTO tasks (task_id, from_name, to_name, priority, status, content, created_at, delivered_at, completed_at, expires_at, network_id)
|
|
72
|
+
SELECT '${prefix}-' || i, 'scale-sender', ${toExpr}, 'normal', ${statusExpr}, 'c',
|
|
73
|
+
datetime('now', '-' || ${ageExpr} || ' seconds'), datetime('now', '-' || ${ageExpr} || ' seconds'),
|
|
74
|
+
CASE WHEN ${doneExpr} THEN datetime('now', '-' || (${ageExpr} - 120) || ' seconds') END,
|
|
75
|
+
datetime('now', '-' || (${ageExpr} - 3600) || ' seconds'), ?2
|
|
76
|
+
FROM seq`,
|
|
77
|
+
[count, NET],
|
|
78
|
+
);
|
|
79
|
+
const NODE_OF = (expr: string) => `'scale-node-' || ((${expr}) % ${NODES})`;
|
|
80
|
+
// Throwaway DB only: no fsync, and a page cache big enough that the seed transaction never spills to the WAL
|
|
81
|
+
// mid-way (test638 runs this file under `strace -f`, where every write syscall is expensive).
|
|
82
|
+
regMs = performance.now() - seedStart;
|
|
83
|
+
db.exec("PRAGMA synchronous = OFF; PRAGMA cache_size = -131072;");
|
|
84
|
+
db.transaction(() => {
|
|
85
|
+
db.run(`${SEQ}
|
|
86
|
+
INSERT INTO sessions (resume_id, alias, status, agent, network_id, node_id, updated_at, last_seen_at, task)
|
|
87
|
+
SELECT 'sdk-scale-' || i, 'scale-node-' || i, 'idle', 'agent-node:codex', ?2, 'n_scale_' || i, datetime('now'), datetime('now'), 'x' FROM seq`,
|
|
88
|
+
[NODES, NET]);
|
|
89
|
+
// Two thirds of the stuck rows on one node, the rest spread out.
|
|
90
|
+
insertTasks("acked", STUCK_ACKED, `CASE WHEN i % 3 = 0 THEN ${NODE_OF("i")} ELSE '${HOT}' END`, `'acked'`, OLD_AGE, "0");
|
|
91
|
+
insertTasks("running", STUCK_RUNNING, NODE_OF("i"), `'running'`, OLD_AGE, "0");
|
|
92
|
+
insertTasks("done", OLD_DONE, NODE_OF("i"), `CASE i % 5 WHEN 3 THEN 'failed' WHEN 4 THEN 'expired' ELSE 'replied' END`, OLD_AGE, "1");
|
|
93
|
+
// Last 24 h: i % OPEN_EVERY = 3 / 4 / 5 → delivered / running / acked (open), else failed (i % 7 = 6) or replied;
|
|
94
|
+
// age 0 … 23 h.
|
|
95
|
+
insertTasks("recent", RECENT, NODE_OF("i * 13"),
|
|
96
|
+
`CASE i % ${OPEN_EVERY} WHEN 3 THEN 'delivered' WHEN 4 THEN 'running' WHEN 5 THEN 'acked' ELSE CASE i % 7 WHEN 6 THEN 'failed' ELSE 'replied' END END`,
|
|
97
|
+
`((i * 37) % 82800)`, `i % ${OPEN_EVERY} NOT IN (3, 4, 5)`);
|
|
98
|
+
});
|
|
99
|
+
for (let i = 0; i < RECENT; i++) {
|
|
100
|
+
if (![3, 4, 5].includes(i % OPEN_EVERY)) continue;
|
|
101
|
+
const to = `scale-node-${(i * 13) % NODES}`;
|
|
102
|
+
expectedOpen.set(to, (expectedOpen.get(to) ?? 0) + 1);
|
|
103
|
+
}
|
|
104
|
+
seedMs = performance.now() - seedStart;
|
|
105
|
+
const srv: any = await import("./server.js");
|
|
106
|
+
hub = srv.bootServer({ port: 0, hostname: "127.0.0.1" });
|
|
107
|
+
BASE = `http://127.0.0.1:${hub.port}`;
|
|
108
|
+
}, 120_000);
|
|
109
|
+
|
|
110
|
+
afterAll(() => {
|
|
111
|
+
try { hub?.stop?.(true); } catch {}
|
|
112
|
+
try { rmSync(DIR, { recursive: true, force: true }); } catch {}
|
|
113
|
+
});
|
|
114
|
+
|
|
115
|
+
describe.skipIf(PG)("#500 queue_depth at production scale (57k tasks, 31k stuck acked)", () => {
|
|
116
|
+
test("plans: never idx_tasks_status; whole table via covering idx_tasks_created_queue, alias list and per-send via idx_tasks_to_created", () => {
|
|
117
|
+
const total = Number(db.get("SELECT COUNT(*) AS n FROM tasks WHERE network_id = ?1", NET).n);
|
|
118
|
+
expect(total).toBe(STUCK_ACKED + STUCK_RUNNING + OLD_DONE + RECENT);
|
|
119
|
+
const stale = Number(db.get("SELECT COUNT(*) AS n FROM tasks WHERE network_id = ?1 AND status IN ('acked','running') AND created_at < datetime('now', '-86400 seconds')", NET).n);
|
|
120
|
+
expect(stale).toBe(STUCK_ACKED + STUCK_RUNNING);
|
|
121
|
+
console.log(`[seed] ${total} tasks in ${seedMs.toFixed(0)} ms (register ${regMs.toFixed(0)} ms)`);
|
|
122
|
+
// Positive control: the shape is one where the #2325 statement really does go through idx_tasks_status.
|
|
123
|
+
expect(plan(SQL_2325, [])).toContain("idx_tasks_status");
|
|
124
|
+
|
|
125
|
+
const whole = mod.queueDepthQuery();
|
|
126
|
+
const wholePlan = plan(whole.sql, whole.params);
|
|
127
|
+
console.log(`[plan] queue_depth whole table: ${wholePlan}`);
|
|
128
|
+
expect(wholePlan).not.toContain("idx_tasks_status");
|
|
129
|
+
expect(wholePlan).toContain("COVERING INDEX idx_tasks_created_queue");
|
|
130
|
+
|
|
131
|
+
// > QUEUE_DEPTH_ALIAS_LIST_MAX aliases → the whole-table form too.
|
|
132
|
+
const many = mod.queueDepthQuery(Array.from({ length: NODES }, (_, i) => `scale-node-${i}`));
|
|
133
|
+
expect(many.sql).toBe(whole.sql);
|
|
134
|
+
|
|
135
|
+
const few = mod.queueDepthQuery([HOT, "scale-node-8"]);
|
|
136
|
+
const fewPlan = plan(few.sql, few.params);
|
|
137
|
+
console.log(`[plan] queue_depth alias list: ${fewPlan}`);
|
|
138
|
+
expect(fewPlan).not.toContain("idx_tasks_status");
|
|
139
|
+
expect(fewPlan).toContain("idx_tasks_to_created");
|
|
140
|
+
|
|
141
|
+
for (const net of [NET, null]) {
|
|
142
|
+
const send = mod.openTaskCountsQuery(HOT, net);
|
|
143
|
+
const sendPlan = plan(send.sql, send.params);
|
|
144
|
+
console.log(`[plan] per-send open count (net=${net ? "set" : "none"}): ${sendPlan}`);
|
|
145
|
+
expect(sendPlan).not.toContain("idx_tasks_status");
|
|
146
|
+
expect(sendPlan).toContain("idx_tasks_to_created");
|
|
147
|
+
}
|
|
148
|
+
});
|
|
149
|
+
|
|
150
|
+
test("counts stay right at scale: stale acked rows are outside the 24 h window", async () => {
|
|
151
|
+
const depth: Map<string, number> = mod.queueDepthByNode();
|
|
152
|
+
for (const [alias, n] of expectedOpen) expect(depth.get(mod.queueDepthKey(NET, alias))).toBe(n);
|
|
153
|
+
expect(depth.get(mod.queueDepthKey(NET, HOT)) ?? 0).toBe(expectedOpen.get(HOT) ?? 0);
|
|
154
|
+
const r = await fetch(`${BASE}/api/status?network_id=${NET}&alias=${HOT}`, { headers: { Authorization: `Bearer ${token}` } });
|
|
155
|
+
const body = await r.json() as any;
|
|
156
|
+
expect(body.sessions[0].queue_depth).toBe(expectedOpen.get(HOT) ?? 0);
|
|
157
|
+
expect(mod.openTaskCounts(HOT, NET).open).toBe(expectedOpen.get(HOT) ?? 0);
|
|
158
|
+
// And a node that does have open tasks in the window (HOT's open rows are all > 24 h old).
|
|
159
|
+
expect(expectedOpen.size).toBeGreaterThan(100);
|
|
160
|
+
const [busy, busyN] = [...expectedOpen].sort((a, b) => b[1] - a[1])[0];
|
|
161
|
+
expect(busyN).toBeGreaterThan(1);
|
|
162
|
+
const rb = await fetch(`${BASE}/api/status?network_id=${NET}&alias=${busy}`, { headers: { Authorization: `Bearer ${token}` } });
|
|
163
|
+
expect(((await rb.json()) as any).sessions[0].queue_depth).toBe(busyN);
|
|
164
|
+
expect(mod.openTaskCounts(busy, NET).open).toBe(busyN);
|
|
165
|
+
});
|
|
166
|
+
|
|
167
|
+
test("cost (relative, robust on slow runners): ≤ half the #2325 statement on the same DB, and ≤ +50% on a full /api/status read vs the same read without queue_depth", async () => {
|
|
168
|
+
// Interleave the two statements so a slow / noisy runner slows both alike.
|
|
169
|
+
const fixed: number[] = [], old: number[] = [];
|
|
170
|
+
for (let i = 0; i < N + 3; i++) {
|
|
171
|
+
let s0 = performance.now(); mod.queueDepthByNode(); const a = performance.now() - s0;
|
|
172
|
+
s0 = performance.now(); db.all(SQL_2325); const b = performance.now() - s0;
|
|
173
|
+
if (i >= 3) { fixed.push(a); old.push(b); }
|
|
174
|
+
}
|
|
175
|
+
// Measured locally (this fixture, covering index): ≈ 0.15× (1.0 vs 6.5 ms); on the production copy 0.3 vs 55 ms.
|
|
176
|
+
const qP50 = pct(fixed, 0.5), oldP50 = pct(old, 0.5);
|
|
177
|
+
console.log(`[perf] queue_depth p50=${qP50.toFixed(2)}ms vs #2325 statement p50=${oldP50.toFixed(2)}ms`);
|
|
178
|
+
expect(qP50).toBeLessThanOrEqual(0.5 * oldP50);
|
|
179
|
+
|
|
180
|
+
// Second budget: queue_depth adds at most +50 % to a full /api/status read (cache bypassed) — the same read
|
|
181
|
+
// with it switched off is the pre-#2325 read. #517: the first version compared the p50 of `with` and `without`
|
|
182
|
+
// on a ~4 ms read and asserted the *difference*; under test638's concurrent aggregates a difference of two
|
|
183
|
+
// noisy p50s is mostly noise (CI #2355: statement 0.64 ms, yet with−without = 2.4 ms > 2.03 ms budget). Now:
|
|
184
|
+
// - each pass alternates A/B request by request (order flipped every pair) and also times the bare
|
|
185
|
+
// queue_depth statement between the reads, so all three series share one time window;
|
|
186
|
+
// - it compares a low percentile (LOW_P) rather than the p50: scheduler noise only ever adds time, so the
|
|
187
|
+
// lower tail is the closest a busy runner gets to the real cost;
|
|
188
|
+
// - it asserts both what the comment claims — the statement's own cost ≤ 50 % of the read without it — and
|
|
189
|
+
// the end-to-end with/without delta on the same low percentile;
|
|
190
|
+
// - a failing pass is re-measured up to RETRIES more times; the real regression (the #2325 statement through
|
|
191
|
+
// idx_tasks_status, 5–8× the fixed cost) fails every pass, noise does not.
|
|
192
|
+
const cache = await import("./status-read-cache.js");
|
|
193
|
+
cache.__setStatusCacheBypassForTest(true);
|
|
194
|
+
const readOnce = async (depthOn: boolean) => {
|
|
195
|
+
mod.__setQueueDepthOffForTest(!depthOn);
|
|
196
|
+
const s0 = performance.now();
|
|
197
|
+
await (await fetch(`${BASE}/api/status?network_id=${NET}`, { headers: { Authorization: `Bearer ${token}` } })).text();
|
|
198
|
+
return performance.now() - s0;
|
|
199
|
+
};
|
|
200
|
+
const abPass = async () => {
|
|
201
|
+
const on: number[] = [], off: number[] = [], q: number[] = [];
|
|
202
|
+
for (let i = 0; i < AB_PAIRS + 3; i++) {
|
|
203
|
+
const first = i % 2 === 0;
|
|
204
|
+
const a = await readOnce(first), b = await readOnce(!first);
|
|
205
|
+
mod.__setQueueDepthOffForTest(false);
|
|
206
|
+
const s0 = performance.now(); mod.queueDepthByNode(); const c = performance.now() - s0;
|
|
207
|
+
if (i >= 3) { (first ? on : off).push(a); (first ? off : on).push(b); q.push(c); }
|
|
208
|
+
}
|
|
209
|
+
const onLow = pct(on, LOW_P), offLow = pct(off, LOW_P), qLow = pct(q, LOW_P);
|
|
210
|
+
return { onLow, offLow, qLow, onP50: pct(on, 0.5), offP50: pct(off, 0.5),
|
|
211
|
+
okDirect: qLow <= 0.5 * offLow, okRead: onLow - offLow <= 0.5 * offLow };
|
|
212
|
+
};
|
|
213
|
+
let r: Awaited<ReturnType<typeof abPass>> | null = null;
|
|
214
|
+
try {
|
|
215
|
+
for (let attempt = 0; attempt <= RETRIES; attempt++) {
|
|
216
|
+
r = await abPass();
|
|
217
|
+
console.log(`[perf] pass ${attempt + 1}: p${LOW_P * 100} queue_depth=${r.qLow.toFixed(2)}ms; full /api/status (bypass) p${LOW_P * 100} with=${r.onLow.toFixed(2)}ms without=${r.offLow.toFixed(2)}ms (+${(100 * (r.onLow - r.offLow) / r.offLow).toFixed(0)}%); p50 with=${r.onP50.toFixed(2)}ms without=${r.offP50.toFixed(2)}ms; direct ${r.okDirect ? "ok" : "OVER"} read ${r.okRead ? "ok" : "OVER"}`);
|
|
218
|
+
if (r.okDirect && r.okRead) break;
|
|
219
|
+
}
|
|
220
|
+
} finally {
|
|
221
|
+
mod.__setQueueDepthOffForTest(false);
|
|
222
|
+
cache.__setStatusCacheBypassForTest(false);
|
|
223
|
+
}
|
|
224
|
+
// What the read budget was measured at (2026-10-04, p50s, before #517):
|
|
225
|
+
// production copy (VACUUM INTO of the live DB: 306 sessions, 57k tasks, 64 open in the last 24 h on 4 nodes),
|
|
226
|
+
// full /api/status p50, cache bypassed, 3 runs each —
|
|
227
|
+
// pre-#2325 10.5–11.3 ms · #2325 68–71 ms · first fix (idx_tasks_created + row lookups) 16.1–17.1 ms (+50 %,
|
|
228
|
+
// statement 3.1 ms) · covering idx_tasks_created_queue 11.5–12.2 ms; this A/B on that copy: +7 / +8 / +9 %.
|
|
229
|
+
// this fixture: +17–31 %. It is harsher than production: ~190 open tasks in the window spread over ~150 nodes
|
|
230
|
+
// (3× production's open count, ~38× its groups) and one-word session rows that make the rest of the read cheap.
|
|
231
|
+
// With the original 1-in-7 open share (813 open) it measured +55–62 % even with the covering index — the
|
|
232
|
+
// GROUP BY over 300 groups, not the index.
|
|
233
|
+
expect(r!.qLow).toBeLessThanOrEqual(0.5 * r!.offLow);
|
|
234
|
+
expect(r!.onLow - r!.offLow).toBeLessThanOrEqual(0.5 * r!.offLow);
|
|
235
|
+
}, 120_000);
|
|
236
|
+
});
|
|
@@ -0,0 +1,186 @@
|
|
|
1
|
+
// #500 step 2 —— 派活之前 / 派活那一刻,让发送方看见目标节点的队列。
|
|
2
|
+
//
|
|
3
|
+
// 第一步(#2322)在任务过期时通知发送方;这一步在 send_task / POST /api/task 的响应里直接告诉它
|
|
4
|
+
// 「前面还有几个、目标正在忙没有、大概要等多久」,并在 /api/status(全量)和 get_all_status 的每一行
|
|
5
|
+
// 带上 queue_depth。只是建议:**从不拒绝派活**,老客户端不认识的字段直接忽略。
|
|
6
|
+
//
|
|
7
|
+
// 定义(与 task-expiry-notice.ts 的 openTasksAhead 同一组状态):
|
|
8
|
+
// 开着的任务 = tasks 行,status ∈ {created, delivered, acked, running},且 created_at 在最近 24 小时内。
|
|
9
|
+
// - created / delivered:还没被消费的,以及已经被运行时取走(consumed_at)但状态还没前移的;
|
|
10
|
+
// - acked / running:已经开工、还没出终态的。
|
|
11
|
+
// - 消息(send_message / broadcast)只写 inbox、不写 tasks,天然不算。终态(replied / failed / cancelled /
|
|
12
|
+
// expired)不算。
|
|
13
|
+
// - 24 小时窗口:TTL 上限就是 24 小时,超过 24 小时还没终态的行几乎都是被遗弃的(节点重启丢了、
|
|
14
|
+
// running 从来没回来;#2322 起 consumed 的 delivered 行也不再被巡检过期),不算进队列,否则队列只增不减。
|
|
15
|
+
// queue_ahead = 派这一条**之前**目标上开着的任务数(按网络;没有网络 = 不加网络条件,与 pending 计数同口径)。
|
|
16
|
+
// target_busy = 目标上有一条已开工(acked / running,或带 started_at / consumed_at)的开着的任务,或者会话状态是
|
|
17
|
+
// working / busy / running。
|
|
18
|
+
// est_wait_minutes = 目标最近 24 小时 replied 任务的耗时中位数 × queue_ahead,四舍五入、上限 1440;
|
|
19
|
+
// 样本 < 3 条 → null(不知道);queue_ahead = 0 → 0(前面没人)。
|
|
20
|
+
// 耗时 = completed_at − (started_at ?? 不晚于 completed_at 的 consumed_at ?? delivered_at ?? created_at)。
|
|
21
|
+
// 开工时间戳很稀疏,回落到派出时间就把排队也算进去了 —— 所以只是**粗估**,偏大。
|
|
22
|
+
// warning = queue_ahead ≥ 3 或 est_wait_minutes > 30 时给一句人话,建议换个空闲节点 / 合并请求 / 加长 ttl_seconds。
|
|
23
|
+
//
|
|
24
|
+
// 查询:派活时一条(计数)+ 必要时一条(耗时样本,LIMIT 200,走 idx_tasks_to_created);状态读一条 GROUP BY。
|
|
25
|
+
// 条件分支各自拼占位符,PostgreSQL 上不会有「绑了没用」的参数。
|
|
26
|
+
|
|
27
|
+
import { db } from "./db.js";
|
|
28
|
+
import { parseDbTimestampMs } from "./db-timestamp.js";
|
|
29
|
+
|
|
30
|
+
export const QUEUE_OPEN_STATUSES_SQL = "('created', 'delivered', 'acked', 'running')";
|
|
31
|
+
/** 开着的任务只看最近这么久派出的(= TTL 上限)。 */
|
|
32
|
+
export const QUEUE_HORIZON_SQL = "datetime('now', '-86400 seconds')";
|
|
33
|
+
export const QUEUE_WARN_AHEAD = 3;
|
|
34
|
+
export const QUEUE_WARN_WAIT_MINUTES = 30;
|
|
35
|
+
export const EST_WAIT_MIN_SAMPLES = 3;
|
|
36
|
+
export const EST_WAIT_CAP_MINUTES = 1440;
|
|
37
|
+
const EST_WAIT_SAMPLE_LIMIT = 200;
|
|
38
|
+
const BUSY_SESSION_STATUSES = new Set(["working", "busy", "running"]);
|
|
39
|
+
|
|
40
|
+
export type DispatchQueueInfo = {
|
|
41
|
+
queue_ahead: number;
|
|
42
|
+
target_busy: boolean;
|
|
43
|
+
est_wait_minutes: number | null;
|
|
44
|
+
warning?: string;
|
|
45
|
+
};
|
|
46
|
+
|
|
47
|
+
function scoped(sql: string, params: any[], networkId: string | null | undefined): string {
|
|
48
|
+
if (!networkId) return sql;
|
|
49
|
+
params.push(networkId);
|
|
50
|
+
return `${sql} AND network_id = ?${params.length}`;
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
/** 目标上开着的任务数 + 其中已开工的数。一条查询。 */
|
|
54
|
+
export function openTaskCounts(alias: string, networkId: string | null | undefined): { open: number; started: number } {
|
|
55
|
+
const q = openTaskCountsQuery(alias, networkId);
|
|
56
|
+
const row = db.get<{ open_n: number | string | null; started_n: number | string | null }>(q.sql, ...q.params);
|
|
57
|
+
return { open: Number(row?.open_n ?? 0) || 0, started: Number(row?.started_n ?? 0) || 0 };
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
/**
|
|
61
|
+
* The statement openTaskCounts runs (exported for the plan assertion). Unlike queueDepthQuery below it needs no
|
|
62
|
+
* index hint: the to_name equality + created_at range always wins idx_tasks_to_created, even for a node with tens of
|
|
63
|
+
* thousands of stuck `acked` rows (task-queue-ahead-scale.test.ts pins that plan).
|
|
64
|
+
*/
|
|
65
|
+
export function openTaskCountsQuery(alias: string, networkId: string | null | undefined): { sql: string; params: any[] } {
|
|
66
|
+
const params: any[] = [alias];
|
|
67
|
+
let sql = `SELECT COUNT(*) AS open_n,
|
|
68
|
+
SUM(CASE WHEN status IN ('acked', 'running') OR started_at IS NOT NULL OR consumed_at IS NOT NULL THEN 1 ELSE 0 END) AS started_n
|
|
69
|
+
FROM tasks WHERE to_name = ?1 AND status IN ${QUEUE_OPEN_STATUSES_SQL} AND created_at >= ${QUEUE_HORIZON_SQL}`;
|
|
70
|
+
sql = scoped(sql, params, networkId);
|
|
71
|
+
return { sql, params };
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
/** 目标最近 24 小时 replied 任务的耗时中位数(分钟);样本不够 → null。一条查询。 */
|
|
75
|
+
export function medianTaskMinutes(alias: string, networkId: string | null | undefined): number | null {
|
|
76
|
+
const params: any[] = [alias];
|
|
77
|
+
let sql = `SELECT created_at, delivered_at, started_at, consumed_at, completed_at FROM tasks
|
|
78
|
+
WHERE to_name = ?1 AND status = 'replied' AND completed_at IS NOT NULL AND created_at >= ${QUEUE_HORIZON_SQL}`;
|
|
79
|
+
sql = scoped(sql, params, networkId);
|
|
80
|
+
sql += ` ORDER BY created_at DESC LIMIT ${EST_WAIT_SAMPLE_LIMIT}`;
|
|
81
|
+
const rows = db.all<{ created_at: string | null; delivered_at: string | null; started_at: string | null; consumed_at: string | null; completed_at: string | null }>(sql, ...params);
|
|
82
|
+
const mins: number[] = [];
|
|
83
|
+
for (const r of rows) {
|
|
84
|
+
const end = r.completed_at ? parseDbTimestampMs(r.completed_at) : NaN;
|
|
85
|
+
if (!Number.isFinite(end)) continue;
|
|
86
|
+
const consumed = r.consumed_at ? parseDbTimestampMs(r.consumed_at) : NaN;
|
|
87
|
+
const startText = r.started_at ?? (Number.isFinite(consumed) && consumed <= end ? r.consumed_at : null) ?? r.delivered_at ?? r.created_at;
|
|
88
|
+
const start = startText ? parseDbTimestampMs(startText) : NaN;
|
|
89
|
+
if (!Number.isFinite(start) || end < start) continue;
|
|
90
|
+
mins.push((end - start) / 60_000);
|
|
91
|
+
}
|
|
92
|
+
if (mins.length < EST_WAIT_MIN_SAMPLES) return null;
|
|
93
|
+
mins.sort((a, b) => a - b);
|
|
94
|
+
const mid = mins.length >> 1;
|
|
95
|
+
return mins.length % 2 ? mins[mid] : (mins[mid - 1] + mins[mid]) / 2;
|
|
96
|
+
}
|
|
97
|
+
|
|
98
|
+
export function queueWarning(alias: string, info: Omit<DispatchQueueInfo, "warning">, ttlSeconds: number): string | undefined {
|
|
99
|
+
const waitHigh = info.est_wait_minutes != null && info.est_wait_minutes > QUEUE_WARN_WAIT_MINUTES;
|
|
100
|
+
if (info.queue_ahead < QUEUE_WARN_AHEAD && !waitHigh) return undefined;
|
|
101
|
+
const ttlMin = Math.round(ttlSeconds / 60);
|
|
102
|
+
const wait = info.est_wait_minutes == null ? "unknown wait" : `rough wait ~${info.est_wait_minutes} min`;
|
|
103
|
+
const expiry = info.est_wait_minutes != null && info.est_wait_minutes >= ttlMin
|
|
104
|
+
? ` It may expire before it starts (ttl ${ttlMin} min).`
|
|
105
|
+
: ` It expires if not started within ${ttlMin} min.`;
|
|
106
|
+
return `${alias} already has ${info.queue_ahead} open task(s) ahead of this one (${wait}${info.target_busy ? ", busy now" : ""}). `
|
|
107
|
+
+ `Your task is queued, not refused.${expiry} Consider an idle node instead (get_all_status: status=idle, queue_depth=0), `
|
|
108
|
+
+ `merging your asks, or a longer ttl_seconds. Do not resend the same task.`;
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
/**
|
|
112
|
+
* 派活那一刻的队列信息。在写 inbox / tasks **之前**调:那时开着的就是排在它前面的。
|
|
113
|
+
* 任何异常都吞掉、返回 null —— 这些字段只是建议,不能让派活失败。
|
|
114
|
+
*/
|
|
115
|
+
export function dispatchQueueInfo(alias: string, networkId: string | null | undefined, sessionStatus: string | null | undefined, ttlSeconds: number): DispatchQueueInfo | null {
|
|
116
|
+
try {
|
|
117
|
+
const { open, started } = openTaskCounts(alias, networkId);
|
|
118
|
+
const target_busy = started > 0 || BUSY_SESSION_STATUSES.has(String(sessionStatus ?? "").toLowerCase());
|
|
119
|
+
let est_wait_minutes: number | null = 0;
|
|
120
|
+
if (open > 0) {
|
|
121
|
+
const median = medianTaskMinutes(alias, networkId);
|
|
122
|
+
est_wait_minutes = median == null ? null : Math.min(EST_WAIT_CAP_MINUTES, Math.round(median * open));
|
|
123
|
+
}
|
|
124
|
+
const base = { queue_ahead: open, target_busy, est_wait_minutes };
|
|
125
|
+
const warning = queueWarning(alias, base, ttlSeconds);
|
|
126
|
+
return warning ? { ...base, warning } : base;
|
|
127
|
+
} catch {
|
|
128
|
+
return null;
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
let queueDepthOffForTest = false;
|
|
133
|
+
/**
|
|
134
|
+
* Test-only: queueDepthByNode answers an empty map without querying (every row's queue_depth = 0), i.e. a full
|
|
135
|
+
* /api/status read does exactly the pre-#2325 work. The scale test alternates reads with it on and off against the
|
|
136
|
+
* same server so the queue_depth share is measured A/B in the same time window (robust to a busy CI runner).
|
|
137
|
+
*/
|
|
138
|
+
export function __setQueueDepthOffForTest(on: boolean): void { queueDepthOffForTest = on; }
|
|
139
|
+
|
|
140
|
+
export const queueDepthKey = (networkId: string | null | undefined, alias: string) => `${networkId ?? ""}\u0000${alias}`;
|
|
141
|
+
|
|
142
|
+
/** 别名不多于这个数时按别名查(走 idx_tasks_to_created);多了就扫最近 24 小时派出的任务(走覆盖索引 idx_tasks_created_queue)。 */
|
|
143
|
+
export const QUEUE_DEPTH_ALIAS_LIST_MAX = 50;
|
|
144
|
+
|
|
145
|
+
/**
|
|
146
|
+
* 每个 (网络, 别名) 上开着的任务数。一条 GROUP BY;没有开着任务的节点不在表里(= 0)。
|
|
147
|
+
* `aliases` = 这次要标注的行:App 的 `?alias=` / `?node_id=` 单行读只查那一两个别名,不扫全库的开着任务。
|
|
148
|
+
* 返回的计数按 (network_id, to_name) 分组,调用方按行自己的 network_id 取,所以别的网络的同名节点不会串。
|
|
149
|
+
*/
|
|
150
|
+
export function queueDepthByNode(aliases?: Array<string | null | undefined>): Map<string, number> {
|
|
151
|
+
const out = new Map<string, number>();
|
|
152
|
+
if (queueDepthOffForTest) return out;
|
|
153
|
+
const q = queueDepthQuery(aliases);
|
|
154
|
+
if (!q) return out;
|
|
155
|
+
const rows = db.all<{ network_id: string | null; to_name: string; n: number | string }>(q.sql, ...q.params);
|
|
156
|
+
for (const r of rows) out.set(queueDepthKey(r.network_id, r.to_name), Number(r.n) || 0);
|
|
157
|
+
return out;
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
/** The statement queueDepthByNode runs (exported so the scale test can EXPLAIN exactly this SQL). null = nothing to ask. */
|
|
161
|
+
export function queueDepthQuery(aliases?: Array<string | null | undefined>): { sql: string; params: any[] } | null {
|
|
162
|
+
const wanted = aliases ? [...new Set(aliases.filter((a): a is string => typeof a === "string" && a.length > 0))] : null;
|
|
163
|
+
if (wanted && wanted.length === 0) return null;
|
|
164
|
+
const params: any[] = [];
|
|
165
|
+
// 🔴 The `|| ''` wrappers are deliberate — they are index hints, not data transforms (#500 follow-up to #2325).
|
|
166
|
+
// On a production-shaped DB (57k tasks, 31k stuck in `acked` for weeks, ~1.9k created in the last 24 h) the
|
|
167
|
+
// planner picked idx_tasks_status for `status IN (…)` and walked every open-status row ever written (~32k) to
|
|
168
|
+
// keep the 24 h window: full /api/status went from ~8 ms to ~70 ms p50. Wrapping `status` makes that index
|
|
169
|
+
// unusable, so the scan is driven by the 24 h range on the covering idx_tasks_created_queue (created_at, status,
|
|
170
|
+
// network_id, to_name) (EXPLAIN: `SEARCH tasks USING COVERING INDEX idx_tasks_created_queue (created_at>?)`, plus a
|
|
171
|
+
// temp b-tree for the GROUP BY; no task row is read) — or, for the alias list, by idx_tasks_to_created. Without the
|
|
172
|
+
// covering index it falls back to idx_tasks_created + one row lookup per task: ~3 ms on the production copy, which
|
|
173
|
+
// was still +50 % on a full /api/status read; covering, ~0.3 ms. The GROUP BY must be wrapped too: grouping on the bare columns lets SQLite pick
|
|
174
|
+
// idx_tasks_network to avoid the sort, which is another full walk. `|| ''` is portable (SQLite and PostgreSQL;
|
|
175
|
+
// a CTE/subquery gets flattened back, `+status` / `LIMIT -1` are SQLite-only). NULL || '' stays NULL on both, and
|
|
176
|
+
// the selected expressions are the grouped ones (PostgreSQL requires that). task-queue-ahead-scale.test.ts pins
|
|
177
|
+
// the plan and a latency budget.
|
|
178
|
+
let sql = `SELECT (network_id || '') AS network_id, (to_name || '') AS to_name, COUNT(*) AS n FROM tasks
|
|
179
|
+
WHERE (status || '') IN ${QUEUE_OPEN_STATUSES_SQL} AND created_at >= ${QUEUE_HORIZON_SQL}`;
|
|
180
|
+
if (wanted && wanted.length <= QUEUE_DEPTH_ALIAS_LIST_MAX) {
|
|
181
|
+
sql += ` AND to_name IN (${wanted.map((_, i) => `?${i + 1}`).join(", ")})`;
|
|
182
|
+
params.push(...wanted);
|
|
183
|
+
}
|
|
184
|
+
sql += " GROUP BY (network_id || ''), (to_name || '')";
|
|
185
|
+
return { sql, params };
|
|
186
|
+
}
|
|
@@ -0,0 +1,160 @@
|
|
|
1
|
+
// #552 — every rejected REST POST /api/task leaves exactly one hub log line.
|
|
2
|
+
//
|
|
3
|
+
// Incident: a restricted member's app sends were refused and the hub log
|
|
4
|
+
// showed nothing, so "never arrived" and "rejected" were indistinguishable.
|
|
5
|
+
// Real Bun.serve on a private port + temp DB; console.log is captured around
|
|
6
|
+
// each request. The line must carry status + error + user + requested
|
|
7
|
+
// alias/network/client_request_id, and must NOT carry the bearer token or the
|
|
8
|
+
// task text.
|
|
9
|
+
|
|
10
|
+
import { describe, expect, test, beforeAll, afterAll } from "bun:test";
|
|
11
|
+
import { mkdtempSync, rmSync } from "fs";
|
|
12
|
+
import { join } from "path";
|
|
13
|
+
import { tmpdir } from "os";
|
|
14
|
+
import { register } from "./auth.js";
|
|
15
|
+
import { db } from "./db.js";
|
|
16
|
+
import { formatTaskRejection } from "./task-rejection-log.js";
|
|
17
|
+
|
|
18
|
+
const DIR = mkdtempSync(join(tmpdir(), "anet-task-reject-log-"));
|
|
19
|
+
let BASE = "";
|
|
20
|
+
let hub: any = null;
|
|
21
|
+
const PW = "RejectLogPassw0rd!x";
|
|
22
|
+
let adminToken = "";
|
|
23
|
+
let adminName = "";
|
|
24
|
+
let NET = "";
|
|
25
|
+
let aliceToken = "";
|
|
26
|
+
const AGENT = { alias: "rejlog-agent", node: "node_rejlog" };
|
|
27
|
+
const SECRET_TEXT = "TOP-SECRET-TASK-BODY-552";
|
|
28
|
+
|
|
29
|
+
async function postTask(token: string | null, body: string): Promise<{ status: number; json: any; lines: string[] }> {
|
|
30
|
+
const lines: string[] = [];
|
|
31
|
+
const orig = console.log;
|
|
32
|
+
console.log = (...args: any[]) => { lines.push(args.map(String).join(" ")); };
|
|
33
|
+
try {
|
|
34
|
+
const headers: Record<string, string> = { "Content-Type": "application/json" };
|
|
35
|
+
if (token) headers.Authorization = `Bearer ${token}`;
|
|
36
|
+
const res = await fetch(`${BASE}/api/task`, { method: "POST", headers, body });
|
|
37
|
+
const text = await res.text();
|
|
38
|
+
let json: any = text;
|
|
39
|
+
try { json = JSON.parse(text); } catch {}
|
|
40
|
+
return { status: res.status, json, lines };
|
|
41
|
+
} finally {
|
|
42
|
+
console.log = orig;
|
|
43
|
+
}
|
|
44
|
+
}
|
|
45
|
+
const rejectedLines = (lines: string[]) => lines.filter(l => l.includes("→ /api/task →") && l.includes("REJECTED"));
|
|
46
|
+
|
|
47
|
+
beforeAll(async () => {
|
|
48
|
+
process.env.COMMHUB_DB ||= join(DIR, "hub.db");
|
|
49
|
+
process.env.COMMHUB_UPLOADS_DIR = join(DIR, "uploads");
|
|
50
|
+
process.env.HOST = "127.0.0.1";
|
|
51
|
+
|
|
52
|
+
const admin = register(`rejlog_admin_${Date.now()}`, PW, undefined, "Admin");
|
|
53
|
+
expect(admin.ok).toBe(true);
|
|
54
|
+
adminToken = admin.token!;
|
|
55
|
+
adminName = admin.user!.username;
|
|
56
|
+
NET = admin.network_id!;
|
|
57
|
+
db.run("UPDATE users SET role = 'admin' WHERE user_id = ?1", [admin.user!.user_id]);
|
|
58
|
+
db.run(
|
|
59
|
+
`INSERT INTO nodes (node_id, node_name, alias, network_id, owner_user_id, updated_at) VALUES (?1, ?2, ?2, ?3, ?4, datetime('now'))`,
|
|
60
|
+
[AGENT.node, AGENT.alias, NET, admin.user!.user_id],
|
|
61
|
+
);
|
|
62
|
+
db.run(
|
|
63
|
+
`INSERT INTO sessions (resume_id, alias, node_id, status, network_id, updated_at, last_seen_at)
|
|
64
|
+
VALUES (?1, ?2, ?3, 'idle', ?4, datetime('now'), datetime('now'))`,
|
|
65
|
+
[`resume_${AGENT.node}`, AGENT.alias, AGENT.node, NET],
|
|
66
|
+
);
|
|
67
|
+
|
|
68
|
+
const mod: any = await import("./server.js");
|
|
69
|
+
hub = mod.bootServer({ port: 0, hostname: "127.0.0.1" });
|
|
70
|
+
BASE = `http://127.0.0.1:${hub.port}`;
|
|
71
|
+
|
|
72
|
+
// A restricted member (default agent_access='granted', zero grants) who also
|
|
73
|
+
// owns a personal network → two memberships, like the #552 app user.
|
|
74
|
+
const created = await fetch(`${BASE}/api/admin/users`, {
|
|
75
|
+
method: "POST",
|
|
76
|
+
headers: { Authorization: `Bearer ${adminToken}`, "Content-Type": "application/json" },
|
|
77
|
+
body: JSON.stringify({ username: "rejlog_alice", password: PW, network_id: NET, role: "member" }),
|
|
78
|
+
});
|
|
79
|
+
expect(created.status).toBe(200);
|
|
80
|
+
const login = await fetch(`${BASE}/api/auth/login`, {
|
|
81
|
+
method: "POST", headers: { "Content-Type": "application/json" },
|
|
82
|
+
body: JSON.stringify({ username: "rejlog_alice", password: PW }),
|
|
83
|
+
});
|
|
84
|
+
expect(login.status).toBe(200);
|
|
85
|
+
aliceToken = (await login.json() as any).token;
|
|
86
|
+
}, 30_000);
|
|
87
|
+
|
|
88
|
+
afterAll(() => {
|
|
89
|
+
try { hub?.stop?.(true); } catch {}
|
|
90
|
+
try { rmSync(DIR, { recursive: true, force: true }); } catch {}
|
|
91
|
+
});
|
|
92
|
+
|
|
93
|
+
function assertNoSecrets(line: string, token: string) {
|
|
94
|
+
expect(line).not.toContain(token);
|
|
95
|
+
expect(line).not.toContain(SECRET_TEXT);
|
|
96
|
+
expect(line).not.toContain("file_");
|
|
97
|
+
}
|
|
98
|
+
|
|
99
|
+
describe("#552 REST POST /api/task rejections are logged", () => {
|
|
100
|
+
test("alias_not_found (404): one line with status, error, user, alias, network, client_request_id", async () => {
|
|
101
|
+
const r = await postTask(adminToken, JSON.stringify({ alias: "no-such-agent", task: SECRET_TEXT, network_id: NET, meta: { client_request_id: "crid-552-a" } }));
|
|
102
|
+
expect(r.status).toBe(404);
|
|
103
|
+
expect(r.json.error).toBe("alias_not_found");
|
|
104
|
+
const lines = rejectedLines(r.lines);
|
|
105
|
+
expect(lines.length).toBe(1);
|
|
106
|
+
expect(lines[0]).toMatch(/^\[\d\d:\d\d:\d\d\] /);
|
|
107
|
+
expect(lines[0]).toContain(`${adminName} → /api/task → no-such-agent: REJECTED 404 alias_not_found`);
|
|
108
|
+
expect(lines[0]).toContain(`net=${NET}`);
|
|
109
|
+
expect(lines[0]).toContain("crid=crid-552-a");
|
|
110
|
+
assertNoSecrets(lines[0], adminToken);
|
|
111
|
+
});
|
|
112
|
+
|
|
113
|
+
test("network_id_required (400) for a member of two networks who omits network_id", async () => {
|
|
114
|
+
const r = await postTask(aliceToken, JSON.stringify({ alias: AGENT.alias, task: SECRET_TEXT }));
|
|
115
|
+
expect(r.status).toBe(400);
|
|
116
|
+
expect(r.json.error).toBe("network_id_required");
|
|
117
|
+
const lines = rejectedLines(r.lines);
|
|
118
|
+
expect(lines.length).toBe(1);
|
|
119
|
+
expect(lines[0]).toContain(`rejlog_alice → /api/task → ${AGENT.alias}: REJECTED 400 network_id_required (net=-)`);
|
|
120
|
+
assertNoSecrets(lines[0], aliceToken);
|
|
121
|
+
});
|
|
122
|
+
|
|
123
|
+
test("agent_not_granted (403) — the restricted-member case from #552", async () => {
|
|
124
|
+
const r = await postTask(aliceToken, JSON.stringify({ alias: AGENT.alias, task: SECRET_TEXT, network_id: NET, attachments: [{ file_id: "file_deadbeef" }] }));
|
|
125
|
+
expect(r.status).toBe(403);
|
|
126
|
+
expect(r.json.error).toBe("agent_not_granted");
|
|
127
|
+
const lines = rejectedLines(r.lines);
|
|
128
|
+
expect(lines.length).toBe(1);
|
|
129
|
+
expect(lines[0]).toContain(`rejlog_alice → /api/task → ${AGENT.alias}: REJECTED 403 agent_not_granted (net=${NET})`);
|
|
130
|
+
assertNoSecrets(lines[0], aliceToken);
|
|
131
|
+
});
|
|
132
|
+
|
|
133
|
+
test("invalid JSON (400) is logged with the user and '-' placeholders", async () => {
|
|
134
|
+
const r = await postTask(aliceToken, "{not json");
|
|
135
|
+
expect(r.status).toBe(400);
|
|
136
|
+
const lines = rejectedLines(r.lines);
|
|
137
|
+
expect(lines.length).toBe(1);
|
|
138
|
+
expect(lines[0]).toContain("rejlog_alice → /api/task → -: REJECTED 400 invalid JSON (net=-)");
|
|
139
|
+
});
|
|
140
|
+
|
|
141
|
+
test("a successful send logs no REJECTED line", async () => {
|
|
142
|
+
const r = await postTask(adminToken, JSON.stringify({ alias: AGENT.alias, task: "hello", network_id: NET }));
|
|
143
|
+
expect(r.status).toBe(200);
|
|
144
|
+
expect(rejectedLines(r.lines).length).toBe(0);
|
|
145
|
+
});
|
|
146
|
+
});
|
|
147
|
+
|
|
148
|
+
describe("formatTaskRejection", () => {
|
|
149
|
+
test("request-supplied values cannot inject extra log lines and are clipped", () => {
|
|
150
|
+
const line = formatTaskRejection({
|
|
151
|
+
status: 404, error: "alias_not_found", username: null,
|
|
152
|
+
body: { alias: "evil\n[00:00:00] admin → forged", network_id: "n".repeat(200), meta: { client_request_id: "x\r\ny" } },
|
|
153
|
+
});
|
|
154
|
+
expect(line).not.toContain("\n");
|
|
155
|
+
expect(line).not.toContain("\r");
|
|
156
|
+
expect(line).toContain("anon → /api/task → evil [00:00:00] admin → forged: REJECTED 404 alias_not_found");
|
|
157
|
+
expect(line).toContain(`net=${"n".repeat(64)}…`);
|
|
158
|
+
expect(line).toContain("crid=x y");
|
|
159
|
+
});
|
|
160
|
+
});
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
// #552 — one log line for every non-2xx answer from REST POST /api/task.
|
|
2
|
+
//
|
|
3
|
+
// A restricted member's app sends were rejected and the hub log showed
|
|
4
|
+
// nothing, so "the request never arrived" and "the hub refused it" looked
|
|
5
|
+
// identical from the server side. Every rejection branch now goes through
|
|
6
|
+
// logTaskRejection().
|
|
7
|
+
//
|
|
8
|
+
// What is logged is deliberately narrow: status, error code, the
|
|
9
|
+
// authenticated username, and the *requested* alias / network_id /
|
|
10
|
+
// client_request_id. Never the bearer token, the task text, or attachment
|
|
11
|
+
// ids. Request-supplied values are clipped and stripped of control
|
|
12
|
+
// characters so a client cannot forge extra log lines.
|
|
13
|
+
|
|
14
|
+
function ts(): string {
|
|
15
|
+
return new Date().toTimeString().slice(0, 8);
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
function clip(value: unknown, max = 64): string {
|
|
19
|
+
if (typeof value !== "string" || !value) return "-";
|
|
20
|
+
const clean = value.replace(/[\u0000-\u001f\u007f\s]+/g, " ").trim();
|
|
21
|
+
if (!clean) return "-";
|
|
22
|
+
return clean.length > max ? `${clean.slice(0, max)}…` : clean;
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
export interface TaskRejectionContext {
|
|
26
|
+
status: number;
|
|
27
|
+
error: unknown;
|
|
28
|
+
username: string | null | undefined;
|
|
29
|
+
/** The raw request body (or null when it did not parse). Only alias, network_id and meta.client_request_id are read. */
|
|
30
|
+
body: unknown;
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
export function formatTaskRejection(ctx: TaskRejectionContext): string {
|
|
34
|
+
const b = ctx.body && typeof ctx.body === "object" ? (ctx.body as Record<string, any>) : {};
|
|
35
|
+
const meta = b.meta && typeof b.meta === "object" ? b.meta : {};
|
|
36
|
+
const crid = typeof meta.client_request_id === "string" ? ` crid=${clip(meta.client_request_id)}` : "";
|
|
37
|
+
return `[${ts()}] ${clip(ctx.username ?? "anon")} → /api/task → ${clip(b.alias)}: REJECTED ${ctx.status} ${clip(ctx.error)} (net=${clip(b.network_id)}${crid})`;
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
/** Logs one line for a non-2xx response and returns the response unchanged. */
|
|
41
|
+
export async function logTaskRejection(res: Response, username: string | null | undefined, body: unknown): Promise<Response> {
|
|
42
|
+
if (res.status >= 200 && res.status < 300) return res;
|
|
43
|
+
let error: unknown = null;
|
|
44
|
+
try {
|
|
45
|
+
error = ((await res.clone().json()) as any)?.error;
|
|
46
|
+
} catch {}
|
|
47
|
+
console.log(formatTaskRejection({ status: res.status, error: error ?? "unknown", username: username || "anon", body }));
|
|
48
|
+
return res;
|
|
49
|
+
}
|