mixdog 0.7.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +31 -0
- package/.claude-plugin/plugin.json +20 -0
- package/.gitattributes +34 -0
- package/.mcp.json +14 -0
- package/ARCHITECTURE.md +77 -0
- package/CHANGELOG.md +7 -0
- package/CONTRIBUTING.md +45 -0
- package/DATA-FLOW.md +79 -0
- package/LICENSE +21 -0
- package/README.md +389 -0
- package/SECURITY.md +138 -0
- package/UNINSTALL.md +112 -0
- package/agents/maintenance.md +5 -0
- package/agents/memory-classification.md +30 -0
- package/agents/scheduler-task.md +18 -0
- package/agents/webhook-handler.md +27 -0
- package/agents/worker.md +24 -0
- package/bin/bridge +133 -0
- package/bin/statusline-launcher.mjs +78 -0
- package/bin/statusline-lib.mjs +550 -0
- package/bin/statusline.mjs +607 -0
- package/bun.lock +802 -0
- package/commands/config.md +16 -0
- package/commands/doctor.md +13 -0
- package/commands/setup.md +17 -0
- package/defaults/cycle3-review-prompt.md +90 -0
- package/defaults/hidden-roles.json +65 -0
- package/defaults/memory-chunk-prompt.md +63 -0
- package/defaults/memory-promote-prompt.md +135 -0
- package/defaults/mixdog-config.template.json +27 -0
- package/defaults/user-workflow.json +8 -0
- package/defaults/user-workflow.md +12 -0
- package/hooks/hooks.json +73 -0
- package/hooks/lib/active-instance.cjs +77 -0
- package/hooks/lib/permission-evaluator.cjs +411 -0
- package/hooks/lib/permission-route.cjs +63 -0
- package/hooks/lib/permission-rules.cjs +170 -0
- package/hooks/lib/settings-loader.cjs +116 -0
- package/hooks/post-tool-use.cjs +84 -0
- package/hooks/pre-mcp-sandbox.cjs +158 -0
- package/hooks/pre-tool-subagent.cjs +253 -0
- package/hooks/session-start.cjs +1372 -0
- package/hooks/turn-timer.cjs +82 -0
- package/lib/claude-md-writer.cjs +386 -0
- package/lib/config-cjs.cjs +61 -0
- package/lib/hook-pipe-path.cjs +10 -0
- package/lib/keychain-cjs.cjs +263 -0
- package/lib/plugin-paths.cjs +61 -0
- package/lib/rules-builder.cjs +241 -0
- package/lib/text-utils.cjs +61 -0
- package/native/README.md +117 -0
- package/native/prebuilt/linux-aarch64/mixdog-shim +0 -0
- package/native/prebuilt/linux-x86_64/mixdog-shim +0 -0
- package/native/prebuilt/macos-aarch64/mixdog-shim +0 -0
- package/native/prebuilt/macos-x86_64/mixdog-shim +0 -0
- package/native/prebuilt/windows-x86_64/mixdog-shim.exe +0 -0
- package/package.json +107 -0
- package/prompts/code-review.txt +16 -0
- package/prompts/security-audit.txt +17 -0
- package/rules/bridge/00-common.md +39 -0
- package/rules/bridge/20-skip-protocol.md +18 -0
- package/rules/bridge/30-explorer.md +33 -0
- package/rules/bridge/40-cycle1-agent.md +52 -0
- package/rules/bridge/41-cycle2-agent.md +62 -0
- package/rules/bridge/42-cycle3-agent.md +44 -0
- package/rules/lead/00-tool-lead.md +61 -0
- package/rules/lead/01-general.md +23 -0
- package/rules/lead/02-channels.md +49 -0
- package/rules/lead/03-team.md +27 -0
- package/rules/lead/04-workflow.md +20 -0
- package/rules/shared/00-language.md +14 -0
- package/rules/shared/01-tool.md +138 -0
- package/scripts/bootstrap.mjs +184 -0
- package/scripts/bridge-unify-smoke.mjs +308 -0
- package/scripts/build-runtime-linux.sh +348 -0
- package/scripts/build-runtime-macos.sh +217 -0
- package/scripts/build-runtime-windows.ps1 +242 -0
- package/scripts/builtin-utils-smoke.mjs +392 -0
- package/scripts/check-json.mjs +45 -0
- package/scripts/check-syntax-changed.mjs +102 -0
- package/scripts/check-syntax.mjs +58 -0
- package/scripts/code-graph-batch.test.mjs +33 -0
- package/scripts/config-preserve-smoke.mjs +180 -0
- package/scripts/doctor.mjs +484 -0
- package/scripts/edit-normalize-fuzz.mjs +130 -0
- package/scripts/edit-normalize-smoke.mjs +401 -0
- package/scripts/edit-operation-smoke.mjs +369 -0
- package/scripts/edit2-smoke.mjs +63 -0
- package/scripts/fuzzy-e2e.mjs +28 -0
- package/scripts/fuzzy-smoke.mjs +26 -0
- package/scripts/generate-runtime-manifest.mjs +166 -0
- package/scripts/guard-smoke.mjs +66 -0
- package/scripts/hidden-role-schema-smoke.mjs +162 -0
- package/scripts/hook-routing-smoke.mjs +29 -0
- package/scripts/inject-input.ps1 +204 -0
- package/scripts/io-complex-smoke.mjs +667 -0
- package/scripts/io-explore-bench.mjs +424 -0
- package/scripts/io-guardrails-smoke.mjs +205 -0
- package/scripts/io-mini-bench-baseline.json +11 -0
- package/scripts/io-mini-bench.mjs +216 -0
- package/scripts/io-route-harness.mjs +933 -0
- package/scripts/io-telemetry-report.mjs +691 -0
- package/scripts/mutation-bench.mjs +564 -0
- package/scripts/mutation-io-smoke.mjs +1081 -0
- package/scripts/native-patch-bridge-smoke.mjs +288 -0
- package/scripts/native-patch-smoke.mjs +304 -0
- package/scripts/patch-interior-context-smoke.mjs +49 -0
- package/scripts/patch-newline-utf8-smoke.mjs +157 -0
- package/scripts/perf-hook-smoke.mjs +71 -0
- package/scripts/permission-eval-smoke.mjs +426 -0
- package/scripts/prep-patch.mjs +53 -0
- package/scripts/prep-shim.mjs +96 -0
- package/scripts/provider-cache-smoke.mjs +687 -0
- package/scripts/report-runtime-health.mjs +132 -0
- package/scripts/run-mcp.mjs +1547 -0
- package/scripts/salvage-v4a-shatter.test.mjs +58 -0
- package/scripts/scoped-cache-io-smoke.mjs +103 -0
- package/scripts/shell-policy-round3-smoke.mjs +46 -0
- package/scripts/smoke-runtime-negative.ps1 +100 -0
- package/scripts/smoke-runtime-negative.sh +95 -0
- package/scripts/stall-policy-smoke.mjs +50 -0
- package/scripts/start-memory-worker.mjs +23 -0
- package/scripts/statusline-launcher-smoke.mjs +82 -0
- package/scripts/stress-atomic-write.mjs +1028 -0
- package/scripts/test-config-rmw-restore.mjs +122 -0
- package/scripts/test-fault-inject.mjs +164 -0
- package/scripts/test-large-file.mjs +174 -0
- package/scripts/tool-edge-smoke.mjs +209 -0
- package/scripts/uninstall.mjs +201 -0
- package/scripts/webhook-selfheal-smoke.mjs +29 -0
- package/scripts/write-overwrite-guard-smoke.mjs +56 -0
- package/server-main.mjs +3055 -0
- package/server.mjs +468 -0
- package/setup/config-merge.mjs +254 -0
- package/setup/install.mjs +120 -0
- package/setup/launch-core.mjs +507 -0
- package/setup/launch.mjs +101 -0
- package/setup/setup-server.mjs +3206 -0
- package/setup/setup.html +3693 -0
- package/skills/retro-skill-proposer/SKILL.md +92 -0
- package/skills/schedule-add/SKILL.md +77 -0
- package/skills/setup/SKILL.md +346 -0
- package/skills/webhook-add/SKILL.md +81 -0
- package/src/agent/bridge-stall-watchdog.mjs +337 -0
- package/src/agent/index.mjs +2138 -0
- package/src/agent/orchestrator/activity-bus.mjs +38 -0
- package/src/agent/orchestrator/ai-wrapped-dispatch.mjs +1010 -0
- package/src/agent/orchestrator/bridge-retry.mjs +220 -0
- package/src/agent/orchestrator/bridge-trace.mjs +583 -0
- package/src/agent/orchestrator/cache-mtime.mjs +58 -0
- package/src/agent/orchestrator/config.mjs +358 -0
- package/src/agent/orchestrator/context/collect.mjs +651 -0
- package/src/agent/orchestrator/dispatch-persist.mjs +549 -0
- package/src/agent/orchestrator/drain-registry.mjs +50 -0
- package/src/agent/orchestrator/explore-validator.mjs +8 -0
- package/src/agent/orchestrator/internal-roles.mjs +118 -0
- package/src/agent/orchestrator/internal-tools.mjs +88 -0
- package/src/agent/orchestrator/jobs.mjs +116 -0
- package/src/agent/orchestrator/mcp/client.mjs +364 -0
- package/src/agent/orchestrator/providers/anthropic-betas.mjs +21 -0
- package/src/agent/orchestrator/providers/anthropic-oauth.mjs +1745 -0
- package/src/agent/orchestrator/providers/anthropic.mjs +437 -0
- package/src/agent/orchestrator/providers/gemini.mjs +1175 -0
- package/src/agent/orchestrator/providers/grok-oauth.mjs +782 -0
- package/src/agent/orchestrator/providers/model-catalog.mjs +241 -0
- package/src/agent/orchestrator/providers/openai-compat.mjs +1467 -0
- package/src/agent/orchestrator/providers/openai-oauth-ws.mjs +1890 -0
- package/src/agent/orchestrator/providers/openai-oauth.mjs +1307 -0
- package/src/agent/orchestrator/providers/openai-ws.mjs +104 -0
- package/src/agent/orchestrator/providers/registry.mjs +192 -0
- package/src/agent/orchestrator/providers/retry-classifier.mjs +325 -0
- package/src/agent/orchestrator/session/abort-lookup.mjs +13 -0
- package/src/agent/orchestrator/session/cache/post-edit-marks.mjs +42 -0
- package/src/agent/orchestrator/session/cache/prefetch-cache.mjs +142 -0
- package/src/agent/orchestrator/session/cache/read-cache.mjs +319 -0
- package/src/agent/orchestrator/session/cache/scoped-cache-outcome.mjs +11 -0
- package/src/agent/orchestrator/session/cache/scoped-cache.mjs +361 -0
- package/src/agent/orchestrator/session/cache/util.mjs +49 -0
- package/src/agent/orchestrator/session/loop.mjs +1478 -0
- package/src/agent/orchestrator/session/manager.mjs +1975 -0
- package/src/agent/orchestrator/session/read-dedup.mjs +6 -0
- package/src/agent/orchestrator/session/result-classification.mjs +65 -0
- package/src/agent/orchestrator/session/save-session-worker.mjs +18 -0
- package/src/agent/orchestrator/session/store.mjs +624 -0
- package/src/agent/orchestrator/session/stream-watchdog.mjs +130 -0
- package/src/agent/orchestrator/session/tool-result-offload.mjs +166 -0
- package/src/agent/orchestrator/session/trim.mjs +491 -0
- package/src/agent/orchestrator/smart-bridge/CACHE-SHARD.md +115 -0
- package/src/agent/orchestrator/smart-bridge/bridge-llm.mjs +327 -0
- package/src/agent/orchestrator/smart-bridge/cache-obs.mjs +150 -0
- package/src/agent/orchestrator/smart-bridge/cache-strategy.mjs +228 -0
- package/src/agent/orchestrator/smart-bridge/index.mjs +215 -0
- package/src/agent/orchestrator/smart-bridge/profiles.mjs +37 -0
- package/src/agent/orchestrator/smart-bridge/registry.mjs +348 -0
- package/src/agent/orchestrator/smart-bridge/session-builder.mjs +116 -0
- package/src/agent/orchestrator/stall-policy.mjs +195 -0
- package/src/agent/orchestrator/tool-loop-guard.mjs +75 -0
- package/src/agent/orchestrator/tools/bash-policy-scan.mjs +77 -0
- package/src/agent/orchestrator/tools/bash-session.mjs +721 -0
- package/src/agent/orchestrator/tools/builtin/advisory-lock.mjs +171 -0
- package/src/agent/orchestrator/tools/builtin/arg-guard.mjs +455 -0
- package/src/agent/orchestrator/tools/builtin/atomic-write.mjs +236 -0
- package/src/agent/orchestrator/tools/builtin/bash-tool.mjs +480 -0
- package/src/agent/orchestrator/tools/builtin/binary-file.mjs +76 -0
- package/src/agent/orchestrator/tools/builtin/builtin-tools.mjs +256 -0
- package/src/agent/orchestrator/tools/builtin/cache-layers.mjs +386 -0
- package/src/agent/orchestrator/tools/builtin/cwd-utils.mjs +37 -0
- package/src/agent/orchestrator/tools/builtin/device-paths.mjs +154 -0
- package/src/agent/orchestrator/tools/builtin/diagnostics-tool.mjs +292 -0
- package/src/agent/orchestrator/tools/builtin/diff-utils.mjs +109 -0
- package/src/agent/orchestrator/tools/builtin/edit-base-guard.mjs +58 -0
- package/src/agent/orchestrator/tools/builtin/edit-byte-plan.mjs +240 -0
- package/src/agent/orchestrator/tools/builtin/edit-byte-utils.mjs +113 -0
- package/src/agent/orchestrator/tools/builtin/edit-commit.mjs +74 -0
- package/src/agent/orchestrator/tools/builtin/edit-context-utils.mjs +242 -0
- package/src/agent/orchestrator/tools/builtin/edit-diagnostics.mjs +211 -0
- package/src/agent/orchestrator/tools/builtin/edit-engine.mjs +1364 -0
- package/src/agent/orchestrator/tools/builtin/edit-failure-context.mjs +126 -0
- package/src/agent/orchestrator/tools/builtin/edit-hint.mjs +141 -0
- package/src/agent/orchestrator/tools/builtin/edit-match-utils.mjs +194 -0
- package/src/agent/orchestrator/tools/builtin/edit-partial-write.mjs +60 -0
- package/src/agent/orchestrator/tools/builtin/edit-stale-refresh.mjs +168 -0
- package/src/agent/orchestrator/tools/builtin/edit-tool.mjs +173 -0
- package/src/agent/orchestrator/tools/builtin/edit-utf8-guard.mjs +48 -0
- package/src/agent/orchestrator/tools/builtin/fs-reachability.mjs +48 -0
- package/src/agent/orchestrator/tools/builtin/fuzzy-match.mjs +99 -0
- package/src/agent/orchestrator/tools/builtin/glob-walk.mjs +170 -0
- package/src/agent/orchestrator/tools/builtin/grep-formatting.mjs +113 -0
- package/src/agent/orchestrator/tools/builtin/hash-utils.mjs +6 -0
- package/src/agent/orchestrator/tools/builtin/list-formatting.mjs +7 -0
- package/src/agent/orchestrator/tools/builtin/list-tool.mjs +593 -0
- package/src/agent/orchestrator/tools/builtin/native-edit-runner.mjs +89 -0
- package/src/agent/orchestrator/tools/builtin/notebook-edit-tool.mjs +300 -0
- package/src/agent/orchestrator/tools/builtin/open-config-tool.mjs +26 -0
- package/src/agent/orchestrator/tools/builtin/path-diagnostics.mjs +152 -0
- package/src/agent/orchestrator/tools/builtin/path-locks.mjs +35 -0
- package/src/agent/orchestrator/tools/builtin/path-utils.mjs +201 -0
- package/src/agent/orchestrator/tools/builtin/read-args.mjs +103 -0
- package/src/agent/orchestrator/tools/builtin/read-batch.mjs +172 -0
- package/src/agent/orchestrator/tools/builtin/read-constants.mjs +40 -0
- package/src/agent/orchestrator/tools/builtin/read-formatting.mjs +118 -0
- package/src/agent/orchestrator/tools/builtin/read-image-resize.mjs +189 -0
- package/src/agent/orchestrator/tools/builtin/read-image.mjs +88 -0
- package/src/agent/orchestrator/tools/builtin/read-lines.mjs +12 -0
- package/src/agent/orchestrator/tools/builtin/read-mode-tool.mjs +455 -0
- package/src/agent/orchestrator/tools/builtin/read-open.mjs +190 -0
- package/src/agent/orchestrator/tools/builtin/read-range-index.mjs +271 -0
- package/src/agent/orchestrator/tools/builtin/read-ranges.mjs +26 -0
- package/src/agent/orchestrator/tools/builtin/read-single-tool.mjs +728 -0
- package/src/agent/orchestrator/tools/builtin/read-snapshot-runtime.mjs +173 -0
- package/src/agent/orchestrator/tools/builtin/read-special-files.mjs +268 -0
- package/src/agent/orchestrator/tools/builtin/read-streaming.mjs +602 -0
- package/src/agent/orchestrator/tools/builtin/read-tool.mjs +530 -0
- package/src/agent/orchestrator/tools/builtin/read-windows.mjs +107 -0
- package/src/agent/orchestrator/tools/builtin/rename-tool.mjs +196 -0
- package/src/agent/orchestrator/tools/builtin/rg-runner.mjs +422 -0
- package/src/agent/orchestrator/tools/builtin/search-builders.mjs +158 -0
- package/src/agent/orchestrator/tools/builtin/search-tool.mjs +869 -0
- package/src/agent/orchestrator/tools/builtin/shell-analysis.mjs +653 -0
- package/src/agent/orchestrator/tools/builtin/shell-jobs.mjs +936 -0
- package/src/agent/orchestrator/tools/builtin/shell-output.mjs +36 -0
- package/src/agent/orchestrator/tools/builtin/shell-runtime.mjs +214 -0
- package/src/agent/orchestrator/tools/builtin/snapshot-helpers.mjs +143 -0
- package/src/agent/orchestrator/tools/builtin/snapshot-store.mjs +206 -0
- package/src/agent/orchestrator/tools/builtin/snapshot-validation.mjs +98 -0
- package/src/agent/orchestrator/tools/builtin/text-stats.mjs +69 -0
- package/src/agent/orchestrator/tools/builtin/windows-roots.mjs +23 -0
- package/src/agent/orchestrator/tools/builtin/write-tool.mjs +401 -0
- package/src/agent/orchestrator/tools/builtin.mjs +500 -0
- package/src/agent/orchestrator/tools/code-graph-prewarm-worker.mjs +39 -0
- package/src/agent/orchestrator/tools/code-graph-tool-defs.mjs +24 -0
- package/src/agent/orchestrator/tools/code-graph.mjs +4095 -0
- package/src/agent/orchestrator/tools/cwd-tool.mjs +298 -0
- package/src/agent/orchestrator/tools/destructive-warning.mjs +323 -0
- package/src/agent/orchestrator/tools/edit-normalize.mjs +603 -0
- package/src/agent/orchestrator/tools/env-scrub.mjs +100 -0
- package/src/agent/orchestrator/tools/graph-binary-fetcher.mjs +144 -0
- package/src/agent/orchestrator/tools/graph-manifest.json +26 -0
- package/src/agent/orchestrator/tools/host-input.mjs +204 -0
- package/src/agent/orchestrator/tools/mutation-content-cache.mjs +67 -0
- package/src/agent/orchestrator/tools/mutation-planner.mjs +75 -0
- package/src/agent/orchestrator/tools/next-call-utils.mjs +48 -0
- package/src/agent/orchestrator/tools/patch-binary-fetcher.mjs +133 -0
- package/src/agent/orchestrator/tools/patch-manifest.json +26 -0
- package/src/agent/orchestrator/tools/patch-tool-defs.mjs +20 -0
- package/src/agent/orchestrator/tools/patch.mjs +2754 -0
- package/src/agent/orchestrator/tools/progress-message.mjs +118 -0
- package/src/agent/orchestrator/tools/result-compression.mjs +279 -0
- package/src/agent/orchestrator/tools/shell-command.mjs +865 -0
- package/src/agent/orchestrator/tools/shell-exec-policy.mjs +89 -0
- package/src/agent/orchestrator/tools/shell-policy-danger-target.mjs +27 -0
- package/src/agent/orchestrator/tools/shell-policy-imports.mjs +7 -0
- package/src/agent/orchestrator/tools/shell-policy.mjs +345 -0
- package/src/agent/orchestrator/tools/shell-snapshot.mjs +313 -0
- package/src/agent/orchestrator/workflow-store.mjs +93 -0
- package/src/agent/tool-defs.mjs +103 -0
- package/src/channels/backends/discord.mjs +784 -0
- package/src/channels/data/voice-runtime-manifest.json +138 -0
- package/src/channels/index.mjs +3229 -0
- package/src/channels/lib/cli-worker-host.mjs +12 -0
- package/src/channels/lib/config-lock.mjs +13 -0
- package/src/channels/lib/config.mjs +292 -0
- package/src/channels/lib/drop-trace.mjs +71 -0
- package/src/channels/lib/event-pipeline.mjs +81 -0
- package/src/channels/lib/event-queue.mjs +345 -0
- package/src/channels/lib/executor.mjs +168 -0
- package/src/channels/lib/format.mjs +188 -0
- package/src/channels/lib/holidays.mjs +138 -0
- package/src/channels/lib/hook-pipe-server.mjs +802 -0
- package/src/channels/lib/interaction-workflows.mjs +184 -0
- package/src/channels/lib/memory-client.mjs +149 -0
- package/src/channels/lib/output-forwarder.mjs +765 -0
- package/src/channels/lib/runtime-paths.mjs +479 -0
- package/src/channels/lib/scheduler.mjs +723 -0
- package/src/channels/lib/session-control.mjs +36 -0
- package/src/channels/lib/session-discovery.mjs +103 -0
- package/src/channels/lib/settings.mjs +11 -0
- package/src/channels/lib/state-file.mjs +68 -0
- package/src/channels/lib/status-snapshot.mjs +219 -0
- package/src/channels/lib/tool-format.mjs +140 -0
- package/src/channels/lib/transcript-discovery.mjs +195 -0
- package/src/channels/lib/voice-runtime-fetcher.mjs +734 -0
- package/src/channels/lib/webhook.mjs +1179 -0
- package/src/channels/lib/whisper-server.mjs +477 -0
- package/src/channels/tool-defs.mjs +170 -0
- package/src/daemon/host.mjs +118 -0
- package/src/daemon/mcp-transport.mjs +47 -0
- package/src/daemon/session.mjs +100 -0
- package/src/daemon/thin-client.mjs +71 -0
- package/src/daemon/transport.mjs +163 -0
- package/src/memory/data/runtime-manifest.json +40 -0
- package/src/memory/index.mjs +3305 -0
- package/src/memory/lib/agent-ipc.mjs +93 -0
- package/src/memory/lib/bridge-trace-queries.mjs +120 -0
- package/src/memory/lib/core-memory-store.mjs +330 -0
- package/src/memory/lib/embedding-provider.mjs +269 -0
- package/src/memory/lib/embedding-worker.mjs +323 -0
- package/src/memory/lib/llm-worker-host.mjs +17 -0
- package/src/memory/lib/memory-cycle.mjs +11 -0
- package/src/memory/lib/memory-cycle1.mjs +641 -0
- package/src/memory/lib/memory-cycle2.mjs +1284 -0
- package/src/memory/lib/memory-cycle3.mjs +540 -0
- package/src/memory/lib/memory-embed.mjs +299 -0
- package/src/memory/lib/memory-extraction.mjs +5 -0
- package/src/memory/lib/memory-maintenance-store.mjs +32 -0
- package/src/memory/lib/memory-ops-policy.mjs +190 -0
- package/src/memory/lib/memory-recall-id-patch.mjs +15 -0
- package/src/memory/lib/memory-recall-read-query.mjs +7 -0
- package/src/memory/lib/memory-recall-scope-filter.mjs +63 -0
- package/src/memory/lib/memory-recall-store.mjs +621 -0
- package/src/memory/lib/memory-retrievers.mjs +112 -0
- package/src/memory/lib/memory-score.mjs +71 -0
- package/src/memory/lib/memory-text-utils.mjs +58 -0
- package/src/memory/lib/memory.mjs +412 -0
- package/src/memory/lib/model-profile.mjs +85 -0
- package/src/memory/lib/pg/adapter.mjs +308 -0
- package/src/memory/lib/pg/process.mjs +360 -0
- package/src/memory/lib/pg/supervisor.mjs +396 -0
- package/src/memory/lib/project-id-resolver.mjs +86 -0
- package/src/memory/lib/runtime-fetcher.mjs +442 -0
- package/src/memory/lib/trace-store.mjs +728 -0
- package/src/memory/tool-defs.mjs +79 -0
- package/src/search/index.mjs +1173 -0
- package/src/search/lib/backends/anthropic-oauth.mjs +98 -0
- package/src/search/lib/backends/exa.mjs +50 -0
- package/src/search/lib/backends/firecrawl.mjs +61 -0
- package/src/search/lib/backends/gemini-api.mjs +83 -0
- package/src/search/lib/backends/grok-oauth.mjs +86 -0
- package/src/search/lib/backends/index.mjs +150 -0
- package/src/search/lib/backends/openai-api.mjs +144 -0
- package/src/search/lib/backends/openai-oauth.mjs +98 -0
- package/src/search/lib/backends/openai-web-search.mjs +76 -0
- package/src/search/lib/backends/tavily.mjs +55 -0
- package/src/search/lib/backends/xai-api.mjs +113 -0
- package/src/search/lib/cache.mjs +131 -0
- package/src/search/lib/config.mjs +192 -0
- package/src/search/lib/formatter.mjs +115 -0
- package/src/search/lib/provider-usage.mjs +67 -0
- package/src/search/lib/providers.mjs +47 -0
- package/src/search/lib/search-intent.mjs +109 -0
- package/src/search/lib/setup-handler.mjs +261 -0
- package/src/search/lib/state.mjs +201 -0
- package/src/search/lib/web-tools.mjs +1207 -0
- package/src/search/tool-defs.mjs +83 -0
- package/src/setup/defender-exclusion.mjs +183 -0
- package/src/shared/abort-controller.mjs +15 -0
- package/src/shared/atomic-file.mjs +420 -0
- package/src/shared/config.mjs +350 -0
- package/src/shared/daemon-recycle.mjs +108 -0
- package/src/shared/disable-claude-builtins.mjs +88 -0
- package/src/shared/err-text.mjs +12 -0
- package/src/shared/llm/cost.mjs +66 -0
- package/src/shared/llm/http-agent.mjs +123 -0
- package/src/shared/llm/index.mjs +41 -0
- package/src/shared/llm/pid-cleanup.mjs +27 -0
- package/src/shared/llm/usage-log.mjs +47 -0
- package/src/shared/plugin-paths.mjs +58 -0
- package/src/shared/schedules-store.mjs +70 -0
- package/src/shared/seed.mjs +119 -0
- package/src/shared/user-cwd.mjs +213 -0
- package/src/shared/user-data-guard.mjs +238 -0
- package/src/status/aggregator.mjs +584 -0
- package/src/status/server.mjs +413 -0
- package/tools.json +1653 -0
|
@@ -0,0 +1,1890 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* OpenAI Codex OAuth — WebSocket transport.
|
|
3
|
+
*
|
|
4
|
+
* Single dispatch path for the openai-oauth provider (SSE removed in
|
|
5
|
+
* v0.6.117). Uses the `responses_websockets=2026-02-06` beta WebSocket
|
|
6
|
+
* upgrade on chatgpt.com/backend-api/codex/responses. Per-session
|
|
7
|
+
* connections are pooled (5 min idle TTL, up to 8 parallel sockets per
|
|
8
|
+
* key) so subsequent tool-loop iterations can send only the incremental
|
|
9
|
+
* `input` delta plus `previous_response_id`, skipping the full
|
|
10
|
+
* tools/system/history prefix each turn.
|
|
11
|
+
*
|
|
12
|
+
* References:
|
|
13
|
+
* - pi-mono packages/ai/src/providers/openai-codex-responses.ts
|
|
14
|
+
* (acquireWebSocket/release, get_incremental_items delta logic).
|
|
15
|
+
* - openai/codex codex-rs/core/src/client.rs (turn-state echo header).
|
|
16
|
+
*
|
|
17
|
+
* Exposes:
|
|
18
|
+
* sendViaWebSocket({ auth, body, sendOpts, onStreamDelta, onToolCall,
|
|
19
|
+
* onStageChange, externalSignal, poolKey, cacheKey, iteration,
|
|
20
|
+
* useModel, traceCtx })
|
|
21
|
+
*
|
|
22
|
+
* The caller (openai-oauth.mjs) supplies a fully built request body and the
|
|
23
|
+
* auth bundle; this module handles connection caching, delta framing, event
|
|
24
|
+
* parsing, and tracing.
|
|
25
|
+
*/
|
|
26
|
+
import WebSocket from 'ws';
|
|
27
|
+
import { errText } from '../../../shared/err-text.mjs';
|
|
28
|
+
import { createHash, randomBytes } from 'crypto';
|
|
29
|
+
import {
|
|
30
|
+
extractCachedTokens,
|
|
31
|
+
traceBridgeFetch,
|
|
32
|
+
traceBridgeSse,
|
|
33
|
+
traceBridgeUsage,
|
|
34
|
+
appendBridgeTrace,
|
|
35
|
+
} from '../bridge-trace.mjs';
|
|
36
|
+
import { jitterDelayMs, populateHttpStatusFromMessage } from './retry-classifier.mjs';
|
|
37
|
+
import {
|
|
38
|
+
PROVIDER_RETRY_MAX_ATTEMPTS,
|
|
39
|
+
PROVIDER_WS_ACQUIRE_TIMEOUT_MS,
|
|
40
|
+
PROVIDER_WS_FIRST_MEANINGFUL_TIMEOUT_MS,
|
|
41
|
+
PROVIDER_WS_HANDSHAKE_TIMEOUT_MS,
|
|
42
|
+
PROVIDER_WS_INTER_CHUNK_TIMEOUT_MS,
|
|
43
|
+
} from '../stall-policy.mjs';
|
|
44
|
+
|
|
45
|
+
const CODEX_WS_URL = 'wss://chatgpt.com/backend-api/codex/responses';
|
|
46
|
+
const OPENAI_WS_URL = 'wss://api.openai.com/v1/responses';
|
|
47
|
+
const XAI_WS_URL = 'wss://api.x.ai/v1/responses';
|
|
48
|
+
const WS_IDLE_MS = 5 * 60_000;
|
|
49
|
+
const WS_HANDSHAKE_TIMEOUT_MS = PROVIDER_WS_HANDSHAKE_TIMEOUT_MS;
|
|
50
|
+
const WS_ACQUIRE_TIMEOUT_MS = PROVIDER_WS_ACQUIRE_TIMEOUT_MS;
|
|
51
|
+
// Pre-stream watchdog uses the shared provider deadline so it fails before
|
|
52
|
+
// the 5-minute session slow warning.
|
|
53
|
+
const WS_FIRST_MEANINGFUL_MS = PROVIDER_WS_FIRST_MEANINGFUL_TIMEOUT_MS;
|
|
54
|
+
// Pre-`response.created` deadline. Once the socket is open and the
|
|
55
|
+
// response.create frame is sent, a healthy server emits response.created
|
|
56
|
+
// within seconds. If it stalls past this short bound the socket has wedged
|
|
57
|
+
// post-upgrade with zero server events — treat it as a fast, retryable
|
|
58
|
+
// first-byte timeout rather than waiting the longer first-meaningful window.
|
|
59
|
+
// Only this short window is shortened; the post-`response.created`
|
|
60
|
+
// inter-chunk / reasoning span keeps the longer deadlines below.
|
|
61
|
+
const WS_PRE_RESPONSE_CREATED_MS = (() => {
|
|
62
|
+
const raw = process.env.MIXDOG_PROVIDER_WS_PRE_RESPONSE_CREATED_TIMEOUT_MS;
|
|
63
|
+
const n = Number(raw);
|
|
64
|
+
if (Number.isFinite(n) && n > 0) return Math.min(Math.max(n, 1_000), 120_000);
|
|
65
|
+
return 10_000;
|
|
66
|
+
})();
|
|
67
|
+
// Inter-chunk inactivity after first meaningful output.
|
|
68
|
+
const WS_INTER_CHUNK_MS = PROVIDER_WS_INTER_CHUNK_TIMEOUT_MS;
|
|
69
|
+
const MIDSTREAM_WS_TRANSIENT_RETRY_LIMIT = 2;
|
|
70
|
+
const MIDSTREAM_DEFAULT_RETRY_LIMIT = 1;
|
|
71
|
+
const MIDSTREAM_BACKOFF_MS = [250, 1000];
|
|
72
|
+
|
|
73
|
+
// Handshake retry policy. The `ws` library surfaces a bare
|
|
74
|
+
// `Opening handshake has timed out` Error after handshakeTimeout; transient
|
|
75
|
+
// network blips (DNS, reset, 5xx) similarly produce single-shot failures that
|
|
76
|
+
// waste the caller's turn when they'd succeed on retry. We wrap the acquire
|
|
77
|
+
// step with bounded exponential backoff. Permanent auth/quota (4xx) must NOT
|
|
78
|
+
// retry because a second attempt will hit the same deterministic server
|
|
79
|
+
// decision and just double the user-visible latency.
|
|
80
|
+
// Aligned to the cross-provider default (retry-classifier DEFAULT_MAX_ATTEMPTS=5,
|
|
81
|
+
// anthropic-oauth MAX_ATTEMPTS=5, withRetry-using providers all default to 5).
|
|
82
|
+
// Previously 3 — bumped for parity so every provider exhausts the same number
|
|
83
|
+
// of transient-5xx attempts before surfacing failure to the caller.
|
|
84
|
+
const HANDSHAKE_MAX_ATTEMPTS = PROVIDER_RETRY_MAX_ATTEMPTS;
|
|
85
|
+
const HANDSHAKE_BACKOFF_BASE_MS = 500;
|
|
86
|
+
const HANDSHAKE_BACKOFF_CAP_MS = 5000;
|
|
87
|
+
// WS socket pool buckets are keyed by `poolKey` (the per-call sessionId)
|
|
88
|
+
// to isolate parallel bridge invocations — each gets its own socket so
|
|
89
|
+
// a second caller cannot grab a sibling's mid-turn entry (Codex would
|
|
90
|
+
// otherwise reject the new response.create with "No tool output found
|
|
91
|
+
// for function call ..."). The Codex handshake `session_id` header/URL
|
|
92
|
+
// uses `cacheKey` — a provider-scoped unified key (e.g. 'mixdog-codex')
|
|
93
|
+
// built in manager.mjs via providerCacheKey(). All orchestrator-internal
|
|
94
|
+
// dispatches targeting this provider share the same cacheKey, so the
|
|
95
|
+
// server-side prompt-cache shard is shared across every role/source.
|
|
96
|
+
// Codex dedupes cache by handshake session_id, not by
|
|
97
|
+
// body.prompt_cache_key alone (measured 2026-04-19 after the v0.6.151
|
|
98
|
+
// regression).
|
|
99
|
+
const MAX_POOLED_SOCKETS_PER_KEY = 8;
|
|
100
|
+
|
|
101
|
+
// poolKey -> Entry[]
|
|
102
|
+
// Entry: { socket, busy, idleTimer, lastResponseId, lastRequestSansInput,
|
|
103
|
+
// lastInputLen, turnState, closing, ephemeral }
|
|
104
|
+
const _wsPool = new Map();
|
|
105
|
+
|
|
106
|
+
function _getPoolArr(poolKey) {
|
|
107
|
+
if (!poolKey) return null;
|
|
108
|
+
let arr = _wsPool.get(poolKey);
|
|
109
|
+
if (!arr) {
|
|
110
|
+
arr = [];
|
|
111
|
+
_wsPool.set(poolKey, arr);
|
|
112
|
+
}
|
|
113
|
+
return arr;
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
function _removeFromPool(poolKey, entry) {
|
|
117
|
+
if (!poolKey) return;
|
|
118
|
+
const arr = _wsPool.get(poolKey);
|
|
119
|
+
if (!arr) return;
|
|
120
|
+
const idx = arr.indexOf(entry);
|
|
121
|
+
if (idx >= 0) arr.splice(idx, 1);
|
|
122
|
+
if (arr.length === 0) _wsPool.delete(poolKey);
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
function _scheduleIdleClose(poolKey, entry) {
|
|
126
|
+
if (!entry) return;
|
|
127
|
+
if (entry.idleTimer) clearTimeout(entry.idleTimer);
|
|
128
|
+
entry.idleTimer = setTimeout(() => {
|
|
129
|
+
if (entry.busy) return;
|
|
130
|
+
try { entry.socket.close(1000, 'idle_timeout'); } catch {}
|
|
131
|
+
_removeFromPool(poolKey, entry);
|
|
132
|
+
}, WS_IDLE_MS);
|
|
133
|
+
try { entry.idleTimer.unref?.(); } catch {}
|
|
134
|
+
}
|
|
135
|
+
|
|
136
|
+
function _clearIdle(entry) {
|
|
137
|
+
if (entry?.idleTimer) {
|
|
138
|
+
clearTimeout(entry.idleTimer);
|
|
139
|
+
entry.idleTimer = null;
|
|
140
|
+
}
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
function _isOpen(entry) {
|
|
144
|
+
return entry?.socket?.readyState === WebSocket.OPEN;
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
// Awaited frame send. Asserts the socket is OPEN and resolves only after
|
|
148
|
+
// the underlying transport reports the buffered write succeeded (or fails)
|
|
149
|
+
// via the WebSocket send callback. Raw `socket.send(JSON.stringify(...))`
|
|
150
|
+
// is fire-and-forget — a wedged or half-closed socket silently queues the
|
|
151
|
+
// payload and the caller assumes it landed, then later times out waiting
|
|
152
|
+
// for a server event that will never arrive. Tag any failure with
|
|
153
|
+
// `wsSendFailed=true` so _classifyMidstreamError routes the next attempt
|
|
154
|
+
// through a fresh socket.
|
|
155
|
+
function _sendFrame(entry, frame) {
|
|
156
|
+
return new Promise((resolve, reject) => {
|
|
157
|
+
const socket = entry?.socket;
|
|
158
|
+
if (!socket || socket.readyState !== WebSocket.OPEN) {
|
|
159
|
+
const err = new Error(`WS send: socket not OPEN (readyState=${socket?.readyState ?? 'n/a'})`);
|
|
160
|
+
err.wsSendFailed = true;
|
|
161
|
+
reject(err);
|
|
162
|
+
return;
|
|
163
|
+
}
|
|
164
|
+
let payload;
|
|
165
|
+
try { payload = JSON.stringify(frame); }
|
|
166
|
+
catch (e) {
|
|
167
|
+
const err = e instanceof Error ? e : new Error(String(e));
|
|
168
|
+
err.wsSendFailed = true;
|
|
169
|
+
reject(err);
|
|
170
|
+
return;
|
|
171
|
+
}
|
|
172
|
+
try {
|
|
173
|
+
// Do NOT await the send callback: on a wedged-but-OPEN socket the
|
|
174
|
+
// ws write callback may never fire, which would hang this Promise
|
|
175
|
+
// before _streamResponse arms its first-byte watchdog. Fire and
|
|
176
|
+
// resolve immediately; transport failures surface via the socket
|
|
177
|
+
// 'error'/'close' handlers and the first-byte watchdog.
|
|
178
|
+
socket.send(payload, () => {});
|
|
179
|
+
resolve();
|
|
180
|
+
} catch (e) {
|
|
181
|
+
const err = e instanceof Error ? e : new Error(String(e));
|
|
182
|
+
err.wsSendFailed = true;
|
|
183
|
+
reject(err);
|
|
184
|
+
}
|
|
185
|
+
});
|
|
186
|
+
}
|
|
187
|
+
|
|
188
|
+
function _buildHandshakeHeaders({ auth, sessionToken, turnState, cacheKey }) {
|
|
189
|
+
// xAI WS: do NOT pin x-grok-conv-id. Measured parallel runs show that
|
|
190
|
+
// forcing a routing shard via that header alternates cold caches across
|
|
191
|
+
// parallel workers; the automatic prompt-prefix cache holds up better
|
|
192
|
+
// when each handshake is unpinned. Reference: vercel/ai xai provider.
|
|
193
|
+
const headers = auth.type === 'xai'
|
|
194
|
+
? {
|
|
195
|
+
'Authorization': `Bearer ${auth.apiKey}`,
|
|
196
|
+
}
|
|
197
|
+
: auth.type === 'openai-direct'
|
|
198
|
+
? {
|
|
199
|
+
'Authorization': `Bearer ${auth.apiKey}`,
|
|
200
|
+
'OpenAI-Beta': 'responses_websockets=2026-02-06',
|
|
201
|
+
}
|
|
202
|
+
: {
|
|
203
|
+
'Authorization': `Bearer ${auth.access_token}`,
|
|
204
|
+
'chatgpt-account-id': auth.account_id || '',
|
|
205
|
+
'originator': 'mixdog',
|
|
206
|
+
'OpenAI-Beta': 'responses_websockets=2026-02-06',
|
|
207
|
+
};
|
|
208
|
+
if (sessionToken) {
|
|
209
|
+
const sid = String(sessionToken);
|
|
210
|
+
headers['session_id'] = sid;
|
|
211
|
+
}
|
|
212
|
+
// x-client-request-id must be a per-request value so server-side request
|
|
213
|
+
// traces stay distinguishable across retries / reconnects sharing the same
|
|
214
|
+
// session_id. Reusing sessionToken (= cacheKey) collapsed every request
|
|
215
|
+
// for the same conversation onto one trace bucket.
|
|
216
|
+
headers['x-client-request-id'] = randomBytes(16).toString('hex');
|
|
217
|
+
if (turnState) headers['x-codex-turn-state'] = turnState;
|
|
218
|
+
return headers;
|
|
219
|
+
}
|
|
220
|
+
|
|
221
|
+
// handshake session_id is the conversation slot Codex uses for in-memory
|
|
222
|
+
// prefix state. All orchestrator-internal dispatches for this provider share
|
|
223
|
+
// the same cacheKey (built in manager.mjs via providerCacheKey()), so they
|
|
224
|
+
// share the server-side prefix-cache shard across roles/sources.
|
|
225
|
+
function _mintSessionToken(cacheKey, auth) {
|
|
226
|
+
// xAI's public WebSocket endpoint uses the open connection plus
|
|
227
|
+
// response ids for continuation; unlike Codex, it does not need the
|
|
228
|
+
// Codex-specific session_id handshake shard.
|
|
229
|
+
if (auth?.type === 'xai') return null;
|
|
230
|
+
return cacheKey || 'mixdog-default';
|
|
231
|
+
}
|
|
232
|
+
|
|
233
|
+
function _openSocket({ auth, sessionToken, turnState, externalSignal, cacheKey }) {
|
|
234
|
+
const headers = _buildHandshakeHeaders({ auth, sessionToken, turnState, cacheKey });
|
|
235
|
+
const baseUrl = auth.type === 'xai'
|
|
236
|
+
? XAI_WS_URL
|
|
237
|
+
: auth.type === 'openai-direct'
|
|
238
|
+
? OPENAI_WS_URL
|
|
239
|
+
: CODEX_WS_URL;
|
|
240
|
+
const _wsOpenStart = Date.now();
|
|
241
|
+
if (process.env.MIXDOG_DEBUG_BRIDGE) {
|
|
242
|
+
process.stderr.write(`[bridge-trace] ws-open-start url=${baseUrl} tokenHash=${createHash('sha256').update(String(sessionToken)).digest('hex').slice(0, 8)} ts=${_wsOpenStart}\n`);
|
|
243
|
+
}
|
|
244
|
+
const url = baseUrl + (sessionToken ? `?session_id=${encodeURIComponent(String(sessionToken))}` : '');
|
|
245
|
+
return new Promise((resolve, reject) => {
|
|
246
|
+
let settled = false;
|
|
247
|
+
let abortListener = null;
|
|
248
|
+
let acquireTimer = null;
|
|
249
|
+
const settle = (ok, val) => {
|
|
250
|
+
if (settled) return;
|
|
251
|
+
settled = true;
|
|
252
|
+
if (acquireTimer) {
|
|
253
|
+
clearTimeout(acquireTimer);
|
|
254
|
+
acquireTimer = null;
|
|
255
|
+
}
|
|
256
|
+
if (abortListener && externalSignal) {
|
|
257
|
+
try { externalSignal.removeEventListener('abort', abortListener); } catch {}
|
|
258
|
+
}
|
|
259
|
+
(ok ? resolve : reject)(val);
|
|
260
|
+
};
|
|
261
|
+
const socket = new WebSocket(url, { headers, handshakeTimeout: WS_HANDSHAKE_TIMEOUT_MS });
|
|
262
|
+
acquireTimer = setTimeout(() => {
|
|
263
|
+
if (settled) return;
|
|
264
|
+
if (process.env.MIXDOG_DEBUG_BRIDGE) {
|
|
265
|
+
process.stderr.write(`[bridge-trace] ws-open-fail kind=acquire_timeout timeoutMs=${WS_ACQUIRE_TIMEOUT_MS} elapsed=${Date.now() - _wsOpenStart}ms\n`);
|
|
266
|
+
}
|
|
267
|
+
try { socket.terminate(); } catch {}
|
|
268
|
+
settle(false, Object.assign(
|
|
269
|
+
new Error(`${_wsErrLabel(auth?.type === 'xai' ? 'xai' : auth?.type === 'openai-direct' ? 'openai-direct' : 'openai-oauth')} acquire timed out before open (${WS_ACQUIRE_TIMEOUT_MS}ms)`),
|
|
270
|
+
{ code: 'EWSACQUIRETIMEOUT', acquireTimeoutMs: WS_ACQUIRE_TIMEOUT_MS },
|
|
271
|
+
));
|
|
272
|
+
}, WS_ACQUIRE_TIMEOUT_MS);
|
|
273
|
+
try { acquireTimer.unref?.(); } catch {}
|
|
274
|
+
const capturedHeaders = { turnState: null };
|
|
275
|
+
socket.once('upgrade', (res) => {
|
|
276
|
+
try {
|
|
277
|
+
const ts = res?.headers?.['x-codex-turn-state'];
|
|
278
|
+
if (typeof ts === 'string' && ts.length) capturedHeaders.turnState = ts;
|
|
279
|
+
} catch {}
|
|
280
|
+
});
|
|
281
|
+
socket.once('open', () => {
|
|
282
|
+
if (process.env.MIXDOG_DEBUG_BRIDGE) {
|
|
283
|
+
process.stderr.write(`[bridge-trace] ws-open-ok elapsed=${Date.now() - _wsOpenStart}ms\n`);
|
|
284
|
+
}
|
|
285
|
+
settle(true, { socket, turnState: capturedHeaders.turnState });
|
|
286
|
+
});
|
|
287
|
+
socket.once('error', (err) => {
|
|
288
|
+
if (process.env.MIXDOG_DEBUG_BRIDGE) {
|
|
289
|
+
process.stderr.write(`[bridge-trace] ws-open-fail kind=error msg=${String(err?.message || err).slice(0, 120)} elapsed=${Date.now() - _wsOpenStart}ms\n`);
|
|
290
|
+
}
|
|
291
|
+
try { socket.terminate(); } catch {}
|
|
292
|
+
settle(false, err instanceof Error ? err : Object.assign(new Error(errText(err) || 'openai-oauth WS error'), { wsErrorEvent: true, original: err }));
|
|
293
|
+
});
|
|
294
|
+
socket.once('close', (code, reason) => {
|
|
295
|
+
// Half-open handshake: the peer closed before 'open'/'error' fired
|
|
296
|
+
// (TCP RST / TLS edge). Without this the connect Promise never
|
|
297
|
+
// settles and only the 600s outer watchdog can break the stall
|
|
298
|
+
// (observed stage=requesting 601s hang). Open-path closes are
|
|
299
|
+
// no-ops here because settle() has already flipped `settled`.
|
|
300
|
+
if (settled) return;
|
|
301
|
+
try { socket.terminate(); } catch {}
|
|
302
|
+
settle(false, Object.assign(
|
|
303
|
+
new Error(`${_wsErrLabel(auth?.type === 'xai' ? 'xai' : auth?.type === 'openai-direct' ? 'openai-direct' : 'openai-oauth')} handshake closed before open (code=${code})`),
|
|
304
|
+
{ wsCloseCode: code, wsCloseReason: (reason && reason.toString) ? reason.toString('utf-8') : '' }));
|
|
305
|
+
});
|
|
306
|
+
socket.once('unexpected-response', (_req, res) => {
|
|
307
|
+
if (settled) return;
|
|
308
|
+
const status = res?.statusCode || 0;
|
|
309
|
+
let body = '';
|
|
310
|
+
res.on('data', c => { if (body.length < 2048) body += c.toString('utf-8'); });
|
|
311
|
+
res.on('end', () => {
|
|
312
|
+
if (process.env.MIXDOG_DEBUG_BRIDGE) {
|
|
313
|
+
process.stderr.write(`[bridge-trace] ws-open-fail kind=http status=${status} body=${body.slice(0, 120)} elapsed=${Date.now() - _wsOpenStart}ms\n`);
|
|
314
|
+
}
|
|
315
|
+
try { socket.terminate(); } catch {}
|
|
316
|
+
settle(false, Object.assign(new Error(`${_wsErrLabel(auth?.type === 'xai' ? 'xai' : auth?.type === 'openai-direct' ? 'openai-direct' : 'openai-oauth')} handshake ${status}: ${body.slice(0, 200)}`), { httpStatus: status, httpBody: body }));
|
|
317
|
+
});
|
|
318
|
+
});
|
|
319
|
+
if (externalSignal) {
|
|
320
|
+
const onAbort = () => {
|
|
321
|
+
try { socket.terminate(); } catch {}
|
|
322
|
+
const reason = externalSignal.reason;
|
|
323
|
+
settle(false, reason instanceof Error ? reason : new Error(`${_wsErrLabel(auth?.type === 'xai' ? 'xai' : auth?.type === 'openai-direct' ? 'openai-direct' : 'openai-oauth')} handshake aborted`));
|
|
324
|
+
};
|
|
325
|
+
if (externalSignal.aborted) { onAbort(); return; }
|
|
326
|
+
abortListener = onAbort;
|
|
327
|
+
externalSignal.addEventListener('abort', onAbort, { once: true });
|
|
328
|
+
}
|
|
329
|
+
});
|
|
330
|
+
}
|
|
331
|
+
|
|
332
|
+
async function acquireWebSocket({ auth, poolKey, cacheKey, forceFresh, externalSignal }) {
|
|
333
|
+
const _acqStart = Date.now();
|
|
334
|
+
if (process.env.MIXDOG_DEBUG_BRIDGE) {
|
|
335
|
+
process.stderr.write(`[bridge-trace] acquire-start poolKey=${poolKey} cacheKey=${cacheKey} forceFresh=${forceFresh} externalAborted=${!!externalSignal?.aborted} ts=${_acqStart}\n`);
|
|
336
|
+
}
|
|
337
|
+
if (externalSignal?.aborted) {
|
|
338
|
+
const reason = externalSignal.reason;
|
|
339
|
+
throw reason instanceof Error ? reason : new Error('Codex WS acquire aborted');
|
|
340
|
+
}
|
|
341
|
+
if (poolKey && !forceFresh) {
|
|
342
|
+
const arr = _wsPool.get(poolKey) || [];
|
|
343
|
+
// Prune dead entries first.
|
|
344
|
+
for (let i = arr.length - 1; i >= 0; i--) {
|
|
345
|
+
if (!_isOpen(arr[i]) || arr[i].closing) {
|
|
346
|
+
_clearIdle(arr[i]);
|
|
347
|
+
arr.splice(i, 1);
|
|
348
|
+
}
|
|
349
|
+
}
|
|
350
|
+
if (arr.length === 0) _wsPool.delete(poolKey);
|
|
351
|
+
// Reuse any idle open entry (cache-warm path).
|
|
352
|
+
const idle = arr.find(e => !e.busy);
|
|
353
|
+
if (idle) {
|
|
354
|
+
_clearIdle(idle);
|
|
355
|
+
idle.busy = true;
|
|
356
|
+
// Defensive: pre-existing pooled entries created before the
|
|
357
|
+
// prefix-hash field was introduced may not have it set. Normalize
|
|
358
|
+
// to null so the first delta check reads a deterministic value
|
|
359
|
+
// (and falls back to full-create instead of silently passing).
|
|
360
|
+
if (idle.lastInputPrefixHash === undefined) idle.lastInputPrefixHash = null;
|
|
361
|
+
if (process.env.MIXDOG_DEBUG_BRIDGE) {
|
|
362
|
+
process.stderr.write(`[bridge-trace] acquire-reuse poolKey=${poolKey} openSockets=${arr.length} elapsed=${Date.now() - _acqStart}ms\n`);
|
|
363
|
+
}
|
|
364
|
+
return { entry: idle, reused: true };
|
|
365
|
+
}
|
|
366
|
+
// All entries busy and bucket at cap: fall through to ephemeral socket.
|
|
367
|
+
if (arr.length >= MAX_POOLED_SOCKETS_PER_KEY) {
|
|
368
|
+
if (process.env.MIXDOG_DEBUG_BRIDGE) {
|
|
369
|
+
process.stderr.write(`[bridge-trace] acquire-ephemeral cacheKey=${cacheKey} reason=cap elapsed=${Date.now() - _acqStart}ms\n`);
|
|
370
|
+
}
|
|
371
|
+
const ephSessionToken = _mintSessionToken(cacheKey, auth);
|
|
372
|
+
const { socket, turnState } = await _openSocket({ auth, sessionToken: ephSessionToken, turnState: null, externalSignal, cacheKey });
|
|
373
|
+
// Drain-complete fence: same invariant as the normal acquire path —
|
|
374
|
+
// if drain fired during the await, do NOT push an ephemeral entry
|
|
375
|
+
// back into the pool.
|
|
376
|
+
if (_drainComplete) {
|
|
377
|
+
try { socket.close(1000, 'drain-complete'); } catch {}
|
|
378
|
+
throw new Error('WS pool drained — process exiting');
|
|
379
|
+
}
|
|
380
|
+
const entry = {
|
|
381
|
+
socket,
|
|
382
|
+
busy: true,
|
|
383
|
+
idleTimer: null,
|
|
384
|
+
lastResponseId: null,
|
|
385
|
+
lastRequestSansInput: null,
|
|
386
|
+
lastInputLen: 0,
|
|
387
|
+
lastInputPrefixHash: null,
|
|
388
|
+
turnState: turnState || null,
|
|
389
|
+
closing: false,
|
|
390
|
+
ephemeral: true,
|
|
391
|
+
sessionToken: ephSessionToken,
|
|
392
|
+
};
|
|
393
|
+
socket.on('close', () => { entry.closing = true; });
|
|
394
|
+
return { entry, reused: false };
|
|
395
|
+
}
|
|
396
|
+
}
|
|
397
|
+
// Parallel sockets must not inherit sibling turnState or the Codex server
|
|
398
|
+
// treats the new request as a continuation of another in-flight turn and
|
|
399
|
+
// returns "No tool output found for function call …". turnState only
|
|
400
|
+
// propagates within a single entry across its own iterations.
|
|
401
|
+
const sessionToken = _mintSessionToken(cacheKey, auth);
|
|
402
|
+
if (process.env.MIXDOG_DEBUG_BRIDGE) {
|
|
403
|
+
process.stderr.write(`[bridge-trace] acquire-new tokenHash=${createHash('sha256').update(String(sessionToken)).digest('hex').slice(0, 8)} elapsed=${Date.now() - _acqStart}ms\n`);
|
|
404
|
+
}
|
|
405
|
+
const { socket, turnState } = await _openSocket({ auth, sessionToken, turnState: null, externalSignal, cacheKey });
|
|
406
|
+
const entry = {
|
|
407
|
+
socket,
|
|
408
|
+
busy: true,
|
|
409
|
+
idleTimer: null,
|
|
410
|
+
lastResponseId: null,
|
|
411
|
+
lastRequestSansInput: null,
|
|
412
|
+
lastInputLen: 0,
|
|
413
|
+
lastInputPrefixHash: null,
|
|
414
|
+
turnState: turnState || null,
|
|
415
|
+
closing: false,
|
|
416
|
+
ephemeral: false,
|
|
417
|
+
sessionToken,
|
|
418
|
+
};
|
|
419
|
+
if (poolKey && !forceFresh) _getPoolArr(poolKey).push(entry);
|
|
420
|
+
socket.on('close', () => {
|
|
421
|
+
entry.closing = true;
|
|
422
|
+
_removeFromPool(poolKey, entry);
|
|
423
|
+
});
|
|
424
|
+
return { entry, reused: false };
|
|
425
|
+
}
|
|
426
|
+
|
|
427
|
+
function releaseWebSocket({ entry, poolKey, keep }) {
|
|
428
|
+
if (!entry) return;
|
|
429
|
+
entry.busy = false;
|
|
430
|
+
if (!keep || !_isOpen(entry) || !poolKey || entry.ephemeral) {
|
|
431
|
+
try { entry.socket.close(1000, keep ? 'no_session' : 'release_no_keep'); } catch {}
|
|
432
|
+
_removeFromPool(poolKey, entry);
|
|
433
|
+
return;
|
|
434
|
+
}
|
|
435
|
+
_scheduleIdleClose(poolKey, entry);
|
|
436
|
+
}
|
|
437
|
+
|
|
438
|
+
// Port of pi-mono get_incremental_items: if the cached request (sans input)
|
|
439
|
+
// matches the current one and the current input starts with the cached input,
|
|
440
|
+
// return only the tail. Otherwise return the full input (fresh turn).
|
|
441
|
+
function _sansInput(body) {
|
|
442
|
+
const { input: _ignored, ...rest } = body;
|
|
443
|
+
return rest;
|
|
444
|
+
}
|
|
445
|
+
|
|
446
|
+
function _stableStringify(obj) {
|
|
447
|
+
// Shallow stable-ish: JSON.stringify with sorted top-level keys. Nested
|
|
448
|
+
// arrays (tools, include) are order-sensitive and reflect intent, so we
|
|
449
|
+
// do not sort them.
|
|
450
|
+
if (obj === null || typeof obj !== 'object' || Array.isArray(obj)) return JSON.stringify(obj);
|
|
451
|
+
const keys = Object.keys(obj).sort();
|
|
452
|
+
const parts = [];
|
|
453
|
+
for (const k of keys) parts.push(JSON.stringify(k) + ':' + _stableStringify(obj[k]));
|
|
454
|
+
return '{' + parts.join(',') + '}';
|
|
455
|
+
}
|
|
456
|
+
|
|
457
|
+
function _computeDelta({ entry, body }) {
|
|
458
|
+
if (!entry || !entry.lastRequestSansInput || !entry.lastResponseId) {
|
|
459
|
+
return { mode: 'full', frame: { type: 'response.create', ...body } };
|
|
460
|
+
}
|
|
461
|
+
const curSans = _stableStringify(_sansInput(body));
|
|
462
|
+
if (curSans !== entry.lastRequestSansInput) {
|
|
463
|
+
return { mode: 'full', frame: { type: 'response.create', ...body } };
|
|
464
|
+
}
|
|
465
|
+
const prevLen = entry.lastInputLen | 0;
|
|
466
|
+
const curInput = Array.isArray(body.input) ? body.input : [];
|
|
467
|
+
if (curInput.length < prevLen) {
|
|
468
|
+
return { mode: 'full', frame: { type: 'response.create', ...body } };
|
|
469
|
+
}
|
|
470
|
+
// Prefix integrity guard: the cached state on `entry` only stays valid
|
|
471
|
+
// if the current request's first `prevLen` items are byte-identical to
|
|
472
|
+
// the prior full input. If anything in the prefix mutated (a tool result
|
|
473
|
+
// got rewritten, a reasoning item dropped, a system note rotated) the
|
|
474
|
+
// server would mis-anchor the delta. Compare a sha256 of the serialized
|
|
475
|
+
// prefix against the hash captured on the previous success. Mismatch →
|
|
476
|
+
// fall back to a full create for this turn only; the entry itself stays
|
|
477
|
+
// in the pool so the next turn can retry the delta path after
|
|
478
|
+
// sendViaWebSocket refreshes the cache state.
|
|
479
|
+
// Without a hash baseline prefix integrity is unprovable — force full
|
|
480
|
+
// so sendViaWebSocket can seed the hash on success.
|
|
481
|
+
if (entry.lastInputPrefixHash == null) {
|
|
482
|
+
return { mode: 'full', frame: { type: 'response.create', ...body } };
|
|
483
|
+
}
|
|
484
|
+
const curPrefixHash = createHash('sha256')
|
|
485
|
+
.update(JSON.stringify(curInput.slice(0, prevLen)))
|
|
486
|
+
.digest('hex');
|
|
487
|
+
if (curPrefixHash !== entry.lastInputPrefixHash) {
|
|
488
|
+
return { mode: 'full', frame: { type: 'response.create', ...body } };
|
|
489
|
+
}
|
|
490
|
+
const tail = curInput.slice(prevLen);
|
|
491
|
+
return {
|
|
492
|
+
mode: 'delta',
|
|
493
|
+
frame: {
|
|
494
|
+
...body,
|
|
495
|
+
type: 'response.create',
|
|
496
|
+
previous_response_id: entry.lastResponseId,
|
|
497
|
+
input: tail,
|
|
498
|
+
},
|
|
499
|
+
};
|
|
500
|
+
}
|
|
501
|
+
|
|
502
|
+
function _estimateFrameTokens(frame) {
|
|
503
|
+
try {
|
|
504
|
+
const s = JSON.stringify(frame);
|
|
505
|
+
return Math.ceil(s.length / 4);
|
|
506
|
+
} catch { return 0; }
|
|
507
|
+
}
|
|
508
|
+
|
|
509
|
+
function _usageNum(value) {
|
|
510
|
+
const n = Number(value || 0);
|
|
511
|
+
return Number.isFinite(n) ? n : 0;
|
|
512
|
+
}
|
|
513
|
+
|
|
514
|
+
function _combineUsageWithWarmup(actual, warmup) {
|
|
515
|
+
if (!warmup) return actual;
|
|
516
|
+
if (!actual) return warmup;
|
|
517
|
+
const actualRaw = actual.raw || {};
|
|
518
|
+
const warmupRaw = warmup.raw || {};
|
|
519
|
+
const actualTicks = _usageNum(actualRaw.cost_in_usd_ticks);
|
|
520
|
+
const warmupTicks = _usageNum(warmupRaw.cost_in_usd_ticks);
|
|
521
|
+
return {
|
|
522
|
+
...actual,
|
|
523
|
+
inputTokens: _usageNum(actual.inputTokens) + _usageNum(warmup.inputTokens),
|
|
524
|
+
outputTokens: _usageNum(actual.outputTokens) + _usageNum(warmup.outputTokens),
|
|
525
|
+
cachedTokens: _usageNum(actual.cachedTokens) + _usageNum(warmup.cachedTokens),
|
|
526
|
+
promptTokens: _usageNum(actual.promptTokens) + _usageNum(warmup.promptTokens),
|
|
527
|
+
warmupInputTokens: _usageNum(warmup.inputTokens),
|
|
528
|
+
warmupCachedTokens: _usageNum(warmup.cachedTokens),
|
|
529
|
+
warmupOutputTokens: _usageNum(warmup.outputTokens),
|
|
530
|
+
raw: {
|
|
531
|
+
...actualRaw,
|
|
532
|
+
warmup_usage: warmupRaw,
|
|
533
|
+
...(actualTicks || warmupTicks ? { cost_in_usd_ticks: actualTicks + warmupTicks } : {}),
|
|
534
|
+
},
|
|
535
|
+
};
|
|
536
|
+
}
|
|
537
|
+
|
|
538
|
+
function _parseEvent(raw) {
|
|
539
|
+
try { return JSON.parse(raw); } catch { return null; }
|
|
540
|
+
}
|
|
541
|
+
|
|
542
|
+
function _httpStatusFromWsClose(code, reason) {
|
|
543
|
+
const n = Number(code || 0);
|
|
544
|
+
const r = String(reason || '').toLowerCase();
|
|
545
|
+
if (n === 4401
|
|
546
|
+
|| /\b(?:unauthorized|unauthorised|authentication|auth(?:enticated?)?|not authenticated|token expired|access token)\b/.test(r)) {
|
|
547
|
+
return 401;
|
|
548
|
+
}
|
|
549
|
+
if (n === 4403 || /\b(?:forbidden|policy|permission denied)\b/.test(r)) return 403;
|
|
550
|
+
if (n === 4429 || /\b(?:rate limit|quota)\b/.test(r)) return 429;
|
|
551
|
+
return 0;
|
|
552
|
+
}
|
|
553
|
+
|
|
554
|
+
function _wsErrLabel(p) {
|
|
555
|
+
if (p === 'xai') return 'xAI WS';
|
|
556
|
+
if (p === 'openai-direct' || p === 'openai') return 'OpenAI WS';
|
|
557
|
+
return 'Codex WS';
|
|
558
|
+
}
|
|
559
|
+
async function _streamResponse({ entry, externalSignal, onStreamDelta, onToolCall, state, logSuppressedReasoningDeltas = true, traceProvider = 'openai-oauth' }) {
|
|
560
|
+
const errLabel = _wsErrLabel(traceProvider);
|
|
561
|
+
const socket = entry.socket;
|
|
562
|
+
const _streamingStart = Date.now();
|
|
563
|
+
let _firstDeltaEmitted = false;
|
|
564
|
+
let content = '';
|
|
565
|
+
let model = '';
|
|
566
|
+
let responseId = '';
|
|
567
|
+
let responseServiceTier = '';
|
|
568
|
+
const toolCalls = [];
|
|
569
|
+
const webSearchCalls = [];
|
|
570
|
+
const webSearchCallKeys = new Set();
|
|
571
|
+
const citations = [];
|
|
572
|
+
const citationKeys = new Set();
|
|
573
|
+
const pendingCalls = new Map();
|
|
574
|
+
// Reasoning items collected from response.output_item.done (or salvaged
|
|
575
|
+
// from response.completed.response.output). The request still includes
|
|
576
|
+
// `reasoning.encrypted_content` so the server keeps emitting the blobs,
|
|
577
|
+
// but explicit input-side replay is INTENTIONALLY OMITTED in
|
|
578
|
+
// convertMessagesToResponsesInput (openai-oauth.mjs:233-238) — Codex
|
|
579
|
+
// rejects the same `rs_*` id twice in one handshake session_id with a
|
|
580
|
+
// "Duplicate item" error. Server-side conversation state already carries
|
|
581
|
+
// the prefix forward across the WS_IDLE_MS window. The collected
|
|
582
|
+
// reasoningItems below are surfaced for trace/debugging only; they do
|
|
583
|
+
// not feed back into the next request body.
|
|
584
|
+
const reasoningItems = [];
|
|
585
|
+
let reasoningTextDeltaCount = 0;
|
|
586
|
+
let reasoningSummaryTextDeltaCount = 0;
|
|
587
|
+
let reasoningOtherDeltaCount = 0;
|
|
588
|
+
let reasoningDeltaLogEmitted = false;
|
|
589
|
+
const pushReasoningItem = (item) => {
|
|
590
|
+
if (!item || item.type !== 'reasoning') return;
|
|
591
|
+
if (!item.encrypted_content) return;
|
|
592
|
+
reasoningItems.push({
|
|
593
|
+
id: item.id || '',
|
|
594
|
+
encrypted_content: item.encrypted_content,
|
|
595
|
+
summary: Array.isArray(item.summary) ? item.summary : [],
|
|
596
|
+
});
|
|
597
|
+
};
|
|
598
|
+
const pushCitation = (raw, fallbackTitle = '') => {
|
|
599
|
+
const url = raw?.url || raw?.uri || raw?.href || '';
|
|
600
|
+
if (!url || citationKeys.has(url)) return;
|
|
601
|
+
citationKeys.add(url);
|
|
602
|
+
citations.push({
|
|
603
|
+
title: raw?.title || fallbackTitle || '',
|
|
604
|
+
url,
|
|
605
|
+
snippet: raw?.snippet || raw?.text || raw?.description || '',
|
|
606
|
+
source: 'openai-oauth',
|
|
607
|
+
});
|
|
608
|
+
};
|
|
609
|
+
const pushOutputTextAnnotations = (contentPart) => {
|
|
610
|
+
const annotations = Array.isArray(contentPart?.annotations) ? contentPart.annotations : [];
|
|
611
|
+
for (const annotation of annotations) pushCitation(annotation);
|
|
612
|
+
};
|
|
613
|
+
const pushWebSearchCall = (item) => {
|
|
614
|
+
if (!item || item.type !== 'web_search_call') return;
|
|
615
|
+
let key = item.id || '';
|
|
616
|
+
if (!key) {
|
|
617
|
+
try { key = JSON.stringify(item.action || item); } catch { key = `${webSearchCalls.length}`; }
|
|
618
|
+
}
|
|
619
|
+
if (webSearchCallKeys.has(key)) return;
|
|
620
|
+
webSearchCallKeys.add(key);
|
|
621
|
+
webSearchCalls.push({
|
|
622
|
+
id: item.id || '',
|
|
623
|
+
status: item.status || '',
|
|
624
|
+
action: item.action || null,
|
|
625
|
+
});
|
|
626
|
+
const action = item.action || {};
|
|
627
|
+
if (action.url) pushCitation({ url: action.url, title: action.query || '' });
|
|
628
|
+
if (Array.isArray(action.urls)) {
|
|
629
|
+
for (const url of action.urls) pushCitation({ url, title: action.query || '' });
|
|
630
|
+
}
|
|
631
|
+
};
|
|
632
|
+
const logReasoningDeltaSuppression = () => {
|
|
633
|
+
if (!logSuppressedReasoningDeltas) return;
|
|
634
|
+
const total = reasoningTextDeltaCount + reasoningSummaryTextDeltaCount + reasoningOtherDeltaCount;
|
|
635
|
+
if (reasoningDeltaLogEmitted || total === 0) return;
|
|
636
|
+
reasoningDeltaLogEmitted = true;
|
|
637
|
+
process.stderr.write(`[openai-oauth-ws] suppressed reasoning text deltas from user content count=${total} text=${reasoningTextDeltaCount} summary=${reasoningSummaryTextDeltaCount} other=${reasoningOtherDeltaCount}\n`);
|
|
638
|
+
};
|
|
639
|
+
let usage;
|
|
640
|
+
let done = false;
|
|
641
|
+
let terminalError = null;
|
|
642
|
+
// Mid-stream retry classifier needs to distinguish "stream died before we
|
|
643
|
+
// even saw response.created" from "stream died after we had a partial
|
|
644
|
+
// response but before completion". Mutate the shared state object so the
|
|
645
|
+
// caller can inspect flags on the error path without us having to attach
|
|
646
|
+
// them manually at every reject site.
|
|
647
|
+
const midState = state || {};
|
|
648
|
+
midState.sawResponseCreated = midState.sawResponseCreated || false;
|
|
649
|
+
midState.sawCompleted = midState.sawCompleted || false;
|
|
650
|
+
midState.wsCloseCode = null;
|
|
651
|
+
midState.responseFailedPayload = null;
|
|
652
|
+
let idleTimer = null;
|
|
653
|
+
let keepaliveTimer = null;
|
|
654
|
+
let abortHandler = null;
|
|
655
|
+
let messageHandler = null;
|
|
656
|
+
let closeHandler = null;
|
|
657
|
+
let errorHandler = null;
|
|
658
|
+
|
|
659
|
+
return new Promise((resolve, reject) => {
|
|
660
|
+
// Pre-stream watchdog: the timer fires if the server never sends a
|
|
661
|
+
// first event (response.created) within WS_PRE_RESPONSE_CREATED_MS
|
|
662
|
+
// after our last frame. The socket is open and the response.create
|
|
663
|
+
// frame was sent, but no server event has come back — a wedged
|
|
664
|
+
// post-upgrade socket. Healthy servers ack within seconds, so this
|
|
665
|
+
// window is intentionally short (~25s). Once response.created (or
|
|
666
|
+
// any other meaningful event) arrives, the timer is cancelled and
|
|
667
|
+
// the longer inter-chunk inactivity watchdog takes over — silent
|
|
668
|
+
// gaps mid-reasoning (Codex spending 50s+ producing reasoning
|
|
669
|
+
// tokens) are normal and should not abort the turn.
|
|
670
|
+
const armPreStreamWatchdog = () => {
|
|
671
|
+
if (idleTimer) clearTimeout(idleTimer);
|
|
672
|
+
idleTimer = setTimeout(() => {
|
|
673
|
+
if (process.env.MIXDOG_DEBUG_BRIDGE) {
|
|
674
|
+
process.stderr.write(`[bridge-trace] ws-timeout kind=first-byte afterMs=${WS_PRE_RESPONSE_CREATED_MS}\n`);
|
|
675
|
+
}
|
|
676
|
+
const err = new Error(`WS stream: no first server event within ${WS_PRE_RESPONSE_CREATED_MS}ms`);
|
|
677
|
+
// Tag the close code so _classifyMidstreamError sees a 4000
|
|
678
|
+
// (our local pre-stream watchdog code) and routes through
|
|
679
|
+
// the post-upgrade-no-first-event retryable bucket.
|
|
680
|
+
err.wsCloseCode = 4000;
|
|
681
|
+
// Tag the error object itself (not just midState): the warmup
|
|
682
|
+
// path streams under a separate warmupState and rethrows on
|
|
683
|
+
// timeout BEFORE it can copy flags to the outer midState, so the
|
|
684
|
+
// outer catch's _classifyMidstreamError would otherwise see
|
|
685
|
+
// sawResponseCreated=false + close 4000 and hit the pre-created
|
|
686
|
+
// deny gate. err.firstByteTimeout makes both paths retryable.
|
|
687
|
+
err.firstByteTimeout = true;
|
|
688
|
+
midState.firstByteTimeout = true;
|
|
689
|
+
terminalError = err;
|
|
690
|
+
try { socket.close(4000, 'first_byte_timeout'); } catch {}
|
|
691
|
+
// socket.close() may not settle a half-open WS (closeHandler never
|
|
692
|
+
// fires) — reject directly so the turn retries instead of hanging
|
|
693
|
+
// until the 600s watchdog. finish() is idempotent (Promise settles
|
|
694
|
+
// once; cleanup is null-safe).
|
|
695
|
+
finish();
|
|
696
|
+
}, WS_PRE_RESPONSE_CREATED_MS);
|
|
697
|
+
};
|
|
698
|
+
let interChunkTimer = null;
|
|
699
|
+
let firstMeaningfulSeen = false;
|
|
700
|
+
const resetInterChunk = () => {
|
|
701
|
+
if (interChunkTimer) clearTimeout(interChunkTimer);
|
|
702
|
+
interChunkTimer = setTimeout(() => {
|
|
703
|
+
if (process.env.MIXDOG_DEBUG_BRIDGE) {
|
|
704
|
+
process.stderr.write(`[bridge-trace] ws-timeout kind=inter-chunk afterMs=${WS_INTER_CHUNK_MS}\n`);
|
|
705
|
+
}
|
|
706
|
+
terminalError = new Error(`WS stream: inter-chunk inactivity for ${WS_INTER_CHUNK_MS}ms`);
|
|
707
|
+
try { socket.close(4000, 'inter_chunk_timeout'); } catch {}
|
|
708
|
+
// Same half-open guard as the pre-stream watchdog: reject directly
|
|
709
|
+
// so a stuck socket.close() cannot leave the Promise pending.
|
|
710
|
+
finish();
|
|
711
|
+
}, WS_INTER_CHUNK_MS);
|
|
712
|
+
};
|
|
713
|
+
// Called on every event that carries real output tokens or tool progress.
|
|
714
|
+
// Disarms the pre-stream watchdog on first occurrence; thereafter resets
|
|
715
|
+
// the rolling inter-chunk inactivity timer.
|
|
716
|
+
const onMeaningfulOutput = () => {
|
|
717
|
+
if (!firstMeaningfulSeen) {
|
|
718
|
+
firstMeaningfulSeen = true;
|
|
719
|
+
if (idleTimer) { clearTimeout(idleTimer); idleTimer = null; }
|
|
720
|
+
}
|
|
721
|
+
resetInterChunk();
|
|
722
|
+
};
|
|
723
|
+
// resetIdle kept for compat; metadata frames no longer disarm pre-stream watchdog.
|
|
724
|
+
const resetIdle = () => { /* noop — only onMeaningfulOutput() disarms */ };
|
|
725
|
+
const cleanup = () => {
|
|
726
|
+
if (idleTimer) clearTimeout(idleTimer);
|
|
727
|
+
if (interChunkTimer) { clearTimeout(interChunkTimer); interChunkTimer = null; }
|
|
728
|
+
if (keepaliveTimer) { clearInterval(keepaliveTimer); keepaliveTimer = null; }
|
|
729
|
+
if (messageHandler) socket.off('message', messageHandler);
|
|
730
|
+
if (closeHandler) socket.off('close', closeHandler);
|
|
731
|
+
if (errorHandler) socket.off('error', errorHandler);
|
|
732
|
+
if (abortHandler && externalSignal) externalSignal.removeEventListener('abort', abortHandler);
|
|
733
|
+
};
|
|
734
|
+
const finish = () => {
|
|
735
|
+
logReasoningDeltaSuppression();
|
|
736
|
+
cleanup();
|
|
737
|
+
if (terminalError) { reject(terminalError); return; }
|
|
738
|
+
resolve({
|
|
739
|
+
content,
|
|
740
|
+
model,
|
|
741
|
+
reasoningItems: reasoningItems.length ? reasoningItems : undefined,
|
|
742
|
+
toolCalls: toolCalls.length ? toolCalls : undefined,
|
|
743
|
+
citations: citations.length ? citations : undefined,
|
|
744
|
+
webSearchCalls: webSearchCalls.length ? webSearchCalls : undefined,
|
|
745
|
+
usage,
|
|
746
|
+
responseId: responseId || undefined,
|
|
747
|
+
serviceTier: responseServiceTier || undefined,
|
|
748
|
+
});
|
|
749
|
+
};
|
|
750
|
+
|
|
751
|
+
messageHandler = (data) => {
|
|
752
|
+
resetIdle();
|
|
753
|
+
// Do NOT call onStreamDelta for every frame — metadata/keepalive frames
|
|
754
|
+
// must not reset bridge-stall-watchdog's lastStreamDeltaAt. Only
|
|
755
|
+
// meaningful output (text delta / tool call) updates that timestamp.
|
|
756
|
+
const text = typeof data === 'string' ? data : data.toString('utf-8');
|
|
757
|
+
const event = _parseEvent(text);
|
|
758
|
+
if (!event) return;
|
|
759
|
+
if (event.error) {
|
|
760
|
+
const err = new Error(event.error.message || 'Responses WS error');
|
|
761
|
+
try {
|
|
762
|
+
err.payload = event.error;
|
|
763
|
+
populateHttpStatusFromMessage(err);
|
|
764
|
+
} catch {}
|
|
765
|
+
terminalError = err;
|
|
766
|
+
finish();
|
|
767
|
+
return;
|
|
768
|
+
}
|
|
769
|
+
if (typeof event.type !== 'string') return;
|
|
770
|
+
switch (event.type) {
|
|
771
|
+
case 'response.created':
|
|
772
|
+
midState.sawResponseCreated = true;
|
|
773
|
+
if (event.response?.model) model = event.response.model;
|
|
774
|
+
if (event.response?.id) responseId = event.response.id;
|
|
775
|
+
// Server ack — cancel the short pre-`response.created`
|
|
776
|
+
// watchdog and arm the longer inter-chunk inactivity
|
|
777
|
+
// timer for the remainder of the stream. Reasoning
|
|
778
|
+
// silences post-ack are normal and must not trip the
|
|
779
|
+
// 25s first-byte deadline.
|
|
780
|
+
onMeaningfulOutput();
|
|
781
|
+
break;
|
|
782
|
+
case 'response.output_text.delta':
|
|
783
|
+
content += event.delta || '';
|
|
784
|
+
try {
|
|
785
|
+
if (!_firstDeltaEmitted) {
|
|
786
|
+
_firstDeltaEmitted = true;
|
|
787
|
+
if (process.env.MIXDOG_DEBUG_BRIDGE) {
|
|
788
|
+
process.stderr.write(`[bridge-trace] ws-first-delta sinceStreaming=${Date.now() - _streamingStart}ms\n`);
|
|
789
|
+
}
|
|
790
|
+
}
|
|
791
|
+
onStreamDelta?.();
|
|
792
|
+
} catch {}
|
|
793
|
+
onMeaningfulOutput();
|
|
794
|
+
break;
|
|
795
|
+
case 'response.reasoning_text.delta':
|
|
796
|
+
case 'response.reasoning_summary_text.delta':
|
|
797
|
+
if (event.type === 'response.reasoning_text.delta') reasoningTextDeltaCount += 1;
|
|
798
|
+
else reasoningSummaryTextDeltaCount += 1;
|
|
799
|
+
// Reasoning text is live model progress — refresh
|
|
800
|
+
// lastStreamDeltaAt so stream-watchdog does not flag a
|
|
801
|
+
// long reasoning span as a stall. It also counts as
|
|
802
|
+
// liveness for the local pre-stream / inter-chunk
|
|
803
|
+
// watchdogs: a long reasoning span without any
|
|
804
|
+
// output_text delta would otherwise trip the
|
|
805
|
+
// first-meaningful timer and abort an otherwise healthy
|
|
806
|
+
// stream. Reasoning is still suppressed from user
|
|
807
|
+
// content (no `content +=` here) — only the watchdog
|
|
808
|
+
// timers are reset.
|
|
809
|
+
try { onStreamDelta?.(); } catch {}
|
|
810
|
+
onMeaningfulOutput();
|
|
811
|
+
break;
|
|
812
|
+
case 'response.output_item.added':
|
|
813
|
+
if (event.item?.type === 'function_call') {
|
|
814
|
+
pendingCalls.set(event.item.id || '', {
|
|
815
|
+
name: event.item.name || '',
|
|
816
|
+
callId: event.item.call_id || '',
|
|
817
|
+
});
|
|
818
|
+
}
|
|
819
|
+
break;
|
|
820
|
+
case 'response.function_call_arguments.delta':
|
|
821
|
+
try { onStreamDelta?.(); } catch {}
|
|
822
|
+
onMeaningfulOutput();
|
|
823
|
+
break;
|
|
824
|
+
case 'response.function_call_arguments.done': {
|
|
825
|
+
const itemId = event.item_id || '';
|
|
826
|
+
const pending = pendingCalls.get(itemId);
|
|
827
|
+
let args = {};
|
|
828
|
+
try { args = JSON.parse(event.arguments || '{}'); } catch {}
|
|
829
|
+
if (pending?.callId && pending?.name) {
|
|
830
|
+
const call = { id: pending.callId, name: pending.name, arguments: args };
|
|
831
|
+
toolCalls.push(call);
|
|
832
|
+
midState.emittedToolCall = true;
|
|
833
|
+
try { onToolCall?.(call); } catch {}
|
|
834
|
+
} else {
|
|
835
|
+
// Synthesizing a `tc_${Date.now()}` callId here would
|
|
836
|
+
// make the next turn fail to match the model's
|
|
837
|
+
// function_call_output reference. Defer instead and
|
|
838
|
+
// salvage call_id/name from the final
|
|
839
|
+
// response.completed.output bundle below. If salvage
|
|
840
|
+
// also fails we fail the stream explicitly — masking
|
|
841
|
+
// the gap with a synthetic id just shifts the failure
|
|
842
|
+
// one turn later under a confusing "No tool output
|
|
843
|
+
// found for function call" error.
|
|
844
|
+
toolCalls.push({
|
|
845
|
+
id: pending?.callId || '',
|
|
846
|
+
name: pending?.name || '',
|
|
847
|
+
arguments: args,
|
|
848
|
+
_pendingItemId: itemId,
|
|
849
|
+
_deferred: true,
|
|
850
|
+
});
|
|
851
|
+
}
|
|
852
|
+
try { onStreamDelta?.(); } catch {}
|
|
853
|
+
break;
|
|
854
|
+
}
|
|
855
|
+
case 'response.output_item.done':
|
|
856
|
+
// function_call / output_text already captured via their
|
|
857
|
+
// dedicated streaming events. The one shape we still need
|
|
858
|
+
// here is `reasoning` — carries encrypted_content that
|
|
859
|
+
// must be replayed on the next input to keep the Codex
|
|
860
|
+
// server-side prompt cache prefix warm.
|
|
861
|
+
if (event.item?.type === 'reasoning') pushReasoningItem(event.item);
|
|
862
|
+
if (event.item?.type === 'web_search_call') pushWebSearchCall(event.item);
|
|
863
|
+
break;
|
|
864
|
+
case 'response.completed': {
|
|
865
|
+
const completedServiceTier = event.response?.service_tier || event.response?.serviceTier || '';
|
|
866
|
+
if (completedServiceTier) responseServiceTier = String(completedServiceTier);
|
|
867
|
+
if (event.response?.usage) {
|
|
868
|
+
const u = event.response.usage;
|
|
869
|
+
const rawUsage = responseServiceTier
|
|
870
|
+
? { ...u, service_tier: responseServiceTier }
|
|
871
|
+
: u;
|
|
872
|
+
usage = {
|
|
873
|
+
inputTokens: u.input_tokens || 0,
|
|
874
|
+
outputTokens: u.output_tokens || 0,
|
|
875
|
+
cachedTokens: extractCachedTokens(u),
|
|
876
|
+
// OpenAI Codex reports input_tokens as the total
|
|
877
|
+
// prompt volume (cached portion is a subset, not
|
|
878
|
+
// additive). Alias into the cross-provider
|
|
879
|
+
// `promptTokens` field so downstream loggers have
|
|
880
|
+
// uniform semantics.
|
|
881
|
+
promptTokens: u.input_tokens || 0,
|
|
882
|
+
raw: rawUsage,
|
|
883
|
+
};
|
|
884
|
+
}
|
|
885
|
+
if (!model && event.response?.model) model = event.response.model;
|
|
886
|
+
if (!responseId && event.response?.id) responseId = event.response.id;
|
|
887
|
+
if (event.response?.output) {
|
|
888
|
+
for (const item of event.response.output) {
|
|
889
|
+
if (!content && item.type === 'message') {
|
|
890
|
+
for (const c of item.content || []) {
|
|
891
|
+
if (c.type === 'output_text') {
|
|
892
|
+
content += c.text || '';
|
|
893
|
+
pushOutputTextAnnotations(c);
|
|
894
|
+
}
|
|
895
|
+
}
|
|
896
|
+
}
|
|
897
|
+
if (item.type === 'message') {
|
|
898
|
+
for (const c of item.content || []) {
|
|
899
|
+
if (c.type === 'output_text') pushOutputTextAnnotations(c);
|
|
900
|
+
}
|
|
901
|
+
}
|
|
902
|
+
if (item.type === 'web_search_call') pushWebSearchCall(item);
|
|
903
|
+
// Salvage path: some streams emit reasoning only
|
|
904
|
+
// inside the final response.completed.output
|
|
905
|
+
// bundle (no per-item .done event). Dedup by id.
|
|
906
|
+
if (item.type === 'reasoning'
|
|
907
|
+
&& !reasoningItems.some(r => r.id === item.id)) {
|
|
908
|
+
pushReasoningItem(item);
|
|
909
|
+
}
|
|
910
|
+
// Salvage path for function_call: when
|
|
911
|
+
// arguments.done fired before (or without) a
|
|
912
|
+
// matching output_item.added, the deferred tool
|
|
913
|
+
// call placeholder has empty id/name. The
|
|
914
|
+
// completed.output bundle carries the canonical
|
|
915
|
+
// call_id/name; fill them in and emit onToolCall.
|
|
916
|
+
if (item.type === 'function_call') {
|
|
917
|
+
const tc = toolCalls.find(
|
|
918
|
+
(t) => t._deferred && t._pendingItemId === (item.id || ''),
|
|
919
|
+
);
|
|
920
|
+
if (tc) {
|
|
921
|
+
if (!tc.id && item.call_id) tc.id = item.call_id;
|
|
922
|
+
if (!tc.name && item.name) tc.name = item.name;
|
|
923
|
+
if (tc.id && tc.name) {
|
|
924
|
+
delete tc._deferred;
|
|
925
|
+
delete tc._pendingItemId;
|
|
926
|
+
midState.emittedToolCall = true;
|
|
927
|
+
try { onToolCall?.(tc); } catch {}
|
|
928
|
+
}
|
|
929
|
+
}
|
|
930
|
+
}
|
|
931
|
+
}
|
|
932
|
+
}
|
|
933
|
+
// Salvage validation. Any deferred call still missing
|
|
934
|
+
// id/name would propagate to the next turn as a
|
|
935
|
+
// function_call_output the server can't anchor. Fail the
|
|
936
|
+
// stream now so the caller sees a deterministic error
|
|
937
|
+
// instead of a cryptic mismatch one turn later.
|
|
938
|
+
const unresolved = toolCalls.find((t) => t._deferred);
|
|
939
|
+
if (unresolved) {
|
|
940
|
+
terminalError = new Error(
|
|
941
|
+
`${errLabel} function_call salvage failed: missing call_id/name for item_id=${unresolved._pendingItemId || '?'}`,
|
|
942
|
+
);
|
|
943
|
+
finish();
|
|
944
|
+
break;
|
|
945
|
+
}
|
|
946
|
+
midState.sawCompleted = true;
|
|
947
|
+
done = true;
|
|
948
|
+
finish();
|
|
949
|
+
break;
|
|
950
|
+
}
|
|
951
|
+
case 'response.done': {
|
|
952
|
+
// response.done is the terminal frame for some Codex
|
|
953
|
+
// streams that never emit a separate response.completed.
|
|
954
|
+
// Route through the same completed/failed/incomplete
|
|
955
|
+
// normalization based on event.response.status so a
|
|
956
|
+
// server-side abort (incomplete / failed) does not slip
|
|
957
|
+
// through as success. status === 'completed' falls
|
|
958
|
+
// through to the success path with sawCompleted set;
|
|
959
|
+
// anything else is converted into a terminal error.
|
|
960
|
+
const status = event.response?.status || '';
|
|
961
|
+
if (status === 'failed') {
|
|
962
|
+
midState.responseFailedPayload = event;
|
|
963
|
+
const msg = event.response?.error?.message
|
|
964
|
+
|| event.error?.message
|
|
965
|
+
|| event.message
|
|
966
|
+
|| 'response.done failed';
|
|
967
|
+
terminalError = Object.assign(
|
|
968
|
+
new Error(`${errLabel} response.done failed: ${msg}`),
|
|
969
|
+
{ responseFailed: event },
|
|
970
|
+
);
|
|
971
|
+
populateHttpStatusFromMessage(terminalError, msg);
|
|
972
|
+
done = true;
|
|
973
|
+
finish();
|
|
974
|
+
break;
|
|
975
|
+
}
|
|
976
|
+
if (status === 'incomplete') {
|
|
977
|
+
const reasonObj = event.response?.incomplete_details
|
|
978
|
+
|| event.incomplete_details
|
|
979
|
+
|| event.response?.status_details
|
|
980
|
+
|| null;
|
|
981
|
+
const reasonStr = reasonObj?.reason
|
|
982
|
+
|| event.response?.status
|
|
983
|
+
|| 'incomplete';
|
|
984
|
+
terminalError = Object.assign(
|
|
985
|
+
new Error(`${errLabel} response.done incomplete: ${reasonStr}`),
|
|
986
|
+
{ responseIncomplete: event, incompleteReason: reasonStr },
|
|
987
|
+
);
|
|
988
|
+
done = true;
|
|
989
|
+
finish();
|
|
990
|
+
break;
|
|
991
|
+
}
|
|
992
|
+
if (status && status !== 'completed') {
|
|
993
|
+
terminalError = Object.assign(
|
|
994
|
+
new Error(`${errLabel} response.done unexpected status: ${status}`),
|
|
995
|
+
{ responseDoneStatus: status },
|
|
996
|
+
);
|
|
997
|
+
done = true;
|
|
998
|
+
finish();
|
|
999
|
+
break;
|
|
1000
|
+
}
|
|
1001
|
+
midState.sawCompleted = true;
|
|
1002
|
+
done = true;
|
|
1003
|
+
finish();
|
|
1004
|
+
break;
|
|
1005
|
+
}
|
|
1006
|
+
case 'response.incomplete': {
|
|
1007
|
+
// response.incomplete is a server-side abort (max_output_tokens,
|
|
1008
|
+
// content filter, length, etc.). Surfacing it as success silently
|
|
1009
|
+
// truncates the turn; convert to a terminal error so the caller
|
|
1010
|
+
// can decide whether to retry / surface to user.
|
|
1011
|
+
const reasonObj = event.response?.incomplete_details
|
|
1012
|
+
|| event.incomplete_details
|
|
1013
|
+
|| event.response?.status_details
|
|
1014
|
+
|| null;
|
|
1015
|
+
const reasonStr = reasonObj?.reason
|
|
1016
|
+
|| event.response?.status
|
|
1017
|
+
|| 'incomplete';
|
|
1018
|
+
terminalError = Object.assign(
|
|
1019
|
+
new Error(`${errLabel} response.incomplete: ${reasonStr}`),
|
|
1020
|
+
{ responseIncomplete: event, incompleteReason: reasonStr },
|
|
1021
|
+
);
|
|
1022
|
+
finish();
|
|
1023
|
+
break;
|
|
1024
|
+
}
|
|
1025
|
+
case 'response.failed': {
|
|
1026
|
+
// Stash the payload so the mid-stream classifier can sniff
|
|
1027
|
+
// network_error / stream_disconnected without re-parsing.
|
|
1028
|
+
midState.responseFailedPayload = event;
|
|
1029
|
+
const msg = event.response?.error?.message
|
|
1030
|
+
|| event.error?.message
|
|
1031
|
+
|| event.message
|
|
1032
|
+
|| 'response.failed';
|
|
1033
|
+
terminalError = Object.assign(new Error(`${errLabel} response.failed: ${msg}`), {
|
|
1034
|
+
responseFailed: event,
|
|
1035
|
+
});
|
|
1036
|
+
// Sniff the server message for transient/auth/permanent
|
|
1037
|
+
// hints so the handshake / mid-stream retry classifiers
|
|
1038
|
+
// can route by httpStatus. Without this, server-side
|
|
1039
|
+
// events like "Our servers are currently overloaded"
|
|
1040
|
+
// surfaced as unclassified errors and skipped the
|
|
1041
|
+
// 5xx retry bucket entirely.
|
|
1042
|
+
populateHttpStatusFromMessage(terminalError, msg);
|
|
1043
|
+
finish();
|
|
1044
|
+
break;
|
|
1045
|
+
}
|
|
1046
|
+
case 'error': {
|
|
1047
|
+
const errMsg = String(event.message || event.error?.message || 'unknown');
|
|
1048
|
+
terminalError = new Error(`${errLabel} error: ${errMsg}`);
|
|
1049
|
+
populateHttpStatusFromMessage(terminalError, errMsg);
|
|
1050
|
+
finish();
|
|
1051
|
+
break;
|
|
1052
|
+
}
|
|
1053
|
+
default:
|
|
1054
|
+
// Catch any other reasoning-delta variants (e.g.
|
|
1055
|
+
// `response.reasoning.<sub>.delta`) so they are counted
|
|
1056
|
+
// and suppressed, never reaching the user content buffer.
|
|
1057
|
+
if (typeof event.type === 'string'
|
|
1058
|
+
&& event.type.startsWith('response.reasoning')
|
|
1059
|
+
&& event.type.endsWith('.delta')) {
|
|
1060
|
+
reasoningOtherDeltaCount += 1;
|
|
1061
|
+
}
|
|
1062
|
+
// Trace-only events (response.in_progress, etc.)
|
|
1063
|
+
break;
|
|
1064
|
+
}
|
|
1065
|
+
};
|
|
1066
|
+
closeHandler = (code, reason) => {
|
|
1067
|
+
if (done) return;
|
|
1068
|
+
midState.wsCloseCode = code;
|
|
1069
|
+
if (!terminalError) {
|
|
1070
|
+
const r = reason?.toString?.('utf-8') || '';
|
|
1071
|
+
const httpStatus = _httpStatusFromWsClose(code, r);
|
|
1072
|
+
terminalError = Object.assign(
|
|
1073
|
+
new Error(`Codex WS closed before response.completed (code=${code}${r ? `, reason=${r}` : ''})`),
|
|
1074
|
+
{ wsCloseCode: code, wsCloseReason: r, ...(httpStatus ? { httpStatus } : {}) },
|
|
1075
|
+
);
|
|
1076
|
+
} else if (terminalError && !terminalError.wsCloseCode) {
|
|
1077
|
+
try { terminalError.wsCloseCode = code; } catch {}
|
|
1078
|
+
try { terminalError.httpStatus = terminalError.httpStatus || _httpStatusFromWsClose(code, reason?.toString?.('utf-8') || ''); } catch {}
|
|
1079
|
+
}
|
|
1080
|
+
finish();
|
|
1081
|
+
};
|
|
1082
|
+
errorHandler = (err) => {
|
|
1083
|
+
if (done) return;
|
|
1084
|
+
const wrapped = err instanceof Error ? err : new Error(String(err));
|
|
1085
|
+
if (terminalError) {
|
|
1086
|
+
// Preserve the first terminalError; chain the later socket
|
|
1087
|
+
// error in via `cause` (or `suppressed` if cause already set)
|
|
1088
|
+
// so diagnostics keep the original failure visible.
|
|
1089
|
+
try {
|
|
1090
|
+
if (!terminalError.cause) terminalError.cause = wrapped;
|
|
1091
|
+
else {
|
|
1092
|
+
const list = Array.isArray(terminalError.suppressed)
|
|
1093
|
+
? terminalError.suppressed
|
|
1094
|
+
: [];
|
|
1095
|
+
list.push(wrapped);
|
|
1096
|
+
terminalError.suppressed = list;
|
|
1097
|
+
}
|
|
1098
|
+
} catch {}
|
|
1099
|
+
} else {
|
|
1100
|
+
terminalError = wrapped;
|
|
1101
|
+
}
|
|
1102
|
+
try { socket.close(4001, 'stream_error'); } catch {}
|
|
1103
|
+
finish();
|
|
1104
|
+
};
|
|
1105
|
+
if (externalSignal) {
|
|
1106
|
+
abortHandler = () => {
|
|
1107
|
+
if (done) return;
|
|
1108
|
+
const reason = externalSignal.reason;
|
|
1109
|
+
terminalError = reason instanceof Error ? reason : new Error('Codex WS aborted by session close');
|
|
1110
|
+
// Tag: was this a user/caller abort, or a watchdog abort?
|
|
1111
|
+
// Mid-stream retry must skip user aborts but may retry watchdog
|
|
1112
|
+
// aborts. The caller-owned AbortController surfaces through
|
|
1113
|
+
// externalSignal; bridge-stall-watchdog signals via a reason
|
|
1114
|
+
// object whose name === 'BridgeStallAbortError'. stream-watchdog
|
|
1115
|
+
// uses StreamStalledAbortError. Anything else → treat as user.
|
|
1116
|
+
const reasonName = reason?.name || '';
|
|
1117
|
+
if (reasonName === 'BridgeStallAbortError'
|
|
1118
|
+
|| reasonName === 'StreamStalledAbortError') {
|
|
1119
|
+
midState.watchdogAbort = reasonName;
|
|
1120
|
+
} else {
|
|
1121
|
+
midState.userAbort = true;
|
|
1122
|
+
}
|
|
1123
|
+
try { socket.close(4002, 'aborted'); } catch {}
|
|
1124
|
+
finish();
|
|
1125
|
+
};
|
|
1126
|
+
if (externalSignal.aborted) { abortHandler(); return; }
|
|
1127
|
+
externalSignal.addEventListener('abort', abortHandler, { once: true });
|
|
1128
|
+
}
|
|
1129
|
+
socket.on('message', messageHandler);
|
|
1130
|
+
socket.on('close', closeHandler);
|
|
1131
|
+
socket.on('error', errorHandler);
|
|
1132
|
+
armPreStreamWatchdog();
|
|
1133
|
+
// Periodic client-side WS ping while the stream is active. Codex's
|
|
1134
|
+
// server closes with 1011 "keepalive ping timeout" when it thinks the
|
|
1135
|
+
// peer is silent during long reasoning windows where no data frames
|
|
1136
|
+
// flow. Sending a ping every 18s from our side keeps the socket warm.
|
|
1137
|
+
// The interval is unref'd so it never holds the event loop open, and
|
|
1138
|
+
// cleanup() clears it on every terminal path (completed / close /
|
|
1139
|
+
// error / abort / mid-stream retry teardown).
|
|
1140
|
+
keepaliveTimer = setInterval(() => {
|
|
1141
|
+
try {
|
|
1142
|
+
if (socket.readyState !== WebSocket.OPEN) return;
|
|
1143
|
+
socket.ping();
|
|
1144
|
+
} catch {}
|
|
1145
|
+
}, 18_000);
|
|
1146
|
+
try { keepaliveTimer.unref?.(); } catch {}
|
|
1147
|
+
});
|
|
1148
|
+
}
|
|
1149
|
+
|
|
1150
|
+
/**
|
|
1151
|
+
* Classify a handshake error for retry eligibility.
|
|
1152
|
+
*
|
|
1153
|
+
* Default-deny: anything we don't recognize as transient returns null (treat
|
|
1154
|
+
* as permanent). Permanent buckets (401/403/404/429) also return null — the
|
|
1155
|
+
* server has made a deterministic decision that a retry can't change.
|
|
1156
|
+
*
|
|
1157
|
+
* Returns one of:
|
|
1158
|
+
* 'timeout' — `ws` handshakeTimeout fired
|
|
1159
|
+
* 'reset' — ECONNRESET / socket hang up
|
|
1160
|
+
* 'dns' — EAI_AGAIN / ENOTFOUND / EAI_NODATA
|
|
1161
|
+
* 'refused' — ECONNREFUSED
|
|
1162
|
+
* 'network' — ENETUNREACH / EHOSTUNREACH / EPIPE
|
|
1163
|
+
* 'acquire_timeout' — hard client-side open/acquire deadline fired
|
|
1164
|
+
* 'http_5xx' (with specific status e.g. 'http_503') — server overload
|
|
1165
|
+
* null — not retryable
|
|
1166
|
+
*/
|
|
1167
|
+
export function _classifyHandshakeError(err) {
|
|
1168
|
+
if (!err) return null;
|
|
1169
|
+
const code = err.code || '';
|
|
1170
|
+
const msg = String(err.message || '');
|
|
1171
|
+
const status = Number(err.httpStatus || 0);
|
|
1172
|
+
|
|
1173
|
+
// Permanent HTTP (auth / quota / not-found) short-circuits.
|
|
1174
|
+
if (status === 401 || status === 403 || status === 404 || status === 429) {
|
|
1175
|
+
return null;
|
|
1176
|
+
}
|
|
1177
|
+
// 5xx transient.
|
|
1178
|
+
if (status >= 500 && status < 600) {
|
|
1179
|
+
return `http_${status}`;
|
|
1180
|
+
}
|
|
1181
|
+
|
|
1182
|
+
// Node errno codes.
|
|
1183
|
+
if (code === 'ECONNRESET') return 'reset';
|
|
1184
|
+
if (code === 'EAI_AGAIN' || code === 'ENOTFOUND' || code === 'EAI_NODATA') return 'dns';
|
|
1185
|
+
if (code === 'ECONNREFUSED') return 'refused';
|
|
1186
|
+
if (code === 'ETIMEDOUT' || code === 'ESOCKETTIMEDOUT') return 'timeout';
|
|
1187
|
+
if (code === 'EWSACQUIRETIMEOUT') return 'acquire_timeout';
|
|
1188
|
+
if (code === 'ENETUNREACH' || code === 'EHOSTUNREACH' || code === 'EPIPE') return 'network';
|
|
1189
|
+
|
|
1190
|
+
// `ws` library's handshake-timeout path: thrown as a bare Error.
|
|
1191
|
+
if (/opening handshake has timed out/i.test(msg)) return 'timeout';
|
|
1192
|
+
if (/socket hang up/i.test(msg)) return 'reset';
|
|
1193
|
+
|
|
1194
|
+
return null;
|
|
1195
|
+
}
|
|
1196
|
+
|
|
1197
|
+
/**
|
|
1198
|
+
* Classify a mid-stream error for bounded retry eligibility.
|
|
1199
|
+
*
|
|
1200
|
+
* Only fires AFTER `response.created` is observed and BEFORE
|
|
1201
|
+
* `response.completed`. The window is narrow on purpose: retrying a handshake
|
|
1202
|
+
* or a pre-create connect failure is owned by _acquireWithRetry; retrying
|
|
1203
|
+
* after completion would replay a finished turn.
|
|
1204
|
+
*
|
|
1205
|
+
* Retry buckets:
|
|
1206
|
+
* 'bridge_stall' — BridgeStallAbortError from bridge-stall-watchdog
|
|
1207
|
+
* 'stream_stalled' — StreamStalledAbortError from stream-watchdog
|
|
1208
|
+
* 'ws_1006' — abnormal close (connection lost)
|
|
1209
|
+
* 'ws_1011' — server unexpected condition
|
|
1210
|
+
* 'ws_1012' — service restart
|
|
1211
|
+
* 'ws_4000' — our armPreStreamWatchdog close with idle_timeout
|
|
1212
|
+
* 'ws_1000' — server-side normal close fired after response.created
|
|
1213
|
+
* but before response.completed (truncated stream)
|
|
1214
|
+
* 'first_byte_timeout' — post-upgrade-no-first-event: socket opened, our
|
|
1215
|
+
* response.create frame sent, but the server never
|
|
1216
|
+
* emitted response.created within the short
|
|
1217
|
+
* pre-stream deadline. Fast-fail retryable.
|
|
1218
|
+
* 'response_failed_network' — response.failed with network_error
|
|
1219
|
+
* 'response_failed_disconnected' — response.failed with stream_disconnected
|
|
1220
|
+
*
|
|
1221
|
+
* Deny buckets (return null):
|
|
1222
|
+
* - externalSignal aborted by user (state.userAbort)
|
|
1223
|
+
* - state.sawCompleted === true (already done)
|
|
1224
|
+
* - state.sawResponseCreated === false (still pre-stream; handshake retry
|
|
1225
|
+
* owns that window) — EXCEPT for WS close 1011/1012, which can fire
|
|
1226
|
+
* after the 101 upgrade but before the first response.created event,
|
|
1227
|
+
* AND the pre-`response.created` first-byte timeout
|
|
1228
|
+
* (state.firstByteTimeout), which is permitted a bounded retry here
|
|
1229
|
+
* - HTTP 401 / 403 / 429 surfaced on the error
|
|
1230
|
+
* - state.attemptIndex has reached the classifier-specific retry budget
|
|
1231
|
+
*/
|
|
1232
|
+
export function _classifyMidstreamError(err, state) {
|
|
1233
|
+
if (!state) return null;
|
|
1234
|
+
const attemptIndex = state.attemptIndex | 0;
|
|
1235
|
+
// Already completed (shouldn't throw, but defensive).
|
|
1236
|
+
if (state.sawCompleted) return null;
|
|
1237
|
+
// Any tool call already surfaced to the caller — retrying would
|
|
1238
|
+
// normally duplicate the side effect. EXCEPTION: ws_1000 truncation
|
|
1239
|
+
// (server-side normal close after response.created, before completion)
|
|
1240
|
+
// leaves the caller with an orphaned tool_use that the next turn cannot
|
|
1241
|
+
// pair to a tool_result, which the provider rejects with a hard 400.
|
|
1242
|
+
// The duplicate-side-effect risk is preferable to deterministic worker
|
|
1243
|
+
// death, especially for detached bridges that re-dispatch idempotently.
|
|
1244
|
+
if (state.emittedToolCall) {
|
|
1245
|
+
const _cc = Number(err?.wsCloseCode || state.wsCloseCode || 0);
|
|
1246
|
+
if (!(_cc === 1000 && state.sawResponseCreated && !state.sawCompleted)) return null;
|
|
1247
|
+
}
|
|
1248
|
+
// Post-upgrade-no-first-event: the socket opened, our response.create
|
|
1249
|
+
// frame was sent, but the server never emitted a single event before
|
|
1250
|
+
// the short pre-`response.created` watchdog fired. The handshake retry
|
|
1251
|
+
// layer only sees pre-upgrade failures and the legacy pre-stream gate
|
|
1252
|
+
// below would deny this case (sawResponseCreated === false). Tag it
|
|
1253
|
+
// here as a fast retryable bucket so the worker reconnects within
|
|
1254
|
+
// seconds instead of stalling for the full first-meaningful window.
|
|
1255
|
+
if (state.firstByteTimeout || err?.firstByteTimeout) {
|
|
1256
|
+
return _allowMidstreamRetry('first_byte_timeout', attemptIndex);
|
|
1257
|
+
}
|
|
1258
|
+
// _sendFrame failure (socket not OPEN, send callback errored, JSON
|
|
1259
|
+
// serialize threw). Always retryable: caller will forceFresh next
|
|
1260
|
+
// attempt so the wedged socket is dropped.
|
|
1261
|
+
if (err?.wsSendFailed || state.wsSendFailed) {
|
|
1262
|
+
return _allowMidstreamRetry('ws_send_failed', attemptIndex);
|
|
1263
|
+
}
|
|
1264
|
+
// Pre-stream failures normally belong to the handshake retry layer. BUT
|
|
1265
|
+
// WS close 1011 / 1012 can fire after the 101 upgrade but BEFORE the
|
|
1266
|
+
// first response.created event when the server's keepalive times out or
|
|
1267
|
+
// the service restarts. Neither the handshake retry layer (it only sees
|
|
1268
|
+
// pre-upgrade failures) nor the existing mid-stream gate covers this
|
|
1269
|
+
// window, so permit bounded retry here for those two codes only.
|
|
1270
|
+
if (!state.sawResponseCreated) {
|
|
1271
|
+
const closeCode = Number(err?.wsCloseCode || state.wsCloseCode || 0);
|
|
1272
|
+
if (closeCode !== 1011 && closeCode !== 1012) return null;
|
|
1273
|
+
}
|
|
1274
|
+
// User/caller abort — never retry.
|
|
1275
|
+
if (state.userAbort) return null;
|
|
1276
|
+
|
|
1277
|
+
if (!err) return null;
|
|
1278
|
+
const status = Number(err?.httpStatus || 0);
|
|
1279
|
+
if (status === 401 || status === 403 || status === 429) return null;
|
|
1280
|
+
// Transient 5xx surfaced via populateHttpStatusFromMessage (case 'error'
|
|
1281
|
+
// and case 'response.failed' branches sniff server-supplied text like
|
|
1282
|
+
// "Our servers are currently overloaded" and assign httpStatus=503).
|
|
1283
|
+
// Allow one bounded mid-stream retry on the same budget as the WS close-
|
|
1284
|
+
// code buckets above so server-side overload no longer leaks straight
|
|
1285
|
+
// to the caller without a single retry attempt.
|
|
1286
|
+
if (status >= 500 && status < 600) {
|
|
1287
|
+
return _allowMidstreamRetry(`http_${status}`, attemptIndex);
|
|
1288
|
+
}
|
|
1289
|
+
|
|
1290
|
+
const name = err?.name || '';
|
|
1291
|
+
if (name === 'BridgeStallAbortError') return _allowMidstreamRetry('bridge_stall', attemptIndex);
|
|
1292
|
+
if (name === 'StreamStalledAbortError') return _allowMidstreamRetry('stream_stalled', attemptIndex);
|
|
1293
|
+
|
|
1294
|
+
// Watchdog abort surfaced via externalSignal handler → err is the reason
|
|
1295
|
+
// itself. state.watchdogAbort captures the class name when the error
|
|
1296
|
+
// shape was preserved but the name was stripped by some wrapper.
|
|
1297
|
+
if (state.watchdogAbort === 'BridgeStallAbortError') return _allowMidstreamRetry('bridge_stall', attemptIndex);
|
|
1298
|
+
if (state.watchdogAbort === 'StreamStalledAbortError') return _allowMidstreamRetry('stream_stalled', attemptIndex);
|
|
1299
|
+
|
|
1300
|
+
// WS close codes: prefer the decorated property, fall back to state.
|
|
1301
|
+
const closeCode = Number(err?.wsCloseCode || state.wsCloseCode || 0);
|
|
1302
|
+
if (closeCode === 1006) return _allowMidstreamRetry('ws_1006', attemptIndex);
|
|
1303
|
+
if (closeCode === 1011) return _allowMidstreamRetry('ws_1011', attemptIndex);
|
|
1304
|
+
if (closeCode === 1012) return _allowMidstreamRetry('ws_1012', attemptIndex);
|
|
1305
|
+
// Private 4xxx codes from a server/proxy are auth/policy/application closes;
|
|
1306
|
+
// never treat them as transient. 4000 is our local pre-stream watchdog code.
|
|
1307
|
+
if (closeCode >= 4000 && closeCode < 5000 && closeCode !== 4000) return null;
|
|
1308
|
+
if (closeCode === 4000) return _allowMidstreamRetry('ws_4000', attemptIndex);
|
|
1309
|
+
// Server-side normal close (1000) AFTER response.created but BEFORE
|
|
1310
|
+
// response.completed = truncated stream; legitimate transient. The
|
|
1311
|
+
// pre-stream gate above already rejects 1000 before sawResponseCreated
|
|
1312
|
+
// (handshake retry layer owns that window).
|
|
1313
|
+
if (closeCode === 1000 && state.sawResponseCreated && !state.sawCompleted) return _allowMidstreamRetry('ws_1000', attemptIndex);
|
|
1314
|
+
|
|
1315
|
+
// response.failed payload mentioning network_error / stream_disconnected.
|
|
1316
|
+
// xAI's gRPC backend periodically rotates auth context (server-side TTL)
|
|
1317
|
+
// and surfaces "Auth context expired" as a response.failed event. The
|
|
1318
|
+
// attemptIndex > 0 path in sendViaWebSocket forces a fresh WS handshake,
|
|
1319
|
+
// which re-authenticates — so a single bounded retry recovers the turn
|
|
1320
|
+
// instead of letting the worker die mid-session.
|
|
1321
|
+
const failed = err?.responseFailed || state.responseFailedPayload;
|
|
1322
|
+
if (failed) {
|
|
1323
|
+
try {
|
|
1324
|
+
const blob = JSON.stringify(failed).toLowerCase();
|
|
1325
|
+
if (blob.includes('stream_disconnected')) return _allowMidstreamRetry('response_failed_disconnected', attemptIndex);
|
|
1326
|
+
if (blob.includes('network_error')) return _allowMidstreamRetry('response_failed_network', attemptIndex);
|
|
1327
|
+
if (blob.includes('auth context expired')) return _allowMidstreamRetry('response_failed_auth_expired', attemptIndex);
|
|
1328
|
+
} catch {}
|
|
1329
|
+
}
|
|
1330
|
+
|
|
1331
|
+
// Unknown → default-deny (don't risk a second full-cost turn for an error
|
|
1332
|
+
// class we haven't proven is transient).
|
|
1333
|
+
return null;
|
|
1334
|
+
}
|
|
1335
|
+
|
|
1336
|
+
function _midstreamRetryLimit(classifier) {
|
|
1337
|
+
return classifier === 'ws_1006' || classifier === 'ws_1011'
|
|
1338
|
+
? MIDSTREAM_WS_TRANSIENT_RETRY_LIMIT
|
|
1339
|
+
: MIDSTREAM_DEFAULT_RETRY_LIMIT;
|
|
1340
|
+
}
|
|
1341
|
+
|
|
1342
|
+
function _allowMidstreamRetry(classifier, attemptIndex) {
|
|
1343
|
+
return attemptIndex < _midstreamRetryLimit(classifier) ? classifier : null;
|
|
1344
|
+
}
|
|
1345
|
+
|
|
1346
|
+
function _midstreamBackoffFor(retryNumber) {
|
|
1347
|
+
return MIDSTREAM_BACKOFF_MS[Math.min(Math.max(retryNumber, 1), MIDSTREAM_BACKOFF_MS.length) - 1];
|
|
1348
|
+
}
|
|
1349
|
+
|
|
1350
|
+
function _backoffFor(attempt) {
|
|
1351
|
+
// attempt is 1-based. retry 1 → 500, retry 2 → 1000, retry 3 → 2000 … capped.
|
|
1352
|
+
const raw = HANDSHAKE_BACKOFF_BASE_MS * (1 << (attempt - 1));
|
|
1353
|
+
return jitterDelayMs(Math.min(raw, HANDSHAKE_BACKOFF_CAP_MS));
|
|
1354
|
+
}
|
|
1355
|
+
|
|
1356
|
+
const _defaultSleep = (ms) => new Promise((r) => setTimeout(r, ms));
|
|
1357
|
+
|
|
1358
|
+
async function _sleepWithAbort(ms, externalSignal, sleepFn = _defaultSleep) {
|
|
1359
|
+
if (!ms) return;
|
|
1360
|
+
if (!externalSignal) {
|
|
1361
|
+
await sleepFn(ms);
|
|
1362
|
+
return;
|
|
1363
|
+
}
|
|
1364
|
+
await new Promise((resolve, reject) => {
|
|
1365
|
+
const t = setTimeout(() => {
|
|
1366
|
+
externalSignal.removeEventListener('abort', onAbort);
|
|
1367
|
+
resolve();
|
|
1368
|
+
}, ms);
|
|
1369
|
+
const onAbort = () => {
|
|
1370
|
+
clearTimeout(t);
|
|
1371
|
+
const reason = externalSignal.reason;
|
|
1372
|
+
reject(reason instanceof Error ? reason : new Error('Codex WS retry backoff aborted'));
|
|
1373
|
+
};
|
|
1374
|
+
if (externalSignal.aborted) { onAbort(); return; }
|
|
1375
|
+
externalSignal.addEventListener('abort', onAbort, { once: true });
|
|
1376
|
+
});
|
|
1377
|
+
}
|
|
1378
|
+
|
|
1379
|
+
/**
|
|
1380
|
+
* Run `_acquire({auth, poolKey, cacheKey})` with bounded exponential-backoff
|
|
1381
|
+
* retry on transient handshake failures. The injection seams (`_acquire`,
|
|
1382
|
+
* `_sleepFn`, `onRetry`) let unit tests drive the state machine without
|
|
1383
|
+
* opening real sockets.
|
|
1384
|
+
*
|
|
1385
|
+
* On exhaustion the thrown error is tagged with:
|
|
1386
|
+
* err.attempts — 1..HANDSHAKE_MAX_ATTEMPTS
|
|
1387
|
+
* err.retryClassifier — final classifier string, or null for permanent
|
|
1388
|
+
*/
|
|
1389
|
+
export async function _acquireWithRetry({
|
|
1390
|
+
auth,
|
|
1391
|
+
poolKey,
|
|
1392
|
+
cacheKey,
|
|
1393
|
+
forceFresh,
|
|
1394
|
+
onRetry,
|
|
1395
|
+
externalSignal,
|
|
1396
|
+
_acquire = acquireWebSocket,
|
|
1397
|
+
_sleepFn = _defaultSleep,
|
|
1398
|
+
} = {}) {
|
|
1399
|
+
let lastErr = null;
|
|
1400
|
+
let lastClassifier = null;
|
|
1401
|
+
for (let attempt = 1; attempt <= HANDSHAKE_MAX_ATTEMPTS; attempt++) {
|
|
1402
|
+
if (externalSignal?.aborted) {
|
|
1403
|
+
const reason = externalSignal.reason;
|
|
1404
|
+
throw reason instanceof Error ? reason : new Error('Codex WS acquire aborted');
|
|
1405
|
+
}
|
|
1406
|
+
try {
|
|
1407
|
+
if (attempt > 1) {
|
|
1408
|
+
if (process.env.MIXDOG_DEBUG_BRIDGE) {
|
|
1409
|
+
process.stderr.write(`[bridge-trace] ws-handshake-attempt n=${attempt}\n`);
|
|
1410
|
+
}
|
|
1411
|
+
}
|
|
1412
|
+
return await _acquire({ auth, poolKey, cacheKey, forceFresh, externalSignal });
|
|
1413
|
+
} catch (err) {
|
|
1414
|
+
lastErr = err;
|
|
1415
|
+
const classifier = _classifyHandshakeError(err);
|
|
1416
|
+
lastClassifier = classifier;
|
|
1417
|
+
// Permanent (or unknown → default-deny): stop immediately.
|
|
1418
|
+
if (!classifier) {
|
|
1419
|
+
if (err && typeof err === 'object') {
|
|
1420
|
+
try { err.attempts = attempt; } catch {}
|
|
1421
|
+
try { err.retryClassifier = null; } catch {}
|
|
1422
|
+
}
|
|
1423
|
+
throw err;
|
|
1424
|
+
}
|
|
1425
|
+
// Transient but exhausted: surface with tagging.
|
|
1426
|
+
if (attempt >= HANDSHAKE_MAX_ATTEMPTS) {
|
|
1427
|
+
if (err && typeof err === 'object') {
|
|
1428
|
+
try { err.attempts = attempt; } catch {}
|
|
1429
|
+
try { err.retryClassifier = classifier; } catch {}
|
|
1430
|
+
}
|
|
1431
|
+
try {
|
|
1432
|
+
process.stderr.write(
|
|
1433
|
+
`[openai-oauth-ws] handshake failed after ${attempt}/${HANDSHAKE_MAX_ATTEMPTS} attempts: ${err?.message || err}\n`,
|
|
1434
|
+
);
|
|
1435
|
+
} catch {}
|
|
1436
|
+
throw err;
|
|
1437
|
+
}
|
|
1438
|
+
// Schedule backoff and emit progress.
|
|
1439
|
+
const backoff = _backoffFor(attempt);
|
|
1440
|
+
try {
|
|
1441
|
+
process.stderr.write(
|
|
1442
|
+
`[openai-oauth-ws] worker retry ${attempt}/${HANDSHAKE_MAX_ATTEMPTS} (transient: ${classifier}, backoff ${backoff}ms)\n`,
|
|
1443
|
+
);
|
|
1444
|
+
} catch {}
|
|
1445
|
+
try {
|
|
1446
|
+
onRetry?.({
|
|
1447
|
+
attempt,
|
|
1448
|
+
max: HANDSHAKE_MAX_ATTEMPTS,
|
|
1449
|
+
classifier,
|
|
1450
|
+
backoffMs: backoff,
|
|
1451
|
+
error: err,
|
|
1452
|
+
});
|
|
1453
|
+
} catch {}
|
|
1454
|
+
// Sleep is abort-aware: an abort during backoff rejects immediately
|
|
1455
|
+
// instead of burning the remaining wait.
|
|
1456
|
+
if (externalSignal) {
|
|
1457
|
+
await new Promise((resolve, reject) => {
|
|
1458
|
+
const t = setTimeout(() => {
|
|
1459
|
+
externalSignal.removeEventListener('abort', onAbort);
|
|
1460
|
+
resolve();
|
|
1461
|
+
}, backoff);
|
|
1462
|
+
const onAbort = () => {
|
|
1463
|
+
clearTimeout(t);
|
|
1464
|
+
const reason = externalSignal.reason;
|
|
1465
|
+
reject(reason instanceof Error ? reason : new Error('Codex WS acquire aborted'));
|
|
1466
|
+
};
|
|
1467
|
+
if (externalSignal.aborted) { onAbort(); return; }
|
|
1468
|
+
externalSignal.addEventListener('abort', onAbort, { once: true });
|
|
1469
|
+
});
|
|
1470
|
+
} else {
|
|
1471
|
+
await _sleepFn(backoff);
|
|
1472
|
+
}
|
|
1473
|
+
}
|
|
1474
|
+
}
|
|
1475
|
+
// Unreachable — the loop either returns or throws above — but keep the
|
|
1476
|
+
// typing honest.
|
|
1477
|
+
if (lastErr && typeof lastErr === 'object') {
|
|
1478
|
+
try { lastErr.attempts = HANDSHAKE_MAX_ATTEMPTS; } catch {}
|
|
1479
|
+
try { lastErr.retryClassifier = lastClassifier; } catch {}
|
|
1480
|
+
}
|
|
1481
|
+
throw lastErr || new Error('acquireWithRetry: unreachable');
|
|
1482
|
+
}
|
|
1483
|
+
|
|
1484
|
+
/**
|
|
1485
|
+
* Dispatch one tool-loop iteration over a per-session cached WebSocket.
|
|
1486
|
+
* Returns the same shape as the SSE path: { content, model, toolCalls, usage }.
|
|
1487
|
+
*/
|
|
1488
|
+
export async function sendViaWebSocket({
|
|
1489
|
+
auth,
|
|
1490
|
+
body,
|
|
1491
|
+
sendOpts,
|
|
1492
|
+
onStreamDelta,
|
|
1493
|
+
onToolCall,
|
|
1494
|
+
onStageChange,
|
|
1495
|
+
externalSignal,
|
|
1496
|
+
poolKey,
|
|
1497
|
+
cacheKey,
|
|
1498
|
+
iteration,
|
|
1499
|
+
useModel,
|
|
1500
|
+
displayModel,
|
|
1501
|
+
forceFresh = false,
|
|
1502
|
+
includeResponseId = false,
|
|
1503
|
+
traceProvider = 'openai-oauth',
|
|
1504
|
+
logSuppressedReasoningDeltas = true,
|
|
1505
|
+
warmupBody = null,
|
|
1506
|
+
// Test seams (undefined in production). Let the unit test drive the
|
|
1507
|
+
// retry state machine without opening real sockets or touching the
|
|
1508
|
+
// handshake-retry layer.
|
|
1509
|
+
_acquireWithRetryFn = _acquireWithRetry,
|
|
1510
|
+
_streamFn = _streamResponse,
|
|
1511
|
+
_sendFrameFn = _sendFrame,
|
|
1512
|
+
_sleepFn = _defaultSleep,
|
|
1513
|
+
}) {
|
|
1514
|
+
// Bounded mid-stream retry: if an attempt's stream dies after
|
|
1515
|
+
// response.created but before response.completed from a transient cause
|
|
1516
|
+
// (watchdog abort / ws 1006/1011/1012/4000 / response.failed with network
|
|
1517
|
+
// error), tear down the socket and reissue the full request from scratch
|
|
1518
|
+
// with a classifier-specific budget. ws_1006/ws_1011 get two retries with
|
|
1519
|
+
// 250ms/1s backoff; other legacy transient buckets keep the prior one retry.
|
|
1520
|
+
// No delta resume — content restarts, which is the accepted tradeoff for
|
|
1521
|
+
// reviewer/worker flows that need the complete answer.
|
|
1522
|
+
// Retries are layered ABOVE the handshake retry loop (_acquireWithRetry
|
|
1523
|
+
// owns connect-level transience); the two never interleave because we
|
|
1524
|
+
// force a brand-new acquire for the retry attempt.
|
|
1525
|
+
const MAX_MIDSTREAM_RETRIES = MIDSTREAM_WS_TRANSIENT_RETRY_LIMIT;
|
|
1526
|
+
let firstAttemptError = null;
|
|
1527
|
+
let firstAttemptClassifier = null;
|
|
1528
|
+
// Server-side xAI conversation anchor preserved across mid-stream
|
|
1529
|
+
// retries. xAI keys its conversation by previous_response_id alone
|
|
1530
|
+
// (sessionToken is null for xAI in _mintSessionToken); a forceFresh
|
|
1531
|
+
// socket on retry would otherwise drop prev_id and cold-start a new
|
|
1532
|
+
// server-side conversation, evicting every prefix the prior attempts
|
|
1533
|
+
// warmed. Codex / openai-direct anchor by per-socket session_id, where
|
|
1534
|
+
// this carry-forward would not help and is therefore gated to xAI.
|
|
1535
|
+
let carryForwardCache = null;
|
|
1536
|
+
const emittedProgress = [];
|
|
1537
|
+
|
|
1538
|
+
for (let attemptIndex = 0; attemptIndex <= MAX_MIDSTREAM_RETRIES; attemptIndex++) {
|
|
1539
|
+
const handshakeStart = Date.now();
|
|
1540
|
+
let acquired;
|
|
1541
|
+
let handshakeRetries = 0;
|
|
1542
|
+
const handshakeRetryClassifiers = [];
|
|
1543
|
+
try { onStageChange?.('requesting'); } catch {}
|
|
1544
|
+
try {
|
|
1545
|
+
acquired = await _acquireWithRetryFn({
|
|
1546
|
+
auth,
|
|
1547
|
+
poolKey,
|
|
1548
|
+
cacheKey,
|
|
1549
|
+
// Retry attempt must not reuse a pooled socket — the prior
|
|
1550
|
+
// one is either torn down or in an unknown state.
|
|
1551
|
+
forceFresh: forceFresh || attemptIndex > 0,
|
|
1552
|
+
externalSignal,
|
|
1553
|
+
onRetry: (info) => {
|
|
1554
|
+
handshakeRetries += 1;
|
|
1555
|
+
if (info?.classifier) handshakeRetryClassifiers.push(info.classifier);
|
|
1556
|
+
},
|
|
1557
|
+
});
|
|
1558
|
+
} catch (err) {
|
|
1559
|
+
const classifier = err?.retryClassifier || (err?.code === 'EWSACQUIRETIMEOUT' ? 'acquire_timeout' : null);
|
|
1560
|
+
const classifiers = [...handshakeRetryClassifiers];
|
|
1561
|
+
if (classifier && !classifiers.includes(classifier)) classifiers.push(classifier);
|
|
1562
|
+
if (err?.httpStatus != null || classifier || handshakeRetries > 0 || classifiers.length > 0) {
|
|
1563
|
+
traceBridgeFetch({
|
|
1564
|
+
sessionId: poolKey,
|
|
1565
|
+
headersMs: Date.now() - handshakeStart,
|
|
1566
|
+
httpStatus: Number(err?.httpStatus || 0),
|
|
1567
|
+
provider: traceProvider,
|
|
1568
|
+
model: useModel,
|
|
1569
|
+
transport: 'websocket',
|
|
1570
|
+
handshakeRetries: err?.attempts ? Math.max(Number(err.attempts) - 1, 0) : handshakeRetries,
|
|
1571
|
+
handshakeRetryClassifiers: classifiers,
|
|
1572
|
+
});
|
|
1573
|
+
}
|
|
1574
|
+
// Handshake-layer failure. Don't double-wrap: if this is the retry
|
|
1575
|
+
// attempt, surface the ORIGINAL first-attempt error (which is what
|
|
1576
|
+
// the caller's turn actually tripped on).
|
|
1577
|
+
if (attemptIndex > 0 && firstAttemptError) {
|
|
1578
|
+
try { firstAttemptError.midstreamRetries = attemptIndex; } catch {}
|
|
1579
|
+
throw firstAttemptError;
|
|
1580
|
+
}
|
|
1581
|
+
throw err;
|
|
1582
|
+
}
|
|
1583
|
+
const { entry, reused } = acquired;
|
|
1584
|
+
// Re-seed the retry attempt's fresh entry with the prior attempt's
|
|
1585
|
+
// last successful anchor so _computeDelta sees a non-null
|
|
1586
|
+
// lastInputPrefixHash and prev_response_id, keeping the same xAI
|
|
1587
|
+
// conversation slot warm instead of cold-starting one per retry.
|
|
1588
|
+
if (carryForwardCache && auth?.type === 'xai' && !reused) {
|
|
1589
|
+
entry.lastResponseId = carryForwardCache.lastResponseId;
|
|
1590
|
+
entry.lastInputPrefixHash = carryForwardCache.lastInputPrefixHash;
|
|
1591
|
+
entry.lastInputLen = carryForwardCache.lastInputLen;
|
|
1592
|
+
entry.lastRequestSansInput = carryForwardCache.lastRequestSansInput;
|
|
1593
|
+
}
|
|
1594
|
+
traceBridgeFetch({
|
|
1595
|
+
sessionId: poolKey,
|
|
1596
|
+
headersMs: Date.now() - handshakeStart,
|
|
1597
|
+
httpStatus: reused ? 0 : 101,
|
|
1598
|
+
provider: traceProvider,
|
|
1599
|
+
model: useModel,
|
|
1600
|
+
transport: 'websocket',
|
|
1601
|
+
handshakeRetries,
|
|
1602
|
+
handshakeRetryClassifiers,
|
|
1603
|
+
});
|
|
1604
|
+
|
|
1605
|
+
let requestBody = body;
|
|
1606
|
+
// Mid-stream retry: pin prev_id in the body so _computeDelta's
|
|
1607
|
+
// mode='full' fallback (triggered when the carried prefix hash no
|
|
1608
|
+
// longer matches the current input) still carries the conversation
|
|
1609
|
+
// anchor. The delta path overwrites this from entry.lastResponseId,
|
|
1610
|
+
// which equals the carried value, so the two paths agree.
|
|
1611
|
+
if (carryForwardCache && auth?.type === 'xai' && attemptIndex > 0 && !body.previous_response_id) {
|
|
1612
|
+
requestBody = { ...body, previous_response_id: carryForwardCache.lastResponseId };
|
|
1613
|
+
}
|
|
1614
|
+
let warmupResult = null;
|
|
1615
|
+
// midState is shared between warmup and the main stream so warmup
|
|
1616
|
+
// failures (first-byte timeout, send-failure, ws_4000) flow through
|
|
1617
|
+
// the SAME mid-stream classifier as the main send. A wedged warmup
|
|
1618
|
+
// socket must not bypass the retry loop and surface raw to the
|
|
1619
|
+
// caller — release the entry, force a fresh acquire, and retry.
|
|
1620
|
+
const midState = {
|
|
1621
|
+
attemptIndex,
|
|
1622
|
+
sawResponseCreated: false,
|
|
1623
|
+
sawCompleted: false,
|
|
1624
|
+
};
|
|
1625
|
+
const sseStart = Date.now();
|
|
1626
|
+
let mode = 'full';
|
|
1627
|
+
let frame = null;
|
|
1628
|
+
let deltaTokens = 0;
|
|
1629
|
+
let result;
|
|
1630
|
+
try {
|
|
1631
|
+
if (warmupBody && typeof warmupBody === 'object' && attemptIndex === 0) {
|
|
1632
|
+
const warmupFrame = { type: 'response.create', ...warmupBody };
|
|
1633
|
+
await _sendFrameFn(entry, warmupFrame);
|
|
1634
|
+
const warmupStart = Date.now();
|
|
1635
|
+
const warmupState = {
|
|
1636
|
+
attemptIndex,
|
|
1637
|
+
sawResponseCreated: false,
|
|
1638
|
+
sawCompleted: false,
|
|
1639
|
+
};
|
|
1640
|
+
warmupResult = await _streamFn({
|
|
1641
|
+
entry,
|
|
1642
|
+
externalSignal,
|
|
1643
|
+
onStreamDelta: null,
|
|
1644
|
+
onToolCall: null,
|
|
1645
|
+
state: warmupState,
|
|
1646
|
+
logSuppressedReasoningDeltas,
|
|
1647
|
+
traceProvider,
|
|
1648
|
+
});
|
|
1649
|
+
// Surface warmup-time first-event timeout / send-failure
|
|
1650
|
+
// flags onto the shared midState so the outer catch's
|
|
1651
|
+
// classifier sees them. (warmupResult itself only resolves
|
|
1652
|
+
// on success; failures throw and skip this block.)
|
|
1653
|
+
if (warmupState.firstByteTimeout) midState.firstByteTimeout = true;
|
|
1654
|
+
if (warmupState.wsSendFailed) midState.wsSendFailed = true;
|
|
1655
|
+
if (!warmupResult?.responseId) {
|
|
1656
|
+
throw new Error('Responses WS warmup completed without response id');
|
|
1657
|
+
}
|
|
1658
|
+
entry.lastResponseId = warmupResult.responseId;
|
|
1659
|
+
entry.lastRequestSansInput = _stableStringify(_sansInput(warmupBody));
|
|
1660
|
+
const warmupInputArr = Array.isArray(warmupBody.input) ? warmupBody.input : [];
|
|
1661
|
+
entry.lastInputLen = warmupInputArr.length;
|
|
1662
|
+
entry.lastInputPrefixHash = createHash('sha256')
|
|
1663
|
+
.update(JSON.stringify(warmupInputArr))
|
|
1664
|
+
.digest('hex');
|
|
1665
|
+
try {
|
|
1666
|
+
const warmupPayload = {
|
|
1667
|
+
provider: traceProvider,
|
|
1668
|
+
transport: 'websocket',
|
|
1669
|
+
event: 'warmup_completed',
|
|
1670
|
+
response_id: warmupResult.responseId,
|
|
1671
|
+
elapsed_ms: Date.now() - warmupStart,
|
|
1672
|
+
input_tokens: warmupResult.usage?.inputTokens || 0,
|
|
1673
|
+
cached_tokens: warmupResult.usage?.cachedTokens || 0,
|
|
1674
|
+
output_tokens: warmupResult.usage?.outputTokens || 0,
|
|
1675
|
+
prompt_tokens: warmupResult.usage?.promptTokens || 0,
|
|
1676
|
+
};
|
|
1677
|
+
appendBridgeTrace({
|
|
1678
|
+
sessionId: poolKey,
|
|
1679
|
+
iteration,
|
|
1680
|
+
kind: 'cache_warmup',
|
|
1681
|
+
...warmupPayload,
|
|
1682
|
+
payload: warmupPayload,
|
|
1683
|
+
});
|
|
1684
|
+
} catch {}
|
|
1685
|
+
requestBody = { ...body, previous_response_id: warmupResult.responseId };
|
|
1686
|
+
delete requestBody.instructions;
|
|
1687
|
+
delete requestBody.generate;
|
|
1688
|
+
}
|
|
1689
|
+
|
|
1690
|
+
({ mode, frame } = _computeDelta({ entry, body: requestBody }));
|
|
1691
|
+
deltaTokens = _estimateFrameTokens(frame);
|
|
1692
|
+
|
|
1693
|
+
// Re-check abort after acquire/warmup — narrow window where
|
|
1694
|
+
// externalSignal could fire between successful acquire and
|
|
1695
|
+
// send(). Without this gate an aborted request could still
|
|
1696
|
+
// emit one frame to the provider.
|
|
1697
|
+
if (externalSignal?.aborted) {
|
|
1698
|
+
throw new Error('Aborted');
|
|
1699
|
+
}
|
|
1700
|
+
await _sendFrameFn(entry, frame);
|
|
1701
|
+
|
|
1702
|
+
if (process.env.MIXDOG_DEBUG_BRIDGE) {
|
|
1703
|
+
process.stderr.write(`[bridge-trace] ws-streaming-start sinceAcquire=${Date.now() - handshakeStart}ms\n`);
|
|
1704
|
+
}
|
|
1705
|
+
try { onStageChange?.('streaming'); } catch {}
|
|
1706
|
+
result = await _streamFn({
|
|
1707
|
+
entry,
|
|
1708
|
+
externalSignal,
|
|
1709
|
+
onStreamDelta,
|
|
1710
|
+
onToolCall,
|
|
1711
|
+
state: midState,
|
|
1712
|
+
logSuppressedReasoningDeltas,
|
|
1713
|
+
traceProvider,
|
|
1714
|
+
});
|
|
1715
|
+
} catch (err) {
|
|
1716
|
+
// Snapshot the xAI conversation anchor BEFORE releasing the
|
|
1717
|
+
// entry. release closes the socket but leaves state fields
|
|
1718
|
+
// intact; the next forceFresh acquire creates a new entry into
|
|
1719
|
+
// which we manually carry the anchor so the retry continues the
|
|
1720
|
+
// same conversation instead of cold-starting one.
|
|
1721
|
+
if (auth?.type === 'xai' && entry.lastResponseId) {
|
|
1722
|
+
carryForwardCache = {
|
|
1723
|
+
lastResponseId: entry.lastResponseId,
|
|
1724
|
+
lastInputPrefixHash: entry.lastInputPrefixHash,
|
|
1725
|
+
lastInputLen: entry.lastInputLen,
|
|
1726
|
+
lastRequestSansInput: entry.lastRequestSansInput,
|
|
1727
|
+
};
|
|
1728
|
+
}
|
|
1729
|
+
releaseWebSocket({ entry, poolKey, keep: false });
|
|
1730
|
+
// Mid-stream classification.
|
|
1731
|
+
const classifier = _classifyMidstreamError(err, midState);
|
|
1732
|
+
const retryLimit = classifier ? _midstreamRetryLimit(classifier) : 0;
|
|
1733
|
+
if (classifier && attemptIndex < retryLimit) {
|
|
1734
|
+
// Retry-eligible: stash the first-attempt error, emit progress,
|
|
1735
|
+
// and loop. The subsequent acquire uses forceFresh so no socket
|
|
1736
|
+
// is shared between attempts.
|
|
1737
|
+
firstAttemptError = err;
|
|
1738
|
+
firstAttemptClassifier = classifier;
|
|
1739
|
+
try { err.midstreamClassifier = classifier; } catch {}
|
|
1740
|
+
const retryNumber = attemptIndex + 1;
|
|
1741
|
+
const backoff = _midstreamBackoffFor(retryNumber);
|
|
1742
|
+
try {
|
|
1743
|
+
const line = `[openai-oauth-ws] mid-stream recovered: retry ${retryNumber}/${retryLimit} (cause: ${classifier}, backoff ${backoff}ms)\n`;
|
|
1744
|
+
process.stderr.write(line);
|
|
1745
|
+
emittedProgress.push(line);
|
|
1746
|
+
} catch {}
|
|
1747
|
+
await _sleepWithAbort(backoff, externalSignal, _sleepFn);
|
|
1748
|
+
continue;
|
|
1749
|
+
}
|
|
1750
|
+
// Not retryable, OR we've already exhausted the retry budget.
|
|
1751
|
+
if (attemptIndex > 0 && firstAttemptError) {
|
|
1752
|
+
// Exhausted path: surface the first-attempt error (the one
|
|
1753
|
+
// the user's turn actually tripped on), tag actual retry count.
|
|
1754
|
+
try { firstAttemptError.midstreamRetries = attemptIndex; } catch {}
|
|
1755
|
+
try { firstAttemptError.midstreamClassifier = firstAttemptClassifier; } catch {}
|
|
1756
|
+
// Attach the retry attempt's error so post-mortem diagnostics
|
|
1757
|
+
// can see WHY the retry also failed instead of silently
|
|
1758
|
+
// dropping it. Use `cause` if free, else `suppressed`.
|
|
1759
|
+
try {
|
|
1760
|
+
if (!firstAttemptError.cause) firstAttemptError.cause = err;
|
|
1761
|
+
else {
|
|
1762
|
+
const list = Array.isArray(firstAttemptError.suppressed)
|
|
1763
|
+
? firstAttemptError.suppressed
|
|
1764
|
+
: [];
|
|
1765
|
+
list.push(err);
|
|
1766
|
+
firstAttemptError.suppressed = list;
|
|
1767
|
+
}
|
|
1768
|
+
} catch {}
|
|
1769
|
+
throw firstAttemptError;
|
|
1770
|
+
}
|
|
1771
|
+
throw err;
|
|
1772
|
+
}
|
|
1773
|
+
const liveModel = result.model || useModel;
|
|
1774
|
+
traceBridgeSse({
|
|
1775
|
+
sessionId: poolKey,
|
|
1776
|
+
sseParseMs: Date.now() - sseStart,
|
|
1777
|
+
provider: traceProvider,
|
|
1778
|
+
model: liveModel,
|
|
1779
|
+
transport: 'websocket',
|
|
1780
|
+
});
|
|
1781
|
+
|
|
1782
|
+
// Update cache state for the next iteration in this session.
|
|
1783
|
+
if (result.responseId) {
|
|
1784
|
+
entry.lastResponseId = result.responseId;
|
|
1785
|
+
entry.lastRequestSansInput = _stableStringify(_sansInput(requestBody));
|
|
1786
|
+
const inputArr = Array.isArray(requestBody.input) ? requestBody.input : [];
|
|
1787
|
+
entry.lastInputLen = inputArr.length;
|
|
1788
|
+
// Capture the prefix hash for the next turn's delta integrity
|
|
1789
|
+
// check. Serialize the full input (matching what _computeDelta
|
|
1790
|
+
// will hash on the next turn's first prevLen items, where
|
|
1791
|
+
// prevLen === inputArr.length).
|
|
1792
|
+
entry.lastInputPrefixHash = createHash('sha256')
|
|
1793
|
+
.update(JSON.stringify(inputArr))
|
|
1794
|
+
.digest('hex');
|
|
1795
|
+
}
|
|
1796
|
+
|
|
1797
|
+
if (warmupResult?.usage) {
|
|
1798
|
+
result.usage = _combineUsageWithWarmup(result.usage, warmupResult.usage);
|
|
1799
|
+
}
|
|
1800
|
+
|
|
1801
|
+
const requestedServiceTier = body?.service_tier || null;
|
|
1802
|
+
const responseServiceTier = result.serviceTier || result.usage?.raw?.service_tier || null;
|
|
1803
|
+
traceBridgeUsage({
|
|
1804
|
+
sessionId: poolKey,
|
|
1805
|
+
iteration,
|
|
1806
|
+
inputTokens: result.usage?.inputTokens || 0,
|
|
1807
|
+
outputTokens: result.usage?.outputTokens || 0,
|
|
1808
|
+
cachedTokens: result.usage?.cachedTokens || 0,
|
|
1809
|
+
promptTokens: result.usage?.promptTokens || 0,
|
|
1810
|
+
model: liveModel,
|
|
1811
|
+
modelDisplay: displayModel ? displayModel(liveModel) : liveModel,
|
|
1812
|
+
responseId: result.responseId || null,
|
|
1813
|
+
rawUsage: result.usage?.raw || null,
|
|
1814
|
+
provider: traceProvider,
|
|
1815
|
+
serviceTier: responseServiceTier,
|
|
1816
|
+
});
|
|
1817
|
+
// Extra WS-specific observability: transport + per-iteration delta bytes.
|
|
1818
|
+
try {
|
|
1819
|
+
const transportPayload = {
|
|
1820
|
+
provider: traceProvider,
|
|
1821
|
+
transport: 'websocket',
|
|
1822
|
+
ws_mode: mode,
|
|
1823
|
+
iteration_delta_tokens: deltaTokens,
|
|
1824
|
+
reused_connection: reused,
|
|
1825
|
+
requested_service_tier: requestedServiceTier,
|
|
1826
|
+
response_service_tier: responseServiceTier,
|
|
1827
|
+
handshake_retries: handshakeRetries,
|
|
1828
|
+
handshake_retry_classifiers: handshakeRetryClassifiers,
|
|
1829
|
+
midstream_retries: attemptIndex,
|
|
1830
|
+
response_id: result.responseId || null,
|
|
1831
|
+
request_has_previous_response_id: typeof frame.previous_response_id === 'string' && frame.previous_response_id.length > 0,
|
|
1832
|
+
body_input_items: Array.isArray(requestBody.input) ? requestBody.input.length : null,
|
|
1833
|
+
frame_input_items: Array.isArray(frame.input) ? frame.input.length : null,
|
|
1834
|
+
frame_has_instructions: typeof frame.instructions === 'string' && frame.instructions.length > 0,
|
|
1835
|
+
warmup_used: !!warmupResult,
|
|
1836
|
+
warmup_response_id: warmupResult?.responseId || null,
|
|
1837
|
+
};
|
|
1838
|
+
appendBridgeTrace({
|
|
1839
|
+
sessionId: poolKey,
|
|
1840
|
+
iteration,
|
|
1841
|
+
kind: 'transport',
|
|
1842
|
+
...transportPayload,
|
|
1843
|
+
payload: transportPayload,
|
|
1844
|
+
});
|
|
1845
|
+
} catch {}
|
|
1846
|
+
|
|
1847
|
+
releaseWebSocket({ entry, poolKey, keep: true });
|
|
1848
|
+
const { responseId: _ignored, ...out } = result;
|
|
1849
|
+
if (includeResponseId && result.responseId) out.responseId = result.responseId;
|
|
1850
|
+
if (warmupResult) {
|
|
1851
|
+
try {
|
|
1852
|
+
Object.defineProperty(out, '__warmup', {
|
|
1853
|
+
value: {
|
|
1854
|
+
requestBody,
|
|
1855
|
+
responseId: warmupResult.responseId,
|
|
1856
|
+
usage: warmupResult.usage,
|
|
1857
|
+
},
|
|
1858
|
+
enumerable: false,
|
|
1859
|
+
});
|
|
1860
|
+
} catch {}
|
|
1861
|
+
}
|
|
1862
|
+
// Leave a breadcrumb on the result so downstream callers can observe
|
|
1863
|
+
// that a retry was used (0 = first-try success, up to 2 for ws_1006/1011).
|
|
1864
|
+
try { Object.defineProperty(out, '__midstreamRetries', { value: attemptIndex, enumerable: false }); } catch {}
|
|
1865
|
+
return out;
|
|
1866
|
+
}
|
|
1867
|
+
// Unreachable — the loop either returns or throws above.
|
|
1868
|
+
throw firstAttemptError || new Error('sendViaWebSocket: unreachable');
|
|
1869
|
+
}
|
|
1870
|
+
|
|
1871
|
+
// Drain-complete fence — set true once _closeAllPooledSockets runs so any
|
|
1872
|
+
// in-flight acquire that resumes after drain throws instead of pushing a
|
|
1873
|
+
// fresh socket into the cleared pool. Single-set, mirrors drain-registry's
|
|
1874
|
+
// process-lifetime atomic invariant.
|
|
1875
|
+
let _drainComplete = false;
|
|
1876
|
+
|
|
1877
|
+
// Drain hook — registered in drain-registry external-resource bucket.
|
|
1878
|
+
// Force-closes pooled sockets and fences subsequent acquires.
|
|
1879
|
+
// `drainOpenaiWsPool` alias matches the registry's `drain*` naming convention;
|
|
1880
|
+
// `_closeAllPooledSockets` kept for backward compat with existing call sites.
|
|
1881
|
+
export function _closeAllPooledSockets(reason = 'shutdown') {
|
|
1882
|
+
_drainComplete = true;
|
|
1883
|
+
for (const arr of _wsPool.values()) {
|
|
1884
|
+
for (const entry of arr) {
|
|
1885
|
+
try { entry.socket.close(1000, reason); } catch {}
|
|
1886
|
+
}
|
|
1887
|
+
}
|
|
1888
|
+
_wsPool.clear();
|
|
1889
|
+
}
|
|
1890
|
+
export const drainOpenaiWsPool = _closeAllPooledSockets;
|