mixdog 0.8.1 → 0.9.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +47 -23
- package/package.json +34 -28
- package/scripts/_test-folder-dialog.mjs +30 -0
- package/scripts/agent-parallel-smoke.mjs +388 -0
- package/scripts/agent-tag-reuse-smoke.mjs +183 -0
- package/scripts/background-task-meta-smoke.mjs +38 -0
- package/scripts/boot-smoke.mjs +52 -9
- package/scripts/build-runtime-linux.sh +348 -0
- package/scripts/build-runtime-macos.sh +217 -0
- package/scripts/build-runtime-windows.ps1 +242 -0
- package/scripts/compact-active-turn-test.mjs +68 -0
- package/scripts/compact-smoke.mjs +859 -129
- package/scripts/compact-trigger-migration-smoke.mjs +187 -0
- package/scripts/fix-brief-fn.mjs +35 -0
- package/scripts/fix-format-tool-surface.mjs +24 -0
- package/scripts/fix-tool-exec-visible.mjs +42 -0
- package/scripts/generate-runtime-manifest.mjs +166 -0
- package/scripts/hook-bus-test.mjs +330 -0
- package/scripts/lead-workflow-smoke.mjs +33 -39
- package/scripts/live-worker-smoke.mjs +43 -37
- package/scripts/llm-trace-summary.mjs +315 -0
- package/scripts/memory-meta-concurrency-test.mjs +20 -0
- package/scripts/output-style-smoke.mjs +56 -15
- package/scripts/parent-abort-link-test.mjs +44 -0
- package/scripts/patch-agent-brief.mjs +48 -0
- package/scripts/patch-app.mjs +21 -0
- package/scripts/patch-app2.mjs +18 -0
- package/scripts/patch-dist-brief.mjs +96 -0
- package/scripts/patch-tool-exec.mjs +70 -0
- package/scripts/pretool-ask-runtime-test.mjs +54 -0
- package/scripts/provider-toolcall-test.mjs +376 -0
- package/scripts/reactive-compact-persist-smoke.mjs +124 -0
- package/scripts/sanitize-tool-pairs-test.mjs +260 -0
- package/scripts/session-context-bench.mjs +205 -33
- package/scripts/session-ingest-smoke.mjs +177 -0
- package/scripts/set-effort-config-test.mjs +41 -0
- package/scripts/smoke-runtime-negative.ps1 +106 -0
- package/scripts/smoke-runtime-negative.sh +97 -0
- package/scripts/smoke.mjs +25 -0
- package/scripts/tool-result-hook-test.mjs +48 -0
- package/scripts/tool-smoke.mjs +1222 -90
- package/scripts/toolcall-args-test.mjs +150 -0
- package/scripts/tui-background-failure-smoke.mjs +73 -0
- package/scripts/usage-metrics-epoch-smoke.mjs +114 -0
- package/src/agents/debugger/AGENT.md +8 -0
- package/src/agents/explore/AGENT.md +4 -0
- package/src/agents/heavy-worker/AGENT.md +9 -3
- package/src/agents/maintainer/AGENT.md +4 -0
- package/src/agents/reviewer/AGENT.md +8 -0
- package/src/agents/scheduler-task/AGENT.md +12 -0
- package/src/agents/scheduler-task/agent.json +6 -0
- package/src/agents/webhook-handler/AGENT.md +12 -0
- package/src/agents/webhook-handler/agent.json +6 -0
- package/src/agents/worker/AGENT.md +9 -3
- package/src/app.mjs +77 -3
- package/src/defaults/hidden-roles.json +17 -12
- package/src/headless-role.mjs +117 -0
- package/src/help.mjs +30 -0
- package/src/hooks/lib/permission-evaluator.cjs +11 -475
- package/src/lib/keychain-cjs.cjs +9 -1
- package/src/lib/mixdog-debug.cjs +0 -29
- package/src/lib/plugin-paths.cjs +1 -7
- package/src/lib/rules-builder.cjs +242 -98
- package/src/lib/text-utils.cjs +1 -1
- package/src/mixdog-session-runtime.mjs +2211 -446
- package/src/output-styles/default.md +12 -28
- package/src/output-styles/extreme-simple.md +9 -6
- package/src/output-styles/simple.md +22 -9
- package/src/repl.mjs +108 -59
- package/src/rules/agent/00-common.md +15 -0
- package/src/rules/{bridge → agent}/20-skip-protocol.md +1 -2
- package/src/rules/agent/30-explorer.md +22 -0
- package/src/rules/{bridge → agent}/40-cycle1-agent.md +7 -0
- package/src/rules/{bridge → agent}/41-cycle2-agent.md +7 -0
- package/src/rules/{bridge → agent}/42-cycle3-agent.md +7 -0
- package/src/rules/lead/01-general.md +9 -5
- package/src/rules/lead/04-workflow.md +51 -12
- package/src/rules/lead/lead-tool.md +6 -0
- package/src/rules/shared/01-tool.md +12 -1
- package/src/runtime/agent/orchestrator/activity-bus.mjs +7 -18
- package/src/runtime/agent/orchestrator/agent-owner.mjs +11 -0
- package/src/runtime/agent/orchestrator/{smart-bridge/bridge-llm.mjs → agent-runtime/agent-dispatch.mjs} +138 -111
- package/src/runtime/agent/orchestrator/agent-runtime/agent-progress-watchdog.mjs +94 -0
- package/src/runtime/agent/orchestrator/{smart-bridge → agent-runtime}/cache-strategy.mjs +32 -23
- package/src/runtime/agent/orchestrator/{smart-bridge → agent-runtime}/session-builder.mjs +33 -27
- package/src/runtime/agent/orchestrator/{bridge-trace.mjs → agent-trace.mjs} +132 -81
- package/src/runtime/agent/orchestrator/cache-mtime.mjs +0 -21
- package/src/runtime/agent/orchestrator/config.mjs +174 -55
- package/src/runtime/agent/orchestrator/context/collect.mjs +195 -487
- package/src/runtime/agent/orchestrator/dispatch-persist.mjs +1 -1
- package/src/runtime/agent/orchestrator/internal-roles.mjs +77 -29
- package/src/runtime/agent/orchestrator/internal-tools.mjs +5 -6
- package/src/runtime/agent/orchestrator/mcp/client.mjs +15 -9
- package/src/runtime/agent/orchestrator/providers/anthropic-betas.mjs +7 -1
- package/src/runtime/agent/orchestrator/providers/anthropic-oauth.mjs +380 -268
- package/src/runtime/agent/orchestrator/providers/anthropic.mjs +131 -93
- package/src/runtime/agent/orchestrator/providers/api-usage.mjs +236 -4
- package/src/runtime/agent/orchestrator/providers/custom-tool-wire.mjs +49 -0
- package/src/runtime/agent/orchestrator/providers/gemini.mjs +58 -13
- package/src/runtime/agent/orchestrator/providers/grok-oauth.mjs +67 -149
- package/src/runtime/agent/orchestrator/providers/media-normalization.mjs +132 -2
- package/src/runtime/agent/orchestrator/providers/model-catalog.mjs +4 -1
- package/src/runtime/agent/orchestrator/providers/oauth-credential-probes.mjs +45 -0
- package/src/runtime/agent/orchestrator/providers/oauth-usage.mjs +61 -116
- package/src/runtime/agent/orchestrator/providers/openai-compat-presets.mjs +25 -0
- package/src/runtime/agent/orchestrator/providers/openai-compat-stream.mjs +79 -255
- package/src/runtime/agent/orchestrator/providers/openai-compat.mjs +203 -71
- package/src/runtime/agent/orchestrator/providers/openai-oauth-ws.mjs +512 -312
- package/src/runtime/agent/orchestrator/providers/openai-oauth.mjs +343 -518
- package/src/runtime/agent/orchestrator/providers/openai-ws.mjs +6 -6
- package/src/runtime/agent/orchestrator/providers/registry.mjs +88 -51
- package/src/runtime/agent/orchestrator/providers/retry-classifier.mjs +289 -11
- package/src/runtime/agent/orchestrator/providers/statusline-route-meta.mjs +41 -8
- package/src/runtime/agent/orchestrator/session/compact.mjs +1173 -267
- package/src/runtime/agent/orchestrator/session/context-utils.mjs +199 -36
- package/src/runtime/agent/orchestrator/session/loop.mjs +840 -612
- package/src/runtime/agent/orchestrator/session/manager.mjs +1574 -459
- package/src/runtime/agent/orchestrator/session/manager.reactive-persist.test.mjs +107 -0
- package/src/runtime/agent/orchestrator/session/store.mjs +67 -29
- package/src/runtime/agent/orchestrator/session/tool-result-offload.mjs +2 -2
- package/src/runtime/agent/orchestrator/stall-policy.mjs +31 -16
- package/src/runtime/agent/orchestrator/tool-loop-guard.mjs +3 -219
- package/src/runtime/agent/orchestrator/tools/bash-session.mjs +35 -8
- package/src/runtime/agent/orchestrator/tools/builtin/arg-guard.mjs +19 -8
- package/src/runtime/agent/orchestrator/tools/builtin/bash-tool.mjs +10 -53
- package/src/runtime/agent/orchestrator/tools/builtin/builtin-tools.mjs +60 -37
- package/src/runtime/agent/orchestrator/tools/builtin/cache-layers.mjs +21 -2
- package/src/runtime/agent/orchestrator/tools/builtin/device-paths.mjs +1 -1
- package/src/runtime/agent/orchestrator/tools/builtin/diagnostics-tool.mjs +0 -7
- package/src/runtime/agent/orchestrator/tools/builtin/glob-walk.mjs +1 -3
- package/src/runtime/agent/orchestrator/tools/builtin/list-tool.mjs +36 -12
- package/src/runtime/agent/orchestrator/tools/builtin/read-args.mjs +2 -0
- package/src/runtime/agent/orchestrator/tools/builtin/read-constants.mjs +2 -2
- package/src/runtime/agent/orchestrator/tools/builtin/read-formatting.mjs +5 -12
- package/src/runtime/agent/orchestrator/tools/builtin/read-image-resize.mjs +1 -1
- package/src/runtime/agent/orchestrator/tools/builtin/read-single-tool.mjs +4 -36
- package/src/runtime/agent/orchestrator/tools/builtin/read-tool.mjs +2 -40
- package/src/runtime/agent/orchestrator/tools/builtin/rg-runner.mjs +148 -27
- package/src/runtime/agent/orchestrator/tools/builtin/search-builders.mjs +2 -2
- package/src/runtime/agent/orchestrator/tools/builtin/search-tool.mjs +43 -75
- package/src/runtime/agent/orchestrator/tools/builtin/shell-jobs.mjs +90 -20
- package/src/runtime/agent/orchestrator/tools/builtin.mjs +59 -5
- package/src/runtime/agent/orchestrator/tools/code-graph-state.mjs +86 -0
- package/src/runtime/agent/orchestrator/tools/code-graph-tool-defs.mjs +11 -11
- package/src/runtime/agent/orchestrator/tools/code-graph.mjs +4106 -4019
- package/src/runtime/agent/orchestrator/tools/patch-tool-defs.mjs +33 -4
- package/src/runtime/agent/orchestrator/tools/patch.mjs +90 -6
- package/src/runtime/agent/orchestrator/tools/progress-message.mjs +6 -4
- package/src/runtime/agent/orchestrator/tools/result-compression.mjs +4 -4
- package/src/runtime/agent/orchestrator/tools/shell-command.mjs +8 -1
- package/src/runtime/agent/orchestrator/tools/shell-snapshot.mjs +4 -4
- package/src/runtime/channels/index.mjs +149 -51
- package/src/runtime/channels/lib/cli-worker-host.mjs +1 -8
- package/src/runtime/channels/lib/config.mjs +0 -1
- package/src/runtime/channels/lib/drop-trace.mjs +1 -1
- package/src/runtime/channels/lib/executor.mjs +0 -3
- package/src/runtime/channels/lib/memory-client.mjs +0 -38
- package/src/runtime/channels/lib/output-forwarder.mjs +1 -8
- package/src/runtime/channels/lib/runtime-paths.mjs +0 -6
- package/src/runtime/channels/lib/scheduler.mjs +18 -14
- package/src/runtime/channels/lib/session-discovery.mjs +3 -6
- package/src/runtime/channels/lib/tool-format.mjs +0 -1
- package/src/runtime/channels/lib/transcript-discovery.mjs +4 -12
- package/src/runtime/channels/lib/webhook.mjs +1 -1
- package/src/runtime/channels/tool-defs.mjs +29 -29
- package/src/runtime/lib/keychain-cjs.cjs +0 -1
- package/src/runtime/memory/data/runtime-manifest.json +6 -7
- package/src/runtime/memory/index.mjs +519 -118
- package/src/runtime/memory/lib/agent-ipc.mjs +29 -12
- package/src/runtime/memory/lib/core-memory-store.mjs +2 -2
- package/src/runtime/memory/lib/embedding-model-config.mjs +55 -0
- package/src/runtime/memory/lib/embedding-provider.mjs +31 -4
- package/src/runtime/memory/lib/embedding-worker.mjs +19 -10
- package/src/runtime/memory/lib/llm-worker-host.mjs +0 -4
- package/src/runtime/memory/lib/memory-cycle1.mjs +28 -7
- package/src/runtime/memory/lib/memory-cycle2.mjs +4 -5
- package/src/runtime/memory/lib/memory-cycle3.mjs +2 -2
- package/src/runtime/memory/lib/memory-ops-policy.mjs +2 -2
- package/src/runtime/memory/lib/memory-session-merge.mjs +38 -0
- package/src/runtime/memory/lib/memory.mjs +88 -9
- package/src/runtime/memory/lib/model-profile.mjs +1 -1
- package/src/runtime/memory/lib/pg/adapter.mjs +1 -1
- package/src/runtime/memory/lib/pg/supervisor.mjs +12 -0
- package/src/runtime/memory/lib/runtime-fetcher.mjs +80 -21
- package/src/runtime/memory/lib/session-ingest.mjs +196 -0
- package/src/runtime/memory/lib/trace-store.mjs +96 -51
- package/src/runtime/memory/tool-defs.mjs +46 -37
- package/src/runtime/search/index.mjs +104 -473
- package/src/runtime/search/lib/config.mjs +0 -4
- package/src/runtime/search/lib/state.mjs +1 -15
- package/src/runtime/search/lib/web-tools.mjs +44 -25
- package/src/runtime/search/tool-defs.mjs +16 -23
- package/src/runtime/shared/abort-controller.mjs +1 -1
- package/src/runtime/shared/atomic-file.mjs +4 -3
- package/src/runtime/shared/background-tasks.mjs +122 -11
- package/src/runtime/shared/child-spawn-gate.mjs +139 -0
- package/src/runtime/shared/config.mjs +7 -4
- package/src/runtime/shared/err-text.mjs +131 -4
- package/src/runtime/shared/llm/cost.mjs +2 -2
- package/src/runtime/shared/llm/http-agent.mjs +23 -7
- package/src/runtime/shared/llm/index.mjs +34 -11
- package/src/runtime/shared/llm/usage-log.mjs +4 -4
- package/src/runtime/shared/markdown-frontmatter.mjs +56 -0
- package/src/runtime/shared/singleton-owner.mjs +104 -0
- package/src/runtime/shared/tool-execution-contract.mjs +199 -20
- package/src/runtime/shared/tool-execution-contract.test.mjs +183 -0
- package/src/runtime/shared/tool-surface.mjs +624 -95
- package/src/runtime/shared/user-data-guard.mjs +0 -2
- package/src/standalone/agent-task-status.mjs +203 -0
- package/src/standalone/agent-task-status.test.mjs +76 -0
- package/src/standalone/agent-tool.mjs +1913 -0
- package/src/standalone/channel-worker.mjs +370 -14
- package/src/standalone/explore-tool.mjs +165 -70
- package/src/standalone/folder-dialog.mjs +314 -0
- package/src/standalone/hook-bus.mjs +898 -22
- package/src/standalone/memory-runtime-proxy.mjs +320 -0
- package/src/standalone/projects.mjs +226 -0
- package/src/standalone/provider-admin.mjs +41 -24
- package/src/standalone/seeds.mjs +1 -78
- package/src/standalone/usage-dashboard.mjs +96 -8
- package/src/tui/App.jsx +4800 -2140
- package/src/tui/components/AnsiText.jsx +39 -28
- package/src/tui/components/ContextPanel.jsx +14 -4
- package/src/tui/components/Markdown.jsx +43 -77
- package/src/tui/components/MarkdownTable.jsx +9 -184
- package/src/tui/components/Message.jsx +28 -11
- package/src/tui/components/Picker.jsx +95 -56
- package/src/tui/components/PromptInput.jsx +428 -239
- package/src/tui/components/QueuedCommands.jsx +1 -1
- package/src/tui/components/SlashCommandPalette.jsx +27 -21
- package/src/tui/components/Spinner.jsx +67 -38
- package/src/tui/components/StatusLine.jsx +606 -38
- package/src/tui/components/TextEntryPanel.jsx +128 -9
- package/src/tui/components/ToolExecution.jsx +597 -362
- package/src/tui/components/TurnDone.jsx +3 -3
- package/src/tui/components/UsagePanel.jsx +3 -5
- package/src/tui/components/tool-output-format.mjs +499 -0
- package/src/tui/components/tool-output-format.test.mjs +312 -0
- package/src/tui/dist/index.mjs +9192 -2399
- package/src/tui/engine-runtime-notification.test.mjs +115 -0
- package/src/tui/engine-tool-result-text.test.mjs +75 -0
- package/src/tui/engine.mjs +1455 -279
- package/src/tui/figures.mjs +21 -40
- package/src/tui/index.jsx +75 -31
- package/src/tui/input-editing.mjs +25 -0
- package/src/tui/markdown/format-token.mjs +671 -69
- package/src/tui/markdown/format-token.test.mjs +312 -0
- package/src/tui/markdown/render-ansi.mjs +94 -0
- package/src/tui/markdown/render-ansi.test.mjs +108 -0
- package/src/tui/markdown/stream-fence.mjs +34 -0
- package/src/tui/markdown/stream-fence.test.mjs +26 -0
- package/src/tui/markdown/table-layout.mjs +250 -0
- package/src/tui/paste-attachments.mjs +0 -7
- package/src/tui/spinner-verbs.mjs +1 -2
- package/src/tui/statusline-ansi-bridge.mjs +172 -0
- package/src/tui/statusline-ansi-bridge.test.mjs +159 -0
- package/src/tui/theme.mjs +756 -24
- package/src/tui/time-format.mjs +1 -1
- package/src/tui/transcript-tool-failures.mjs +67 -0
- package/src/tui/transcript-tool-failures.test.mjs +111 -0
- package/src/ui/ansi.mjs +1 -2
- package/src/ui/markdown.mjs +85 -26
- package/src/ui/markdown.test.mjs +70 -0
- package/src/ui/model-display.mjs +121 -0
- package/src/ui/session-stats.mjs +44 -0
- package/src/ui/statusline-context-label.test.mjs +15 -0
- package/src/ui/statusline.mjs +386 -178
- package/src/ui/tool-card.mjs +2 -13
- package/src/vendor/statusline/bin/statusline-lib.mjs +1 -620
- package/src/vendor/statusline/bin/statusline-route.mjs +169 -37
- package/src/vendor/statusline/bin/statusline-route.test.mjs +80 -0
- package/src/vendor/statusline/scripts/lib/gateway-settings.mjs +3 -3
- package/src/vendor/statusline/src/gateway/claude-current.mjs +1 -1
- package/src/vendor/statusline/src/gateway/route-meta.mjs +44 -6
- package/src/vendor/statusline/src/gateway/session-routes.mjs +1 -1
- package/src/workflows/default/WORKFLOW.md +12 -5
- package/src/workflows/default/workflow.json +0 -1
- package/src/workflows/solo/WORKFLOW.md +15 -0
- package/src/workflows/solo/workflow.json +7 -0
- package/vendor/ink/build/ink.js +54 -8
- package/vendor/ink/build/output.js +6 -1
- package/src/agents/scheduler-task.md +0 -3
- package/src/agents/web-researcher/AGENT.md +0 -3
- package/src/agents/web-researcher/agent.json +0 -6
- package/src/agents/webhook-handler.md +0 -3
- package/src/hooks/lib/permission-rules.cjs +0 -170
- package/src/hooks/lib/settings-loader.cjs +0 -112
- package/src/lib/hook-pipe-path.cjs +0 -10
- package/src/rules/bridge/00-common.md +0 -5
- package/src/rules/bridge/30-explorer.md +0 -4
- package/src/rules/lead/00-tool-lead.md +0 -5
- package/src/rules/shared/00-language.md +0 -3
- package/src/runtime/agent/orchestrator/tools/builtin/native-edit-runner.mjs +0 -110
- package/src/runtime/agent/orchestrator/tools/mutation-content-cache.mjs +0 -67
- package/src/runtime/channels/lib/hook-pipe-server.mjs +0 -671
- package/src/runtime/memory/lib/bridge-trace-queries.mjs +0 -120
- package/src/runtime/shared/llm/pid-cleanup.mjs +0 -27
- package/src/standalone/bridge-tool.mjs +0 -1414
- package/src/tui/runtime/shared/process-shutdown.mjs +0 -1
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* OpenAI
|
|
2
|
+
* OpenAI OAuth subscription — WebSocket transport.
|
|
3
3
|
*
|
|
4
4
|
* Single dispatch path for the openai-oauth provider (SSE removed in
|
|
5
5
|
* v0.6.117). Uses the `responses_websockets=2026-02-06` beta WebSocket
|
|
@@ -28,12 +28,21 @@ import { errText } from '../../../shared/err-text.mjs';
|
|
|
28
28
|
import { createHash, randomBytes } from 'crypto';
|
|
29
29
|
import {
|
|
30
30
|
extractCachedTokens,
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
} from '../
|
|
36
|
-
import {
|
|
31
|
+
traceAgentFetch,
|
|
32
|
+
traceAgentSse,
|
|
33
|
+
traceAgentUsage,
|
|
34
|
+
appendAgentTrace,
|
|
35
|
+
} from '../agent-trace.mjs';
|
|
36
|
+
import {
|
|
37
|
+
classifyHandshakeError,
|
|
38
|
+
classifyMidstreamError,
|
|
39
|
+
createStreamSafetyStamps,
|
|
40
|
+
jitterDelayMs,
|
|
41
|
+
MIDSTREAM_RETRY_POLICY,
|
|
42
|
+
populateHttpStatusFromMessage,
|
|
43
|
+
sleepWithAbort,
|
|
44
|
+
} from './retry-classifier.mjs';
|
|
45
|
+
import { makeInvalidToolArgsMarker } from './openai-compat-stream.mjs';
|
|
37
46
|
import {
|
|
38
47
|
PROVIDER_RETRY_MAX_ATTEMPTS,
|
|
39
48
|
PROVIDER_WS_ACQUIRE_TIMEOUT_MS,
|
|
@@ -41,6 +50,7 @@ import {
|
|
|
41
50
|
PROVIDER_WS_HANDSHAKE_TIMEOUT_MS,
|
|
42
51
|
PROVIDER_WS_INTER_CHUNK_TIMEOUT_MS,
|
|
43
52
|
} from '../stall-policy.mjs';
|
|
53
|
+
import { customToolCallFromResponseItem } from './custom-tool-wire.mjs';
|
|
44
54
|
|
|
45
55
|
globalThis.__mixdogOpenaiWsRuntimeLoaded = true;
|
|
46
56
|
|
|
@@ -69,9 +79,18 @@ const WS_PRE_RESPONSE_CREATED_MS = (() => {
|
|
|
69
79
|
})();
|
|
70
80
|
// Inter-chunk inactivity after first meaningful output.
|
|
71
81
|
const WS_INTER_CHUNK_MS = PROVIDER_WS_INTER_CHUNK_TIMEOUT_MS;
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
82
|
+
// Mid-stream retry budgets + backoff now live in the shared MIDSTREAM_RETRY_POLICY
|
|
83
|
+
// table (retry-classifier.mjs). These aliases keep the local call sites readable
|
|
84
|
+
// and ensure the numbers exist in exactly ONE place.
|
|
85
|
+
const MIDSTREAM_WS_TRANSIENT_RETRY_LIMIT = MIDSTREAM_RETRY_POLICY.ws.transientCloseRetries;
|
|
86
|
+
const MIDSTREAM_DEFAULT_RETRY_LIMIT = MIDSTREAM_RETRY_POLICY.ws.defaultRetries;
|
|
87
|
+
const MIDSTREAM_BACKOFF_MS = MIDSTREAM_RETRY_POLICY.ws.backoff;
|
|
88
|
+
// Policy object passed to the shared classifyMidstreamError for the WS path.
|
|
89
|
+
const WS_MIDSTREAM_POLICY = {
|
|
90
|
+
mode: 'ws',
|
|
91
|
+
transientCloseRetries: MIDSTREAM_RETRY_POLICY.ws.transientCloseRetries,
|
|
92
|
+
defaultRetries: MIDSTREAM_RETRY_POLICY.ws.defaultRetries,
|
|
93
|
+
};
|
|
75
94
|
|
|
76
95
|
// Handshake retry policy. The `ws` library surfaces a bare
|
|
77
96
|
// `Opening handshake has timed out` Error after handshakeTimeout; transient
|
|
@@ -88,14 +107,14 @@ const HANDSHAKE_MAX_ATTEMPTS = PROVIDER_RETRY_MAX_ATTEMPTS;
|
|
|
88
107
|
const HANDSHAKE_BACKOFF_BASE_MS = 500;
|
|
89
108
|
const HANDSHAKE_BACKOFF_CAP_MS = 5000;
|
|
90
109
|
// WS socket pool buckets are keyed by `poolKey` (the per-call sessionId)
|
|
91
|
-
// to isolate parallel
|
|
92
|
-
// a second caller cannot grab a sibling's mid-turn entry (
|
|
110
|
+
// to isolate parallel agent invocations — each gets its own socket so
|
|
111
|
+
// a second caller cannot grab a sibling's mid-turn entry (openai-oauth would
|
|
93
112
|
// otherwise reject the new response.create with "No tool output found
|
|
94
|
-
// for function call ..."). The
|
|
113
|
+
// for function call ..."). The handshake `session_id` header/URL
|
|
95
114
|
// uses `cacheKey` — a prefix-scoped cache key derived from the configured
|
|
96
115
|
// provider namespace plus model/system/tools hash. Same-prefix sessions share
|
|
97
116
|
// server-side prompt cache, while unrelated main/worker prefixes no longer
|
|
98
|
-
// evict each other inside one static provider lane.
|
|
117
|
+
// evict each other inside one static provider lane. The backend dedupes cache by
|
|
99
118
|
// handshake session_id, not by body.prompt_cache_key alone (measured
|
|
100
119
|
// 2026-04-19 after the v0.6.151 regression).
|
|
101
120
|
const MAX_POOLED_SOCKETS_PER_KEY = 8;
|
|
@@ -106,10 +125,10 @@ const MAX_POOLED_SOCKETS_PER_KEY = 8;
|
|
|
106
125
|
// closing, ephemeral }
|
|
107
126
|
const _wsPool = new Map();
|
|
108
127
|
|
|
109
|
-
// Final prompt_cache_key/session_id lane guard for OpenAI
|
|
128
|
+
// Final prompt_cache_key/session_id lane guard for OpenAI OAuth transports.
|
|
110
129
|
// The provider code may shard one logical prefix into N cache keys for 10+
|
|
111
130
|
// total parallelism; inside each final key we still serialize requests because
|
|
112
|
-
//
|
|
131
|
+
// Live probes show same-key concurrent WebSockets can randomly miss the
|
|
113
132
|
// server prompt cache even after warm-up.
|
|
114
133
|
const _openAiPromptCacheLanes = new Map();
|
|
115
134
|
const _openAiPromptCacheLaneRates = new Map();
|
|
@@ -121,6 +140,11 @@ function _positiveInt(value, fallback = 0) {
|
|
|
121
140
|
return Number.isFinite(n) && n > 0 ? Math.floor(n) : fallback;
|
|
122
141
|
}
|
|
123
142
|
|
|
143
|
+
function _nonNegativeInt(value, fallback = 0) {
|
|
144
|
+
const n = Number(value);
|
|
145
|
+
return Number.isFinite(n) && n >= 0 ? Math.floor(n) : fallback;
|
|
146
|
+
}
|
|
147
|
+
|
|
124
148
|
function _cacheLaneHash(value) {
|
|
125
149
|
return createHash('sha256').update(String(value || '')).digest('hex').slice(0, 12);
|
|
126
150
|
}
|
|
@@ -146,7 +170,7 @@ function _openAiPromptCacheLaneQueueTimeoutMs(sendOpts = {}) {
|
|
|
146
170
|
}
|
|
147
171
|
|
|
148
172
|
function _openAiPromptCacheLaneRateLimitPerMin(sendOpts = {}) {
|
|
149
|
-
return
|
|
173
|
+
return _nonNegativeInt(
|
|
150
174
|
sendOpts?.openaiCacheLaneRateLimitPerMin
|
|
151
175
|
?? sendOpts?.promptCacheLaneRateLimitPerMin
|
|
152
176
|
?? process.env.MIXDOG_OPENAI_CACHE_LANE_RPM
|
|
@@ -155,6 +179,45 @@ function _openAiPromptCacheLaneRateLimitPerMin(sendOpts = {}) {
|
|
|
155
179
|
);
|
|
156
180
|
}
|
|
157
181
|
|
|
182
|
+
function _openAiPromptCacheLaneDeltaRateLimitPerMin(sendOpts = {}) {
|
|
183
|
+
return _nonNegativeInt(
|
|
184
|
+
sendOpts?.openaiCacheLaneDeltaRateLimitPerMin
|
|
185
|
+
?? sendOpts?.promptCacheLaneDeltaRateLimitPerMin
|
|
186
|
+
?? process.env.MIXDOG_OPENAI_CACHE_LANE_DELTA_RPM
|
|
187
|
+
?? process.env.MIXDOG_OPENAI_CACHE_DELTA_RPM,
|
|
188
|
+
60,
|
|
189
|
+
);
|
|
190
|
+
}
|
|
191
|
+
|
|
192
|
+
function _openAiPromptCacheLaneDeltaMaxItems(sendOpts = {}) {
|
|
193
|
+
return _nonNegativeInt(
|
|
194
|
+
sendOpts?.openaiCacheLaneDeltaMaxItems
|
|
195
|
+
?? sendOpts?.promptCacheLaneDeltaMaxItems
|
|
196
|
+
?? process.env.MIXDOG_OPENAI_CACHE_LANE_DELTA_MAX_ITEMS
|
|
197
|
+
?? process.env.MIXDOG_OPENAI_CACHE_DELTA_MAX_ITEMS,
|
|
198
|
+
8,
|
|
199
|
+
);
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
function _openAiPromptCacheLaneDeltaMaxTokens(sendOpts = {}) {
|
|
203
|
+
return _nonNegativeInt(
|
|
204
|
+
sendOpts?.openaiCacheLaneDeltaMaxTokens
|
|
205
|
+
?? sendOpts?.promptCacheLaneDeltaMaxTokens
|
|
206
|
+
?? process.env.MIXDOG_OPENAI_CACHE_LANE_DELTA_MAX_TOKENS
|
|
207
|
+
?? process.env.MIXDOG_OPENAI_CACHE_DELTA_MAX_TOKENS,
|
|
208
|
+
20_000,
|
|
209
|
+
);
|
|
210
|
+
}
|
|
211
|
+
|
|
212
|
+
function _openAiPromptCacheLaneSlowTraceMs(sendOpts = {}) {
|
|
213
|
+
return _positiveInt(
|
|
214
|
+
sendOpts?.openaiCacheLaneSlowTraceMs
|
|
215
|
+
?? sendOpts?.promptCacheLaneSlowTraceMs
|
|
216
|
+
?? process.env.MIXDOG_OPENAI_CACHE_LANE_SLOW_MS,
|
|
217
|
+
3000,
|
|
218
|
+
);
|
|
219
|
+
}
|
|
220
|
+
|
|
158
221
|
function _isOpenAiPromptCacheLaneAuth(auth) {
|
|
159
222
|
return auth?.type !== 'xai';
|
|
160
223
|
}
|
|
@@ -229,12 +292,13 @@ function _sleepWithSignal(ms, signal) {
|
|
|
229
292
|
});
|
|
230
293
|
}
|
|
231
294
|
|
|
232
|
-
async function _reserveOpenAiPromptCacheLaneRate({ key, limitPerMin, signal }) {
|
|
295
|
+
async function _reserveOpenAiPromptCacheLaneRate({ key, limitPerMin, signal, beforeWait }) {
|
|
233
296
|
if (!limitPerMin || limitPerMin <= 0) {
|
|
234
297
|
return { rateLimitPerMin: 0, rateWaitMs: 0, rateWindowCount: 0 };
|
|
235
298
|
}
|
|
236
299
|
const state = _getOpenAiPromptCacheLaneRateState(key);
|
|
237
300
|
const startedAt = Date.now();
|
|
301
|
+
let beforeWaitCalled = false;
|
|
238
302
|
while (true) {
|
|
239
303
|
const now = Date.now();
|
|
240
304
|
_pruneOpenAiPromptCacheLaneRateState(state, now);
|
|
@@ -247,10 +311,57 @@ async function _reserveOpenAiPromptCacheLaneRate({ key, limitPerMin, signal }) {
|
|
|
247
311
|
};
|
|
248
312
|
}
|
|
249
313
|
const waitMs = Math.max(25, state.starts[0] + OPENAI_PROMPT_CACHE_LANE_RATE_WINDOW_MS - now);
|
|
314
|
+
if (!beforeWaitCalled && typeof beforeWait === 'function') {
|
|
315
|
+
beforeWaitCalled = true;
|
|
316
|
+
await beforeWait({
|
|
317
|
+
waitMs,
|
|
318
|
+
rateLimitPerMin: limitPerMin,
|
|
319
|
+
rateWindowCount: state.starts.length,
|
|
320
|
+
});
|
|
321
|
+
}
|
|
250
322
|
await _sleepWithSignal(waitMs, signal);
|
|
251
323
|
}
|
|
252
324
|
}
|
|
253
325
|
|
|
326
|
+
export function _resolveOpenAiPromptCacheRatePolicy(sendOpts = {}, info = {}) {
|
|
327
|
+
const mode = String(info?.mode || '').toLowerCase();
|
|
328
|
+
const frameInputItems = Number(info?.frameInputItems);
|
|
329
|
+
const deltaTokens = Number(info?.deltaTokens);
|
|
330
|
+
const hasPreviousResponseId = info?.hasPreviousResponseId === true;
|
|
331
|
+
const fullLimitPerMin = _openAiPromptCacheLaneRateLimitPerMin(sendOpts);
|
|
332
|
+
const deltaLimitPerMin = _openAiPromptCacheLaneDeltaRateLimitPerMin(sendOpts);
|
|
333
|
+
const deltaMaxItems = _openAiPromptCacheLaneDeltaMaxItems(sendOpts);
|
|
334
|
+
const deltaMaxTokens = _openAiPromptCacheLaneDeltaMaxTokens(sendOpts);
|
|
335
|
+
const itemCount = Number.isFinite(frameInputItems) ? frameInputItems : null;
|
|
336
|
+
const tokenCount = Number.isFinite(deltaTokens) ? deltaTokens : null;
|
|
337
|
+
const smallDeltaItems = itemCount == null || deltaMaxItems <= 0 || itemCount <= deltaMaxItems;
|
|
338
|
+
const smallDeltaTokens = tokenCount == null || deltaMaxTokens <= 0 || tokenCount <= deltaMaxTokens;
|
|
339
|
+
|
|
340
|
+
if (mode === 'delta' && hasPreviousResponseId && smallDeltaItems && smallDeltaTokens) {
|
|
341
|
+
return {
|
|
342
|
+
policy: deltaLimitPerMin > 0 ? 'delta_relaxed' : 'delta_unlimited',
|
|
343
|
+
limitPerMin: deltaLimitPerMin,
|
|
344
|
+
fullLimitPerMin,
|
|
345
|
+
deltaLimitPerMin,
|
|
346
|
+
deltaMaxItems,
|
|
347
|
+
deltaMaxTokens,
|
|
348
|
+
frameInputItems: itemCount,
|
|
349
|
+
deltaTokens: tokenCount,
|
|
350
|
+
};
|
|
351
|
+
}
|
|
352
|
+
|
|
353
|
+
return {
|
|
354
|
+
policy: mode === 'delta' ? 'delta_guarded' : 'full_guard',
|
|
355
|
+
limitPerMin: fullLimitPerMin,
|
|
356
|
+
fullLimitPerMin,
|
|
357
|
+
deltaLimitPerMin,
|
|
358
|
+
deltaMaxItems,
|
|
359
|
+
deltaMaxTokens,
|
|
360
|
+
frameInputItems: itemCount,
|
|
361
|
+
deltaTokens: tokenCount,
|
|
362
|
+
};
|
|
363
|
+
}
|
|
364
|
+
|
|
254
365
|
function _removeQueuedOpenAiPromptCacheLaneRequest(state, request) {
|
|
255
366
|
const index = state.queue.indexOf(request);
|
|
256
367
|
if (index >= 0) state.queue.splice(index, 1);
|
|
@@ -332,29 +443,10 @@ async function _withOpenAiPromptCacheLane({ auth, cacheKey, sendOpts, poolKey, i
|
|
|
332
443
|
}
|
|
333
444
|
const laneKey = `openai-prompt:${traceProvider || 'openai'}:${useModel || 'default'}:${cacheKey}`;
|
|
334
445
|
const laneKeyHash = _cacheLaneHash(laneKey);
|
|
335
|
-
const rateMeta = await _reserveOpenAiPromptCacheLaneRate({
|
|
336
|
-
key: laneKey,
|
|
337
|
-
limitPerMin: _openAiPromptCacheLaneRateLimitPerMin(sendOpts),
|
|
338
|
-
signal: externalSignal,
|
|
339
|
-
});
|
|
340
|
-
if (rateMeta.rateWaitMs > 0) {
|
|
341
|
-
appendBridgeTrace({
|
|
342
|
-
sessionId: poolKey,
|
|
343
|
-
iteration,
|
|
344
|
-
kind: 'cache_lane',
|
|
345
|
-
provider: traceProvider,
|
|
346
|
-
model: useModel,
|
|
347
|
-
event: 'rate_wait',
|
|
348
|
-
lane_key_hash: laneKeyHash,
|
|
349
|
-
rate_limit_per_min: rateMeta.rateLimitPerMin,
|
|
350
|
-
rate_wait_ms: rateMeta.rateWaitMs,
|
|
351
|
-
rate_window_count: rateMeta.rateWindowCount,
|
|
352
|
-
});
|
|
353
|
-
}
|
|
354
446
|
const state = _getOpenAiPromptCacheLaneState(laneKey, maxInFlight);
|
|
355
447
|
const queued = state.active >= state.maxInFlight;
|
|
356
448
|
if (queued) {
|
|
357
|
-
|
|
449
|
+
appendAgentTrace({
|
|
358
450
|
sessionId: poolKey,
|
|
359
451
|
iteration,
|
|
360
452
|
kind: 'cache_lane',
|
|
@@ -368,23 +460,139 @@ async function _withOpenAiPromptCacheLane({ auth, cacheKey, sendOpts, poolKey, i
|
|
|
368
460
|
});
|
|
369
461
|
}
|
|
370
462
|
const timeoutMs = _openAiPromptCacheLaneQueueTimeoutMs(sendOpts);
|
|
371
|
-
|
|
463
|
+
let handle = await _acquireOpenAiPromptCacheLane({ key: laneKey, maxInFlight, signal: externalSignal, timeoutMs });
|
|
464
|
+
let handleActive = true;
|
|
372
465
|
const laneMeta = {
|
|
373
466
|
enabled: true,
|
|
374
467
|
laneKeyHash,
|
|
375
468
|
maxInFlight,
|
|
376
|
-
|
|
377
|
-
|
|
378
|
-
|
|
469
|
+
ratePolicy: 'pending',
|
|
470
|
+
rateLimitPerMin: 0,
|
|
471
|
+
rateWaitMs: 0,
|
|
472
|
+
rateWindowCount: 0,
|
|
473
|
+
rateReleasedForWait: false,
|
|
474
|
+
rateReacquireWaitMs: 0,
|
|
379
475
|
queued: queued || handle.queued === true,
|
|
380
476
|
waitMs: handle.waitedMs,
|
|
381
477
|
activeAfterAcquire: handle.activeCount,
|
|
382
478
|
queueDepthAfterAcquire: handle.queueDepth,
|
|
479
|
+
async reserveRate(info = {}) {
|
|
480
|
+
if (laneMeta.ratePolicy !== 'pending') return laneMeta;
|
|
481
|
+
const policy = _resolveOpenAiPromptCacheRatePolicy(sendOpts, info);
|
|
482
|
+
let releasedForRateWait = false;
|
|
483
|
+
const rateMeta = await _reserveOpenAiPromptCacheLaneRate({
|
|
484
|
+
key: laneKey,
|
|
485
|
+
limitPerMin: policy.limitPerMin,
|
|
486
|
+
signal: externalSignal,
|
|
487
|
+
beforeWait: () => {
|
|
488
|
+
if (!handleActive) return;
|
|
489
|
+
handle.release();
|
|
490
|
+
handleActive = false;
|
|
491
|
+
releasedForRateWait = true;
|
|
492
|
+
},
|
|
493
|
+
});
|
|
494
|
+
let reacquireWaitMs = 0;
|
|
495
|
+
if (releasedForRateWait) {
|
|
496
|
+
const reacquired = await _acquireOpenAiPromptCacheLane({
|
|
497
|
+
key: laneKey,
|
|
498
|
+
maxInFlight,
|
|
499
|
+
signal: externalSignal,
|
|
500
|
+
timeoutMs,
|
|
501
|
+
});
|
|
502
|
+
handle = reacquired;
|
|
503
|
+
handleActive = true;
|
|
504
|
+
reacquireWaitMs = reacquired.waitedMs;
|
|
505
|
+
laneMeta.queued = laneMeta.queued || reacquired.queued === true;
|
|
506
|
+
laneMeta.waitMs = (Number(laneMeta.waitMs) || 0) + reacquireWaitMs;
|
|
507
|
+
laneMeta.activeAfterAcquire = reacquired.activeCount;
|
|
508
|
+
laneMeta.queueDepthAfterAcquire = reacquired.queueDepth;
|
|
509
|
+
}
|
|
510
|
+
Object.assign(laneMeta, {
|
|
511
|
+
ratePolicy: policy.policy,
|
|
512
|
+
rateLimitPerMin: rateMeta.rateLimitPerMin,
|
|
513
|
+
rateWaitMs: rateMeta.rateWaitMs,
|
|
514
|
+
rateWindowCount: rateMeta.rateWindowCount,
|
|
515
|
+
rateReleasedForWait: releasedForRateWait,
|
|
516
|
+
rateReacquireWaitMs: reacquireWaitMs,
|
|
517
|
+
rateFullLimitPerMin: policy.fullLimitPerMin,
|
|
518
|
+
rateDeltaLimitPerMin: policy.deltaLimitPerMin,
|
|
519
|
+
rateDeltaMaxItems: policy.deltaMaxItems,
|
|
520
|
+
rateDeltaMaxTokens: policy.deltaMaxTokens,
|
|
521
|
+
ratePolicyFrameInputItems: policy.frameInputItems,
|
|
522
|
+
ratePolicyDeltaTokens: policy.deltaTokens,
|
|
523
|
+
});
|
|
524
|
+
if (rateMeta.rateWaitMs > 0) {
|
|
525
|
+
appendAgentTrace({
|
|
526
|
+
sessionId: poolKey,
|
|
527
|
+
iteration,
|
|
528
|
+
kind: 'cache_lane',
|
|
529
|
+
provider: traceProvider,
|
|
530
|
+
model: useModel,
|
|
531
|
+
event: 'rate_wait',
|
|
532
|
+
lane_key_hash: laneKeyHash,
|
|
533
|
+
rate_policy: laneMeta.ratePolicy,
|
|
534
|
+
rate_limit_per_min: rateMeta.rateLimitPerMin,
|
|
535
|
+
rate_wait_ms: rateMeta.rateWaitMs,
|
|
536
|
+
rate_window_count: rateMeta.rateWindowCount,
|
|
537
|
+
released_for_rate_wait: releasedForRateWait,
|
|
538
|
+
reacquire_wait_ms: reacquireWaitMs,
|
|
539
|
+
frame_input_items: policy.frameInputItems,
|
|
540
|
+
delta_tokens: policy.deltaTokens,
|
|
541
|
+
});
|
|
542
|
+
}
|
|
543
|
+
return laneMeta;
|
|
544
|
+
},
|
|
383
545
|
};
|
|
384
546
|
try {
|
|
385
547
|
return await fn(laneMeta);
|
|
386
548
|
} finally {
|
|
387
|
-
|
|
549
|
+
const slowTraceMs = _openAiPromptCacheLaneSlowTraceMs(sendOpts);
|
|
550
|
+
const slowWaitMs = Math.max(Number(laneMeta.rateWaitMs) || 0, Number(laneMeta.waitMs) || 0);
|
|
551
|
+
if (slowTraceMs > 0 && slowWaitMs >= slowTraceMs) {
|
|
552
|
+
appendAgentTrace({
|
|
553
|
+
sessionId: poolKey,
|
|
554
|
+
iteration,
|
|
555
|
+
kind: 'cache_lane_slow',
|
|
556
|
+
provider: traceProvider,
|
|
557
|
+
model: useModel,
|
|
558
|
+
event: laneMeta.rateWaitMs > 0 && laneMeta.waitMs > 0
|
|
559
|
+
? 'rate_and_queue_wait'
|
|
560
|
+
: laneMeta.rateWaitMs > 0
|
|
561
|
+
? 'rate_wait'
|
|
562
|
+
: 'queue_wait',
|
|
563
|
+
lane_key_hash: laneKeyHash,
|
|
564
|
+
payload: {
|
|
565
|
+
event: laneMeta.rateWaitMs > 0 && laneMeta.waitMs > 0
|
|
566
|
+
? 'rate_and_queue_wait'
|
|
567
|
+
: laneMeta.rateWaitMs > 0
|
|
568
|
+
? 'rate_wait'
|
|
569
|
+
: 'queue_wait',
|
|
570
|
+
provider: traceProvider,
|
|
571
|
+
model: useModel,
|
|
572
|
+
lane_key_hash: laneKeyHash,
|
|
573
|
+
threshold_ms: slowTraceMs,
|
|
574
|
+
max_wait_ms: slowWaitMs,
|
|
575
|
+
rate_policy: laneMeta.ratePolicy,
|
|
576
|
+
rate_limit_per_min: laneMeta.rateLimitPerMin,
|
|
577
|
+
rate_wait_ms: laneMeta.rateWaitMs,
|
|
578
|
+
rate_window_count: laneMeta.rateWindowCount,
|
|
579
|
+
released_for_rate_wait: laneMeta.rateReleasedForWait,
|
|
580
|
+
reacquire_wait_ms: laneMeta.rateReacquireWaitMs,
|
|
581
|
+
rate_full_limit_per_min: laneMeta.rateFullLimitPerMin,
|
|
582
|
+
rate_delta_limit_per_min: laneMeta.rateDeltaLimitPerMin,
|
|
583
|
+
rate_delta_max_items: laneMeta.rateDeltaMaxItems,
|
|
584
|
+
rate_delta_max_tokens: laneMeta.rateDeltaMaxTokens,
|
|
585
|
+
rate_policy_frame_input_items: laneMeta.ratePolicyFrameInputItems,
|
|
586
|
+
rate_policy_delta_tokens: laneMeta.ratePolicyDeltaTokens,
|
|
587
|
+
max_in_flight: laneMeta.maxInFlight,
|
|
588
|
+
queued: laneMeta.queued,
|
|
589
|
+
wait_ms: laneMeta.waitMs,
|
|
590
|
+
active_after_acquire: laneMeta.activeAfterAcquire,
|
|
591
|
+
queue_depth_after_acquire: laneMeta.queueDepthAfterAcquire,
|
|
592
|
+
},
|
|
593
|
+
});
|
|
594
|
+
}
|
|
595
|
+
if (handleActive) handle.release();
|
|
388
596
|
}
|
|
389
597
|
}
|
|
390
598
|
|
|
@@ -503,14 +711,14 @@ function _buildHandshakeHeaders({ auth, sessionToken, turnState, cacheKey: _cach
|
|
|
503
711
|
return headers;
|
|
504
712
|
}
|
|
505
713
|
|
|
506
|
-
// handshake session_id is the conversation slot
|
|
714
|
+
// handshake session_id is the conversation slot openai-oauth uses for in-memory
|
|
507
715
|
// prefix state. All orchestrator-internal dispatches for this provider share
|
|
508
716
|
// the same cacheKey (built in manager.mjs via providerCacheKey()), so they
|
|
509
717
|
// share the server-side prefix-cache shard across roles/sources.
|
|
510
718
|
function _mintSessionToken(cacheKey, auth) {
|
|
511
719
|
// xAI's public WebSocket endpoint uses the open connection plus
|
|
512
|
-
// response ids for continuation; unlike
|
|
513
|
-
//
|
|
720
|
+
// response ids for continuation; unlike openai-oauth, it does not need the
|
|
721
|
+
// OAuth-specific session_id handshake shard.
|
|
514
722
|
if (auth?.type === 'xai') return null;
|
|
515
723
|
return cacheKey || 'mixdog-default';
|
|
516
724
|
}
|
|
@@ -523,8 +731,8 @@ function _openSocket({ auth, sessionToken, turnState, externalSignal, cacheKey }
|
|
|
523
731
|
? OPENAI_WS_URL
|
|
524
732
|
: CODEX_WS_URL;
|
|
525
733
|
const _wsOpenStart = Date.now();
|
|
526
|
-
if (process.env.
|
|
527
|
-
process.stderr.write(`[
|
|
734
|
+
if (process.env.MIXDOG_DEBUG_AGENT) {
|
|
735
|
+
process.stderr.write(`[agent-trace] ws-open-start url=${baseUrl} tokenHash=${createHash('sha256').update(String(sessionToken)).digest('hex').slice(0, 8)} ts=${_wsOpenStart}\n`);
|
|
528
736
|
}
|
|
529
737
|
const url = baseUrl + (sessionToken ? `?session_id=${encodeURIComponent(String(sessionToken))}` : '');
|
|
530
738
|
return new Promise((resolve, reject) => {
|
|
@@ -546,8 +754,8 @@ function _openSocket({ auth, sessionToken, turnState, externalSignal, cacheKey }
|
|
|
546
754
|
const socket = new WebSocket(url, { headers, handshakeTimeout: WS_HANDSHAKE_TIMEOUT_MS });
|
|
547
755
|
acquireTimer = setTimeout(() => {
|
|
548
756
|
if (settled) return;
|
|
549
|
-
if (process.env.
|
|
550
|
-
process.stderr.write(`[
|
|
757
|
+
if (process.env.MIXDOG_DEBUG_AGENT) {
|
|
758
|
+
process.stderr.write(`[agent-trace] ws-open-fail kind=acquire_timeout timeoutMs=${WS_ACQUIRE_TIMEOUT_MS} elapsed=${Date.now() - _wsOpenStart}ms\n`);
|
|
551
759
|
}
|
|
552
760
|
try { socket.terminate(); } catch {}
|
|
553
761
|
settle(false, Object.assign(
|
|
@@ -564,14 +772,14 @@ function _openSocket({ auth, sessionToken, turnState, externalSignal, cacheKey }
|
|
|
564
772
|
} catch {}
|
|
565
773
|
});
|
|
566
774
|
socket.once('open', () => {
|
|
567
|
-
if (process.env.
|
|
568
|
-
process.stderr.write(`[
|
|
775
|
+
if (process.env.MIXDOG_DEBUG_AGENT) {
|
|
776
|
+
process.stderr.write(`[agent-trace] ws-open-ok elapsed=${Date.now() - _wsOpenStart}ms\n`);
|
|
569
777
|
}
|
|
570
778
|
settle(true, { socket, turnState: capturedHeaders.turnState });
|
|
571
779
|
});
|
|
572
780
|
socket.once('error', (err) => {
|
|
573
|
-
if (process.env.
|
|
574
|
-
process.stderr.write(`[
|
|
781
|
+
if (process.env.MIXDOG_DEBUG_AGENT) {
|
|
782
|
+
process.stderr.write(`[agent-trace] ws-open-fail kind=error msg=${String(err?.message || err).slice(0, 120)} elapsed=${Date.now() - _wsOpenStart}ms\n`);
|
|
575
783
|
}
|
|
576
784
|
try { socket.terminate(); } catch {}
|
|
577
785
|
settle(false, err instanceof Error ? err : Object.assign(new Error(errText(err) || 'openai-oauth WS error'), { wsErrorEvent: true, original: err }));
|
|
@@ -594,8 +802,8 @@ function _openSocket({ auth, sessionToken, turnState, externalSignal, cacheKey }
|
|
|
594
802
|
let body = '';
|
|
595
803
|
res.on('data', c => { if (body.length < 2048) body += c.toString('utf-8'); });
|
|
596
804
|
res.on('end', () => {
|
|
597
|
-
if (process.env.
|
|
598
|
-
process.stderr.write(`[
|
|
805
|
+
if (process.env.MIXDOG_DEBUG_AGENT) {
|
|
806
|
+
process.stderr.write(`[agent-trace] ws-open-fail kind=http status=${status} body=${body.slice(0, 120)} elapsed=${Date.now() - _wsOpenStart}ms\n`);
|
|
599
807
|
}
|
|
600
808
|
try { socket.terminate(); } catch {}
|
|
601
809
|
settle(false, Object.assign(new Error(`${_wsErrLabel(auth?.type === 'xai' ? 'xai' : auth?.type === 'openai-direct' ? 'openai-direct' : 'openai-oauth')} handshake ${status}: ${body.slice(0, 200)}`), { httpStatus: status, httpBody: body }));
|
|
@@ -616,12 +824,12 @@ function _openSocket({ auth, sessionToken, turnState, externalSignal, cacheKey }
|
|
|
616
824
|
|
|
617
825
|
async function acquireWebSocket({ auth, poolKey, cacheKey, forceFresh, externalSignal }) {
|
|
618
826
|
const _acqStart = Date.now();
|
|
619
|
-
if (process.env.
|
|
620
|
-
process.stderr.write(`[
|
|
827
|
+
if (process.env.MIXDOG_DEBUG_AGENT) {
|
|
828
|
+
process.stderr.write(`[agent-trace] acquire-start poolKey=${poolKey} cacheKey=${cacheKey} forceFresh=${forceFresh} externalAborted=${!!externalSignal?.aborted} ts=${_acqStart}\n`);
|
|
621
829
|
}
|
|
622
830
|
if (externalSignal?.aborted) {
|
|
623
831
|
const reason = externalSignal.reason;
|
|
624
|
-
throw reason instanceof Error ? reason : new Error('
|
|
832
|
+
throw reason instanceof Error ? reason : new Error('OpenAI OAuth WS acquire aborted');
|
|
625
833
|
}
|
|
626
834
|
if (poolKey && !forceFresh) {
|
|
627
835
|
const arr = _wsPool.get(poolKey) || [];
|
|
@@ -645,15 +853,15 @@ async function acquireWebSocket({ auth, poolKey, cacheKey, forceFresh, externalS
|
|
|
645
853
|
if (idle.lastInputPrefixHash === undefined) idle.lastInputPrefixHash = null;
|
|
646
854
|
if (idle.lastRequestInput === undefined) idle.lastRequestInput = null;
|
|
647
855
|
if (idle.lastResponseItems === undefined) idle.lastResponseItems = null;
|
|
648
|
-
if (process.env.
|
|
649
|
-
process.stderr.write(`[
|
|
856
|
+
if (process.env.MIXDOG_DEBUG_AGENT) {
|
|
857
|
+
process.stderr.write(`[agent-trace] acquire-reuse poolKey=${poolKey} openSockets=${arr.length} elapsed=${Date.now() - _acqStart}ms\n`);
|
|
650
858
|
}
|
|
651
859
|
return { entry: idle, reused: true };
|
|
652
860
|
}
|
|
653
861
|
// All entries busy and bucket at cap: fall through to ephemeral socket.
|
|
654
862
|
if (arr.length >= MAX_POOLED_SOCKETS_PER_KEY) {
|
|
655
|
-
if (process.env.
|
|
656
|
-
process.stderr.write(`[
|
|
863
|
+
if (process.env.MIXDOG_DEBUG_AGENT) {
|
|
864
|
+
process.stderr.write(`[agent-trace] acquire-ephemeral cacheKey=${cacheKey} reason=cap elapsed=${Date.now() - _acqStart}ms\n`);
|
|
657
865
|
}
|
|
658
866
|
const ephSessionToken = _mintSessionToken(cacheKey, auth);
|
|
659
867
|
const { socket, turnState } = await _openSocket({ auth, sessionToken: ephSessionToken, turnState: null, externalSignal, cacheKey });
|
|
@@ -683,13 +891,13 @@ async function acquireWebSocket({ auth, poolKey, cacheKey, forceFresh, externalS
|
|
|
683
891
|
return { entry, reused: false };
|
|
684
892
|
}
|
|
685
893
|
}
|
|
686
|
-
// Parallel sockets must not inherit sibling turnState or the
|
|
894
|
+
// Parallel sockets must not inherit sibling turnState or the openai-oauth server
|
|
687
895
|
// treats the new request as a continuation of another in-flight turn and
|
|
688
896
|
// returns "No tool output found for function call …". turnState only
|
|
689
897
|
// propagates within a single entry across its own iterations.
|
|
690
898
|
const sessionToken = _mintSessionToken(cacheKey, auth);
|
|
691
|
-
if (process.env.
|
|
692
|
-
process.stderr.write(`[
|
|
899
|
+
if (process.env.MIXDOG_DEBUG_AGENT) {
|
|
900
|
+
process.stderr.write(`[agent-trace] acquire-new tokenHash=${createHash('sha256').update(String(sessionToken)).digest('hex').slice(0, 8)} elapsed=${Date.now() - _acqStart}ms\n`);
|
|
693
901
|
}
|
|
694
902
|
const { socket, turnState } = await _openSocket({ auth, sessionToken, turnState: null, externalSignal, cacheKey });
|
|
695
903
|
const entry = {
|
|
@@ -780,16 +988,41 @@ function _contentPartsEqual(a, b) {
|
|
|
780
988
|
return _stableStringify(aa) === _stableStringify(bb);
|
|
781
989
|
}
|
|
782
990
|
|
|
783
|
-
function _logicalResponseItemMatch(inputItem, responseItem) {
|
|
991
|
+
export function _logicalResponseItemMatch(inputItem, responseItem) {
|
|
784
992
|
if (!inputItem || !responseItem) return false;
|
|
785
993
|
const inputType = inputItem.type || (inputItem.role === 'assistant' ? 'message' : '');
|
|
786
994
|
const responseType = responseItem.type || '';
|
|
787
995
|
if (responseType === 'function_call') {
|
|
788
|
-
|
|
789
|
-
|
|
790
|
-
|
|
996
|
+
if (inputType !== 'function_call') return false;
|
|
997
|
+
const inputCallId = String(inputItem.call_id || '');
|
|
998
|
+
const responseCallId = String(responseItem.call_id || '');
|
|
999
|
+
const inputName = String(inputItem.name || '');
|
|
1000
|
+
const responseName = String(responseItem.name || '');
|
|
1001
|
+
if (inputCallId && responseCallId) {
|
|
1002
|
+
// call_id is the server-side anchor. The replayed history may carry
|
|
1003
|
+
// locally compacted arguments, but previous_response_id already
|
|
1004
|
+
// points at the canonical output item.
|
|
1005
|
+
return inputCallId === responseCallId && inputName === responseName;
|
|
1006
|
+
}
|
|
1007
|
+
return inputName === responseName
|
|
791
1008
|
&& _normalizeArguments(inputItem.arguments) === _normalizeArguments(responseItem.arguments);
|
|
792
1009
|
}
|
|
1010
|
+
if (responseType === 'tool_search_call') {
|
|
1011
|
+
if (inputType !== 'tool_search_call') return false;
|
|
1012
|
+
const inputCallId = String(inputItem.call_id || '');
|
|
1013
|
+
const responseCallId = String(responseItem.call_id || '');
|
|
1014
|
+
if (inputCallId && responseCallId) return inputCallId === responseCallId;
|
|
1015
|
+
return _normalizeArguments(inputItem.arguments) === _normalizeArguments(responseItem.arguments);
|
|
1016
|
+
}
|
|
1017
|
+
if (responseType === 'custom_tool_call') {
|
|
1018
|
+
if (inputType !== 'custom_tool_call') return false;
|
|
1019
|
+
const inputCallId = String(inputItem.call_id || '');
|
|
1020
|
+
const responseCallId = String(responseItem.call_id || '');
|
|
1021
|
+
const inputName = String(inputItem.name || '');
|
|
1022
|
+
const responseName = String(responseItem.name || '');
|
|
1023
|
+
if (inputCallId && responseCallId) return inputCallId === responseCallId && inputName === responseName;
|
|
1024
|
+
return inputName === responseName && String(inputItem.input || '') === String(responseItem.input || '');
|
|
1025
|
+
}
|
|
793
1026
|
if (responseType === 'message') {
|
|
794
1027
|
const inputRole = inputItem.role || (inputType === 'message' ? 'assistant' : '');
|
|
795
1028
|
const responseRole = responseItem.role || 'assistant';
|
|
@@ -838,6 +1071,7 @@ function _isReplayLikeHead(item, responseItem) {
|
|
|
838
1071
|
const responseType = responseItem.type || '';
|
|
839
1072
|
if (responseType === 'message') return inputType === 'message';
|
|
840
1073
|
if (responseType === 'function_call') return inputType === 'function_call';
|
|
1074
|
+
if (responseType === 'tool_search_call') return inputType === 'tool_search_call';
|
|
841
1075
|
return inputType === responseType;
|
|
842
1076
|
}
|
|
843
1077
|
|
|
@@ -969,7 +1203,25 @@ function _httpStatusFromWsClose(code, reason) {
|
|
|
969
1203
|
function _wsErrLabel(p) {
|
|
970
1204
|
if (p === 'xai') return 'xAI WS';
|
|
971
1205
|
if (p === 'openai-direct' || p === 'openai') return 'OpenAI WS';
|
|
972
|
-
return '
|
|
1206
|
+
return 'OpenAI OAuth WS';
|
|
1207
|
+
}
|
|
1208
|
+
// tool_search_call.arguments parse. Module-scope (exported) for direct test
|
|
1209
|
+
// coverage. Native convergence (openai-oauth / anthropic-oauth / opencode): same policy
|
|
1210
|
+
// as the function_call_arguments.done path and openai-oauth _parseJsonObject —
|
|
1211
|
+
// object passes through; null/non-string/empty/whitespace → {} (no args); a
|
|
1212
|
+
// non-empty string that fails JSON.parse is deterministic bad JSON, surfaced
|
|
1213
|
+
// as an invalid-args MARKER (not silently swallowed to {}) so the dispatch
|
|
1214
|
+
// loop returns an is_error tool_result and the model self-corrects in the same
|
|
1215
|
+
// turn.
|
|
1216
|
+
export function parseToolSearchArgs(value) {
|
|
1217
|
+
if (value && typeof value === 'object') return value;
|
|
1218
|
+
if (typeof value !== 'string' || !value.trim()) return {};
|
|
1219
|
+
try {
|
|
1220
|
+
const parsed = JSON.parse(value);
|
|
1221
|
+
return parsed && typeof parsed === 'object' ? parsed : {};
|
|
1222
|
+
} catch (err) {
|
|
1223
|
+
return makeInvalidToolArgsMarker(value, err instanceof Error ? err.message : String(err));
|
|
1224
|
+
}
|
|
973
1225
|
}
|
|
974
1226
|
export async function _streamResponse({
|
|
975
1227
|
entry,
|
|
@@ -996,7 +1248,6 @@ export async function _streamResponse({
|
|
|
996
1248
|
const toolCalls = [];
|
|
997
1249
|
const webSearchCalls = [];
|
|
998
1250
|
const webSearchCallKeys = new Set();
|
|
999
|
-
const compactionItems = [];
|
|
1000
1251
|
const responseItemsAdded = [];
|
|
1001
1252
|
const responseItemKeys = new Set();
|
|
1002
1253
|
const citations = [];
|
|
@@ -1006,7 +1257,7 @@ export async function _streamResponse({
|
|
|
1006
1257
|
// from response.completed.response.output). The request still includes
|
|
1007
1258
|
// `reasoning.encrypted_content` so the server keeps emitting the blobs,
|
|
1008
1259
|
// but explicit input-side replay is INTENTIONALLY OMITTED in
|
|
1009
|
-
// convertMessagesToResponsesInput (openai-oauth.mjs:233-238) —
|
|
1260
|
+
// convertMessagesToResponsesInput (openai-oauth.mjs:233-238) — openai-oauth
|
|
1010
1261
|
// rejects the same `rs_*` id twice in one handshake session_id with a
|
|
1011
1262
|
// "Duplicate item" error. Server-side conversation state already carries
|
|
1012
1263
|
// the prefix forward across the WS_IDLE_MS window. The collected
|
|
@@ -1085,10 +1336,26 @@ export async function _streamResponse({
|
|
|
1085
1336
|
for (const url of action.urls) pushCitation({ url, title: action.query || '' });
|
|
1086
1337
|
}
|
|
1087
1338
|
};
|
|
1088
|
-
const
|
|
1089
|
-
|
|
1090
|
-
if (!
|
|
1091
|
-
|
|
1339
|
+
const pushCustomToolCall = (item) => {
|
|
1340
|
+
const call = customToolCallFromResponseItem(item);
|
|
1341
|
+
if (!call || toolCalls.some((existing) => existing.id === call.id)) return;
|
|
1342
|
+
toolCalls.push(call);
|
|
1343
|
+
midState.emittedToolCall = true;
|
|
1344
|
+
try { onToolCall?.(call); } catch {}
|
|
1345
|
+
};
|
|
1346
|
+
const pushToolSearchCall = (item) => {
|
|
1347
|
+
if (!item || item.type !== 'tool_search_call') return;
|
|
1348
|
+
const callId = item.call_id || item.id || '';
|
|
1349
|
+
if (!callId || toolCalls.some((call) => call.id === callId)) return;
|
|
1350
|
+
const call = {
|
|
1351
|
+
id: callId,
|
|
1352
|
+
name: 'tool_search',
|
|
1353
|
+
arguments: parseToolSearchArgs(item.arguments),
|
|
1354
|
+
nativeType: 'tool_search_call',
|
|
1355
|
+
};
|
|
1356
|
+
toolCalls.push(call);
|
|
1357
|
+
midState.emittedToolCall = true;
|
|
1358
|
+
try { onToolCall?.(call); } catch {}
|
|
1092
1359
|
};
|
|
1093
1360
|
const logReasoningDeltaSuppression = () => {
|
|
1094
1361
|
if (!logSuppressedReasoningDeltas) return;
|
|
@@ -1128,13 +1395,13 @@ export async function _streamResponse({
|
|
|
1128
1395
|
// window is intentionally short (~25s). Once response.created (or
|
|
1129
1396
|
// any other meaningful event) arrives, the timer is cancelled and
|
|
1130
1397
|
// the longer inter-chunk inactivity watchdog takes over — silent
|
|
1131
|
-
// gaps mid-reasoning (
|
|
1398
|
+
// gaps mid-reasoning (openai-oauth spending 50s+ producing reasoning
|
|
1132
1399
|
// tokens) are normal and should not abort the turn.
|
|
1133
1400
|
const armPreStreamWatchdog = () => {
|
|
1134
1401
|
if (idleTimer) clearTimeout(idleTimer);
|
|
1135
1402
|
idleTimer = setTimeout(() => {
|
|
1136
|
-
if (process.env.
|
|
1137
|
-
process.stderr.write(`[
|
|
1403
|
+
if (process.env.MIXDOG_DEBUG_AGENT) {
|
|
1404
|
+
process.stderr.write(`[agent-trace] ws-timeout kind=first-byte afterMs=${preResponseCreatedMs}\n`);
|
|
1138
1405
|
}
|
|
1139
1406
|
traceWsTimeout('first_byte_timeout', preResponseCreatedMs);
|
|
1140
1407
|
const err = new Error(`WS stream: no first server event within ${preResponseCreatedMs}ms`);
|
|
@@ -1178,7 +1445,7 @@ export async function _streamResponse({
|
|
|
1178
1445
|
saw_response_created: midState.sawResponseCreated === true,
|
|
1179
1446
|
first_meaningful_seen: firstMeaningfulSeen === true,
|
|
1180
1447
|
};
|
|
1181
|
-
|
|
1448
|
+
appendAgentTrace({
|
|
1182
1449
|
sessionId: midState.sessionId || null,
|
|
1183
1450
|
iteration: Number.isFinite(iteration) ? iteration : null,
|
|
1184
1451
|
kind: 'ws_timeout',
|
|
@@ -1203,8 +1470,8 @@ export async function _streamResponse({
|
|
|
1203
1470
|
if (firstMeaningfulSeen || firstMeaningfulMs <= 0) return;
|
|
1204
1471
|
clearFirstMeaningfulWatchdog();
|
|
1205
1472
|
firstMeaningfulTimer = setTimeout(() => {
|
|
1206
|
-
if (process.env.
|
|
1207
|
-
process.stderr.write(`[
|
|
1473
|
+
if (process.env.MIXDOG_DEBUG_AGENT) {
|
|
1474
|
+
process.stderr.write(`[agent-trace] ws-timeout kind=first-meaningful afterMs=${firstMeaningfulMs}\n`);
|
|
1208
1475
|
}
|
|
1209
1476
|
traceWsTimeout('first_meaningful_timeout', firstMeaningfulMs);
|
|
1210
1477
|
const err = new Error(`WS stream: no meaningful output within ${firstMeaningfulMs}ms after response.created`);
|
|
@@ -1219,8 +1486,8 @@ export async function _streamResponse({
|
|
|
1219
1486
|
const resetInterChunk = () => {
|
|
1220
1487
|
if (interChunkTimer) clearTimeout(interChunkTimer);
|
|
1221
1488
|
interChunkTimer = setTimeout(() => {
|
|
1222
|
-
if (process.env.
|
|
1223
|
-
process.stderr.write(`[
|
|
1489
|
+
if (process.env.MIXDOG_DEBUG_AGENT) {
|
|
1490
|
+
process.stderr.write(`[agent-trace] ws-timeout kind=inter-chunk afterMs=${interChunkMs}\n`);
|
|
1224
1491
|
}
|
|
1225
1492
|
traceWsTimeout('inter_chunk_timeout', interChunkMs);
|
|
1226
1493
|
terminalError = new Error(`WS stream: inter-chunk inactivity for ${interChunkMs}ms`);
|
|
@@ -1237,7 +1504,7 @@ export async function _streamResponse({
|
|
|
1237
1504
|
};
|
|
1238
1505
|
// Called on every event that carries real output tokens or tool
|
|
1239
1506
|
// progress. `response.created` is only an ACK and must not count here:
|
|
1240
|
-
// a wedged
|
|
1507
|
+
// a wedged openai-oauth stream can ACK immediately and then never produce
|
|
1241
1508
|
// text/reasoning/tool deltas, holding the prompt-cache lane for the
|
|
1242
1509
|
// full inter-chunk window.
|
|
1243
1510
|
const onMeaningfulOutput = () => {
|
|
@@ -1263,19 +1530,12 @@ export async function _streamResponse({
|
|
|
1263
1530
|
const finish = () => {
|
|
1264
1531
|
logReasoningDeltaSuppression();
|
|
1265
1532
|
cleanup();
|
|
1266
|
-
if (!terminalError && midState.expectCompaction === true && compactionItems.length !== 1) {
|
|
1267
|
-
terminalError = new Error(
|
|
1268
|
-
`${errLabel} remote compaction expected exactly one compaction output item, got ${compactionItems.length}`,
|
|
1269
|
-
);
|
|
1270
|
-
}
|
|
1271
1533
|
if (terminalError) { reject(terminalError); return; }
|
|
1272
1534
|
resolve({
|
|
1273
1535
|
content,
|
|
1274
1536
|
model,
|
|
1275
1537
|
reasoningItems: reasoningItems.length ? reasoningItems : undefined,
|
|
1276
1538
|
responseItems: responseItemsAdded.length ? responseItemsAdded : undefined,
|
|
1277
|
-
compactionItem: compactionItems.length === 1 ? compactionItems[0] : undefined,
|
|
1278
|
-
compactionItems: compactionItems.length ? compactionItems : undefined,
|
|
1279
1539
|
toolCalls: toolCalls.length ? toolCalls : undefined,
|
|
1280
1540
|
citations: citations.length ? citations : undefined,
|
|
1281
1541
|
webSearchCalls: webSearchCalls.length ? webSearchCalls : undefined,
|
|
@@ -1290,7 +1550,7 @@ export async function _streamResponse({
|
|
|
1290
1550
|
messageHandler = (data) => {
|
|
1291
1551
|
resetIdle();
|
|
1292
1552
|
// Do NOT call onStreamDelta for every frame — metadata/keepalive frames
|
|
1293
|
-
// must not reset
|
|
1553
|
+
// must not reset the agent stall watchdog's lastStreamDeltaAt. Only
|
|
1294
1554
|
// meaningful output (text delta / tool call) updates that timestamp.
|
|
1295
1555
|
const text = typeof data === 'string' ? data : data.toString('utf-8');
|
|
1296
1556
|
const event = _parseEvent(text);
|
|
@@ -1321,8 +1581,8 @@ export async function _streamResponse({
|
|
|
1321
1581
|
try {
|
|
1322
1582
|
if (!_firstDeltaEmitted) {
|
|
1323
1583
|
_firstDeltaEmitted = true;
|
|
1324
|
-
if (process.env.
|
|
1325
|
-
process.stderr.write(`[
|
|
1584
|
+
if (process.env.MIXDOG_DEBUG_AGENT) {
|
|
1585
|
+
process.stderr.write(`[agent-trace] ws-first-delta sinceStreaming=${Date.now() - _streamingStart}ms\n`);
|
|
1326
1586
|
}
|
|
1327
1587
|
}
|
|
1328
1588
|
onStreamDelta?.();
|
|
@@ -1370,11 +1630,30 @@ export async function _streamResponse({
|
|
|
1370
1630
|
try { onStreamDelta?.(); } catch {}
|
|
1371
1631
|
onMeaningfulOutput();
|
|
1372
1632
|
break;
|
|
1633
|
+
case 'response.custom_tool_call_input.delta':
|
|
1634
|
+
try { onStreamDelta?.(); } catch {}
|
|
1635
|
+
onMeaningfulOutput();
|
|
1636
|
+
break;
|
|
1373
1637
|
case 'response.function_call_arguments.done': {
|
|
1374
1638
|
const itemId = event.item_id || '';
|
|
1375
1639
|
const pending = pendingCalls.get(itemId);
|
|
1640
|
+
// function_call_arguments.done is a completion signal:
|
|
1641
|
+
// empty/whitespace → no args ({}); a non-empty string that
|
|
1642
|
+
// fails JSON.parse is deterministic bad JSON. Native
|
|
1643
|
+
// convergence: surface an invalid-args MARKER (not silent
|
|
1644
|
+
// {}) so the dispatch loop returns an is_error tool_result
|
|
1645
|
+
// and the model re-issues valid JSON in the same turn.
|
|
1376
1646
|
let args = {};
|
|
1377
|
-
|
|
1647
|
+
{
|
|
1648
|
+
const _argText = typeof event.arguments === 'string' ? event.arguments : '';
|
|
1649
|
+
if (_argText.trim() !== '') {
|
|
1650
|
+
try {
|
|
1651
|
+
args = JSON.parse(_argText);
|
|
1652
|
+
} catch (err) {
|
|
1653
|
+
args = makeInvalidToolArgsMarker(_argText, err instanceof Error ? err.message : String(err));
|
|
1654
|
+
}
|
|
1655
|
+
}
|
|
1656
|
+
}
|
|
1378
1657
|
enrichFunctionCallResponseItem({
|
|
1379
1658
|
itemId,
|
|
1380
1659
|
callId: pending?.callId || event.call_id || '',
|
|
@@ -1413,12 +1692,16 @@ export async function _streamResponse({
|
|
|
1413
1692
|
// function_call / output_text already captured via their
|
|
1414
1693
|
// dedicated streaming events. The one shape we still need
|
|
1415
1694
|
// here is `reasoning` — carries encrypted_content that
|
|
1416
|
-
// must be replayed on the next input to keep the
|
|
1695
|
+
// must be replayed on the next input to keep the openai-oauth
|
|
1417
1696
|
// server-side prompt cache prefix warm.
|
|
1418
1697
|
if (event.item?.type === 'reasoning') pushReasoningItem(event.item);
|
|
1419
1698
|
if (event.item?.type === 'web_search_call') pushWebSearchCall(event.item);
|
|
1420
|
-
if (
|
|
1421
|
-
|
|
1699
|
+
if (event.item?.type === 'tool_search_call') {
|
|
1700
|
+
pushToolSearchCall(event.item);
|
|
1701
|
+
onMeaningfulOutput();
|
|
1702
|
+
}
|
|
1703
|
+
if (event.item?.type === 'custom_tool_call') {
|
|
1704
|
+
pushCustomToolCall(event.item);
|
|
1422
1705
|
onMeaningfulOutput();
|
|
1423
1706
|
}
|
|
1424
1707
|
break;
|
|
@@ -1434,7 +1717,7 @@ export async function _streamResponse({
|
|
|
1434
1717
|
inputTokens: u.input_tokens || 0,
|
|
1435
1718
|
outputTokens: u.output_tokens || 0,
|
|
1436
1719
|
cachedTokens: extractCachedTokens(u),
|
|
1437
|
-
//
|
|
1720
|
+
// openai-oauth reports input_tokens as the total
|
|
1438
1721
|
// prompt volume (cached portion is a subset, not
|
|
1439
1722
|
// additive). Alias into the cross-provider
|
|
1440
1723
|
// `promptTokens` field so downstream loggers have
|
|
@@ -1462,9 +1745,8 @@ export async function _streamResponse({
|
|
|
1462
1745
|
}
|
|
1463
1746
|
}
|
|
1464
1747
|
if (item.type === 'web_search_call') pushWebSearchCall(item);
|
|
1465
|
-
if (
|
|
1466
|
-
|
|
1467
|
-
}
|
|
1748
|
+
if (item.type === 'tool_search_call') pushToolSearchCall(item);
|
|
1749
|
+
if (item.type === 'custom_tool_call') pushCustomToolCall(item);
|
|
1468
1750
|
// Salvage path: some streams emit reasoning only
|
|
1469
1751
|
// inside the final response.completed.output
|
|
1470
1752
|
// bundle (no per-item .done event). Dedup by id.
|
|
@@ -1514,7 +1796,7 @@ export async function _streamResponse({
|
|
|
1514
1796
|
break;
|
|
1515
1797
|
}
|
|
1516
1798
|
case 'response.done': {
|
|
1517
|
-
// response.done is the terminal frame for some
|
|
1799
|
+
// response.done is the terminal frame for some openai-oauth
|
|
1518
1800
|
// streams that never emit a separate response.completed.
|
|
1519
1801
|
// Route through the same completed/failed/incomplete
|
|
1520
1802
|
// normalization based on event.response.status so a
|
|
@@ -1638,7 +1920,7 @@ export async function _streamResponse({
|
|
|
1638
1920
|
const r = reason?.toString?.('utf-8') || '';
|
|
1639
1921
|
const httpStatus = _httpStatusFromWsClose(code, r);
|
|
1640
1922
|
terminalError = Object.assign(
|
|
1641
|
-
new Error(`
|
|
1923
|
+
new Error(`OpenAI OAuth WS closed before response.completed (code=${code}${r ? `, reason=${r}` : ''})`),
|
|
1642
1924
|
{ wsCloseCode: code, wsCloseReason: r, ...(httpStatus ? { httpStatus } : {}) },
|
|
1643
1925
|
);
|
|
1644
1926
|
} else if (terminalError && !terminalError.wsCloseCode) {
|
|
@@ -1674,15 +1956,15 @@ export async function _streamResponse({
|
|
|
1674
1956
|
abortHandler = () => {
|
|
1675
1957
|
if (done) return;
|
|
1676
1958
|
const reason = externalSignal.reason;
|
|
1677
|
-
terminalError = reason instanceof Error ? reason : new Error('
|
|
1959
|
+
terminalError = reason instanceof Error ? reason : new Error('OpenAI OAuth WS aborted by session close');
|
|
1678
1960
|
// Tag: was this a user/caller abort, or a watchdog abort?
|
|
1679
1961
|
// Mid-stream retry must skip user aborts but may retry watchdog
|
|
1680
1962
|
// aborts. The caller-owned AbortController surfaces through
|
|
1681
|
-
// externalSignal;
|
|
1682
|
-
// object whose name === '
|
|
1963
|
+
// externalSignal; the agent stall watchdog signals via a reason
|
|
1964
|
+
// object whose name === 'AgentStallAbortError'. stream-watchdog
|
|
1683
1965
|
// uses StreamStalledAbortError. Anything else → treat as user.
|
|
1684
1966
|
const reasonName = reason?.name || '';
|
|
1685
|
-
if (reasonName === '
|
|
1967
|
+
if (reasonName === 'AgentStallAbortError'
|
|
1686
1968
|
|| reasonName === 'StreamStalledAbortError') {
|
|
1687
1969
|
midState.watchdogAbort = reasonName;
|
|
1688
1970
|
} else {
|
|
@@ -1698,10 +1980,10 @@ export async function _streamResponse({
|
|
|
1698
1980
|
socket.on('close', closeHandler);
|
|
1699
1981
|
socket.on('error', errorHandler);
|
|
1700
1982
|
armPreStreamWatchdog();
|
|
1701
|
-
// Periodic client-side WS ping while the stream is active.
|
|
1983
|
+
// Periodic client-side WS ping while the stream is active. The server's
|
|
1702
1984
|
// server closes with 1011 "keepalive ping timeout" when it thinks the
|
|
1703
1985
|
// peer is silent during long reasoning windows where no data frames
|
|
1704
|
-
// flow. Sending a ping every
|
|
1986
|
+
// flow. Sending a ping every 10s from our side keeps the socket warm.
|
|
1705
1987
|
// The interval is unref'd so it never holds the event loop open, and
|
|
1706
1988
|
// cleanup() clears it on every terminal path (completed / close /
|
|
1707
1989
|
// error / abort / mid-stream retry teardown).
|
|
@@ -1710,7 +1992,7 @@ export async function _streamResponse({
|
|
|
1710
1992
|
if (socket.readyState !== WebSocket.OPEN) return;
|
|
1711
1993
|
socket.ping();
|
|
1712
1994
|
} catch {}
|
|
1713
|
-
},
|
|
1995
|
+
}, 10_000);
|
|
1714
1996
|
try { keepaliveTimer.unref?.(); } catch {}
|
|
1715
1997
|
});
|
|
1716
1998
|
}
|
|
@@ -1732,34 +2014,12 @@ export async function _streamResponse({
|
|
|
1732
2014
|
* 'http_5xx' (with specific status e.g. 'http_503') — server overload
|
|
1733
2015
|
* null — not retryable
|
|
1734
2016
|
*/
|
|
2017
|
+
// Thin re-export wrapper: handshake classification now lives in the shared
|
|
2018
|
+
// retry-classifier (classifyHandshakeError). Kept here as a named export so
|
|
2019
|
+
// internal call sites (_acquireWithRetry) and any external importer keep
|
|
2020
|
+
// resolving the same symbol.
|
|
1735
2021
|
export function _classifyHandshakeError(err) {
|
|
1736
|
-
|
|
1737
|
-
const code = err.code || '';
|
|
1738
|
-
const msg = String(err.message || '');
|
|
1739
|
-
const status = Number(err.httpStatus || 0);
|
|
1740
|
-
|
|
1741
|
-
// Permanent HTTP (auth / quota / not-found) short-circuits.
|
|
1742
|
-
if (status === 401 || status === 403 || status === 404 || status === 429) {
|
|
1743
|
-
return null;
|
|
1744
|
-
}
|
|
1745
|
-
// 5xx transient.
|
|
1746
|
-
if (status >= 500 && status < 600) {
|
|
1747
|
-
return `http_${status}`;
|
|
1748
|
-
}
|
|
1749
|
-
|
|
1750
|
-
// Node errno codes.
|
|
1751
|
-
if (code === 'ECONNRESET') return 'reset';
|
|
1752
|
-
if (code === 'EAI_AGAIN' || code === 'ENOTFOUND' || code === 'EAI_NODATA') return 'dns';
|
|
1753
|
-
if (code === 'ECONNREFUSED') return 'refused';
|
|
1754
|
-
if (code === 'ETIMEDOUT' || code === 'ESOCKETTIMEDOUT') return 'timeout';
|
|
1755
|
-
if (code === 'EWSACQUIRETIMEOUT') return 'acquire_timeout';
|
|
1756
|
-
if (code === 'ENETUNREACH' || code === 'EHOSTUNREACH' || code === 'EPIPE') return 'network';
|
|
1757
|
-
|
|
1758
|
-
// `ws` library's handshake-timeout path: thrown as a bare Error.
|
|
1759
|
-
if (/opening handshake has timed out/i.test(msg)) return 'timeout';
|
|
1760
|
-
if (/socket hang up/i.test(msg)) return 'reset';
|
|
1761
|
-
|
|
1762
|
-
return null;
|
|
2022
|
+
return classifyHandshakeError(err);
|
|
1763
2023
|
}
|
|
1764
2024
|
|
|
1765
2025
|
/**
|
|
@@ -1771,7 +2031,7 @@ export function _classifyHandshakeError(err) {
|
|
|
1771
2031
|
* after completion would replay a finished turn.
|
|
1772
2032
|
*
|
|
1773
2033
|
* Retry buckets:
|
|
1774
|
-
* '
|
|
2034
|
+
* 'agent_stall' — AgentStallAbortError from agent stall watchdog
|
|
1775
2035
|
* 'stream_stalled' — StreamStalledAbortError from stream-watchdog
|
|
1776
2036
|
* 'ws_1006' — abnormal close (connection lost)
|
|
1777
2037
|
* 'ws_1011' — server unexpected condition
|
|
@@ -1800,130 +2060,28 @@ export function _classifyHandshakeError(err) {
|
|
|
1800
2060
|
* - HTTP 401 / 403 / 429 surfaced on the error
|
|
1801
2061
|
* - state.attemptIndex has reached the classifier-specific retry budget
|
|
1802
2062
|
*/
|
|
2063
|
+
// Thin wrapper: the full WS mid-stream decision tree now lives in the shared
|
|
2064
|
+
// classifyMidstreamError (retry-classifier.mjs, policy.mode='ws'). Kept as a
|
|
2065
|
+
// named export so internal call sites and any external importer keep resolving
|
|
2066
|
+
// the same symbol. Behavior is byte-identical — the shared function is the
|
|
2067
|
+
// relocated original, with the per-classifier budget gating supplied by
|
|
2068
|
+
// WS_MIDSTREAM_POLICY (transientCloseRetries=4, defaultRetries=2).
|
|
1803
2069
|
export function _classifyMidstreamError(err, state) {
|
|
1804
|
-
|
|
1805
|
-
const attemptIndex = state.attemptIndex | 0;
|
|
1806
|
-
// Already completed (shouldn't throw, but defensive).
|
|
1807
|
-
if (state.sawCompleted) return null;
|
|
1808
|
-
// Any tool call already surfaced to the caller — retrying would
|
|
1809
|
-
// normally duplicate the side effect. EXCEPTION: ws_1000 truncation
|
|
1810
|
-
// (server-side normal close after response.created, before completion)
|
|
1811
|
-
// leaves the caller with an orphaned tool_use that the next turn cannot
|
|
1812
|
-
// pair to a tool_result, which the provider rejects with a hard 400.
|
|
1813
|
-
// The duplicate-side-effect risk is preferable to deterministic worker
|
|
1814
|
-
// death, especially for detached bridges that re-dispatch idempotently.
|
|
1815
|
-
if (state.emittedToolCall) {
|
|
1816
|
-
const _cc = Number(err?.wsCloseCode || state.wsCloseCode || 0);
|
|
1817
|
-
if (!(_cc === 1000 && state.sawResponseCreated && !state.sawCompleted)) return null;
|
|
1818
|
-
}
|
|
1819
|
-
// Live-text invariant: once a non-empty text chunk has been relayed to the
|
|
1820
|
-
// client (gateway live mode), the rendered output cannot be withdrawn and a
|
|
1821
|
-
// retry would concatenate a second attempt. Treat every subsequent
|
|
1822
|
-
// mid-stream/truncated failure as final — never retry.
|
|
1823
|
-
if (state.emittedText || err?.liveTextEmitted) return null;
|
|
1824
|
-
// Post-upgrade-no-first-event: the socket opened, our response.create
|
|
1825
|
-
// frame was sent, but the server never emitted a single event before
|
|
1826
|
-
// the short pre-`response.created` watchdog fired. The handshake retry
|
|
1827
|
-
// layer only sees pre-upgrade failures and the legacy pre-stream gate
|
|
1828
|
-
// below would deny this case (sawResponseCreated === false). Tag it
|
|
1829
|
-
// here as a fast retryable bucket so the worker reconnects within
|
|
1830
|
-
// seconds instead of stalling for the full first-meaningful window.
|
|
1831
|
-
if (state.firstByteTimeout || err?.firstByteTimeout) {
|
|
1832
|
-
return _allowMidstreamRetry('first_byte_timeout', attemptIndex);
|
|
1833
|
-
}
|
|
1834
|
-
if (state.firstMeaningfulTimeout || err?.firstMeaningfulTimeout) {
|
|
1835
|
-
return _allowMidstreamRetry('first_meaningful_timeout', attemptIndex);
|
|
1836
|
-
}
|
|
1837
|
-
// _sendFrame failure (socket not OPEN, send callback errored, JSON
|
|
1838
|
-
// serialize threw). Always retryable: caller will forceFresh next
|
|
1839
|
-
// attempt so the wedged socket is dropped.
|
|
1840
|
-
if (err?.wsSendFailed || state.wsSendFailed) {
|
|
1841
|
-
return _allowMidstreamRetry('ws_send_failed', attemptIndex);
|
|
1842
|
-
}
|
|
1843
|
-
// Pre-stream failures normally belong to the handshake retry layer. BUT
|
|
1844
|
-
// WS close 1011 / 1012 can fire after the 101 upgrade but BEFORE the
|
|
1845
|
-
// first response.created event when the server's keepalive times out or
|
|
1846
|
-
// the service restarts. Neither the handshake retry layer (it only sees
|
|
1847
|
-
// pre-upgrade failures) nor the existing mid-stream gate covers this
|
|
1848
|
-
// window, so permit bounded retry here for those two codes only.
|
|
1849
|
-
if (!state.sawResponseCreated) {
|
|
1850
|
-
const closeCode = Number(err?.wsCloseCode || state.wsCloseCode || 0);
|
|
1851
|
-
if (closeCode !== 1011 && closeCode !== 1012) return null;
|
|
1852
|
-
}
|
|
1853
|
-
// User/caller abort — never retry.
|
|
1854
|
-
if (state.userAbort) return null;
|
|
1855
|
-
|
|
1856
|
-
if (!err) return null;
|
|
1857
|
-
const status = Number(err?.httpStatus || 0);
|
|
1858
|
-
if (status === 401 || status === 403 || status === 429) return null;
|
|
1859
|
-
// Transient 5xx surfaced via populateHttpStatusFromMessage (case 'error'
|
|
1860
|
-
// and case 'response.failed' branches sniff server-supplied text like
|
|
1861
|
-
// "Our servers are currently overloaded" and assign httpStatus=503).
|
|
1862
|
-
// Allow one bounded mid-stream retry on the same budget as the WS close-
|
|
1863
|
-
// code buckets above so server-side overload no longer leaks straight
|
|
1864
|
-
// to the caller without a single retry attempt.
|
|
1865
|
-
if (status >= 500 && status < 600) {
|
|
1866
|
-
return _allowMidstreamRetry(`http_${status}`, attemptIndex);
|
|
1867
|
-
}
|
|
1868
|
-
|
|
1869
|
-
const name = err?.name || '';
|
|
1870
|
-
if (name === 'BridgeStallAbortError') return _allowMidstreamRetry('bridge_stall', attemptIndex);
|
|
1871
|
-
if (name === 'StreamStalledAbortError') return _allowMidstreamRetry('stream_stalled', attemptIndex);
|
|
1872
|
-
|
|
1873
|
-
// Watchdog abort surfaced via externalSignal handler → err is the reason
|
|
1874
|
-
// itself. state.watchdogAbort captures the class name when the error
|
|
1875
|
-
// shape was preserved but the name was stripped by some wrapper.
|
|
1876
|
-
if (state.watchdogAbort === 'BridgeStallAbortError') return _allowMidstreamRetry('bridge_stall', attemptIndex);
|
|
1877
|
-
if (state.watchdogAbort === 'StreamStalledAbortError') return _allowMidstreamRetry('stream_stalled', attemptIndex);
|
|
1878
|
-
|
|
1879
|
-
// WS close codes: prefer the decorated property, fall back to state.
|
|
1880
|
-
const closeCode = Number(err?.wsCloseCode || state.wsCloseCode || 0);
|
|
1881
|
-
if (closeCode === 1006) return _allowMidstreamRetry('ws_1006', attemptIndex);
|
|
1882
|
-
if (closeCode === 1011) return _allowMidstreamRetry('ws_1011', attemptIndex);
|
|
1883
|
-
if (closeCode === 1012) return _allowMidstreamRetry('ws_1012', attemptIndex);
|
|
1884
|
-
// Private 4xxx codes from a server/proxy are auth/policy/application closes;
|
|
1885
|
-
// never treat them as transient. 4000 is our local pre-stream watchdog code.
|
|
1886
|
-
if (closeCode >= 4000 && closeCode < 5000 && closeCode !== 4000) return null;
|
|
1887
|
-
if (closeCode === 4000) return _allowMidstreamRetry('ws_4000', attemptIndex);
|
|
1888
|
-
// Server-side normal close (1000) AFTER response.created but BEFORE
|
|
1889
|
-
// response.completed = truncated stream; legitimate transient. The
|
|
1890
|
-
// pre-stream gate above already rejects 1000 before sawResponseCreated
|
|
1891
|
-
// (handshake retry layer owns that window).
|
|
1892
|
-
if (closeCode === 1000 && state.sawResponseCreated && !state.sawCompleted) return _allowMidstreamRetry('ws_1000', attemptIndex);
|
|
1893
|
-
|
|
1894
|
-
// response.failed payload mentioning network_error / stream_disconnected.
|
|
1895
|
-
// xAI's gRPC backend periodically rotates auth context (server-side TTL)
|
|
1896
|
-
// and surfaces "Auth context expired" as a response.failed event. The
|
|
1897
|
-
// attemptIndex > 0 path in sendViaWebSocket forces a fresh WS handshake,
|
|
1898
|
-
// which re-authenticates — so a single bounded retry recovers the turn
|
|
1899
|
-
// instead of letting the worker die mid-session.
|
|
1900
|
-
const failed = err?.responseFailed || state.responseFailedPayload;
|
|
1901
|
-
if (failed) {
|
|
1902
|
-
try {
|
|
1903
|
-
const blob = JSON.stringify(failed).toLowerCase();
|
|
1904
|
-
if (blob.includes('stream_disconnected')) return _allowMidstreamRetry('response_failed_disconnected', attemptIndex);
|
|
1905
|
-
if (blob.includes('network_error')) return _allowMidstreamRetry('response_failed_network', attemptIndex);
|
|
1906
|
-
if (blob.includes('auth context expired')) return _allowMidstreamRetry('response_failed_auth_expired', attemptIndex);
|
|
1907
|
-
} catch {}
|
|
1908
|
-
}
|
|
1909
|
-
|
|
1910
|
-
// Unknown → default-deny (don't risk a second full-cost turn for an error
|
|
1911
|
-
// class we haven't proven is transient).
|
|
1912
|
-
return null;
|
|
2070
|
+
return classifyMidstreamError(err, state, WS_MIDSTREAM_POLICY);
|
|
1913
2071
|
}
|
|
1914
2072
|
|
|
2073
|
+
// Per-classifier retry budget, used by the sendViaWebSocket loop to bound the
|
|
2074
|
+
// attempt count once classifyMidstreamError returns a bucket. Mirrors the
|
|
2075
|
+
// shared _midstreamLimitFor(ws) — the numbers come from MIDSTREAM_RETRY_POLICY.
|
|
1915
2076
|
function _midstreamRetryLimit(classifier) {
|
|
1916
2077
|
return classifier === 'ws_1006' || classifier === 'ws_1011'
|
|
1917
2078
|
? MIDSTREAM_WS_TRANSIENT_RETRY_LIMIT
|
|
1918
2079
|
: MIDSTREAM_DEFAULT_RETRY_LIMIT;
|
|
1919
2080
|
}
|
|
1920
2081
|
|
|
1921
|
-
function _allowMidstreamRetry(classifier, attemptIndex) {
|
|
1922
|
-
return attemptIndex < _midstreamRetryLimit(classifier) ? classifier : null;
|
|
1923
|
-
}
|
|
1924
|
-
|
|
1925
2082
|
function _midstreamBackoffFor(retryNumber) {
|
|
1926
|
-
|
|
2083
|
+
const raw = MIDSTREAM_BACKOFF_MS[Math.min(Math.max(retryNumber, 1), MIDSTREAM_BACKOFF_MS.length) - 1];
|
|
2084
|
+
return jitterDelayMs(raw);
|
|
1927
2085
|
}
|
|
1928
2086
|
|
|
1929
2087
|
function _backoffFor(attempt) {
|
|
@@ -1934,25 +2092,11 @@ function _backoffFor(attempt) {
|
|
|
1934
2092
|
|
|
1935
2093
|
const _defaultSleep = (ms) => new Promise((r) => setTimeout(r, ms));
|
|
1936
2094
|
|
|
1937
|
-
|
|
1938
|
-
|
|
1939
|
-
|
|
1940
|
-
|
|
1941
|
-
|
|
1942
|
-
}
|
|
1943
|
-
await new Promise((resolve, reject) => {
|
|
1944
|
-
const t = setTimeout(() => {
|
|
1945
|
-
externalSignal.removeEventListener('abort', onAbort);
|
|
1946
|
-
resolve();
|
|
1947
|
-
}, ms);
|
|
1948
|
-
const onAbort = () => {
|
|
1949
|
-
clearTimeout(t);
|
|
1950
|
-
const reason = externalSignal.reason;
|
|
1951
|
-
reject(reason instanceof Error ? reason : new Error('Codex WS retry backoff aborted'));
|
|
1952
|
-
};
|
|
1953
|
-
if (externalSignal.aborted) { onAbort(); return; }
|
|
1954
|
-
externalSignal.addEventListener('abort', onAbort, { once: true });
|
|
1955
|
-
});
|
|
2095
|
+
// Abort-aware backoff sleep → shared sleepWithAbort (retry-classifier.mjs). The
|
|
2096
|
+
// abortMessage preserves the prior fallback text when the abort reason is not an
|
|
2097
|
+
// Error; _sleepFn (test seam) is threaded through as the no-signal sleep impl.
|
|
2098
|
+
function _sleepWithAbort(ms, externalSignal, sleepFn = _defaultSleep) {
|
|
2099
|
+
return sleepWithAbort(ms, externalSignal, sleepFn, 'OpenAI OAuth WS retry backoff aborted');
|
|
1956
2100
|
}
|
|
1957
2101
|
|
|
1958
2102
|
/**
|
|
@@ -1980,12 +2124,12 @@ export async function _acquireWithRetry({
|
|
|
1980
2124
|
for (let attempt = 1; attempt <= HANDSHAKE_MAX_ATTEMPTS; attempt++) {
|
|
1981
2125
|
if (externalSignal?.aborted) {
|
|
1982
2126
|
const reason = externalSignal.reason;
|
|
1983
|
-
throw reason instanceof Error ? reason : new Error('
|
|
2127
|
+
throw reason instanceof Error ? reason : new Error('OpenAI OAuth WS acquire aborted');
|
|
1984
2128
|
}
|
|
1985
2129
|
try {
|
|
1986
2130
|
if (attempt > 1) {
|
|
1987
|
-
if (process.env.
|
|
1988
|
-
process.stderr.write(`[
|
|
2131
|
+
if (process.env.MIXDOG_DEBUG_AGENT) {
|
|
2132
|
+
process.stderr.write(`[agent-trace] ws-handshake-attempt n=${attempt}\n`);
|
|
1989
2133
|
}
|
|
1990
2134
|
}
|
|
1991
2135
|
return await _acquire({ auth, poolKey, cacheKey, forceFresh, externalSignal });
|
|
@@ -2041,7 +2185,7 @@ export async function _acquireWithRetry({
|
|
|
2041
2185
|
const onAbort = () => {
|
|
2042
2186
|
clearTimeout(t);
|
|
2043
2187
|
const reason = externalSignal.reason;
|
|
2044
|
-
reject(reason instanceof Error ? reason : new Error('
|
|
2188
|
+
reject(reason instanceof Error ? reason : new Error('OpenAI OAuth WS acquire aborted'));
|
|
2045
2189
|
};
|
|
2046
2190
|
if (externalSignal.aborted) { onAbort(); return; }
|
|
2047
2191
|
externalSignal.addEventListener('abort', onAbort, { once: true });
|
|
@@ -2113,20 +2257,21 @@ export async function sendViaWebSocket({
|
|
|
2113
2257
|
// already-rendered output. A text-emitting attempt is never retry-eligible
|
|
2114
2258
|
// (_classifyMidstreamError returns null on emittedText), so the surfaced
|
|
2115
2259
|
// error is frequently an EARLIER attempt's firstAttemptError that never saw
|
|
2116
|
-
// the marker;
|
|
2117
|
-
|
|
2118
|
-
|
|
2119
|
-
|
|
2120
|
-
|
|
2121
|
-
|
|
2122
|
-
|
|
2123
|
-
|
|
2260
|
+
// the marker; stampText re-applies it on every throw path. The latch state
|
|
2261
|
+
// + stamp semantics now come from the shared createStreamSafetyStamps()
|
|
2262
|
+
// factory (retry-classifier.mjs) — identical to the former _stampLiveText /
|
|
2263
|
+
// _stampTool closures. markText()/markTool() set the latch (replacing the
|
|
2264
|
+
// liveTextEmittedAcrossAttempts / toolEmittedAcrossAttempts booleans);
|
|
2265
|
+
// stampText/stampTool re-apply the markers on every throw.
|
|
2266
|
+
const _safetyStamps = createStreamSafetyStamps();
|
|
2267
|
+
const _stampLiveText = _safetyStamps.stampText;
|
|
2268
|
+
const _stampTool = _safetyStamps.stampTool;
|
|
2124
2269
|
// Server-side xAI conversation anchor preserved across mid-stream
|
|
2125
2270
|
// retries. xAI keys its conversation by previous_response_id alone
|
|
2126
2271
|
// (sessionToken is null for xAI in _mintSessionToken); a forceFresh
|
|
2127
2272
|
// socket on retry would otherwise drop prev_id and cold-start a new
|
|
2128
2273
|
// server-side conversation, evicting every prefix the prior attempts
|
|
2129
|
-
// warmed.
|
|
2274
|
+
// warmed. openai-oauth / openai-direct anchor by per-socket session_id, where
|
|
2130
2275
|
// this carry-forward would not help and is therefore gated to xAI.
|
|
2131
2276
|
let carryForwardCache = null;
|
|
2132
2277
|
const emittedProgress = [];
|
|
@@ -2166,7 +2311,7 @@ export async function sendViaWebSocket({
|
|
|
2166
2311
|
const classifiers = [...handshakeRetryClassifiers];
|
|
2167
2312
|
if (classifier && !classifiers.includes(classifier)) classifiers.push(classifier);
|
|
2168
2313
|
if (err?.httpStatus != null || classifier || handshakeRetries > 0 || classifiers.length > 0) {
|
|
2169
|
-
|
|
2314
|
+
traceAgentFetch({
|
|
2170
2315
|
sessionId: poolKey,
|
|
2171
2316
|
headersMs: Date.now() - handshakeStart,
|
|
2172
2317
|
httpStatus: Number(err?.httpStatus || 0),
|
|
@@ -2182,9 +2327,9 @@ export async function sendViaWebSocket({
|
|
|
2182
2327
|
// the caller's turn actually tripped on).
|
|
2183
2328
|
if (attemptIndex > 0 && firstAttemptError) {
|
|
2184
2329
|
try { firstAttemptError.midstreamRetries = attemptIndex; } catch {}
|
|
2185
|
-
throw _stampLiveText(firstAttemptError);
|
|
2330
|
+
throw _stampTool(_stampLiveText(firstAttemptError));
|
|
2186
2331
|
}
|
|
2187
|
-
throw _stampLiveText(err);
|
|
2332
|
+
throw _stampTool(_stampLiveText(err));
|
|
2188
2333
|
}
|
|
2189
2334
|
const { entry, reused } = acquired;
|
|
2190
2335
|
// Re-seed the retry attempt's fresh entry with the prior attempt's
|
|
@@ -2199,7 +2344,7 @@ export async function sendViaWebSocket({
|
|
|
2199
2344
|
entry.lastRequestInput = carryForwardCache.lastRequestInput;
|
|
2200
2345
|
entry.lastResponseItems = carryForwardCache.lastResponseItems;
|
|
2201
2346
|
}
|
|
2202
|
-
|
|
2347
|
+
traceAgentFetch({
|
|
2203
2348
|
sessionId: poolKey,
|
|
2204
2349
|
headersMs: Date.now() - handshakeStart,
|
|
2205
2350
|
httpStatus: reused ? 0 : 101,
|
|
@@ -2245,14 +2390,15 @@ export async function sendViaWebSocket({
|
|
|
2245
2390
|
let strippedResponseItems = 0;
|
|
2246
2391
|
let skippedResponseItems = 0;
|
|
2247
2392
|
let result;
|
|
2248
|
-
const streamTimeouts =
|
|
2249
|
-
? {
|
|
2250
|
-
firstMeaningfulMs: WS_INTER_CHUNK_MS,
|
|
2251
|
-
interChunkMs: WS_INTER_CHUNK_MS,
|
|
2252
|
-
}
|
|
2253
|
-
: null;
|
|
2393
|
+
const streamTimeouts = null;
|
|
2254
2394
|
try {
|
|
2255
2395
|
if (warmupBody && typeof warmupBody === 'object' && attemptIndex === 0) {
|
|
2396
|
+
await promptCacheLane?.reserveRate?.({
|
|
2397
|
+
mode: 'full',
|
|
2398
|
+
frameInputItems: Array.isArray(warmupBody.input) ? warmupBody.input.length : null,
|
|
2399
|
+
deltaTokens: _estimateFrameTokens({ type: 'response.create', ...warmupBody }),
|
|
2400
|
+
hasPreviousResponseId: false,
|
|
2401
|
+
});
|
|
2256
2402
|
const warmupFrame = { type: 'response.create', ...warmupBody };
|
|
2257
2403
|
await _sendFrameFn(entry, warmupFrame);
|
|
2258
2404
|
const warmupStart = Date.now();
|
|
@@ -2306,7 +2452,7 @@ export async function sendViaWebSocket({
|
|
|
2306
2452
|
output_tokens: warmupResult.usage?.outputTokens || 0,
|
|
2307
2453
|
prompt_tokens: warmupResult.usage?.promptTokens || 0,
|
|
2308
2454
|
};
|
|
2309
|
-
|
|
2455
|
+
appendAgentTrace({
|
|
2310
2456
|
sessionId: poolKey,
|
|
2311
2457
|
iteration,
|
|
2312
2458
|
kind: 'cache_warmup',
|
|
@@ -2329,6 +2475,12 @@ export async function sendViaWebSocket({
|
|
|
2329
2475
|
strippedResponseItems = delta.strippedResponseItems || 0;
|
|
2330
2476
|
skippedResponseItems = delta.skippedResponseItems || 0;
|
|
2331
2477
|
deltaTokens = _estimateFrameTokens(frame);
|
|
2478
|
+
await promptCacheLane?.reserveRate?.({
|
|
2479
|
+
mode,
|
|
2480
|
+
frameInputItems: Array.isArray(frame.input) ? frame.input.length : null,
|
|
2481
|
+
deltaTokens,
|
|
2482
|
+
hasPreviousResponseId: typeof frame.previous_response_id === 'string' && frame.previous_response_id.length > 0,
|
|
2483
|
+
});
|
|
2332
2484
|
|
|
2333
2485
|
// Re-check abort after acquire/warmup — narrow window where
|
|
2334
2486
|
// externalSignal could fire between successful acquire and
|
|
@@ -2343,8 +2495,8 @@ export async function sendViaWebSocket({
|
|
|
2343
2495
|
}
|
|
2344
2496
|
await _sendFrameFn(entry, frame);
|
|
2345
2497
|
|
|
2346
|
-
if (process.env.
|
|
2347
|
-
process.stderr.write(`[
|
|
2498
|
+
if (process.env.MIXDOG_DEBUG_AGENT) {
|
|
2499
|
+
process.stderr.write(`[agent-trace] ws-streaming-start sinceAcquire=${Date.now() - handshakeStart}ms\n`);
|
|
2348
2500
|
}
|
|
2349
2501
|
try { onStageChange?.('streaming'); } catch {}
|
|
2350
2502
|
result = await _streamFn({
|
|
@@ -2383,9 +2535,13 @@ export async function sendViaWebSocket({
|
|
|
2383
2535
|
// Latch across attempts: even though THIS error is never
|
|
2384
2536
|
// retry-eligible once text is out, a later/earlier surfaced
|
|
2385
2537
|
// error (firstAttemptError) must still carry the marker.
|
|
2386
|
-
|
|
2538
|
+
_safetyStamps.markText();
|
|
2539
|
+
}
|
|
2540
|
+
if (midState.emittedToolCall) {
|
|
2541
|
+
_safetyStamps.markTool();
|
|
2387
2542
|
}
|
|
2388
2543
|
_stampLiveText(err);
|
|
2544
|
+
_stampTool(err);
|
|
2389
2545
|
const classifier = _classifyMidstreamError(err, midState);
|
|
2390
2546
|
const retryLimit = classifier ? _midstreamRetryLimit(classifier) : 0;
|
|
2391
2547
|
if (classifier && attemptIndex < retryLimit) {
|
|
@@ -2424,12 +2580,12 @@ export async function sendViaWebSocket({
|
|
|
2424
2580
|
firstAttemptError.suppressed = list;
|
|
2425
2581
|
}
|
|
2426
2582
|
} catch {}
|
|
2427
|
-
throw _stampLiveText(firstAttemptError);
|
|
2583
|
+
throw _stampTool(_stampLiveText(firstAttemptError));
|
|
2428
2584
|
}
|
|
2429
|
-
throw _stampLiveText(err);
|
|
2585
|
+
throw _stampTool(_stampLiveText(err));
|
|
2430
2586
|
}
|
|
2431
2587
|
const liveModel = result.model || useModel;
|
|
2432
|
-
|
|
2588
|
+
traceAgentSse({
|
|
2433
2589
|
sessionId: poolKey,
|
|
2434
2590
|
sseParseMs: Date.now() - sseStart,
|
|
2435
2591
|
provider: traceProvider,
|
|
@@ -2438,10 +2594,21 @@ export async function sendViaWebSocket({
|
|
|
2438
2594
|
});
|
|
2439
2595
|
|
|
2440
2596
|
const resultToolCallCount = Array.isArray(result.toolCalls) ? result.toolCalls.length : 0;
|
|
2441
|
-
|
|
2597
|
+
// Keep the conversation chain whenever the server gave us a response id.
|
|
2598
|
+
// `incompleteReason` is ONLY ever set for max_output_tokens-class
|
|
2599
|
+
// truncation (every other incomplete status throws upstream), and in
|
|
2600
|
+
// that case the response IS valid and the server preserves its
|
|
2601
|
+
// response_id as a continuation anchor. Dropping the chain here forced
|
|
2602
|
+
// the NEXT turn to cold-start (no_anchor → full resend), which the
|
|
2603
|
+
// trace logs showed repeating 50-78x in long max-output sessions. If a
|
|
2604
|
+
// truncated turn's response items don't line up next turn,
|
|
2605
|
+
// _stripResponseItemsFromHead still falls back to a full send on its
|
|
2606
|
+
// own, so retaining the anchor cannot corrupt the cache — it only adds
|
|
2607
|
+
// a delta fast-path when the items DO match.
|
|
2608
|
+
const keepResponseChain = !!result.responseId;
|
|
2442
2609
|
const keepSocket = true;
|
|
2443
2610
|
|
|
2444
|
-
// Update cache state for the next iteration in this session.
|
|
2611
|
+
// Update cache state for the next iteration in this session. openai-oauth
|
|
2445
2612
|
// keeps the previous response anchor even when the model emitted tool
|
|
2446
2613
|
// calls: the next request is previous input + server output items
|
|
2447
2614
|
// + tool results, and _computeDelta strips the first two parts so the
|
|
@@ -2473,7 +2640,7 @@ export async function sendViaWebSocket({
|
|
|
2473
2640
|
|
|
2474
2641
|
const requestedServiceTier = body?.service_tier || null;
|
|
2475
2642
|
const responseServiceTier = result.serviceTier || result.usage?.raw?.service_tier || null;
|
|
2476
|
-
|
|
2643
|
+
traceAgentUsage({
|
|
2477
2644
|
sessionId: poolKey,
|
|
2478
2645
|
iteration,
|
|
2479
2646
|
inputTokens: result.usage?.inputTokens || 0,
|
|
@@ -2489,6 +2656,9 @@ export async function sendViaWebSocket({
|
|
|
2489
2656
|
});
|
|
2490
2657
|
// Extra WS-specific observability: transport + per-iteration delta bytes.
|
|
2491
2658
|
try {
|
|
2659
|
+
const transportCacheKeyHash = cacheKey
|
|
2660
|
+
? createHash('sha256').update(String(cacheKey)).digest('hex').slice(0, 12)
|
|
2661
|
+
: null;
|
|
2492
2662
|
const transportPayload = {
|
|
2493
2663
|
provider: traceProvider,
|
|
2494
2664
|
transport: 'websocket',
|
|
@@ -2504,15 +2674,18 @@ export async function sendViaWebSocket({
|
|
|
2504
2674
|
handshake_retry_classifiers: handshakeRetryClassifiers,
|
|
2505
2675
|
midstream_retries: attemptIndex,
|
|
2506
2676
|
response_id: result.responseId || null,
|
|
2507
|
-
cache_key_hash:
|
|
2508
|
-
? createHash('sha256').update(String(cacheKey)).digest('hex').slice(0, 12)
|
|
2509
|
-
: null,
|
|
2677
|
+
cache_key_hash: transportCacheKeyHash,
|
|
2510
2678
|
cache_lane_enabled: promptCacheLane?.enabled === true,
|
|
2511
2679
|
cache_lane_key_hash: promptCacheLane?.laneKeyHash || null,
|
|
2680
|
+
cache_lane_rate_policy: promptCacheLane?.ratePolicy || null,
|
|
2512
2681
|
cache_lane_max_in_flight: Number.isFinite(Number(promptCacheLane?.maxInFlight)) ? Number(promptCacheLane.maxInFlight) : null,
|
|
2513
2682
|
cache_lane_rate_limit_per_min: Number.isFinite(Number(promptCacheLane?.rateLimitPerMin)) ? Number(promptCacheLane.rateLimitPerMin) : null,
|
|
2683
|
+
cache_lane_rate_full_limit_per_min: Number.isFinite(Number(promptCacheLane?.rateFullLimitPerMin)) ? Number(promptCacheLane.rateFullLimitPerMin) : null,
|
|
2684
|
+
cache_lane_rate_delta_limit_per_min: Number.isFinite(Number(promptCacheLane?.rateDeltaLimitPerMin)) ? Number(promptCacheLane.rateDeltaLimitPerMin) : null,
|
|
2514
2685
|
cache_lane_rate_wait_ms: Number.isFinite(Number(promptCacheLane?.rateWaitMs)) ? Number(promptCacheLane.rateWaitMs) : null,
|
|
2515
2686
|
cache_lane_rate_window_count: Number.isFinite(Number(promptCacheLane?.rateWindowCount)) ? Number(promptCacheLane.rateWindowCount) : null,
|
|
2687
|
+
cache_lane_rate_released_for_wait: promptCacheLane?.rateReleasedForWait === true,
|
|
2688
|
+
cache_lane_rate_reacquire_wait_ms: Number.isFinite(Number(promptCacheLane?.rateReacquireWaitMs)) ? Number(promptCacheLane.rateReacquireWaitMs) : null,
|
|
2516
2689
|
cache_lane_wait_ms: Number.isFinite(Number(promptCacheLane?.waitMs)) ? Number(promptCacheLane.waitMs) : null,
|
|
2517
2690
|
cache_lane_queued: promptCacheLane?.queued === true,
|
|
2518
2691
|
cache_lane_active: Number.isFinite(Number(promptCacheLane?.activeAfterAcquire)) ? Number(promptCacheLane.activeAfterAcquire) : null,
|
|
@@ -2531,13 +2704,40 @@ export async function sendViaWebSocket({
|
|
|
2531
2704
|
keep_socket: keepSocket,
|
|
2532
2705
|
keep_response_chain: keepResponseChain,
|
|
2533
2706
|
};
|
|
2534
|
-
|
|
2707
|
+
appendAgentTrace({
|
|
2535
2708
|
sessionId: poolKey,
|
|
2536
2709
|
iteration,
|
|
2537
2710
|
kind: 'transport',
|
|
2538
2711
|
...transportPayload,
|
|
2539
2712
|
payload: transportPayload,
|
|
2540
2713
|
});
|
|
2714
|
+
if (mode !== 'delta' || deltaReason) {
|
|
2715
|
+
appendAgentTrace({
|
|
2716
|
+
sessionId: poolKey,
|
|
2717
|
+
iteration,
|
|
2718
|
+
kind: 'cache_break',
|
|
2719
|
+
provider: traceProvider,
|
|
2720
|
+
model: liveModel,
|
|
2721
|
+
payload: {
|
|
2722
|
+
provider: traceProvider,
|
|
2723
|
+
model: liveModel,
|
|
2724
|
+
transport: 'websocket',
|
|
2725
|
+
ws_mode: mode,
|
|
2726
|
+
reason: mode === 'delta' ? deltaReason : (deltaReason || 'full_frame'),
|
|
2727
|
+
cache_key_hash: transportCacheKeyHash,
|
|
2728
|
+
cache_lane_key_hash: promptCacheLane?.laneKeyHash || null,
|
|
2729
|
+
request_has_previous_response_id: transportPayload.request_has_previous_response_id,
|
|
2730
|
+
chain_stripped_response_items: strippedResponseItems,
|
|
2731
|
+
chain_skipped_response_items: skippedResponseItems,
|
|
2732
|
+
chain_response_items: Array.isArray(result.responseItems) ? result.responseItems.length : 0,
|
|
2733
|
+
body_input_items: Array.isArray(requestBody.input) ? requestBody.input.length : null,
|
|
2734
|
+
frame_input_items: Array.isArray(frame.input) ? frame.input.length : null,
|
|
2735
|
+
frame_has_instructions: transportPayload.frame_has_instructions,
|
|
2736
|
+
keep_response_chain: keepResponseChain,
|
|
2737
|
+
tool_call_count: resultToolCallCount,
|
|
2738
|
+
},
|
|
2739
|
+
});
|
|
2740
|
+
}
|
|
2541
2741
|
} catch {}
|
|
2542
2742
|
|
|
2543
2743
|
releaseWebSocket({ entry, poolKey, keep: keepSocket });
|
|
@@ -2561,7 +2761,7 @@ export async function sendViaWebSocket({
|
|
|
2561
2761
|
return out;
|
|
2562
2762
|
}
|
|
2563
2763
|
// Unreachable — the loop either returns or throws above.
|
|
2564
|
-
throw _stampLiveText(firstAttemptError || new Error('sendViaWebSocket: unreachable'));
|
|
2764
|
+
throw _stampTool(_stampLiveText(firstAttemptError || new Error('sendViaWebSocket: unreachable')));
|
|
2565
2765
|
});
|
|
2566
2766
|
}
|
|
2567
2767
|
|