@bitkyc08/opencodex 2.65.0 → 2.66.0-preview.20260925
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +14 -0
- package/gui/dist/assets/App-BVjA0T6O.js +51 -0
- package/gui/dist/assets/App-gkoJIEOV.css +1 -0
- package/gui/dist/assets/{Tray-D4bA1vff.js → Tray-BCFtiWwD.js} +1 -1
- package/gui/dist/assets/index-ComVsdSk.css +1 -0
- package/gui/dist/assets/index-vO4dgfiU.js +86 -0
- package/gui/dist/assets/{tray-data-BZXhB3qz.js → tray-data-C4FiNaxk.js} +1 -1
- package/gui/dist/index.html +2 -2
- package/package.json +1 -2
- package/src/adapters/anthropic/beta-allowlist.ts +80 -0
- package/src/adapters/anthropic/passthrough.ts +221 -0
- package/src/adapters/anthropic-image-codec.ts +56 -19
- package/src/adapters/anthropic-image-normalize.ts +20 -10
- package/src/adapters/anthropic.ts +55 -24
- package/src/adapters/coding-agent/protocol.ts +29 -19
- package/src/adapters/cursor/live-transport.ts +18 -3
- package/src/adapters/cursor/native-exec-shell.ts +18 -63
- package/src/adapters/cursor/native-exec.ts +5 -2
- package/src/adapters/cursor/native-foreground-shell.ts +44 -0
- package/src/adapters/devin/cloud-direct/chat.ts +7 -1
- package/src/adapters/exec-tool-result-normalize.ts +4 -1
- package/src/adapters/google.ts +12 -0
- package/src/adapters/ollama-native.ts +32 -1
- package/src/adapters/openai-chat/serialized-tool-call-content.ts +131 -34
- package/src/adapters/openai-chat.ts +6 -4
- package/src/adapters/openai-responses/passthrough.ts +1 -0
- package/src/adapters/openai-responses/reasoning.ts +45 -1
- package/src/bridge/response-json.ts +10 -2
- package/src/chat/outbound.ts +104 -52
- package/src/claude/claude-code-block.ts +16 -0
- package/src/claude/context-windows.ts +52 -4
- package/src/claude/desktop-first-party.ts +25 -8
- package/src/claude/inbound.ts +22 -0
- package/src/claude/intercept/connect-proxy.ts +37 -4
- package/src/claude/intercept/proxy-auth.ts +166 -0
- package/src/claude/intercept/runtime.ts +21 -0
- package/src/claude/intercept/settings.ts +33 -12
- package/src/claude/outbound.ts +45 -27
- package/src/cli/account-extended.ts +47 -15
- package/src/cli/account-target.ts +115 -0
- package/src/cli/account.ts +20 -28
- package/src/cli/agent.ts +10 -1
- package/src/cli/api-protocols.ts +206 -0
- package/src/cli/capabilities.ts +86 -0
- package/src/cli/combo.ts +2 -2
- package/src/cli/connect.ts +127 -3
- package/src/cli/dispatch.ts +8 -0
- package/src/cli/doctor.ts +33 -0
- package/src/cli/help.ts +2 -0
- package/src/cli/link.ts +288 -0
- package/src/cli/registry.ts +23 -0
- package/src/cli/runtime-api.ts +79 -0
- package/src/cli/system-command.ts +21 -2
- package/src/client/connect.ts +125 -23
- package/src/client/hub-relay.ts +12 -8
- package/src/client/link-join.ts +323 -0
- package/src/client/link-relay.ts +241 -0
- package/src/client/link-state.ts +110 -0
- package/src/client/link-teardown.ts +63 -0
- package/src/client/link-tunnel.ts +465 -0
- package/src/client/machine-listener.ts +14 -5
- package/src/client/runtime.ts +91 -42
- package/src/client/state.ts +4 -0
- package/src/codex/catalog/build-entries.ts +4 -0
- package/src/codex/catalog/derive-entry.ts +5 -0
- package/src/codex/catalog/parsing.ts +4 -0
- package/src/codex/catalog/retained-sync.ts +4 -1
- package/src/codex/desktop-switches.ts +88 -8
- package/src/codex/inject/plan.ts +16 -3
- package/src/codex/inject.ts +3 -0
- package/src/codex/internal/catalog-writer.ts +13 -2
- package/src/combos/failover.ts +3 -2
- package/src/combos/index.ts +12 -0
- package/src/combos/jev.ts +646 -0
- package/src/combos/request.ts +5 -0
- package/src/combos/resolve.ts +17 -15
- package/src/combos/types.ts +47 -3
- package/src/config/atomic-write.ts +3 -0
- package/src/config/live-reconcile.ts +20 -4
- package/src/config/persist-unlocked.ts +20 -7
- package/src/config/schema/config-schema.ts +16 -0
- package/src/config/schema/leaf-validators.ts +38 -1
- package/src/generated/compatibility-version.json +416 -128
- package/src/integrations/omp-yaml-source.ts +109 -4
- package/src/lab/automation/orchestrator.ts +60 -4
- package/src/lab/events/limits.ts +1 -1
- package/src/lib/bounded-body.ts +55 -38
- package/src/lib/config-ownership.ts +11 -22
- package/src/lib/local-desktop-snapshot-capability.ts +56 -0
- package/src/lib/local-management-capability.ts +25 -3
- package/src/lib/optional-shutdown-hooks.ts +17 -0
- package/src/lib/state-store-registrations.ts +2 -2
- package/src/lib/windows-elevation.ts +62 -25
- package/src/lib/windows-secret-acl.ts +114 -23
- package/src/link/admission-wait.ts +73 -0
- package/src/link/compensation.ts +100 -0
- package/src/link/fingerprint.ts +21 -0
- package/src/link/paths.ts +19 -0
- package/src/link/ports.ts +12 -0
- package/src/link/routes.ts +32 -0
- package/src/link/ssh-argv.ts +170 -0
- package/src/link/ssh-config.ts +152 -0
- package/src/link/ssh-runner.ts +161 -0
- package/src/link/status-projection.ts +96 -0
- package/src/link/store.ts +139 -0
- package/src/link/supervisor.ts +404 -0
- package/src/link/tunnel-state.ts +91 -0
- package/src/protocols/baseline.ts +88 -0
- package/src/protocols/codecs/chat.ts +17 -0
- package/src/protocols/codecs/messages.ts +18 -0
- package/src/protocols/codecs/responses.ts +17 -0
- package/src/protocols/contract.ts +185 -0
- package/src/protocols/dto.ts +267 -0
- package/src/protocols/encoders/adapter-events.ts +876 -0
- package/src/protocols/encoders/chat.ts +243 -0
- package/src/protocols/encoders/messages.ts +462 -0
- package/src/protocols/envelope.ts +43 -0
- package/src/protocols/features.ts +295 -0
- package/src/protocols/guard.ts +33 -0
- package/src/protocols/opaque-state.ts +132 -0
- package/src/protocols/path.ts +39 -0
- package/src/protocols/plan-snapshot.ts +226 -0
- package/src/protocols/plan.ts +192 -0
- package/src/protocols/provider-summary.ts +101 -0
- package/src/protocols/settings.ts +118 -0
- package/src/protocols/shadow-plan.ts +36 -0
- package/src/protocols/shadow.ts +62 -0
- package/src/protocols/trace.ts +335 -0
- package/src/providers/model-rename-migration.ts +3 -3
- package/src/providers/registry/entries-core.ts +6 -1
- package/src/providers/registry/entries-extended.ts +15 -0
- package/src/providers/registry/model-ids.ts +1 -0
- package/src/providers/registry/types.ts +6 -0
- package/src/providers/registry.ts +1 -1
- package/src/remote-control/workspace-hub.ts +5 -4
- package/src/responses/apply-patch-envelope.ts +6 -1
- package/src/responses/code-mode-shell-input.ts +4 -2
- package/src/responses/freeform-wrapper-scan.ts +51 -24
- package/src/responses/progressive-freeform-input.ts +14 -20
- package/src/responses/spill-store.ts +187 -33
- package/src/responses/state/snapshot-select.ts +49 -0
- package/src/responses/state/spill-inspect.ts +46 -0
- package/src/responses/state/spill-queue.ts +16 -0
- package/src/responses/state.ts +51 -27
- package/src/server/audio-client.ts +8 -3
- package/src/server/audio-upstream.ts +9 -4
- package/src/server/auth-cors.ts +25 -8
- package/src/server/chat-completions.ts +71 -11
- package/src/server/chat-native-eligibility.ts +74 -0
- package/src/server/chat-native.ts +139 -74
- package/src/server/claude-messages.ts +140 -22
- package/src/server/hub-usage.ts +4 -3
- package/src/server/index/link-listener.ts +195 -0
- package/src/server/index/optional-listeners.ts +115 -0
- package/src/server/index/serve-options.ts +44 -10
- package/src/server/index.ts +10 -9
- package/src/server/inference/attempt.ts +51 -0
- package/src/server/inference/client-encoder-delivery.ts +235 -0
- package/src/server/inference/client-wire-log.ts +49 -0
- package/src/server/inference/client-wire.ts +69 -0
- package/src/server/inference/context.ts +15 -0
- package/src/server/inference/final-log.ts +37 -0
- package/src/server/local-desktop-snapshot-auth.ts +57 -0
- package/src/server/management/agent-settings-routes.ts +15 -13
- package/src/server/management/api-access.ts +11 -1
- package/src/server/management/config-routes.ts +3 -6
- package/src/server/management/context.ts +25 -0
- package/src/server/management/link-routes.ts +535 -0
- package/src/server/management/logs-usage-routes.ts +31 -1
- package/src/server/management/native-integration-routes.ts +6 -11
- package/src/server/management/oauth-account-routes.ts +42 -16
- package/src/server/management/protocol-routes.ts +159 -0
- package/src/server/management/protocol-settings-patch.ts +145 -0
- package/src/server/management/provider-patch-transaction.ts +61 -0
- package/src/server/management/provider-routes.ts +39 -18
- package/src/server/management/route-registry.ts +12 -0
- package/src/server/management/sidebar-routes.ts +8 -3
- package/src/server/management/system-routes.ts +18 -1
- package/src/server/management/usage-aggregate-cache.ts +206 -6
- package/src/server/management-api.ts +26 -1
- package/src/server/management-auth.ts +19 -5
- package/src/server/messages-native-eligibility.ts +161 -0
- package/src/server/messages-native-oauth.ts +149 -0
- package/src/server/messages-native.ts +790 -0
- package/src/server/relay.ts +9 -0
- package/src/server/request-log-filter.ts +92 -0
- package/src/server/request-log.ts +31 -70
- package/src/server/responses/adapter-delivery.ts +43 -16
- package/src/server/responses/agent-task-recovery.ts +90 -22
- package/src/server/responses/codex-ws-exchange.ts +4 -0
- package/src/server/responses/codex-ws-wire.ts +10 -1
- package/src/server/responses/core-combo-native.ts +323 -0
- package/src/server/responses/core-combo.ts +224 -13
- package/src/server/responses/core-options.ts +22 -0
- package/src/server/responses/core.ts +14 -26
- package/src/server/responses/request-sidecar-auth.ts +8 -3
- package/src/server/responses/run-turn-execution.ts +257 -63
- package/src/server/responses/sidecar-execution.ts +7 -4
- package/src/server/system-env.ts +7 -0
- package/src/service/diagnostics.ts +7 -1
- package/src/service/launchd.ts +26 -37
- package/src/service/windows-ops.ts +24 -20
- package/src/types/config.ts +58 -1
- package/src/types.ts +2 -0
- package/src/update/transactional-install.d.mts +0 -1
- package/src/update/transactional-install.mjs +17 -17
- package/src/usage/jev-stats.ts +495 -0
- package/src/usage/log.ts +24 -3
- package/src/web-search/run-turn-loop.ts +568 -0
- package/gui/dist/assets/App-8NMiZxT0.css +0 -1
- package/gui/dist/assets/App-DGLte6IR.js +0 -50
- package/gui/dist/assets/index--EWgGQvZ.css +0 -1
- package/gui/dist/assets/index-Bi1K37Wl.js +0 -86
package/src/server/relay.ts
CHANGED
|
@@ -32,6 +32,8 @@ import {
|
|
|
32
32
|
} from "./sse-frame-buffer";
|
|
33
33
|
import { replaceSseDataPayload, sseDataPayload } from "./sse-payload-rewrite";
|
|
34
34
|
import { createBoundedResponseLogBody } from "./response-log-body";
|
|
35
|
+
import { clientWireLogOf } from "./inference/client-wire";
|
|
36
|
+
import { recordClientWireRequestLog } from "./inference/client-wire-log";
|
|
35
37
|
|
|
36
38
|
const nativePassthroughSseResponses = new WeakSet<Response>();
|
|
37
39
|
const eagerRelaySseResponses = new WeakSet<Response>();
|
|
@@ -817,6 +819,13 @@ export function responseWithDeferredRequestLog(
|
|
|
817
819
|
if (isNativePassthroughSseResponse(response)) {
|
|
818
820
|
return response;
|
|
819
821
|
}
|
|
822
|
+
// A body already in the client's wire is not Responses SSE or JSON; its producer reports the
|
|
823
|
+
// facts the tap below would read (PF-09 direct encoders).
|
|
824
|
+
const clientWireLog = clientWireLogOf(response);
|
|
825
|
+
if (clientWireLog) {
|
|
826
|
+
recordClientWireRequestLog(clientWireLog, requestId, start, logCtx, addLog);
|
|
827
|
+
return response;
|
|
828
|
+
}
|
|
820
829
|
if (!response.body || !contentType.includes("text/event-stream")) {
|
|
821
830
|
if (response.body && (contentType.includes("application/json") || response.status >= 400)) {
|
|
822
831
|
const body = createBoundedResponseLogBody(response.body, {
|
|
@@ -0,0 +1,92 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `/api/logs` query filters over the in-memory request log. Split out of `request-log.ts`
|
|
3
|
+
* (which sits at its file-size threshold) so query clauses can grow here.
|
|
4
|
+
*/
|
|
5
|
+
import type { RequestLogEntry } from "./request-log";
|
|
6
|
+
import { matchesLogConversationId } from "./request-log-conversation";
|
|
7
|
+
import { isDeliveryMode } from "../protocols/contract";
|
|
8
|
+
|
|
9
|
+
/** Capacity of the in-memory request log ring; also the upper bound of `tail` and `limit`. */
|
|
10
|
+
export const MAX_LOG_SIZE = 2000;
|
|
11
|
+
|
|
12
|
+
/** `protocolMode=none` selects rows with no observed protocol trace. */
|
|
13
|
+
export const REQUEST_LOG_PROTOCOL_MODE_NONE = "none";
|
|
14
|
+
|
|
15
|
+
/**
|
|
16
|
+
* Whether a row matches a `protocolMode` query value: the final trace mode, or `none` for a row
|
|
17
|
+
* without a trace. An unrecognised value matches nothing rather than being ignored, for the
|
|
18
|
+
* reason #2704 gives for `model`.
|
|
19
|
+
*/
|
|
20
|
+
export function matchesProtocolMode(entry: Pick<RequestLogEntry, "protocolTrace">, mode: string): boolean {
|
|
21
|
+
if (mode === REQUEST_LOG_PROTOCOL_MODE_NONE) return entry.protocolTrace === undefined;
|
|
22
|
+
return isDeliveryMode(mode) && entry.protocolTrace?.mode === mode;
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
export function filterRequestLogs(logs: RequestLogEntry[], params: URLSearchParams): RequestLogEntry[] {
|
|
26
|
+
let filtered = logs;
|
|
27
|
+
const provider = params.get("provider")?.trim();
|
|
28
|
+
if (provider) {
|
|
29
|
+
filtered = filtered.filter(entry => entry.provider === provider
|
|
30
|
+
|| entry.attempts?.some(attempt => attempt.provider === provider));
|
|
31
|
+
}
|
|
32
|
+
const conversationId = params.get("conversationId")?.trim() || params.get("conversation")?.trim();
|
|
33
|
+
if (conversationId) {
|
|
34
|
+
filtered = filtered.filter(entry => matchesLogConversationId(entry.conversationId, conversationId));
|
|
35
|
+
}
|
|
36
|
+
// #2704: there was no `model` clause at all, so `?model=x` was ACCEPTED and silently
|
|
37
|
+
// ignored -- worse than an error, because it yields wrong conclusions from output that
|
|
38
|
+
// looks correct. Attempts are matched for the same reason `provider` matches them: a
|
|
39
|
+
// request that failed over should be findable by the model that actually served it.
|
|
40
|
+
const model = params.get("model")?.trim();
|
|
41
|
+
if (model) {
|
|
42
|
+
filtered = filtered.filter(entry => entry.model === model
|
|
43
|
+
|| entry.attempts?.some(attempt => attempt.model === model));
|
|
44
|
+
}
|
|
45
|
+
// #4057: "which account served this request" is the first question asked when one provider
|
|
46
|
+
// holds several accounts, and until now the only way to answer it was to grep usage.jsonl by
|
|
47
|
+
// hand. Attempts are matched for the same reason `provider` and `model` match them: when a
|
|
48
|
+
// request failed over between pool accounts, a search for the account that finally served it
|
|
49
|
+
// has to find that request, not only the account that first refused it.
|
|
50
|
+
const account = params.get("account")?.trim();
|
|
51
|
+
if (account) {
|
|
52
|
+
filtered = filtered.filter(entry => entry.accountLogLabel === account
|
|
53
|
+
|| entry.attempts?.some(attempt => attempt.accountLogLabel === account));
|
|
54
|
+
}
|
|
55
|
+
const protocolMode = params.get("protocolMode")?.trim().toLowerCase();
|
|
56
|
+
if (protocolMode) filtered = filtered.filter(entry => matchesProtocolMode(entry, protocolMode));
|
|
57
|
+
const status = params.get("status")?.trim().toLowerCase();
|
|
58
|
+
if (status) {
|
|
59
|
+
filtered = /^[1-5]xx$/.test(status)
|
|
60
|
+
? filtered.filter(entry => Math.floor(entry.status / 100) === Number(status[0]))
|
|
61
|
+
: filtered.filter(entry => String(entry.status) === status);
|
|
62
|
+
}
|
|
63
|
+
const tailRaw = params.get("tail")?.trim();
|
|
64
|
+
if (tailRaw) {
|
|
65
|
+
const tail = Number.parseInt(tailRaw, 10);
|
|
66
|
+
if (Number.isFinite(tail) && tail > 0) filtered = filtered.slice(-Math.min(tail, MAX_LOG_SIZE));
|
|
67
|
+
}
|
|
68
|
+
const offsetRaw = params.get("offset")?.trim();
|
|
69
|
+
const limitRaw = params.get("limit")?.trim();
|
|
70
|
+
if (limitRaw) {
|
|
71
|
+
const limit = Number.parseInt(limitRaw, 10);
|
|
72
|
+
const offset = offsetRaw ? Number.parseInt(offsetRaw, 10) : 0;
|
|
73
|
+
if (Number.isFinite(limit) && limit > 0) {
|
|
74
|
+
const capped = Math.min(limit, MAX_LOG_SIZE);
|
|
75
|
+
const startOffset = Number.isFinite(offset) && offset > 0 ? offset : 0;
|
|
76
|
+
const end = filtered.length - startOffset;
|
|
77
|
+
if (end <= 0) filtered = [];
|
|
78
|
+
else {
|
|
79
|
+
const begin = Math.max(0, end - capped);
|
|
80
|
+
filtered = filtered.slice(begin, end);
|
|
81
|
+
}
|
|
82
|
+
}
|
|
83
|
+
}
|
|
84
|
+
return filtered;
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
export function filteredRequestLogCount(logs: RequestLogEntry[], params: URLSearchParams): number {
|
|
88
|
+
const withoutPagination = new URLSearchParams(params);
|
|
89
|
+
withoutPagination.delete("limit");
|
|
90
|
+
withoutPagination.delete("offset");
|
|
91
|
+
return filterRequestLogs(logs, withoutPagination).length;
|
|
92
|
+
}
|
|
@@ -22,6 +22,8 @@ import type { CodexAffinityMove, CodexAffinityReason } from "../codex/routing";
|
|
|
22
22
|
import { readCodexCatalogPath } from "../codex/catalog";
|
|
23
23
|
import type { AttemptTierOutcome, OcxProviderConfig, OcxUsage } from "../types";
|
|
24
24
|
import { normalizeRouteDecisionTrace, type RouteDecisionTraceV1 } from "../routing/trace";
|
|
25
|
+
import { parseProtocolTraceV1, type ProtocolTraceV1 } from "../protocols/dto";
|
|
26
|
+
import { protocolTraceForRequest } from "../protocols/trace";
|
|
25
27
|
import type { AdapterRequest } from "../adapters/base";
|
|
26
28
|
import type { RequestSpendSettlement } from "./responses/request-spend";
|
|
27
29
|
import type { AdapterTierMetadata } from "../providers/fastwire";
|
|
@@ -70,13 +72,18 @@ import {
|
|
|
70
72
|
USAGE_DEBUG_BODY_SAMPLE_BYTES,
|
|
71
73
|
type UsageDebugBodyKind,
|
|
72
74
|
} from "../usage/debug";
|
|
73
|
-
import {
|
|
75
|
+
import { MAX_LOG_SIZE } from "./request-log-filter";
|
|
76
|
+
export { filterRequestLogs, filteredRequestLogCount } from "./request-log-filter";
|
|
74
77
|
import { enforceAppOwnedMemoryBudget, type RetainedStoreSnapshot } from "../lib/app-owned-memory";
|
|
75
78
|
import { capEstimateAtContextWindow } from "../lib/token-estimate";
|
|
76
79
|
import { inferCursorContextWindow } from "../adapters/cursor/discovery";
|
|
77
80
|
import { KIRO_MODEL_CONTEXT_WINDOWS, normalizeKiroModelId } from "../providers/kiro-models";
|
|
78
81
|
import { DEVIN_MODEL_CONTEXT_WINDOWS } from "../adapters/devin/live-models";
|
|
79
82
|
import { modelRecordValue } from "../reasoning-effort";
|
|
83
|
+
import {
|
|
84
|
+
normalizePersistedJevDecision,
|
|
85
|
+
type PersistedJevDecisionV1,
|
|
86
|
+
} from "../usage/jev-stats";
|
|
80
87
|
import type { RequestMetricsRecorder } from "./request-metrics";
|
|
81
88
|
import type {
|
|
82
89
|
CacheDiagnosticDraft,
|
|
@@ -239,6 +246,8 @@ export interface RequestLogContext {
|
|
|
239
246
|
terminalSource?: "upstream" | "synthetic";
|
|
240
247
|
/** Bounded route-decision trace (RI-01); never contains secrets. */
|
|
241
248
|
routeDecision?: RouteDecisionTraceV1;
|
|
249
|
+
/** Privacy-bounded JEV selection metadata; downstream usage is recorded on attempts[]. */
|
|
250
|
+
jevDecision?: PersistedJevDecisionV1;
|
|
242
251
|
/** Opt-in shadow evidence, normalized again at the logging boundary. */
|
|
243
252
|
claudeCompatibility?: PersistedClaudeCompatibilityLog;
|
|
244
253
|
}
|
|
@@ -349,6 +358,8 @@ export interface RequestLogEntry {
|
|
|
349
358
|
terminalSource?: "upstream" | "synthetic";
|
|
350
359
|
/** Bounded route-decision trace (RI-01); never contains secrets. */
|
|
351
360
|
routeDecision?: RouteDecisionTraceV1;
|
|
361
|
+
/** Privacy-bounded JEV selection metadata; downstream usage is recorded on attempts[]. */
|
|
362
|
+
jevDecision?: PersistedJevDecisionV1;
|
|
352
363
|
/** Closed Claude protocol codes; no request or header values. */
|
|
353
364
|
claudeCompatibility?: PersistedClaudeCompatibilityLog;
|
|
354
365
|
/**
|
|
@@ -358,11 +369,12 @@ export interface RequestLogEntry {
|
|
|
358
369
|
*/
|
|
359
370
|
failureStage?: RequestFailureStage;
|
|
360
371
|
failureCause?: RequestFailureCause;
|
|
372
|
+
/** Observed protocol path (PF-02, `src/protocols/trace.ts`); absent when nothing was observed. */
|
|
373
|
+
protocolTrace?: ProtocolTraceV1;
|
|
361
374
|
}
|
|
362
375
|
|
|
363
376
|
const requestLog: RequestLogEntry[] = [];
|
|
364
377
|
const requestLogObserversForTests = new Set<(entry: RequestLogEntry) => void>();
|
|
365
|
-
const MAX_LOG_SIZE = 2000;
|
|
366
378
|
const requestLogEntryBytes = new WeakMap<RequestLogEntry, number>();
|
|
367
379
|
let requestLogBytes = 0;
|
|
368
380
|
/** True after hydrateRequestLogsFromDisk ran once in this process. */
|
|
@@ -427,8 +439,10 @@ export function requestLogEntryFromPersistedUsage(entry: PersistedUsageEntry): R
|
|
|
427
439
|
const terminalStatus = asTerminalStatus(entry.terminalStatus);
|
|
428
440
|
const closeReason = asCloseReason(entry.closeReason);
|
|
429
441
|
const routeDecision = normalizeRouteDecisionTraceForLog(entry.routeDecision);
|
|
442
|
+
const jevDecision = normalizePersistedJevDecision(entry.jevDecision);
|
|
430
443
|
const claudeCompatibility = normalizeClaudeCompatibilityUsageLog(entry.claudeCompatibility);
|
|
431
444
|
const spend = normalizeRequestSpend(entry.spend);
|
|
445
|
+
const protocolTrace = parseProtocolTraceV1(entry.protocolTrace);
|
|
432
446
|
return {
|
|
433
447
|
requestId: entry.requestId,
|
|
434
448
|
...(isLogicalRequestId(entry.logicalRequestId) ? { logicalRequestId: entry.logicalRequestId } : {}),
|
|
@@ -479,11 +493,13 @@ export function requestLogEntryFromPersistedUsage(entry: PersistedUsageEntry): R
|
|
|
479
493
|
...(isKnownTransportPhase(entry.transportPhase) ? { transportPhase: entry.transportPhase } : {}),
|
|
480
494
|
...(isKnownTerminalSource(entry.terminalSource) ? { terminalSource: entry.terminalSource } : {}),
|
|
481
495
|
...(routeDecision ? { routeDecision } : {}),
|
|
496
|
+
...(jevDecision ? { jevDecision } : {}),
|
|
482
497
|
...(claudeCompatibility ? { claudeCompatibility } : {}),
|
|
483
498
|
...(entry.conversationStateScrub === "account-change"
|
|
484
499
|
? { conversationStateScrub: "account-change" }
|
|
485
500
|
: {}),
|
|
486
501
|
...normalizeRequestFailureAttribution(entry),
|
|
502
|
+
...(protocolTrace ? { protocolTrace } : {}),
|
|
487
503
|
};
|
|
488
504
|
}
|
|
489
505
|
|
|
@@ -574,8 +590,11 @@ export function addRequestLog(entry: RequestLogEntry) {
|
|
|
574
590
|
const shadowCallRewrittenFrom = sanitizeLogMetadataString(entry.shadowCallRewrittenFrom);
|
|
575
591
|
const servedModel = modelIdentityLogFields(entry).servedModel;
|
|
576
592
|
const claudeCompatibility = normalizeClaudeCompatibilityUsageLog(entry.claudeCompatibility);
|
|
593
|
+
const jevDecision = normalizePersistedJevDecision(entry.jevDecision);
|
|
577
594
|
const retained: RequestLogEntry = shadowCallRewrittenFrom === entry.shadowCallRewrittenFrom
|
|
578
|
-
&& servedModel === entry.servedModel
|
|
595
|
+
&& servedModel === entry.servedModel
|
|
596
|
+
&& entry.claudeCompatibility === undefined
|
|
597
|
+
&& entry.jevDecision === undefined
|
|
579
598
|
? entry
|
|
580
599
|
: { ...entry, ...(shadowCallRewrittenFrom ? { shadowCallRewrittenFrom } : {}) };
|
|
581
600
|
if (!shadowCallRewrittenFrom && retained !== entry) delete retained.shadowCallRewrittenFrom;
|
|
@@ -585,6 +604,8 @@ export function addRequestLog(entry: RequestLogEntry) {
|
|
|
585
604
|
}
|
|
586
605
|
if (claudeCompatibility) retained.claudeCompatibility = claudeCompatibility;
|
|
587
606
|
else if (retained !== entry) delete retained.claudeCompatibility;
|
|
607
|
+
if (jevDecision) retained.jevDecision = jevDecision;
|
|
608
|
+
else if (retained !== entry) delete retained.jevDecision;
|
|
588
609
|
entry = retained;
|
|
589
610
|
retainRequestLogEntry(entry);
|
|
590
611
|
for (const observer of requestLogObserversForTests) {
|
|
@@ -661,7 +682,9 @@ export function addRequestLog(entry: RequestLogEntry) {
|
|
|
661
682
|
// usage.jsonl, which is the surface the derived failure projection reads.
|
|
662
683
|
...normalizeRequestFailureAttribution(entry),
|
|
663
684
|
...(entry.routeDecision ? { routeDecision: entry.routeDecision } : {}),
|
|
685
|
+
...(entry.jevDecision ? { jevDecision: entry.jevDecision } : {}),
|
|
664
686
|
...(entry.claudeCompatibility ? { claudeCompatibility: entry.claudeCompatibility } : {}),
|
|
687
|
+
...(entry.protocolTrace ? { protocolTrace: entry.protocolTrace } : {}),
|
|
665
688
|
...(entry.conversationStateScrub === "account-change"
|
|
666
689
|
? { conversationStateScrub: "account-change" }
|
|
667
690
|
: {}),
|
|
@@ -1504,6 +1527,9 @@ export function addFinalRequestLog(
|
|
|
1504
1527
|
// the in-memory /api/logs row matches what usage.jsonl already stores.
|
|
1505
1528
|
const shadowCallRewrittenFrom = sanitizeLogMetadataString(logCtx.shadowCallRewrittenFrom);
|
|
1506
1529
|
const claudeCompatibility = normalizeClaudeCompatibilityUsageLog(logCtx.claudeCompatibility);
|
|
1530
|
+
const jevDecision = normalizePersistedJevDecision(logCtx.jevDecision);
|
|
1531
|
+
// Keyed by the live attempt objects, not the detached copies above.
|
|
1532
|
+
const protocolTrace = protocolTraceForRequest(logCtx, logCtx.attempts);
|
|
1507
1533
|
addLog({
|
|
1508
1534
|
requestId,
|
|
1509
1535
|
...(isLogicalRequestId(logicalRequestId) ? { logicalRequestId } : {}),
|
|
@@ -1563,7 +1589,9 @@ export function addFinalRequestLog(
|
|
|
1563
1589
|
...(logCtx.transportPhase ? { transportPhase: logCtx.transportPhase } : {}),
|
|
1564
1590
|
...(logCtx.terminalSource ? { terminalSource: logCtx.terminalSource } : {}),
|
|
1565
1591
|
...(logCtx.routeDecision ? { routeDecision: logCtx.routeDecision } : {}),
|
|
1592
|
+
...(jevDecision ? { jevDecision } : {}),
|
|
1566
1593
|
...(claudeCompatibility ? { claudeCompatibility } : {}),
|
|
1594
|
+
...(protocolTrace ? { protocolTrace } : {}),
|
|
1567
1595
|
...attribution,
|
|
1568
1596
|
});
|
|
1569
1597
|
// Formatted from the finalized snapshot, so the ring shows exactly what the ledger holds.
|
|
@@ -1583,73 +1611,6 @@ export function addFinalRequestLog(
|
|
|
1583
1611
|
}
|
|
1584
1612
|
}
|
|
1585
1613
|
|
|
1586
|
-
export function filterRequestLogs(logs: RequestLogEntry[], params: URLSearchParams): RequestLogEntry[] {
|
|
1587
|
-
let filtered = logs;
|
|
1588
|
-
const provider = params.get("provider")?.trim();
|
|
1589
|
-
if (provider) {
|
|
1590
|
-
filtered = filtered.filter(entry => entry.provider === provider
|
|
1591
|
-
|| entry.attempts?.some(attempt => attempt.provider === provider));
|
|
1592
|
-
}
|
|
1593
|
-
const conversationId = params.get("conversationId")?.trim() || params.get("conversation")?.trim();
|
|
1594
|
-
if (conversationId) {
|
|
1595
|
-
filtered = filtered.filter(entry => matchesLogConversationId(entry.conversationId, conversationId));
|
|
1596
|
-
}
|
|
1597
|
-
// #2704: there was no `model` clause at all, so `?model=x` was ACCEPTED and silently
|
|
1598
|
-
// ignored -- worse than an error, because it yields wrong conclusions from output that
|
|
1599
|
-
// looks correct. Attempts are matched for the same reason `provider` matches them: a
|
|
1600
|
-
// request that failed over should be findable by the model that actually served it.
|
|
1601
|
-
const model = params.get("model")?.trim();
|
|
1602
|
-
if (model) {
|
|
1603
|
-
filtered = filtered.filter(entry => entry.model === model
|
|
1604
|
-
|| entry.attempts?.some(attempt => attempt.model === model));
|
|
1605
|
-
}
|
|
1606
|
-
// #4057: "which account served this request" is the first question asked when one provider
|
|
1607
|
-
// holds several accounts, and until now the only way to answer it was to grep usage.jsonl by
|
|
1608
|
-
// hand. Attempts are matched for the same reason `provider` and `model` match them: when a
|
|
1609
|
-
// request failed over between pool accounts, a search for the account that finally served it
|
|
1610
|
-
// has to find that request, not only the account that first refused it.
|
|
1611
|
-
const account = params.get("account")?.trim();
|
|
1612
|
-
if (account) {
|
|
1613
|
-
filtered = filtered.filter(entry => entry.accountLogLabel === account
|
|
1614
|
-
|| entry.attempts?.some(attempt => attempt.accountLogLabel === account));
|
|
1615
|
-
}
|
|
1616
|
-
const status = params.get("status")?.trim().toLowerCase();
|
|
1617
|
-
if (status) {
|
|
1618
|
-
filtered = /^[1-5]xx$/.test(status)
|
|
1619
|
-
? filtered.filter(entry => Math.floor(entry.status / 100) === Number(status[0]))
|
|
1620
|
-
: filtered.filter(entry => String(entry.status) === status);
|
|
1621
|
-
}
|
|
1622
|
-
const tailRaw = params.get("tail")?.trim();
|
|
1623
|
-
if (tailRaw) {
|
|
1624
|
-
const tail = Number.parseInt(tailRaw, 10);
|
|
1625
|
-
if (Number.isFinite(tail) && tail > 0) filtered = filtered.slice(-Math.min(tail, MAX_LOG_SIZE));
|
|
1626
|
-
}
|
|
1627
|
-
const offsetRaw = params.get("offset")?.trim();
|
|
1628
|
-
const limitRaw = params.get("limit")?.trim();
|
|
1629
|
-
if (limitRaw) {
|
|
1630
|
-
const limit = Number.parseInt(limitRaw, 10);
|
|
1631
|
-
const offset = offsetRaw ? Number.parseInt(offsetRaw, 10) : 0;
|
|
1632
|
-
if (Number.isFinite(limit) && limit > 0) {
|
|
1633
|
-
const capped = Math.min(limit, MAX_LOG_SIZE);
|
|
1634
|
-
const startOffset = Number.isFinite(offset) && offset > 0 ? offset : 0;
|
|
1635
|
-
const end = filtered.length - startOffset;
|
|
1636
|
-
if (end <= 0) filtered = [];
|
|
1637
|
-
else {
|
|
1638
|
-
const begin = Math.max(0, end - capped);
|
|
1639
|
-
filtered = filtered.slice(begin, end);
|
|
1640
|
-
}
|
|
1641
|
-
}
|
|
1642
|
-
}
|
|
1643
|
-
return filtered;
|
|
1644
|
-
}
|
|
1645
|
-
|
|
1646
|
-
export function filteredRequestLogCount(logs: RequestLogEntry[], params: URLSearchParams): number {
|
|
1647
|
-
const withoutPagination = new URLSearchParams(params);
|
|
1648
|
-
withoutPagination.delete("limit");
|
|
1649
|
-
withoutPagination.delete("offset");
|
|
1650
|
-
return filterRequestLogs(logs, withoutPagination).length;
|
|
1651
|
-
}
|
|
1652
|
-
|
|
1653
1614
|
interface FinalizedUsageResult {
|
|
1654
1615
|
usage?: OcxUsage;
|
|
1655
1616
|
status: UsageStatus;
|
|
@@ -20,6 +20,7 @@ import {
|
|
|
20
20
|
ResponseBodyInactivityError,
|
|
21
21
|
} from "../../lib/response-body-inactivity";
|
|
22
22
|
import { resolveStallTimeoutSec } from "../../stall-timeout";
|
|
23
|
+
import { clientEncoderForDelivery, deliverClientEncodedResponse } from "../inference/client-encoder-delivery";
|
|
23
24
|
|
|
24
25
|
/** One responsibility of the Responses request pipeline; state owners are explicit. */
|
|
25
26
|
export async function deliverAdapterResponse(
|
|
@@ -114,6 +115,47 @@ export async function deliverAdapterResponse(
|
|
|
114
115
|
})
|
|
115
116
|
: eventStream;
|
|
116
117
|
const { toolNsMap, declaredToolNames, toolParameterSchemas, freeformToolNames, toolSearchToolNames } = toolBridgeMaps;
|
|
118
|
+
// One completion owner for both deliveries: the bridge calls it from its terminal, the
|
|
119
|
+
// direct client encoder from the fold of the same events.
|
|
120
|
+
const onCompletedResponse = (response: Record<string, unknown>, providerState?: OcxProviderContinuationState) => {
|
|
121
|
+
commitReasoningReplayServingRoute();
|
|
122
|
+
rememberKiroDeliveredFinalAnswer(transportState.activeAdapter.name, response);
|
|
123
|
+
// Compaction turns must NOT enter the continuation cache: _rawBody still holds the full
|
|
124
|
+
// PRE-compaction history, and a later previous_response_id expansion would rehydrate the
|
|
125
|
+
// giant stale chain Codex just replaced.
|
|
126
|
+
if (!routedCompaction) {
|
|
127
|
+
rememberResponseState(
|
|
128
|
+
parsed._rawBody,
|
|
129
|
+
response,
|
|
130
|
+
continuationStateForResponse(providerState),
|
|
131
|
+
responseStateOptions(transportState.activeAdapter.name === "kiro"),
|
|
132
|
+
);
|
|
133
|
+
}
|
|
134
|
+
notifyResponseComplete(response);
|
|
135
|
+
};
|
|
136
|
+
const clientEncoder = clientEncoderForDelivery(options, logCtx, !!routedCompaction, transportState.activeAdapter.name);
|
|
137
|
+
if (clientEncoder) {
|
|
138
|
+
return deliverClientEncodedResponse({
|
|
139
|
+
encoder: clientEncoder,
|
|
140
|
+
events: guardedEventStream,
|
|
141
|
+
logCtx,
|
|
142
|
+
translatorBudget,
|
|
143
|
+
responseModelId: parsed._responseModelId ?? parsed.modelId,
|
|
144
|
+
adapterName: transportState.activeAdapter.name,
|
|
145
|
+
fold: {
|
|
146
|
+
replayCacheScope: parsed._reasoningReplayScope,
|
|
147
|
+
hideThinkingSummary: parsed.options.hideThinkingSummary,
|
|
148
|
+
toolNsMap, declaredToolNames, toolParameterSchemas, freeformToolNames, toolSearchToolNames,
|
|
149
|
+
},
|
|
150
|
+
stallTimeoutSec: config.stallTimeoutSec,
|
|
151
|
+
turnAdmissionLease: options.turnAdmissionLease,
|
|
152
|
+
...(options.onFirstOutput ? { onFirstOutput: options.onFirstOutput } : {}),
|
|
153
|
+
stopUpstream: () => { cancelResponseCompletion(); upstream.abort(); },
|
|
154
|
+
onStreamDone: cleanupUpstreamAbort,
|
|
155
|
+
onCompletedResponse,
|
|
156
|
+
bindUsage: usage => transportState.bindKeyUsageFromBridge(usage),
|
|
157
|
+
});
|
|
158
|
+
}
|
|
117
159
|
const sseStream = bridgeToResponsesSSE(
|
|
118
160
|
guardedEventStream, parsed._responseModelId ?? parsed.modelId, toolNsMap, freeformToolNames, toolSearchToolNames,
|
|
119
161
|
() => { cancelResponseCompletion(); upstream.abort(); }, 2_000,
|
|
@@ -134,22 +176,7 @@ export async function deliverAdapterResponse(
|
|
|
134
176
|
// Raw adapter usage, pre wire-normalization (see the runTurn branch above).
|
|
135
177
|
transportState.bindKeyUsageFromBridge(usage);
|
|
136
178
|
},
|
|
137
|
-
onCompletedResponse
|
|
138
|
-
commitReasoningReplayServingRoute();
|
|
139
|
-
rememberKiroDeliveredFinalAnswer(transportState.activeAdapter.name, response);
|
|
140
|
-
// Compaction turns must NOT enter the continuation cache: _rawBody still holds the full
|
|
141
|
-
// PRE-compaction history, and a later previous_response_id expansion would rehydrate the
|
|
142
|
-
// giant stale chain Codex just replaced.
|
|
143
|
-
if (!routedCompaction) {
|
|
144
|
-
rememberResponseState(
|
|
145
|
-
parsed._rawBody,
|
|
146
|
-
response,
|
|
147
|
-
continuationStateForResponse(providerState),
|
|
148
|
-
responseStateOptions(transportState.activeAdapter.name === "kiro"),
|
|
149
|
-
);
|
|
150
|
-
}
|
|
151
|
-
notifyResponseComplete(response);
|
|
152
|
-
},
|
|
179
|
+
onCompletedResponse,
|
|
153
180
|
},
|
|
154
181
|
);
|
|
155
182
|
const bridgeTurnAc = new AbortController();
|
|
@@ -2,6 +2,7 @@ import { createHash, createHmac, randomBytes } from "node:crypto";
|
|
|
2
2
|
import { decodeJwtPayload, extractAccountId } from "../../oauth/chatgpt";
|
|
3
3
|
import type { OcxConfig } from "../../types";
|
|
4
4
|
import { boundedBodyDecodeFailure, readBoundedResponseBody } from "../../lib/bounded-body";
|
|
5
|
+
import { isTransientUpstreamStatus, retryBackoffDelayMs, sleepWithAbort } from "../../lib/upstream-retry";
|
|
5
6
|
import { isApiAuthRequired, isProxyAdmissionSecret } from "../auth-cors";
|
|
6
7
|
import { MAX_AGENT_TASK_CIPHERTEXT_BYTES, MAX_AGENT_TASK_ENCRYPTED_PARTS, structurallyValidFernetTokens } from "./encrypted-payload";
|
|
7
8
|
import {
|
|
@@ -34,12 +35,18 @@ const OPENAI_TOKEN_AUDIENCE = "https://api.openai.com/v1";
|
|
|
34
35
|
const MAX_ASSIGNMENT_BYTES = 2 * 1024 * 1024;
|
|
35
36
|
const MAX_RECOVERY_RESPONSE_BYTES = 4 * 1024 * 1024;
|
|
36
37
|
const CACHE_SCOPE_KEY = randomBytes(32);
|
|
38
|
+
// 1 initial send + up to 2 retries matches TRANSIENT_RETRY_MAX_ATTEMPTS; retries stay inside
|
|
39
|
+
// the caller's deadline, admission scope, and shared flight (#3661).
|
|
40
|
+
const MAX_RECOVERY_RETRIES = 2;
|
|
41
|
+
const RECOVERY_RETRY_BASE_DELAY_MS = 500;
|
|
42
|
+
const RECOVERY_RETRY_MAX_DELAY_MS = 2_000;
|
|
37
43
|
|
|
38
44
|
export interface AgentTaskRecoveryOptions {
|
|
39
45
|
enabled?: boolean;
|
|
40
46
|
model?: string;
|
|
41
47
|
timeoutMs?: number;
|
|
42
48
|
cacheEntries?: number;
|
|
49
|
+
retries?: number;
|
|
43
50
|
}
|
|
44
51
|
|
|
45
52
|
export type AgentTaskRecoveryFailureReason =
|
|
@@ -67,6 +74,9 @@ export function agentTaskRecoveryConfig(config: OcxConfig): AgentTaskRecoveryOpt
|
|
|
67
74
|
cacheEntries: Number.isFinite(raw.cacheEntries) && (raw.cacheEntries ?? 0) >= 1
|
|
68
75
|
? Math.min(512, Math.floor(raw.cacheEntries!))
|
|
69
76
|
: 200,
|
|
77
|
+
retries: Number.isFinite(raw.retries) && (raw.retries ?? 0) >= 0
|
|
78
|
+
? Math.min(MAX_RECOVERY_RETRIES, Math.floor(raw.retries!))
|
|
79
|
+
: 0,
|
|
70
80
|
};
|
|
71
81
|
}
|
|
72
82
|
|
|
@@ -448,20 +458,24 @@ function assignmentFromRecoverySse(raw: string, envelope: AgentEnvelope): string
|
|
|
448
458
|
: null;
|
|
449
459
|
}
|
|
450
460
|
|
|
451
|
-
|
|
461
|
+
interface RecoveryAttempt {
|
|
462
|
+
resolution: AgentTaskRecoveryResolution;
|
|
463
|
+
retryable: boolean;
|
|
464
|
+
retryHeaders?: Headers;
|
|
465
|
+
}
|
|
466
|
+
|
|
467
|
+
async function attemptRecovery(
|
|
452
468
|
admission: RecoveryAdmission,
|
|
453
469
|
envelope: AgentEnvelope,
|
|
454
470
|
options: AgentTaskRecoveryOptions,
|
|
455
|
-
|
|
456
|
-
|
|
457
|
-
|
|
458
|
-
|
|
459
|
-
|
|
460
|
-
|
|
461
|
-
|
|
462
|
-
|
|
463
|
-
? AbortSignal.any([abortSignal, controller.signal])
|
|
464
|
-
: controller.signal;
|
|
471
|
+
signal: AbortSignal,
|
|
472
|
+
abortSignal: AbortSignal | undefined,
|
|
473
|
+
deadlineSignal: AbortSignal,
|
|
474
|
+
): Promise<RecoveryAttempt> {
|
|
475
|
+
const terminal = (resolution: AgentTaskRecoveryResolution): RecoveryAttempt => ({
|
|
476
|
+
resolution,
|
|
477
|
+
retryable: false,
|
|
478
|
+
});
|
|
465
479
|
try {
|
|
466
480
|
const response = await fetch(RECOVERY_ENDPOINT, {
|
|
467
481
|
method: "POST",
|
|
@@ -473,9 +487,13 @@ async function requestRecovery(
|
|
|
473
487
|
if (!response.ok) {
|
|
474
488
|
// A rejected or never-settling cancellation must not extend the recovery deadline.
|
|
475
489
|
try { void response.body?.cancel().catch(() => undefined); } catch { /* already closed */ }
|
|
476
|
-
if (abortSignal?.aborted) return { recovered: false, reason: "recovery_aborted" };
|
|
477
|
-
if (
|
|
478
|
-
return {
|
|
490
|
+
if (abortSignal?.aborted) return terminal({ recovered: false, reason: "recovery_aborted" });
|
|
491
|
+
if (deadlineSignal.aborted) return terminal({ recovered: false, reason: "recovery_timeout" });
|
|
492
|
+
return {
|
|
493
|
+
resolution: { recovered: false, reason: "recovery_http_rejected" },
|
|
494
|
+
retryable: isTransientUpstreamStatus(response.status),
|
|
495
|
+
retryHeaders: response.headers,
|
|
496
|
+
};
|
|
479
497
|
}
|
|
480
498
|
const body = await readBoundedResponseBody(response, {
|
|
481
499
|
signal,
|
|
@@ -485,18 +503,68 @@ async function requestRecovery(
|
|
|
485
503
|
inactivityTimeoutMs: options.timeoutMs ?? 45_000,
|
|
486
504
|
firstByteTimeoutMs: options.timeoutMs ?? 45_000,
|
|
487
505
|
});
|
|
488
|
-
if (abortSignal?.aborted) return { recovered: false, reason: "recovery_aborted" };
|
|
489
|
-
if (
|
|
490
|
-
if (body.truncated || body.oversized || !body.displaySafe) return { recovered: false, reason: "recovery_invalid_output" };
|
|
506
|
+
if (abortSignal?.aborted) return terminal({ recovered: false, reason: "recovery_aborted" });
|
|
507
|
+
if (deadlineSignal.aborted || body.timedOut) return terminal({ recovered: false, reason: "recovery_timeout" });
|
|
508
|
+
if (body.truncated || body.oversized || !body.displaySafe) return terminal({ recovered: false, reason: "recovery_invalid_output" });
|
|
491
509
|
const assignment = assignmentFromRecoverySse(body.text, envelope);
|
|
492
|
-
return assignment === null
|
|
510
|
+
return terminal(assignment === null
|
|
493
511
|
? { recovered: false, reason: "recovery_invalid_output" }
|
|
494
|
-
: { recovered: true, assignment };
|
|
512
|
+
: { recovered: true, assignment });
|
|
495
513
|
} catch (error) {
|
|
496
|
-
if (abortSignal?.aborted) return { recovered: false, reason: "recovery_aborted" };
|
|
514
|
+
if (abortSignal?.aborted) return terminal({ recovered: false, reason: "recovery_aborted" });
|
|
497
515
|
const decodeFailure = boundedBodyDecodeFailure(error);
|
|
498
|
-
if (
|
|
499
|
-
return
|
|
516
|
+
if (deadlineSignal.aborted || decodeFailure === "timeout") return terminal({ recovered: false, reason: "recovery_timeout" });
|
|
517
|
+
return decodeFailure === "invalid_utf8"
|
|
518
|
+
? terminal({ recovered: false, reason: "recovery_invalid_output" })
|
|
519
|
+
: { resolution: { recovered: false, reason: "recovery_transport_error" }, retryable: true };
|
|
520
|
+
}
|
|
521
|
+
}
|
|
522
|
+
|
|
523
|
+
async function requestRecovery(
|
|
524
|
+
admission: RecoveryAdmission,
|
|
525
|
+
envelope: AgentEnvelope,
|
|
526
|
+
options: AgentTaskRecoveryOptions,
|
|
527
|
+
abortSignal?: AbortSignal,
|
|
528
|
+
): Promise<AgentTaskRecoveryResolution> {
|
|
529
|
+
const controller = new AbortController();
|
|
530
|
+
const timeoutMs = options.timeoutMs ?? 45_000;
|
|
531
|
+
const deadline = Date.now() + timeoutMs;
|
|
532
|
+
const timeout = setTimeout(
|
|
533
|
+
() => controller.abort(new DOMException("Agent task recovery timed out", "TimeoutError")),
|
|
534
|
+
timeoutMs,
|
|
535
|
+
);
|
|
536
|
+
const signal = abortSignal
|
|
537
|
+
? AbortSignal.any([abortSignal, controller.signal])
|
|
538
|
+
: controller.signal;
|
|
539
|
+
try {
|
|
540
|
+
// The configured bound is re-clamped here: callers may pass options that never went
|
|
541
|
+
// through agentTaskRecoveryConfig, and a larger value must not widen the outage window.
|
|
542
|
+
const retries = Number.isFinite(options.retries)
|
|
543
|
+
? Math.max(0, Math.min(MAX_RECOVERY_RETRIES, Math.floor(options.retries!)))
|
|
544
|
+
: 0;
|
|
545
|
+
for (let attempt = 0; ; attempt += 1) {
|
|
546
|
+
const { resolution, retryable, retryHeaders } = await attemptRecovery(
|
|
547
|
+
admission, envelope, options, signal, abortSignal, controller.signal,
|
|
548
|
+
);
|
|
549
|
+
if (resolution.recovered || !retryable || attempt >= retries) return resolution;
|
|
550
|
+
// Retry-After is the provider's floor, not something our backoff cap may shorten.
|
|
551
|
+
// If honouring it would outlive the shared deadline, end with this failure rather
|
|
552
|
+
// than resend early into a refusal.
|
|
553
|
+
const delayMs = retryBackoffDelayMs(attempt, {
|
|
554
|
+
baseDelayMs: RECOVERY_RETRY_BASE_DELAY_MS,
|
|
555
|
+
maxDelayMs: RECOVERY_RETRY_MAX_DELAY_MS,
|
|
556
|
+
headers: retryHeaders,
|
|
557
|
+
retryAfterIsLowerBound: true,
|
|
558
|
+
});
|
|
559
|
+
if (delayMs >= deadline - Date.now()) return resolution;
|
|
560
|
+
try {
|
|
561
|
+
await sleepWithAbort(delayMs, signal);
|
|
562
|
+
} catch {
|
|
563
|
+
return abortSignal?.aborted
|
|
564
|
+
? { recovered: false, reason: "recovery_aborted" }
|
|
565
|
+
: { recovered: false, reason: "recovery_timeout" };
|
|
566
|
+
}
|
|
567
|
+
}
|
|
500
568
|
} finally {
|
|
501
569
|
clearTimeout(timeout);
|
|
502
570
|
}
|
|
@@ -114,6 +114,7 @@ export function codexWsExchange(options: ExchangeOptions): Promise<Response> {
|
|
|
114
114
|
let pongs = 0;
|
|
115
115
|
let sentAt: number | null = null;
|
|
116
116
|
let firstFrameAt: number | null = null;
|
|
117
|
+
let firstResponseAt: number | null = null;
|
|
117
118
|
// Numeric close code for the durable stage record; the reason string stays
|
|
118
119
|
// out of it on purpose (#4191 content-free contract).
|
|
119
120
|
let closeCode: number | null = null;
|
|
@@ -170,6 +171,7 @@ export function codexWsExchange(options: ExchangeOptions): Promise<Response> {
|
|
|
170
171
|
controlFrames,
|
|
171
172
|
relayedEvents,
|
|
172
173
|
firstFrameMs: sentAt !== null && firstFrameAt !== null ? Math.max(0, firstFrameAt - sentAt) : null,
|
|
174
|
+
firstResponseMs: sentAt !== null && firstResponseAt !== null ? Math.max(0, firstResponseAt - sentAt) : null,
|
|
173
175
|
elapsedMs: sentAt !== null ? Math.max(0, Date.now() - sentAt) : null,
|
|
174
176
|
pings,
|
|
175
177
|
pongs,
|
|
@@ -188,6 +190,7 @@ export function codexWsExchange(options: ExchangeOptions): Promise<Response> {
|
|
|
188
190
|
controlFrames,
|
|
189
191
|
relayedEvents,
|
|
190
192
|
firstFrameMs: sentAt !== null && firstFrameAt !== null ? Math.max(0, firstFrameAt - sentAt) : null,
|
|
193
|
+
firstResponseMs: sentAt !== null && firstResponseAt !== null ? Math.max(0, firstResponseAt - sentAt) : null,
|
|
191
194
|
elapsedMs: sentAt !== null ? Math.max(0, Date.now() - sentAt) : null,
|
|
192
195
|
pings,
|
|
193
196
|
pongs,
|
|
@@ -481,6 +484,7 @@ export function codexWsExchange(options: ExchangeOptions): Promise<Response> {
|
|
|
481
484
|
if (!controlFrame && !type.startsWith("response.") && type !== "error") return;
|
|
482
485
|
let steeringEnded = false;
|
|
483
486
|
if (!controlFrame) {
|
|
487
|
+
firstResponseAt ??= Date.now();
|
|
484
488
|
try {
|
|
485
489
|
if (nativeControl) steeringEnded = nativeControl.observe(normalized.payload);
|
|
486
490
|
else correlation?.accept(normalized.payload);
|
|
@@ -176,6 +176,14 @@ export type CodexWsFailureStage = {
|
|
|
176
176
|
relayedEvents: number;
|
|
177
177
|
/** Milliseconds from send to the first upstream frame; null when none arrived. */
|
|
178
178
|
firstFrameMs: number | null;
|
|
179
|
+
/**
|
|
180
|
+
* Milliseconds from send to the first non-control Responses event (a
|
|
181
|
+
* `response.*` or `error` frame the relay would hand downstream); null when
|
|
182
|
+
* none arrived. `firstFrameMs` alone cannot separate "the peer answered with
|
|
183
|
+
* quota metadata and went quiet" from "the peer was still working": the
|
|
184
|
+
* first measures any frame, this one measures the turn's own start (#4191).
|
|
185
|
+
*/
|
|
186
|
+
firstResponseMs: number | null;
|
|
179
187
|
/** Milliseconds from send to this failure; null when the failure predates the send. */
|
|
180
188
|
elapsedMs: number | null;
|
|
181
189
|
/** Liveness pings the exchange sent while waiting for the first response event. */
|
|
@@ -271,7 +279,8 @@ export function codexWsFailureDetail(stage: CodexWsFailureStage): string {
|
|
|
271
279
|
return ` [cause=${classifyCodexWsFailure(stage)} request=${stage.requestBytes}B`
|
|
272
280
|
+ ` sent=${stage.sent ? "yes" : "no"} frames=${stage.upstreamFrames}`
|
|
273
281
|
+ ` control=${stage.controlFrames} relayed=${stage.relayedEvents}`
|
|
274
|
-
+ ` first-frame=${duration(stage.firstFrameMs)}
|
|
282
|
+
+ ` first-frame=${duration(stage.firstFrameMs)} first-response=${duration(stage.firstResponseMs)}`
|
|
283
|
+
+ ` elapsed=${duration(stage.elapsedMs)}`
|
|
275
284
|
+ ` pings=${stage.pings} pongs=${stage.pongs}]`;
|
|
276
285
|
}
|
|
277
286
|
|