@iislee/opencodex 2.11.0 → 2.35.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/AGENTS_INSTALL.md +109 -0
- package/README.md +117 -22
- package/bin/ocx.mjs +245 -88
- package/bin/package-main.mjs +1 -1
- package/gui/dist/assets/index-Be1sDpWw.css +1 -0
- package/gui/dist/assets/index-D4OKgUV4.js +114 -0
- package/gui/dist/index.html +2 -2
- package/gui/dist/provider-icons/alibaba-color.svg +1 -1
- package/gui/dist/provider-icons/antigravity-color.svg +1 -1
- package/gui/dist/provider-icons/claude-color.svg +1 -1
- package/gui/dist/provider-icons/cline-color.svg +16 -0
- package/gui/dist/provider-icons/cloudflare-ai-gateway-color.svg +1 -1
- package/gui/dist/provider-icons/commandcode-color.svg +1 -0
- package/gui/dist/provider-icons/copilot-color.svg +1 -1
- package/gui/dist/provider-icons/cursor-color.svg +1 -1
- package/gui/dist/provider-icons/deepseek-color.svg +1 -1
- package/gui/dist/provider-icons/firepass-color.svg +1 -1
- package/gui/dist/provider-icons/fireworks-color.svg +1 -1
- package/gui/dist/provider-icons/gemini-color.svg +1 -1
- package/gui/dist/provider-icons/github-copilot-color.svg +1 -1
- package/gui/dist/provider-icons/gitlab-duo-color.svg +1 -1
- package/gui/dist/provider-icons/grok.svg +1 -1
- package/gui/dist/provider-icons/groq-color.svg +1 -1
- package/gui/dist/provider-icons/huggingface-color.svg +1 -1
- package/gui/dist/provider-icons/kimi-color.svg +1 -1
- package/gui/dist/provider-icons/kiro-color.svg +2 -2
- package/gui/dist/provider-icons/lm-studio-color.svg +1 -1
- package/gui/dist/provider-icons/mistral-color.svg +1 -1
- package/gui/dist/provider-icons/moonshot-color.svg +1 -1
- package/gui/dist/provider-icons/nvidia-color.svg +1 -1
- package/gui/dist/provider-icons/ollama-color.svg +1 -1
- package/gui/dist/provider-icons/openai.svg +1 -1
- package/gui/dist/provider-icons/opencode.svg +2 -1
- package/gui/dist/provider-icons/openrouter-color.svg +1 -1
- package/gui/dist/provider-icons/pi.svg +2 -2
- package/gui/dist/provider-icons/qianfan-color.svg +1 -1
- package/gui/dist/provider-icons/qwen-portal-color.svg +1 -1
- package/gui/dist/provider-icons/vercel-ai-gateway-color.svg +1 -1
- package/gui/dist/provider-icons/vllm-color.svg +1 -1
- package/gui/dist/provider-icons/xiaomi-color.svg +1 -1
- package/package.json +21 -10
- package/src/adapters/anthropic-output-schema.ts +137 -0
- package/src/adapters/anthropic.ts +475 -62
- package/src/adapters/base.ts +82 -7
- package/src/adapters/client-fingerprint.ts +18 -12
- package/src/adapters/cline-pass-deepseek-v4-tool-replay.ts +69 -0
- package/src/adapters/command-code.ts +637 -0
- package/src/adapters/cursor/call-id.ts +44 -0
- package/src/adapters/cursor/catalog.ts +541 -0
- package/src/adapters/cursor/checkpoint-store.ts +308 -0
- package/src/adapters/cursor/cursor-errors.ts +144 -6
- package/src/adapters/cursor/discovery.ts +122 -14
- package/src/adapters/cursor/effort-map.ts +106 -4
- package/src/adapters/cursor/envelope-echo.ts +290 -0
- package/src/adapters/cursor/framing.ts +39 -0
- package/src/adapters/cursor/h2-pool.ts +123 -0
- package/src/adapters/cursor/http1-bidi.ts +361 -0
- package/src/adapters/cursor/images.ts +704 -0
- package/src/adapters/cursor/live-models.ts +180 -59
- package/src/adapters/cursor/live-transport.ts +623 -170
- package/src/adapters/cursor/message-mapper.ts +4 -1
- package/src/adapters/cursor/native-exec-common.ts +23 -2
- package/src/adapters/cursor/native-exec-desktop.ts +23 -0
- package/src/adapters/cursor/native-exec-fs.ts +10 -7
- package/src/adapters/cursor/native-exec-network.ts +1 -1
- package/src/adapters/cursor/native-exec-shell.ts +5 -3
- package/src/adapters/cursor/native-exec.ts +121 -14
- package/src/adapters/cursor/protobuf-events.ts +829 -11
- package/src/adapters/cursor/protobuf-request.ts +447 -75
- package/src/adapters/cursor/request-builder.ts +271 -35
- package/src/adapters/cursor/tool-definitions.ts +252 -12
- package/src/adapters/cursor/tool-result-normalize.ts +115 -0
- package/src/adapters/cursor/transport.ts +22 -0
- package/src/adapters/cursor/types.ts +48 -1
- package/src/adapters/cursor.ts +336 -39
- package/src/adapters/exec-tool-result-normalize.ts +99 -0
- package/src/adapters/google-antigravity-replay.ts +713 -41
- package/src/adapters/google-antigravity-wire.ts +48 -10
- package/src/adapters/google-errors.ts +44 -12
- package/src/adapters/google-http.ts +38 -10
- package/src/adapters/google-tool-schema.ts +80 -15
- package/src/adapters/google-truncation.ts +11 -0
- package/src/adapters/google.ts +742 -86
- package/src/adapters/identity.ts +39 -6
- package/src/adapters/image.ts +1 -1
- package/src/adapters/kiro-constants.ts +27 -0
- package/src/adapters/kiro-errors.ts +11 -0
- package/src/adapters/kiro-events.ts +19 -1
- package/src/adapters/kiro-thinking.ts +18 -2
- package/src/adapters/kiro-tools.ts +65 -17
- package/src/adapters/kiro.ts +270 -73
- package/src/adapters/mimo-free.ts +17 -0
- package/src/adapters/openai-chat-url.ts +11 -0
- package/src/adapters/openai-chat.ts +1377 -344
- package/src/adapters/openai-responses-url.ts +16 -0
- package/src/adapters/openai-responses.ts +1164 -64
- package/src/adapters/registry.ts +175 -0
- package/src/adapters/responses-tool-schema.ts +67 -0
- package/src/adapters/run-turn-queue.ts +36 -1
- package/src/adapters/tool-call-id.ts +119 -0
- package/src/adapters/tool-catalog-nudge.ts +105 -21
- package/src/adapters/xai-tool-schema.ts +436 -0
- package/src/adapters/xai-web-search.ts +186 -0
- package/src/bridge.ts +512 -92
- package/src/chat/inbound.ts +57 -20
- package/src/chat/outbound.ts +97 -33
- package/src/claude/agents-inject.ts +40 -10
- package/src/claude/context-windows.ts +37 -14
- package/src/claude/desktop-3p.ts +243 -9
- package/src/claude/gateway-cache.ts +41 -4
- package/src/claude/inbound.ts +72 -3
- package/src/claude/model-info.ts +38 -15
- package/src/claude/outbound.ts +88 -21
- package/src/cli/access.ts +46 -3
- package/src/cli/account-api.ts +150 -19
- package/src/cli/account-auth.ts +33 -6
- package/src/cli/account-catalog-refresh.ts +14 -0
- package/src/cli/account-extended.ts +648 -28
- package/src/cli/account-main.ts +317 -0
- package/src/cli/account.ts +97 -19
- package/src/cli/agent.ts +53 -2
- package/src/cli/alias.ts +66 -0
- package/src/cli/capabilities-command.ts +94 -0
- package/src/cli/capabilities.ts +496 -0
- package/src/cli/claude-agent-startup-sync.ts +73 -0
- package/src/cli/claude-desktop.ts +60 -15
- package/src/cli/claude.ts +129 -35
- package/src/cli/codex-log-guard-doctor.ts +103 -0
- package/src/cli/codex-shim-autorestore.ts +2 -0
- package/src/cli/codex-shim-readiness.ts +76 -0
- package/src/cli/combo.ts +8 -0
- package/src/cli/config-command.ts +74 -10
- package/src/cli/dispatch.ts +821 -0
- package/src/cli/doctor.ts +440 -56
- package/src/cli/ensure-desired-integrations.ts +152 -0
- package/src/cli/export-command.ts +46 -20
- package/src/cli/help.ts +34 -274
- package/src/cli/index.ts +417 -528
- package/src/cli/init.ts +5 -17
- package/src/cli/inspect.ts +230 -0
- package/src/cli/integrations.ts +120 -2
- package/src/cli/lab.ts +607 -0
- package/src/cli/launcher-context.ts +77 -0
- package/src/cli/minimax.ts +497 -0
- package/src/cli/models-runtime.ts +130 -2
- package/src/cli/models.ts +118 -16
- package/src/cli/observe.ts +123 -11
- package/src/cli/opencode.ts +4 -2
- package/src/cli/provider-runtime.ts +34 -3
- package/src/cli/provider.ts +25 -3
- package/src/cli/ready.ts +301 -0
- package/src/cli/registry.ts +476 -0
- package/src/cli/root.ts +86 -0
- package/src/cli/route-policy.ts +92 -0
- package/src/cli/runtime-api.ts +57 -10
- package/src/cli/star-prompt.ts +71 -15
- package/src/cli/status.ts +51 -4
- package/src/cli/storage.ts +234 -0
- package/src/cli/system-command.ts +16 -0
- package/src/cli/system-restart-client.ts +146 -0
- package/src/cli/tray-proxy.ts +153 -6
- package/src/cli/usage-report.ts +184 -0
- package/src/cli/v2.ts +137 -18
- package/src/cli/version-skew.ts +46 -0
- package/src/cli.ts +1 -1
- package/src/clients/config-export.ts +1442 -23
- package/src/clients/effective-status.ts +30 -2
- package/src/clients/probes/cline.ts +81 -0
- package/src/codex/account-label.ts +35 -1
- package/src/codex/account-lifecycle.ts +130 -13
- package/src/codex/account-namespaces.ts +49 -3
- package/src/codex/account-pause.ts +2 -1
- package/src/codex/account-priority.ts +84 -0
- package/src/codex/account-store.ts +29 -2
- package/src/codex/account-usability.ts +25 -2
- package/src/codex/admission.ts +256 -0
- package/src/codex/affinity-debug.ts +162 -0
- package/src/codex/app-server-processes.ts +570 -107
- package/src/codex/app-server-restart-service.ts +232 -0
- package/src/codex/auth-api.ts +893 -246
- package/src/codex/auth-collision.ts +5 -3
- package/src/codex/auth-context.ts +348 -32
- package/src/codex/autostart-health.ts +24 -1
- package/src/codex/catalog/account-models.ts +67 -0
- package/src/codex/catalog/aggregation.ts +86 -10
- package/src/codex/catalog/bundled.ts +331 -33
- package/src/codex/catalog/effort.ts +138 -32
- package/src/codex/catalog/filesystem-evidence.ts +302 -0
- package/src/codex/catalog/kinds.ts +2 -0
- package/src/codex/catalog/metadata.ts +555 -45
- package/src/codex/catalog/native-models.ts +75 -0
- package/src/codex/catalog/parsing.ts +269 -30
- package/src/codex/catalog/provider-fetch.ts +1626 -144
- package/src/codex/catalog/sync.ts +1576 -187
- package/src/codex/catalog-admission.ts +199 -0
- package/src/codex/catalog-refresh-status.ts +105 -0
- package/src/codex/catalog-write-serialization.ts +242 -0
- package/src/codex/catalog.ts +6 -3
- package/src/codex/codex-write-lock.ts +386 -0
- package/src/codex/convergence-types.ts +614 -0
- package/src/codex/convergence.ts +676 -0
- package/src/codex/coordinator-doctor.ts +332 -0
- package/src/codex/custom-model-catalog-migration.ts +176 -0
- package/src/codex/desired-state.ts +230 -0
- package/src/codex/desktop-app-restart.ts +355 -0
- package/src/codex/features.ts +636 -39
- package/src/codex/generation.ts +202 -0
- package/src/codex/history-job.ts +436 -0
- package/src/codex/history-lock.ts +242 -0
- package/src/codex/history-manifest.ts +112 -0
- package/src/codex/history-migration-guardian.ts +30 -24
- package/src/codex/history-provider.ts +1016 -235
- package/src/codex/history-transition.ts +105 -0
- package/src/codex/history-worker.ts +223 -0
- package/src/codex/inject-coordination.ts +373 -0
- package/src/codex/inject.ts +1114 -152
- package/src/codex/injected-marker.ts +37 -3
- package/src/codex/integration-record.ts +266 -0
- package/src/codex/internal/catalog-writer.ts +203 -0
- package/src/codex/internal/history-writer.ts +81 -0
- package/src/codex/journal.ts +66 -4
- package/src/codex/log-guard/inspect.ts +524 -0
- package/src/codex/log-guard/lock.ts +150 -0
- package/src/codex/log-guard/maintenance.ts +403 -0
- package/src/codex/log-guard/path-safety.ts +88 -0
- package/src/codex/log-guard/policy.ts +44 -0
- package/src/codex/log-guard/processes.ts +205 -0
- package/src/codex/log-guard/protection.ts +489 -0
- package/src/codex/log-guard/sqlite-errors.ts +9 -0
- package/src/codex/main-account-cache.ts +24 -0
- package/src/codex/main-account.ts +29 -1
- package/src/codex/management-convergence.ts +167 -0
- package/src/codex/model-cache.ts +56 -10
- package/src/codex/model-entitlements.ts +360 -0
- package/src/codex/native-main-admission.ts +47 -0
- package/src/codex/native-main-auth-temp.ts +187 -0
- package/src/codex/native-main-claim.ts +178 -0
- package/src/codex/native-main-lock-file.ts +162 -0
- package/src/codex/native-main-owner.ts +329 -0
- package/src/codex/native-profile-api.ts +247 -0
- package/src/codex/native-profile-manager.ts +1531 -0
- package/src/codex/native-profile-processes.ts +121 -0
- package/src/codex/native-profile-recovery.ts +99 -0
- package/src/codex/native-profile-stage-store.ts +387 -0
- package/src/codex/native-profile-startup.ts +622 -0
- package/src/codex/native-profile-store.ts +855 -0
- package/src/codex/native-profile-types.ts +120 -0
- package/src/codex/native-residue.ts +675 -0
- package/src/codex/paths.ts +80 -1
- package/src/codex/plan-from-token.ts +140 -0
- package/src/codex/plan.ts +40 -0
- package/src/codex/plugins-doctor.ts +1 -1
- package/src/codex/pool-rotation.ts +74 -4
- package/src/codex/project-config-warnings.ts +20 -6
- package/src/codex/prompt-journal.ts +352 -0
- package/src/codex/prompt-layers.ts +1313 -0
- package/src/codex/prompt-lock.ts +143 -0
- package/src/codex/prompt-text-probe.ts +238 -0
- package/src/codex/quota-rejection.ts +298 -0
- package/src/codex/quota.ts +264 -21
- package/src/codex/refresh.ts +11 -2
- package/src/codex/reset-credit-recovery.ts +1044 -0
- package/src/codex/routing.ts +514 -94
- package/src/codex/runtime.ts +159 -38
- package/src/codex/shim.ts +1065 -31
- package/src/codex/subagent-model-fallback.ts +437 -43
- package/src/codex/sync.ts +191 -2
- package/src/codex/transition-state.ts +720 -0
- package/src/codex/upstream-host-health.ts +368 -0
- package/src/codex/user-identity.ts +557 -0
- package/src/codex/warmup.ts +187 -81
- package/src/codex/write-coordination.ts +114 -0
- package/src/combos/failover.ts +47 -0
- package/src/combos/index.ts +4 -0
- package/src/combos/request.ts +32 -0
- package/src/combos/types.ts +91 -9
- package/src/compatibility/index.ts +26 -0
- package/src/compatibility/manifest.ts +253 -0
- package/src/compatibility/openai-responses.ts +121 -0
- package/src/config/atomic-write.ts +219 -0
- package/src/config/paths.ts +40 -0
- package/src/config/process-state.ts +309 -0
- package/src/config/provider-name.ts +24 -0
- package/src/config/provider-validation.ts +177 -0
- package/src/config/rebase-provenance.ts +68 -0
- package/src/config.ts +1713 -709
- package/src/generated/compatibility-version.json +3324 -0
- package/src/generated/{jawcode-model-metadata.ts → model-metadata.ts} +19 -17
- package/src/grok/inject.ts +16 -5
- package/src/grok/inspect.ts +45 -0
- package/src/grok/sync.ts +2 -2
- package/src/images/loop.ts +157 -33
- package/src/images/plan.ts +24 -13
- package/src/integrations/config-io.ts +269 -0
- package/src/integrations/journal.ts +315 -0
- package/src/integrations/merge.ts +135 -0
- package/src/integrations/mutation-flight.ts +71 -0
- package/src/integrations/native/ownership-preflight.ts +202 -0
- package/src/integrations/omp-yaml-source.ts +358 -0
- package/src/integrations/owned-refresh.ts +74 -0
- package/src/integrations/ownership-policy.ts +160 -0
- package/src/integrations/ownership.ts +155 -0
- package/src/integrations/registry.ts +166 -0
- package/src/integrations/serialize.ts +314 -0
- package/src/integrations/state.ts +433 -0
- package/src/integrations/store.ts +103 -0
- package/src/integrations/writer-lock.ts +98 -0
- package/src/integrations/writer.ts +715 -0
- package/src/lab/artifacts/sanitize.ts +586 -0
- package/src/lab/artifacts/secure-fs.ts +475 -0
- package/src/lab/artifacts/store.ts +310 -0
- package/src/lab/automation/budgets.ts +78 -0
- package/src/lab/automation/config-persistence.ts +256 -0
- package/src/lab/automation/constants.ts +39 -0
- package/src/lab/automation/cooldown.ts +103 -0
- package/src/lab/automation/dispatch.ts +211 -0
- package/src/lab/automation/index.ts +13 -0
- package/src/lab/automation/orchestrator.ts +499 -0
- package/src/lab/automation/persistence.ts +512 -0
- package/src/lab/automation/planner.ts +371 -0
- package/src/lab/automation/policy.ts +136 -0
- package/src/lab/automation/queue.ts +191 -0
- package/src/lab/automation/recovery.ts +24 -0
- package/src/lab/automation/route-context.ts +21 -0
- package/src/lab/automation/run-key.ts +44 -0
- package/src/lab/automation/runs-query.ts +34 -0
- package/src/lab/automation/types.ts +160 -0
- package/src/lab/conformance/assertion.ts +325 -0
- package/src/lab/conformance/digest.ts +22 -0
- package/src/lab/conformance/executor.ts +741 -0
- package/src/lab/conformance/fixture-provider.ts +27 -0
- package/src/lab/conformance/fixtures/live-v1-cases.json +175 -0
- package/src/lab/conformance/fixtures/protocol-v1-cases.json +461 -0
- package/src/lab/conformance/harness-budget.ts +47 -0
- package/src/lab/conformance/index.ts +5 -0
- package/src/lab/conformance/jcs.ts +64 -0
- package/src/lab/conformance/json-pointer.ts +39 -0
- package/src/lab/conformance/manifest.ts +180 -0
- package/src/lab/conformance/mcp-stub.ts +179 -0
- package/src/lab/conformance/negative-controls.ts +164 -0
- package/src/lab/conformance/observation.ts +355 -0
- package/src/lab/conformance/runner.ts +68 -0
- package/src/lab/conformance/sse-normalize.ts +59 -0
- package/src/lab/conformance/suite-manifest.ts +78 -0
- package/src/lab/conformance/types.ts +214 -0
- package/src/lab/constants.ts +126 -0
- package/src/lab/digest.ts +64 -0
- package/src/lab/events/errors.ts +9 -0
- package/src/lab/events/limits.ts +117 -0
- package/src/lab/events/types.ts +229 -0
- package/src/lab/events/validate.ts +781 -0
- package/src/lab/fabric/constants.ts +40 -0
- package/src/lab/fabric/executor.ts +492 -0
- package/src/lab/fabric/index.ts +80 -0
- package/src/lab/fabric/manifest.ts +222 -0
- package/src/lab/fabric/observe.ts +489 -0
- package/src/lab/fabric/patch.ts +79 -0
- package/src/lab/fabric/producer-child.ts +139 -0
- package/src/lab/fabric/producer-isolate.ts +276 -0
- package/src/lab/fabric/producer-protocol.ts +61 -0
- package/src/lab/fabric/scratch.ts +439 -0
- package/src/lab/fabric/subject.ts +106 -0
- package/src/lab/fabric/types.ts +134 -0
- package/src/lab/fabric/verifier.ts +98 -0
- package/src/lab/index.ts +54 -0
- package/src/lab/ledger/artifact-refs.ts +127 -0
- package/src/lab/ledger/invalidation.ts +136 -0
- package/src/lab/ledger/purge.ts +310 -0
- package/src/lab/ledger/store.ts +532 -0
- package/src/lab/live/credential-lease.ts +53 -0
- package/src/lab/live/destination.ts +155 -0
- package/src/lab/live/executor.ts +336 -0
- package/src/lab/live/inert-tools.ts +56 -0
- package/src/lab/live/manifest.ts +85 -0
- package/src/lab/live/mcp-loopback.ts +57 -0
- package/src/lab/live/runner.ts +19 -0
- package/src/lab/live/sandbox.ts +61 -0
- package/src/lab/live/suite-manifest.ts +41 -0
- package/src/lab/live/transport.ts +118 -0
- package/src/lab/live/types.ts +197 -0
- package/src/lab/observe/from-conformance.ts +301 -0
- package/src/lab/observe/from-live.ts +117 -0
- package/src/lab/paths.ts +153 -0
- package/src/lab/projection/rebuild.ts +495 -0
- package/src/lab/projection/schema.ts +135 -0
- package/src/lab/projection/verdicts.ts +474 -0
- package/src/lab/projection/verification.ts +412 -0
- package/src/lab/public/bundle.ts +217 -0
- package/src/lab/public/community-authority.ts +175 -0
- package/src/lab/public/community-files.ts +29 -0
- package/src/lab/public/community.ts +479 -0
- package/src/lab/public/file-safety.ts +155 -0
- package/src/lab/public/ids.ts +26 -0
- package/src/lab/public/index.ts +16 -0
- package/src/lab/public/mutation-lock.ts +424 -0
- package/src/lab/public/operator.ts +353 -0
- package/src/lab/public/origin-purge.ts +79 -0
- package/src/lab/public/origin.ts +203 -0
- package/src/lab/public/privacy.ts +143 -0
- package/src/lab/public/private-file.ts +261 -0
- package/src/lab/public/project.ts +124 -0
- package/src/lab/public/purge-test-fault.ts +21 -0
- package/src/lab/public/purge.ts +223 -0
- package/src/lab/public/registry.ts +44 -0
- package/src/lab/public/revocation.ts +252 -0
- package/src/lab/public/signature.ts +243 -0
- package/src/lab/public/storage.ts +105 -0
- package/src/lab/public/strict-json.ts +206 -0
- package/src/lab/public/time.ts +26 -0
- package/src/lab/public/types.ts +172 -0
- package/src/lab/public/validate.ts +391 -0
- package/src/lab/query/catalog.ts +101 -0
- package/src/lab/query/connection.ts +107 -0
- package/src/lab/query/constants.ts +4 -0
- package/src/lab/query/cursor.ts +132 -0
- package/src/lab/query/dto-map.ts +277 -0
- package/src/lab/query/errors.ts +22 -0
- package/src/lab/query/freshness.ts +53 -0
- package/src/lab/query/index.ts +45 -0
- package/src/lab/query/latest-observation.ts +59 -0
- package/src/lab/query/passive-production.ts +159 -0
- package/src/lab/query/queries.ts +444 -0
- package/src/lab/query/types.ts +266 -0
- package/src/lab/subject/behavior-fingerprint.ts +77 -0
- package/src/lab/subject/installation-salt.ts +112 -0
- package/src/lab/subject/protocol-subject.ts +80 -0
- package/src/lab/subject/route-subject.ts +74 -0
- package/src/lib/admin-secrets.ts +24 -0
- package/src/lib/app-owned-memory-stores.ts +22 -0
- package/src/lib/bounded-body.ts +167 -11
- package/src/lib/bun-runtime.ts +125 -12
- package/src/lib/bun-stream-caps.ts +13 -9
- package/src/lib/codex-restart-contract.ts +120 -0
- package/src/lib/config-ownership.ts +26 -2
- package/src/lib/destination-policy.ts +65 -1
- package/src/lib/errors.ts +80 -5
- package/src/lib/fabric-task-execution-authority.ts +7 -0
- package/src/lib/fabric-task-host.ts +29 -0
- package/src/lib/lab-activation.ts +223 -0
- package/src/lib/lab-live-execution-authority.ts +13 -0
- package/src/lib/lab-live-host.ts +30 -0
- package/src/lib/lab-live-pinned-sender.ts +56 -0
- package/src/lib/lab-live-route-production.ts +130 -0
- package/src/lib/lab-passive-linker-registration.ts +26 -0
- package/src/lib/local-management-attestation.ts +51 -0
- package/src/lib/local-management-capability.ts +100 -0
- package/src/lib/local-provider-reload-contract.ts +100 -0
- package/src/lib/optional-shutdown-hooks.ts +57 -0
- package/src/lib/package-tree-integrity.ts +101 -0
- package/src/lib/pinned-http.ts +145 -26
- package/src/lib/process-control.ts +6 -2
- package/src/lib/provider-outbound.ts +49 -9
- package/src/lib/redact.ts +419 -3
- package/src/lib/self-launch-argv.ts +15 -0
- package/src/lib/server-resource-ownership.ts +71 -0
- package/src/lib/service-secrets.ts +15 -0
- package/src/lib/shadow-call.ts +35 -4
- package/src/lib/sse-decoder.ts +41 -0
- package/src/lib/state-store-registrations.ts +10 -2
- package/src/lib/system-restart-contract.ts +73 -0
- package/src/lib/token-estimate.ts +19 -2
- package/src/lib/tool-argument-integers.ts +253 -0
- package/src/lib/translator-budget.ts +44 -0
- package/src/lib/upstream-http-version.ts +57 -0
- package/src/lib/upstream-reachability.ts +95 -0
- package/src/lib/upstream-retry.ts +156 -3
- package/src/lib/windows-atomic-replace.ts +155 -0
- package/src/lib/windows-elevation.ts +70 -2
- package/src/lib/windows-secret-acl.ts +409 -69
- package/src/lib/windows-service-wrappers.ts +72 -0
- package/src/lib/windows-text.ts +106 -0
- package/src/lib/windows-user-principal.ts +341 -0
- package/src/lib/winsw.ts +33 -5
- package/src/oauth/account-import/google-antigravity-adapter.ts +74 -0
- package/src/oauth/account-import/index.ts +15 -0
- package/src/oauth/account-import/parser.ts +83 -0
- package/src/oauth/account-import/registry.ts +18 -0
- package/src/oauth/account-import/service.ts +75 -0
- package/src/oauth/account-import/types.ts +91 -0
- package/src/oauth/anthropic.ts +12 -1
- package/src/oauth/callback-server.ts +30 -4
- package/src/oauth/chatgpt.ts +12 -1
- package/src/oauth/cline.ts +203 -0
- package/src/oauth/command-code.ts +239 -0
- package/src/oauth/cursor.ts +46 -5
- package/src/oauth/generic-account-failover.ts +231 -0
- package/src/oauth/google-antigravity.ts +35 -3
- package/src/oauth/health.ts +20 -12
- package/src/oauth/index.ts +462 -71
- package/src/oauth/key-providers.ts +25 -0
- package/src/oauth/kimi.ts +25 -3
- package/src/oauth/kiro.ts +95 -6
- package/src/oauth/local-token-detect.ts +13 -2
- package/src/oauth/log.ts +3 -1
- package/src/oauth/login-cli.ts +88 -28
- package/src/oauth/nous.ts +798 -0
- package/src/oauth/open-browser-choice.ts +26 -0
- package/src/oauth/store.ts +133 -26
- package/src/oauth/token-guardian.ts +9 -3
- package/src/oauth/types.ts +15 -0
- package/src/pi/models.ts +2 -2
- package/src/providers/alibaba-region-migration.ts +1 -1
- package/src/providers/antigravity-models.ts +521 -31
- package/src/providers/auto-compact-budget.ts +65 -0
- package/src/providers/base-url-choices.ts +10 -0
- package/src/providers/codex-capacity.ts +292 -0
- package/src/providers/command-code-efforts.ts +176 -0
- package/src/providers/context-cap.ts +26 -8
- package/src/providers/cursor-pool.ts +72 -0
- package/src/providers/default-aliases.ts +65 -0
- package/src/providers/derive.ts +281 -6
- package/src/providers/fastwire.ts +507 -0
- package/src/providers/free-directory.ts +10 -7
- package/src/providers/google-vertex-location.ts +14 -0
- package/src/providers/key-failover.ts +71 -3
- package/src/providers/label.ts +35 -2
- package/src/providers/model-discovery-limits.ts +16 -0
- package/src/providers/model-discovery.ts +115 -22
- package/src/providers/model-presets.ts +119 -0
- package/src/providers/model-rename-migration.ts +255 -0
- package/src/providers/model-rename-startup.ts +28 -0
- package/src/providers/new-model-policy.ts +146 -0
- package/src/providers/openai-sidecar.ts +72 -4
- package/src/providers/openai-tier-startup.ts +31 -2
- package/src/providers/openai-tiers.ts +119 -4
- package/src/providers/openai-virtual-models.ts +1 -0
- package/src/providers/opencode-zen-rate-limit.ts +102 -0
- package/src/providers/provider-id-rewrite.ts +30 -0
- package/src/providers/quota.ts +1379 -40
- package/src/providers/registry.ts +1545 -112
- package/src/providers/request-pacing.ts +310 -0
- package/src/providers/service-tier.ts +277 -0
- package/src/providers/slug-codec.ts +94 -6
- package/src/providers/static-model-discovery.ts +86 -0
- package/src/providers/xai-responses-opt-in.ts +15 -0
- package/src/providers/xai-transport.ts +32 -4
- package/src/reasoning-effort.ts +68 -3
- package/src/responses/apply-patch-envelope.ts +63 -0
- package/src/responses/code-mode-helper-compat.ts +50 -0
- package/src/responses/compaction.ts +26 -1
- package/src/responses/custom-tool-compat.ts +384 -0
- package/src/responses/hosted-tool-policy.ts +9 -0
- package/src/responses/namespace-tool-compat.ts +435 -0
- package/src/responses/parser.ts +225 -38
- package/src/responses/provider-continuation.ts +98 -0
- package/src/responses/provider-opaque-metadata.ts +73 -0
- package/src/responses/reasoning-envelope.ts +9 -1
- package/src/responses/reasoning-replay-cache.ts +504 -0
- package/src/responses/schema.ts +12 -2
- package/src/responses/spill-store.ts +75 -10
- package/src/responses/state.ts +659 -31
- package/src/responses/thought-signature-replay.ts +364 -0
- package/src/responses/tool-search-compat.ts +301 -0
- package/src/responses/truncated-stop-reason.ts +60 -0
- package/src/responses/turn-termination.ts +107 -0
- package/src/router.ts +405 -31
- package/src/routing/analytics.ts +379 -0
- package/src/routing/capability.ts +244 -0
- package/src/routing/compatibility/assemble.ts +73 -0
- package/src/routing/compatibility/behavior.ts +278 -0
- package/src/routing/compatibility/catalog.ts +99 -0
- package/src/routing/compatibility/endpoint.ts +52 -0
- package/src/routing/compatibility/lab-evidence-provider.ts +130 -0
- package/src/routing/compatibility/policy.ts +181 -0
- package/src/routing/compatibility/provider-slot.ts +56 -0
- package/src/routing/compatibility/reader.ts +110 -0
- package/src/routing/compatibility/subject.ts +191 -0
- package/src/routing/compatibility/types.ts +64 -0
- package/src/routing/compatibility/version.ts +104 -0
- package/src/routing/cost.ts +77 -0
- package/src/routing/evaluator.ts +495 -0
- package/src/routing/health.ts +412 -0
- package/src/routing/history/cursor.ts +43 -0
- package/src/routing/history/indexer.ts +605 -0
- package/src/routing/history/schema.ts +72 -0
- package/src/routing/profile-namespace.ts +15 -0
- package/src/routing/profile.ts +547 -0
- package/src/routing/quota.ts +155 -0
- package/src/routing/request-evidence.ts +45 -0
- package/src/routing/trace.ts +776 -0
- package/src/server/adapter-resolve.ts +2 -29
- package/src/server/auth-cors.ts +289 -48
- package/src/server/background-lifecycle.ts +182 -0
- package/src/server/chat-completions.ts +154 -72
- package/src/server/chat-native-sse.ts +331 -0
- package/src/server/chat-native.ts +445 -0
- package/src/server/claude-messages.ts +160 -44
- package/src/server/direct-local-http.ts +347 -0
- package/src/server/effort-policy.ts +18 -0
- package/src/server/github-copilot-responses-repair.ts +338 -0
- package/src/server/gui-static.ts +39 -10
- package/src/server/images.ts +94 -12
- package/src/server/index.ts +1021 -190
- package/src/server/lifecycle.ts +336 -14
- package/src/server/live.ts +136 -17
- package/src/server/local-management-read-client.ts +90 -0
- package/src/server/local-provider-reload-client.ts +137 -0
- package/src/server/management/agent-settings-routes.ts +433 -116
- package/src/server/management/api-key-usage.ts +31 -5
- package/src/server/management/body.ts +6 -0
- package/src/server/management/codex-prompt-routes.ts +570 -0
- package/src/server/management/combo-routes.ts +64 -25
- package/src/server/management/config-routes.ts +511 -51
- package/src/server/management/context.ts +89 -2
- package/src/server/management/integration-routes.ts +498 -0
- package/src/server/management/lab-automation-routes.ts +206 -0
- package/src/server/management/lab-routes.ts +563 -0
- package/src/server/management/logs-usage-routes.ts +127 -52
- package/src/server/management/model-routes.ts +461 -133
- package/src/server/management/model-rows.ts +167 -0
- package/src/server/management/native-integration-routes.ts +770 -0
- package/src/server/management/oauth-account-routes.ts +90 -8
- package/src/server/management/provider-capability-config.ts +48 -0
- package/src/server/management/provider-routes.ts +836 -159
- package/src/server/management/request-history-routes.ts +191 -0
- package/src/server/management/route-registry.ts +332 -0
- package/src/server/management/routing-analytics-routes.ts +74 -0
- package/src/server/management/routing-profile-routes.ts +380 -0
- package/src/server/management/shared.ts +27 -11
- package/src/server/management/sidebar-routes.ts +47 -31
- package/src/server/management/storage-log-guard-routes.ts +186 -0
- package/src/server/management/sync-response.ts +69 -0
- package/src/server/management/system-restart.ts +289 -33
- package/src/server/management/system-routes.ts +77 -0
- package/src/server/management/usage-summary-cache.ts +9 -1
- package/src/server/management/vision-sidecar-options.ts +167 -0
- package/src/server/management/web-search-sidecar-options.ts +120 -0
- package/src/server/management-api.ts +126 -23
- package/src/server/management-auth.ts +220 -5
- package/src/server/passive-route-linker.ts +66 -0
- package/src/server/port-reclaim.ts +1 -1
- package/src/server/ports.ts +41 -1
- package/src/server/proxy-liveness.ts +161 -10
- package/src/server/readiness.ts +99 -0
- package/src/server/relay-eager.ts +221 -54
- package/src/server/relay.ts +479 -87
- package/src/server/request-decompress.ts +113 -6
- package/src/server/request-log-conversation.ts +51 -0
- package/src/server/request-log.ts +298 -31
- package/src/server/responses/agent-task-recovery-cache.ts +143 -0
- package/src/server/responses/agent-task-recovery.ts +465 -0
- package/src/server/responses/codex-auth-error.ts +55 -0
- package/src/server/responses/collaboration.ts +239 -36
- package/src/server/responses/combo-stream-preflight.ts +178 -0
- package/src/server/responses/compact.ts +444 -72
- package/src/server/responses/core.ts +3579 -436
- package/src/server/responses/empty-completion-guard.ts +333 -0
- package/src/server/responses/encrypted-payload.ts +62 -39
- package/src/server/responses/fetch-helpers.ts +89 -99
- package/src/server/responses/input-admission.ts +187 -0
- package/src/server/responses/pacing-overload.ts +13 -0
- package/src/server/responses/passthrough-error.ts +33 -9
- package/src/server/responses/policy-fallback.ts +178 -0
- package/src/server/responses/responses-field-backfill.ts +343 -0
- package/src/server/responses/terminal-guard.ts +26 -5
- package/src/server/responses/upstream-error.ts +5 -0
- package/src/server/responses/ws-upstream.ts +381 -0
- package/src/server/responses-custom-tool-repair.ts +363 -0
- package/src/server/responses-item-id-repair.ts +54 -6
- package/src/server/responses-json-events.ts +90 -0
- package/src/server/responses-model-rewrite.ts +29 -0
- package/src/server/responses-reasoning-summary-rewrite.ts +178 -0
- package/src/server/responses-snapshot-repair.ts +621 -0
- package/src/server/responses-terminal-repair.ts +363 -0
- package/src/server/responses-tool-search-repair.ts +267 -0
- package/src/server/responses-undeclared-tool-guard.ts +376 -0
- package/src/server/responses.ts +18 -2
- package/src/server/search.ts +78 -13
- package/src/server/sse-frame-buffer.ts +319 -0
- package/src/server/sse-payload-rewrite.ts +110 -22
- package/src/server/startup-action-control.ts +8 -1
- package/src/server/startup-health-cache.ts +19 -1
- package/src/server/system-env.ts +80 -9
- package/src/server/ws-bridge.ts +53 -40
- package/src/service-manager-probe.ts +892 -0
- package/src/service.ts +1153 -94
- package/src/sidecar/auth.ts +92 -0
- package/src/sidecar/candidates.ts +83 -0
- package/src/storage/cleanup.ts +2 -2
- package/src/storage/policy-job.ts +14 -4
- package/src/storage/policy.ts +88 -23
- package/src/storage/scanner.ts +1 -1
- package/src/storage/worker-lifecycle.ts +14 -14
- package/src/tray/windows-tray.ps1 +83 -9
- package/src/tray/windows.ts +43 -16
- package/src/types/accounts.ts +37 -0
- package/src/types/config.ts +911 -0
- package/src/types/provider.ts +614 -0
- package/src/types/request.ts +397 -0
- package/src/types/tools.ts +236 -0
- package/src/types/wire.ts +80 -0
- package/src/types.ts +107 -1237
- package/src/update/index.ts +37 -23
- package/src/update/job.ts +445 -68
- package/src/update/notify.ts +12 -6
- package/src/update/npm-cache-preflight.d.mts +47 -0
- package/src/update/npm-cache-preflight.mjs +201 -0
- package/src/update/transactional-install.d.mts +22 -0
- package/src/update/transactional-install.mjs +266 -0
- package/src/usage/cost.ts +0 -0
- package/src/usage/expected-prices.ts +287 -16
- package/src/usage/log.ts +666 -50
- package/src/usage/summary.ts +388 -17
- package/src/usage/user-cost-overlay-reconciler.ts +313 -0
- package/src/usage/user-cost-overlays.ts +314 -0
- package/src/vision/anthropic-describe.ts +10 -6
- package/src/vision/backends.ts +97 -0
- package/src/vision/describe.ts +27 -16
- package/src/vision/eligibility.ts +250 -0
- package/src/vision/index.ts +238 -24
- package/src/vision/reasoning.ts +55 -0
- package/src/vision/routed-describe.ts +175 -0
- package/src/vision/timeout-bounds.ts +9 -0
- package/src/web-search/anthropic-executor.ts +13 -7
- package/src/web-search/backends.ts +108 -0
- package/src/web-search/exa-executor.ts +119 -0
- package/src/web-search/executor.ts +21 -6
- package/src/web-search/gemini-executor.ts +141 -0
- package/src/web-search/index.ts +150 -15
- package/src/web-search/loop.ts +284 -54
- package/src/web-search/parse.ts +125 -30
- package/src/web-search/sources.ts +60 -0
- package/src/web-search/xai-executor.ts +219 -0
- package/gui/dist/assets/index-DTpMHS4F.js +0 -67
- package/gui/dist/assets/index-ZNVDE3C7.css +0 -1
- package/gui/dist/provider-icons/antigravity.svg +0 -1
- package/gui/dist/provider-icons/claude.svg +0 -1
- package/gui/dist/provider-icons/copilot.svg +0 -1
- package/gui/dist/provider-icons/cursor.svg +0 -2
- package/gui/dist/provider-icons/gemini.svg +0 -1
- package/gui/dist/provider-icons/grok-color.svg +0 -1
- package/gui/dist/provider-icons/kiro.svg +0 -14
- package/src/cli/internal-dispatch.ts +0 -20
package/src/usage/log.ts
CHANGED
|
@@ -1,19 +1,58 @@
|
|
|
1
|
+
import { createHash, type Hash } from "node:crypto";
|
|
1
2
|
import { chmodSync, closeSync, existsSync, fstatSync, mkdirSync, openSync, readFileSync, readSync, appendFileSync } from "node:fs";
|
|
2
3
|
import { join } from "node:path";
|
|
3
4
|
import { getConfigDir } from "../config";
|
|
5
|
+
import { enforceAppOwnedMemoryBudget } from "../lib/app-owned-memory";
|
|
4
6
|
import { recordOwnedConfigPath } from "../lib/config-ownership";
|
|
7
|
+
import { sanitizeLogMetadataString } from "../lib/redact";
|
|
5
8
|
import { usageDisplayTotalTokens } from "./totals";
|
|
6
|
-
import type { OcxUsage } from "../types";
|
|
9
|
+
import type { AttemptTierOutcome, OcxUsage } from "../types";
|
|
10
|
+
import { normalizeRouteDecisionTrace, type RouteDecisionTraceV1 } from "../routing/trace";
|
|
11
|
+
import { ACCOUNT_LOG_LABEL_RE, CODEX_ACCOUNT_LOG_LABEL_RE } from "../codex/account-label";
|
|
7
12
|
|
|
8
13
|
export type UsageStatus = "reported" | "unreported" | "unsupported" | "estimated";
|
|
14
|
+
/**
|
|
15
|
+
* A persisted account label: a Codex pool account (`main`/`p<hex6>`) or a non-Codex OAuth
|
|
16
|
+
* provider account (`o<hex6>`, #2699).
|
|
17
|
+
*
|
|
18
|
+
* The old name `CodexUsageAccountLogLabel` is kept as an alias because it is exported and used
|
|
19
|
+
* across modules; the two predicates below are what callers should choose between.
|
|
20
|
+
*/
|
|
21
|
+
export type UsageAccountLogLabel = "main" | `p${string}` | `o${string}`;
|
|
22
|
+
export type CodexUsageAccountLogLabel = UsageAccountLogLabel;
|
|
23
|
+
|
|
24
|
+
/**
|
|
25
|
+
* Accepts EITHER label family. This is the predicate the persistence writers use, so widening
|
|
26
|
+
* it here is what stops six separate call sites from silently dropping an `o`-label -- including
|
|
27
|
+
* two in the live request path (`request-log.ts:972` and `:1187`).
|
|
28
|
+
*
|
|
29
|
+
* The name is unchanged deliberately: renaming it would touch every call site for no behavior,
|
|
30
|
+
* and the widened contract is what every one of those sites wanted.
|
|
31
|
+
*/
|
|
32
|
+
export function isCodexUsageAccountLogLabel(value: unknown): value is UsageAccountLogLabel {
|
|
33
|
+
return value === "main" || (typeof value === "string" && ACCOUNT_LOG_LABEL_RE.test(value));
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
/** Strictly a Codex pool label. Use when the Codex-only distinction actually matters. */
|
|
37
|
+
export function isCodexPoolAccountLogLabel(value: unknown): value is "main" | `p${string}` {
|
|
38
|
+
return value === "main" || (typeof value === "string" && CODEX_ACCOUNT_LOG_LABEL_RE.test(value));
|
|
39
|
+
}
|
|
9
40
|
|
|
41
|
+
/**
|
|
42
|
+
* Recovery kinds recorded per attempt in the usage log; the GUI renders localized labels
|
|
43
|
+
* for these wire values.
|
|
44
|
+
*/
|
|
10
45
|
export type AttemptRecoveryKind =
|
|
11
46
|
| "transient-5xx"
|
|
12
47
|
| "connection-reset"
|
|
13
48
|
| "oauth-401"
|
|
14
49
|
| "key-429"
|
|
50
|
+
| "rate-limit-429"
|
|
15
51
|
| "anthropic-oauth-429"
|
|
16
|
-
| "
|
|
52
|
+
| "oauth-account-429"
|
|
53
|
+
| "image-413"
|
|
54
|
+
| "opaque-blob-rejection"
|
|
55
|
+
| "empty-completion";
|
|
17
56
|
|
|
18
57
|
export interface PersistedUsageAttempt {
|
|
19
58
|
ordinal: number;
|
|
@@ -22,23 +61,43 @@ export interface PersistedUsageAttempt {
|
|
|
22
61
|
adapter: string;
|
|
23
62
|
status: number;
|
|
24
63
|
durationMs: number;
|
|
64
|
+
/**
|
|
65
|
+
* True only when the upstream stream died after its 200 head was committed,
|
|
66
|
+
* so the row must not meter as a success the client never received.
|
|
67
|
+
* Absent on ordinary attempts so old rows keep their exact shape.
|
|
68
|
+
*/
|
|
69
|
+
streamAborted?: boolean;
|
|
25
70
|
/** TTFT relative to THIS attempt's start (WP4); unset for non-streaming/tool-only. */
|
|
26
71
|
firstOutputMs?: number;
|
|
27
72
|
sendCount: number;
|
|
28
73
|
recoveryKinds: AttemptRecoveryKind[];
|
|
29
74
|
usageStatus: UsageStatus;
|
|
75
|
+
/**
|
|
76
|
+
* True when the proxy answered this turn locally and issued no upstream request. It travels on
|
|
77
|
+
* the attempt itself rather than as a `finishRequestAttempt` argument because that function is
|
|
78
|
+
* called from six places, and a new parameter would silently default to the wrong answer at any
|
|
79
|
+
* one of them that was missed. Absent on ordinary attempts so old rows keep their exact shape.
|
|
80
|
+
*/
|
|
81
|
+
locallyAnswered?: boolean;
|
|
82
|
+
/** Stable non-PII identity for the Codex pool account that served this attempt. */
|
|
83
|
+
accountLogLabel?: CodexUsageAccountLogLabel;
|
|
30
84
|
inputTokenEstimate?: number;
|
|
31
85
|
usage?: OcxUsage;
|
|
32
86
|
totalTokens?: number;
|
|
33
87
|
errorCode?: string;
|
|
88
|
+
/** Installation-local exact Compatibility Lab route-subject digest for this attempt. */
|
|
89
|
+
labRouteSubjectId?: string;
|
|
34
90
|
/** Target-specific reasoning intent and exact adapter-normalized wire parameter. */
|
|
35
91
|
requestedEffort?: string;
|
|
36
92
|
effectiveEffort?: string;
|
|
37
93
|
reasoningWireField?: string;
|
|
38
|
-
reasoningWireValue?: string | number;
|
|
94
|
+
reasoningWireValue?: string | number | boolean;
|
|
95
|
+
/** Adapter-produced tier fact for this physical attempt; absent on pre-B0 rows. */
|
|
96
|
+
tierOutcome?: AttemptTierOutcome;
|
|
39
97
|
}
|
|
40
98
|
|
|
41
99
|
export interface PersistedUsageEntry {
|
|
100
|
+
requestedAlias?: string;
|
|
42
101
|
requestId: string;
|
|
43
102
|
timestamp: number;
|
|
44
103
|
provider: string;
|
|
@@ -50,22 +109,30 @@ export interface PersistedUsageEntry {
|
|
|
50
109
|
admissionKind?: "configured" | "environment" | "loopback";
|
|
51
110
|
/** The inbound wire, not the client product — see `surface`. */
|
|
52
111
|
inboundProtocol?: "responses" | "chat" | "messages";
|
|
112
|
+
/** Stable non-PII identity for Codex Pool usage; absent for Direct/non-Codex traffic. */
|
|
113
|
+
accountLogLabel?: CodexUsageAccountLogLabel;
|
|
53
114
|
/** Best-effort chat/session correlation for Logs grouping (#330). */
|
|
54
115
|
conversationId?: string;
|
|
55
116
|
resolvedModel?: string;
|
|
56
117
|
requestedModel?: string;
|
|
118
|
+
/** Original bare helper model when the opt-in shadow-call route rewrote this request. */
|
|
119
|
+
shadowCallRewrittenFrom?: string;
|
|
57
120
|
/** Reasoning effort / service-tier metadata for GUI Logs after restart. */
|
|
58
121
|
requestedEffort?: string;
|
|
59
122
|
/** Adapter-normalized tier and exact upstream parameter emitted for this request. */
|
|
60
123
|
effectiveEffort?: string;
|
|
61
124
|
reasoningWireField?: string;
|
|
62
|
-
reasoningWireValue?: string | number;
|
|
125
|
+
reasoningWireValue?: string | number | boolean;
|
|
126
|
+
/** Raw caller tier captured before routing, sanitized and bounded for durable logs. */
|
|
127
|
+
callerServiceTier?: string;
|
|
63
128
|
requestedServiceTier?: string;
|
|
64
129
|
requestedSpeedLabel?: string;
|
|
65
130
|
configuredServiceTier?: string;
|
|
66
131
|
configuredSpeedLabel?: string;
|
|
67
132
|
modelSupportsServiceTier?: boolean;
|
|
68
133
|
responseServiceTier?: string;
|
|
134
|
+
/** Summary of the final physical attempt for dashboard consumers. */
|
|
135
|
+
tierOutcome?: AttemptTierOutcome;
|
|
69
136
|
status: number;
|
|
70
137
|
durationMs: number;
|
|
71
138
|
/** TTFT relative to the request start (WP4); unset for non-streaming/tool-only. */
|
|
@@ -81,6 +148,12 @@ export interface PersistedUsageEntry {
|
|
|
81
148
|
closeReason?: "terminal" | "client_cancel" | "non_stream" | "body_stall" | "body_overflow";
|
|
82
149
|
/** Already redacted + capped at capture (request-log.ts redactSecretString().slice(0,500)). */
|
|
83
150
|
upstreamError?: string;
|
|
151
|
+
/**
|
|
152
|
+
* Bounded route-decision trace (RI-01): why this provider/model/account was
|
|
153
|
+
* selected. Additive field; old rows without it parse unchanged. Never
|
|
154
|
+
* contains prompts, credentials, or hidden reasoning.
|
|
155
|
+
*/
|
|
156
|
+
routeDecision?: RouteDecisionTraceV1;
|
|
84
157
|
}
|
|
85
158
|
|
|
86
159
|
const KNOWN_USAGE_SURFACES = new Set<NonNullable<PersistedUsageEntry["surface"]>>([
|
|
@@ -118,8 +191,8 @@ export function isKnownInboundProtocol(value: unknown): value is NonNullable<Per
|
|
|
118
191
|
return typeof value === "string" && KNOWN_INBOUND_PROTOCOLS.has(value as NonNullable<PersistedUsageEntry["inboundProtocol"]>);
|
|
119
192
|
}
|
|
120
193
|
|
|
121
|
-
export function usageLogPath(): string {
|
|
122
|
-
return join(getConfigDir(), "usage.jsonl");
|
|
194
|
+
export function usageLogPath(configDir?: string): string {
|
|
195
|
+
return join(configDir ?? getConfigDir(), "usage.jsonl");
|
|
123
196
|
}
|
|
124
197
|
|
|
125
198
|
export function usageTotalTokens(usage: OcxUsage | undefined): number | undefined {
|
|
@@ -136,8 +209,20 @@ function isEstimatedUsageProvider(providerOrAdapter: string): boolean {
|
|
|
136
209
|
|| providerOrAdapter === "cursor" || providerOrAdapter.startsWith("cursor-");
|
|
137
210
|
}
|
|
138
211
|
|
|
139
|
-
export function usageForFinalLog(
|
|
212
|
+
export function usageForFinalLog(
|
|
213
|
+
provider: string,
|
|
214
|
+
usage: OcxUsage | undefined,
|
|
215
|
+
/**
|
|
216
|
+
* True when the proxy answered this turn locally and issued no upstream request. Such a turn's
|
|
217
|
+
* zero counts are EXACT, so the provider-wide estimated marking must not apply: Kiro and Cursor
|
|
218
|
+
* are marked estimated because their adapters can only guess a real inference's usage, and a
|
|
219
|
+
* turn with no inference has nothing to guess. Without this, a no-send turn is indistinguishable
|
|
220
|
+
* from a real one whose usage frame never arrived.
|
|
221
|
+
*/
|
|
222
|
+
locallyAnswered = false,
|
|
223
|
+
): OcxUsage | undefined {
|
|
140
224
|
if (!usage) return undefined;
|
|
225
|
+
if (locallyAnswered) return usage;
|
|
141
226
|
if (usage.estimated || isEstimatedUsageProvider(provider)) return { ...usage, estimated: true };
|
|
142
227
|
return usage;
|
|
143
228
|
}
|
|
@@ -173,8 +258,12 @@ const ATTEMPT_RECOVERY_KINDS = new Set<AttemptRecoveryKind>([
|
|
|
173
258
|
"connection-reset",
|
|
174
259
|
"oauth-401",
|
|
175
260
|
"key-429",
|
|
261
|
+
"rate-limit-429",
|
|
176
262
|
"anthropic-oauth-429",
|
|
263
|
+
"oauth-account-429",
|
|
177
264
|
"image-413",
|
|
265
|
+
"opaque-blob-rejection",
|
|
266
|
+
"empty-completion",
|
|
178
267
|
]);
|
|
179
268
|
const USAGE_STATUSES = new Set<UsageStatus>([
|
|
180
269
|
"reported",
|
|
@@ -182,6 +271,20 @@ const USAGE_STATUSES = new Set<UsageStatus>([
|
|
|
182
271
|
"unsupported",
|
|
183
272
|
"estimated",
|
|
184
273
|
]);
|
|
274
|
+
const LAB_ROUTE_SUBJECT_ID_RE = /^[0-9a-f]{64}$/;
|
|
275
|
+
const FAST_OUTCOMES = new Set<AttemptTierOutcome["fastOutcome"]>([
|
|
276
|
+
"not-requested", "applied", "downgraded", "unknown",
|
|
277
|
+
]);
|
|
278
|
+
const TIER_CONFIRMATIONS = new Set<AttemptTierOutcome["confirmation"]>([
|
|
279
|
+
"confirmed", "assumed", "downgraded", "unknown",
|
|
280
|
+
]);
|
|
281
|
+
const FAST_DOWNGRADE_REASONS = new Set<NonNullable<AttemptTierOutcome["fastDowngradeReason"]>>([
|
|
282
|
+
"route-unsupported", "wire-unavailable", "response-declined",
|
|
283
|
+
]);
|
|
284
|
+
|
|
285
|
+
export function isLabRouteSubjectId(value: unknown): value is string {
|
|
286
|
+
return typeof value === "string" && LAB_ROUTE_SUBJECT_ID_RE.test(value);
|
|
287
|
+
}
|
|
185
288
|
|
|
186
289
|
function isNonNegativeFiniteNumber(value: unknown): value is number {
|
|
187
290
|
return typeof value === "number" && Number.isFinite(value) && value >= 0;
|
|
@@ -206,6 +309,53 @@ function normalizeAttemptUsage(raw: unknown): OcxUsage | null {
|
|
|
206
309
|
return normalizeUsageValue(usage as unknown as OcxUsage) ?? null;
|
|
207
310
|
}
|
|
208
311
|
|
|
312
|
+
function normalizeAttemptTierOutcome(raw: unknown): AttemptTierOutcome | null {
|
|
313
|
+
if (!raw || typeof raw !== "object" || Array.isArray(raw)) return null;
|
|
314
|
+
const outcome = raw as Record<string, unknown>;
|
|
315
|
+
if (typeof outcome.fastOutcome !== "string"
|
|
316
|
+
|| !FAST_OUTCOMES.has(outcome.fastOutcome as AttemptTierOutcome["fastOutcome"])
|
|
317
|
+
|| typeof outcome.confirmation !== "string"
|
|
318
|
+
|| !TIER_CONFIRMATIONS.has(outcome.confirmation as AttemptTierOutcome["confirmation"])) {
|
|
319
|
+
return null;
|
|
320
|
+
}
|
|
321
|
+
if ("canonical" in outcome && outcome.canonical !== "priority") return null;
|
|
322
|
+
if ("wireKind" in outcome
|
|
323
|
+
&& outcome.wireKind !== null
|
|
324
|
+
&& outcome.wireKind !== "service-tier"
|
|
325
|
+
&& outcome.wireKind !== "anthropic-speed") return null;
|
|
326
|
+
if ("wireValue" in outcome && outcome.wireValue !== null && typeof outcome.wireValue !== "string") return null;
|
|
327
|
+
if ("fastDowngradeReason" in outcome
|
|
328
|
+
&& (typeof outcome.fastDowngradeReason !== "string"
|
|
329
|
+
|| !FAST_DOWNGRADE_REASONS.has(outcome.fastDowngradeReason as NonNullable<AttemptTierOutcome["fastDowngradeReason"]>))) {
|
|
330
|
+
return null;
|
|
331
|
+
}
|
|
332
|
+
if ("callerTierDropped" in outcome && typeof outcome.callerTierDropped !== "boolean") return null;
|
|
333
|
+
if ("callerFastSuppressedByConfig" in outcome
|
|
334
|
+
&& typeof outcome.callerFastSuppressedByConfig !== "boolean") return null;
|
|
335
|
+
if ("responseServiceTier" in outcome && typeof outcome.responseServiceTier !== "string") return null;
|
|
336
|
+
const wireValue = sanitizeLogMetadataString(outcome.wireValue);
|
|
337
|
+
const responseServiceTier = sanitizeLogMetadataString(outcome.responseServiceTier);
|
|
338
|
+
return {
|
|
339
|
+
...(outcome.canonical === "priority" ? { canonical: "priority" as const } : {}),
|
|
340
|
+
...(outcome.wireKind === null || outcome.wireKind === "service-tier" || outcome.wireKind === "anthropic-speed"
|
|
341
|
+
? { wireKind: outcome.wireKind }
|
|
342
|
+
: {}),
|
|
343
|
+
...(outcome.wireValue === null
|
|
344
|
+
? { wireValue: null }
|
|
345
|
+
: wireValue ? { wireValue } : {}),
|
|
346
|
+
fastOutcome: outcome.fastOutcome as AttemptTierOutcome["fastOutcome"],
|
|
347
|
+
...(typeof outcome.fastDowngradeReason === "string"
|
|
348
|
+
? { fastDowngradeReason: outcome.fastDowngradeReason as NonNullable<AttemptTierOutcome["fastDowngradeReason"]> }
|
|
349
|
+
: {}),
|
|
350
|
+
...(typeof outcome.callerTierDropped === "boolean" ? { callerTierDropped: outcome.callerTierDropped } : {}),
|
|
351
|
+
...(typeof outcome.callerFastSuppressedByConfig === "boolean"
|
|
352
|
+
? { callerFastSuppressedByConfig: outcome.callerFastSuppressedByConfig }
|
|
353
|
+
: {}),
|
|
354
|
+
confirmation: outcome.confirmation as AttemptTierOutcome["confirmation"],
|
|
355
|
+
...(responseServiceTier ? { responseServiceTier } : {}),
|
|
356
|
+
};
|
|
357
|
+
}
|
|
358
|
+
|
|
209
359
|
function normalizeUsageAttempt(raw: unknown): PersistedUsageAttempt | null {
|
|
210
360
|
if (!raw || typeof raw !== "object" || Array.isArray(raw)) return null;
|
|
211
361
|
const attempt = raw as Record<string, unknown>;
|
|
@@ -232,6 +382,9 @@ function normalizeUsageAttempt(raw: unknown): PersistedUsageAttempt | null {
|
|
|
232
382
|
&& !isNonNegativeFiniteNumber(attempt.totalTokens)) return null;
|
|
233
383
|
const usage = "usage" in attempt ? normalizeAttemptUsage(attempt.usage) : undefined;
|
|
234
384
|
if ("usage" in attempt && usage === null) return null;
|
|
385
|
+
const tierOutcome = "tierOutcome" in attempt
|
|
386
|
+
? normalizeAttemptTierOutcome(attempt.tierOutcome)
|
|
387
|
+
: undefined;
|
|
235
388
|
const recoveryKinds = Array.isArray(attempt.recoveryKinds)
|
|
236
389
|
? [...new Set(attempt.recoveryKinds.filter(
|
|
237
390
|
(value): value is AttemptRecoveryKind => typeof value === "string"
|
|
@@ -245,12 +398,17 @@ function normalizeUsageAttempt(raw: unknown): PersistedUsageAttempt | null {
|
|
|
245
398
|
adapter: attempt.adapter,
|
|
246
399
|
status: attempt.status,
|
|
247
400
|
durationMs: attempt.durationMs,
|
|
401
|
+
// Absent by default; only the literal `true` marker survives the round trip.
|
|
402
|
+
...(attempt.streamAborted === true ? { streamAborted: true } : {}),
|
|
248
403
|
...(isNonNegativeFiniteNumber(attempt.firstOutputMs)
|
|
249
404
|
? { firstOutputMs: attempt.firstOutputMs }
|
|
250
405
|
: {}),
|
|
251
406
|
sendCount: attempt.sendCount as number,
|
|
252
407
|
recoveryKinds,
|
|
253
408
|
usageStatus: attempt.usageStatus as UsageStatus,
|
|
409
|
+
...(isCodexUsageAccountLogLabel(attempt.accountLogLabel)
|
|
410
|
+
? { accountLogLabel: attempt.accountLogLabel }
|
|
411
|
+
: {}),
|
|
254
412
|
...(isNonNegativeFiniteNumber(attempt.inputTokenEstimate)
|
|
255
413
|
? { inputTokenEstimate: attempt.inputTokenEstimate }
|
|
256
414
|
: {}),
|
|
@@ -259,6 +417,9 @@ function normalizeUsageAttempt(raw: unknown): PersistedUsageAttempt | null {
|
|
|
259
417
|
? { totalTokens: attempt.totalTokens }
|
|
260
418
|
: {}),
|
|
261
419
|
...(typeof attempt.errorCode === "string" ? { errorCode: attempt.errorCode } : {}),
|
|
420
|
+
...(isLabRouteSubjectId(attempt.labRouteSubjectId)
|
|
421
|
+
? { labRouteSubjectId: attempt.labRouteSubjectId }
|
|
422
|
+
: {}),
|
|
262
423
|
...(typeof attempt.requestedEffort === "string" && attempt.requestedEffort
|
|
263
424
|
? { requestedEffort: capMetadataString(attempt.requestedEffort) }
|
|
264
425
|
: {}),
|
|
@@ -268,14 +429,30 @@ function normalizeUsageAttempt(raw: unknown): PersistedUsageAttempt | null {
|
|
|
268
429
|
...(typeof attempt.reasoningWireField === "string" && attempt.reasoningWireField
|
|
269
430
|
? { reasoningWireField: capMetadataString(attempt.reasoningWireField) }
|
|
270
431
|
: {}),
|
|
271
|
-
...(
|
|
272
|
-
?
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
432
|
+
...(isValidReasoningWireValue(attempt.reasoningWireField, attempt.reasoningWireValue)
|
|
433
|
+
? typeof attempt.reasoningWireValue === "string"
|
|
434
|
+
? { reasoningWireValue: capMetadataString(attempt.reasoningWireValue) }
|
|
435
|
+
: { reasoningWireValue: attempt.reasoningWireValue }
|
|
436
|
+
: {}),
|
|
437
|
+
...(tierOutcome ? { tierOutcome } : {}),
|
|
276
438
|
};
|
|
277
439
|
}
|
|
278
440
|
|
|
441
|
+
/**
|
|
442
|
+
* Pairing rule for reasoning diagnostics, shared with the live request-log capture path:
|
|
443
|
+
* a non-empty string, a non-negative finite number, or a boolean only for
|
|
444
|
+
* `reasoning.enabled`. The field name itself is validated separately at capture time;
|
|
445
|
+
* persisted rows may carry legacy field names, so this checks only the value shape.
|
|
446
|
+
*/
|
|
447
|
+
export function isValidReasoningWireValue(
|
|
448
|
+
wireField: unknown,
|
|
449
|
+
wireValue: unknown,
|
|
450
|
+
): wireValue is string | number | boolean {
|
|
451
|
+
return (typeof wireValue === "string" && wireValue.length > 0)
|
|
452
|
+
|| (typeof wireValue === "number" && Number.isFinite(wireValue) && wireValue >= 0)
|
|
453
|
+
|| (wireField === "reasoning.enabled" && typeof wireValue === "boolean");
|
|
454
|
+
}
|
|
455
|
+
|
|
279
456
|
function normalizedAttempts(raw: unknown): PersistedUsageAttempt[] {
|
|
280
457
|
if (!Array.isArray(raw)) return [];
|
|
281
458
|
return raw.map(normalizeUsageAttempt)
|
|
@@ -294,6 +471,13 @@ export function normalizeUsageEntryForTest(entry: PersistedUsageEntry): Persiste
|
|
|
294
471
|
|
|
295
472
|
function normalizeUsageEntry(entry: PersistedUsageEntry): PersistedUsageEntry {
|
|
296
473
|
const attempts = normalizedAttempts(entry.attempts);
|
|
474
|
+
const tierOutcome = entry.tierOutcome ? normalizeAttemptTierOutcome(entry.tierOutcome) : undefined;
|
|
475
|
+
const callerServiceTier = sanitizeLogMetadataString(entry.callerServiceTier);
|
|
476
|
+
const responseServiceTier = sanitizeLogMetadataString(entry.responseServiceTier);
|
|
477
|
+
const shadowCallRewrittenFrom = sanitizeLogMetadataString(entry.shadowCallRewrittenFrom);
|
|
478
|
+
const routeDecision = entry.routeDecision
|
|
479
|
+
? normalizeRouteDecisionTrace(entry.routeDecision)
|
|
480
|
+
: undefined;
|
|
297
481
|
return {
|
|
298
482
|
requestId: entry.requestId,
|
|
299
483
|
timestamp: entry.timestamp,
|
|
@@ -309,11 +493,15 @@ function normalizeUsageEntry(entry: PersistedUsageEntry): PersistedUsageEntry {
|
|
|
309
493
|
: {}),
|
|
310
494
|
...(isKnownAdmissionKind(entry.admissionKind) ? { admissionKind: entry.admissionKind } : {}),
|
|
311
495
|
...(isKnownInboundProtocol(entry.inboundProtocol) ? { inboundProtocol: entry.inboundProtocol } : {}),
|
|
496
|
+
...(isCodexUsageAccountLogLabel(entry.accountLogLabel)
|
|
497
|
+
? { accountLogLabel: entry.accountLogLabel }
|
|
498
|
+
: {}),
|
|
312
499
|
...(typeof entry.conversationId === "string" && entry.conversationId.trim()
|
|
313
500
|
? { conversationId: entry.conversationId.trim().slice(0, 128) }
|
|
314
501
|
: {}),
|
|
315
502
|
...(entry.resolvedModel ? { resolvedModel: entry.resolvedModel } : {}),
|
|
316
503
|
...(entry.requestedModel ? { requestedModel: entry.requestedModel } : {}),
|
|
504
|
+
...(shadowCallRewrittenFrom ? { shadowCallRewrittenFrom } : {}),
|
|
317
505
|
...(typeof entry.requestedEffort === "string" && entry.requestedEffort
|
|
318
506
|
? { requestedEffort: capMetadataString(entry.requestedEffort) }
|
|
319
507
|
: {}),
|
|
@@ -323,11 +511,12 @@ function normalizeUsageEntry(entry: PersistedUsageEntry): PersistedUsageEntry {
|
|
|
323
511
|
...(typeof entry.reasoningWireField === "string" && entry.reasoningWireField
|
|
324
512
|
? { reasoningWireField: capMetadataString(entry.reasoningWireField) }
|
|
325
513
|
: {}),
|
|
326
|
-
...(
|
|
327
|
-
?
|
|
328
|
-
|
|
329
|
-
|
|
330
|
-
|
|
514
|
+
...(isValidReasoningWireValue(entry.reasoningWireField, entry.reasoningWireValue)
|
|
515
|
+
? typeof entry.reasoningWireValue === "string"
|
|
516
|
+
? { reasoningWireValue: capMetadataString(entry.reasoningWireValue) }
|
|
517
|
+
: { reasoningWireValue: entry.reasoningWireValue }
|
|
518
|
+
: {}),
|
|
519
|
+
...(callerServiceTier ? { callerServiceTier } : {}),
|
|
331
520
|
...(typeof entry.requestedServiceTier === "string" && entry.requestedServiceTier
|
|
332
521
|
? { requestedServiceTier: capMetadataString(entry.requestedServiceTier) }
|
|
333
522
|
: {}),
|
|
@@ -343,9 +532,8 @@ function normalizeUsageEntry(entry: PersistedUsageEntry): PersistedUsageEntry {
|
|
|
343
532
|
...(typeof entry.modelSupportsServiceTier === "boolean"
|
|
344
533
|
? { modelSupportsServiceTier: entry.modelSupportsServiceTier }
|
|
345
534
|
: {}),
|
|
346
|
-
...(
|
|
347
|
-
|
|
348
|
-
: {}),
|
|
535
|
+
...(responseServiceTier ? { responseServiceTier } : {}),
|
|
536
|
+
...(tierOutcome ? { tierOutcome } : {}),
|
|
349
537
|
status: entry.status,
|
|
350
538
|
durationMs: entry.durationMs,
|
|
351
539
|
...(isNonNegativeFiniteNumber(entry.firstOutputMs)
|
|
@@ -354,11 +542,12 @@ function normalizeUsageEntry(entry: PersistedUsageEntry): PersistedUsageEntry {
|
|
|
354
542
|
usageStatus: entry.usageStatus,
|
|
355
543
|
...(entry.usage ? { usage: normalizeUsageValue(entry.usage) } : {}),
|
|
356
544
|
...(typeof entry.totalTokens === "number" ? { totalTokens: entry.totalTokens } : {}),
|
|
357
|
-
...(attempts
|
|
545
|
+
...(Array.isArray(entry.attempts) ? { attempts } : {}),
|
|
358
546
|
...(entry.errorCode ? { errorCode: entry.errorCode } : {}),
|
|
359
547
|
...(entry.terminalStatus ? { terminalStatus: entry.terminalStatus } : {}),
|
|
360
548
|
...(entry.closeReason ? { closeReason: entry.closeReason } : {}),
|
|
361
549
|
...(entry.upstreamError ? { upstreamError: entry.upstreamError } : {}),
|
|
550
|
+
...(routeDecision ? { routeDecision } : {}),
|
|
362
551
|
};
|
|
363
552
|
}
|
|
364
553
|
|
|
@@ -388,8 +577,18 @@ export type UsageLogRevision = {
|
|
|
388
577
|
|
|
389
578
|
let usageReadCacheStats = { fullReads: 0, tailReads: 0, parsedLines: 0 };
|
|
390
579
|
const MANAGEMENT_USAGE_MAX_READ_BYTES = 64 * 1024 * 1024;
|
|
580
|
+
const RECENT_USAGE_MAX_READ_BYTES = 64 * 1024 * 1024;
|
|
391
581
|
const MANAGEMENT_USAGE_READ_CHUNK_BYTES = 1024 * 1024;
|
|
392
|
-
const
|
|
582
|
+
const MANAGEMENT_USAGE_MAX_ENTRIES_DEFAULT = 500_000;
|
|
583
|
+
/**
|
|
584
|
+
* Row cap for a management snapshot. Overridable only so tests can reach the cap without
|
|
585
|
+
* building a half-million-row fixture; production always uses the default.
|
|
586
|
+
*/
|
|
587
|
+
let MANAGEMENT_USAGE_MAX_ENTRIES = MANAGEMENT_USAGE_MAX_ENTRIES_DEFAULT;
|
|
588
|
+
|
|
589
|
+
export function setManagementUsageMaxEntriesForTests(value: number | null): void {
|
|
590
|
+
MANAGEMENT_USAGE_MAX_ENTRIES = value ?? MANAGEMENT_USAGE_MAX_ENTRIES_DEFAULT;
|
|
591
|
+
}
|
|
393
592
|
const MANAGEMENT_USAGE_FLIGHT_STALE_MS = 30_000;
|
|
394
593
|
export interface ManagementUsageSnapshot {
|
|
395
594
|
entries: PersistedUsageEntry[];
|
|
@@ -397,14 +596,156 @@ export interface ManagementUsageSnapshot {
|
|
|
397
596
|
truncatedPrefixBytes: number;
|
|
398
597
|
entriesTruncated: boolean;
|
|
399
598
|
entriesDropped: number;
|
|
599
|
+
/** Digest of the covered prefix, used to detect an in-place rewrite before reuse. */
|
|
600
|
+
prefixDigest: string;
|
|
601
|
+
/**
|
|
602
|
+
* Byte offset where the RETAINED ROWS begin.
|
|
603
|
+
*
|
|
604
|
+
* Distinct from `truncatedPrefixBytes`, which is the API-visible "bytes skipped by the
|
|
605
|
+
* byte window" signal and must stay independent of entry-count truncation. When the
|
|
606
|
+
* entry cap drops rows, those bytes are not window truncation, but the retained rows do
|
|
607
|
+
* start later -- this field tracks that so the byte accounting stays exact.
|
|
608
|
+
*/
|
|
609
|
+
rowsBeginAtBytes: number;
|
|
610
|
+
/**
|
|
611
|
+
* Byte length of each returned row, in order, including its newline.
|
|
612
|
+
*
|
|
613
|
+
* Lets a later read trim rows that have fallen out of the bounded window without
|
|
614
|
+
* re-reading or re-parsing them, which is what keeps the returned set equal to the
|
|
615
|
+
* window the caller asked for.
|
|
616
|
+
*/
|
|
617
|
+
entryLengths: number[];
|
|
618
|
+
/** Unparseable bytes after the last returned row; the next read folds them forward. */
|
|
619
|
+
trailingSkippedBytes: number;
|
|
400
620
|
}
|
|
401
621
|
let managementUsageReadInflight: {
|
|
402
622
|
key: string;
|
|
623
|
+
openedSize: number;
|
|
403
624
|
promise: Promise<ManagementUsageSnapshot>;
|
|
404
625
|
startedAt: number;
|
|
405
626
|
abort: AbortController;
|
|
406
627
|
} | null = null;
|
|
407
628
|
|
|
629
|
+
/**
|
|
630
|
+
* Append-tolerant snapshot of the last management read.
|
|
631
|
+
*
|
|
632
|
+
* The management reader parses a 64 MiB tail into ~53k objects, which costs roughly
|
|
633
|
+
* 640 MB of transient RSS per cold call. The JS objects are collected promptly, but
|
|
634
|
+
* the allocator does not return those pages, so every cold miss ratchets process RSS
|
|
635
|
+
* upward and never comes back down (observed: 7.9 GiB RSS against a 130 MB JS heap).
|
|
636
|
+
*
|
|
637
|
+
* Reparsing an unchanged prefix is what makes that transient recur. `usage.jsonl` is
|
|
638
|
+
* append-only under a stable identity, so when the file has only grown we keep the
|
|
639
|
+
* previously parsed rows and parse just the appended bytes. This is retained state, so
|
|
640
|
+
* it is registered with the app-owned memory budget and is evictable under pressure.
|
|
641
|
+
*/
|
|
642
|
+
interface RetainedUsageSnapshot {
|
|
643
|
+
identityKey: string;
|
|
644
|
+
maxReadBytes: number;
|
|
645
|
+
/** Absolute end offset in the file that `entries` already covers. */
|
|
646
|
+
coveredThroughBytes: number;
|
|
647
|
+
/**
|
|
648
|
+
* Digest of the last bytes of the covered prefix, re-verified before extending.
|
|
649
|
+
*
|
|
650
|
+
* Identity (path/dev/ino/birthtime) intentionally ignores size and mtime so appends
|
|
651
|
+
* can share work, which also means an in-place rewrite that keeps the inode is
|
|
652
|
+
* invisible to it. A hand-edit or external compaction can therefore replace history
|
|
653
|
+
* under a stable identity without shrinking the file. Re-reading this trailing window
|
|
654
|
+
* catches that: if the bytes behind `coveredThroughBytes` changed, the retained rows
|
|
655
|
+
* no longer describe the file and must not be extended.
|
|
656
|
+
*/
|
|
657
|
+
prefixDigest: string;
|
|
658
|
+
/** Bytes of the file skipped ahead of the retained window. */
|
|
659
|
+
truncatedPrefixBytes: number;
|
|
660
|
+
/** Byte length of each retained row, so out-of-window rows can be trimmed exactly. */
|
|
661
|
+
entryLengths: number[];
|
|
662
|
+
/** Unparseable bytes after the last retained row; folded into the next row's span. */
|
|
663
|
+
trailingSkippedBytes: number;
|
|
664
|
+
/** Byte offset where the retained rows begin; see ManagementUsageSnapshot. */
|
|
665
|
+
rowsBeginAtBytes: number;
|
|
666
|
+
entries: PersistedUsageEntry[];
|
|
667
|
+
entriesTruncated: boolean;
|
|
668
|
+
entriesDropped: number;
|
|
669
|
+
revision: UsageLogRevision;
|
|
670
|
+
retainedAt: number;
|
|
671
|
+
approxBytes: number;
|
|
672
|
+
}
|
|
673
|
+
let retainedUsageSnapshot: RetainedUsageSnapshot | null = null;
|
|
674
|
+
|
|
675
|
+
/** Rough per-row retained cost; exact sizing would cost another full serialization pass. */
|
|
676
|
+
const RETAINED_USAGE_ENTRY_BYTES = 512;
|
|
677
|
+
|
|
678
|
+
/** Chunk size used when digesting a retained region. */
|
|
679
|
+
const RETAINED_USAGE_DIGEST_CHUNK_BYTES = 1024 * 1024;
|
|
680
|
+
|
|
681
|
+
/**
|
|
682
|
+
* Digest a byte range into `hash`; false when it cannot be read.
|
|
683
|
+
*
|
|
684
|
+
* Deliberately not sampled. A sampled digest covers a vanishing fraction of a large
|
|
685
|
+
* prefix (32 KiB of 64 MiB is 0.05%), so an ordinary fixed-width in-place edit -- a
|
|
686
|
+
* redaction script fixing one field, a compaction rewriting a middle region -- lands in
|
|
687
|
+
* a gap by default and the stale rows are served.
|
|
688
|
+
*
|
|
689
|
+
* Hashing from byte 0 on every call is also wrong: that is O(file) per poll while the
|
|
690
|
+
* read it protects is capped at maxReadBytes, so the ratio degrades as the ledger grows
|
|
691
|
+
* and becomes SLOWER than a full read past roughly 1-2 GB. Only the retained REGION
|
|
692
|
+
* (truncatedPrefixBytes..coveredThroughBytes) is hashed. It is never wider than the
|
|
693
|
+
* window, and bytes below the retained start describe no retained row, so reading them
|
|
694
|
+
* would prove nothing.
|
|
695
|
+
*/
|
|
696
|
+
function updateUsageDigest(hash: Hash, fd: number, from: number, to: number): boolean {
|
|
697
|
+
if (to <= from) return true;
|
|
698
|
+
const buffer = Buffer.allocUnsafe(Math.min(RETAINED_USAGE_DIGEST_CHUNK_BYTES, to - from));
|
|
699
|
+
for (let position = from; position < to;) {
|
|
700
|
+
const length = Math.min(buffer.byteLength, to - position);
|
|
701
|
+
let offset = 0;
|
|
702
|
+
while (offset < length) {
|
|
703
|
+
const read = readSync(fd, buffer, offset, length - offset, position + offset);
|
|
704
|
+
if (read === 0) return false;
|
|
705
|
+
offset += read;
|
|
706
|
+
}
|
|
707
|
+
hash.update(buffer.subarray(0, length));
|
|
708
|
+
position += length;
|
|
709
|
+
}
|
|
710
|
+
return true;
|
|
711
|
+
}
|
|
712
|
+
|
|
713
|
+
/**
|
|
714
|
+
* Digest of `from`..`to`; null when it cannot be read.
|
|
715
|
+
*
|
|
716
|
+
* The range is bound into the digest so a region cannot be confused with an equal-length
|
|
717
|
+
* region at a different offset.
|
|
718
|
+
*/
|
|
719
|
+
function usageRegionDigest(fd: number, from: number, to: number): string | null {
|
|
720
|
+
if (to <= from) return `${from}:${to}:empty`;
|
|
721
|
+
const hash = createHash("sha256");
|
|
722
|
+
if (!updateUsageDigest(hash, fd, from, to)) return null;
|
|
723
|
+
return `${from}:${to}:${hash.digest("hex")}`;
|
|
724
|
+
}
|
|
725
|
+
|
|
726
|
+
function retainedUsageSnapshotBytes(entries: PersistedUsageEntry[]): number {
|
|
727
|
+
return entries.length * RETAINED_USAGE_ENTRY_BYTES;
|
|
728
|
+
}
|
|
729
|
+
|
|
730
|
+
export function discardRetainedUsageSnapshot(): number {
|
|
731
|
+
const released = retainedUsageSnapshot?.approxBytes ?? 0;
|
|
732
|
+
retainedUsageSnapshot = null;
|
|
733
|
+
return released;
|
|
734
|
+
}
|
|
735
|
+
|
|
736
|
+
export function retainedUsageSnapshotStats(): {
|
|
737
|
+
count: number;
|
|
738
|
+
bytes: number;
|
|
739
|
+
oldestAt: number | null;
|
|
740
|
+
} {
|
|
741
|
+
if (!retainedUsageSnapshot) return { count: 0, bytes: 0, oldestAt: null };
|
|
742
|
+
return {
|
|
743
|
+
count: 1,
|
|
744
|
+
bytes: retainedUsageSnapshot.approxBytes,
|
|
745
|
+
oldestAt: retainedUsageSnapshot.retainedAt,
|
|
746
|
+
};
|
|
747
|
+
}
|
|
748
|
+
|
|
408
749
|
/** Test-only observability for proving that unchanged prefixes are not reparsed. */
|
|
409
750
|
export function usageReadCacheStatsForTests(): Readonly<typeof usageReadCacheStats> {
|
|
410
751
|
return { ...usageReadCacheStats };
|
|
@@ -414,6 +755,7 @@ export function resetUsageReadCacheForTests(): void {
|
|
|
414
755
|
usageReadCacheStats = { fullReads: 0, tailReads: 0, parsedLines: 0 };
|
|
415
756
|
managementUsageReadInflight?.abort.abort();
|
|
416
757
|
managementUsageReadInflight = null;
|
|
758
|
+
retainedUsageSnapshot = null;
|
|
417
759
|
}
|
|
418
760
|
|
|
419
761
|
function readExactly(fd: number, length: number, position: number): Buffer | null {
|
|
@@ -453,6 +795,12 @@ export function usageLogRevisionKey(revision: UsageLogRevision | null): string {
|
|
|
453
795
|
].join("\0");
|
|
454
796
|
}
|
|
455
797
|
|
|
798
|
+
/** Identity of the usage ledger file, excluding size/mtime/ctime so appends can share work. */
|
|
799
|
+
export function usageLogIdentityKey(revision: UsageLogRevision | null): string {
|
|
800
|
+
if (!revision) return "missing";
|
|
801
|
+
return [revision.path, revision.dev, revision.ino, revision.birthtimeMs].join("\0");
|
|
802
|
+
}
|
|
803
|
+
|
|
456
804
|
export function currentUsageLogRevision(): UsageLogRevision | null {
|
|
457
805
|
const path = usageLogPath();
|
|
458
806
|
if (!existsSync(path)) return null;
|
|
@@ -468,23 +816,71 @@ export function currentUsageLogRevision(): UsageLogRevision | null {
|
|
|
468
816
|
async function parseUsageTextCooperatively(text: string, signal: AbortSignal): Promise<{
|
|
469
817
|
entries: PersistedUsageEntry[];
|
|
470
818
|
entriesDropped: number;
|
|
819
|
+
entryLengths: number[];
|
|
820
|
+
/** Bytes of unparseable lines after the final accepted row. */
|
|
821
|
+
trailingSkippedBytes: number;
|
|
822
|
+
/** Bytes of rows removed by the entry cap, which move into the skipped prefix. */
|
|
823
|
+
cappedPrefixBytes: number;
|
|
471
824
|
}> {
|
|
472
|
-
|
|
825
|
+
// Split on "\n" only, so a CRLF line keeps its "\r" and its byte length stays exact.
|
|
826
|
+
// Splitting on /\r?\n/ consumes two bytes but leaves no way to tell that it did, which
|
|
827
|
+
// made the recorded lengths short by one byte per line on a CRLF ledger and failed the
|
|
828
|
+
// accounting self-check. JSON.parse tolerates the trailing "\r".
|
|
829
|
+
const lines = text.split("\n");
|
|
473
830
|
usageReadCacheStats.parsedLines += lines.filter(line => line.trim()).length;
|
|
474
831
|
const entries: PersistedUsageEntry[] = [];
|
|
832
|
+
// Byte length of each accepted row including its newline, so a later read can trim
|
|
833
|
+
// rows that fall out of the bounded window without re-reading the file.
|
|
834
|
+
const entryLengths: number[] = [];
|
|
475
835
|
const batchSize = 1_000;
|
|
836
|
+
// Bytes of lines that did not yield an entry (malformed JSON, missing requestId, a
|
|
837
|
+
// torn final write). They still occupy space in the file, so they are folded into the
|
|
838
|
+
// next accepted row's recorded length. Dropping them would make the recorded lengths
|
|
839
|
+
// sum to less than the real byte span, and the window trim -- which walks forward by
|
|
840
|
+
// summing those lengths -- would consume extra rows to reach the window start,
|
|
841
|
+
// silently hiding history and desynchronizing truncatedPrefixBytes.
|
|
842
|
+
let pendingSkippedBytes = 0;
|
|
476
843
|
for (let offset = 0; offset < lines.length; offset += batchSize) {
|
|
477
844
|
if (signal.aborted) throw signal.reason;
|
|
478
|
-
|
|
845
|
+
const batch = lines.slice(offset, offset + batchSize);
|
|
846
|
+
for (let index = 0; index < batch.length; index++) {
|
|
847
|
+
const line = batch[index]!;
|
|
848
|
+
// The split leaves a trailing "" after the final newline; it occupies no bytes.
|
|
849
|
+
const isLastLine = offset + index === lines.length - 1;
|
|
850
|
+
const lineBytes = Buffer.byteLength(line, "utf-8") + (isLastLine && line === "" ? 0 : 1);
|
|
851
|
+
const parsed = parseUsageLines([line]);
|
|
852
|
+
if (parsed.length === 0) {
|
|
853
|
+
pendingSkippedBytes += lineBytes;
|
|
854
|
+
continue;
|
|
855
|
+
}
|
|
856
|
+
entries.push(parsed[0]!);
|
|
857
|
+
entryLengths.push(lineBytes + pendingSkippedBytes);
|
|
858
|
+
pendingSkippedBytes = 0;
|
|
859
|
+
}
|
|
479
860
|
if (offset + batchSize < lines.length) {
|
|
480
861
|
// JSON parsing dominates large-log startup. Yield between bounded batches so
|
|
481
862
|
// Bun can continue serving health and settings requests on the same thread.
|
|
482
863
|
await new Promise<void>(resolve => setTimeout(resolve, 0));
|
|
483
864
|
}
|
|
484
865
|
}
|
|
485
|
-
|
|
866
|
+
// Skipped bytes AFTER the last accepted row belong to no entry length, so report them
|
|
867
|
+
// separately; the trim arithmetic adds them back to keep lengths summing to the span.
|
|
868
|
+
const trailingSkippedBytes = pendingSkippedBytes;
|
|
869
|
+
if (entries.length <= MANAGEMENT_USAGE_MAX_ENTRIES) {
|
|
870
|
+
return { entries, entriesDropped: 0, entryLengths, trailingSkippedBytes, cappedPrefixBytes: 0 };
|
|
871
|
+
}
|
|
486
872
|
const entriesDropped = entries.length - MANAGEMENT_USAGE_MAX_ENTRIES;
|
|
487
|
-
return {
|
|
873
|
+
return {
|
|
874
|
+
entries: entries.slice(-MANAGEMENT_USAGE_MAX_ENTRIES),
|
|
875
|
+
entriesDropped,
|
|
876
|
+
entryLengths: entryLengths.slice(-MANAGEMENT_USAGE_MAX_ENTRIES),
|
|
877
|
+
trailingSkippedBytes,
|
|
878
|
+
// Bytes of the rows the cap removed. The caller adds them to its skipped prefix so
|
|
879
|
+
// the recorded lengths keep summing to the byte span they describe.
|
|
880
|
+
cappedPrefixBytes: entryLengths
|
|
881
|
+
.slice(0, entryLengths.length - MANAGEMENT_USAGE_MAX_ENTRIES)
|
|
882
|
+
.reduce((total, length) => total + length, 0),
|
|
883
|
+
};
|
|
488
884
|
}
|
|
489
885
|
|
|
490
886
|
async function readUsageEntriesFullCooperatively(
|
|
@@ -525,12 +921,185 @@ async function readUsageEntriesFullCooperatively(
|
|
|
525
921
|
}
|
|
526
922
|
const parsed = await parseUsageTextCooperatively(bytes.toString("utf-8"), signal);
|
|
527
923
|
usageReadCacheStats.fullReads += 1;
|
|
924
|
+
// Rows removed by the entry cap start the retained rows later in the file. That is
|
|
925
|
+
// NOT byte-window truncation, so it must not move truncatedPrefixBytes -- the two
|
|
926
|
+
// signals are independent in the API. It is tracked separately for the byte
|
|
927
|
+
// accounting the incremental reader relies on.
|
|
928
|
+
const rowsBeginAtBytes = truncatedPrefixBytes + parsed.cappedPrefixBytes;
|
|
929
|
+
// Digest the exact prefix these rows describe, so a later incremental read can
|
|
930
|
+
// prove the file was appended to rather than rewritten under the same inode.
|
|
931
|
+
const prefixDigest = usageRegionDigest(fd, rowsBeginAtBytes, Number(stat.size));
|
|
932
|
+
if (prefixDigest === null) throw new Error("usage log changed while it was being read");
|
|
528
933
|
return {
|
|
529
934
|
entries: parsed.entries,
|
|
530
935
|
revision: usageLogRevision(path, stat),
|
|
531
936
|
truncatedPrefixBytes,
|
|
532
937
|
entriesTruncated: parsed.entriesDropped > 0,
|
|
533
938
|
entriesDropped: parsed.entriesDropped,
|
|
939
|
+
prefixDigest,
|
|
940
|
+
entryLengths: parsed.entryLengths,
|
|
941
|
+
trailingSkippedBytes: parsed.trailingSkippedBytes,
|
|
942
|
+
rowsBeginAtBytes,
|
|
943
|
+
};
|
|
944
|
+
} finally {
|
|
945
|
+
if (fd !== undefined) closeSync(fd);
|
|
946
|
+
}
|
|
947
|
+
}
|
|
948
|
+
|
|
949
|
+
/**
|
|
950
|
+
* Parse only the bytes appended since the retained snapshot's covered offset.
|
|
951
|
+
*
|
|
952
|
+
* Returns null when the retained snapshot cannot be extended safely — a different
|
|
953
|
+
* identity or read window, a file that shrank (replacement/truncation), or a covered
|
|
954
|
+
* offset that no longer sits on a record boundary. Callers then fall back to a full
|
|
955
|
+
* bounded read.
|
|
956
|
+
*/
|
|
957
|
+
async function readUsageEntriesIncrementally(
|
|
958
|
+
path: string,
|
|
959
|
+
signal: AbortSignal,
|
|
960
|
+
maxReadBytes: number,
|
|
961
|
+
retained: RetainedUsageSnapshot,
|
|
962
|
+
): Promise<ManagementUsageSnapshot | null> {
|
|
963
|
+
let fd: number | undefined;
|
|
964
|
+
try {
|
|
965
|
+
fd = openSync(path, "r");
|
|
966
|
+
const stat = fstatSync(fd);
|
|
967
|
+
const revision = usageLogRevision(path, stat);
|
|
968
|
+
if (usageLogIdentityKey(revision) !== retained.identityKey) return null;
|
|
969
|
+
const size = Number(stat.size);
|
|
970
|
+
// A shrink means truncation or replacement-in-place; the retained rows may no
|
|
971
|
+
// longer correspond to file contents, so refuse to extend them.
|
|
972
|
+
if (size < retained.coveredThroughBytes) return null;
|
|
973
|
+
// Retained-state reuse is only an optimization. A burst larger than the configured
|
|
974
|
+
// window must re-anchor through the bounded full-tail reader instead of reading and
|
|
975
|
+
// parsing every byte appended since the previous poll.
|
|
976
|
+
if (size - retained.coveredThroughBytes > maxReadBytes) return null;
|
|
977
|
+
// Verify the retained REGION is unchanged before anything is reused. Identity keeps
|
|
978
|
+
// dev/ino/birthtime, and an append and an in-place rewrite both move mtime/ctime
|
|
979
|
+
// forward, so only the bytes themselves settle it.
|
|
980
|
+
//
|
|
981
|
+
// Only `truncatedPrefixBytes..coveredThroughBytes` is hashed: that is exactly the
|
|
982
|
+
// span the retained rows were parsed from, and after trimming it is never wider than
|
|
983
|
+
// maxReadBytes. Hashing from byte 0 instead would make every poll O(file) -- cheaper
|
|
984
|
+
// than a reparse on a 245 MB ledger but MORE expensive past roughly 1-2 GB, turning
|
|
985
|
+
// this optimization into a pessimization on exactly the growth curve an append-only
|
|
986
|
+
// ledger follows. Bytes before the retained start are not described by any retained
|
|
987
|
+
// row, so re-reading them proves nothing.
|
|
988
|
+
const covered = usageRegionDigest(fd, retained.rowsBeginAtBytes, retained.coveredThroughBytes);
|
|
989
|
+
if (covered === null || covered !== retained.prefixDigest) return null;
|
|
990
|
+
// Read and parse ONLY the appended bytes.
|
|
991
|
+
let appendedEntries: PersistedUsageEntry[] = [];
|
|
992
|
+
let appendedLengths: number[] = [];
|
|
993
|
+
let appendedDropped = 0;
|
|
994
|
+
let appendedTrailingSkipped = retained.trailingSkippedBytes;
|
|
995
|
+
if (size > retained.coveredThroughBytes) {
|
|
996
|
+
// The covered offset must land immediately after a newline, or the retained rows
|
|
997
|
+
// and the appended text do not join on a record boundary.
|
|
998
|
+
if (retained.coveredThroughBytes > 0) {
|
|
999
|
+
const preceding = readExactly(fd, 1, retained.coveredThroughBytes - 1);
|
|
1000
|
+
if (preceding === null || preceding[0] !== 0x0a) return null;
|
|
1001
|
+
}
|
|
1002
|
+
const chunks: Buffer[] = [];
|
|
1003
|
+
for (let position = retained.coveredThroughBytes; position < size;) {
|
|
1004
|
+
if (signal.aborted) throw signal.reason;
|
|
1005
|
+
const length = Math.min(MANAGEMENT_USAGE_READ_CHUNK_BYTES, size - position);
|
|
1006
|
+
const chunk = readExactly(fd, length, position);
|
|
1007
|
+
if (chunk === null) throw new Error("usage log changed while it was being read");
|
|
1008
|
+
chunks.push(chunk);
|
|
1009
|
+
position += length;
|
|
1010
|
+
}
|
|
1011
|
+
const appended = await parseUsageTextCooperatively(Buffer.concat(chunks).toString("utf-8"), signal);
|
|
1012
|
+
appendedEntries = appended.entries;
|
|
1013
|
+
appendedLengths = appended.entryLengths;
|
|
1014
|
+
appendedDropped = appended.entriesDropped;
|
|
1015
|
+
// A capped appended chunk is not joinable: its dropped rows sit between the
|
|
1016
|
+
// retained rows and the kept ones, so the lengths no longer describe a contiguous
|
|
1017
|
+
// span. Fall back to a full read.
|
|
1018
|
+
if (appended.cappedPrefixBytes > 0) return null;
|
|
1019
|
+
// If the appended chunk produced rows, its own trailing skipped bytes become the
|
|
1020
|
+
// new trailing remainder; otherwise the earlier remainder still stands and the new
|
|
1021
|
+
// skipped bytes add to it.
|
|
1022
|
+
appendedTrailingSkipped = appended.entries.length > 0
|
|
1023
|
+
? appended.trailingSkippedBytes
|
|
1024
|
+
: retained.trailingSkippedBytes + appended.trailingSkippedBytes;
|
|
1025
|
+
}
|
|
1026
|
+
// Re-anchor the window in place. Rows that have fallen outside `size - maxReadBytes`
|
|
1027
|
+
// are dropped using their recorded byte lengths, so the result is exactly the rows a
|
|
1028
|
+
// fresh bounded read would load -- no superset, and truncatedPrefixBytes and
|
|
1029
|
+
// snapshotWindow keep describing the read honestly. Refusing here instead would make
|
|
1030
|
+
// this path dead code on any ledger past the window, which is precisely the case it
|
|
1031
|
+
// exists for.
|
|
1032
|
+
const windowStart = Math.max(0, size - maxReadBytes);
|
|
1033
|
+
let entries = retained.entries.concat(appendedEntries);
|
|
1034
|
+
// Skipped bytes trailing the retained rows sit BETWEEN them and the appended rows, so
|
|
1035
|
+
// they belong to the first appended row's span. Folding them in keeps the recorded
|
|
1036
|
+
// lengths summing to the true byte distance, which is what the trim walk relies on.
|
|
1037
|
+
const joinedLengths = appendedLengths.slice();
|
|
1038
|
+
if (retained.trailingSkippedBytes > 0 && joinedLengths.length > 0) {
|
|
1039
|
+
joinedLengths[0] = joinedLengths[0]! + retained.trailingSkippedBytes;
|
|
1040
|
+
}
|
|
1041
|
+
let lengths = retained.entryLengths.concat(joinedLengths);
|
|
1042
|
+
let rowsBeginAtBytes = retained.rowsBeginAtBytes;
|
|
1043
|
+
// Byte-window truncation advances ONLY here, so it stays exactly what a cold read of
|
|
1044
|
+
// this window reports. The entry cap below is entry-count truncation and must not
|
|
1045
|
+
// move it -- the two are independent signals in the API.
|
|
1046
|
+
let windowTruncatedBytes = retained.truncatedPrefixBytes;
|
|
1047
|
+
let dropIndex = 0;
|
|
1048
|
+
while (dropIndex < lengths.length && rowsBeginAtBytes < windowStart) {
|
|
1049
|
+
rowsBeginAtBytes += lengths[dropIndex]!;
|
|
1050
|
+
windowTruncatedBytes += lengths[dropIndex]!;
|
|
1051
|
+
dropIndex += 1;
|
|
1052
|
+
}
|
|
1053
|
+
// If every row is gone and a trailing unparseable remainder still sits before the
|
|
1054
|
+
// window start, nothing is left to advance the offset with: the retained span would
|
|
1055
|
+
// keep growing past maxReadBytes on each malformed-only append while the accounting
|
|
1056
|
+
// still balanced. Re-anchor with a full read instead.
|
|
1057
|
+
if (rowsBeginAtBytes < windowStart) return null;
|
|
1058
|
+
if (dropIndex > 0) {
|
|
1059
|
+
entries = entries.slice(dropIndex);
|
|
1060
|
+
lengths = lengths.slice(dropIndex);
|
|
1061
|
+
}
|
|
1062
|
+
let entriesDropped = retained.entriesDropped + appendedDropped;
|
|
1063
|
+
if (entries.length > MANAGEMENT_USAGE_MAX_ENTRIES) {
|
|
1064
|
+
// A cold read applies the entry cap to the whole window and reports byte
|
|
1065
|
+
// truncation for the window boundary alone. An incremental read arrives at the cap
|
|
1066
|
+
// by a different route and cannot reconstruct that ordering from retained state, so
|
|
1067
|
+
// continuing here would report a truncatedPrefixBytes that disagrees with a fresh
|
|
1068
|
+
// read of the same window. Re-anchor instead.
|
|
1069
|
+
//
|
|
1070
|
+
// This is reachable in production, not a theoretical branch: rows average ~118
|
|
1071
|
+
// bytes on a real ledger, so 500,000 of them occupy ~56 MiB and fit inside the
|
|
1072
|
+
// 64 MiB window. Both truncations can therefore apply at once.
|
|
1073
|
+
return null;
|
|
1074
|
+
}
|
|
1075
|
+
// The recorded lengths plus the trailing remainder must account for every byte from
|
|
1076
|
+
// the retained rows' start to EOF; if they do not, the lengths and the file have
|
|
1077
|
+
// diverged and the retained rows cannot be trusted.
|
|
1078
|
+
let accounted = appendedTrailingSkipped;
|
|
1079
|
+
for (const length of lengths) accounted += length;
|
|
1080
|
+
if (rowsBeginAtBytes + accounted !== size) return null;
|
|
1081
|
+
usageReadCacheStats.tailReads += 1;
|
|
1082
|
+
// Byte-window truncation is what the API reports, and it stays independent of
|
|
1083
|
+
// entry-count truncation. A cold read reports the record boundary it actually landed
|
|
1084
|
+
// on, which is where the rows begin MINUS whatever the entry cap removed -- the cap
|
|
1085
|
+
// is not window truncation. When nothing was skipped by the window at all, a cold
|
|
1086
|
+
// read reports 0.
|
|
1087
|
+
const truncatedPrefixBytes = windowTruncatedBytes;
|
|
1088
|
+
return {
|
|
1089
|
+
entries,
|
|
1090
|
+
revision,
|
|
1091
|
+
truncatedPrefixBytes,
|
|
1092
|
+
// ENTRY-count truncation only. Byte-window truncation is reported by
|
|
1093
|
+
// truncatedPrefixBytes, and the route ORs the two itself; folding bytes in here
|
|
1094
|
+
// would make a byte-truncated read claim rows were dropped when none were.
|
|
1095
|
+
entriesTruncated: entriesDropped > 0,
|
|
1096
|
+
entriesDropped,
|
|
1097
|
+
// The digest must describe exactly the region the returned rows came from, which
|
|
1098
|
+
// is the post-trim window, not the pre-trim one.
|
|
1099
|
+
prefixDigest: usageRegionDigest(fd, rowsBeginAtBytes, size) ?? "",
|
|
1100
|
+
entryLengths: lengths,
|
|
1101
|
+
trailingSkippedBytes: appendedTrailingSkipped,
|
|
1102
|
+
rowsBeginAtBytes,
|
|
534
1103
|
};
|
|
535
1104
|
} finally {
|
|
536
1105
|
if (fd !== undefined) closeSync(fd);
|
|
@@ -539,8 +1108,10 @@ async function readUsageEntriesFullCooperatively(
|
|
|
539
1108
|
|
|
540
1109
|
/**
|
|
541
1110
|
* Management API reader: full parses yield between bounded batches and concurrent
|
|
542
|
-
* callers share work
|
|
543
|
-
*
|
|
1111
|
+
* callers share work when they observe the same ledger identity and byte window.
|
|
1112
|
+
* Appends keep that identity; replacements (inode/birthtime change) start a new flight.
|
|
1113
|
+
* The parsed tail is retained under the app-owned memory budget so an append reparses
|
|
1114
|
+
* only the appended bytes; the retained rows are evictable and are copied per caller.
|
|
544
1115
|
*/
|
|
545
1116
|
export async function readUsageSnapshotForManagement(maxReadBytes = MANAGEMENT_USAGE_MAX_READ_BYTES): Promise<{
|
|
546
1117
|
entries: PersistedUsageEntry[];
|
|
@@ -553,18 +1124,57 @@ export async function readUsageSnapshotForManagement(maxReadBytes = MANAGEMENT_U
|
|
|
553
1124
|
const path = usageLogPath();
|
|
554
1125
|
if (!existsSync(path)) return { entries: [], revision: null, truncatedPrefixBytes: 0, entriesTruncated: false, entriesDropped: 0 };
|
|
555
1126
|
const observed = currentUsageLogRevision();
|
|
556
|
-
const key = `${
|
|
1127
|
+
const key = `${usageLogIdentityKey(observed)}\0${maxReadBytes}`;
|
|
1128
|
+
const observedSize = observed?.size ?? 0;
|
|
557
1129
|
const existing = managementUsageReadInflight;
|
|
558
|
-
|
|
1130
|
+
const replacement = Boolean(existing && observedSize < existing.openedSize);
|
|
1131
|
+
if (!replacement && existing?.key === key && Date.now() - existing.startedAt <= MANAGEMENT_USAGE_FLIGHT_STALE_MS) {
|
|
1132
|
+
const shared = await existing.promise;
|
|
1133
|
+
return { ...shared, entries: shared.entries.slice() };
|
|
1134
|
+
}
|
|
1135
|
+
if (existing && (existing.key !== key || replacement || Date.now() - existing.startedAt > MANAGEMENT_USAGE_FLIGHT_STALE_MS)) {
|
|
1136
|
+
existing.abort.abort(new Error("management usage read superseded"));
|
|
1137
|
+
} else if (existing) {
|
|
559
1138
|
const shared = await existing.promise;
|
|
560
1139
|
return { ...shared, entries: shared.entries.slice() };
|
|
561
1140
|
}
|
|
562
|
-
existing?.abort.abort(new Error("management usage read superseded"));
|
|
563
1141
|
const abort = new AbortController();
|
|
564
|
-
const
|
|
565
|
-
|
|
1142
|
+
const retained = retainedUsageSnapshot;
|
|
1143
|
+
const reusable = retained
|
|
1144
|
+
&& retained.identityKey === usageLogIdentityKey(observed)
|
|
1145
|
+
&& retained.maxReadBytes === maxReadBytes
|
|
1146
|
+
? retained
|
|
1147
|
+
: null;
|
|
1148
|
+
const promise = (async (): Promise<ManagementUsageSnapshot> => {
|
|
1149
|
+
if (reusable) {
|
|
1150
|
+
const incremental = await readUsageEntriesIncrementally(path, abort.signal, maxReadBytes, reusable);
|
|
1151
|
+
if (incremental) return incremental;
|
|
1152
|
+
// The retained rows could not be extended safely; drop them before the full read
|
|
1153
|
+
// so a stale window is never combined with freshly parsed bytes.
|
|
1154
|
+
discardRetainedUsageSnapshot();
|
|
1155
|
+
}
|
|
1156
|
+
return readUsageEntriesFullCooperatively(path, abort.signal, maxReadBytes);
|
|
1157
|
+
})();
|
|
1158
|
+
managementUsageReadInflight = { key, openedSize: observedSize, promise, startedAt: Date.now(), abort };
|
|
566
1159
|
try {
|
|
567
1160
|
const snapshot = await promise;
|
|
1161
|
+
retainedUsageSnapshot = {
|
|
1162
|
+
identityKey: usageLogIdentityKey(snapshot.revision),
|
|
1163
|
+
maxReadBytes,
|
|
1164
|
+
coveredThroughBytes: snapshot.revision.size,
|
|
1165
|
+
prefixDigest: snapshot.prefixDigest,
|
|
1166
|
+
truncatedPrefixBytes: snapshot.truncatedPrefixBytes,
|
|
1167
|
+
entryLengths: snapshot.entryLengths,
|
|
1168
|
+
trailingSkippedBytes: snapshot.trailingSkippedBytes,
|
|
1169
|
+
rowsBeginAtBytes: snapshot.rowsBeginAtBytes,
|
|
1170
|
+
entries: snapshot.entries,
|
|
1171
|
+
entriesTruncated: snapshot.entriesTruncated,
|
|
1172
|
+
entriesDropped: snapshot.entriesDropped,
|
|
1173
|
+
revision: snapshot.revision,
|
|
1174
|
+
retainedAt: Date.now(),
|
|
1175
|
+
approxBytes: retainedUsageSnapshotBytes(snapshot.entries),
|
|
1176
|
+
};
|
|
1177
|
+
enforceAppOwnedMemoryBudget();
|
|
568
1178
|
return { ...snapshot, entries: snapshot.entries.slice() };
|
|
569
1179
|
} finally {
|
|
570
1180
|
if (managementUsageReadInflight?.promise === promise) managementUsageReadInflight = null;
|
|
@@ -575,6 +1185,14 @@ export async function readUsageEntriesForManagement(): Promise<PersistedUsageEnt
|
|
|
575
1185
|
return (await readUsageSnapshotForManagement()).entries;
|
|
576
1186
|
}
|
|
577
1187
|
|
|
1188
|
+
/** Keep legacy optional fields permissive, but reject rows that cannot be safely attributed. */
|
|
1189
|
+
function normalizePersistedUsageRow(value: unknown): PersistedUsageEntry | undefined {
|
|
1190
|
+
if (value === null || typeof value !== "object" || Array.isArray(value)) return undefined;
|
|
1191
|
+
const row = value as Record<string, unknown>;
|
|
1192
|
+
if (typeof row.requestId !== "string" || typeof row.provider !== "string") return undefined;
|
|
1193
|
+
return normalizeUsageEntry(row as unknown as PersistedUsageEntry);
|
|
1194
|
+
}
|
|
1195
|
+
|
|
578
1196
|
export function readUsageEntries(): PersistedUsageEntry[] {
|
|
579
1197
|
const path = usageLogPath();
|
|
580
1198
|
if (!existsSync(path)) return [];
|
|
@@ -583,10 +1201,8 @@ export function readUsageEntries(): PersistedUsageEntry[] {
|
|
|
583
1201
|
for (const line of lines) {
|
|
584
1202
|
if (!line.trim()) continue;
|
|
585
1203
|
try {
|
|
586
|
-
const parsed = JSON.parse(line)
|
|
587
|
-
if (parsed
|
|
588
|
-
entries.push(normalizeUsageEntry(parsed));
|
|
589
|
-
}
|
|
1204
|
+
const parsed = normalizePersistedUsageRow(JSON.parse(line));
|
|
1205
|
+
if (parsed) entries.push(parsed);
|
|
590
1206
|
} catch {
|
|
591
1207
|
/* keep reading after a partially written or hand-edited line */
|
|
592
1208
|
}
|
|
@@ -599,10 +1215,8 @@ function parseUsageLines(lines: string[]): PersistedUsageEntry[] {
|
|
|
599
1215
|
for (const line of lines) {
|
|
600
1216
|
if (!line.trim()) continue;
|
|
601
1217
|
try {
|
|
602
|
-
const parsed = JSON.parse(line)
|
|
603
|
-
if (parsed
|
|
604
|
-
entries.push(normalizeUsageEntry(parsed));
|
|
605
|
-
}
|
|
1218
|
+
const parsed = normalizePersistedUsageRow(JSON.parse(line));
|
|
1219
|
+
if (parsed) entries.push(parsed);
|
|
606
1220
|
} catch {
|
|
607
1221
|
/* skip partial / hand-edited lines */
|
|
608
1222
|
}
|
|
@@ -614,18 +1228,20 @@ function parseUsageLines(lines: string[]): PersistedUsageEntry[] {
|
|
|
614
1228
|
* Read only the newest `limit` usage.jsonl rows without loading the whole append-only
|
|
615
1229
|
* file into memory. Used by request-log hydration on `ocx start`.
|
|
616
1230
|
*/
|
|
617
|
-
export function readRecentUsageEntries(limit: number): PersistedUsageEntry[] {
|
|
1231
|
+
export function readRecentUsageEntries(limit: number, configDir?: string): PersistedUsageEntry[] {
|
|
618
1232
|
if (!Number.isFinite(limit) || limit <= 0) return [];
|
|
619
|
-
const path = usageLogPath();
|
|
1233
|
+
const path = usageLogPath(configDir);
|
|
620
1234
|
if (!existsSync(path)) return [];
|
|
621
1235
|
let fd: number | undefined;
|
|
622
1236
|
try {
|
|
623
1237
|
fd = openSync(path, "r");
|
|
624
1238
|
const size = fstatSync(fd).size;
|
|
625
1239
|
if (size <= 0) return [];
|
|
626
|
-
//
|
|
627
|
-
|
|
628
|
-
|
|
1240
|
+
// Trace-sized rows need a larger per-row budget than the pre-trace ledger,
|
|
1241
|
+
// but startup hydration must never grow into an unbounded whole-file read.
|
|
1242
|
+
const maxWindowBytes = Math.min(size, RECENT_USAGE_MAX_READ_BYTES);
|
|
1243
|
+
let windowBytes = Math.min(maxWindowBytes, Math.max(64 * 1024, Math.ceil(limit) * 20 * 1024));
|
|
1244
|
+
while (true) {
|
|
629
1245
|
const start = Math.max(0, size - windowBytes);
|
|
630
1246
|
const buf = Buffer.alloc(size - start);
|
|
631
1247
|
readSync(fd, buf, 0, buf.length, start);
|
|
@@ -633,8 +1249,8 @@ export function readRecentUsageEntries(limit: number): PersistedUsageEntry[] {
|
|
|
633
1249
|
if (start > 0) {
|
|
634
1250
|
const nl = text.indexOf("\n");
|
|
635
1251
|
if (nl < 0) {
|
|
636
|
-
if (start === 0) break;
|
|
637
|
-
windowBytes = Math.min(
|
|
1252
|
+
if (start === 0 || windowBytes >= maxWindowBytes) break;
|
|
1253
|
+
windowBytes = Math.min(maxWindowBytes, windowBytes * 4);
|
|
638
1254
|
continue;
|
|
639
1255
|
}
|
|
640
1256
|
text = text.slice(nl + 1);
|
|
@@ -644,8 +1260,8 @@ export function readRecentUsageEntries(limit: number): PersistedUsageEntry[] {
|
|
|
644
1260
|
// or partial lines are filtered out during parsing and we always return the
|
|
645
1261
|
// most recent N valid rows (not N physical lines minus corrupt ones).
|
|
646
1262
|
const entries = parseUsageLines(lines);
|
|
647
|
-
if (entries.length >= limit || start === 0 || windowBytes >=
|
|
648
|
-
windowBytes = Math.min(
|
|
1263
|
+
if (entries.length >= limit || start === 0 || windowBytes >= maxWindowBytes) return entries.slice(-limit);
|
|
1264
|
+
windowBytes = Math.min(maxWindowBytes, windowBytes * 4);
|
|
649
1265
|
}
|
|
650
1266
|
return [];
|
|
651
1267
|
} catch {
|