@openai/codex-security 0.8.3 → 0.10.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +36 -746
- package/_bundled_plugin/.codex-plugin/plugin.json +2 -0
- package/_bundled_plugin/.mcp.json +7 -1
- package/_bundled_plugin/examples/completed-scan/findings.json +16 -4
- package/_bundled_plugin/examples/completed-scan/report.md +11 -8
- package/_bundled_plugin/examples/completed-scan/scan-manifest.json +4 -2
- package/_bundled_plugin/mcp/fzstd-LICENSE.txt +21 -0
- package/_bundled_plugin/mcp/helpers.mjs +2 -11
- package/_bundled_plugin/mcp/helpers.mjs.br +0 -0
- package/_bundled_plugin/mcp/native/THIRD_PARTY_NOTICES.txt +3 -210
- package/_bundled_plugin/mcp/native/darwin-arm64/unix.node +0 -0
- package/_bundled_plugin/mcp/native/darwin-x64/unix.node +0 -0
- package/_bundled_plugin/mcp/native/linux-arm64-gnu/unix.node +0 -0
- package/_bundled_plugin/mcp/native/linux-arm64-musl/unix.node +0 -0
- package/_bundled_plugin/mcp/native/linux-x64-gnu/unix.node +0 -0
- package/_bundled_plugin/mcp/native/linux-x64-musl/unix.node +0 -0
- package/_bundled_plugin/mcp/native/win32-arm64/windows.node +0 -0
- package/_bundled_plugin/mcp/native/win32-x64/windows.node +0 -0
- package/_bundled_plugin/mcp/package.json +1 -0
- package/_bundled_plugin/mcp/permission-profile-preflight.mjs +5 -2
- package/_bundled_plugin/mcp/server.mjs +2 -11
- package/_bundled_plugin/mcp/server.mjs.br +0 -0
- package/_bundled_plugin/mcp/session-decompression.mjs +712 -0
- package/_bundled_plugin/references/artifact-storage.md +1 -1
- package/_bundled_plugin/references/baseline-auditor.md +18 -0
- package/_bundled_plugin/references/core-scan.md +5 -42
- package/_bundled_plugin/references/default-severity-rubric.md +9 -4
- package/_bundled_plugin/references/final-report.md +5 -3
- package/_bundled_plugin/references/finding-detail-fields.md +18 -18
- package/_bundled_plugin/references/focused-investigator.md +13 -0
- package/_bundled_plugin/references/sarif-adapter.md +8 -2
- package/_bundled_plugin/references/scan-artifacts.md +21 -12
- package/_bundled_plugin/references/scan-contract.md +27 -33
- package/_bundled_plugin/references/scan-prologue.md +2 -2
- package/_bundled_plugin/references/scan-runtime.md +4 -2
- package/_bundled_plugin/references/security-guidance.md +14 -1
- package/_bundled_plugin/references/validate-scan-findings.md +3 -0
- package/_bundled_plugin/schemas/tools/scan-draft.schema.json +2 -2
- package/_bundled_plugin/scripts/check_deep_scan_target.py +97 -0
- package/_bundled_plugin/scripts/codex_profile.mjs +7 -1
- package/_bundled_plugin/scripts/deep_scan_config.py +20 -1
- package/_bundled_plugin/scripts/deep_scan_workbench.py +192 -352
- package/_bundled_plugin/scripts/dispatch_inventory_tool +7 -0
- package/_bundled_plugin/scripts/filesystem_identity.py +1 -0
- package/_bundled_plugin/scripts/finalize_scan_contract.py +182 -230
- package/_bundled_plugin/scripts/finding_preview.py +20 -0
- package/_bundled_plugin/scripts/launch_codex_security_mcp +55 -16
- package/_bundled_plugin/scripts/legacy_findings.py +148 -0
- package/_bundled_plugin/scripts/posix_scan_local_files.py +37 -1
- package/_bundled_plugin/scripts/read_codex_session.mjs +21 -0
- package/_bundled_plugin/scripts/report_projection.py +81 -43
- package/_bundled_plugin/scripts/review_state.py +8 -2
- package/_bundled_plugin/scripts/servicenow_avit.py +20 -0
- package/_bundled_plugin/scripts/snapshot_sqlite.py +9 -2
- package/_bundled_plugin/scripts/threat_model_projection.py +22 -26
- package/_bundled_plugin/scripts/validate_scan_contract.py +7 -1
- package/_bundled_plugin/scripts/windows_scan_local_files.py +582 -20
- package/_bundled_plugin/scripts/workbench/__init__.py +1 -1
- package/_bundled_plugin/scripts/workbench/handoff.py +100 -23
- package/_bundled_plugin/scripts/workbench/retained_coverage.py +30 -51
- package/_bundled_plugin/scripts/workbench/retained_records.py +18 -30
- package/_bundled_plugin/scripts/workbench/runtime_toml.py +4 -1
- package/_bundled_plugin/scripts/workbench/saved_coverage.py +9 -17
- package/_bundled_plugin/scripts/workbench/saved_outputs.py +9 -0
- package/_bundled_plugin/scripts/workbench/saved_sources.py +47 -40
- package/_bundled_plugin/scripts/workbench/storage.py +91 -7
- package/_bundled_plugin/scripts/workbench_cli.py +54 -14
- package/_bundled_plugin/scripts/workbench_constants.py +12 -0
- package/_bundled_plugin/scripts/workbench_coverage_receipts.py +3 -3
- package/_bundled_plugin/scripts/workbench_db.py +562 -303
- package/_bundled_plugin/scripts/workbench_feedback.py +57 -11
- package/_bundled_plugin/scripts/workbench_finding_index.py +1 -0
- package/_bundled_plugin/scripts/workbench_finding_workflows.py +4 -0
- package/_bundled_plugin/scripts/workbench_native_indexes.py +238 -41
- package/_bundled_plugin/scripts/workbench_progress.py +169 -4
- package/_bundled_plugin/scripts/workbench_publication.py +16 -13
- package/_bundled_plugin/scripts/workbench_remediation.py +155 -2
- package/_bundled_plugin/scripts/workbench_saved_results.py +176 -81
- package/_bundled_plugin/scripts/workbench_scan_history.py +398 -174
- package/_bundled_plugin/scripts/workbench_scan_start.py +224 -31
- package/_bundled_plugin/scripts/workbench_scan_usage.py +675 -73
- package/_bundled_plugin/scripts/workbench_schema.py +258 -63
- package/_bundled_plugin/scripts/workbench_setup.py +14 -2
- package/_bundled_plugin/scripts/workbench_source_excerpt.py +13 -0
- package/_bundled_plugin/scripts/workbench_source_scopes.py +18 -0
- package/_bundled_plugin/scripts/workbench_target.py +594 -136
- package/_bundled_plugin/scripts/workbench_target_state.py +1110 -21
- package/_bundled_plugin/scripts/workbench_validation.py +43 -3
- package/_bundled_plugin/shared/workbench-history-repairs.json +77 -35
- package/_bundled_plugin/shared/workbench-migrations.json +100 -0
- package/_bundled_plugin/skills/cloud-findings/SKILL.md +5 -5
- package/_bundled_plugin/skills/deep-security-scan/SKILL.md +7 -50
- package/_bundled_plugin/skills/fix-finding/SKILL.md +2 -1
- package/_bundled_plugin/skills/publish-findings/SKILL.md +24 -153
- package/_bundled_plugin/skills/publish-findings/agents/openai.yaml +2 -2
- package/_bundled_plugin/skills/publish-findings/references/checkmarx-import.md +37 -0
- package/_bundled_plugin/skills/publish-findings/references/wiz-import.md +3 -3
- package/_bundled_plugin/skills/publish-findings/scripts/collect_checkmarx.py +409 -0
- package/_bundled_plugin/skills/publish-findings/scripts/collect_continuum.py +224 -0
- package/_bundled_plugin/skills/publish-findings/scripts/collector_paths.py +63 -0
- package/_bundled_plugin/skills/security-scan/SKILL.md +9 -46
- package/_bundled_plugin/skills/security-scan/agents/openai.yaml +2 -2
- package/_bundled_plugin/skills/security-scan/references/desktop-scan.md +3 -21
- package/_bundled_plugin/skills/security-scan/references/managed-scan.md +11 -0
- package/_bundled_plugin/skills/security-scan/references/scan-results.md +15 -0
- package/_bundled_plugin/skills/security-scan/references/scoped-paths.md +23 -0
- package/_bundled_plugin/skills/security-scan/references/sdk-files.md +24 -0
- package/_bundled_plugin/skills/threat-model-visualize/scripts/build_explorer.py +22 -1
- package/_bundled_plugin/skills/triage-finding/references/ticket-intake.md +6 -2
- package/bin/codex-security.mjs +2 -2
- package/dist/api.d.ts +45 -5
- package/dist/api.d.ts.map +1 -1
- package/dist/api.js +1192 -427
- package/dist/api.js.map +1 -1
- package/dist/armorcode-findings.d.ts +9 -0
- package/dist/armorcode-findings.d.ts.map +1 -0
- package/dist/armorcode-findings.js +145 -0
- package/dist/armorcode-findings.js.map +1 -0
- package/dist/artifact-export.d.ts +2 -0
- package/dist/artifact-export.d.ts.map +1 -1
- package/dist/artifact-export.js +29 -13
- package/dist/artifact-export.js.map +1 -1
- package/dist/auth.d.ts +6 -2
- package/dist/auth.d.ts.map +1 -1
- package/dist/auth.js +37 -18
- package/dist/auth.js.map +1 -1
- package/dist/batch-cost.d.ts +18 -0
- package/dist/batch-cost.d.ts.map +1 -0
- package/dist/batch-cost.js +41 -0
- package/dist/batch-cost.js.map +1 -0
- package/dist/bedrock-oss-pricing.d.ts +6 -0
- package/dist/bedrock-oss-pricing.d.ts.map +1 -0
- package/dist/bedrock-oss-pricing.js +58 -0
- package/dist/bedrock-oss-pricing.js.map +1 -0
- package/dist/bedrock-pricing.d.ts +15 -0
- package/dist/bedrock-pricing.d.ts.map +1 -0
- package/dist/bedrock-pricing.js +117 -0
- package/dist/bedrock-pricing.js.map +1 -0
- package/dist/blackduck-findings.d.ts +1 -1
- package/dist/blackduck-findings.d.ts.map +1 -1
- package/dist/blackduck-findings.js +24 -22
- package/dist/blackduck-findings.js.map +1 -1
- package/dist/bulk-scan-discovery.d.ts.map +1 -1
- package/dist/bulk-scan-discovery.js +4 -4
- package/dist/bulk-scan-discovery.js.map +1 -1
- package/dist/bundled-plugin.d.ts +4 -0
- package/dist/bundled-plugin.d.ts.map +1 -0
- package/dist/bundled-plugin.js +23 -0
- package/dist/bundled-plugin.js.map +1 -0
- package/dist/checkmarx-evidence.d.ts +19 -0
- package/dist/checkmarx-evidence.d.ts.map +1 -0
- package/dist/checkmarx-evidence.js +36 -0
- package/dist/checkmarx-evidence.js.map +1 -0
- package/dist/checkmarx-findings.d.ts +4 -0
- package/dist/checkmarx-findings.d.ts.map +1 -0
- package/dist/checkmarx-findings.js +27 -0
- package/dist/checkmarx-findings.js.map +1 -0
- package/dist/checkmarx-one.d.ts +4 -0
- package/dist/checkmarx-one.d.ts.map +1 -0
- package/dist/checkmarx-one.js +53 -0
- package/dist/checkmarx-one.js.map +1 -0
- package/dist/checkmarx-repository.d.ts +12 -0
- package/dist/checkmarx-repository.d.ts.map +1 -0
- package/dist/checkmarx-repository.js +181 -0
- package/dist/checkmarx-repository.js.map +1 -0
- package/dist/checkmarx-xml.d.ts +12 -0
- package/dist/checkmarx-xml.d.ts.map +1 -0
- package/dist/checkmarx-xml.js +196 -0
- package/dist/checkmarx-xml.js.map +1 -0
- package/dist/classify-scan-severity.d.ts +1 -1
- package/dist/classify-scan-severity.d.ts.map +1 -1
- package/dist/classify-scan-severity.js +11 -3
- package/dist/classify-scan-severity.js.map +1 -1
- package/dist/classify-severity-records.d.ts +1 -9
- package/dist/classify-severity-records.d.ts.map +1 -1
- package/dist/classify-severity-records.js +3 -13
- package/dist/classify-severity-records.js.map +1 -1
- package/dist/classify-severity.d.ts.map +1 -1
- package/dist/classify-severity.js +11 -10
- package/dist/classify-severity.js.map +1 -1
- package/dist/cli-argument.d.ts +4 -0
- package/dist/cli-argument.d.ts.map +1 -0
- package/dist/cli-argument.js +17 -0
- package/dist/cli-argument.js.map +1 -0
- package/dist/cli-help.d.ts +1 -1
- package/dist/cli-help.d.ts.map +1 -1
- package/dist/cli-help.js +71 -29
- package/dist/cli-help.js.map +1 -1
- package/dist/cli-hook.d.ts +3 -0
- package/dist/cli-hook.d.ts.map +1 -0
- package/dist/cli-hook.js +87 -0
- package/dist/cli-hook.js.map +1 -0
- package/dist/cli-integrations.d.ts +13 -0
- package/dist/cli-integrations.d.ts.map +1 -0
- package/dist/cli-integrations.js +333 -0
- package/dist/cli-integrations.js.map +1 -0
- package/dist/cli-mcp.d.ts +15 -0
- package/dist/cli-mcp.d.ts.map +1 -0
- package/dist/cli-mcp.js +178 -0
- package/dist/cli-mcp.js.map +1 -0
- package/dist/cli-token-view.d.ts +10 -0
- package/dist/cli-token-view.d.ts.map +1 -0
- package/dist/cli-token-view.js +41 -0
- package/dist/cli-token-view.js.map +1 -0
- package/dist/cli.d.ts +16 -8
- package/dist/cli.d.ts.map +1 -1
- package/dist/cli.js +3382 -2017
- package/dist/cli.js.map +1 -1
- package/dist/cloud-publish.d.ts.map +1 -1
- package/dist/cloud-publish.js +35 -33
- package/dist/cloud-publish.js.map +1 -1
- package/dist/codex-sdk-environment.d.ts +6 -0
- package/dist/codex-sdk-environment.d.ts.map +1 -1
- package/dist/codex-sdk-environment.js +19 -0
- package/dist/codex-sdk-environment.js.map +1 -1
- package/dist/command-words.d.ts +3 -0
- package/dist/command-words.d.ts.map +1 -0
- package/dist/command-words.js +46 -0
- package/dist/command-words.js.map +1 -0
- package/dist/component-scan.d.ts +19 -4
- package/dist/component-scan.d.ts.map +1 -1
- package/dist/component-scan.js +67 -9
- package/dist/component-scan.js.map +1 -1
- package/dist/config.d.ts +1 -1
- package/dist/config.d.ts.map +1 -1
- package/dist/config.js +130 -67
- package/dist/config.js.map +1 -1
- package/dist/continuum-findings.d.ts +1 -1
- package/dist/continuum-findings.d.ts.map +1 -1
- package/dist/continuum-findings.js +149 -19
- package/dist/continuum-findings.js.map +1 -1
- package/dist/contract-values.d.ts +2 -0
- package/dist/contract-values.d.ts.map +1 -1
- package/dist/contract-values.js +12 -9
- package/dist/contract-values.js.map +1 -1
- package/dist/cost-model.d.ts +13 -1
- package/dist/cost-model.d.ts.map +1 -1
- package/dist/cost-model.js +66 -112
- package/dist/cost-model.js.map +1 -1
- package/dist/cost.d.ts +42 -3
- package/dist/cost.d.ts.map +1 -1
- package/dist/cost.js +484 -149
- package/dist/cost.js.map +1 -1
- package/dist/custom-publish.d.ts +4 -0
- package/dist/custom-publish.d.ts.map +1 -1
- package/dist/custom-publish.js +40 -3
- package/dist/custom-publish.js.map +1 -1
- package/dist/custom-validation-prompt.d.ts +1 -1
- package/dist/custom-validation-prompt.d.ts.map +1 -1
- package/dist/custom-validation-prompt.js +29 -19
- package/dist/custom-validation-prompt.js.map +1 -1
- package/dist/deduplication/local.d.ts +4 -2
- package/dist/deduplication/local.d.ts.map +1 -1
- package/dist/deduplication/local.js +9 -2
- package/dist/deduplication/local.js.map +1 -1
- package/dist/deduplication/refusal.d.ts.map +1 -1
- package/dist/deduplication/refusal.js +26 -5
- package/dist/deduplication/refusal.js.map +1 -1
- package/dist/deduplication/scan.d.ts +5 -0
- package/dist/deduplication/scan.d.ts.map +1 -1
- package/dist/deduplication/scan.js +24 -7
- package/dist/deduplication/scan.js.map +1 -1
- package/dist/deep-config.d.ts +0 -1
- package/dist/deep-config.d.ts.map +1 -1
- package/dist/deep-config.js +22 -33
- package/dist/deep-config.js.map +1 -1
- package/dist/deep-scan-defaults.d.ts +2 -0
- package/dist/deep-scan-defaults.d.ts.map +1 -1
- package/dist/deep-scan-defaults.js +28 -1
- package/dist/deep-scan-defaults.js.map +1 -1
- package/dist/deep-scan.d.ts +2 -0
- package/dist/deep-scan.d.ts.map +1 -1
- package/dist/deep-scan.js +44 -19
- package/dist/deep-scan.js.map +1 -1
- package/dist/default-severity-rubric.d.ts +1 -1
- package/dist/default-severity-rubric.d.ts.map +1 -1
- package/dist/default-severity-rubric.js +1 -1
- package/dist/default-severity-rubric.js.map +1 -1
- package/dist/endor-findings.d.ts +3 -1
- package/dist/endor-findings.d.ts.map +1 -1
- package/dist/endor-findings.js +86 -45
- package/dist/endor-findings.js.map +1 -1
- package/dist/errors.d.ts +4 -2
- package/dist/errors.d.ts.map +1 -1
- package/dist/errors.js +15 -4
- package/dist/errors.js.map +1 -1
- package/dist/executable-path.d.ts +3 -0
- package/dist/executable-path.d.ts.map +1 -1
- package/dist/executable-path.js +26 -0
- package/dist/executable-path.js.map +1 -1
- package/dist/external-findings-import.d.ts +5 -3
- package/dist/external-findings-import.d.ts.map +1 -1
- package/dist/external-findings-import.js +19 -7
- package/dist/external-findings-import.js.map +1 -1
- package/dist/external-findings-local.d.ts +69 -5
- package/dist/external-findings-local.d.ts.map +1 -1
- package/dist/external-findings-local.js +537 -108
- package/dist/external-findings-local.js.map +1 -1
- package/dist/external-findings-publish.d.ts +13 -6
- package/dist/external-findings-publish.d.ts.map +1 -1
- package/dist/external-findings-publish.js +895 -1142
- package/dist/external-findings-publish.js.map +1 -1
- package/dist/external-findings-read.d.ts +21 -2
- package/dist/external-findings-read.d.ts.map +1 -1
- package/dist/external-findings-read.js +130 -68
- package/dist/external-findings-read.js.map +1 -1
- package/dist/external-findings.d.ts +15 -3
- package/dist/external-findings.d.ts.map +1 -1
- package/dist/external-findings.js +54 -2
- package/dist/external-findings.js.map +1 -1
- package/dist/external-import-contract.d.ts +8 -12
- package/dist/external-import-contract.d.ts.map +1 -1
- package/dist/external-import-contract.js +51 -184
- package/dist/external-import-contract.js.map +1 -1
- package/dist/external-import-models.d.ts +1 -1
- package/dist/external-import-models.d.ts.map +1 -1
- package/dist/external-import-models.js.map +1 -1
- package/dist/feedback.d.ts.map +1 -1
- package/dist/feedback.js +5 -4
- package/dist/feedback.js.map +1 -1
- package/dist/finding-prompt.d.ts +5 -0
- package/dist/finding-prompt.d.ts.map +1 -0
- package/dist/finding-prompt.js +32 -0
- package/dist/finding-prompt.js.map +1 -0
- package/dist/findings-client.d.ts +1 -0
- package/dist/findings-client.d.ts.map +1 -1
- package/dist/findings-client.js +27 -3
- package/dist/findings-client.js.map +1 -1
- package/dist/github-advisories.d.ts.map +1 -1
- package/dist/github-advisories.js +0 -2
- package/dist/github-advisories.js.map +1 -1
- package/dist/github-dependabot-findings.d.ts.map +1 -1
- package/dist/github-dependabot-findings.js +18 -8
- package/dist/github-dependabot-findings.js.map +1 -1
- package/dist/github-findings.d.ts +4 -1
- package/dist/github-findings.d.ts.map +1 -1
- package/dist/github-findings.js +73 -316
- package/dist/github-findings.js.map +1 -1
- package/dist/hosted-git-remotes.d.ts +5 -0
- package/dist/hosted-git-remotes.d.ts.map +1 -0
- package/dist/hosted-git-remotes.js +285 -0
- package/dist/hosted-git-remotes.js.map +1 -0
- package/dist/hosted-git-repository.d.ts +28 -0
- package/dist/hosted-git-repository.d.ts.map +1 -0
- package/dist/hosted-git-repository.js +222 -0
- package/dist/hosted-git-repository.js.map +1 -0
- package/dist/hosted-remediation.d.ts +1 -0
- package/dist/hosted-remediation.d.ts.map +1 -1
- package/dist/hosted-remediation.js +23 -3
- package/dist/hosted-remediation.js.map +1 -1
- package/dist/hosted-scan-protocol.js +2 -2
- package/dist/hosted-scan-protocol.js.map +1 -1
- package/dist/hosted-scan.d.ts.map +1 -1
- package/dist/hosted-scan.js +16 -11
- package/dist/hosted-scan.js.map +1 -1
- package/dist/hosted-scanner-http.d.ts +21 -0
- package/dist/hosted-scanner-http.d.ts.map +1 -0
- package/dist/hosted-scanner-http.js +204 -0
- package/dist/hosted-scanner-http.js.map +1 -0
- package/dist/import-scan.d.ts.map +1 -1
- package/dist/import-scan.js +3 -0
- package/dist/import-scan.js.map +1 -1
- package/dist/index.d.ts +21 -9
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +12 -3
- package/dist/index.js.map +1 -1
- package/dist/inspector-findings.d.ts.map +1 -1
- package/dist/inspector-findings.js +52 -86
- package/dist/inspector-findings.js.map +1 -1
- package/dist/knowledge-base.d.ts +1 -1
- package/dist/knowledge-base.d.ts.map +1 -1
- package/dist/knowledge-base.js +102 -32
- package/dist/knowledge-base.js.map +1 -1
- package/dist/linear.d.ts +11 -0
- package/dist/linear.d.ts.map +1 -1
- package/dist/linear.js.map +1 -1
- package/dist/local-git-metadata.d.ts +5 -2
- package/dist/local-git-metadata.d.ts.map +1 -1
- package/dist/local-git-metadata.js +8 -5
- package/dist/local-git-metadata.js.map +1 -1
- package/dist/multiscan.d.ts +10 -4
- package/dist/multiscan.d.ts.map +1 -1
- package/dist/multiscan.js +217 -102
- package/dist/multiscan.js.map +1 -1
- package/dist/native-json-findings.d.ts +2 -1
- package/dist/native-json-findings.d.ts.map +1 -1
- package/dist/native-json-findings.js +12 -2
- package/dist/native-json-findings.js.map +1 -1
- package/dist/opengrep-findings.d.ts +5 -2
- package/dist/opengrep-findings.d.ts.map +1 -1
- package/dist/opengrep-findings.js +54 -33
- package/dist/opengrep-findings.js.map +1 -1
- package/dist/operation-telemetry.d.ts +104 -0
- package/dist/operation-telemetry.d.ts.map +1 -0
- package/dist/operation-telemetry.js +293 -0
- package/dist/operation-telemetry.js.map +1 -0
- package/dist/plugin-python-probe.d.ts +4 -0
- package/dist/plugin-python-probe.d.ts.map +1 -0
- package/dist/plugin-python-probe.js +10 -0
- package/dist/plugin-python-probe.js.map +1 -0
- package/dist/private-finding-import.d.ts +99 -9
- package/dist/private-finding-import.d.ts.map +1 -1
- package/dist/private-finding-import.js +116 -152
- package/dist/private-finding-import.js.map +1 -1
- package/dist/project-config.d.ts.map +1 -1
- package/dist/project-config.js +53 -13
- package/dist/project-config.js.map +1 -1
- package/dist/prompt-files.d.ts +7 -2
- package/dist/prompt-files.d.ts.map +1 -1
- package/dist/prompt-files.js +42 -20
- package/dist/prompt-files.js.map +1 -1
- package/dist/provider-profile.d.ts +1 -1
- package/dist/provider-profile.d.ts.map +1 -1
- package/dist/provider-profile.js +12 -10
- package/dist/provider-profile.js.map +1 -1
- package/dist/publication-telemetry.d.ts +45 -0
- package/dist/publication-telemetry.d.ts.map +1 -0
- package/dist/publication-telemetry.js +50 -0
- package/dist/publication-telemetry.js.map +1 -0
- package/dist/publication.d.ts +1 -1
- package/dist/publication.d.ts.map +1 -1
- package/dist/publication.js.map +1 -1
- package/dist/publish.d.ts +4 -0
- package/dist/publish.d.ts.map +1 -1
- package/dist/publish.js +42 -2
- package/dist/publish.js.map +1 -1
- package/dist/record.d.ts +1 -5
- package/dist/record.d.ts.map +1 -1
- package/dist/record.js +3 -51
- package/dist/record.js.map +1 -1
- package/dist/remediation.d.ts +1 -4
- package/dist/remediation.d.ts.map +1 -1
- package/dist/remediation.js +6 -8
- package/dist/remediation.js.map +1 -1
- package/dist/request-metadata.d.ts +3 -0
- package/dist/request-metadata.d.ts.map +1 -1
- package/dist/request-metadata.js +9 -0
- package/dist/request-metadata.js.map +1 -1
- package/dist/result.d.ts +4 -0
- package/dist/result.d.ts.map +1 -1
- package/dist/result.js +1 -1
- package/dist/result.js.map +1 -1
- package/dist/retry-windows-rename.d.ts +8 -0
- package/dist/retry-windows-rename.d.ts.map +1 -0
- package/dist/retry-windows-rename.js +24 -0
- package/dist/retry-windows-rename.js.map +1 -0
- package/dist/runtime.d.ts +54 -6
- package/dist/runtime.d.ts.map +1 -1
- package/dist/runtime.js +1084 -256
- package/dist/runtime.js.map +1 -1
- package/dist/saved-scan-bootstrap.d.ts +7 -1
- package/dist/saved-scan-bootstrap.d.ts.map +1 -1
- package/dist/saved-scan-bootstrap.js +269 -114
- package/dist/saved-scan-bootstrap.js.map +1 -1
- package/dist/scan-activity.d.ts.map +1 -1
- package/dist/scan-activity.js +141 -56
- package/dist/scan-activity.js.map +1 -1
- package/dist/scan-archive-path.d.ts +2 -0
- package/dist/scan-archive-path.d.ts.map +1 -0
- package/dist/scan-archive-path.js +5 -0
- package/dist/scan-archive-path.js.map +1 -0
- package/dist/scan-comparison.d.ts +10 -5
- package/dist/scan-comparison.d.ts.map +1 -1
- package/dist/scan-comparison.js +66 -17
- package/dist/scan-comparison.js.map +1 -1
- package/dist/scan-dashboard.d.ts +7 -6
- package/dist/scan-dashboard.d.ts.map +1 -1
- package/dist/scan-dashboard.js +92 -43
- package/dist/scan-dashboard.js.map +1 -1
- package/dist/scan-executor.d.ts +10 -1
- package/dist/scan-executor.d.ts.map +1 -1
- package/dist/scan-executor.js +24 -2
- package/dist/scan-executor.js.map +1 -1
- package/dist/scan-history-renderer.d.ts +1 -0
- package/dist/scan-history-renderer.d.ts.map +1 -1
- package/dist/scan-history-renderer.js +34 -28
- package/dist/scan-history-renderer.js.map +1 -1
- package/dist/scan-logs.d.ts +12 -0
- package/dist/scan-logs.d.ts.map +1 -1
- package/dist/scan-logs.js +131 -78
- package/dist/scan-logs.js.map +1 -1
- package/dist/scan-measurement.d.ts +26 -0
- package/dist/scan-measurement.d.ts.map +1 -0
- package/dist/scan-measurement.js +77 -0
- package/dist/scan-measurement.js.map +1 -0
- package/dist/scan-progress.d.ts +1 -0
- package/dist/scan-progress.d.ts.map +1 -1
- package/dist/scan-progress.js +3 -0
- package/dist/scan-progress.js.map +1 -1
- package/dist/scan-sessions.d.ts.map +1 -1
- package/dist/scan-sessions.js +4 -1
- package/dist/scan-sessions.js.map +1 -1
- package/dist/scan-settings.d.ts +1 -1
- package/dist/scan-settings.d.ts.map +1 -1
- package/dist/scan-settings.js +1 -24
- package/dist/scan-settings.js.map +1 -1
- package/dist/scanner-import.d.ts +25 -13
- package/dist/scanner-import.d.ts.map +1 -1
- package/dist/scanner-import.js +216 -97
- package/dist/scanner-import.js.map +1 -1
- package/dist/scanner-repository-url.d.ts +4 -1
- package/dist/scanner-repository-url.d.ts.map +1 -1
- package/dist/scanner-repository-url.js +10 -6
- package/dist/scanner-repository-url.js.map +1 -1
- package/dist/security-policy.d.ts +1 -1
- package/dist/security-policy.d.ts.map +1 -1
- package/dist/security-policy.js +26 -6
- package/dist/security-policy.js.map +1 -1
- package/dist/semgrep-findings.d.ts +6 -3
- package/dist/semgrep-findings.d.ts.map +1 -1
- package/dist/semgrep-findings.js +72 -42
- package/dist/semgrep-findings.js.map +1 -1
- package/dist/semgrep-guardian-findings.d.ts +38 -0
- package/dist/semgrep-guardian-findings.d.ts.map +1 -0
- package/dist/semgrep-guardian-findings.js +165 -0
- package/dist/semgrep-guardian-findings.js.map +1 -0
- package/dist/semgrep-platform-collector.d.ts +26 -0
- package/dist/semgrep-platform-collector.d.ts.map +1 -0
- package/dist/semgrep-platform-collector.js +108 -0
- package/dist/semgrep-platform-collector.js.map +1 -0
- package/dist/semgrep-platform-findings.d.ts +157 -6
- package/dist/semgrep-platform-findings.d.ts.map +1 -1
- package/dist/semgrep-platform-findings.js +766 -103
- package/dist/semgrep-platform-findings.js.map +1 -1
- package/dist/server/api.d.ts +1 -1
- package/dist/server/api.d.ts.map +1 -1
- package/dist/server/api.js.map +1 -1
- package/dist/server/embedding-tokenizer.cdx.json +33 -0
- package/dist/server/embedding-tokenizer.d.ts +2 -0
- package/dist/server/embedding-tokenizer.d.ts.map +1 -0
- package/dist/server/embedding-tokenizer.js +322 -0
- package/dist/server/embedding-tokenizer.js.map +6 -0
- package/dist/server/embeddings.d.ts +16 -2
- package/dist/server/embeddings.d.ts.map +1 -1
- package/dist/server/embeddings.js +35 -5
- package/dist/server/embeddings.js.map +1 -1
- package/dist/session-lines.d.ts +5 -0
- package/dist/session-lines.d.ts.map +1 -0
- package/dist/session-lines.js +60 -0
- package/dist/session-lines.js.map +1 -0
- package/dist/severity-review.d.ts +3 -1
- package/dist/severity-review.d.ts.map +1 -1
- package/dist/severity-review.js +3 -2
- package/dist/severity-review.js.map +1 -1
- package/dist/skill-authentication.d.ts +1 -0
- package/dist/skill-authentication.d.ts.map +1 -1
- package/dist/skill-authentication.js +1 -0
- package/dist/skill-authentication.js.map +1 -1
- package/dist/snyk-findings.d.ts +9 -2
- package/dist/snyk-findings.d.ts.map +1 -1
- package/dist/snyk-findings.js +90 -70
- package/dist/snyk-findings.js.map +1 -1
- package/dist/snyk-rest-findings.d.ts +59 -0
- package/dist/snyk-rest-findings.d.ts.map +1 -0
- package/dist/snyk-rest-findings.js +879 -0
- package/dist/snyk-rest-findings.js.map +1 -0
- package/dist/snyk-token-origin.d.ts +4 -0
- package/dist/snyk-token-origin.d.ts.map +1 -0
- package/dist/snyk-token-origin.js +82 -0
- package/dist/snyk-token-origin.js.map +1 -0
- package/dist/sonar-collector.d.ts.map +1 -1
- package/dist/sonar-collector.js +0 -3
- package/dist/sonar-collector.js.map +1 -1
- package/dist/sonar-findings.d.ts +1 -1
- package/dist/sonar-findings.d.ts.map +1 -1
- package/dist/sonar-findings.js +2 -4
- package/dist/sonar-findings.js.map +1 -1
- package/dist/sonar-import.d.ts +3 -0
- package/dist/sonar-import.d.ts.map +1 -1
- package/dist/sonar-import.js +30 -34
- package/dist/sonar-import.js.map +1 -1
- package/dist/stored-finding-projection.d.ts +6 -0
- package/dist/stored-finding-projection.d.ts.map +1 -0
- package/dist/stored-finding-projection.js +104 -0
- package/dist/stored-finding-projection.js.map +1 -0
- package/dist/stored-findings-publication.d.ts +23 -0
- package/dist/stored-findings-publication.d.ts.map +1 -0
- package/dist/stored-findings-publication.js +169 -0
- package/dist/stored-findings-publication.js.map +1 -0
- package/dist/target-drift.d.ts +9 -0
- package/dist/target-drift.d.ts.map +1 -0
- package/dist/target-drift.js +27 -0
- package/dist/target-drift.js.map +1 -0
- package/dist/targets.d.ts +8 -1
- package/dist/targets.d.ts.map +1 -1
- package/dist/targets.js +131 -42
- package/dist/targets.js.map +1 -1
- package/dist/tenable-findings.d.ts +6 -0
- package/dist/tenable-findings.d.ts.map +1 -0
- package/dist/tenable-findings.js +122 -0
- package/dist/tenable-findings.js.map +1 -0
- package/dist/terminal-stream.d.ts +13 -0
- package/dist/terminal-stream.d.ts.map +1 -0
- package/dist/terminal-stream.js +35 -0
- package/dist/terminal-stream.js.map +1 -0
- package/dist/trivy-findings.d.ts +19 -0
- package/dist/trivy-findings.d.ts.map +1 -0
- package/dist/trivy-findings.js +424 -0
- package/dist/trivy-findings.js.map +1 -0
- package/dist/trivy-import.d.ts +19 -0
- package/dist/trivy-import.d.ts.map +1 -0
- package/dist/trivy-import.js +151 -0
- package/dist/trivy-import.js.map +1 -0
- package/dist/trusted-executable.d.ts +6 -0
- package/dist/trusted-executable.d.ts.map +1 -1
- package/dist/trusted-executable.js +140 -37
- package/dist/trusted-executable.js.map +1 -1
- package/dist/validation-records.js +1 -1
- package/dist/validation-records.js.map +1 -1
- package/dist/veracode-findings.d.ts +1 -1
- package/dist/veracode-findings.d.ts.map +1 -1
- package/dist/veracode-findings.js +19 -6
- package/dist/veracode-findings.js.map +1 -1
- package/dist/version.d.ts +3 -1
- package/dist/version.d.ts.map +1 -1
- package/dist/version.js +184 -32
- package/dist/version.js.map +1 -1
- package/dist/vertex-pricing.d.ts +8 -0
- package/dist/vertex-pricing.d.ts.map +1 -0
- package/dist/vertex-pricing.js +20 -0
- package/dist/vertex-pricing.js.map +1 -0
- package/dist/windows-path.d.ts +1 -0
- package/dist/windows-path.d.ts.map +1 -1
- package/dist/windows-path.js +26 -1
- package/dist/windows-path.js.map +1 -1
- package/dist/wiz-findings.d.ts +6 -2
- package/dist/wiz-findings.d.ts.map +1 -1
- package/dist/wiz-findings.js +65 -89
- package/dist/wiz-findings.js.map +1 -1
- package/docs/classify-severity-records.md +27 -12
- package/docs/cli.md +853 -639
- package/docs/dedupe-records.md +44 -24
- package/docs/findings-service.md +84 -44
- package/docs/hosted-remediation.md +27 -11
- package/docs/hosted-scans.md +50 -42
- package/docs/sdk.md +903 -0
- package/docs/telemetry.md +236 -0
- package/docs/validate-records.md +78 -44
- package/package.json +23 -16
- package/schemas/external-findings.schema.json +4 -1
- package/_bundled_plugin/mcp/helpers.mjs.br.part-000 +0 -0
- package/_bundled_plugin/mcp/helpers.mjs.br.part-001 +0 -0
- package/_bundled_plugin/mcp/server.mjs.br.part-000 +0 -0
- package/_bundled_plugin/mcp/server.mjs.br.part-001 +0 -0
- package/_bundled_plugin/mcp/server.mjs.br.part-002 +0 -0
- package/_bundled_plugin/schemas/tools/deep-reducer.schema.json +0 -96
- package/dist/external-imports-feature.d.ts +0 -6
- package/dist/external-imports-feature.d.ts.map +0 -1
- package/dist/external-imports-feature.js +0 -51
- package/dist/external-imports-feature.js.map +0 -1
- package/dist/finding-normalization.d.ts +0 -5
- package/dist/finding-normalization.d.ts.map +0 -1
- package/dist/finding-normalization.js +0 -155
- package/dist/finding-normalization.js.map +0 -1
- package/dist/reader-selection.d.ts +0 -13
- package/dist/reader-selection.d.ts.map +0 -1
- package/dist/reader-selection.js +0 -81
- package/dist/reader-selection.js.map +0 -1
|
@@ -8,13 +8,19 @@ import math
|
|
|
8
8
|
import os
|
|
9
9
|
import re
|
|
10
10
|
import sqlite3
|
|
11
|
+
import subprocess
|
|
11
12
|
import sys
|
|
13
|
+
import tempfile
|
|
12
14
|
import uuid
|
|
15
|
+
from contextlib import contextmanager
|
|
13
16
|
from dataclasses import dataclass
|
|
14
17
|
from datetime import datetime, timezone
|
|
15
18
|
from pathlib import Path
|
|
16
19
|
from typing import Any, Mapping
|
|
17
|
-
|
|
20
|
+
|
|
21
|
+
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
|
22
|
+
from snapshot_sqlite import readonly_sqlite_uri
|
|
23
|
+
from workbench.storage import archive_source_directory, worker_usage_directory
|
|
18
24
|
|
|
19
25
|
TOKEN_FIELDS = {
|
|
20
26
|
"input_tokens": "inputTokens",
|
|
@@ -30,6 +36,8 @@ STATE_DATABASE_TIMEOUT_SECONDS = 1.0
|
|
|
30
36
|
|
|
31
37
|
@dataclass(frozen=True)
|
|
32
38
|
class RolloutSession:
|
|
39
|
+
"""Associate a thread and its optional parent with the path of the rollout used for usage accounting."""
|
|
40
|
+
|
|
33
41
|
thread_id: str
|
|
34
42
|
parent_thread_id: str | None
|
|
35
43
|
path: Path
|
|
@@ -37,6 +45,8 @@ class RolloutSession:
|
|
|
37
45
|
|
|
38
46
|
@dataclass
|
|
39
47
|
class RolloutUsage:
|
|
48
|
+
"""Collect token counts, their update time, warnings, and account rate-limit observations for a rollout."""
|
|
49
|
+
|
|
40
50
|
tokens: dict[str, int]
|
|
41
51
|
warnings: set[str]
|
|
42
52
|
tokens_updated_at: datetime | None
|
|
@@ -66,13 +76,27 @@ def reconcile_completed_scan_cost(
|
|
|
66
76
|
) -> None:
|
|
67
77
|
"""Persist authoritative SDK cost without discarding measured worker usage."""
|
|
68
78
|
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
79
|
+
stored = stored_scan_cost_fields(scan["cost_json"])
|
|
80
|
+
completion = stored.get("usage", {}).get("completion")
|
|
81
|
+
native = collect_scan_usage_safely(
|
|
82
|
+
connection,
|
|
83
|
+
scan,
|
|
84
|
+
thread_id=completion["threadId"] if completion else None,
|
|
85
|
+
completed_turn_id=completion["turnId"] if completion and completion["finalized"] else None,
|
|
86
|
+
)
|
|
87
|
+
fields = merge_scan_usage_fields(stored, stored_scan_cost_fields(cost_json), {"usage": native})
|
|
88
|
+
if completion:
|
|
89
|
+
usage = {**fields["usage"], "completion": completion}
|
|
90
|
+
if not completion["finalized"]:
|
|
91
|
+
if usage["coverage"] == "complete":
|
|
92
|
+
usage["coverage"] = "partial"
|
|
93
|
+
usage["warnings"] = sorted({*usage.get("warnings", []), "turn_in_progress"})
|
|
94
|
+
fields["usage"] = usage
|
|
95
|
+
cost_json = json.dumps(
|
|
96
|
+
fields if "usage" in fields else fields["cost"],
|
|
97
|
+
separators=(",", ":"),
|
|
98
|
+
allow_nan=False,
|
|
99
|
+
)
|
|
76
100
|
connection.execute("BEGIN IMMEDIATE")
|
|
77
101
|
with connection:
|
|
78
102
|
connection.execute(
|
|
@@ -81,52 +105,219 @@ def reconcile_completed_scan_cost(
|
|
|
81
105
|
)
|
|
82
106
|
|
|
83
107
|
|
|
108
|
+
def merge_scan_usage_fields(*observations: Mapping[str, Any]) -> dict[str, Any]:
|
|
109
|
+
"""Reconcile overlapping measurements without summing the same execution twice."""
|
|
110
|
+
|
|
111
|
+
merged: dict[str, Any] = {}
|
|
112
|
+
latest_coverage: dict[str, Any] | None = None
|
|
113
|
+
rate_limits: dict[str, dict[str, Any]] = {}
|
|
114
|
+
for observation in observations:
|
|
115
|
+
if isinstance(observation.get("cost"), dict):
|
|
116
|
+
merged["cost"] = observation["cost"]
|
|
117
|
+
usage = observation.get("usage")
|
|
118
|
+
if not isinstance(usage, dict):
|
|
119
|
+
continue
|
|
120
|
+
for snapshot in usage.get("accountRateLimits", []):
|
|
121
|
+
key = snapshot["limitId"]
|
|
122
|
+
if key not in rate_limits or snapshot["observedAt"] >= rate_limits[key]["observedAt"]:
|
|
123
|
+
rate_limits[key] = snapshot
|
|
124
|
+
previous = merged.get("usage")
|
|
125
|
+
if previous is None or _usage_quality(usage) >= _usage_quality(previous):
|
|
126
|
+
merged["usage"] = usage
|
|
127
|
+
if usage.get("coverage") not in ("complete", "partial"):
|
|
128
|
+
continue
|
|
129
|
+
latest_coverage = (
|
|
130
|
+
usage if latest_coverage is None else _merge_usage_coverage(latest_coverage, usage)
|
|
131
|
+
)
|
|
132
|
+
cost = merged.get("cost")
|
|
133
|
+
usage = merged.get("usage")
|
|
134
|
+
if usage is not None and latest_coverage is not None and latest_coverage is not usage:
|
|
135
|
+
usage = {**usage}
|
|
136
|
+
for field in (
|
|
137
|
+
"coverage",
|
|
138
|
+
"coverageScope",
|
|
139
|
+
"threadCount",
|
|
140
|
+
"missingThreadCount",
|
|
141
|
+
"warnings",
|
|
142
|
+
"observedAt",
|
|
143
|
+
"completion",
|
|
144
|
+
"byRole",
|
|
145
|
+
):
|
|
146
|
+
if field in latest_coverage:
|
|
147
|
+
usage[field] = latest_coverage[field]
|
|
148
|
+
else:
|
|
149
|
+
usage.pop(field, None)
|
|
150
|
+
# Counts describe the coverage observation, even when its scope narrows.
|
|
151
|
+
# Its whole role breakdown must still sum to the retained token subtotal.
|
|
152
|
+
if "byRole" in usage and any(
|
|
153
|
+
usage.get(field) != latest_coverage.get(field) for field in TOKEN_FIELDS.values()
|
|
154
|
+
):
|
|
155
|
+
usage.pop("byRole")
|
|
156
|
+
merged["usage"] = usage
|
|
157
|
+
cost_warning = "cost_estimate_covers_earlier_usage"
|
|
158
|
+
if (
|
|
159
|
+
cost is not None
|
|
160
|
+
and usage is not None
|
|
161
|
+
and usage.get("coverage") in ("complete", "partial")
|
|
162
|
+
and type(usage.get("totalTokens")) is int
|
|
163
|
+
):
|
|
164
|
+
warnings = set(usage.get("warnings", []))
|
|
165
|
+
warnings.discard(cost_warning)
|
|
166
|
+
if usage["totalTokens"] > cost["inputTokens"] + cost["outputTokens"]:
|
|
167
|
+
warnings.add(cost_warning)
|
|
168
|
+
if warnings != set(usage.get("warnings", [])):
|
|
169
|
+
updated = {**usage}
|
|
170
|
+
updated["coverage"] = (
|
|
171
|
+
"partial"
|
|
172
|
+
if warnings - {cost_warning} or usage.get("missingThreadCount", 0)
|
|
173
|
+
else "complete"
|
|
174
|
+
)
|
|
175
|
+
if warnings:
|
|
176
|
+
updated["warnings"] = sorted(warnings)
|
|
177
|
+
else:
|
|
178
|
+
updated.pop("warnings", None)
|
|
179
|
+
merged["usage"] = updated
|
|
180
|
+
if rate_limits:
|
|
181
|
+
merged["usage"] = {
|
|
182
|
+
**merged["usage"],
|
|
183
|
+
"accountRateLimits": list(rate_limits.values()),
|
|
184
|
+
}
|
|
185
|
+
return merged
|
|
186
|
+
|
|
187
|
+
|
|
188
|
+
def _merge_usage_coverage(previous: dict[str, Any], observed: dict[str, Any]) -> dict[str, Any]:
|
|
189
|
+
if previous == observed:
|
|
190
|
+
return observed
|
|
191
|
+
previous_at = _timestamp(previous.get("observedAt"))
|
|
192
|
+
observed_at = _timestamp(observed.get("observedAt"))
|
|
193
|
+
if previous_at is not None and observed_at is not None and previous_at > observed_at:
|
|
194
|
+
previous, observed = observed, previous
|
|
195
|
+
previous_scope = previous.get("coverageScope")
|
|
196
|
+
observed_scope = observed.get("coverageScope")
|
|
197
|
+
covers_previous = (
|
|
198
|
+
previous_scope is not None
|
|
199
|
+
and observed_scope is not None
|
|
200
|
+
and set(previous_scope).issubset(observed_scope)
|
|
201
|
+
)
|
|
202
|
+
scope_gap = previous_scope is not None and observed_scope is not None and not covers_previous
|
|
203
|
+
unresolved_previous = (previous["coverage"] == "partial" and not covers_previous) or scope_gap
|
|
204
|
+
merged = {**observed}
|
|
205
|
+
if unresolved_previous:
|
|
206
|
+
merged["coverage"] = "partial"
|
|
207
|
+
merged["missingThreadCount"] = max(
|
|
208
|
+
previous.get("missingThreadCount", 0), observed.get("missingThreadCount", 0)
|
|
209
|
+
)
|
|
210
|
+
warnings = set(previous.get("warnings", [])) | set(observed.get("warnings", []))
|
|
211
|
+
if scope_gap:
|
|
212
|
+
# A newer subset can grow while still staying below the older
|
|
213
|
+
# aggregate. Neither subtotal proves complete usage for the union.
|
|
214
|
+
warnings.add("usage_scope_incomplete")
|
|
215
|
+
if warnings:
|
|
216
|
+
merged["warnings"] = sorted(warnings)
|
|
217
|
+
if previous.get("completion", {}).get("finalized") is False:
|
|
218
|
+
merged["completion"] = previous["completion"]
|
|
219
|
+
if previous_scope is not None and observed_scope is not None:
|
|
220
|
+
merged["coverageScope"] = sorted(set(previous_scope) | set(observed_scope))
|
|
221
|
+
elif previous_scope is not None and observed["coverage"] == "complete":
|
|
222
|
+
merged["coverageScope"] = previous_scope
|
|
223
|
+
elif unresolved_previous:
|
|
224
|
+
# An older subtotal without scope cannot prove which gaps a later
|
|
225
|
+
# collector has resolved. Retain that uncertainty across future merges.
|
|
226
|
+
merged.pop("coverageScope", None)
|
|
227
|
+
return merged
|
|
228
|
+
|
|
229
|
+
|
|
230
|
+
def _usage_quality(
|
|
231
|
+
usage: Mapping[str, Any], *, observed_at: datetime | None = None
|
|
232
|
+
) -> tuple[int, bool, datetime]:
|
|
233
|
+
return (
|
|
234
|
+
usage.get("totalTokens", -1),
|
|
235
|
+
usage.get("cacheWriteInputTokensReported") is not False,
|
|
236
|
+
observed_at
|
|
237
|
+
or _timestamp(usage.get("observedAt"))
|
|
238
|
+
or datetime.min.replace(tzinfo=timezone.utc),
|
|
239
|
+
)
|
|
240
|
+
|
|
241
|
+
|
|
242
|
+
def collect_scan_usage_safely(
|
|
243
|
+
connection: sqlite3.Connection, scan: sqlite3.Row, **options: Any
|
|
244
|
+
) -> dict[str, Any]:
|
|
245
|
+
try:
|
|
246
|
+
return collect_scan_usage(connection, scan, **options)
|
|
247
|
+
except (Exception, SystemExit) as exc:
|
|
248
|
+
try:
|
|
249
|
+
print(f"Could not collect scan token usage: {exc}", file=sys.stderr)
|
|
250
|
+
except OSError:
|
|
251
|
+
# A closed diagnostic stream must not replace the unavailable-usage result.
|
|
252
|
+
pass
|
|
253
|
+
return _unavailable_usage("usage_collection_failed")
|
|
254
|
+
|
|
255
|
+
|
|
84
256
|
def collect_scan_usage(
|
|
85
257
|
connection: sqlite3.Connection,
|
|
86
258
|
scan: sqlite3.Row,
|
|
87
259
|
*,
|
|
88
260
|
thread_id: str | None = None,
|
|
261
|
+
validation_thread_id: str | None = None,
|
|
89
262
|
completed_at: str | None = None,
|
|
90
263
|
completed_turn_id: str | None = None,
|
|
91
264
|
) -> dict[str, Any]:
|
|
92
|
-
"""Count attributable receipts
|
|
265
|
+
"""Count attributable session receipts during the scan window and report missing coverage.
|
|
266
|
+
|
|
267
|
+
The optional thread and completion arguments identify the finishing turn when
|
|
268
|
+
the scan record alone is insufficient. Return an unavailable usage record when
|
|
269
|
+
the window, Codex state, or relevant threads cannot be established.
|
|
270
|
+
"""
|
|
93
271
|
|
|
94
272
|
roots = _scan_root_thread_ids(connection, scan, thread_id)
|
|
95
273
|
parent_ids = set(_scan_root_thread_ids(connection, scan, thread_id, include_workers=False))
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
if state_database is None:
|
|
102
|
-
return _unavailable_usage("codex_state_unavailable")
|
|
103
|
-
|
|
274
|
+
if validation_thread_id:
|
|
275
|
+
roots.append(validation_thread_id)
|
|
276
|
+
parent_ids.add(validation_thread_id)
|
|
277
|
+
warnings: set[str] = set()
|
|
278
|
+
worker_directories = _scan_worker_directories(connection, scan, warnings)
|
|
104
279
|
started_at = _timestamp(scan["started_at"])
|
|
105
280
|
observed_at = datetime.now(timezone.utc)
|
|
106
281
|
stopped_at = _timestamp(completed_at or scan["completed_at"]) or observed_at
|
|
107
282
|
if started_at is None:
|
|
108
283
|
return _unavailable_usage("scan_window_unavailable")
|
|
109
284
|
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
)
|
|
120
|
-
|
|
121
|
-
|
|
285
|
+
receipts, missing_receipts, receipt_scope = _worker_usage_receipts(
|
|
286
|
+
scan, started_at, stopped_at, warnings
|
|
287
|
+
)
|
|
288
|
+
state_database = _codex_state_database()
|
|
289
|
+
sessions: list[RolloutSession] = []
|
|
290
|
+
missing_thread_ids: set[str] = set()
|
|
291
|
+
if state_database is None:
|
|
292
|
+
if not receipts:
|
|
293
|
+
return _unavailable_usage("codex_state_unavailable", warnings=warnings)
|
|
294
|
+
warnings.add("codex_state_unavailable")
|
|
295
|
+
missing_thread_ids.update(roots)
|
|
296
|
+
else:
|
|
297
|
+
try:
|
|
298
|
+
sessions, missing_thread_ids = _discover_rollout_sessions(
|
|
299
|
+
state_database,
|
|
300
|
+
roots,
|
|
301
|
+
warnings,
|
|
302
|
+
worker_directories=worker_directories,
|
|
303
|
+
scan_directory=Path(scan["scan_dir"]) if worker_directories else None,
|
|
304
|
+
started_at=started_at,
|
|
305
|
+
completed_at=stopped_at,
|
|
306
|
+
)
|
|
307
|
+
except (OSError, sqlite3.Error, ValueError):
|
|
308
|
+
if not receipts:
|
|
309
|
+
return _unavailable_usage("codex_state_unavailable", warnings=warnings)
|
|
310
|
+
warnings.add("codex_state_unavailable")
|
|
311
|
+
missing_thread_ids.update(roots)
|
|
122
312
|
|
|
123
|
-
if not sessions:
|
|
313
|
+
if not sessions and not receipts:
|
|
124
314
|
return _unavailable_usage("scan_thread_unavailable", warnings=warnings)
|
|
125
315
|
|
|
126
|
-
by_role = {role: {**_empty_token_usage(), "threadCount": 0} for role in ("parent", "workers")}
|
|
127
316
|
rate_limits: dict[str, dict[str, Any]] = {}
|
|
128
317
|
tokens_updated_at = None
|
|
129
318
|
accepted_thread_ids: set[str] = set()
|
|
319
|
+
measured_threads: dict[str, dict[str, int]] = {}
|
|
320
|
+
complete_native_thread_ids: set[str] = set()
|
|
130
321
|
excluded_thread_ids: set[str] = set()
|
|
131
322
|
for session in sessions:
|
|
132
323
|
if session.parent_thread_id in excluded_thread_ids:
|
|
@@ -145,6 +336,12 @@ def collect_scan_usage(
|
|
|
145
336
|
started_at=started_at,
|
|
146
337
|
completed_at=stopped_at,
|
|
147
338
|
completed_turn_id=completed_turn_id if session.thread_id == thread_id else None,
|
|
339
|
+
resume_owner=(
|
|
340
|
+
scan["mode"] == "standard"
|
|
341
|
+
and "recipe_json" in scan.keys()
|
|
342
|
+
and scan["recipe_json"] is not None
|
|
343
|
+
and session.thread_id == scan["continuation_thread_id"]
|
|
344
|
+
),
|
|
148
345
|
is_scan_owner=(
|
|
149
346
|
scan["mode"] == "deep"
|
|
150
347
|
and session.thread_id == scan["deep_scan_owner_thread_id"]
|
|
@@ -167,25 +364,60 @@ def collect_scan_usage(
|
|
|
167
364
|
for key, snapshot in usage.rate_limits.items():
|
|
168
365
|
if key not in rate_limits or snapshot["observedAt"] > rate_limits[key]["observedAt"]:
|
|
169
366
|
rate_limits[key] = snapshot
|
|
170
|
-
if "token_usage_unavailable"
|
|
367
|
+
if usage.warnings & {"token_usage_unavailable", "token_breakdown_unavailable"}:
|
|
171
368
|
missing_thread_ids.add(session.thread_id)
|
|
172
369
|
continue
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
370
|
+
measured_threads[session.thread_id] = usage.tokens
|
|
371
|
+
if not usage.warnings:
|
|
372
|
+
complete_native_thread_ids.add(session.thread_id)
|
|
176
373
|
if usage.tokens_updated_at is not None:
|
|
177
374
|
tokens_updated_at = max(
|
|
178
375
|
tokens_updated_at or usage.tokens_updated_at, usage.tokens_updated_at
|
|
179
376
|
)
|
|
180
377
|
|
|
378
|
+
runtime_receipts_used = False
|
|
379
|
+
for receipt_thread_id, receipt in receipts.items():
|
|
380
|
+
native = measured_threads.get(receipt_thread_id)
|
|
381
|
+
extends_native = native is None or receipt.tokens["totalTokens"] > native["totalTokens"]
|
|
382
|
+
fills_cache_write_usage = (
|
|
383
|
+
native is not None
|
|
384
|
+
and receipt.tokens["totalTokens"] == native["totalTokens"]
|
|
385
|
+
and native.get("cacheWriteInputTokensReported") is False
|
|
386
|
+
and receipt.tokens.get("cacheWriteInputTokensReported") is not False
|
|
387
|
+
)
|
|
388
|
+
if extends_native or fills_cache_write_usage:
|
|
389
|
+
measured_threads[receipt_thread_id] = receipt.tokens
|
|
390
|
+
accepted_thread_ids.add(receipt_thread_id)
|
|
391
|
+
runtime_receipts_used = True
|
|
392
|
+
if extends_native:
|
|
393
|
+
complete_native_thread_ids.discard(receipt_thread_id)
|
|
394
|
+
# Numeric SDK subtotals cannot establish that descendants and
|
|
395
|
+
# compaction requests are present when their rollouts are absent.
|
|
396
|
+
missing_thread_ids.add(receipt_thread_id)
|
|
397
|
+
warnings.add("worker_rollout_usage_incomplete")
|
|
398
|
+
if receipt.tokens_updated_at is not None:
|
|
399
|
+
tokens_updated_at = max(
|
|
400
|
+
tokens_updated_at or receipt.tokens_updated_at, receipt.tokens_updated_at
|
|
401
|
+
)
|
|
402
|
+
unresolved_receipts = missing_receipts - complete_native_thread_ids
|
|
403
|
+
if unresolved_receipts:
|
|
404
|
+
missing_thread_ids.update(unresolved_receipts)
|
|
405
|
+
warnings.add("worker_usage_unavailable")
|
|
181
406
|
if not accepted_thread_ids:
|
|
182
407
|
return _unavailable_usage("scan_thread_unavailable", warnings=warnings)
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
"
|
|
408
|
+
if not measured_threads:
|
|
409
|
+
reason = (
|
|
410
|
+
"token_breakdown_unavailable"
|
|
411
|
+
if "token_breakdown_unavailable" in warnings
|
|
412
|
+
else "token_usage_unavailable"
|
|
187
413
|
)
|
|
414
|
+
return _unavailable_usage(reason, warnings=warnings, rate_limits=rate_limits)
|
|
188
415
|
|
|
416
|
+
by_role = {role: {**_empty_token_usage(), "threadCount": 0} for role in ("parent", "workers")}
|
|
417
|
+
for measured_id, tokens in measured_threads.items():
|
|
418
|
+
role = by_role["parent" if measured_id in parent_ids else "workers"]
|
|
419
|
+
_add_token_usage(role, tokens)
|
|
420
|
+
role["threadCount"] += 1
|
|
189
421
|
if any(
|
|
190
422
|
usage["cachedInputTokens"] + usage["cacheWriteInputTokens"] > usage["inputTokens"]
|
|
191
423
|
or usage["totalTokens"] != usage["inputTokens"] + usage["outputTokens"]
|
|
@@ -195,18 +427,28 @@ def collect_scan_usage(
|
|
|
195
427
|
"token_breakdown_unavailable", warnings=warnings, rate_limits=rate_limits
|
|
196
428
|
)
|
|
197
429
|
|
|
198
|
-
if completed_turn_id is not None and tokens_updated_at is not None:
|
|
430
|
+
if (completed_turn_id is not None or runtime_receipts_used) and tokens_updated_at is not None:
|
|
199
431
|
stopped_at = max(stopped_at, tokens_updated_at)
|
|
200
432
|
result: dict[str, Any] = {
|
|
201
433
|
"coverage": "partial" if missing_thread_ids or warnings else "complete",
|
|
202
|
-
"source": "codex_rollout",
|
|
434
|
+
"source": "codex_runtime" if runtime_receipts_used else "codex_rollout",
|
|
203
435
|
**{key: sum(role[key] for role in by_role.values()) for key in TOKEN_FIELDS.values()},
|
|
204
|
-
"threadCount":
|
|
436
|
+
"threadCount": len(measured_threads),
|
|
437
|
+
"coverageScope": sorted(
|
|
438
|
+
receipt_scope
|
|
439
|
+
| {f"thread:{identity}" for identity in measured_threads}
|
|
440
|
+
| {
|
|
441
|
+
identity if identity.startswith("worker-attempt:") else f"thread:{identity}"
|
|
442
|
+
for identity in missing_thread_ids
|
|
443
|
+
}
|
|
444
|
+
),
|
|
205
445
|
"byRole": by_role,
|
|
206
446
|
"window": {"startedAt": started_at.isoformat(), "endedAt": stopped_at.isoformat()},
|
|
207
447
|
"observedAt": observed_at.isoformat(),
|
|
208
448
|
"accountRateLimits": [rate_limits[key] for key in sorted(rate_limits)],
|
|
209
449
|
}
|
|
450
|
+
if any(role.get("cacheWriteInputTokensReported") is False for role in by_role.values()):
|
|
451
|
+
result["cacheWriteInputTokensReported"] = False
|
|
210
452
|
if tokens_updated_at is not None:
|
|
211
453
|
result["tokensUpdatedAt"] = tokens_updated_at.isoformat()
|
|
212
454
|
if missing_thread_ids:
|
|
@@ -216,6 +458,79 @@ def collect_scan_usage(
|
|
|
216
458
|
return result
|
|
217
459
|
|
|
218
460
|
|
|
461
|
+
def _worker_usage_receipts(
|
|
462
|
+
scan: Mapping[str, Any],
|
|
463
|
+
started_at: datetime,
|
|
464
|
+
completed_at: datetime,
|
|
465
|
+
warnings: set[str],
|
|
466
|
+
) -> tuple[dict[str, RolloutUsage], set[str], set[str]]:
|
|
467
|
+
if scan["mode"] != "deep":
|
|
468
|
+
return {}, set(), set()
|
|
469
|
+
directory = worker_usage_directory(scan["id"])
|
|
470
|
+
try:
|
|
471
|
+
if directory.resolve(strict=True) != directory:
|
|
472
|
+
raise ValueError("Worker usage directory is not a canonical protected directory.")
|
|
473
|
+
except FileNotFoundError:
|
|
474
|
+
return {}, set(), set()
|
|
475
|
+
except (OSError, RuntimeError, ValueError):
|
|
476
|
+
warnings.add("worker_usage_receipt_unavailable")
|
|
477
|
+
return {}, {"worker-receipts"}, {"worker-receipts"}
|
|
478
|
+
receipts: dict[str, RolloutUsage] = {}
|
|
479
|
+
missing: set[str] = set()
|
|
480
|
+
attempts: set[str] = set()
|
|
481
|
+
scope: set[str] = set()
|
|
482
|
+
for path in sorted(directory.glob("*.json")):
|
|
483
|
+
try:
|
|
484
|
+
if _rollout_path(str(path)) is None:
|
|
485
|
+
raise ValueError("Worker usage receipt path is unavailable.")
|
|
486
|
+
receipt = json.loads(path.read_text(encoding="utf-8"))
|
|
487
|
+
if (
|
|
488
|
+
not isinstance(receipt, dict)
|
|
489
|
+
or receipt.get("schemaVersion") != 1
|
|
490
|
+
or receipt.get("source") != "codex_sdk"
|
|
491
|
+
or receipt.get("usageScope") != "thread_cumulative"
|
|
492
|
+
or receipt.get("scanId") != scan["id"]
|
|
493
|
+
or receipt.get("kind") not in ("discovery", "dedup", "compare")
|
|
494
|
+
or receipt.get("attemptId") != path.stem
|
|
495
|
+
):
|
|
496
|
+
raise ValueError("Worker usage receipt identity is invalid.")
|
|
497
|
+
attempt_id = receipt["attemptId"]
|
|
498
|
+
begin = _timestamp(receipt.get("startedAt"))
|
|
499
|
+
observed = _timestamp(receipt.get("observedAt"))
|
|
500
|
+
if begin is None or observed is None:
|
|
501
|
+
raise ValueError("Worker usage receipt timestamps are invalid.")
|
|
502
|
+
if begin < started_at or begin > completed_at or attempt_id in attempts:
|
|
503
|
+
continue
|
|
504
|
+
attempts.add(attempt_id)
|
|
505
|
+
scope.add(f"worker-attempt:{attempt_id}")
|
|
506
|
+
thread_id = receipt.get("threadId")
|
|
507
|
+
if isinstance(thread_id, str) and thread_id:
|
|
508
|
+
scope.add(f"thread:{thread_id}")
|
|
509
|
+
tokens = _usage_counts(receipt.get("usage"))
|
|
510
|
+
if not isinstance(thread_id, str) or not thread_id or tokens is None:
|
|
511
|
+
missing.add(
|
|
512
|
+
thread_id
|
|
513
|
+
if isinstance(thread_id, str) and thread_id
|
|
514
|
+
else f"worker-attempt:{attempt_id}"
|
|
515
|
+
)
|
|
516
|
+
continue
|
|
517
|
+
previous = receipts.get(thread_id)
|
|
518
|
+
if previous is None or _usage_quality(tokens, observed_at=observed) > _usage_quality(
|
|
519
|
+
previous.tokens, observed_at=previous.tokens_updated_at
|
|
520
|
+
):
|
|
521
|
+
receipts[thread_id] = RolloutUsage(tokens, set(), observed, {})
|
|
522
|
+
except (OSError, UnicodeError, ValueError) as exc:
|
|
523
|
+
warnings.add("worker_usage_receipt_unavailable")
|
|
524
|
+
missing.add(f"worker-attempt:{path.stem}")
|
|
525
|
+
scope.add(f"worker-attempt:{path.stem}")
|
|
526
|
+
try:
|
|
527
|
+
print(f"Could not read worker token usage: {exc}", file=sys.stderr)
|
|
528
|
+
except OSError:
|
|
529
|
+
# The receipt warning and missing-attempt scope are already retained.
|
|
530
|
+
pass
|
|
531
|
+
return receipts, missing, scope
|
|
532
|
+
|
|
533
|
+
|
|
219
534
|
def _scan_root_thread_ids(
|
|
220
535
|
connection: sqlite3.Connection,
|
|
221
536
|
scan: sqlite3.Row,
|
|
@@ -266,25 +581,41 @@ def _scan_execution_thread_ids(connection: sqlite3.Connection, scan: sqlite3.Row
|
|
|
266
581
|
)
|
|
267
582
|
|
|
268
583
|
|
|
269
|
-
def _scan_worker_directories(
|
|
584
|
+
def _scan_worker_directories(
|
|
585
|
+
connection: sqlite3.Connection, scan: sqlite3.Row, warnings: set[str]
|
|
586
|
+
) -> dict[str, bool]:
|
|
270
587
|
if scan["mode"] != "deep":
|
|
271
|
-
return
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
588
|
+
return {}
|
|
589
|
+
if connection.execute(
|
|
590
|
+
"SELECT 1 FROM deep_scan_runs WHERE scan_id = ? AND worker_usage_gap = 1", (scan["id"],)
|
|
591
|
+
).fetchone():
|
|
592
|
+
warnings.add("worker_thread_unavailable")
|
|
593
|
+
artifacts = Path(scan["scan_dir"]) / "artifacts"
|
|
594
|
+
# Comparison turns run in this directory without a discovery worker row.
|
|
595
|
+
directories = {str(artifacts / "deep_discovery"): False}
|
|
596
|
+
for row in connection.execute(
|
|
597
|
+
"SELECT kind, artifact_dir, started_at, sdk_thread_id, attempt FROM deep_scan_workers "
|
|
598
|
+
"WHERE scan_id = ? AND kind IN ('discovery', 'dedup')",
|
|
599
|
+
(scan["id"],),
|
|
600
|
+
):
|
|
601
|
+
if row["kind"] == "discovery":
|
|
602
|
+
if row["sdk_thread_id"] and row["attempt"] == 1:
|
|
603
|
+
continue # The first attempt already has an exact thread identity.
|
|
604
|
+
# An attempt that started without a saved ID must be found by its assigned cwd.
|
|
605
|
+
missing_thread = bool(row["started_at"] and not row["sdk_thread_id"])
|
|
606
|
+
directories[row["artifact_dir"]] = missing_thread
|
|
607
|
+
else:
|
|
608
|
+
directories[str(artifacts)] = False
|
|
609
|
+
if row["attempt"] > 1 and row["started_at"] and not row["sdk_thread_id"]:
|
|
610
|
+
# Retries share a cwd. A rollout from an older attempt cannot prove the new one.
|
|
611
|
+
warnings.add("worker_thread_unavailable")
|
|
612
|
+
return directories
|
|
282
613
|
|
|
283
614
|
|
|
284
615
|
def _codex_state_database() -> Path | None:
|
|
285
|
-
configured_database = os.environ.get("CODEX_STATE_DB", "")
|
|
286
|
-
if configured_database:
|
|
287
|
-
path = Path(
|
|
616
|
+
configured_database = os.environ.get("CODEX_STATE_DB", "")
|
|
617
|
+
if configured_database.strip():
|
|
618
|
+
path = Path(os.path.expanduser(configured_database))
|
|
288
619
|
return path.resolve() if path.is_file() and os.access(path, os.R_OK) else None
|
|
289
620
|
|
|
290
621
|
configured_home = os.environ.get("CODEX_HOME", "")
|
|
@@ -323,12 +654,13 @@ def _discover_rollout_sessions(
|
|
|
323
654
|
roots: list[str],
|
|
324
655
|
warnings: set[str],
|
|
325
656
|
*,
|
|
326
|
-
worker_directories:
|
|
657
|
+
worker_directories: dict[str, bool],
|
|
658
|
+
scan_directory: Path | None = None,
|
|
327
659
|
started_at: datetime,
|
|
328
660
|
completed_at: datetime | None,
|
|
329
661
|
) -> tuple[list[RolloutSession], set[str]]:
|
|
330
662
|
database = sqlite3.connect(
|
|
331
|
-
|
|
663
|
+
readonly_sqlite_uri(state_database),
|
|
332
664
|
uri=True,
|
|
333
665
|
timeout=STATE_DATABASE_TIMEOUT_SECONDS,
|
|
334
666
|
)
|
|
@@ -342,9 +674,12 @@ def _discover_rollout_sessions(
|
|
|
342
674
|
columns = {str(row["name"]) for row in database.execute(f"PRAGMA table_info({table})")}
|
|
343
675
|
if not required.issubset(columns):
|
|
344
676
|
raise ValueError("Codex state graph does not expose the required thread columns.")
|
|
677
|
+
if table == "threads" and worker_directories and not {"cwd", "created_at"} <= columns:
|
|
678
|
+
worker_directories = {}
|
|
679
|
+
warnings.add("worker_directories_unavailable")
|
|
345
680
|
sessions: dict[str, RolloutSession] = {}
|
|
346
681
|
missing_thread_ids: set[str] = set()
|
|
347
|
-
root_ids = dict.fromkeys(roots)
|
|
682
|
+
root_ids: dict[str, str | None] = dict.fromkeys(roots)
|
|
348
683
|
started_millisecond = started_at.replace(microsecond=started_at.microsecond // 1000 * 1000)
|
|
349
684
|
completed_timestamp = int(completed_at.timestamp()) if completed_at is not None else None
|
|
350
685
|
directory_expression = "cwd"
|
|
@@ -355,16 +690,54 @@ def _discover_rollout_sessions(
|
|
|
355
690
|
"codex_security_path_key", 1, _normalized_windows_path, deterministic=True
|
|
356
691
|
)
|
|
357
692
|
directory_expression = "codex_security_path_key(cwd)"
|
|
358
|
-
worker_directories = {
|
|
359
|
-
|
|
693
|
+
worker_directories = {
|
|
694
|
+
_normalized_windows_path(path): required
|
|
695
|
+
for path, required in worker_directories.items()
|
|
696
|
+
}
|
|
697
|
+
if scan_directory is not None:
|
|
698
|
+
scan_directory = Path(_normalized_windows_path(scan_directory))
|
|
699
|
+
unmatched = {path for path, required in worker_directories.items() if required}
|
|
700
|
+
directory_sources = {path: {path} for path in worker_directories}
|
|
701
|
+
if worker_directories and scan_directory and ".previous-" in scan_directory.name:
|
|
702
|
+
# Archived files move, but session working directories describe where execution ran.
|
|
703
|
+
# Confirm the original output with an already-bound session before including peers.
|
|
704
|
+
bound_directories = {
|
|
705
|
+
Path(row[0])
|
|
706
|
+
for row in database.execute(
|
|
707
|
+
f"SELECT DISTINCT {directory_expression} FROM threads WHERE cwd IS NOT NULL "
|
|
708
|
+
"AND id IN (SELECT value FROM json_each(?))",
|
|
709
|
+
(json.dumps(roots),),
|
|
710
|
+
)
|
|
711
|
+
}
|
|
712
|
+
relatives = {
|
|
713
|
+
Path(path).relative_to(scan_directory): path
|
|
714
|
+
for path in worker_directories
|
|
715
|
+
if Path(path).is_relative_to(scan_directory)
|
|
716
|
+
}
|
|
717
|
+
originals = {
|
|
718
|
+
original
|
|
719
|
+
for path in bound_directories | {Path(path) for path in worker_directories}
|
|
720
|
+
if (original := archive_source_directory(scan_directory, path)) is not None
|
|
721
|
+
}
|
|
722
|
+
for original in originals:
|
|
723
|
+
candidates = {original / relative: path for relative, path in relatives.items()}
|
|
724
|
+
# Older archives kept absolute discovery paths in their original directory.
|
|
725
|
+
if (
|
|
726
|
+
original in bound_directories
|
|
727
|
+
or candidates.keys() & bound_directories
|
|
728
|
+
or any(Path(path).is_relative_to(original) for path in worker_directories)
|
|
729
|
+
):
|
|
730
|
+
for candidate, path in candidates.items():
|
|
731
|
+
directory_sources.setdefault(str(candidate), set()).add(path)
|
|
732
|
+
for directory in sorted(directory_sources):
|
|
360
733
|
for row in database.execute(
|
|
361
734
|
"SELECT id FROM threads WHERE cwd IS NOT NULL "
|
|
362
735
|
f"AND {directory_expression} = ? AND created_at >= ? "
|
|
363
736
|
"AND (? IS NULL OR created_at <= ?) ORDER BY id",
|
|
364
737
|
(directory, int(started_at.timestamp()), completed_timestamp, completed_timestamp),
|
|
365
738
|
):
|
|
366
|
-
root_ids[row["id"]] =
|
|
367
|
-
for root in root_ids:
|
|
739
|
+
root_ids[row["id"]] = directory
|
|
740
|
+
for root, directory in root_ids.items():
|
|
368
741
|
row = database.execute(
|
|
369
742
|
"SELECT id, rollout_path FROM threads WHERE id = ?",
|
|
370
743
|
(root,),
|
|
@@ -380,8 +753,8 @@ def _discover_rollout_sessions(
|
|
|
380
753
|
continue
|
|
381
754
|
if root not in roots:
|
|
382
755
|
try:
|
|
383
|
-
with path
|
|
384
|
-
metadata = json.loads(source
|
|
756
|
+
with _rollout_lines(path) as source:
|
|
757
|
+
metadata = json.loads(next(source, b""))
|
|
385
758
|
except (OSError, ValueError):
|
|
386
759
|
missing_thread_ids.add(root)
|
|
387
760
|
warnings.add("rollout_unavailable")
|
|
@@ -392,9 +765,15 @@ def _discover_rollout_sessions(
|
|
|
392
765
|
and isinstance(payload := metadata.get("payload"), dict)
|
|
393
766
|
and (payload.get("id") or payload.get("session_id")) == root
|
|
394
767
|
and (created_at := _timestamp(payload.get("timestamp"))) is not None
|
|
395
|
-
and
|
|
768
|
+
and (
|
|
769
|
+
created_at < started_millisecond
|
|
770
|
+
or completed_at is not None
|
|
771
|
+
and created_at > completed_at
|
|
772
|
+
)
|
|
396
773
|
):
|
|
397
774
|
continue
|
|
775
|
+
if directory is not None:
|
|
776
|
+
unmatched.difference_update(directory_sources[directory])
|
|
398
777
|
sessions[root] = RolloutSession(root, None, path)
|
|
399
778
|
descendants = database.execute(
|
|
400
779
|
"""
|
|
@@ -452,6 +831,8 @@ def _discover_rollout_sessions(
|
|
|
452
831
|
warnings.add("rollout_unavailable")
|
|
453
832
|
continue
|
|
454
833
|
sessions[child_id] = RolloutSession(child_id, parent_id, path)
|
|
834
|
+
if unmatched:
|
|
835
|
+
warnings.add("worker_thread_unavailable")
|
|
455
836
|
return list(sessions.values()), missing_thread_ids
|
|
456
837
|
finally:
|
|
457
838
|
database.close()
|
|
@@ -464,6 +845,13 @@ def _rollout_path(value: object) -> Path | None:
|
|
|
464
845
|
if not candidate.is_absolute():
|
|
465
846
|
return None
|
|
466
847
|
try:
|
|
848
|
+
try:
|
|
849
|
+
candidate.lstat()
|
|
850
|
+
except FileNotFoundError:
|
|
851
|
+
# Native cold storage may compress an indexed JSONL file in place.
|
|
852
|
+
if candidate.suffix != ".jsonl":
|
|
853
|
+
return None
|
|
854
|
+
candidate = candidate.with_suffix(".jsonl.zst")
|
|
467
855
|
resolved = candidate.resolve(strict=True)
|
|
468
856
|
if not resolved.is_file():
|
|
469
857
|
return None
|
|
@@ -480,6 +868,63 @@ def _rollout_path(value: object) -> Path | None:
|
|
|
480
868
|
return None
|
|
481
869
|
|
|
482
870
|
|
|
871
|
+
@contextmanager
|
|
872
|
+
def _rollout_lines(path: Path, *, verify_end: bool = False):
|
|
873
|
+
if path.suffix != ".zst":
|
|
874
|
+
with path.open("rb") as source:
|
|
875
|
+
yield source
|
|
876
|
+
return
|
|
877
|
+
try:
|
|
878
|
+
from compression import zstd # Python 3.14+
|
|
879
|
+
except ImportError:
|
|
880
|
+
try:
|
|
881
|
+
from backports import zstd
|
|
882
|
+
except ImportError:
|
|
883
|
+
zstd = None
|
|
884
|
+
if zstd is not None:
|
|
885
|
+
try:
|
|
886
|
+
with zstd.open(path, "rb") as source:
|
|
887
|
+
yield source
|
|
888
|
+
# A requested completion turn may stop parsing before the frame checksum.
|
|
889
|
+
if verify_end:
|
|
890
|
+
while source.read(64 * 1024):
|
|
891
|
+
pass
|
|
892
|
+
except (zstd.ZstdError, EOFError) as error:
|
|
893
|
+
raise ValueError("Invalid compressed Codex rollout.") from error
|
|
894
|
+
return
|
|
895
|
+
with tempfile.TemporaryFile() as errors:
|
|
896
|
+
with subprocess.Popen(
|
|
897
|
+
[
|
|
898
|
+
os.environ.get("CODEX_MCP_NODE_PATH") or "node",
|
|
899
|
+
str(Path(__file__).with_name("read_codex_session.mjs")),
|
|
900
|
+
str(path),
|
|
901
|
+
],
|
|
902
|
+
stdout=subprocess.PIPE,
|
|
903
|
+
stderr=errors,
|
|
904
|
+
) as child:
|
|
905
|
+
exhausted = False
|
|
906
|
+
|
|
907
|
+
def lines():
|
|
908
|
+
nonlocal exhausted
|
|
909
|
+
yield from child.stdout
|
|
910
|
+
exhausted = True
|
|
911
|
+
|
|
912
|
+
try:
|
|
913
|
+
iterator = lines()
|
|
914
|
+
yield iterator
|
|
915
|
+
if verify_end:
|
|
916
|
+
for _ in iterator:
|
|
917
|
+
pass
|
|
918
|
+
finally:
|
|
919
|
+
if not exhausted and child.poll() is None:
|
|
920
|
+
child.terminate()
|
|
921
|
+
child.stdout.close()
|
|
922
|
+
status = child.wait()
|
|
923
|
+
if exhausted and status != 0:
|
|
924
|
+
errors.seek(0)
|
|
925
|
+
raise OSError(errors.read().decode("utf-8", errors="replace"))
|
|
926
|
+
|
|
927
|
+
|
|
483
928
|
def _read_rollout_usage(
|
|
484
929
|
session: RolloutSession,
|
|
485
930
|
*,
|
|
@@ -487,13 +932,19 @@ def _read_rollout_usage(
|
|
|
487
932
|
completed_at: datetime | None,
|
|
488
933
|
completed_turn_id: str | None = None,
|
|
489
934
|
is_scan_owner: bool = False,
|
|
935
|
+
resume_owner: bool = False,
|
|
490
936
|
) -> RolloutUsage:
|
|
491
937
|
result = RolloutUsage(_empty_token_usage(), set(), None, {})
|
|
492
938
|
total = result.tokens
|
|
493
939
|
warnings = result.warnings
|
|
494
940
|
previous = _empty_token_usage()
|
|
941
|
+
previous_cache_writes_reported = True
|
|
942
|
+
prefix_unclassified = 0
|
|
943
|
+
prefix_missing_cache: set[str] = set()
|
|
944
|
+
ambiguous_legacy_cache = False
|
|
495
945
|
boundary_reached = False
|
|
496
946
|
legacy_usage = _empty_token_usage()
|
|
947
|
+
completed_legacy_usage = _empty_token_usage()
|
|
497
948
|
response_usage: dict[str, int] | None = None
|
|
498
949
|
observed_response_usage = _empty_token_usage()
|
|
499
950
|
response_ids: set[str] = set()
|
|
@@ -503,6 +954,7 @@ def _read_rollout_usage(
|
|
|
503
954
|
started_turns: set[str] = set()
|
|
504
955
|
open_turns: set[str] = set()
|
|
505
956
|
closed_turns: set[str] = set()
|
|
957
|
+
failed_turns: set[str] = set()
|
|
506
958
|
measured_turns: set[str] = set()
|
|
507
959
|
|
|
508
960
|
def past_scan_end(timestamp: datetime, turn_id: object) -> bool:
|
|
@@ -513,7 +965,7 @@ def _read_rollout_usage(
|
|
|
513
965
|
and (completed_turn_id is None or turn_id != completed_turn_id)
|
|
514
966
|
)
|
|
515
967
|
|
|
516
|
-
with session.path
|
|
968
|
+
with _rollout_lines(session.path, verify_end=True) as source:
|
|
517
969
|
for line_number, raw_line in enumerate(source, start=1):
|
|
518
970
|
if not raw_line.endswith(b"\n"):
|
|
519
971
|
warnings.add("rollout_record_incomplete")
|
|
@@ -575,7 +1027,14 @@ def _read_rollout_usage(
|
|
|
575
1027
|
elif event.get("type") == "event_msg" and payload.get("type") == "token_count":
|
|
576
1028
|
inherited_usage = _token_snapshot(payload)
|
|
577
1029
|
if inherited_usage is not None:
|
|
578
|
-
|
|
1030
|
+
missing_cache = _missing_cache_fields(payload)
|
|
1031
|
+
baseline = _retain_cache_baseline(previous, inherited_usage, missing_cache)
|
|
1032
|
+
if baseline is None:
|
|
1033
|
+
warnings.add("token_breakdown_unavailable")
|
|
1034
|
+
else:
|
|
1035
|
+
previous = baseline
|
|
1036
|
+
prefix_unclassified = _uncached_input(baseline)
|
|
1037
|
+
prefix_missing_cache = missing_cache
|
|
579
1038
|
if not boundary_reached:
|
|
580
1039
|
continue
|
|
581
1040
|
if event.get("type") == "event_msg" and payload.get("type") == "task_started":
|
|
@@ -593,6 +1052,11 @@ def _read_rollout_usage(
|
|
|
593
1052
|
_add_token_usage(
|
|
594
1053
|
total, response_usage if response_usage is not None else legacy_usage
|
|
595
1054
|
)
|
|
1055
|
+
if response_usage is None:
|
|
1056
|
+
_add_token_usage(completed_legacy_usage, legacy_usage)
|
|
1057
|
+
if ambiguous_legacy_cache:
|
|
1058
|
+
warnings.add("token_breakdown_unavailable")
|
|
1059
|
+
ambiguous_legacy_cache = False
|
|
596
1060
|
legacy_usage = _empty_token_usage()
|
|
597
1061
|
response_usage = None
|
|
598
1062
|
if payload.get("turn_id") != current_turn_id:
|
|
@@ -621,6 +1085,8 @@ def _read_rollout_usage(
|
|
|
621
1085
|
if timestamp >= started_at:
|
|
622
1086
|
if terminal_turn_id in started_turns:
|
|
623
1087
|
closed_turns.add(terminal_turn_id)
|
|
1088
|
+
if payload.get("type") == "task_complete" and payload.get("error"):
|
|
1089
|
+
failed_turns.add(terminal_turn_id)
|
|
624
1090
|
if payload.get("type") == "turn_aborted":
|
|
625
1091
|
# Later successful work does not recover interrupted request usage.
|
|
626
1092
|
warnings.add("turn_interrupted")
|
|
@@ -687,6 +1153,20 @@ def _read_rollout_usage(
|
|
|
687
1153
|
warnings.add("token_record_incomplete")
|
|
688
1154
|
result.tokens_updated_at = max(result.tokens_updated_at or timestamp, timestamp)
|
|
689
1155
|
_add_token_usage(response_usage, usage)
|
|
1156
|
+
if (
|
|
1157
|
+
resume_owner
|
|
1158
|
+
and receipt_turn_id == current_turn_id
|
|
1159
|
+
and current_turn_id in started_turns
|
|
1160
|
+
and (open_turns - {current_turn_id} or failed_turns)
|
|
1161
|
+
):
|
|
1162
|
+
# Matching thread totals account for turns left open or failed before resume.
|
|
1163
|
+
cumulative = _usage_counts(payload.get("thread_token_usage"))
|
|
1164
|
+
if cumulative is not None and all(
|
|
1165
|
+
cumulative[key] == total[key] + response_usage[key]
|
|
1166
|
+
for key in TOKEN_FIELDS.values()
|
|
1167
|
+
):
|
|
1168
|
+
open_turns.intersection_update({current_turn_id})
|
|
1169
|
+
closed_turns.difference_update(failed_turns - {current_turn_id})
|
|
690
1170
|
continue
|
|
691
1171
|
if event.get("type") != "event_msg" or payload.get("type") != "token_count":
|
|
692
1172
|
continue
|
|
@@ -711,25 +1191,98 @@ def _read_rollout_usage(
|
|
|
711
1191
|
if timestamp is None or snapshot is None:
|
|
712
1192
|
warnings.add("token_record_invalid")
|
|
713
1193
|
continue
|
|
1194
|
+
reset = snapshot["totalTokens"] < previous["totalTokens"]
|
|
1195
|
+
if reset:
|
|
1196
|
+
prefix_unclassified = 0
|
|
1197
|
+
missing_cache = _missing_cache_fields(payload)
|
|
1198
|
+
cache_writes_reported = "cacheWriteInputTokens" not in missing_cache
|
|
1199
|
+
baseline = _retain_cache_baseline(previous, snapshot, missing_cache)
|
|
1200
|
+
if baseline is None:
|
|
1201
|
+
if not past_scan_end(timestamp, current_turn_id):
|
|
1202
|
+
warnings.add("token_breakdown_unavailable")
|
|
1203
|
+
continue
|
|
1204
|
+
if baseline is previous:
|
|
1205
|
+
cache_writes_reported = previous_cache_writes_reported and cache_writes_reported
|
|
1206
|
+
refines_classification = (
|
|
1207
|
+
baseline["inputTokens"] >= previous["inputTokens"]
|
|
1208
|
+
and baseline["outputTokens"] >= previous["outputTokens"]
|
|
1209
|
+
and baseline["cachedInputTokens"] >= previous["cachedInputTokens"]
|
|
1210
|
+
and baseline["cacheWriteInputTokens"] >= previous["cacheWriteInputTokens"]
|
|
1211
|
+
and (
|
|
1212
|
+
(not previous_cache_writes_reported and cache_writes_reported)
|
|
1213
|
+
or baseline["cachedInputTokens"] > previous["cachedInputTokens"]
|
|
1214
|
+
or baseline["cacheWriteInputTokens"] > previous["cacheWriteInputTokens"]
|
|
1215
|
+
)
|
|
1216
|
+
)
|
|
1217
|
+
previous_cache_writes_reported = cache_writes_reported
|
|
714
1218
|
delta = {
|
|
715
|
-
key:
|
|
716
|
-
|
|
1219
|
+
key: snapshot[key] - previous[key]
|
|
1220
|
+
if not reset and snapshot[key] >= previous[key]
|
|
1221
|
+
else snapshot[key]
|
|
1222
|
+
for key in TOKEN_FIELDS.values()
|
|
717
1223
|
}
|
|
718
|
-
|
|
1224
|
+
if (
|
|
1225
|
+
snapshot.get("cacheWriteInputTokensReported") is False
|
|
1226
|
+
or previous.get("cacheWriteInputTokensReported") is False
|
|
1227
|
+
):
|
|
1228
|
+
delta["cacheWriteInputTokensReported"] = False
|
|
1229
|
+
previous = baseline
|
|
719
1230
|
if past_scan_end(timestamp, current_turn_id):
|
|
720
1231
|
continue
|
|
721
1232
|
if isinstance(current_turn_id, str):
|
|
722
1233
|
measured_turns.add(current_turn_id)
|
|
723
1234
|
if timestamp < started_at:
|
|
1235
|
+
prefix_unclassified = _uncached_input(baseline)
|
|
1236
|
+
prefix_missing_cache = missing_cache
|
|
724
1237
|
continue
|
|
725
1238
|
if completed_turn_id is not None and current_turn_id == completed_turn_id:
|
|
726
1239
|
completed_turn_measured = True
|
|
727
1240
|
result.tokens_updated_at = max(result.tokens_updated_at or timestamp, timestamp)
|
|
728
|
-
|
|
1241
|
+
delta["totalTokens"] = delta["inputTokens"] + delta["outputTokens"]
|
|
1242
|
+
if delta["totalTokens"] <= 0 and not refines_classification:
|
|
729
1243
|
continue
|
|
1244
|
+
cache_write_capacity = (
|
|
1245
|
+
_uncached_input(legacy_usage) + delta["inputTokens"]
|
|
1246
|
+
if refines_classification
|
|
1247
|
+
else delta["inputTokens"]
|
|
1248
|
+
)
|
|
1249
|
+
# Late cumulative cache details can classify an earlier legacy turn.
|
|
1250
|
+
completed_capacity = (
|
|
1251
|
+
_uncached_input(completed_legacy_usage)
|
|
1252
|
+
if refines_classification and response_usage is None
|
|
1253
|
+
else 0
|
|
1254
|
+
)
|
|
1255
|
+
if (
|
|
1256
|
+
prefix_unclassified > 0
|
|
1257
|
+
and refines_classification
|
|
1258
|
+
and (
|
|
1259
|
+
any(delta[key] > 0 for key in prefix_missing_cache)
|
|
1260
|
+
# Even explicit counters can revise earlier input when the
|
|
1261
|
+
# newly classified tokens exceed all in-scan capacity.
|
|
1262
|
+
or delta["cachedInputTokens"] + delta["cacheWriteInputTokens"]
|
|
1263
|
+
> cache_write_capacity + completed_capacity
|
|
1264
|
+
)
|
|
1265
|
+
):
|
|
1266
|
+
# The newly classified tokens could have occurred before this scan.
|
|
1267
|
+
ambiguous_legacy_cache = True
|
|
1268
|
+
for key in ("cacheWriteInputTokens", "cachedInputTokens"):
|
|
1269
|
+
current = min(delta[key], cache_write_capacity)
|
|
1270
|
+
completed = min(delta[key] - current, completed_capacity)
|
|
1271
|
+
if completed and ambiguous_legacy_cache:
|
|
1272
|
+
warnings.add("token_breakdown_unavailable")
|
|
1273
|
+
delta[key] = current
|
|
1274
|
+
cache_write_capacity -= current
|
|
1275
|
+
completed_capacity -= completed
|
|
1276
|
+
completed_legacy_usage[key] += completed
|
|
1277
|
+
total[key] += completed
|
|
1278
|
+
delta["reasoningOutputTokens"] = min(
|
|
1279
|
+
delta["reasoningOutputTokens"], delta["outputTokens"]
|
|
1280
|
+
)
|
|
730
1281
|
_add_token_usage(legacy_usage, delta)
|
|
731
1282
|
|
|
732
1283
|
_add_token_usage(total, response_usage if response_usage is not None else legacy_usage)
|
|
1284
|
+
if response_usage is None and ambiguous_legacy_cache:
|
|
1285
|
+
warnings.add("token_breakdown_unavailable")
|
|
733
1286
|
if not boundary_reached:
|
|
734
1287
|
warnings.add("thread_ownership_unavailable")
|
|
735
1288
|
elif result.tokens_updated_at is None and not direct_control_session:
|
|
@@ -800,6 +1353,47 @@ def valid_account_rate_limit(value: object) -> bool:
|
|
|
800
1353
|
return True
|
|
801
1354
|
|
|
802
1355
|
|
|
1356
|
+
def _missing_cache_fields(payload: Mapping[str, Any]) -> set[str]:
|
|
1357
|
+
usage = payload["info"]["total_token_usage"]
|
|
1358
|
+
missing: set[str] = set()
|
|
1359
|
+
if "cached_input_tokens" not in usage:
|
|
1360
|
+
missing.add("cachedInputTokens")
|
|
1361
|
+
if "cache_write_input_tokens" not in usage and "cache_write_tokens" not in usage:
|
|
1362
|
+
missing.add("cacheWriteInputTokens")
|
|
1363
|
+
return missing
|
|
1364
|
+
|
|
1365
|
+
|
|
1366
|
+
def _uncached_input(usage: dict[str, int]) -> int:
|
|
1367
|
+
return usage["inputTokens"] - usage["cachedInputTokens"] - usage["cacheWriteInputTokens"]
|
|
1368
|
+
|
|
1369
|
+
|
|
1370
|
+
def _retain_cache_baseline(
|
|
1371
|
+
previous: dict[str, int], snapshot: dict[str, int], missing: set[str]
|
|
1372
|
+
) -> dict[str, int] | None:
|
|
1373
|
+
baseline = snapshot
|
|
1374
|
+
if (
|
|
1375
|
+
missing
|
|
1376
|
+
and snapshot["inputTokens"] >= previous["inputTokens"]
|
|
1377
|
+
and snapshot["outputTokens"] >= previous["outputTokens"]
|
|
1378
|
+
):
|
|
1379
|
+
baseline = {**snapshot, **{key: previous[key] for key in missing}}
|
|
1380
|
+
if (
|
|
1381
|
+
baseline["cachedInputTokens"] + baseline["cacheWriteInputTokens"]
|
|
1382
|
+
> baseline["inputTokens"]
|
|
1383
|
+
):
|
|
1384
|
+
return None
|
|
1385
|
+
if (
|
|
1386
|
+
baseline["inputTokens"] == previous["inputTokens"]
|
|
1387
|
+
and baseline["outputTokens"] == previous["outputTokens"]
|
|
1388
|
+
and (
|
|
1389
|
+
baseline["cachedInputTokens"] < previous["cachedInputTokens"]
|
|
1390
|
+
or baseline["cacheWriteInputTokens"] < previous["cacheWriteInputTokens"]
|
|
1391
|
+
)
|
|
1392
|
+
):
|
|
1393
|
+
return previous
|
|
1394
|
+
return baseline
|
|
1395
|
+
|
|
1396
|
+
|
|
803
1397
|
def _session_parent_thread_id(payload: Mapping[str, Any]) -> str | None:
|
|
804
1398
|
source = payload.get("source")
|
|
805
1399
|
if isinstance(source, dict):
|
|
@@ -882,6 +1476,12 @@ def _usage_counts(usage: object) -> dict[str, int] | None:
|
|
|
882
1476
|
if result["cachedInputTokens"] + result["cacheWriteInputTokens"] > result["inputTokens"]:
|
|
883
1477
|
return None
|
|
884
1478
|
result["totalTokens"] = result["inputTokens"] + result["outputTokens"]
|
|
1479
|
+
if (
|
|
1480
|
+
usage.get("cache_write_input_tokens_reported") is False
|
|
1481
|
+
or usage.get("cache_write_input_tokens") is None
|
|
1482
|
+
and usage.get("cache_write_tokens") is None
|
|
1483
|
+
):
|
|
1484
|
+
result["cacheWriteInputTokensReported"] = False
|
|
885
1485
|
return result
|
|
886
1486
|
|
|
887
1487
|
|
|
@@ -892,6 +1492,8 @@ def _empty_token_usage() -> dict[str, int]:
|
|
|
892
1492
|
def _add_token_usage(target: dict[str, int], addition: Mapping[str, int]) -> None:
|
|
893
1493
|
for key in TOKEN_FIELDS.values():
|
|
894
1494
|
target[key] += addition[key]
|
|
1495
|
+
if addition.get("cacheWriteInputTokensReported") is False:
|
|
1496
|
+
target["cacheWriteInputTokensReported"] = False
|
|
895
1497
|
|
|
896
1498
|
|
|
897
1499
|
def _timestamp(value: object) -> datetime | None:
|