@openai/codex-security 0.8.2 → 0.9.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +201 -178
- package/_bundled_plugin/.app.json +4 -0
- package/_bundled_plugin/.mcp.json +6 -1
- package/_bundled_plugin/examples/completed-scan/findings.json +16 -4
- package/_bundled_plugin/examples/completed-scan/report.md +11 -8
- package/_bundled_plugin/examples/completed-scan/scan-manifest.json +4 -2
- package/_bundled_plugin/mcp/fzstd-LICENSE.txt +21 -0
- package/_bundled_plugin/mcp/helpers.mjs +2 -11
- package/_bundled_plugin/mcp/helpers.mjs.br +0 -0
- package/_bundled_plugin/mcp/native/THIRD_PARTY_NOTICES.txt +3 -210
- package/_bundled_plugin/mcp/native/darwin-arm64/unix.node +0 -0
- package/_bundled_plugin/mcp/native/darwin-x64/unix.node +0 -0
- package/_bundled_plugin/mcp/native/linux-arm64-gnu/unix.node +0 -0
- package/_bundled_plugin/mcp/native/linux-arm64-musl/unix.node +0 -0
- package/_bundled_plugin/mcp/native/linux-x64-gnu/unix.node +0 -0
- package/_bundled_plugin/mcp/native/linux-x64-musl/unix.node +0 -0
- package/_bundled_plugin/mcp/native/win32-arm64/windows.node +0 -0
- package/_bundled_plugin/mcp/native/win32-x64/windows.node +0 -0
- package/_bundled_plugin/mcp/package.json +1 -0
- package/_bundled_plugin/mcp/permission-profile-preflight.mjs +51 -14
- package/_bundled_plugin/mcp/server.mjs +2 -11
- package/_bundled_plugin/mcp/server.mjs.br +0 -0
- package/_bundled_plugin/mcp/session-decompression.mjs +712 -0
- package/_bundled_plugin/references/artifact-storage.md +1 -1
- package/_bundled_plugin/references/core-scan.md +14 -5
- package/_bundled_plugin/references/default-severity-rubric.md +33 -0
- package/_bundled_plugin/references/final-report.md +5 -3
- package/_bundled_plugin/references/finding-detail-fields.md +19 -18
- package/_bundled_plugin/references/sarif-adapter.md +8 -2
- package/_bundled_plugin/references/scan-artifacts.md +21 -12
- package/_bundled_plugin/references/scan-contract.md +27 -33
- package/_bundled_plugin/references/threat-model.md +1 -0
- package/_bundled_plugin/schemas/tools/scan-draft.schema.json +2 -2
- package/_bundled_plugin/scripts/check_deep_scan_target.py +97 -0
- package/_bundled_plugin/scripts/codex_profile.mjs +7 -1
- package/_bundled_plugin/scripts/deep_scan_config.py +20 -1
- package/_bundled_plugin/scripts/deep_scan_workbench.py +188 -454
- package/_bundled_plugin/scripts/filesystem_identity.py +1 -0
- package/_bundled_plugin/scripts/finalize_scan_contract.py +317 -377
- package/_bundled_plugin/scripts/finding_preview.py +25 -29
- package/_bundled_plugin/scripts/launch_codex_security_mcp +1 -0
- package/_bundled_plugin/scripts/legacy_findings.py +148 -0
- package/_bundled_plugin/scripts/posix_scan_local_files.py +198 -0
- package/_bundled_plugin/scripts/read_codex_session.mjs +21 -0
- package/_bundled_plugin/scripts/report_projection.py +167 -178
- package/_bundled_plugin/scripts/review_state.py +8 -2
- package/_bundled_plugin/scripts/servicenow_avit.py +20 -0
- package/_bundled_plugin/scripts/snapshot_sqlite.py +1 -0
- package/_bundled_plugin/scripts/threat_model_projection.py +22 -26
- package/_bundled_plugin/scripts/validate_scan_contract.py +9 -7
- package/_bundled_plugin/scripts/windows_scan_local_files.py +600 -72
- package/_bundled_plugin/scripts/workbench/__init__.py +1 -1
- package/_bundled_plugin/scripts/workbench/handoff.py +47 -0
- package/_bundled_plugin/scripts/workbench/retained_coverage.py +38 -75
- package/_bundled_plugin/scripts/workbench/retained_records.py +18 -30
- package/_bundled_plugin/scripts/workbench/runtime_toml.py +4 -1
- package/_bundled_plugin/scripts/workbench/saved_coverage.py +152 -248
- package/_bundled_plugin/scripts/workbench/saved_outputs.py +9 -0
- package/_bundled_plugin/scripts/workbench/saved_sources.py +47 -40
- package/_bundled_plugin/scripts/workbench/storage.py +71 -7
- package/_bundled_plugin/scripts/workbench_cli.py +121 -14
- package/_bundled_plugin/scripts/workbench_constants.py +19 -0
- package/_bundled_plugin/scripts/workbench_coverage_receipts.py +3 -3
- package/_bundled_plugin/scripts/workbench_db.py +550 -492
- package/_bundled_plugin/scripts/workbench_feedback.py +8 -7
- package/_bundled_plugin/scripts/workbench_finding_index.py +1 -0
- package/_bundled_plugin/scripts/workbench_finding_workflows.py +90 -4
- package/_bundled_plugin/scripts/workbench_native_indexes.py +29 -33
- package/_bundled_plugin/scripts/workbench_progress.py +10 -1
- package/_bundled_plugin/scripts/workbench_publication.py +14 -10
- package/_bundled_plugin/scripts/workbench_remediation.py +508 -11
- package/_bundled_plugin/scripts/workbench_saved_results.py +132 -77
- package/_bundled_plugin/scripts/workbench_scan_history.py +139 -80
- package/_bundled_plugin/scripts/workbench_scan_start.py +84 -25
- package/_bundled_plugin/scripts/workbench_scan_usage.py +666 -117
- package/_bundled_plugin/scripts/workbench_schema.py +44 -55
- package/_bundled_plugin/scripts/workbench_setup.py +251 -0
- package/_bundled_plugin/scripts/workbench_source_excerpt.py +13 -0
- package/_bundled_plugin/scripts/workbench_source_scopes.py +33 -23
- package/_bundled_plugin/scripts/workbench_target.py +330 -66
- package/_bundled_plugin/scripts/workbench_target_state.py +3 -0
- package/_bundled_plugin/scripts/workbench_validation.py +44 -12
- package/_bundled_plugin/shared/workbench-history-repairs.json +19 -16
- package/_bundled_plugin/shared/workbench-migrations.json +53 -0
- package/_bundled_plugin/skills/attack-path-analysis/SKILL.md +1 -1
- package/_bundled_plugin/skills/attack-path-analysis/references/severity-policy.md +4 -68
- package/_bundled_plugin/skills/check-skills/SKILL.md +43 -0
- package/_bundled_plugin/skills/check-skills/agents/openai.yaml +4 -0
- package/_bundled_plugin/skills/cloud-findings/SKILL.md +6 -4
- package/_bundled_plugin/skills/deep-security-scan/SKILL.md +7 -50
- package/_bundled_plugin/skills/fix-finding/SKILL.md +19 -2
- package/_bundled_plugin/skills/publish-findings/SKILL.md +81 -45
- package/_bundled_plugin/skills/publish-findings/references/checkmarx-import.md +37 -0
- package/_bundled_plugin/skills/publish-findings/references/wiz-import.md +5 -24
- package/_bundled_plugin/skills/publish-findings/scripts/collect_checkmarx.py +464 -0
- package/_bundled_plugin/skills/threat-model-visualize/scripts/build_explorer.py +36 -9
- package/_bundled_plugin/skills/track-findings/SKILL.md +5 -3
- package/_bundled_plugin/skills/triage-finding/references/github-rest-intake.md +22 -68
- package/_bundled_plugin/skills/triage-finding/references/ticket-intake.md +12 -6
- package/dist/api.d.ts +45 -5
- package/dist/api.d.ts.map +1 -1
- package/dist/api.js +1144 -411
- package/dist/api.js.map +1 -1
- package/dist/auth.d.ts.map +1 -1
- package/dist/auth.js +10 -7
- package/dist/auth.js.map +1 -1
- package/dist/bedrock-oss-pricing.d.ts +6 -0
- package/dist/bedrock-oss-pricing.d.ts.map +1 -0
- package/dist/bedrock-oss-pricing.js +58 -0
- package/dist/bedrock-oss-pricing.js.map +1 -0
- package/dist/bedrock-pricing.d.ts +15 -0
- package/dist/bedrock-pricing.d.ts.map +1 -0
- package/dist/bedrock-pricing.js +117 -0
- package/dist/bedrock-pricing.js.map +1 -0
- package/dist/blackduck-findings.d.ts +8 -0
- package/dist/blackduck-findings.d.ts.map +1 -0
- package/dist/blackduck-findings.js +320 -0
- package/dist/blackduck-findings.js.map +1 -0
- package/dist/bulk-scan-discovery.d.ts.map +1 -1
- package/dist/bulk-scan-discovery.js +51 -36
- package/dist/bulk-scan-discovery.js.map +1 -1
- package/dist/checkmarx-evidence.d.ts +19 -0
- package/dist/checkmarx-evidence.d.ts.map +1 -0
- package/dist/checkmarx-evidence.js +36 -0
- package/dist/checkmarx-evidence.js.map +1 -0
- package/dist/checkmarx-findings.d.ts +4 -0
- package/dist/checkmarx-findings.d.ts.map +1 -0
- package/dist/checkmarx-findings.js +27 -0
- package/dist/checkmarx-findings.js.map +1 -0
- package/dist/checkmarx-one.d.ts +4 -0
- package/dist/checkmarx-one.d.ts.map +1 -0
- package/dist/checkmarx-one.js +53 -0
- package/dist/checkmarx-one.js.map +1 -0
- package/dist/checkmarx-repository.d.ts +12 -0
- package/dist/checkmarx-repository.d.ts.map +1 -0
- package/dist/checkmarx-repository.js +181 -0
- package/dist/checkmarx-repository.js.map +1 -0
- package/dist/checkmarx-xml.d.ts +12 -0
- package/dist/checkmarx-xml.d.ts.map +1 -0
- package/dist/checkmarx-xml.js +196 -0
- package/dist/checkmarx-xml.js.map +1 -0
- package/dist/classify-severity-records.d.ts +6 -9
- package/dist/classify-severity-records.d.ts.map +1 -1
- package/dist/classify-severity-records.js +9 -21
- package/dist/classify-severity-records.js.map +1 -1
- package/dist/classify-severity.d.ts +4 -1
- package/dist/classify-severity.d.ts.map +1 -1
- package/dist/classify-severity.js +30 -13
- package/dist/classify-severity.js.map +1 -1
- package/dist/cli.d.ts +14 -5
- package/dist/cli.d.ts.map +1 -1
- package/dist/cli.js +2112 -1266
- package/dist/cli.js.map +1 -1
- package/dist/cloud-publish.js +2 -2
- package/dist/cloud-publish.js.map +1 -1
- package/dist/codex-prompt.d.ts +2 -0
- package/dist/codex-prompt.d.ts.map +1 -1
- package/dist/codex-prompt.js +6 -0
- package/dist/codex-prompt.js.map +1 -1
- package/dist/component-scan.d.ts +16 -3
- package/dist/component-scan.d.ts.map +1 -1
- package/dist/component-scan.js +32 -6
- package/dist/component-scan.js.map +1 -1
- package/dist/config.d.ts +2 -1
- package/dist/config.d.ts.map +1 -1
- package/dist/config.js +221 -65
- package/dist/config.js.map +1 -1
- package/dist/continuum-findings.d.ts +7 -0
- package/dist/continuum-findings.d.ts.map +1 -0
- package/dist/continuum-findings.js +225 -0
- package/dist/continuum-findings.js.map +1 -0
- package/dist/contract-values.d.ts +30 -0
- package/dist/contract-values.d.ts.map +1 -0
- package/dist/contract-values.js +78 -0
- package/dist/contract-values.js.map +1 -0
- package/dist/contract.d.ts +2 -20
- package/dist/contract.d.ts.map +1 -1
- package/dist/contract.js +58 -669
- package/dist/contract.js.map +1 -1
- package/dist/cost-model.d.ts +13 -1
- package/dist/cost-model.d.ts.map +1 -1
- package/dist/cost-model.js +66 -115
- package/dist/cost-model.js.map +1 -1
- package/dist/cost.d.ts +61 -4
- package/dist/cost.d.ts.map +1 -1
- package/dist/cost.js +2560 -100
- package/dist/cost.js.map +1 -1
- package/dist/custom-publish.d.ts +4 -0
- package/dist/custom-publish.d.ts.map +1 -1
- package/dist/custom-publish.js +40 -3
- package/dist/custom-publish.js.map +1 -1
- package/dist/custom-validation-prompt.d.ts +1 -1
- package/dist/custom-validation-prompt.d.ts.map +1 -1
- package/dist/custom-validation-prompt.js +5 -0
- package/dist/custom-validation-prompt.js.map +1 -1
- package/dist/custom-validation.d.ts +2 -0
- package/dist/custom-validation.d.ts.map +1 -1
- package/dist/custom-validation.js +4 -0
- package/dist/custom-validation.js.map +1 -1
- package/dist/deduplication/deduplication-prompts.d.ts +4 -4
- package/dist/deduplication/deduplication-prompts.d.ts.map +1 -1
- package/dist/deduplication/deduplication-prompts.js +2 -2
- package/dist/deduplication/deduplication-prompts.js.map +1 -1
- package/dist/deduplication/deduplication-reviewer.d.ts +14 -11
- package/dist/deduplication/deduplication-reviewer.d.ts.map +1 -1
- package/dist/deduplication/deduplication-reviewer.js +11 -4
- package/dist/deduplication/deduplication-reviewer.js.map +1 -1
- package/dist/deduplication/deduplication.js.map +1 -1
- package/dist/deduplication/finding-schema.d.ts +19 -1
- package/dist/deduplication/finding-schema.d.ts.map +1 -1
- package/dist/deduplication/finding-schema.js +25 -0
- package/dist/deduplication/finding-schema.js.map +1 -1
- package/dist/deduplication/local.d.ts +2 -2
- package/dist/deduplication/local.d.ts.map +1 -1
- package/dist/deduplication/local.js +5 -10
- package/dist/deduplication/local.js.map +1 -1
- package/dist/deduplication/records.d.ts +8 -2
- package/dist/deduplication/records.d.ts.map +1 -1
- package/dist/deduplication/records.js +36 -8
- package/dist/deduplication/records.js.map +1 -1
- package/dist/deduplication/refusal.d.ts.map +1 -1
- package/dist/deduplication/refusal.js +25 -5
- package/dist/deduplication/refusal.js.map +1 -1
- package/dist/deduplication/scan.d.ts +3 -0
- package/dist/deduplication/scan.d.ts.map +1 -1
- package/dist/deduplication/scan.js +1 -1
- package/dist/deduplication/scan.js.map +1 -1
- package/dist/deep-config.d.ts +0 -1
- package/dist/deep-config.d.ts.map +1 -1
- package/dist/deep-config.js +22 -33
- package/dist/deep-config.js.map +1 -1
- package/dist/deep-scan-defaults.d.ts +2 -0
- package/dist/deep-scan-defaults.d.ts.map +1 -1
- package/dist/deep-scan-defaults.js +28 -1
- package/dist/deep-scan-defaults.js.map +1 -1
- package/dist/deep-scan.d.ts +13 -0
- package/dist/deep-scan.d.ts.map +1 -1
- package/dist/deep-scan.js +262 -7
- package/dist/deep-scan.js.map +1 -1
- package/dist/default-severity-rubric.d.ts +1 -2
- package/dist/default-severity-rubric.d.ts.map +1 -1
- package/dist/default-severity-rubric.js +2 -66
- package/dist/default-severity-rubric.js.map +1 -1
- package/dist/endor-findings.d.ts +3 -1
- package/dist/endor-findings.d.ts.map +1 -1
- package/dist/endor-findings.js +108 -80
- package/dist/endor-findings.js.map +1 -1
- package/dist/errors.d.ts +1 -1
- package/dist/errors.d.ts.map +1 -1
- package/dist/errors.js +9 -2
- package/dist/errors.js.map +1 -1
- package/dist/external-findings-import.d.ts +5 -3
- package/dist/external-findings-import.d.ts.map +1 -1
- package/dist/external-findings-import.js +11 -16
- package/dist/external-findings-import.js.map +1 -1
- package/dist/external-findings-local.d.ts +15 -4
- package/dist/external-findings-local.d.ts.map +1 -1
- package/dist/external-findings-local.js +619 -225
- package/dist/external-findings-local.js.map +1 -1
- package/dist/external-findings-publish.d.ts +20 -6
- package/dist/external-findings-publish.d.ts.map +1 -1
- package/dist/external-findings-publish.js +2356 -308
- package/dist/external-findings-publish.js.map +1 -1
- package/dist/external-findings-read.d.ts +28 -2
- package/dist/external-findings-read.d.ts.map +1 -1
- package/dist/external-findings-read.js +174 -81
- package/dist/external-findings-read.js.map +1 -1
- package/dist/external-findings.d.ts +17 -3
- package/dist/external-findings.d.ts.map +1 -1
- package/dist/external-findings.js +153 -1
- package/dist/external-findings.js.map +1 -1
- package/dist/external-import-contract.d.ts +33 -1
- package/dist/external-import-contract.d.ts.map +1 -1
- package/dist/external-import-contract.js +335 -51
- package/dist/external-import-contract.js.map +1 -1
- package/dist/external-import-models.d.ts +1 -1
- package/dist/external-import-models.d.ts.map +1 -1
- package/dist/external-import-models.js.map +1 -1
- package/dist/feedback.d.ts.map +1 -1
- package/dist/feedback.js +5 -4
- package/dist/feedback.js.map +1 -1
- package/dist/finding-prompt.d.ts +5 -0
- package/dist/finding-prompt.d.ts.map +1 -0
- package/dist/finding-prompt.js +32 -0
- package/dist/finding-prompt.js.map +1 -0
- package/dist/finding-retrieval.d.ts +3 -3
- package/dist/finding-retrieval.d.ts.map +1 -1
- package/dist/finding-retrieval.js.map +1 -1
- package/dist/finding-workflow.d.ts +6 -1
- package/dist/finding-workflow.d.ts.map +1 -1
- package/dist/finding-workflow.js +2 -2
- package/dist/finding-workflow.js.map +1 -1
- package/dist/findings-client.d.ts +1 -0
- package/dist/findings-client.d.ts.map +1 -1
- package/dist/findings-client.js +27 -3
- package/dist/findings-client.js.map +1 -1
- package/dist/git-remote.d.ts +4 -0
- package/dist/git-remote.d.ts.map +1 -1
- package/dist/git-remote.js +20 -0
- package/dist/git-remote.js.map +1 -1
- package/dist/github-advisories.d.ts +3 -12
- package/dist/github-advisories.d.ts.map +1 -1
- package/dist/github-advisories.js +9 -31
- package/dist/github-advisories.js.map +1 -1
- package/dist/github-dependabot-findings.d.ts.map +1 -1
- package/dist/github-dependabot-findings.js +22 -20
- package/dist/github-dependabot-findings.js.map +1 -1
- package/dist/github-findings.d.ts +4 -1
- package/dist/github-findings.d.ts.map +1 -1
- package/dist/github-findings.js +392 -79
- package/dist/github-findings.js.map +1 -1
- package/dist/github.d.ts +17 -7
- package/dist/github.d.ts.map +1 -1
- package/dist/github.js +43 -30
- package/dist/github.js.map +1 -1
- package/dist/host-protocol.d.ts +2 -2
- package/dist/host-protocol.d.ts.map +1 -1
- package/dist/host-protocol.js.map +1 -1
- package/dist/host-records-protocol.d.ts +10 -11
- package/dist/host-records-protocol.d.ts.map +1 -1
- package/dist/host-records-protocol.js +69 -9
- package/dist/host-records-protocol.js.map +1 -1
- package/dist/hosted-git-remotes.d.ts +5 -0
- package/dist/hosted-git-remotes.d.ts.map +1 -0
- package/dist/hosted-git-remotes.js +285 -0
- package/dist/hosted-git-remotes.js.map +1 -0
- package/dist/hosted-git-repository.d.ts +28 -0
- package/dist/hosted-git-repository.d.ts.map +1 -0
- package/dist/hosted-git-repository.js +222 -0
- package/dist/hosted-git-repository.js.map +1 -0
- package/dist/hosted-scan-protocol.js +2 -2
- package/dist/hosted-scan-protocol.js.map +1 -1
- package/dist/hosted-scan.d.ts +15 -0
- package/dist/hosted-scan.d.ts.map +1 -1
- package/dist/hosted-scan.js +71 -44
- package/dist/hosted-scan.js.map +1 -1
- package/dist/hosted-scanner-http.d.ts +21 -0
- package/dist/hosted-scanner-http.d.ts.map +1 -0
- package/dist/hosted-scanner-http.js +204 -0
- package/dist/hosted-scanner-http.js.map +1 -0
- package/dist/hosted-scanner-v1.d.ts +6 -0
- package/dist/hosted-scanner-v1.d.ts.map +1 -0
- package/dist/hosted-scanner-v1.js +60 -0
- package/dist/hosted-scanner-v1.js.map +1 -0
- package/dist/import-scan.d.ts.map +1 -1
- package/dist/import-scan.js +3 -0
- package/dist/import-scan.js.map +1 -1
- package/dist/index.d.ts +24 -6
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +15 -1
- package/dist/index.js.map +1 -1
- package/dist/inspector-findings.d.ts.map +1 -1
- package/dist/inspector-findings.js +58 -81
- package/dist/inspector-findings.js.map +1 -1
- package/dist/knowledge-base.d.ts.map +1 -1
- package/dist/knowledge-base.js +86 -10
- package/dist/knowledge-base.js.map +1 -1
- package/dist/local-git-metadata.d.ts +5 -0
- package/dist/local-git-metadata.d.ts.map +1 -0
- package/dist/local-git-metadata.js +130 -0
- package/dist/local-git-metadata.js.map +1 -0
- package/dist/multiscan.d.ts +6 -4
- package/dist/multiscan.d.ts.map +1 -1
- package/dist/multiscan.js +160 -77
- package/dist/multiscan.js.map +1 -1
- package/dist/native-json-findings.d.ts +2 -1
- package/dist/native-json-findings.d.ts.map +1 -1
- package/dist/native-json-findings.js +12 -2
- package/dist/native-json-findings.js.map +1 -1
- package/dist/opengrep-findings.d.ts +1 -0
- package/dist/opengrep-findings.d.ts.map +1 -1
- package/dist/opengrep-findings.js +49 -31
- package/dist/opengrep-findings.js.map +1 -1
- package/dist/operation-telemetry.d.ts +104 -0
- package/dist/operation-telemetry.d.ts.map +1 -0
- package/dist/operation-telemetry.js +293 -0
- package/dist/operation-telemetry.js.map +1 -0
- package/dist/patch-tui.d.ts.map +1 -1
- package/dist/patch-tui.js +90 -35
- package/dist/patch-tui.js.map +1 -1
- package/dist/plugin-python-probe.d.ts +4 -0
- package/dist/plugin-python-probe.d.ts.map +1 -0
- package/dist/plugin-python-probe.js +10 -0
- package/dist/plugin-python-probe.js.map +1 -0
- package/dist/private-finding-import.d.ts +341 -0
- package/dist/private-finding-import.d.ts.map +1 -0
- package/dist/private-finding-import.js +411 -0
- package/dist/private-finding-import.js.map +1 -0
- package/dist/project-config.d.ts.map +1 -1
- package/dist/project-config.js +2 -2
- package/dist/project-config.js.map +1 -1
- package/dist/provider-profile.d.ts +11 -3
- package/dist/provider-profile.d.ts.map +1 -1
- package/dist/provider-profile.js +80 -11
- package/dist/provider-profile.js.map +1 -1
- package/dist/publication-telemetry.d.ts +45 -0
- package/dist/publication-telemetry.d.ts.map +1 -0
- package/dist/publication-telemetry.js +50 -0
- package/dist/publication-telemetry.js.map +1 -0
- package/dist/publication.js +7 -1
- package/dist/publication.js.map +1 -1
- package/dist/publish.d.ts +4 -0
- package/dist/publish.d.ts.map +1 -1
- package/dist/publish.js +42 -2
- package/dist/publish.js.map +1 -1
- package/dist/reader-selection.d.ts +15 -0
- package/dist/reader-selection.d.ts.map +1 -0
- package/dist/reader-selection.js +89 -0
- package/dist/reader-selection.js.map +1 -0
- package/dist/record.d.ts +6 -1
- package/dist/record.d.ts.map +1 -1
- package/dist/record.js +53 -2
- package/dist/record.js.map +1 -1
- package/dist/remediation.d.ts +3 -1
- package/dist/remediation.d.ts.map +1 -1
- package/dist/remediation.js +28 -2
- package/dist/remediation.js.map +1 -1
- package/dist/request-metadata.d.ts +3 -0
- package/dist/request-metadata.d.ts.map +1 -1
- package/dist/request-metadata.js +9 -0
- package/dist/request-metadata.js.map +1 -1
- package/dist/result.d.ts +6 -0
- package/dist/result.d.ts.map +1 -1
- package/dist/result.js +3 -1
- package/dist/result.js.map +1 -1
- package/dist/retry-windows-rename.d.ts +8 -0
- package/dist/retry-windows-rename.d.ts.map +1 -0
- package/dist/retry-windows-rename.js +24 -0
- package/dist/retry-windows-rename.js.map +1 -0
- package/dist/runtime.d.ts +22 -1
- package/dist/runtime.d.ts.map +1 -1
- package/dist/runtime.js +408 -162
- package/dist/runtime.js.map +1 -1
- package/dist/saved-scan-bootstrap.d.ts +1 -0
- package/dist/saved-scan-bootstrap.d.ts.map +1 -1
- package/dist/saved-scan-bootstrap.js +18 -15
- package/dist/saved-scan-bootstrap.js.map +1 -1
- package/dist/scan-activity.d.ts.map +1 -1
- package/dist/scan-activity.js +172 -64
- package/dist/scan-activity.js.map +1 -1
- package/dist/scan-comparison.d.ts +10 -5
- package/dist/scan-comparison.d.ts.map +1 -1
- package/dist/scan-comparison.js +68 -25
- package/dist/scan-comparison.js.map +1 -1
- package/dist/scan-dashboard.d.ts +7 -6
- package/dist/scan-dashboard.d.ts.map +1 -1
- package/dist/scan-dashboard.js +91 -79
- package/dist/scan-dashboard.js.map +1 -1
- package/dist/scan-executor.d.ts +12 -1
- package/dist/scan-executor.d.ts.map +1 -1
- package/dist/scan-executor.js +22 -2
- package/dist/scan-executor.js.map +1 -1
- package/dist/scan-history-renderer.d.ts.map +1 -1
- package/dist/scan-history-renderer.js +31 -28
- package/dist/scan-history-renderer.js.map +1 -1
- package/dist/scan-logs.d.ts +12 -0
- package/dist/scan-logs.d.ts.map +1 -1
- package/dist/scan-logs.js +106 -77
- package/dist/scan-logs.js.map +1 -1
- package/dist/scan-measurement.d.ts +26 -0
- package/dist/scan-measurement.d.ts.map +1 -0
- package/dist/scan-measurement.js +77 -0
- package/dist/scan-measurement.js.map +1 -0
- package/dist/scan-progress.d.ts +1 -0
- package/dist/scan-progress.d.ts.map +1 -1
- package/dist/scan-progress.js +3 -0
- package/dist/scan-progress.js.map +1 -1
- package/dist/scan-sessions.d.ts.map +1 -1
- package/dist/scan-sessions.js +4 -1
- package/dist/scan-sessions.js.map +1 -1
- package/dist/scan-settings.d.ts +1 -1
- package/dist/scan-settings.d.ts.map +1 -1
- package/dist/scan-settings.js +1 -24
- package/dist/scan-settings.js.map +1 -1
- package/dist/scanner-import.d.ts +64 -6
- package/dist/scanner-import.d.ts.map +1 -1
- package/dist/scanner-import.js +406 -48
- package/dist/scanner-import.js.map +1 -1
- package/dist/scanner-repository-url.d.ts +17 -0
- package/dist/scanner-repository-url.d.ts.map +1 -0
- package/dist/scanner-repository-url.js +71 -0
- package/dist/scanner-repository-url.js.map +1 -0
- package/dist/security-policy.d.ts +1 -1
- package/dist/security-policy.d.ts.map +1 -1
- package/dist/security-policy.js.map +1 -1
- package/dist/semgrep-findings.d.ts +6 -3
- package/dist/semgrep-findings.d.ts.map +1 -1
- package/dist/semgrep-findings.js +124 -88
- package/dist/semgrep-findings.js.map +1 -1
- package/dist/semgrep-guardian-findings.d.ts +38 -0
- package/dist/semgrep-guardian-findings.d.ts.map +1 -0
- package/dist/semgrep-guardian-findings.js +165 -0
- package/dist/semgrep-guardian-findings.js.map +1 -0
- package/dist/semgrep-platform-collector.d.ts +26 -0
- package/dist/semgrep-platform-collector.d.ts.map +1 -0
- package/dist/semgrep-platform-collector.js +108 -0
- package/dist/semgrep-platform-collector.js.map +1 -0
- package/dist/semgrep-platform-findings.d.ts +213 -0
- package/dist/semgrep-platform-findings.d.ts.map +1 -0
- package/dist/semgrep-platform-findings.js +1078 -0
- package/dist/semgrep-platform-findings.js.map +1 -0
- package/dist/semgrep-platform-v1.d.ts +6 -0
- package/dist/semgrep-platform-v1.d.ts.map +1 -0
- package/dist/semgrep-platform-v1.js +83 -0
- package/dist/semgrep-platform-v1.js.map +1 -0
- package/dist/server/api.d.ts +1 -1
- package/dist/server/api.d.ts.map +1 -1
- package/dist/server/api.js.map +1 -1
- package/dist/server/embedding-tokenizer.cdx.json +33 -0
- package/dist/server/embedding-tokenizer.d.ts +2 -0
- package/dist/server/embedding-tokenizer.d.ts.map +1 -0
- package/dist/server/embedding-tokenizer.js +322 -0
- package/dist/server/embedding-tokenizer.js.map +6 -0
- package/dist/server/embeddings.d.ts +16 -2
- package/dist/server/embeddings.d.ts.map +1 -1
- package/dist/server/embeddings.js +35 -5
- package/dist/server/embeddings.js.map +1 -1
- package/dist/session-lines.d.ts +5 -0
- package/dist/session-lines.d.ts.map +1 -0
- package/dist/session-lines.js +60 -0
- package/dist/session-lines.js.map +1 -0
- package/dist/severity-review.d.ts +3 -1
- package/dist/severity-review.d.ts.map +1 -1
- package/dist/severity-review.js +3 -2
- package/dist/severity-review.js.map +1 -1
- package/dist/severity-rubric.d.ts +20 -0
- package/dist/severity-rubric.d.ts.map +1 -0
- package/dist/severity-rubric.js +73 -0
- package/dist/severity-rubric.js.map +1 -0
- package/dist/severity-store.js +15 -5
- package/dist/severity-store.js.map +1 -1
- package/dist/skill-authentication.d.ts +1 -0
- package/dist/skill-authentication.d.ts.map +1 -1
- package/dist/skill-authentication.js +1 -0
- package/dist/skill-authentication.js.map +1 -1
- package/dist/snyk-findings.d.ts +7 -2
- package/dist/snyk-findings.d.ts.map +1 -1
- package/dist/snyk-findings.js +108 -84
- package/dist/snyk-findings.js.map +1 -1
- package/dist/snyk-rest-findings.d.ts +76 -0
- package/dist/snyk-rest-findings.d.ts.map +1 -0
- package/dist/snyk-rest-findings.js +1051 -0
- package/dist/snyk-rest-findings.js.map +1 -0
- package/dist/snyk-rest-v1.d.ts +28 -0
- package/dist/snyk-rest-v1.d.ts.map +1 -0
- package/dist/snyk-rest-v1.js +132 -0
- package/dist/snyk-rest-v1.js.map +1 -0
- package/dist/snyk-token-origin.d.ts +4 -0
- package/dist/snyk-token-origin.d.ts.map +1 -0
- package/dist/snyk-token-origin.js +82 -0
- package/dist/snyk-token-origin.js.map +1 -0
- package/dist/sonar-collector.d.ts +16 -0
- package/dist/sonar-collector.d.ts.map +1 -0
- package/dist/sonar-collector.js +380 -0
- package/dist/sonar-collector.js.map +1 -0
- package/dist/sonar-findings.d.ts +49 -0
- package/dist/sonar-findings.d.ts.map +1 -0
- package/dist/sonar-findings.js +288 -0
- package/dist/sonar-findings.js.map +1 -0
- package/dist/sonar-import.d.ts +111 -0
- package/dist/sonar-import.d.ts.map +1 -0
- package/dist/sonar-import.js +277 -0
- package/dist/sonar-import.js.map +1 -0
- package/dist/suggest-owners.d.ts.map +1 -1
- package/dist/suggest-owners.js +2 -1
- package/dist/suggest-owners.js.map +1 -1
- package/dist/target-drift.d.ts +9 -0
- package/dist/target-drift.d.ts.map +1 -0
- package/dist/target-drift.js +27 -0
- package/dist/target-drift.js.map +1 -0
- package/dist/targets.d.ts +5 -0
- package/dist/targets.d.ts.map +1 -1
- package/dist/targets.js +73 -31
- package/dist/targets.js.map +1 -1
- package/dist/tenable-findings.d.ts +9 -0
- package/dist/tenable-findings.d.ts.map +1 -0
- package/dist/tenable-findings.js +122 -0
- package/dist/tenable-findings.js.map +1 -0
- package/dist/terminal-columns.d.ts +4 -0
- package/dist/terminal-columns.d.ts.map +1 -0
- package/dist/terminal-columns.js +19 -0
- package/dist/terminal-columns.js.map +1 -0
- package/dist/terminal-stream.d.ts +13 -0
- package/dist/terminal-stream.d.ts.map +1 -0
- package/dist/terminal-stream.js +35 -0
- package/dist/terminal-stream.js.map +1 -0
- package/dist/trivy-findings.d.ts +19 -0
- package/dist/trivy-findings.d.ts.map +1 -0
- package/dist/trivy-findings.js +435 -0
- package/dist/trivy-findings.js.map +1 -0
- package/dist/trivy-import.d.ts +19 -0
- package/dist/trivy-import.d.ts.map +1 -0
- package/dist/trivy-import.js +151 -0
- package/dist/trivy-import.js.map +1 -0
- package/dist/validation-records.d.ts +0 -3
- package/dist/validation-records.d.ts.map +1 -1
- package/dist/validation-records.js +1 -17
- package/dist/validation-records.js.map +1 -1
- package/dist/veracode-findings.d.ts +9 -0
- package/dist/veracode-findings.d.ts.map +1 -0
- package/dist/veracode-findings.js +409 -0
- package/dist/veracode-findings.js.map +1 -0
- package/dist/vertex-pricing.d.ts +8 -0
- package/dist/vertex-pricing.d.ts.map +1 -0
- package/dist/vertex-pricing.js +20 -0
- package/dist/vertex-pricing.js.map +1 -0
- package/dist/windows-path.d.ts +1 -0
- package/dist/windows-path.d.ts.map +1 -1
- package/dist/windows-path.js +27 -1
- package/dist/windows-path.js.map +1 -1
- package/dist/wiz-csv.d.ts +2 -2
- package/dist/wiz-csv.d.ts.map +1 -1
- package/dist/wiz-csv.js +1 -3
- package/dist/wiz-csv.js.map +1 -1
- package/dist/wiz-findings.d.ts +14 -2
- package/dist/wiz-findings.d.ts.map +1 -1
- package/dist/wiz-findings.js +143 -50
- package/dist/wiz-findings.js.map +1 -1
- package/docs/classify-severity-records.md +28 -13
- package/docs/cli.md +623 -843
- package/docs/dedupe-records.md +54 -24
- package/docs/findings-service.md +87 -47
- package/docs/hosted-remediation.md +14 -7
- package/docs/hosted-scans.md +64 -41
- package/docs/telemetry.md +236 -0
- package/docs/validate-records.md +78 -44
- package/package.json +19 -17
- package/schemas/external-findings.schema.json +14 -1
- package/schemas/hosted-scan-v2.fixture.json +16 -1
- package/_bundled_plugin/mcp/helpers.mjs.br.part-000 +0 -0
- package/_bundled_plugin/mcp/helpers.mjs.br.part-001 +0 -0
- package/_bundled_plugin/mcp/server.mjs.br.part-000 +0 -0
- package/_bundled_plugin/mcp/server.mjs.br.part-001 +0 -0
- package/_bundled_plugin/mcp/server.mjs.br.part-002 +0 -0
- package/_bundled_plugin/schemas/tools/deep-reducer.schema.json +0 -96
- package/_bundled_plugin/scripts/validate_tracking_source.py +0 -82
- package/_bundled_plugin/scripts/workbench_local_dedupe.py +0 -136
- package/dist/deduplication/records-protocol.d.ts +0 -4
- package/dist/deduplication/records-protocol.d.ts.map +0 -1
- package/dist/deduplication/records-protocol.js +0 -49
- package/dist/deduplication/records-protocol.js.map +0 -1
- package/dist/external-imports-feature.d.ts +0 -6
- package/dist/external-imports-feature.d.ts.map +0 -1
- package/dist/external-imports-feature.js +0 -51
- package/dist/external-imports-feature.js.map +0 -1
- package/dist/finding-normalization.d.ts +0 -5
- package/dist/finding-normalization.d.ts.map +0 -1
- package/dist/finding-normalization.js +0 -120
- package/dist/finding-normalization.js.map +0 -1
|
@@ -8,14 +8,20 @@ import math
|
|
|
8
8
|
import os
|
|
9
9
|
import re
|
|
10
10
|
import sqlite3
|
|
11
|
+
import subprocess
|
|
11
12
|
import sys
|
|
13
|
+
import tempfile
|
|
12
14
|
import uuid
|
|
15
|
+
from contextlib import contextmanager
|
|
13
16
|
from dataclasses import dataclass
|
|
14
17
|
from datetime import datetime, timezone
|
|
15
18
|
from pathlib import Path
|
|
16
19
|
from typing import Any, Mapping
|
|
17
20
|
from urllib.parse import quote
|
|
18
21
|
|
|
22
|
+
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
|
23
|
+
from workbench.storage import worker_usage_directory
|
|
24
|
+
|
|
19
25
|
TOKEN_FIELDS = {
|
|
20
26
|
"input_tokens": "inputTokens",
|
|
21
27
|
"cached_input_tokens": "cachedInputTokens",
|
|
@@ -30,6 +36,8 @@ STATE_DATABASE_TIMEOUT_SECONDS = 1.0
|
|
|
30
36
|
|
|
31
37
|
@dataclass(frozen=True)
|
|
32
38
|
class RolloutSession:
|
|
39
|
+
"""Associate a thread and its optional parent with the path of the rollout used for usage accounting."""
|
|
40
|
+
|
|
33
41
|
thread_id: str
|
|
34
42
|
parent_thread_id: str | None
|
|
35
43
|
path: Path
|
|
@@ -37,6 +45,8 @@ class RolloutSession:
|
|
|
37
45
|
|
|
38
46
|
@dataclass
|
|
39
47
|
class RolloutUsage:
|
|
48
|
+
"""Collect token counts, their update time, warnings, and account rate-limit observations for a rollout."""
|
|
49
|
+
|
|
40
50
|
tokens: dict[str, int]
|
|
41
51
|
warnings: set[str]
|
|
42
52
|
tokens_updated_at: datetime | None
|
|
@@ -66,13 +76,27 @@ def reconcile_completed_scan_cost(
|
|
|
66
76
|
) -> None:
|
|
67
77
|
"""Persist authoritative SDK cost without discarding measured worker usage."""
|
|
68
78
|
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
79
|
+
stored = stored_scan_cost_fields(scan["cost_json"])
|
|
80
|
+
completion = stored.get("usage", {}).get("completion")
|
|
81
|
+
native = collect_scan_usage_safely(
|
|
82
|
+
connection,
|
|
83
|
+
scan,
|
|
84
|
+
thread_id=completion["threadId"] if completion else None,
|
|
85
|
+
completed_turn_id=completion["turnId"] if completion and completion["finalized"] else None,
|
|
86
|
+
)
|
|
87
|
+
fields = merge_scan_usage_fields(stored, stored_scan_cost_fields(cost_json), {"usage": native})
|
|
88
|
+
if completion:
|
|
89
|
+
usage = {**fields["usage"], "completion": completion}
|
|
90
|
+
if not completion["finalized"]:
|
|
91
|
+
if usage["coverage"] == "complete":
|
|
92
|
+
usage["coverage"] = "partial"
|
|
93
|
+
usage["warnings"] = sorted({*usage.get("warnings", []), "turn_in_progress"})
|
|
94
|
+
fields["usage"] = usage
|
|
95
|
+
cost_json = json.dumps(
|
|
96
|
+
fields if "usage" in fields else fields["cost"],
|
|
97
|
+
separators=(",", ":"),
|
|
98
|
+
allow_nan=False,
|
|
99
|
+
)
|
|
76
100
|
connection.execute("BEGIN IMMEDIATE")
|
|
77
101
|
with connection:
|
|
78
102
|
connection.execute(
|
|
@@ -81,53 +105,219 @@ def reconcile_completed_scan_cost(
|
|
|
81
105
|
)
|
|
82
106
|
|
|
83
107
|
|
|
108
|
+
def merge_scan_usage_fields(*observations: Mapping[str, Any]) -> dict[str, Any]:
|
|
109
|
+
"""Reconcile overlapping measurements without summing the same execution twice."""
|
|
110
|
+
|
|
111
|
+
merged: dict[str, Any] = {}
|
|
112
|
+
latest_coverage: dict[str, Any] | None = None
|
|
113
|
+
rate_limits: dict[str, dict[str, Any]] = {}
|
|
114
|
+
for observation in observations:
|
|
115
|
+
if isinstance(observation.get("cost"), dict):
|
|
116
|
+
merged["cost"] = observation["cost"]
|
|
117
|
+
usage = observation.get("usage")
|
|
118
|
+
if not isinstance(usage, dict):
|
|
119
|
+
continue
|
|
120
|
+
for snapshot in usage.get("accountRateLimits", []):
|
|
121
|
+
key = snapshot["limitId"]
|
|
122
|
+
if key not in rate_limits or snapshot["observedAt"] >= rate_limits[key]["observedAt"]:
|
|
123
|
+
rate_limits[key] = snapshot
|
|
124
|
+
previous = merged.get("usage")
|
|
125
|
+
if previous is None or _usage_quality(usage) >= _usage_quality(previous):
|
|
126
|
+
merged["usage"] = usage
|
|
127
|
+
if usage.get("coverage") not in ("complete", "partial"):
|
|
128
|
+
continue
|
|
129
|
+
latest_coverage = (
|
|
130
|
+
usage if latest_coverage is None else _merge_usage_coverage(latest_coverage, usage)
|
|
131
|
+
)
|
|
132
|
+
cost = merged.get("cost")
|
|
133
|
+
usage = merged.get("usage")
|
|
134
|
+
if usage is not None and latest_coverage is not None and latest_coverage is not usage:
|
|
135
|
+
usage = {**usage}
|
|
136
|
+
for field in (
|
|
137
|
+
"coverage",
|
|
138
|
+
"coverageScope",
|
|
139
|
+
"threadCount",
|
|
140
|
+
"missingThreadCount",
|
|
141
|
+
"warnings",
|
|
142
|
+
"observedAt",
|
|
143
|
+
"completion",
|
|
144
|
+
"byRole",
|
|
145
|
+
):
|
|
146
|
+
if field in latest_coverage:
|
|
147
|
+
usage[field] = latest_coverage[field]
|
|
148
|
+
else:
|
|
149
|
+
usage.pop(field, None)
|
|
150
|
+
# Counts describe the coverage observation, even when its scope narrows.
|
|
151
|
+
# Its whole role breakdown must still sum to the retained token subtotal.
|
|
152
|
+
if "byRole" in usage and any(
|
|
153
|
+
usage.get(field) != latest_coverage.get(field) for field in TOKEN_FIELDS.values()
|
|
154
|
+
):
|
|
155
|
+
usage.pop("byRole")
|
|
156
|
+
merged["usage"] = usage
|
|
157
|
+
cost_warning = "cost_estimate_covers_earlier_usage"
|
|
158
|
+
if (
|
|
159
|
+
cost is not None
|
|
160
|
+
and usage is not None
|
|
161
|
+
and usage.get("coverage") in ("complete", "partial")
|
|
162
|
+
and type(usage.get("totalTokens")) is int
|
|
163
|
+
):
|
|
164
|
+
warnings = set(usage.get("warnings", []))
|
|
165
|
+
warnings.discard(cost_warning)
|
|
166
|
+
if usage["totalTokens"] > cost["inputTokens"] + cost["outputTokens"]:
|
|
167
|
+
warnings.add(cost_warning)
|
|
168
|
+
if warnings != set(usage.get("warnings", [])):
|
|
169
|
+
updated = {**usage}
|
|
170
|
+
updated["coverage"] = (
|
|
171
|
+
"partial"
|
|
172
|
+
if warnings - {cost_warning} or usage.get("missingThreadCount", 0)
|
|
173
|
+
else "complete"
|
|
174
|
+
)
|
|
175
|
+
if warnings:
|
|
176
|
+
updated["warnings"] = sorted(warnings)
|
|
177
|
+
else:
|
|
178
|
+
updated.pop("warnings", None)
|
|
179
|
+
merged["usage"] = updated
|
|
180
|
+
if rate_limits:
|
|
181
|
+
merged["usage"] = {
|
|
182
|
+
**merged["usage"],
|
|
183
|
+
"accountRateLimits": list(rate_limits.values()),
|
|
184
|
+
}
|
|
185
|
+
return merged
|
|
186
|
+
|
|
187
|
+
|
|
188
|
+
def _merge_usage_coverage(previous: dict[str, Any], observed: dict[str, Any]) -> dict[str, Any]:
|
|
189
|
+
if previous == observed:
|
|
190
|
+
return observed
|
|
191
|
+
previous_at = _timestamp(previous.get("observedAt"))
|
|
192
|
+
observed_at = _timestamp(observed.get("observedAt"))
|
|
193
|
+
if previous_at is not None and observed_at is not None and previous_at > observed_at:
|
|
194
|
+
previous, observed = observed, previous
|
|
195
|
+
previous_scope = previous.get("coverageScope")
|
|
196
|
+
observed_scope = observed.get("coverageScope")
|
|
197
|
+
covers_previous = (
|
|
198
|
+
previous_scope is not None
|
|
199
|
+
and observed_scope is not None
|
|
200
|
+
and set(previous_scope).issubset(observed_scope)
|
|
201
|
+
)
|
|
202
|
+
scope_gap = previous_scope is not None and observed_scope is not None and not covers_previous
|
|
203
|
+
unresolved_previous = (previous["coverage"] == "partial" and not covers_previous) or scope_gap
|
|
204
|
+
merged = {**observed}
|
|
205
|
+
if unresolved_previous:
|
|
206
|
+
merged["coverage"] = "partial"
|
|
207
|
+
merged["missingThreadCount"] = max(
|
|
208
|
+
previous.get("missingThreadCount", 0), observed.get("missingThreadCount", 0)
|
|
209
|
+
)
|
|
210
|
+
warnings = set(previous.get("warnings", [])) | set(observed.get("warnings", []))
|
|
211
|
+
if scope_gap:
|
|
212
|
+
# A newer subset can grow while still staying below the older
|
|
213
|
+
# aggregate. Neither subtotal proves complete usage for the union.
|
|
214
|
+
warnings.add("usage_scope_incomplete")
|
|
215
|
+
if warnings:
|
|
216
|
+
merged["warnings"] = sorted(warnings)
|
|
217
|
+
if previous.get("completion", {}).get("finalized") is False:
|
|
218
|
+
merged["completion"] = previous["completion"]
|
|
219
|
+
if previous_scope is not None and observed_scope is not None:
|
|
220
|
+
merged["coverageScope"] = sorted(set(previous_scope) | set(observed_scope))
|
|
221
|
+
elif previous_scope is not None and observed["coverage"] == "complete":
|
|
222
|
+
merged["coverageScope"] = previous_scope
|
|
223
|
+
elif unresolved_previous:
|
|
224
|
+
# An older subtotal without scope cannot prove which gaps a later
|
|
225
|
+
# collector has resolved. Retain that uncertainty across future merges.
|
|
226
|
+
merged.pop("coverageScope", None)
|
|
227
|
+
return merged
|
|
228
|
+
|
|
229
|
+
|
|
230
|
+
def _usage_quality(
|
|
231
|
+
usage: Mapping[str, Any], *, observed_at: datetime | None = None
|
|
232
|
+
) -> tuple[int, bool, datetime]:
|
|
233
|
+
return (
|
|
234
|
+
usage.get("totalTokens", -1),
|
|
235
|
+
usage.get("cacheWriteInputTokensReported") is not False,
|
|
236
|
+
observed_at
|
|
237
|
+
or _timestamp(usage.get("observedAt"))
|
|
238
|
+
or datetime.min.replace(tzinfo=timezone.utc),
|
|
239
|
+
)
|
|
240
|
+
|
|
241
|
+
|
|
242
|
+
def collect_scan_usage_safely(
|
|
243
|
+
connection: sqlite3.Connection, scan: sqlite3.Row, **options: Any
|
|
244
|
+
) -> dict[str, Any]:
|
|
245
|
+
try:
|
|
246
|
+
return collect_scan_usage(connection, scan, **options)
|
|
247
|
+
except (Exception, SystemExit) as exc:
|
|
248
|
+
try:
|
|
249
|
+
print(f"Could not collect scan token usage: {exc}", file=sys.stderr)
|
|
250
|
+
except OSError:
|
|
251
|
+
# A closed diagnostic stream must not replace the unavailable-usage result.
|
|
252
|
+
pass
|
|
253
|
+
return _unavailable_usage("usage_collection_failed")
|
|
254
|
+
|
|
255
|
+
|
|
84
256
|
def collect_scan_usage(
|
|
85
257
|
connection: sqlite3.Connection,
|
|
86
258
|
scan: sqlite3.Row,
|
|
87
259
|
*,
|
|
88
260
|
thread_id: str | None = None,
|
|
261
|
+
validation_thread_id: str | None = None,
|
|
89
262
|
completed_at: str | None = None,
|
|
90
263
|
completed_turn_id: str | None = None,
|
|
91
264
|
) -> dict[str, Any]:
|
|
92
|
-
"""Count attributable receipts
|
|
265
|
+
"""Count attributable session receipts during the scan window and report missing coverage.
|
|
266
|
+
|
|
267
|
+
The optional thread and completion arguments identify the finishing turn when
|
|
268
|
+
the scan record alone is insufficient. Return an unavailable usage record when
|
|
269
|
+
the window, Codex state, or relevant threads cannot be established.
|
|
270
|
+
"""
|
|
93
271
|
|
|
94
272
|
roots = _scan_root_thread_ids(connection, scan, thread_id)
|
|
95
273
|
parent_ids = set(_scan_root_thread_ids(connection, scan, thread_id, include_workers=False))
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
if state_database is None:
|
|
102
|
-
return _unavailable_usage("codex_state_unavailable")
|
|
103
|
-
|
|
274
|
+
if validation_thread_id:
|
|
275
|
+
roots.append(validation_thread_id)
|
|
276
|
+
parent_ids.add(validation_thread_id)
|
|
277
|
+
warnings: set[str] = set()
|
|
278
|
+
worker_directories = _scan_worker_directories(connection, scan, warnings)
|
|
104
279
|
started_at = _timestamp(scan["started_at"])
|
|
105
280
|
observed_at = datetime.now(timezone.utc)
|
|
106
281
|
stopped_at = _timestamp(completed_at or scan["completed_at"]) or observed_at
|
|
107
282
|
if started_at is None:
|
|
108
283
|
return _unavailable_usage("scan_window_unavailable")
|
|
109
284
|
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
)
|
|
120
|
-
|
|
121
|
-
|
|
285
|
+
receipts, missing_receipts, receipt_scope = _worker_usage_receipts(
|
|
286
|
+
scan, started_at, stopped_at, warnings
|
|
287
|
+
)
|
|
288
|
+
state_database = _codex_state_database()
|
|
289
|
+
sessions: list[RolloutSession] = []
|
|
290
|
+
missing_thread_ids: set[str] = set()
|
|
291
|
+
if state_database is None:
|
|
292
|
+
if not receipts:
|
|
293
|
+
return _unavailable_usage("codex_state_unavailable", warnings=warnings)
|
|
294
|
+
warnings.add("codex_state_unavailable")
|
|
295
|
+
missing_thread_ids.update(roots)
|
|
296
|
+
else:
|
|
297
|
+
try:
|
|
298
|
+
sessions, missing_thread_ids = _discover_rollout_sessions(
|
|
299
|
+
state_database,
|
|
300
|
+
roots,
|
|
301
|
+
warnings,
|
|
302
|
+
worker_directories=worker_directories,
|
|
303
|
+
scan_directory=Path(scan["scan_dir"]) if worker_directories else None,
|
|
304
|
+
started_at=started_at,
|
|
305
|
+
completed_at=stopped_at,
|
|
306
|
+
)
|
|
307
|
+
except (OSError, sqlite3.Error, ValueError):
|
|
308
|
+
if not receipts:
|
|
309
|
+
return _unavailable_usage("codex_state_unavailable", warnings=warnings)
|
|
310
|
+
warnings.add("codex_state_unavailable")
|
|
311
|
+
missing_thread_ids.update(roots)
|
|
122
312
|
|
|
123
|
-
if not sessions:
|
|
313
|
+
if not sessions and not receipts:
|
|
124
314
|
return _unavailable_usage("scan_thread_unavailable", warnings=warnings)
|
|
125
315
|
|
|
126
|
-
by_role = {role: {**_empty_token_usage(), "threadCount": 0} for role in ("parent", "workers")}
|
|
127
316
|
rate_limits: dict[str, dict[str, Any]] = {}
|
|
128
317
|
tokens_updated_at = None
|
|
129
318
|
accepted_thread_ids: set[str] = set()
|
|
130
|
-
|
|
319
|
+
measured_threads: dict[str, dict[str, int]] = {}
|
|
320
|
+
complete_native_thread_ids: set[str] = set()
|
|
131
321
|
excluded_thread_ids: set[str] = set()
|
|
132
322
|
for session in sessions:
|
|
133
323
|
if session.parent_thread_id in excluded_thread_ids:
|
|
@@ -168,25 +358,60 @@ def collect_scan_usage(
|
|
|
168
358
|
for key, snapshot in usage.rate_limits.items():
|
|
169
359
|
if key not in rate_limits or snapshot["observedAt"] > rate_limits[key]["observedAt"]:
|
|
170
360
|
rate_limits[key] = snapshot
|
|
171
|
-
if "token_usage_unavailable"
|
|
361
|
+
if usage.warnings & {"token_usage_unavailable", "token_breakdown_unavailable"}:
|
|
172
362
|
missing_thread_ids.add(session.thread_id)
|
|
173
363
|
continue
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
role["threadCount"] += 1
|
|
364
|
+
measured_threads[session.thread_id] = usage.tokens
|
|
365
|
+
if not usage.warnings:
|
|
366
|
+
complete_native_thread_ids.add(session.thread_id)
|
|
178
367
|
if usage.tokens_updated_at is not None:
|
|
179
368
|
tokens_updated_at = max(
|
|
180
369
|
tokens_updated_at or usage.tokens_updated_at, usage.tokens_updated_at
|
|
181
370
|
)
|
|
182
371
|
|
|
372
|
+
runtime_receipts_used = False
|
|
373
|
+
for receipt_thread_id, receipt in receipts.items():
|
|
374
|
+
native = measured_threads.get(receipt_thread_id)
|
|
375
|
+
extends_native = native is None or receipt.tokens["totalTokens"] > native["totalTokens"]
|
|
376
|
+
fills_cache_write_usage = (
|
|
377
|
+
native is not None
|
|
378
|
+
and receipt.tokens["totalTokens"] == native["totalTokens"]
|
|
379
|
+
and native.get("cacheWriteInputTokensReported") is False
|
|
380
|
+
and receipt.tokens.get("cacheWriteInputTokensReported") is not False
|
|
381
|
+
)
|
|
382
|
+
if extends_native or fills_cache_write_usage:
|
|
383
|
+
measured_threads[receipt_thread_id] = receipt.tokens
|
|
384
|
+
accepted_thread_ids.add(receipt_thread_id)
|
|
385
|
+
runtime_receipts_used = True
|
|
386
|
+
if extends_native:
|
|
387
|
+
complete_native_thread_ids.discard(receipt_thread_id)
|
|
388
|
+
# Numeric SDK subtotals cannot establish that descendants and
|
|
389
|
+
# compaction requests are present when their rollouts are absent.
|
|
390
|
+
missing_thread_ids.add(receipt_thread_id)
|
|
391
|
+
warnings.add("worker_rollout_usage_incomplete")
|
|
392
|
+
if receipt.tokens_updated_at is not None:
|
|
393
|
+
tokens_updated_at = max(
|
|
394
|
+
tokens_updated_at or receipt.tokens_updated_at, receipt.tokens_updated_at
|
|
395
|
+
)
|
|
396
|
+
unresolved_receipts = missing_receipts - complete_native_thread_ids
|
|
397
|
+
if unresolved_receipts:
|
|
398
|
+
missing_thread_ids.update(unresolved_receipts)
|
|
399
|
+
warnings.add("worker_usage_unavailable")
|
|
183
400
|
if not accepted_thread_ids:
|
|
184
401
|
return _unavailable_usage("scan_thread_unavailable", warnings=warnings)
|
|
185
|
-
if not
|
|
186
|
-
|
|
187
|
-
"
|
|
402
|
+
if not measured_threads:
|
|
403
|
+
reason = (
|
|
404
|
+
"token_breakdown_unavailable"
|
|
405
|
+
if "token_breakdown_unavailable" in warnings
|
|
406
|
+
else "token_usage_unavailable"
|
|
188
407
|
)
|
|
408
|
+
return _unavailable_usage(reason, warnings=warnings, rate_limits=rate_limits)
|
|
189
409
|
|
|
410
|
+
by_role = {role: {**_empty_token_usage(), "threadCount": 0} for role in ("parent", "workers")}
|
|
411
|
+
for measured_id, tokens in measured_threads.items():
|
|
412
|
+
role = by_role["parent" if measured_id in parent_ids else "workers"]
|
|
413
|
+
_add_token_usage(role, tokens)
|
|
414
|
+
role["threadCount"] += 1
|
|
190
415
|
if any(
|
|
191
416
|
usage["cachedInputTokens"] + usage["cacheWriteInputTokens"] > usage["inputTokens"]
|
|
192
417
|
or usage["totalTokens"] != usage["inputTokens"] + usage["outputTokens"]
|
|
@@ -196,18 +421,28 @@ def collect_scan_usage(
|
|
|
196
421
|
"token_breakdown_unavailable", warnings=warnings, rate_limits=rate_limits
|
|
197
422
|
)
|
|
198
423
|
|
|
199
|
-
if completed_turn_id is not None and tokens_updated_at is not None:
|
|
424
|
+
if (completed_turn_id is not None or runtime_receipts_used) and tokens_updated_at is not None:
|
|
200
425
|
stopped_at = max(stopped_at, tokens_updated_at)
|
|
201
426
|
result: dict[str, Any] = {
|
|
202
427
|
"coverage": "partial" if missing_thread_ids or warnings else "complete",
|
|
203
|
-
"source": "codex_rollout",
|
|
428
|
+
"source": "codex_runtime" if runtime_receipts_used else "codex_rollout",
|
|
204
429
|
**{key: sum(role[key] for role in by_role.values()) for key in TOKEN_FIELDS.values()},
|
|
205
|
-
"threadCount": len(
|
|
430
|
+
"threadCount": len(measured_threads),
|
|
431
|
+
"coverageScope": sorted(
|
|
432
|
+
receipt_scope
|
|
433
|
+
| {f"thread:{identity}" for identity in measured_threads}
|
|
434
|
+
| {
|
|
435
|
+
identity if identity.startswith("worker-attempt:") else f"thread:{identity}"
|
|
436
|
+
for identity in missing_thread_ids
|
|
437
|
+
}
|
|
438
|
+
),
|
|
206
439
|
"byRole": by_role,
|
|
207
440
|
"window": {"startedAt": started_at.isoformat(), "endedAt": stopped_at.isoformat()},
|
|
208
441
|
"observedAt": observed_at.isoformat(),
|
|
209
442
|
"accountRateLimits": [rate_limits[key] for key in sorted(rate_limits)],
|
|
210
443
|
}
|
|
444
|
+
if any(role.get("cacheWriteInputTokensReported") is False for role in by_role.values()):
|
|
445
|
+
result["cacheWriteInputTokensReported"] = False
|
|
211
446
|
if tokens_updated_at is not None:
|
|
212
447
|
result["tokensUpdatedAt"] = tokens_updated_at.isoformat()
|
|
213
448
|
if missing_thread_ids:
|
|
@@ -217,6 +452,79 @@ def collect_scan_usage(
|
|
|
217
452
|
return result
|
|
218
453
|
|
|
219
454
|
|
|
455
|
+
def _worker_usage_receipts(
|
|
456
|
+
scan: Mapping[str, Any],
|
|
457
|
+
started_at: datetime,
|
|
458
|
+
completed_at: datetime,
|
|
459
|
+
warnings: set[str],
|
|
460
|
+
) -> tuple[dict[str, RolloutUsage], set[str], set[str]]:
|
|
461
|
+
if scan["mode"] != "deep":
|
|
462
|
+
return {}, set(), set()
|
|
463
|
+
directory = worker_usage_directory(scan["id"])
|
|
464
|
+
try:
|
|
465
|
+
if directory.resolve(strict=True) != directory:
|
|
466
|
+
raise ValueError("Worker usage directory is not a canonical protected directory.")
|
|
467
|
+
except FileNotFoundError:
|
|
468
|
+
return {}, set(), set()
|
|
469
|
+
except (OSError, RuntimeError, ValueError):
|
|
470
|
+
warnings.add("worker_usage_receipt_unavailable")
|
|
471
|
+
return {}, {"worker-receipts"}, {"worker-receipts"}
|
|
472
|
+
receipts: dict[str, RolloutUsage] = {}
|
|
473
|
+
missing: set[str] = set()
|
|
474
|
+
attempts: set[str] = set()
|
|
475
|
+
scope: set[str] = set()
|
|
476
|
+
for path in sorted(directory.glob("*.json")):
|
|
477
|
+
try:
|
|
478
|
+
if _rollout_path(str(path)) is None:
|
|
479
|
+
raise ValueError("Worker usage receipt path is unavailable.")
|
|
480
|
+
receipt = json.loads(path.read_text(encoding="utf-8"))
|
|
481
|
+
if (
|
|
482
|
+
not isinstance(receipt, dict)
|
|
483
|
+
or receipt.get("schemaVersion") != 1
|
|
484
|
+
or receipt.get("source") != "codex_sdk"
|
|
485
|
+
or receipt.get("usageScope") != "thread_cumulative"
|
|
486
|
+
or receipt.get("scanId") != scan["id"]
|
|
487
|
+
or receipt.get("kind") not in ("discovery", "dedup", "compare")
|
|
488
|
+
or receipt.get("attemptId") != path.stem
|
|
489
|
+
):
|
|
490
|
+
raise ValueError("Worker usage receipt identity is invalid.")
|
|
491
|
+
attempt_id = receipt["attemptId"]
|
|
492
|
+
begin = _timestamp(receipt.get("startedAt"))
|
|
493
|
+
observed = _timestamp(receipt.get("observedAt"))
|
|
494
|
+
if begin is None or observed is None:
|
|
495
|
+
raise ValueError("Worker usage receipt timestamps are invalid.")
|
|
496
|
+
if begin < started_at or begin > completed_at or attempt_id in attempts:
|
|
497
|
+
continue
|
|
498
|
+
attempts.add(attempt_id)
|
|
499
|
+
scope.add(f"worker-attempt:{attempt_id}")
|
|
500
|
+
thread_id = receipt.get("threadId")
|
|
501
|
+
if isinstance(thread_id, str) and thread_id:
|
|
502
|
+
scope.add(f"thread:{thread_id}")
|
|
503
|
+
tokens = _usage_counts(receipt.get("usage"))
|
|
504
|
+
if not isinstance(thread_id, str) or not thread_id or tokens is None:
|
|
505
|
+
missing.add(
|
|
506
|
+
thread_id
|
|
507
|
+
if isinstance(thread_id, str) and thread_id
|
|
508
|
+
else f"worker-attempt:{attempt_id}"
|
|
509
|
+
)
|
|
510
|
+
continue
|
|
511
|
+
previous = receipts.get(thread_id)
|
|
512
|
+
if previous is None or _usage_quality(tokens, observed_at=observed) > _usage_quality(
|
|
513
|
+
previous.tokens, observed_at=previous.tokens_updated_at
|
|
514
|
+
):
|
|
515
|
+
receipts[thread_id] = RolloutUsage(tokens, set(), observed, {})
|
|
516
|
+
except (OSError, UnicodeError, ValueError) as exc:
|
|
517
|
+
warnings.add("worker_usage_receipt_unavailable")
|
|
518
|
+
missing.add(f"worker-attempt:{path.stem}")
|
|
519
|
+
scope.add(f"worker-attempt:{path.stem}")
|
|
520
|
+
try:
|
|
521
|
+
print(f"Could not read worker token usage: {exc}", file=sys.stderr)
|
|
522
|
+
except OSError:
|
|
523
|
+
# The receipt warning and missing-attempt scope are already retained.
|
|
524
|
+
pass
|
|
525
|
+
return receipts, missing, scope
|
|
526
|
+
|
|
527
|
+
|
|
220
528
|
def _scan_root_thread_ids(
|
|
221
529
|
connection: sqlite3.Connection,
|
|
222
530
|
scan: sqlite3.Row,
|
|
@@ -267,19 +575,35 @@ def _scan_execution_thread_ids(connection: sqlite3.Connection, scan: sqlite3.Row
|
|
|
267
575
|
)
|
|
268
576
|
|
|
269
577
|
|
|
270
|
-
def _scan_worker_directories(
|
|
578
|
+
def _scan_worker_directories(
|
|
579
|
+
connection: sqlite3.Connection, scan: sqlite3.Row, warnings: set[str]
|
|
580
|
+
) -> dict[str, bool]:
|
|
271
581
|
if scan["mode"] != "deep":
|
|
272
|
-
return
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
282
|
-
|
|
582
|
+
return {}
|
|
583
|
+
if connection.execute(
|
|
584
|
+
"SELECT 1 FROM deep_scan_runs WHERE scan_id = ? AND worker_usage_gap = 1", (scan["id"],)
|
|
585
|
+
).fetchone():
|
|
586
|
+
warnings.add("worker_thread_unavailable")
|
|
587
|
+
artifacts = Path(scan["scan_dir"]) / "artifacts"
|
|
588
|
+
# Comparison turns run in this directory without a discovery worker row.
|
|
589
|
+
directories = {str(artifacts / "deep_discovery"): False}
|
|
590
|
+
for row in connection.execute(
|
|
591
|
+
"SELECT kind, artifact_dir, started_at, sdk_thread_id, attempt FROM deep_scan_workers "
|
|
592
|
+
"WHERE scan_id = ? AND kind IN ('discovery', 'dedup')",
|
|
593
|
+
(scan["id"],),
|
|
594
|
+
):
|
|
595
|
+
if row["kind"] == "discovery":
|
|
596
|
+
if row["sdk_thread_id"] and row["attempt"] == 1:
|
|
597
|
+
continue # The first attempt already has an exact thread identity.
|
|
598
|
+
# An attempt that started without a saved ID must be found by its assigned cwd.
|
|
599
|
+
missing_thread = bool(row["started_at"] and not row["sdk_thread_id"])
|
|
600
|
+
directories[row["artifact_dir"]] = missing_thread
|
|
601
|
+
else:
|
|
602
|
+
directories[str(artifacts)] = False
|
|
603
|
+
if row["attempt"] > 1 and row["started_at"] and not row["sdk_thread_id"]:
|
|
604
|
+
# Retries share a cwd. A rollout from an older attempt cannot prove the new one.
|
|
605
|
+
warnings.add("worker_thread_unavailable")
|
|
606
|
+
return directories
|
|
283
607
|
|
|
284
608
|
|
|
285
609
|
def _codex_state_database() -> Path | None:
|
|
@@ -324,7 +648,8 @@ def _discover_rollout_sessions(
|
|
|
324
648
|
roots: list[str],
|
|
325
649
|
warnings: set[str],
|
|
326
650
|
*,
|
|
327
|
-
worker_directories:
|
|
651
|
+
worker_directories: dict[str, bool],
|
|
652
|
+
scan_directory: Path | None = None,
|
|
328
653
|
started_at: datetime,
|
|
329
654
|
completed_at: datetime | None,
|
|
330
655
|
) -> tuple[list[RolloutSession], set[str]]:
|
|
@@ -343,9 +668,12 @@ def _discover_rollout_sessions(
|
|
|
343
668
|
columns = {str(row["name"]) for row in database.execute(f"PRAGMA table_info({table})")}
|
|
344
669
|
if not required.issubset(columns):
|
|
345
670
|
raise ValueError("Codex state graph does not expose the required thread columns.")
|
|
671
|
+
if table == "threads" and worker_directories and not {"cwd", "created_at"} <= columns:
|
|
672
|
+
worker_directories = {}
|
|
673
|
+
warnings.add("worker_directories_unavailable")
|
|
346
674
|
sessions: dict[str, RolloutSession] = {}
|
|
347
675
|
missing_thread_ids: set[str] = set()
|
|
348
|
-
root_ids = dict.fromkeys(roots)
|
|
676
|
+
root_ids: dict[str, str | None] = dict.fromkeys(roots)
|
|
349
677
|
started_millisecond = started_at.replace(microsecond=started_at.microsecond // 1000 * 1000)
|
|
350
678
|
completed_timestamp = int(completed_at.timestamp()) if completed_at is not None else None
|
|
351
679
|
directory_expression = "cwd"
|
|
@@ -356,16 +684,52 @@ def _discover_rollout_sessions(
|
|
|
356
684
|
"codex_security_path_key", 1, _normalized_windows_path, deterministic=True
|
|
357
685
|
)
|
|
358
686
|
directory_expression = "codex_security_path_key(cwd)"
|
|
359
|
-
worker_directories = {
|
|
360
|
-
|
|
687
|
+
worker_directories = {
|
|
688
|
+
_normalized_windows_path(path): required
|
|
689
|
+
for path, required in worker_directories.items()
|
|
690
|
+
}
|
|
691
|
+
if scan_directory is not None:
|
|
692
|
+
scan_directory = Path(_normalized_windows_path(scan_directory))
|
|
693
|
+
unmatched = {path for path, required in worker_directories.items() if required}
|
|
694
|
+
directory_sources = {path: {path} for path in worker_directories}
|
|
695
|
+
if worker_directories and scan_directory and ".previous-" in scan_directory.name:
|
|
696
|
+
# Archived files move, but session working directories describe where execution ran.
|
|
697
|
+
# Confirm the original output with an already-bound session before including peers.
|
|
698
|
+
bound_directories = {
|
|
699
|
+
Path(row[0])
|
|
700
|
+
for row in database.execute(
|
|
701
|
+
f"SELECT DISTINCT {directory_expression} FROM threads WHERE cwd IS NOT NULL "
|
|
702
|
+
"AND id IN (SELECT value FROM json_each(?))",
|
|
703
|
+
(json.dumps(roots),),
|
|
704
|
+
)
|
|
705
|
+
}
|
|
706
|
+
relatives = {
|
|
707
|
+
Path(path).relative_to(scan_directory): path
|
|
708
|
+
for path in worker_directories
|
|
709
|
+
if Path(path).is_relative_to(scan_directory)
|
|
710
|
+
}
|
|
711
|
+
for marker in re.finditer(r"\.previous-", scan_directory.name):
|
|
712
|
+
if marker.start() == 0:
|
|
713
|
+
continue
|
|
714
|
+
original = scan_directory.with_name(scan_directory.name[: marker.start()])
|
|
715
|
+
candidates = {original / relative: path for relative, path in relatives.items()}
|
|
716
|
+
# Older archives kept absolute discovery paths in their original directory.
|
|
717
|
+
if (
|
|
718
|
+
original in bound_directories
|
|
719
|
+
or candidates.keys() & bound_directories
|
|
720
|
+
or any(Path(path).is_relative_to(original) for path in worker_directories)
|
|
721
|
+
):
|
|
722
|
+
for candidate, path in candidates.items():
|
|
723
|
+
directory_sources.setdefault(str(candidate), set()).add(path)
|
|
724
|
+
for directory in sorted(directory_sources):
|
|
361
725
|
for row in database.execute(
|
|
362
726
|
"SELECT id FROM threads WHERE cwd IS NOT NULL "
|
|
363
727
|
f"AND {directory_expression} = ? AND created_at >= ? "
|
|
364
728
|
"AND (? IS NULL OR created_at <= ?) ORDER BY id",
|
|
365
729
|
(directory, int(started_at.timestamp()), completed_timestamp, completed_timestamp),
|
|
366
730
|
):
|
|
367
|
-
root_ids[row["id"]] =
|
|
368
|
-
for root in root_ids:
|
|
731
|
+
root_ids[row["id"]] = directory
|
|
732
|
+
for root, directory in root_ids.items():
|
|
369
733
|
row = database.execute(
|
|
370
734
|
"SELECT id, rollout_path FROM threads WHERE id = ?",
|
|
371
735
|
(root,),
|
|
@@ -381,8 +745,8 @@ def _discover_rollout_sessions(
|
|
|
381
745
|
continue
|
|
382
746
|
if root not in roots:
|
|
383
747
|
try:
|
|
384
|
-
with path
|
|
385
|
-
metadata = json.loads(source
|
|
748
|
+
with _rollout_lines(path) as source:
|
|
749
|
+
metadata = json.loads(next(source, b""))
|
|
386
750
|
except (OSError, ValueError):
|
|
387
751
|
missing_thread_ids.add(root)
|
|
388
752
|
warnings.add("rollout_unavailable")
|
|
@@ -393,9 +757,15 @@ def _discover_rollout_sessions(
|
|
|
393
757
|
and isinstance(payload := metadata.get("payload"), dict)
|
|
394
758
|
and (payload.get("id") or payload.get("session_id")) == root
|
|
395
759
|
and (created_at := _timestamp(payload.get("timestamp"))) is not None
|
|
396
|
-
and
|
|
760
|
+
and (
|
|
761
|
+
created_at < started_millisecond
|
|
762
|
+
or completed_at is not None
|
|
763
|
+
and created_at > completed_at
|
|
764
|
+
)
|
|
397
765
|
):
|
|
398
766
|
continue
|
|
767
|
+
if directory is not None:
|
|
768
|
+
unmatched.difference_update(directory_sources[directory])
|
|
399
769
|
sessions[root] = RolloutSession(root, None, path)
|
|
400
770
|
descendants = database.execute(
|
|
401
771
|
"""
|
|
@@ -453,6 +823,8 @@ def _discover_rollout_sessions(
|
|
|
453
823
|
warnings.add("rollout_unavailable")
|
|
454
824
|
continue
|
|
455
825
|
sessions[child_id] = RolloutSession(child_id, parent_id, path)
|
|
826
|
+
if unmatched:
|
|
827
|
+
warnings.add("worker_thread_unavailable")
|
|
456
828
|
return list(sessions.values()), missing_thread_ids
|
|
457
829
|
finally:
|
|
458
830
|
database.close()
|
|
@@ -465,6 +837,13 @@ def _rollout_path(value: object) -> Path | None:
|
|
|
465
837
|
if not candidate.is_absolute():
|
|
466
838
|
return None
|
|
467
839
|
try:
|
|
840
|
+
try:
|
|
841
|
+
candidate.lstat()
|
|
842
|
+
except FileNotFoundError:
|
|
843
|
+
# Native cold storage may compress an indexed JSONL file in place.
|
|
844
|
+
if candidate.suffix != ".jsonl":
|
|
845
|
+
return None
|
|
846
|
+
candidate = candidate.with_suffix(".jsonl.zst")
|
|
468
847
|
resolved = candidate.resolve(strict=True)
|
|
469
848
|
if not resolved.is_file():
|
|
470
849
|
return None
|
|
@@ -481,6 +860,63 @@ def _rollout_path(value: object) -> Path | None:
|
|
|
481
860
|
return None
|
|
482
861
|
|
|
483
862
|
|
|
863
|
+
@contextmanager
|
|
864
|
+
def _rollout_lines(path: Path, *, verify_end: bool = False):
|
|
865
|
+
if path.suffix != ".zst":
|
|
866
|
+
with path.open("rb") as source:
|
|
867
|
+
yield source
|
|
868
|
+
return
|
|
869
|
+
try:
|
|
870
|
+
from compression import zstd # Python 3.14+
|
|
871
|
+
except ImportError:
|
|
872
|
+
try:
|
|
873
|
+
from backports import zstd
|
|
874
|
+
except ImportError:
|
|
875
|
+
zstd = None
|
|
876
|
+
if zstd is not None:
|
|
877
|
+
try:
|
|
878
|
+
with zstd.open(path, "rb") as source:
|
|
879
|
+
yield source
|
|
880
|
+
# A requested completion turn may stop parsing before the frame checksum.
|
|
881
|
+
if verify_end:
|
|
882
|
+
while source.read(64 * 1024):
|
|
883
|
+
pass
|
|
884
|
+
except (zstd.ZstdError, EOFError) as error:
|
|
885
|
+
raise ValueError("Invalid compressed Codex rollout.") from error
|
|
886
|
+
return
|
|
887
|
+
with tempfile.TemporaryFile() as errors:
|
|
888
|
+
with subprocess.Popen(
|
|
889
|
+
[
|
|
890
|
+
os.environ.get("CODEX_MCP_NODE_PATH") or "node",
|
|
891
|
+
str(Path(__file__).with_name("read_codex_session.mjs")),
|
|
892
|
+
str(path),
|
|
893
|
+
],
|
|
894
|
+
stdout=subprocess.PIPE,
|
|
895
|
+
stderr=errors,
|
|
896
|
+
) as child:
|
|
897
|
+
exhausted = False
|
|
898
|
+
|
|
899
|
+
def lines():
|
|
900
|
+
nonlocal exhausted
|
|
901
|
+
yield from child.stdout
|
|
902
|
+
exhausted = True
|
|
903
|
+
|
|
904
|
+
try:
|
|
905
|
+
iterator = lines()
|
|
906
|
+
yield iterator
|
|
907
|
+
if verify_end:
|
|
908
|
+
for _ in iterator:
|
|
909
|
+
pass
|
|
910
|
+
finally:
|
|
911
|
+
if not exhausted and child.poll() is None:
|
|
912
|
+
child.terminate()
|
|
913
|
+
child.stdout.close()
|
|
914
|
+
status = child.wait()
|
|
915
|
+
if exhausted and status != 0:
|
|
916
|
+
errors.seek(0)
|
|
917
|
+
raise OSError(errors.read().decode("utf-8", errors="replace"))
|
|
918
|
+
|
|
919
|
+
|
|
484
920
|
def _read_rollout_usage(
|
|
485
921
|
session: RolloutSession,
|
|
486
922
|
*,
|
|
@@ -493,8 +929,13 @@ def _read_rollout_usage(
|
|
|
493
929
|
total = result.tokens
|
|
494
930
|
warnings = result.warnings
|
|
495
931
|
previous = _empty_token_usage()
|
|
932
|
+
previous_cache_writes_reported = True
|
|
933
|
+
prefix_unclassified = 0
|
|
934
|
+
prefix_missing_cache: set[str] = set()
|
|
935
|
+
ambiguous_legacy_cache = False
|
|
496
936
|
boundary_reached = False
|
|
497
937
|
legacy_usage = _empty_token_usage()
|
|
938
|
+
completed_legacy_usage = _empty_token_usage()
|
|
498
939
|
response_usage: dict[str, int] | None = None
|
|
499
940
|
observed_response_usage = _empty_token_usage()
|
|
500
941
|
response_ids: set[str] = set()
|
|
@@ -506,7 +947,15 @@ def _read_rollout_usage(
|
|
|
506
947
|
closed_turns: set[str] = set()
|
|
507
948
|
measured_turns: set[str] = set()
|
|
508
949
|
|
|
509
|
-
|
|
950
|
+
def past_scan_end(timestamp: datetime, turn_id: object) -> bool:
|
|
951
|
+
# The requested completion turn may finish after the scan was marked complete.
|
|
952
|
+
return (
|
|
953
|
+
completed_at is not None
|
|
954
|
+
and timestamp > completed_at
|
|
955
|
+
and (completed_turn_id is None or turn_id != completed_turn_id)
|
|
956
|
+
)
|
|
957
|
+
|
|
958
|
+
with _rollout_lines(session.path, verify_end=True) as source:
|
|
510
959
|
for line_number, raw_line in enumerate(source, start=1):
|
|
511
960
|
if not raw_line.endswith(b"\n"):
|
|
512
961
|
warnings.add("rollout_record_incomplete")
|
|
@@ -559,10 +1008,8 @@ def _read_rollout_usage(
|
|
|
559
1008
|
if task_started_at is None:
|
|
560
1009
|
warnings.add("thread_ownership_unavailable")
|
|
561
1010
|
return result
|
|
562
|
-
if task_started_at < started_at or (
|
|
563
|
-
|
|
564
|
-
and task_started_at > completed_at
|
|
565
|
-
and payload.get("turn_id") != completed_turn_id
|
|
1011
|
+
if task_started_at < started_at or past_scan_end(
|
|
1012
|
+
task_started_at, payload.get("turn_id")
|
|
566
1013
|
):
|
|
567
1014
|
warnings.add("thread_outside_scan_window")
|
|
568
1015
|
return result
|
|
@@ -570,22 +1017,21 @@ def _read_rollout_usage(
|
|
|
570
1017
|
elif event.get("type") == "event_msg" and payload.get("type") == "token_count":
|
|
571
1018
|
inherited_usage = _token_snapshot(payload)
|
|
572
1019
|
if inherited_usage is not None:
|
|
573
|
-
|
|
1020
|
+
missing_cache = _missing_cache_fields(payload)
|
|
1021
|
+
baseline = _retain_cache_baseline(previous, inherited_usage, missing_cache)
|
|
1022
|
+
if baseline is None:
|
|
1023
|
+
warnings.add("token_breakdown_unavailable")
|
|
1024
|
+
else:
|
|
1025
|
+
previous = baseline
|
|
1026
|
+
prefix_unclassified = _uncached_input(baseline)
|
|
1027
|
+
prefix_missing_cache = missing_cache
|
|
574
1028
|
if not boundary_reached:
|
|
575
1029
|
continue
|
|
576
1030
|
if event.get("type") == "event_msg" and payload.get("type") == "task_started":
|
|
577
1031
|
# The same owner can resume as a model-backed coordinator. Only
|
|
578
1032
|
# a provably post-scan turn leaves the original control zero intact.
|
|
579
|
-
|
|
580
|
-
if (
|
|
581
|
-
completed_at is None
|
|
582
|
-
or task_started_at is None
|
|
583
|
-
or task_started_at <= completed_at
|
|
584
|
-
or (
|
|
585
|
-
completed_turn_id is not None
|
|
586
|
-
and payload.get("turn_id") == completed_turn_id
|
|
587
|
-
)
|
|
588
|
-
):
|
|
1033
|
+
timestamp = _timestamp(event.get("timestamp"))
|
|
1034
|
+
if timestamp is None or not past_scan_end(timestamp, payload.get("turn_id")):
|
|
589
1035
|
direct_control_session = False
|
|
590
1036
|
if (
|
|
591
1037
|
completed_turn_id is not None
|
|
@@ -596,19 +1042,19 @@ def _read_rollout_usage(
|
|
|
596
1042
|
_add_token_usage(
|
|
597
1043
|
total, response_usage if response_usage is not None else legacy_usage
|
|
598
1044
|
)
|
|
1045
|
+
if response_usage is None:
|
|
1046
|
+
_add_token_usage(completed_legacy_usage, legacy_usage)
|
|
1047
|
+
if ambiguous_legacy_cache:
|
|
1048
|
+
warnings.add("token_breakdown_unavailable")
|
|
1049
|
+
ambiguous_legacy_cache = False
|
|
599
1050
|
legacy_usage = _empty_token_usage()
|
|
600
1051
|
response_usage = None
|
|
601
1052
|
if payload.get("turn_id") != current_turn_id:
|
|
602
1053
|
observed_response_usage = _empty_token_usage()
|
|
603
1054
|
current_turn_id = payload.get("turn_id")
|
|
604
|
-
timestamp = _timestamp(event.get("timestamp"))
|
|
605
1055
|
if not isinstance(current_turn_id, str) or not current_turn_id or timestamp is None:
|
|
606
1056
|
warnings.add("turn_record_invalid")
|
|
607
|
-
elif (
|
|
608
|
-
completed_at is None
|
|
609
|
-
or timestamp <= completed_at
|
|
610
|
-
or current_turn_id == completed_turn_id
|
|
611
|
-
):
|
|
1057
|
+
elif not past_scan_end(timestamp, current_turn_id):
|
|
612
1058
|
started_turns.add(current_turn_id)
|
|
613
1059
|
open_turns.add(current_turn_id)
|
|
614
1060
|
continue
|
|
@@ -624,11 +1070,7 @@ def _read_rollout_usage(
|
|
|
624
1070
|
or timestamp is None
|
|
625
1071
|
):
|
|
626
1072
|
warnings.add("turn_record_invalid")
|
|
627
|
-
elif (
|
|
628
|
-
completed_at is None
|
|
629
|
-
or timestamp <= completed_at
|
|
630
|
-
or terminal_turn_id == completed_turn_id
|
|
631
|
-
):
|
|
1073
|
+
elif not past_scan_end(timestamp, terminal_turn_id):
|
|
632
1074
|
open_turns.discard(terminal_turn_id)
|
|
633
1075
|
if timestamp >= started_at:
|
|
634
1076
|
if terminal_turn_id in started_turns:
|
|
@@ -648,11 +1090,7 @@ def _read_rollout_usage(
|
|
|
648
1090
|
if (
|
|
649
1091
|
timestamp is not None
|
|
650
1092
|
and timestamp >= started_at
|
|
651
|
-
and (
|
|
652
|
-
completed_at is None
|
|
653
|
-
or timestamp <= completed_at
|
|
654
|
-
or (completed_turn_id is not None and current_turn_id == completed_turn_id)
|
|
655
|
-
)
|
|
1093
|
+
and not past_scan_end(timestamp, current_turn_id)
|
|
656
1094
|
and "compaction_response_id" not in payload
|
|
657
1095
|
and response_usage is None
|
|
658
1096
|
):
|
|
@@ -675,15 +1113,13 @@ def _read_rollout_usage(
|
|
|
675
1113
|
):
|
|
676
1114
|
warnings.add("token_record_invalid")
|
|
677
1115
|
continue
|
|
678
|
-
|
|
679
|
-
|
|
680
|
-
)
|
|
681
|
-
if completed_at is not None and timestamp > completed_at and not is_completion_turn:
|
|
1116
|
+
receipt_turn_id = payload.get("turn_id")
|
|
1117
|
+
if past_scan_end(timestamp, receipt_turn_id):
|
|
682
1118
|
continue
|
|
683
1119
|
if response_usage is None:
|
|
684
1120
|
response_usage = _empty_token_usage()
|
|
685
1121
|
if current_turn_id is None:
|
|
686
|
-
current_turn_id =
|
|
1122
|
+
current_turn_id = receipt_turn_id
|
|
687
1123
|
if isinstance(current_turn_id, str):
|
|
688
1124
|
open_turns.add(current_turn_id)
|
|
689
1125
|
if response_id in response_ids:
|
|
@@ -691,14 +1127,13 @@ def _read_rollout_usage(
|
|
|
691
1127
|
response_ids.add(response_id)
|
|
692
1128
|
_add_token_usage(observed_response_usage, usage)
|
|
693
1129
|
turn_usage = _usage_counts(payload.get("turn_token_usage"))
|
|
694
|
-
receipt_turn_id = payload.get("turn_id")
|
|
695
1130
|
if isinstance(receipt_turn_id, str):
|
|
696
1131
|
measured_turns.add(receipt_turn_id)
|
|
697
1132
|
if timestamp < started_at:
|
|
698
1133
|
if turn_usage is not None:
|
|
699
1134
|
observed_response_usage = turn_usage
|
|
700
1135
|
continue
|
|
701
|
-
if
|
|
1136
|
+
if completed_turn_id is not None and receipt_turn_id == completed_turn_id:
|
|
702
1137
|
completed_turn_measured = True
|
|
703
1138
|
if turn_usage is not None and any(
|
|
704
1139
|
turn_usage[key] > observed_response_usage[key] for key in TOKEN_FIELDS.values()
|
|
@@ -713,11 +1148,7 @@ def _read_rollout_usage(
|
|
|
713
1148
|
in_window = (
|
|
714
1149
|
timestamp is not None
|
|
715
1150
|
and timestamp >= started_at
|
|
716
|
-
and (
|
|
717
|
-
completed_at is None
|
|
718
|
-
or timestamp <= completed_at
|
|
719
|
-
or (completed_turn_id is not None and current_turn_id == completed_turn_id)
|
|
720
|
-
)
|
|
1151
|
+
and not past_scan_end(timestamp, current_turn_id)
|
|
721
1152
|
)
|
|
722
1153
|
if in_window:
|
|
723
1154
|
rate_limits = _rate_limit_snapshot(payload.get("rate_limits"), timestamp)
|
|
@@ -730,33 +1161,102 @@ def _read_rollout_usage(
|
|
|
730
1161
|
result.rate_limits[key] = rate_limits
|
|
731
1162
|
if "info" in payload and payload["info"] is None:
|
|
732
1163
|
continue
|
|
733
|
-
timestamp = _timestamp(event.get("timestamp"))
|
|
734
1164
|
snapshot = _token_snapshot(payload)
|
|
735
1165
|
if timestamp is None or snapshot is None:
|
|
736
1166
|
warnings.add("token_record_invalid")
|
|
737
1167
|
continue
|
|
1168
|
+
reset = snapshot["totalTokens"] < previous["totalTokens"]
|
|
1169
|
+
if reset:
|
|
1170
|
+
prefix_unclassified = 0
|
|
1171
|
+
missing_cache = _missing_cache_fields(payload)
|
|
1172
|
+
cache_writes_reported = "cacheWriteInputTokens" not in missing_cache
|
|
1173
|
+
baseline = _retain_cache_baseline(previous, snapshot, missing_cache)
|
|
1174
|
+
if baseline is None:
|
|
1175
|
+
if not past_scan_end(timestamp, current_turn_id):
|
|
1176
|
+
warnings.add("token_breakdown_unavailable")
|
|
1177
|
+
continue
|
|
1178
|
+
if baseline is previous:
|
|
1179
|
+
cache_writes_reported = previous_cache_writes_reported and cache_writes_reported
|
|
1180
|
+
refines_classification = (
|
|
1181
|
+
baseline["inputTokens"] >= previous["inputTokens"]
|
|
1182
|
+
and baseline["outputTokens"] >= previous["outputTokens"]
|
|
1183
|
+
and baseline["cachedInputTokens"] >= previous["cachedInputTokens"]
|
|
1184
|
+
and baseline["cacheWriteInputTokens"] >= previous["cacheWriteInputTokens"]
|
|
1185
|
+
and (
|
|
1186
|
+
(not previous_cache_writes_reported and cache_writes_reported)
|
|
1187
|
+
or baseline["cachedInputTokens"] > previous["cachedInputTokens"]
|
|
1188
|
+
or baseline["cacheWriteInputTokens"] > previous["cacheWriteInputTokens"]
|
|
1189
|
+
)
|
|
1190
|
+
)
|
|
1191
|
+
previous_cache_writes_reported = cache_writes_reported
|
|
738
1192
|
delta = {
|
|
739
|
-
key:
|
|
740
|
-
|
|
1193
|
+
key: snapshot[key] - previous[key]
|
|
1194
|
+
if not reset and snapshot[key] >= previous[key]
|
|
1195
|
+
else snapshot[key]
|
|
1196
|
+
for key in TOKEN_FIELDS.values()
|
|
741
1197
|
}
|
|
742
|
-
|
|
743
|
-
|
|
744
|
-
|
|
745
|
-
)
|
|
746
|
-
|
|
1198
|
+
if (
|
|
1199
|
+
snapshot.get("cacheWriteInputTokensReported") is False
|
|
1200
|
+
or previous.get("cacheWriteInputTokensReported") is False
|
|
1201
|
+
):
|
|
1202
|
+
delta["cacheWriteInputTokensReported"] = False
|
|
1203
|
+
previous = baseline
|
|
1204
|
+
if past_scan_end(timestamp, current_turn_id):
|
|
747
1205
|
continue
|
|
748
1206
|
if isinstance(current_turn_id, str):
|
|
749
1207
|
measured_turns.add(current_turn_id)
|
|
750
1208
|
if timestamp < started_at:
|
|
1209
|
+
prefix_unclassified = _uncached_input(baseline)
|
|
1210
|
+
prefix_missing_cache = missing_cache
|
|
751
1211
|
continue
|
|
752
|
-
if
|
|
1212
|
+
if completed_turn_id is not None and current_turn_id == completed_turn_id:
|
|
753
1213
|
completed_turn_measured = True
|
|
754
1214
|
result.tokens_updated_at = max(result.tokens_updated_at or timestamp, timestamp)
|
|
755
|
-
|
|
1215
|
+
delta["totalTokens"] = delta["inputTokens"] + delta["outputTokens"]
|
|
1216
|
+
if delta["totalTokens"] <= 0 and not refines_classification:
|
|
756
1217
|
continue
|
|
1218
|
+
cache_write_capacity = (
|
|
1219
|
+
_uncached_input(legacy_usage) + delta["inputTokens"]
|
|
1220
|
+
if refines_classification
|
|
1221
|
+
else delta["inputTokens"]
|
|
1222
|
+
)
|
|
1223
|
+
# Late cumulative cache details can classify an earlier legacy turn.
|
|
1224
|
+
completed_capacity = (
|
|
1225
|
+
_uncached_input(completed_legacy_usage)
|
|
1226
|
+
if refines_classification and response_usage is None
|
|
1227
|
+
else 0
|
|
1228
|
+
)
|
|
1229
|
+
if (
|
|
1230
|
+
prefix_unclassified > 0
|
|
1231
|
+
and refines_classification
|
|
1232
|
+
and (
|
|
1233
|
+
any(delta[key] > 0 for key in prefix_missing_cache)
|
|
1234
|
+
# Even explicit counters can revise earlier input when the
|
|
1235
|
+
# newly classified tokens exceed all in-scan capacity.
|
|
1236
|
+
or delta["cachedInputTokens"] + delta["cacheWriteInputTokens"]
|
|
1237
|
+
> cache_write_capacity + completed_capacity
|
|
1238
|
+
)
|
|
1239
|
+
):
|
|
1240
|
+
# The newly classified tokens could have occurred before this scan.
|
|
1241
|
+
ambiguous_legacy_cache = True
|
|
1242
|
+
for key in ("cacheWriteInputTokens", "cachedInputTokens"):
|
|
1243
|
+
current = min(delta[key], cache_write_capacity)
|
|
1244
|
+
completed = min(delta[key] - current, completed_capacity)
|
|
1245
|
+
if completed and ambiguous_legacy_cache:
|
|
1246
|
+
warnings.add("token_breakdown_unavailable")
|
|
1247
|
+
delta[key] = current
|
|
1248
|
+
cache_write_capacity -= current
|
|
1249
|
+
completed_capacity -= completed
|
|
1250
|
+
completed_legacy_usage[key] += completed
|
|
1251
|
+
total[key] += completed
|
|
1252
|
+
delta["reasoningOutputTokens"] = min(
|
|
1253
|
+
delta["reasoningOutputTokens"], delta["outputTokens"]
|
|
1254
|
+
)
|
|
757
1255
|
_add_token_usage(legacy_usage, delta)
|
|
758
1256
|
|
|
759
1257
|
_add_token_usage(total, response_usage if response_usage is not None else legacy_usage)
|
|
1258
|
+
if response_usage is None and ambiguous_legacy_cache:
|
|
1259
|
+
warnings.add("token_breakdown_unavailable")
|
|
760
1260
|
if not boundary_reached:
|
|
761
1261
|
warnings.add("thread_ownership_unavailable")
|
|
762
1262
|
elif result.tokens_updated_at is None and not direct_control_session:
|
|
@@ -827,6 +1327,47 @@ def valid_account_rate_limit(value: object) -> bool:
|
|
|
827
1327
|
return True
|
|
828
1328
|
|
|
829
1329
|
|
|
1330
|
+
def _missing_cache_fields(payload: Mapping[str, Any]) -> set[str]:
|
|
1331
|
+
usage = payload["info"]["total_token_usage"]
|
|
1332
|
+
missing: set[str] = set()
|
|
1333
|
+
if "cached_input_tokens" not in usage:
|
|
1334
|
+
missing.add("cachedInputTokens")
|
|
1335
|
+
if "cache_write_input_tokens" not in usage and "cache_write_tokens" not in usage:
|
|
1336
|
+
missing.add("cacheWriteInputTokens")
|
|
1337
|
+
return missing
|
|
1338
|
+
|
|
1339
|
+
|
|
1340
|
+
def _uncached_input(usage: dict[str, int]) -> int:
|
|
1341
|
+
return usage["inputTokens"] - usage["cachedInputTokens"] - usage["cacheWriteInputTokens"]
|
|
1342
|
+
|
|
1343
|
+
|
|
1344
|
+
def _retain_cache_baseline(
|
|
1345
|
+
previous: dict[str, int], snapshot: dict[str, int], missing: set[str]
|
|
1346
|
+
) -> dict[str, int] | None:
|
|
1347
|
+
baseline = snapshot
|
|
1348
|
+
if (
|
|
1349
|
+
missing
|
|
1350
|
+
and snapshot["inputTokens"] >= previous["inputTokens"]
|
|
1351
|
+
and snapshot["outputTokens"] >= previous["outputTokens"]
|
|
1352
|
+
):
|
|
1353
|
+
baseline = {**snapshot, **{key: previous[key] for key in missing}}
|
|
1354
|
+
if (
|
|
1355
|
+
baseline["cachedInputTokens"] + baseline["cacheWriteInputTokens"]
|
|
1356
|
+
> baseline["inputTokens"]
|
|
1357
|
+
):
|
|
1358
|
+
return None
|
|
1359
|
+
if (
|
|
1360
|
+
baseline["inputTokens"] == previous["inputTokens"]
|
|
1361
|
+
and baseline["outputTokens"] == previous["outputTokens"]
|
|
1362
|
+
and (
|
|
1363
|
+
baseline["cachedInputTokens"] < previous["cachedInputTokens"]
|
|
1364
|
+
or baseline["cacheWriteInputTokens"] < previous["cacheWriteInputTokens"]
|
|
1365
|
+
)
|
|
1366
|
+
):
|
|
1367
|
+
return previous
|
|
1368
|
+
return baseline
|
|
1369
|
+
|
|
1370
|
+
|
|
830
1371
|
def _session_parent_thread_id(payload: Mapping[str, Any]) -> str | None:
|
|
831
1372
|
source = payload.get("source")
|
|
832
1373
|
if isinstance(source, dict):
|
|
@@ -909,6 +1450,12 @@ def _usage_counts(usage: object) -> dict[str, int] | None:
|
|
|
909
1450
|
if result["cachedInputTokens"] + result["cacheWriteInputTokens"] > result["inputTokens"]:
|
|
910
1451
|
return None
|
|
911
1452
|
result["totalTokens"] = result["inputTokens"] + result["outputTokens"]
|
|
1453
|
+
if (
|
|
1454
|
+
usage.get("cache_write_input_tokens_reported") is False
|
|
1455
|
+
or usage.get("cache_write_input_tokens") is None
|
|
1456
|
+
and usage.get("cache_write_tokens") is None
|
|
1457
|
+
):
|
|
1458
|
+
result["cacheWriteInputTokensReported"] = False
|
|
912
1459
|
return result
|
|
913
1460
|
|
|
914
1461
|
|
|
@@ -919,6 +1466,8 @@ def _empty_token_usage() -> dict[str, int]:
|
|
|
919
1466
|
def _add_token_usage(target: dict[str, int], addition: Mapping[str, int]) -> None:
|
|
920
1467
|
for key in TOKEN_FIELDS.values():
|
|
921
1468
|
target[key] += addition[key]
|
|
1469
|
+
if addition.get("cacheWriteInputTokensReported") is False:
|
|
1470
|
+
target["cacheWriteInputTokensReported"] = False
|
|
922
1471
|
|
|
923
1472
|
|
|
924
1473
|
def _timestamp(value: object) -> datetime | None:
|