@openai/codex-security 0.8.3 → 0.9.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +192 -151
- package/_bundled_plugin/.mcp.json +5 -1
- package/_bundled_plugin/examples/completed-scan/findings.json +16 -4
- package/_bundled_plugin/examples/completed-scan/report.md +11 -8
- package/_bundled_plugin/examples/completed-scan/scan-manifest.json +4 -2
- package/_bundled_plugin/mcp/fzstd-LICENSE.txt +21 -0
- package/_bundled_plugin/mcp/helpers.mjs +2 -11
- package/_bundled_plugin/mcp/helpers.mjs.br +0 -0
- package/_bundled_plugin/mcp/native/THIRD_PARTY_NOTICES.txt +3 -210
- package/_bundled_plugin/mcp/native/darwin-arm64/unix.node +0 -0
- package/_bundled_plugin/mcp/native/darwin-x64/unix.node +0 -0
- package/_bundled_plugin/mcp/native/linux-arm64-gnu/unix.node +0 -0
- package/_bundled_plugin/mcp/native/linux-arm64-musl/unix.node +0 -0
- package/_bundled_plugin/mcp/native/linux-x64-gnu/unix.node +0 -0
- package/_bundled_plugin/mcp/native/linux-x64-musl/unix.node +0 -0
- package/_bundled_plugin/mcp/native/win32-arm64/windows.node +0 -0
- package/_bundled_plugin/mcp/native/win32-x64/windows.node +0 -0
- package/_bundled_plugin/mcp/package.json +1 -0
- package/_bundled_plugin/mcp/permission-profile-preflight.mjs +5 -2
- package/_bundled_plugin/mcp/server.mjs +2 -11
- package/_bundled_plugin/mcp/server.mjs.br +0 -0
- package/_bundled_plugin/mcp/session-decompression.mjs +712 -0
- package/_bundled_plugin/references/default-severity-rubric.md +9 -4
- package/_bundled_plugin/references/final-report.md +5 -3
- package/_bundled_plugin/references/finding-detail-fields.md +18 -18
- package/_bundled_plugin/references/sarif-adapter.md +8 -2
- package/_bundled_plugin/references/scan-artifacts.md +21 -12
- package/_bundled_plugin/references/scan-contract.md +27 -33
- package/_bundled_plugin/schemas/tools/scan-draft.schema.json +2 -2
- package/_bundled_plugin/scripts/check_deep_scan_target.py +97 -0
- package/_bundled_plugin/scripts/codex_profile.mjs +7 -1
- package/_bundled_plugin/scripts/deep_scan_config.py +20 -1
- package/_bundled_plugin/scripts/deep_scan_workbench.py +165 -342
- package/_bundled_plugin/scripts/filesystem_identity.py +1 -0
- package/_bundled_plugin/scripts/finalize_scan_contract.py +153 -227
- package/_bundled_plugin/scripts/finding_preview.py +20 -0
- package/_bundled_plugin/scripts/launch_codex_security_mcp +1 -0
- package/_bundled_plugin/scripts/legacy_findings.py +148 -0
- package/_bundled_plugin/scripts/posix_scan_local_files.py +11 -0
- package/_bundled_plugin/scripts/read_codex_session.mjs +21 -0
- package/_bundled_plugin/scripts/report_projection.py +81 -43
- package/_bundled_plugin/scripts/review_state.py +8 -2
- package/_bundled_plugin/scripts/servicenow_avit.py +20 -0
- package/_bundled_plugin/scripts/snapshot_sqlite.py +1 -0
- package/_bundled_plugin/scripts/threat_model_projection.py +22 -26
- package/_bundled_plugin/scripts/validate_scan_contract.py +7 -1
- package/_bundled_plugin/scripts/windows_scan_local_files.py +388 -17
- package/_bundled_plugin/scripts/workbench/__init__.py +1 -1
- package/_bundled_plugin/scripts/workbench/handoff.py +47 -0
- package/_bundled_plugin/scripts/workbench/retained_coverage.py +30 -51
- package/_bundled_plugin/scripts/workbench/retained_records.py +18 -30
- package/_bundled_plugin/scripts/workbench/runtime_toml.py +4 -1
- package/_bundled_plugin/scripts/workbench/saved_coverage.py +9 -17
- package/_bundled_plugin/scripts/workbench/saved_outputs.py +9 -0
- package/_bundled_plugin/scripts/workbench/saved_sources.py +47 -40
- package/_bundled_plugin/scripts/workbench/storage.py +71 -7
- package/_bundled_plugin/scripts/workbench_cli.py +30 -12
- package/_bundled_plugin/scripts/workbench_constants.py +10 -0
- package/_bundled_plugin/scripts/workbench_coverage_receipts.py +3 -3
- package/_bundled_plugin/scripts/workbench_db.py +369 -137
- package/_bundled_plugin/scripts/workbench_feedback.py +8 -7
- package/_bundled_plugin/scripts/workbench_finding_index.py +1 -0
- package/_bundled_plugin/scripts/workbench_finding_workflows.py +4 -0
- package/_bundled_plugin/scripts/workbench_native_indexes.py +12 -12
- package/_bundled_plugin/scripts/workbench_progress.py +10 -1
- package/_bundled_plugin/scripts/workbench_publication.py +14 -10
- package/_bundled_plugin/scripts/workbench_remediation.py +13 -0
- package/_bundled_plugin/scripts/workbench_saved_results.py +132 -77
- package/_bundled_plugin/scripts/workbench_scan_history.py +117 -55
- package/_bundled_plugin/scripts/workbench_scan_start.py +83 -24
- package/_bundled_plugin/scripts/workbench_scan_usage.py +644 -68
- package/_bundled_plugin/scripts/workbench_schema.py +44 -55
- package/_bundled_plugin/scripts/workbench_setup.py +13 -1
- package/_bundled_plugin/scripts/workbench_source_excerpt.py +13 -0
- package/_bundled_plugin/scripts/workbench_source_scopes.py +18 -0
- package/_bundled_plugin/scripts/workbench_target.py +127 -56
- package/_bundled_plugin/scripts/workbench_target_state.py +3 -0
- package/_bundled_plugin/scripts/workbench_validation.py +43 -3
- package/_bundled_plugin/shared/workbench-history-repairs.json +19 -23
- package/_bundled_plugin/shared/workbench-migrations.json +34 -0
- package/_bundled_plugin/skills/cloud-findings/SKILL.md +5 -5
- package/_bundled_plugin/skills/deep-security-scan/SKILL.md +7 -50
- package/_bundled_plugin/skills/fix-finding/SKILL.md +1 -1
- package/_bundled_plugin/skills/publish-findings/SKILL.md +32 -25
- package/_bundled_plugin/skills/publish-findings/references/checkmarx-import.md +37 -0
- package/_bundled_plugin/skills/publish-findings/references/wiz-import.md +3 -3
- package/_bundled_plugin/skills/publish-findings/scripts/collect_checkmarx.py +464 -0
- package/_bundled_plugin/skills/threat-model-visualize/scripts/build_explorer.py +22 -1
- package/_bundled_plugin/skills/triage-finding/references/ticket-intake.md +6 -2
- package/dist/api.d.ts +39 -4
- package/dist/api.d.ts.map +1 -1
- package/dist/api.js +774 -292
- package/dist/api.js.map +1 -1
- package/dist/auth.d.ts.map +1 -1
- package/dist/auth.js +4 -6
- package/dist/auth.js.map +1 -1
- package/dist/bedrock-oss-pricing.d.ts +6 -0
- package/dist/bedrock-oss-pricing.d.ts.map +1 -0
- package/dist/bedrock-oss-pricing.js +58 -0
- package/dist/bedrock-oss-pricing.js.map +1 -0
- package/dist/bedrock-pricing.d.ts +15 -0
- package/dist/bedrock-pricing.d.ts.map +1 -0
- package/dist/bedrock-pricing.js +117 -0
- package/dist/bedrock-pricing.js.map +1 -0
- package/dist/blackduck-findings.d.ts.map +1 -1
- package/dist/blackduck-findings.js +35 -20
- package/dist/blackduck-findings.js.map +1 -1
- package/dist/bulk-scan-discovery.d.ts.map +1 -1
- package/dist/bulk-scan-discovery.js +4 -4
- package/dist/bulk-scan-discovery.js.map +1 -1
- package/dist/checkmarx-evidence.d.ts +19 -0
- package/dist/checkmarx-evidence.d.ts.map +1 -0
- package/dist/checkmarx-evidence.js +36 -0
- package/dist/checkmarx-evidence.js.map +1 -0
- package/dist/checkmarx-findings.d.ts +4 -0
- package/dist/checkmarx-findings.d.ts.map +1 -0
- package/dist/checkmarx-findings.js +27 -0
- package/dist/checkmarx-findings.js.map +1 -0
- package/dist/checkmarx-one.d.ts +4 -0
- package/dist/checkmarx-one.d.ts.map +1 -0
- package/dist/checkmarx-one.js +53 -0
- package/dist/checkmarx-one.js.map +1 -0
- package/dist/checkmarx-repository.d.ts +12 -0
- package/dist/checkmarx-repository.d.ts.map +1 -0
- package/dist/checkmarx-repository.js +181 -0
- package/dist/checkmarx-repository.js.map +1 -0
- package/dist/checkmarx-xml.d.ts +12 -0
- package/dist/checkmarx-xml.d.ts.map +1 -0
- package/dist/checkmarx-xml.js +196 -0
- package/dist/checkmarx-xml.js.map +1 -0
- package/dist/classify-severity-records.d.ts +1 -9
- package/dist/classify-severity-records.d.ts.map +1 -1
- package/dist/classify-severity-records.js +3 -13
- package/dist/classify-severity-records.js.map +1 -1
- package/dist/classify-severity.d.ts.map +1 -1
- package/dist/classify-severity.js +2 -4
- package/dist/classify-severity.js.map +1 -1
- package/dist/cli.d.ts +11 -5
- package/dist/cli.d.ts.map +1 -1
- package/dist/cli.js +1734 -1238
- package/dist/cli.js.map +1 -1
- package/dist/cloud-publish.js +2 -2
- package/dist/cloud-publish.js.map +1 -1
- package/dist/component-scan.d.ts +16 -3
- package/dist/component-scan.d.ts.map +1 -1
- package/dist/component-scan.js +27 -2
- package/dist/component-scan.js.map +1 -1
- package/dist/config.d.ts.map +1 -1
- package/dist/config.js +100 -66
- package/dist/config.js.map +1 -1
- package/dist/continuum-findings.d.ts.map +1 -1
- package/dist/continuum-findings.js +80 -12
- package/dist/continuum-findings.js.map +1 -1
- package/dist/contract-values.d.ts +2 -0
- package/dist/contract-values.d.ts.map +1 -1
- package/dist/contract-values.js +12 -9
- package/dist/contract-values.js.map +1 -1
- package/dist/cost-model.d.ts +13 -1
- package/dist/cost-model.d.ts.map +1 -1
- package/dist/cost-model.js +66 -112
- package/dist/cost-model.js.map +1 -1
- package/dist/cost.d.ts +42 -3
- package/dist/cost.d.ts.map +1 -1
- package/dist/cost.js +483 -104
- package/dist/cost.js.map +1 -1
- package/dist/custom-publish.d.ts +4 -0
- package/dist/custom-publish.d.ts.map +1 -1
- package/dist/custom-publish.js +40 -3
- package/dist/custom-publish.js.map +1 -1
- package/dist/deduplication/local.d.ts +2 -2
- package/dist/deduplication/local.d.ts.map +1 -1
- package/dist/deduplication/local.js +2 -2
- package/dist/deduplication/local.js.map +1 -1
- package/dist/deduplication/refusal.d.ts.map +1 -1
- package/dist/deduplication/refusal.js +25 -5
- package/dist/deduplication/refusal.js.map +1 -1
- package/dist/deduplication/scan.d.ts +3 -0
- package/dist/deduplication/scan.d.ts.map +1 -1
- package/dist/deduplication/scan.js +1 -1
- package/dist/deduplication/scan.js.map +1 -1
- package/dist/deep-config.d.ts +0 -1
- package/dist/deep-config.d.ts.map +1 -1
- package/dist/deep-config.js +22 -33
- package/dist/deep-config.js.map +1 -1
- package/dist/deep-scan-defaults.d.ts +2 -0
- package/dist/deep-scan-defaults.d.ts.map +1 -1
- package/dist/deep-scan-defaults.js +28 -1
- package/dist/deep-scan-defaults.js.map +1 -1
- package/dist/deep-scan.d.ts +2 -0
- package/dist/deep-scan.d.ts.map +1 -1
- package/dist/deep-scan.js +28 -4
- package/dist/deep-scan.js.map +1 -1
- package/dist/default-severity-rubric.d.ts +1 -1
- package/dist/default-severity-rubric.d.ts.map +1 -1
- package/dist/default-severity-rubric.js +1 -1
- package/dist/default-severity-rubric.js.map +1 -1
- package/dist/endor-findings.d.ts +3 -1
- package/dist/endor-findings.d.ts.map +1 -1
- package/dist/endor-findings.js +103 -45
- package/dist/endor-findings.js.map +1 -1
- package/dist/errors.d.ts +1 -1
- package/dist/errors.d.ts.map +1 -1
- package/dist/errors.js +8 -1
- package/dist/errors.js.map +1 -1
- package/dist/external-findings-import.d.ts +5 -3
- package/dist/external-findings-import.d.ts.map +1 -1
- package/dist/external-findings-import.js +7 -6
- package/dist/external-findings-import.js.map +1 -1
- package/dist/external-findings-local.d.ts +7 -5
- package/dist/external-findings-local.d.ts.map +1 -1
- package/dist/external-findings-local.js +337 -89
- package/dist/external-findings-local.js.map +1 -1
- package/dist/external-findings-publish.d.ts +8 -6
- package/dist/external-findings-publish.d.ts.map +1 -1
- package/dist/external-findings-publish.js +1318 -258
- package/dist/external-findings-publish.js.map +1 -1
- package/dist/external-findings-read.d.ts +20 -2
- package/dist/external-findings-read.d.ts.map +1 -1
- package/dist/external-findings-read.js +148 -66
- package/dist/external-findings-read.js.map +1 -1
- package/dist/external-findings.d.ts +15 -3
- package/dist/external-findings.d.ts.map +1 -1
- package/dist/external-findings.js +42 -1
- package/dist/external-findings.js.map +1 -1
- package/dist/external-import-contract.d.ts +5 -1
- package/dist/external-import-contract.d.ts.map +1 -1
- package/dist/external-import-contract.js +23 -9
- package/dist/external-import-contract.js.map +1 -1
- package/dist/external-import-models.d.ts +1 -1
- package/dist/external-import-models.d.ts.map +1 -1
- package/dist/external-import-models.js.map +1 -1
- package/dist/feedback.d.ts.map +1 -1
- package/dist/feedback.js +5 -4
- package/dist/feedback.js.map +1 -1
- package/dist/finding-prompt.d.ts +5 -0
- package/dist/finding-prompt.d.ts.map +1 -0
- package/dist/finding-prompt.js +32 -0
- package/dist/finding-prompt.js.map +1 -0
- package/dist/findings-client.d.ts +1 -0
- package/dist/findings-client.d.ts.map +1 -1
- package/dist/findings-client.js +27 -3
- package/dist/findings-client.js.map +1 -1
- package/dist/github-advisories.d.ts.map +1 -1
- package/dist/github-advisories.js +0 -2
- package/dist/github-advisories.js.map +1 -1
- package/dist/github-dependabot-findings.d.ts.map +1 -1
- package/dist/github-dependabot-findings.js +15 -4
- package/dist/github-dependabot-findings.js.map +1 -1
- package/dist/github-findings.d.ts +4 -1
- package/dist/github-findings.d.ts.map +1 -1
- package/dist/github-findings.js +55 -30
- package/dist/github-findings.js.map +1 -1
- package/dist/hosted-git-remotes.d.ts +5 -0
- package/dist/hosted-git-remotes.d.ts.map +1 -0
- package/dist/hosted-git-remotes.js +285 -0
- package/dist/hosted-git-remotes.js.map +1 -0
- package/dist/hosted-git-repository.d.ts +28 -0
- package/dist/hosted-git-repository.d.ts.map +1 -0
- package/dist/hosted-git-repository.js +222 -0
- package/dist/hosted-git-repository.js.map +1 -0
- package/dist/hosted-scan-protocol.js +2 -2
- package/dist/hosted-scan-protocol.js.map +1 -1
- package/dist/hosted-scan.d.ts.map +1 -1
- package/dist/hosted-scan.js +16 -11
- package/dist/hosted-scan.js.map +1 -1
- package/dist/hosted-scanner-http.d.ts +21 -0
- package/dist/hosted-scanner-http.d.ts.map +1 -0
- package/dist/hosted-scanner-http.js +204 -0
- package/dist/hosted-scanner-http.js.map +1 -0
- package/dist/hosted-scanner-v1.d.ts +6 -0
- package/dist/hosted-scanner-v1.d.ts.map +1 -0
- package/dist/hosted-scanner-v1.js +60 -0
- package/dist/hosted-scanner-v1.js.map +1 -0
- package/dist/import-scan.d.ts.map +1 -1
- package/dist/import-scan.js +3 -0
- package/dist/import-scan.js.map +1 -1
- package/dist/index.d.ts +18 -7
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +10 -2
- package/dist/index.js.map +1 -1
- package/dist/inspector-findings.d.ts.map +1 -1
- package/dist/inspector-findings.js +47 -65
- package/dist/inspector-findings.js.map +1 -1
- package/dist/knowledge-base.d.ts.map +1 -1
- package/dist/knowledge-base.js +68 -5
- package/dist/knowledge-base.js.map +1 -1
- package/dist/local-git-metadata.d.ts +1 -1
- package/dist/local-git-metadata.d.ts.map +1 -1
- package/dist/local-git-metadata.js +1 -2
- package/dist/local-git-metadata.js.map +1 -1
- package/dist/multiscan.d.ts +6 -4
- package/dist/multiscan.d.ts.map +1 -1
- package/dist/multiscan.js +138 -65
- package/dist/multiscan.js.map +1 -1
- package/dist/native-json-findings.d.ts +2 -1
- package/dist/native-json-findings.d.ts.map +1 -1
- package/dist/native-json-findings.js +12 -2
- package/dist/native-json-findings.js.map +1 -1
- package/dist/opengrep-findings.d.ts +1 -0
- package/dist/opengrep-findings.d.ts.map +1 -1
- package/dist/opengrep-findings.js +17 -13
- package/dist/opengrep-findings.js.map +1 -1
- package/dist/operation-telemetry.d.ts +104 -0
- package/dist/operation-telemetry.d.ts.map +1 -0
- package/dist/operation-telemetry.js +293 -0
- package/dist/operation-telemetry.js.map +1 -0
- package/dist/plugin-python-probe.d.ts +4 -0
- package/dist/plugin-python-probe.d.ts.map +1 -0
- package/dist/plugin-python-probe.js +10 -0
- package/dist/plugin-python-probe.js.map +1 -0
- package/dist/private-finding-import.d.ts +97 -2
- package/dist/private-finding-import.d.ts.map +1 -1
- package/dist/private-finding-import.js +58 -34
- package/dist/private-finding-import.js.map +1 -1
- package/dist/provider-profile.d.ts +1 -1
- package/dist/provider-profile.d.ts.map +1 -1
- package/dist/provider-profile.js +8 -1
- package/dist/provider-profile.js.map +1 -1
- package/dist/publication-telemetry.d.ts +45 -0
- package/dist/publication-telemetry.d.ts.map +1 -0
- package/dist/publication-telemetry.js +50 -0
- package/dist/publication-telemetry.js.map +1 -0
- package/dist/publish.d.ts +4 -0
- package/dist/publish.d.ts.map +1 -1
- package/dist/publish.js +42 -2
- package/dist/publish.js.map +1 -1
- package/dist/reader-selection.d.ts +2 -0
- package/dist/reader-selection.d.ts.map +1 -1
- package/dist/reader-selection.js +8 -0
- package/dist/reader-selection.js.map +1 -1
- package/dist/remediation.d.ts +1 -4
- package/dist/remediation.d.ts.map +1 -1
- package/dist/remediation.js +3 -7
- package/dist/remediation.js.map +1 -1
- package/dist/request-metadata.d.ts +3 -0
- package/dist/request-metadata.d.ts.map +1 -1
- package/dist/request-metadata.js +9 -0
- package/dist/request-metadata.js.map +1 -1
- package/dist/result.d.ts +4 -0
- package/dist/result.d.ts.map +1 -1
- package/dist/result.js +1 -1
- package/dist/result.js.map +1 -1
- package/dist/retry-windows-rename.d.ts +8 -0
- package/dist/retry-windows-rename.d.ts.map +1 -0
- package/dist/retry-windows-rename.js +24 -0
- package/dist/retry-windows-rename.js.map +1 -0
- package/dist/runtime.d.ts +20 -0
- package/dist/runtime.d.ts.map +1 -1
- package/dist/runtime.js +313 -110
- package/dist/runtime.js.map +1 -1
- package/dist/saved-scan-bootstrap.d.ts +1 -0
- package/dist/saved-scan-bootstrap.d.ts.map +1 -1
- package/dist/saved-scan-bootstrap.js +18 -15
- package/dist/saved-scan-bootstrap.js.map +1 -1
- package/dist/scan-activity.d.ts.map +1 -1
- package/dist/scan-activity.js +141 -56
- package/dist/scan-activity.js.map +1 -1
- package/dist/scan-comparison.d.ts +10 -5
- package/dist/scan-comparison.d.ts.map +1 -1
- package/dist/scan-comparison.js +60 -15
- package/dist/scan-comparison.js.map +1 -1
- package/dist/scan-dashboard.d.ts +6 -6
- package/dist/scan-dashboard.d.ts.map +1 -1
- package/dist/scan-dashboard.js +75 -42
- package/dist/scan-dashboard.js.map +1 -1
- package/dist/scan-executor.d.ts +10 -1
- package/dist/scan-executor.d.ts.map +1 -1
- package/dist/scan-executor.js +22 -2
- package/dist/scan-executor.js.map +1 -1
- package/dist/scan-history-renderer.d.ts.map +1 -1
- package/dist/scan-history-renderer.js +9 -19
- package/dist/scan-history-renderer.js.map +1 -1
- package/dist/scan-logs.d.ts +12 -0
- package/dist/scan-logs.d.ts.map +1 -1
- package/dist/scan-logs.js +106 -77
- package/dist/scan-logs.js.map +1 -1
- package/dist/scan-measurement.d.ts +26 -0
- package/dist/scan-measurement.d.ts.map +1 -0
- package/dist/scan-measurement.js +77 -0
- package/dist/scan-measurement.js.map +1 -0
- package/dist/scan-progress.d.ts +1 -0
- package/dist/scan-progress.d.ts.map +1 -1
- package/dist/scan-progress.js +3 -0
- package/dist/scan-progress.js.map +1 -1
- package/dist/scan-sessions.d.ts.map +1 -1
- package/dist/scan-sessions.js +4 -1
- package/dist/scan-sessions.js.map +1 -1
- package/dist/scan-settings.d.ts +1 -1
- package/dist/scan-settings.d.ts.map +1 -1
- package/dist/scan-settings.js +1 -24
- package/dist/scan-settings.js.map +1 -1
- package/dist/scanner-import.d.ts +20 -2
- package/dist/scanner-import.d.ts.map +1 -1
- package/dist/scanner-import.js +200 -8
- package/dist/scanner-import.js.map +1 -1
- package/dist/scanner-repository-url.d.ts +4 -1
- package/dist/scanner-repository-url.d.ts.map +1 -1
- package/dist/scanner-repository-url.js +10 -6
- package/dist/scanner-repository-url.js.map +1 -1
- package/dist/security-policy.d.ts +1 -1
- package/dist/security-policy.d.ts.map +1 -1
- package/dist/security-policy.js.map +1 -1
- package/dist/semgrep-findings.d.ts +6 -3
- package/dist/semgrep-findings.d.ts.map +1 -1
- package/dist/semgrep-findings.js +107 -43
- package/dist/semgrep-findings.js.map +1 -1
- package/dist/semgrep-guardian-findings.d.ts +38 -0
- package/dist/semgrep-guardian-findings.d.ts.map +1 -0
- package/dist/semgrep-guardian-findings.js +165 -0
- package/dist/semgrep-guardian-findings.js.map +1 -0
- package/dist/semgrep-platform-collector.d.ts +26 -0
- package/dist/semgrep-platform-collector.d.ts.map +1 -0
- package/dist/semgrep-platform-collector.js +108 -0
- package/dist/semgrep-platform-collector.js.map +1 -0
- package/dist/semgrep-platform-findings.d.ts +207 -6
- package/dist/semgrep-platform-findings.d.ts.map +1 -1
- package/dist/semgrep-platform-findings.js +983 -103
- package/dist/semgrep-platform-findings.js.map +1 -1
- package/dist/semgrep-platform-v1.d.ts +6 -0
- package/dist/semgrep-platform-v1.d.ts.map +1 -0
- package/dist/semgrep-platform-v1.js +83 -0
- package/dist/semgrep-platform-v1.js.map +1 -0
- package/dist/server/api.d.ts +1 -1
- package/dist/server/api.d.ts.map +1 -1
- package/dist/server/api.js.map +1 -1
- package/dist/server/embedding-tokenizer.cdx.json +33 -0
- package/dist/server/embedding-tokenizer.d.ts +2 -0
- package/dist/server/embedding-tokenizer.d.ts.map +1 -0
- package/dist/server/embedding-tokenizer.js +322 -0
- package/dist/server/embedding-tokenizer.js.map +6 -0
- package/dist/server/embeddings.d.ts +16 -2
- package/dist/server/embeddings.d.ts.map +1 -1
- package/dist/server/embeddings.js +35 -5
- package/dist/server/embeddings.js.map +1 -1
- package/dist/session-lines.d.ts +5 -0
- package/dist/session-lines.d.ts.map +1 -0
- package/dist/session-lines.js +60 -0
- package/dist/session-lines.js.map +1 -0
- package/dist/severity-review.d.ts +3 -1
- package/dist/severity-review.d.ts.map +1 -1
- package/dist/severity-review.js +3 -2
- package/dist/severity-review.js.map +1 -1
- package/dist/skill-authentication.d.ts +1 -0
- package/dist/skill-authentication.d.ts.map +1 -1
- package/dist/skill-authentication.js +1 -0
- package/dist/skill-authentication.js.map +1 -1
- package/dist/snyk-findings.d.ts +7 -2
- package/dist/snyk-findings.d.ts.map +1 -1
- package/dist/snyk-findings.js +93 -70
- package/dist/snyk-findings.js.map +1 -1
- package/dist/snyk-rest-findings.d.ts +76 -0
- package/dist/snyk-rest-findings.d.ts.map +1 -0
- package/dist/snyk-rest-findings.js +1051 -0
- package/dist/snyk-rest-findings.js.map +1 -0
- package/dist/snyk-rest-v1.d.ts +28 -0
- package/dist/snyk-rest-v1.d.ts.map +1 -0
- package/dist/snyk-rest-v1.js +132 -0
- package/dist/snyk-rest-v1.js.map +1 -0
- package/dist/snyk-token-origin.d.ts +4 -0
- package/dist/snyk-token-origin.d.ts.map +1 -0
- package/dist/snyk-token-origin.js +82 -0
- package/dist/snyk-token-origin.js.map +1 -0
- package/dist/sonar-collector.d.ts.map +1 -1
- package/dist/sonar-collector.js +0 -3
- package/dist/sonar-collector.js.map +1 -1
- package/dist/sonar-import.d.ts.map +1 -1
- package/dist/sonar-import.js +2 -4
- package/dist/sonar-import.js.map +1 -1
- package/dist/target-drift.d.ts +9 -0
- package/dist/target-drift.d.ts.map +1 -0
- package/dist/target-drift.js +27 -0
- package/dist/target-drift.js.map +1 -0
- package/dist/targets.d.ts +2 -0
- package/dist/targets.d.ts.map +1 -1
- package/dist/targets.js +56 -20
- package/dist/targets.js.map +1 -1
- package/dist/tenable-findings.d.ts +9 -0
- package/dist/tenable-findings.d.ts.map +1 -0
- package/dist/tenable-findings.js +122 -0
- package/dist/tenable-findings.js.map +1 -0
- package/dist/terminal-stream.d.ts +13 -0
- package/dist/terminal-stream.d.ts.map +1 -0
- package/dist/terminal-stream.js +35 -0
- package/dist/terminal-stream.js.map +1 -0
- package/dist/trivy-findings.d.ts +19 -0
- package/dist/trivy-findings.d.ts.map +1 -0
- package/dist/trivy-findings.js +435 -0
- package/dist/trivy-findings.js.map +1 -0
- package/dist/trivy-import.d.ts +19 -0
- package/dist/trivy-import.d.ts.map +1 -0
- package/dist/trivy-import.js +151 -0
- package/dist/trivy-import.js.map +1 -0
- package/dist/validation-records.js +1 -1
- package/dist/validation-records.js.map +1 -1
- package/dist/veracode-findings.d.ts +2 -0
- package/dist/veracode-findings.d.ts.map +1 -1
- package/dist/veracode-findings.js +49 -11
- package/dist/veracode-findings.js.map +1 -1
- package/dist/vertex-pricing.d.ts +8 -0
- package/dist/vertex-pricing.d.ts.map +1 -0
- package/dist/vertex-pricing.js +20 -0
- package/dist/vertex-pricing.js.map +1 -0
- package/dist/windows-path.d.ts +1 -0
- package/dist/windows-path.d.ts.map +1 -1
- package/dist/windows-path.js +26 -1
- package/dist/windows-path.js.map +1 -1
- package/dist/wiz-findings.d.ts +4 -0
- package/dist/wiz-findings.d.ts.map +1 -1
- package/dist/wiz-findings.js +57 -3
- package/dist/wiz-findings.js.map +1 -1
- package/docs/classify-severity-records.md +27 -12
- package/docs/cli.md +497 -550
- package/docs/dedupe-records.md +44 -24
- package/docs/findings-service.md +66 -39
- package/docs/hosted-remediation.md +14 -7
- package/docs/hosted-scans.md +50 -42
- package/docs/telemetry.md +236 -0
- package/docs/validate-records.md +78 -44
- package/package.json +18 -15
- package/schemas/external-findings.schema.json +3 -1
- package/_bundled_plugin/mcp/helpers.mjs.br.part-000 +0 -0
- package/_bundled_plugin/mcp/helpers.mjs.br.part-001 +0 -0
- package/_bundled_plugin/mcp/server.mjs.br.part-000 +0 -0
- package/_bundled_plugin/mcp/server.mjs.br.part-001 +0 -0
- package/_bundled_plugin/mcp/server.mjs.br.part-002 +0 -0
- package/_bundled_plugin/schemas/tools/deep-reducer.schema.json +0 -96
- package/dist/external-imports-feature.d.ts +0 -6
- package/dist/external-imports-feature.d.ts.map +0 -1
- package/dist/external-imports-feature.js +0 -51
- package/dist/external-imports-feature.js.map +0 -1
- package/dist/finding-normalization.d.ts +0 -5
- package/dist/finding-normalization.d.ts.map +0 -1
- package/dist/finding-normalization.js +0 -155
- package/dist/finding-normalization.js.map +0 -1
|
@@ -8,14 +8,20 @@ import math
|
|
|
8
8
|
import os
|
|
9
9
|
import re
|
|
10
10
|
import sqlite3
|
|
11
|
+
import subprocess
|
|
11
12
|
import sys
|
|
13
|
+
import tempfile
|
|
12
14
|
import uuid
|
|
15
|
+
from contextlib import contextmanager
|
|
13
16
|
from dataclasses import dataclass
|
|
14
17
|
from datetime import datetime, timezone
|
|
15
18
|
from pathlib import Path
|
|
16
19
|
from typing import Any, Mapping
|
|
17
20
|
from urllib.parse import quote
|
|
18
21
|
|
|
22
|
+
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
|
23
|
+
from workbench.storage import worker_usage_directory
|
|
24
|
+
|
|
19
25
|
TOKEN_FIELDS = {
|
|
20
26
|
"input_tokens": "inputTokens",
|
|
21
27
|
"cached_input_tokens": "cachedInputTokens",
|
|
@@ -30,6 +36,8 @@ STATE_DATABASE_TIMEOUT_SECONDS = 1.0
|
|
|
30
36
|
|
|
31
37
|
@dataclass(frozen=True)
|
|
32
38
|
class RolloutSession:
|
|
39
|
+
"""Associate a thread and its optional parent with the path of the rollout used for usage accounting."""
|
|
40
|
+
|
|
33
41
|
thread_id: str
|
|
34
42
|
parent_thread_id: str | None
|
|
35
43
|
path: Path
|
|
@@ -37,6 +45,8 @@ class RolloutSession:
|
|
|
37
45
|
|
|
38
46
|
@dataclass
|
|
39
47
|
class RolloutUsage:
|
|
48
|
+
"""Collect token counts, their update time, warnings, and account rate-limit observations for a rollout."""
|
|
49
|
+
|
|
40
50
|
tokens: dict[str, int]
|
|
41
51
|
warnings: set[str]
|
|
42
52
|
tokens_updated_at: datetime | None
|
|
@@ -66,13 +76,27 @@ def reconcile_completed_scan_cost(
|
|
|
66
76
|
) -> None:
|
|
67
77
|
"""Persist authoritative SDK cost without discarding measured worker usage."""
|
|
68
78
|
|
|
69
|
-
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
79
|
+
stored = stored_scan_cost_fields(scan["cost_json"])
|
|
80
|
+
completion = stored.get("usage", {}).get("completion")
|
|
81
|
+
native = collect_scan_usage_safely(
|
|
82
|
+
connection,
|
|
83
|
+
scan,
|
|
84
|
+
thread_id=completion["threadId"] if completion else None,
|
|
85
|
+
completed_turn_id=completion["turnId"] if completion and completion["finalized"] else None,
|
|
86
|
+
)
|
|
87
|
+
fields = merge_scan_usage_fields(stored, stored_scan_cost_fields(cost_json), {"usage": native})
|
|
88
|
+
if completion:
|
|
89
|
+
usage = {**fields["usage"], "completion": completion}
|
|
90
|
+
if not completion["finalized"]:
|
|
91
|
+
if usage["coverage"] == "complete":
|
|
92
|
+
usage["coverage"] = "partial"
|
|
93
|
+
usage["warnings"] = sorted({*usage.get("warnings", []), "turn_in_progress"})
|
|
94
|
+
fields["usage"] = usage
|
|
95
|
+
cost_json = json.dumps(
|
|
96
|
+
fields if "usage" in fields else fields["cost"],
|
|
97
|
+
separators=(",", ":"),
|
|
98
|
+
allow_nan=False,
|
|
99
|
+
)
|
|
76
100
|
connection.execute("BEGIN IMMEDIATE")
|
|
77
101
|
with connection:
|
|
78
102
|
connection.execute(
|
|
@@ -81,52 +105,219 @@ def reconcile_completed_scan_cost(
|
|
|
81
105
|
)
|
|
82
106
|
|
|
83
107
|
|
|
108
|
+
def merge_scan_usage_fields(*observations: Mapping[str, Any]) -> dict[str, Any]:
|
|
109
|
+
"""Reconcile overlapping measurements without summing the same execution twice."""
|
|
110
|
+
|
|
111
|
+
merged: dict[str, Any] = {}
|
|
112
|
+
latest_coverage: dict[str, Any] | None = None
|
|
113
|
+
rate_limits: dict[str, dict[str, Any]] = {}
|
|
114
|
+
for observation in observations:
|
|
115
|
+
if isinstance(observation.get("cost"), dict):
|
|
116
|
+
merged["cost"] = observation["cost"]
|
|
117
|
+
usage = observation.get("usage")
|
|
118
|
+
if not isinstance(usage, dict):
|
|
119
|
+
continue
|
|
120
|
+
for snapshot in usage.get("accountRateLimits", []):
|
|
121
|
+
key = snapshot["limitId"]
|
|
122
|
+
if key not in rate_limits or snapshot["observedAt"] >= rate_limits[key]["observedAt"]:
|
|
123
|
+
rate_limits[key] = snapshot
|
|
124
|
+
previous = merged.get("usage")
|
|
125
|
+
if previous is None or _usage_quality(usage) >= _usage_quality(previous):
|
|
126
|
+
merged["usage"] = usage
|
|
127
|
+
if usage.get("coverage") not in ("complete", "partial"):
|
|
128
|
+
continue
|
|
129
|
+
latest_coverage = (
|
|
130
|
+
usage if latest_coverage is None else _merge_usage_coverage(latest_coverage, usage)
|
|
131
|
+
)
|
|
132
|
+
cost = merged.get("cost")
|
|
133
|
+
usage = merged.get("usage")
|
|
134
|
+
if usage is not None and latest_coverage is not None and latest_coverage is not usage:
|
|
135
|
+
usage = {**usage}
|
|
136
|
+
for field in (
|
|
137
|
+
"coverage",
|
|
138
|
+
"coverageScope",
|
|
139
|
+
"threadCount",
|
|
140
|
+
"missingThreadCount",
|
|
141
|
+
"warnings",
|
|
142
|
+
"observedAt",
|
|
143
|
+
"completion",
|
|
144
|
+
"byRole",
|
|
145
|
+
):
|
|
146
|
+
if field in latest_coverage:
|
|
147
|
+
usage[field] = latest_coverage[field]
|
|
148
|
+
else:
|
|
149
|
+
usage.pop(field, None)
|
|
150
|
+
# Counts describe the coverage observation, even when its scope narrows.
|
|
151
|
+
# Its whole role breakdown must still sum to the retained token subtotal.
|
|
152
|
+
if "byRole" in usage and any(
|
|
153
|
+
usage.get(field) != latest_coverage.get(field) for field in TOKEN_FIELDS.values()
|
|
154
|
+
):
|
|
155
|
+
usage.pop("byRole")
|
|
156
|
+
merged["usage"] = usage
|
|
157
|
+
cost_warning = "cost_estimate_covers_earlier_usage"
|
|
158
|
+
if (
|
|
159
|
+
cost is not None
|
|
160
|
+
and usage is not None
|
|
161
|
+
and usage.get("coverage") in ("complete", "partial")
|
|
162
|
+
and type(usage.get("totalTokens")) is int
|
|
163
|
+
):
|
|
164
|
+
warnings = set(usage.get("warnings", []))
|
|
165
|
+
warnings.discard(cost_warning)
|
|
166
|
+
if usage["totalTokens"] > cost["inputTokens"] + cost["outputTokens"]:
|
|
167
|
+
warnings.add(cost_warning)
|
|
168
|
+
if warnings != set(usage.get("warnings", [])):
|
|
169
|
+
updated = {**usage}
|
|
170
|
+
updated["coverage"] = (
|
|
171
|
+
"partial"
|
|
172
|
+
if warnings - {cost_warning} or usage.get("missingThreadCount", 0)
|
|
173
|
+
else "complete"
|
|
174
|
+
)
|
|
175
|
+
if warnings:
|
|
176
|
+
updated["warnings"] = sorted(warnings)
|
|
177
|
+
else:
|
|
178
|
+
updated.pop("warnings", None)
|
|
179
|
+
merged["usage"] = updated
|
|
180
|
+
if rate_limits:
|
|
181
|
+
merged["usage"] = {
|
|
182
|
+
**merged["usage"],
|
|
183
|
+
"accountRateLimits": list(rate_limits.values()),
|
|
184
|
+
}
|
|
185
|
+
return merged
|
|
186
|
+
|
|
187
|
+
|
|
188
|
+
def _merge_usage_coverage(previous: dict[str, Any], observed: dict[str, Any]) -> dict[str, Any]:
|
|
189
|
+
if previous == observed:
|
|
190
|
+
return observed
|
|
191
|
+
previous_at = _timestamp(previous.get("observedAt"))
|
|
192
|
+
observed_at = _timestamp(observed.get("observedAt"))
|
|
193
|
+
if previous_at is not None and observed_at is not None and previous_at > observed_at:
|
|
194
|
+
previous, observed = observed, previous
|
|
195
|
+
previous_scope = previous.get("coverageScope")
|
|
196
|
+
observed_scope = observed.get("coverageScope")
|
|
197
|
+
covers_previous = (
|
|
198
|
+
previous_scope is not None
|
|
199
|
+
and observed_scope is not None
|
|
200
|
+
and set(previous_scope).issubset(observed_scope)
|
|
201
|
+
)
|
|
202
|
+
scope_gap = previous_scope is not None and observed_scope is not None and not covers_previous
|
|
203
|
+
unresolved_previous = (previous["coverage"] == "partial" and not covers_previous) or scope_gap
|
|
204
|
+
merged = {**observed}
|
|
205
|
+
if unresolved_previous:
|
|
206
|
+
merged["coverage"] = "partial"
|
|
207
|
+
merged["missingThreadCount"] = max(
|
|
208
|
+
previous.get("missingThreadCount", 0), observed.get("missingThreadCount", 0)
|
|
209
|
+
)
|
|
210
|
+
warnings = set(previous.get("warnings", [])) | set(observed.get("warnings", []))
|
|
211
|
+
if scope_gap:
|
|
212
|
+
# A newer subset can grow while still staying below the older
|
|
213
|
+
# aggregate. Neither subtotal proves complete usage for the union.
|
|
214
|
+
warnings.add("usage_scope_incomplete")
|
|
215
|
+
if warnings:
|
|
216
|
+
merged["warnings"] = sorted(warnings)
|
|
217
|
+
if previous.get("completion", {}).get("finalized") is False:
|
|
218
|
+
merged["completion"] = previous["completion"]
|
|
219
|
+
if previous_scope is not None and observed_scope is not None:
|
|
220
|
+
merged["coverageScope"] = sorted(set(previous_scope) | set(observed_scope))
|
|
221
|
+
elif previous_scope is not None and observed["coverage"] == "complete":
|
|
222
|
+
merged["coverageScope"] = previous_scope
|
|
223
|
+
elif unresolved_previous:
|
|
224
|
+
# An older subtotal without scope cannot prove which gaps a later
|
|
225
|
+
# collector has resolved. Retain that uncertainty across future merges.
|
|
226
|
+
merged.pop("coverageScope", None)
|
|
227
|
+
return merged
|
|
228
|
+
|
|
229
|
+
|
|
230
|
+
def _usage_quality(
|
|
231
|
+
usage: Mapping[str, Any], *, observed_at: datetime | None = None
|
|
232
|
+
) -> tuple[int, bool, datetime]:
|
|
233
|
+
return (
|
|
234
|
+
usage.get("totalTokens", -1),
|
|
235
|
+
usage.get("cacheWriteInputTokensReported") is not False,
|
|
236
|
+
observed_at
|
|
237
|
+
or _timestamp(usage.get("observedAt"))
|
|
238
|
+
or datetime.min.replace(tzinfo=timezone.utc),
|
|
239
|
+
)
|
|
240
|
+
|
|
241
|
+
|
|
242
|
+
def collect_scan_usage_safely(
|
|
243
|
+
connection: sqlite3.Connection, scan: sqlite3.Row, **options: Any
|
|
244
|
+
) -> dict[str, Any]:
|
|
245
|
+
try:
|
|
246
|
+
return collect_scan_usage(connection, scan, **options)
|
|
247
|
+
except (Exception, SystemExit) as exc:
|
|
248
|
+
try:
|
|
249
|
+
print(f"Could not collect scan token usage: {exc}", file=sys.stderr)
|
|
250
|
+
except OSError:
|
|
251
|
+
# A closed diagnostic stream must not replace the unavailable-usage result.
|
|
252
|
+
pass
|
|
253
|
+
return _unavailable_usage("usage_collection_failed")
|
|
254
|
+
|
|
255
|
+
|
|
84
256
|
def collect_scan_usage(
|
|
85
257
|
connection: sqlite3.Connection,
|
|
86
258
|
scan: sqlite3.Row,
|
|
87
259
|
*,
|
|
88
260
|
thread_id: str | None = None,
|
|
261
|
+
validation_thread_id: str | None = None,
|
|
89
262
|
completed_at: str | None = None,
|
|
90
263
|
completed_turn_id: str | None = None,
|
|
91
264
|
) -> dict[str, Any]:
|
|
92
|
-
"""Count attributable receipts
|
|
265
|
+
"""Count attributable session receipts during the scan window and report missing coverage.
|
|
266
|
+
|
|
267
|
+
The optional thread and completion arguments identify the finishing turn when
|
|
268
|
+
the scan record alone is insufficient. Return an unavailable usage record when
|
|
269
|
+
the window, Codex state, or relevant threads cannot be established.
|
|
270
|
+
"""
|
|
93
271
|
|
|
94
272
|
roots = _scan_root_thread_ids(connection, scan, thread_id)
|
|
95
273
|
parent_ids = set(_scan_root_thread_ids(connection, scan, thread_id, include_workers=False))
|
|
96
|
-
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
if state_database is None:
|
|
102
|
-
return _unavailable_usage("codex_state_unavailable")
|
|
103
|
-
|
|
274
|
+
if validation_thread_id:
|
|
275
|
+
roots.append(validation_thread_id)
|
|
276
|
+
parent_ids.add(validation_thread_id)
|
|
277
|
+
warnings: set[str] = set()
|
|
278
|
+
worker_directories = _scan_worker_directories(connection, scan, warnings)
|
|
104
279
|
started_at = _timestamp(scan["started_at"])
|
|
105
280
|
observed_at = datetime.now(timezone.utc)
|
|
106
281
|
stopped_at = _timestamp(completed_at or scan["completed_at"]) or observed_at
|
|
107
282
|
if started_at is None:
|
|
108
283
|
return _unavailable_usage("scan_window_unavailable")
|
|
109
284
|
|
|
110
|
-
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
117
|
-
|
|
118
|
-
|
|
119
|
-
)
|
|
120
|
-
|
|
121
|
-
|
|
285
|
+
receipts, missing_receipts, receipt_scope = _worker_usage_receipts(
|
|
286
|
+
scan, started_at, stopped_at, warnings
|
|
287
|
+
)
|
|
288
|
+
state_database = _codex_state_database()
|
|
289
|
+
sessions: list[RolloutSession] = []
|
|
290
|
+
missing_thread_ids: set[str] = set()
|
|
291
|
+
if state_database is None:
|
|
292
|
+
if not receipts:
|
|
293
|
+
return _unavailable_usage("codex_state_unavailable", warnings=warnings)
|
|
294
|
+
warnings.add("codex_state_unavailable")
|
|
295
|
+
missing_thread_ids.update(roots)
|
|
296
|
+
else:
|
|
297
|
+
try:
|
|
298
|
+
sessions, missing_thread_ids = _discover_rollout_sessions(
|
|
299
|
+
state_database,
|
|
300
|
+
roots,
|
|
301
|
+
warnings,
|
|
302
|
+
worker_directories=worker_directories,
|
|
303
|
+
scan_directory=Path(scan["scan_dir"]) if worker_directories else None,
|
|
304
|
+
started_at=started_at,
|
|
305
|
+
completed_at=stopped_at,
|
|
306
|
+
)
|
|
307
|
+
except (OSError, sqlite3.Error, ValueError):
|
|
308
|
+
if not receipts:
|
|
309
|
+
return _unavailable_usage("codex_state_unavailable", warnings=warnings)
|
|
310
|
+
warnings.add("codex_state_unavailable")
|
|
311
|
+
missing_thread_ids.update(roots)
|
|
122
312
|
|
|
123
|
-
if not sessions:
|
|
313
|
+
if not sessions and not receipts:
|
|
124
314
|
return _unavailable_usage("scan_thread_unavailable", warnings=warnings)
|
|
125
315
|
|
|
126
|
-
by_role = {role: {**_empty_token_usage(), "threadCount": 0} for role in ("parent", "workers")}
|
|
127
316
|
rate_limits: dict[str, dict[str, Any]] = {}
|
|
128
317
|
tokens_updated_at = None
|
|
129
318
|
accepted_thread_ids: set[str] = set()
|
|
319
|
+
measured_threads: dict[str, dict[str, int]] = {}
|
|
320
|
+
complete_native_thread_ids: set[str] = set()
|
|
130
321
|
excluded_thread_ids: set[str] = set()
|
|
131
322
|
for session in sessions:
|
|
132
323
|
if session.parent_thread_id in excluded_thread_ids:
|
|
@@ -167,25 +358,60 @@ def collect_scan_usage(
|
|
|
167
358
|
for key, snapshot in usage.rate_limits.items():
|
|
168
359
|
if key not in rate_limits or snapshot["observedAt"] > rate_limits[key]["observedAt"]:
|
|
169
360
|
rate_limits[key] = snapshot
|
|
170
|
-
if "token_usage_unavailable"
|
|
361
|
+
if usage.warnings & {"token_usage_unavailable", "token_breakdown_unavailable"}:
|
|
171
362
|
missing_thread_ids.add(session.thread_id)
|
|
172
363
|
continue
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
364
|
+
measured_threads[session.thread_id] = usage.tokens
|
|
365
|
+
if not usage.warnings:
|
|
366
|
+
complete_native_thread_ids.add(session.thread_id)
|
|
176
367
|
if usage.tokens_updated_at is not None:
|
|
177
368
|
tokens_updated_at = max(
|
|
178
369
|
tokens_updated_at or usage.tokens_updated_at, usage.tokens_updated_at
|
|
179
370
|
)
|
|
180
371
|
|
|
372
|
+
runtime_receipts_used = False
|
|
373
|
+
for receipt_thread_id, receipt in receipts.items():
|
|
374
|
+
native = measured_threads.get(receipt_thread_id)
|
|
375
|
+
extends_native = native is None or receipt.tokens["totalTokens"] > native["totalTokens"]
|
|
376
|
+
fills_cache_write_usage = (
|
|
377
|
+
native is not None
|
|
378
|
+
and receipt.tokens["totalTokens"] == native["totalTokens"]
|
|
379
|
+
and native.get("cacheWriteInputTokensReported") is False
|
|
380
|
+
and receipt.tokens.get("cacheWriteInputTokensReported") is not False
|
|
381
|
+
)
|
|
382
|
+
if extends_native or fills_cache_write_usage:
|
|
383
|
+
measured_threads[receipt_thread_id] = receipt.tokens
|
|
384
|
+
accepted_thread_ids.add(receipt_thread_id)
|
|
385
|
+
runtime_receipts_used = True
|
|
386
|
+
if extends_native:
|
|
387
|
+
complete_native_thread_ids.discard(receipt_thread_id)
|
|
388
|
+
# Numeric SDK subtotals cannot establish that descendants and
|
|
389
|
+
# compaction requests are present when their rollouts are absent.
|
|
390
|
+
missing_thread_ids.add(receipt_thread_id)
|
|
391
|
+
warnings.add("worker_rollout_usage_incomplete")
|
|
392
|
+
if receipt.tokens_updated_at is not None:
|
|
393
|
+
tokens_updated_at = max(
|
|
394
|
+
tokens_updated_at or receipt.tokens_updated_at, receipt.tokens_updated_at
|
|
395
|
+
)
|
|
396
|
+
unresolved_receipts = missing_receipts - complete_native_thread_ids
|
|
397
|
+
if unresolved_receipts:
|
|
398
|
+
missing_thread_ids.update(unresolved_receipts)
|
|
399
|
+
warnings.add("worker_usage_unavailable")
|
|
181
400
|
if not accepted_thread_ids:
|
|
182
401
|
return _unavailable_usage("scan_thread_unavailable", warnings=warnings)
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
|
|
186
|
-
"
|
|
402
|
+
if not measured_threads:
|
|
403
|
+
reason = (
|
|
404
|
+
"token_breakdown_unavailable"
|
|
405
|
+
if "token_breakdown_unavailable" in warnings
|
|
406
|
+
else "token_usage_unavailable"
|
|
187
407
|
)
|
|
408
|
+
return _unavailable_usage(reason, warnings=warnings, rate_limits=rate_limits)
|
|
188
409
|
|
|
410
|
+
by_role = {role: {**_empty_token_usage(), "threadCount": 0} for role in ("parent", "workers")}
|
|
411
|
+
for measured_id, tokens in measured_threads.items():
|
|
412
|
+
role = by_role["parent" if measured_id in parent_ids else "workers"]
|
|
413
|
+
_add_token_usage(role, tokens)
|
|
414
|
+
role["threadCount"] += 1
|
|
189
415
|
if any(
|
|
190
416
|
usage["cachedInputTokens"] + usage["cacheWriteInputTokens"] > usage["inputTokens"]
|
|
191
417
|
or usage["totalTokens"] != usage["inputTokens"] + usage["outputTokens"]
|
|
@@ -195,18 +421,28 @@ def collect_scan_usage(
|
|
|
195
421
|
"token_breakdown_unavailable", warnings=warnings, rate_limits=rate_limits
|
|
196
422
|
)
|
|
197
423
|
|
|
198
|
-
if completed_turn_id is not None and tokens_updated_at is not None:
|
|
424
|
+
if (completed_turn_id is not None or runtime_receipts_used) and tokens_updated_at is not None:
|
|
199
425
|
stopped_at = max(stopped_at, tokens_updated_at)
|
|
200
426
|
result: dict[str, Any] = {
|
|
201
427
|
"coverage": "partial" if missing_thread_ids or warnings else "complete",
|
|
202
|
-
"source": "codex_rollout",
|
|
428
|
+
"source": "codex_runtime" if runtime_receipts_used else "codex_rollout",
|
|
203
429
|
**{key: sum(role[key] for role in by_role.values()) for key in TOKEN_FIELDS.values()},
|
|
204
|
-
"threadCount":
|
|
430
|
+
"threadCount": len(measured_threads),
|
|
431
|
+
"coverageScope": sorted(
|
|
432
|
+
receipt_scope
|
|
433
|
+
| {f"thread:{identity}" for identity in measured_threads}
|
|
434
|
+
| {
|
|
435
|
+
identity if identity.startswith("worker-attempt:") else f"thread:{identity}"
|
|
436
|
+
for identity in missing_thread_ids
|
|
437
|
+
}
|
|
438
|
+
),
|
|
205
439
|
"byRole": by_role,
|
|
206
440
|
"window": {"startedAt": started_at.isoformat(), "endedAt": stopped_at.isoformat()},
|
|
207
441
|
"observedAt": observed_at.isoformat(),
|
|
208
442
|
"accountRateLimits": [rate_limits[key] for key in sorted(rate_limits)],
|
|
209
443
|
}
|
|
444
|
+
if any(role.get("cacheWriteInputTokensReported") is False for role in by_role.values()):
|
|
445
|
+
result["cacheWriteInputTokensReported"] = False
|
|
210
446
|
if tokens_updated_at is not None:
|
|
211
447
|
result["tokensUpdatedAt"] = tokens_updated_at.isoformat()
|
|
212
448
|
if missing_thread_ids:
|
|
@@ -216,6 +452,79 @@ def collect_scan_usage(
|
|
|
216
452
|
return result
|
|
217
453
|
|
|
218
454
|
|
|
455
|
+
def _worker_usage_receipts(
|
|
456
|
+
scan: Mapping[str, Any],
|
|
457
|
+
started_at: datetime,
|
|
458
|
+
completed_at: datetime,
|
|
459
|
+
warnings: set[str],
|
|
460
|
+
) -> tuple[dict[str, RolloutUsage], set[str], set[str]]:
|
|
461
|
+
if scan["mode"] != "deep":
|
|
462
|
+
return {}, set(), set()
|
|
463
|
+
directory = worker_usage_directory(scan["id"])
|
|
464
|
+
try:
|
|
465
|
+
if directory.resolve(strict=True) != directory:
|
|
466
|
+
raise ValueError("Worker usage directory is not a canonical protected directory.")
|
|
467
|
+
except FileNotFoundError:
|
|
468
|
+
return {}, set(), set()
|
|
469
|
+
except (OSError, RuntimeError, ValueError):
|
|
470
|
+
warnings.add("worker_usage_receipt_unavailable")
|
|
471
|
+
return {}, {"worker-receipts"}, {"worker-receipts"}
|
|
472
|
+
receipts: dict[str, RolloutUsage] = {}
|
|
473
|
+
missing: set[str] = set()
|
|
474
|
+
attempts: set[str] = set()
|
|
475
|
+
scope: set[str] = set()
|
|
476
|
+
for path in sorted(directory.glob("*.json")):
|
|
477
|
+
try:
|
|
478
|
+
if _rollout_path(str(path)) is None:
|
|
479
|
+
raise ValueError("Worker usage receipt path is unavailable.")
|
|
480
|
+
receipt = json.loads(path.read_text(encoding="utf-8"))
|
|
481
|
+
if (
|
|
482
|
+
not isinstance(receipt, dict)
|
|
483
|
+
or receipt.get("schemaVersion") != 1
|
|
484
|
+
or receipt.get("source") != "codex_sdk"
|
|
485
|
+
or receipt.get("usageScope") != "thread_cumulative"
|
|
486
|
+
or receipt.get("scanId") != scan["id"]
|
|
487
|
+
or receipt.get("kind") not in ("discovery", "dedup", "compare")
|
|
488
|
+
or receipt.get("attemptId") != path.stem
|
|
489
|
+
):
|
|
490
|
+
raise ValueError("Worker usage receipt identity is invalid.")
|
|
491
|
+
attempt_id = receipt["attemptId"]
|
|
492
|
+
begin = _timestamp(receipt.get("startedAt"))
|
|
493
|
+
observed = _timestamp(receipt.get("observedAt"))
|
|
494
|
+
if begin is None or observed is None:
|
|
495
|
+
raise ValueError("Worker usage receipt timestamps are invalid.")
|
|
496
|
+
if begin < started_at or begin > completed_at or attempt_id in attempts:
|
|
497
|
+
continue
|
|
498
|
+
attempts.add(attempt_id)
|
|
499
|
+
scope.add(f"worker-attempt:{attempt_id}")
|
|
500
|
+
thread_id = receipt.get("threadId")
|
|
501
|
+
if isinstance(thread_id, str) and thread_id:
|
|
502
|
+
scope.add(f"thread:{thread_id}")
|
|
503
|
+
tokens = _usage_counts(receipt.get("usage"))
|
|
504
|
+
if not isinstance(thread_id, str) or not thread_id or tokens is None:
|
|
505
|
+
missing.add(
|
|
506
|
+
thread_id
|
|
507
|
+
if isinstance(thread_id, str) and thread_id
|
|
508
|
+
else f"worker-attempt:{attempt_id}"
|
|
509
|
+
)
|
|
510
|
+
continue
|
|
511
|
+
previous = receipts.get(thread_id)
|
|
512
|
+
if previous is None or _usage_quality(tokens, observed_at=observed) > _usage_quality(
|
|
513
|
+
previous.tokens, observed_at=previous.tokens_updated_at
|
|
514
|
+
):
|
|
515
|
+
receipts[thread_id] = RolloutUsage(tokens, set(), observed, {})
|
|
516
|
+
except (OSError, UnicodeError, ValueError) as exc:
|
|
517
|
+
warnings.add("worker_usage_receipt_unavailable")
|
|
518
|
+
missing.add(f"worker-attempt:{path.stem}")
|
|
519
|
+
scope.add(f"worker-attempt:{path.stem}")
|
|
520
|
+
try:
|
|
521
|
+
print(f"Could not read worker token usage: {exc}", file=sys.stderr)
|
|
522
|
+
except OSError:
|
|
523
|
+
# The receipt warning and missing-attempt scope are already retained.
|
|
524
|
+
pass
|
|
525
|
+
return receipts, missing, scope
|
|
526
|
+
|
|
527
|
+
|
|
219
528
|
def _scan_root_thread_ids(
|
|
220
529
|
connection: sqlite3.Connection,
|
|
221
530
|
scan: sqlite3.Row,
|
|
@@ -266,19 +575,35 @@ def _scan_execution_thread_ids(connection: sqlite3.Connection, scan: sqlite3.Row
|
|
|
266
575
|
)
|
|
267
576
|
|
|
268
577
|
|
|
269
|
-
def _scan_worker_directories(
|
|
578
|
+
def _scan_worker_directories(
|
|
579
|
+
connection: sqlite3.Connection, scan: sqlite3.Row, warnings: set[str]
|
|
580
|
+
) -> dict[str, bool]:
|
|
270
581
|
if scan["mode"] != "deep":
|
|
271
|
-
return
|
|
272
|
-
|
|
273
|
-
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
|
|
280
|
-
|
|
281
|
-
|
|
582
|
+
return {}
|
|
583
|
+
if connection.execute(
|
|
584
|
+
"SELECT 1 FROM deep_scan_runs WHERE scan_id = ? AND worker_usage_gap = 1", (scan["id"],)
|
|
585
|
+
).fetchone():
|
|
586
|
+
warnings.add("worker_thread_unavailable")
|
|
587
|
+
artifacts = Path(scan["scan_dir"]) / "artifacts"
|
|
588
|
+
# Comparison turns run in this directory without a discovery worker row.
|
|
589
|
+
directories = {str(artifacts / "deep_discovery"): False}
|
|
590
|
+
for row in connection.execute(
|
|
591
|
+
"SELECT kind, artifact_dir, started_at, sdk_thread_id, attempt FROM deep_scan_workers "
|
|
592
|
+
"WHERE scan_id = ? AND kind IN ('discovery', 'dedup')",
|
|
593
|
+
(scan["id"],),
|
|
594
|
+
):
|
|
595
|
+
if row["kind"] == "discovery":
|
|
596
|
+
if row["sdk_thread_id"] and row["attempt"] == 1:
|
|
597
|
+
continue # The first attempt already has an exact thread identity.
|
|
598
|
+
# An attempt that started without a saved ID must be found by its assigned cwd.
|
|
599
|
+
missing_thread = bool(row["started_at"] and not row["sdk_thread_id"])
|
|
600
|
+
directories[row["artifact_dir"]] = missing_thread
|
|
601
|
+
else:
|
|
602
|
+
directories[str(artifacts)] = False
|
|
603
|
+
if row["attempt"] > 1 and row["started_at"] and not row["sdk_thread_id"]:
|
|
604
|
+
# Retries share a cwd. A rollout from an older attempt cannot prove the new one.
|
|
605
|
+
warnings.add("worker_thread_unavailable")
|
|
606
|
+
return directories
|
|
282
607
|
|
|
283
608
|
|
|
284
609
|
def _codex_state_database() -> Path | None:
|
|
@@ -323,7 +648,8 @@ def _discover_rollout_sessions(
|
|
|
323
648
|
roots: list[str],
|
|
324
649
|
warnings: set[str],
|
|
325
650
|
*,
|
|
326
|
-
worker_directories:
|
|
651
|
+
worker_directories: dict[str, bool],
|
|
652
|
+
scan_directory: Path | None = None,
|
|
327
653
|
started_at: datetime,
|
|
328
654
|
completed_at: datetime | None,
|
|
329
655
|
) -> tuple[list[RolloutSession], set[str]]:
|
|
@@ -342,9 +668,12 @@ def _discover_rollout_sessions(
|
|
|
342
668
|
columns = {str(row["name"]) for row in database.execute(f"PRAGMA table_info({table})")}
|
|
343
669
|
if not required.issubset(columns):
|
|
344
670
|
raise ValueError("Codex state graph does not expose the required thread columns.")
|
|
671
|
+
if table == "threads" and worker_directories and not {"cwd", "created_at"} <= columns:
|
|
672
|
+
worker_directories = {}
|
|
673
|
+
warnings.add("worker_directories_unavailable")
|
|
345
674
|
sessions: dict[str, RolloutSession] = {}
|
|
346
675
|
missing_thread_ids: set[str] = set()
|
|
347
|
-
root_ids = dict.fromkeys(roots)
|
|
676
|
+
root_ids: dict[str, str | None] = dict.fromkeys(roots)
|
|
348
677
|
started_millisecond = started_at.replace(microsecond=started_at.microsecond // 1000 * 1000)
|
|
349
678
|
completed_timestamp = int(completed_at.timestamp()) if completed_at is not None else None
|
|
350
679
|
directory_expression = "cwd"
|
|
@@ -355,16 +684,52 @@ def _discover_rollout_sessions(
|
|
|
355
684
|
"codex_security_path_key", 1, _normalized_windows_path, deterministic=True
|
|
356
685
|
)
|
|
357
686
|
directory_expression = "codex_security_path_key(cwd)"
|
|
358
|
-
worker_directories = {
|
|
359
|
-
|
|
687
|
+
worker_directories = {
|
|
688
|
+
_normalized_windows_path(path): required
|
|
689
|
+
for path, required in worker_directories.items()
|
|
690
|
+
}
|
|
691
|
+
if scan_directory is not None:
|
|
692
|
+
scan_directory = Path(_normalized_windows_path(scan_directory))
|
|
693
|
+
unmatched = {path for path, required in worker_directories.items() if required}
|
|
694
|
+
directory_sources = {path: {path} for path in worker_directories}
|
|
695
|
+
if worker_directories and scan_directory and ".previous-" in scan_directory.name:
|
|
696
|
+
# Archived files move, but session working directories describe where execution ran.
|
|
697
|
+
# Confirm the original output with an already-bound session before including peers.
|
|
698
|
+
bound_directories = {
|
|
699
|
+
Path(row[0])
|
|
700
|
+
for row in database.execute(
|
|
701
|
+
f"SELECT DISTINCT {directory_expression} FROM threads WHERE cwd IS NOT NULL "
|
|
702
|
+
"AND id IN (SELECT value FROM json_each(?))",
|
|
703
|
+
(json.dumps(roots),),
|
|
704
|
+
)
|
|
705
|
+
}
|
|
706
|
+
relatives = {
|
|
707
|
+
Path(path).relative_to(scan_directory): path
|
|
708
|
+
for path in worker_directories
|
|
709
|
+
if Path(path).is_relative_to(scan_directory)
|
|
710
|
+
}
|
|
711
|
+
for marker in re.finditer(r"\.previous-", scan_directory.name):
|
|
712
|
+
if marker.start() == 0:
|
|
713
|
+
continue
|
|
714
|
+
original = scan_directory.with_name(scan_directory.name[: marker.start()])
|
|
715
|
+
candidates = {original / relative: path for relative, path in relatives.items()}
|
|
716
|
+
# Older archives kept absolute discovery paths in their original directory.
|
|
717
|
+
if (
|
|
718
|
+
original in bound_directories
|
|
719
|
+
or candidates.keys() & bound_directories
|
|
720
|
+
or any(Path(path).is_relative_to(original) for path in worker_directories)
|
|
721
|
+
):
|
|
722
|
+
for candidate, path in candidates.items():
|
|
723
|
+
directory_sources.setdefault(str(candidate), set()).add(path)
|
|
724
|
+
for directory in sorted(directory_sources):
|
|
360
725
|
for row in database.execute(
|
|
361
726
|
"SELECT id FROM threads WHERE cwd IS NOT NULL "
|
|
362
727
|
f"AND {directory_expression} = ? AND created_at >= ? "
|
|
363
728
|
"AND (? IS NULL OR created_at <= ?) ORDER BY id",
|
|
364
729
|
(directory, int(started_at.timestamp()), completed_timestamp, completed_timestamp),
|
|
365
730
|
):
|
|
366
|
-
root_ids[row["id"]] =
|
|
367
|
-
for root in root_ids:
|
|
731
|
+
root_ids[row["id"]] = directory
|
|
732
|
+
for root, directory in root_ids.items():
|
|
368
733
|
row = database.execute(
|
|
369
734
|
"SELECT id, rollout_path FROM threads WHERE id = ?",
|
|
370
735
|
(root,),
|
|
@@ -380,8 +745,8 @@ def _discover_rollout_sessions(
|
|
|
380
745
|
continue
|
|
381
746
|
if root not in roots:
|
|
382
747
|
try:
|
|
383
|
-
with path
|
|
384
|
-
metadata = json.loads(source
|
|
748
|
+
with _rollout_lines(path) as source:
|
|
749
|
+
metadata = json.loads(next(source, b""))
|
|
385
750
|
except (OSError, ValueError):
|
|
386
751
|
missing_thread_ids.add(root)
|
|
387
752
|
warnings.add("rollout_unavailable")
|
|
@@ -392,9 +757,15 @@ def _discover_rollout_sessions(
|
|
|
392
757
|
and isinstance(payload := metadata.get("payload"), dict)
|
|
393
758
|
and (payload.get("id") or payload.get("session_id")) == root
|
|
394
759
|
and (created_at := _timestamp(payload.get("timestamp"))) is not None
|
|
395
|
-
and
|
|
760
|
+
and (
|
|
761
|
+
created_at < started_millisecond
|
|
762
|
+
or completed_at is not None
|
|
763
|
+
and created_at > completed_at
|
|
764
|
+
)
|
|
396
765
|
):
|
|
397
766
|
continue
|
|
767
|
+
if directory is not None:
|
|
768
|
+
unmatched.difference_update(directory_sources[directory])
|
|
398
769
|
sessions[root] = RolloutSession(root, None, path)
|
|
399
770
|
descendants = database.execute(
|
|
400
771
|
"""
|
|
@@ -452,6 +823,8 @@ def _discover_rollout_sessions(
|
|
|
452
823
|
warnings.add("rollout_unavailable")
|
|
453
824
|
continue
|
|
454
825
|
sessions[child_id] = RolloutSession(child_id, parent_id, path)
|
|
826
|
+
if unmatched:
|
|
827
|
+
warnings.add("worker_thread_unavailable")
|
|
455
828
|
return list(sessions.values()), missing_thread_ids
|
|
456
829
|
finally:
|
|
457
830
|
database.close()
|
|
@@ -464,6 +837,13 @@ def _rollout_path(value: object) -> Path | None:
|
|
|
464
837
|
if not candidate.is_absolute():
|
|
465
838
|
return None
|
|
466
839
|
try:
|
|
840
|
+
try:
|
|
841
|
+
candidate.lstat()
|
|
842
|
+
except FileNotFoundError:
|
|
843
|
+
# Native cold storage may compress an indexed JSONL file in place.
|
|
844
|
+
if candidate.suffix != ".jsonl":
|
|
845
|
+
return None
|
|
846
|
+
candidate = candidate.with_suffix(".jsonl.zst")
|
|
467
847
|
resolved = candidate.resolve(strict=True)
|
|
468
848
|
if not resolved.is_file():
|
|
469
849
|
return None
|
|
@@ -480,6 +860,63 @@ def _rollout_path(value: object) -> Path | None:
|
|
|
480
860
|
return None
|
|
481
861
|
|
|
482
862
|
|
|
863
|
+
@contextmanager
|
|
864
|
+
def _rollout_lines(path: Path, *, verify_end: bool = False):
|
|
865
|
+
if path.suffix != ".zst":
|
|
866
|
+
with path.open("rb") as source:
|
|
867
|
+
yield source
|
|
868
|
+
return
|
|
869
|
+
try:
|
|
870
|
+
from compression import zstd # Python 3.14+
|
|
871
|
+
except ImportError:
|
|
872
|
+
try:
|
|
873
|
+
from backports import zstd
|
|
874
|
+
except ImportError:
|
|
875
|
+
zstd = None
|
|
876
|
+
if zstd is not None:
|
|
877
|
+
try:
|
|
878
|
+
with zstd.open(path, "rb") as source:
|
|
879
|
+
yield source
|
|
880
|
+
# A requested completion turn may stop parsing before the frame checksum.
|
|
881
|
+
if verify_end:
|
|
882
|
+
while source.read(64 * 1024):
|
|
883
|
+
pass
|
|
884
|
+
except (zstd.ZstdError, EOFError) as error:
|
|
885
|
+
raise ValueError("Invalid compressed Codex rollout.") from error
|
|
886
|
+
return
|
|
887
|
+
with tempfile.TemporaryFile() as errors:
|
|
888
|
+
with subprocess.Popen(
|
|
889
|
+
[
|
|
890
|
+
os.environ.get("CODEX_MCP_NODE_PATH") or "node",
|
|
891
|
+
str(Path(__file__).with_name("read_codex_session.mjs")),
|
|
892
|
+
str(path),
|
|
893
|
+
],
|
|
894
|
+
stdout=subprocess.PIPE,
|
|
895
|
+
stderr=errors,
|
|
896
|
+
) as child:
|
|
897
|
+
exhausted = False
|
|
898
|
+
|
|
899
|
+
def lines():
|
|
900
|
+
nonlocal exhausted
|
|
901
|
+
yield from child.stdout
|
|
902
|
+
exhausted = True
|
|
903
|
+
|
|
904
|
+
try:
|
|
905
|
+
iterator = lines()
|
|
906
|
+
yield iterator
|
|
907
|
+
if verify_end:
|
|
908
|
+
for _ in iterator:
|
|
909
|
+
pass
|
|
910
|
+
finally:
|
|
911
|
+
if not exhausted and child.poll() is None:
|
|
912
|
+
child.terminate()
|
|
913
|
+
child.stdout.close()
|
|
914
|
+
status = child.wait()
|
|
915
|
+
if exhausted and status != 0:
|
|
916
|
+
errors.seek(0)
|
|
917
|
+
raise OSError(errors.read().decode("utf-8", errors="replace"))
|
|
918
|
+
|
|
919
|
+
|
|
483
920
|
def _read_rollout_usage(
|
|
484
921
|
session: RolloutSession,
|
|
485
922
|
*,
|
|
@@ -492,8 +929,13 @@ def _read_rollout_usage(
|
|
|
492
929
|
total = result.tokens
|
|
493
930
|
warnings = result.warnings
|
|
494
931
|
previous = _empty_token_usage()
|
|
932
|
+
previous_cache_writes_reported = True
|
|
933
|
+
prefix_unclassified = 0
|
|
934
|
+
prefix_missing_cache: set[str] = set()
|
|
935
|
+
ambiguous_legacy_cache = False
|
|
495
936
|
boundary_reached = False
|
|
496
937
|
legacy_usage = _empty_token_usage()
|
|
938
|
+
completed_legacy_usage = _empty_token_usage()
|
|
497
939
|
response_usage: dict[str, int] | None = None
|
|
498
940
|
observed_response_usage = _empty_token_usage()
|
|
499
941
|
response_ids: set[str] = set()
|
|
@@ -513,7 +955,7 @@ def _read_rollout_usage(
|
|
|
513
955
|
and (completed_turn_id is None or turn_id != completed_turn_id)
|
|
514
956
|
)
|
|
515
957
|
|
|
516
|
-
with session.path
|
|
958
|
+
with _rollout_lines(session.path, verify_end=True) as source:
|
|
517
959
|
for line_number, raw_line in enumerate(source, start=1):
|
|
518
960
|
if not raw_line.endswith(b"\n"):
|
|
519
961
|
warnings.add("rollout_record_incomplete")
|
|
@@ -575,7 +1017,14 @@ def _read_rollout_usage(
|
|
|
575
1017
|
elif event.get("type") == "event_msg" and payload.get("type") == "token_count":
|
|
576
1018
|
inherited_usage = _token_snapshot(payload)
|
|
577
1019
|
if inherited_usage is not None:
|
|
578
|
-
|
|
1020
|
+
missing_cache = _missing_cache_fields(payload)
|
|
1021
|
+
baseline = _retain_cache_baseline(previous, inherited_usage, missing_cache)
|
|
1022
|
+
if baseline is None:
|
|
1023
|
+
warnings.add("token_breakdown_unavailable")
|
|
1024
|
+
else:
|
|
1025
|
+
previous = baseline
|
|
1026
|
+
prefix_unclassified = _uncached_input(baseline)
|
|
1027
|
+
prefix_missing_cache = missing_cache
|
|
579
1028
|
if not boundary_reached:
|
|
580
1029
|
continue
|
|
581
1030
|
if event.get("type") == "event_msg" and payload.get("type") == "task_started":
|
|
@@ -593,6 +1042,11 @@ def _read_rollout_usage(
|
|
|
593
1042
|
_add_token_usage(
|
|
594
1043
|
total, response_usage if response_usage is not None else legacy_usage
|
|
595
1044
|
)
|
|
1045
|
+
if response_usage is None:
|
|
1046
|
+
_add_token_usage(completed_legacy_usage, legacy_usage)
|
|
1047
|
+
if ambiguous_legacy_cache:
|
|
1048
|
+
warnings.add("token_breakdown_unavailable")
|
|
1049
|
+
ambiguous_legacy_cache = False
|
|
596
1050
|
legacy_usage = _empty_token_usage()
|
|
597
1051
|
response_usage = None
|
|
598
1052
|
if payload.get("turn_id") != current_turn_id:
|
|
@@ -711,25 +1165,98 @@ def _read_rollout_usage(
|
|
|
711
1165
|
if timestamp is None or snapshot is None:
|
|
712
1166
|
warnings.add("token_record_invalid")
|
|
713
1167
|
continue
|
|
1168
|
+
reset = snapshot["totalTokens"] < previous["totalTokens"]
|
|
1169
|
+
if reset:
|
|
1170
|
+
prefix_unclassified = 0
|
|
1171
|
+
missing_cache = _missing_cache_fields(payload)
|
|
1172
|
+
cache_writes_reported = "cacheWriteInputTokens" not in missing_cache
|
|
1173
|
+
baseline = _retain_cache_baseline(previous, snapshot, missing_cache)
|
|
1174
|
+
if baseline is None:
|
|
1175
|
+
if not past_scan_end(timestamp, current_turn_id):
|
|
1176
|
+
warnings.add("token_breakdown_unavailable")
|
|
1177
|
+
continue
|
|
1178
|
+
if baseline is previous:
|
|
1179
|
+
cache_writes_reported = previous_cache_writes_reported and cache_writes_reported
|
|
1180
|
+
refines_classification = (
|
|
1181
|
+
baseline["inputTokens"] >= previous["inputTokens"]
|
|
1182
|
+
and baseline["outputTokens"] >= previous["outputTokens"]
|
|
1183
|
+
and baseline["cachedInputTokens"] >= previous["cachedInputTokens"]
|
|
1184
|
+
and baseline["cacheWriteInputTokens"] >= previous["cacheWriteInputTokens"]
|
|
1185
|
+
and (
|
|
1186
|
+
(not previous_cache_writes_reported and cache_writes_reported)
|
|
1187
|
+
or baseline["cachedInputTokens"] > previous["cachedInputTokens"]
|
|
1188
|
+
or baseline["cacheWriteInputTokens"] > previous["cacheWriteInputTokens"]
|
|
1189
|
+
)
|
|
1190
|
+
)
|
|
1191
|
+
previous_cache_writes_reported = cache_writes_reported
|
|
714
1192
|
delta = {
|
|
715
|
-
key:
|
|
716
|
-
|
|
1193
|
+
key: snapshot[key] - previous[key]
|
|
1194
|
+
if not reset and snapshot[key] >= previous[key]
|
|
1195
|
+
else snapshot[key]
|
|
1196
|
+
for key in TOKEN_FIELDS.values()
|
|
717
1197
|
}
|
|
718
|
-
|
|
1198
|
+
if (
|
|
1199
|
+
snapshot.get("cacheWriteInputTokensReported") is False
|
|
1200
|
+
or previous.get("cacheWriteInputTokensReported") is False
|
|
1201
|
+
):
|
|
1202
|
+
delta["cacheWriteInputTokensReported"] = False
|
|
1203
|
+
previous = baseline
|
|
719
1204
|
if past_scan_end(timestamp, current_turn_id):
|
|
720
1205
|
continue
|
|
721
1206
|
if isinstance(current_turn_id, str):
|
|
722
1207
|
measured_turns.add(current_turn_id)
|
|
723
1208
|
if timestamp < started_at:
|
|
1209
|
+
prefix_unclassified = _uncached_input(baseline)
|
|
1210
|
+
prefix_missing_cache = missing_cache
|
|
724
1211
|
continue
|
|
725
1212
|
if completed_turn_id is not None and current_turn_id == completed_turn_id:
|
|
726
1213
|
completed_turn_measured = True
|
|
727
1214
|
result.tokens_updated_at = max(result.tokens_updated_at or timestamp, timestamp)
|
|
728
|
-
|
|
1215
|
+
delta["totalTokens"] = delta["inputTokens"] + delta["outputTokens"]
|
|
1216
|
+
if delta["totalTokens"] <= 0 and not refines_classification:
|
|
729
1217
|
continue
|
|
1218
|
+
cache_write_capacity = (
|
|
1219
|
+
_uncached_input(legacy_usage) + delta["inputTokens"]
|
|
1220
|
+
if refines_classification
|
|
1221
|
+
else delta["inputTokens"]
|
|
1222
|
+
)
|
|
1223
|
+
# Late cumulative cache details can classify an earlier legacy turn.
|
|
1224
|
+
completed_capacity = (
|
|
1225
|
+
_uncached_input(completed_legacy_usage)
|
|
1226
|
+
if refines_classification and response_usage is None
|
|
1227
|
+
else 0
|
|
1228
|
+
)
|
|
1229
|
+
if (
|
|
1230
|
+
prefix_unclassified > 0
|
|
1231
|
+
and refines_classification
|
|
1232
|
+
and (
|
|
1233
|
+
any(delta[key] > 0 for key in prefix_missing_cache)
|
|
1234
|
+
# Even explicit counters can revise earlier input when the
|
|
1235
|
+
# newly classified tokens exceed all in-scan capacity.
|
|
1236
|
+
or delta["cachedInputTokens"] + delta["cacheWriteInputTokens"]
|
|
1237
|
+
> cache_write_capacity + completed_capacity
|
|
1238
|
+
)
|
|
1239
|
+
):
|
|
1240
|
+
# The newly classified tokens could have occurred before this scan.
|
|
1241
|
+
ambiguous_legacy_cache = True
|
|
1242
|
+
for key in ("cacheWriteInputTokens", "cachedInputTokens"):
|
|
1243
|
+
current = min(delta[key], cache_write_capacity)
|
|
1244
|
+
completed = min(delta[key] - current, completed_capacity)
|
|
1245
|
+
if completed and ambiguous_legacy_cache:
|
|
1246
|
+
warnings.add("token_breakdown_unavailable")
|
|
1247
|
+
delta[key] = current
|
|
1248
|
+
cache_write_capacity -= current
|
|
1249
|
+
completed_capacity -= completed
|
|
1250
|
+
completed_legacy_usage[key] += completed
|
|
1251
|
+
total[key] += completed
|
|
1252
|
+
delta["reasoningOutputTokens"] = min(
|
|
1253
|
+
delta["reasoningOutputTokens"], delta["outputTokens"]
|
|
1254
|
+
)
|
|
730
1255
|
_add_token_usage(legacy_usage, delta)
|
|
731
1256
|
|
|
732
1257
|
_add_token_usage(total, response_usage if response_usage is not None else legacy_usage)
|
|
1258
|
+
if response_usage is None and ambiguous_legacy_cache:
|
|
1259
|
+
warnings.add("token_breakdown_unavailable")
|
|
733
1260
|
if not boundary_reached:
|
|
734
1261
|
warnings.add("thread_ownership_unavailable")
|
|
735
1262
|
elif result.tokens_updated_at is None and not direct_control_session:
|
|
@@ -800,6 +1327,47 @@ def valid_account_rate_limit(value: object) -> bool:
|
|
|
800
1327
|
return True
|
|
801
1328
|
|
|
802
1329
|
|
|
1330
|
+
def _missing_cache_fields(payload: Mapping[str, Any]) -> set[str]:
|
|
1331
|
+
usage = payload["info"]["total_token_usage"]
|
|
1332
|
+
missing: set[str] = set()
|
|
1333
|
+
if "cached_input_tokens" not in usage:
|
|
1334
|
+
missing.add("cachedInputTokens")
|
|
1335
|
+
if "cache_write_input_tokens" not in usage and "cache_write_tokens" not in usage:
|
|
1336
|
+
missing.add("cacheWriteInputTokens")
|
|
1337
|
+
return missing
|
|
1338
|
+
|
|
1339
|
+
|
|
1340
|
+
def _uncached_input(usage: dict[str, int]) -> int:
|
|
1341
|
+
return usage["inputTokens"] - usage["cachedInputTokens"] - usage["cacheWriteInputTokens"]
|
|
1342
|
+
|
|
1343
|
+
|
|
1344
|
+
def _retain_cache_baseline(
|
|
1345
|
+
previous: dict[str, int], snapshot: dict[str, int], missing: set[str]
|
|
1346
|
+
) -> dict[str, int] | None:
|
|
1347
|
+
baseline = snapshot
|
|
1348
|
+
if (
|
|
1349
|
+
missing
|
|
1350
|
+
and snapshot["inputTokens"] >= previous["inputTokens"]
|
|
1351
|
+
and snapshot["outputTokens"] >= previous["outputTokens"]
|
|
1352
|
+
):
|
|
1353
|
+
baseline = {**snapshot, **{key: previous[key] for key in missing}}
|
|
1354
|
+
if (
|
|
1355
|
+
baseline["cachedInputTokens"] + baseline["cacheWriteInputTokens"]
|
|
1356
|
+
> baseline["inputTokens"]
|
|
1357
|
+
):
|
|
1358
|
+
return None
|
|
1359
|
+
if (
|
|
1360
|
+
baseline["inputTokens"] == previous["inputTokens"]
|
|
1361
|
+
and baseline["outputTokens"] == previous["outputTokens"]
|
|
1362
|
+
and (
|
|
1363
|
+
baseline["cachedInputTokens"] < previous["cachedInputTokens"]
|
|
1364
|
+
or baseline["cacheWriteInputTokens"] < previous["cacheWriteInputTokens"]
|
|
1365
|
+
)
|
|
1366
|
+
):
|
|
1367
|
+
return previous
|
|
1368
|
+
return baseline
|
|
1369
|
+
|
|
1370
|
+
|
|
803
1371
|
def _session_parent_thread_id(payload: Mapping[str, Any]) -> str | None:
|
|
804
1372
|
source = payload.get("source")
|
|
805
1373
|
if isinstance(source, dict):
|
|
@@ -882,6 +1450,12 @@ def _usage_counts(usage: object) -> dict[str, int] | None:
|
|
|
882
1450
|
if result["cachedInputTokens"] + result["cacheWriteInputTokens"] > result["inputTokens"]:
|
|
883
1451
|
return None
|
|
884
1452
|
result["totalTokens"] = result["inputTokens"] + result["outputTokens"]
|
|
1453
|
+
if (
|
|
1454
|
+
usage.get("cache_write_input_tokens_reported") is False
|
|
1455
|
+
or usage.get("cache_write_input_tokens") is None
|
|
1456
|
+
and usage.get("cache_write_tokens") is None
|
|
1457
|
+
):
|
|
1458
|
+
result["cacheWriteInputTokensReported"] = False
|
|
885
1459
|
return result
|
|
886
1460
|
|
|
887
1461
|
|
|
@@ -892,6 +1466,8 @@ def _empty_token_usage() -> dict[str, int]:
|
|
|
892
1466
|
def _add_token_usage(target: dict[str, int], addition: Mapping[str, int]) -> None:
|
|
893
1467
|
for key in TOKEN_FIELDS.values():
|
|
894
1468
|
target[key] += addition[key]
|
|
1469
|
+
if addition.get("cacheWriteInputTokensReported") is False:
|
|
1470
|
+
target["cacheWriteInputTokensReported"] = False
|
|
895
1471
|
|
|
896
1472
|
|
|
897
1473
|
def _timestamp(value: object) -> datetime | None:
|