@openai/codex-security 0.2.0 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +437 -2692
- package/_bundled_plugin/.app.json +5 -1
- package/_bundled_plugin/.codex-plugin/plugin.json +4 -0
- package/_bundled_plugin/.mcp.json +2 -0
- package/_bundled_plugin/examples/completed-scan/coverage.json +0 -1
- package/_bundled_plugin/examples/completed-scan/report.md +28 -26
- package/_bundled_plugin/examples/completed-scan/scan-manifest.json +1 -1
- package/_bundled_plugin/mcp/helpers.mjs +1 -0
- package/_bundled_plugin/mcp/helpers.mjs.br.part-000 +0 -0
- package/_bundled_plugin/mcp/native/THIRD_PARTY_NOTICES.txt +6 -213
- package/_bundled_plugin/mcp/native/darwin-arm64/unix.node +0 -0
- package/_bundled_plugin/mcp/native/darwin-x64/unix.node +0 -0
- package/_bundled_plugin/mcp/native/linux-arm64-gnu/unix.node +0 -0
- package/_bundled_plugin/mcp/native/linux-arm64-musl/unix.node +0 -0
- package/_bundled_plugin/mcp/native/linux-x64-gnu/unix.node +0 -0
- package/_bundled_plugin/mcp/native/linux-x64-musl/unix.node +0 -0
- package/_bundled_plugin/mcp/native/win32-arm64/windows.node +0 -0
- package/_bundled_plugin/mcp/native/win32-x64/windows.node +0 -0
- package/_bundled_plugin/mcp/permission-profile-preflight.mjs +556 -0
- package/_bundled_plugin/mcp/server.mjs +1 -0
- package/_bundled_plugin/mcp/server.mjs.br.part-000 +0 -0
- package/_bundled_plugin/mcp/server.mjs.br.part-001 +0 -0
- package/_bundled_plugin/preflight/capability-profiles.toml +0 -13
- package/_bundled_plugin/references/artifact-storage.md +2 -2
- package/_bundled_plugin/references/config-preflight.md +0 -4
- package/_bundled_plugin/references/core-scan.md +21 -9
- package/_bundled_plugin/references/final-report.md +11 -13
- package/_bundled_plugin/references/finding-detail-fields.md +90 -80
- package/_bundled_plugin/references/sarif-adapter.md +1 -1
- package/_bundled_plugin/references/scan-artifacts.md +1 -2
- package/_bundled_plugin/references/scan-contract.md +9 -4
- package/_bundled_plugin/references/scan-prologue.md +12 -0
- package/_bundled_plugin/references/security-guidance.md +2 -2
- package/_bundled_plugin/references/standalone-validation.md +11 -0
- package/_bundled_plugin/references/threat-model.md +1 -1
- package/_bundled_plugin/schemas/coverage.schema.json +43 -170
- package/_bundled_plugin/schemas/definitions/artifact-common.schema.json +2 -24
- package/_bundled_plugin/schemas/definitions/discovery-candidate.schema.json +49 -25
- package/_bundled_plugin/schemas/findings.schema.json +159 -567
- package/_bundled_plugin/schemas/patch-risk-assessment.schema.json +106 -119
- package/_bundled_plugin/schemas/scan-manifest.schema.json +78 -220
- package/_bundled_plugin/schemas/tools/candidate-attack-paths.schema.json +91 -83
- package/_bundled_plugin/schemas/tools/candidate-validations.schema.json +35 -64
- package/_bundled_plugin/schemas/tools/deep-reducer.schema.json +5 -2
- package/_bundled_plugin/schemas/tools/discovery-candidates.schema.json +6 -55
- package/_bundled_plugin/schemas/tools/review-items.schema.json +2 -38
- package/_bundled_plugin/schemas/tools/scan-draft.schema.json +169 -294
- package/_bundled_plugin/scripts/codex_profile.mjs +239 -0
- package/_bundled_plugin/scripts/codex_session.mjs +53 -0
- package/_bundled_plugin/scripts/config_preflight.py +26 -62
- package/_bundled_plugin/scripts/deep_scan_config.py +9 -11
- package/_bundled_plugin/scripts/deep_scan_workbench.py +129 -150
- package/_bundled_plugin/scripts/finalize_scan_contract.py +321 -237
- package/_bundled_plugin/scripts/finding_preview.py +39 -9
- package/_bundled_plugin/scripts/generate_in_scope_files.py +27 -77
- package/_bundled_plugin/scripts/generate_rank_input.py +115 -926
- package/_bundled_plugin/scripts/launch_codex_security_mcp +35 -10
- package/_bundled_plugin/scripts/launch_codex_security_mcp.cmd +5 -5
- package/_bundled_plugin/scripts/rank_preview.py +42 -918
- package/_bundled_plugin/scripts/report_projection.py +150 -183
- package/_bundled_plugin/scripts/snapshot_sqlite.py +6 -1
- package/_bundled_plugin/scripts/threat_model_projection.py +4 -7
- package/_bundled_plugin/scripts/validate_scan_contract.py +4 -34
- package/_bundled_plugin/scripts/validate_tracking_source.py +3 -13
- package/_bundled_plugin/scripts/windows_scan_local_files.py +48 -3
- package/_bundled_plugin/scripts/workbench/handoff.py +25 -50
- package/_bundled_plugin/scripts/workbench/runtime_toml.py +22 -0
- package/_bundled_plugin/scripts/workbench/storage.py +10 -6
- package/_bundled_plugin/scripts/workbench_cli.py +102 -184
- package/_bundled_plugin/scripts/workbench_db.py +463 -464
- package/_bundled_plugin/scripts/workbench_feedback.py +6 -3
- package/_bundled_plugin/scripts/workbench_finding_index.py +19 -1
- package/_bundled_plugin/scripts/workbench_finding_workflows.py +18 -4
- package/_bundled_plugin/scripts/workbench_local_dedupe.py +136 -0
- package/_bundled_plugin/scripts/workbench_native_indexes.py +27 -14
- package/_bundled_plugin/scripts/workbench_progress.py +14 -58
- package/_bundled_plugin/scripts/workbench_publication.py +234 -289
- package/_bundled_plugin/scripts/workbench_remediation.py +12 -0
- package/_bundled_plugin/scripts/workbench_saved_results.py +172 -49
- package/_bundled_plugin/scripts/workbench_scan_history.py +57 -57
- package/_bundled_plugin/scripts/workbench_scan_start.py +105 -41
- package/_bundled_plugin/scripts/workbench_scan_usage.py +194 -42
- package/_bundled_plugin/scripts/workbench_schema.py +37 -1151
- package/_bundled_plugin/scripts/workbench_source_excerpt.py +11 -1
- package/_bundled_plugin/scripts/workbench_target.py +430 -142
- package/_bundled_plugin/scripts/workbench_validation.py +31 -28
- package/_bundled_plugin/shared/workbench-history-repairs.json +218 -0
- package/_bundled_plugin/shared/workbench-migrations.json +414 -0
- package/_bundled_plugin/skills/assess-patch-risk/SKILL.md +15 -4
- package/_bundled_plugin/skills/attack-path-analysis/SKILL.md +1 -0
- package/_bundled_plugin/skills/cloud-findings/SKILL.md +54 -0
- package/_bundled_plugin/skills/cloud-findings/agents/openai.yaml +4 -0
- package/_bundled_plugin/skills/deep-security-scan/SKILL.md +3 -3
- package/_bundled_plugin/skills/finding-discovery/SKILL.md +3 -3
- package/_bundled_plugin/skills/propose-security-hardening/references/proposal-format.md +50 -20
- package/_bundled_plugin/skills/security-diff-scan/SKILL.md +8 -14
- package/_bundled_plugin/skills/security-scan/SKILL.md +22 -7
- package/_bundled_plugin/skills/security-scan/references/desktop-scan.md +2 -2
- package/_bundled_plugin/skills/security-scan/references/scan-artifacts-and-ledger.md +15 -1
- package/_bundled_plugin/skills/threat-model/SKILL.md +1 -1
- package/_bundled_plugin/skills/track-findings/SKILL.md +48 -176
- package/_bundled_plugin/skills/track-findings/references/github-security-advisories.md +16 -24
- package/_bundled_plugin/skills/track-findings/references/jira.md +15 -61
- package/_bundled_plugin/skills/triage-finding/SKILL.md +50 -289
- package/_bundled_plugin/skills/triage-finding/agents/openai.yaml +2 -2
- package/_bundled_plugin/skills/triage-finding/references/ticket-intake.md +23 -43
- package/_bundled_plugin/skills/triage-finding/references/triage-result-contract.md +7 -1
- package/bin/codex-security.mjs +9 -2
- package/dist/api.d.ts +63 -10
- package/dist/api.d.ts.map +1 -1
- package/dist/api.js +1015 -383
- package/dist/api.js.map +1 -1
- package/dist/artifact-export.d.ts +1 -1
- package/dist/artifact-export.d.ts.map +1 -1
- package/dist/artifact-export.js +28 -13
- package/dist/artifact-export.js.map +1 -1
- package/dist/auth.d.ts +2 -6
- package/dist/auth.d.ts.map +1 -1
- package/dist/auth.js +19 -9
- package/dist/auth.js.map +1 -1
- package/dist/bulk-scan-discovery.d.ts.map +1 -1
- package/dist/bulk-scan-discovery.js +1 -0
- package/dist/bulk-scan-discovery.js.map +1 -1
- package/dist/classify-severity.d.ts +26 -9
- package/dist/classify-severity.d.ts.map +1 -1
- package/dist/classify-severity.js +13 -30
- package/dist/classify-severity.js.map +1 -1
- package/dist/cli-help.d.ts.map +1 -1
- package/dist/cli-help.js +17 -13
- package/dist/cli-help.js.map +1 -1
- package/dist/cli-signals.d.ts +8 -0
- package/dist/cli-signals.d.ts.map +1 -0
- package/dist/cli-signals.js +28 -0
- package/dist/cli-signals.js.map +1 -0
- package/dist/cli.d.ts +4 -3
- package/dist/cli.d.ts.map +1 -1
- package/dist/cli.js +1102 -905
- package/dist/cli.js.map +1 -1
- package/dist/cloud-publish.d.ts.map +1 -1
- package/dist/cloud-publish.js +10 -9
- package/dist/cloud-publish.js.map +1 -1
- package/dist/codeowners.d.ts +19 -0
- package/dist/codeowners.d.ts.map +1 -0
- package/dist/codeowners.js +138 -0
- package/dist/codeowners.js.map +1 -0
- package/dist/codex-sdk-environment.d.ts +1 -0
- package/dist/codex-sdk-environment.d.ts.map +1 -1
- package/dist/codex-sdk-environment.js +40 -2
- package/dist/codex-sdk-environment.js.map +1 -1
- package/dist/component-plan.d.ts +2 -5
- package/dist/component-plan.d.ts.map +1 -1
- package/dist/component-plan.js +39 -31
- package/dist/component-plan.js.map +1 -1
- package/dist/component-scan.d.ts +6 -5
- package/dist/component-scan.d.ts.map +1 -1
- package/dist/component-scan.js +25 -36
- package/dist/component-scan.js.map +1 -1
- package/dist/config.d.ts.map +1 -1
- package/dist/config.js +64 -74
- package/dist/config.js.map +1 -1
- package/dist/contract.d.ts +16 -3
- package/dist/contract.d.ts.map +1 -1
- package/dist/contract.js +91 -255
- package/dist/contract.js.map +1 -1
- package/dist/cost-model.d.ts +13 -6
- package/dist/cost-model.d.ts.map +1 -1
- package/dist/cost-model.js +14 -4
- package/dist/cost-model.js.map +1 -1
- package/dist/cost.d.ts +10 -2
- package/dist/cost.d.ts.map +1 -1
- package/dist/cost.js +102 -104
- package/dist/cost.js.map +1 -1
- package/dist/custom-validation-prompt.d.ts.map +1 -1
- package/dist/custom-validation-prompt.js +6 -5
- package/dist/custom-validation-prompt.js.map +1 -1
- package/dist/custom-validation.d.ts +1 -2
- package/dist/custom-validation.d.ts.map +1 -1
- package/dist/custom-validation.js +9 -3
- package/dist/custom-validation.js.map +1 -1
- package/dist/deduplication/checkpointed-review.d.ts +4 -2
- package/dist/deduplication/checkpointed-review.d.ts.map +1 -1
- package/dist/deduplication/checkpointed-review.js +23 -9
- package/dist/deduplication/checkpointed-review.js.map +1 -1
- package/dist/deduplication/codex-review.d.ts +8 -6
- package/dist/deduplication/codex-review.d.ts.map +1 -1
- package/dist/deduplication/codex-review.js +128 -19
- package/dist/deduplication/codex-review.js.map +1 -1
- package/dist/deduplication/deduplication-prompts.d.ts +5 -2
- package/dist/deduplication/deduplication-prompts.d.ts.map +1 -1
- package/dist/deduplication/deduplication-prompts.js +18 -4
- package/dist/deduplication/deduplication-prompts.js.map +1 -1
- package/dist/deduplication/deduplication-reviewer.d.ts +27 -5
- package/dist/deduplication/deduplication-reviewer.d.ts.map +1 -1
- package/dist/deduplication/deduplication-reviewer.js +58 -48
- package/dist/deduplication/deduplication-reviewer.js.map +1 -1
- package/dist/deduplication/deduplication.d.ts.map +1 -1
- package/dist/deduplication/deduplication.js +19 -51
- package/dist/deduplication/deduplication.js.map +1 -1
- package/dist/deduplication/diagnostics.d.ts +17 -0
- package/dist/deduplication/diagnostics.d.ts.map +1 -0
- package/dist/deduplication/diagnostics.js +5 -0
- package/dist/deduplication/diagnostics.js.map +1 -0
- package/dist/deduplication/finding-schema.d.ts.map +1 -1
- package/dist/deduplication/finding-schema.js +7 -1
- package/dist/deduplication/finding-schema.js.map +1 -1
- package/dist/deduplication/local.d.ts +31 -0
- package/dist/deduplication/local.d.ts.map +1 -0
- package/dist/deduplication/local.js +166 -0
- package/dist/deduplication/local.js.map +1 -0
- package/dist/deduplication/records-protocol.d.ts +3 -8
- package/dist/deduplication/records-protocol.d.ts.map +1 -1
- package/dist/deduplication/records-protocol.js +11 -184
- package/dist/deduplication/records-protocol.js.map +1 -1
- package/dist/deduplication/records.d.ts +186 -11
- package/dist/deduplication/records.d.ts.map +1 -1
- package/dist/deduplication/records.js +10 -18
- package/dist/deduplication/records.js.map +1 -1
- package/dist/deduplication/scan.d.ts +9 -3
- package/dist/deduplication/scan.d.ts.map +1 -1
- package/dist/deduplication/scan.js +158 -39
- package/dist/deduplication/scan.js.map +1 -1
- package/dist/deep-config.d.ts +2 -1
- package/dist/deep-config.d.ts.map +1 -1
- package/dist/deep-config.js +4 -4
- package/dist/deep-config.js.map +1 -1
- package/dist/deep-progress.d.ts +17 -1
- package/dist/deep-progress.d.ts.map +1 -1
- package/dist/deep-progress.js +19 -17
- package/dist/deep-progress.js.map +1 -1
- package/dist/deep-scan.d.ts +20 -0
- package/dist/deep-scan.d.ts.map +1 -0
- package/dist/deep-scan.js +113 -0
- package/dist/deep-scan.js.map +1 -0
- package/dist/errors.d.ts +2 -0
- package/dist/errors.d.ts.map +1 -1
- package/dist/errors.js +2 -0
- package/dist/errors.js.map +1 -1
- package/dist/executable-path.d.ts +2 -0
- package/dist/executable-path.d.ts.map +1 -0
- package/dist/executable-path.js +12 -0
- package/dist/executable-path.js.map +1 -0
- package/dist/feedback.d.ts +1 -4
- package/dist/feedback.d.ts.map +1 -1
- package/dist/feedback.js +54 -22
- package/dist/feedback.js.map +1 -1
- package/dist/finding-catalogue.d.ts.map +1 -1
- package/dist/finding-catalogue.js +4 -8
- package/dist/finding-catalogue.js.map +1 -1
- package/dist/finding-normalization.d.ts +5 -0
- package/dist/finding-normalization.d.ts.map +1 -0
- package/dist/finding-normalization.js +120 -0
- package/dist/finding-normalization.js.map +1 -0
- package/dist/finding-retrieval.d.ts +9 -0
- package/dist/finding-retrieval.d.ts.map +1 -1
- package/dist/finding-retrieval.js.map +1 -1
- package/dist/finding-workflow.d.ts +13 -8
- package/dist/finding-workflow.d.ts.map +1 -1
- package/dist/finding-workflow.js +14 -6
- package/dist/finding-workflow.js.map +1 -1
- package/dist/findings-client.d.ts +2 -1
- package/dist/findings-client.d.ts.map +1 -1
- package/dist/findings-client.js +36 -9
- package/dist/findings-client.js.map +1 -1
- package/dist/findings-errors.d.ts +16 -0
- package/dist/findings-errors.d.ts.map +1 -0
- package/dist/findings-errors.js +24 -0
- package/dist/findings-errors.js.map +1 -0
- package/dist/findings-import.d.ts +1 -1
- package/dist/findings-import.d.ts.map +1 -1
- package/dist/findings-import.js +12 -30
- package/dist/findings-import.js.map +1 -1
- package/dist/github.d.ts.map +1 -1
- package/dist/github.js +7 -1
- package/dist/github.js.map +1 -1
- package/dist/host-protocol.d.ts +19 -0
- package/dist/host-protocol.d.ts.map +1 -0
- package/dist/host-protocol.js +270 -0
- package/dist/host-protocol.js.map +1 -0
- package/dist/hosted-scan-protocol.d.ts +4 -0
- package/dist/hosted-scan-protocol.d.ts.map +1 -0
- package/dist/hosted-scan-protocol.js +25 -0
- package/dist/hosted-scan-protocol.js.map +1 -0
- package/dist/hosted-scan.d.ts +117 -0
- package/dist/hosted-scan.d.ts.map +1 -0
- package/dist/hosted-scan.js +260 -0
- package/dist/hosted-scan.js.map +1 -0
- package/dist/import-scan.d.ts.map +1 -1
- package/dist/import-scan.js +48 -38
- package/dist/import-scan.js.map +1 -1
- package/dist/index.d.ts +11 -4
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +2 -1
- package/dist/index.js.map +1 -1
- package/dist/knowledge-base.d.ts +7 -1
- package/dist/knowledge-base.d.ts.map +1 -1
- package/dist/knowledge-base.js +114 -46
- package/dist/knowledge-base.js.map +1 -1
- package/dist/linear.d.ts.map +1 -1
- package/dist/linear.js +2 -7
- package/dist/linear.js.map +1 -1
- package/dist/mock-scan.d.ts.map +1 -1
- package/dist/mock-scan.js +0 -1
- package/dist/mock-scan.js.map +1 -1
- package/dist/models.d.ts +0 -2
- package/dist/models.d.ts.map +1 -1
- package/dist/models.js.map +1 -1
- package/dist/multiscan.d.ts +9 -3
- package/dist/multiscan.d.ts.map +1 -1
- package/dist/multiscan.js +240 -135
- package/dist/multiscan.js.map +1 -1
- package/dist/owner-evidence.d.ts +10 -2
- package/dist/owner-evidence.d.ts.map +1 -1
- package/dist/owner-evidence.js +67 -8
- package/dist/owner-evidence.js.map +1 -1
- package/dist/patch-tui.d.ts.map +1 -1
- package/dist/patch-tui.js +3 -2
- package/dist/patch-tui.js.map +1 -1
- package/dist/project-config.js +3 -3
- package/dist/project-config.js.map +1 -1
- package/dist/provider-profile.d.ts +21 -0
- package/dist/provider-profile.d.ts.map +1 -0
- package/dist/provider-profile.js +133 -0
- package/dist/provider-profile.js.map +1 -0
- package/dist/publication-events.d.ts +6 -8
- package/dist/publication-events.d.ts.map +1 -1
- package/dist/publication-events.js +25 -50
- package/dist/publication-events.js.map +1 -1
- package/dist/publication-store.d.ts.map +1 -1
- package/dist/publication-store.js +95 -109
- package/dist/publication-store.js.map +1 -1
- package/dist/publication.d.ts +1 -4
- package/dist/publication.d.ts.map +1 -1
- package/dist/publication.js +2 -13
- package/dist/publication.js.map +1 -1
- package/dist/publish.d.ts +1 -5
- package/dist/publish.d.ts.map +1 -1
- package/dist/publish.js +109 -160
- package/dist/publish.js.map +1 -1
- package/dist/request-metadata.d.ts +9 -0
- package/dist/request-metadata.d.ts.map +1 -0
- package/dist/request-metadata.js +13 -0
- package/dist/request-metadata.js.map +1 -0
- package/dist/runtime.d.ts +20 -6
- package/dist/runtime.d.ts.map +1 -1
- package/dist/runtime.js +407 -148
- package/dist/runtime.js.map +1 -1
- package/dist/saved-scan-bootstrap.d.ts +14 -0
- package/dist/saved-scan-bootstrap.d.ts.map +1 -0
- package/dist/saved-scan-bootstrap.js +261 -0
- package/dist/saved-scan-bootstrap.js.map +1 -0
- package/dist/saved-scan.d.ts +1 -1
- package/dist/saved-scan.d.ts.map +1 -1
- package/dist/saved-scan.js +6 -12
- package/dist/saved-scan.js.map +1 -1
- package/dist/scan-activity.js +11 -8
- package/dist/scan-activity.js.map +1 -1
- package/dist/scan-comparison.d.ts +28 -15
- package/dist/scan-comparison.d.ts.map +1 -1
- package/dist/scan-comparison.js +276 -281
- package/dist/scan-comparison.js.map +1 -1
- package/dist/scan-dashboard.d.ts +3 -0
- package/dist/scan-dashboard.d.ts.map +1 -1
- package/dist/scan-dashboard.js +345 -77
- package/dist/scan-dashboard.js.map +1 -1
- package/dist/scan-executor.d.ts +104 -0
- package/dist/scan-executor.d.ts.map +1 -0
- package/dist/scan-executor.js +62 -0
- package/dist/scan-executor.js.map +1 -0
- package/dist/scan-history-renderer.d.ts.map +1 -1
- package/dist/scan-history-renderer.js +54 -13
- package/dist/scan-history-renderer.js.map +1 -1
- package/dist/scan-inputs.d.ts +6 -0
- package/dist/scan-inputs.d.ts.map +1 -0
- package/dist/scan-inputs.js +71 -0
- package/dist/scan-inputs.js.map +1 -0
- package/dist/scan-logs.d.ts +2 -3
- package/dist/scan-logs.d.ts.map +1 -1
- package/dist/scan-logs.js +12 -6
- package/dist/scan-logs.js.map +1 -1
- package/dist/scan-sessions.d.ts +4 -0
- package/dist/scan-sessions.d.ts.map +1 -1
- package/dist/scan-sessions.js +17 -0
- package/dist/scan-sessions.js.map +1 -1
- package/dist/scan-skill-paths.d.ts +8 -0
- package/dist/scan-skill-paths.d.ts.map +1 -0
- package/dist/scan-skill-paths.js +47 -0
- package/dist/scan-skill-paths.js.map +1 -0
- package/dist/security-policy-cli.d.ts.map +1 -1
- package/dist/security-policy-cli.js +7 -27
- package/dist/security-policy-cli.js.map +1 -1
- package/dist/security-policy.d.ts +9 -5
- package/dist/security-policy.d.ts.map +1 -1
- package/dist/security-policy.js +33 -14
- package/dist/security-policy.js.map +1 -1
- package/dist/server/api.d.ts +0 -1
- package/dist/server/api.d.ts.map +1 -1
- package/dist/server/api.js +0 -1
- package/dist/server/api.js.map +1 -1
- package/dist/server/embeddings.d.ts +3 -1
- package/dist/server/embeddings.d.ts.map +1 -1
- package/dist/server/embeddings.js +10 -3
- package/dist/server/embeddings.js.map +1 -1
- package/dist/server/errors.d.ts +5 -2
- package/dist/server/errors.d.ts.map +1 -1
- package/dist/server/errors.js +7 -0
- package/dist/server/errors.js.map +1 -1
- package/dist/server/sqlite-store.d.ts +0 -2
- package/dist/server/sqlite-store.d.ts.map +1 -1
- package/dist/server/sqlite-store.js +54 -44
- package/dist/server/sqlite-store.js.map +1 -1
- package/dist/server/storage.d.ts +0 -2
- package/dist/server/storage.d.ts.map +1 -1
- package/dist/server/storage.js.map +1 -1
- package/dist/severity-store.js +19 -24
- package/dist/severity-store.js.map +1 -1
- package/dist/standalone-validation-prompt.d.ts +2 -0
- package/dist/standalone-validation-prompt.d.ts.map +1 -0
- package/dist/standalone-validation-prompt.js +10 -0
- package/dist/standalone-validation-prompt.js.map +1 -0
- package/dist/suggest-owners.d.ts +4 -2
- package/dist/suggest-owners.d.ts.map +1 -1
- package/dist/suggest-owners.js +11 -5
- package/dist/suggest-owners.js.map +1 -1
- package/dist/targets.d.ts +11 -0
- package/dist/targets.d.ts.map +1 -1
- package/dist/targets.js +123 -72
- package/dist/targets.js.map +1 -1
- package/dist/trusted-executable.d.ts +2 -2
- package/dist/trusted-executable.d.ts.map +1 -1
- package/dist/trusted-executable.js +8 -6
- package/dist/trusted-executable.js.map +1 -1
- package/dist/value.js +7 -0
- package/dist/value.js.map +1 -1
- package/dist/version.js.map +1 -1
- package/dist/workbench-arguments.d.ts +3 -0
- package/dist/workbench-arguments.d.ts.map +1 -0
- package/dist/workbench-arguments.js +15 -0
- package/dist/workbench-arguments.js.map +1 -0
- package/dist/worker-progress.d.ts +44 -11
- package/dist/worker-progress.d.ts.map +1 -1
- package/dist/worker-progress.js +48 -49
- package/dist/worker-progress.js.map +1 -1
- package/docs/cli.md +1788 -0
- package/docs/findings-service.md +402 -0
- package/docs/hosted-scans.md +242 -0
- package/package.json +23 -22
- package/schemas/hosted-scan-v2.fixture.json +145 -0
- package/_bundled_plugin/schemas/tools/worker-threat-model.schema.json +0 -18
- package/_bundled_plugin/scripts/workbench_dashboard.py +0 -159
- package/_bundled_plugin/scripts/workbench_findings.py +0 -221
- package/_bundled_plugin/scripts/workbench_severity.py +0 -122
- package/_bundled_plugin/skills/assess-patch-risk/scripts/validate_patch_risk_assessment.py +0 -163
- package/dist/server/dashboard/THIRD_PARTY_NOTICES.txt +0 -276
- package/dist/server/dashboard/app.css +0 -2
- package/dist/server/dashboard/app.js +0 -66
- package/dist/server/dashboard/index.html +0 -18
- package/dist/server/dashboard-types.d.ts +0 -48
- package/dist/server/dashboard-types.d.ts.map +0 -1
- package/dist/server/dashboard-types.js +0 -2
- package/dist/server/dashboard-types.js.map +0 -1
- package/dist/server/dashboard.d.ts +0 -6
- package/dist/server/dashboard.d.ts.map +0 -1
- package/dist/server/dashboard.js +0 -59
- package/dist/server/dashboard.js.map +0 -1
- package/dist/server/index.d.ts +0 -2
- package/dist/server/index.d.ts.map +0 -1
- package/dist/server/index.js +0 -6
- package/dist/server/index.js.map +0 -1
- package/dist/server/routes.d.ts +0 -7
- package/dist/server/routes.d.ts.map +0 -1
- package/dist/server/routes.js +0 -98
- package/dist/server/routes.js.map +0 -1
- package/dist/server/serve.d.ts +0 -2
- package/dist/server/serve.d.ts.map +0 -1
- package/dist/server/serve.js +0 -28
- package/dist/server/serve.js.map +0 -1
- package/dist/server/server.d.ts +0 -10
- package/dist/server/server.d.ts.map +0 -1
- package/dist/server/server.js +0 -16
- package/dist/server/server.js.map +0 -1
- package/dist/server/validation.d.ts +0 -17
- package/dist/server/validation.d.ts.map +0 -1
- package/dist/server/validation.js +0 -53
- package/dist/server/validation.js.map +0 -1
|
@@ -5,10 +5,12 @@ from __future__ import annotations
|
|
|
5
5
|
import argparse
|
|
6
6
|
import json
|
|
7
7
|
import os
|
|
8
|
+
import re
|
|
8
9
|
import sqlite3
|
|
9
10
|
import sys
|
|
10
11
|
import tempfile
|
|
11
|
-
from collections.abc import Callable
|
|
12
|
+
from collections.abc import Callable, Iterator
|
|
13
|
+
from contextlib import contextmanager
|
|
12
14
|
from datetime import datetime, timezone
|
|
13
15
|
from pathlib import Path
|
|
14
16
|
|
|
@@ -16,6 +18,7 @@ from pathlib import Path
|
|
|
16
18
|
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
|
17
19
|
from filesystem_identity import serialize_filesystem_identity
|
|
18
20
|
from finalize_scan_contract import write_scan_local_bytes
|
|
21
|
+
from workbench.storage import state_dir
|
|
19
22
|
from workbench_feedback import get_scan_feedback
|
|
20
23
|
from workbench_target import (
|
|
21
24
|
directory_content_digest,
|
|
@@ -26,10 +29,7 @@ from workbench_validation import optional_text
|
|
|
26
29
|
|
|
27
30
|
|
|
28
31
|
def safe_segment(value: str) -> str:
|
|
29
|
-
|
|
30
|
-
character if character.isalnum() or character in "._-" else "-" for character in value
|
|
31
|
-
)
|
|
32
|
-
return segment.strip("-") or "scan"
|
|
32
|
+
return re.sub(r"[^\w.-]", "-", value).strip("-") or "scan"
|
|
33
33
|
|
|
34
34
|
|
|
35
35
|
def compact_timestamp() -> str:
|
|
@@ -86,13 +86,16 @@ def stored_diff_target(row: sqlite3.Row) -> dict[str, str] | None:
|
|
|
86
86
|
return target
|
|
87
87
|
|
|
88
88
|
|
|
89
|
+
@contextmanager
|
|
89
90
|
def archive_scan(
|
|
90
91
|
connection: sqlite3.Connection,
|
|
91
92
|
args: argparse.Namespace,
|
|
92
93
|
scan_dir: Path,
|
|
93
94
|
timestamp: str,
|
|
94
95
|
canonical_directory: Callable[[Path], Path],
|
|
95
|
-
|
|
96
|
+
*,
|
|
97
|
+
before_archive: Callable[[], None] | None = None,
|
|
98
|
+
) -> Iterator[Path | None]:
|
|
96
99
|
archived_scan_dir = (
|
|
97
100
|
canonical_directory(Path(args.archived_scan_dir).expanduser())
|
|
98
101
|
if args.archived_scan_dir is not None
|
|
@@ -108,44 +111,105 @@ def archive_scan(
|
|
|
108
111
|
previous_scan = connection.execute(
|
|
109
112
|
"SELECT id, status FROM scans WHERE scan_dir = ?", (str(scan_dir),)
|
|
110
113
|
).fetchone()
|
|
111
|
-
if previous_scan is None:
|
|
112
|
-
|
|
113
|
-
if not args.archive_existing:
|
|
114
|
-
raise SystemExit(
|
|
115
|
-
"The scan artifact directory belongs to an existing scan. "
|
|
116
|
-
"Use --archive-existing to preserve that scan and start a new one."
|
|
117
|
-
)
|
|
118
|
-
if previous_scan["status"] == "running":
|
|
119
|
-
raise SystemExit("Cannot archive the output of a running scan.")
|
|
120
|
-
artifacts = connection.execute(
|
|
121
|
-
"SELECT kind, path FROM scan_artifacts WHERE scan_id = ?",
|
|
122
|
-
(previous_scan["id"],),
|
|
123
|
-
).fetchall()
|
|
124
|
-
if archived_scan_dir is None:
|
|
125
|
-
if artifacts:
|
|
114
|
+
if previous_scan is not None:
|
|
115
|
+
if not args.archive_existing:
|
|
126
116
|
raise SystemExit(
|
|
127
|
-
"The
|
|
117
|
+
"The scan artifact directory belongs to an existing scan. "
|
|
118
|
+
"Use --archive-existing to preserve that scan and start a new one."
|
|
128
119
|
)
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
"
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
relative_path = Path(artifact["path"]).relative_to(scan_dir)
|
|
139
|
-
except ValueError:
|
|
140
|
-
continue
|
|
120
|
+
if previous_scan["status"] == "running":
|
|
121
|
+
raise SystemExit("Cannot archive the output of a running scan.")
|
|
122
|
+
has_contents = next(scan_dir.iterdir(), None) is not None
|
|
123
|
+
if has_contents and (not args.archive_existing or archived_scan_dir is not None):
|
|
124
|
+
raise SystemExit("The scan artifact directory must be empty before the scan starts.")
|
|
125
|
+
if previous_scan is None and not (args.archive_existing and has_contents):
|
|
126
|
+
yield archived_scan_dir
|
|
127
|
+
return
|
|
128
|
+
artifacts = (
|
|
141
129
|
connection.execute(
|
|
142
|
-
"
|
|
143
|
-
|
|
144
|
-
|
|
145
|
-
|
|
146
|
-
|
|
147
|
-
|
|
148
|
-
|
|
130
|
+
"SELECT kind, path FROM scan_artifacts WHERE scan_id = ?", (previous_scan["id"],)
|
|
131
|
+
).fetchall()
|
|
132
|
+
if previous_scan is not None
|
|
133
|
+
else []
|
|
134
|
+
)
|
|
135
|
+
moved = False
|
|
136
|
+
try:
|
|
137
|
+
if archived_scan_dir is None:
|
|
138
|
+
configured_state = state_dir(canonical=False)
|
|
139
|
+
protected_directories = [
|
|
140
|
+
*(path.resolve() for path in (configured_state, *configured_state.parents)),
|
|
141
|
+
*(
|
|
142
|
+
Path(row[2]).resolve().parent
|
|
143
|
+
for row in connection.execute("PRAGMA database_list")
|
|
144
|
+
if row[2]
|
|
145
|
+
),
|
|
146
|
+
]
|
|
147
|
+
if any(path == scan_dir or scan_dir in path.parents for path in protected_directories):
|
|
148
|
+
raise SystemExit(
|
|
149
|
+
"Cannot archive output containing the workbench state or active database."
|
|
150
|
+
)
|
|
151
|
+
if artifacts and not has_contents:
|
|
152
|
+
raise SystemExit(
|
|
153
|
+
"The archived scan directory is required to preserve existing scan artifacts."
|
|
154
|
+
)
|
|
155
|
+
if before_archive is not None:
|
|
156
|
+
before_archive()
|
|
157
|
+
archived_scan_dir = Path(
|
|
158
|
+
tempfile.mkdtemp(prefix=f"{scan_dir.name}.previous-", dir=scan_dir.parent)
|
|
159
|
+
).resolve()
|
|
160
|
+
archived_scan_dir.rmdir()
|
|
161
|
+
scan_dir.rename(archived_scan_dir)
|
|
162
|
+
moved = True
|
|
163
|
+
scan_dir.mkdir(mode=0o700)
|
|
164
|
+
scan_dir.chmod(0o700)
|
|
165
|
+
if previous_scan is not None:
|
|
166
|
+
connection.execute(
|
|
167
|
+
"UPDATE scans SET scan_dir = ?, updated_at = ? WHERE id = ?",
|
|
168
|
+
(str(archived_scan_dir), timestamp, previous_scan["id"]),
|
|
169
|
+
)
|
|
170
|
+
connection.execute(
|
|
171
|
+
"""UPDATE finding_workflows SET scan_dir = ?,
|
|
172
|
+
results_json = CASE WHEN json_extract(results_json, '$.scan.sarifPath') = ?
|
|
173
|
+
THEN json_set(results_json, '$.scan.sarifPath', ?) ELSE results_json END
|
|
174
|
+
WHERE scan_id = ? AND scan_dir = ?""",
|
|
175
|
+
(
|
|
176
|
+
str(archived_scan_dir),
|
|
177
|
+
str(scan_dir / "exports" / "results.sarif"),
|
|
178
|
+
str(archived_scan_dir / "exports" / "results.sarif"),
|
|
179
|
+
previous_scan["id"],
|
|
180
|
+
str(scan_dir),
|
|
181
|
+
),
|
|
182
|
+
)
|
|
183
|
+
for artifact in artifacts:
|
|
184
|
+
try:
|
|
185
|
+
relative_path = Path(artifact["path"]).relative_to(scan_dir)
|
|
186
|
+
except ValueError:
|
|
187
|
+
continue
|
|
188
|
+
connection.execute(
|
|
189
|
+
"UPDATE scan_artifacts SET path = ? WHERE scan_id = ? AND kind = ?",
|
|
190
|
+
(str(archived_scan_dir / relative_path), previous_scan["id"], artifact["kind"]),
|
|
191
|
+
)
|
|
192
|
+
# The caller commits registration before leaving this context. Older SDKs
|
|
193
|
+
# supplied an already-moved directory; only restore moves we own.
|
|
194
|
+
yield archived_scan_dir
|
|
195
|
+
except BaseException:
|
|
196
|
+
if moved:
|
|
197
|
+
if not connection.in_transaction:
|
|
198
|
+
current_owner = connection.execute(
|
|
199
|
+
"SELECT id FROM scans WHERE scan_dir = ?", (str(scan_dir),)
|
|
200
|
+
).fetchone()
|
|
201
|
+
if current_owner is not None and (
|
|
202
|
+
previous_scan is None or current_owner["id"] != previous_scan["id"]
|
|
203
|
+
):
|
|
204
|
+
# A post-commit interrupt must not undo the saved archive location.
|
|
205
|
+
raise
|
|
206
|
+
try:
|
|
207
|
+
scan_dir.rmdir()
|
|
208
|
+
except FileNotFoundError:
|
|
209
|
+
# Registration may fail before recreating the empty output directory.
|
|
210
|
+
pass
|
|
211
|
+
archived_scan_dir.rename(scan_dir)
|
|
212
|
+
raise
|
|
149
213
|
|
|
150
214
|
|
|
151
215
|
def insert_running_scan(
|
|
@@ -13,6 +13,7 @@ from dataclasses import dataclass
|
|
|
13
13
|
from datetime import datetime, timezone
|
|
14
14
|
from pathlib import Path
|
|
15
15
|
from typing import Any, Mapping
|
|
16
|
+
from urllib.parse import quote
|
|
16
17
|
|
|
17
18
|
TOKEN_FIELDS = {
|
|
18
19
|
"input_tokens": "inputTokens",
|
|
@@ -83,11 +84,13 @@ def collect_scan_usage(
|
|
|
83
84
|
*,
|
|
84
85
|
thread_id: str | None = None,
|
|
85
86
|
completed_at: str | None = None,
|
|
87
|
+
completed_turn_id: str | None = None,
|
|
86
88
|
) -> dict[str, Any]:
|
|
87
89
|
"""Count only complete, attributable rollout events inside this scan's window."""
|
|
88
90
|
|
|
89
91
|
roots = _scan_root_thread_ids(connection, scan, thread_id)
|
|
90
|
-
|
|
92
|
+
worker_directories = _scan_worker_directories(connection, scan)
|
|
93
|
+
if not roots and not worker_directories:
|
|
91
94
|
return _unavailable_usage("scan_thread_unavailable")
|
|
92
95
|
|
|
93
96
|
state_database = _codex_state_database()
|
|
@@ -105,6 +108,9 @@ def collect_scan_usage(
|
|
|
105
108
|
state_database,
|
|
106
109
|
roots,
|
|
107
110
|
warnings,
|
|
111
|
+
worker_directories=worker_directories,
|
|
112
|
+
started_at=started_at,
|
|
113
|
+
completed_at=stopped_at,
|
|
108
114
|
)
|
|
109
115
|
except (OSError, sqlite3.Error, ValueError):
|
|
110
116
|
return _unavailable_usage("codex_state_unavailable")
|
|
@@ -131,6 +137,7 @@ def collect_scan_usage(
|
|
|
131
137
|
session,
|
|
132
138
|
started_at=started_at,
|
|
133
139
|
completed_at=stopped_at,
|
|
140
|
+
completed_turn_id=completed_turn_id if session.thread_id == thread_id else None,
|
|
134
141
|
)
|
|
135
142
|
except (OSError, UnicodeError, ValueError):
|
|
136
143
|
missing_thread_ids.add(session.thread_id)
|
|
@@ -213,23 +220,40 @@ def _scan_execution_thread_ids(connection: sqlite3.Connection, scan: sqlite3.Row
|
|
|
213
220
|
)
|
|
214
221
|
|
|
215
222
|
|
|
223
|
+
def _scan_worker_directories(connection: sqlite3.Connection, scan: sqlite3.Row) -> set[str]:
|
|
224
|
+
if scan["mode"] != "deep":
|
|
225
|
+
return set()
|
|
226
|
+
return {
|
|
227
|
+
row["artifact_dir"]
|
|
228
|
+
if row["kind"] == "discovery"
|
|
229
|
+
else str(Path(scan["scan_dir"]) / "artifacts")
|
|
230
|
+
for row in connection.execute(
|
|
231
|
+
"SELECT kind, artifact_dir FROM deep_scan_workers "
|
|
232
|
+
"WHERE scan_id = ? AND kind IN ('discovery', 'dedup')",
|
|
233
|
+
(scan["id"],),
|
|
234
|
+
)
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
|
|
216
238
|
def _codex_state_database() -> Path | None:
|
|
217
239
|
configured_database = os.environ.get("CODEX_STATE_DB", "").strip()
|
|
218
240
|
if configured_database:
|
|
219
241
|
path = Path(configured_database).expanduser()
|
|
220
242
|
return path.resolve() if path.is_file() and os.access(path, os.R_OK) else None
|
|
221
243
|
|
|
222
|
-
configured_home = os.environ.get("CODEX_HOME", "")
|
|
223
|
-
codex_home = Path(configured_home)
|
|
224
|
-
configured_sqlite_home = os.environ.get("CODEX_SQLITE_HOME", "")
|
|
244
|
+
configured_home = os.environ.get("CODEX_HOME", "")
|
|
245
|
+
codex_home = Path(configured_home) if configured_home.strip() else Path.home() / ".codex"
|
|
246
|
+
configured_sqlite_home = os.environ.get("CODEX_SQLITE_HOME", "")
|
|
225
247
|
search_roots = [
|
|
226
|
-
*([Path(configured_sqlite_home)
|
|
248
|
+
*([Path(configured_sqlite_home)] if configured_sqlite_home.strip() else []),
|
|
227
249
|
codex_home,
|
|
228
250
|
codex_home / "sqlite",
|
|
229
251
|
]
|
|
230
252
|
seen: set[Path] = set()
|
|
231
253
|
for search_root in search_roots:
|
|
232
254
|
try:
|
|
255
|
+
if search_root.parts[:1] == ("~",):
|
|
256
|
+
search_root = search_root.expanduser()
|
|
233
257
|
resolved_root = search_root.resolve()
|
|
234
258
|
if resolved_root in seen:
|
|
235
259
|
continue
|
|
@@ -252,25 +276,49 @@ def _discover_rollout_sessions(
|
|
|
252
276
|
state_database: Path,
|
|
253
277
|
roots: list[str],
|
|
254
278
|
warnings: set[str],
|
|
279
|
+
*,
|
|
280
|
+
worker_directories: set[str],
|
|
281
|
+
started_at: datetime,
|
|
282
|
+
completed_at: datetime | None,
|
|
255
283
|
) -> tuple[list[RolloutSession], set[str]]:
|
|
256
284
|
database = sqlite3.connect(
|
|
257
|
-
state_database
|
|
285
|
+
f"file:{quote(str(state_database), safe='')}?mode=ro",
|
|
258
286
|
uri=True,
|
|
259
287
|
timeout=STATE_DATABASE_TIMEOUT_SECONDS,
|
|
260
288
|
)
|
|
261
289
|
try:
|
|
262
290
|
database.row_factory = sqlite3.Row
|
|
263
291
|
database.execute("PRAGMA query_only = ON")
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
267
|
-
|
|
268
|
-
{"
|
|
269
|
-
|
|
270
|
-
|
|
271
|
-
|
|
292
|
+
for table, required in (
|
|
293
|
+
("threads", {"id", "rollout_path"}),
|
|
294
|
+
("thread_spawn_edges", {"parent_thread_id", "child_thread_id"}),
|
|
295
|
+
):
|
|
296
|
+
columns = {str(row["name"]) for row in database.execute(f"PRAGMA table_info({table})")}
|
|
297
|
+
if not required.issubset(columns):
|
|
298
|
+
raise ValueError("Codex state graph does not expose the required thread columns.")
|
|
299
|
+
sessions: dict[str, RolloutSession] = {}
|
|
272
300
|
missing_thread_ids: set[str] = set()
|
|
273
|
-
|
|
301
|
+
root_ids = dict.fromkeys(roots)
|
|
302
|
+
started_millisecond = started_at.replace(microsecond=started_at.microsecond // 1000 * 1000)
|
|
303
|
+
completed_timestamp = int(completed_at.timestamp()) if completed_at is not None else None
|
|
304
|
+
directory_expression = "cwd"
|
|
305
|
+
if worker_directories and os.name == "nt":
|
|
306
|
+
from windows_scan_local_files import _normalized_windows_path
|
|
307
|
+
|
|
308
|
+
database.create_function(
|
|
309
|
+
"codex_security_path_key", 1, _normalized_windows_path, deterministic=True
|
|
310
|
+
)
|
|
311
|
+
directory_expression = "codex_security_path_key(cwd)"
|
|
312
|
+
worker_directories = {_normalized_windows_path(value) for value in worker_directories}
|
|
313
|
+
for directory in sorted(worker_directories):
|
|
314
|
+
for row in database.execute(
|
|
315
|
+
"SELECT id FROM threads WHERE cwd IS NOT NULL "
|
|
316
|
+
f"AND {directory_expression} = ? AND created_at >= ? "
|
|
317
|
+
"AND (? IS NULL OR created_at <= ?) ORDER BY id",
|
|
318
|
+
(directory, int(started_at.timestamp()), completed_timestamp, completed_timestamp),
|
|
319
|
+
):
|
|
320
|
+
root_ids[row["id"]] = None
|
|
321
|
+
for root in root_ids:
|
|
274
322
|
row = database.execute(
|
|
275
323
|
"SELECT id, rollout_path FROM threads WHERE id = ?",
|
|
276
324
|
(root,),
|
|
@@ -279,14 +327,29 @@ def _discover_rollout_sessions(
|
|
|
279
327
|
missing_thread_ids.add(root)
|
|
280
328
|
warnings.add("scan_root_unavailable")
|
|
281
329
|
continue
|
|
282
|
-
|
|
283
|
-
|
|
284
|
-
|
|
330
|
+
path = _rollout_path(row["rollout_path"])
|
|
331
|
+
if path is None:
|
|
332
|
+
missing_thread_ids.add(root)
|
|
333
|
+
warnings.add("rollout_unavailable")
|
|
334
|
+
continue
|
|
335
|
+
if root not in roots:
|
|
336
|
+
try:
|
|
337
|
+
with path.open("rb") as source:
|
|
338
|
+
metadata = json.loads(source.readline())
|
|
339
|
+
except (OSError, ValueError):
|
|
285
340
|
missing_thread_ids.add(root)
|
|
286
341
|
warnings.add("rollout_unavailable")
|
|
287
342
|
continue
|
|
288
|
-
|
|
289
|
-
|
|
343
|
+
if (
|
|
344
|
+
isinstance(metadata, dict)
|
|
345
|
+
and metadata.get("type") == "session_meta"
|
|
346
|
+
and isinstance(payload := metadata.get("payload"), dict)
|
|
347
|
+
and (payload.get("id") or payload.get("session_id")) == root
|
|
348
|
+
and (created_at := _timestamp(payload.get("timestamp"))) is not None
|
|
349
|
+
and created_at < started_millisecond
|
|
350
|
+
):
|
|
351
|
+
continue
|
|
352
|
+
sessions[root] = RolloutSession(root, None, path)
|
|
290
353
|
descendants = database.execute(
|
|
291
354
|
"""
|
|
292
355
|
WITH RECURSIVE descendants(
|
|
@@ -335,34 +398,19 @@ def _discover_rollout_sessions(
|
|
|
335
398
|
missing_thread_ids.add(child_id)
|
|
336
399
|
warnings.add("thread_lineage_cycle")
|
|
337
400
|
continue
|
|
338
|
-
if child_id in
|
|
401
|
+
if child_id in sessions:
|
|
339
402
|
continue
|
|
340
403
|
path = _rollout_path(descendant["rollout_path"])
|
|
341
404
|
if path is None:
|
|
342
405
|
missing_thread_ids.add(child_id)
|
|
343
406
|
warnings.add("rollout_unavailable")
|
|
344
407
|
continue
|
|
345
|
-
sessions
|
|
346
|
-
|
|
347
|
-
return sessions, missing_thread_ids
|
|
408
|
+
sessions[child_id] = RolloutSession(child_id, parent_id, path)
|
|
409
|
+
return list(sessions.values()), missing_thread_ids
|
|
348
410
|
finally:
|
|
349
411
|
database.close()
|
|
350
412
|
|
|
351
413
|
|
|
352
|
-
def _require_state_columns(
|
|
353
|
-
connection: sqlite3.Connection,
|
|
354
|
-
table: str,
|
|
355
|
-
required: set[str],
|
|
356
|
-
) -> None:
|
|
357
|
-
statements = {
|
|
358
|
-
"threads": "PRAGMA table_info(threads)",
|
|
359
|
-
"thread_spawn_edges": "PRAGMA table_info(thread_spawn_edges)",
|
|
360
|
-
}
|
|
361
|
-
columns = {str(row["name"]) for row in connection.execute(statements[table])}
|
|
362
|
-
if not required.issubset(columns):
|
|
363
|
-
raise ValueError("Codex state graph does not expose the required thread columns.")
|
|
364
|
-
|
|
365
|
-
|
|
366
414
|
def _rollout_path(value: object) -> Path | None:
|
|
367
415
|
if not isinstance(value, str) or not value:
|
|
368
416
|
return None
|
|
@@ -391,11 +439,18 @@ def _read_rollout_usage(
|
|
|
391
439
|
*,
|
|
392
440
|
started_at: datetime,
|
|
393
441
|
completed_at: datetime | None,
|
|
442
|
+
completed_turn_id: str | None = None,
|
|
394
443
|
) -> tuple[dict[str, int], set[str]]:
|
|
395
444
|
total = _empty_token_usage()
|
|
396
445
|
warnings: set[str] = set()
|
|
397
446
|
previous = _empty_token_usage()
|
|
398
447
|
boundary_reached = False
|
|
448
|
+
legacy_usage = _empty_token_usage()
|
|
449
|
+
response_usage: dict[str, int] | None = None
|
|
450
|
+
observed_response_usage = _empty_token_usage()
|
|
451
|
+
response_ids: set[str] = set()
|
|
452
|
+
current_turn_id: str | None = None
|
|
453
|
+
completed_turn_measured = False
|
|
399
454
|
|
|
400
455
|
with session.path.open("rb") as source:
|
|
401
456
|
for line_number, raw_line in enumerate(source, start=1):
|
|
@@ -444,18 +499,104 @@ def _read_rollout_usage(
|
|
|
444
499
|
warnings.add("thread_ownership_unavailable")
|
|
445
500
|
return total, warnings
|
|
446
501
|
if task_started_at < started_at or (
|
|
447
|
-
completed_at is not None
|
|
502
|
+
completed_at is not None
|
|
503
|
+
and task_started_at > completed_at
|
|
504
|
+
and payload.get("turn_id") != completed_turn_id
|
|
448
505
|
):
|
|
449
506
|
warnings.add("thread_outside_scan_window")
|
|
450
507
|
return total, warnings
|
|
451
508
|
boundary_reached = True
|
|
509
|
+
current_turn_id = payload.get("turn_id")
|
|
452
510
|
elif event.get("type") == "event_msg" and payload.get("type") == "token_count":
|
|
453
511
|
inherited_usage = _token_snapshot(payload)
|
|
454
512
|
if inherited_usage is not None:
|
|
455
513
|
previous = inherited_usage
|
|
456
514
|
continue
|
|
515
|
+
if event.get("type") == "event_msg" and payload.get("type") == "task_started":
|
|
516
|
+
if (
|
|
517
|
+
completed_turn_id is not None
|
|
518
|
+
and current_turn_id == completed_turn_id
|
|
519
|
+
and payload.get("turn_id") != completed_turn_id
|
|
520
|
+
):
|
|
521
|
+
break
|
|
522
|
+
_add_token_usage(
|
|
523
|
+
total, response_usage if response_usage is not None else legacy_usage
|
|
524
|
+
)
|
|
525
|
+
legacy_usage = _empty_token_usage()
|
|
526
|
+
response_usage = None
|
|
527
|
+
if payload.get("turn_id") != current_turn_id:
|
|
528
|
+
observed_response_usage = _empty_token_usage()
|
|
529
|
+
current_turn_id = payload.get("turn_id")
|
|
530
|
+
continue
|
|
531
|
+
if (
|
|
532
|
+
completed_turn_id is not None
|
|
533
|
+
and event.get("type") == "event_msg"
|
|
534
|
+
and payload.get("type") == "task_complete"
|
|
535
|
+
and payload.get("turn_id") == completed_turn_id
|
|
536
|
+
):
|
|
537
|
+
break
|
|
538
|
+
if event.get("type") == "compacted":
|
|
539
|
+
timestamp = _timestamp(event.get("timestamp"))
|
|
540
|
+
if (
|
|
541
|
+
timestamp is not None
|
|
542
|
+
and timestamp >= started_at
|
|
543
|
+
and (
|
|
544
|
+
completed_at is None
|
|
545
|
+
or timestamp <= completed_at
|
|
546
|
+
or (completed_turn_id is not None and current_turn_id == completed_turn_id)
|
|
547
|
+
)
|
|
548
|
+
and "compaction_response_id" not in payload
|
|
549
|
+
and response_usage is None
|
|
550
|
+
):
|
|
551
|
+
warnings.add("compaction_usage_unavailable")
|
|
552
|
+
continue
|
|
553
|
+
if event.get("type") == "token_usage_record":
|
|
554
|
+
if not isinstance(payload.get("thread_id"), str):
|
|
555
|
+
warnings.add("token_record_invalid")
|
|
556
|
+
continue
|
|
557
|
+
if payload.get("thread_id") != session.thread_id:
|
|
558
|
+
continue
|
|
559
|
+
timestamp = _timestamp(event.get("timestamp"))
|
|
560
|
+
usage = _usage_counts(payload.get("usage"))
|
|
561
|
+
response_id = payload.get("response_id")
|
|
562
|
+
if (
|
|
563
|
+
timestamp is None
|
|
564
|
+
or usage is None
|
|
565
|
+
or not isinstance(response_id, str)
|
|
566
|
+
or not response_id
|
|
567
|
+
):
|
|
568
|
+
warnings.add("token_record_invalid")
|
|
569
|
+
continue
|
|
570
|
+
is_completion_turn = (
|
|
571
|
+
completed_turn_id is not None and payload.get("turn_id") == completed_turn_id
|
|
572
|
+
)
|
|
573
|
+
if completed_at is not None and timestamp > completed_at and not is_completion_turn:
|
|
574
|
+
continue
|
|
575
|
+
if response_usage is None:
|
|
576
|
+
response_usage = _empty_token_usage()
|
|
577
|
+
if current_turn_id is None:
|
|
578
|
+
current_turn_id = payload.get("turn_id")
|
|
579
|
+
if response_id in response_ids:
|
|
580
|
+
continue
|
|
581
|
+
response_ids.add(response_id)
|
|
582
|
+
_add_token_usage(observed_response_usage, usage)
|
|
583
|
+
turn_usage = _usage_counts(payload.get("turn_token_usage"))
|
|
584
|
+
if timestamp < started_at:
|
|
585
|
+
if turn_usage is not None:
|
|
586
|
+
observed_response_usage = turn_usage
|
|
587
|
+
continue
|
|
588
|
+
if is_completion_turn:
|
|
589
|
+
completed_turn_measured = True
|
|
590
|
+
if turn_usage is not None and any(
|
|
591
|
+
turn_usage[key] > observed_response_usage[key] for key in TOKEN_FIELDS.values()
|
|
592
|
+
):
|
|
593
|
+
warnings.add("token_record_incomplete")
|
|
594
|
+
_add_token_usage(response_usage, usage)
|
|
595
|
+
continue
|
|
457
596
|
if event.get("type") != "event_msg" or payload.get("type") != "token_count":
|
|
458
597
|
continue
|
|
598
|
+
if "info" in payload and payload["info"] is None:
|
|
599
|
+
continue
|
|
459
600
|
timestamp = _timestamp(event.get("timestamp"))
|
|
460
601
|
snapshot = _token_snapshot(payload)
|
|
461
602
|
if timestamp is None or snapshot is None:
|
|
@@ -468,14 +609,22 @@ def _read_rollout_usage(
|
|
|
468
609
|
previous = snapshot
|
|
469
610
|
if timestamp < started_at:
|
|
470
611
|
continue
|
|
471
|
-
|
|
612
|
+
is_completion_turn = (
|
|
613
|
+
completed_turn_id is not None and current_turn_id == completed_turn_id
|
|
614
|
+
)
|
|
615
|
+
if completed_at is not None and timestamp > completed_at and not is_completion_turn:
|
|
472
616
|
continue
|
|
617
|
+
if is_completion_turn:
|
|
618
|
+
completed_turn_measured = True
|
|
473
619
|
if delta["totalTokens"] <= 0:
|
|
474
620
|
continue
|
|
475
|
-
_add_token_usage(
|
|
621
|
+
_add_token_usage(legacy_usage, delta)
|
|
476
622
|
|
|
623
|
+
_add_token_usage(total, response_usage if response_usage is not None else legacy_usage)
|
|
477
624
|
if not boundary_reached:
|
|
478
625
|
warnings.add("thread_ownership_unavailable")
|
|
626
|
+
if completed_turn_id is not None and not completed_turn_measured:
|
|
627
|
+
warnings.add("completion_turn_unavailable")
|
|
479
628
|
return total, warnings
|
|
480
629
|
|
|
481
630
|
|
|
@@ -526,7 +675,10 @@ def _token_snapshot(payload: Mapping[str, Any]) -> dict[str, int] | None:
|
|
|
526
675
|
info = payload.get("info")
|
|
527
676
|
if not isinstance(info, dict):
|
|
528
677
|
return None
|
|
529
|
-
|
|
678
|
+
return _usage_counts(info.get("total_token_usage"))
|
|
679
|
+
|
|
680
|
+
|
|
681
|
+
def _usage_counts(usage: object) -> dict[str, int] | None:
|
|
530
682
|
if not isinstance(usage, dict):
|
|
531
683
|
return None
|
|
532
684
|
cache_write = usage.get("cache_write_input_tokens", usage.get("cache_write_tokens", 0))
|