@openai/codex-security 0.2.0 → 0.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +437 -2692
- package/_bundled_plugin/.app.json +5 -1
- package/_bundled_plugin/.codex-plugin/plugin.json +9 -5
- package/_bundled_plugin/.mcp.json +2 -0
- package/_bundled_plugin/examples/completed-scan/coverage.json +0 -1
- package/_bundled_plugin/examples/completed-scan/report.md +28 -26
- package/_bundled_plugin/examples/completed-scan/scan-manifest.json +1 -1
- package/_bundled_plugin/mcp/helpers.mjs +1 -0
- package/_bundled_plugin/mcp/helpers.mjs.br.part-000 +0 -0
- package/_bundled_plugin/mcp/native/THIRD_PARTY_NOTICES.txt +6 -213
- package/_bundled_plugin/mcp/native/darwin-arm64/unix.node +0 -0
- package/_bundled_plugin/mcp/native/darwin-x64/unix.node +0 -0
- package/_bundled_plugin/mcp/native/linux-arm64-gnu/unix.node +0 -0
- package/_bundled_plugin/mcp/native/linux-arm64-musl/unix.node +0 -0
- package/_bundled_plugin/mcp/native/linux-x64-gnu/unix.node +0 -0
- package/_bundled_plugin/mcp/native/linux-x64-musl/unix.node +0 -0
- package/_bundled_plugin/mcp/native/win32-arm64/windows.node +0 -0
- package/_bundled_plugin/mcp/native/win32-x64/windows.node +0 -0
- package/_bundled_plugin/mcp/permission-profile-preflight.mjs +563 -0
- package/_bundled_plugin/mcp/server.mjs +1 -0
- package/_bundled_plugin/mcp/server.mjs.br.part-000 +0 -0
- package/_bundled_plugin/mcp/server.mjs.br.part-001 +0 -0
- package/_bundled_plugin/preflight/capability-profiles.toml +0 -13
- package/_bundled_plugin/references/artifact-storage.md +2 -2
- package/_bundled_plugin/references/config-preflight.md +0 -4
- package/_bundled_plugin/references/core-scan.md +21 -9
- package/_bundled_plugin/references/final-report.md +10 -12
- package/_bundled_plugin/references/finding-detail-fields.md +90 -80
- package/_bundled_plugin/references/sarif-adapter.md +1 -1
- package/_bundled_plugin/references/scan-artifacts.md +1 -2
- package/_bundled_plugin/references/scan-contract.md +9 -4
- package/_bundled_plugin/references/scan-prologue.md +12 -0
- package/_bundled_plugin/references/security-guidance.md +2 -2
- package/_bundled_plugin/references/standalone-validation.md +11 -0
- package/_bundled_plugin/references/threat-model.md +1 -1
- package/_bundled_plugin/schemas/coverage.schema.json +1 -47
- package/_bundled_plugin/schemas/definitions/artifact-common.schema.json +2 -24
- package/_bundled_plugin/schemas/definitions/discovery-candidate.schema.json +49 -25
- package/_bundled_plugin/schemas/findings.schema.json +71 -192
- package/_bundled_plugin/schemas/scan-manifest.schema.json +18 -45
- package/_bundled_plugin/schemas/tools/candidate-attack-paths.schema.json +91 -83
- package/_bundled_plugin/schemas/tools/candidate-validations.schema.json +35 -64
- package/_bundled_plugin/schemas/tools/deep-reducer.schema.json +5 -2
- package/_bundled_plugin/schemas/tools/discovery-candidates.schema.json +6 -55
- package/_bundled_plugin/schemas/tools/review-items.schema.json +2 -38
- package/_bundled_plugin/schemas/tools/scan-draft.schema.json +169 -294
- package/_bundled_plugin/scripts/codex_profile.mjs +239 -0
- package/_bundled_plugin/scripts/codex_session.mjs +53 -0
- package/_bundled_plugin/scripts/config_preflight.py +26 -62
- package/_bundled_plugin/scripts/deep_scan_config.py +9 -11
- package/_bundled_plugin/scripts/deep_scan_workbench.py +129 -150
- package/_bundled_plugin/scripts/finalize_scan_contract.py +315 -232
- package/_bundled_plugin/scripts/finding_preview.py +39 -9
- package/_bundled_plugin/scripts/generate_in_scope_files.py +27 -77
- package/_bundled_plugin/scripts/generate_rank_input.py +99 -829
- package/_bundled_plugin/scripts/launch_codex_security_mcp +35 -10
- package/_bundled_plugin/scripts/launch_codex_security_mcp.cmd +5 -5
- package/_bundled_plugin/scripts/rank_preview.py +42 -918
- package/_bundled_plugin/scripts/report_projection.py +117 -172
- package/_bundled_plugin/scripts/snapshot_sqlite.py +6 -1
- package/_bundled_plugin/scripts/threat_model_projection.py +4 -7
- package/_bundled_plugin/scripts/validate_scan_contract.py +4 -34
- package/_bundled_plugin/scripts/validate_tracking_source.py +3 -13
- package/_bundled_plugin/scripts/windows_scan_local_files.py +48 -3
- package/_bundled_plugin/scripts/workbench/handoff.py +25 -50
- package/_bundled_plugin/scripts/workbench/runtime_toml.py +22 -0
- package/_bundled_plugin/scripts/workbench/storage.py +10 -6
- package/_bundled_plugin/scripts/workbench_cli.py +94 -179
- package/_bundled_plugin/scripts/workbench_db.py +427 -463
- package/_bundled_plugin/scripts/workbench_feedback.py +6 -3
- package/_bundled_plugin/scripts/workbench_finding_index.py +19 -1
- package/_bundled_plugin/scripts/workbench_finding_workflows.py +18 -4
- package/_bundled_plugin/scripts/workbench_local_dedupe.py +136 -0
- package/_bundled_plugin/scripts/workbench_native_indexes.py +27 -14
- package/_bundled_plugin/scripts/workbench_progress.py +14 -58
- package/_bundled_plugin/scripts/workbench_publication.py +33 -57
- package/_bundled_plugin/scripts/workbench_remediation.py +12 -0
- package/_bundled_plugin/scripts/workbench_saved_results.py +172 -49
- package/_bundled_plugin/scripts/workbench_scan_history.py +57 -57
- package/_bundled_plugin/scripts/workbench_scan_start.py +105 -41
- package/_bundled_plugin/scripts/workbench_scan_usage.py +23 -34
- package/_bundled_plugin/scripts/workbench_schema.py +37 -1151
- package/_bundled_plugin/scripts/workbench_source_excerpt.py +11 -1
- package/_bundled_plugin/scripts/workbench_target.py +430 -142
- package/_bundled_plugin/scripts/workbench_validation.py +16 -27
- package/_bundled_plugin/shared/workbench-history-repairs.json +218 -0
- package/_bundled_plugin/shared/workbench-migrations.json +397 -0
- package/_bundled_plugin/skills/assess-patch-risk/SKILL.md +15 -4
- package/_bundled_plugin/skills/attack-path-analysis/SKILL.md +1 -0
- package/_bundled_plugin/skills/cloud-findings/SKILL.md +54 -0
- package/_bundled_plugin/skills/cloud-findings/agents/openai.yaml +4 -0
- package/_bundled_plugin/skills/deep-security-scan/SKILL.md +3 -3
- package/_bundled_plugin/skills/deep-security-scan/agents/openai.yaml +1 -1
- package/_bundled_plugin/skills/finding-discovery/SKILL.md +3 -3
- package/_bundled_plugin/skills/propose-security-hardening/references/proposal-format.md +50 -20
- package/_bundled_plugin/skills/security-diff-scan/SKILL.md +8 -14
- package/_bundled_plugin/skills/security-diff-scan/agents/openai.yaml +1 -1
- package/_bundled_plugin/skills/security-scan/SKILL.md +22 -7
- package/_bundled_plugin/skills/security-scan/agents/openai.yaml +1 -1
- package/_bundled_plugin/skills/security-scan/references/desktop-scan.md +1 -1
- package/_bundled_plugin/skills/security-scan/references/scan-artifacts-and-ledger.md +13 -1
- package/_bundled_plugin/skills/threat-model/SKILL.md +1 -1
- package/_bundled_plugin/skills/track-findings/SKILL.md +46 -178
- package/_bundled_plugin/skills/track-findings/agents/openai.yaml +1 -1
- package/_bundled_plugin/skills/track-findings/references/github-security-advisories.md +16 -24
- package/_bundled_plugin/skills/track-findings/references/jira.md +15 -61
- package/_bundled_plugin/skills/triage-finding/SKILL.md +50 -289
- package/_bundled_plugin/skills/triage-finding/agents/openai.yaml +2 -2
- package/_bundled_plugin/skills/triage-finding/references/ticket-intake.md +23 -43
- package/_bundled_plugin/skills/triage-finding/references/triage-result-contract.md +7 -1
- package/bin/codex-security.mjs +9 -2
- package/dist/api.d.ts +60 -10
- package/dist/api.d.ts.map +1 -1
- package/dist/api.js +803 -380
- package/dist/api.js.map +1 -1
- package/dist/artifact-export.d.ts +1 -1
- package/dist/artifact-export.d.ts.map +1 -1
- package/dist/artifact-export.js +28 -13
- package/dist/artifact-export.js.map +1 -1
- package/dist/auth.d.ts +2 -6
- package/dist/auth.d.ts.map +1 -1
- package/dist/auth.js +19 -9
- package/dist/auth.js.map +1 -1
- package/dist/bulk-scan-discovery.d.ts.map +1 -1
- package/dist/bulk-scan-discovery.js +1 -0
- package/dist/bulk-scan-discovery.js.map +1 -1
- package/dist/classify-severity.d.ts +26 -9
- package/dist/classify-severity.d.ts.map +1 -1
- package/dist/classify-severity.js +13 -30
- package/dist/classify-severity.js.map +1 -1
- package/dist/cli-help.d.ts.map +1 -1
- package/dist/cli-help.js +17 -13
- package/dist/cli-help.js.map +1 -1
- package/dist/cli-signals.d.ts +8 -0
- package/dist/cli-signals.d.ts.map +1 -0
- package/dist/cli-signals.js +28 -0
- package/dist/cli-signals.js.map +1 -0
- package/dist/cli.d.ts +4 -3
- package/dist/cli.d.ts.map +1 -1
- package/dist/cli.js +1087 -900
- package/dist/cli.js.map +1 -1
- package/dist/cloud-publish.d.ts.map +1 -1
- package/dist/cloud-publish.js +10 -9
- package/dist/cloud-publish.js.map +1 -1
- package/dist/codeowners.d.ts +19 -0
- package/dist/codeowners.d.ts.map +1 -0
- package/dist/codeowners.js +138 -0
- package/dist/codeowners.js.map +1 -0
- package/dist/codex-sdk-environment.d.ts +1 -0
- package/dist/codex-sdk-environment.d.ts.map +1 -1
- package/dist/codex-sdk-environment.js +40 -2
- package/dist/codex-sdk-environment.js.map +1 -1
- package/dist/component-plan.d.ts +2 -5
- package/dist/component-plan.d.ts.map +1 -1
- package/dist/component-plan.js +38 -30
- package/dist/component-plan.js.map +1 -1
- package/dist/component-scan.d.ts +6 -5
- package/dist/component-scan.d.ts.map +1 -1
- package/dist/component-scan.js +23 -35
- package/dist/component-scan.js.map +1 -1
- package/dist/config.d.ts.map +1 -1
- package/dist/config.js +64 -74
- package/dist/config.js.map +1 -1
- package/dist/contract.d.ts +15 -3
- package/dist/contract.d.ts.map +1 -1
- package/dist/contract.js +87 -252
- package/dist/contract.js.map +1 -1
- package/dist/cost-model.d.ts +13 -6
- package/dist/cost-model.d.ts.map +1 -1
- package/dist/cost-model.js +14 -4
- package/dist/cost-model.js.map +1 -1
- package/dist/cost.d.ts +10 -2
- package/dist/cost.d.ts.map +1 -1
- package/dist/cost.js +102 -104
- package/dist/cost.js.map +1 -1
- package/dist/custom-validation-prompt.d.ts.map +1 -1
- package/dist/custom-validation-prompt.js +6 -5
- package/dist/custom-validation-prompt.js.map +1 -1
- package/dist/custom-validation.d.ts +1 -2
- package/dist/custom-validation.d.ts.map +1 -1
- package/dist/custom-validation.js +9 -3
- package/dist/custom-validation.js.map +1 -1
- package/dist/deduplication/checkpointed-review.d.ts +4 -2
- package/dist/deduplication/checkpointed-review.d.ts.map +1 -1
- package/dist/deduplication/checkpointed-review.js +23 -9
- package/dist/deduplication/checkpointed-review.js.map +1 -1
- package/dist/deduplication/codex-review.d.ts +7 -1
- package/dist/deduplication/codex-review.d.ts.map +1 -1
- package/dist/deduplication/codex-review.js +126 -17
- package/dist/deduplication/codex-review.js.map +1 -1
- package/dist/deduplication/deduplication-prompts.d.ts +5 -2
- package/dist/deduplication/deduplication-prompts.d.ts.map +1 -1
- package/dist/deduplication/deduplication-prompts.js +18 -4
- package/dist/deduplication/deduplication-prompts.js.map +1 -1
- package/dist/deduplication/deduplication-reviewer.d.ts +26 -4
- package/dist/deduplication/deduplication-reviewer.d.ts.map +1 -1
- package/dist/deduplication/deduplication-reviewer.js +57 -47
- package/dist/deduplication/deduplication-reviewer.js.map +1 -1
- package/dist/deduplication/deduplication.d.ts.map +1 -1
- package/dist/deduplication/deduplication.js +19 -51
- package/dist/deduplication/deduplication.js.map +1 -1
- package/dist/deduplication/diagnostics.d.ts +17 -0
- package/dist/deduplication/diagnostics.d.ts.map +1 -0
- package/dist/deduplication/diagnostics.js +5 -0
- package/dist/deduplication/diagnostics.js.map +1 -0
- package/dist/deduplication/finding-schema.d.ts.map +1 -1
- package/dist/deduplication/finding-schema.js +7 -1
- package/dist/deduplication/finding-schema.js.map +1 -1
- package/dist/deduplication/local.d.ts +31 -0
- package/dist/deduplication/local.d.ts.map +1 -0
- package/dist/deduplication/local.js +166 -0
- package/dist/deduplication/local.js.map +1 -0
- package/dist/deduplication/records-protocol.d.ts.map +1 -1
- package/dist/deduplication/records-protocol.js +11 -24
- package/dist/deduplication/records-protocol.js.map +1 -1
- package/dist/deduplication/records.d.ts +186 -11
- package/dist/deduplication/records.d.ts.map +1 -1
- package/dist/deduplication/records.js +10 -18
- package/dist/deduplication/records.js.map +1 -1
- package/dist/deduplication/scan.d.ts +9 -3
- package/dist/deduplication/scan.d.ts.map +1 -1
- package/dist/deduplication/scan.js +158 -39
- package/dist/deduplication/scan.js.map +1 -1
- package/dist/deep-config.d.ts +2 -1
- package/dist/deep-config.d.ts.map +1 -1
- package/dist/deep-config.js +4 -4
- package/dist/deep-config.js.map +1 -1
- package/dist/deep-progress.d.ts +2 -0
- package/dist/deep-progress.d.ts.map +1 -1
- package/dist/deep-progress.js +13 -10
- package/dist/deep-progress.js.map +1 -1
- package/dist/deep-scan.d.ts +20 -0
- package/dist/deep-scan.d.ts.map +1 -0
- package/dist/deep-scan.js +113 -0
- package/dist/deep-scan.js.map +1 -0
- package/dist/errors.d.ts +2 -0
- package/dist/errors.d.ts.map +1 -1
- package/dist/errors.js +2 -0
- package/dist/errors.js.map +1 -1
- package/dist/feedback.d.ts.map +1 -1
- package/dist/feedback.js +53 -21
- package/dist/feedback.js.map +1 -1
- package/dist/finding-catalogue.d.ts.map +1 -1
- package/dist/finding-catalogue.js +4 -8
- package/dist/finding-catalogue.js.map +1 -1
- package/dist/finding-normalization.d.ts +5 -0
- package/dist/finding-normalization.d.ts.map +1 -0
- package/dist/finding-normalization.js +120 -0
- package/dist/finding-normalization.js.map +1 -0
- package/dist/finding-retrieval.d.ts +9 -0
- package/dist/finding-retrieval.d.ts.map +1 -1
- package/dist/finding-retrieval.js.map +1 -1
- package/dist/finding-workflow.d.ts +13 -8
- package/dist/finding-workflow.d.ts.map +1 -1
- package/dist/finding-workflow.js +14 -6
- package/dist/finding-workflow.js.map +1 -1
- package/dist/findings-client.d.ts +2 -1
- package/dist/findings-client.d.ts.map +1 -1
- package/dist/findings-client.js +36 -9
- package/dist/findings-client.js.map +1 -1
- package/dist/findings-errors.d.ts +16 -0
- package/dist/findings-errors.d.ts.map +1 -0
- package/dist/findings-errors.js +24 -0
- package/dist/findings-errors.js.map +1 -0
- package/dist/findings-import.d.ts +1 -1
- package/dist/findings-import.d.ts.map +1 -1
- package/dist/findings-import.js +12 -30
- package/dist/findings-import.js.map +1 -1
- package/dist/github.d.ts.map +1 -1
- package/dist/github.js +7 -1
- package/dist/github.js.map +1 -1
- package/dist/import-scan.d.ts.map +1 -1
- package/dist/import-scan.js +48 -38
- package/dist/import-scan.js.map +1 -1
- package/dist/index.d.ts +8 -4
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1 -1
- package/dist/index.js.map +1 -1
- package/dist/knowledge-base.d.ts +7 -1
- package/dist/knowledge-base.d.ts.map +1 -1
- package/dist/knowledge-base.js +114 -46
- package/dist/knowledge-base.js.map +1 -1
- package/dist/linear.d.ts.map +1 -1
- package/dist/linear.js +2 -7
- package/dist/linear.js.map +1 -1
- package/dist/mock-scan.d.ts.map +1 -1
- package/dist/mock-scan.js +0 -1
- package/dist/mock-scan.js.map +1 -1
- package/dist/models.d.ts +0 -2
- package/dist/models.d.ts.map +1 -1
- package/dist/models.js.map +1 -1
- package/dist/multiscan.d.ts +9 -3
- package/dist/multiscan.d.ts.map +1 -1
- package/dist/multiscan.js +193 -126
- package/dist/multiscan.js.map +1 -1
- package/dist/owner-evidence.d.ts +10 -2
- package/dist/owner-evidence.d.ts.map +1 -1
- package/dist/owner-evidence.js +67 -8
- package/dist/owner-evidence.js.map +1 -1
- package/dist/patch-tui.d.ts.map +1 -1
- package/dist/patch-tui.js +3 -2
- package/dist/patch-tui.js.map +1 -1
- package/dist/project-config.js +3 -3
- package/dist/project-config.js.map +1 -1
- package/dist/provider-profile.d.ts +21 -0
- package/dist/provider-profile.d.ts.map +1 -0
- package/dist/provider-profile.js +133 -0
- package/dist/provider-profile.js.map +1 -0
- package/dist/publication-events.d.ts +6 -8
- package/dist/publication-events.d.ts.map +1 -1
- package/dist/publication-events.js +25 -50
- package/dist/publication-events.js.map +1 -1
- package/dist/publication-store.d.ts.map +1 -1
- package/dist/publication-store.js +40 -64
- package/dist/publication-store.js.map +1 -1
- package/dist/publication.d.ts +1 -2
- package/dist/publication.d.ts.map +1 -1
- package/dist/publication.js +2 -8
- package/dist/publication.js.map +1 -1
- package/dist/publish.d.ts +1 -5
- package/dist/publish.d.ts.map +1 -1
- package/dist/publish.js +109 -160
- package/dist/publish.js.map +1 -1
- package/dist/request-metadata.d.ts +9 -0
- package/dist/request-metadata.d.ts.map +1 -0
- package/dist/request-metadata.js +13 -0
- package/dist/request-metadata.js.map +1 -0
- package/dist/runtime.d.ts +15 -4
- package/dist/runtime.d.ts.map +1 -1
- package/dist/runtime.js +430 -138
- package/dist/runtime.js.map +1 -1
- package/dist/saved-scan-bootstrap.d.ts +14 -0
- package/dist/saved-scan-bootstrap.d.ts.map +1 -0
- package/dist/saved-scan-bootstrap.js +261 -0
- package/dist/saved-scan-bootstrap.js.map +1 -0
- package/dist/saved-scan.d.ts +1 -1
- package/dist/saved-scan.d.ts.map +1 -1
- package/dist/saved-scan.js +6 -12
- package/dist/saved-scan.js.map +1 -1
- package/dist/scan-activity.js +11 -8
- package/dist/scan-activity.js.map +1 -1
- package/dist/scan-comparison.d.ts +28 -15
- package/dist/scan-comparison.d.ts.map +1 -1
- package/dist/scan-comparison.js +276 -281
- package/dist/scan-comparison.js.map +1 -1
- package/dist/scan-dashboard.d.ts +3 -0
- package/dist/scan-dashboard.d.ts.map +1 -1
- package/dist/scan-dashboard.js +345 -77
- package/dist/scan-dashboard.js.map +1 -1
- package/dist/scan-history-renderer.d.ts.map +1 -1
- package/dist/scan-history-renderer.js +54 -13
- package/dist/scan-history-renderer.js.map +1 -1
- package/dist/scan-inputs.d.ts +6 -0
- package/dist/scan-inputs.d.ts.map +1 -0
- package/dist/scan-inputs.js +71 -0
- package/dist/scan-inputs.js.map +1 -0
- package/dist/scan-logs.d.ts +2 -3
- package/dist/scan-logs.d.ts.map +1 -1
- package/dist/scan-logs.js +12 -6
- package/dist/scan-logs.js.map +1 -1
- package/dist/scan-sessions.d.ts +4 -0
- package/dist/scan-sessions.d.ts.map +1 -1
- package/dist/scan-sessions.js +17 -0
- package/dist/scan-sessions.js.map +1 -1
- package/dist/scan-skill-paths.d.ts +8 -0
- package/dist/scan-skill-paths.d.ts.map +1 -0
- package/dist/scan-skill-paths.js +47 -0
- package/dist/scan-skill-paths.js.map +1 -0
- package/dist/security-policy-cli.d.ts.map +1 -1
- package/dist/security-policy-cli.js +7 -27
- package/dist/security-policy-cli.js.map +1 -1
- package/dist/security-policy.d.ts +9 -5
- package/dist/security-policy.d.ts.map +1 -1
- package/dist/security-policy.js +33 -14
- package/dist/security-policy.js.map +1 -1
- package/dist/server/api.d.ts +0 -1
- package/dist/server/api.d.ts.map +1 -1
- package/dist/server/api.js +0 -1
- package/dist/server/api.js.map +1 -1
- package/dist/server/embeddings.d.ts +3 -1
- package/dist/server/embeddings.d.ts.map +1 -1
- package/dist/server/embeddings.js +10 -3
- package/dist/server/embeddings.js.map +1 -1
- package/dist/server/errors.d.ts +5 -2
- package/dist/server/errors.d.ts.map +1 -1
- package/dist/server/errors.js +7 -0
- package/dist/server/errors.js.map +1 -1
- package/dist/server/sqlite-store.d.ts +0 -2
- package/dist/server/sqlite-store.d.ts.map +1 -1
- package/dist/server/sqlite-store.js +54 -44
- package/dist/server/sqlite-store.js.map +1 -1
- package/dist/server/storage.d.ts +0 -2
- package/dist/server/storage.d.ts.map +1 -1
- package/dist/server/storage.js.map +1 -1
- package/dist/severity-store.js +19 -24
- package/dist/severity-store.js.map +1 -1
- package/dist/standalone-validation-prompt.d.ts +2 -0
- package/dist/standalone-validation-prompt.d.ts.map +1 -0
- package/dist/standalone-validation-prompt.js +10 -0
- package/dist/standalone-validation-prompt.js.map +1 -0
- package/dist/suggest-owners.d.ts +4 -2
- package/dist/suggest-owners.d.ts.map +1 -1
- package/dist/suggest-owners.js +11 -5
- package/dist/suggest-owners.js.map +1 -1
- package/dist/targets.d.ts +11 -0
- package/dist/targets.d.ts.map +1 -1
- package/dist/targets.js +106 -47
- package/dist/targets.js.map +1 -1
- package/dist/trusted-executable.d.ts +2 -2
- package/dist/trusted-executable.d.ts.map +1 -1
- package/dist/trusted-executable.js +8 -6
- package/dist/trusted-executable.js.map +1 -1
- package/dist/value.js +7 -0
- package/dist/value.js.map +1 -1
- package/dist/version.js.map +1 -1
- package/dist/workbench-arguments.d.ts +3 -0
- package/dist/workbench-arguments.d.ts.map +1 -0
- package/dist/workbench-arguments.js +15 -0
- package/dist/workbench-arguments.js.map +1 -0
- package/dist/worker-progress.d.ts +1 -0
- package/dist/worker-progress.d.ts.map +1 -1
- package/dist/worker-progress.js +27 -12
- package/dist/worker-progress.js.map +1 -1
- package/docs/cli.md +1778 -0
- package/docs/findings-service.md +402 -0
- package/package.json +18 -19
- package/_bundled_plugin/schemas/tools/worker-threat-model.schema.json +0 -18
- package/_bundled_plugin/scripts/workbench_dashboard.py +0 -159
- package/_bundled_plugin/scripts/workbench_findings.py +0 -221
- package/_bundled_plugin/scripts/workbench_severity.py +0 -122
- package/_bundled_plugin/skills/assess-patch-risk/scripts/validate_patch_risk_assessment.py +0 -163
- package/dist/server/dashboard/THIRD_PARTY_NOTICES.txt +0 -276
- package/dist/server/dashboard/app.css +0 -2
- package/dist/server/dashboard/app.js +0 -66
- package/dist/server/dashboard/index.html +0 -18
- package/dist/server/dashboard-types.d.ts +0 -48
- package/dist/server/dashboard-types.d.ts.map +0 -1
- package/dist/server/dashboard-types.js +0 -2
- package/dist/server/dashboard-types.js.map +0 -1
- package/dist/server/dashboard.d.ts +0 -6
- package/dist/server/dashboard.d.ts.map +0 -1
- package/dist/server/dashboard.js +0 -59
- package/dist/server/dashboard.js.map +0 -1
- package/dist/server/index.d.ts +0 -2
- package/dist/server/index.d.ts.map +0 -1
- package/dist/server/index.js +0 -6
- package/dist/server/index.js.map +0 -1
- package/dist/server/routes.d.ts +0 -7
- package/dist/server/routes.d.ts.map +0 -1
- package/dist/server/routes.js +0 -98
- package/dist/server/routes.js.map +0 -1
- package/dist/server/serve.d.ts +0 -2
- package/dist/server/serve.d.ts.map +0 -1
- package/dist/server/serve.js +0 -28
- package/dist/server/serve.js.map +0 -1
- package/dist/server/server.d.ts +0 -10
- package/dist/server/server.d.ts.map +0 -1
- package/dist/server/server.js +0 -16
- package/dist/server/server.js.map +0 -1
- package/dist/server/validation.d.ts +0 -17
- package/dist/server/validation.d.ts.map +0 -1
- package/dist/server/validation.js +0 -53
- package/dist/server/validation.js.map +0 -1
|
@@ -8,31 +8,19 @@ This script stays deliberately model-free:
|
|
|
8
8
|
- `make-diff-rank-input` creates the deterministic diff-scoped JSONL candidate
|
|
9
9
|
worklist from Git changed paths. It supports committed revision diffs and
|
|
10
10
|
local working-tree patches.
|
|
11
|
-
|
|
12
|
-
|
|
13
|
-
|
|
14
|
-
- `validate-rank-worker` validates one worker slot and emits a content-bound
|
|
15
|
-
completion receipt.
|
|
16
|
-
- `validate-rank-shard` validates one completed worker output before the
|
|
17
|
-
coordinator accepts it.
|
|
18
|
-
- `validate-rank-pool` validates the pool plan and every assigned shard output.
|
|
19
|
-
- `merge-rank-outputs` validates and combines worker-local shard outputs.
|
|
20
|
-
- `copy-deep-review-input` copies every candidate into the deep-review worklist
|
|
21
|
-
for exhaustive mode.
|
|
22
|
-
- `select-deep-review-input` selects the ranked rows for deep review.
|
|
11
|
+
|
|
12
|
+
Candidate selection uses repository scope, native tool ignore rules, and binary detection,
|
|
13
|
+
not filename or directory classifications. Ranking inputs use bounded source previews.
|
|
23
14
|
"""
|
|
24
15
|
|
|
25
16
|
from __future__ import annotations
|
|
26
17
|
|
|
27
18
|
import argparse
|
|
28
|
-
import hashlib
|
|
29
19
|
import json
|
|
30
20
|
import os
|
|
31
|
-
import re
|
|
32
21
|
import subprocess
|
|
33
22
|
import sys
|
|
34
|
-
from collections import
|
|
35
|
-
from collections.abc import Callable, KeysView
|
|
23
|
+
from collections.abc import Iterable
|
|
36
24
|
from pathlib import Path
|
|
37
25
|
|
|
38
26
|
# Some plugin hosts launch Python with safe-path isolation enabled.
|
|
@@ -40,96 +28,13 @@ sys.path.insert(0, str(Path(__file__).resolve().parent))
|
|
|
40
28
|
from generate_in_scope_files import windows_stream_component
|
|
41
29
|
from rank_preview import (
|
|
42
30
|
DEFAULT_PREVIEW_BYTES,
|
|
43
|
-
|
|
31
|
+
is_binary_file,
|
|
44
32
|
preview_for,
|
|
45
33
|
preview_for_bytes,
|
|
46
34
|
)
|
|
47
|
-
from workbench_target import
|
|
48
|
-
|
|
49
|
-
EXCLUDED_DIRS = {
|
|
50
|
-
".cache",
|
|
51
|
-
".circleci",
|
|
52
|
-
".devcontainer",
|
|
53
|
-
".git",
|
|
54
|
-
".github",
|
|
55
|
-
".idea",
|
|
56
|
-
".mypy_cache",
|
|
57
|
-
".pytest_cache",
|
|
58
|
-
".ruff_cache",
|
|
59
|
-
".tox",
|
|
60
|
-
".venv",
|
|
61
|
-
".vscode",
|
|
62
|
-
"__pycache__",
|
|
63
|
-
"bench",
|
|
64
|
-
"benchmark",
|
|
65
|
-
"bintest",
|
|
66
|
-
"build",
|
|
67
|
-
"build_config",
|
|
68
|
-
"build_configs",
|
|
69
|
-
"build-tools",
|
|
70
|
-
"build_tools",
|
|
71
|
-
"ci",
|
|
72
|
-
"coverage",
|
|
73
|
-
"deps",
|
|
74
|
-
"dev",
|
|
75
|
-
"dist",
|
|
76
|
-
"doc",
|
|
77
|
-
"docs",
|
|
78
|
-
"example",
|
|
79
|
-
"examples",
|
|
80
|
-
"external",
|
|
81
|
-
"extern",
|
|
82
|
-
"fixture",
|
|
83
|
-
"fixtures",
|
|
84
|
-
"generated",
|
|
85
|
-
"node_modules",
|
|
86
|
-
"sample",
|
|
87
|
-
"samples",
|
|
88
|
-
"target",
|
|
89
|
-
"test",
|
|
90
|
-
"tests",
|
|
91
|
-
"testing",
|
|
92
|
-
"third-party",
|
|
93
|
-
"third_party",
|
|
94
|
-
"tmp",
|
|
95
|
-
"vendor",
|
|
96
|
-
}
|
|
97
|
-
|
|
98
|
-
EXCLUDED_FILENAMES = {
|
|
99
|
-
".DS_Store",
|
|
100
|
-
"CHANGELOG",
|
|
101
|
-
"CHANGELOG.md",
|
|
102
|
-
"CONTRIBUTING.md",
|
|
103
|
-
"Dockerfile",
|
|
104
|
-
"Gemfile",
|
|
105
|
-
"Gemfile.lock",
|
|
106
|
-
"LICENSE",
|
|
107
|
-
"LICENSE.md",
|
|
108
|
-
"Makefile",
|
|
109
|
-
"NEWS",
|
|
110
|
-
"NEWS.md",
|
|
111
|
-
"NOTICE",
|
|
112
|
-
"README",
|
|
113
|
-
"README.md",
|
|
114
|
-
"README.rst",
|
|
115
|
-
"Rakefile",
|
|
116
|
-
"SECURITY.md",
|
|
117
|
-
"TODO",
|
|
118
|
-
"TODO.md",
|
|
119
|
-
"docker-compose.yml",
|
|
120
|
-
"package-lock.json",
|
|
121
|
-
"pnpm-lock.yaml",
|
|
122
|
-
"yarn.lock",
|
|
123
|
-
}
|
|
35
|
+
from workbench_target import git_blob_samples, git_command, git_directory_snapshot_paths
|
|
124
36
|
|
|
125
|
-
SHARD_INPUT_GLOB = "rank-shard-*.input.jsonl"
|
|
126
|
-
SHARD_OUTPUT_GLOB = "rank-shard-*.output.jsonl"
|
|
127
|
-
SHARD_INPUT_PATTERN = re.compile(r"^rank-shard-([0-9]{4,})\.input\.jsonl$")
|
|
128
|
-
RANK_POOL_PLAN_SCHEMA_VERSION = 1
|
|
129
|
-
RANK_POOL_STRATEGY = "round_robin"
|
|
130
|
-
RANK_POOL_WORKER_CAP = 6
|
|
131
37
|
JsonRow = dict[str, object]
|
|
132
|
-
RowValidator = Callable[[JsonRow, Path, int], None]
|
|
133
38
|
|
|
134
39
|
|
|
135
40
|
def parse_args() -> argparse.Namespace:
|
|
@@ -171,17 +76,9 @@ def parse_args() -> argparse.Namespace:
|
|
|
171
76
|
)
|
|
172
77
|
scoped.add_argument("--out", required=True, help="Output scoped-source-input.jsonl path.")
|
|
173
78
|
|
|
174
|
-
bind = subparsers.add_parser(
|
|
175
|
-
"bind-repo-scopes",
|
|
176
|
-
help="Copy SDK scoped-path targets into the unsealed manifest and coverage documents.",
|
|
177
|
-
)
|
|
178
|
-
bind.add_argument("--scopes-file", required=True, help="JSON array of requested scopes.")
|
|
179
|
-
bind.add_argument("--manifest", required=True, help="Unsealed scan-manifest.json path.")
|
|
180
|
-
bind.add_argument("--coverage", required=True, help="Unsealed coverage.json path.")
|
|
181
|
-
|
|
182
79
|
diff = subparsers.add_parser(
|
|
183
80
|
"make-diff-rank-input",
|
|
184
|
-
help="Create rank_input.jsonl from Git changed
|
|
81
|
+
help="Create rank_input.jsonl from Git changed text files.",
|
|
185
82
|
)
|
|
186
83
|
diff.add_argument("--repo", required=True, help="Repository root.")
|
|
187
84
|
diff.add_argument("--base", required=True, help="Git diff base revision.")
|
|
@@ -201,104 +98,9 @@ def parse_args() -> argparse.Namespace:
|
|
|
201
98
|
help=f"Maximum UTF-8 bytes in each preview. Defaults to {DEFAULT_PREVIEW_BYTES}.",
|
|
202
99
|
)
|
|
203
100
|
|
|
204
|
-
shards = subparsers.add_parser(
|
|
205
|
-
"make-rank-shards",
|
|
206
|
-
help="Partition rank_input.jsonl into deterministic worker input shards.",
|
|
207
|
-
)
|
|
208
|
-
shards.add_argument("--rank-input", required=True, help="Deterministic rank input JSONL.")
|
|
209
|
-
shards.add_argument("--out-dir", required=True, help="Directory for worker input shards.")
|
|
210
|
-
shards.add_argument(
|
|
211
|
-
"--max-rows",
|
|
212
|
-
type=int,
|
|
213
|
-
default=150,
|
|
214
|
-
help="Maximum rows per shard. Defaults to 150.",
|
|
215
|
-
)
|
|
216
|
-
|
|
217
|
-
pool_plan = subparsers.add_parser(
|
|
218
|
-
"make-rank-pool-plan",
|
|
219
|
-
help="Assign rank shards to a deterministic bounded worker pool.",
|
|
220
|
-
)
|
|
221
|
-
pool_plan.add_argument("--shard-dir", required=True, help="Directory of rank shards.")
|
|
222
|
-
pool_plan.add_argument(
|
|
223
|
-
"--usable-worker-slots",
|
|
224
|
-
required=True,
|
|
225
|
-
type=int,
|
|
226
|
-
help="Usable ranking-worker slots reported by capability preflight; capped at 6.",
|
|
227
|
-
)
|
|
228
|
-
pool_plan.add_argument("--out", required=True, help="Output rank_worker_assignments.json path.")
|
|
229
|
-
|
|
230
|
-
validate_shard = subparsers.add_parser(
|
|
231
|
-
"validate-rank-shard",
|
|
232
|
-
help="Validate one worker output against its rank input shard.",
|
|
233
|
-
)
|
|
234
|
-
validate_shard.add_argument("--input", required=True, help="Worker rank input shard.")
|
|
235
|
-
validate_shard.add_argument("--output", required=True, help="Worker rank output shard.")
|
|
236
|
-
|
|
237
|
-
validate_worker = subparsers.add_parser(
|
|
238
|
-
"validate-rank-worker",
|
|
239
|
-
help="Validate one assigned ranking-worker slot and emit its completion receipt.",
|
|
240
|
-
)
|
|
241
|
-
validate_worker.add_argument("--plan", required=True, help="Rank pool plan JSON path.")
|
|
242
|
-
validate_worker.add_argument("--shard-dir", required=True, help="Directory of rank shards.")
|
|
243
|
-
validate_worker.add_argument(
|
|
244
|
-
"--slot",
|
|
245
|
-
required=True,
|
|
246
|
-
type=int,
|
|
247
|
-
help="One-based ranking-worker slot from the rank pool plan.",
|
|
248
|
-
)
|
|
249
|
-
|
|
250
|
-
validate_pool = subparsers.add_parser(
|
|
251
|
-
"validate-rank-pool",
|
|
252
|
-
help="Validate a rank pool plan and every assigned shard output.",
|
|
253
|
-
)
|
|
254
|
-
validate_pool.add_argument("--plan", required=True, help="Rank pool plan JSON path.")
|
|
255
|
-
validate_pool.add_argument("--shard-dir", required=True, help="Directory of rank shards.")
|
|
256
|
-
|
|
257
|
-
merge = subparsers.add_parser(
|
|
258
|
-
"merge-rank-outputs",
|
|
259
|
-
help="Validate worker shard outputs and create rank_output.jsonl.",
|
|
260
|
-
)
|
|
261
|
-
merge.add_argument("--rank-input", required=True, help="Authoritative rank input JSONL.")
|
|
262
|
-
merge.add_argument("--shard-dir", required=True, help="Directory of input and output shards.")
|
|
263
|
-
merge.add_argument("--out", required=True, help="Output rank_output.jsonl path.")
|
|
264
|
-
|
|
265
|
-
copy = subparsers.add_parser(
|
|
266
|
-
"copy-deep-review-input",
|
|
267
|
-
help="Create deep_review_input.jsonl directly from rank_input.jsonl.",
|
|
268
|
-
)
|
|
269
|
-
copy.add_argument("--rank-input", required=True, help="Deterministic rank input JSONL.")
|
|
270
|
-
copy.add_argument("--out", required=True, help="Output deep_review_input.jsonl path.")
|
|
271
|
-
|
|
272
|
-
select = subparsers.add_parser(
|
|
273
|
-
"select-deep-review-input",
|
|
274
|
-
help="Create deep_review_input.jsonl from worker-produced rank_output.jsonl.",
|
|
275
|
-
)
|
|
276
|
-
select.add_argument("--rank-output", required=True, help="Worker ranking output JSONL.")
|
|
277
|
-
select.add_argument("--out", required=True, help="Output deep_review_input.jsonl path.")
|
|
278
|
-
select.add_argument(
|
|
279
|
-
"--top-percent",
|
|
280
|
-
type=int,
|
|
281
|
-
default=100,
|
|
282
|
-
help="Percent of included files to keep for deep review.",
|
|
283
|
-
)
|
|
284
101
|
return parser.parse_args()
|
|
285
102
|
|
|
286
103
|
|
|
287
|
-
def path_is_excluded(path: Path) -> bool:
|
|
288
|
-
if any(part in EXCLUDED_DIRS for part in path.parts):
|
|
289
|
-
return True
|
|
290
|
-
if path.name in EXCLUDED_FILENAMES:
|
|
291
|
-
return True
|
|
292
|
-
return path.name.endswith((".min.js", ".map"))
|
|
293
|
-
|
|
294
|
-
|
|
295
|
-
def path_is_diff_excluded(path: Path) -> bool:
|
|
296
|
-
"""Apply repository exclusions while retaining changed workflow files."""
|
|
297
|
-
if path.parts[:2] == (".github", "workflows"):
|
|
298
|
-
return False
|
|
299
|
-
return path_is_excluded(path)
|
|
300
|
-
|
|
301
|
-
|
|
302
104
|
def resolve_scope(
|
|
303
105
|
repo: Path,
|
|
304
106
|
scope: str,
|
|
@@ -351,14 +153,6 @@ def write_jsonl(output: Path, rows: list[JsonRow]) -> None:
|
|
|
351
153
|
handle.write("\n")
|
|
352
154
|
|
|
353
155
|
|
|
354
|
-
def write_json(output: Path, payload: dict[str, object]) -> None:
|
|
355
|
-
output.parent.mkdir(parents=True, exist_ok=True)
|
|
356
|
-
output.write_text(
|
|
357
|
-
json.dumps(payload, ensure_ascii=False, indent=2, sort_keys=True) + "\n",
|
|
358
|
-
encoding="utf-8",
|
|
359
|
-
)
|
|
360
|
-
|
|
361
|
-
|
|
362
156
|
def load_scopes_file(scopes_file: Path) -> list[str]:
|
|
363
157
|
try:
|
|
364
158
|
loaded: object = json.loads(scopes_file.read_text(encoding="utf-8"))
|
|
@@ -373,75 +167,58 @@ def load_scopes_file(scopes_file: Path) -> list[str]:
|
|
|
373
167
|
return loaded
|
|
374
168
|
|
|
375
169
|
|
|
376
|
-
def
|
|
377
|
-
|
|
378
|
-
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
|
|
384
|
-
|
|
385
|
-
|
|
386
|
-
|
|
387
|
-
|
|
388
|
-
|
|
389
|
-
|
|
390
|
-
|
|
391
|
-
|
|
392
|
-
|
|
393
|
-
|
|
394
|
-
|
|
395
|
-
|
|
396
|
-
|
|
397
|
-
|
|
398
|
-
|
|
399
|
-
|
|
400
|
-
|
|
401
|
-
|
|
402
|
-
|
|
403
|
-
|
|
404
|
-
|
|
405
|
-
|
|
406
|
-
|
|
407
|
-
|
|
408
|
-
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
414
|
-
|
|
415
|
-
|
|
416
|
-
|
|
417
|
-
|
|
418
|
-
|
|
419
|
-
|
|
420
|
-
|
|
421
|
-
|
|
422
|
-
|
|
423
|
-
|
|
424
|
-
|
|
425
|
-
def validate_rank_output_row(row: JsonRow, path: Path, line_number: int) -> None:
|
|
426
|
-
require_exact_fields(row, {"path", "area", "score", "include", "reason"}, path, line_number)
|
|
427
|
-
require_string(row, "path", path, line_number, allow_empty=False)
|
|
428
|
-
require_string(row, "area", path, line_number, allow_empty=True)
|
|
429
|
-
score = row["score"]
|
|
430
|
-
if isinstance(score, bool) or not isinstance(score, int):
|
|
431
|
-
raise SystemExit(f"{path}:{line_number}: score must be an integer from 1 through 10")
|
|
432
|
-
if not 1 <= score <= 10:
|
|
433
|
-
raise SystemExit(f"{path}:{line_number}: score must be from 1 through 10")
|
|
434
|
-
if not isinstance(row["include"], bool):
|
|
435
|
-
raise SystemExit(f"{path}:{line_number}: include must be a boolean")
|
|
436
|
-
require_string(row, "reason", path, line_number, allow_empty=False)
|
|
437
|
-
|
|
170
|
+
def scope_candidates(repo: Path, scope_path: Path) -> Iterable[Path]:
|
|
171
|
+
"""Use Git's inventory in worktrees and ripgrep's ignore rules elsewhere.
|
|
172
|
+
|
|
173
|
+
Git retains tracked files and applies its standard exclusions to untracked files.
|
|
174
|
+
Outside Git, ripgrep also honors .ignore and .rgignore alongside .gitignore.
|
|
175
|
+
Explicit file scopes bypass directory ignore rules.
|
|
176
|
+
"""
|
|
177
|
+
if scope_path.is_file():
|
|
178
|
+
return (scope_path,)
|
|
179
|
+
git_candidates = git_directory_snapshot_paths(scope_path)
|
|
180
|
+
if git_candidates is not None:
|
|
181
|
+
return git_candidates
|
|
182
|
+
|
|
183
|
+
command = [
|
|
184
|
+
"rg",
|
|
185
|
+
"--files",
|
|
186
|
+
"--hidden",
|
|
187
|
+
"--no-require-git",
|
|
188
|
+
"--null",
|
|
189
|
+
# Also exclude descendants when the scope starts inside .git.
|
|
190
|
+
"--glob",
|
|
191
|
+
"!**/.git",
|
|
192
|
+
"--glob",
|
|
193
|
+
"!**/.git/**",
|
|
194
|
+
"--",
|
|
195
|
+
str(scope_path.relative_to(repo)),
|
|
196
|
+
]
|
|
197
|
+
try:
|
|
198
|
+
result = subprocess.run(command, cwd=repo, capture_output=True, check=False)
|
|
199
|
+
except OSError as exc:
|
|
200
|
+
ignore_names = (".gitignore", ".ignore", ".rgignore")
|
|
201
|
+
ancestors = (scope_path, *scope_path.parents)
|
|
202
|
+
has_ignore_rules = (
|
|
203
|
+
any((ancestor / ".git").exists() for ancestor in (repo, *repo.parents))
|
|
204
|
+
or any(
|
|
205
|
+
(ancestor / name).is_file()
|
|
206
|
+
for ancestor in ancestors
|
|
207
|
+
if ancestor == repo or repo in ancestor.parents
|
|
208
|
+
for name in ignore_names
|
|
209
|
+
)
|
|
210
|
+
or any(path.name in ignore_names for path in scope_path.rglob("*") if path.is_file())
|
|
211
|
+
)
|
|
212
|
+
if has_ignore_rules:
|
|
213
|
+
raise SystemExit(
|
|
214
|
+
"Could not safely enumerate ignored scoped files without Git or ripgrep."
|
|
215
|
+
) from exc
|
|
216
|
+
return scope_path.rglob("*")
|
|
438
217
|
|
|
439
|
-
|
|
440
|
-
|
|
441
|
-
|
|
442
|
-
|
|
443
|
-
raise SystemExit(f"{label} contains duplicate paths: {duplicates}")
|
|
444
|
-
return counts.keys()
|
|
218
|
+
if result.returncode not in (0, 1):
|
|
219
|
+
detail = result.stderr.decode("utf-8", errors="replace").strip()
|
|
220
|
+
raise SystemExit(f"Could not enumerate scoped repository files: {detail}")
|
|
221
|
+
return (repo / os.fsdecode(path) for path in result.stdout.split(b"\0") if path)
|
|
445
222
|
|
|
446
223
|
|
|
447
224
|
def make_repo_rank_input(args: argparse.Namespace) -> None:
|
|
@@ -463,7 +240,7 @@ def make_repo_rank_input(args: argparse.Namespace) -> None:
|
|
|
463
240
|
for scope_abs in resolved_scopes:
|
|
464
241
|
scope_rel = scope_abs.relative_to(repo)
|
|
465
242
|
area = args.area or scope_rel.as_posix()
|
|
466
|
-
candidates = (
|
|
243
|
+
candidates = scope_candidates(repo, scope_abs)
|
|
467
244
|
for path in candidates:
|
|
468
245
|
try:
|
|
469
246
|
if path.is_symlink() or not path.is_file():
|
|
@@ -473,26 +250,14 @@ def make_repo_rank_input(args: argparse.Namespace) -> None:
|
|
|
473
250
|
continue
|
|
474
251
|
rel = path.relative_to(repo)
|
|
475
252
|
directly_requested = path in directly_requested_files
|
|
476
|
-
|
|
477
|
-
path.relative_to(scope_abs if scope_abs.is_dir() else scope_abs.parent)
|
|
478
|
-
if explicit_scopes
|
|
479
|
-
else rel
|
|
480
|
-
)
|
|
481
|
-
if not directly_requested and (
|
|
482
|
-
path_is_excluded(excluded_path) or path.suffix.lower() not in TEXT_CODE_EXTENSIONS
|
|
483
|
-
):
|
|
253
|
+
if ".git" in rel.parts:
|
|
484
254
|
continue
|
|
485
255
|
|
|
486
|
-
|
|
487
|
-
|
|
488
|
-
|
|
489
|
-
and path.name not in EXCLUDED_FILENAMES
|
|
490
|
-
):
|
|
491
|
-
preview = ""
|
|
492
|
-
else:
|
|
493
|
-
preview, is_binary = preview_for(path, args.preview_bytes)
|
|
494
|
-
if is_binary and not directly_requested:
|
|
256
|
+
preview, is_binary = preview_for(path, args.preview_bytes)
|
|
257
|
+
if is_binary or is_binary_file(path):
|
|
258
|
+
if not directly_requested:
|
|
495
259
|
continue
|
|
260
|
+
preview = ""
|
|
496
261
|
rows_by_path.setdefault(
|
|
497
262
|
rel.as_posix(),
|
|
498
263
|
{"path": rel.as_posix(), "area": area, "preview": preview},
|
|
@@ -513,58 +278,7 @@ def make_repo_scope_input(args: argparse.Namespace) -> None:
|
|
|
513
278
|
rows_by_path: dict[str, JsonRow] = {}
|
|
514
279
|
for scope in scopes:
|
|
515
280
|
scope_path = resolve_scope(repo, scope, expand_user=False, reject_symlinks=True)
|
|
516
|
-
|
|
517
|
-
candidates = (scope_path,)
|
|
518
|
-
else:
|
|
519
|
-
git_candidates = git_directory_snapshot_paths(scope_path)
|
|
520
|
-
if git_candidates is not None:
|
|
521
|
-
candidates = git_candidates
|
|
522
|
-
else:
|
|
523
|
-
command = [
|
|
524
|
-
"rg",
|
|
525
|
-
"--files",
|
|
526
|
-
"--hidden",
|
|
527
|
-
"--no-require-git",
|
|
528
|
-
"--null",
|
|
529
|
-
# Also exclude descendants when the scope starts inside .git.
|
|
530
|
-
"--glob",
|
|
531
|
-
"!**/.git",
|
|
532
|
-
"--glob",
|
|
533
|
-
"!**/.git/**",
|
|
534
|
-
"--",
|
|
535
|
-
str(scope_path.relative_to(repo)),
|
|
536
|
-
]
|
|
537
|
-
try:
|
|
538
|
-
result = subprocess.run(command, cwd=repo, capture_output=True, check=False)
|
|
539
|
-
except OSError as exc:
|
|
540
|
-
ignore_names = (".gitignore", ".ignore", ".rgignore")
|
|
541
|
-
ancestors = (scope_path, *scope_path.parents)
|
|
542
|
-
has_ignore_rules = (
|
|
543
|
-
any((ancestor / ".git").exists() for ancestor in (repo, *repo.parents))
|
|
544
|
-
or any(
|
|
545
|
-
(ancestor / name).is_file()
|
|
546
|
-
for ancestor in ancestors
|
|
547
|
-
if ancestor == repo or repo in ancestor.parents
|
|
548
|
-
for name in ignore_names
|
|
549
|
-
)
|
|
550
|
-
or any(
|
|
551
|
-
path.name in ignore_names
|
|
552
|
-
for path in scope_path.rglob("*")
|
|
553
|
-
if path.is_file()
|
|
554
|
-
)
|
|
555
|
-
)
|
|
556
|
-
if has_ignore_rules:
|
|
557
|
-
raise SystemExit(
|
|
558
|
-
"Could not safely enumerate ignored scoped files without Git or ripgrep."
|
|
559
|
-
) from exc
|
|
560
|
-
candidates = scope_path.rglob("*")
|
|
561
|
-
else:
|
|
562
|
-
if result.returncode not in (0, 1):
|
|
563
|
-
detail = result.stderr.decode("utf-8", errors="replace").strip()
|
|
564
|
-
raise SystemExit(f"Could not enumerate scoped repository files: {detail}")
|
|
565
|
-
candidates = (
|
|
566
|
-
repo / os.fsdecode(path) for path in result.stdout.split(b"\0") if path
|
|
567
|
-
)
|
|
281
|
+
candidates = scope_candidates(repo, scope_path)
|
|
568
282
|
for path in candidates:
|
|
569
283
|
try:
|
|
570
284
|
if path.is_symlink() or not path.is_file():
|
|
@@ -582,41 +296,15 @@ def make_repo_scope_input(args: argparse.Namespace) -> None:
|
|
|
582
296
|
print(f"Wrote {len(rows)} scoped paths to {output}")
|
|
583
297
|
|
|
584
298
|
|
|
585
|
-
def bind_repo_scopes(args: argparse.Namespace) -> None:
|
|
586
|
-
scopes = load_scopes_file(Path(args.scopes_file).expanduser())
|
|
587
|
-
manifest_path = Path(args.manifest).expanduser()
|
|
588
|
-
coverage_path = Path(args.coverage).expanduser()
|
|
589
|
-
try:
|
|
590
|
-
manifest: object = json.loads(manifest_path.read_text(encoding="utf-8"))
|
|
591
|
-
coverage: object = json.loads(coverage_path.read_text(encoding="utf-8"))
|
|
592
|
-
if not isinstance(manifest, dict) or not isinstance(coverage, dict):
|
|
593
|
-
raise ValueError("expected JSON objects")
|
|
594
|
-
scan = manifest.get("scan")
|
|
595
|
-
if not isinstance(scan, dict):
|
|
596
|
-
raise ValueError("manifest.scan must be an object")
|
|
597
|
-
scope = scan.get("scope")
|
|
598
|
-
if not isinstance(scope, dict):
|
|
599
|
-
raise ValueError("manifest.scan.scope must be an object")
|
|
600
|
-
except (OSError, UnicodeError, json.JSONDecodeError, ValueError) as exc:
|
|
601
|
-
raise SystemExit("Unable to bind requested scopes into the scan contract") from exc
|
|
602
|
-
scope["includePaths"] = scopes
|
|
603
|
-
coverage["includePaths"] = scopes
|
|
604
|
-
manifest_path.write_text(
|
|
605
|
-
json.dumps(manifest, ensure_ascii=True, indent=2) + "\n", encoding="utf-8"
|
|
606
|
-
)
|
|
607
|
-
coverage_path.write_text(
|
|
608
|
-
json.dumps(coverage, ensure_ascii=True, indent=2) + "\n", encoding="utf-8"
|
|
609
|
-
)
|
|
610
|
-
print(f"Bound {len(scopes)} requested scopes into the scan contract")
|
|
611
|
-
|
|
612
|
-
|
|
613
299
|
def run_git_changed_paths(repo: Path, diff_args: list[str]) -> list[tuple[Path, str]]:
|
|
300
|
+
"""Return changed regular files from the selected side of each change."""
|
|
614
301
|
result = git_command(
|
|
615
302
|
repo,
|
|
616
303
|
"diff",
|
|
617
|
-
"--
|
|
304
|
+
"--ignore-submodules=all",
|
|
305
|
+
"--raw",
|
|
618
306
|
"-z",
|
|
619
|
-
"--diff-filter=
|
|
307
|
+
"--diff-filter=ACMRDT",
|
|
620
308
|
*diff_args,
|
|
621
309
|
text=False,
|
|
622
310
|
)
|
|
@@ -628,13 +316,16 @@ def run_git_changed_paths(repo: Path, diff_args: list[str]) -> list[tuple[Path,
|
|
|
628
316
|
changed: list[tuple[Path, str]] = []
|
|
629
317
|
index = 0
|
|
630
318
|
while index < len(fields):
|
|
631
|
-
|
|
319
|
+
metadata = fields[index].split()
|
|
320
|
+
status = chr(metadata[-1][0])
|
|
632
321
|
index += 1
|
|
633
322
|
if status in {"C", "R"}:
|
|
634
323
|
index += 1
|
|
635
324
|
path = repo / os.fsdecode(fields[index])
|
|
636
325
|
index += 1
|
|
637
|
-
|
|
326
|
+
selected_mode = metadata[0].removeprefix(b":") if status == "D" else metadata[1]
|
|
327
|
+
if selected_mode.startswith(b"100"):
|
|
328
|
+
changed.append((path, status))
|
|
638
329
|
return changed
|
|
639
330
|
|
|
640
331
|
|
|
@@ -658,9 +349,13 @@ def git_changed_paths(repo: Path, base: str, head: str, mode: str) -> list[tuple
|
|
|
658
349
|
combined.update(
|
|
659
350
|
(repo / os.fsdecode(relative), "A")
|
|
660
351
|
for relative in untracked.stdout.split(b"\0")
|
|
661
|
-
if relative
|
|
352
|
+
if relative and not relative.endswith(b"/")
|
|
353
|
+
)
|
|
354
|
+
return sorted(
|
|
355
|
+
(path, status)
|
|
356
|
+
for path, status in combined.items()
|
|
357
|
+
if status == "D" or (not path.is_symlink() and path.is_file())
|
|
662
358
|
)
|
|
663
|
-
return sorted(combined.items())
|
|
664
359
|
raise SystemExit(f"Unknown diff mode: {mode}")
|
|
665
360
|
|
|
666
361
|
|
|
@@ -669,25 +364,16 @@ def make_diff_rank_input(args: argparse.Namespace) -> None:
|
|
|
669
364
|
if not repo.is_dir():
|
|
670
365
|
raise SystemExit(f"Repo path not found: {repo}")
|
|
671
366
|
|
|
672
|
-
changed =
|
|
673
|
-
|
|
674
|
-
|
|
675
|
-
|
|
676
|
-
and path.suffix.lower() in TEXT_CODE_EXTENSIONS
|
|
677
|
-
]
|
|
678
|
-
revision_paths = [
|
|
679
|
-
path.relative_to(repo)
|
|
680
|
-
for path, status in changed
|
|
681
|
-
if args.mode == "revisions" and status != "D"
|
|
682
|
-
]
|
|
683
|
-
revision_blobs = dict(
|
|
684
|
-
zip(
|
|
685
|
-
revision_paths,
|
|
686
|
-
git_blob_bytes(
|
|
687
|
-
repo,
|
|
688
|
-
[f"{args.head}:{path.as_posix()}" for path in revision_paths],
|
|
689
|
-
),
|
|
367
|
+
changed = git_changed_paths(repo, args.base, args.head, args.mode)
|
|
368
|
+
revision_refs = {
|
|
369
|
+
path.relative_to(repo): (
|
|
370
|
+
f"{args.base if status == 'D' else args.head}:{path.relative_to(repo).as_posix()}"
|
|
690
371
|
)
|
|
372
|
+
for path, status in changed
|
|
373
|
+
if args.mode == "revisions" or status == "D"
|
|
374
|
+
}
|
|
375
|
+
revision_samples = dict(
|
|
376
|
+
zip(revision_refs, git_blob_samples(repo, list(revision_refs.values())))
|
|
691
377
|
)
|
|
692
378
|
|
|
693
379
|
rows: list[JsonRow] = []
|
|
@@ -695,23 +381,24 @@ def make_diff_rank_input(args: argparse.Namespace) -> None:
|
|
|
695
381
|
rel = path.relative_to(repo)
|
|
696
382
|
|
|
697
383
|
preview = ""
|
|
698
|
-
if
|
|
699
|
-
|
|
700
|
-
if
|
|
701
|
-
|
|
702
|
-
|
|
703
|
-
|
|
704
|
-
preview, is_binary = preview_for_bytes(rel, content, args.preview_bytes)
|
|
384
|
+
if args.mode == "revisions" or status == "D":
|
|
385
|
+
sample = revision_samples[rel]
|
|
386
|
+
if sample is None:
|
|
387
|
+
revision = args.base if status == "D" else args.head
|
|
388
|
+
raise SystemExit(f"Unable to read committed diff blob: {revision}:{rel.as_posix()}")
|
|
389
|
+
content, is_binary = sample
|
|
705
390
|
if is_binary:
|
|
706
391
|
continue
|
|
707
|
-
|
|
392
|
+
if status != "D":
|
|
393
|
+
preview, _ = preview_for_bytes(content, args.preview_bytes)
|
|
394
|
+
elif not path.is_symlink() and path.is_file():
|
|
708
395
|
try:
|
|
709
396
|
path.resolve(strict=True).relative_to(repo)
|
|
710
397
|
except (OSError, ValueError):
|
|
711
398
|
preview = ""
|
|
712
399
|
else:
|
|
713
400
|
preview, is_binary = preview_for(path, args.preview_bytes)
|
|
714
|
-
if is_binary:
|
|
401
|
+
if is_binary or is_binary_file(path):
|
|
715
402
|
continue
|
|
716
403
|
rows.append({"path": rel.as_posix(), "area": args.area, "preview": preview})
|
|
717
404
|
|
|
@@ -721,431 +408,14 @@ def make_diff_rank_input(args: argparse.Namespace) -> None:
|
|
|
721
408
|
print(f"Wrote {len(rows)} rows to {output}")
|
|
722
409
|
|
|
723
410
|
|
|
724
|
-
def make_rank_shards(args: argparse.Namespace) -> None:
|
|
725
|
-
if args.max_rows < 1:
|
|
726
|
-
raise SystemExit("--max-rows must be at least 1")
|
|
727
|
-
|
|
728
|
-
rank_input = Path(args.rank_input).expanduser()
|
|
729
|
-
rows = load_jsonl(rank_input, "Rank input", validate_rank_input_row)
|
|
730
|
-
require_unique_paths(rows, "Rank input")
|
|
731
|
-
|
|
732
|
-
output_dir = Path(args.out_dir).expanduser()
|
|
733
|
-
output_dir.mkdir(parents=True, exist_ok=True)
|
|
734
|
-
existing = sorted((*output_dir.glob(SHARD_INPUT_GLOB), *output_dir.glob(SHARD_OUTPUT_GLOB)))
|
|
735
|
-
if existing:
|
|
736
|
-
raise SystemExit(f"Rank shard directory already contains shard files: {output_dir}")
|
|
737
|
-
|
|
738
|
-
shard_count = 0
|
|
739
|
-
for shard_count, start in enumerate(range(0, len(rows), args.max_rows), start=1):
|
|
740
|
-
shard_path = output_dir / f"rank-shard-{shard_count:04d}.input.jsonl"
|
|
741
|
-
write_jsonl(shard_path, rows[start : start + args.max_rows])
|
|
742
|
-
|
|
743
|
-
print(f"Wrote {shard_count} rank shards to {output_dir}")
|
|
744
|
-
|
|
745
|
-
|
|
746
|
-
def discover_input_shards(shard_dir: Path) -> list[Path]:
|
|
747
|
-
if not shard_dir.is_dir():
|
|
748
|
-
raise SystemExit(f"Rank shard directory missing: {shard_dir}")
|
|
749
|
-
|
|
750
|
-
numbered_shards: list[tuple[int, Path]] = []
|
|
751
|
-
for path in shard_dir.glob(SHARD_INPUT_GLOB):
|
|
752
|
-
match = SHARD_INPUT_PATTERN.fullmatch(path.name)
|
|
753
|
-
if match is None:
|
|
754
|
-
raise SystemExit(f"Rank input shard has invalid name: {path.name}")
|
|
755
|
-
numbered_shards.append((int(match.group(1)), path))
|
|
756
|
-
numbered_shards.sort(key=lambda item: (item[0], item[1].name))
|
|
757
|
-
input_shards = [path for _, path in numbered_shards]
|
|
758
|
-
expected_names = [
|
|
759
|
-
f"rank-shard-{index:04d}.input.jsonl" for index in range(1, len(input_shards) + 1)
|
|
760
|
-
]
|
|
761
|
-
actual_names = [path.name for path in input_shards]
|
|
762
|
-
if actual_names != expected_names:
|
|
763
|
-
raise SystemExit(
|
|
764
|
-
"Rank input shards must use contiguous canonical names; "
|
|
765
|
-
f"expected={expected_names}; actual={actual_names}"
|
|
766
|
-
)
|
|
767
|
-
return input_shards
|
|
768
|
-
|
|
769
|
-
|
|
770
|
-
def output_name_for(input_name: str) -> str:
|
|
771
|
-
return input_name.replace(".input.jsonl", ".output.jsonl")
|
|
772
|
-
|
|
773
|
-
|
|
774
|
-
def require_plan_shard_dir(plan_path: Path, shard_dir: Path) -> None:
|
|
775
|
-
expected = plan_path.parent / "rank_shards"
|
|
776
|
-
if shard_dir.resolve() != expected.resolve():
|
|
777
|
-
raise SystemExit(
|
|
778
|
-
"Rank shard directory must be the assignment plan's sibling rank_shards "
|
|
779
|
-
f"directory; expected={expected}; actual={shard_dir}"
|
|
780
|
-
)
|
|
781
|
-
|
|
782
|
-
|
|
783
|
-
def require_no_misplaced_rank_shards(plan_path: Path) -> None:
|
|
784
|
-
misplaced = sorted(
|
|
785
|
-
(
|
|
786
|
-
*plan_path.parent.glob(SHARD_INPUT_GLOB),
|
|
787
|
-
*plan_path.parent.glob(SHARD_OUTPUT_GLOB),
|
|
788
|
-
),
|
|
789
|
-
key=lambda path: path.name,
|
|
790
|
-
)
|
|
791
|
-
if misplaced:
|
|
792
|
-
raise SystemExit(
|
|
793
|
-
"Rank shard artifacts must be stored in the assignment plan's sibling "
|
|
794
|
-
f"rank_shards directory; misplaced={[path.name for path in misplaced]}"
|
|
795
|
-
)
|
|
796
|
-
|
|
797
|
-
|
|
798
|
-
def make_rank_pool_plan(args: argparse.Namespace) -> None:
|
|
799
|
-
if args.usable_worker_slots < 1:
|
|
800
|
-
raise SystemExit("--usable-worker-slots must be at least 1")
|
|
801
|
-
|
|
802
|
-
shard_dir = Path(args.shard_dir).expanduser()
|
|
803
|
-
output = Path(args.out).expanduser()
|
|
804
|
-
require_plan_shard_dir(output, shard_dir)
|
|
805
|
-
input_shards = discover_input_shards(shard_dir)
|
|
806
|
-
worker_count = min(len(input_shards), args.usable_worker_slots, RANK_POOL_WORKER_CAP)
|
|
807
|
-
workers: list[dict[str, object]] = []
|
|
808
|
-
for worker_index in range(worker_count):
|
|
809
|
-
assigned_inputs = [path.name for path in input_shards[worker_index::worker_count]]
|
|
810
|
-
workers.append(
|
|
811
|
-
{
|
|
812
|
-
"slot": worker_index + 1,
|
|
813
|
-
"input_shards": assigned_inputs,
|
|
814
|
-
"output_shards": [output_name_for(name) for name in assigned_inputs],
|
|
815
|
-
}
|
|
816
|
-
)
|
|
817
|
-
|
|
818
|
-
plan: dict[str, object] = {
|
|
819
|
-
"schema_version": RANK_POOL_PLAN_SCHEMA_VERSION,
|
|
820
|
-
"strategy": RANK_POOL_STRATEGY,
|
|
821
|
-
"shard_count": len(input_shards),
|
|
822
|
-
"ranking_worker_count": worker_count,
|
|
823
|
-
"workers": workers,
|
|
824
|
-
}
|
|
825
|
-
write_json(output, plan)
|
|
826
|
-
print(f"Assigned {len(input_shards)} rank shards to {worker_count} ranking workers in {output}")
|
|
827
|
-
|
|
828
|
-
|
|
829
|
-
def load_rank_pool_plan(plan_path: Path) -> tuple[dict[str, object], bytes]:
|
|
830
|
-
if not plan_path.exists():
|
|
831
|
-
raise SystemExit(f"Rank pool plan missing: {plan_path}")
|
|
832
|
-
plan_bytes = plan_path.read_bytes()
|
|
833
|
-
try:
|
|
834
|
-
payload: object = json.loads(plan_bytes)
|
|
835
|
-
except json.JSONDecodeError as exc:
|
|
836
|
-
raise SystemExit(f"{plan_path}: invalid JSON: {exc.msg}") from exc
|
|
837
|
-
if not isinstance(payload, dict):
|
|
838
|
-
raise SystemExit(f"{plan_path}: expected a JSON object")
|
|
839
|
-
return payload, plan_bytes
|
|
840
|
-
|
|
841
|
-
|
|
842
|
-
def require_integer(value: object, label: str, *, minimum: int) -> int:
|
|
843
|
-
if isinstance(value, bool) or not isinstance(value, int) or value < minimum:
|
|
844
|
-
raise SystemExit(f"{label} must be an integer of at least {minimum}")
|
|
845
|
-
return value
|
|
846
|
-
|
|
847
|
-
|
|
848
|
-
def require_string_list(value: object, label: str) -> list[str]:
|
|
849
|
-
if not isinstance(value, list) or not value:
|
|
850
|
-
raise SystemExit(f"{label} must be a non-empty list")
|
|
851
|
-
if any(not isinstance(item, str) or not item for item in value):
|
|
852
|
-
raise SystemExit(f"{label} entries must be non-empty strings")
|
|
853
|
-
return [item for item in value if isinstance(item, str)]
|
|
854
|
-
|
|
855
|
-
|
|
856
|
-
def validate_rank_pool_plan(
|
|
857
|
-
plan_path: Path, shard_dir: Path
|
|
858
|
-
) -> tuple[list[Path], list[tuple[list[str], list[str]]], bytes]:
|
|
859
|
-
require_plan_shard_dir(plan_path, shard_dir)
|
|
860
|
-
require_no_misplaced_rank_shards(plan_path)
|
|
861
|
-
input_shards = discover_input_shards(shard_dir)
|
|
862
|
-
input_names = [path.name for path in input_shards]
|
|
863
|
-
plan, plan_bytes = load_rank_pool_plan(plan_path)
|
|
864
|
-
expected_fields = {
|
|
865
|
-
"schema_version",
|
|
866
|
-
"strategy",
|
|
867
|
-
"shard_count",
|
|
868
|
-
"ranking_worker_count",
|
|
869
|
-
"workers",
|
|
870
|
-
}
|
|
871
|
-
actual_fields = set(plan)
|
|
872
|
-
if actual_fields != expected_fields:
|
|
873
|
-
raise SystemExit(
|
|
874
|
-
f"{plan_path}: rank pool plan fields do not match schema; "
|
|
875
|
-
f"missing={sorted(expected_fields - actual_fields)}; "
|
|
876
|
-
f"unexpected={sorted(actual_fields - expected_fields)}"
|
|
877
|
-
)
|
|
878
|
-
schema_version = require_integer(
|
|
879
|
-
plan["schema_version"], f"{plan_path}: schema_version", minimum=1
|
|
880
|
-
)
|
|
881
|
-
if schema_version != RANK_POOL_PLAN_SCHEMA_VERSION:
|
|
882
|
-
raise SystemExit(f"{plan_path}: schema_version must be {RANK_POOL_PLAN_SCHEMA_VERSION}")
|
|
883
|
-
if plan["strategy"] != RANK_POOL_STRATEGY:
|
|
884
|
-
raise SystemExit(f"{plan_path}: strategy must be {RANK_POOL_STRATEGY}")
|
|
885
|
-
|
|
886
|
-
shard_count = require_integer(plan["shard_count"], f"{plan_path}: shard_count", minimum=0)
|
|
887
|
-
if shard_count != len(input_shards):
|
|
888
|
-
raise SystemExit(
|
|
889
|
-
f"{plan_path}: shard_count does not match input shards; "
|
|
890
|
-
f"plan={shard_count}; actual={len(input_shards)}"
|
|
891
|
-
)
|
|
892
|
-
worker_count = require_integer(
|
|
893
|
-
plan["ranking_worker_count"], f"{plan_path}: ranking_worker_count", minimum=0
|
|
894
|
-
)
|
|
895
|
-
if shard_count > 0 and worker_count == 0:
|
|
896
|
-
raise SystemExit(
|
|
897
|
-
f"{plan_path}: ranking_worker_count must be at least 1 when input shards exist"
|
|
898
|
-
)
|
|
899
|
-
if worker_count > shard_count:
|
|
900
|
-
raise SystemExit(f"{plan_path}: ranking_worker_count cannot exceed shard_count")
|
|
901
|
-
if worker_count > RANK_POOL_WORKER_CAP:
|
|
902
|
-
raise SystemExit(f"{plan_path}: ranking_worker_count cannot exceed {RANK_POOL_WORKER_CAP}")
|
|
903
|
-
|
|
904
|
-
workers = plan["workers"]
|
|
905
|
-
if not isinstance(workers, list) or len(workers) != worker_count:
|
|
906
|
-
raise SystemExit(
|
|
907
|
-
f"{plan_path}: workers must contain exactly {worker_count} worker assignments"
|
|
908
|
-
)
|
|
909
|
-
|
|
910
|
-
counts: Counter[str] = Counter()
|
|
911
|
-
parsed_workers: list[tuple[list[str], list[str]]] = []
|
|
912
|
-
worker_fields = {"slot", "input_shards", "output_shards"}
|
|
913
|
-
for worker_index, worker in enumerate(workers):
|
|
914
|
-
label = f"{plan_path}: workers[{worker_index}]"
|
|
915
|
-
if not isinstance(worker, dict):
|
|
916
|
-
raise SystemExit(f"{label} must be a JSON object")
|
|
917
|
-
if set(worker) != worker_fields:
|
|
918
|
-
raise SystemExit(
|
|
919
|
-
f"{label} fields do not match schema; "
|
|
920
|
-
f"missing={sorted(worker_fields - set(worker))}; "
|
|
921
|
-
f"unexpected={sorted(set(worker) - worker_fields)}"
|
|
922
|
-
)
|
|
923
|
-
slot = require_integer(worker["slot"], f"{label}.slot", minimum=1)
|
|
924
|
-
if slot != worker_index + 1:
|
|
925
|
-
raise SystemExit(f"{label}.slot must be {worker_index + 1}")
|
|
926
|
-
worker_inputs = require_string_list(worker["input_shards"], f"{label}.input_shards")
|
|
927
|
-
worker_outputs = require_string_list(worker["output_shards"], f"{label}.output_shards")
|
|
928
|
-
if len(worker_inputs) != len(worker_outputs):
|
|
929
|
-
raise SystemExit(f"{label} input_shards and output_shards lengths must match")
|
|
930
|
-
expected_worker_outputs = [output_name_for(name) for name in worker_inputs]
|
|
931
|
-
if worker_outputs != expected_worker_outputs:
|
|
932
|
-
raise SystemExit(f"{label}.output_shards do not match its input_shards")
|
|
933
|
-
counts.update(worker_inputs)
|
|
934
|
-
parsed_workers.append((worker_inputs, worker_outputs))
|
|
935
|
-
|
|
936
|
-
# Canonical input names map one-to-one to the per-worker outputs checked above.
|
|
937
|
-
duplicates = sorted(name for name, count in counts.items() if count > 1)
|
|
938
|
-
expected = set(input_names)
|
|
939
|
-
missing = sorted(expected - counts.keys())
|
|
940
|
-
unexpected = sorted(counts.keys() - expected)
|
|
941
|
-
if missing or duplicates or unexpected:
|
|
942
|
-
raise SystemExit(
|
|
943
|
-
f"{plan_path}: pool plan must assign each input shard exactly once; "
|
|
944
|
-
f"missing={missing}; duplicates={duplicates}; unexpected={unexpected}"
|
|
945
|
-
)
|
|
946
|
-
for worker_index, (worker_inputs, _) in enumerate(parsed_workers):
|
|
947
|
-
expected_inputs = input_names[worker_index::worker_count]
|
|
948
|
-
if worker_inputs != expected_inputs:
|
|
949
|
-
raise SystemExit(
|
|
950
|
-
f"{plan_path}: worker slot {worker_index + 1} does not match the deterministic "
|
|
951
|
-
f"{RANK_POOL_STRATEGY} assignment"
|
|
952
|
-
)
|
|
953
|
-
return input_shards, parsed_workers, plan_bytes
|
|
954
|
-
|
|
955
|
-
|
|
956
|
-
def validate_rank_worker_command(args: argparse.Namespace) -> None:
|
|
957
|
-
plan_path = Path(args.plan).expanduser()
|
|
958
|
-
shard_dir = Path(args.shard_dir).expanduser()
|
|
959
|
-
_, workers, plan_bytes = validate_rank_pool_plan(plan_path, shard_dir)
|
|
960
|
-
|
|
961
|
-
slot = require_integer(args.slot, "--slot", minimum=1)
|
|
962
|
-
worker_count = len(workers)
|
|
963
|
-
if slot > worker_count:
|
|
964
|
-
raise SystemExit(f"--slot must be at most {worker_count}")
|
|
965
|
-
|
|
966
|
-
input_names, output_names = workers[slot - 1]
|
|
967
|
-
|
|
968
|
-
row_count = 0
|
|
969
|
-
outputs_digest = hashlib.sha256()
|
|
970
|
-
for input_name, output_name in zip(input_names, output_names, strict=True):
|
|
971
|
-
input_shard = shard_dir / input_name
|
|
972
|
-
output_shard = shard_dir / output_name
|
|
973
|
-
_, output_rows = validate_rank_shard(input_shard, output_shard)
|
|
974
|
-
output_bytes = output_shard.read_bytes()
|
|
975
|
-
row_count += len(output_rows)
|
|
976
|
-
outputs_digest.update(output_name.encode("utf-8"))
|
|
977
|
-
outputs_digest.update(b"\0")
|
|
978
|
-
outputs_digest.update(output_bytes)
|
|
979
|
-
outputs_digest.update(b"\0")
|
|
980
|
-
|
|
981
|
-
receipt: dict[str, object] = {
|
|
982
|
-
"schema_version": 1,
|
|
983
|
-
"plan_sha256": hashlib.sha256(plan_bytes).hexdigest(),
|
|
984
|
-
"slot": slot,
|
|
985
|
-
"ranking_worker_count": worker_count,
|
|
986
|
-
"output_shards": len(output_names),
|
|
987
|
-
"rows": row_count,
|
|
988
|
-
"outputs_sha256": outputs_digest.hexdigest(),
|
|
989
|
-
"status": "complete",
|
|
990
|
-
}
|
|
991
|
-
print("RANK_WORKER_RECEIPT " + json.dumps(receipt, sort_keys=True, separators=(",", ":")))
|
|
992
|
-
|
|
993
|
-
|
|
994
|
-
def validate_rank_pool_command(args: argparse.Namespace) -> None:
|
|
995
|
-
plan_path = Path(args.plan).expanduser()
|
|
996
|
-
shard_dir = Path(args.shard_dir).expanduser()
|
|
997
|
-
input_shards, workers, _ = validate_rank_pool_plan(plan_path, shard_dir)
|
|
998
|
-
|
|
999
|
-
actual_output_names = {path.name for path in shard_dir.glob(SHARD_OUTPUT_GLOB)}
|
|
1000
|
-
expected_outputs = {output_name_for(path.name) for path in input_shards}
|
|
1001
|
-
if actual_output_names != expected_outputs:
|
|
1002
|
-
missing = sorted(expected_outputs - actual_output_names)
|
|
1003
|
-
unexpected = sorted(actual_output_names - expected_outputs)
|
|
1004
|
-
raise SystemExit(
|
|
1005
|
-
"Rank pool outputs are incomplete; "
|
|
1006
|
-
f"missing output shards={missing}; unexpected output shards={unexpected}"
|
|
1007
|
-
)
|
|
1008
|
-
|
|
1009
|
-
row_count = 0
|
|
1010
|
-
for input_shard in input_shards:
|
|
1011
|
-
output_shard = input_shard.with_name(output_name_for(input_shard.name))
|
|
1012
|
-
_, shard_outputs = validate_rank_shard(input_shard, output_shard)
|
|
1013
|
-
row_count += len(shard_outputs)
|
|
1014
|
-
print(
|
|
1015
|
-
f"Validated {len(workers)} ranking workers, "
|
|
1016
|
-
f"{len(input_shards)} shards, and {row_count} ranking rows"
|
|
1017
|
-
)
|
|
1018
|
-
|
|
1019
|
-
|
|
1020
|
-
def validate_rank_shard(
|
|
1021
|
-
input_shard: Path, output_shard: Path
|
|
1022
|
-
) -> tuple[list[JsonRow], list[JsonRow]]:
|
|
1023
|
-
shard_inputs = load_jsonl(input_shard, "Rank input shard", validate_rank_input_row)
|
|
1024
|
-
expected_paths = require_unique_paths(shard_inputs, f"Rank input shard {input_shard.name}")
|
|
1025
|
-
shard_outputs = load_jsonl(output_shard, "Rank output shard", validate_rank_output_row)
|
|
1026
|
-
actual_paths = require_unique_paths(shard_outputs, f"Rank output shard {output_shard.name}")
|
|
1027
|
-
|
|
1028
|
-
if expected_paths != actual_paths:
|
|
1029
|
-
missing = sorted(expected_paths - actual_paths)
|
|
1030
|
-
unknown = sorted(actual_paths - expected_paths)
|
|
1031
|
-
raise SystemExit(
|
|
1032
|
-
f"{output_shard}: paths do not match its input shard; "
|
|
1033
|
-
f"missing={missing}; unknown={unknown}"
|
|
1034
|
-
)
|
|
1035
|
-
|
|
1036
|
-
area_by_path = {str(row["path"]): row["area"] for row in shard_inputs}
|
|
1037
|
-
for row in shard_outputs:
|
|
1038
|
-
row_path = str(row["path"])
|
|
1039
|
-
if row["area"] != area_by_path[row_path]:
|
|
1040
|
-
raise SystemExit(f"{output_shard}: area does not match rank input for {row_path}")
|
|
1041
|
-
return shard_inputs, shard_outputs
|
|
1042
|
-
|
|
1043
|
-
|
|
1044
|
-
def validate_rank_shard_command(args: argparse.Namespace) -> None:
|
|
1045
|
-
input_shard = Path(args.input).expanduser()
|
|
1046
|
-
output_shard = Path(args.output).expanduser()
|
|
1047
|
-
_, output_rows = validate_rank_shard(input_shard, output_shard)
|
|
1048
|
-
print(f"Validated {len(output_rows)} ranking rows in {output_shard}")
|
|
1049
|
-
|
|
1050
|
-
|
|
1051
|
-
def merge_rank_outputs(args: argparse.Namespace) -> None:
|
|
1052
|
-
rank_input = Path(args.rank_input).expanduser()
|
|
1053
|
-
authoritative_rows = load_jsonl(rank_input, "Rank input", validate_rank_input_row)
|
|
1054
|
-
require_unique_paths(authoritative_rows, "Rank input")
|
|
1055
|
-
|
|
1056
|
-
shard_dir = Path(args.shard_dir).expanduser()
|
|
1057
|
-
input_shards = discover_input_shards(shard_dir)
|
|
1058
|
-
output_shards = sorted(shard_dir.glob(SHARD_OUTPUT_GLOB))
|
|
1059
|
-
expected_output_names = {
|
|
1060
|
-
path.name.replace(".input.jsonl", ".output.jsonl") for path in input_shards
|
|
1061
|
-
}
|
|
1062
|
-
actual_output_names = {path.name for path in output_shards}
|
|
1063
|
-
if expected_output_names != actual_output_names:
|
|
1064
|
-
missing = sorted(expected_output_names - actual_output_names)
|
|
1065
|
-
unexpected = sorted(actual_output_names - expected_output_names)
|
|
1066
|
-
details: list[str] = []
|
|
1067
|
-
if missing:
|
|
1068
|
-
details.append(f"missing output shards {missing}")
|
|
1069
|
-
if unexpected:
|
|
1070
|
-
details.append(f"unexpected output shards {unexpected}")
|
|
1071
|
-
raise SystemExit(f"Rank shard outputs are incomplete: {'; '.join(details)}")
|
|
1072
|
-
|
|
1073
|
-
sharded_inputs: list[JsonRow] = []
|
|
1074
|
-
output_by_path: dict[str, JsonRow] = {}
|
|
1075
|
-
for input_shard in input_shards:
|
|
1076
|
-
output_shard = input_shard.with_name(
|
|
1077
|
-
input_shard.name.replace(".input.jsonl", ".output.jsonl")
|
|
1078
|
-
)
|
|
1079
|
-
shard_inputs, shard_outputs = validate_rank_shard(input_shard, output_shard)
|
|
1080
|
-
sharded_inputs.extend(shard_inputs)
|
|
1081
|
-
for row in shard_outputs:
|
|
1082
|
-
row_path = str(row["path"])
|
|
1083
|
-
if row_path in output_by_path:
|
|
1084
|
-
raise SystemExit(f"Rank outputs contain duplicate path: {row_path}")
|
|
1085
|
-
output_by_path[row_path] = row
|
|
1086
|
-
|
|
1087
|
-
if sharded_inputs != authoritative_rows:
|
|
1088
|
-
raise SystemExit("Rank input shards do not exactly partition the authoritative rank input")
|
|
1089
|
-
|
|
1090
|
-
merged = [output_by_path[str(row["path"])] for row in authoritative_rows]
|
|
1091
|
-
output = Path(args.out).expanduser()
|
|
1092
|
-
write_jsonl(output, merged)
|
|
1093
|
-
print(f"Merged {len(merged)} ranking rows into {output}")
|
|
1094
|
-
|
|
1095
|
-
|
|
1096
|
-
def copy_deep_review_input(args: argparse.Namespace) -> None:
|
|
1097
|
-
rank_input = Path(args.rank_input).expanduser()
|
|
1098
|
-
rows = load_jsonl(rank_input, "Rank input", validate_rank_input_row)
|
|
1099
|
-
require_unique_paths(rows, "Rank input")
|
|
1100
|
-
selected = [{"path": row["path"], "area": row["area"]} for row in rows]
|
|
1101
|
-
|
|
1102
|
-
output = Path(args.out).expanduser()
|
|
1103
|
-
write_jsonl(output, selected)
|
|
1104
|
-
print(f"Copied {len(selected)} rows into {output}")
|
|
1105
|
-
|
|
1106
|
-
|
|
1107
|
-
def select_deep_review_input(args: argparse.Namespace) -> None:
|
|
1108
|
-
rank_output = Path(args.rank_output).expanduser()
|
|
1109
|
-
rows = load_jsonl(rank_output, "Rank output", validate_rank_output_row)
|
|
1110
|
-
require_unique_paths(rows, "Rank output")
|
|
1111
|
-
|
|
1112
|
-
included = [row for row in rows if row["include"]]
|
|
1113
|
-
base_rows = included if included else rows
|
|
1114
|
-
base_rows.sort(key=lambda row: (-int(row["score"]), str(row["path"])))
|
|
1115
|
-
keep = max(1, int(len(base_rows) * (args.top_percent / 100.0))) if base_rows else 0
|
|
1116
|
-
selected = [{"path": row["path"], "area": row["area"]} for row in base_rows[:keep]]
|
|
1117
|
-
|
|
1118
|
-
output = Path(args.out).expanduser()
|
|
1119
|
-
write_jsonl(output, selected)
|
|
1120
|
-
print(f"Selected {len(selected)} of {len(base_rows)} rows into {output}")
|
|
1121
|
-
|
|
1122
|
-
|
|
1123
411
|
def main() -> None:
|
|
1124
412
|
args = parse_args()
|
|
1125
413
|
if args.command == "make-repo-rank-input":
|
|
1126
414
|
make_repo_rank_input(args)
|
|
1127
415
|
elif args.command == "make-repo-scope-input":
|
|
1128
416
|
make_repo_scope_input(args)
|
|
1129
|
-
elif args.command == "bind-repo-scopes":
|
|
1130
|
-
bind_repo_scopes(args)
|
|
1131
417
|
elif args.command == "make-diff-rank-input":
|
|
1132
418
|
make_diff_rank_input(args)
|
|
1133
|
-
elif args.command == "make-rank-shards":
|
|
1134
|
-
make_rank_shards(args)
|
|
1135
|
-
elif args.command == "make-rank-pool-plan":
|
|
1136
|
-
make_rank_pool_plan(args)
|
|
1137
|
-
elif args.command == "validate-rank-shard":
|
|
1138
|
-
validate_rank_shard_command(args)
|
|
1139
|
-
elif args.command == "validate-rank-worker":
|
|
1140
|
-
validate_rank_worker_command(args)
|
|
1141
|
-
elif args.command == "validate-rank-pool":
|
|
1142
|
-
validate_rank_pool_command(args)
|
|
1143
|
-
elif args.command == "merge-rank-outputs":
|
|
1144
|
-
merge_rank_outputs(args)
|
|
1145
|
-
elif args.command == "copy-deep-review-input":
|
|
1146
|
-
copy_deep_review_input(args)
|
|
1147
|
-
elif args.command == "select-deep-review-input":
|
|
1148
|
-
select_deep_review_input(args)
|
|
1149
419
|
else:
|
|
1150
420
|
raise SystemExit(f"Unknown command: {args.command}")
|
|
1151
421
|
|