@openai/codex-security 0.5.1 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +151 -36
- package/_bundled_plugin/.mcp.json +1 -0
- package/_bundled_plugin/mcp/helpers.mjs.br.part-000 +0 -0
- package/_bundled_plugin/mcp/helpers.mjs.br.part-001 +0 -0
- package/_bundled_plugin/mcp/native/THIRD_PARTY_NOTICES.txt +1 -1
- package/_bundled_plugin/mcp/native/darwin-arm64/unix.node +0 -0
- package/_bundled_plugin/mcp/native/darwin-x64/unix.node +0 -0
- package/_bundled_plugin/mcp/native/linux-arm64-gnu/unix.node +0 -0
- package/_bundled_plugin/mcp/native/linux-arm64-musl/unix.node +0 -0
- package/_bundled_plugin/mcp/native/linux-x64-gnu/unix.node +0 -0
- package/_bundled_plugin/mcp/native/win32-arm64/windows.node +0 -0
- package/_bundled_plugin/mcp/native/win32-x64/windows.node +0 -0
- package/_bundled_plugin/mcp/server.mjs.br.part-000 +0 -0
- package/_bundled_plugin/mcp/server.mjs.br.part-001 +0 -0
- package/_bundled_plugin/mcp/server.mjs.br.part-002 +0 -0
- package/_bundled_plugin/references/access-advisory.md +11 -0
- package/_bundled_plugin/references/core-scan.md +30 -77
- package/_bundled_plugin/references/final-report.md +45 -189
- package/_bundled_plugin/references/scan-contract.md +2 -0
- package/_bundled_plugin/references/scan-prologue.md +1 -1
- package/_bundled_plugin/references/scan-runtime.md +1 -5
- package/_bundled_plugin/references/threat-model.md +33 -41
- package/_bundled_plugin/schemas/coverage.schema.json +12 -1
- package/_bundled_plugin/schemas/scan-manifest.schema.json +1 -0
- package/_bundled_plugin/schemas/tools/deep-reducer.schema.json +38 -1
- package/_bundled_plugin/scripts/deep_scan_config.py +39 -39
- package/_bundled_plugin/scripts/deep_scan_workbench.py +145 -85
- package/_bundled_plugin/scripts/finalize_scan_contract.py +189 -57
- package/_bundled_plugin/scripts/report_projection.py +44 -15
- package/_bundled_plugin/scripts/threat_model_projection.py +4 -2
- package/_bundled_plugin/scripts/windows_scan_local_files.py +43 -14
- package/_bundled_plugin/scripts/workbench/retained_coverage.py +3218 -0
- package/_bundled_plugin/scripts/workbench/retained_records.py +627 -0
- package/_bundled_plugin/scripts/workbench/saved_coverage.py +644 -1
- package/_bundled_plugin/scripts/workbench/saved_outputs.py +72 -0
- package/_bundled_plugin/scripts/workbench/saved_sources.py +712 -0
- package/_bundled_plugin/scripts/workbench_cli.py +3 -0
- package/_bundled_plugin/scripts/workbench_coverage_receipts.py +479 -0
- package/_bundled_plugin/scripts/workbench_db.py +51 -22
- package/_bundled_plugin/scripts/workbench_feedback.py +0 -1
- package/_bundled_plugin/scripts/workbench_finding_workflows.py +53 -9
- package/_bundled_plugin/scripts/workbench_native_indexes.py +56 -44
- package/_bundled_plugin/scripts/workbench_saved_results.py +641 -2387
- package/_bundled_plugin/scripts/workbench_scan_history.py +72 -7
- package/_bundled_plugin/scripts/workbench_schema.py +1 -1
- package/_bundled_plugin/scripts/workbench_source_scopes.py +4 -1
- package/_bundled_plugin/scripts/workbench_target.py +142 -17
- package/_bundled_plugin/scripts/workbench_validation.py +9 -2
- package/_bundled_plugin/shared/workbench-history-repairs.json +21 -0
- package/_bundled_plugin/shared/workbench-migrations.json +44 -0
- package/_bundled_plugin/skills/assess-patch-risk/SKILL.md +3 -14
- package/_bundled_plugin/skills/assess-patch-risk/references/windows-validation.md +14 -0
- package/_bundled_plugin/skills/attack-path-analysis/SKILL.md +38 -103
- package/_bundled_plugin/skills/attack-path-analysis/references/attack-path-facts.md +11 -73
- package/_bundled_plugin/skills/attack-path-analysis/references/severity-policy.md +57 -116
- package/_bundled_plugin/skills/deep-security-scan/SKILL.md +28 -51
- package/_bundled_plugin/skills/define-security-policy/SKILL.md +9 -9
- package/_bundled_plugin/skills/finding-discovery/SKILL.md +26 -140
- package/_bundled_plugin/skills/fix-finding/SKILL.md +19 -30
- package/_bundled_plugin/skills/propose-security-hardening/SKILL.md +28 -218
- package/_bundled_plugin/skills/propose-security-hardening/references/proposal-format.md +29 -424
- package/_bundled_plugin/skills/publish-findings/SKILL.md +134 -0
- package/_bundled_plugin/skills/publish-findings/agents/openai.yaml +6 -0
- package/_bundled_plugin/skills/publish-findings/references/wiz-import.md +64 -0
- package/_bundled_plugin/skills/security-diff-scan/SKILL.md +12 -12
- package/_bundled_plugin/skills/security-scan/SKILL.md +28 -16
- package/_bundled_plugin/skills/security-scan/references/desktop-scan.md +11 -17
- package/_bundled_plugin/skills/security-scan/references/scan-artifacts-and-ledger.md +52 -69
- package/_bundled_plugin/skills/threat-model/SKILL.md +16 -9
- package/_bundled_plugin/skills/triage-finding/SKILL.md +5 -15
- package/_bundled_plugin/skills/triage-finding/references/ticket-intake.md +6 -6
- package/_bundled_plugin/skills/validation/SKILL.md +42 -104
- package/_bundled_plugin/skills/validation/references/validation-guidance.md +21 -106
- package/_bundled_plugin/skills/verify-fix/SKILL.md +8 -4
- package/_bundled_plugin/skills/vulnerability-writeup/SKILL.md +21 -159
- package/_bundled_plugin/skills/vulnerability-writeup/references/report-format.md +18 -60
- package/dist/api.d.ts +8 -3
- package/dist/api.d.ts.map +1 -1
- package/dist/api.js +414 -72
- package/dist/api.js.map +1 -1
- package/dist/auth.d.ts.map +1 -1
- package/dist/auth.js +4 -1
- package/dist/auth.js.map +1 -1
- package/dist/classify-severity-records.d.ts +5 -2
- package/dist/classify-severity-records.d.ts.map +1 -1
- package/dist/classify-severity-records.js +40 -34
- package/dist/classify-severity-records.js.map +1 -1
- package/dist/classify-severity.d.ts +5 -1
- package/dist/classify-severity.d.ts.map +1 -1
- package/dist/classify-severity.js +20 -30
- package/dist/classify-severity.js.map +1 -1
- package/dist/cli-help.d.ts.map +1 -1
- package/dist/cli-help.js +1 -0
- package/dist/cli-help.js.map +1 -1
- package/dist/cli.d.ts +9 -2
- package/dist/cli.d.ts.map +1 -1
- package/dist/cli.js +802 -149
- package/dist/cli.js.map +1 -1
- package/dist/cloud-endpoint.d.ts +6 -0
- package/dist/cloud-endpoint.d.ts.map +1 -0
- package/dist/cloud-endpoint.js +51 -0
- package/dist/cloud-endpoint.js.map +1 -0
- package/dist/cloud-publish.d.ts.map +1 -1
- package/dist/cloud-publish.js +7 -5
- package/dist/cloud-publish.js.map +1 -1
- package/dist/component-scan.d.ts +8 -2
- package/dist/component-scan.d.ts.map +1 -1
- package/dist/component-scan.js +2 -0
- package/dist/component-scan.js.map +1 -1
- package/dist/contract.d.ts +2 -0
- package/dist/contract.d.ts.map +1 -1
- package/dist/contract.js +67 -33
- package/dist/contract.js.map +1 -1
- package/dist/cost.d.ts +18 -0
- package/dist/cost.d.ts.map +1 -1
- package/dist/cost.js +107 -12
- package/dist/cost.js.map +1 -1
- package/dist/coverage-presentation.d.ts +6 -0
- package/dist/coverage-presentation.d.ts.map +1 -0
- package/dist/coverage-presentation.js +34 -0
- package/dist/coverage-presentation.js.map +1 -0
- package/dist/custom-validation-prompt.js +3 -3
- package/dist/custom-validation-prompt.js.map +1 -1
- package/dist/custom-validation.d.ts.map +1 -1
- package/dist/custom-validation.js +30 -4
- package/dist/custom-validation.js.map +1 -1
- package/dist/deduplication/checkpointed-review.d.ts.map +1 -1
- package/dist/deduplication/checkpointed-review.js +3 -1
- package/dist/deduplication/checkpointed-review.js.map +1 -1
- package/dist/deduplication/scan.d.ts.map +1 -1
- package/dist/deduplication/scan.js +7 -5
- package/dist/deduplication/scan.js.map +1 -1
- package/dist/default-severity-rubric.d.ts +3 -0
- package/dist/default-severity-rubric.d.ts.map +1 -0
- package/dist/default-severity-rubric.js +67 -0
- package/dist/default-severity-rubric.js.map +1 -0
- package/dist/endor-findings.d.ts +8 -0
- package/dist/endor-findings.d.ts.map +1 -0
- package/dist/endor-findings.js +238 -0
- package/dist/endor-findings.js.map +1 -0
- package/dist/external-findings-import.d.ts +5 -0
- package/dist/external-findings-import.d.ts.map +1 -0
- package/dist/external-findings-import.js +46 -0
- package/dist/external-findings-import.js.map +1 -0
- package/dist/external-findings-local.d.ts +63 -0
- package/dist/external-findings-local.d.ts.map +1 -0
- package/dist/external-findings-local.js +344 -0
- package/dist/external-findings-local.js.map +1 -0
- package/dist/external-findings-publish.d.ts +73 -0
- package/dist/external-findings-publish.d.ts.map +1 -0
- package/dist/external-findings-publish.js +721 -0
- package/dist/external-findings-publish.js.map +1 -0
- package/dist/external-findings-read.d.ts +16 -0
- package/dist/external-findings-read.d.ts.map +1 -0
- package/dist/external-findings-read.js +97 -0
- package/dist/external-findings-read.js.map +1 -0
- package/dist/external-findings.d.ts +21 -0
- package/dist/external-findings.d.ts.map +1 -0
- package/dist/external-findings.js +2 -0
- package/dist/external-findings.js.map +1 -0
- package/dist/external-import-contract.d.ts +15 -0
- package/dist/external-import-contract.d.ts.map +1 -0
- package/dist/external-import-contract.js +262 -0
- package/dist/external-import-contract.js.map +1 -0
- package/dist/external-import-models.d.ts +362 -0
- package/dist/external-import-models.d.ts.map +1 -0
- package/dist/external-import-models.js +3 -0
- package/dist/external-import-models.js.map +1 -0
- package/dist/external-imports-feature.d.ts +6 -0
- package/dist/external-imports-feature.d.ts.map +1 -0
- package/dist/external-imports-feature.js +51 -0
- package/dist/external-imports-feature.js.map +1 -0
- package/dist/finding-workflow.d.ts +23 -6
- package/dist/finding-workflow.d.ts.map +1 -1
- package/dist/finding-workflow.js +23 -10
- package/dist/finding-workflow.js.map +1 -1
- package/dist/github-advisories.d.ts +386 -0
- package/dist/github-advisories.d.ts.map +1 -0
- package/dist/github-advisories.js +133 -0
- package/dist/github-advisories.js.map +1 -0
- package/dist/github-findings.d.ts +11 -0
- package/dist/github-findings.d.ts.map +1 -0
- package/dist/github-findings.js +258 -0
- package/dist/github-findings.js.map +1 -0
- package/dist/hosted-remediation-protocol.d.ts +2 -1
- package/dist/hosted-remediation-protocol.d.ts.map +1 -1
- package/dist/hosted-remediation-protocol.js +17 -11
- package/dist/hosted-remediation-protocol.js.map +1 -1
- package/dist/hosted-remediation.d.ts +4 -0
- package/dist/hosted-remediation.d.ts.map +1 -1
- package/dist/hosted-remediation.js +22 -3
- package/dist/hosted-remediation.js.map +1 -1
- package/dist/hosted-scan.d.ts +16 -0
- package/dist/hosted-scan.d.ts.map +1 -1
- package/dist/hosted-scan.js +9 -2
- package/dist/hosted-scan.js.map +1 -1
- package/dist/import-scan.js +1 -1
- package/dist/import-scan.js.map +1 -1
- package/dist/index.d.ts +16 -3
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +9 -1
- package/dist/index.js.map +1 -1
- package/dist/knowledge-base.d.ts +1 -0
- package/dist/knowledge-base.d.ts.map +1 -1
- package/dist/knowledge-base.js +17 -8
- package/dist/knowledge-base.js.map +1 -1
- package/dist/models.d.ts +1 -0
- package/dist/models.d.ts.map +1 -1
- package/dist/models.js.map +1 -1
- package/dist/owner-evidence.d.ts +2 -0
- package/dist/owner-evidence.d.ts.map +1 -1
- package/dist/owner-evidence.js +2 -1
- package/dist/owner-evidence.js.map +1 -1
- package/dist/project-config-schema.d.ts +2 -0
- package/dist/project-config-schema.d.ts.map +1 -1
- package/dist/project-config-schema.js +1 -0
- package/dist/project-config-schema.js.map +1 -1
- package/dist/project-config.d.ts +1 -0
- package/dist/project-config.d.ts.map +1 -1
- package/dist/project-config.js +2 -0
- package/dist/project-config.js.map +1 -1
- package/dist/publish.d.ts +2 -0
- package/dist/publish.d.ts.map +1 -1
- package/dist/publish.js +1 -1
- package/dist/publish.js.map +1 -1
- package/dist/remediation.d.ts +7 -25
- package/dist/remediation.d.ts.map +1 -1
- package/dist/remediation.js +18 -28
- package/dist/remediation.js.map +1 -1
- package/dist/result.d.ts +4 -0
- package/dist/result.d.ts.map +1 -1
- package/dist/result.js +10 -0
- package/dist/result.js.map +1 -1
- package/dist/runtime.d.ts +1 -0
- package/dist/runtime.d.ts.map +1 -1
- package/dist/runtime.js +10 -8
- package/dist/runtime.js.map +1 -1
- package/dist/scan-dashboard.d.ts +5 -3
- package/dist/scan-dashboard.d.ts.map +1 -1
- package/dist/scan-dashboard.js +79 -26
- package/dist/scan-dashboard.js.map +1 -1
- package/dist/scan-history-renderer.d.ts.map +1 -1
- package/dist/scan-history-renderer.js +8 -2
- package/dist/scan-history-renderer.js.map +1 -1
- package/dist/scan-owner-suggestions.d.ts +8 -0
- package/dist/scan-owner-suggestions.d.ts.map +1 -0
- package/dist/scan-owner-suggestions.js +33 -0
- package/dist/scan-owner-suggestions.js.map +1 -0
- package/dist/scan-settings.d.ts +1 -0
- package/dist/scan-settings.d.ts.map +1 -1
- package/dist/scan-settings.js +4 -0
- package/dist/scan-settings.js.map +1 -1
- package/dist/scan-usage-display.d.ts +8 -0
- package/dist/scan-usage-display.d.ts.map +1 -0
- package/dist/scan-usage-display.js +46 -0
- package/dist/scan-usage-display.js.map +1 -0
- package/dist/scanner-import.d.ts +20 -0
- package/dist/scanner-import.d.ts.map +1 -0
- package/dist/scanner-import.js +208 -0
- package/dist/scanner-import.js.map +1 -0
- package/dist/security-policy-diff.d.ts +2 -0
- package/dist/security-policy-diff.d.ts.map +1 -0
- package/dist/security-policy-diff.js +197 -0
- package/dist/security-policy-diff.js.map +1 -0
- package/dist/security-policy.d.ts +7 -3
- package/dist/security-policy.d.ts.map +1 -1
- package/dist/security-policy.js +11 -76
- package/dist/security-policy.js.map +1 -1
- package/dist/semgrep-findings.d.ts +10 -0
- package/dist/semgrep-findings.d.ts.map +1 -0
- package/dist/semgrep-findings.js +293 -0
- package/dist/semgrep-findings.js.map +1 -0
- package/dist/severity-review.d.ts +3 -0
- package/dist/severity-review.d.ts.map +1 -1
- package/dist/severity-review.js +19 -0
- package/dist/severity-review.js.map +1 -1
- package/dist/severity-store.js +7 -2
- package/dist/severity-store.js.map +1 -1
- package/dist/skill-authentication.d.ts +18 -0
- package/dist/skill-authentication.d.ts.map +1 -0
- package/dist/skill-authentication.js +82 -0
- package/dist/skill-authentication.js.map +1 -0
- package/dist/snyk-findings.d.ts +10 -0
- package/dist/snyk-findings.d.ts.map +1 -0
- package/dist/snyk-findings.js +350 -0
- package/dist/snyk-findings.js.map +1 -0
- package/dist/suggest-owners.d.ts +4 -1
- package/dist/suggest-owners.d.ts.map +1 -1
- package/dist/suggest-owners.js +53 -21
- package/dist/suggest-owners.js.map +1 -1
- package/dist/targets.d.ts +2 -2
- package/dist/targets.d.ts.map +1 -1
- package/dist/targets.js +16 -3
- package/dist/targets.js.map +1 -1
- package/dist/wiz-csv.d.ts +9 -0
- package/dist/wiz-csv.d.ts.map +1 -0
- package/dist/wiz-csv.js +116 -0
- package/dist/wiz-csv.js.map +1 -0
- package/dist/wiz-findings.d.ts +5 -0
- package/dist/wiz-findings.d.ts.map +1 -0
- package/dist/wiz-findings.js +464 -0
- package/dist/wiz-findings.js.map +1 -0
- package/docs/classify-severity-records.md +30 -13
- package/docs/cli.md +835 -47
- package/docs/findings-service.md +119 -118
- package/docs/hosted-remediation.md +43 -24
- package/docs/hosted-scans.md +38 -0
- package/package.json +11 -9
- package/schemas/external-findings.schema.json +1421 -0
- package/schemas/project-config.schema.json +4 -0
- package/_bundled_plugin/scripts/generate_in_scope_files.py +0 -277
- package/_bundled_plugin/scripts/generate_rank_input.py +0 -350
- package/_bundled_plugin/scripts/rank_preview.py +0 -129
|
@@ -31,6 +31,10 @@
|
|
|
31
31
|
"type": "string",
|
|
32
32
|
"enum": ["standard", "daybreak_blue", "daybreak_red"]
|
|
33
33
|
},
|
|
34
|
+
"owner_suggestions": {
|
|
35
|
+
"description": "Generate optional owner-suggestions.json after a committed scan (default: false).",
|
|
36
|
+
"type": "boolean"
|
|
37
|
+
},
|
|
34
38
|
"scope": {
|
|
35
39
|
"description": "One scope variant. Omit for the whole repository. Mode compatibility is checked after overrides.",
|
|
36
40
|
"anyOf": [
|
|
@@ -1,277 +0,0 @@
|
|
|
1
|
-
#!/usr/bin/env python3
|
|
2
|
-
"""Generate the shared, deterministically ordered security-scan file inventory."""
|
|
3
|
-
|
|
4
|
-
from __future__ import annotations
|
|
5
|
-
|
|
6
|
-
import argparse
|
|
7
|
-
import os
|
|
8
|
-
import subprocess
|
|
9
|
-
import sys
|
|
10
|
-
import tempfile
|
|
11
|
-
from pathlib import Path
|
|
12
|
-
|
|
13
|
-
# Some plugin hosts launch Python with safe-path isolation enabled.
|
|
14
|
-
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
|
15
|
-
from workbench_target import SOURCE_METADATA_DIRECTORIES, git_blob_samples, git_command
|
|
16
|
-
|
|
17
|
-
|
|
18
|
-
class InventoryError(ValueError):
|
|
19
|
-
"""Raised when the repository, scope, or inventory cannot be used safely."""
|
|
20
|
-
|
|
21
|
-
|
|
22
|
-
def windows_stream_component(path: Path) -> str | None:
|
|
23
|
-
"""Return the first NTFS alternate-data-stream component."""
|
|
24
|
-
|
|
25
|
-
if os.name != "nt":
|
|
26
|
-
return None
|
|
27
|
-
return next(
|
|
28
|
-
(component for component in path.parts if component != path.anchor and ":" in component),
|
|
29
|
-
None,
|
|
30
|
-
)
|
|
31
|
-
|
|
32
|
-
|
|
33
|
-
def resolve_repository(value: str) -> Path:
|
|
34
|
-
"""Resolve the repository once so every scope is bound to its real root."""
|
|
35
|
-
try:
|
|
36
|
-
repository = Path(value).expanduser().resolve(strict=True)
|
|
37
|
-
except (OSError, ValueError) as error:
|
|
38
|
-
raise InventoryError(f"--repo: cannot resolve repository: {value}") from error
|
|
39
|
-
if not repository.is_dir():
|
|
40
|
-
raise InventoryError(f"--repo: expected a directory: {repository}")
|
|
41
|
-
return repository
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
def resolve_scope(repository: Path, value: str) -> str:
|
|
45
|
-
"""Preserve ripgrep's relative path spelling while rejecting escaped scopes."""
|
|
46
|
-
if not value or "\0" in value:
|
|
47
|
-
raise InventoryError("--scope: expected a non-empty file or directory")
|
|
48
|
-
|
|
49
|
-
requested = Path(value).expanduser()
|
|
50
|
-
stream = windows_stream_component(requested)
|
|
51
|
-
if stream is not None:
|
|
52
|
-
raise InventoryError(f"--scope: NTFS alternate data streams are not supported: {stream}")
|
|
53
|
-
scope = requested if requested.is_absolute() else repository / requested
|
|
54
|
-
try:
|
|
55
|
-
resolved = scope.resolve(strict=True)
|
|
56
|
-
except (OSError, ValueError) as error:
|
|
57
|
-
raise InventoryError(f"--scope: path does not exist: {value}") from error
|
|
58
|
-
|
|
59
|
-
try:
|
|
60
|
-
relative = resolved.relative_to(repository)
|
|
61
|
-
except ValueError as error:
|
|
62
|
-
raise InventoryError(f"--scope: path must remain inside --repo: {value}") from error
|
|
63
|
-
|
|
64
|
-
if not resolved.is_dir() and not resolved.is_file():
|
|
65
|
-
raise InventoryError(f"--scope: expected a file or directory: {value}")
|
|
66
|
-
|
|
67
|
-
if requested.is_absolute():
|
|
68
|
-
return relative.as_posix() if relative.parts else "."
|
|
69
|
-
return value
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
def resolve_output(value: str) -> Path:
|
|
73
|
-
"""Reject direct symlink outputs without constraining the artifact root."""
|
|
74
|
-
if not value or "\0" in value:
|
|
75
|
-
raise InventoryError("--out: expected an inventory file path")
|
|
76
|
-
requested = Path(value).expanduser()
|
|
77
|
-
if requested.is_symlink():
|
|
78
|
-
raise InventoryError("--out: refusing to replace a symbolic link")
|
|
79
|
-
try:
|
|
80
|
-
output = requested.resolve(strict=False)
|
|
81
|
-
except (OSError, ValueError) as error:
|
|
82
|
-
raise InventoryError(f"--out: cannot resolve inventory path: {value}") from error
|
|
83
|
-
if output.exists() and not output.is_file():
|
|
84
|
-
raise InventoryError(f"--out: expected a regular file path: {output}")
|
|
85
|
-
return output
|
|
86
|
-
|
|
87
|
-
|
|
88
|
-
def generate_in_scope_files(repository: Path, scope: str, output: Path) -> int:
|
|
89
|
-
"""Atomically write the exact ripgrep inventory sorted as ``LC_ALL=C``."""
|
|
90
|
-
command = [
|
|
91
|
-
"rg",
|
|
92
|
-
"--files",
|
|
93
|
-
"--null",
|
|
94
|
-
"--hidden",
|
|
95
|
-
"--path-separator",
|
|
96
|
-
"/",
|
|
97
|
-
# Prune metadata and its contents even when the scope starts inside it.
|
|
98
|
-
"--glob",
|
|
99
|
-
"!**/{.git,.svn}",
|
|
100
|
-
"--glob",
|
|
101
|
-
"!**/{.git,.svn}/**",
|
|
102
|
-
"--",
|
|
103
|
-
scope,
|
|
104
|
-
]
|
|
105
|
-
with tempfile.TemporaryFile(mode="w+b") as inventory:
|
|
106
|
-
try:
|
|
107
|
-
result = subprocess.run(
|
|
108
|
-
command,
|
|
109
|
-
cwd=repository,
|
|
110
|
-
stdout=inventory,
|
|
111
|
-
stderr=subprocess.PIPE,
|
|
112
|
-
check=False,
|
|
113
|
-
)
|
|
114
|
-
except OSError as error:
|
|
115
|
-
raise InventoryError(f"could not run ripgrep: {error}") from error
|
|
116
|
-
|
|
117
|
-
if result.returncode not in (0, 1):
|
|
118
|
-
detail = result.stderr.decode("utf-8", errors="replace").strip()
|
|
119
|
-
message = f"ripgrep exited with status {result.returncode}"
|
|
120
|
-
if detail:
|
|
121
|
-
message = f"{message}: {detail}"
|
|
122
|
-
raise InventoryError(message)
|
|
123
|
-
|
|
124
|
-
if any((parent / ".git").exists() for parent in (repository, *repository.parents)):
|
|
125
|
-
try:
|
|
126
|
-
tracked = git_command(
|
|
127
|
-
repository,
|
|
128
|
-
"ls-files",
|
|
129
|
-
"--cached",
|
|
130
|
-
"--ignored",
|
|
131
|
-
"--exclude-standard",
|
|
132
|
-
"-z",
|
|
133
|
-
"--",
|
|
134
|
-
scope,
|
|
135
|
-
text=False,
|
|
136
|
-
)
|
|
137
|
-
except OSError:
|
|
138
|
-
tracked = None
|
|
139
|
-
if tracked is not None and tracked.returncode == 0:
|
|
140
|
-
prefix = b"./" if scope == "." or scope.startswith("./") else b""
|
|
141
|
-
for path in tracked.stdout.split(b"\0"):
|
|
142
|
-
candidate = repository / os.fsdecode(path)
|
|
143
|
-
if path and candidate.is_file() and not candidate.is_symlink():
|
|
144
|
-
inventory.write(prefix + path + b"\0")
|
|
145
|
-
|
|
146
|
-
inventory.seek(0)
|
|
147
|
-
rows: list[bytes] = []
|
|
148
|
-
for path in inventory.read().split(b"\0"):
|
|
149
|
-
if not path or not SOURCE_METADATA_DIRECTORIES.isdisjoint(os.fsdecode(path).split("/")):
|
|
150
|
-
continue
|
|
151
|
-
if b"\n" in path or b"\r" in path:
|
|
152
|
-
raise InventoryError(
|
|
153
|
-
"Repository contains a path that cannot fit in the file inventory"
|
|
154
|
-
)
|
|
155
|
-
rows.append(path + b"\n")
|
|
156
|
-
|
|
157
|
-
return write_inventory(output, sorted(set(rows)))
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
def generate_diff_in_scope_files(
|
|
161
|
-
repository: Path,
|
|
162
|
-
base: str,
|
|
163
|
-
head: str,
|
|
164
|
-
mode: str,
|
|
165
|
-
output: Path,
|
|
166
|
-
) -> int:
|
|
167
|
-
"""Reuse the existing diff selection without generating previews or duplicate worklists."""
|
|
168
|
-
from generate_rank_input import git_changed_paths
|
|
169
|
-
from rank_preview import is_binary_file
|
|
170
|
-
|
|
171
|
-
rows: list[bytes] = []
|
|
172
|
-
try:
|
|
173
|
-
changed = git_changed_paths(repository, base, head, mode)
|
|
174
|
-
revision_refs = {
|
|
175
|
-
path.relative_to(repository): (
|
|
176
|
-
f"{base if status == 'D' else head}:{path.relative_to(repository).as_posix()}"
|
|
177
|
-
)
|
|
178
|
-
for path, status in changed
|
|
179
|
-
if mode == "revisions" or status == "D"
|
|
180
|
-
}
|
|
181
|
-
revision_samples = dict(
|
|
182
|
-
zip(revision_refs, git_blob_samples(repository, list(revision_refs.values())))
|
|
183
|
-
)
|
|
184
|
-
|
|
185
|
-
for path, status in changed:
|
|
186
|
-
relative = path.relative_to(repository)
|
|
187
|
-
if mode == "revisions" or status == "D":
|
|
188
|
-
sample = revision_samples[relative]
|
|
189
|
-
if sample is None:
|
|
190
|
-
revision = base if status == "D" else head
|
|
191
|
-
raise InventoryError(
|
|
192
|
-
f"could not read committed diff blob: {revision}:{relative.as_posix()}"
|
|
193
|
-
)
|
|
194
|
-
_, is_binary = sample
|
|
195
|
-
if is_binary:
|
|
196
|
-
continue
|
|
197
|
-
elif path.is_symlink() or not path.is_file() or is_binary_file(path):
|
|
198
|
-
continue
|
|
199
|
-
relative_path = os.fsencode(relative.as_posix())
|
|
200
|
-
if b"\n" in relative_path or b"\r" in relative_path:
|
|
201
|
-
raise InventoryError(
|
|
202
|
-
"Git changes contain a path that cannot fit in the file inventory"
|
|
203
|
-
)
|
|
204
|
-
rows.append(relative_path + b"\n")
|
|
205
|
-
except (OSError, subprocess.CalledProcessError) as error:
|
|
206
|
-
detail = getattr(error, "stderr", None)
|
|
207
|
-
if isinstance(detail, bytes):
|
|
208
|
-
detail = detail.decode("utf-8", errors="replace")
|
|
209
|
-
message = detail.strip() if isinstance(detail, str) and detail.strip() else str(error)
|
|
210
|
-
raise InventoryError(f"could not resolve the selected Git changes: {message}") from error
|
|
211
|
-
|
|
212
|
-
return write_inventory(output, sorted(set(rows)))
|
|
213
|
-
|
|
214
|
-
|
|
215
|
-
def write_inventory(output: Path, rows: list[bytes]) -> int:
|
|
216
|
-
"""Replace a complete inventory atomically, keeping failures from corrupting the old one."""
|
|
217
|
-
output.parent.mkdir(parents=True, exist_ok=True)
|
|
218
|
-
temporary: Path | None = None
|
|
219
|
-
try:
|
|
220
|
-
with tempfile.NamedTemporaryFile(
|
|
221
|
-
mode="wb",
|
|
222
|
-
dir=output.parent,
|
|
223
|
-
# Including the output name can exceed the filesystem's 255-byte name limit.
|
|
224
|
-
prefix=".",
|
|
225
|
-
suffix=".tmp",
|
|
226
|
-
delete=False,
|
|
227
|
-
) as handle:
|
|
228
|
-
temporary = Path(handle.name)
|
|
229
|
-
handle.writelines(rows)
|
|
230
|
-
temporary.replace(output)
|
|
231
|
-
finally:
|
|
232
|
-
if temporary is not None:
|
|
233
|
-
temporary.unlink(missing_ok=True)
|
|
234
|
-
|
|
235
|
-
return len(rows)
|
|
236
|
-
|
|
237
|
-
|
|
238
|
-
def main() -> None:
|
|
239
|
-
parser = argparse.ArgumentParser(description=__doc__)
|
|
240
|
-
parser.add_argument("--repo", required=True, help="Repository root.")
|
|
241
|
-
parser.add_argument("--scope", required=True, help="File or directory within the repository.")
|
|
242
|
-
parser.add_argument("--out", required=True, help="Destination for the file inventory.")
|
|
243
|
-
parser.add_argument("--diff-base", help="Authoritative Git base for a changed-file inventory.")
|
|
244
|
-
parser.add_argument("--diff-head", default="HEAD", help="Authoritative Git head revision.")
|
|
245
|
-
parser.add_argument(
|
|
246
|
-
"--diff-mode",
|
|
247
|
-
choices=("revisions", "local-patch"),
|
|
248
|
-
default="revisions",
|
|
249
|
-
help="Use committed revisions or the current staged and unstaged patch.",
|
|
250
|
-
)
|
|
251
|
-
args = parser.parse_args()
|
|
252
|
-
|
|
253
|
-
try:
|
|
254
|
-
repository = resolve_repository(args.repo)
|
|
255
|
-
scope = resolve_scope(repository, args.scope)
|
|
256
|
-
output = resolve_output(args.out)
|
|
257
|
-
if args.diff_base is None:
|
|
258
|
-
count = generate_in_scope_files(repository, scope, output)
|
|
259
|
-
elif scope not in (".", "./"):
|
|
260
|
-
raise InventoryError("--scope: diff scans must use the repository root")
|
|
261
|
-
else:
|
|
262
|
-
count = generate_diff_in_scope_files(
|
|
263
|
-
repository,
|
|
264
|
-
args.diff_base,
|
|
265
|
-
args.diff_head,
|
|
266
|
-
args.diff_mode,
|
|
267
|
-
output,
|
|
268
|
-
)
|
|
269
|
-
except (OSError, ValueError) as error:
|
|
270
|
-
print(f"generate_in_scope_files: {error}", file=sys.stderr)
|
|
271
|
-
raise SystemExit(2) from error
|
|
272
|
-
|
|
273
|
-
print(f"Recorded {count} in-scope files.")
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
if __name__ == "__main__":
|
|
277
|
-
main()
|
|
@@ -1,350 +0,0 @@
|
|
|
1
|
-
#!/usr/bin/env python3
|
|
2
|
-
"""Generate and post-process Codex Security scan worklists.
|
|
3
|
-
|
|
4
|
-
This script stays deliberately model-free:
|
|
5
|
-
|
|
6
|
-
- `make-repo-scope-input` lists every explicitly scoped repository file.
|
|
7
|
-
- `make-diff-rank-input` creates the deterministic diff-scoped JSONL candidate
|
|
8
|
-
worklist from Git changed paths. It supports committed revision diffs and
|
|
9
|
-
local working-tree patches.
|
|
10
|
-
|
|
11
|
-
Candidate selection uses repository scope, native tool ignore rules, and binary detection,
|
|
12
|
-
not filename or directory classifications. Diff inputs use bounded source previews.
|
|
13
|
-
"""
|
|
14
|
-
|
|
15
|
-
from __future__ import annotations
|
|
16
|
-
|
|
17
|
-
import argparse
|
|
18
|
-
import json
|
|
19
|
-
import os
|
|
20
|
-
import subprocess
|
|
21
|
-
import sys
|
|
22
|
-
from collections.abc import Iterable
|
|
23
|
-
from pathlib import Path
|
|
24
|
-
|
|
25
|
-
# Some plugin hosts launch Python with safe-path isolation enabled.
|
|
26
|
-
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
|
27
|
-
from generate_in_scope_files import windows_stream_component
|
|
28
|
-
from rank_preview import (
|
|
29
|
-
DEFAULT_PREVIEW_BYTES,
|
|
30
|
-
is_binary_file,
|
|
31
|
-
preview_for,
|
|
32
|
-
preview_for_bytes,
|
|
33
|
-
)
|
|
34
|
-
from workbench_target import (
|
|
35
|
-
SOURCE_METADATA_DIRECTORIES,
|
|
36
|
-
git_blob_samples,
|
|
37
|
-
git_command,
|
|
38
|
-
git_directory_snapshot_paths,
|
|
39
|
-
)
|
|
40
|
-
|
|
41
|
-
JsonRow = dict[str, object]
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
def parse_args() -> argparse.Namespace:
|
|
45
|
-
parser = argparse.ArgumentParser(description="Codex Security scan worklist helper.")
|
|
46
|
-
subparsers = parser.add_subparsers(dest="command", required=True)
|
|
47
|
-
|
|
48
|
-
scoped = subparsers.add_parser(
|
|
49
|
-
"make-repo-scope-input",
|
|
50
|
-
help="List every explicitly scoped file without ranking or reading its contents.",
|
|
51
|
-
)
|
|
52
|
-
scoped.add_argument("--repo", required=True, help="Repository root.")
|
|
53
|
-
scoped.add_argument(
|
|
54
|
-
"--scopes-file",
|
|
55
|
-
required=True,
|
|
56
|
-
help="JSON array of repository-relative files and directories to scan together.",
|
|
57
|
-
)
|
|
58
|
-
scoped.add_argument("--out", required=True, help="Output scoped-source-input.jsonl path.")
|
|
59
|
-
|
|
60
|
-
diff = subparsers.add_parser(
|
|
61
|
-
"make-diff-rank-input",
|
|
62
|
-
help="Create rank_input.jsonl from Git changed text files.",
|
|
63
|
-
)
|
|
64
|
-
diff.add_argument("--repo", required=True, help="Repository root.")
|
|
65
|
-
diff.add_argument("--base", required=True, help="Git diff base revision.")
|
|
66
|
-
diff.add_argument(
|
|
67
|
-
"--mode",
|
|
68
|
-
choices=("revisions", "local-patch"),
|
|
69
|
-
default="revisions",
|
|
70
|
-
help="Git diff mode: committed revisions or staged plus unstaged local patch.",
|
|
71
|
-
)
|
|
72
|
-
diff.add_argument("--head", default="HEAD", help="Git diff head revision.")
|
|
73
|
-
diff.add_argument("--out", required=True, help="Output rank_input.jsonl path.")
|
|
74
|
-
diff.add_argument("--area", default="diff", help="Area label for changed-file rows.")
|
|
75
|
-
diff.add_argument(
|
|
76
|
-
"--preview-bytes",
|
|
77
|
-
type=int,
|
|
78
|
-
default=DEFAULT_PREVIEW_BYTES,
|
|
79
|
-
help=f"Maximum UTF-8 bytes in each preview. Defaults to {DEFAULT_PREVIEW_BYTES}.",
|
|
80
|
-
)
|
|
81
|
-
|
|
82
|
-
return parser.parse_args()
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
def resolve_scope(repo: Path, scope: str) -> Path:
|
|
86
|
-
scope_path = Path(scope)
|
|
87
|
-
stream = windows_stream_component(scope_path)
|
|
88
|
-
if stream is not None:
|
|
89
|
-
raise SystemExit(f"Scope must not use an NTFS alternate data stream: {stream}")
|
|
90
|
-
if not scope_path.is_absolute():
|
|
91
|
-
scope_path = repo / scope_path
|
|
92
|
-
repository = repo.resolve()
|
|
93
|
-
try:
|
|
94
|
-
relative = scope_path.relative_to(repository)
|
|
95
|
-
except ValueError as exc:
|
|
96
|
-
raise SystemExit(f"Scope must be inside repo: {scope_path}") from exc
|
|
97
|
-
ancestor = repository
|
|
98
|
-
for part in relative.parts:
|
|
99
|
-
if part == "..":
|
|
100
|
-
if ancestor == repository:
|
|
101
|
-
raise SystemExit(f"Scope must be inside repo: {scope_path}")
|
|
102
|
-
ancestor = ancestor.parent
|
|
103
|
-
continue
|
|
104
|
-
ancestor /= part
|
|
105
|
-
try:
|
|
106
|
-
metadata = ancestor.stat(follow_symlinks=False)
|
|
107
|
-
except OSError as exc:
|
|
108
|
-
raise SystemExit(f"Scope path not found: {ancestor}") from exc
|
|
109
|
-
if ancestor.is_symlink() or getattr(metadata, "st_reparse_tag", 0) & 0x20000000:
|
|
110
|
-
raise SystemExit(f"Requested scope must not contain symbolic links: {ancestor}")
|
|
111
|
-
scope_path = scope_path.resolve()
|
|
112
|
-
repo_resolved = repo.resolve()
|
|
113
|
-
try:
|
|
114
|
-
scope_path.relative_to(repo_resolved)
|
|
115
|
-
except ValueError as exc:
|
|
116
|
-
raise SystemExit(f"Scope must be inside repo: {scope_path}") from exc
|
|
117
|
-
if not scope_path.is_dir() and not scope_path.is_file():
|
|
118
|
-
raise SystemExit(f"Scope path not found: {scope_path}")
|
|
119
|
-
return scope_path
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
def write_jsonl(output: Path, rows: list[JsonRow]) -> None:
|
|
123
|
-
output.parent.mkdir(parents=True, exist_ok=True)
|
|
124
|
-
with output.open("w", encoding="utf-8") as handle:
|
|
125
|
-
for row in rows:
|
|
126
|
-
handle.write(json.dumps(row, ensure_ascii=True, separators=(",", ":")))
|
|
127
|
-
handle.write("\n")
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
def load_scopes_file(scopes_file: Path) -> list[str]:
|
|
131
|
-
try:
|
|
132
|
-
loaded: object = json.loads(scopes_file.read_text(encoding="utf-8"))
|
|
133
|
-
except (OSError, UnicodeError, json.JSONDecodeError) as exc:
|
|
134
|
-
raise SystemExit(f"Unable to read scopes file: {scopes_file}") from exc
|
|
135
|
-
if (
|
|
136
|
-
not isinstance(loaded, list)
|
|
137
|
-
or not loaded
|
|
138
|
-
or any(not isinstance(scope, str) or not scope for scope in loaded)
|
|
139
|
-
):
|
|
140
|
-
raise SystemExit(f"Scopes file must contain a non-empty JSON string array: {scopes_file}")
|
|
141
|
-
return loaded
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
def scope_candidates(repo: Path, scope_path: Path) -> Iterable[Path]:
|
|
145
|
-
"""Use Git's inventory in worktrees and ripgrep's ignore rules elsewhere.
|
|
146
|
-
|
|
147
|
-
Git retains tracked files and applies its standard exclusions to untracked files.
|
|
148
|
-
Outside Git, ripgrep also honors .ignore and .rgignore alongside .gitignore.
|
|
149
|
-
Explicit file scopes bypass directory ignore rules.
|
|
150
|
-
"""
|
|
151
|
-
if scope_path.is_file():
|
|
152
|
-
return (scope_path,)
|
|
153
|
-
git_candidates = git_directory_snapshot_paths(scope_path)
|
|
154
|
-
if git_candidates is not None:
|
|
155
|
-
return git_candidates
|
|
156
|
-
|
|
157
|
-
command = [
|
|
158
|
-
"rg",
|
|
159
|
-
"--files",
|
|
160
|
-
"--hidden",
|
|
161
|
-
"--no-require-git",
|
|
162
|
-
"--null",
|
|
163
|
-
# Also exclude descendants when the scope starts inside metadata.
|
|
164
|
-
"--glob",
|
|
165
|
-
"!**/{.git,.svn}",
|
|
166
|
-
"--glob",
|
|
167
|
-
"!**/{.git,.svn}/**",
|
|
168
|
-
"--",
|
|
169
|
-
str(scope_path.relative_to(repo)),
|
|
170
|
-
]
|
|
171
|
-
try:
|
|
172
|
-
result = subprocess.run(command, cwd=repo, capture_output=True, check=False)
|
|
173
|
-
except OSError as exc:
|
|
174
|
-
ignore_names = (".gitignore", ".ignore", ".rgignore")
|
|
175
|
-
ancestors = (scope_path, *scope_path.parents)
|
|
176
|
-
has_ignore_rules = (
|
|
177
|
-
any((ancestor / ".git").exists() for ancestor in (repo, *repo.parents))
|
|
178
|
-
or any(
|
|
179
|
-
(ancestor / name).is_file()
|
|
180
|
-
for ancestor in ancestors
|
|
181
|
-
if ancestor == repo or repo in ancestor.parents
|
|
182
|
-
for name in ignore_names
|
|
183
|
-
)
|
|
184
|
-
or any(path.name in ignore_names for path in scope_path.rglob("*") if path.is_file())
|
|
185
|
-
)
|
|
186
|
-
if has_ignore_rules:
|
|
187
|
-
raise SystemExit(
|
|
188
|
-
"Could not safely enumerate ignored scoped files without Git or ripgrep."
|
|
189
|
-
) from exc
|
|
190
|
-
return scope_path.rglob("*")
|
|
191
|
-
|
|
192
|
-
if result.returncode not in (0, 1):
|
|
193
|
-
detail = result.stderr.decode("utf-8", errors="replace").strip()
|
|
194
|
-
raise SystemExit(f"Could not enumerate scoped repository files: {detail}")
|
|
195
|
-
return (repo / os.fsdecode(path) for path in result.stdout.split(b"\0") if path)
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
def make_repo_scope_input(args: argparse.Namespace) -> None:
|
|
199
|
-
repo = Path(args.repo).expanduser().resolve()
|
|
200
|
-
if not repo.is_dir():
|
|
201
|
-
raise SystemExit(f"Repo path not found: {repo}")
|
|
202
|
-
|
|
203
|
-
scopes = load_scopes_file(Path(args.scopes_file).expanduser())
|
|
204
|
-
rows_by_path: dict[str, JsonRow] = {}
|
|
205
|
-
for scope in scopes:
|
|
206
|
-
scope_path = resolve_scope(repo, scope)
|
|
207
|
-
candidates = scope_candidates(repo, scope_path)
|
|
208
|
-
for path in candidates:
|
|
209
|
-
try:
|
|
210
|
-
if path.is_symlink() or not path.is_file():
|
|
211
|
-
continue
|
|
212
|
-
relative = path.resolve(strict=True).relative_to(repo)
|
|
213
|
-
except (OSError, ValueError):
|
|
214
|
-
continue
|
|
215
|
-
if not SOURCE_METADATA_DIRECTORIES.isdisjoint(relative.parts):
|
|
216
|
-
continue
|
|
217
|
-
rows_by_path.setdefault(relative.as_posix(), {"path": relative.as_posix()})
|
|
218
|
-
|
|
219
|
-
rows = sorted(rows_by_path.values(), key=lambda row: str(row["path"]))
|
|
220
|
-
output = Path(args.out).expanduser()
|
|
221
|
-
write_jsonl(output, rows)
|
|
222
|
-
print(f"Wrote {len(rows)} scoped paths to {output}")
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
def run_git_changed_paths(repo: Path, diff_args: list[str]) -> list[tuple[Path, str]]:
|
|
226
|
-
"""Return changed regular files from the selected side of each change."""
|
|
227
|
-
result = git_command(
|
|
228
|
-
repo,
|
|
229
|
-
"diff",
|
|
230
|
-
"--ignore-submodules=all",
|
|
231
|
-
"--raw",
|
|
232
|
-
"-z",
|
|
233
|
-
"--diff-filter=ACMRDT",
|
|
234
|
-
*diff_args,
|
|
235
|
-
"--",
|
|
236
|
-
text=False,
|
|
237
|
-
)
|
|
238
|
-
result.check_returncode()
|
|
239
|
-
fields = result.stdout.split(b"\0")
|
|
240
|
-
if fields and not fields[-1]:
|
|
241
|
-
fields.pop()
|
|
242
|
-
|
|
243
|
-
changed: list[tuple[Path, str]] = []
|
|
244
|
-
index = 0
|
|
245
|
-
while index < len(fields):
|
|
246
|
-
metadata = fields[index].split()
|
|
247
|
-
status = chr(metadata[-1][0])
|
|
248
|
-
index += 1
|
|
249
|
-
if status in {"C", "R"}:
|
|
250
|
-
index += 1
|
|
251
|
-
path = repo / os.fsdecode(fields[index])
|
|
252
|
-
index += 1
|
|
253
|
-
selected_mode = metadata[0].removeprefix(b":") if status == "D" else metadata[1]
|
|
254
|
-
if selected_mode.startswith(b"100"):
|
|
255
|
-
changed.append((path, status))
|
|
256
|
-
return changed
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
def git_changed_paths(repo: Path, base: str, head: str, mode: str) -> list[tuple[Path, str]]:
|
|
260
|
-
if mode == "revisions":
|
|
261
|
-
return run_git_changed_paths(repo, [f"{base}..{head}"])
|
|
262
|
-
if mode == "local-patch":
|
|
263
|
-
unstaged = run_git_changed_paths(repo, [base])
|
|
264
|
-
staged = run_git_changed_paths(repo, ["--cached", base])
|
|
265
|
-
untracked = git_command(
|
|
266
|
-
repo,
|
|
267
|
-
"ls-files",
|
|
268
|
-
"--others",
|
|
269
|
-
"--exclude-standard",
|
|
270
|
-
"-z",
|
|
271
|
-
text=False,
|
|
272
|
-
)
|
|
273
|
-
untracked.check_returncode()
|
|
274
|
-
combined = dict(staged)
|
|
275
|
-
combined.update(unstaged)
|
|
276
|
-
combined.update(
|
|
277
|
-
(repo / os.fsdecode(relative), "A")
|
|
278
|
-
for relative in untracked.stdout.split(b"\0")
|
|
279
|
-
if relative and not relative.endswith(b"/")
|
|
280
|
-
)
|
|
281
|
-
return sorted(
|
|
282
|
-
(path, status)
|
|
283
|
-
for path, status in combined.items()
|
|
284
|
-
if SOURCE_METADATA_DIRECTORIES.isdisjoint(path.relative_to(repo).parts)
|
|
285
|
-
and (status == "D" or (not path.is_symlink() and path.is_file()))
|
|
286
|
-
)
|
|
287
|
-
raise SystemExit(f"Unknown diff mode: {mode}")
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
def make_diff_rank_input(args: argparse.Namespace) -> None:
|
|
291
|
-
repo = Path(args.repo).expanduser().resolve()
|
|
292
|
-
if not repo.is_dir():
|
|
293
|
-
raise SystemExit(f"Repo path not found: {repo}")
|
|
294
|
-
|
|
295
|
-
changed = git_changed_paths(repo, args.base, args.head, args.mode)
|
|
296
|
-
revision_refs = {
|
|
297
|
-
path.relative_to(repo): (
|
|
298
|
-
f"{args.base if status == 'D' else args.head}:{path.relative_to(repo).as_posix()}"
|
|
299
|
-
)
|
|
300
|
-
for path, status in changed
|
|
301
|
-
if args.mode == "revisions" or status == "D"
|
|
302
|
-
}
|
|
303
|
-
revision_samples = dict(
|
|
304
|
-
zip(revision_refs, git_blob_samples(repo, list(revision_refs.values())))
|
|
305
|
-
)
|
|
306
|
-
|
|
307
|
-
rows: list[JsonRow] = []
|
|
308
|
-
for path, status in changed:
|
|
309
|
-
rel = path.relative_to(repo)
|
|
310
|
-
|
|
311
|
-
preview = ""
|
|
312
|
-
if args.mode == "revisions" or status == "D":
|
|
313
|
-
sample = revision_samples[rel]
|
|
314
|
-
if sample is None:
|
|
315
|
-
revision = args.base if status == "D" else args.head
|
|
316
|
-
raise SystemExit(f"Unable to read committed diff blob: {revision}:{rel.as_posix()}")
|
|
317
|
-
content, is_binary = sample
|
|
318
|
-
if is_binary:
|
|
319
|
-
continue
|
|
320
|
-
if status != "D":
|
|
321
|
-
preview, _ = preview_for_bytes(content, args.preview_bytes)
|
|
322
|
-
elif not path.is_symlink() and path.is_file():
|
|
323
|
-
try:
|
|
324
|
-
path.resolve(strict=True).relative_to(repo)
|
|
325
|
-
except (OSError, ValueError):
|
|
326
|
-
preview = ""
|
|
327
|
-
else:
|
|
328
|
-
preview, is_binary = preview_for(path, args.preview_bytes)
|
|
329
|
-
if is_binary or is_binary_file(path):
|
|
330
|
-
continue
|
|
331
|
-
rows.append({"path": rel.as_posix(), "area": args.area, "preview": preview})
|
|
332
|
-
|
|
333
|
-
rows.sort(key=lambda row: str(row["path"]))
|
|
334
|
-
output = Path(args.out).expanduser()
|
|
335
|
-
write_jsonl(output, rows)
|
|
336
|
-
print(f"Wrote {len(rows)} rows to {output}")
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
def main() -> None:
|
|
340
|
-
args = parse_args()
|
|
341
|
-
if args.command == "make-repo-scope-input":
|
|
342
|
-
make_repo_scope_input(args)
|
|
343
|
-
elif args.command == "make-diff-rank-input":
|
|
344
|
-
make_diff_rank_input(args)
|
|
345
|
-
else:
|
|
346
|
-
raise SystemExit(f"Unknown command: {args.command}")
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
if __name__ == "__main__":
|
|
350
|
-
main()
|