@openai/codex-security 0.6.0 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +68 -37
- package/_bundled_plugin/.mcp.json +1 -0
- package/_bundled_plugin/mcp/helpers.mjs.br.part-000 +0 -0
- package/_bundled_plugin/mcp/helpers.mjs.br.part-001 +0 -0
- package/_bundled_plugin/mcp/native/THIRD_PARTY_NOTICES.txt +1 -1
- package/_bundled_plugin/mcp/native/darwin-arm64/unix.node +0 -0
- package/_bundled_plugin/mcp/native/darwin-x64/unix.node +0 -0
- package/_bundled_plugin/mcp/native/linux-arm64-gnu/unix.node +0 -0
- package/_bundled_plugin/mcp/native/linux-arm64-musl/unix.node +0 -0
- package/_bundled_plugin/mcp/native/linux-x64-gnu/unix.node +0 -0
- package/_bundled_plugin/mcp/native/win32-arm64/windows.node +0 -0
- package/_bundled_plugin/mcp/native/win32-x64/windows.node +0 -0
- package/_bundled_plugin/mcp/server.mjs.br.part-000 +0 -0
- package/_bundled_plugin/mcp/server.mjs.br.part-001 +0 -0
- package/_bundled_plugin/mcp/server.mjs.br.part-002 +0 -0
- package/_bundled_plugin/references/access-advisory.md +11 -0
- package/_bundled_plugin/references/core-scan.md +30 -77
- package/_bundled_plugin/references/final-report.md +45 -189
- package/_bundled_plugin/references/scan-contract.md +2 -0
- package/_bundled_plugin/references/scan-prologue.md +1 -1
- package/_bundled_plugin/references/scan-runtime.md +1 -5
- package/_bundled_plugin/references/threat-model.md +33 -41
- package/_bundled_plugin/schemas/coverage.schema.json +12 -1
- package/_bundled_plugin/schemas/scan-manifest.schema.json +1 -0
- package/_bundled_plugin/schemas/tools/deep-reducer.schema.json +38 -1
- package/_bundled_plugin/scripts/deep_scan_config.py +39 -39
- package/_bundled_plugin/scripts/deep_scan_workbench.py +145 -85
- package/_bundled_plugin/scripts/finalize_scan_contract.py +189 -57
- package/_bundled_plugin/scripts/report_projection.py +44 -15
- package/_bundled_plugin/scripts/threat_model_projection.py +4 -2
- package/_bundled_plugin/scripts/windows_scan_local_files.py +43 -14
- package/_bundled_plugin/scripts/workbench/retained_coverage.py +3218 -0
- package/_bundled_plugin/scripts/workbench/retained_records.py +627 -0
- package/_bundled_plugin/scripts/workbench/saved_coverage.py +644 -1
- package/_bundled_plugin/scripts/workbench/saved_outputs.py +72 -0
- package/_bundled_plugin/scripts/workbench/saved_sources.py +712 -0
- package/_bundled_plugin/scripts/workbench_cli.py +3 -0
- package/_bundled_plugin/scripts/workbench_coverage_receipts.py +479 -0
- package/_bundled_plugin/scripts/workbench_db.py +51 -22
- package/_bundled_plugin/scripts/workbench_feedback.py +0 -1
- package/_bundled_plugin/scripts/workbench_finding_workflows.py +51 -9
- package/_bundled_plugin/scripts/workbench_native_indexes.py +56 -44
- package/_bundled_plugin/scripts/workbench_saved_results.py +641 -2387
- package/_bundled_plugin/scripts/workbench_scan_history.py +72 -7
- package/_bundled_plugin/scripts/workbench_schema.py +1 -1
- package/_bundled_plugin/scripts/workbench_source_scopes.py +4 -1
- package/_bundled_plugin/scripts/workbench_target.py +142 -17
- package/_bundled_plugin/scripts/workbench_validation.py +9 -2
- package/_bundled_plugin/shared/workbench-history-repairs.json +21 -0
- package/_bundled_plugin/shared/workbench-migrations.json +28 -0
- package/_bundled_plugin/skills/assess-patch-risk/SKILL.md +3 -14
- package/_bundled_plugin/skills/assess-patch-risk/references/windows-validation.md +14 -0
- package/_bundled_plugin/skills/attack-path-analysis/SKILL.md +38 -103
- package/_bundled_plugin/skills/attack-path-analysis/references/attack-path-facts.md +11 -73
- package/_bundled_plugin/skills/attack-path-analysis/references/severity-policy.md +57 -116
- package/_bundled_plugin/skills/deep-security-scan/SKILL.md +28 -51
- package/_bundled_plugin/skills/define-security-policy/SKILL.md +9 -9
- package/_bundled_plugin/skills/finding-discovery/SKILL.md +26 -140
- package/_bundled_plugin/skills/fix-finding/SKILL.md +9 -28
- package/_bundled_plugin/skills/propose-security-hardening/SKILL.md +28 -218
- package/_bundled_plugin/skills/propose-security-hardening/references/proposal-format.md +29 -424
- package/_bundled_plugin/skills/publish-findings/SKILL.md +28 -24
- package/_bundled_plugin/skills/publish-findings/references/wiz-import.md +64 -0
- package/_bundled_plugin/skills/security-diff-scan/SKILL.md +12 -12
- package/_bundled_plugin/skills/security-scan/SKILL.md +28 -16
- package/_bundled_plugin/skills/security-scan/references/desktop-scan.md +11 -17
- package/_bundled_plugin/skills/security-scan/references/scan-artifacts-and-ledger.md +52 -69
- package/_bundled_plugin/skills/threat-model/SKILL.md +16 -9
- package/_bundled_plugin/skills/triage-finding/SKILL.md +5 -15
- package/_bundled_plugin/skills/triage-finding/references/ticket-intake.md +6 -6
- package/_bundled_plugin/skills/validation/SKILL.md +42 -104
- package/_bundled_plugin/skills/validation/references/validation-guidance.md +21 -106
- package/_bundled_plugin/skills/verify-fix/SKILL.md +4 -2
- package/_bundled_plugin/skills/vulnerability-writeup/SKILL.md +21 -159
- package/_bundled_plugin/skills/vulnerability-writeup/references/report-format.md +18 -60
- package/dist/api.d.ts +5 -3
- package/dist/api.d.ts.map +1 -1
- package/dist/api.js +341 -37
- package/dist/api.js.map +1 -1
- package/dist/auth.d.ts.map +1 -1
- package/dist/auth.js +4 -1
- package/dist/auth.js.map +1 -1
- package/dist/cli.d.ts +2 -2
- package/dist/cli.d.ts.map +1 -1
- package/dist/cli.js +674 -419
- package/dist/cli.js.map +1 -1
- package/dist/component-scan.d.ts +8 -2
- package/dist/component-scan.d.ts.map +1 -1
- package/dist/component-scan.js +2 -0
- package/dist/component-scan.js.map +1 -1
- package/dist/contract.d.ts +2 -0
- package/dist/contract.d.ts.map +1 -1
- package/dist/contract.js +67 -33
- package/dist/contract.js.map +1 -1
- package/dist/cost.d.ts +18 -0
- package/dist/cost.d.ts.map +1 -1
- package/dist/cost.js +107 -12
- package/dist/cost.js.map +1 -1
- package/dist/coverage-presentation.d.ts +6 -0
- package/dist/coverage-presentation.d.ts.map +1 -0
- package/dist/coverage-presentation.js +34 -0
- package/dist/coverage-presentation.js.map +1 -0
- package/dist/custom-validation-prompt.js +3 -3
- package/dist/custom-validation-prompt.js.map +1 -1
- package/dist/custom-validation.d.ts.map +1 -1
- package/dist/custom-validation.js +30 -4
- package/dist/custom-validation.js.map +1 -1
- package/dist/deduplication/checkpointed-review.d.ts.map +1 -1
- package/dist/deduplication/checkpointed-review.js +3 -1
- package/dist/deduplication/checkpointed-review.js.map +1 -1
- package/dist/deduplication/scan.d.ts.map +1 -1
- package/dist/deduplication/scan.js +7 -5
- package/dist/deduplication/scan.js.map +1 -1
- package/dist/endor-findings.d.ts +4 -3
- package/dist/endor-findings.d.ts.map +1 -1
- package/dist/endor-findings.js +16 -11
- package/dist/endor-findings.js.map +1 -1
- package/dist/external-findings-import.d.ts.map +1 -1
- package/dist/external-findings-import.js +13 -5
- package/dist/external-findings-import.js.map +1 -1
- package/dist/external-findings-local.d.ts.map +1 -1
- package/dist/external-findings-local.js +8 -1
- package/dist/external-findings-local.js.map +1 -1
- package/dist/external-findings-publish.d.ts +1 -1
- package/dist/external-findings-publish.d.ts.map +1 -1
- package/dist/external-findings-publish.js +11 -3
- package/dist/external-findings-publish.js.map +1 -1
- package/dist/external-findings-read.d.ts +11 -1
- package/dist/external-findings-read.d.ts.map +1 -1
- package/dist/external-findings-read.js +15 -6
- package/dist/external-findings-read.js.map +1 -1
- package/dist/external-import-contract.d.ts +3 -1
- package/dist/external-import-contract.d.ts.map +1 -1
- package/dist/external-import-contract.js +33 -25
- package/dist/external-import-contract.js.map +1 -1
- package/dist/external-imports-feature.d.ts +6 -0
- package/dist/external-imports-feature.d.ts.map +1 -0
- package/dist/external-imports-feature.js +51 -0
- package/dist/external-imports-feature.js.map +1 -0
- package/dist/finding-workflow.d.ts +22 -5
- package/dist/finding-workflow.d.ts.map +1 -1
- package/dist/finding-workflow.js +21 -9
- package/dist/finding-workflow.js.map +1 -1
- package/dist/github-advisories.d.ts.map +1 -1
- package/dist/github-advisories.js +3 -1
- package/dist/github-advisories.js.map +1 -1
- package/dist/github-findings.d.ts +4 -3
- package/dist/github-findings.d.ts.map +1 -1
- package/dist/github-findings.js +16 -14
- package/dist/github-findings.js.map +1 -1
- package/dist/hosted-remediation.d.ts.map +1 -1
- package/dist/hosted-remediation.js +19 -2
- package/dist/hosted-remediation.js.map +1 -1
- package/dist/hosted-scan.d.ts +1 -1
- package/dist/hosted-scan.d.ts.map +1 -1
- package/dist/hosted-scan.js +0 -5
- package/dist/hosted-scan.js.map +1 -1
- package/dist/import-scan.js +1 -1
- package/dist/import-scan.js.map +1 -1
- package/dist/index.d.ts +3 -3
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1 -1
- package/dist/index.js.map +1 -1
- package/dist/knowledge-base.d.ts +1 -0
- package/dist/knowledge-base.d.ts.map +1 -1
- package/dist/knowledge-base.js +17 -8
- package/dist/knowledge-base.js.map +1 -1
- package/dist/models.d.ts +1 -0
- package/dist/models.d.ts.map +1 -1
- package/dist/models.js.map +1 -1
- package/dist/owner-evidence.d.ts +2 -0
- package/dist/owner-evidence.d.ts.map +1 -1
- package/dist/owner-evidence.js +2 -1
- package/dist/owner-evidence.js.map +1 -1
- package/dist/publish.d.ts +2 -0
- package/dist/publish.d.ts.map +1 -1
- package/dist/publish.js +1 -1
- package/dist/publish.js.map +1 -1
- package/dist/remediation.d.ts +1 -25
- package/dist/remediation.d.ts.map +1 -1
- package/dist/remediation.js +3 -24
- package/dist/remediation.js.map +1 -1
- package/dist/runtime.d.ts.map +1 -1
- package/dist/runtime.js +7 -7
- package/dist/runtime.js.map +1 -1
- package/dist/scan-dashboard.d.ts +5 -3
- package/dist/scan-dashboard.d.ts.map +1 -1
- package/dist/scan-dashboard.js +79 -26
- package/dist/scan-dashboard.js.map +1 -1
- package/dist/scan-history-renderer.d.ts.map +1 -1
- package/dist/scan-history-renderer.js +8 -2
- package/dist/scan-history-renderer.js.map +1 -1
- package/dist/scan-owner-suggestions.d.ts +1 -1
- package/dist/scan-owner-suggestions.d.ts.map +1 -1
- package/dist/scan-owner-suggestions.js +4 -4
- package/dist/scan-owner-suggestions.js.map +1 -1
- package/dist/scan-usage-display.d.ts +8 -0
- package/dist/scan-usage-display.d.ts.map +1 -0
- package/dist/scan-usage-display.js +46 -0
- package/dist/scan-usage-display.js.map +1 -0
- package/dist/scanner-import.d.ts +1 -1
- package/dist/scanner-import.d.ts.map +1 -1
- package/dist/scanner-import.js +7 -3
- package/dist/scanner-import.js.map +1 -1
- package/dist/security-policy.d.ts.map +1 -1
- package/dist/security-policy.js +1 -20
- package/dist/security-policy.js.map +1 -1
- package/dist/semgrep-findings.d.ts +4 -3
- package/dist/semgrep-findings.d.ts.map +1 -1
- package/dist/semgrep-findings.js +9 -9
- package/dist/semgrep-findings.js.map +1 -1
- package/dist/skill-authentication.d.ts +18 -0
- package/dist/skill-authentication.d.ts.map +1 -0
- package/dist/skill-authentication.js +82 -0
- package/dist/skill-authentication.js.map +1 -0
- package/dist/snyk-findings.d.ts +5 -4
- package/dist/snyk-findings.d.ts.map +1 -1
- package/dist/snyk-findings.js +18 -17
- package/dist/snyk-findings.js.map +1 -1
- package/dist/suggest-owners.d.ts +4 -1
- package/dist/suggest-owners.d.ts.map +1 -1
- package/dist/suggest-owners.js +53 -21
- package/dist/suggest-owners.js.map +1 -1
- package/dist/targets.d.ts.map +1 -1
- package/dist/targets.js +4 -1
- package/dist/targets.js.map +1 -1
- package/dist/wiz-csv.d.ts +2 -1
- package/dist/wiz-csv.d.ts.map +1 -1
- package/dist/wiz-csv.js +3 -3
- package/dist/wiz-csv.js.map +1 -1
- package/dist/wiz-findings.d.ts +2 -1
- package/dist/wiz-findings.d.ts.map +1 -1
- package/dist/wiz-findings.js +27 -23
- package/dist/wiz-findings.js.map +1 -1
- package/docs/classify-severity-records.md +3 -4
- package/docs/cli.md +155 -38
- package/docs/findings-service.md +119 -118
- package/docs/hosted-remediation.md +19 -17
- package/docs/hosted-scans.md +37 -11
- package/package.json +8 -8
- package/_bundled_plugin/scripts/generate_in_scope_files.py +0 -277
- package/_bundled_plugin/scripts/generate_rank_input.py +0 -350
- package/_bundled_plugin/scripts/rank_preview.py +0 -129
|
@@ -1,350 +0,0 @@
|
|
|
1
|
-
#!/usr/bin/env python3
|
|
2
|
-
"""Generate and post-process Codex Security scan worklists.
|
|
3
|
-
|
|
4
|
-
This script stays deliberately model-free:
|
|
5
|
-
|
|
6
|
-
- `make-repo-scope-input` lists every explicitly scoped repository file.
|
|
7
|
-
- `make-diff-rank-input` creates the deterministic diff-scoped JSONL candidate
|
|
8
|
-
worklist from Git changed paths. It supports committed revision diffs and
|
|
9
|
-
local working-tree patches.
|
|
10
|
-
|
|
11
|
-
Candidate selection uses repository scope, native tool ignore rules, and binary detection,
|
|
12
|
-
not filename or directory classifications. Diff inputs use bounded source previews.
|
|
13
|
-
"""
|
|
14
|
-
|
|
15
|
-
from __future__ import annotations
|
|
16
|
-
|
|
17
|
-
import argparse
|
|
18
|
-
import json
|
|
19
|
-
import os
|
|
20
|
-
import subprocess
|
|
21
|
-
import sys
|
|
22
|
-
from collections.abc import Iterable
|
|
23
|
-
from pathlib import Path
|
|
24
|
-
|
|
25
|
-
# Some plugin hosts launch Python with safe-path isolation enabled.
|
|
26
|
-
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
|
27
|
-
from generate_in_scope_files import windows_stream_component
|
|
28
|
-
from rank_preview import (
|
|
29
|
-
DEFAULT_PREVIEW_BYTES,
|
|
30
|
-
is_binary_file,
|
|
31
|
-
preview_for,
|
|
32
|
-
preview_for_bytes,
|
|
33
|
-
)
|
|
34
|
-
from workbench_target import (
|
|
35
|
-
SOURCE_METADATA_DIRECTORIES,
|
|
36
|
-
git_blob_samples,
|
|
37
|
-
git_command,
|
|
38
|
-
git_directory_snapshot_paths,
|
|
39
|
-
)
|
|
40
|
-
|
|
41
|
-
JsonRow = dict[str, object]
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
def parse_args() -> argparse.Namespace:
|
|
45
|
-
parser = argparse.ArgumentParser(description="Codex Security scan worklist helper.")
|
|
46
|
-
subparsers = parser.add_subparsers(dest="command", required=True)
|
|
47
|
-
|
|
48
|
-
scoped = subparsers.add_parser(
|
|
49
|
-
"make-repo-scope-input",
|
|
50
|
-
help="List every explicitly scoped file without ranking or reading its contents.",
|
|
51
|
-
)
|
|
52
|
-
scoped.add_argument("--repo", required=True, help="Repository root.")
|
|
53
|
-
scoped.add_argument(
|
|
54
|
-
"--scopes-file",
|
|
55
|
-
required=True,
|
|
56
|
-
help="JSON array of repository-relative files and directories to scan together.",
|
|
57
|
-
)
|
|
58
|
-
scoped.add_argument("--out", required=True, help="Output scoped-source-input.jsonl path.")
|
|
59
|
-
|
|
60
|
-
diff = subparsers.add_parser(
|
|
61
|
-
"make-diff-rank-input",
|
|
62
|
-
help="Create rank_input.jsonl from Git changed text files.",
|
|
63
|
-
)
|
|
64
|
-
diff.add_argument("--repo", required=True, help="Repository root.")
|
|
65
|
-
diff.add_argument("--base", required=True, help="Git diff base revision.")
|
|
66
|
-
diff.add_argument(
|
|
67
|
-
"--mode",
|
|
68
|
-
choices=("revisions", "local-patch"),
|
|
69
|
-
default="revisions",
|
|
70
|
-
help="Git diff mode: committed revisions or staged plus unstaged local patch.",
|
|
71
|
-
)
|
|
72
|
-
diff.add_argument("--head", default="HEAD", help="Git diff head revision.")
|
|
73
|
-
diff.add_argument("--out", required=True, help="Output rank_input.jsonl path.")
|
|
74
|
-
diff.add_argument("--area", default="diff", help="Area label for changed-file rows.")
|
|
75
|
-
diff.add_argument(
|
|
76
|
-
"--preview-bytes",
|
|
77
|
-
type=int,
|
|
78
|
-
default=DEFAULT_PREVIEW_BYTES,
|
|
79
|
-
help=f"Maximum UTF-8 bytes in each preview. Defaults to {DEFAULT_PREVIEW_BYTES}.",
|
|
80
|
-
)
|
|
81
|
-
|
|
82
|
-
return parser.parse_args()
|
|
83
|
-
|
|
84
|
-
|
|
85
|
-
def resolve_scope(repo: Path, scope: str) -> Path:
|
|
86
|
-
scope_path = Path(scope)
|
|
87
|
-
stream = windows_stream_component(scope_path)
|
|
88
|
-
if stream is not None:
|
|
89
|
-
raise SystemExit(f"Scope must not use an NTFS alternate data stream: {stream}")
|
|
90
|
-
if not scope_path.is_absolute():
|
|
91
|
-
scope_path = repo / scope_path
|
|
92
|
-
repository = repo.resolve()
|
|
93
|
-
try:
|
|
94
|
-
relative = scope_path.relative_to(repository)
|
|
95
|
-
except ValueError as exc:
|
|
96
|
-
raise SystemExit(f"Scope must be inside repo: {scope_path}") from exc
|
|
97
|
-
ancestor = repository
|
|
98
|
-
for part in relative.parts:
|
|
99
|
-
if part == "..":
|
|
100
|
-
if ancestor == repository:
|
|
101
|
-
raise SystemExit(f"Scope must be inside repo: {scope_path}")
|
|
102
|
-
ancestor = ancestor.parent
|
|
103
|
-
continue
|
|
104
|
-
ancestor /= part
|
|
105
|
-
try:
|
|
106
|
-
metadata = ancestor.stat(follow_symlinks=False)
|
|
107
|
-
except OSError as exc:
|
|
108
|
-
raise SystemExit(f"Scope path not found: {ancestor}") from exc
|
|
109
|
-
if ancestor.is_symlink() or getattr(metadata, "st_reparse_tag", 0) & 0x20000000:
|
|
110
|
-
raise SystemExit(f"Requested scope must not contain symbolic links: {ancestor}")
|
|
111
|
-
scope_path = scope_path.resolve()
|
|
112
|
-
repo_resolved = repo.resolve()
|
|
113
|
-
try:
|
|
114
|
-
scope_path.relative_to(repo_resolved)
|
|
115
|
-
except ValueError as exc:
|
|
116
|
-
raise SystemExit(f"Scope must be inside repo: {scope_path}") from exc
|
|
117
|
-
if not scope_path.is_dir() and not scope_path.is_file():
|
|
118
|
-
raise SystemExit(f"Scope path not found: {scope_path}")
|
|
119
|
-
return scope_path
|
|
120
|
-
|
|
121
|
-
|
|
122
|
-
def write_jsonl(output: Path, rows: list[JsonRow]) -> None:
|
|
123
|
-
output.parent.mkdir(parents=True, exist_ok=True)
|
|
124
|
-
with output.open("w", encoding="utf-8") as handle:
|
|
125
|
-
for row in rows:
|
|
126
|
-
handle.write(json.dumps(row, ensure_ascii=True, separators=(",", ":")))
|
|
127
|
-
handle.write("\n")
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
def load_scopes_file(scopes_file: Path) -> list[str]:
|
|
131
|
-
try:
|
|
132
|
-
loaded: object = json.loads(scopes_file.read_text(encoding="utf-8"))
|
|
133
|
-
except (OSError, UnicodeError, json.JSONDecodeError) as exc:
|
|
134
|
-
raise SystemExit(f"Unable to read scopes file: {scopes_file}") from exc
|
|
135
|
-
if (
|
|
136
|
-
not isinstance(loaded, list)
|
|
137
|
-
or not loaded
|
|
138
|
-
or any(not isinstance(scope, str) or not scope for scope in loaded)
|
|
139
|
-
):
|
|
140
|
-
raise SystemExit(f"Scopes file must contain a non-empty JSON string array: {scopes_file}")
|
|
141
|
-
return loaded
|
|
142
|
-
|
|
143
|
-
|
|
144
|
-
def scope_candidates(repo: Path, scope_path: Path) -> Iterable[Path]:
|
|
145
|
-
"""Use Git's inventory in worktrees and ripgrep's ignore rules elsewhere.
|
|
146
|
-
|
|
147
|
-
Git retains tracked files and applies its standard exclusions to untracked files.
|
|
148
|
-
Outside Git, ripgrep also honors .ignore and .rgignore alongside .gitignore.
|
|
149
|
-
Explicit file scopes bypass directory ignore rules.
|
|
150
|
-
"""
|
|
151
|
-
if scope_path.is_file():
|
|
152
|
-
return (scope_path,)
|
|
153
|
-
git_candidates = git_directory_snapshot_paths(scope_path)
|
|
154
|
-
if git_candidates is not None:
|
|
155
|
-
return git_candidates
|
|
156
|
-
|
|
157
|
-
command = [
|
|
158
|
-
"rg",
|
|
159
|
-
"--files",
|
|
160
|
-
"--hidden",
|
|
161
|
-
"--no-require-git",
|
|
162
|
-
"--null",
|
|
163
|
-
# Also exclude descendants when the scope starts inside metadata.
|
|
164
|
-
"--glob",
|
|
165
|
-
"!**/{.git,.svn}",
|
|
166
|
-
"--glob",
|
|
167
|
-
"!**/{.git,.svn}/**",
|
|
168
|
-
"--",
|
|
169
|
-
str(scope_path.relative_to(repo)),
|
|
170
|
-
]
|
|
171
|
-
try:
|
|
172
|
-
result = subprocess.run(command, cwd=repo, capture_output=True, check=False)
|
|
173
|
-
except OSError as exc:
|
|
174
|
-
ignore_names = (".gitignore", ".ignore", ".rgignore")
|
|
175
|
-
ancestors = (scope_path, *scope_path.parents)
|
|
176
|
-
has_ignore_rules = (
|
|
177
|
-
any((ancestor / ".git").exists() for ancestor in (repo, *repo.parents))
|
|
178
|
-
or any(
|
|
179
|
-
(ancestor / name).is_file()
|
|
180
|
-
for ancestor in ancestors
|
|
181
|
-
if ancestor == repo or repo in ancestor.parents
|
|
182
|
-
for name in ignore_names
|
|
183
|
-
)
|
|
184
|
-
or any(path.name in ignore_names for path in scope_path.rglob("*") if path.is_file())
|
|
185
|
-
)
|
|
186
|
-
if has_ignore_rules:
|
|
187
|
-
raise SystemExit(
|
|
188
|
-
"Could not safely enumerate ignored scoped files without Git or ripgrep."
|
|
189
|
-
) from exc
|
|
190
|
-
return scope_path.rglob("*")
|
|
191
|
-
|
|
192
|
-
if result.returncode not in (0, 1):
|
|
193
|
-
detail = result.stderr.decode("utf-8", errors="replace").strip()
|
|
194
|
-
raise SystemExit(f"Could not enumerate scoped repository files: {detail}")
|
|
195
|
-
return (repo / os.fsdecode(path) for path in result.stdout.split(b"\0") if path)
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
def make_repo_scope_input(args: argparse.Namespace) -> None:
|
|
199
|
-
repo = Path(args.repo).expanduser().resolve()
|
|
200
|
-
if not repo.is_dir():
|
|
201
|
-
raise SystemExit(f"Repo path not found: {repo}")
|
|
202
|
-
|
|
203
|
-
scopes = load_scopes_file(Path(args.scopes_file).expanduser())
|
|
204
|
-
rows_by_path: dict[str, JsonRow] = {}
|
|
205
|
-
for scope in scopes:
|
|
206
|
-
scope_path = resolve_scope(repo, scope)
|
|
207
|
-
candidates = scope_candidates(repo, scope_path)
|
|
208
|
-
for path in candidates:
|
|
209
|
-
try:
|
|
210
|
-
if path.is_symlink() or not path.is_file():
|
|
211
|
-
continue
|
|
212
|
-
relative = path.resolve(strict=True).relative_to(repo)
|
|
213
|
-
except (OSError, ValueError):
|
|
214
|
-
continue
|
|
215
|
-
if not SOURCE_METADATA_DIRECTORIES.isdisjoint(relative.parts):
|
|
216
|
-
continue
|
|
217
|
-
rows_by_path.setdefault(relative.as_posix(), {"path": relative.as_posix()})
|
|
218
|
-
|
|
219
|
-
rows = sorted(rows_by_path.values(), key=lambda row: str(row["path"]))
|
|
220
|
-
output = Path(args.out).expanduser()
|
|
221
|
-
write_jsonl(output, rows)
|
|
222
|
-
print(f"Wrote {len(rows)} scoped paths to {output}")
|
|
223
|
-
|
|
224
|
-
|
|
225
|
-
def run_git_changed_paths(repo: Path, diff_args: list[str]) -> list[tuple[Path, str]]:
|
|
226
|
-
"""Return changed regular files from the selected side of each change."""
|
|
227
|
-
result = git_command(
|
|
228
|
-
repo,
|
|
229
|
-
"diff",
|
|
230
|
-
"--ignore-submodules=all",
|
|
231
|
-
"--raw",
|
|
232
|
-
"-z",
|
|
233
|
-
"--diff-filter=ACMRDT",
|
|
234
|
-
*diff_args,
|
|
235
|
-
"--",
|
|
236
|
-
text=False,
|
|
237
|
-
)
|
|
238
|
-
result.check_returncode()
|
|
239
|
-
fields = result.stdout.split(b"\0")
|
|
240
|
-
if fields and not fields[-1]:
|
|
241
|
-
fields.pop()
|
|
242
|
-
|
|
243
|
-
changed: list[tuple[Path, str]] = []
|
|
244
|
-
index = 0
|
|
245
|
-
while index < len(fields):
|
|
246
|
-
metadata = fields[index].split()
|
|
247
|
-
status = chr(metadata[-1][0])
|
|
248
|
-
index += 1
|
|
249
|
-
if status in {"C", "R"}:
|
|
250
|
-
index += 1
|
|
251
|
-
path = repo / os.fsdecode(fields[index])
|
|
252
|
-
index += 1
|
|
253
|
-
selected_mode = metadata[0].removeprefix(b":") if status == "D" else metadata[1]
|
|
254
|
-
if selected_mode.startswith(b"100"):
|
|
255
|
-
changed.append((path, status))
|
|
256
|
-
return changed
|
|
257
|
-
|
|
258
|
-
|
|
259
|
-
def git_changed_paths(repo: Path, base: str, head: str, mode: str) -> list[tuple[Path, str]]:
|
|
260
|
-
if mode == "revisions":
|
|
261
|
-
return run_git_changed_paths(repo, [f"{base}..{head}"])
|
|
262
|
-
if mode == "local-patch":
|
|
263
|
-
unstaged = run_git_changed_paths(repo, [base])
|
|
264
|
-
staged = run_git_changed_paths(repo, ["--cached", base])
|
|
265
|
-
untracked = git_command(
|
|
266
|
-
repo,
|
|
267
|
-
"ls-files",
|
|
268
|
-
"--others",
|
|
269
|
-
"--exclude-standard",
|
|
270
|
-
"-z",
|
|
271
|
-
text=False,
|
|
272
|
-
)
|
|
273
|
-
untracked.check_returncode()
|
|
274
|
-
combined = dict(staged)
|
|
275
|
-
combined.update(unstaged)
|
|
276
|
-
combined.update(
|
|
277
|
-
(repo / os.fsdecode(relative), "A")
|
|
278
|
-
for relative in untracked.stdout.split(b"\0")
|
|
279
|
-
if relative and not relative.endswith(b"/")
|
|
280
|
-
)
|
|
281
|
-
return sorted(
|
|
282
|
-
(path, status)
|
|
283
|
-
for path, status in combined.items()
|
|
284
|
-
if SOURCE_METADATA_DIRECTORIES.isdisjoint(path.relative_to(repo).parts)
|
|
285
|
-
and (status == "D" or (not path.is_symlink() and path.is_file()))
|
|
286
|
-
)
|
|
287
|
-
raise SystemExit(f"Unknown diff mode: {mode}")
|
|
288
|
-
|
|
289
|
-
|
|
290
|
-
def make_diff_rank_input(args: argparse.Namespace) -> None:
|
|
291
|
-
repo = Path(args.repo).expanduser().resolve()
|
|
292
|
-
if not repo.is_dir():
|
|
293
|
-
raise SystemExit(f"Repo path not found: {repo}")
|
|
294
|
-
|
|
295
|
-
changed = git_changed_paths(repo, args.base, args.head, args.mode)
|
|
296
|
-
revision_refs = {
|
|
297
|
-
path.relative_to(repo): (
|
|
298
|
-
f"{args.base if status == 'D' else args.head}:{path.relative_to(repo).as_posix()}"
|
|
299
|
-
)
|
|
300
|
-
for path, status in changed
|
|
301
|
-
if args.mode == "revisions" or status == "D"
|
|
302
|
-
}
|
|
303
|
-
revision_samples = dict(
|
|
304
|
-
zip(revision_refs, git_blob_samples(repo, list(revision_refs.values())))
|
|
305
|
-
)
|
|
306
|
-
|
|
307
|
-
rows: list[JsonRow] = []
|
|
308
|
-
for path, status in changed:
|
|
309
|
-
rel = path.relative_to(repo)
|
|
310
|
-
|
|
311
|
-
preview = ""
|
|
312
|
-
if args.mode == "revisions" or status == "D":
|
|
313
|
-
sample = revision_samples[rel]
|
|
314
|
-
if sample is None:
|
|
315
|
-
revision = args.base if status == "D" else args.head
|
|
316
|
-
raise SystemExit(f"Unable to read committed diff blob: {revision}:{rel.as_posix()}")
|
|
317
|
-
content, is_binary = sample
|
|
318
|
-
if is_binary:
|
|
319
|
-
continue
|
|
320
|
-
if status != "D":
|
|
321
|
-
preview, _ = preview_for_bytes(content, args.preview_bytes)
|
|
322
|
-
elif not path.is_symlink() and path.is_file():
|
|
323
|
-
try:
|
|
324
|
-
path.resolve(strict=True).relative_to(repo)
|
|
325
|
-
except (OSError, ValueError):
|
|
326
|
-
preview = ""
|
|
327
|
-
else:
|
|
328
|
-
preview, is_binary = preview_for(path, args.preview_bytes)
|
|
329
|
-
if is_binary or is_binary_file(path):
|
|
330
|
-
continue
|
|
331
|
-
rows.append({"path": rel.as_posix(), "area": args.area, "preview": preview})
|
|
332
|
-
|
|
333
|
-
rows.sort(key=lambda row: str(row["path"]))
|
|
334
|
-
output = Path(args.out).expanduser()
|
|
335
|
-
write_jsonl(output, rows)
|
|
336
|
-
print(f"Wrote {len(rows)} rows to {output}")
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
def main() -> None:
|
|
340
|
-
args = parse_args()
|
|
341
|
-
if args.command == "make-repo-scope-input":
|
|
342
|
-
make_repo_scope_input(args)
|
|
343
|
-
elif args.command == "make-diff-rank-input":
|
|
344
|
-
make_diff_rank_input(args)
|
|
345
|
-
else:
|
|
346
|
-
raise SystemExit(f"Unknown command: {args.command}")
|
|
347
|
-
|
|
348
|
-
|
|
349
|
-
if __name__ == "__main__":
|
|
350
|
-
main()
|
|
@@ -1,129 +0,0 @@
|
|
|
1
|
-
"""Build bounded source previews for rank inputs regardless of language."""
|
|
2
|
-
|
|
3
|
-
from __future__ import annotations
|
|
4
|
-
|
|
5
|
-
import argparse
|
|
6
|
-
from bisect import bisect_right
|
|
7
|
-
from pathlib import Path
|
|
8
|
-
|
|
9
|
-
DEFAULT_PREVIEW_BYTES = 1024
|
|
10
|
-
DEFAULT_PREVIEW_READ_BYTES = 64 * 1024
|
|
11
|
-
PREVIEW_HEAD_LINES = 12
|
|
12
|
-
PREVIEW_SAMPLE_LINES = 10
|
|
13
|
-
_UTF16_BOMS = (b"\xff\xfe", b"\xfe\xff")
|
|
14
|
-
|
|
15
|
-
|
|
16
|
-
def _decode_source(data: bytes) -> str:
|
|
17
|
-
encoding = "utf-16" if data.startswith(_UTF16_BOMS) else "utf-8-sig"
|
|
18
|
-
return data.decode(encoding, errors="ignore")
|
|
19
|
-
|
|
20
|
-
|
|
21
|
-
def is_binary_sample(data: bytes) -> bool:
|
|
22
|
-
if data.startswith(_UTF16_BOMS):
|
|
23
|
-
return "\0" in _decode_source(data)
|
|
24
|
-
return b"\0" in data
|
|
25
|
-
|
|
26
|
-
|
|
27
|
-
def is_binary_file(path: Path) -> bool:
|
|
28
|
-
"""Classify the whole file in bounded chunks."""
|
|
29
|
-
try:
|
|
30
|
-
with path.open("rb") as source:
|
|
31
|
-
sample = source.read(DEFAULT_PREVIEW_READ_BYTES)
|
|
32
|
-
if is_binary_sample(sample):
|
|
33
|
-
return True
|
|
34
|
-
bom = sample[:2] if sample.startswith(_UTF16_BOMS) else b""
|
|
35
|
-
while chunk := source.read(DEFAULT_PREVIEW_READ_BYTES):
|
|
36
|
-
# Even-sized reads preserve UTF-16 code-unit alignment and byte order.
|
|
37
|
-
if is_binary_sample(bom + chunk) if bom else b"\0" in chunk:
|
|
38
|
-
return True
|
|
39
|
-
except OSError:
|
|
40
|
-
return True
|
|
41
|
-
return False
|
|
42
|
-
|
|
43
|
-
|
|
44
|
-
def truncate_utf8(text: str, max_bytes: int) -> str:
|
|
45
|
-
if max_bytes <= 0:
|
|
46
|
-
return ""
|
|
47
|
-
encoded = text.encode("utf-8")
|
|
48
|
-
if len(encoded) <= max_bytes:
|
|
49
|
-
return text
|
|
50
|
-
return encoded[:max_bytes].decode("utf-8", errors="ignore")
|
|
51
|
-
|
|
52
|
-
|
|
53
|
-
def select_preview_lines(lines: list[str]) -> list[str]:
|
|
54
|
-
nonblank = [line for line in lines if line.strip()]
|
|
55
|
-
head = nonblank[:PREVIEW_HEAD_LINES]
|
|
56
|
-
remainder = nonblank[PREVIEW_HEAD_LINES:]
|
|
57
|
-
if len(remainder) <= PREVIEW_SAMPLE_LINES:
|
|
58
|
-
return [*head, *remainder]
|
|
59
|
-
|
|
60
|
-
last_index = len(remainder) - 1
|
|
61
|
-
sampled = [
|
|
62
|
-
remainder[index * last_index // (PREVIEW_SAMPLE_LINES - 1)]
|
|
63
|
-
for index in range(PREVIEW_SAMPLE_LINES)
|
|
64
|
-
]
|
|
65
|
-
return [*head, "...", *sampled]
|
|
66
|
-
|
|
67
|
-
|
|
68
|
-
def fit_preview_lines(lines: list[str], max_bytes: int) -> str:
|
|
69
|
-
if not lines or max_bytes <= 0:
|
|
70
|
-
return ""
|
|
71
|
-
|
|
72
|
-
full_preview = "\n".join(lines)
|
|
73
|
-
if len(full_preview.encode("utf-8")) <= max_bytes:
|
|
74
|
-
return full_preview
|
|
75
|
-
|
|
76
|
-
def render(line_bytes: int) -> str:
|
|
77
|
-
return "\n".join(
|
|
78
|
-
line if line == "..." else truncate_utf8(line, line_bytes) for line in lines
|
|
79
|
-
)
|
|
80
|
-
|
|
81
|
-
high = max(len(line.encode("utf-8")) for line in lines)
|
|
82
|
-
limit = bisect_right(
|
|
83
|
-
range(high + 1), max_bytes, key=lambda size: len(render(size).encode("utf-8"))
|
|
84
|
-
)
|
|
85
|
-
best = render(limit - 1) if limit else ""
|
|
86
|
-
if best.strip() not in {"", "..."}:
|
|
87
|
-
return best
|
|
88
|
-
return truncate_utf8(full_preview, max_bytes)
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
def preview_for(
|
|
92
|
-
path: Path, preview_bytes: int, *, max_read_bytes: int = DEFAULT_PREVIEW_READ_BYTES
|
|
93
|
-
) -> tuple[str, bool]:
|
|
94
|
-
"""Preview the first 64 KiB by default, including the binary-detection sample."""
|
|
95
|
-
if max_read_bytes <= 0:
|
|
96
|
-
raise ValueError("max_read_bytes must be positive")
|
|
97
|
-
try:
|
|
98
|
-
with path.open("rb") as source:
|
|
99
|
-
sample = source.read(min(4096, max_read_bytes))
|
|
100
|
-
if is_binary_sample(sample):
|
|
101
|
-
return "", True
|
|
102
|
-
remaining = source.read(max_read_bytes - len(sample))
|
|
103
|
-
data = sample + remaining
|
|
104
|
-
except OSError:
|
|
105
|
-
return "", True
|
|
106
|
-
return preview_for_bytes(data, preview_bytes)
|
|
107
|
-
|
|
108
|
-
|
|
109
|
-
def preview_for_bytes(data: bytes, preview_bytes: int) -> tuple[str, bool]:
|
|
110
|
-
if is_binary_sample(data):
|
|
111
|
-
return "", True
|
|
112
|
-
lines = _decode_source(data).replace("\r\n", "\n").replace("\r", "\n").split("\n")
|
|
113
|
-
start, end = 0, len(lines)
|
|
114
|
-
while start < end and not lines[start].strip():
|
|
115
|
-
start += 1
|
|
116
|
-
while end > start and not lines[end - 1].strip():
|
|
117
|
-
end -= 1
|
|
118
|
-
lines = lines[start:end]
|
|
119
|
-
text = "\n".join(lines)
|
|
120
|
-
if len(text.encode("utf-8")) <= preview_bytes:
|
|
121
|
-
return text, False
|
|
122
|
-
return fit_preview_lines(select_preview_lines(lines), preview_bytes), False
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
if __name__ == "__main__":
|
|
126
|
-
parser = argparse.ArgumentParser(
|
|
127
|
-
description="Build bounded source previews for Codex Security rank inputs."
|
|
128
|
-
)
|
|
129
|
-
parser.parse_args()
|