@openai/codex-security 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +201 -0
- package/README.md +306 -0
- package/_bundled_plugin/.app.json +20 -0
- package/_bundled_plugin/.codex-plugin/plugin.json +41 -0
- package/_bundled_plugin/.mcp.json +20 -0
- package/_bundled_plugin/assets/logo.png +0 -0
- package/_bundled_plugin/examples/completed-scan/coverage.json +22 -0
- package/_bundled_plugin/examples/completed-scan/findings.json +51 -0
- package/_bundled_plugin/examples/completed-scan/scan-manifest.json +43 -0
- package/_bundled_plugin/mcp/mcp-app.html.br +0 -0
- package/_bundled_plugin/mcp/server.mjs +24 -0
- package/_bundled_plugin/mcp/server.mjs.br.part-000 +0 -0
- package/_bundled_plugin/mcp/server.mjs.br.part-001 +0 -0
- package/_bundled_plugin/preflight/capability-profiles.toml +179 -0
- package/_bundled_plugin/references/config-preflight.md +115 -0
- package/_bundled_plugin/references/final-report.md +205 -0
- package/_bundled_plugin/references/finding-detail-fields.md +162 -0
- package/_bundled_plugin/references/sarif-adapter.md +26 -0
- package/_bundled_plugin/references/scan-artifacts.md +116 -0
- package/_bundled_plugin/references/scan-contract.md +164 -0
- package/_bundled_plugin/references/security-guidance.md +15 -0
- package/_bundled_plugin/references/shared-hard-rules.md +15 -0
- package/_bundled_plugin/references/static-finding-assessment.md +75 -0
- package/_bundled_plugin/schemas/coverage.schema.json +228 -0
- package/_bundled_plugin/schemas/findings.schema.json +359 -0
- package/_bundled_plugin/schemas/scan-manifest.schema.json +332 -0
- package/_bundled_plugin/scripts/config_preflight.py +901 -0
- package/_bundled_plugin/scripts/deep_scan_config.py +108 -0
- package/_bundled_plugin/scripts/deep_scan_workbench.py +1547 -0
- package/_bundled_plugin/scripts/filesystem_identity.py +30 -0
- package/_bundled_plugin/scripts/finalize_scan_contract.py +2208 -0
- package/_bundled_plugin/scripts/finding_preview.py +270 -0
- package/_bundled_plugin/scripts/generate_rank_input.py +1013 -0
- package/_bundled_plugin/scripts/normalize_candidates.py +320 -0
- package/_bundled_plugin/scripts/rank_preview.py +968 -0
- package/_bundled_plugin/scripts/report_projection.py +844 -0
- package/_bundled_plugin/scripts/resolve_security_md.py +158 -0
- package/_bundled_plugin/scripts/snapshot_sqlite.py +26 -0
- package/_bundled_plugin/scripts/validate_report_format.py +138 -0
- package/_bundled_plugin/scripts/validate_scan_contract.py +108 -0
- package/_bundled_plugin/scripts/validate_tracking_source.py +92 -0
- package/_bundled_plugin/scripts/windows_scan_local_files.py +612 -0
- package/_bundled_plugin/scripts/workbench/__init__.py +1 -0
- package/_bundled_plugin/scripts/workbench/handoff.py +238 -0
- package/_bundled_plugin/scripts/workbench_cli.py +325 -0
- package/_bundled_plugin/scripts/workbench_constants.py +81 -0
- package/_bundled_plugin/scripts/workbench_db.py +3728 -0
- package/_bundled_plugin/scripts/workbench_native_indexes.py +242 -0
- package/_bundled_plugin/scripts/workbench_progress.py +226 -0
- package/_bundled_plugin/scripts/workbench_remediation.py +125 -0
- package/_bundled_plugin/scripts/workbench_scan_history.py +829 -0
- package/_bundled_plugin/scripts/workbench_scan_start.py +149 -0
- package/_bundled_plugin/scripts/workbench_schema.py +714 -0
- package/_bundled_plugin/scripts/workbench_source_excerpt.py +99 -0
- package/_bundled_plugin/scripts/workbench_target.py +469 -0
- package/_bundled_plugin/scripts/workbench_target_state.py +63 -0
- package/_bundled_plugin/scripts/workbench_validation.py +294 -0
- package/_bundled_plugin/skills/attack-path-analysis/SKILL.md +117 -0
- package/_bundled_plugin/skills/attack-path-analysis/agents/openai.yaml +4 -0
- package/_bundled_plugin/skills/attack-path-analysis/references/attack-path-facts.md +76 -0
- package/_bundled_plugin/skills/attack-path-analysis/references/severity-policy.md +153 -0
- package/_bundled_plugin/skills/deep-security-scan/SKILL.md +166 -0
- package/_bundled_plugin/skills/deep-security-scan/agents/openai.yaml +4 -0
- package/_bundled_plugin/skills/define-security-policy/SKILL.md +93 -0
- package/_bundled_plugin/skills/define-security-policy/agents/openai.yaml +4 -0
- package/_bundled_plugin/skills/finding-discovery/SKILL.md +155 -0
- package/_bundled_plugin/skills/finding-discovery/agents/openai.yaml +4 -0
- package/_bundled_plugin/skills/fix-finding/SKILL.md +112 -0
- package/_bundled_plugin/skills/fix-finding/agents/openai.yaml +4 -0
- package/_bundled_plugin/skills/propose-security-hardening/SKILL.md +240 -0
- package/_bundled_plugin/skills/propose-security-hardening/agents/openai.yaml +4 -0
- package/_bundled_plugin/skills/propose-security-hardening/references/proposal-format.md +424 -0
- package/_bundled_plugin/skills/security-diff-scan/SKILL.md +167 -0
- package/_bundled_plugin/skills/security-diff-scan/agents/openai.yaml +4 -0
- package/_bundled_plugin/skills/security-scan/SKILL.md +42 -0
- package/_bundled_plugin/skills/security-scan/agents/openai.yaml +4 -0
- package/_bundled_plugin/skills/security-scan/references/repository-wide-scan.md +48 -0
- package/_bundled_plugin/skills/security-scan/references/scan-artifacts-and-ledger.md +85 -0
- package/_bundled_plugin/skills/threat-model/SKILL.md +59 -0
- package/_bundled_plugin/skills/threat-model/agents/openai.yaml +4 -0
- package/_bundled_plugin/skills/threat-model/references/threat-model-guidance.md +48 -0
- package/_bundled_plugin/skills/track-findings/SKILL.md +222 -0
- package/_bundled_plugin/skills/track-findings/agents/openai.yaml +6 -0
- package/_bundled_plugin/skills/track-findings/references/github-security-advisories.md +54 -0
- package/_bundled_plugin/skills/track-findings/references/jira.md +77 -0
- package/_bundled_plugin/skills/triage-finding/SKILL.md +342 -0
- package/_bundled_plugin/skills/triage-finding/agents/openai.yaml +4 -0
- package/_bundled_plugin/skills/triage-finding/references/github-rest-intake.md +168 -0
- package/_bundled_plugin/skills/triage-finding/references/ticket-intake.md +62 -0
- package/_bundled_plugin/skills/triage-finding/references/triage-result-contract.md +108 -0
- package/_bundled_plugin/skills/validation/SKILL.md +110 -0
- package/_bundled_plugin/skills/validation/agents/openai.yaml +4 -0
- package/_bundled_plugin/skills/validation/references/validation-guidance.md +122 -0
- package/_bundled_plugin/skills/vulnerability-writeup/SKILL.md +263 -0
- package/_bundled_plugin/skills/vulnerability-writeup/agents/openai.yaml +4 -0
- package/_bundled_plugin/skills/vulnerability-writeup/references/report-format.md +104 -0
- package/bin/codex-security.mjs +26 -0
- package/dist/api.d.ts +112 -0
- package/dist/api.d.ts.map +1 -0
- package/dist/api.js +1051 -0
- package/dist/api.js.map +1 -0
- package/dist/auth.d.ts +29 -0
- package/dist/auth.d.ts.map +1 -0
- package/dist/auth.js +246 -0
- package/dist/auth.js.map +1 -0
- package/dist/bulk-scan-discovery.d.ts +35 -0
- package/dist/bulk-scan-discovery.d.ts.map +1 -0
- package/dist/bulk-scan-discovery.js +217 -0
- package/dist/bulk-scan-discovery.js.map +1 -0
- package/dist/cli.d.ts +67 -0
- package/dist/cli.d.ts.map +1 -0
- package/dist/cli.js +2128 -0
- package/dist/cli.js.map +1 -0
- package/dist/config.d.ts +19 -0
- package/dist/config.d.ts.map +1 -0
- package/dist/config.js +199 -0
- package/dist/config.js.map +1 -0
- package/dist/contract.d.ts +21 -0
- package/dist/contract.d.ts.map +1 -0
- package/dist/contract.js +848 -0
- package/dist/contract.js.map +1 -0
- package/dist/cost.d.ts +30 -0
- package/dist/cost.d.ts.map +1 -0
- package/dist/cost.js +280 -0
- package/dist/cost.js.map +1 -0
- package/dist/errors.d.ts +38 -0
- package/dist/errors.d.ts.map +1 -0
- package/dist/errors.js +52 -0
- package/dist/errors.js.map +1 -0
- package/dist/index.d.ts +22 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +11 -0
- package/dist/index.js.map +1 -0
- package/dist/knowledge-base.d.ts +7 -0
- package/dist/knowledge-base.d.ts.map +1 -0
- package/dist/knowledge-base.js +181 -0
- package/dist/knowledge-base.js.map +1 -0
- package/dist/models.d.ts +231 -0
- package/dist/models.d.ts.map +1 -0
- package/dist/models.js +3 -0
- package/dist/models.js.map +1 -0
- package/dist/multiscan.d.ts +30 -0
- package/dist/multiscan.d.ts.map +1 -0
- package/dist/multiscan.js +397 -0
- package/dist/multiscan.js.map +1 -0
- package/dist/result.d.ts +39 -0
- package/dist/result.d.ts.map +1 -0
- package/dist/result.js +71 -0
- package/dist/result.js.map +1 -0
- package/dist/runtime.d.ts +69 -0
- package/dist/runtime.d.ts.map +1 -0
- package/dist/runtime.js +997 -0
- package/dist/runtime.js.map +1 -0
- package/dist/scan-comparison.d.ts +42 -0
- package/dist/scan-comparison.d.ts.map +1 -0
- package/dist/scan-comparison.js +153 -0
- package/dist/scan-comparison.js.map +1 -0
- package/dist/scan-history-renderer.d.ts +13 -0
- package/dist/scan-history-renderer.d.ts.map +1 -0
- package/dist/scan-history-renderer.js +297 -0
- package/dist/scan-history-renderer.js.map +1 -0
- package/dist/targets.d.ts +39 -0
- package/dist/targets.d.ts.map +1 -0
- package/dist/targets.js +295 -0
- package/dist/targets.js.map +1 -0
- package/dist/trusted-executable.d.ts +6 -0
- package/dist/trusted-executable.d.ts.map +1 -0
- package/dist/trusted-executable.js +77 -0
- package/dist/trusted-executable.js.map +1 -0
- package/dist/version.d.ts +6 -0
- package/dist/version.d.ts.map +1 -0
- package/dist/version.js +42 -0
- package/dist/version.js.map +1 -0
- package/dist/worker-progress.d.ts +13 -0
- package/dist/worker-progress.d.ts.map +1 -0
- package/dist/worker-progress.js +113 -0
- package/dist/worker-progress.js.map +1 -0
- package/package.json +74 -0
|
@@ -0,0 +1,320 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""Validate and combine security-scan candidates into deterministic JSONL."""
|
|
3
|
+
|
|
4
|
+
from __future__ import annotations
|
|
5
|
+
|
|
6
|
+
import argparse
|
|
7
|
+
import hashlib
|
|
8
|
+
import json
|
|
9
|
+
import re
|
|
10
|
+
import sys
|
|
11
|
+
import tempfile
|
|
12
|
+
from pathlib import Path, PurePosixPath
|
|
13
|
+
from typing import Any
|
|
14
|
+
|
|
15
|
+
CWE = re.compile(r"(?i)CWE-(\d+)")
|
|
16
|
+
ROLES = {
|
|
17
|
+
"entrypoint": 0,
|
|
18
|
+
"entrypoint/wrapper": 1,
|
|
19
|
+
"source": 2,
|
|
20
|
+
"root_control": 3,
|
|
21
|
+
"sink": 4,
|
|
22
|
+
"concrete_implementation": 5,
|
|
23
|
+
"evidence": 6,
|
|
24
|
+
}
|
|
25
|
+
FIELDS = {
|
|
26
|
+
"candidate_id",
|
|
27
|
+
"cwe_ids",
|
|
28
|
+
"locations",
|
|
29
|
+
"summary",
|
|
30
|
+
"evidence",
|
|
31
|
+
"context",
|
|
32
|
+
"instance",
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def text_field(row: dict[str, Any], field: str, *, required: bool = True) -> str | None:
|
|
37
|
+
value = row.get(field)
|
|
38
|
+
if value is None and not required:
|
|
39
|
+
return None
|
|
40
|
+
if not isinstance(value, str) or not value.strip():
|
|
41
|
+
raise ValueError(f"{field}: expected a non-empty string")
|
|
42
|
+
return value.strip()
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def cwe_ids(row: dict[str, Any]) -> list[str]:
|
|
46
|
+
value = row.get("cwe_ids")
|
|
47
|
+
if not isinstance(value, list):
|
|
48
|
+
raise ValueError("cwe_ids: expected an array")
|
|
49
|
+
found: set[int] = set()
|
|
50
|
+
for item in value:
|
|
51
|
+
if not isinstance(item, str):
|
|
52
|
+
raise ValueError("cwe_ids: expected CWE strings")
|
|
53
|
+
match = CWE.fullmatch(item.strip())
|
|
54
|
+
if match is None or int(match[1]) < 1:
|
|
55
|
+
raise ValueError(f"cwe_ids: unsupported value {item!r}")
|
|
56
|
+
found.add(int(match[1]))
|
|
57
|
+
return [f"CWE-{number}" for number in sorted(found)]
|
|
58
|
+
|
|
59
|
+
|
|
60
|
+
def relative_file(value: Any, repo_root: Path) -> tuple[str, Path]:
|
|
61
|
+
if not isinstance(value, str) or not value or "\0" in value:
|
|
62
|
+
raise ValueError("path: expected a non-empty repository-relative path")
|
|
63
|
+
raw = value
|
|
64
|
+
if sys.platform == "win32":
|
|
65
|
+
raw = raw.replace("\\", "/")
|
|
66
|
+
path = PurePosixPath(raw)
|
|
67
|
+
if (
|
|
68
|
+
path.is_absolute()
|
|
69
|
+
or ".." in path.parts
|
|
70
|
+
or (sys.platform == "win32" and re.match(r"^[A-Za-z]:", raw))
|
|
71
|
+
):
|
|
72
|
+
raise ValueError("path: expected a repository-relative path without traversal")
|
|
73
|
+
resolved = (repo_root / raw).resolve(strict=True)
|
|
74
|
+
try:
|
|
75
|
+
relative = resolved.relative_to(repo_root).as_posix()
|
|
76
|
+
except ValueError as error:
|
|
77
|
+
raise ValueError("path: must resolve inside --repo-root") from error
|
|
78
|
+
if not resolved.is_file():
|
|
79
|
+
raise ValueError("path: expected a regular file")
|
|
80
|
+
return relative, resolved
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
def positive_line(value: Any, field: str) -> int:
|
|
84
|
+
if not isinstance(value, int) or isinstance(value, bool) or value < 1:
|
|
85
|
+
raise ValueError(f"{field}: expected a positive integer")
|
|
86
|
+
return value
|
|
87
|
+
|
|
88
|
+
|
|
89
|
+
def normalize_locations(
|
|
90
|
+
row: dict[str, Any], repo_root: Path, line_counts: dict[Path, int]
|
|
91
|
+
) -> list[dict[str, Any]]:
|
|
92
|
+
value = row.get("locations")
|
|
93
|
+
if not isinstance(value, list) or not value:
|
|
94
|
+
raise ValueError("locations: expected a non-empty array")
|
|
95
|
+
normalized: dict[tuple[str, int, int, str], dict[str, Any]] = {}
|
|
96
|
+
for item in value:
|
|
97
|
+
if not isinstance(item, dict):
|
|
98
|
+
raise ValueError("locations: expected location objects")
|
|
99
|
+
unknown = set(item) - {"path", "start_line", "end_line", "role"}
|
|
100
|
+
if unknown:
|
|
101
|
+
raise ValueError(f"locations: unsupported fields {', '.join(sorted(unknown))}")
|
|
102
|
+
relative, source = relative_file(item.get("path"), repo_root)
|
|
103
|
+
if (
|
|
104
|
+
not relative.strip()
|
|
105
|
+
or "\\" in relative
|
|
106
|
+
or any(":" in part for part in PurePosixPath(relative).parts)
|
|
107
|
+
):
|
|
108
|
+
raise ValueError("path: expected a safe repository-relative POSIX path")
|
|
109
|
+
start = positive_line(item.get("start_line"), "start_line")
|
|
110
|
+
end = positive_line(item.get("end_line", start), "end_line")
|
|
111
|
+
if end < start:
|
|
112
|
+
raise ValueError("end_line: must be greater than or equal to start_line")
|
|
113
|
+
if source not in line_counts:
|
|
114
|
+
line_counts[source] = len(source.read_bytes().splitlines())
|
|
115
|
+
if end > line_counts[source]:
|
|
116
|
+
raise ValueError(f"line range {start}-{end} exceeds {relative}:{line_counts[source]}")
|
|
117
|
+
role = item.get("role")
|
|
118
|
+
if not isinstance(role, str) or role not in ROLES:
|
|
119
|
+
raise ValueError(f"role: unsupported value {role!r}")
|
|
120
|
+
key = (relative, start, end, role)
|
|
121
|
+
normalized[key] = {
|
|
122
|
+
"path": relative,
|
|
123
|
+
"start_line": start,
|
|
124
|
+
"end_line": end,
|
|
125
|
+
"role": role,
|
|
126
|
+
}
|
|
127
|
+
return sorted(
|
|
128
|
+
normalized.values(),
|
|
129
|
+
key=lambda item: (
|
|
130
|
+
ROLES[item["role"]],
|
|
131
|
+
item["path"],
|
|
132
|
+
item["start_line"],
|
|
133
|
+
item["end_line"],
|
|
134
|
+
),
|
|
135
|
+
)
|
|
136
|
+
|
|
137
|
+
|
|
138
|
+
def read_scope(path: Path, repo_root: Path) -> set[str]:
|
|
139
|
+
contents = path.read_bytes().decode("utf-8")
|
|
140
|
+
lines = contents.split("\n")
|
|
141
|
+
listed_rows = set(lines)
|
|
142
|
+
|
|
143
|
+
def is_scope_file(value: str) -> bool:
|
|
144
|
+
try:
|
|
145
|
+
relative_file(value, repo_root)
|
|
146
|
+
except (OSError, ValueError):
|
|
147
|
+
return False
|
|
148
|
+
return True
|
|
149
|
+
|
|
150
|
+
carriage_rows = {
|
|
151
|
+
line: (is_scope_file(line), is_scope_file(line.removesuffix("\r")))
|
|
152
|
+
for line in lines
|
|
153
|
+
if line.endswith("\r") and line != "\r" and sys.platform != "win32"
|
|
154
|
+
}
|
|
155
|
+
crlf_evidence = any(line == "\r" for line in lines) or any(
|
|
156
|
+
stripped and not literal for literal, stripped in carriage_rows.values()
|
|
157
|
+
)
|
|
158
|
+
literal_evidence = any(literal and not stripped for literal, stripped in carriage_rows.values())
|
|
159
|
+
|
|
160
|
+
scope: set[str] = set()
|
|
161
|
+
for number, line in enumerate(lines, 1):
|
|
162
|
+
if sys.platform == "win32" or line == "\r":
|
|
163
|
+
line = line.removesuffix("\r")
|
|
164
|
+
elif line.endswith("\r"):
|
|
165
|
+
literal, stripped = carriage_rows[line]
|
|
166
|
+
if stripped and not literal:
|
|
167
|
+
line = line.removesuffix("\r")
|
|
168
|
+
elif stripped and literal:
|
|
169
|
+
if number == len(lines) and not contents.endswith("\n"):
|
|
170
|
+
pass
|
|
171
|
+
elif line.removesuffix("\r") in listed_rows:
|
|
172
|
+
pass
|
|
173
|
+
elif crlf_evidence and not literal_evidence:
|
|
174
|
+
line = line.removesuffix("\r")
|
|
175
|
+
elif literal_evidence and not crlf_evidence:
|
|
176
|
+
pass
|
|
177
|
+
else:
|
|
178
|
+
raise ValueError(f"in-scope file row {number}: ambiguous carriage-return paths")
|
|
179
|
+
elif not literal and crlf_evidence:
|
|
180
|
+
line = line.removesuffix("\r")
|
|
181
|
+
if not line:
|
|
182
|
+
continue
|
|
183
|
+
try:
|
|
184
|
+
relative, _ = relative_file(line, repo_root)
|
|
185
|
+
except (OSError, ValueError) as error:
|
|
186
|
+
raise ValueError(f"in-scope file row {number}: {error}") from error
|
|
187
|
+
scope.add(relative)
|
|
188
|
+
return scope
|
|
189
|
+
|
|
190
|
+
|
|
191
|
+
def normalize_candidate(
|
|
192
|
+
row: dict[str, Any], repo_root: Path, scope: set[str], line_counts: dict[Path, int]
|
|
193
|
+
) -> dict[str, Any]:
|
|
194
|
+
unknown = set(row) - FIELDS
|
|
195
|
+
if unknown:
|
|
196
|
+
raise ValueError(f"unsupported fields {', '.join(sorted(unknown))}")
|
|
197
|
+
if "candidate_id" in row:
|
|
198
|
+
text_field(row, "candidate_id")
|
|
199
|
+
candidate_locations = normalize_locations(row, repo_root, line_counts)
|
|
200
|
+
if not any(item["path"] in scope for item in candidate_locations):
|
|
201
|
+
raise ValueError("locations: expected at least one in-scope file")
|
|
202
|
+
result: dict[str, Any] = {
|
|
203
|
+
"cwe_ids": cwe_ids(row),
|
|
204
|
+
"locations": candidate_locations,
|
|
205
|
+
"summary": text_field(row, "summary"),
|
|
206
|
+
"evidence": text_field(row, "evidence"),
|
|
207
|
+
}
|
|
208
|
+
context = text_field(row, "context", required=False)
|
|
209
|
+
if context is not None:
|
|
210
|
+
result["context"] = context
|
|
211
|
+
instance = text_field(row, "instance", required=False)
|
|
212
|
+
if instance is not None:
|
|
213
|
+
result["instance"] = instance
|
|
214
|
+
return result
|
|
215
|
+
|
|
216
|
+
|
|
217
|
+
def identity(row: dict[str, Any]) -> str:
|
|
218
|
+
return json.dumps(
|
|
219
|
+
{
|
|
220
|
+
"cwe_ids": row["cwe_ids"],
|
|
221
|
+
"locations": row["locations"],
|
|
222
|
+
"instance": row.get("instance"),
|
|
223
|
+
},
|
|
224
|
+
ensure_ascii=False,
|
|
225
|
+
separators=(",", ":"),
|
|
226
|
+
sort_keys=True,
|
|
227
|
+
)
|
|
228
|
+
|
|
229
|
+
|
|
230
|
+
def merged_text(group: list[dict[str, Any]], field: str) -> str:
|
|
231
|
+
return "\n".join(sorted({item[field] for item in group if field in item}))
|
|
232
|
+
|
|
233
|
+
|
|
234
|
+
def combine(rows: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
|
235
|
+
groups: dict[str, list[dict[str, Any]]] = {}
|
|
236
|
+
for row in rows:
|
|
237
|
+
groups.setdefault(identity(row), []).append(row)
|
|
238
|
+
combined: list[dict[str, Any]] = []
|
|
239
|
+
for key, group in sorted(groups.items()):
|
|
240
|
+
candidate_id = hashlib.sha256(key.encode()).hexdigest()[:16]
|
|
241
|
+
|
|
242
|
+
result = {
|
|
243
|
+
"candidate_id": f"candidate-{candidate_id}",
|
|
244
|
+
"cwe_ids": group[0]["cwe_ids"],
|
|
245
|
+
"locations": group[0]["locations"],
|
|
246
|
+
"summary": merged_text(group, "summary"),
|
|
247
|
+
"evidence": merged_text(group, "evidence"),
|
|
248
|
+
}
|
|
249
|
+
context = merged_text(group, "context")
|
|
250
|
+
if context:
|
|
251
|
+
result["context"] = context
|
|
252
|
+
if "instance" in group[0]:
|
|
253
|
+
result["instance"] = group[0]["instance"]
|
|
254
|
+
combined.append(result)
|
|
255
|
+
return combined
|
|
256
|
+
|
|
257
|
+
|
|
258
|
+
def main() -> None:
|
|
259
|
+
parser = argparse.ArgumentParser(description=__doc__)
|
|
260
|
+
parser.add_argument("--input", nargs="+", required=True, help="Candidate JSONL inputs.")
|
|
261
|
+
parser.add_argument("--out", required=True, help="Combined candidate JSONL output.")
|
|
262
|
+
parser.add_argument("--repo-root", required=True, help="Repository root for candidate paths.")
|
|
263
|
+
parser.add_argument("--in-scope-files", required=True, help="Repository-relative file list.")
|
|
264
|
+
args = parser.parse_args()
|
|
265
|
+
try:
|
|
266
|
+
repo_root = Path(args.repo_root).expanduser().resolve(strict=True)
|
|
267
|
+
if not repo_root.is_dir():
|
|
268
|
+
raise ValueError("--repo-root: expected a directory")
|
|
269
|
+
output = Path(args.out).expanduser().resolve(strict=False)
|
|
270
|
+
scope_path = Path(args.in_scope_files).expanduser().resolve(strict=True)
|
|
271
|
+
inputs = sorted({Path(value).expanduser().resolve(strict=True) for value in args.input})
|
|
272
|
+
if output in inputs:
|
|
273
|
+
raise ValueError("--out: must not also be an input")
|
|
274
|
+
if output == scope_path:
|
|
275
|
+
raise ValueError("--out: must not replace --in-scope-files")
|
|
276
|
+
scope = read_scope(scope_path, repo_root)
|
|
277
|
+
line_counts: dict[Path, int] = {}
|
|
278
|
+
rows: list[dict[str, Any]] = []
|
|
279
|
+
for source in inputs:
|
|
280
|
+
with source.open(encoding="utf-8") as handle:
|
|
281
|
+
for number, line in enumerate(handle, 1):
|
|
282
|
+
if not line.strip():
|
|
283
|
+
continue
|
|
284
|
+
try:
|
|
285
|
+
row = json.loads(line)
|
|
286
|
+
if not isinstance(row, dict):
|
|
287
|
+
raise ValueError("expected a JSON object")
|
|
288
|
+
rows.append(normalize_candidate(row, repo_root, scope, line_counts))
|
|
289
|
+
except (ValueError, TypeError, OSError) as error:
|
|
290
|
+
raise ValueError(f"{source} row {number}: {error}") from error
|
|
291
|
+
combined = combine(rows)
|
|
292
|
+
output.parent.mkdir(parents=True, exist_ok=True)
|
|
293
|
+
payload = "".join(
|
|
294
|
+
json.dumps(row, ensure_ascii=False, separators=(",", ":"), sort_keys=True) + "\n"
|
|
295
|
+
for row in combined
|
|
296
|
+
)
|
|
297
|
+
temporary: Path | None = None
|
|
298
|
+
try:
|
|
299
|
+
with tempfile.NamedTemporaryFile(
|
|
300
|
+
mode="w",
|
|
301
|
+
encoding="utf-8",
|
|
302
|
+
dir=output.parent,
|
|
303
|
+
prefix=f".{output.name}.",
|
|
304
|
+
suffix=".tmp",
|
|
305
|
+
delete=False,
|
|
306
|
+
) as handle:
|
|
307
|
+
temporary = Path(handle.name)
|
|
308
|
+
handle.write(payload)
|
|
309
|
+
temporary.replace(output)
|
|
310
|
+
finally:
|
|
311
|
+
if temporary is not None:
|
|
312
|
+
temporary.unlink(missing_ok=True)
|
|
313
|
+
print(f"Combined {len(rows)} candidate rows into {len(combined)} rows in {output}")
|
|
314
|
+
except (OSError, ValueError) as error:
|
|
315
|
+
print(f"normalize_candidates: {error}", file=sys.stderr)
|
|
316
|
+
raise SystemExit(2) from error
|
|
317
|
+
|
|
318
|
+
|
|
319
|
+
if __name__ == "__main__":
|
|
320
|
+
main()
|