@openai/codex-security 0.1.11 → 0.1.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +195 -49
- package/_bundled_plugin/.codex-plugin/plugin.json +1 -1
- package/_bundled_plugin/.mcp.json +2 -0
- package/_bundled_plugin/examples/completed-scan/coverage.json +1 -1
- package/_bundled_plugin/examples/completed-scan/scan-manifest.json +2 -2
- package/_bundled_plugin/mcp/server.mjs.br.part-000 +0 -0
- package/_bundled_plugin/mcp/server.mjs.br.part-001 +0 -0
- package/_bundled_plugin/preflight/capability-profiles.toml +1 -34
- package/_bundled_plugin/references/config-preflight.md +11 -13
- package/_bundled_plugin/references/core-scan.md +94 -0
- package/_bundled_plugin/references/final-report.md +6 -6
- package/_bundled_plugin/references/scan-artifacts.md +15 -14
- package/_bundled_plugin/references/scan-prologue.md +27 -0
- package/_bundled_plugin/schemas/tools/deep-reducer.schema.json +1 -106
- package/_bundled_plugin/scripts/config_preflight.py +40 -3
- package/_bundled_plugin/scripts/deep_scan_config.py +8 -5
- package/_bundled_plugin/scripts/deep_scan_workbench.py +103 -35
- package/_bundled_plugin/scripts/finalize_scan_contract.py +29 -17
- package/_bundled_plugin/scripts/finding_preview.py +122 -6
- package/_bundled_plugin/scripts/generate_in_scope_files.py +39 -17
- package/_bundled_plugin/scripts/generate_rank_input.py +21 -3
- package/_bundled_plugin/scripts/normalize_candidates.py +5 -5
- package/_bundled_plugin/scripts/report_projection.py +1 -19
- package/_bundled_plugin/scripts/workbench_cli.py +4 -0
- package/_bundled_plugin/scripts/workbench_db.py +280 -29
- package/_bundled_plugin/scripts/workbench_progress.py +0 -5
- package/_bundled_plugin/scripts/workbench_scan_usage.py +69 -8
- package/_bundled_plugin/scripts/workbench_schema.py +41 -20
- package/_bundled_plugin/skills/attack-path-analysis/SKILL.md +8 -11
- package/_bundled_plugin/skills/deep-security-scan/SKILL.md +24 -110
- package/_bundled_plugin/skills/security-scan/SKILL.md +5 -88
- package/_bundled_plugin/skills/threat-model/SKILL.md +4 -2
- package/_bundled_plugin/skills/track-findings/SKILL.md +1 -1
- package/_bundled_plugin/skills/triage-finding/SKILL.md +5 -6
- package/_bundled_plugin/skills/triage-finding/references/github-rest-intake.md +6 -7
- package/_bundled_plugin/skills/validation/SKILL.md +9 -12
- package/_bundled_plugin/skills/vulnerability-writeup/SKILL.md +3 -2
- package/_bundled_plugin/skills/vulnerability-writeup/references/report-format.md +1 -1
- package/dist/api.d.ts +1 -0
- package/dist/api.d.ts.map +1 -1
- package/dist/api.js +195 -59
- package/dist/api.js.map +1 -1
- package/dist/bulk-scan-discovery.d.ts +5 -1
- package/dist/bulk-scan-discovery.d.ts.map +1 -1
- package/dist/bulk-scan-discovery.js +38 -9
- package/dist/bulk-scan-discovery.js.map +1 -1
- package/dist/cli.d.ts +9 -3
- package/dist/cli.d.ts.map +1 -1
- package/dist/cli.js +764 -60
- package/dist/cli.js.map +1 -1
- package/dist/config.d.ts.map +1 -1
- package/dist/config.js +1 -0
- package/dist/config.js.map +1 -1
- package/dist/contract.d.ts +12 -0
- package/dist/contract.d.ts.map +1 -1
- package/dist/contract.js +30 -2
- package/dist/contract.js.map +1 -1
- package/dist/cost.d.ts.map +1 -1
- package/dist/cost.js +10 -1
- package/dist/cost.js.map +1 -1
- package/dist/index.d.ts +2 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1 -0
- package/dist/index.js.map +1 -1
- package/dist/knowledge-base.d.ts.map +1 -1
- package/dist/knowledge-base.js +10 -3
- package/dist/knowledge-base.js.map +1 -1
- package/dist/multiscan.d.ts +1 -0
- package/dist/multiscan.d.ts.map +1 -1
- package/dist/multiscan.js +77 -18
- package/dist/multiscan.js.map +1 -1
- package/dist/publication-events.d.ts +16 -0
- package/dist/publication-events.d.ts.map +1 -0
- package/dist/publication-events.js +141 -0
- package/dist/publication-events.js.map +1 -0
- package/dist/publication-store.d.ts +5 -0
- package/dist/publication-store.d.ts.map +1 -0
- package/dist/publication-store.js +102 -0
- package/dist/publication-store.js.map +1 -0
- package/dist/publication.d.ts +27 -0
- package/dist/publication.d.ts.map +1 -0
- package/dist/publication.js +207 -0
- package/dist/publication.js.map +1 -0
- package/dist/publish.d.ts +77 -0
- package/dist/publish.d.ts.map +1 -0
- package/dist/publish.js +748 -0
- package/dist/publish.js.map +1 -0
- package/dist/runtime.d.ts.map +1 -1
- package/dist/runtime.js +15 -44
- package/dist/runtime.js.map +1 -1
- package/dist/scan-dashboard.d.ts +7 -1
- package/dist/scan-dashboard.d.ts.map +1 -1
- package/dist/scan-dashboard.js +86 -20
- package/dist/scan-dashboard.js.map +1 -1
- package/dist/scan-logs.d.ts +2 -0
- package/dist/scan-logs.d.ts.map +1 -1
- package/dist/scan-logs.js +97 -37
- package/dist/scan-logs.js.map +1 -1
- package/dist/targets.d.ts.map +1 -1
- package/dist/targets.js +20 -11
- package/dist/targets.js.map +1 -1
- package/dist/trusted-executable.js +1 -1
- package/dist/trusted-executable.js.map +1 -1
- package/dist/version.d.ts +3 -3
- package/dist/version.d.ts.map +1 -1
- package/dist/version.js +6 -4
- package/dist/version.js.map +1 -1
- package/package.json +5 -4
- package/_bundled_plugin/references/shared-hard-rules.md +0 -15
- package/_bundled_plugin/scripts/validate_report_format.py +0 -138
- package/_bundled_plugin/skills/security-scan/references/repository-wide-scan.md +0 -33
|
@@ -17,6 +17,7 @@ from typing import Any, Callable
|
|
|
17
17
|
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
|
18
18
|
from deep_scan_config import resolve_deep_scan_config
|
|
19
19
|
from filesystem_identity import serialize_filesystem_identity
|
|
20
|
+
from finalize_scan_contract import _read_scan_local_json
|
|
20
21
|
from workbench.handoff import require_current_continuation
|
|
21
22
|
from workbench_target import (
|
|
22
23
|
directory_content_digest,
|
|
@@ -358,31 +359,25 @@ def deep_scan_state(connection: sqlite3.Connection, scan_id: str) -> dict[str, A
|
|
|
358
359
|
""",
|
|
359
360
|
(run["scan_id"],),
|
|
360
361
|
)
|
|
361
|
-
successful_reducer = connection.execute(
|
|
362
|
-
"""
|
|
363
|
-
SELECT 1 FROM deep_scan_workers
|
|
364
|
-
WHERE scan_id = ? AND kind = 'dedup' AND status = 'succeeded'
|
|
365
|
-
LIMIT 1
|
|
366
|
-
""",
|
|
367
|
-
(run["scan_id"],),
|
|
368
|
-
).fetchone()
|
|
369
362
|
canonical_artifacts = None
|
|
370
|
-
if
|
|
371
|
-
|
|
372
|
-
|
|
373
|
-
|
|
374
|
-
|
|
375
|
-
|
|
363
|
+
if (
|
|
364
|
+
run["canonical_inventory_path"] is None
|
|
365
|
+
and run["status"] == "succeeded"
|
|
366
|
+
and run["manifest_path"] is not None
|
|
367
|
+
and run["manifest_path"] != str(Path(scan["scan_dir"]) / "scan-manifest.json")
|
|
368
|
+
and (Path(scan["scan_dir"]) / "artifacts" / "02_discovery" / "in_scope_files.txt").exists()
|
|
369
|
+
):
|
|
370
|
+
canonical_artifacts = canonical_discovery_artifacts(scan)
|
|
371
|
+
if (
|
|
372
|
+
run["terminal_reason"] == "capped"
|
|
376
373
|
and run["completion_sequence"] == 0
|
|
377
374
|
and deep_scan_deadline_reached(run)
|
|
375
|
+
and Path(canonical_artifacts["candidateLedgerPath"]).stat().st_size != 0
|
|
378
376
|
):
|
|
379
|
-
|
|
380
|
-
|
|
381
|
-
|
|
382
|
-
|
|
383
|
-
"candidate ledger."
|
|
384
|
-
)
|
|
385
|
-
canonical_artifacts = candidate_artifacts
|
|
377
|
+
raise SystemExit(
|
|
378
|
+
"A capped Deep Scan without completed discoveries requires an empty "
|
|
379
|
+
"candidate ledger."
|
|
380
|
+
)
|
|
386
381
|
return {
|
|
387
382
|
"scanId": run["scan_id"],
|
|
388
383
|
"targetPath": scan["target_path"],
|
|
@@ -1549,7 +1544,6 @@ def commit_deep_scan_dedup_locked(
|
|
|
1549
1544
|
"Canonical candidate ledger path",
|
|
1550
1545
|
)
|
|
1551
1546
|
else:
|
|
1552
|
-
canonical_discovery_artifacts(scan)
|
|
1553
1547
|
candidate_ledger_path = None
|
|
1554
1548
|
canonical_candidate_ledger_path = None
|
|
1555
1549
|
result_manifest_path = deep_scan_path(
|
|
@@ -1657,6 +1651,65 @@ def finish_deep_scan_locked(
|
|
|
1657
1651
|
scan, args.manifest_path, "Deep Scan coordinator manifest path", kind="file"
|
|
1658
1652
|
)
|
|
1659
1653
|
)
|
|
1654
|
+
standard_scan_manifest = manifest_path == str(Path(scan["scan_dir"]) / "scan-manifest.json")
|
|
1655
|
+
|
|
1656
|
+
failure_capped = False
|
|
1657
|
+
if (
|
|
1658
|
+
standard_scan_manifest
|
|
1659
|
+
and args.terminal_reason == "capped"
|
|
1660
|
+
and (run["status"] == "running" or omitted_worker_ids)
|
|
1661
|
+
):
|
|
1662
|
+
for artifact_name in ("scan-manifest.json", "findings.json", "coverage.json"):
|
|
1663
|
+
deep_scan_path(
|
|
1664
|
+
scan,
|
|
1665
|
+
str(Path(scan["scan_dir"]) / artifact_name),
|
|
1666
|
+
f"Canonical parent {artifact_name}",
|
|
1667
|
+
kind="file",
|
|
1668
|
+
)
|
|
1669
|
+
coverage = _read_scan_local_json(
|
|
1670
|
+
Path(scan["scan_dir"]), "coverage.json", "Canonical parent coverage.json"
|
|
1671
|
+
)
|
|
1672
|
+
deferred = coverage.get("deferred")
|
|
1673
|
+
failure_capped = (
|
|
1674
|
+
coverage.get("completeness") == "partial"
|
|
1675
|
+
and isinstance(deferred, list)
|
|
1676
|
+
and any(
|
|
1677
|
+
isinstance(item, dict)
|
|
1678
|
+
and isinstance(item.get("reason"), str)
|
|
1679
|
+
and item["reason"].startswith("Deep Scan stopped before completion: ")
|
|
1680
|
+
for item in deferred
|
|
1681
|
+
)
|
|
1682
|
+
and connection.execute(
|
|
1683
|
+
"""
|
|
1684
|
+
SELECT 1 FROM deep_scan_workers
|
|
1685
|
+
WHERE scan_id = ? AND kind = 'dedup' AND status = 'succeeded'
|
|
1686
|
+
LIMIT 1
|
|
1687
|
+
""",
|
|
1688
|
+
(scan_id,),
|
|
1689
|
+
).fetchone()
|
|
1690
|
+
is not None
|
|
1691
|
+
)
|
|
1692
|
+
if failure_capped and run["status"] == "running":
|
|
1693
|
+
require_running_deep_scan(connection, scan_id)
|
|
1694
|
+
connection.execute(
|
|
1695
|
+
"""
|
|
1696
|
+
UPDATE deep_scan_workers
|
|
1697
|
+
SET merge_state = 'buffered', updated_at = ?
|
|
1698
|
+
WHERE scan_id = ? AND kind = 'discovery' AND status = 'succeeded'
|
|
1699
|
+
AND merge_state = 'merging'
|
|
1700
|
+
AND id IN (
|
|
1701
|
+
SELECT inputs.discovery_worker_id
|
|
1702
|
+
FROM deep_scan_dedup_inputs AS inputs
|
|
1703
|
+
JOIN deep_scan_workers AS reducers
|
|
1704
|
+
ON reducers.id = inputs.dedup_worker_id
|
|
1705
|
+
AND reducers.scan_id = inputs.scan_id
|
|
1706
|
+
WHERE inputs.scan_id = ?
|
|
1707
|
+
AND reducers.kind = 'dedup'
|
|
1708
|
+
AND reducers.status IN ('failed', 'canceled')
|
|
1709
|
+
)
|
|
1710
|
+
""",
|
|
1711
|
+
(now(), scan_id, scan_id),
|
|
1712
|
+
)
|
|
1660
1713
|
buffered_worker_ids = [
|
|
1661
1714
|
row["id"]
|
|
1662
1715
|
for row in connection.execute(
|
|
@@ -1671,7 +1724,7 @@ def finish_deep_scan_locked(
|
|
|
1671
1724
|
]
|
|
1672
1725
|
omissions_match = (
|
|
1673
1726
|
set(omitted_worker_ids) == set(buffered_worker_ids)
|
|
1674
|
-
if args.terminal_reason == "saturated"
|
|
1727
|
+
if args.terminal_reason == "saturated" or failure_capped
|
|
1675
1728
|
else not omitted_worker_ids and not buffered_worker_ids
|
|
1676
1729
|
)
|
|
1677
1730
|
if run["status"] == "succeeded":
|
|
@@ -1703,16 +1756,27 @@ def finish_deep_scan_locked(
|
|
|
1703
1756
|
args.terminal_reason == "capped"
|
|
1704
1757
|
and run["discovery_runs_dispatched"] < run["max_discovery_runs"]
|
|
1705
1758
|
and not deep_scan_deadline_reached(run)
|
|
1759
|
+
and not failure_capped
|
|
1706
1760
|
):
|
|
1707
1761
|
raise SystemExit(
|
|
1708
1762
|
"Deep Scan cannot finish capped before reaching its configured maximum."
|
|
1709
1763
|
)
|
|
1710
|
-
|
|
1711
|
-
|
|
1712
|
-
|
|
1713
|
-
|
|
1714
|
-
|
|
1715
|
-
|
|
1764
|
+
canonical_artifacts = None
|
|
1765
|
+
if standard_scan_manifest:
|
|
1766
|
+
for artifact_name in ("scan-manifest.json", "findings.json", "coverage.json"):
|
|
1767
|
+
deep_scan_path(
|
|
1768
|
+
scan,
|
|
1769
|
+
str(Path(scan["scan_dir"]) / artifact_name),
|
|
1770
|
+
f"Canonical parent {artifact_name}",
|
|
1771
|
+
kind="file",
|
|
1772
|
+
)
|
|
1773
|
+
else:
|
|
1774
|
+
try:
|
|
1775
|
+
canonical_artifacts = canonical_discovery_artifacts(scan)
|
|
1776
|
+
except SystemExit as exc:
|
|
1777
|
+
raise SystemExit(
|
|
1778
|
+
f"Deep Scan cannot finish without canonical discovery artifacts: {exc}"
|
|
1779
|
+
) from exc
|
|
1716
1780
|
successful_reducer = connection.execute(
|
|
1717
1781
|
"""
|
|
1718
1782
|
SELECT 1 FROM deep_scan_workers
|
|
@@ -1725,7 +1789,11 @@ def finish_deep_scan_locked(
|
|
|
1725
1789
|
args.terminal_reason == "capped"
|
|
1726
1790
|
and deep_scan_deadline_reached(run)
|
|
1727
1791
|
and run["completion_sequence"] == 0
|
|
1728
|
-
and
|
|
1792
|
+
and (
|
|
1793
|
+
standard_scan_manifest
|
|
1794
|
+
or canonical_artifacts is not None
|
|
1795
|
+
and Path(canonical_artifacts["candidateLedgerPath"]).stat().st_size == 0
|
|
1796
|
+
)
|
|
1729
1797
|
)
|
|
1730
1798
|
if successful_reducer is None and not zero_discovery_deadline:
|
|
1731
1799
|
raise SystemExit("Deep Scan cannot finish without a successful dedup worker.")
|
|
@@ -1760,7 +1828,7 @@ def finish_deep_scan_locked(
|
|
|
1760
1828
|
""",
|
|
1761
1829
|
(scan_id,),
|
|
1762
1830
|
).fetchone()
|
|
1763
|
-
if failed_worker is not None:
|
|
1831
|
+
if failed_worker is not None and not failure_capped:
|
|
1764
1832
|
raise SystemExit("Deep Scan cannot finish after a worker has failed.")
|
|
1765
1833
|
active_worker = connection.execute(
|
|
1766
1834
|
"""
|
|
@@ -1782,15 +1850,15 @@ def finish_deep_scan_locked(
|
|
|
1782
1850
|
).fetchone()
|
|
1783
1851
|
if merging_worker is not None:
|
|
1784
1852
|
raise SystemExit("Deep Scan cannot finish while discovery output is merging.")
|
|
1785
|
-
if args.terminal_reason == "capped" and omitted_worker_ids:
|
|
1853
|
+
if args.terminal_reason == "capped" and omitted_worker_ids and not failure_capped:
|
|
1786
1854
|
raise SystemExit("Deep Scan capped completion cannot declare omitted buffered workers.")
|
|
1787
|
-
if args.terminal_reason == "capped" and buffered_worker_ids:
|
|
1855
|
+
if args.terminal_reason == "capped" and buffered_worker_ids and not failure_capped:
|
|
1788
1856
|
raise SystemExit(
|
|
1789
1857
|
"Deep Scan cannot finish capped while discovery output remains buffered."
|
|
1790
1858
|
)
|
|
1791
|
-
if args.terminal_reason == "saturated" and not omissions_match:
|
|
1859
|
+
if (args.terminal_reason == "saturated" or failure_capped) and not omissions_match:
|
|
1792
1860
|
raise SystemExit(
|
|
1793
|
-
"Deep Scan
|
|
1861
|
+
f"Deep Scan {args.terminal_reason} completion must exactly identify all buffered discovery "
|
|
1794
1862
|
"workers with --omitted-worker-id."
|
|
1795
1863
|
)
|
|
1796
1864
|
if args.staged_manifest_path:
|
|
@@ -17,6 +17,7 @@ import re
|
|
|
17
17
|
import secrets
|
|
18
18
|
import stat
|
|
19
19
|
import sys
|
|
20
|
+
import time
|
|
20
21
|
from collections.abc import Iterator
|
|
21
22
|
from datetime import datetime, timezone
|
|
22
23
|
from pathlib import Path, PurePosixPath
|
|
@@ -71,6 +72,10 @@ class ContractError(ValueError):
|
|
|
71
72
|
"""Raised when a completed scan does not satisfy the additive contract."""
|
|
72
73
|
|
|
73
74
|
|
|
75
|
+
class RecoverableContractError(ContractError):
|
|
76
|
+
"""Raised when report projection can safely be retried before publication."""
|
|
77
|
+
|
|
78
|
+
|
|
74
79
|
def _reject_non_finite_json(value: str) -> None:
|
|
75
80
|
raise ValueError(f"non-finite JSON number {value!r} is not supported")
|
|
76
81
|
|
|
@@ -94,11 +99,6 @@ def _read_json(path: Path) -> dict[str, Any]:
|
|
|
94
99
|
return payload
|
|
95
100
|
|
|
96
101
|
|
|
97
|
-
def _write_json(path: Path, payload: Any) -> None:
|
|
98
|
-
path.parent.mkdir(parents=True, exist_ok=True)
|
|
99
|
-
path.write_bytes(_json_bytes(payload))
|
|
100
|
-
|
|
101
|
-
|
|
102
102
|
def _generate_report_projection(
|
|
103
103
|
manifest: dict[str, Any],
|
|
104
104
|
findings: dict[str, Any],
|
|
@@ -110,10 +110,21 @@ def _generate_report_projection(
|
|
|
110
110
|
raise ContractError(f"could not load report projection helper: {script}")
|
|
111
111
|
module = importlib.util.module_from_spec(spec)
|
|
112
112
|
spec.loader.exec_module(module)
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
113
|
+
attempts = (
|
|
114
|
+
getattr(sys.modules.get("workbench_constants"), "SQLITE_RETRY_ATTEMPTS", 1)
|
|
115
|
+
if coverage.get("mode") == "deep_repository"
|
|
116
|
+
else 1
|
|
117
|
+
)
|
|
118
|
+
for attempt in range(attempts):
|
|
119
|
+
try:
|
|
120
|
+
return module.generate_report_markdown(manifest, findings, coverage)
|
|
121
|
+
except OSError as exc:
|
|
122
|
+
if attempt == attempts - 1:
|
|
123
|
+
raise RecoverableContractError(f"report projection failed: {exc}") from exc
|
|
124
|
+
time.sleep(0.05 * (2**attempt))
|
|
125
|
+
except ValueError as exc:
|
|
126
|
+
raise ContractError(f"report projection failed: {exc}") from exc
|
|
127
|
+
raise AssertionError("Report projection retry loop exhausted unexpectedly.")
|
|
117
128
|
|
|
118
129
|
|
|
119
130
|
def _validate_report_output_paths(scan_dir: Path) -> None:
|
|
@@ -178,14 +189,6 @@ def _sha256_bytes(value: bytes) -> str:
|
|
|
178
189
|
return hashlib.sha256(value).hexdigest()
|
|
179
190
|
|
|
180
191
|
|
|
181
|
-
def _sha256_file(path: Path) -> str:
|
|
182
|
-
digest = hashlib.sha256()
|
|
183
|
-
with path.open("rb") as handle:
|
|
184
|
-
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
|
|
185
|
-
digest.update(chunk)
|
|
186
|
-
return digest.hexdigest()
|
|
187
|
-
|
|
188
|
-
|
|
189
192
|
def _require_dict(payload: dict[str, Any], key: str, context: str) -> dict[str, Any]:
|
|
190
193
|
value = payload.get(key)
|
|
191
194
|
if not isinstance(value, dict):
|
|
@@ -858,6 +861,15 @@ def _recover_unsealed_coverage(
|
|
|
858
861
|
partial = completeness not in ("complete", "partial", "unknown")
|
|
859
862
|
if partial:
|
|
860
863
|
warnings.append("Recovered malformed coverage completeness; marked coverage as partial.")
|
|
864
|
+
if (
|
|
865
|
+
coverage.get("mode") == "deep_repository"
|
|
866
|
+
and coverage.get("inventoryStrategy") != "repository"
|
|
867
|
+
):
|
|
868
|
+
coverage["inventoryStrategy"] = "repository"
|
|
869
|
+
warnings.append(
|
|
870
|
+
"Recovered malformed Deep Scan inventory strategy; marked coverage as partial."
|
|
871
|
+
)
|
|
872
|
+
partial = True
|
|
861
873
|
|
|
862
874
|
surface_ids: set[str] = set()
|
|
863
875
|
for field, label in (
|
|
@@ -111,8 +111,14 @@ def bounded_finding_details(value: Any) -> dict[str, Any]:
|
|
|
111
111
|
):
|
|
112
112
|
key = next((alias for alias in aliases if alias in value), None)
|
|
113
113
|
if key is not None:
|
|
114
|
+
section = value[key]
|
|
115
|
+
if key == "attackPath" and isinstance(section, dict):
|
|
116
|
+
section = dict(section)
|
|
117
|
+
for assessment in ("impact", "likelihood"):
|
|
118
|
+
if isinstance(section.get(assessment), str):
|
|
119
|
+
section[assessment] = {"level": section[assessment]}
|
|
114
120
|
prepared[key] = bounded_finding_section(
|
|
115
|
-
|
|
121
|
+
section,
|
|
116
122
|
maximum_bytes,
|
|
117
123
|
priority_keys,
|
|
118
124
|
reserved_fields,
|
|
@@ -140,6 +146,8 @@ def bounded_finding_details(value: Any) -> dict[str, Any]:
|
|
|
140
146
|
"severity",
|
|
141
147
|
"status",
|
|
142
148
|
"taxonomy",
|
|
149
|
+
"preventiveControls",
|
|
150
|
+
"remediationTests",
|
|
143
151
|
):
|
|
144
152
|
if key in value:
|
|
145
153
|
prepared[key] = (
|
|
@@ -148,8 +156,64 @@ def bounded_finding_details(value: Any) -> dict[str, Any]:
|
|
|
148
156
|
else value[key]
|
|
149
157
|
)
|
|
150
158
|
|
|
151
|
-
|
|
152
|
-
|
|
159
|
+
guidance = {
|
|
160
|
+
key: prepared[key]
|
|
161
|
+
for key in ("remediationTests", "preventiveControls")
|
|
162
|
+
if key in prepared and isinstance(prepared[key], list)
|
|
163
|
+
}
|
|
164
|
+
diagnostics = (
|
|
165
|
+
"rootCause",
|
|
166
|
+
"root_cause",
|
|
167
|
+
"validation",
|
|
168
|
+
"attackPath",
|
|
169
|
+
"codeEvidence",
|
|
170
|
+
"code_evidence",
|
|
171
|
+
)
|
|
172
|
+
core_keys = (
|
|
173
|
+
"writeup",
|
|
174
|
+
*diagnostics,
|
|
175
|
+
"confidence",
|
|
176
|
+
"detectedAt",
|
|
177
|
+
"identity",
|
|
178
|
+
"provenance",
|
|
179
|
+
"ruleId",
|
|
180
|
+
"severity",
|
|
181
|
+
"status",
|
|
182
|
+
"taxonomy",
|
|
183
|
+
"evidence",
|
|
184
|
+
"evidenceExcerpt",
|
|
185
|
+
)
|
|
186
|
+
core = {key: prepared[key] for key in core_keys if key in prepared}
|
|
187
|
+
extras = {
|
|
188
|
+
key: item
|
|
189
|
+
for key, item in prepared.items()
|
|
190
|
+
if key not in core and key not in guidance
|
|
191
|
+
}
|
|
192
|
+
complete_guidance = {key: items[:1] for key, items in guidance.items()}
|
|
193
|
+
minimum_guidance = {
|
|
194
|
+
key: [items[0][:1]] if items and isinstance(items[0], str) else []
|
|
195
|
+
for key, items in guidance.items()
|
|
196
|
+
}
|
|
197
|
+
projected_core = {}
|
|
198
|
+
for selected_guidance in (complete_guidance, minimum_guidance):
|
|
199
|
+
reserved = (
|
|
200
|
+
len(json.dumps(selected_guidance, separators=(",", ":")).encode("utf-8")) - 1
|
|
201
|
+
if selected_guidance
|
|
202
|
+
else 0
|
|
203
|
+
)
|
|
204
|
+
if reserved >= FINDING_DETAILS_PREVIEW_BYTES:
|
|
205
|
+
continue
|
|
206
|
+
projected_core = bounded_json_value(
|
|
207
|
+
core,
|
|
208
|
+
[FINDING_DETAILS_PREVIEW_BYTES - reserved],
|
|
209
|
+
)
|
|
210
|
+
if all(key in projected_core for key in core):
|
|
211
|
+
break
|
|
212
|
+
ordered_guidance = dict(sorted(guidance.items(), key=lambda entry: bool(entry[1])))
|
|
213
|
+
bounded = bounded_json_value(
|
|
214
|
+
{**projected_core, **ordered_guidance, **extras},
|
|
215
|
+
[FINDING_DETAILS_PREVIEW_BYTES],
|
|
216
|
+
)
|
|
153
217
|
return bounded if isinstance(bounded, dict) else {}
|
|
154
218
|
|
|
155
219
|
|
|
@@ -215,11 +279,20 @@ def bounded_json_value(value: Any, budget: list[int], *, depth: int = 0) -> Any:
|
|
|
215
279
|
if not consume_json_budget(budget, 2):
|
|
216
280
|
return []
|
|
217
281
|
result = []
|
|
218
|
-
for item in value
|
|
282
|
+
for item in value:
|
|
283
|
+
remaining = budget[0]
|
|
219
284
|
separator = 0 if not result else 1
|
|
220
285
|
if not consume_json_budget(budget, separator):
|
|
221
286
|
break
|
|
222
|
-
|
|
287
|
+
bounded_item = bounded_json_value(item, budget, depth=depth + 1)
|
|
288
|
+
size = len(json.dumps(bounded_item, separators=(",", ":")).encode("utf-8"))
|
|
289
|
+
if separator + size > remaining or (
|
|
290
|
+
isinstance(item, str) and item and bounded_item == ""
|
|
291
|
+
):
|
|
292
|
+
budget[0] = remaining
|
|
293
|
+
break
|
|
294
|
+
budget[0] = remaining - separator - size
|
|
295
|
+
result.append(bounded_item)
|
|
223
296
|
return result
|
|
224
297
|
if isinstance(value, dict):
|
|
225
298
|
if not consume_json_budget(budget, 2):
|
|
@@ -228,13 +301,56 @@ def bounded_json_value(value: Any, budget: list[int], *, depth: int = 0) -> Any:
|
|
|
228
301
|
for key, item in list(value.items())[:20]:
|
|
229
302
|
if budget[0] <= 0 or not isinstance(key, str):
|
|
230
303
|
break
|
|
304
|
+
remaining = budget[0]
|
|
231
305
|
separator = 0 if not result else 1
|
|
232
306
|
if not consume_json_budget(budget, separator):
|
|
307
|
+
budget[0] = remaining
|
|
233
308
|
break
|
|
234
309
|
bounded_key, key_size = bounded_json_text(key, min(budget[0], 512))
|
|
235
310
|
if not consume_json_budget(budget, key_size + 1):
|
|
311
|
+
budget[0] = remaining
|
|
312
|
+
break
|
|
313
|
+
item_budget = budget
|
|
314
|
+
if depth == 0 and key == "remediationTests":
|
|
315
|
+
controls = value.get("preventiveControls")
|
|
316
|
+
if (
|
|
317
|
+
isinstance(item, list)
|
|
318
|
+
and item
|
|
319
|
+
and isinstance(item[0], str)
|
|
320
|
+
and item[0]
|
|
321
|
+
and isinstance(controls, list)
|
|
322
|
+
and controls
|
|
323
|
+
and isinstance(controls[0], str)
|
|
324
|
+
and controls[0]
|
|
325
|
+
):
|
|
326
|
+
minimum_tests = len(
|
|
327
|
+
json.dumps([item[0][0]], separators=(",", ":")).encode("utf-8")
|
|
328
|
+
)
|
|
329
|
+
for control in (controls[0], controls[0][0]):
|
|
330
|
+
reserved = (
|
|
331
|
+
len(
|
|
332
|
+
json.dumps(
|
|
333
|
+
{"preventiveControls": [control]},
|
|
334
|
+
separators=(",", ":"),
|
|
335
|
+
).encode("utf-8")
|
|
336
|
+
)
|
|
337
|
+
- 1
|
|
338
|
+
)
|
|
339
|
+
if budget[0] >= minimum_tests + reserved:
|
|
340
|
+
item_budget = [budget[0] - reserved]
|
|
341
|
+
break
|
|
342
|
+
bounded_item = bounded_json_value(item, item_budget, depth=depth + 1)
|
|
343
|
+
size = (
|
|
344
|
+
separator
|
|
345
|
+
+ key_size
|
|
346
|
+
+ 1
|
|
347
|
+
+ len(json.dumps(bounded_item, separators=(",", ":")).encode("utf-8"))
|
|
348
|
+
)
|
|
349
|
+
if size > remaining or (isinstance(item, str) and item and bounded_item == ""):
|
|
350
|
+
budget[0] = remaining
|
|
236
351
|
break
|
|
237
|
-
|
|
352
|
+
budget[0] = remaining - size
|
|
353
|
+
result[bounded_key] = bounded_item
|
|
238
354
|
return result
|
|
239
355
|
consume_json_budget(budget, 4)
|
|
240
356
|
return None
|
|
@@ -4,6 +4,7 @@
|
|
|
4
4
|
from __future__ import annotations
|
|
5
5
|
|
|
6
6
|
import argparse
|
|
7
|
+
import os
|
|
7
8
|
import subprocess
|
|
8
9
|
import sys
|
|
9
10
|
import tempfile
|
|
@@ -105,6 +106,38 @@ def generate_in_scope_files(repository: Path, scope: str, output: Path) -> int:
|
|
|
105
106
|
return write_inventory(output, rows)
|
|
106
107
|
|
|
107
108
|
|
|
109
|
+
def committed_changed_paths(repository: Path, base: str, head: str) -> list[tuple[Path, str]]:
|
|
110
|
+
result = subprocess.run(
|
|
111
|
+
[
|
|
112
|
+
"git",
|
|
113
|
+
"-C",
|
|
114
|
+
str(repository),
|
|
115
|
+
"diff",
|
|
116
|
+
"--raw",
|
|
117
|
+
"-z",
|
|
118
|
+
"--diff-filter=ACMRD",
|
|
119
|
+
f"{base}..{head}",
|
|
120
|
+
],
|
|
121
|
+
capture_output=True,
|
|
122
|
+
check=True,
|
|
123
|
+
)
|
|
124
|
+
fields = result.stdout.split(b"\0")
|
|
125
|
+
changed: list[tuple[Path, str]] = []
|
|
126
|
+
index = 0
|
|
127
|
+
while index < len(fields) - 1:
|
|
128
|
+
metadata = fields[index].split()
|
|
129
|
+
status = chr(metadata[-1][0])
|
|
130
|
+
index += 1
|
|
131
|
+
if status in {"C", "R"}:
|
|
132
|
+
index += 1
|
|
133
|
+
path = os.fsdecode(fields[index])
|
|
134
|
+
index += 1
|
|
135
|
+
selected_mode = metadata[0].removeprefix(b":") if status == "D" else metadata[1]
|
|
136
|
+
if selected_mode != b"120000":
|
|
137
|
+
changed.append((repository / path, status))
|
|
138
|
+
return changed
|
|
139
|
+
|
|
140
|
+
|
|
108
141
|
def generate_diff_in_scope_files(
|
|
109
142
|
repository: Path,
|
|
110
143
|
base: str,
|
|
@@ -114,7 +147,7 @@ def generate_diff_in_scope_files(
|
|
|
114
147
|
) -> int:
|
|
115
148
|
"""Reuse the existing diff selection without generating previews or duplicate worklists."""
|
|
116
149
|
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
|
117
|
-
from generate_rank_input import git_changed_paths, path_is_excluded
|
|
150
|
+
from generate_rank_input import git_changed_paths, path_is_excluded
|
|
118
151
|
from rank_preview import (
|
|
119
152
|
DEFAULT_PREVIEW_BYTES,
|
|
120
153
|
TEXT_CODE_EXTENSIONS,
|
|
@@ -125,22 +158,11 @@ def generate_diff_in_scope_files(
|
|
|
125
158
|
|
|
126
159
|
rows: list[bytes] = []
|
|
127
160
|
try:
|
|
128
|
-
|
|
129
|
-
|
|
130
|
-
|
|
131
|
-
|
|
132
|
-
|
|
133
|
-
text=True,
|
|
134
|
-
check=True,
|
|
135
|
-
)
|
|
136
|
-
changed.extend(
|
|
137
|
-
(repository / relative, "A")
|
|
138
|
-
for relative in untracked.stdout.split("\0")
|
|
139
|
-
if relative
|
|
140
|
-
)
|
|
141
|
-
else:
|
|
142
|
-
changed = git_changed_paths(repository, base, head, mode)
|
|
143
|
-
|
|
161
|
+
changed = (
|
|
162
|
+
committed_changed_paths(repository, base, head)
|
|
163
|
+
if mode == "revisions"
|
|
164
|
+
else git_changed_paths(repository, base, head, mode)
|
|
165
|
+
)
|
|
144
166
|
eligible = [
|
|
145
167
|
(path, status)
|
|
146
168
|
for path, status in changed
|
|
@@ -646,8 +646,19 @@ def git_changed_paths(repo: Path, base: str, head: str, mode: str) -> list[tuple
|
|
|
646
646
|
if mode == "local-patch":
|
|
647
647
|
unstaged = run_git_changed_paths(repo, [base])
|
|
648
648
|
staged = run_git_changed_paths(repo, ["--cached", base])
|
|
649
|
+
untracked = subprocess.run(
|
|
650
|
+
["git", "-C", str(repo), "ls-files", "--others", "--exclude-standard", "-z"],
|
|
651
|
+
capture_output=True,
|
|
652
|
+
text=True,
|
|
653
|
+
check=True,
|
|
654
|
+
)
|
|
649
655
|
combined = dict(staged)
|
|
650
656
|
combined.update(unstaged)
|
|
657
|
+
combined.update(
|
|
658
|
+
(repo / relative, "A")
|
|
659
|
+
for relative in untracked.stdout.split("\0")
|
|
660
|
+
if relative
|
|
661
|
+
)
|
|
651
662
|
return sorted(combined.items())
|
|
652
663
|
raise SystemExit(f"Unknown diff mode: {mode}")
|
|
653
664
|
|
|
@@ -693,10 +704,17 @@ def make_diff_rank_input(args: argparse.Namespace) -> None:
|
|
|
693
704
|
preview, is_binary = preview_for_bytes(rel, content, args.preview_bytes)
|
|
694
705
|
if is_binary:
|
|
695
706
|
continue
|
|
707
|
+
elif path.is_symlink():
|
|
708
|
+
preview = ""
|
|
696
709
|
elif path.is_file():
|
|
697
|
-
|
|
698
|
-
|
|
699
|
-
|
|
710
|
+
try:
|
|
711
|
+
path.resolve(strict=True).relative_to(repo)
|
|
712
|
+
except (OSError, ValueError):
|
|
713
|
+
preview = ""
|
|
714
|
+
else:
|
|
715
|
+
preview, is_binary = preview_for(path, args.preview_bytes)
|
|
716
|
+
if is_binary:
|
|
717
|
+
continue
|
|
700
718
|
else:
|
|
701
719
|
preview = ""
|
|
702
720
|
rows.append({"path": rel.as_posix(), "area": args.area, "preview": preview})
|
|
@@ -308,10 +308,6 @@ def main() -> None:
|
|
|
308
308
|
raise ValueError(f"{source} row {number}: {error}") from error
|
|
309
309
|
combined = combine(rows)
|
|
310
310
|
output.parent.mkdir(parents=True, exist_ok=True)
|
|
311
|
-
payload = "".join(
|
|
312
|
-
json.dumps(row, ensure_ascii=False, separators=(",", ":"), sort_keys=True) + "\n"
|
|
313
|
-
for row in combined
|
|
314
|
-
)
|
|
315
311
|
temporary: Path | None = None
|
|
316
312
|
try:
|
|
317
313
|
with tempfile.NamedTemporaryFile(
|
|
@@ -323,7 +319,11 @@ def main() -> None:
|
|
|
323
319
|
delete=False,
|
|
324
320
|
) as handle:
|
|
325
321
|
temporary = Path(handle.name)
|
|
326
|
-
|
|
322
|
+
for row in combined:
|
|
323
|
+
handle.write(
|
|
324
|
+
json.dumps(row, ensure_ascii=False, separators=(",", ":"), sort_keys=True)
|
|
325
|
+
+ "\n"
|
|
326
|
+
)
|
|
327
327
|
temporary.replace(output)
|
|
328
328
|
finally:
|
|
329
329
|
if temporary is not None:
|
|
@@ -4,11 +4,8 @@
|
|
|
4
4
|
from __future__ import annotations
|
|
5
5
|
|
|
6
6
|
import argparse
|
|
7
|
-
import importlib.util
|
|
8
7
|
import re
|
|
9
8
|
from collections import Counter
|
|
10
|
-
from pathlib import Path
|
|
11
|
-
from types import ModuleType
|
|
12
9
|
from typing import Any
|
|
13
10
|
|
|
14
11
|
SEVERITY_ORDER = {"critical": 0, "high": 1, "medium": 2, "low": 3, "informational": 4}
|
|
@@ -28,16 +25,6 @@ class ReportProjectionError(ValueError):
|
|
|
28
25
|
"""Raised when a canonical scan cannot be projected into a valid report."""
|
|
29
26
|
|
|
30
27
|
|
|
31
|
-
def _load_script(name: str) -> ModuleType:
|
|
32
|
-
path = Path(__file__).resolve().parent / f"{name}.py"
|
|
33
|
-
spec = importlib.util.spec_from_file_location(f"codex_security_{name}", path)
|
|
34
|
-
if spec is None or spec.loader is None:
|
|
35
|
-
raise ReportProjectionError(f"could not load report helper: {path}")
|
|
36
|
-
module = importlib.util.module_from_spec(spec)
|
|
37
|
-
spec.loader.exec_module(module)
|
|
38
|
-
return module
|
|
39
|
-
|
|
40
|
-
|
|
41
28
|
def _text(value: Any, fallback: str) -> str:
|
|
42
29
|
candidate = value if isinstance(value, str) and value.strip() else fallback
|
|
43
30
|
normalized = " ".join(candidate.split())
|
|
@@ -864,12 +851,7 @@ def generate_report_markdown(
|
|
|
864
851
|
findings: dict[str, Any],
|
|
865
852
|
coverage: dict[str, Any],
|
|
866
853
|
) -> bytes:
|
|
867
|
-
|
|
868
|
-
validator = _load_script("validate_report_format")
|
|
869
|
-
errors = validator.validate_report(markdown)
|
|
870
|
-
if errors:
|
|
871
|
-
raise ReportProjectionError("generated report failed validation: " + "; ".join(errors))
|
|
872
|
-
return markdown.encode("utf-8")
|
|
854
|
+
return build_report_markdown(manifest, findings, coverage).encode("utf-8")
|
|
873
855
|
|
|
874
856
|
|
|
875
857
|
def main() -> int:
|
|
@@ -304,6 +304,10 @@ def parse_args(description: str) -> argparse.Namespace:
|
|
|
304
304
|
export_findings.add_argument("--scan-id", required=True)
|
|
305
305
|
export_findings.add_argument("--format", choices=EXPORT_FORMATS, required=True)
|
|
306
306
|
|
|
307
|
+
for command in ("prepare-linear-publication", "record-linear-publications"):
|
|
308
|
+
publication = subparsers.add_parser(command)
|
|
309
|
+
publication.add_argument("--input-file", required=True)
|
|
310
|
+
|
|
307
311
|
subparsers.add_parser("database-info")
|
|
308
312
|
return parser.parse_args()
|
|
309
313
|
|