@openai/codex-security 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +201 -0
- package/README.md +306 -0
- package/_bundled_plugin/.app.json +20 -0
- package/_bundled_plugin/.codex-plugin/plugin.json +41 -0
- package/_bundled_plugin/.mcp.json +20 -0
- package/_bundled_plugin/assets/logo.png +0 -0
- package/_bundled_plugin/examples/completed-scan/coverage.json +22 -0
- package/_bundled_plugin/examples/completed-scan/findings.json +51 -0
- package/_bundled_plugin/examples/completed-scan/scan-manifest.json +43 -0
- package/_bundled_plugin/mcp/mcp-app.html.br +0 -0
- package/_bundled_plugin/mcp/server.mjs +24 -0
- package/_bundled_plugin/mcp/server.mjs.br.part-000 +0 -0
- package/_bundled_plugin/mcp/server.mjs.br.part-001 +0 -0
- package/_bundled_plugin/preflight/capability-profiles.toml +179 -0
- package/_bundled_plugin/references/config-preflight.md +115 -0
- package/_bundled_plugin/references/final-report.md +205 -0
- package/_bundled_plugin/references/finding-detail-fields.md +162 -0
- package/_bundled_plugin/references/sarif-adapter.md +26 -0
- package/_bundled_plugin/references/scan-artifacts.md +116 -0
- package/_bundled_plugin/references/scan-contract.md +164 -0
- package/_bundled_plugin/references/security-guidance.md +15 -0
- package/_bundled_plugin/references/shared-hard-rules.md +15 -0
- package/_bundled_plugin/references/static-finding-assessment.md +75 -0
- package/_bundled_plugin/schemas/coverage.schema.json +228 -0
- package/_bundled_plugin/schemas/findings.schema.json +359 -0
- package/_bundled_plugin/schemas/scan-manifest.schema.json +332 -0
- package/_bundled_plugin/scripts/config_preflight.py +901 -0
- package/_bundled_plugin/scripts/deep_scan_config.py +108 -0
- package/_bundled_plugin/scripts/deep_scan_workbench.py +1547 -0
- package/_bundled_plugin/scripts/filesystem_identity.py +30 -0
- package/_bundled_plugin/scripts/finalize_scan_contract.py +2208 -0
- package/_bundled_plugin/scripts/finding_preview.py +270 -0
- package/_bundled_plugin/scripts/generate_rank_input.py +1013 -0
- package/_bundled_plugin/scripts/normalize_candidates.py +320 -0
- package/_bundled_plugin/scripts/rank_preview.py +968 -0
- package/_bundled_plugin/scripts/report_projection.py +844 -0
- package/_bundled_plugin/scripts/resolve_security_md.py +158 -0
- package/_bundled_plugin/scripts/snapshot_sqlite.py +26 -0
- package/_bundled_plugin/scripts/validate_report_format.py +138 -0
- package/_bundled_plugin/scripts/validate_scan_contract.py +108 -0
- package/_bundled_plugin/scripts/validate_tracking_source.py +92 -0
- package/_bundled_plugin/scripts/windows_scan_local_files.py +612 -0
- package/_bundled_plugin/scripts/workbench/__init__.py +1 -0
- package/_bundled_plugin/scripts/workbench/handoff.py +238 -0
- package/_bundled_plugin/scripts/workbench_cli.py +325 -0
- package/_bundled_plugin/scripts/workbench_constants.py +81 -0
- package/_bundled_plugin/scripts/workbench_db.py +3728 -0
- package/_bundled_plugin/scripts/workbench_native_indexes.py +242 -0
- package/_bundled_plugin/scripts/workbench_progress.py +226 -0
- package/_bundled_plugin/scripts/workbench_remediation.py +125 -0
- package/_bundled_plugin/scripts/workbench_scan_history.py +829 -0
- package/_bundled_plugin/scripts/workbench_scan_start.py +149 -0
- package/_bundled_plugin/scripts/workbench_schema.py +714 -0
- package/_bundled_plugin/scripts/workbench_source_excerpt.py +99 -0
- package/_bundled_plugin/scripts/workbench_target.py +469 -0
- package/_bundled_plugin/scripts/workbench_target_state.py +63 -0
- package/_bundled_plugin/scripts/workbench_validation.py +294 -0
- package/_bundled_plugin/skills/attack-path-analysis/SKILL.md +117 -0
- package/_bundled_plugin/skills/attack-path-analysis/agents/openai.yaml +4 -0
- package/_bundled_plugin/skills/attack-path-analysis/references/attack-path-facts.md +76 -0
- package/_bundled_plugin/skills/attack-path-analysis/references/severity-policy.md +153 -0
- package/_bundled_plugin/skills/deep-security-scan/SKILL.md +166 -0
- package/_bundled_plugin/skills/deep-security-scan/agents/openai.yaml +4 -0
- package/_bundled_plugin/skills/define-security-policy/SKILL.md +93 -0
- package/_bundled_plugin/skills/define-security-policy/agents/openai.yaml +4 -0
- package/_bundled_plugin/skills/finding-discovery/SKILL.md +155 -0
- package/_bundled_plugin/skills/finding-discovery/agents/openai.yaml +4 -0
- package/_bundled_plugin/skills/fix-finding/SKILL.md +112 -0
- package/_bundled_plugin/skills/fix-finding/agents/openai.yaml +4 -0
- package/_bundled_plugin/skills/propose-security-hardening/SKILL.md +240 -0
- package/_bundled_plugin/skills/propose-security-hardening/agents/openai.yaml +4 -0
- package/_bundled_plugin/skills/propose-security-hardening/references/proposal-format.md +424 -0
- package/_bundled_plugin/skills/security-diff-scan/SKILL.md +167 -0
- package/_bundled_plugin/skills/security-diff-scan/agents/openai.yaml +4 -0
- package/_bundled_plugin/skills/security-scan/SKILL.md +42 -0
- package/_bundled_plugin/skills/security-scan/agents/openai.yaml +4 -0
- package/_bundled_plugin/skills/security-scan/references/repository-wide-scan.md +48 -0
- package/_bundled_plugin/skills/security-scan/references/scan-artifacts-and-ledger.md +85 -0
- package/_bundled_plugin/skills/threat-model/SKILL.md +59 -0
- package/_bundled_plugin/skills/threat-model/agents/openai.yaml +4 -0
- package/_bundled_plugin/skills/threat-model/references/threat-model-guidance.md +48 -0
- package/_bundled_plugin/skills/track-findings/SKILL.md +222 -0
- package/_bundled_plugin/skills/track-findings/agents/openai.yaml +6 -0
- package/_bundled_plugin/skills/track-findings/references/github-security-advisories.md +54 -0
- package/_bundled_plugin/skills/track-findings/references/jira.md +77 -0
- package/_bundled_plugin/skills/triage-finding/SKILL.md +342 -0
- package/_bundled_plugin/skills/triage-finding/agents/openai.yaml +4 -0
- package/_bundled_plugin/skills/triage-finding/references/github-rest-intake.md +168 -0
- package/_bundled_plugin/skills/triage-finding/references/ticket-intake.md +62 -0
- package/_bundled_plugin/skills/triage-finding/references/triage-result-contract.md +108 -0
- package/_bundled_plugin/skills/validation/SKILL.md +110 -0
- package/_bundled_plugin/skills/validation/agents/openai.yaml +4 -0
- package/_bundled_plugin/skills/validation/references/validation-guidance.md +122 -0
- package/_bundled_plugin/skills/vulnerability-writeup/SKILL.md +263 -0
- package/_bundled_plugin/skills/vulnerability-writeup/agents/openai.yaml +4 -0
- package/_bundled_plugin/skills/vulnerability-writeup/references/report-format.md +104 -0
- package/bin/codex-security.mjs +26 -0
- package/dist/api.d.ts +112 -0
- package/dist/api.d.ts.map +1 -0
- package/dist/api.js +1051 -0
- package/dist/api.js.map +1 -0
- package/dist/auth.d.ts +29 -0
- package/dist/auth.d.ts.map +1 -0
- package/dist/auth.js +246 -0
- package/dist/auth.js.map +1 -0
- package/dist/bulk-scan-discovery.d.ts +35 -0
- package/dist/bulk-scan-discovery.d.ts.map +1 -0
- package/dist/bulk-scan-discovery.js +217 -0
- package/dist/bulk-scan-discovery.js.map +1 -0
- package/dist/cli.d.ts +67 -0
- package/dist/cli.d.ts.map +1 -0
- package/dist/cli.js +2128 -0
- package/dist/cli.js.map +1 -0
- package/dist/config.d.ts +19 -0
- package/dist/config.d.ts.map +1 -0
- package/dist/config.js +199 -0
- package/dist/config.js.map +1 -0
- package/dist/contract.d.ts +21 -0
- package/dist/contract.d.ts.map +1 -0
- package/dist/contract.js +848 -0
- package/dist/contract.js.map +1 -0
- package/dist/cost.d.ts +30 -0
- package/dist/cost.d.ts.map +1 -0
- package/dist/cost.js +280 -0
- package/dist/cost.js.map +1 -0
- package/dist/errors.d.ts +38 -0
- package/dist/errors.d.ts.map +1 -0
- package/dist/errors.js +52 -0
- package/dist/errors.js.map +1 -0
- package/dist/index.d.ts +22 -0
- package/dist/index.d.ts.map +1 -0
- package/dist/index.js +11 -0
- package/dist/index.js.map +1 -0
- package/dist/knowledge-base.d.ts +7 -0
- package/dist/knowledge-base.d.ts.map +1 -0
- package/dist/knowledge-base.js +181 -0
- package/dist/knowledge-base.js.map +1 -0
- package/dist/models.d.ts +231 -0
- package/dist/models.d.ts.map +1 -0
- package/dist/models.js +3 -0
- package/dist/models.js.map +1 -0
- package/dist/multiscan.d.ts +30 -0
- package/dist/multiscan.d.ts.map +1 -0
- package/dist/multiscan.js +397 -0
- package/dist/multiscan.js.map +1 -0
- package/dist/result.d.ts +39 -0
- package/dist/result.d.ts.map +1 -0
- package/dist/result.js +71 -0
- package/dist/result.js.map +1 -0
- package/dist/runtime.d.ts +69 -0
- package/dist/runtime.d.ts.map +1 -0
- package/dist/runtime.js +997 -0
- package/dist/runtime.js.map +1 -0
- package/dist/scan-comparison.d.ts +42 -0
- package/dist/scan-comparison.d.ts.map +1 -0
- package/dist/scan-comparison.js +153 -0
- package/dist/scan-comparison.js.map +1 -0
- package/dist/scan-history-renderer.d.ts +13 -0
- package/dist/scan-history-renderer.d.ts.map +1 -0
- package/dist/scan-history-renderer.js +297 -0
- package/dist/scan-history-renderer.js.map +1 -0
- package/dist/targets.d.ts +39 -0
- package/dist/targets.d.ts.map +1 -0
- package/dist/targets.js +295 -0
- package/dist/targets.js.map +1 -0
- package/dist/trusted-executable.d.ts +6 -0
- package/dist/trusted-executable.d.ts.map +1 -0
- package/dist/trusted-executable.js +77 -0
- package/dist/trusted-executable.js.map +1 -0
- package/dist/version.d.ts +6 -0
- package/dist/version.d.ts.map +1 -0
- package/dist/version.js +42 -0
- package/dist/version.js.map +1 -0
- package/dist/worker-progress.d.ts +13 -0
- package/dist/worker-progress.d.ts.map +1 -0
- package/dist/worker-progress.js +113 -0
- package/dist/worker-progress.js.map +1 -0
- package/package.json +74 -0
|
@@ -0,0 +1,2208 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""Validate and seal additive Codex Security scan-contract artifacts."""
|
|
3
|
+
|
|
4
|
+
from __future__ import annotations
|
|
5
|
+
|
|
6
|
+
import argparse
|
|
7
|
+
import copy
|
|
8
|
+
import csv
|
|
9
|
+
import errno
|
|
10
|
+
import hashlib
|
|
11
|
+
import importlib.util
|
|
12
|
+
import io
|
|
13
|
+
import json
|
|
14
|
+
import math
|
|
15
|
+
import os
|
|
16
|
+
import re
|
|
17
|
+
import secrets
|
|
18
|
+
import stat
|
|
19
|
+
import sys
|
|
20
|
+
from collections.abc import Iterator
|
|
21
|
+
from datetime import datetime, timezone
|
|
22
|
+
from pathlib import Path, PurePosixPath
|
|
23
|
+
from typing import Any, BinaryIO, TextIO
|
|
24
|
+
from urllib.parse import quote, urlsplit
|
|
25
|
+
|
|
26
|
+
SCHEMA_VERSION = "1.0"
|
|
27
|
+
PRODUCER_NAME = "codex-security-plugin"
|
|
28
|
+
FINGERPRINT_ALGORITHM = "codex-security/v1"
|
|
29
|
+
SARIF_SCHEMA = "https://docs.oasis-open.org/sarif/sarif/v2.1.0/os/schemas/sarif-schema-2.1.0.json"
|
|
30
|
+
SEVERITIES = {"critical", "high", "medium", "low", "informational"}
|
|
31
|
+
CONFIDENCES = {"high", "medium", "low"}
|
|
32
|
+
TARGET_KINDS = {"git_revision", "git_worktree", "git_diff", "directory_snapshot"}
|
|
33
|
+
TARGET_COORDINATE_FIELDS = {
|
|
34
|
+
"revision",
|
|
35
|
+
"baseRevision",
|
|
36
|
+
"headRevision",
|
|
37
|
+
"snapshotDigest",
|
|
38
|
+
}
|
|
39
|
+
TARGET_REQUIRED_COORDINATE_FIELDS = {
|
|
40
|
+
"git_revision": {"revision"},
|
|
41
|
+
"git_worktree": {"snapshotDigest"},
|
|
42
|
+
"git_diff": {"snapshotDigest"},
|
|
43
|
+
"directory_snapshot": {"snapshotDigest"},
|
|
44
|
+
}
|
|
45
|
+
DISPOSITIONS = {"reported", "no_issue_found", "rejected", "not_applicable", "needs_follow_up"}
|
|
46
|
+
SARIF_LEVELS = {
|
|
47
|
+
"critical": "error",
|
|
48
|
+
"high": "error",
|
|
49
|
+
"medium": "warning",
|
|
50
|
+
"low": "note",
|
|
51
|
+
"informational": "note",
|
|
52
|
+
}
|
|
53
|
+
SLUG_RE = re.compile(r"^[a-z0-9][a-z0-9._/-]*$")
|
|
54
|
+
RFC3339_RE = re.compile(
|
|
55
|
+
r"^\d{4}-\d{2}-\d{2}[Tt]\d{2}:\d{2}:\d{2}(?:\.\d+)?(?:[Zz]|[+-]\d{2}:\d{2})$"
|
|
56
|
+
)
|
|
57
|
+
GITHUB_HASH_BLOCK_SIZE = 100
|
|
58
|
+
GITHUB_HASH_MOD = 37
|
|
59
|
+
GITHUB_HASH_MASK = (1 << 64) - 1
|
|
60
|
+
GITHUB_HASH_EOF = 65535
|
|
61
|
+
GITHUB_HASH_MAX_LINES = 100_000
|
|
62
|
+
SOURCE_READ_CHUNK_SIZE = 64 * 1024
|
|
63
|
+
SOURCE_READ_MAX_BYTES = 10 * 1024 * 1024
|
|
64
|
+
CONTRACT_DOCUMENT_MAX_BYTES = {
|
|
65
|
+
"scan-manifest.json": 16 * 1024 * 1024,
|
|
66
|
+
"findings.json": 128 * 1024 * 1024,
|
|
67
|
+
"coverage.json": 32 * 1024 * 1024,
|
|
68
|
+
}
|
|
69
|
+
SCHEMA_DOCUMENT_MAX_BYTES = 4 * 1024 * 1024
|
|
70
|
+
JSON_DOCUMENT_READ_CHUNK_SIZE = 64 * 1024
|
|
71
|
+
MAX_JSON_DEPTH = 256
|
|
72
|
+
MAX_JSON_INTEGER = (1 << 53) - 1
|
|
73
|
+
MAX_SCHEMA_NODES = 8192
|
|
74
|
+
MAX_SCHEMA_COLLECTION_ENTRIES = 4096
|
|
75
|
+
MAX_SCHEMA_APPLICATOR_EDGES = 128
|
|
76
|
+
SAFE_SCHEMA_PATTERNS = {
|
|
77
|
+
r"^(?![^:/?#]+://[^/?#]*@)[^?#]+$",
|
|
78
|
+
r"^codex-security-snapshot/v1:sha256:[a-f0-9]{64}$",
|
|
79
|
+
r"^(?!/)(?!.*(?:^|/)\.\.(?:/|$))(?!.*\\).+$",
|
|
80
|
+
r"^[a-f0-9]{64}$",
|
|
81
|
+
r"^(?!.*(?:^|/)\.\.(?:/|$))(?!.*\\)artifacts/.+$",
|
|
82
|
+
r"^csf_[a-f0-9]{24}$",
|
|
83
|
+
r"^occ_[a-f0-9]{24}$",
|
|
84
|
+
r"^[a-z0-9][a-z0-9._/-]*$",
|
|
85
|
+
r"^codex-security/v1:sha256:[a-f0-9]{64}$",
|
|
86
|
+
r"^findings/([a-z0-9][a-z0-9._-]*)/\1\.md$",
|
|
87
|
+
}
|
|
88
|
+
EXPORT_PATHS = {
|
|
89
|
+
"csv": "exports/findings.csv",
|
|
90
|
+
"json": "exports/findings.json",
|
|
91
|
+
"sarif": "exports/results.sarif",
|
|
92
|
+
}
|
|
93
|
+
|
|
94
|
+
|
|
95
|
+
class ContractError(ValueError):
|
|
96
|
+
"""Raised when a completed scan does not satisfy the additive contract."""
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def _reject_non_finite_json(value: str) -> None:
|
|
100
|
+
raise ValueError(f"non-finite JSON number {value!r} is not supported")
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
def _loads_json(value: str | bytes) -> Any:
|
|
104
|
+
return json.loads(value, parse_constant=_reject_non_finite_json)
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
def _read_json(path: Path) -> dict[str, Any]:
|
|
108
|
+
try:
|
|
109
|
+
with path.open("rb") as handle:
|
|
110
|
+
raw = _read_bounded_json_document(handle, str(path), SCHEMA_DOCUMENT_MAX_BYTES)
|
|
111
|
+
payload = _loads_json(raw.decode("utf-8"))
|
|
112
|
+
_require_safe_json_value(payload, str(path))
|
|
113
|
+
except FileNotFoundError as exc:
|
|
114
|
+
raise ContractError(f"missing required contract artifact: {path}") from exc
|
|
115
|
+
except ContractError:
|
|
116
|
+
raise
|
|
117
|
+
except ValueError as exc:
|
|
118
|
+
raise ContractError(f"{path}: invalid JSON: {exc}") from exc
|
|
119
|
+
if not isinstance(payload, dict):
|
|
120
|
+
raise ContractError(f"{path}: expected a JSON object")
|
|
121
|
+
return payload
|
|
122
|
+
|
|
123
|
+
|
|
124
|
+
def _write_json(path: Path, payload: Any) -> None:
|
|
125
|
+
path.parent.mkdir(parents=True, exist_ok=True)
|
|
126
|
+
path.write_bytes(_json_bytes(payload))
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
def _generate_report_projection(
|
|
130
|
+
manifest: dict[str, Any],
|
|
131
|
+
findings: dict[str, Any],
|
|
132
|
+
coverage: dict[str, Any],
|
|
133
|
+
) -> bytes:
|
|
134
|
+
script = Path(__file__).resolve().parent / "report_projection.py"
|
|
135
|
+
spec = importlib.util.spec_from_file_location("codex_security_report_projection", script)
|
|
136
|
+
if spec is None or spec.loader is None:
|
|
137
|
+
raise ContractError(f"could not load report projection helper: {script}")
|
|
138
|
+
module = importlib.util.module_from_spec(spec)
|
|
139
|
+
spec.loader.exec_module(module)
|
|
140
|
+
try:
|
|
141
|
+
return module.generate_report_markdown(manifest, findings, coverage)
|
|
142
|
+
except (OSError, ValueError) as exc:
|
|
143
|
+
raise ContractError(f"report projection failed: {exc}") from exc
|
|
144
|
+
|
|
145
|
+
|
|
146
|
+
def _validate_report_output_paths(scan_dir: Path) -> None:
|
|
147
|
+
_validate_scan_local_output_path(scan_dir, scan_dir / "report.md", "report.md")
|
|
148
|
+
|
|
149
|
+
|
|
150
|
+
def _json_bytes(payload: Any) -> bytes:
|
|
151
|
+
try:
|
|
152
|
+
encoded = json.dumps(payload, allow_nan=False, indent=2, sort_keys=True)
|
|
153
|
+
except ValueError as exc:
|
|
154
|
+
raise ContractError(f"cannot encode canonical JSON: {exc}") from exc
|
|
155
|
+
return (encoded + "\n").encode("utf-8")
|
|
156
|
+
|
|
157
|
+
|
|
158
|
+
def _contract_json_bytes(relative_path: str, payload: Any) -> bytes:
|
|
159
|
+
_require_safe_json_value(payload, relative_path)
|
|
160
|
+
encoded = _json_bytes(payload)
|
|
161
|
+
maximum = CONTRACT_DOCUMENT_MAX_BYTES.get(relative_path)
|
|
162
|
+
if maximum is not None and len(encoded) > maximum:
|
|
163
|
+
raise ContractError(f"{relative_path}: JSON document exceeds the {maximum}-byte limit")
|
|
164
|
+
return encoded
|
|
165
|
+
|
|
166
|
+
|
|
167
|
+
def _read_bounded_json_document(handle: BinaryIO, context: str, maximum: int) -> bytes:
|
|
168
|
+
if os.fstat(handle.fileno()).st_size > maximum:
|
|
169
|
+
raise ContractError(f"{context}: JSON document exceeds the {maximum}-byte limit")
|
|
170
|
+
chunks: list[bytes] = []
|
|
171
|
+
length = 0
|
|
172
|
+
while length <= maximum:
|
|
173
|
+
chunk = handle.read(min(JSON_DOCUMENT_READ_CHUNK_SIZE, maximum + 1 - length))
|
|
174
|
+
if not chunk:
|
|
175
|
+
break
|
|
176
|
+
length += len(chunk)
|
|
177
|
+
if length > maximum:
|
|
178
|
+
raise ContractError(f"{context}: JSON document exceeds the {maximum}-byte limit")
|
|
179
|
+
chunks.append(chunk)
|
|
180
|
+
result = b"".join(chunks)
|
|
181
|
+
_require_json_nesting(result, context)
|
|
182
|
+
return result
|
|
183
|
+
|
|
184
|
+
|
|
185
|
+
def _require_json_nesting(value: bytes, context: str) -> None:
|
|
186
|
+
depth = 0
|
|
187
|
+
in_string = False
|
|
188
|
+
escaped = False
|
|
189
|
+
for character in value:
|
|
190
|
+
if in_string:
|
|
191
|
+
if escaped:
|
|
192
|
+
escaped = False
|
|
193
|
+
elif character == ord("\\"):
|
|
194
|
+
escaped = True
|
|
195
|
+
elif character == ord('"'):
|
|
196
|
+
in_string = False
|
|
197
|
+
continue
|
|
198
|
+
if character == ord('"'):
|
|
199
|
+
in_string = True
|
|
200
|
+
elif character in (ord("{"), ord("[")):
|
|
201
|
+
depth += 1
|
|
202
|
+
if depth > MAX_JSON_DEPTH + 1:
|
|
203
|
+
raise ContractError(
|
|
204
|
+
f"{context}: JSON document exceeds the {MAX_JSON_DEPTH}-level nesting limit"
|
|
205
|
+
)
|
|
206
|
+
elif character in (ord("}"), ord("]")):
|
|
207
|
+
depth -= 1
|
|
208
|
+
|
|
209
|
+
|
|
210
|
+
def _require_safe_json_value(value: Any, context: str, *, validate_strings: bool = True) -> None:
|
|
211
|
+
def visit(item: Any, location: str, depth: int) -> None:
|
|
212
|
+
if depth > MAX_JSON_DEPTH:
|
|
213
|
+
raise ContractError(
|
|
214
|
+
f"{location}: JSON document exceeds the {MAX_JSON_DEPTH}-level nesting limit"
|
|
215
|
+
)
|
|
216
|
+
if isinstance(item, dict):
|
|
217
|
+
for key, child in item.items():
|
|
218
|
+
if not isinstance(key, str):
|
|
219
|
+
raise ContractError(f"{location}: expected string JSON property names")
|
|
220
|
+
if validate_strings:
|
|
221
|
+
_require_safe_json_string(key, location)
|
|
222
|
+
visit(child, f"{location}.<property>", depth + 1)
|
|
223
|
+
elif isinstance(item, list):
|
|
224
|
+
for index, child in enumerate(item):
|
|
225
|
+
visit(child, f"{location}[{index}]", depth + 1)
|
|
226
|
+
elif isinstance(item, str) and validate_strings:
|
|
227
|
+
_require_safe_json_string(item, location)
|
|
228
|
+
elif isinstance(item, int) and not isinstance(item, bool):
|
|
229
|
+
if abs(item) > MAX_JSON_INTEGER:
|
|
230
|
+
raise ContractError(
|
|
231
|
+
f"{location}: unsafe integer-valued JSON numbers are not supported"
|
|
232
|
+
)
|
|
233
|
+
elif isinstance(item, float):
|
|
234
|
+
if not math.isfinite(item):
|
|
235
|
+
raise ContractError(f"{location}: non-finite JSON numbers are not supported")
|
|
236
|
+
if item.is_integer() and abs(item) > MAX_JSON_INTEGER:
|
|
237
|
+
raise ContractError(
|
|
238
|
+
f"{location}: unsafe integer-valued JSON numbers are not supported"
|
|
239
|
+
)
|
|
240
|
+
|
|
241
|
+
visit(value, context, 0)
|
|
242
|
+
|
|
243
|
+
|
|
244
|
+
def _require_safe_json_string(value: str, context: str) -> None:
|
|
245
|
+
try:
|
|
246
|
+
value.encode("utf-8")
|
|
247
|
+
except UnicodeEncodeError as exc:
|
|
248
|
+
raise ContractError(f"{context}: expected well-formed Unicode JSON strings") from exc
|
|
249
|
+
|
|
250
|
+
|
|
251
|
+
def _sha256_text(value: str) -> str:
|
|
252
|
+
return hashlib.sha256(value.encode("utf-8")).hexdigest()
|
|
253
|
+
|
|
254
|
+
|
|
255
|
+
def _sha256_bytes(value: bytes) -> str:
|
|
256
|
+
return hashlib.sha256(value).hexdigest()
|
|
257
|
+
|
|
258
|
+
|
|
259
|
+
def _sha256_file(path: Path) -> str:
|
|
260
|
+
digest = hashlib.sha256()
|
|
261
|
+
with path.open("rb") as handle:
|
|
262
|
+
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
|
|
263
|
+
digest.update(chunk)
|
|
264
|
+
return digest.hexdigest()
|
|
265
|
+
|
|
266
|
+
|
|
267
|
+
def _require_dict(payload: dict[str, Any], key: str, context: str) -> dict[str, Any]:
|
|
268
|
+
value = payload.get(key)
|
|
269
|
+
if not isinstance(value, dict):
|
|
270
|
+
raise ContractError(f"{context}.{key}: expected an object")
|
|
271
|
+
return value
|
|
272
|
+
|
|
273
|
+
|
|
274
|
+
def _require_list(payload: dict[str, Any], key: str, context: str) -> list[Any]:
|
|
275
|
+
value = payload.get(key)
|
|
276
|
+
if not isinstance(value, list):
|
|
277
|
+
raise ContractError(f"{context}.{key}: expected an array")
|
|
278
|
+
return value
|
|
279
|
+
|
|
280
|
+
|
|
281
|
+
def _require_str(payload: dict[str, Any], key: str, context: str) -> str:
|
|
282
|
+
value = payload.get(key)
|
|
283
|
+
if not isinstance(value, str) or not value.strip():
|
|
284
|
+
raise ContractError(f"{context}.{key}: expected a non-empty string")
|
|
285
|
+
return value
|
|
286
|
+
|
|
287
|
+
|
|
288
|
+
def _require_safe_relative_path(value: str, context: str, *, allow_dot: bool = False) -> str:
|
|
289
|
+
try:
|
|
290
|
+
value.encode("utf-8")
|
|
291
|
+
except UnicodeEncodeError as exc:
|
|
292
|
+
raise ContractError(f"{context}: expected a safe repository-relative POSIX path") from exc
|
|
293
|
+
path = PurePosixPath(value)
|
|
294
|
+
normalized = path.as_posix()
|
|
295
|
+
if (
|
|
296
|
+
not value
|
|
297
|
+
or (normalized == "." and not allow_dot)
|
|
298
|
+
or "\\" in value
|
|
299
|
+
or "\0" in value
|
|
300
|
+
or path.is_absolute()
|
|
301
|
+
or ".." in path.parts
|
|
302
|
+
):
|
|
303
|
+
raise ContractError(f"{context}: expected a safe repository-relative POSIX path")
|
|
304
|
+
return normalized
|
|
305
|
+
|
|
306
|
+
|
|
307
|
+
def _require_scan_directory(scan_dir: Path) -> Path:
|
|
308
|
+
scan_dir = scan_dir.absolute()
|
|
309
|
+
try:
|
|
310
|
+
metadata = scan_dir.lstat()
|
|
311
|
+
except OSError as exc:
|
|
312
|
+
raise ContractError("scan directory: expected an existing non-symlink directory") from exc
|
|
313
|
+
if not stat.S_ISDIR(metadata.st_mode):
|
|
314
|
+
raise ContractError("scan directory: expected an existing non-symlink directory")
|
|
315
|
+
try:
|
|
316
|
+
resolved = scan_dir.resolve(strict=True)
|
|
317
|
+
except OSError as exc:
|
|
318
|
+
raise ContractError("scan directory: expected an existing non-symlink directory") from exc
|
|
319
|
+
if resolved != scan_dir:
|
|
320
|
+
raise ContractError("scan directory: expected a canonical non-symlink directory")
|
|
321
|
+
return resolved
|
|
322
|
+
|
|
323
|
+
|
|
324
|
+
def _validate_scan_local_output_path(scan_dir: Path, path: Path, relative_path: str) -> None:
|
|
325
|
+
try:
|
|
326
|
+
resolved_parent = path.parent.resolve(strict=True)
|
|
327
|
+
resolved_parent.relative_to(scan_dir)
|
|
328
|
+
except (OSError, RuntimeError, ValueError) as exc:
|
|
329
|
+
raise ContractError(f"{relative_path}: expected a path inside the scan directory") from exc
|
|
330
|
+
if resolved_parent != path.parent or path.is_symlink():
|
|
331
|
+
raise ContractError(
|
|
332
|
+
f"{relative_path}: expected a non-symlink path inside the scan directory"
|
|
333
|
+
)
|
|
334
|
+
if path.exists() and not path.is_file():
|
|
335
|
+
raise ContractError(f"{relative_path}: expected a regular file")
|
|
336
|
+
|
|
337
|
+
|
|
338
|
+
def _descriptor_relative_reads_available() -> bool:
|
|
339
|
+
return os.open in os.supports_dir_fd and hasattr(os, "O_NOFOLLOW")
|
|
340
|
+
|
|
341
|
+
|
|
342
|
+
def _is_windows() -> bool:
|
|
343
|
+
return os.name == "nt"
|
|
344
|
+
|
|
345
|
+
|
|
346
|
+
def _descriptor_relative_writes_available() -> bool:
|
|
347
|
+
# os.replace accepts src_dir_fd/dst_dir_fd wherever descriptor-relative
|
|
348
|
+
# os.rename is supported, but Python lists only os.rename in supports_dir_fd.
|
|
349
|
+
required_operations = (os.mkdir, os.open, os.rename, os.stat, os.unlink)
|
|
350
|
+
return hasattr(os, "O_NOFOLLOW") and all(
|
|
351
|
+
operation in os.supports_dir_fd for operation in required_operations
|
|
352
|
+
)
|
|
353
|
+
|
|
354
|
+
|
|
355
|
+
_WINDOWS_SCAN_LOCAL_FILES: Any | None = None
|
|
356
|
+
|
|
357
|
+
|
|
358
|
+
def _windows_scan_local_files() -> Any:
|
|
359
|
+
"""Load the Win32 backend only on runtimes that need it."""
|
|
360
|
+
|
|
361
|
+
global _WINDOWS_SCAN_LOCAL_FILES
|
|
362
|
+
if _WINDOWS_SCAN_LOCAL_FILES is None:
|
|
363
|
+
script = Path(__file__).resolve().with_name("windows_scan_local_files.py")
|
|
364
|
+
spec = importlib.util.spec_from_file_location("codex_security_windows_scan_files", script)
|
|
365
|
+
if spec is None or spec.loader is None:
|
|
366
|
+
raise ContractError(f"could not load Windows scan-local file helper: {script}")
|
|
367
|
+
module = importlib.util.module_from_spec(spec)
|
|
368
|
+
spec.loader.exec_module(module)
|
|
369
|
+
_WINDOWS_SCAN_LOCAL_FILES = module
|
|
370
|
+
return _WINDOWS_SCAN_LOCAL_FILES
|
|
371
|
+
|
|
372
|
+
|
|
373
|
+
def _open_verified_scan_directory(scan_dir: Path) -> int:
|
|
374
|
+
scan_dir = scan_dir.absolute()
|
|
375
|
+
try:
|
|
376
|
+
expected = scan_dir.lstat()
|
|
377
|
+
canonical = _require_scan_directory(scan_dir)
|
|
378
|
+
descriptor = os.open(
|
|
379
|
+
canonical,
|
|
380
|
+
os.O_RDONLY | getattr(os, "O_DIRECTORY", 0) | getattr(os, "O_NOFOLLOW", 0),
|
|
381
|
+
)
|
|
382
|
+
except OSError as exc:
|
|
383
|
+
raise ContractError("scan directory: expected an existing non-symlink directory") from exc
|
|
384
|
+
opened = os.fstat(descriptor)
|
|
385
|
+
if (opened.st_dev, opened.st_ino) != (expected.st_dev, expected.st_ino):
|
|
386
|
+
os.close(descriptor)
|
|
387
|
+
raise ContractError("scan directory: changed while it was being opened")
|
|
388
|
+
return descriptor
|
|
389
|
+
|
|
390
|
+
|
|
391
|
+
def _open_scan_local_directory(root_fd: int, parts: tuple[str, ...], *, create: bool) -> int:
|
|
392
|
+
descriptor = os.dup(root_fd)
|
|
393
|
+
try:
|
|
394
|
+
for part in parts:
|
|
395
|
+
if create:
|
|
396
|
+
try:
|
|
397
|
+
os.mkdir(part, mode=0o700, dir_fd=descriptor)
|
|
398
|
+
except FileExistsError:
|
|
399
|
+
pass
|
|
400
|
+
next_descriptor = os.open(
|
|
401
|
+
part,
|
|
402
|
+
os.O_RDONLY | getattr(os, "O_DIRECTORY", 0) | getattr(os, "O_NOFOLLOW", 0),
|
|
403
|
+
dir_fd=descriptor,
|
|
404
|
+
)
|
|
405
|
+
os.close(descriptor)
|
|
406
|
+
descriptor = next_descriptor
|
|
407
|
+
return descriptor
|
|
408
|
+
except BaseException:
|
|
409
|
+
os.close(descriptor)
|
|
410
|
+
raise
|
|
411
|
+
|
|
412
|
+
|
|
413
|
+
def open_scan_local_file_descriptor(scan_dir: Path, relative_path: str, context: str) -> int:
|
|
414
|
+
scan_dir = _require_scan_directory(scan_dir)
|
|
415
|
+
relative_path = _require_safe_relative_path(relative_path, context)
|
|
416
|
+
if not _descriptor_relative_reads_available():
|
|
417
|
+
if not _is_windows():
|
|
418
|
+
raise ContractError("scan-local input requires descriptor-relative file operations")
|
|
419
|
+
try:
|
|
420
|
+
return _windows_scan_local_files().open_read_fd(scan_dir, relative_path, context)
|
|
421
|
+
except OSError as exc:
|
|
422
|
+
raise ContractError(str(exc)) from exc
|
|
423
|
+
root_fd: int | None = None
|
|
424
|
+
parent_fd: int | None = None
|
|
425
|
+
descriptor: int | None = None
|
|
426
|
+
try:
|
|
427
|
+
root_fd = _open_verified_scan_directory(scan_dir)
|
|
428
|
+
parts = PurePosixPath(relative_path).parts
|
|
429
|
+
try:
|
|
430
|
+
parent_fd = _open_scan_local_directory(root_fd, parts[:-1], create=False)
|
|
431
|
+
descriptor = os.open(
|
|
432
|
+
parts[-1],
|
|
433
|
+
os.O_RDONLY | os.O_NOFOLLOW | getattr(os, "O_NONBLOCK", 0),
|
|
434
|
+
dir_fd=parent_fd,
|
|
435
|
+
)
|
|
436
|
+
except OSError as exc:
|
|
437
|
+
if exc.errno == errno.ELOOP:
|
|
438
|
+
try:
|
|
439
|
+
link_target = Path(os.readlink(parts[-1], dir_fd=parent_fd))
|
|
440
|
+
if not link_target.is_absolute():
|
|
441
|
+
link_target = scan_dir.joinpath(*parts[:-1], link_target)
|
|
442
|
+
link_target.resolve(strict=False).relative_to(scan_dir)
|
|
443
|
+
except (OSError, RuntimeError, ValueError):
|
|
444
|
+
raise ContractError(
|
|
445
|
+
f"{context}: expected a file inside the scan directory"
|
|
446
|
+
) from exc
|
|
447
|
+
raise ContractError(f"{context}: expected a regular non-symlink file") from exc
|
|
448
|
+
raise ContractError(f"{context}: expected a file inside the scan directory") from exc
|
|
449
|
+
metadata = os.fstat(descriptor)
|
|
450
|
+
if not stat.S_ISREG(metadata.st_mode):
|
|
451
|
+
raise ContractError(f"{context}: expected a regular non-symlink file")
|
|
452
|
+
result = descriptor
|
|
453
|
+
descriptor = None
|
|
454
|
+
return result
|
|
455
|
+
finally:
|
|
456
|
+
if descriptor is not None:
|
|
457
|
+
os.close(descriptor)
|
|
458
|
+
if parent_fd is not None:
|
|
459
|
+
os.close(parent_fd)
|
|
460
|
+
if root_fd is not None:
|
|
461
|
+
os.close(root_fd)
|
|
462
|
+
|
|
463
|
+
|
|
464
|
+
def _require_scan_local_file(scan_dir: Path, relative_path: str, context: str) -> None:
|
|
465
|
+
descriptor = open_scan_local_file_descriptor(scan_dir, relative_path, context)
|
|
466
|
+
os.close(descriptor)
|
|
467
|
+
|
|
468
|
+
|
|
469
|
+
def _require_derived_writeup_files(scan_dir: Path, findings: dict[str, Any]) -> None:
|
|
470
|
+
for index, finding in enumerate(findings.get("findings", [])):
|
|
471
|
+
if not isinstance(finding, dict):
|
|
472
|
+
continue
|
|
473
|
+
writeup = finding.get("writeup")
|
|
474
|
+
if not isinstance(writeup, dict):
|
|
475
|
+
continue
|
|
476
|
+
report_path = writeup.get("reportPath")
|
|
477
|
+
if isinstance(report_path, str):
|
|
478
|
+
_require_scan_local_file(
|
|
479
|
+
scan_dir,
|
|
480
|
+
report_path,
|
|
481
|
+
f"findings[{index}].writeup.reportPath",
|
|
482
|
+
)
|
|
483
|
+
|
|
484
|
+
|
|
485
|
+
def _require_hardening_portfolio_file(scan_dir: Path, scan: dict[str, Any]) -> None:
|
|
486
|
+
hardening = scan.get("hardening")
|
|
487
|
+
if not isinstance(hardening, dict):
|
|
488
|
+
return
|
|
489
|
+
portfolio_path = hardening.get("portfolioPath")
|
|
490
|
+
if isinstance(portfolio_path, str):
|
|
491
|
+
_require_scan_local_file(
|
|
492
|
+
scan_dir,
|
|
493
|
+
portfolio_path,
|
|
494
|
+
"manifest.scan.hardening.portfolioPath",
|
|
495
|
+
)
|
|
496
|
+
|
|
497
|
+
|
|
498
|
+
def _read_scan_local_json_bytes(
|
|
499
|
+
scan_dir: Path, relative_path: str, context: str
|
|
500
|
+
) -> tuple[dict[str, Any], bytes]:
|
|
501
|
+
descriptor = open_scan_local_file_descriptor(scan_dir, relative_path, context)
|
|
502
|
+
try:
|
|
503
|
+
with os.fdopen(descriptor, "rb") as handle:
|
|
504
|
+
descriptor = -1
|
|
505
|
+
maximum = CONTRACT_DOCUMENT_MAX_BYTES.get(relative_path)
|
|
506
|
+
raw = (
|
|
507
|
+
handle.read()
|
|
508
|
+
if maximum is None
|
|
509
|
+
else _read_bounded_json_document(handle, context, maximum)
|
|
510
|
+
)
|
|
511
|
+
try:
|
|
512
|
+
payload = _loads_json(raw.decode("utf-8"))
|
|
513
|
+
except (UnicodeDecodeError, ValueError) as exc:
|
|
514
|
+
raise ContractError(f"{context}: invalid JSON: {exc}") from exc
|
|
515
|
+
if not isinstance(payload, dict):
|
|
516
|
+
raise ContractError(f"{context}: expected a JSON object")
|
|
517
|
+
_require_safe_json_value(payload, context, validate_strings=False)
|
|
518
|
+
finally:
|
|
519
|
+
if descriptor >= 0:
|
|
520
|
+
os.close(descriptor)
|
|
521
|
+
return payload, raw
|
|
522
|
+
|
|
523
|
+
|
|
524
|
+
def _read_scan_local_json(scan_dir: Path, relative_path: str, context: str) -> dict[str, Any]:
|
|
525
|
+
payload, _ = _read_scan_local_json_bytes(scan_dir, relative_path, context)
|
|
526
|
+
return payload
|
|
527
|
+
|
|
528
|
+
|
|
529
|
+
def _sha256_scan_local_file(scan_dir: Path, relative_path: str, context: str) -> str:
|
|
530
|
+
descriptor = open_scan_local_file_descriptor(scan_dir, relative_path, context)
|
|
531
|
+
digest = hashlib.sha256()
|
|
532
|
+
try:
|
|
533
|
+
with os.fdopen(descriptor, "rb") as handle:
|
|
534
|
+
descriptor = -1
|
|
535
|
+
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
|
|
536
|
+
digest.update(chunk)
|
|
537
|
+
finally:
|
|
538
|
+
if descriptor >= 0:
|
|
539
|
+
os.close(descriptor)
|
|
540
|
+
return digest.hexdigest()
|
|
541
|
+
|
|
542
|
+
|
|
543
|
+
def write_scan_local_bytes(
|
|
544
|
+
scan_dir: Path, relative_path: str, payload: bytes, *, external_name: bool = False
|
|
545
|
+
) -> None:
|
|
546
|
+
scan_dir = _require_scan_directory(scan_dir)
|
|
547
|
+
if external_name:
|
|
548
|
+
if relative_path in {"", ".", ".."} or "/" in relative_path or "\0" in relative_path:
|
|
549
|
+
raise ContractError("external output path: expected a safe file name")
|
|
550
|
+
else:
|
|
551
|
+
relative_path = _require_safe_relative_path(relative_path, "scan-local output path")
|
|
552
|
+
path = scan_dir / relative_path
|
|
553
|
+
if not _descriptor_relative_writes_available():
|
|
554
|
+
if not _is_windows():
|
|
555
|
+
raise ContractError("scan-local output requires descriptor-relative file operations")
|
|
556
|
+
try:
|
|
557
|
+
_windows_scan_local_files().atomic_write(scan_dir, relative_path, payload)
|
|
558
|
+
except OSError as exc:
|
|
559
|
+
raise ContractError(f"{relative_path}: {exc}") from exc
|
|
560
|
+
return
|
|
561
|
+
root_fd: int | None = None
|
|
562
|
+
parent_fd: int | None = None
|
|
563
|
+
temp_name: str | None = None
|
|
564
|
+
try:
|
|
565
|
+
root_fd = _open_verified_scan_directory(scan_dir)
|
|
566
|
+
parts = PurePosixPath(relative_path).parts
|
|
567
|
+
try:
|
|
568
|
+
parent_fd = _open_scan_local_directory(root_fd, parts[:-1], create=True)
|
|
569
|
+
except OSError as exc:
|
|
570
|
+
raise ContractError(
|
|
571
|
+
f"{relative_path}: expected a path inside the scan directory"
|
|
572
|
+
) from exc
|
|
573
|
+
try:
|
|
574
|
+
metadata = os.stat(parts[-1], dir_fd=parent_fd, follow_symlinks=False)
|
|
575
|
+
except FileNotFoundError:
|
|
576
|
+
pass
|
|
577
|
+
else:
|
|
578
|
+
if not stat.S_ISREG(metadata.st_mode):
|
|
579
|
+
raise ContractError(f"{relative_path}: expected a regular non-symlink file")
|
|
580
|
+
temp_name = f".{path.name}.{secrets.token_hex(8)}.tmp"
|
|
581
|
+
temp_fd = os.open(temp_name, os.O_WRONLY | os.O_CREAT | os.O_EXCL, 0o600, dir_fd=parent_fd)
|
|
582
|
+
with os.fdopen(temp_fd, "wb") as handle:
|
|
583
|
+
handle.write(payload)
|
|
584
|
+
handle.flush()
|
|
585
|
+
os.fsync(handle.fileno())
|
|
586
|
+
os.replace(temp_name, parts[-1], src_dir_fd=parent_fd, dst_dir_fd=parent_fd)
|
|
587
|
+
temp_name = None
|
|
588
|
+
finally:
|
|
589
|
+
if temp_name is not None and parent_fd is not None:
|
|
590
|
+
try:
|
|
591
|
+
os.unlink(temp_name, dir_fd=parent_fd)
|
|
592
|
+
except FileNotFoundError:
|
|
593
|
+
pass
|
|
594
|
+
if parent_fd is not None:
|
|
595
|
+
os.close(parent_fd)
|
|
596
|
+
if root_fd is not None:
|
|
597
|
+
os.close(root_fd)
|
|
598
|
+
|
|
599
|
+
|
|
600
|
+
def _remove_scan_local_file_if_exists(scan_dir: Path, relative_path: str) -> None:
|
|
601
|
+
scan_dir = _require_scan_directory(scan_dir)
|
|
602
|
+
relative_path = _require_safe_relative_path(relative_path, "scan-local cleanup path")
|
|
603
|
+
if not _descriptor_relative_writes_available():
|
|
604
|
+
if not _is_windows():
|
|
605
|
+
raise ContractError("scan-local cleanup requires descriptor-relative file operations")
|
|
606
|
+
try:
|
|
607
|
+
_windows_scan_local_files().unlink_if_exists(scan_dir, relative_path)
|
|
608
|
+
except OSError as exc:
|
|
609
|
+
raise ContractError(f"{relative_path}: {exc}") from exc
|
|
610
|
+
return
|
|
611
|
+
root_fd: int | None = None
|
|
612
|
+
parent_fd: int | None = None
|
|
613
|
+
try:
|
|
614
|
+
root_fd = _open_verified_scan_directory(scan_dir)
|
|
615
|
+
parts = PurePosixPath(relative_path).parts
|
|
616
|
+
parent_fd = _open_scan_local_directory(root_fd, parts[:-1], create=False)
|
|
617
|
+
try:
|
|
618
|
+
metadata = os.stat(parts[-1], dir_fd=parent_fd, follow_symlinks=False)
|
|
619
|
+
except FileNotFoundError:
|
|
620
|
+
return
|
|
621
|
+
if not (stat.S_ISREG(metadata.st_mode) or stat.S_ISLNK(metadata.st_mode)):
|
|
622
|
+
raise ContractError(f"{relative_path}: expected a regular file or symlink")
|
|
623
|
+
os.unlink(parts[-1], dir_fd=parent_fd)
|
|
624
|
+
finally:
|
|
625
|
+
if parent_fd is not None:
|
|
626
|
+
os.close(parent_fd)
|
|
627
|
+
if root_fd is not None:
|
|
628
|
+
os.close(root_fd)
|
|
629
|
+
|
|
630
|
+
|
|
631
|
+
def _write_scan_local_json(scan_dir: Path, relative_path: str, payload: Any) -> None:
|
|
632
|
+
write_scan_local_bytes(scan_dir, relative_path, _contract_json_bytes(relative_path, payload))
|
|
633
|
+
|
|
634
|
+
|
|
635
|
+
def _validate_remote(remote: str, context: str) -> None:
|
|
636
|
+
parsed = urlsplit(remote)
|
|
637
|
+
if not parsed.scheme or not parsed.netloc:
|
|
638
|
+
raise ContractError(f"{context}: expected a sanitized canonical absolute URL")
|
|
639
|
+
if parsed.username or parsed.password or parsed.query or parsed.fragment:
|
|
640
|
+
raise ContractError(
|
|
641
|
+
f"{context}: remote URL must not contain credentials, query, or fragment"
|
|
642
|
+
)
|
|
643
|
+
|
|
644
|
+
|
|
645
|
+
def _validate_date_time(value: str, context: str) -> None:
|
|
646
|
+
if not RFC3339_RE.fullmatch(value):
|
|
647
|
+
raise ContractError(f"{context}: expected an RFC 3339 timestamp")
|
|
648
|
+
try:
|
|
649
|
+
parsed = datetime.fromisoformat(value[:-1] + "+00:00" if value[-1] in "Zz" else value)
|
|
650
|
+
except ValueError as exc:
|
|
651
|
+
raise ContractError(f"{context}: expected an RFC 3339 timestamp") from exc
|
|
652
|
+
if parsed.tzinfo is None:
|
|
653
|
+
raise ContractError(f"{context}: expected an RFC 3339 timestamp")
|
|
654
|
+
|
|
655
|
+
|
|
656
|
+
def _validate_target(target: dict[str, Any]) -> None:
|
|
657
|
+
kind = _require_str(target, "kind", "scan.target")
|
|
658
|
+
if kind not in TARGET_KINDS:
|
|
659
|
+
raise ContractError(f"scan.target.kind: unsupported target kind: {kind}")
|
|
660
|
+
_require_str(target, "targetId", "scan.target")
|
|
661
|
+
_require_str(target, "displayName", "scan.target")
|
|
662
|
+
remote = target.get("remote")
|
|
663
|
+
if remote is not None:
|
|
664
|
+
if not isinstance(remote, str):
|
|
665
|
+
raise ContractError("scan.target.remote: expected a string")
|
|
666
|
+
_validate_remote(remote, "scan.target.remote")
|
|
667
|
+
if kind == "git_revision":
|
|
668
|
+
_require_str(target, "revision", "scan.target")
|
|
669
|
+
elif kind == "git_worktree":
|
|
670
|
+
_require_str(target, "snapshotDigest", "scan.target")
|
|
671
|
+
elif kind == "git_diff":
|
|
672
|
+
_require_str(target, "snapshotDigest", "scan.target")
|
|
673
|
+
elif kind == "directory_snapshot":
|
|
674
|
+
_require_str(target, "snapshotDigest", "scan.target")
|
|
675
|
+
|
|
676
|
+
|
|
677
|
+
def _fingerprint(target_id: str, finding: dict[str, Any]) -> str:
|
|
678
|
+
identity = _require_dict(finding, "identity", "finding")
|
|
679
|
+
anchor = _require_str(identity, "anchor", "finding.identity")
|
|
680
|
+
if not SLUG_RE.fullmatch(anchor):
|
|
681
|
+
raise ContractError("finding.identity.anchor: expected a stable lowercase semantic slug")
|
|
682
|
+
instance = identity.get("instance", "")
|
|
683
|
+
if not isinstance(instance, str):
|
|
684
|
+
raise ContractError("finding.identity.instance: expected a string")
|
|
685
|
+
if instance and not SLUG_RE.fullmatch(instance):
|
|
686
|
+
raise ContractError("finding.identity.instance: expected a stable lowercase semantic slug")
|
|
687
|
+
rule_id = _require_str(finding, "ruleId", "finding")
|
|
688
|
+
if not SLUG_RE.fullmatch(rule_id):
|
|
689
|
+
raise ContractError("finding.ruleId: expected a stable lowercase rule slug")
|
|
690
|
+
material = "\0".join((FINGERPRINT_ALGORITHM, target_id, rule_id, anchor, instance))
|
|
691
|
+
return f"{FINGERPRINT_ALGORITHM}:sha256:{_sha256_text(material)}"
|
|
692
|
+
|
|
693
|
+
|
|
694
|
+
def _stable_id(prefix: str, *parts: str) -> str:
|
|
695
|
+
return f"{prefix}_{_sha256_text(chr(0).join(parts))[:24]}"
|
|
696
|
+
|
|
697
|
+
|
|
698
|
+
def _validate_location(location: dict[str, Any], context: str) -> None:
|
|
699
|
+
_require_safe_relative_path(_require_str(location, "path", context), f"{context}.path")
|
|
700
|
+
start = location.get("startLine")
|
|
701
|
+
end = location.get("endLine", start)
|
|
702
|
+
if not isinstance(start, int) or start < 1:
|
|
703
|
+
raise ContractError(f"{context}.startLine: expected a positive integer")
|
|
704
|
+
if not isinstance(end, int) or end < start:
|
|
705
|
+
raise ContractError(f"{context}.endLine: expected an integer >= startLine")
|
|
706
|
+
role = location.get("role")
|
|
707
|
+
if role is not None and (not isinstance(role, str) or not role):
|
|
708
|
+
raise ContractError(f"{context}.role: expected a non-empty string")
|
|
709
|
+
|
|
710
|
+
|
|
711
|
+
def _derived_finding_identity_rows(
|
|
712
|
+
manifest: dict[str, Any],
|
|
713
|
+
findings: dict[str, Any],
|
|
714
|
+
) -> list[tuple[str, dict[str, Any], str, str, dict[str, str]]]:
|
|
715
|
+
scan = _require_dict(manifest, "scan", "manifest")
|
|
716
|
+
scan_id = _require_str(scan, "id", "manifest.scan")
|
|
717
|
+
target_id = _require_str(
|
|
718
|
+
_require_dict(scan, "target", "manifest.scan"), "targetId", "scan.target"
|
|
719
|
+
)
|
|
720
|
+
if findings.get("scanId") != scan_id:
|
|
721
|
+
raise ContractError("findings.scanId: must match manifest scan id")
|
|
722
|
+
|
|
723
|
+
finding_ids: set[str] = set()
|
|
724
|
+
occurrence_ids: set[str] = set()
|
|
725
|
+
rows: list[tuple[str, dict[str, Any], str, str, dict[str, str]]] = []
|
|
726
|
+
for index, finding in enumerate(_require_list(findings, "findings", "findings")):
|
|
727
|
+
context = f"findings.findings[{index}]"
|
|
728
|
+
if not isinstance(finding, dict):
|
|
729
|
+
raise ContractError(f"{context}: expected an object")
|
|
730
|
+
fingerprint = _fingerprint(target_id, finding)
|
|
731
|
+
expected_finding_id = _stable_id("csf", fingerprint)
|
|
732
|
+
expected_occurrence_id = _stable_id("occ", scan_id, fingerprint)
|
|
733
|
+
expected_fingerprints = {"algorithm": FINGERPRINT_ALGORITHM, "primary": fingerprint}
|
|
734
|
+
rows.append(
|
|
735
|
+
(
|
|
736
|
+
context,
|
|
737
|
+
finding,
|
|
738
|
+
expected_finding_id,
|
|
739
|
+
expected_occurrence_id,
|
|
740
|
+
expected_fingerprints,
|
|
741
|
+
)
|
|
742
|
+
)
|
|
743
|
+
finding_ids.add(expected_finding_id)
|
|
744
|
+
if expected_occurrence_id in occurrence_ids:
|
|
745
|
+
raise ContractError(
|
|
746
|
+
f"{context}: duplicate occurrence identity; use identity.instance to split siblings"
|
|
747
|
+
)
|
|
748
|
+
occurrence_ids.add(expected_occurrence_id)
|
|
749
|
+
|
|
750
|
+
if len(finding_ids) != len(occurrence_ids):
|
|
751
|
+
raise ContractError("findings: duplicate logical findings in one scan")
|
|
752
|
+
return rows
|
|
753
|
+
|
|
754
|
+
|
|
755
|
+
def _populate_unsealed_finding_identities(
|
|
756
|
+
manifest: dict[str, Any],
|
|
757
|
+
findings: dict[str, Any],
|
|
758
|
+
) -> None:
|
|
759
|
+
"""Replace draft-owned finding identities with deterministic finalizer values."""
|
|
760
|
+
|
|
761
|
+
for _, finding, finding_id, occurrence_id, fingerprints in _derived_finding_identity_rows(
|
|
762
|
+
manifest,
|
|
763
|
+
findings,
|
|
764
|
+
):
|
|
765
|
+
finding["findingId"] = finding_id
|
|
766
|
+
finding["occurrenceId"] = occurrence_id
|
|
767
|
+
finding["fingerprints"] = fingerprints
|
|
768
|
+
|
|
769
|
+
|
|
770
|
+
def _validate_derived_finding_identities(
|
|
771
|
+
manifest: dict[str, Any],
|
|
772
|
+
findings: dict[str, Any],
|
|
773
|
+
) -> None:
|
|
774
|
+
"""Require sealed finding identities to equal their deterministic values."""
|
|
775
|
+
|
|
776
|
+
for context, finding, finding_id, occurrence_id, fingerprints in _derived_finding_identity_rows(
|
|
777
|
+
manifest,
|
|
778
|
+
findings,
|
|
779
|
+
):
|
|
780
|
+
if finding.get("findingId") != finding_id:
|
|
781
|
+
raise ContractError(f"{context}.findingId: does not match derived fingerprint identity")
|
|
782
|
+
if finding.get("occurrenceId") != occurrence_id:
|
|
783
|
+
raise ContractError(f"{context}.occurrenceId: does not match scan occurrence identity")
|
|
784
|
+
if finding.get("fingerprints") != fingerprints:
|
|
785
|
+
raise ContractError(f"{context}.fingerprints: does not match derived fingerprint")
|
|
786
|
+
|
|
787
|
+
|
|
788
|
+
def _populate_unsealed_manifest_envelope(
|
|
789
|
+
manifest: dict[str, Any],
|
|
790
|
+
scan: dict[str, Any],
|
|
791
|
+
completion_binding: dict[str, Any] | None,
|
|
792
|
+
) -> None:
|
|
793
|
+
"""Populate non-semantic draft fields owned by finalization or the workbench."""
|
|
794
|
+
|
|
795
|
+
manifest["documentType"] = "codex-security.scan-manifest"
|
|
796
|
+
manifest["schemaVersion"] = SCHEMA_VERSION
|
|
797
|
+
scan["status"] = "completed"
|
|
798
|
+
scan["coverageRef"] = "coverage.json"
|
|
799
|
+
scan["findingsRef"] = "findings.json"
|
|
800
|
+
if completion_binding is None:
|
|
801
|
+
started_at = os.environ.get("CODEX_SECURITY_STARTED_AT")
|
|
802
|
+
if started_at is not None:
|
|
803
|
+
_validate_date_time(started_at, "CODEX_SECURITY_STARTED_AT")
|
|
804
|
+
scan["startedAt"] = started_at
|
|
805
|
+
scan["completedAt"] = datetime.now(timezone.utc).isoformat().replace("+00:00", "Z")
|
|
806
|
+
return
|
|
807
|
+
|
|
808
|
+
scan["id"] = completion_binding["scanId"]
|
|
809
|
+
scan["startedAt"] = completion_binding["startedAt"]
|
|
810
|
+
scan["completedAt"] = completion_binding["completedAt"]
|
|
811
|
+
scan["producer"] = copy.deepcopy(completion_binding["producer"])
|
|
812
|
+
|
|
813
|
+
target = scan.get("target")
|
|
814
|
+
if isinstance(target, dict):
|
|
815
|
+
_populate_unsealed_target_binding(target, completion_binding["target"])
|
|
816
|
+
|
|
817
|
+
scope = scan.get("scope")
|
|
818
|
+
if isinstance(scope, dict):
|
|
819
|
+
scope.update(copy.deepcopy(completion_binding["scope"]))
|
|
820
|
+
|
|
821
|
+
|
|
822
|
+
def _populate_unsealed_target_binding(
|
|
823
|
+
target: dict[str, Any],
|
|
824
|
+
target_binding: dict[str, Any],
|
|
825
|
+
) -> None:
|
|
826
|
+
"""Replace workbench-owned target coordinates without retaining incompatible drafts."""
|
|
827
|
+
|
|
828
|
+
target_kind = target.get("kind")
|
|
829
|
+
required_coordinates = (
|
|
830
|
+
TARGET_REQUIRED_COORDINATE_FIELDS.get(target_kind, set())
|
|
831
|
+
if isinstance(target_kind, str)
|
|
832
|
+
else set()
|
|
833
|
+
)
|
|
834
|
+
for field in TARGET_COORDINATE_FIELDS:
|
|
835
|
+
if field not in target_binding and field not in required_coordinates:
|
|
836
|
+
target.pop(field, None)
|
|
837
|
+
target.update(copy.deepcopy(target_binding))
|
|
838
|
+
|
|
839
|
+
|
|
840
|
+
def _populate_unsealed_artifact_envelope(
|
|
841
|
+
manifest: dict[str, Any],
|
|
842
|
+
findings: dict[str, Any],
|
|
843
|
+
coverage: dict[str, Any],
|
|
844
|
+
completion_binding: dict[str, Any] | None,
|
|
845
|
+
) -> None:
|
|
846
|
+
"""Populate deterministic top-level draft fields after refs have been resolved."""
|
|
847
|
+
|
|
848
|
+
findings["documentType"] = "codex-security.findings"
|
|
849
|
+
findings["schemaVersion"] = SCHEMA_VERSION
|
|
850
|
+
coverage["documentType"] = "codex-security.coverage"
|
|
851
|
+
coverage["schemaVersion"] = SCHEMA_VERSION
|
|
852
|
+
if completion_binding is None:
|
|
853
|
+
return
|
|
854
|
+
|
|
855
|
+
scan_id = completion_binding["scanId"]
|
|
856
|
+
findings["scanId"] = scan_id
|
|
857
|
+
coverage["scanId"] = scan_id
|
|
858
|
+
coverage["mode"] = completion_binding["coverageMode"]
|
|
859
|
+
|
|
860
|
+
scan = _require_dict(manifest, "scan", "manifest")
|
|
861
|
+
scope = _require_dict(scan, "scope", "manifest.scan")
|
|
862
|
+
if "includePaths" in scope:
|
|
863
|
+
coverage["includePaths"] = copy.deepcopy(scope["includePaths"])
|
|
864
|
+
if "excludePaths" in scope:
|
|
865
|
+
coverage["excludePaths"] = copy.deepcopy(scope["excludePaths"])
|
|
866
|
+
|
|
867
|
+
|
|
868
|
+
def _normalize_unsealed_open_questions(coverage: dict[str, Any]) -> None:
|
|
869
|
+
"""Keep only schema-valid optional open-question rows without inventing content."""
|
|
870
|
+
|
|
871
|
+
open_questions = coverage.get("openQuestions")
|
|
872
|
+
if not isinstance(open_questions, list):
|
|
873
|
+
coverage.pop("openQuestions", None)
|
|
874
|
+
return
|
|
875
|
+
|
|
876
|
+
normalized: list[Any] = []
|
|
877
|
+
for item in open_questions:
|
|
878
|
+
if isinstance(item, str):
|
|
879
|
+
question = item.strip()
|
|
880
|
+
if question:
|
|
881
|
+
normalized.append({"question": question})
|
|
882
|
+
continue
|
|
883
|
+
if not isinstance(item, dict):
|
|
884
|
+
continue
|
|
885
|
+
|
|
886
|
+
question = item.get("question")
|
|
887
|
+
if not isinstance(question, str) or not question.strip():
|
|
888
|
+
continue
|
|
889
|
+
row = dict(item)
|
|
890
|
+
row["question"] = question.strip()
|
|
891
|
+
follow_up = row.get("followUpPrompt")
|
|
892
|
+
if not isinstance(follow_up, str) or not follow_up.strip():
|
|
893
|
+
row.pop("followUpPrompt", None)
|
|
894
|
+
normalized.append(row)
|
|
895
|
+
coverage["openQuestions"] = normalized
|
|
896
|
+
|
|
897
|
+
|
|
898
|
+
def _normalize_unsealed_deep_repository_inventory_strategy(
|
|
899
|
+
coverage: dict[str, Any],
|
|
900
|
+
*,
|
|
901
|
+
expected_coverage_mode: str | None,
|
|
902
|
+
) -> None:
|
|
903
|
+
"""Normalize the old Deep workflow label to the ordinary repository inventory."""
|
|
904
|
+
|
|
905
|
+
if (
|
|
906
|
+
expected_coverage_mode == "deep_repository"
|
|
907
|
+
and coverage.get("inventoryStrategy") == "deep_repository_repeated_discovery"
|
|
908
|
+
):
|
|
909
|
+
coverage["inventoryStrategy"] = "repository"
|
|
910
|
+
|
|
911
|
+
|
|
912
|
+
def _validate_completion_binding(
|
|
913
|
+
manifest: dict[str, Any],
|
|
914
|
+
findings: dict[str, Any],
|
|
915
|
+
coverage: dict[str, Any],
|
|
916
|
+
completion_binding: dict[str, Any] | None,
|
|
917
|
+
) -> None:
|
|
918
|
+
"""Verify populated workbench-owned fields before an unsealed draft is written."""
|
|
919
|
+
|
|
920
|
+
if completion_binding is None:
|
|
921
|
+
return
|
|
922
|
+
scan = _require_dict(manifest, "scan", "manifest")
|
|
923
|
+
if scan.get("id") != completion_binding["scanId"]:
|
|
924
|
+
raise ContractError("manifest.scan.id: must match the workbench scan")
|
|
925
|
+
if scan.get("startedAt") != completion_binding["startedAt"]:
|
|
926
|
+
raise ContractError("manifest.scan.startedAt: must match the workbench scan")
|
|
927
|
+
if scan.get("completedAt") != completion_binding["completedAt"]:
|
|
928
|
+
raise ContractError("manifest.scan.completedAt: must match the workbench completion")
|
|
929
|
+
if scan.get("producer") != completion_binding["producer"]:
|
|
930
|
+
raise ContractError("manifest.scan.producer: must match the workbench producer")
|
|
931
|
+
target = _require_dict(scan, "target", "manifest.scan")
|
|
932
|
+
allowed_target_kinds = completion_binding["allowedTargetKinds"]
|
|
933
|
+
if target.get("kind") not in allowed_target_kinds:
|
|
934
|
+
raise ContractError("scan.target.kind: must match the workbench target")
|
|
935
|
+
for key, expected in completion_binding["target"].items():
|
|
936
|
+
if target.get(key) != expected:
|
|
937
|
+
raise ContractError(f"scan.target.{key}: must match the workbench target")
|
|
938
|
+
scope = _require_dict(scan, "scope", "manifest.scan")
|
|
939
|
+
for key, expected in completion_binding["scope"].items():
|
|
940
|
+
if scope.get(key) != expected:
|
|
941
|
+
raise ContractError(f"manifest.scan.scope.{key}: must match the workbench scan")
|
|
942
|
+
if findings.get("scanId") != completion_binding["scanId"]:
|
|
943
|
+
raise ContractError("findings.scanId: must match the workbench scan")
|
|
944
|
+
if coverage.get("scanId") != completion_binding["scanId"]:
|
|
945
|
+
raise ContractError("coverage.scanId: must match the workbench scan")
|
|
946
|
+
if coverage.get("mode") != completion_binding["coverageMode"]:
|
|
947
|
+
raise ContractError("coverage.mode: must match the workbench scan")
|
|
948
|
+
for key, expected in completion_binding["scope"].items():
|
|
949
|
+
if coverage.get(key) != expected:
|
|
950
|
+
raise ContractError(f"coverage.{key}: must match the workbench scan")
|
|
951
|
+
|
|
952
|
+
|
|
953
|
+
def _validate_finding(finding: dict[str, Any], context: str) -> None:
|
|
954
|
+
for key in ("findingId", "occurrenceId", "ruleId", "title", "summary", "remediation"):
|
|
955
|
+
_require_str(finding, key, context)
|
|
956
|
+
_require_dict(finding, "identity", context)
|
|
957
|
+
fingerprints = _require_dict(finding, "fingerprints", context)
|
|
958
|
+
if fingerprints.get("algorithm") != FINGERPRINT_ALGORITHM:
|
|
959
|
+
raise ContractError(f"{context}.fingerprints.algorithm: unsupported algorithm")
|
|
960
|
+
_require_str(fingerprints, "primary", f"{context}.fingerprints")
|
|
961
|
+
|
|
962
|
+
severity = _require_dict(finding, "severity", context)
|
|
963
|
+
level = _require_str(severity, "level", f"{context}.severity")
|
|
964
|
+
if level not in SEVERITIES:
|
|
965
|
+
raise ContractError(f"{context}.severity.level: unsupported severity: {level}")
|
|
966
|
+
score = severity.get("score")
|
|
967
|
+
if score is not None:
|
|
968
|
+
if not isinstance(score, (int, float)) or isinstance(score, bool) or not 0 <= score <= 10:
|
|
969
|
+
raise ContractError(f"{context}.severity.score: expected a number from 0 through 10")
|
|
970
|
+
_require_str(severity, "scoringSystem", f"{context}.severity")
|
|
971
|
+
|
|
972
|
+
confidence = _require_dict(finding, "confidence", context)
|
|
973
|
+
confidence_level = _require_str(confidence, "level", f"{context}.confidence")
|
|
974
|
+
if confidence_level not in CONFIDENCES:
|
|
975
|
+
raise ContractError(
|
|
976
|
+
f"{context}.confidence.level: unsupported confidence: {confidence_level}"
|
|
977
|
+
)
|
|
978
|
+
_require_str(confidence, "rationale", f"{context}.confidence")
|
|
979
|
+
|
|
980
|
+
taxonomy = _require_dict(finding, "taxonomy", context)
|
|
981
|
+
_require_str(taxonomy, "category", f"{context}.taxonomy")
|
|
982
|
+
cwe = taxonomy.get("cwe", [])
|
|
983
|
+
if not isinstance(cwe, list) or any(not isinstance(item, str) or not item for item in cwe):
|
|
984
|
+
raise ContractError(f"{context}.taxonomy.cwe: expected an array of strings")
|
|
985
|
+
|
|
986
|
+
locations = _require_list(finding, "locations", context)
|
|
987
|
+
if not locations:
|
|
988
|
+
raise ContractError(f"{context}.locations: expected at least one location")
|
|
989
|
+
for index, location in enumerate(locations):
|
|
990
|
+
if not isinstance(location, dict):
|
|
991
|
+
raise ContractError(f"{context}.locations[{index}]: expected an object")
|
|
992
|
+
_validate_location(location, f"{context}.locations[{index}]")
|
|
993
|
+
|
|
994
|
+
evidence_ids: set[str] = set()
|
|
995
|
+
code_evidence = finding.get("codeEvidence")
|
|
996
|
+
if code_evidence is not None:
|
|
997
|
+
if not isinstance(code_evidence, list):
|
|
998
|
+
raise ContractError(f"{context}.codeEvidence: expected an array")
|
|
999
|
+
for index, evidence in enumerate(code_evidence):
|
|
1000
|
+
evidence_context = f"{context}.codeEvidence[{index}]"
|
|
1001
|
+
if not isinstance(evidence, dict):
|
|
1002
|
+
raise ContractError(f"{evidence_context}: expected an object")
|
|
1003
|
+
evidence_id = _require_str(evidence, "id", evidence_context)
|
|
1004
|
+
if evidence_id in evidence_ids:
|
|
1005
|
+
raise ContractError(f"{evidence_context}.id: duplicate code-evidence id")
|
|
1006
|
+
evidence_ids.add(evidence_id)
|
|
1007
|
+
_require_str(evidence, "code", evidence_context)
|
|
1008
|
+
|
|
1009
|
+
for section_name in ("rootCause", "validation", "attackPath"):
|
|
1010
|
+
section = finding.get(section_name)
|
|
1011
|
+
if not isinstance(section, dict) or "evidenceRefs" not in section:
|
|
1012
|
+
continue
|
|
1013
|
+
refs = section["evidenceRefs"]
|
|
1014
|
+
if not isinstance(refs, list) or any(not isinstance(ref, str) or not ref for ref in refs):
|
|
1015
|
+
raise ContractError(f"{context}.{section_name}.evidenceRefs: expected strings")
|
|
1016
|
+
unknown_refs = sorted(set(refs) - evidence_ids)
|
|
1017
|
+
if unknown_refs:
|
|
1018
|
+
raise ContractError(
|
|
1019
|
+
f"{context}.{section_name}.evidenceRefs: unknown code-evidence ids: "
|
|
1020
|
+
+ ", ".join(unknown_refs)
|
|
1021
|
+
)
|
|
1022
|
+
|
|
1023
|
+
provenance = _require_dict(finding, "provenance", context)
|
|
1024
|
+
_require_str(provenance, "source", f"{context}.provenance")
|
|
1025
|
+
extensions = finding.get("extensions")
|
|
1026
|
+
if extensions is not None and not isinstance(extensions, dict):
|
|
1027
|
+
raise ContractError(f"{context}.extensions: expected an object")
|
|
1028
|
+
|
|
1029
|
+
|
|
1030
|
+
def _validate_coverage(manifest: dict[str, Any], coverage: dict[str, Any], scan_dir: Path) -> None:
|
|
1031
|
+
scan = _require_dict(manifest, "scan", "manifest")
|
|
1032
|
+
scan_id = _require_str(scan, "id", "manifest.scan")
|
|
1033
|
+
if coverage.get("scanId") != scan_id:
|
|
1034
|
+
raise ContractError("coverage.scanId: must match manifest scan id")
|
|
1035
|
+
_require_str(coverage, "mode", "coverage")
|
|
1036
|
+
completeness = _require_str(coverage, "completeness", "coverage")
|
|
1037
|
+
_require_str(coverage, "inventoryStrategy", "coverage")
|
|
1038
|
+
scope = _require_dict(scan, "scope", "manifest.scan")
|
|
1039
|
+
if coverage.get("includePaths") != scope.get("includePaths"):
|
|
1040
|
+
raise ContractError("coverage.includePaths: must match manifest scope")
|
|
1041
|
+
if coverage.get("excludePaths") != scope.get("excludePaths"):
|
|
1042
|
+
raise ContractError("coverage.excludePaths: must match manifest scope")
|
|
1043
|
+
surface_ids: set[str] = set()
|
|
1044
|
+
has_needs_follow_up = False
|
|
1045
|
+
for index, surface in enumerate(_require_list(coverage, "surfaces", "coverage")):
|
|
1046
|
+
context = f"coverage.surfaces[{index}]"
|
|
1047
|
+
if not isinstance(surface, dict):
|
|
1048
|
+
raise ContractError(f"{context}: expected an object")
|
|
1049
|
+
surface_id = _require_str(surface, "id", context)
|
|
1050
|
+
if surface_id in surface_ids:
|
|
1051
|
+
raise ContractError(f"{context}.id: duplicate surface id")
|
|
1052
|
+
surface_ids.add(surface_id)
|
|
1053
|
+
_require_str(surface, "label", context)
|
|
1054
|
+
disposition = _require_str(surface, "disposition", context)
|
|
1055
|
+
if disposition not in DISPOSITIONS:
|
|
1056
|
+
raise ContractError(f"{context}.disposition: unsupported disposition: {disposition}")
|
|
1057
|
+
has_needs_follow_up = has_needs_follow_up or disposition == "needs_follow_up"
|
|
1058
|
+
receipt_refs = surface.get("receiptRefs", [])
|
|
1059
|
+
if not isinstance(receipt_refs, list):
|
|
1060
|
+
raise ContractError(f"{context}.receiptRefs: expected an array")
|
|
1061
|
+
for ref_index, ref in enumerate(receipt_refs):
|
|
1062
|
+
if not isinstance(ref, str):
|
|
1063
|
+
raise ContractError(f"{context}.receiptRefs[{ref_index}]: expected a string")
|
|
1064
|
+
normalized_ref = _require_safe_relative_path(ref, f"{context}.receiptRefs[{ref_index}]")
|
|
1065
|
+
if not normalized_ref.startswith("artifacts/"):
|
|
1066
|
+
raise ContractError(
|
|
1067
|
+
f"{context}.receiptRefs[{ref_index}]: expected a file under artifacts/"
|
|
1068
|
+
)
|
|
1069
|
+
receipt_refs[ref_index] = normalized_ref
|
|
1070
|
+
_require_scan_local_file(
|
|
1071
|
+
scan_dir, normalized_ref, f"{context}.receiptRefs[{ref_index}]"
|
|
1072
|
+
)
|
|
1073
|
+
for field in ("explicitExclusions", "deferred"):
|
|
1074
|
+
if not isinstance(coverage.get(field, []), list):
|
|
1075
|
+
raise ContractError(f"coverage.{field}: expected an array")
|
|
1076
|
+
if completeness == "complete" and (has_needs_follow_up or coverage.get("deferred")):
|
|
1077
|
+
raise ContractError("coverage.completeness: complete coverage cannot have deferred work")
|
|
1078
|
+
_require_safe_json_value(coverage, "coverage.json")
|
|
1079
|
+
|
|
1080
|
+
|
|
1081
|
+
def _validate_manifest(manifest: dict[str, Any]) -> None:
|
|
1082
|
+
if manifest.get("documentType") != "codex-security.scan-manifest":
|
|
1083
|
+
raise ContractError("manifest.documentType: expected codex-security.scan-manifest")
|
|
1084
|
+
if manifest.get("schemaVersion") != SCHEMA_VERSION:
|
|
1085
|
+
raise ContractError(f"manifest.schemaVersion: expected {SCHEMA_VERSION}")
|
|
1086
|
+
scan = _require_dict(manifest, "scan", "manifest")
|
|
1087
|
+
for key in ("id", "startedAt", "completedAt", "sealedAt"):
|
|
1088
|
+
_require_str(scan, key, "manifest.scan")
|
|
1089
|
+
if scan.get("status") != "completed":
|
|
1090
|
+
raise ContractError("manifest.scan.status: expected completed")
|
|
1091
|
+
producer = _require_dict(scan, "producer", "manifest.scan")
|
|
1092
|
+
_require_str(producer, "name", "manifest.scan.producer")
|
|
1093
|
+
_require_str(producer, "version", "manifest.scan.producer")
|
|
1094
|
+
_validate_target(_require_dict(scan, "target", "manifest.scan"))
|
|
1095
|
+
scope = _require_dict(scan, "scope", "manifest.scan")
|
|
1096
|
+
for field in ("includePaths", "excludePaths"):
|
|
1097
|
+
values = _require_list(scope, field, "manifest.scan.scope")
|
|
1098
|
+
for index, value in enumerate(values):
|
|
1099
|
+
if not isinstance(value, str):
|
|
1100
|
+
raise ContractError(f"manifest.scan.scope.{field}[{index}]: expected a string")
|
|
1101
|
+
_require_safe_relative_path(
|
|
1102
|
+
value, f"manifest.scan.scope.{field}[{index}]", allow_dot=True
|
|
1103
|
+
)
|
|
1104
|
+
_validate_contract_refs(scan)
|
|
1105
|
+
artifacts = _require_list(scan, "artifacts", "manifest.scan")
|
|
1106
|
+
if not artifacts:
|
|
1107
|
+
raise ContractError("manifest.scan.artifacts: expected generated artifact records")
|
|
1108
|
+
artifact_paths: set[str] = set()
|
|
1109
|
+
for index, artifact in enumerate(artifacts):
|
|
1110
|
+
context = f"manifest.scan.artifacts[{index}]"
|
|
1111
|
+
if not isinstance(artifact, dict):
|
|
1112
|
+
raise ContractError(f"{context}: expected an object")
|
|
1113
|
+
path = _require_safe_relative_path(
|
|
1114
|
+
_require_str(artifact, "path", context), f"{context}.path"
|
|
1115
|
+
)
|
|
1116
|
+
if path in artifact_paths:
|
|
1117
|
+
raise ContractError(f"{context}.path: duplicate artifact path")
|
|
1118
|
+
artifact_paths.add(path)
|
|
1119
|
+
_require_str(artifact, "sha256", context)
|
|
1120
|
+
_require_str(artifact, "mediaType", context)
|
|
1121
|
+
for required_path in ("findings.json", "coverage.json"):
|
|
1122
|
+
if required_path not in artifact_paths:
|
|
1123
|
+
raise ContractError(
|
|
1124
|
+
f"manifest.scan.artifacts: missing required artifact: {required_path}"
|
|
1125
|
+
)
|
|
1126
|
+
_require_safe_json_value(manifest, "scan-manifest.json")
|
|
1127
|
+
|
|
1128
|
+
|
|
1129
|
+
def _validate_findings(manifest: dict[str, Any], findings: dict[str, Any]) -> None:
|
|
1130
|
+
if findings.get("documentType") != "codex-security.findings":
|
|
1131
|
+
raise ContractError("findings.documentType: expected codex-security.findings")
|
|
1132
|
+
if findings.get("schemaVersion") != SCHEMA_VERSION:
|
|
1133
|
+
raise ContractError(f"findings.schemaVersion: expected {SCHEMA_VERSION}")
|
|
1134
|
+
scan_id = _require_str(_require_dict(manifest, "scan", "manifest"), "id", "manifest.scan")
|
|
1135
|
+
if findings.get("scanId") != scan_id:
|
|
1136
|
+
raise ContractError("findings.scanId: must match manifest scan id")
|
|
1137
|
+
finding_ids: set[str] = set()
|
|
1138
|
+
occurrence_ids: set[str] = set()
|
|
1139
|
+
for index, finding in enumerate(_require_list(findings, "findings", "findings")):
|
|
1140
|
+
context = f"findings.findings[{index}]"
|
|
1141
|
+
if not isinstance(finding, dict):
|
|
1142
|
+
raise ContractError(f"{context}: expected an object")
|
|
1143
|
+
_validate_finding(finding, context)
|
|
1144
|
+
finding_id = str(finding["findingId"])
|
|
1145
|
+
occurrence_id = str(finding["occurrenceId"])
|
|
1146
|
+
if finding_id in finding_ids or occurrence_id in occurrence_ids:
|
|
1147
|
+
raise ContractError(f"{context}: duplicate finding or occurrence id")
|
|
1148
|
+
finding_ids.add(finding_id)
|
|
1149
|
+
occurrence_ids.add(occurrence_id)
|
|
1150
|
+
_require_safe_json_value(findings, "findings.json")
|
|
1151
|
+
|
|
1152
|
+
|
|
1153
|
+
def _schema_type_matches(value: Any, expected: str) -> bool:
|
|
1154
|
+
return {
|
|
1155
|
+
"array": isinstance(value, list),
|
|
1156
|
+
"boolean": isinstance(value, bool),
|
|
1157
|
+
"integer": isinstance(value, int) and not isinstance(value, bool),
|
|
1158
|
+
"number": isinstance(value, (int, float)) and not isinstance(value, bool),
|
|
1159
|
+
"object": isinstance(value, dict),
|
|
1160
|
+
"string": isinstance(value, str),
|
|
1161
|
+
"null": value is None,
|
|
1162
|
+
}[expected]
|
|
1163
|
+
|
|
1164
|
+
|
|
1165
|
+
def _validate_schema_node(value: Any, schema: dict[str, Any], context: str) -> None:
|
|
1166
|
+
expected = schema.get("type")
|
|
1167
|
+
if isinstance(expected, list):
|
|
1168
|
+
if not any(_schema_type_matches(value, item) for item in expected):
|
|
1169
|
+
raise ContractError(f"{context}: does not match schema type {expected}")
|
|
1170
|
+
elif isinstance(expected, str) and not _schema_type_matches(value, expected):
|
|
1171
|
+
raise ContractError(f"{context}: expected schema type {expected}")
|
|
1172
|
+
if "const" in schema and value != schema["const"]:
|
|
1173
|
+
raise ContractError(f"{context}: expected {schema['const']!r}")
|
|
1174
|
+
if "enum" in schema and value not in schema["enum"]:
|
|
1175
|
+
raise ContractError(f"{context}: unsupported value {value!r}")
|
|
1176
|
+
if isinstance(value, str):
|
|
1177
|
+
if schema.get("minLength", 0) and len(value) < schema["minLength"]:
|
|
1178
|
+
raise ContractError(f"{context}: string is too short")
|
|
1179
|
+
if "pattern" in schema and not re.fullmatch(schema["pattern"], value):
|
|
1180
|
+
raise ContractError(f"{context}: string does not match schema pattern")
|
|
1181
|
+
if schema.get("format") == "date-time":
|
|
1182
|
+
_validate_date_time(value, context)
|
|
1183
|
+
if isinstance(value, (int, float)) and not isinstance(value, bool):
|
|
1184
|
+
if "minimum" in schema and value < schema["minimum"]:
|
|
1185
|
+
raise ContractError(f"{context}: value is below schema minimum")
|
|
1186
|
+
if "maximum" in schema and value > schema["maximum"]:
|
|
1187
|
+
raise ContractError(f"{context}: value is above schema maximum")
|
|
1188
|
+
if isinstance(value, list):
|
|
1189
|
+
if "minItems" in schema and len(value) < schema["minItems"]:
|
|
1190
|
+
raise ContractError(f"{context}: array has too few items")
|
|
1191
|
+
contains = schema.get("contains")
|
|
1192
|
+
if isinstance(contains, dict):
|
|
1193
|
+
matches = 0
|
|
1194
|
+
for item in value:
|
|
1195
|
+
try:
|
|
1196
|
+
_validate_schema_node(item, contains, context)
|
|
1197
|
+
except ContractError:
|
|
1198
|
+
pass
|
|
1199
|
+
else:
|
|
1200
|
+
matches += 1
|
|
1201
|
+
if matches < schema.get("minContains", 1):
|
|
1202
|
+
raise ContractError(f"{context}: array contains too few matching items")
|
|
1203
|
+
if "maxContains" in schema and matches > schema["maxContains"]:
|
|
1204
|
+
raise ContractError(f"{context}: array contains too many matching items")
|
|
1205
|
+
item_schema = schema.get("items")
|
|
1206
|
+
if isinstance(item_schema, dict):
|
|
1207
|
+
for index, item in enumerate(value):
|
|
1208
|
+
_validate_schema_node(item, item_schema, f"{context}[{index}]")
|
|
1209
|
+
if isinstance(value, dict):
|
|
1210
|
+
for item_schema in schema.get("allOf", []):
|
|
1211
|
+
_validate_schema_node(value, item_schema, context)
|
|
1212
|
+
condition = schema.get("if")
|
|
1213
|
+
if isinstance(condition, dict):
|
|
1214
|
+
try:
|
|
1215
|
+
_validate_schema_node(value, condition, context)
|
|
1216
|
+
except ContractError:
|
|
1217
|
+
pass
|
|
1218
|
+
else:
|
|
1219
|
+
then_schema = schema.get("then")
|
|
1220
|
+
if isinstance(then_schema, dict):
|
|
1221
|
+
_validate_schema_node(value, then_schema, context)
|
|
1222
|
+
for key in schema.get("required", []):
|
|
1223
|
+
if key not in value:
|
|
1224
|
+
raise ContractError(f"{context}.{key}: missing required schema property")
|
|
1225
|
+
properties = schema.get("properties", {})
|
|
1226
|
+
for key, item in value.items():
|
|
1227
|
+
item_schema = properties.get(key)
|
|
1228
|
+
if isinstance(item_schema, dict):
|
|
1229
|
+
_validate_schema_node(item, item_schema, f"{context}.{key}")
|
|
1230
|
+
elif schema.get("additionalProperties") is False:
|
|
1231
|
+
raise ContractError(f"{context}.{key}: unexpected schema property")
|
|
1232
|
+
|
|
1233
|
+
|
|
1234
|
+
def validate_against_schema(payload: dict[str, Any], schema_path: Path) -> None:
|
|
1235
|
+
schema = _read_json(schema_path)
|
|
1236
|
+
_require_safe_schema(schema, schema_path.name)
|
|
1237
|
+
_validate_schema_node(payload, schema, schema_path.stem)
|
|
1238
|
+
|
|
1239
|
+
|
|
1240
|
+
def _require_safe_schema(schema: dict[str, Any], context: str) -> None:
|
|
1241
|
+
pending: list[tuple[Any, bool]] = [(schema, True)]
|
|
1242
|
+
nodes = 0
|
|
1243
|
+
applicator_edges = 0
|
|
1244
|
+
unsupported_keywords = {
|
|
1245
|
+
"$async",
|
|
1246
|
+
"$ref",
|
|
1247
|
+
"$dynamicRef",
|
|
1248
|
+
"$recursiveRef",
|
|
1249
|
+
"prefixItems",
|
|
1250
|
+
"patternProperties",
|
|
1251
|
+
"propertyNames",
|
|
1252
|
+
"dependentSchemas",
|
|
1253
|
+
"dependencies",
|
|
1254
|
+
"uniqueItems",
|
|
1255
|
+
}
|
|
1256
|
+
while pending:
|
|
1257
|
+
value, is_schema = pending.pop()
|
|
1258
|
+
nodes += 1
|
|
1259
|
+
if nodes > MAX_SCHEMA_NODES:
|
|
1260
|
+
raise ContractError(
|
|
1261
|
+
f"{context}: JSON Schema exceeds the {MAX_SCHEMA_NODES}-node complexity limit"
|
|
1262
|
+
)
|
|
1263
|
+
if isinstance(value, list):
|
|
1264
|
+
if len(value) > MAX_SCHEMA_COLLECTION_ENTRIES:
|
|
1265
|
+
raise ContractError(
|
|
1266
|
+
f"{context}: JSON Schema exceeds the "
|
|
1267
|
+
f"{MAX_SCHEMA_COLLECTION_ENTRIES}-entry collection limit"
|
|
1268
|
+
)
|
|
1269
|
+
pending.extend((child, is_schema) for child in value)
|
|
1270
|
+
continue
|
|
1271
|
+
if not isinstance(value, dict):
|
|
1272
|
+
continue
|
|
1273
|
+
if len(value) > MAX_SCHEMA_COLLECTION_ENTRIES:
|
|
1274
|
+
raise ContractError(
|
|
1275
|
+
f"{context}: JSON Schema exceeds the "
|
|
1276
|
+
f"{MAX_SCHEMA_COLLECTION_ENTRIES}-entry collection limit"
|
|
1277
|
+
)
|
|
1278
|
+
for keyword, child in value.items():
|
|
1279
|
+
if not is_schema:
|
|
1280
|
+
pending.append((child, False))
|
|
1281
|
+
continue
|
|
1282
|
+
if keyword in unsupported_keywords:
|
|
1283
|
+
raise ContractError(f"{context}: unsupported JSON Schema keyword")
|
|
1284
|
+
edges = 0
|
|
1285
|
+
if keyword in {"allOf", "anyOf", "oneOf"} and isinstance(child, list):
|
|
1286
|
+
edges = len(child)
|
|
1287
|
+
elif keyword in {
|
|
1288
|
+
"if",
|
|
1289
|
+
"then",
|
|
1290
|
+
"else",
|
|
1291
|
+
"not",
|
|
1292
|
+
"items",
|
|
1293
|
+
"contains",
|
|
1294
|
+
"additionalProperties",
|
|
1295
|
+
"unevaluatedProperties",
|
|
1296
|
+
"unevaluatedItems",
|
|
1297
|
+
} and isinstance(child, (dict, bool)):
|
|
1298
|
+
edges = 1
|
|
1299
|
+
elif keyword in {"properties", "$defs", "definitions"} and isinstance(child, dict):
|
|
1300
|
+
edges = len(child)
|
|
1301
|
+
applicator_edges += edges
|
|
1302
|
+
if applicator_edges > MAX_SCHEMA_APPLICATOR_EDGES:
|
|
1303
|
+
raise ContractError(
|
|
1304
|
+
f"{context}: JSON Schema exceeds the "
|
|
1305
|
+
f"{MAX_SCHEMA_APPLICATOR_EDGES}-edge applicator limit"
|
|
1306
|
+
)
|
|
1307
|
+
if keyword == "pattern" and isinstance(child, str):
|
|
1308
|
+
if child not in SAFE_SCHEMA_PATTERNS:
|
|
1309
|
+
raise ContractError(f"{context}: unsupported JSON Schema pattern")
|
|
1310
|
+
if keyword in {"properties", "$defs", "definitions"} and isinstance(child, dict):
|
|
1311
|
+
pending.extend((child_schema, True) for child_schema in child.values())
|
|
1312
|
+
continue
|
|
1313
|
+
pending.append(
|
|
1314
|
+
(
|
|
1315
|
+
child,
|
|
1316
|
+
keyword not in {"const", "enum", "default", "examples", "dependentRequired"},
|
|
1317
|
+
)
|
|
1318
|
+
)
|
|
1319
|
+
|
|
1320
|
+
|
|
1321
|
+
def _validate_canonical_schemas_before_projection(
|
|
1322
|
+
manifest: dict[str, Any],
|
|
1323
|
+
findings: dict[str, Any],
|
|
1324
|
+
coverage: dict[str, Any],
|
|
1325
|
+
schema_dir: Path,
|
|
1326
|
+
) -> None:
|
|
1327
|
+
provisional_manifest = copy.deepcopy(manifest)
|
|
1328
|
+
provisional_scan = _require_dict(provisional_manifest, "scan", "manifest")
|
|
1329
|
+
provisional_scan["artifacts"] = [
|
|
1330
|
+
{"path": "findings.json", "sha256": "0" * 64, "mediaType": "application/json"},
|
|
1331
|
+
{"path": "coverage.json", "sha256": "0" * 64, "mediaType": "application/json"},
|
|
1332
|
+
]
|
|
1333
|
+
validate_against_schema(provisional_manifest, schema_dir / "scan-manifest.schema.json")
|
|
1334
|
+
validate_against_schema(findings, schema_dir / "findings.schema.json")
|
|
1335
|
+
validate_against_schema(coverage, schema_dir / "coverage.schema.json")
|
|
1336
|
+
|
|
1337
|
+
|
|
1338
|
+
def _validate_contract_refs(scan: dict[str, Any]) -> None:
|
|
1339
|
+
for field, expected in (
|
|
1340
|
+
("coverageRef", "coverage.json"),
|
|
1341
|
+
("findingsRef", "findings.json"),
|
|
1342
|
+
):
|
|
1343
|
+
actual = _require_str(scan, field, "manifest.scan")
|
|
1344
|
+
if actual != expected:
|
|
1345
|
+
raise ContractError(f"manifest.scan.{field}: expected {expected!r}")
|
|
1346
|
+
|
|
1347
|
+
|
|
1348
|
+
def _sarif_rule(rule_id: str) -> dict[str, Any]:
|
|
1349
|
+
return {
|
|
1350
|
+
"id": rule_id,
|
|
1351
|
+
"name": rule_id,
|
|
1352
|
+
"shortDescription": {"text": rule_id},
|
|
1353
|
+
"properties": {"tags": ["security"]},
|
|
1354
|
+
}
|
|
1355
|
+
|
|
1356
|
+
|
|
1357
|
+
def _utf16_code_units(value: str) -> Iterator[int]:
|
|
1358
|
+
encoded = value.encode("utf-16-le")
|
|
1359
|
+
for index in range(0, len(encoded), 2):
|
|
1360
|
+
yield int.from_bytes(encoded[index : index + 2], "little")
|
|
1361
|
+
|
|
1362
|
+
|
|
1363
|
+
def _github_line_hashes(
|
|
1364
|
+
handle: TextIO,
|
|
1365
|
+
requested_lines: set[int] | None = None,
|
|
1366
|
+
source_read_budget: list[int] | None = None,
|
|
1367
|
+
) -> dict[int, str] | None:
|
|
1368
|
+
if source_read_budget is not None and source_read_budget[0] <= 0:
|
|
1369
|
+
return None
|
|
1370
|
+
window = [0] * GITHUB_HASH_BLOCK_SIZE
|
|
1371
|
+
line_numbers = [-1] * GITHUB_HASH_BLOCK_SIZE
|
|
1372
|
+
hash_counts: dict[str, int] = {}
|
|
1373
|
+
hashes: dict[int, str] = {}
|
|
1374
|
+
first_mod = pow(GITHUB_HASH_MOD, GITHUB_HASH_BLOCK_SIZE, 1 << 64)
|
|
1375
|
+
hash_raw = 0
|
|
1376
|
+
index = 0
|
|
1377
|
+
line_number = 0
|
|
1378
|
+
line_start = True
|
|
1379
|
+
previous_was_cr = False
|
|
1380
|
+
source_bytes = 0
|
|
1381
|
+
|
|
1382
|
+
def output_hash() -> None:
|
|
1383
|
+
nonlocal index
|
|
1384
|
+
hash_value = format(hash_raw & GITHUB_HASH_MASK, "x")
|
|
1385
|
+
hash_counts[hash_value] = hash_counts.get(hash_value, 0) + 1
|
|
1386
|
+
line_number = line_numbers[index]
|
|
1387
|
+
if requested_lines is None or line_number in requested_lines:
|
|
1388
|
+
hashes[line_number] = f"{hash_value}:{hash_counts[hash_value]}"
|
|
1389
|
+
line_numbers[index] = -1
|
|
1390
|
+
|
|
1391
|
+
def update_hash(current: int) -> None:
|
|
1392
|
+
nonlocal hash_raw, index
|
|
1393
|
+
beginning = window[index]
|
|
1394
|
+
window[index] = current
|
|
1395
|
+
hash_raw = (GITHUB_HASH_MOD * hash_raw + current - first_mod * beginning) & GITHUB_HASH_MASK
|
|
1396
|
+
index = (index + 1) % GITHUB_HASH_BLOCK_SIZE
|
|
1397
|
+
|
|
1398
|
+
def process_character(current: int) -> bool:
|
|
1399
|
+
nonlocal line_number, line_start, previous_was_cr
|
|
1400
|
+
if current in {ord(" "), ord("\t")} or (previous_was_cr and current == ord("\n")):
|
|
1401
|
+
previous_was_cr = False
|
|
1402
|
+
return True
|
|
1403
|
+
if current == ord("\r"):
|
|
1404
|
+
current = ord("\n")
|
|
1405
|
+
previous_was_cr = True
|
|
1406
|
+
else:
|
|
1407
|
+
previous_was_cr = False
|
|
1408
|
+
if line_numbers[index] != -1:
|
|
1409
|
+
output_hash()
|
|
1410
|
+
if line_start:
|
|
1411
|
+
line_start = False
|
|
1412
|
+
line_number += 1
|
|
1413
|
+
if line_number > GITHUB_HASH_MAX_LINES:
|
|
1414
|
+
return False
|
|
1415
|
+
line_numbers[index] = line_number
|
|
1416
|
+
if current == ord("\n"):
|
|
1417
|
+
line_start = True
|
|
1418
|
+
update_hash(current)
|
|
1419
|
+
return True
|
|
1420
|
+
|
|
1421
|
+
while chunk := handle.read(SOURCE_READ_CHUNK_SIZE):
|
|
1422
|
+
chunk_bytes = len(chunk.encode("utf-8", errors="replace"))
|
|
1423
|
+
source_bytes += chunk_bytes
|
|
1424
|
+
if source_bytes > SOURCE_READ_MAX_BYTES:
|
|
1425
|
+
return None
|
|
1426
|
+
if source_read_budget is not None:
|
|
1427
|
+
source_read_budget[0] -= chunk_bytes
|
|
1428
|
+
if source_read_budget[0] < 0:
|
|
1429
|
+
return None
|
|
1430
|
+
for code_unit in _utf16_code_units(chunk):
|
|
1431
|
+
if not process_character(code_unit):
|
|
1432
|
+
return None
|
|
1433
|
+
if not process_character(GITHUB_HASH_EOF):
|
|
1434
|
+
return None
|
|
1435
|
+
for _ in range(GITHUB_HASH_BLOCK_SIZE):
|
|
1436
|
+
if line_numbers[index] != -1:
|
|
1437
|
+
output_hash()
|
|
1438
|
+
update_hash(0)
|
|
1439
|
+
return hashes
|
|
1440
|
+
|
|
1441
|
+
|
|
1442
|
+
def _open_source_file(source_root: Path, relative_path: str) -> TextIO | None:
|
|
1443
|
+
file_fd: int | None = None
|
|
1444
|
+
try:
|
|
1445
|
+
file_fd = open_scan_local_file_descriptor(
|
|
1446
|
+
source_root, relative_path, f"source file {relative_path}"
|
|
1447
|
+
)
|
|
1448
|
+
handle = os.fdopen(file_fd, "r", encoding="utf-8", errors="replace")
|
|
1449
|
+
file_fd = None
|
|
1450
|
+
return handle
|
|
1451
|
+
except (ContractError, OSError, ValueError):
|
|
1452
|
+
return None
|
|
1453
|
+
finally:
|
|
1454
|
+
if file_fd is not None:
|
|
1455
|
+
os.close(file_fd)
|
|
1456
|
+
|
|
1457
|
+
|
|
1458
|
+
def _github_line_hashes_for_source(
|
|
1459
|
+
source_root: Path,
|
|
1460
|
+
relative_path: str,
|
|
1461
|
+
requested_lines: set[int] | None = None,
|
|
1462
|
+
source_read_budget: list[int] | None = None,
|
|
1463
|
+
) -> dict[int, str] | None:
|
|
1464
|
+
handle = _open_source_file(source_root, relative_path)
|
|
1465
|
+
if handle is None:
|
|
1466
|
+
return None
|
|
1467
|
+
try:
|
|
1468
|
+
with handle:
|
|
1469
|
+
return _github_line_hashes(handle, requested_lines, source_read_budget)
|
|
1470
|
+
except OSError:
|
|
1471
|
+
return None
|
|
1472
|
+
|
|
1473
|
+
|
|
1474
|
+
def _sarif_primary_location(finding: dict[str, Any]) -> dict[str, Any]:
|
|
1475
|
+
return next(
|
|
1476
|
+
(location for location in finding["locations"] if location.get("role") == "root_control"),
|
|
1477
|
+
finding["locations"][0],
|
|
1478
|
+
)
|
|
1479
|
+
|
|
1480
|
+
|
|
1481
|
+
def _sarif_locations(finding: dict[str, Any]) -> list[dict[str, Any]]:
|
|
1482
|
+
primary = _sarif_primary_location(finding)
|
|
1483
|
+
locations = [
|
|
1484
|
+
primary,
|
|
1485
|
+
*(location for location in finding["locations"] if location is not primary),
|
|
1486
|
+
]
|
|
1487
|
+
locations.extend(
|
|
1488
|
+
{
|
|
1489
|
+
"path": evidence["path"],
|
|
1490
|
+
"startLine": evidence["startLine"],
|
|
1491
|
+
"endLine": evidence.get("endLine", evidence["startLine"]),
|
|
1492
|
+
"role": f"evidence:{evidence['id']}",
|
|
1493
|
+
}
|
|
1494
|
+
for evidence in finding.get("codeEvidence", [])
|
|
1495
|
+
)
|
|
1496
|
+
unique: dict[tuple[str, int, int], dict[str, Any]] = {}
|
|
1497
|
+
for location in locations:
|
|
1498
|
+
key = (
|
|
1499
|
+
location["path"],
|
|
1500
|
+
location["startLine"],
|
|
1501
|
+
location.get("endLine", location["startLine"]),
|
|
1502
|
+
)
|
|
1503
|
+
unique.setdefault(key, location)
|
|
1504
|
+
return list(unique.values())
|
|
1505
|
+
|
|
1506
|
+
|
|
1507
|
+
def _github_primary_location_line_hash(
|
|
1508
|
+
finding: dict[str, Any],
|
|
1509
|
+
source_root: Path | None,
|
|
1510
|
+
line_hash_cache: dict[tuple[Path, int], str | None] | None = None,
|
|
1511
|
+
) -> str | None:
|
|
1512
|
+
if source_root is None:
|
|
1513
|
+
return None
|
|
1514
|
+
primary_location = _sarif_primary_location(finding)
|
|
1515
|
+
try:
|
|
1516
|
+
source_root = source_root.resolve(strict=True)
|
|
1517
|
+
except (OSError, RuntimeError):
|
|
1518
|
+
return None
|
|
1519
|
+
relative_path = _require_safe_relative_path(primary_location["path"], "SARIF source location")
|
|
1520
|
+
source_path = source_root / relative_path
|
|
1521
|
+
start_line = primary_location["startLine"]
|
|
1522
|
+
cache_key = (source_path, start_line)
|
|
1523
|
+
if line_hash_cache is not None and cache_key in line_hash_cache:
|
|
1524
|
+
return line_hash_cache[cache_key]
|
|
1525
|
+
line_hashes = _github_line_hashes_for_source(source_root, relative_path, {start_line})
|
|
1526
|
+
line_hash = None if line_hashes is None else line_hashes.get(start_line)
|
|
1527
|
+
if line_hash_cache is not None:
|
|
1528
|
+
line_hash_cache[cache_key] = line_hash
|
|
1529
|
+
return line_hash
|
|
1530
|
+
|
|
1531
|
+
|
|
1532
|
+
def _github_line_hash_cache(
|
|
1533
|
+
findings: list[dict[str, Any]], source_root: Path | None
|
|
1534
|
+
) -> dict[tuple[Path, int], str | None]:
|
|
1535
|
+
if source_root is None:
|
|
1536
|
+
return {}
|
|
1537
|
+
try:
|
|
1538
|
+
source_root = source_root.resolve(strict=True)
|
|
1539
|
+
except (OSError, RuntimeError):
|
|
1540
|
+
return {}
|
|
1541
|
+
requested_lines_by_path: dict[str, set[int]] = {}
|
|
1542
|
+
for finding in findings:
|
|
1543
|
+
primary_location = _sarif_primary_location(finding)
|
|
1544
|
+
relative_path = _require_safe_relative_path(
|
|
1545
|
+
primary_location["path"], "SARIF source location"
|
|
1546
|
+
)
|
|
1547
|
+
requested_lines_by_path.setdefault(relative_path, set()).add(primary_location["startLine"])
|
|
1548
|
+
line_hash_cache: dict[tuple[Path, int], str | None] = {}
|
|
1549
|
+
source_read_budget = [SOURCE_READ_MAX_BYTES]
|
|
1550
|
+
for relative_path, requested_lines in requested_lines_by_path.items():
|
|
1551
|
+
line_hashes = (
|
|
1552
|
+
None
|
|
1553
|
+
if source_read_budget[0] <= 0
|
|
1554
|
+
else _github_line_hashes_for_source(
|
|
1555
|
+
source_root, relative_path, requested_lines, source_read_budget
|
|
1556
|
+
)
|
|
1557
|
+
)
|
|
1558
|
+
source_path = source_root / relative_path
|
|
1559
|
+
for line_number in requested_lines:
|
|
1560
|
+
line_hash_cache[(source_path, line_number)] = (
|
|
1561
|
+
None if line_hashes is None else line_hashes.get(line_number)
|
|
1562
|
+
)
|
|
1563
|
+
return line_hash_cache
|
|
1564
|
+
|
|
1565
|
+
|
|
1566
|
+
def _sarif_location(location: dict[str, Any], location_id: int | None = None) -> dict[str, Any]:
|
|
1567
|
+
sarif_location: dict[str, Any] = {
|
|
1568
|
+
"physicalLocation": {
|
|
1569
|
+
"artifactLocation": {
|
|
1570
|
+
"uri": quote(location["path"], safe="/"),
|
|
1571
|
+
},
|
|
1572
|
+
"region": {
|
|
1573
|
+
"startLine": location["startLine"],
|
|
1574
|
+
"endLine": location.get("endLine", location["startLine"]),
|
|
1575
|
+
},
|
|
1576
|
+
}
|
|
1577
|
+
}
|
|
1578
|
+
if location_id is not None:
|
|
1579
|
+
sarif_location["id"] = location_id
|
|
1580
|
+
if location.get("role"):
|
|
1581
|
+
sarif_location["message"] = {"text": location["role"]}
|
|
1582
|
+
return sarif_location
|
|
1583
|
+
|
|
1584
|
+
|
|
1585
|
+
def _sarif_result(
|
|
1586
|
+
finding: dict[str, Any],
|
|
1587
|
+
rule_index: int,
|
|
1588
|
+
source_root: Path | None = None,
|
|
1589
|
+
line_hash_cache: dict[tuple[Path, int], str | None] | None = None,
|
|
1590
|
+
) -> dict[str, Any]:
|
|
1591
|
+
properties = {
|
|
1592
|
+
"category": finding["taxonomy"]["category"],
|
|
1593
|
+
"confidence": finding["confidence"]["level"],
|
|
1594
|
+
"findingId": finding["findingId"],
|
|
1595
|
+
"occurrenceId": finding["occurrenceId"],
|
|
1596
|
+
"severity": finding["severity"]["level"],
|
|
1597
|
+
}
|
|
1598
|
+
extensions = finding.get("extensions")
|
|
1599
|
+
candidate_id = extensions.get("candidateId") if isinstance(extensions, dict) else None
|
|
1600
|
+
if isinstance(candidate_id, str) and candidate_id:
|
|
1601
|
+
properties["candidateId"] = candidate_id
|
|
1602
|
+
partial_fingerprints = {
|
|
1603
|
+
"codexSecurity/v1": finding["fingerprints"]["primary"],
|
|
1604
|
+
}
|
|
1605
|
+
line_hash = _github_primary_location_line_hash(finding, source_root, line_hash_cache)
|
|
1606
|
+
if line_hash is not None:
|
|
1607
|
+
partial_fingerprints["primaryLocationLineHash"] = line_hash
|
|
1608
|
+
result = {
|
|
1609
|
+
"ruleId": finding["ruleId"],
|
|
1610
|
+
"ruleIndex": rule_index,
|
|
1611
|
+
"level": SARIF_LEVELS[finding["severity"]["level"]],
|
|
1612
|
+
"message": {"text": finding["summary"]},
|
|
1613
|
+
"locations": [_sarif_location(location) for location in _sarif_locations(finding)],
|
|
1614
|
+
"partialFingerprints": partial_fingerprints,
|
|
1615
|
+
"properties": properties,
|
|
1616
|
+
}
|
|
1617
|
+
return result
|
|
1618
|
+
|
|
1619
|
+
|
|
1620
|
+
def build_sarif(
|
|
1621
|
+
manifest: dict[str, Any], findings: dict[str, Any], source_root: Path | None = None
|
|
1622
|
+
) -> dict[str, Any]:
|
|
1623
|
+
scan = manifest["scan"]
|
|
1624
|
+
target = scan["target"]
|
|
1625
|
+
ordered_findings = sorted(findings["findings"], key=lambda finding: finding["occurrenceId"])
|
|
1626
|
+
findings_by_rule: dict[str, list[dict[str, Any]]] = {}
|
|
1627
|
+
for finding in ordered_findings:
|
|
1628
|
+
findings_by_rule.setdefault(finding["ruleId"], []).append(finding)
|
|
1629
|
+
ordered_rule_ids = sorted(findings_by_rule)
|
|
1630
|
+
rule_index = {rule_id: index for index, rule_id in enumerate(ordered_rule_ids)}
|
|
1631
|
+
line_hash_cache = _github_line_hash_cache(ordered_findings, source_root)
|
|
1632
|
+
run: dict[str, Any] = {
|
|
1633
|
+
"tool": {
|
|
1634
|
+
"driver": {
|
|
1635
|
+
"name": "Codex Security",
|
|
1636
|
+
"version": scan["producer"]["version"],
|
|
1637
|
+
"rules": [_sarif_rule(rule_id) for rule_id in ordered_rule_ids],
|
|
1638
|
+
}
|
|
1639
|
+
},
|
|
1640
|
+
"automationDetails": {"id": scan["id"]},
|
|
1641
|
+
"results": [
|
|
1642
|
+
_sarif_result(finding, rule_index[finding["ruleId"]], source_root, line_hash_cache)
|
|
1643
|
+
for finding in ordered_findings
|
|
1644
|
+
],
|
|
1645
|
+
"properties": {
|
|
1646
|
+
"codexSecuritySchemaVersion": manifest["schemaVersion"],
|
|
1647
|
+
"codexSecurityTargetKind": target["kind"],
|
|
1648
|
+
},
|
|
1649
|
+
}
|
|
1650
|
+
if target["kind"] == "git_revision" and target.get("remote") and target.get("revision"):
|
|
1651
|
+
run["versionControlProvenance"] = [
|
|
1652
|
+
{
|
|
1653
|
+
"repositoryUri": target["remote"],
|
|
1654
|
+
"revisionId": target["revision"],
|
|
1655
|
+
}
|
|
1656
|
+
]
|
|
1657
|
+
return {
|
|
1658
|
+
"$schema": SARIF_SCHEMA,
|
|
1659
|
+
"version": "2.1.0",
|
|
1660
|
+
"runs": [run],
|
|
1661
|
+
}
|
|
1662
|
+
|
|
1663
|
+
|
|
1664
|
+
def _validate_sarif(sarif: dict[str, Any]) -> None:
|
|
1665
|
+
if sarif.get("version") != "2.1.0":
|
|
1666
|
+
raise ContractError("SARIF: expected version 2.1.0")
|
|
1667
|
+
runs = sarif.get("runs")
|
|
1668
|
+
if not isinstance(runs, list) or len(runs) != 1:
|
|
1669
|
+
raise ContractError("SARIF: expected exactly one run")
|
|
1670
|
+
run = runs[0]
|
|
1671
|
+
if not isinstance(run, dict):
|
|
1672
|
+
raise ContractError("SARIF: expected a run object")
|
|
1673
|
+
rule_ids = [rule["id"] for rule in run["tool"]["driver"]["rules"]]
|
|
1674
|
+
for result in run["results"]:
|
|
1675
|
+
if result["ruleId"] not in rule_ids:
|
|
1676
|
+
raise ContractError("SARIF: result references an unknown rule")
|
|
1677
|
+
if not result.get("partialFingerprints"):
|
|
1678
|
+
raise ContractError("SARIF: result is missing partialFingerprints")
|
|
1679
|
+
|
|
1680
|
+
|
|
1681
|
+
def _artifact_record(
|
|
1682
|
+
scan_dir: Path, relative_path: str, media_type: str, contents: bytes | None = None
|
|
1683
|
+
) -> dict[str, str]:
|
|
1684
|
+
relative_path = _require_safe_relative_path(relative_path, "artifact path")
|
|
1685
|
+
if contents is not None:
|
|
1686
|
+
_require_scan_local_file(scan_dir, relative_path, relative_path)
|
|
1687
|
+
return {
|
|
1688
|
+
"mediaType": media_type,
|
|
1689
|
+
"path": relative_path,
|
|
1690
|
+
"sha256": (
|
|
1691
|
+
_sha256_bytes(contents)
|
|
1692
|
+
if contents is not None
|
|
1693
|
+
else _sha256_scan_local_file(scan_dir, relative_path, relative_path)
|
|
1694
|
+
),
|
|
1695
|
+
}
|
|
1696
|
+
|
|
1697
|
+
|
|
1698
|
+
def _coverage_receipt_refs(coverage: dict[str, Any]) -> list[str]:
|
|
1699
|
+
refs = {ref for surface in coverage["surfaces"] for ref in surface.get("receiptRefs", [])}
|
|
1700
|
+
return sorted(refs)
|
|
1701
|
+
|
|
1702
|
+
|
|
1703
|
+
def _validate_sealed_coverage_receipts(scan: dict[str, Any], coverage: dict[str, Any]) -> None:
|
|
1704
|
+
artifact_paths = {
|
|
1705
|
+
_require_safe_relative_path(artifact["path"], "sealed artifact path")
|
|
1706
|
+
for artifact in scan["artifacts"]
|
|
1707
|
+
}
|
|
1708
|
+
for ref in _coverage_receipt_refs(coverage):
|
|
1709
|
+
if ref not in artifact_paths:
|
|
1710
|
+
raise ContractError(f"coverage receipt is missing from sealed artifacts: {ref}")
|
|
1711
|
+
|
|
1712
|
+
|
|
1713
|
+
def _validate_existing_seal(
|
|
1714
|
+
scan_dir: Path,
|
|
1715
|
+
scan: dict[str, Any],
|
|
1716
|
+
*,
|
|
1717
|
+
artifact_contents: dict[str, bytes] | None = None,
|
|
1718
|
+
) -> None:
|
|
1719
|
+
sealed_at = scan.get("sealedAt")
|
|
1720
|
+
artifacts = scan.get("artifacts")
|
|
1721
|
+
if sealed_at is None and artifacts is None:
|
|
1722
|
+
return
|
|
1723
|
+
if sealed_at != scan.get("completedAt"):
|
|
1724
|
+
raise ContractError("manifest.scan.sealedAt: must match completedAt")
|
|
1725
|
+
if not isinstance(artifacts, list) or not artifacts:
|
|
1726
|
+
raise ContractError("manifest.scan.artifacts: sealed manifest requires artifact records")
|
|
1727
|
+
artifact_paths: set[str] = set()
|
|
1728
|
+
for index, artifact in enumerate(artifacts):
|
|
1729
|
+
context = f"manifest.scan.artifacts[{index}]"
|
|
1730
|
+
if not isinstance(artifact, dict):
|
|
1731
|
+
raise ContractError(f"{context}: expected an object")
|
|
1732
|
+
path = _require_safe_relative_path(
|
|
1733
|
+
_require_str(artifact, "path", context), f"{context}.path"
|
|
1734
|
+
)
|
|
1735
|
+
if path in artifact_paths:
|
|
1736
|
+
raise ContractError(f"{context}.path: duplicate artifact path")
|
|
1737
|
+
artifact_paths.add(path)
|
|
1738
|
+
expected_sha256 = _require_str(artifact, "sha256", context)
|
|
1739
|
+
contents = (artifact_contents or {}).get(path)
|
|
1740
|
+
actual_sha256 = (
|
|
1741
|
+
_sha256_bytes(contents)
|
|
1742
|
+
if contents is not None
|
|
1743
|
+
else _sha256_scan_local_file(scan_dir, path, context)
|
|
1744
|
+
)
|
|
1745
|
+
if actual_sha256 != expected_sha256:
|
|
1746
|
+
raise ContractError(f"{context}: sealed artifact changed or is missing")
|
|
1747
|
+
|
|
1748
|
+
|
|
1749
|
+
def _read_sealed_scan(
|
|
1750
|
+
scan_dir: Path, schema_dir: Path | None, required_for: str
|
|
1751
|
+
) -> tuple[dict[str, Any], dict[str, Any], dict[str, Any], bytes]:
|
|
1752
|
+
scan_dir = _require_scan_directory(scan_dir)
|
|
1753
|
+
schema_dir = schema_dir or Path(__file__).resolve().parent.parent / "schemas"
|
|
1754
|
+
manifest = _read_scan_local_json(scan_dir, "scan-manifest.json", "scan-manifest.json")
|
|
1755
|
+
scan = _require_dict(manifest, "scan", "manifest")
|
|
1756
|
+
_validate_contract_refs(scan)
|
|
1757
|
+
if scan.get("sealedAt") is None or scan.get("artifacts") is None:
|
|
1758
|
+
raise ContractError(f"manifest.scan: {required_for} requires a sealed scan")
|
|
1759
|
+
findings, findings_bytes = _read_scan_local_json_bytes(
|
|
1760
|
+
scan_dir, scan["findingsRef"], scan["findingsRef"]
|
|
1761
|
+
)
|
|
1762
|
+
coverage, coverage_bytes = _read_scan_local_json_bytes(
|
|
1763
|
+
scan_dir, scan["coverageRef"], scan["coverageRef"]
|
|
1764
|
+
)
|
|
1765
|
+
_validate_existing_seal(
|
|
1766
|
+
scan_dir,
|
|
1767
|
+
scan,
|
|
1768
|
+
artifact_contents={
|
|
1769
|
+
scan["findingsRef"]: findings_bytes,
|
|
1770
|
+
scan["coverageRef"]: coverage_bytes,
|
|
1771
|
+
},
|
|
1772
|
+
)
|
|
1773
|
+
_validate_manifest(manifest)
|
|
1774
|
+
_validate_findings(manifest, findings)
|
|
1775
|
+
_validate_coverage(manifest, coverage, scan_dir)
|
|
1776
|
+
_validate_sealed_coverage_receipts(scan, coverage)
|
|
1777
|
+
validate_against_schema(manifest, schema_dir / "scan-manifest.schema.json")
|
|
1778
|
+
validate_against_schema(findings, schema_dir / "findings.schema.json")
|
|
1779
|
+
validate_against_schema(coverage, schema_dir / "coverage.schema.json")
|
|
1780
|
+
_validate_derived_finding_identities(manifest, findings)
|
|
1781
|
+
return manifest, findings, coverage, findings_bytes
|
|
1782
|
+
|
|
1783
|
+
|
|
1784
|
+
def build_sarif_projection(
|
|
1785
|
+
scan_dir: Path, source_root: Path | None = None, schema_dir: Path | None = None
|
|
1786
|
+
) -> dict[str, Any]:
|
|
1787
|
+
if source_root is not None:
|
|
1788
|
+
try:
|
|
1789
|
+
source_root = source_root.resolve(strict=True)
|
|
1790
|
+
source_root_is_directory = source_root.is_dir()
|
|
1791
|
+
except (OSError, RuntimeError):
|
|
1792
|
+
source_root_is_directory = False
|
|
1793
|
+
if not source_root_is_directory:
|
|
1794
|
+
raise ContractError("source root: expected an existing directory")
|
|
1795
|
+
manifest, findings, _, _ = _read_sealed_scan(scan_dir, schema_dir, "SARIF projection")
|
|
1796
|
+
sarif = build_sarif(manifest, findings, source_root)
|
|
1797
|
+
_validate_sarif(sarif)
|
|
1798
|
+
return sarif
|
|
1799
|
+
|
|
1800
|
+
|
|
1801
|
+
def write_sarif_projection(
|
|
1802
|
+
scan_dir: Path, source_root: Path | None = None, schema_dir: Path | None = None
|
|
1803
|
+
) -> None:
|
|
1804
|
+
sarif = build_sarif_projection(scan_dir, source_root, schema_dir)
|
|
1805
|
+
_write_scan_local_json(scan_dir, "exports/results.sarif", sarif)
|
|
1806
|
+
|
|
1807
|
+
|
|
1808
|
+
def write_sarif_output(scan_dir: Path, output: Path, sarif: dict[str, Any]) -> None:
|
|
1809
|
+
write_export_output(scan_dir, output, "sarif", _json_bytes(sarif))
|
|
1810
|
+
|
|
1811
|
+
|
|
1812
|
+
def csv_cell(value: Any) -> Any:
|
|
1813
|
+
if isinstance(value, str) and (
|
|
1814
|
+
value.startswith(("\t", "\r", "\n"))
|
|
1815
|
+
or value.lstrip().startswith(("=", "+", "-", "@", "=", "+", "-", "@"))
|
|
1816
|
+
):
|
|
1817
|
+
return f"'{value}"
|
|
1818
|
+
return value
|
|
1819
|
+
|
|
1820
|
+
|
|
1821
|
+
def finding_candidate_id(finding: dict[str, Any]) -> str | None:
|
|
1822
|
+
extensions = finding.get("extensions")
|
|
1823
|
+
if not isinstance(extensions, dict):
|
|
1824
|
+
return None
|
|
1825
|
+
return next(
|
|
1826
|
+
(
|
|
1827
|
+
value
|
|
1828
|
+
for field in ("candidateId", "reportId", "ledgerRowId")
|
|
1829
|
+
if isinstance(value := extensions.get(field), str) and value.strip()
|
|
1830
|
+
),
|
|
1831
|
+
None,
|
|
1832
|
+
)
|
|
1833
|
+
|
|
1834
|
+
|
|
1835
|
+
def build_csv_projection(findings: dict[str, Any], coverage: dict[str, Any]) -> bytes:
|
|
1836
|
+
output = io.StringIO(newline="")
|
|
1837
|
+
writer = csv.writer(output)
|
|
1838
|
+
deep_scan = coverage.get("mode") == "deep_repository" or (
|
|
1839
|
+
coverage.get("mode") == "scoped_path"
|
|
1840
|
+
and any(
|
|
1841
|
+
isinstance(finding.get("extensions"), dict)
|
|
1842
|
+
and any(
|
|
1843
|
+
isinstance(finding["extensions"].get(field), str)
|
|
1844
|
+
and finding["extensions"][field].strip()
|
|
1845
|
+
for field in ("candidateId", "reportId")
|
|
1846
|
+
)
|
|
1847
|
+
for finding in findings["findings"]
|
|
1848
|
+
)
|
|
1849
|
+
)
|
|
1850
|
+
writer.writerow(
|
|
1851
|
+
(
|
|
1852
|
+
"occurrence_id",
|
|
1853
|
+
"finding_id",
|
|
1854
|
+
*(("candidate_id",) if deep_scan else ()),
|
|
1855
|
+
"title",
|
|
1856
|
+
"summary",
|
|
1857
|
+
"severity",
|
|
1858
|
+
"confidence",
|
|
1859
|
+
"status",
|
|
1860
|
+
"close_reason",
|
|
1861
|
+
"note",
|
|
1862
|
+
"remediation",
|
|
1863
|
+
"path",
|
|
1864
|
+
"start_line",
|
|
1865
|
+
"end_line",
|
|
1866
|
+
)
|
|
1867
|
+
)
|
|
1868
|
+
for finding in findings["findings"]:
|
|
1869
|
+
locations = finding["locations"]
|
|
1870
|
+
location = next(
|
|
1871
|
+
(candidate for candidate in locations if candidate.get("role") == "root_control"),
|
|
1872
|
+
locations[0],
|
|
1873
|
+
)
|
|
1874
|
+
writer.writerow(
|
|
1875
|
+
(
|
|
1876
|
+
csv_cell(finding["occurrenceId"]),
|
|
1877
|
+
csv_cell(finding["findingId"]),
|
|
1878
|
+
*((csv_cell(finding_candidate_id(finding)),) if deep_scan else ()),
|
|
1879
|
+
csv_cell(finding["title"]),
|
|
1880
|
+
csv_cell(finding["summary"]),
|
|
1881
|
+
csv_cell(finding["severity"]["level"]),
|
|
1882
|
+
csv_cell(finding["confidence"]["level"]),
|
|
1883
|
+
"open",
|
|
1884
|
+
"",
|
|
1885
|
+
"",
|
|
1886
|
+
csv_cell(finding["remediation"]),
|
|
1887
|
+
csv_cell(location["path"]),
|
|
1888
|
+
location["startLine"],
|
|
1889
|
+
location.get("endLine", location["startLine"]),
|
|
1890
|
+
)
|
|
1891
|
+
)
|
|
1892
|
+
return output.getvalue().encode("utf-8")
|
|
1893
|
+
|
|
1894
|
+
|
|
1895
|
+
def build_findings_export(
|
|
1896
|
+
scan_dir: Path,
|
|
1897
|
+
export_format: str,
|
|
1898
|
+
source_root: Path | None = None,
|
|
1899
|
+
schema_dir: Path | None = None,
|
|
1900
|
+
) -> bytes:
|
|
1901
|
+
if export_format not in EXPORT_PATHS:
|
|
1902
|
+
raise ContractError(f"unsupported export format: {export_format}")
|
|
1903
|
+
if export_format == "sarif":
|
|
1904
|
+
return _json_bytes(build_sarif_projection(scan_dir, source_root, schema_dir))
|
|
1905
|
+
if source_root is not None:
|
|
1906
|
+
raise ContractError("source-root is only supported for SARIF exports")
|
|
1907
|
+
_, findings, coverage, findings_bytes = _read_sealed_scan(
|
|
1908
|
+
scan_dir, schema_dir, f"{export_format.upper()} export"
|
|
1909
|
+
)
|
|
1910
|
+
if export_format == "json":
|
|
1911
|
+
return findings_bytes
|
|
1912
|
+
return build_csv_projection(findings, coverage)
|
|
1913
|
+
|
|
1914
|
+
|
|
1915
|
+
def write_export_output(scan_dir: Path, output: Path, export_format: str, contents: bytes) -> None:
|
|
1916
|
+
if export_format not in EXPORT_PATHS:
|
|
1917
|
+
raise ContractError(f"unsupported export format: {export_format}")
|
|
1918
|
+
scan_dir = _require_scan_directory(scan_dir)
|
|
1919
|
+
output = Path(os.path.abspath(output))
|
|
1920
|
+
try:
|
|
1921
|
+
relative_output = output.relative_to(scan_dir).as_posix()
|
|
1922
|
+
except ValueError:
|
|
1923
|
+
for ancestor in output.parents:
|
|
1924
|
+
try:
|
|
1925
|
+
inside_scan = ancestor.samefile(scan_dir)
|
|
1926
|
+
except FileNotFoundError:
|
|
1927
|
+
continue
|
|
1928
|
+
except OSError as exc:
|
|
1929
|
+
raise ContractError(
|
|
1930
|
+
"export output path: unable to inspect output directory"
|
|
1931
|
+
) from exc
|
|
1932
|
+
if inside_scan:
|
|
1933
|
+
if any(parent.is_symlink() for parent in (ancestor, *ancestor.parents)):
|
|
1934
|
+
raise ContractError(
|
|
1935
|
+
"export output path: symbolic links cannot alias the scan directory"
|
|
1936
|
+
) from None
|
|
1937
|
+
relative_output = output.relative_to(ancestor).as_posix()
|
|
1938
|
+
break
|
|
1939
|
+
else:
|
|
1940
|
+
write_scan_local_bytes(output.parent, output.name, contents, external_name=True)
|
|
1941
|
+
return
|
|
1942
|
+
if relative_output != EXPORT_PATHS[export_format]:
|
|
1943
|
+
raise ContractError(f"{export_format.upper()} output path cannot overwrite a scan artifact")
|
|
1944
|
+
manifest = _read_scan_local_json(scan_dir, "scan-manifest.json", "scan-manifest.json")
|
|
1945
|
+
scan = _require_dict(manifest, "scan", "manifest")
|
|
1946
|
+
artifacts = _require_list(scan, "artifacts", "manifest.scan")
|
|
1947
|
+
artifact_paths = [
|
|
1948
|
+
_require_safe_relative_path(
|
|
1949
|
+
_require_str(artifact, "path", f"manifest.scan.artifacts[{index}]"),
|
|
1950
|
+
f"manifest.scan.artifacts[{index}].path",
|
|
1951
|
+
)
|
|
1952
|
+
for index, artifact in enumerate(artifacts)
|
|
1953
|
+
if isinstance(artifact, dict)
|
|
1954
|
+
]
|
|
1955
|
+
try:
|
|
1956
|
+
output_metadata = output.stat(follow_symlinks=False)
|
|
1957
|
+
except FileNotFoundError:
|
|
1958
|
+
output_metadata = None
|
|
1959
|
+
except OSError as exc:
|
|
1960
|
+
raise ContractError(f"{relative_output}: unable to inspect export output") from exc
|
|
1961
|
+
for artifact_path in artifact_paths:
|
|
1962
|
+
if artifact_path == relative_output:
|
|
1963
|
+
raise ContractError(
|
|
1964
|
+
f"{export_format.upper()} output path cannot overwrite a sealed scan artifact"
|
|
1965
|
+
)
|
|
1966
|
+
if output_metadata is None:
|
|
1967
|
+
continue
|
|
1968
|
+
descriptor = open_scan_local_file_descriptor(
|
|
1969
|
+
scan_dir, artifact_path, f"sealed artifact {artifact_path}"
|
|
1970
|
+
)
|
|
1971
|
+
try:
|
|
1972
|
+
artifact_metadata = os.fstat(descriptor)
|
|
1973
|
+
finally:
|
|
1974
|
+
os.close(descriptor)
|
|
1975
|
+
if os.path.samestat(output_metadata, artifact_metadata):
|
|
1976
|
+
raise ContractError(
|
|
1977
|
+
f"{export_format.upper()} output path cannot overwrite a sealed scan artifact"
|
|
1978
|
+
)
|
|
1979
|
+
write_scan_local_bytes(scan_dir, relative_output, contents)
|
|
1980
|
+
|
|
1981
|
+
|
|
1982
|
+
def _write_sarif_projection_if_possible(
|
|
1983
|
+
scan_dir: Path, source_root: Path | None = None, schema_dir: Path | None = None
|
|
1984
|
+
) -> None:
|
|
1985
|
+
try:
|
|
1986
|
+
write_sarif_projection(scan_dir, source_root, schema_dir)
|
|
1987
|
+
except (ContractError, OSError) as error:
|
|
1988
|
+
print(
|
|
1989
|
+
f"codex-security: warning: automatic SARIF export failed: {error}. "
|
|
1990
|
+
"Run `codex-security export <scan-dir> --export-format sarif` to retry.",
|
|
1991
|
+
file=sys.stderr,
|
|
1992
|
+
)
|
|
1993
|
+
|
|
1994
|
+
|
|
1995
|
+
PreparedScanFinalization = tuple[
|
|
1996
|
+
Path,
|
|
1997
|
+
Path,
|
|
1998
|
+
dict[str, Any],
|
|
1999
|
+
dict[str, Any],
|
|
2000
|
+
dict[str, Any],
|
|
2001
|
+
bool,
|
|
2002
|
+
bytes,
|
|
2003
|
+
]
|
|
2004
|
+
|
|
2005
|
+
|
|
2006
|
+
def _prepare_scan_finalization(
|
|
2007
|
+
scan_dir: Path,
|
|
2008
|
+
schema_dir: Path | None = None,
|
|
2009
|
+
*,
|
|
2010
|
+
expected_coverage_mode: str | None = None,
|
|
2011
|
+
completion_binding: dict[str, Any] | None = None,
|
|
2012
|
+
) -> PreparedScanFinalization:
|
|
2013
|
+
"""Read, populate, and validate a scan without writing any output files."""
|
|
2014
|
+
|
|
2015
|
+
scan_dir = _require_scan_directory(scan_dir)
|
|
2016
|
+
schema_dir = schema_dir or Path(__file__).resolve().parent.parent / "schemas"
|
|
2017
|
+
manifest = _read_scan_local_json(scan_dir, "scan-manifest.json", "scan-manifest.json")
|
|
2018
|
+
scan = _require_dict(manifest, "scan", "manifest")
|
|
2019
|
+
was_sealed = scan.get("sealedAt") is not None or scan.get("artifacts") is not None
|
|
2020
|
+
if not was_sealed:
|
|
2021
|
+
_populate_unsealed_manifest_envelope(manifest, scan, completion_binding)
|
|
2022
|
+
_validate_contract_refs(scan)
|
|
2023
|
+
findings, findings_input_bytes = _read_scan_local_json_bytes(
|
|
2024
|
+
scan_dir, scan["findingsRef"], scan["findingsRef"]
|
|
2025
|
+
)
|
|
2026
|
+
coverage, coverage_input_bytes = _read_scan_local_json_bytes(
|
|
2027
|
+
scan_dir, scan["coverageRef"], scan["coverageRef"]
|
|
2028
|
+
)
|
|
2029
|
+
if not was_sealed:
|
|
2030
|
+
_populate_unsealed_artifact_envelope(manifest, findings, coverage, completion_binding)
|
|
2031
|
+
_normalize_unsealed_deep_repository_inventory_strategy(
|
|
2032
|
+
coverage,
|
|
2033
|
+
expected_coverage_mode=expected_coverage_mode,
|
|
2034
|
+
)
|
|
2035
|
+
_normalize_unsealed_open_questions(coverage)
|
|
2036
|
+
|
|
2037
|
+
if manifest.get("schemaVersion") != SCHEMA_VERSION:
|
|
2038
|
+
raise ContractError(f"manifest.schemaVersion: expected {SCHEMA_VERSION}")
|
|
2039
|
+
if scan.get("status") != "completed":
|
|
2040
|
+
raise ContractError("manifest.scan.status: expected completed before sealing")
|
|
2041
|
+
if (
|
|
2042
|
+
expected_coverage_mode is not None
|
|
2043
|
+
and completion_binding is not None
|
|
2044
|
+
and completion_binding["coverageMode"] != expected_coverage_mode
|
|
2045
|
+
):
|
|
2046
|
+
raise ContractError("completion binding coverage mode does not match expected mode")
|
|
2047
|
+
if expected_coverage_mode is not None and coverage.get("mode") != expected_coverage_mode:
|
|
2048
|
+
raise ContractError(
|
|
2049
|
+
f"coverage.mode: must match selected scan mode {expected_coverage_mode}"
|
|
2050
|
+
)
|
|
2051
|
+
_validate_existing_seal(
|
|
2052
|
+
scan_dir,
|
|
2053
|
+
scan,
|
|
2054
|
+
artifact_contents={
|
|
2055
|
+
scan["findingsRef"]: findings_input_bytes,
|
|
2056
|
+
scan["coverageRef"]: coverage_input_bytes,
|
|
2057
|
+
},
|
|
2058
|
+
)
|
|
2059
|
+
scan["sealedAt"] = _require_str(scan, "completedAt", "manifest.scan")
|
|
2060
|
+
_validate_target(_require_dict(scan, "target", "manifest.scan"))
|
|
2061
|
+
_validate_completion_binding(manifest, findings, coverage, completion_binding)
|
|
2062
|
+
if was_sealed:
|
|
2063
|
+
_validate_findings(manifest, findings)
|
|
2064
|
+
if was_sealed:
|
|
2065
|
+
_validate_derived_finding_identities(manifest, findings)
|
|
2066
|
+
else:
|
|
2067
|
+
_populate_unsealed_finding_identities(manifest, findings)
|
|
2068
|
+
_validate_findings(manifest, findings)
|
|
2069
|
+
_validate_coverage(manifest, coverage, scan_dir)
|
|
2070
|
+
_validate_canonical_schemas_before_projection(manifest, findings, coverage, schema_dir)
|
|
2071
|
+
_require_derived_writeup_files(scan_dir, findings)
|
|
2072
|
+
_require_hardening_portfolio_file(scan_dir, scan)
|
|
2073
|
+
if was_sealed:
|
|
2074
|
+
_validate_sealed_coverage_receipts(scan, coverage)
|
|
2075
|
+
_validate_manifest(manifest)
|
|
2076
|
+
validate_against_schema(manifest, schema_dir / "scan-manifest.schema.json")
|
|
2077
|
+
validate_against_schema(findings, schema_dir / "findings.schema.json")
|
|
2078
|
+
validate_against_schema(coverage, schema_dir / "coverage.schema.json")
|
|
2079
|
+
report_markdown_bytes = _generate_report_projection(manifest, findings, coverage)
|
|
2080
|
+
_validate_report_output_paths(scan_dir)
|
|
2081
|
+
return (
|
|
2082
|
+
scan_dir,
|
|
2083
|
+
schema_dir,
|
|
2084
|
+
manifest,
|
|
2085
|
+
findings,
|
|
2086
|
+
coverage,
|
|
2087
|
+
was_sealed,
|
|
2088
|
+
report_markdown_bytes,
|
|
2089
|
+
)
|
|
2090
|
+
|
|
2091
|
+
findings_bytes = _contract_json_bytes("findings.json", findings)
|
|
2092
|
+
coverage_bytes = _contract_json_bytes("coverage.json", coverage)
|
|
2093
|
+
report_markdown_bytes = _generate_report_projection(manifest, findings, coverage)
|
|
2094
|
+
_validate_report_output_paths(scan_dir)
|
|
2095
|
+
scan["artifacts"] = [
|
|
2096
|
+
_artifact_record(scan_dir, "findings.json", "application/json", findings_bytes),
|
|
2097
|
+
_artifact_record(scan_dir, "coverage.json", "application/json", coverage_bytes),
|
|
2098
|
+
*[
|
|
2099
|
+
_artifact_record(scan_dir, ref, "application/octet-stream")
|
|
2100
|
+
for ref in _coverage_receipt_refs(coverage)
|
|
2101
|
+
],
|
|
2102
|
+
]
|
|
2103
|
+
_validate_sealed_coverage_receipts(scan, coverage)
|
|
2104
|
+
_validate_manifest(manifest)
|
|
2105
|
+
validate_against_schema(manifest, schema_dir / "scan-manifest.schema.json")
|
|
2106
|
+
validate_against_schema(findings, schema_dir / "findings.schema.json")
|
|
2107
|
+
validate_against_schema(coverage, schema_dir / "coverage.schema.json")
|
|
2108
|
+
_contract_json_bytes("scan-manifest.json", manifest)
|
|
2109
|
+
return (
|
|
2110
|
+
scan_dir,
|
|
2111
|
+
schema_dir,
|
|
2112
|
+
manifest,
|
|
2113
|
+
findings,
|
|
2114
|
+
coverage,
|
|
2115
|
+
was_sealed,
|
|
2116
|
+
report_markdown_bytes,
|
|
2117
|
+
)
|
|
2118
|
+
|
|
2119
|
+
|
|
2120
|
+
def _write_prepared_scan_finalization(
|
|
2121
|
+
prepared: PreparedScanFinalization,
|
|
2122
|
+
source_root: Path | None = None,
|
|
2123
|
+
) -> tuple[dict[str, Any], dict[str, Any], dict[str, Any]]:
|
|
2124
|
+
"""Write a previously validated scan finalization result."""
|
|
2125
|
+
|
|
2126
|
+
(
|
|
2127
|
+
scan_dir,
|
|
2128
|
+
schema_dir,
|
|
2129
|
+
manifest,
|
|
2130
|
+
findings,
|
|
2131
|
+
coverage,
|
|
2132
|
+
was_sealed,
|
|
2133
|
+
report_markdown_bytes,
|
|
2134
|
+
) = prepared
|
|
2135
|
+
scan = _require_dict(manifest, "scan", "manifest")
|
|
2136
|
+
if was_sealed:
|
|
2137
|
+
write_scan_local_bytes(scan_dir, "report.md", report_markdown_bytes)
|
|
2138
|
+
_remove_scan_local_file_if_exists(scan_dir, "report.html")
|
|
2139
|
+
_write_sarif_projection_if_possible(scan_dir, source_root, schema_dir)
|
|
2140
|
+
return manifest, findings, coverage
|
|
2141
|
+
|
|
2142
|
+
_write_scan_local_json(scan_dir, "findings.json", findings)
|
|
2143
|
+
_write_scan_local_json(scan_dir, "coverage.json", coverage)
|
|
2144
|
+
write_scan_local_bytes(scan_dir, "report.md", report_markdown_bytes)
|
|
2145
|
+
_remove_scan_local_file_if_exists(scan_dir, "report.html")
|
|
2146
|
+
_write_scan_local_json(scan_dir, "scan-manifest.json", manifest)
|
|
2147
|
+
_validate_existing_seal(scan_dir, scan)
|
|
2148
|
+
_write_sarif_projection_if_possible(scan_dir, source_root, schema_dir)
|
|
2149
|
+
return manifest, findings, coverage
|
|
2150
|
+
|
|
2151
|
+
|
|
2152
|
+
def finalize_scan(
|
|
2153
|
+
scan_dir: Path,
|
|
2154
|
+
schema_dir: Path | None = None,
|
|
2155
|
+
source_root: Path | None = None,
|
|
2156
|
+
*,
|
|
2157
|
+
expected_coverage_mode: str | None = None,
|
|
2158
|
+
completion_binding: dict[str, Any] | None = None,
|
|
2159
|
+
) -> tuple[dict[str, Any], dict[str, Any], dict[str, Any]]:
|
|
2160
|
+
prepared = _prepare_scan_finalization(
|
|
2161
|
+
scan_dir,
|
|
2162
|
+
schema_dir,
|
|
2163
|
+
expected_coverage_mode=expected_coverage_mode,
|
|
2164
|
+
completion_binding=completion_binding,
|
|
2165
|
+
)
|
|
2166
|
+
return _write_prepared_scan_finalization(prepared, source_root)
|
|
2167
|
+
|
|
2168
|
+
|
|
2169
|
+
def main() -> int:
|
|
2170
|
+
parser = argparse.ArgumentParser(description=__doc__)
|
|
2171
|
+
parser.add_argument("--scan-dir", required=True, type=Path)
|
|
2172
|
+
parser.add_argument("--schema-dir", type=Path)
|
|
2173
|
+
parser.add_argument("--source-root", type=Path)
|
|
2174
|
+
parser.add_argument("--sarif-only", action="store_true")
|
|
2175
|
+
parser.add_argument("--sarif-output", type=Path)
|
|
2176
|
+
parser.add_argument("--export-format", choices=sorted(EXPORT_PATHS))
|
|
2177
|
+
parser.add_argument("--export-output", type=Path)
|
|
2178
|
+
args = parser.parse_args()
|
|
2179
|
+
try:
|
|
2180
|
+
if args.sarif_only and args.export_format is not None:
|
|
2181
|
+
parser.error("--sarif-only cannot be combined with --export-format")
|
|
2182
|
+
if args.export_output is not None and args.export_format is None:
|
|
2183
|
+
parser.error("--export-output requires --export-format")
|
|
2184
|
+
if args.sarif_output is not None and not args.sarif_only:
|
|
2185
|
+
parser.error("--sarif-output requires --sarif-only")
|
|
2186
|
+
if args.export_format is not None:
|
|
2187
|
+
contents = build_findings_export(
|
|
2188
|
+
args.scan_dir, args.export_format, args.source_root, args.schema_dir
|
|
2189
|
+
)
|
|
2190
|
+
if args.export_output is None:
|
|
2191
|
+
sys.stdout.buffer.write(contents)
|
|
2192
|
+
else:
|
|
2193
|
+
write_export_output(args.scan_dir, args.export_output, args.export_format, contents)
|
|
2194
|
+
elif args.sarif_only:
|
|
2195
|
+
sarif = build_sarif_projection(args.scan_dir, args.source_root, args.schema_dir)
|
|
2196
|
+
if args.sarif_output is None:
|
|
2197
|
+
sys.stdout.buffer.write(_json_bytes(sarif))
|
|
2198
|
+
else:
|
|
2199
|
+
write_sarif_output(args.scan_dir, args.sarif_output, sarif)
|
|
2200
|
+
else:
|
|
2201
|
+
finalize_scan(args.scan_dir, args.schema_dir, args.source_root)
|
|
2202
|
+
except ContractError as exc:
|
|
2203
|
+
parser.error(str(exc))
|
|
2204
|
+
return 0
|
|
2205
|
+
|
|
2206
|
+
|
|
2207
|
+
if __name__ == "__main__":
|
|
2208
|
+
raise SystemExit(main())
|