@openai/codex-security 0.1.10 → 0.1.12
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +72 -41
- package/_bundled_plugin/.codex-plugin/plugin.json +1 -1
- package/_bundled_plugin/.mcp.json +5 -1
- package/_bundled_plugin/examples/completed-scan/coverage.json +1 -1
- package/_bundled_plugin/examples/completed-scan/scan-manifest.json +2 -2
- package/_bundled_plugin/mcp/server.mjs.br.part-000 +0 -0
- package/_bundled_plugin/mcp/server.mjs.br.part-001 +0 -0
- package/_bundled_plugin/preflight/capability-profiles.toml +1 -34
- package/_bundled_plugin/references/config-preflight.md +11 -13
- package/_bundled_plugin/references/core-scan.md +94 -0
- package/_bundled_plugin/references/final-report.md +6 -6
- package/_bundled_plugin/references/scan-artifacts.md +15 -14
- package/_bundled_plugin/references/scan-prologue.md +27 -0
- package/_bundled_plugin/schemas/tools/deep-reducer.schema.json +1 -106
- package/_bundled_plugin/scripts/config_preflight.py +40 -3
- package/_bundled_plugin/scripts/deep_scan_config.py +29 -6
- package/_bundled_plugin/scripts/deep_scan_workbench.py +132 -34
- package/_bundled_plugin/scripts/finalize_scan_contract.py +44 -224
- package/_bundled_plugin/scripts/finding_preview.py +122 -6
- package/_bundled_plugin/scripts/generate_in_scope_files.py +77 -31
- package/_bundled_plugin/scripts/generate_rank_input.py +58 -7
- package/_bundled_plugin/scripts/normalize_candidates.py +5 -5
- package/_bundled_plugin/scripts/rank_preview.py +4 -0
- package/_bundled_plugin/scripts/report_projection.py +40 -20
- package/_bundled_plugin/scripts/workbench_cli.py +5 -0
- package/_bundled_plugin/scripts/workbench_db.py +304 -32
- package/_bundled_plugin/scripts/workbench_progress.py +0 -5
- package/_bundled_plugin/scripts/workbench_scan_usage.py +43 -8
- package/_bundled_plugin/scripts/workbench_schema.py +16 -20
- package/_bundled_plugin/scripts/workbench_source_excerpt.py +1 -5
- package/_bundled_plugin/scripts/workbench_target.py +82 -0
- package/_bundled_plugin/skills/attack-path-analysis/SKILL.md +8 -11
- package/_bundled_plugin/skills/deep-security-scan/SKILL.md +24 -108
- package/_bundled_plugin/skills/security-scan/SKILL.md +5 -88
- package/_bundled_plugin/skills/threat-model/SKILL.md +4 -2
- package/_bundled_plugin/skills/track-findings/SKILL.md +1 -1
- package/_bundled_plugin/skills/triage-finding/SKILL.md +5 -6
- package/_bundled_plugin/skills/triage-finding/references/github-rest-intake.md +6 -7
- package/_bundled_plugin/skills/validation/SKILL.md +9 -12
- package/_bundled_plugin/skills/vulnerability-writeup/SKILL.md +3 -2
- package/_bundled_plugin/skills/vulnerability-writeup/references/report-format.md +1 -1
- package/dist/api.d.ts +2 -0
- package/dist/api.d.ts.map +1 -1
- package/dist/api.js +393 -145
- package/dist/api.js.map +1 -1
- package/dist/auth.d.ts +2 -3
- package/dist/auth.d.ts.map +1 -1
- package/dist/auth.js +59 -270
- package/dist/auth.js.map +1 -1
- package/dist/bulk-scan-discovery.js +14 -2
- package/dist/bulk-scan-discovery.js.map +1 -1
- package/dist/cli.d.ts +6 -3
- package/dist/cli.d.ts.map +1 -1
- package/dist/cli.js +202 -46
- package/dist/cli.js.map +1 -1
- package/dist/config.d.ts.map +1 -1
- package/dist/config.js +1 -0
- package/dist/config.js.map +1 -1
- package/dist/contract.js +7 -5
- package/dist/contract.js.map +1 -1
- package/dist/cost.d.ts +1 -0
- package/dist/cost.d.ts.map +1 -1
- package/dist/cost.js +33 -2
- package/dist/cost.js.map +1 -1
- package/dist/knowledge-base.d.ts.map +1 -1
- package/dist/knowledge-base.js +12 -4
- package/dist/knowledge-base.js.map +1 -1
- package/dist/multiscan.d.ts +1 -0
- package/dist/multiscan.d.ts.map +1 -1
- package/dist/multiscan.js +77 -17
- package/dist/multiscan.js.map +1 -1
- package/dist/runtime.d.ts +9 -6
- package/dist/runtime.d.ts.map +1 -1
- package/dist/runtime.js +140 -326
- package/dist/runtime.js.map +1 -1
- package/dist/scan-comparison.js +1 -1
- package/dist/scan-comparison.js.map +1 -1
- package/dist/scan-dashboard.d.ts +5 -1
- package/dist/scan-dashboard.d.ts.map +1 -1
- package/dist/scan-dashboard.js +65 -16
- package/dist/scan-dashboard.js.map +1 -1
- package/dist/scan-logs.d.ts +2 -0
- package/dist/scan-logs.d.ts.map +1 -1
- package/dist/scan-logs.js +97 -37
- package/dist/scan-logs.js.map +1 -1
- package/dist/targets.d.ts +1 -0
- package/dist/targets.d.ts.map +1 -1
- package/dist/targets.js +35 -4
- package/dist/targets.js.map +1 -1
- package/dist/trusted-executable.js +1 -1
- package/dist/trusted-executable.js.map +1 -1
- package/dist/version.d.ts +3 -3
- package/dist/version.d.ts.map +1 -1
- package/dist/version.js +6 -4
- package/dist/version.js.map +1 -1
- package/package.json +4 -4
- package/_bundled_plugin/references/shared-hard-rules.md +0 -15
- package/_bundled_plugin/scripts/validate_report_format.py +0 -138
- package/_bundled_plugin/skills/security-scan/references/repository-wide-scan.md +0 -33
|
@@ -17,10 +17,11 @@ import re
|
|
|
17
17
|
import secrets
|
|
18
18
|
import stat
|
|
19
19
|
import sys
|
|
20
|
+
import time
|
|
20
21
|
from collections.abc import Iterator
|
|
21
22
|
from datetime import datetime, timezone
|
|
22
23
|
from pathlib import Path, PurePosixPath
|
|
23
|
-
from typing import Any,
|
|
24
|
+
from typing import Any, TextIO
|
|
24
25
|
from urllib.parse import quote, urlsplit
|
|
25
26
|
|
|
26
27
|
SCHEMA_VERSION = "1.0"
|
|
@@ -58,33 +59,8 @@ GITHUB_HASH_BLOCK_SIZE = 100
|
|
|
58
59
|
GITHUB_HASH_MOD = 37
|
|
59
60
|
GITHUB_HASH_MASK = (1 << 64) - 1
|
|
60
61
|
GITHUB_HASH_EOF = 65535
|
|
61
|
-
GITHUB_HASH_MAX_LINES = 100_000
|
|
62
62
|
SOURCE_READ_CHUNK_SIZE = 64 * 1024
|
|
63
|
-
SOURCE_READ_MAX_BYTES = 10 * 1024 * 1024
|
|
64
|
-
CONTRACT_DOCUMENT_MAX_BYTES = {
|
|
65
|
-
"scan-manifest.json": 16 * 1024 * 1024,
|
|
66
|
-
"findings.json": 128 * 1024 * 1024,
|
|
67
|
-
"coverage.json": 32 * 1024 * 1024,
|
|
68
|
-
}
|
|
69
|
-
SCHEMA_DOCUMENT_MAX_BYTES = 4 * 1024 * 1024
|
|
70
|
-
JSON_DOCUMENT_READ_CHUNK_SIZE = 64 * 1024
|
|
71
|
-
MAX_JSON_DEPTH = 256
|
|
72
63
|
MAX_JSON_INTEGER = (1 << 53) - 1
|
|
73
|
-
MAX_SCHEMA_NODES = 8192
|
|
74
|
-
MAX_SCHEMA_COLLECTION_ENTRIES = 4096
|
|
75
|
-
MAX_SCHEMA_APPLICATOR_EDGES = 128
|
|
76
|
-
SAFE_SCHEMA_PATTERNS = {
|
|
77
|
-
r"^(?![^:/?#]+://[^/?#]*@)[^?#]+$",
|
|
78
|
-
r"^codex-security-snapshot/v1:sha256:[a-f0-9]{64}$",
|
|
79
|
-
r"^(?!/)(?!.*(?:^|/)\.\.(?:/|$))(?!.*\\).+$",
|
|
80
|
-
r"^[a-f0-9]{64}$",
|
|
81
|
-
r"^(?!.*(?:^|/)\.\.(?:/|$))(?!.*\\)artifacts/.+$",
|
|
82
|
-
r"^csf_[a-f0-9]{24}$",
|
|
83
|
-
r"^occ_[a-f0-9]{24}$",
|
|
84
|
-
r"^[a-z0-9][a-z0-9._/-]*$",
|
|
85
|
-
r"^codex-security/v1:sha256:[a-f0-9]{64}$",
|
|
86
|
-
r"^findings/([a-z0-9][a-z0-9._-]*)/\1\.md$",
|
|
87
|
-
}
|
|
88
64
|
EXPORT_PATHS = {
|
|
89
65
|
"csv": "exports/findings.csv",
|
|
90
66
|
"json": "exports/findings.json",
|
|
@@ -96,6 +72,10 @@ class ContractError(ValueError):
|
|
|
96
72
|
"""Raised when a completed scan does not satisfy the additive contract."""
|
|
97
73
|
|
|
98
74
|
|
|
75
|
+
class RecoverableContractError(ContractError):
|
|
76
|
+
"""Raised when report projection can safely be retried before publication."""
|
|
77
|
+
|
|
78
|
+
|
|
99
79
|
def _reject_non_finite_json(value: str) -> None:
|
|
100
80
|
raise ValueError(f"non-finite JSON number {value!r} is not supported")
|
|
101
81
|
|
|
@@ -106,9 +86,7 @@ def _loads_json(value: str | bytes) -> Any:
|
|
|
106
86
|
|
|
107
87
|
def _read_json(path: Path) -> dict[str, Any]:
|
|
108
88
|
try:
|
|
109
|
-
|
|
110
|
-
raw = _read_bounded_json_document(handle, str(path), SCHEMA_DOCUMENT_MAX_BYTES)
|
|
111
|
-
payload = _loads_json(raw.decode("utf-8"))
|
|
89
|
+
payload = _loads_json(path.read_text(encoding="utf-8"))
|
|
112
90
|
_require_safe_json_value(payload, str(path))
|
|
113
91
|
except FileNotFoundError as exc:
|
|
114
92
|
raise ContractError(f"missing required contract artifact: {path}") from exc
|
|
@@ -121,11 +99,6 @@ def _read_json(path: Path) -> dict[str, Any]:
|
|
|
121
99
|
return payload
|
|
122
100
|
|
|
123
101
|
|
|
124
|
-
def _write_json(path: Path, payload: Any) -> None:
|
|
125
|
-
path.parent.mkdir(parents=True, exist_ok=True)
|
|
126
|
-
path.write_bytes(_json_bytes(payload))
|
|
127
|
-
|
|
128
|
-
|
|
129
102
|
def _generate_report_projection(
|
|
130
103
|
manifest: dict[str, Any],
|
|
131
104
|
findings: dict[str, Any],
|
|
@@ -137,10 +110,21 @@ def _generate_report_projection(
|
|
|
137
110
|
raise ContractError(f"could not load report projection helper: {script}")
|
|
138
111
|
module = importlib.util.module_from_spec(spec)
|
|
139
112
|
spec.loader.exec_module(module)
|
|
140
|
-
|
|
141
|
-
|
|
142
|
-
|
|
143
|
-
|
|
113
|
+
attempts = (
|
|
114
|
+
getattr(sys.modules.get("workbench_constants"), "SQLITE_RETRY_ATTEMPTS", 1)
|
|
115
|
+
if coverage.get("mode") == "deep_repository"
|
|
116
|
+
else 1
|
|
117
|
+
)
|
|
118
|
+
for attempt in range(attempts):
|
|
119
|
+
try:
|
|
120
|
+
return module.generate_report_markdown(manifest, findings, coverage)
|
|
121
|
+
except OSError as exc:
|
|
122
|
+
if attempt == attempts - 1:
|
|
123
|
+
raise RecoverableContractError(f"report projection failed: {exc}") from exc
|
|
124
|
+
time.sleep(0.05 * (2**attempt))
|
|
125
|
+
except ValueError as exc:
|
|
126
|
+
raise ContractError(f"report projection failed: {exc}") from exc
|
|
127
|
+
raise AssertionError("Report projection retry loop exhausted unexpectedly.")
|
|
144
128
|
|
|
145
129
|
|
|
146
130
|
def _validate_report_output_paths(scan_dir: Path) -> None:
|
|
@@ -157,72 +141,21 @@ def _json_bytes(payload: Any) -> bytes:
|
|
|
157
141
|
|
|
158
142
|
def _contract_json_bytes(relative_path: str, payload: Any) -> bytes:
|
|
159
143
|
_require_safe_json_value(payload, relative_path)
|
|
160
|
-
|
|
161
|
-
maximum = CONTRACT_DOCUMENT_MAX_BYTES.get(relative_path)
|
|
162
|
-
if maximum is not None and len(encoded) > maximum:
|
|
163
|
-
raise ContractError(f"{relative_path}: JSON document exceeds the {maximum}-byte limit")
|
|
164
|
-
return encoded
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
def _read_bounded_json_document(handle: BinaryIO, context: str, maximum: int) -> bytes:
|
|
168
|
-
if os.fstat(handle.fileno()).st_size > maximum:
|
|
169
|
-
raise ContractError(f"{context}: JSON document exceeds the {maximum}-byte limit")
|
|
170
|
-
chunks: list[bytes] = []
|
|
171
|
-
length = 0
|
|
172
|
-
while length <= maximum:
|
|
173
|
-
chunk = handle.read(min(JSON_DOCUMENT_READ_CHUNK_SIZE, maximum + 1 - length))
|
|
174
|
-
if not chunk:
|
|
175
|
-
break
|
|
176
|
-
length += len(chunk)
|
|
177
|
-
if length > maximum:
|
|
178
|
-
raise ContractError(f"{context}: JSON document exceeds the {maximum}-byte limit")
|
|
179
|
-
chunks.append(chunk)
|
|
180
|
-
result = b"".join(chunks)
|
|
181
|
-
_require_json_nesting(result, context)
|
|
182
|
-
return result
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
def _require_json_nesting(value: bytes, context: str) -> None:
|
|
186
|
-
depth = 0
|
|
187
|
-
in_string = False
|
|
188
|
-
escaped = False
|
|
189
|
-
for character in value:
|
|
190
|
-
if in_string:
|
|
191
|
-
if escaped:
|
|
192
|
-
escaped = False
|
|
193
|
-
elif character == ord("\\"):
|
|
194
|
-
escaped = True
|
|
195
|
-
elif character == ord('"'):
|
|
196
|
-
in_string = False
|
|
197
|
-
continue
|
|
198
|
-
if character == ord('"'):
|
|
199
|
-
in_string = True
|
|
200
|
-
elif character in (ord("{"), ord("[")):
|
|
201
|
-
depth += 1
|
|
202
|
-
if depth > MAX_JSON_DEPTH + 1:
|
|
203
|
-
raise ContractError(
|
|
204
|
-
f"{context}: JSON document exceeds the {MAX_JSON_DEPTH}-level nesting limit"
|
|
205
|
-
)
|
|
206
|
-
elif character in (ord("}"), ord("]")):
|
|
207
|
-
depth -= 1
|
|
144
|
+
return _json_bytes(payload)
|
|
208
145
|
|
|
209
146
|
|
|
210
147
|
def _require_safe_json_value(value: Any, context: str, *, validate_strings: bool = True) -> None:
|
|
211
|
-
def visit(item: Any, location: str
|
|
212
|
-
if depth > MAX_JSON_DEPTH:
|
|
213
|
-
raise ContractError(
|
|
214
|
-
f"{location}: JSON document exceeds the {MAX_JSON_DEPTH}-level nesting limit"
|
|
215
|
-
)
|
|
148
|
+
def visit(item: Any, location: str) -> None:
|
|
216
149
|
if isinstance(item, dict):
|
|
217
150
|
for key, child in item.items():
|
|
218
151
|
if not isinstance(key, str):
|
|
219
152
|
raise ContractError(f"{location}: expected string JSON property names")
|
|
220
153
|
if validate_strings:
|
|
221
154
|
_require_safe_json_string(key, location)
|
|
222
|
-
visit(child, f"{location}.<property>"
|
|
155
|
+
visit(child, f"{location}.<property>")
|
|
223
156
|
elif isinstance(item, list):
|
|
224
157
|
for index, child in enumerate(item):
|
|
225
|
-
visit(child, f"{location}[{index}]"
|
|
158
|
+
visit(child, f"{location}[{index}]")
|
|
226
159
|
elif isinstance(item, str) and validate_strings:
|
|
227
160
|
_require_safe_json_string(item, location)
|
|
228
161
|
elif isinstance(item, int) and not isinstance(item, bool):
|
|
@@ -238,7 +171,7 @@ def _require_safe_json_value(value: Any, context: str, *, validate_strings: bool
|
|
|
238
171
|
f"{location}: unsafe integer-valued JSON numbers are not supported"
|
|
239
172
|
)
|
|
240
173
|
|
|
241
|
-
visit(value, context
|
|
174
|
+
visit(value, context)
|
|
242
175
|
|
|
243
176
|
|
|
244
177
|
def _require_safe_json_string(value: str, context: str) -> None:
|
|
@@ -256,14 +189,6 @@ def _sha256_bytes(value: bytes) -> str:
|
|
|
256
189
|
return hashlib.sha256(value).hexdigest()
|
|
257
190
|
|
|
258
191
|
|
|
259
|
-
def _sha256_file(path: Path) -> str:
|
|
260
|
-
digest = hashlib.sha256()
|
|
261
|
-
with path.open("rb") as handle:
|
|
262
|
-
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
|
|
263
|
-
digest.update(chunk)
|
|
264
|
-
return digest.hexdigest()
|
|
265
|
-
|
|
266
|
-
|
|
267
192
|
def _require_dict(payload: dict[str, Any], key: str, context: str) -> dict[str, Any]:
|
|
268
193
|
value = payload.get(key)
|
|
269
194
|
if not isinstance(value, dict):
|
|
@@ -505,12 +430,7 @@ def _read_scan_local_json_bytes(
|
|
|
505
430
|
try:
|
|
506
431
|
with os.fdopen(descriptor, "rb") as handle:
|
|
507
432
|
descriptor = -1
|
|
508
|
-
|
|
509
|
-
raw = (
|
|
510
|
-
handle.read()
|
|
511
|
-
if maximum is None
|
|
512
|
-
else _read_bounded_json_document(handle, context, maximum)
|
|
513
|
-
)
|
|
433
|
+
raw = handle.read()
|
|
514
434
|
try:
|
|
515
435
|
payload = _loads_json(raw.decode("utf-8"))
|
|
516
436
|
except (UnicodeDecodeError, ValueError) as exc:
|
|
@@ -786,7 +706,6 @@ def _recover_unsealed_findings(
|
|
|
786
706
|
warnings: list[str],
|
|
787
707
|
) -> list[str]:
|
|
788
708
|
schema = _read_json(schema_dir / "findings.schema.json")
|
|
789
|
-
_require_safe_schema(schema, "findings.schema.json")
|
|
790
709
|
properties = _require_dict(schema, "properties", "findings.schema")
|
|
791
710
|
finding_array = _require_dict(properties, "findings", "findings.schema.properties")
|
|
792
711
|
finding_schema = _require_dict(finding_array, "items", "findings.schema.properties.findings")
|
|
@@ -937,12 +856,20 @@ def _recover_unsealed_coverage(
|
|
|
937
856
|
discarded_findings: list[str],
|
|
938
857
|
) -> None:
|
|
939
858
|
schema = _read_json(schema_dir / "coverage.schema.json")
|
|
940
|
-
_require_safe_schema(schema, "coverage.schema.json")
|
|
941
859
|
properties = _require_dict(schema, "properties", "coverage.schema")
|
|
942
860
|
completeness = coverage.get("completeness")
|
|
943
861
|
partial = completeness not in ("complete", "partial", "unknown")
|
|
944
862
|
if partial:
|
|
945
863
|
warnings.append("Recovered malformed coverage completeness; marked coverage as partial.")
|
|
864
|
+
if (
|
|
865
|
+
coverage.get("mode") == "deep_repository"
|
|
866
|
+
and coverage.get("inventoryStrategy") != "repository"
|
|
867
|
+
):
|
|
868
|
+
coverage["inventoryStrategy"] = "repository"
|
|
869
|
+
warnings.append(
|
|
870
|
+
"Recovered malformed Deep Scan inventory strategy; marked coverage as partial."
|
|
871
|
+
)
|
|
872
|
+
partial = True
|
|
946
873
|
|
|
947
874
|
surface_ids: set[str] = set()
|
|
948
875
|
for field, label in (
|
|
@@ -1536,91 +1463,9 @@ def _validate_schema_node(value: Any, schema: dict[str, Any], context: str) -> N
|
|
|
1536
1463
|
|
|
1537
1464
|
def validate_against_schema(payload: dict[str, Any], schema_path: Path) -> None:
|
|
1538
1465
|
schema = _read_json(schema_path)
|
|
1539
|
-
_require_safe_schema(schema, schema_path.name)
|
|
1540
1466
|
_validate_schema_node(payload, schema, schema_path.stem)
|
|
1541
1467
|
|
|
1542
1468
|
|
|
1543
|
-
def _require_safe_schema(schema: dict[str, Any], context: str) -> None:
|
|
1544
|
-
pending: list[tuple[Any, bool]] = [(schema, True)]
|
|
1545
|
-
nodes = 0
|
|
1546
|
-
applicator_edges = 0
|
|
1547
|
-
unsupported_keywords = {
|
|
1548
|
-
"$async",
|
|
1549
|
-
"$ref",
|
|
1550
|
-
"$dynamicRef",
|
|
1551
|
-
"$recursiveRef",
|
|
1552
|
-
"prefixItems",
|
|
1553
|
-
"patternProperties",
|
|
1554
|
-
"propertyNames",
|
|
1555
|
-
"dependentSchemas",
|
|
1556
|
-
"dependencies",
|
|
1557
|
-
"uniqueItems",
|
|
1558
|
-
}
|
|
1559
|
-
while pending:
|
|
1560
|
-
value, is_schema = pending.pop()
|
|
1561
|
-
nodes += 1
|
|
1562
|
-
if nodes > MAX_SCHEMA_NODES:
|
|
1563
|
-
raise ContractError(
|
|
1564
|
-
f"{context}: JSON Schema exceeds the {MAX_SCHEMA_NODES}-node complexity limit"
|
|
1565
|
-
)
|
|
1566
|
-
if isinstance(value, list):
|
|
1567
|
-
if len(value) > MAX_SCHEMA_COLLECTION_ENTRIES:
|
|
1568
|
-
raise ContractError(
|
|
1569
|
-
f"{context}: JSON Schema exceeds the "
|
|
1570
|
-
f"{MAX_SCHEMA_COLLECTION_ENTRIES}-entry collection limit"
|
|
1571
|
-
)
|
|
1572
|
-
pending.extend((child, is_schema) for child in value)
|
|
1573
|
-
continue
|
|
1574
|
-
if not isinstance(value, dict):
|
|
1575
|
-
continue
|
|
1576
|
-
if len(value) > MAX_SCHEMA_COLLECTION_ENTRIES:
|
|
1577
|
-
raise ContractError(
|
|
1578
|
-
f"{context}: JSON Schema exceeds the "
|
|
1579
|
-
f"{MAX_SCHEMA_COLLECTION_ENTRIES}-entry collection limit"
|
|
1580
|
-
)
|
|
1581
|
-
for keyword, child in value.items():
|
|
1582
|
-
if not is_schema:
|
|
1583
|
-
pending.append((child, False))
|
|
1584
|
-
continue
|
|
1585
|
-
if keyword in unsupported_keywords:
|
|
1586
|
-
raise ContractError(f"{context}: unsupported JSON Schema keyword")
|
|
1587
|
-
edges = 0
|
|
1588
|
-
if keyword in {"allOf", "anyOf", "oneOf"} and isinstance(child, list):
|
|
1589
|
-
edges = len(child)
|
|
1590
|
-
elif keyword in {
|
|
1591
|
-
"if",
|
|
1592
|
-
"then",
|
|
1593
|
-
"else",
|
|
1594
|
-
"not",
|
|
1595
|
-
"items",
|
|
1596
|
-
"contains",
|
|
1597
|
-
"additionalProperties",
|
|
1598
|
-
"unevaluatedProperties",
|
|
1599
|
-
"unevaluatedItems",
|
|
1600
|
-
} and isinstance(child, (dict, bool)):
|
|
1601
|
-
edges = 1
|
|
1602
|
-
elif keyword in {"properties", "$defs", "definitions"} and isinstance(child, dict):
|
|
1603
|
-
edges = len(child)
|
|
1604
|
-
applicator_edges += edges
|
|
1605
|
-
if applicator_edges > MAX_SCHEMA_APPLICATOR_EDGES:
|
|
1606
|
-
raise ContractError(
|
|
1607
|
-
f"{context}: JSON Schema exceeds the "
|
|
1608
|
-
f"{MAX_SCHEMA_APPLICATOR_EDGES}-edge applicator limit"
|
|
1609
|
-
)
|
|
1610
|
-
if keyword == "pattern" and isinstance(child, str):
|
|
1611
|
-
if child not in SAFE_SCHEMA_PATTERNS:
|
|
1612
|
-
raise ContractError(f"{context}: unsupported JSON Schema pattern")
|
|
1613
|
-
if keyword in {"properties", "$defs", "definitions"} and isinstance(child, dict):
|
|
1614
|
-
pending.extend((child_schema, True) for child_schema in child.values())
|
|
1615
|
-
continue
|
|
1616
|
-
pending.append(
|
|
1617
|
-
(
|
|
1618
|
-
child,
|
|
1619
|
-
keyword not in {"const", "enum", "default", "examples", "dependentRequired"},
|
|
1620
|
-
)
|
|
1621
|
-
)
|
|
1622
|
-
|
|
1623
|
-
|
|
1624
1469
|
def _validate_canonical_schemas_before_projection(
|
|
1625
1470
|
manifest: dict[str, Any],
|
|
1626
1471
|
findings: dict[str, Any],
|
|
@@ -1666,10 +1511,7 @@ def _utf16_code_units(value: str) -> Iterator[int]:
|
|
|
1666
1511
|
def _github_line_hashes(
|
|
1667
1512
|
handle: TextIO,
|
|
1668
1513
|
requested_lines: set[int] | None = None,
|
|
1669
|
-
|
|
1670
|
-
) -> dict[int, str] | None:
|
|
1671
|
-
if source_read_budget is not None and source_read_budget[0] <= 0:
|
|
1672
|
-
return None
|
|
1514
|
+
) -> dict[int, str]:
|
|
1673
1515
|
window = [0] * GITHUB_HASH_BLOCK_SIZE
|
|
1674
1516
|
line_numbers = [-1] * GITHUB_HASH_BLOCK_SIZE
|
|
1675
1517
|
hash_counts: dict[str, int] = {}
|
|
@@ -1680,7 +1522,6 @@ def _github_line_hashes(
|
|
|
1680
1522
|
line_number = 0
|
|
1681
1523
|
line_start = True
|
|
1682
1524
|
previous_was_cr = False
|
|
1683
|
-
source_bytes = 0
|
|
1684
1525
|
|
|
1685
1526
|
def output_hash() -> None:
|
|
1686
1527
|
nonlocal index
|
|
@@ -1698,11 +1539,11 @@ def _github_line_hashes(
|
|
|
1698
1539
|
hash_raw = (GITHUB_HASH_MOD * hash_raw + current - first_mod * beginning) & GITHUB_HASH_MASK
|
|
1699
1540
|
index = (index + 1) % GITHUB_HASH_BLOCK_SIZE
|
|
1700
1541
|
|
|
1701
|
-
def process_character(current: int) ->
|
|
1542
|
+
def process_character(current: int) -> None:
|
|
1702
1543
|
nonlocal line_number, line_start, previous_was_cr
|
|
1703
1544
|
if current in {ord(" "), ord("\t")} or (previous_was_cr and current == ord("\n")):
|
|
1704
1545
|
previous_was_cr = False
|
|
1705
|
-
return
|
|
1546
|
+
return
|
|
1706
1547
|
if current == ord("\r"):
|
|
1707
1548
|
current = ord("\n")
|
|
1708
1549
|
previous_was_cr = True
|
|
@@ -1713,28 +1554,15 @@ def _github_line_hashes(
|
|
|
1713
1554
|
if line_start:
|
|
1714
1555
|
line_start = False
|
|
1715
1556
|
line_number += 1
|
|
1716
|
-
if line_number > GITHUB_HASH_MAX_LINES:
|
|
1717
|
-
return False
|
|
1718
1557
|
line_numbers[index] = line_number
|
|
1719
1558
|
if current == ord("\n"):
|
|
1720
1559
|
line_start = True
|
|
1721
1560
|
update_hash(current)
|
|
1722
|
-
return True
|
|
1723
1561
|
|
|
1724
1562
|
while chunk := handle.read(SOURCE_READ_CHUNK_SIZE):
|
|
1725
|
-
chunk_bytes = len(chunk.encode("utf-8", errors="replace"))
|
|
1726
|
-
source_bytes += chunk_bytes
|
|
1727
|
-
if source_bytes > SOURCE_READ_MAX_BYTES:
|
|
1728
|
-
return None
|
|
1729
|
-
if source_read_budget is not None:
|
|
1730
|
-
source_read_budget[0] -= chunk_bytes
|
|
1731
|
-
if source_read_budget[0] < 0:
|
|
1732
|
-
return None
|
|
1733
1563
|
for code_unit in _utf16_code_units(chunk):
|
|
1734
|
-
|
|
1735
|
-
|
|
1736
|
-
if not process_character(GITHUB_HASH_EOF):
|
|
1737
|
-
return None
|
|
1564
|
+
process_character(code_unit)
|
|
1565
|
+
process_character(GITHUB_HASH_EOF)
|
|
1738
1566
|
for _ in range(GITHUB_HASH_BLOCK_SIZE):
|
|
1739
1567
|
if line_numbers[index] != -1:
|
|
1740
1568
|
output_hash()
|
|
@@ -1762,14 +1590,13 @@ def _github_line_hashes_for_source(
|
|
|
1762
1590
|
source_root: Path,
|
|
1763
1591
|
relative_path: str,
|
|
1764
1592
|
requested_lines: set[int] | None = None,
|
|
1765
|
-
source_read_budget: list[int] | None = None,
|
|
1766
1593
|
) -> dict[int, str] | None:
|
|
1767
1594
|
handle = _open_source_file(source_root, relative_path)
|
|
1768
1595
|
if handle is None:
|
|
1769
1596
|
return None
|
|
1770
1597
|
try:
|
|
1771
1598
|
with handle:
|
|
1772
|
-
return _github_line_hashes(handle, requested_lines
|
|
1599
|
+
return _github_line_hashes(handle, requested_lines)
|
|
1773
1600
|
except OSError:
|
|
1774
1601
|
return None
|
|
1775
1602
|
|
|
@@ -1849,15 +1676,8 @@ def _github_line_hash_cache(
|
|
|
1849
1676
|
)
|
|
1850
1677
|
requested_lines_by_path.setdefault(relative_path, set()).add(primary_location["startLine"])
|
|
1851
1678
|
line_hash_cache: dict[tuple[Path, int], str | None] = {}
|
|
1852
|
-
source_read_budget = [SOURCE_READ_MAX_BYTES]
|
|
1853
1679
|
for relative_path, requested_lines in requested_lines_by_path.items():
|
|
1854
|
-
line_hashes = (
|
|
1855
|
-
None
|
|
1856
|
-
if source_read_budget[0] <= 0
|
|
1857
|
-
else _github_line_hashes_for_source(
|
|
1858
|
-
source_root, relative_path, requested_lines, source_read_budget
|
|
1859
|
-
)
|
|
1860
|
-
)
|
|
1680
|
+
line_hashes = _github_line_hashes_for_source(source_root, relative_path, requested_lines)
|
|
1861
1681
|
source_path = source_root / relative_path
|
|
1862
1682
|
for line_number in requested_lines:
|
|
1863
1683
|
line_hash_cache[(source_path, line_number)] = (
|
|
@@ -111,8 +111,14 @@ def bounded_finding_details(value: Any) -> dict[str, Any]:
|
|
|
111
111
|
):
|
|
112
112
|
key = next((alias for alias in aliases if alias in value), None)
|
|
113
113
|
if key is not None:
|
|
114
|
+
section = value[key]
|
|
115
|
+
if key == "attackPath" and isinstance(section, dict):
|
|
116
|
+
section = dict(section)
|
|
117
|
+
for assessment in ("impact", "likelihood"):
|
|
118
|
+
if isinstance(section.get(assessment), str):
|
|
119
|
+
section[assessment] = {"level": section[assessment]}
|
|
114
120
|
prepared[key] = bounded_finding_section(
|
|
115
|
-
|
|
121
|
+
section,
|
|
116
122
|
maximum_bytes,
|
|
117
123
|
priority_keys,
|
|
118
124
|
reserved_fields,
|
|
@@ -140,6 +146,8 @@ def bounded_finding_details(value: Any) -> dict[str, Any]:
|
|
|
140
146
|
"severity",
|
|
141
147
|
"status",
|
|
142
148
|
"taxonomy",
|
|
149
|
+
"preventiveControls",
|
|
150
|
+
"remediationTests",
|
|
143
151
|
):
|
|
144
152
|
if key in value:
|
|
145
153
|
prepared[key] = (
|
|
@@ -148,8 +156,64 @@ def bounded_finding_details(value: Any) -> dict[str, Any]:
|
|
|
148
156
|
else value[key]
|
|
149
157
|
)
|
|
150
158
|
|
|
151
|
-
|
|
152
|
-
|
|
159
|
+
guidance = {
|
|
160
|
+
key: prepared[key]
|
|
161
|
+
for key in ("remediationTests", "preventiveControls")
|
|
162
|
+
if key in prepared and isinstance(prepared[key], list)
|
|
163
|
+
}
|
|
164
|
+
diagnostics = (
|
|
165
|
+
"rootCause",
|
|
166
|
+
"root_cause",
|
|
167
|
+
"validation",
|
|
168
|
+
"attackPath",
|
|
169
|
+
"codeEvidence",
|
|
170
|
+
"code_evidence",
|
|
171
|
+
)
|
|
172
|
+
core_keys = (
|
|
173
|
+
"writeup",
|
|
174
|
+
*diagnostics,
|
|
175
|
+
"confidence",
|
|
176
|
+
"detectedAt",
|
|
177
|
+
"identity",
|
|
178
|
+
"provenance",
|
|
179
|
+
"ruleId",
|
|
180
|
+
"severity",
|
|
181
|
+
"status",
|
|
182
|
+
"taxonomy",
|
|
183
|
+
"evidence",
|
|
184
|
+
"evidenceExcerpt",
|
|
185
|
+
)
|
|
186
|
+
core = {key: prepared[key] for key in core_keys if key in prepared}
|
|
187
|
+
extras = {
|
|
188
|
+
key: item
|
|
189
|
+
for key, item in prepared.items()
|
|
190
|
+
if key not in core and key not in guidance
|
|
191
|
+
}
|
|
192
|
+
complete_guidance = {key: items[:1] for key, items in guidance.items()}
|
|
193
|
+
minimum_guidance = {
|
|
194
|
+
key: [items[0][:1]] if items and isinstance(items[0], str) else []
|
|
195
|
+
for key, items in guidance.items()
|
|
196
|
+
}
|
|
197
|
+
projected_core = {}
|
|
198
|
+
for selected_guidance in (complete_guidance, minimum_guidance):
|
|
199
|
+
reserved = (
|
|
200
|
+
len(json.dumps(selected_guidance, separators=(",", ":")).encode("utf-8")) - 1
|
|
201
|
+
if selected_guidance
|
|
202
|
+
else 0
|
|
203
|
+
)
|
|
204
|
+
if reserved >= FINDING_DETAILS_PREVIEW_BYTES:
|
|
205
|
+
continue
|
|
206
|
+
projected_core = bounded_json_value(
|
|
207
|
+
core,
|
|
208
|
+
[FINDING_DETAILS_PREVIEW_BYTES - reserved],
|
|
209
|
+
)
|
|
210
|
+
if all(key in projected_core for key in core):
|
|
211
|
+
break
|
|
212
|
+
ordered_guidance = dict(sorted(guidance.items(), key=lambda entry: bool(entry[1])))
|
|
213
|
+
bounded = bounded_json_value(
|
|
214
|
+
{**projected_core, **ordered_guidance, **extras},
|
|
215
|
+
[FINDING_DETAILS_PREVIEW_BYTES],
|
|
216
|
+
)
|
|
153
217
|
return bounded if isinstance(bounded, dict) else {}
|
|
154
218
|
|
|
155
219
|
|
|
@@ -215,11 +279,20 @@ def bounded_json_value(value: Any, budget: list[int], *, depth: int = 0) -> Any:
|
|
|
215
279
|
if not consume_json_budget(budget, 2):
|
|
216
280
|
return []
|
|
217
281
|
result = []
|
|
218
|
-
for item in value
|
|
282
|
+
for item in value:
|
|
283
|
+
remaining = budget[0]
|
|
219
284
|
separator = 0 if not result else 1
|
|
220
285
|
if not consume_json_budget(budget, separator):
|
|
221
286
|
break
|
|
222
|
-
|
|
287
|
+
bounded_item = bounded_json_value(item, budget, depth=depth + 1)
|
|
288
|
+
size = len(json.dumps(bounded_item, separators=(",", ":")).encode("utf-8"))
|
|
289
|
+
if separator + size > remaining or (
|
|
290
|
+
isinstance(item, str) and item and bounded_item == ""
|
|
291
|
+
):
|
|
292
|
+
budget[0] = remaining
|
|
293
|
+
break
|
|
294
|
+
budget[0] = remaining - separator - size
|
|
295
|
+
result.append(bounded_item)
|
|
223
296
|
return result
|
|
224
297
|
if isinstance(value, dict):
|
|
225
298
|
if not consume_json_budget(budget, 2):
|
|
@@ -228,13 +301,56 @@ def bounded_json_value(value: Any, budget: list[int], *, depth: int = 0) -> Any:
|
|
|
228
301
|
for key, item in list(value.items())[:20]:
|
|
229
302
|
if budget[0] <= 0 or not isinstance(key, str):
|
|
230
303
|
break
|
|
304
|
+
remaining = budget[0]
|
|
231
305
|
separator = 0 if not result else 1
|
|
232
306
|
if not consume_json_budget(budget, separator):
|
|
307
|
+
budget[0] = remaining
|
|
233
308
|
break
|
|
234
309
|
bounded_key, key_size = bounded_json_text(key, min(budget[0], 512))
|
|
235
310
|
if not consume_json_budget(budget, key_size + 1):
|
|
311
|
+
budget[0] = remaining
|
|
312
|
+
break
|
|
313
|
+
item_budget = budget
|
|
314
|
+
if depth == 0 and key == "remediationTests":
|
|
315
|
+
controls = value.get("preventiveControls")
|
|
316
|
+
if (
|
|
317
|
+
isinstance(item, list)
|
|
318
|
+
and item
|
|
319
|
+
and isinstance(item[0], str)
|
|
320
|
+
and item[0]
|
|
321
|
+
and isinstance(controls, list)
|
|
322
|
+
and controls
|
|
323
|
+
and isinstance(controls[0], str)
|
|
324
|
+
and controls[0]
|
|
325
|
+
):
|
|
326
|
+
minimum_tests = len(
|
|
327
|
+
json.dumps([item[0][0]], separators=(",", ":")).encode("utf-8")
|
|
328
|
+
)
|
|
329
|
+
for control in (controls[0], controls[0][0]):
|
|
330
|
+
reserved = (
|
|
331
|
+
len(
|
|
332
|
+
json.dumps(
|
|
333
|
+
{"preventiveControls": [control]},
|
|
334
|
+
separators=(",", ":"),
|
|
335
|
+
).encode("utf-8")
|
|
336
|
+
)
|
|
337
|
+
- 1
|
|
338
|
+
)
|
|
339
|
+
if budget[0] >= minimum_tests + reserved:
|
|
340
|
+
item_budget = [budget[0] - reserved]
|
|
341
|
+
break
|
|
342
|
+
bounded_item = bounded_json_value(item, item_budget, depth=depth + 1)
|
|
343
|
+
size = (
|
|
344
|
+
separator
|
|
345
|
+
+ key_size
|
|
346
|
+
+ 1
|
|
347
|
+
+ len(json.dumps(bounded_item, separators=(",", ":")).encode("utf-8"))
|
|
348
|
+
)
|
|
349
|
+
if size > remaining or (isinstance(item, str) and item and bounded_item == ""):
|
|
350
|
+
budget[0] = remaining
|
|
236
351
|
break
|
|
237
|
-
|
|
352
|
+
budget[0] = remaining - size
|
|
353
|
+
result[bounded_key] = bounded_item
|
|
238
354
|
return result
|
|
239
355
|
consume_json_budget(budget, 4)
|
|
240
356
|
return None
|