@openai/codex-security 0.1.9 → 0.1.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +43 -18
- package/_bundled_plugin/.codex-plugin/plugin.json +1 -1
- package/_bundled_plugin/.mcp.json +3 -1
- package/_bundled_plugin/mcp/server.mjs.br.part-000 +0 -0
- package/_bundled_plugin/mcp/server.mjs.br.part-001 +0 -0
- package/_bundled_plugin/preflight/capability-profiles.toml +1 -1
- package/_bundled_plugin/references/scan-contract.md +1 -1
- package/_bundled_plugin/scripts/deep_scan_config.py +21 -1
- package/_bundled_plugin/scripts/deep_scan_workbench.py +66 -14
- package/_bundled_plugin/scripts/finalize_scan_contract.py +15 -207
- package/_bundled_plugin/scripts/generate_in_scope_files.py +127 -2
- package/_bundled_plugin/scripts/generate_rank_input.py +38 -5
- package/_bundled_plugin/scripts/normalize_candidates.py +20 -2
- package/_bundled_plugin/scripts/rank_preview.py +4 -0
- package/_bundled_plugin/scripts/report_projection.py +39 -1
- package/_bundled_plugin/scripts/workbench_cli.py +9 -0
- package/_bundled_plugin/scripts/workbench_db.py +295 -4
- package/_bundled_plugin/scripts/workbench_native_indexes.py +100 -43
- package/_bundled_plugin/scripts/workbench_schema.py +15 -0
- package/_bundled_plugin/scripts/workbench_source_excerpt.py +1 -5
- package/_bundled_plugin/scripts/workbench_target.py +82 -0
- package/_bundled_plugin/skills/deep-security-scan/SKILL.md +5 -3
- package/_bundled_plugin/skills/finding-discovery/SKILL.md +8 -4
- package/_bundled_plugin/skills/security-diff-scan/SKILL.md +21 -140
- package/dist/api.d.ts +43 -3
- package/dist/api.d.ts.map +1 -1
- package/dist/api.js +319 -135
- package/dist/api.js.map +1 -1
- package/dist/auth.d.ts +2 -3
- package/dist/auth.d.ts.map +1 -1
- package/dist/auth.js +59 -270
- package/dist/auth.js.map +1 -1
- package/dist/cli.d.ts +1 -1
- package/dist/cli.d.ts.map +1 -1
- package/dist/cli.js +107 -43
- package/dist/cli.js.map +1 -1
- package/dist/contract.js +7 -5
- package/dist/contract.js.map +1 -1
- package/dist/cost.d.ts +2 -0
- package/dist/cost.d.ts.map +1 -1
- package/dist/cost.js +24 -2
- package/dist/cost.js.map +1 -1
- package/dist/errors.d.ts +4 -2
- package/dist/errors.d.ts.map +1 -1
- package/dist/errors.js +10 -46
- package/dist/errors.js.map +1 -1
- package/dist/index.d.ts +2 -2
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +1 -1
- package/dist/index.js.map +1 -1
- package/dist/knowledge-base.d.ts.map +1 -1
- package/dist/knowledge-base.js +2 -1
- package/dist/knowledge-base.js.map +1 -1
- package/dist/multiscan.d.ts.map +1 -1
- package/dist/multiscan.js +3 -2
- package/dist/multiscan.js.map +1 -1
- package/dist/result.d.ts +13 -1
- package/dist/result.d.ts.map +1 -1
- package/dist/result.js +3 -0
- package/dist/result.js.map +1 -1
- package/dist/runtime.d.ts +9 -6
- package/dist/runtime.d.ts.map +1 -1
- package/dist/runtime.js +130 -287
- package/dist/runtime.js.map +1 -1
- package/dist/scan-comparison.d.ts +14 -0
- package/dist/scan-comparison.d.ts.map +1 -1
- package/dist/scan-comparison.js +90 -1
- package/dist/scan-comparison.js.map +1 -1
- package/dist/scan-dashboard.d.ts.map +1 -1
- package/dist/scan-dashboard.js +1 -0
- package/dist/scan-dashboard.js.map +1 -1
- package/dist/scan-history-renderer.d.ts +1 -1
- package/dist/scan-history-renderer.d.ts.map +1 -1
- package/dist/scan-history-renderer.js +10 -0
- package/dist/scan-history-renderer.js.map +1 -1
- package/dist/scan-logs.d.ts +17 -0
- package/dist/scan-logs.d.ts.map +1 -0
- package/dist/scan-logs.js +90 -0
- package/dist/scan-logs.js.map +1 -0
- package/dist/targets.d.ts +1 -0
- package/dist/targets.d.ts.map +1 -1
- package/dist/targets.js +22 -0
- package/dist/targets.js.map +1 -1
- package/dist/version.d.ts +1 -1
- package/dist/version.js +1 -1
- package/dist/version.js.map +1 -1
- package/package.json +2 -2
|
@@ -20,7 +20,7 @@ import sys
|
|
|
20
20
|
from collections.abc import Iterator
|
|
21
21
|
from datetime import datetime, timezone
|
|
22
22
|
from pathlib import Path, PurePosixPath
|
|
23
|
-
from typing import Any,
|
|
23
|
+
from typing import Any, TextIO
|
|
24
24
|
from urllib.parse import quote, urlsplit
|
|
25
25
|
|
|
26
26
|
SCHEMA_VERSION = "1.0"
|
|
@@ -58,33 +58,8 @@ GITHUB_HASH_BLOCK_SIZE = 100
|
|
|
58
58
|
GITHUB_HASH_MOD = 37
|
|
59
59
|
GITHUB_HASH_MASK = (1 << 64) - 1
|
|
60
60
|
GITHUB_HASH_EOF = 65535
|
|
61
|
-
GITHUB_HASH_MAX_LINES = 100_000
|
|
62
61
|
SOURCE_READ_CHUNK_SIZE = 64 * 1024
|
|
63
|
-
SOURCE_READ_MAX_BYTES = 10 * 1024 * 1024
|
|
64
|
-
CONTRACT_DOCUMENT_MAX_BYTES = {
|
|
65
|
-
"scan-manifest.json": 16 * 1024 * 1024,
|
|
66
|
-
"findings.json": 128 * 1024 * 1024,
|
|
67
|
-
"coverage.json": 32 * 1024 * 1024,
|
|
68
|
-
}
|
|
69
|
-
SCHEMA_DOCUMENT_MAX_BYTES = 4 * 1024 * 1024
|
|
70
|
-
JSON_DOCUMENT_READ_CHUNK_SIZE = 64 * 1024
|
|
71
|
-
MAX_JSON_DEPTH = 256
|
|
72
62
|
MAX_JSON_INTEGER = (1 << 53) - 1
|
|
73
|
-
MAX_SCHEMA_NODES = 8192
|
|
74
|
-
MAX_SCHEMA_COLLECTION_ENTRIES = 4096
|
|
75
|
-
MAX_SCHEMA_APPLICATOR_EDGES = 128
|
|
76
|
-
SAFE_SCHEMA_PATTERNS = {
|
|
77
|
-
r"^(?![^:/?#]+://[^/?#]*@)[^?#]+$",
|
|
78
|
-
r"^codex-security-snapshot/v1:sha256:[a-f0-9]{64}$",
|
|
79
|
-
r"^(?!/)(?!.*(?:^|/)\.\.(?:/|$))(?!.*\\).+$",
|
|
80
|
-
r"^[a-f0-9]{64}$",
|
|
81
|
-
r"^(?!.*(?:^|/)\.\.(?:/|$))(?!.*\\)artifacts/.+$",
|
|
82
|
-
r"^csf_[a-f0-9]{24}$",
|
|
83
|
-
r"^occ_[a-f0-9]{24}$",
|
|
84
|
-
r"^[a-z0-9][a-z0-9._/-]*$",
|
|
85
|
-
r"^codex-security/v1:sha256:[a-f0-9]{64}$",
|
|
86
|
-
r"^findings/([a-z0-9][a-z0-9._-]*)/\1\.md$",
|
|
87
|
-
}
|
|
88
63
|
EXPORT_PATHS = {
|
|
89
64
|
"csv": "exports/findings.csv",
|
|
90
65
|
"json": "exports/findings.json",
|
|
@@ -106,9 +81,7 @@ def _loads_json(value: str | bytes) -> Any:
|
|
|
106
81
|
|
|
107
82
|
def _read_json(path: Path) -> dict[str, Any]:
|
|
108
83
|
try:
|
|
109
|
-
|
|
110
|
-
raw = _read_bounded_json_document(handle, str(path), SCHEMA_DOCUMENT_MAX_BYTES)
|
|
111
|
-
payload = _loads_json(raw.decode("utf-8"))
|
|
84
|
+
payload = _loads_json(path.read_text(encoding="utf-8"))
|
|
112
85
|
_require_safe_json_value(payload, str(path))
|
|
113
86
|
except FileNotFoundError as exc:
|
|
114
87
|
raise ContractError(f"missing required contract artifact: {path}") from exc
|
|
@@ -157,72 +130,21 @@ def _json_bytes(payload: Any) -> bytes:
|
|
|
157
130
|
|
|
158
131
|
def _contract_json_bytes(relative_path: str, payload: Any) -> bytes:
|
|
159
132
|
_require_safe_json_value(payload, relative_path)
|
|
160
|
-
|
|
161
|
-
maximum = CONTRACT_DOCUMENT_MAX_BYTES.get(relative_path)
|
|
162
|
-
if maximum is not None and len(encoded) > maximum:
|
|
163
|
-
raise ContractError(f"{relative_path}: JSON document exceeds the {maximum}-byte limit")
|
|
164
|
-
return encoded
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
def _read_bounded_json_document(handle: BinaryIO, context: str, maximum: int) -> bytes:
|
|
168
|
-
if os.fstat(handle.fileno()).st_size > maximum:
|
|
169
|
-
raise ContractError(f"{context}: JSON document exceeds the {maximum}-byte limit")
|
|
170
|
-
chunks: list[bytes] = []
|
|
171
|
-
length = 0
|
|
172
|
-
while length <= maximum:
|
|
173
|
-
chunk = handle.read(min(JSON_DOCUMENT_READ_CHUNK_SIZE, maximum + 1 - length))
|
|
174
|
-
if not chunk:
|
|
175
|
-
break
|
|
176
|
-
length += len(chunk)
|
|
177
|
-
if length > maximum:
|
|
178
|
-
raise ContractError(f"{context}: JSON document exceeds the {maximum}-byte limit")
|
|
179
|
-
chunks.append(chunk)
|
|
180
|
-
result = b"".join(chunks)
|
|
181
|
-
_require_json_nesting(result, context)
|
|
182
|
-
return result
|
|
183
|
-
|
|
184
|
-
|
|
185
|
-
def _require_json_nesting(value: bytes, context: str) -> None:
|
|
186
|
-
depth = 0
|
|
187
|
-
in_string = False
|
|
188
|
-
escaped = False
|
|
189
|
-
for character in value:
|
|
190
|
-
if in_string:
|
|
191
|
-
if escaped:
|
|
192
|
-
escaped = False
|
|
193
|
-
elif character == ord("\\"):
|
|
194
|
-
escaped = True
|
|
195
|
-
elif character == ord('"'):
|
|
196
|
-
in_string = False
|
|
197
|
-
continue
|
|
198
|
-
if character == ord('"'):
|
|
199
|
-
in_string = True
|
|
200
|
-
elif character in (ord("{"), ord("[")):
|
|
201
|
-
depth += 1
|
|
202
|
-
if depth > MAX_JSON_DEPTH + 1:
|
|
203
|
-
raise ContractError(
|
|
204
|
-
f"{context}: JSON document exceeds the {MAX_JSON_DEPTH}-level nesting limit"
|
|
205
|
-
)
|
|
206
|
-
elif character in (ord("}"), ord("]")):
|
|
207
|
-
depth -= 1
|
|
133
|
+
return _json_bytes(payload)
|
|
208
134
|
|
|
209
135
|
|
|
210
136
|
def _require_safe_json_value(value: Any, context: str, *, validate_strings: bool = True) -> None:
|
|
211
|
-
def visit(item: Any, location: str
|
|
212
|
-
if depth > MAX_JSON_DEPTH:
|
|
213
|
-
raise ContractError(
|
|
214
|
-
f"{location}: JSON document exceeds the {MAX_JSON_DEPTH}-level nesting limit"
|
|
215
|
-
)
|
|
137
|
+
def visit(item: Any, location: str) -> None:
|
|
216
138
|
if isinstance(item, dict):
|
|
217
139
|
for key, child in item.items():
|
|
218
140
|
if not isinstance(key, str):
|
|
219
141
|
raise ContractError(f"{location}: expected string JSON property names")
|
|
220
142
|
if validate_strings:
|
|
221
143
|
_require_safe_json_string(key, location)
|
|
222
|
-
visit(child, f"{location}.<property>"
|
|
144
|
+
visit(child, f"{location}.<property>")
|
|
223
145
|
elif isinstance(item, list):
|
|
224
146
|
for index, child in enumerate(item):
|
|
225
|
-
visit(child, f"{location}[{index}]"
|
|
147
|
+
visit(child, f"{location}[{index}]")
|
|
226
148
|
elif isinstance(item, str) and validate_strings:
|
|
227
149
|
_require_safe_json_string(item, location)
|
|
228
150
|
elif isinstance(item, int) and not isinstance(item, bool):
|
|
@@ -238,7 +160,7 @@ def _require_safe_json_value(value: Any, context: str, *, validate_strings: bool
|
|
|
238
160
|
f"{location}: unsafe integer-valued JSON numbers are not supported"
|
|
239
161
|
)
|
|
240
162
|
|
|
241
|
-
visit(value, context
|
|
163
|
+
visit(value, context)
|
|
242
164
|
|
|
243
165
|
|
|
244
166
|
def _require_safe_json_string(value: str, context: str) -> None:
|
|
@@ -505,12 +427,7 @@ def _read_scan_local_json_bytes(
|
|
|
505
427
|
try:
|
|
506
428
|
with os.fdopen(descriptor, "rb") as handle:
|
|
507
429
|
descriptor = -1
|
|
508
|
-
|
|
509
|
-
raw = (
|
|
510
|
-
handle.read()
|
|
511
|
-
if maximum is None
|
|
512
|
-
else _read_bounded_json_document(handle, context, maximum)
|
|
513
|
-
)
|
|
430
|
+
raw = handle.read()
|
|
514
431
|
try:
|
|
515
432
|
payload = _loads_json(raw.decode("utf-8"))
|
|
516
433
|
except (UnicodeDecodeError, ValueError) as exc:
|
|
@@ -786,7 +703,6 @@ def _recover_unsealed_findings(
|
|
|
786
703
|
warnings: list[str],
|
|
787
704
|
) -> list[str]:
|
|
788
705
|
schema = _read_json(schema_dir / "findings.schema.json")
|
|
789
|
-
_require_safe_schema(schema, "findings.schema.json")
|
|
790
706
|
properties = _require_dict(schema, "properties", "findings.schema")
|
|
791
707
|
finding_array = _require_dict(properties, "findings", "findings.schema.properties")
|
|
792
708
|
finding_schema = _require_dict(finding_array, "items", "findings.schema.properties.findings")
|
|
@@ -937,7 +853,6 @@ def _recover_unsealed_coverage(
|
|
|
937
853
|
discarded_findings: list[str],
|
|
938
854
|
) -> None:
|
|
939
855
|
schema = _read_json(schema_dir / "coverage.schema.json")
|
|
940
|
-
_require_safe_schema(schema, "coverage.schema.json")
|
|
941
856
|
properties = _require_dict(schema, "properties", "coverage.schema")
|
|
942
857
|
completeness = coverage.get("completeness")
|
|
943
858
|
partial = completeness not in ("complete", "partial", "unknown")
|
|
@@ -1536,91 +1451,9 @@ def _validate_schema_node(value: Any, schema: dict[str, Any], context: str) -> N
|
|
|
1536
1451
|
|
|
1537
1452
|
def validate_against_schema(payload: dict[str, Any], schema_path: Path) -> None:
|
|
1538
1453
|
schema = _read_json(schema_path)
|
|
1539
|
-
_require_safe_schema(schema, schema_path.name)
|
|
1540
1454
|
_validate_schema_node(payload, schema, schema_path.stem)
|
|
1541
1455
|
|
|
1542
1456
|
|
|
1543
|
-
def _require_safe_schema(schema: dict[str, Any], context: str) -> None:
|
|
1544
|
-
pending: list[tuple[Any, bool]] = [(schema, True)]
|
|
1545
|
-
nodes = 0
|
|
1546
|
-
applicator_edges = 0
|
|
1547
|
-
unsupported_keywords = {
|
|
1548
|
-
"$async",
|
|
1549
|
-
"$ref",
|
|
1550
|
-
"$dynamicRef",
|
|
1551
|
-
"$recursiveRef",
|
|
1552
|
-
"prefixItems",
|
|
1553
|
-
"patternProperties",
|
|
1554
|
-
"propertyNames",
|
|
1555
|
-
"dependentSchemas",
|
|
1556
|
-
"dependencies",
|
|
1557
|
-
"uniqueItems",
|
|
1558
|
-
}
|
|
1559
|
-
while pending:
|
|
1560
|
-
value, is_schema = pending.pop()
|
|
1561
|
-
nodes += 1
|
|
1562
|
-
if nodes > MAX_SCHEMA_NODES:
|
|
1563
|
-
raise ContractError(
|
|
1564
|
-
f"{context}: JSON Schema exceeds the {MAX_SCHEMA_NODES}-node complexity limit"
|
|
1565
|
-
)
|
|
1566
|
-
if isinstance(value, list):
|
|
1567
|
-
if len(value) > MAX_SCHEMA_COLLECTION_ENTRIES:
|
|
1568
|
-
raise ContractError(
|
|
1569
|
-
f"{context}: JSON Schema exceeds the "
|
|
1570
|
-
f"{MAX_SCHEMA_COLLECTION_ENTRIES}-entry collection limit"
|
|
1571
|
-
)
|
|
1572
|
-
pending.extend((child, is_schema) for child in value)
|
|
1573
|
-
continue
|
|
1574
|
-
if not isinstance(value, dict):
|
|
1575
|
-
continue
|
|
1576
|
-
if len(value) > MAX_SCHEMA_COLLECTION_ENTRIES:
|
|
1577
|
-
raise ContractError(
|
|
1578
|
-
f"{context}: JSON Schema exceeds the "
|
|
1579
|
-
f"{MAX_SCHEMA_COLLECTION_ENTRIES}-entry collection limit"
|
|
1580
|
-
)
|
|
1581
|
-
for keyword, child in value.items():
|
|
1582
|
-
if not is_schema:
|
|
1583
|
-
pending.append((child, False))
|
|
1584
|
-
continue
|
|
1585
|
-
if keyword in unsupported_keywords:
|
|
1586
|
-
raise ContractError(f"{context}: unsupported JSON Schema keyword")
|
|
1587
|
-
edges = 0
|
|
1588
|
-
if keyword in {"allOf", "anyOf", "oneOf"} and isinstance(child, list):
|
|
1589
|
-
edges = len(child)
|
|
1590
|
-
elif keyword in {
|
|
1591
|
-
"if",
|
|
1592
|
-
"then",
|
|
1593
|
-
"else",
|
|
1594
|
-
"not",
|
|
1595
|
-
"items",
|
|
1596
|
-
"contains",
|
|
1597
|
-
"additionalProperties",
|
|
1598
|
-
"unevaluatedProperties",
|
|
1599
|
-
"unevaluatedItems",
|
|
1600
|
-
} and isinstance(child, (dict, bool)):
|
|
1601
|
-
edges = 1
|
|
1602
|
-
elif keyword in {"properties", "$defs", "definitions"} and isinstance(child, dict):
|
|
1603
|
-
edges = len(child)
|
|
1604
|
-
applicator_edges += edges
|
|
1605
|
-
if applicator_edges > MAX_SCHEMA_APPLICATOR_EDGES:
|
|
1606
|
-
raise ContractError(
|
|
1607
|
-
f"{context}: JSON Schema exceeds the "
|
|
1608
|
-
f"{MAX_SCHEMA_APPLICATOR_EDGES}-edge applicator limit"
|
|
1609
|
-
)
|
|
1610
|
-
if keyword == "pattern" and isinstance(child, str):
|
|
1611
|
-
if child not in SAFE_SCHEMA_PATTERNS:
|
|
1612
|
-
raise ContractError(f"{context}: unsupported JSON Schema pattern")
|
|
1613
|
-
if keyword in {"properties", "$defs", "definitions"} and isinstance(child, dict):
|
|
1614
|
-
pending.extend((child_schema, True) for child_schema in child.values())
|
|
1615
|
-
continue
|
|
1616
|
-
pending.append(
|
|
1617
|
-
(
|
|
1618
|
-
child,
|
|
1619
|
-
keyword not in {"const", "enum", "default", "examples", "dependentRequired"},
|
|
1620
|
-
)
|
|
1621
|
-
)
|
|
1622
|
-
|
|
1623
|
-
|
|
1624
1457
|
def _validate_canonical_schemas_before_projection(
|
|
1625
1458
|
manifest: dict[str, Any],
|
|
1626
1459
|
findings: dict[str, Any],
|
|
@@ -1666,10 +1499,7 @@ def _utf16_code_units(value: str) -> Iterator[int]:
|
|
|
1666
1499
|
def _github_line_hashes(
|
|
1667
1500
|
handle: TextIO,
|
|
1668
1501
|
requested_lines: set[int] | None = None,
|
|
1669
|
-
|
|
1670
|
-
) -> dict[int, str] | None:
|
|
1671
|
-
if source_read_budget is not None and source_read_budget[0] <= 0:
|
|
1672
|
-
return None
|
|
1502
|
+
) -> dict[int, str]:
|
|
1673
1503
|
window = [0] * GITHUB_HASH_BLOCK_SIZE
|
|
1674
1504
|
line_numbers = [-1] * GITHUB_HASH_BLOCK_SIZE
|
|
1675
1505
|
hash_counts: dict[str, int] = {}
|
|
@@ -1680,7 +1510,6 @@ def _github_line_hashes(
|
|
|
1680
1510
|
line_number = 0
|
|
1681
1511
|
line_start = True
|
|
1682
1512
|
previous_was_cr = False
|
|
1683
|
-
source_bytes = 0
|
|
1684
1513
|
|
|
1685
1514
|
def output_hash() -> None:
|
|
1686
1515
|
nonlocal index
|
|
@@ -1698,11 +1527,11 @@ def _github_line_hashes(
|
|
|
1698
1527
|
hash_raw = (GITHUB_HASH_MOD * hash_raw + current - first_mod * beginning) & GITHUB_HASH_MASK
|
|
1699
1528
|
index = (index + 1) % GITHUB_HASH_BLOCK_SIZE
|
|
1700
1529
|
|
|
1701
|
-
def process_character(current: int) ->
|
|
1530
|
+
def process_character(current: int) -> None:
|
|
1702
1531
|
nonlocal line_number, line_start, previous_was_cr
|
|
1703
1532
|
if current in {ord(" "), ord("\t")} or (previous_was_cr and current == ord("\n")):
|
|
1704
1533
|
previous_was_cr = False
|
|
1705
|
-
return
|
|
1534
|
+
return
|
|
1706
1535
|
if current == ord("\r"):
|
|
1707
1536
|
current = ord("\n")
|
|
1708
1537
|
previous_was_cr = True
|
|
@@ -1713,28 +1542,15 @@ def _github_line_hashes(
|
|
|
1713
1542
|
if line_start:
|
|
1714
1543
|
line_start = False
|
|
1715
1544
|
line_number += 1
|
|
1716
|
-
if line_number > GITHUB_HASH_MAX_LINES:
|
|
1717
|
-
return False
|
|
1718
1545
|
line_numbers[index] = line_number
|
|
1719
1546
|
if current == ord("\n"):
|
|
1720
1547
|
line_start = True
|
|
1721
1548
|
update_hash(current)
|
|
1722
|
-
return True
|
|
1723
1549
|
|
|
1724
1550
|
while chunk := handle.read(SOURCE_READ_CHUNK_SIZE):
|
|
1725
|
-
chunk_bytes = len(chunk.encode("utf-8", errors="replace"))
|
|
1726
|
-
source_bytes += chunk_bytes
|
|
1727
|
-
if source_bytes > SOURCE_READ_MAX_BYTES:
|
|
1728
|
-
return None
|
|
1729
|
-
if source_read_budget is not None:
|
|
1730
|
-
source_read_budget[0] -= chunk_bytes
|
|
1731
|
-
if source_read_budget[0] < 0:
|
|
1732
|
-
return None
|
|
1733
1551
|
for code_unit in _utf16_code_units(chunk):
|
|
1734
|
-
|
|
1735
|
-
|
|
1736
|
-
if not process_character(GITHUB_HASH_EOF):
|
|
1737
|
-
return None
|
|
1552
|
+
process_character(code_unit)
|
|
1553
|
+
process_character(GITHUB_HASH_EOF)
|
|
1738
1554
|
for _ in range(GITHUB_HASH_BLOCK_SIZE):
|
|
1739
1555
|
if line_numbers[index] != -1:
|
|
1740
1556
|
output_hash()
|
|
@@ -1762,14 +1578,13 @@ def _github_line_hashes_for_source(
|
|
|
1762
1578
|
source_root: Path,
|
|
1763
1579
|
relative_path: str,
|
|
1764
1580
|
requested_lines: set[int] | None = None,
|
|
1765
|
-
source_read_budget: list[int] | None = None,
|
|
1766
1581
|
) -> dict[int, str] | None:
|
|
1767
1582
|
handle = _open_source_file(source_root, relative_path)
|
|
1768
1583
|
if handle is None:
|
|
1769
1584
|
return None
|
|
1770
1585
|
try:
|
|
1771
1586
|
with handle:
|
|
1772
|
-
return _github_line_hashes(handle, requested_lines
|
|
1587
|
+
return _github_line_hashes(handle, requested_lines)
|
|
1773
1588
|
except OSError:
|
|
1774
1589
|
return None
|
|
1775
1590
|
|
|
@@ -1849,15 +1664,8 @@ def _github_line_hash_cache(
|
|
|
1849
1664
|
)
|
|
1850
1665
|
requested_lines_by_path.setdefault(relative_path, set()).add(primary_location["startLine"])
|
|
1851
1666
|
line_hash_cache: dict[tuple[Path, int], str | None] = {}
|
|
1852
|
-
source_read_budget = [SOURCE_READ_MAX_BYTES]
|
|
1853
1667
|
for relative_path, requested_lines in requested_lines_by_path.items():
|
|
1854
|
-
line_hashes = (
|
|
1855
|
-
None
|
|
1856
|
-
if source_read_budget[0] <= 0
|
|
1857
|
-
else _github_line_hashes_for_source(
|
|
1858
|
-
source_root, relative_path, requested_lines, source_read_budget
|
|
1859
|
-
)
|
|
1860
|
-
)
|
|
1668
|
+
line_hashes = _github_line_hashes_for_source(source_root, relative_path, requested_lines)
|
|
1861
1669
|
source_path = source_root / relative_path
|
|
1862
1670
|
for line_number in requested_lines:
|
|
1863
1671
|
line_hash_cache[(source_path, line_number)] = (
|
|
@@ -68,7 +68,18 @@ def resolve_output(value: str) -> Path:
|
|
|
68
68
|
|
|
69
69
|
def generate_in_scope_files(repository: Path, scope: str, output: Path) -> int:
|
|
70
70
|
"""Atomically write the exact ripgrep inventory sorted as ``LC_ALL=C``."""
|
|
71
|
-
command = [
|
|
71
|
+
command = [
|
|
72
|
+
"rg",
|
|
73
|
+
"--files",
|
|
74
|
+
"--hidden",
|
|
75
|
+
"--no-ignore",
|
|
76
|
+
"--path-separator",
|
|
77
|
+
"/",
|
|
78
|
+
"--glob",
|
|
79
|
+
"!.git/**",
|
|
80
|
+
"--",
|
|
81
|
+
scope,
|
|
82
|
+
]
|
|
72
83
|
with tempfile.TemporaryFile(mode="w+b") as inventory:
|
|
73
84
|
try:
|
|
74
85
|
result = subprocess.run(
|
|
@@ -91,6 +102,101 @@ def generate_in_scope_files(repository: Path, scope: str, output: Path) -> int:
|
|
|
91
102
|
inventory.seek(0)
|
|
92
103
|
rows = sorted(inventory)
|
|
93
104
|
|
|
105
|
+
return write_inventory(output, rows)
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
def generate_diff_in_scope_files(
|
|
109
|
+
repository: Path,
|
|
110
|
+
base: str,
|
|
111
|
+
head: str,
|
|
112
|
+
mode: str,
|
|
113
|
+
output: Path,
|
|
114
|
+
) -> int:
|
|
115
|
+
"""Reuse the existing diff selection without generating previews or duplicate worklists."""
|
|
116
|
+
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
|
117
|
+
from generate_rank_input import git_changed_paths, path_is_excluded, run_git_changed_paths
|
|
118
|
+
from rank_preview import (
|
|
119
|
+
DEFAULT_PREVIEW_BYTES,
|
|
120
|
+
TEXT_CODE_EXTENSIONS,
|
|
121
|
+
is_binary_sample,
|
|
122
|
+
preview_for,
|
|
123
|
+
)
|
|
124
|
+
from workbench_target import git_blob_bytes
|
|
125
|
+
|
|
126
|
+
rows: list[bytes] = []
|
|
127
|
+
try:
|
|
128
|
+
if mode == "local-patch":
|
|
129
|
+
changed = run_git_changed_paths(repository, [base])
|
|
130
|
+
untracked = subprocess.run(
|
|
131
|
+
["git", "-C", str(repository), "ls-files", "--others", "--exclude-standard", "-z"],
|
|
132
|
+
capture_output=True,
|
|
133
|
+
text=True,
|
|
134
|
+
check=True,
|
|
135
|
+
)
|
|
136
|
+
changed.extend(
|
|
137
|
+
(repository / relative, "A")
|
|
138
|
+
for relative in untracked.stdout.split("\0")
|
|
139
|
+
if relative
|
|
140
|
+
)
|
|
141
|
+
else:
|
|
142
|
+
changed = git_changed_paths(repository, base, head, mode)
|
|
143
|
+
|
|
144
|
+
eligible = [
|
|
145
|
+
(path, status)
|
|
146
|
+
for path, status in changed
|
|
147
|
+
if not path_is_excluded(path.relative_to(repository))
|
|
148
|
+
and path.suffix.lower() in TEXT_CODE_EXTENSIONS
|
|
149
|
+
]
|
|
150
|
+
revision_paths = [
|
|
151
|
+
path.relative_to(repository)
|
|
152
|
+
for path, status in eligible
|
|
153
|
+
if mode == "revisions" and status != "D"
|
|
154
|
+
]
|
|
155
|
+
revision_blobs = dict(
|
|
156
|
+
zip(
|
|
157
|
+
revision_paths,
|
|
158
|
+
git_blob_bytes(
|
|
159
|
+
repository,
|
|
160
|
+
[f"{head}:{path.as_posix()}" for path in revision_paths],
|
|
161
|
+
),
|
|
162
|
+
)
|
|
163
|
+
)
|
|
164
|
+
|
|
165
|
+
for path, status in eligible:
|
|
166
|
+
relative = path.relative_to(repository)
|
|
167
|
+
if status != "D":
|
|
168
|
+
if mode == "revisions":
|
|
169
|
+
contents = revision_blobs[relative]
|
|
170
|
+
if contents is None:
|
|
171
|
+
raise InventoryError(
|
|
172
|
+
f"could not read committed diff blob: {head}:{relative.as_posix()}"
|
|
173
|
+
)
|
|
174
|
+
if is_binary_sample(contents):
|
|
175
|
+
continue
|
|
176
|
+
elif (
|
|
177
|
+
path.is_symlink()
|
|
178
|
+
or not path.is_file()
|
|
179
|
+
or preview_for(path, DEFAULT_PREVIEW_BYTES)[1]
|
|
180
|
+
):
|
|
181
|
+
continue
|
|
182
|
+
relative_path = relative.as_posix()
|
|
183
|
+
if "\n" in relative_path or "\r" in relative_path:
|
|
184
|
+
raise InventoryError(
|
|
185
|
+
"Git changes contain a path that cannot fit in the file inventory"
|
|
186
|
+
)
|
|
187
|
+
rows.append(f"{relative_path}\n".encode())
|
|
188
|
+
except (OSError, subprocess.CalledProcessError) as error:
|
|
189
|
+
detail = getattr(error, "stderr", None)
|
|
190
|
+
if isinstance(detail, bytes):
|
|
191
|
+
detail = detail.decode("utf-8", errors="replace")
|
|
192
|
+
message = detail.strip() if isinstance(detail, str) and detail.strip() else str(error)
|
|
193
|
+
raise InventoryError(f"could not resolve the selected Git changes: {message}") from error
|
|
194
|
+
|
|
195
|
+
return write_inventory(output, sorted(set(rows)))
|
|
196
|
+
|
|
197
|
+
|
|
198
|
+
def write_inventory(output: Path, rows: list[bytes]) -> int:
|
|
199
|
+
"""Replace a complete inventory atomically, keeping failures from corrupting the old one."""
|
|
94
200
|
output.parent.mkdir(parents=True, exist_ok=True)
|
|
95
201
|
temporary: Path | None = None
|
|
96
202
|
try:
|
|
@@ -116,13 +222,32 @@ def main() -> None:
|
|
|
116
222
|
parser.add_argument("--repo", required=True, help="Repository root.")
|
|
117
223
|
parser.add_argument("--scope", required=True, help="File or directory within the repository.")
|
|
118
224
|
parser.add_argument("--out", required=True, help="Destination for the file inventory.")
|
|
225
|
+
parser.add_argument("--diff-base", help="Authoritative Git base for a changed-file inventory.")
|
|
226
|
+
parser.add_argument("--diff-head", default="HEAD", help="Authoritative Git head revision.")
|
|
227
|
+
parser.add_argument(
|
|
228
|
+
"--diff-mode",
|
|
229
|
+
choices=("revisions", "local-patch"),
|
|
230
|
+
default="revisions",
|
|
231
|
+
help="Use committed revisions or the current staged and unstaged patch.",
|
|
232
|
+
)
|
|
119
233
|
args = parser.parse_args()
|
|
120
234
|
|
|
121
235
|
try:
|
|
122
236
|
repository = resolve_repository(args.repo)
|
|
123
237
|
scope = resolve_scope(repository, args.scope)
|
|
124
238
|
output = resolve_output(args.out)
|
|
125
|
-
|
|
239
|
+
if args.diff_base is None:
|
|
240
|
+
count = generate_in_scope_files(repository, scope, output)
|
|
241
|
+
elif scope not in (".", "./"):
|
|
242
|
+
raise InventoryError("--scope: diff scans must use the repository root")
|
|
243
|
+
else:
|
|
244
|
+
count = generate_diff_in_scope_files(
|
|
245
|
+
repository,
|
|
246
|
+
args.diff_base,
|
|
247
|
+
args.diff_head,
|
|
248
|
+
args.diff_mode,
|
|
249
|
+
output,
|
|
250
|
+
)
|
|
126
251
|
except (OSError, ValueError) as error:
|
|
127
252
|
print(f"generate_in_scope_files: {error}", file=sys.stderr)
|
|
128
253
|
raise SystemExit(2) from error
|
|
@@ -37,8 +37,13 @@ from pathlib import Path
|
|
|
37
37
|
|
|
38
38
|
# Some plugin hosts launch Python with safe-path isolation enabled.
|
|
39
39
|
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
|
40
|
-
from rank_preview import
|
|
41
|
-
|
|
40
|
+
from rank_preview import (
|
|
41
|
+
DEFAULT_PREVIEW_BYTES,
|
|
42
|
+
TEXT_CODE_EXTENSIONS,
|
|
43
|
+
preview_for,
|
|
44
|
+
preview_for_bytes,
|
|
45
|
+
)
|
|
46
|
+
from workbench_target import git_blob_bytes, git_directory_snapshot_paths
|
|
42
47
|
|
|
43
48
|
EXCLUDED_DIRS = {
|
|
44
49
|
".cache",
|
|
@@ -652,14 +657,42 @@ def make_diff_rank_input(args: argparse.Namespace) -> None:
|
|
|
652
657
|
if not repo.is_dir():
|
|
653
658
|
raise SystemExit(f"Repo path not found: {repo}")
|
|
654
659
|
|
|
660
|
+
changed = [
|
|
661
|
+
(path, status)
|
|
662
|
+
for path, status in git_changed_paths(repo, args.base, args.head, args.mode)
|
|
663
|
+
if not path_is_excluded(path.relative_to(repo))
|
|
664
|
+
and path.suffix.lower() in TEXT_CODE_EXTENSIONS
|
|
665
|
+
]
|
|
666
|
+
revision_paths = [
|
|
667
|
+
path.relative_to(repo)
|
|
668
|
+
for path, status in changed
|
|
669
|
+
if args.mode == "revisions" and status != "D"
|
|
670
|
+
]
|
|
671
|
+
revision_blobs = dict(
|
|
672
|
+
zip(
|
|
673
|
+
revision_paths,
|
|
674
|
+
git_blob_bytes(
|
|
675
|
+
repo,
|
|
676
|
+
[f"{args.head}:{path.as_posix()}" for path in revision_paths],
|
|
677
|
+
),
|
|
678
|
+
)
|
|
679
|
+
)
|
|
680
|
+
|
|
655
681
|
rows: list[JsonRow] = []
|
|
656
|
-
for path, status in
|
|
682
|
+
for path, status in changed:
|
|
657
683
|
rel = path.relative_to(repo)
|
|
658
|
-
if path_is_excluded(rel) or path.suffix.lower() not in TEXT_CODE_EXTENSIONS:
|
|
659
|
-
continue
|
|
660
684
|
|
|
661
685
|
if status == "D":
|
|
662
686
|
preview = ""
|
|
687
|
+
elif args.mode == "revisions":
|
|
688
|
+
content = revision_blobs[rel]
|
|
689
|
+
if content is None:
|
|
690
|
+
raise SystemExit(
|
|
691
|
+
f"Unable to read committed diff blob: {args.head}:{rel.as_posix()}"
|
|
692
|
+
)
|
|
693
|
+
preview, is_binary = preview_for_bytes(rel, content, args.preview_bytes)
|
|
694
|
+
if is_binary:
|
|
695
|
+
continue
|
|
663
696
|
elif path.is_file():
|
|
664
697
|
preview, is_binary = preview_for(path, args.preview_bytes)
|
|
665
698
|
if is_binary:
|
|
@@ -135,7 +135,7 @@ def normalize_locations(
|
|
|
135
135
|
)
|
|
136
136
|
|
|
137
137
|
|
|
138
|
-
def read_scope(path: Path, repo_root: Path) -> set[str]:
|
|
138
|
+
def read_scope(path: Path, repo_root: Path, *, allow_missing: bool = False) -> set[str]:
|
|
139
139
|
contents = path.read_bytes().decode("utf-8")
|
|
140
140
|
lines = contents.split("\n")
|
|
141
141
|
listed_rows = set(lines)
|
|
@@ -183,6 +183,19 @@ def read_scope(path: Path, repo_root: Path) -> set[str]:
|
|
|
183
183
|
try:
|
|
184
184
|
relative, _ = relative_file(line, repo_root)
|
|
185
185
|
except (OSError, ValueError) as error:
|
|
186
|
+
if allow_missing and isinstance(error, FileNotFoundError):
|
|
187
|
+
candidate = PurePosixPath(line)
|
|
188
|
+
if candidate.is_absolute() or ".." in candidate.parts or "\0" in line:
|
|
189
|
+
raise ValueError(f"in-scope file row {number}: unsafe deleted path") from error
|
|
190
|
+
resolved = (repo_root / line).resolve(strict=False)
|
|
191
|
+
try:
|
|
192
|
+
relative = resolved.relative_to(repo_root).as_posix()
|
|
193
|
+
except ValueError as escaped:
|
|
194
|
+
raise ValueError(
|
|
195
|
+
f"in-scope file row {number}: path escapes repository"
|
|
196
|
+
) from escaped
|
|
197
|
+
scope.add(relative)
|
|
198
|
+
continue
|
|
186
199
|
raise ValueError(f"in-scope file row {number}: {error}") from error
|
|
187
200
|
scope.add(relative)
|
|
188
201
|
return scope
|
|
@@ -261,6 +274,11 @@ def main() -> None:
|
|
|
261
274
|
parser.add_argument("--out", required=True, help="Combined candidate JSONL output.")
|
|
262
275
|
parser.add_argument("--repo-root", required=True, help="Repository root for candidate paths.")
|
|
263
276
|
parser.add_argument("--in-scope-files", required=True, help="Repository-relative file list.")
|
|
277
|
+
parser.add_argument(
|
|
278
|
+
"--allow-missing-in-scope",
|
|
279
|
+
action="store_true",
|
|
280
|
+
help="Keep deleted Git paths in a diff inventory while validating existing candidate files.",
|
|
281
|
+
)
|
|
264
282
|
args = parser.parse_args()
|
|
265
283
|
try:
|
|
266
284
|
repo_root = Path(args.repo_root).expanduser().resolve(strict=True)
|
|
@@ -273,7 +291,7 @@ def main() -> None:
|
|
|
273
291
|
raise ValueError("--out: must not also be an input")
|
|
274
292
|
if output == scope_path:
|
|
275
293
|
raise ValueError("--out: must not replace --in-scope-files")
|
|
276
|
-
scope = read_scope(scope_path, repo_root)
|
|
294
|
+
scope = read_scope(scope_path, repo_root, allow_missing=args.allow_missing_in_scope)
|
|
277
295
|
line_counts: dict[Path, int] = {}
|
|
278
296
|
rows: list[dict[str, Any]] = []
|
|
279
297
|
for source in inputs:
|
|
@@ -949,6 +949,10 @@ def preview_for(
|
|
|
949
949
|
data = sample + remaining
|
|
950
950
|
except OSError:
|
|
951
951
|
return "", True
|
|
952
|
+
return preview_for_bytes(path, data, preview_bytes)
|
|
953
|
+
|
|
954
|
+
|
|
955
|
+
def preview_for_bytes(path: Path, data: bytes, preview_bytes: int) -> tuple[str, bool]:
|
|
952
956
|
if is_binary_sample(data):
|
|
953
957
|
return "", True
|
|
954
958
|
text = data.decode("utf-8", errors="ignore")
|
|
@@ -739,11 +739,49 @@ def build_report_markdown(
|
|
|
739
739
|
else:
|
|
740
740
|
lines.extend(["", *_finding_section(number, finding)])
|
|
741
741
|
else:
|
|
742
|
+
deferred = coverage.get("deferred", [])
|
|
743
|
+
no_source_review = (
|
|
744
|
+
coverage.get("completeness") == "partial"
|
|
745
|
+
and isinstance(deferred, list)
|
|
746
|
+
and any(
|
|
747
|
+
isinstance(item, dict)
|
|
748
|
+
and item.get("reason")
|
|
749
|
+
== "The configured discovery time limit elapsed before any source review completed."
|
|
750
|
+
for item in deferred
|
|
751
|
+
)
|
|
752
|
+
)
|
|
753
|
+
budget_exhausted = (
|
|
754
|
+
coverage.get("completeness") == "partial"
|
|
755
|
+
and isinstance(deferred, list)
|
|
756
|
+
and any(
|
|
757
|
+
isinstance(item, dict)
|
|
758
|
+
and isinstance(item.get("reason"), str)
|
|
759
|
+
and (
|
|
760
|
+
item["reason"]
|
|
761
|
+
== "Validation was deferred because the scan reached its cost limit."
|
|
762
|
+
or item["reason"].startswith(
|
|
763
|
+
"Validation was deferred because the scan reached its cost limit: "
|
|
764
|
+
)
|
|
765
|
+
)
|
|
766
|
+
for item in deferred
|
|
767
|
+
)
|
|
768
|
+
)
|
|
742
769
|
lines.extend(
|
|
743
770
|
[
|
|
744
771
|
"### No findings",
|
|
745
772
|
"",
|
|
746
|
-
|
|
773
|
+
(
|
|
774
|
+
"No source review completed before the configured time limit. "
|
|
775
|
+
"No vulnerability conclusion can be drawn."
|
|
776
|
+
if no_source_review
|
|
777
|
+
else (
|
|
778
|
+
"No findings were validated before the scan reached its cost limit. "
|
|
779
|
+
"Review the deferred candidates in Open Questions And Follow Up."
|
|
780
|
+
if budget_exhausted
|
|
781
|
+
else "No reportable findings survived the canonical discovery, validation, "
|
|
782
|
+
"and reportability gates."
|
|
783
|
+
)
|
|
784
|
+
),
|
|
747
785
|
]
|
|
748
786
|
)
|
|
749
787
|
if hardening_portfolio_path is not None:
|