@openai/codex-security 0.1.10 → 0.1.12

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (99) hide show
  1. package/README.md +72 -41
  2. package/_bundled_plugin/.codex-plugin/plugin.json +1 -1
  3. package/_bundled_plugin/.mcp.json +5 -1
  4. package/_bundled_plugin/examples/completed-scan/coverage.json +1 -1
  5. package/_bundled_plugin/examples/completed-scan/scan-manifest.json +2 -2
  6. package/_bundled_plugin/mcp/server.mjs.br.part-000 +0 -0
  7. package/_bundled_plugin/mcp/server.mjs.br.part-001 +0 -0
  8. package/_bundled_plugin/preflight/capability-profiles.toml +1 -34
  9. package/_bundled_plugin/references/config-preflight.md +11 -13
  10. package/_bundled_plugin/references/core-scan.md +94 -0
  11. package/_bundled_plugin/references/final-report.md +6 -6
  12. package/_bundled_plugin/references/scan-artifacts.md +15 -14
  13. package/_bundled_plugin/references/scan-prologue.md +27 -0
  14. package/_bundled_plugin/schemas/tools/deep-reducer.schema.json +1 -106
  15. package/_bundled_plugin/scripts/config_preflight.py +40 -3
  16. package/_bundled_plugin/scripts/deep_scan_config.py +29 -6
  17. package/_bundled_plugin/scripts/deep_scan_workbench.py +132 -34
  18. package/_bundled_plugin/scripts/finalize_scan_contract.py +44 -224
  19. package/_bundled_plugin/scripts/finding_preview.py +122 -6
  20. package/_bundled_plugin/scripts/generate_in_scope_files.py +77 -31
  21. package/_bundled_plugin/scripts/generate_rank_input.py +58 -7
  22. package/_bundled_plugin/scripts/normalize_candidates.py +5 -5
  23. package/_bundled_plugin/scripts/rank_preview.py +4 -0
  24. package/_bundled_plugin/scripts/report_projection.py +40 -20
  25. package/_bundled_plugin/scripts/workbench_cli.py +5 -0
  26. package/_bundled_plugin/scripts/workbench_db.py +304 -32
  27. package/_bundled_plugin/scripts/workbench_progress.py +0 -5
  28. package/_bundled_plugin/scripts/workbench_scan_usage.py +43 -8
  29. package/_bundled_plugin/scripts/workbench_schema.py +16 -20
  30. package/_bundled_plugin/scripts/workbench_source_excerpt.py +1 -5
  31. package/_bundled_plugin/scripts/workbench_target.py +82 -0
  32. package/_bundled_plugin/skills/attack-path-analysis/SKILL.md +8 -11
  33. package/_bundled_plugin/skills/deep-security-scan/SKILL.md +24 -108
  34. package/_bundled_plugin/skills/security-scan/SKILL.md +5 -88
  35. package/_bundled_plugin/skills/threat-model/SKILL.md +4 -2
  36. package/_bundled_plugin/skills/track-findings/SKILL.md +1 -1
  37. package/_bundled_plugin/skills/triage-finding/SKILL.md +5 -6
  38. package/_bundled_plugin/skills/triage-finding/references/github-rest-intake.md +6 -7
  39. package/_bundled_plugin/skills/validation/SKILL.md +9 -12
  40. package/_bundled_plugin/skills/vulnerability-writeup/SKILL.md +3 -2
  41. package/_bundled_plugin/skills/vulnerability-writeup/references/report-format.md +1 -1
  42. package/dist/api.d.ts +2 -0
  43. package/dist/api.d.ts.map +1 -1
  44. package/dist/api.js +393 -145
  45. package/dist/api.js.map +1 -1
  46. package/dist/auth.d.ts +2 -3
  47. package/dist/auth.d.ts.map +1 -1
  48. package/dist/auth.js +59 -270
  49. package/dist/auth.js.map +1 -1
  50. package/dist/bulk-scan-discovery.js +14 -2
  51. package/dist/bulk-scan-discovery.js.map +1 -1
  52. package/dist/cli.d.ts +6 -3
  53. package/dist/cli.d.ts.map +1 -1
  54. package/dist/cli.js +202 -46
  55. package/dist/cli.js.map +1 -1
  56. package/dist/config.d.ts.map +1 -1
  57. package/dist/config.js +1 -0
  58. package/dist/config.js.map +1 -1
  59. package/dist/contract.js +7 -5
  60. package/dist/contract.js.map +1 -1
  61. package/dist/cost.d.ts +1 -0
  62. package/dist/cost.d.ts.map +1 -1
  63. package/dist/cost.js +33 -2
  64. package/dist/cost.js.map +1 -1
  65. package/dist/knowledge-base.d.ts.map +1 -1
  66. package/dist/knowledge-base.js +12 -4
  67. package/dist/knowledge-base.js.map +1 -1
  68. package/dist/multiscan.d.ts +1 -0
  69. package/dist/multiscan.d.ts.map +1 -1
  70. package/dist/multiscan.js +77 -17
  71. package/dist/multiscan.js.map +1 -1
  72. package/dist/runtime.d.ts +9 -6
  73. package/dist/runtime.d.ts.map +1 -1
  74. package/dist/runtime.js +140 -326
  75. package/dist/runtime.js.map +1 -1
  76. package/dist/scan-comparison.js +1 -1
  77. package/dist/scan-comparison.js.map +1 -1
  78. package/dist/scan-dashboard.d.ts +5 -1
  79. package/dist/scan-dashboard.d.ts.map +1 -1
  80. package/dist/scan-dashboard.js +65 -16
  81. package/dist/scan-dashboard.js.map +1 -1
  82. package/dist/scan-logs.d.ts +2 -0
  83. package/dist/scan-logs.d.ts.map +1 -1
  84. package/dist/scan-logs.js +97 -37
  85. package/dist/scan-logs.js.map +1 -1
  86. package/dist/targets.d.ts +1 -0
  87. package/dist/targets.d.ts.map +1 -1
  88. package/dist/targets.js +35 -4
  89. package/dist/targets.js.map +1 -1
  90. package/dist/trusted-executable.js +1 -1
  91. package/dist/trusted-executable.js.map +1 -1
  92. package/dist/version.d.ts +3 -3
  93. package/dist/version.d.ts.map +1 -1
  94. package/dist/version.js +6 -4
  95. package/dist/version.js.map +1 -1
  96. package/package.json +4 -4
  97. package/_bundled_plugin/references/shared-hard-rules.md +0 -15
  98. package/_bundled_plugin/scripts/validate_report_format.py +0 -138
  99. package/_bundled_plugin/skills/security-scan/references/repository-wide-scan.md +0 -33
@@ -17,10 +17,11 @@ import re
17
17
  import secrets
18
18
  import stat
19
19
  import sys
20
+ import time
20
21
  from collections.abc import Iterator
21
22
  from datetime import datetime, timezone
22
23
  from pathlib import Path, PurePosixPath
23
- from typing import Any, BinaryIO, TextIO
24
+ from typing import Any, TextIO
24
25
  from urllib.parse import quote, urlsplit
25
26
 
26
27
  SCHEMA_VERSION = "1.0"
@@ -58,33 +59,8 @@ GITHUB_HASH_BLOCK_SIZE = 100
58
59
  GITHUB_HASH_MOD = 37
59
60
  GITHUB_HASH_MASK = (1 << 64) - 1
60
61
  GITHUB_HASH_EOF = 65535
61
- GITHUB_HASH_MAX_LINES = 100_000
62
62
  SOURCE_READ_CHUNK_SIZE = 64 * 1024
63
- SOURCE_READ_MAX_BYTES = 10 * 1024 * 1024
64
- CONTRACT_DOCUMENT_MAX_BYTES = {
65
- "scan-manifest.json": 16 * 1024 * 1024,
66
- "findings.json": 128 * 1024 * 1024,
67
- "coverage.json": 32 * 1024 * 1024,
68
- }
69
- SCHEMA_DOCUMENT_MAX_BYTES = 4 * 1024 * 1024
70
- JSON_DOCUMENT_READ_CHUNK_SIZE = 64 * 1024
71
- MAX_JSON_DEPTH = 256
72
63
  MAX_JSON_INTEGER = (1 << 53) - 1
73
- MAX_SCHEMA_NODES = 8192
74
- MAX_SCHEMA_COLLECTION_ENTRIES = 4096
75
- MAX_SCHEMA_APPLICATOR_EDGES = 128
76
- SAFE_SCHEMA_PATTERNS = {
77
- r"^(?![^:/?#]+://[^/?#]*@)[^?#]+$",
78
- r"^codex-security-snapshot/v1:sha256:[a-f0-9]{64}$",
79
- r"^(?!/)(?!.*(?:^|/)\.\.(?:/|$))(?!.*\\).+$",
80
- r"^[a-f0-9]{64}$",
81
- r"^(?!.*(?:^|/)\.\.(?:/|$))(?!.*\\)artifacts/.+$",
82
- r"^csf_[a-f0-9]{24}$",
83
- r"^occ_[a-f0-9]{24}$",
84
- r"^[a-z0-9][a-z0-9._/-]*$",
85
- r"^codex-security/v1:sha256:[a-f0-9]{64}$",
86
- r"^findings/([a-z0-9][a-z0-9._-]*)/\1\.md$",
87
- }
88
64
  EXPORT_PATHS = {
89
65
  "csv": "exports/findings.csv",
90
66
  "json": "exports/findings.json",
@@ -96,6 +72,10 @@ class ContractError(ValueError):
96
72
  """Raised when a completed scan does not satisfy the additive contract."""
97
73
 
98
74
 
75
+ class RecoverableContractError(ContractError):
76
+ """Raised when report projection can safely be retried before publication."""
77
+
78
+
99
79
  def _reject_non_finite_json(value: str) -> None:
100
80
  raise ValueError(f"non-finite JSON number {value!r} is not supported")
101
81
 
@@ -106,9 +86,7 @@ def _loads_json(value: str | bytes) -> Any:
106
86
 
107
87
  def _read_json(path: Path) -> dict[str, Any]:
108
88
  try:
109
- with path.open("rb") as handle:
110
- raw = _read_bounded_json_document(handle, str(path), SCHEMA_DOCUMENT_MAX_BYTES)
111
- payload = _loads_json(raw.decode("utf-8"))
89
+ payload = _loads_json(path.read_text(encoding="utf-8"))
112
90
  _require_safe_json_value(payload, str(path))
113
91
  except FileNotFoundError as exc:
114
92
  raise ContractError(f"missing required contract artifact: {path}") from exc
@@ -121,11 +99,6 @@ def _read_json(path: Path) -> dict[str, Any]:
121
99
  return payload
122
100
 
123
101
 
124
- def _write_json(path: Path, payload: Any) -> None:
125
- path.parent.mkdir(parents=True, exist_ok=True)
126
- path.write_bytes(_json_bytes(payload))
127
-
128
-
129
102
  def _generate_report_projection(
130
103
  manifest: dict[str, Any],
131
104
  findings: dict[str, Any],
@@ -137,10 +110,21 @@ def _generate_report_projection(
137
110
  raise ContractError(f"could not load report projection helper: {script}")
138
111
  module = importlib.util.module_from_spec(spec)
139
112
  spec.loader.exec_module(module)
140
- try:
141
- return module.generate_report_markdown(manifest, findings, coverage)
142
- except (OSError, ValueError) as exc:
143
- raise ContractError(f"report projection failed: {exc}") from exc
113
+ attempts = (
114
+ getattr(sys.modules.get("workbench_constants"), "SQLITE_RETRY_ATTEMPTS", 1)
115
+ if coverage.get("mode") == "deep_repository"
116
+ else 1
117
+ )
118
+ for attempt in range(attempts):
119
+ try:
120
+ return module.generate_report_markdown(manifest, findings, coverage)
121
+ except OSError as exc:
122
+ if attempt == attempts - 1:
123
+ raise RecoverableContractError(f"report projection failed: {exc}") from exc
124
+ time.sleep(0.05 * (2**attempt))
125
+ except ValueError as exc:
126
+ raise ContractError(f"report projection failed: {exc}") from exc
127
+ raise AssertionError("Report projection retry loop exhausted unexpectedly.")
144
128
 
145
129
 
146
130
  def _validate_report_output_paths(scan_dir: Path) -> None:
@@ -157,72 +141,21 @@ def _json_bytes(payload: Any) -> bytes:
157
141
 
158
142
  def _contract_json_bytes(relative_path: str, payload: Any) -> bytes:
159
143
  _require_safe_json_value(payload, relative_path)
160
- encoded = _json_bytes(payload)
161
- maximum = CONTRACT_DOCUMENT_MAX_BYTES.get(relative_path)
162
- if maximum is not None and len(encoded) > maximum:
163
- raise ContractError(f"{relative_path}: JSON document exceeds the {maximum}-byte limit")
164
- return encoded
165
-
166
-
167
- def _read_bounded_json_document(handle: BinaryIO, context: str, maximum: int) -> bytes:
168
- if os.fstat(handle.fileno()).st_size > maximum:
169
- raise ContractError(f"{context}: JSON document exceeds the {maximum}-byte limit")
170
- chunks: list[bytes] = []
171
- length = 0
172
- while length <= maximum:
173
- chunk = handle.read(min(JSON_DOCUMENT_READ_CHUNK_SIZE, maximum + 1 - length))
174
- if not chunk:
175
- break
176
- length += len(chunk)
177
- if length > maximum:
178
- raise ContractError(f"{context}: JSON document exceeds the {maximum}-byte limit")
179
- chunks.append(chunk)
180
- result = b"".join(chunks)
181
- _require_json_nesting(result, context)
182
- return result
183
-
184
-
185
- def _require_json_nesting(value: bytes, context: str) -> None:
186
- depth = 0
187
- in_string = False
188
- escaped = False
189
- for character in value:
190
- if in_string:
191
- if escaped:
192
- escaped = False
193
- elif character == ord("\\"):
194
- escaped = True
195
- elif character == ord('"'):
196
- in_string = False
197
- continue
198
- if character == ord('"'):
199
- in_string = True
200
- elif character in (ord("{"), ord("[")):
201
- depth += 1
202
- if depth > MAX_JSON_DEPTH + 1:
203
- raise ContractError(
204
- f"{context}: JSON document exceeds the {MAX_JSON_DEPTH}-level nesting limit"
205
- )
206
- elif character in (ord("}"), ord("]")):
207
- depth -= 1
144
+ return _json_bytes(payload)
208
145
 
209
146
 
210
147
  def _require_safe_json_value(value: Any, context: str, *, validate_strings: bool = True) -> None:
211
- def visit(item: Any, location: str, depth: int) -> None:
212
- if depth > MAX_JSON_DEPTH:
213
- raise ContractError(
214
- f"{location}: JSON document exceeds the {MAX_JSON_DEPTH}-level nesting limit"
215
- )
148
+ def visit(item: Any, location: str) -> None:
216
149
  if isinstance(item, dict):
217
150
  for key, child in item.items():
218
151
  if not isinstance(key, str):
219
152
  raise ContractError(f"{location}: expected string JSON property names")
220
153
  if validate_strings:
221
154
  _require_safe_json_string(key, location)
222
- visit(child, f"{location}.<property>", depth + 1)
155
+ visit(child, f"{location}.<property>")
223
156
  elif isinstance(item, list):
224
157
  for index, child in enumerate(item):
225
- visit(child, f"{location}[{index}]", depth + 1)
158
+ visit(child, f"{location}[{index}]")
226
159
  elif isinstance(item, str) and validate_strings:
227
160
  _require_safe_json_string(item, location)
228
161
  elif isinstance(item, int) and not isinstance(item, bool):
@@ -238,7 +171,7 @@ def _require_safe_json_value(value: Any, context: str, *, validate_strings: bool
238
171
  f"{location}: unsafe integer-valued JSON numbers are not supported"
239
172
  )
240
173
 
241
- visit(value, context, 0)
174
+ visit(value, context)
242
175
 
243
176
 
244
177
  def _require_safe_json_string(value: str, context: str) -> None:
@@ -256,14 +189,6 @@ def _sha256_bytes(value: bytes) -> str:
256
189
  return hashlib.sha256(value).hexdigest()
257
190
 
258
191
 
259
- def _sha256_file(path: Path) -> str:
260
- digest = hashlib.sha256()
261
- with path.open("rb") as handle:
262
- for chunk in iter(lambda: handle.read(1024 * 1024), b""):
263
- digest.update(chunk)
264
- return digest.hexdigest()
265
-
266
-
267
192
  def _require_dict(payload: dict[str, Any], key: str, context: str) -> dict[str, Any]:
268
193
  value = payload.get(key)
269
194
  if not isinstance(value, dict):
@@ -505,12 +430,7 @@ def _read_scan_local_json_bytes(
505
430
  try:
506
431
  with os.fdopen(descriptor, "rb") as handle:
507
432
  descriptor = -1
508
- maximum = CONTRACT_DOCUMENT_MAX_BYTES.get(relative_path)
509
- raw = (
510
- handle.read()
511
- if maximum is None
512
- else _read_bounded_json_document(handle, context, maximum)
513
- )
433
+ raw = handle.read()
514
434
  try:
515
435
  payload = _loads_json(raw.decode("utf-8"))
516
436
  except (UnicodeDecodeError, ValueError) as exc:
@@ -786,7 +706,6 @@ def _recover_unsealed_findings(
786
706
  warnings: list[str],
787
707
  ) -> list[str]:
788
708
  schema = _read_json(schema_dir / "findings.schema.json")
789
- _require_safe_schema(schema, "findings.schema.json")
790
709
  properties = _require_dict(schema, "properties", "findings.schema")
791
710
  finding_array = _require_dict(properties, "findings", "findings.schema.properties")
792
711
  finding_schema = _require_dict(finding_array, "items", "findings.schema.properties.findings")
@@ -937,12 +856,20 @@ def _recover_unsealed_coverage(
937
856
  discarded_findings: list[str],
938
857
  ) -> None:
939
858
  schema = _read_json(schema_dir / "coverage.schema.json")
940
- _require_safe_schema(schema, "coverage.schema.json")
941
859
  properties = _require_dict(schema, "properties", "coverage.schema")
942
860
  completeness = coverage.get("completeness")
943
861
  partial = completeness not in ("complete", "partial", "unknown")
944
862
  if partial:
945
863
  warnings.append("Recovered malformed coverage completeness; marked coverage as partial.")
864
+ if (
865
+ coverage.get("mode") == "deep_repository"
866
+ and coverage.get("inventoryStrategy") != "repository"
867
+ ):
868
+ coverage["inventoryStrategy"] = "repository"
869
+ warnings.append(
870
+ "Recovered malformed Deep Scan inventory strategy; marked coverage as partial."
871
+ )
872
+ partial = True
946
873
 
947
874
  surface_ids: set[str] = set()
948
875
  for field, label in (
@@ -1536,91 +1463,9 @@ def _validate_schema_node(value: Any, schema: dict[str, Any], context: str) -> N
1536
1463
 
1537
1464
  def validate_against_schema(payload: dict[str, Any], schema_path: Path) -> None:
1538
1465
  schema = _read_json(schema_path)
1539
- _require_safe_schema(schema, schema_path.name)
1540
1466
  _validate_schema_node(payload, schema, schema_path.stem)
1541
1467
 
1542
1468
 
1543
- def _require_safe_schema(schema: dict[str, Any], context: str) -> None:
1544
- pending: list[tuple[Any, bool]] = [(schema, True)]
1545
- nodes = 0
1546
- applicator_edges = 0
1547
- unsupported_keywords = {
1548
- "$async",
1549
- "$ref",
1550
- "$dynamicRef",
1551
- "$recursiveRef",
1552
- "prefixItems",
1553
- "patternProperties",
1554
- "propertyNames",
1555
- "dependentSchemas",
1556
- "dependencies",
1557
- "uniqueItems",
1558
- }
1559
- while pending:
1560
- value, is_schema = pending.pop()
1561
- nodes += 1
1562
- if nodes > MAX_SCHEMA_NODES:
1563
- raise ContractError(
1564
- f"{context}: JSON Schema exceeds the {MAX_SCHEMA_NODES}-node complexity limit"
1565
- )
1566
- if isinstance(value, list):
1567
- if len(value) > MAX_SCHEMA_COLLECTION_ENTRIES:
1568
- raise ContractError(
1569
- f"{context}: JSON Schema exceeds the "
1570
- f"{MAX_SCHEMA_COLLECTION_ENTRIES}-entry collection limit"
1571
- )
1572
- pending.extend((child, is_schema) for child in value)
1573
- continue
1574
- if not isinstance(value, dict):
1575
- continue
1576
- if len(value) > MAX_SCHEMA_COLLECTION_ENTRIES:
1577
- raise ContractError(
1578
- f"{context}: JSON Schema exceeds the "
1579
- f"{MAX_SCHEMA_COLLECTION_ENTRIES}-entry collection limit"
1580
- )
1581
- for keyword, child in value.items():
1582
- if not is_schema:
1583
- pending.append((child, False))
1584
- continue
1585
- if keyword in unsupported_keywords:
1586
- raise ContractError(f"{context}: unsupported JSON Schema keyword")
1587
- edges = 0
1588
- if keyword in {"allOf", "anyOf", "oneOf"} and isinstance(child, list):
1589
- edges = len(child)
1590
- elif keyword in {
1591
- "if",
1592
- "then",
1593
- "else",
1594
- "not",
1595
- "items",
1596
- "contains",
1597
- "additionalProperties",
1598
- "unevaluatedProperties",
1599
- "unevaluatedItems",
1600
- } and isinstance(child, (dict, bool)):
1601
- edges = 1
1602
- elif keyword in {"properties", "$defs", "definitions"} and isinstance(child, dict):
1603
- edges = len(child)
1604
- applicator_edges += edges
1605
- if applicator_edges > MAX_SCHEMA_APPLICATOR_EDGES:
1606
- raise ContractError(
1607
- f"{context}: JSON Schema exceeds the "
1608
- f"{MAX_SCHEMA_APPLICATOR_EDGES}-edge applicator limit"
1609
- )
1610
- if keyword == "pattern" and isinstance(child, str):
1611
- if child not in SAFE_SCHEMA_PATTERNS:
1612
- raise ContractError(f"{context}: unsupported JSON Schema pattern")
1613
- if keyword in {"properties", "$defs", "definitions"} and isinstance(child, dict):
1614
- pending.extend((child_schema, True) for child_schema in child.values())
1615
- continue
1616
- pending.append(
1617
- (
1618
- child,
1619
- keyword not in {"const", "enum", "default", "examples", "dependentRequired"},
1620
- )
1621
- )
1622
-
1623
-
1624
1469
  def _validate_canonical_schemas_before_projection(
1625
1470
  manifest: dict[str, Any],
1626
1471
  findings: dict[str, Any],
@@ -1666,10 +1511,7 @@ def _utf16_code_units(value: str) -> Iterator[int]:
1666
1511
  def _github_line_hashes(
1667
1512
  handle: TextIO,
1668
1513
  requested_lines: set[int] | None = None,
1669
- source_read_budget: list[int] | None = None,
1670
- ) -> dict[int, str] | None:
1671
- if source_read_budget is not None and source_read_budget[0] <= 0:
1672
- return None
1514
+ ) -> dict[int, str]:
1673
1515
  window = [0] * GITHUB_HASH_BLOCK_SIZE
1674
1516
  line_numbers = [-1] * GITHUB_HASH_BLOCK_SIZE
1675
1517
  hash_counts: dict[str, int] = {}
@@ -1680,7 +1522,6 @@ def _github_line_hashes(
1680
1522
  line_number = 0
1681
1523
  line_start = True
1682
1524
  previous_was_cr = False
1683
- source_bytes = 0
1684
1525
 
1685
1526
  def output_hash() -> None:
1686
1527
  nonlocal index
@@ -1698,11 +1539,11 @@ def _github_line_hashes(
1698
1539
  hash_raw = (GITHUB_HASH_MOD * hash_raw + current - first_mod * beginning) & GITHUB_HASH_MASK
1699
1540
  index = (index + 1) % GITHUB_HASH_BLOCK_SIZE
1700
1541
 
1701
- def process_character(current: int) -> bool:
1542
+ def process_character(current: int) -> None:
1702
1543
  nonlocal line_number, line_start, previous_was_cr
1703
1544
  if current in {ord(" "), ord("\t")} or (previous_was_cr and current == ord("\n")):
1704
1545
  previous_was_cr = False
1705
- return True
1546
+ return
1706
1547
  if current == ord("\r"):
1707
1548
  current = ord("\n")
1708
1549
  previous_was_cr = True
@@ -1713,28 +1554,15 @@ def _github_line_hashes(
1713
1554
  if line_start:
1714
1555
  line_start = False
1715
1556
  line_number += 1
1716
- if line_number > GITHUB_HASH_MAX_LINES:
1717
- return False
1718
1557
  line_numbers[index] = line_number
1719
1558
  if current == ord("\n"):
1720
1559
  line_start = True
1721
1560
  update_hash(current)
1722
- return True
1723
1561
 
1724
1562
  while chunk := handle.read(SOURCE_READ_CHUNK_SIZE):
1725
- chunk_bytes = len(chunk.encode("utf-8", errors="replace"))
1726
- source_bytes += chunk_bytes
1727
- if source_bytes > SOURCE_READ_MAX_BYTES:
1728
- return None
1729
- if source_read_budget is not None:
1730
- source_read_budget[0] -= chunk_bytes
1731
- if source_read_budget[0] < 0:
1732
- return None
1733
1563
  for code_unit in _utf16_code_units(chunk):
1734
- if not process_character(code_unit):
1735
- return None
1736
- if not process_character(GITHUB_HASH_EOF):
1737
- return None
1564
+ process_character(code_unit)
1565
+ process_character(GITHUB_HASH_EOF)
1738
1566
  for _ in range(GITHUB_HASH_BLOCK_SIZE):
1739
1567
  if line_numbers[index] != -1:
1740
1568
  output_hash()
@@ -1762,14 +1590,13 @@ def _github_line_hashes_for_source(
1762
1590
  source_root: Path,
1763
1591
  relative_path: str,
1764
1592
  requested_lines: set[int] | None = None,
1765
- source_read_budget: list[int] | None = None,
1766
1593
  ) -> dict[int, str] | None:
1767
1594
  handle = _open_source_file(source_root, relative_path)
1768
1595
  if handle is None:
1769
1596
  return None
1770
1597
  try:
1771
1598
  with handle:
1772
- return _github_line_hashes(handle, requested_lines, source_read_budget)
1599
+ return _github_line_hashes(handle, requested_lines)
1773
1600
  except OSError:
1774
1601
  return None
1775
1602
 
@@ -1849,15 +1676,8 @@ def _github_line_hash_cache(
1849
1676
  )
1850
1677
  requested_lines_by_path.setdefault(relative_path, set()).add(primary_location["startLine"])
1851
1678
  line_hash_cache: dict[tuple[Path, int], str | None] = {}
1852
- source_read_budget = [SOURCE_READ_MAX_BYTES]
1853
1679
  for relative_path, requested_lines in requested_lines_by_path.items():
1854
- line_hashes = (
1855
- None
1856
- if source_read_budget[0] <= 0
1857
- else _github_line_hashes_for_source(
1858
- source_root, relative_path, requested_lines, source_read_budget
1859
- )
1860
- )
1680
+ line_hashes = _github_line_hashes_for_source(source_root, relative_path, requested_lines)
1861
1681
  source_path = source_root / relative_path
1862
1682
  for line_number in requested_lines:
1863
1683
  line_hash_cache[(source_path, line_number)] = (
@@ -111,8 +111,14 @@ def bounded_finding_details(value: Any) -> dict[str, Any]:
111
111
  ):
112
112
  key = next((alias for alias in aliases if alias in value), None)
113
113
  if key is not None:
114
+ section = value[key]
115
+ if key == "attackPath" and isinstance(section, dict):
116
+ section = dict(section)
117
+ for assessment in ("impact", "likelihood"):
118
+ if isinstance(section.get(assessment), str):
119
+ section[assessment] = {"level": section[assessment]}
114
120
  prepared[key] = bounded_finding_section(
115
- value[key],
121
+ section,
116
122
  maximum_bytes,
117
123
  priority_keys,
118
124
  reserved_fields,
@@ -140,6 +146,8 @@ def bounded_finding_details(value: Any) -> dict[str, Any]:
140
146
  "severity",
141
147
  "status",
142
148
  "taxonomy",
149
+ "preventiveControls",
150
+ "remediationTests",
143
151
  ):
144
152
  if key in value:
145
153
  prepared[key] = (
@@ -148,8 +156,64 @@ def bounded_finding_details(value: Any) -> dict[str, Any]:
148
156
  else value[key]
149
157
  )
150
158
 
151
- budget = [FINDING_DETAILS_PREVIEW_BYTES]
152
- bounded = bounded_json_value(prepared, budget)
159
+ guidance = {
160
+ key: prepared[key]
161
+ for key in ("remediationTests", "preventiveControls")
162
+ if key in prepared and isinstance(prepared[key], list)
163
+ }
164
+ diagnostics = (
165
+ "rootCause",
166
+ "root_cause",
167
+ "validation",
168
+ "attackPath",
169
+ "codeEvidence",
170
+ "code_evidence",
171
+ )
172
+ core_keys = (
173
+ "writeup",
174
+ *diagnostics,
175
+ "confidence",
176
+ "detectedAt",
177
+ "identity",
178
+ "provenance",
179
+ "ruleId",
180
+ "severity",
181
+ "status",
182
+ "taxonomy",
183
+ "evidence",
184
+ "evidenceExcerpt",
185
+ )
186
+ core = {key: prepared[key] for key in core_keys if key in prepared}
187
+ extras = {
188
+ key: item
189
+ for key, item in prepared.items()
190
+ if key not in core and key not in guidance
191
+ }
192
+ complete_guidance = {key: items[:1] for key, items in guidance.items()}
193
+ minimum_guidance = {
194
+ key: [items[0][:1]] if items and isinstance(items[0], str) else []
195
+ for key, items in guidance.items()
196
+ }
197
+ projected_core = {}
198
+ for selected_guidance in (complete_guidance, minimum_guidance):
199
+ reserved = (
200
+ len(json.dumps(selected_guidance, separators=(",", ":")).encode("utf-8")) - 1
201
+ if selected_guidance
202
+ else 0
203
+ )
204
+ if reserved >= FINDING_DETAILS_PREVIEW_BYTES:
205
+ continue
206
+ projected_core = bounded_json_value(
207
+ core,
208
+ [FINDING_DETAILS_PREVIEW_BYTES - reserved],
209
+ )
210
+ if all(key in projected_core for key in core):
211
+ break
212
+ ordered_guidance = dict(sorted(guidance.items(), key=lambda entry: bool(entry[1])))
213
+ bounded = bounded_json_value(
214
+ {**projected_core, **ordered_guidance, **extras},
215
+ [FINDING_DETAILS_PREVIEW_BYTES],
216
+ )
153
217
  return bounded if isinstance(bounded, dict) else {}
154
218
 
155
219
 
@@ -215,11 +279,20 @@ def bounded_json_value(value: Any, budget: list[int], *, depth: int = 0) -> Any:
215
279
  if not consume_json_budget(budget, 2):
216
280
  return []
217
281
  result = []
218
- for item in value[:20]:
282
+ for item in value:
283
+ remaining = budget[0]
219
284
  separator = 0 if not result else 1
220
285
  if not consume_json_budget(budget, separator):
221
286
  break
222
- result.append(bounded_json_value(item, budget, depth=depth + 1))
287
+ bounded_item = bounded_json_value(item, budget, depth=depth + 1)
288
+ size = len(json.dumps(bounded_item, separators=(",", ":")).encode("utf-8"))
289
+ if separator + size > remaining or (
290
+ isinstance(item, str) and item and bounded_item == ""
291
+ ):
292
+ budget[0] = remaining
293
+ break
294
+ budget[0] = remaining - separator - size
295
+ result.append(bounded_item)
223
296
  return result
224
297
  if isinstance(value, dict):
225
298
  if not consume_json_budget(budget, 2):
@@ -228,13 +301,56 @@ def bounded_json_value(value: Any, budget: list[int], *, depth: int = 0) -> Any:
228
301
  for key, item in list(value.items())[:20]:
229
302
  if budget[0] <= 0 or not isinstance(key, str):
230
303
  break
304
+ remaining = budget[0]
231
305
  separator = 0 if not result else 1
232
306
  if not consume_json_budget(budget, separator):
307
+ budget[0] = remaining
233
308
  break
234
309
  bounded_key, key_size = bounded_json_text(key, min(budget[0], 512))
235
310
  if not consume_json_budget(budget, key_size + 1):
311
+ budget[0] = remaining
312
+ break
313
+ item_budget = budget
314
+ if depth == 0 and key == "remediationTests":
315
+ controls = value.get("preventiveControls")
316
+ if (
317
+ isinstance(item, list)
318
+ and item
319
+ and isinstance(item[0], str)
320
+ and item[0]
321
+ and isinstance(controls, list)
322
+ and controls
323
+ and isinstance(controls[0], str)
324
+ and controls[0]
325
+ ):
326
+ minimum_tests = len(
327
+ json.dumps([item[0][0]], separators=(",", ":")).encode("utf-8")
328
+ )
329
+ for control in (controls[0], controls[0][0]):
330
+ reserved = (
331
+ len(
332
+ json.dumps(
333
+ {"preventiveControls": [control]},
334
+ separators=(",", ":"),
335
+ ).encode("utf-8")
336
+ )
337
+ - 1
338
+ )
339
+ if budget[0] >= minimum_tests + reserved:
340
+ item_budget = [budget[0] - reserved]
341
+ break
342
+ bounded_item = bounded_json_value(item, item_budget, depth=depth + 1)
343
+ size = (
344
+ separator
345
+ + key_size
346
+ + 1
347
+ + len(json.dumps(bounded_item, separators=(",", ":")).encode("utf-8"))
348
+ )
349
+ if size > remaining or (isinstance(item, str) and item and bounded_item == ""):
350
+ budget[0] = remaining
236
351
  break
237
- result[bounded_key] = bounded_json_value(item, budget, depth=depth + 1)
352
+ budget[0] = remaining - size
353
+ result[bounded_key] = bounded_item
238
354
  return result
239
355
  consume_json_budget(budget, 4)
240
356
  return None