@openai/codex-security 0.1.30 → 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (320) hide show
  1. package/README.md +386 -37
  2. package/_bundled_plugin/.mcp.json +1 -0
  3. package/_bundled_plugin/mcp/helpers.mjs.br.part-000 +0 -0
  4. package/_bundled_plugin/mcp/native/THIRD_PARTY_NOTICES.txt +4 -4
  5. package/_bundled_plugin/mcp/native/darwin-arm64/unix.node +0 -0
  6. package/_bundled_plugin/mcp/native/darwin-x64/unix.node +0 -0
  7. package/_bundled_plugin/mcp/native/linux-arm64-gnu/unix.node +0 -0
  8. package/_bundled_plugin/mcp/native/linux-arm64-musl/unix.node +0 -0
  9. package/_bundled_plugin/mcp/native/linux-x64-gnu/unix.node +0 -0
  10. package/_bundled_plugin/mcp/native/linux-x64-musl/unix.node +0 -0
  11. package/_bundled_plugin/mcp/native/win32-arm64/windows.node +0 -0
  12. package/_bundled_plugin/mcp/native/win32-x64/windows.node +0 -0
  13. package/_bundled_plugin/mcp/server.mjs.br.part-000 +0 -0
  14. package/_bundled_plugin/mcp/server.mjs.br.part-001 +0 -0
  15. package/_bundled_plugin/references/artifact-storage.md +4 -4
  16. package/_bundled_plugin/references/config-preflight.md +1 -1
  17. package/_bundled_plugin/references/core-scan.md +14 -2
  18. package/_bundled_plugin/references/final-report.md +4 -2
  19. package/_bundled_plugin/references/scan-artifacts.md +2 -2
  20. package/_bundled_plugin/references/scan-contract.md +2 -2
  21. package/_bundled_plugin/references/threat-model.md +2 -2
  22. package/_bundled_plugin/schemas/coverage.schema.json +22 -0
  23. package/_bundled_plugin/schemas/scan-manifest.schema.json +100 -85
  24. package/_bundled_plugin/schemas/tools/deep-reducer.schema.json +19 -4
  25. package/_bundled_plugin/schemas/tools/scan-draft.schema.json +116 -102
  26. package/_bundled_plugin/scripts/config_preflight.py +4 -8
  27. package/_bundled_plugin/scripts/deep_scan_workbench.py +83 -217
  28. package/_bundled_plugin/scripts/filesystem_identity.py +1 -5
  29. package/_bundled_plugin/scripts/finalize_scan_contract.py +441 -166
  30. package/_bundled_plugin/scripts/finding_preview.py +42 -82
  31. package/_bundled_plugin/scripts/generate_in_scope_files.py +30 -34
  32. package/_bundled_plugin/scripts/generate_rank_input.py +58 -99
  33. package/_bundled_plugin/scripts/launch_codex_security_mcp +5 -1
  34. package/_bundled_plugin/scripts/rank_preview.py +86 -68
  35. package/_bundled_plugin/scripts/report_projection.py +37 -58
  36. package/_bundled_plugin/scripts/threat_model_projection.py +164 -0
  37. package/_bundled_plugin/scripts/windows_scan_local_files.py +11 -6
  38. package/_bundled_plugin/scripts/workbench/storage.py +11 -0
  39. package/_bundled_plugin/scripts/workbench_cli.py +11 -17
  40. package/_bundled_plugin/scripts/workbench_constants.py +6 -14
  41. package/_bundled_plugin/scripts/workbench_db.py +116 -228
  42. package/_bundled_plugin/scripts/workbench_finding_index.py +1 -6
  43. package/_bundled_plugin/scripts/workbench_native_indexes.py +1 -4
  44. package/_bundled_plugin/scripts/workbench_progress.py +12 -34
  45. package/_bundled_plugin/scripts/workbench_publication.py +47 -62
  46. package/_bundled_plugin/scripts/workbench_remediation.py +2 -20
  47. package/_bundled_plugin/scripts/workbench_saved_results.py +1246 -230
  48. package/_bundled_plugin/scripts/workbench_scan_start.py +4 -11
  49. package/_bundled_plugin/scripts/workbench_scan_usage.py +7 -15
  50. package/_bundled_plugin/scripts/workbench_schema.py +86 -271
  51. package/_bundled_plugin/scripts/workbench_severity.py +3 -0
  52. package/_bundled_plugin/scripts/workbench_source_excerpt.py +5 -6
  53. package/_bundled_plugin/scripts/workbench_target.py +98 -10
  54. package/_bundled_plugin/scripts/workbench_target_state.py +1 -5
  55. package/_bundled_plugin/scripts/workbench_validation.py +6 -20
  56. package/_bundled_plugin/skills/deep-security-scan/SKILL.md +3 -3
  57. package/_bundled_plugin/skills/security-diff-scan/SKILL.md +3 -3
  58. package/_bundled_plugin/skills/security-scan/SKILL.md +2 -2
  59. package/_bundled_plugin/skills/security-scan/references/desktop-scan.md +2 -2
  60. package/_bundled_plugin/skills/threat-model/SKILL.md +2 -2
  61. package/dist/api.d.ts +3 -5
  62. package/dist/api.d.ts.map +1 -1
  63. package/dist/api.js +201 -207
  64. package/dist/api.js.map +1 -1
  65. package/dist/artifact-export.d.ts +70 -0
  66. package/dist/artifact-export.d.ts.map +1 -0
  67. package/dist/artifact-export.js +277 -0
  68. package/dist/artifact-export.js.map +1 -0
  69. package/dist/auth.d.ts.map +1 -1
  70. package/dist/auth.js +9 -4
  71. package/dist/auth.js.map +1 -1
  72. package/dist/bulk-scan-discovery.d.ts +1 -0
  73. package/dist/bulk-scan-discovery.d.ts.map +1 -1
  74. package/dist/bulk-scan-discovery.js +8 -9
  75. package/dist/bulk-scan-discovery.js.map +1 -1
  76. package/dist/classify-scan-severity.d.ts.map +1 -1
  77. package/dist/classify-scan-severity.js +2 -5
  78. package/dist/classify-scan-severity.js.map +1 -1
  79. package/dist/classify-severity.d.ts +2 -8
  80. package/dist/classify-severity.d.ts.map +1 -1
  81. package/dist/classify-severity.js +3 -5
  82. package/dist/classify-severity.js.map +1 -1
  83. package/dist/cli-help.d.ts +3 -0
  84. package/dist/cli-help.d.ts.map +1 -0
  85. package/dist/cli-help.js +281 -0
  86. package/dist/cli-help.js.map +1 -0
  87. package/dist/cli.d.ts +21 -22
  88. package/dist/cli.d.ts.map +1 -1
  89. package/dist/cli.js +673 -701
  90. package/dist/cli.js.map +1 -1
  91. package/dist/cloud-publish.d.ts.map +1 -1
  92. package/dist/cloud-publish.js +1 -8
  93. package/dist/cloud-publish.js.map +1 -1
  94. package/dist/codex-prompt.d.ts.map +1 -1
  95. package/dist/codex-prompt.js +1 -2
  96. package/dist/codex-prompt.js.map +1 -1
  97. package/dist/codex-sdk-environment.d.ts +2 -0
  98. package/dist/codex-sdk-environment.d.ts.map +1 -0
  99. package/dist/codex-sdk-environment.js +30 -0
  100. package/dist/codex-sdk-environment.js.map +1 -0
  101. package/dist/component-plan.d.ts.map +1 -1
  102. package/dist/component-plan.js +2 -6
  103. package/dist/component-plan.js.map +1 -1
  104. package/dist/component-scan.d.ts +1 -0
  105. package/dist/component-scan.d.ts.map +1 -1
  106. package/dist/component-scan.js +31 -9
  107. package/dist/component-scan.js.map +1 -1
  108. package/dist/config.d.ts.map +1 -1
  109. package/dist/config.js +32 -33
  110. package/dist/config.js.map +1 -1
  111. package/dist/contract.d.ts +2 -1
  112. package/dist/contract.d.ts.map +1 -1
  113. package/dist/contract.js +63 -122
  114. package/dist/contract.js.map +1 -1
  115. package/dist/cost-model.d.ts.map +1 -1
  116. package/dist/cost-model.js +65 -34
  117. package/dist/cost-model.js.map +1 -1
  118. package/dist/cost.d.ts.map +1 -1
  119. package/dist/cost.js +15 -31
  120. package/dist/cost.js.map +1 -1
  121. package/dist/custom-validation-prompt.js +4 -4
  122. package/dist/custom-validation-prompt.js.map +1 -1
  123. package/dist/custom-validation.d.ts.map +1 -1
  124. package/dist/custom-validation.js +17 -16
  125. package/dist/custom-validation.js.map +1 -1
  126. package/dist/deduplication/checkpointed-review.d.ts.map +1 -1
  127. package/dist/deduplication/checkpointed-review.js +10 -11
  128. package/dist/deduplication/checkpointed-review.js.map +1 -1
  129. package/dist/deduplication/codex-review.d.ts +2 -0
  130. package/dist/deduplication/codex-review.d.ts.map +1 -1
  131. package/dist/deduplication/codex-review.js +3 -3
  132. package/dist/deduplication/codex-review.js.map +1 -1
  133. package/dist/deduplication/deduplication-reviewer.d.ts.map +1 -1
  134. package/dist/deduplication/deduplication-reviewer.js +2 -0
  135. package/dist/deduplication/deduplication-reviewer.js.map +1 -1
  136. package/dist/deduplication/deduplication.js +40 -62
  137. package/dist/deduplication/deduplication.js.map +1 -1
  138. package/dist/deduplication/records-protocol.d.ts.map +1 -1
  139. package/dist/deduplication/records-protocol.js +1 -4
  140. package/dist/deduplication/records-protocol.js.map +1 -1
  141. package/dist/deduplication/records.d.ts +2 -1
  142. package/dist/deduplication/records.d.ts.map +1 -1
  143. package/dist/deduplication/records.js +24 -6
  144. package/dist/deduplication/records.js.map +1 -1
  145. package/dist/deduplication/review.d.ts +7 -0
  146. package/dist/deduplication/review.d.ts.map +1 -1
  147. package/dist/deduplication/review.js.map +1 -1
  148. package/dist/deduplication/scan.js +3 -7
  149. package/dist/deduplication/scan.js.map +1 -1
  150. package/dist/deep-config.d.ts.map +1 -1
  151. package/dist/deep-config.js +3 -19
  152. package/dist/deep-config.js.map +1 -1
  153. package/dist/deep-progress.d.ts +0 -1
  154. package/dist/deep-progress.d.ts.map +1 -1
  155. package/dist/deep-progress.js +22 -40
  156. package/dist/deep-progress.js.map +1 -1
  157. package/dist/errors.d.ts +3 -2
  158. package/dist/errors.d.ts.map +1 -1
  159. package/dist/errors.js +5 -7
  160. package/dist/errors.js.map +1 -1
  161. package/dist/finding-catalogue.d.ts.map +1 -1
  162. package/dist/finding-catalogue.js +22 -22
  163. package/dist/finding-catalogue.js.map +1 -1
  164. package/dist/finding-workflow.d.ts.map +1 -1
  165. package/dist/finding-workflow.js +4 -7
  166. package/dist/finding-workflow.js.map +1 -1
  167. package/dist/findings-import.d.ts.map +1 -1
  168. package/dist/findings-import.js +7 -17
  169. package/dist/findings-import.js.map +1 -1
  170. package/dist/github.d.ts.map +1 -1
  171. package/dist/github.js +2 -3
  172. package/dist/github.js.map +1 -1
  173. package/dist/import-scan.js +2 -2
  174. package/dist/import-scan.js.map +1 -1
  175. package/dist/index.d.ts +6 -2
  176. package/dist/index.d.ts.map +1 -1
  177. package/dist/index.js +3 -1
  178. package/dist/index.js.map +1 -1
  179. package/dist/knowledge-base.js +7 -3
  180. package/dist/knowledge-base.js.map +1 -1
  181. package/dist/linear.d.ts.map +1 -1
  182. package/dist/linear.js +2 -3
  183. package/dist/linear.js.map +1 -1
  184. package/dist/models.d.ts +24 -0
  185. package/dist/models.d.ts.map +1 -1
  186. package/dist/models.js.map +1 -1
  187. package/dist/multiscan.d.ts.map +1 -1
  188. package/dist/multiscan.js +36 -31
  189. package/dist/multiscan.js.map +1 -1
  190. package/dist/owner-evidence.d.ts +29 -0
  191. package/dist/owner-evidence.d.ts.map +1 -0
  192. package/dist/owner-evidence.js +192 -0
  193. package/dist/owner-evidence.js.map +1 -0
  194. package/dist/patch-tui.d.ts.map +1 -1
  195. package/dist/patch-tui.js +2 -4
  196. package/dist/patch-tui.js.map +1 -1
  197. package/dist/project-config-schema.d.ts +10 -0
  198. package/dist/project-config-schema.d.ts.map +1 -1
  199. package/dist/project-config-schema.js +1 -0
  200. package/dist/project-config-schema.js.map +1 -1
  201. package/dist/project-config.d.ts +1 -0
  202. package/dist/project-config.d.ts.map +1 -1
  203. package/dist/project-config.js +2 -0
  204. package/dist/project-config.js.map +1 -1
  205. package/dist/prompt-files.d.ts +0 -1
  206. package/dist/prompt-files.d.ts.map +1 -1
  207. package/dist/prompt-files.js +2 -4
  208. package/dist/prompt-files.js.map +1 -1
  209. package/dist/publication-events.d.ts.map +1 -1
  210. package/dist/publication-events.js +15 -29
  211. package/dist/publication-events.js.map +1 -1
  212. package/dist/publication-store.d.ts.map +1 -1
  213. package/dist/publication-store.js +9 -13
  214. package/dist/publication-store.js.map +1 -1
  215. package/dist/publication.d.ts.map +1 -1
  216. package/dist/publication.js +11 -14
  217. package/dist/publication.js.map +1 -1
  218. package/dist/publish.d.ts.map +1 -1
  219. package/dist/publish.js +31 -41
  220. package/dist/publish.js.map +1 -1
  221. package/dist/record.d.ts +2 -0
  222. package/dist/record.d.ts.map +1 -0
  223. package/dist/record.js +4 -0
  224. package/dist/record.js.map +1 -0
  225. package/dist/result.d.ts +4 -1
  226. package/dist/result.d.ts.map +1 -1
  227. package/dist/result.js +7 -0
  228. package/dist/result.js.map +1 -1
  229. package/dist/runtime.d.ts +10 -13
  230. package/dist/runtime.d.ts.map +1 -1
  231. package/dist/runtime.js +230 -145
  232. package/dist/runtime.js.map +1 -1
  233. package/dist/saved-scan.d.ts.map +1 -1
  234. package/dist/saved-scan.js +10 -20
  235. package/dist/saved-scan.js.map +1 -1
  236. package/dist/scan-activity.d.ts.map +1 -1
  237. package/dist/scan-activity.js +8 -22
  238. package/dist/scan-activity.js.map +1 -1
  239. package/dist/scan-comparison.d.ts +4 -4
  240. package/dist/scan-comparison.d.ts.map +1 -1
  241. package/dist/scan-comparison.js +38 -73
  242. package/dist/scan-comparison.js.map +1 -1
  243. package/dist/scan-dashboard.d.ts +0 -1
  244. package/dist/scan-dashboard.d.ts.map +1 -1
  245. package/dist/scan-dashboard.js +10 -19
  246. package/dist/scan-dashboard.js.map +1 -1
  247. package/dist/scan-history-renderer.d.ts.map +1 -1
  248. package/dist/scan-history-renderer.js +1 -2
  249. package/dist/scan-history-renderer.js.map +1 -1
  250. package/dist/scan-logs.d.ts.map +1 -1
  251. package/dist/scan-logs.js +2 -6
  252. package/dist/scan-logs.js.map +1 -1
  253. package/dist/scan-sessions.d.ts.map +1 -1
  254. package/dist/scan-sessions.js +1 -3
  255. package/dist/scan-sessions.js.map +1 -1
  256. package/dist/scan-settings.d.ts +5 -0
  257. package/dist/scan-settings.d.ts.map +1 -1
  258. package/dist/scan-settings.js +4 -0
  259. package/dist/scan-settings.js.map +1 -1
  260. package/dist/security-policy-cli.d.ts.map +1 -1
  261. package/dist/security-policy-cli.js +8 -6
  262. package/dist/security-policy-cli.js.map +1 -1
  263. package/dist/security-policy.d.ts +8 -4
  264. package/dist/security-policy.d.ts.map +1 -1
  265. package/dist/security-policy.js +98 -84
  266. package/dist/security-policy.js.map +1 -1
  267. package/dist/server/dashboard/app.js +5 -5
  268. package/dist/server/embeddings.d.ts.map +1 -1
  269. package/dist/server/embeddings.js +1 -0
  270. package/dist/server/embeddings.js.map +1 -1
  271. package/dist/server/routes.d.ts +3 -2
  272. package/dist/server/routes.d.ts.map +1 -1
  273. package/dist/server/routes.js +19 -7
  274. package/dist/server/routes.js.map +1 -1
  275. package/dist/server/server.d.ts.map +1 -1
  276. package/dist/server/server.js +2 -3
  277. package/dist/server/server.js.map +1 -1
  278. package/dist/server/sqlite-store.d.ts.map +1 -1
  279. package/dist/server/sqlite-store.js +4 -14
  280. package/dist/server/sqlite-store.js.map +1 -1
  281. package/dist/severity-store.js +8 -15
  282. package/dist/severity-store.js.map +1 -1
  283. package/dist/suggest-owners.d.ts +37 -0
  284. package/dist/suggest-owners.d.ts.map +1 -0
  285. package/dist/suggest-owners.js +111 -0
  286. package/dist/suggest-owners.js.map +1 -0
  287. package/dist/targets.d.ts +7 -2
  288. package/dist/targets.d.ts.map +1 -1
  289. package/dist/targets.js +29 -25
  290. package/dist/targets.js.map +1 -1
  291. package/dist/thread-source.d.ts +1 -0
  292. package/dist/thread-source.d.ts.map +1 -1
  293. package/dist/thread-source.js +1 -0
  294. package/dist/thread-source.js.map +1 -1
  295. package/dist/trusted-executable.d.ts +5 -0
  296. package/dist/trusted-executable.d.ts.map +1 -1
  297. package/dist/trusted-executable.js +22 -5
  298. package/dist/trusted-executable.js.map +1 -1
  299. package/dist/value.d.ts +2 -0
  300. package/dist/value.d.ts.map +1 -0
  301. package/dist/value.js +20 -0
  302. package/dist/value.js.map +1 -0
  303. package/dist/version.d.ts.map +1 -1
  304. package/dist/version.js +3 -6
  305. package/dist/version.js.map +1 -1
  306. package/dist/windows-path.d.ts.map +1 -1
  307. package/dist/windows-path.js +1 -2
  308. package/dist/windows-path.js.map +1 -1
  309. package/dist/worker-progress.d.ts +1 -0
  310. package/dist/worker-progress.d.ts.map +1 -1
  311. package/dist/worker-progress.js +13 -34
  312. package/dist/worker-progress.js.map +1 -1
  313. package/docs/dedupe-records.md +24 -0
  314. package/package.json +15 -13
  315. package/schemas/project-config.schema.json +5 -0
  316. package/_bundled_plugin/scripts/normalize_candidates.py +0 -338
  317. package/dist/server/findings-service.d.ts +0 -20
  318. package/dist/server/findings-service.d.ts.map +0 -1
  319. package/dist/server/findings-service.js +0 -31
  320. package/dist/server/findings-service.js.map +0 -1
@@ -1,338 +0,0 @@
1
- #!/usr/bin/env python3
2
- """Validate and combine security-scan candidates into deterministic JSONL."""
3
-
4
- from __future__ import annotations
5
-
6
- import argparse
7
- import hashlib
8
- import json
9
- import re
10
- import sys
11
- import tempfile
12
- from pathlib import Path, PurePosixPath
13
- from typing import Any
14
-
15
- CWE = re.compile(r"(?i)CWE-(\d+)")
16
- ROLES = {
17
- "entrypoint": 0,
18
- "entrypoint/wrapper": 1,
19
- "source": 2,
20
- "root_control": 3,
21
- "sink": 4,
22
- "concrete_implementation": 5,
23
- "evidence": 6,
24
- }
25
- FIELDS = {
26
- "candidate_id",
27
- "cwe_ids",
28
- "locations",
29
- "summary",
30
- "evidence",
31
- "context",
32
- "instance",
33
- }
34
-
35
-
36
- def text_field(row: dict[str, Any], field: str, *, required: bool = True) -> str | None:
37
- value = row.get(field)
38
- if value is None and not required:
39
- return None
40
- if not isinstance(value, str) or not value.strip():
41
- raise ValueError(f"{field}: expected a non-empty string")
42
- return value.strip()
43
-
44
-
45
- def cwe_ids(row: dict[str, Any]) -> list[str]:
46
- value = row.get("cwe_ids")
47
- if not isinstance(value, list):
48
- raise ValueError("cwe_ids: expected an array")
49
- found: set[int] = set()
50
- for item in value:
51
- if not isinstance(item, str):
52
- raise ValueError("cwe_ids: expected CWE strings")
53
- match = CWE.fullmatch(item.strip())
54
- if match is None or int(match[1]) < 1:
55
- raise ValueError(f"cwe_ids: unsupported value {item!r}")
56
- found.add(int(match[1]))
57
- return [f"CWE-{number}" for number in sorted(found)]
58
-
59
-
60
- def relative_file(value: Any, repo_root: Path) -> tuple[str, Path]:
61
- if not isinstance(value, str) or not value or "\0" in value:
62
- raise ValueError("path: expected a non-empty repository-relative path")
63
- raw = value
64
- if sys.platform == "win32":
65
- raw = raw.replace("\\", "/")
66
- path = PurePosixPath(raw)
67
- if (
68
- path.is_absolute()
69
- or ".." in path.parts
70
- or (sys.platform == "win32" and re.match(r"^[A-Za-z]:", raw))
71
- ):
72
- raise ValueError("path: expected a repository-relative path without traversal")
73
- resolved = (repo_root / raw).resolve(strict=True)
74
- try:
75
- relative = resolved.relative_to(repo_root).as_posix()
76
- except ValueError as error:
77
- raise ValueError("path: must resolve inside --repo-root") from error
78
- if not resolved.is_file():
79
- raise ValueError("path: expected a regular file")
80
- return relative, resolved
81
-
82
-
83
- def positive_line(value: Any, field: str) -> int:
84
- if not isinstance(value, int) or isinstance(value, bool) or value < 1:
85
- raise ValueError(f"{field}: expected a positive integer")
86
- return value
87
-
88
-
89
- def normalize_locations(
90
- row: dict[str, Any], repo_root: Path, line_counts: dict[Path, int]
91
- ) -> list[dict[str, Any]]:
92
- value = row.get("locations")
93
- if not isinstance(value, list) or not value:
94
- raise ValueError("locations: expected a non-empty array")
95
- normalized: dict[tuple[str, int, int, str], dict[str, Any]] = {}
96
- for item in value:
97
- if not isinstance(item, dict):
98
- raise ValueError("locations: expected location objects")
99
- unknown = set(item) - {"path", "start_line", "end_line", "role"}
100
- if unknown:
101
- raise ValueError(f"locations: unsupported fields {', '.join(sorted(unknown))}")
102
- relative, source = relative_file(item.get("path"), repo_root)
103
- if (
104
- not relative.strip()
105
- or "\\" in relative
106
- or any(":" in part for part in PurePosixPath(relative).parts)
107
- ):
108
- raise ValueError("path: expected a safe repository-relative POSIX path")
109
- start = positive_line(item.get("start_line"), "start_line")
110
- end = positive_line(item.get("end_line", start), "end_line")
111
- if end < start:
112
- raise ValueError("end_line: must be greater than or equal to start_line")
113
- if source not in line_counts:
114
- line_counts[source] = len(source.read_bytes().splitlines())
115
- if end > line_counts[source]:
116
- raise ValueError(f"line range {start}-{end} exceeds {relative}:{line_counts[source]}")
117
- role = item.get("role")
118
- if not isinstance(role, str) or role not in ROLES:
119
- raise ValueError(f"role: unsupported value {role!r}")
120
- key = (relative, start, end, role)
121
- normalized[key] = {
122
- "path": relative,
123
- "start_line": start,
124
- "end_line": end,
125
- "role": role,
126
- }
127
- return sorted(
128
- normalized.values(),
129
- key=lambda item: (
130
- ROLES[item["role"]],
131
- item["path"],
132
- item["start_line"],
133
- item["end_line"],
134
- ),
135
- )
136
-
137
-
138
- def read_scope(path: Path, repo_root: Path, *, allow_missing: bool = False) -> set[str]:
139
- contents = path.read_bytes().decode("utf-8")
140
- lines = contents.split("\n")
141
- listed_rows = set(lines)
142
-
143
- def is_scope_file(value: str) -> bool:
144
- try:
145
- relative_file(value, repo_root)
146
- except (OSError, ValueError):
147
- return False
148
- return True
149
-
150
- carriage_rows = {
151
- line: (is_scope_file(line), is_scope_file(line.removesuffix("\r")))
152
- for line in lines
153
- if line.endswith("\r") and line != "\r" and sys.platform != "win32"
154
- }
155
- crlf_evidence = any(line == "\r" for line in lines) or any(
156
- stripped and not literal for literal, stripped in carriage_rows.values()
157
- )
158
- literal_evidence = any(literal and not stripped for literal, stripped in carriage_rows.values())
159
-
160
- scope: set[str] = set()
161
- for number, line in enumerate(lines, 1):
162
- if sys.platform == "win32" or line == "\r":
163
- line = line.removesuffix("\r")
164
- elif line.endswith("\r"):
165
- literal, stripped = carriage_rows[line]
166
- if stripped and not literal:
167
- line = line.removesuffix("\r")
168
- elif stripped and literal:
169
- if number == len(lines) and not contents.endswith("\n"):
170
- pass
171
- elif line.removesuffix("\r") in listed_rows:
172
- pass
173
- elif crlf_evidence and not literal_evidence:
174
- line = line.removesuffix("\r")
175
- elif literal_evidence and not crlf_evidence:
176
- pass
177
- else:
178
- raise ValueError(f"in-scope file row {number}: ambiguous carriage-return paths")
179
- elif not literal and crlf_evidence:
180
- line = line.removesuffix("\r")
181
- if not line:
182
- continue
183
- try:
184
- relative, _ = relative_file(line, repo_root)
185
- except (OSError, ValueError) as error:
186
- if allow_missing and isinstance(error, FileNotFoundError):
187
- candidate = PurePosixPath(line)
188
- if candidate.is_absolute() or ".." in candidate.parts or "\0" in line:
189
- raise ValueError(f"in-scope file row {number}: unsafe deleted path") from error
190
- resolved = (repo_root / line).resolve(strict=False)
191
- try:
192
- relative = resolved.relative_to(repo_root).as_posix()
193
- except ValueError as escaped:
194
- raise ValueError(
195
- f"in-scope file row {number}: path escapes repository"
196
- ) from escaped
197
- scope.add(relative)
198
- continue
199
- raise ValueError(f"in-scope file row {number}: {error}") from error
200
- scope.add(relative)
201
- return scope
202
-
203
-
204
- def normalize_candidate(
205
- row: dict[str, Any], repo_root: Path, scope: set[str], line_counts: dict[Path, int]
206
- ) -> dict[str, Any]:
207
- unknown = set(row) - FIELDS
208
- if unknown:
209
- raise ValueError(f"unsupported fields {', '.join(sorted(unknown))}")
210
- if "candidate_id" in row:
211
- text_field(row, "candidate_id")
212
- candidate_locations = normalize_locations(row, repo_root, line_counts)
213
- if not any(item["path"] in scope for item in candidate_locations):
214
- raise ValueError("locations: expected at least one in-scope file")
215
- result: dict[str, Any] = {
216
- "cwe_ids": cwe_ids(row),
217
- "locations": candidate_locations,
218
- "summary": text_field(row, "summary"),
219
- "evidence": text_field(row, "evidence"),
220
- }
221
- context = text_field(row, "context", required=False)
222
- if context is not None:
223
- result["context"] = context
224
- instance = text_field(row, "instance", required=False)
225
- if instance is not None:
226
- result["instance"] = instance
227
- return result
228
-
229
-
230
- def identity(row: dict[str, Any]) -> str:
231
- return json.dumps(
232
- {
233
- "cwe_ids": row["cwe_ids"],
234
- "locations": row["locations"],
235
- "instance": row.get("instance"),
236
- },
237
- ensure_ascii=False,
238
- separators=(",", ":"),
239
- sort_keys=True,
240
- )
241
-
242
-
243
- def merged_text(group: list[dict[str, Any]], field: str) -> str:
244
- return "\n".join(sorted({item[field] for item in group if field in item}))
245
-
246
-
247
- def combine(rows: list[dict[str, Any]]) -> list[dict[str, Any]]:
248
- groups: dict[str, list[dict[str, Any]]] = {}
249
- for row in rows:
250
- groups.setdefault(identity(row), []).append(row)
251
- combined: list[dict[str, Any]] = []
252
- for key, group in sorted(groups.items()):
253
- candidate_id = hashlib.sha256(key.encode()).hexdigest()[:16]
254
-
255
- result = {
256
- "candidate_id": f"candidate-{candidate_id}",
257
- "cwe_ids": group[0]["cwe_ids"],
258
- "locations": group[0]["locations"],
259
- "summary": merged_text(group, "summary"),
260
- "evidence": merged_text(group, "evidence"),
261
- }
262
- context = merged_text(group, "context")
263
- if context:
264
- result["context"] = context
265
- if "instance" in group[0]:
266
- result["instance"] = group[0]["instance"]
267
- combined.append(result)
268
- return combined
269
-
270
-
271
- def main() -> None:
272
- parser = argparse.ArgumentParser(description=__doc__)
273
- parser.add_argument("--input", nargs="+", required=True, help="Candidate JSONL inputs.")
274
- parser.add_argument("--out", required=True, help="Combined candidate JSONL output.")
275
- parser.add_argument("--repo-root", required=True, help="Repository root for candidate paths.")
276
- parser.add_argument("--in-scope-files", required=True, help="Repository-relative file list.")
277
- parser.add_argument(
278
- "--allow-missing-in-scope",
279
- action="store_true",
280
- help="Keep deleted Git paths in a diff inventory while validating existing candidate files.",
281
- )
282
- args = parser.parse_args()
283
- try:
284
- repo_root = Path(args.repo_root).expanduser().resolve(strict=True)
285
- if not repo_root.is_dir():
286
- raise ValueError("--repo-root: expected a directory")
287
- output = Path(args.out).expanduser().resolve(strict=False)
288
- scope_path = Path(args.in_scope_files).expanduser().resolve(strict=True)
289
- inputs = sorted({Path(value).expanduser().resolve(strict=True) for value in args.input})
290
- if output in inputs:
291
- raise ValueError("--out: must not also be an input")
292
- if output == scope_path:
293
- raise ValueError("--out: must not replace --in-scope-files")
294
- scope = read_scope(scope_path, repo_root, allow_missing=args.allow_missing_in_scope)
295
- line_counts: dict[Path, int] = {}
296
- rows: list[dict[str, Any]] = []
297
- for source in inputs:
298
- with source.open(encoding="utf-8") as handle:
299
- for number, line in enumerate(handle, 1):
300
- if not line.strip():
301
- continue
302
- try:
303
- row = json.loads(line)
304
- if not isinstance(row, dict):
305
- raise ValueError("expected a JSON object")
306
- rows.append(normalize_candidate(row, repo_root, scope, line_counts))
307
- except (ValueError, TypeError, OSError) as error:
308
- raise ValueError(f"{source} row {number}: {error}") from error
309
- combined = combine(rows)
310
- output.parent.mkdir(parents=True, exist_ok=True)
311
- temporary: Path | None = None
312
- try:
313
- with tempfile.NamedTemporaryFile(
314
- mode="w",
315
- encoding="utf-8",
316
- dir=output.parent,
317
- prefix=f".{output.name}.",
318
- suffix=".tmp",
319
- delete=False,
320
- ) as handle:
321
- temporary = Path(handle.name)
322
- for row in combined:
323
- handle.write(
324
- json.dumps(row, ensure_ascii=False, separators=(",", ":"), sort_keys=True)
325
- + "\n"
326
- )
327
- temporary.replace(output)
328
- finally:
329
- if temporary is not None:
330
- temporary.unlink(missing_ok=True)
331
- print(f"Combined {len(rows)} candidate rows into {len(combined)} rows in {output}")
332
- except (OSError, ValueError) as error:
333
- print(f"normalize_candidates: {error}", file=sys.stderr)
334
- raise SystemExit(2) from error
335
-
336
-
337
- if __name__ == "__main__":
338
- main()
@@ -1,20 +0,0 @@
1
- import type { Finding } from "../models.js";
2
- import type { DashboardQuery } from "./dashboard-types.js";
3
- import type { FindingSearchScope } from "../finding-retrieval.js";
4
- import type { FindingEmbedder } from "./embeddings.js";
5
- import type { FindingsPage, FindingsStore } from "./storage.js";
6
- export declare class FindingsService {
7
- private readonly store;
8
- private readonly embeddings;
9
- constructor(store: FindingsStore, embeddings: FindingEmbedder);
10
- insert(findings: readonly Finding[], repositoryId?: string): Promise<string[]>;
11
- potentialDuplicates(findingId: string, scope: FindingSearchScope): Promise<import("../finding-retrieval.js").FindingNeighborhood>;
12
- storeDedupeGroups(groups: readonly string[][]): Promise<import("../finding-dedupe-groups.js").FindingDedupeGroup[]>;
13
- listDedupeGroups(findingId: string): Promise<import("../finding-dedupe-groups.js").FindingDedupeGroup[]>;
14
- list(page: {
15
- limit: number;
16
- offset: number;
17
- }): Promise<FindingsPage>;
18
- dashboard(query: DashboardQuery): Promise<import("./dashboard-types.js").DashboardSnapshot>;
19
- }
20
- //# sourceMappingURL=findings-service.d.ts.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"findings-service.d.ts","sourceRoot":"","sources":["../../src/server/findings-service.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,cAAc,CAAC;AAC5C,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,sBAAsB,CAAC;AAC3D,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,yBAAyB,CAAC;AAClE,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,iBAAiB,CAAC;AACvD,OAAO,KAAK,EAAE,YAAY,EAAE,aAAa,EAAE,MAAM,cAAc,CAAC;AAEhE,qBAAa,eAAe;IAExB,OAAO,CAAC,QAAQ,CAAC,KAAK;IACtB,OAAO,CAAC,QAAQ,CAAC,UAAU;IAF7B,YACmB,KAAK,EAAE,aAAa,EACpB,UAAU,EAAE,eAAe,EAC1C;IAEE,MAAM,CACV,QAAQ,EAAE,SAAS,OAAO,EAAE,EAC5B,YAAY,CAAC,EAAE,MAAM,GACpB,OAAO,CAAC,MAAM,EAAE,CAAC,CASnB;IAEK,mBAAmB,CAAC,SAAS,EAAE,MAAM,EAAE,KAAK,EAAE,kBAAkB,kEAErE;IAEK,iBAAiB,CAAC,MAAM,EAAE,SAAS,MAAM,EAAE,EAAE,uEAElD;IAEK,gBAAgB,CAAC,SAAS,EAAE,MAAM,uEAEvC;IAEK,IAAI,CAAC,IAAI,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,MAAM,CAAA;KAAE,GAAG,OAAO,CAAC,YAAY,CAAC,CAEzE;IAEK,SAAS,CAAC,KAAK,EAAE,cAAc,6DAEpC;CACF"}
@@ -1,31 +0,0 @@
1
- export class FindingsService {
2
- store;
3
- embeddings;
4
- constructor(store, embeddings) {
5
- this.store = store;
6
- this.embeddings = embeddings;
7
- }
8
- async insert(findings, repositoryId) {
9
- const embeddings = await this.embeddings.embed(findings);
10
- return await this.store.insert(findings.map((finding, index) => ({
11
- finding,
12
- embedding: embeddings[index],
13
- })), repositoryId);
14
- }
15
- async potentialDuplicates(findingId, scope) {
16
- return await this.store.findPotentialDuplicates(findingId, scope);
17
- }
18
- async storeDedupeGroups(groups) {
19
- return await this.store.storeDedupeGroups(groups);
20
- }
21
- async listDedupeGroups(findingId) {
22
- return await this.store.listDedupeGroups(findingId);
23
- }
24
- async list(page) {
25
- return await this.store.list(page);
26
- }
27
- async dashboard(query) {
28
- return await this.store.dashboard(query);
29
- }
30
- }
31
- //# sourceMappingURL=findings-service.js.map
@@ -1 +0,0 @@
1
- {"version":3,"file":"findings-service.js","sourceRoot":"","sources":["../../src/server/findings-service.ts"],"names":[],"mappings":"AAMA,MAAM,OAAO,eAAe;IAEP,KAAK;IACL,UAAU;IAF7B,YACmB,KAAoB,EACpB,UAA2B;qBAD3B,KAAK;0BACL,UAAU;IAC1B,CAAC;IAEJ,KAAK,CAAC,MAAM,CACV,QAA4B,EAC5B,YAAqB;QAErB,MAAM,UAAU,GAAG,MAAM,IAAI,CAAC,UAAU,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;QACzD,OAAO,MAAM,IAAI,CAAC,KAAK,CAAC,MAAM,CAC5B,QAAQ,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,KAAK,EAAE,EAAE,CAAC,CAAC;YAChC,OAAO;YACP,SAAS,EAAE,UAAU,CAAC,KAAK,CAAE;SAC9B,CAAC,CAAC,EACH,YAAY,CACb,CAAC;IACJ,CAAC;IAED,KAAK,CAAC,mBAAmB,CAAC,SAAiB,EAAE,KAAyB;QACpE,OAAO,MAAM,IAAI,CAAC,KAAK,CAAC,uBAAuB,CAAC,SAAS,EAAE,KAAK,CAAC,CAAC;IACpE,CAAC;IAED,KAAK,CAAC,iBAAiB,CAAC,MAA2B;QACjD,OAAO,MAAM,IAAI,CAAC,KAAK,CAAC,iBAAiB,CAAC,MAAM,CAAC,CAAC;IACpD,CAAC;IAED,KAAK,CAAC,gBAAgB,CAAC,SAAiB;QACtC,OAAO,MAAM,IAAI,CAAC,KAAK,CAAC,gBAAgB,CAAC,SAAS,CAAC,CAAC;IACtD,CAAC;IAED,KAAK,CAAC,IAAI,CAAC,IAAuC;QAChD,OAAO,MAAM,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IACrC,CAAC;IAED,KAAK,CAAC,SAAS,CAAC,KAAqB;QACnC,OAAO,MAAM,IAAI,CAAC,KAAK,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;IAC3C,CAAC;CACF","sourcesContent":["import type { Finding } from \"../models.js\";\nimport type { DashboardQuery } from \"./dashboard-types.js\";\nimport type { FindingSearchScope } from \"../finding-retrieval.js\";\nimport type { FindingEmbedder } from \"./embeddings.js\";\nimport type { FindingsPage, FindingsStore } from \"./storage.js\";\n\nexport class FindingsService {\n constructor(\n private readonly store: FindingsStore,\n private readonly embeddings: FindingEmbedder,\n ) {}\n\n async insert(\n findings: readonly Finding[],\n repositoryId?: string,\n ): Promise<string[]> {\n const embeddings = await this.embeddings.embed(findings);\n return await this.store.insert(\n findings.map((finding, index) => ({\n finding,\n embedding: embeddings[index]!,\n })),\n repositoryId,\n );\n }\n\n async potentialDuplicates(findingId: string, scope: FindingSearchScope) {\n return await this.store.findPotentialDuplicates(findingId, scope);\n }\n\n async storeDedupeGroups(groups: readonly string[][]) {\n return await this.store.storeDedupeGroups(groups);\n }\n\n async listDedupeGroups(findingId: string) {\n return await this.store.listDedupeGroups(findingId);\n }\n\n async list(page: { limit: number; offset: number }): Promise<FindingsPage> {\n return await this.store.list(page);\n }\n\n async dashboard(query: DashboardQuery) {\n return await this.store.dashboard(query);\n }\n}\n"]}