@openai/codex-security 0.1.21 → 0.1.24

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (183) hide show
  1. package/README.md +549 -1
  2. package/_bundled_plugin/.codex-plugin/plugin.json +1 -1
  3. package/_bundled_plugin/.mcp.json +1 -0
  4. package/_bundled_plugin/mcp/server.mjs.br.part-000 +0 -0
  5. package/_bundled_plugin/mcp/server.mjs.br.part-001 +0 -0
  6. package/_bundled_plugin/references/config-preflight.md +2 -30
  7. package/_bundled_plugin/references/core-scan.md +1 -1
  8. package/_bundled_plugin/references/final-report.md +1 -1
  9. package/_bundled_plugin/references/finding-detail-fields.md +5 -10
  10. package/_bundled_plugin/references/sarif-adapter.md +1 -1
  11. package/_bundled_plugin/references/scan-contract.md +10 -0
  12. package/_bundled_plugin/scripts/config_preflight.py +2 -8
  13. package/_bundled_plugin/scripts/deep_scan_workbench.py +6 -1
  14. package/_bundled_plugin/scripts/finalize_scan_contract.py +115 -76
  15. package/_bundled_plugin/scripts/finding_preview.py +22 -4
  16. package/_bundled_plugin/scripts/generate_in_scope_files.py +29 -9
  17. package/_bundled_plugin/scripts/generate_rank_input.py +1 -5
  18. package/_bundled_plugin/scripts/rank_preview.py +1 -0
  19. package/_bundled_plugin/scripts/report_projection.py +8 -14
  20. package/_bundled_plugin/scripts/validate_scan_contract.py +1 -3
  21. package/_bundled_plugin/scripts/windows_scan_local_files.py +90 -6
  22. package/_bundled_plugin/scripts/workbench_cli.py +27 -1
  23. package/_bundled_plugin/scripts/workbench_dashboard.py +138 -0
  24. package/_bundled_plugin/scripts/workbench_db.py +110 -539
  25. package/_bundled_plugin/scripts/workbench_finding_index.py +42 -24
  26. package/_bundled_plugin/scripts/workbench_finding_workflows.py +208 -0
  27. package/_bundled_plugin/scripts/workbench_findings.py +221 -0
  28. package/_bundled_plugin/scripts/workbench_native_indexes.py +1 -2
  29. package/_bundled_plugin/scripts/workbench_publication.py +520 -0
  30. package/_bundled_plugin/scripts/workbench_remediation.py +37 -0
  31. package/_bundled_plugin/scripts/workbench_saved_results.py +322 -89
  32. package/_bundled_plugin/scripts/workbench_scan_history.py +1 -2
  33. package/_bundled_plugin/scripts/workbench_scan_usage.py +9 -33
  34. package/_bundled_plugin/scripts/workbench_schema.py +276 -58
  35. package/_bundled_plugin/scripts/workbench_target.py +30 -6
  36. package/_bundled_plugin/scripts/workbench_validation.py +1 -5
  37. package/_bundled_plugin/skills/assess-patch-risk/SKILL.md +1 -5
  38. package/_bundled_plugin/skills/attack-path-analysis/references/severity-policy.md +6 -1
  39. package/_bundled_plugin/skills/track-findings/SKILL.md +2 -0
  40. package/dist/api.d.ts +5 -1
  41. package/dist/api.d.ts.map +1 -1
  42. package/dist/api.js +135 -27
  43. package/dist/api.js.map +1 -1
  44. package/dist/cli.d.ts +4 -0
  45. package/dist/cli.d.ts.map +1 -1
  46. package/dist/cli.js +158 -62
  47. package/dist/cli.js.map +1 -1
  48. package/dist/cloud-publish.js +5 -1
  49. package/dist/cloud-publish.js.map +1 -1
  50. package/dist/contract.d.ts +1 -0
  51. package/dist/contract.d.ts.map +1 -1
  52. package/dist/contract.js +9 -0
  53. package/dist/contract.js.map +1 -1
  54. package/dist/custom-publish.d.ts +22 -0
  55. package/dist/custom-publish.d.ts.map +1 -0
  56. package/dist/custom-publish.js +47 -0
  57. package/dist/custom-publish.js.map +1 -0
  58. package/dist/custom-validation-prompt.js +1 -1
  59. package/dist/custom-validation-prompt.js.map +1 -1
  60. package/dist/deduplication/checkpointed-review.d.ts +16 -0
  61. package/dist/deduplication/checkpointed-review.d.ts.map +1 -0
  62. package/dist/deduplication/checkpointed-review.js +76 -0
  63. package/dist/deduplication/checkpointed-review.js.map +1 -0
  64. package/dist/deduplication/codex-review.d.ts +21 -0
  65. package/dist/deduplication/codex-review.d.ts.map +1 -0
  66. package/dist/deduplication/codex-review.js +252 -0
  67. package/dist/deduplication/codex-review.js.map +1 -0
  68. package/dist/deduplication/deduplication-prompts.d.ts +8 -0
  69. package/dist/deduplication/deduplication-prompts.d.ts.map +1 -0
  70. package/dist/deduplication/deduplication-prompts.js +35 -0
  71. package/dist/deduplication/deduplication-prompts.js.map +1 -0
  72. package/dist/deduplication/deduplication-reviewer.d.ts +46 -0
  73. package/dist/deduplication/deduplication-reviewer.d.ts.map +1 -0
  74. package/dist/deduplication/deduplication-reviewer.js +105 -0
  75. package/dist/deduplication/deduplication-reviewer.js.map +1 -0
  76. package/dist/deduplication/deduplication.d.ts +6 -0
  77. package/dist/deduplication/deduplication.d.ts.map +1 -0
  78. package/dist/deduplication/deduplication.js +85 -0
  79. package/dist/deduplication/deduplication.js.map +1 -0
  80. package/dist/deduplication/scan.d.ts +16 -0
  81. package/dist/deduplication/scan.d.ts.map +1 -0
  82. package/dist/deduplication/scan.js +101 -0
  83. package/dist/deduplication/scan.js.map +1 -0
  84. package/dist/deep-progress.d.ts +21 -0
  85. package/dist/deep-progress.d.ts.map +1 -0
  86. package/dist/deep-progress.js +108 -0
  87. package/dist/deep-progress.js.map +1 -0
  88. package/dist/finding-dedupe-groups.d.ts +7 -0
  89. package/dist/finding-dedupe-groups.d.ts.map +1 -0
  90. package/dist/finding-dedupe-groups.js +2 -0
  91. package/dist/finding-dedupe-groups.js.map +1 -0
  92. package/dist/finding-retrieval.d.ts +13 -0
  93. package/dist/finding-retrieval.d.ts.map +1 -0
  94. package/dist/finding-retrieval.js +2 -0
  95. package/dist/finding-retrieval.js.map +1 -0
  96. package/dist/finding-workflow.d.ts +53 -0
  97. package/dist/finding-workflow.d.ts.map +1 -0
  98. package/dist/finding-workflow.js +117 -0
  99. package/dist/finding-workflow.js.map +1 -0
  100. package/dist/findings-client.d.ts +15 -0
  101. package/dist/findings-client.d.ts.map +1 -0
  102. package/dist/findings-client.js +60 -0
  103. package/dist/findings-client.js.map +1 -0
  104. package/dist/index.d.ts +5 -0
  105. package/dist/index.d.ts.map +1 -1
  106. package/dist/index.js +2 -0
  107. package/dist/index.js.map +1 -1
  108. package/dist/publication.js +28 -40
  109. package/dist/publication.js.map +1 -1
  110. package/dist/runtime.d.ts +5 -1
  111. package/dist/runtime.d.ts.map +1 -1
  112. package/dist/runtime.js +119 -13
  113. package/dist/runtime.js.map +1 -1
  114. package/dist/saved-scan.d.ts +15 -0
  115. package/dist/saved-scan.d.ts.map +1 -0
  116. package/dist/saved-scan.js +68 -0
  117. package/dist/saved-scan.js.map +1 -0
  118. package/dist/scan-activity.d.ts.map +1 -1
  119. package/dist/scan-activity.js +12 -2
  120. package/dist/scan-activity.js.map +1 -1
  121. package/dist/scan-comparison.d.ts +3 -2
  122. package/dist/scan-comparison.d.ts.map +1 -1
  123. package/dist/scan-comparison.js +1 -1
  124. package/dist/scan-comparison.js.map +1 -1
  125. package/dist/server/dashboard/THIRD_PARTY_NOTICES.txt +276 -0
  126. package/dist/server/dashboard/app.css +2 -0
  127. package/dist/server/dashboard/app.js +66 -0
  128. package/dist/server/dashboard/index.html +18 -0
  129. package/dist/server/dashboard-types.d.ts +45 -0
  130. package/dist/server/dashboard-types.d.ts.map +1 -0
  131. package/dist/server/dashboard-types.js +2 -0
  132. package/dist/server/dashboard-types.js.map +1 -0
  133. package/dist/server/dashboard.d.ts +6 -0
  134. package/dist/server/dashboard.d.ts.map +1 -0
  135. package/dist/server/dashboard.js +45 -0
  136. package/dist/server/dashboard.js.map +1 -0
  137. package/dist/server/embeddings.d.ts +16 -0
  138. package/dist/server/embeddings.d.ts.map +1 -0
  139. package/dist/server/embeddings.js +114 -0
  140. package/dist/server/embeddings.js.map +1 -0
  141. package/dist/server/errors.d.ts +5 -0
  142. package/dist/server/errors.d.ts.map +1 -0
  143. package/dist/server/errors.js +8 -0
  144. package/dist/server/errors.js.map +1 -0
  145. package/dist/server/findings-service.d.ts +20 -0
  146. package/dist/server/findings-service.d.ts.map +1 -0
  147. package/dist/server/findings-service.js +31 -0
  148. package/dist/server/findings-service.js.map +1 -0
  149. package/dist/server/index.d.ts +2 -0
  150. package/dist/server/index.d.ts.map +1 -0
  151. package/dist/server/index.js +6 -0
  152. package/dist/server/index.js.map +1 -0
  153. package/dist/server/routes.d.ts +6 -0
  154. package/dist/server/routes.d.ts.map +1 -0
  155. package/dist/server/routes.js +86 -0
  156. package/dist/server/routes.js.map +1 -0
  157. package/dist/server/serve.d.ts +2 -0
  158. package/dist/server/serve.d.ts.map +1 -0
  159. package/dist/server/serve.js +28 -0
  160. package/dist/server/serve.js.map +1 -0
  161. package/dist/server/server.d.ts +10 -0
  162. package/dist/server/server.d.ts.map +1 -0
  163. package/dist/server/server.js +17 -0
  164. package/dist/server/server.js.map +1 -0
  165. package/dist/server/sqlite-store.d.ts +22 -0
  166. package/dist/server/sqlite-store.d.ts.map +1 -0
  167. package/dist/server/sqlite-store.js +82 -0
  168. package/dist/server/sqlite-store.js.map +1 -0
  169. package/dist/server/storage.d.ts +32 -0
  170. package/dist/server/storage.d.ts.map +1 -0
  171. package/dist/server/storage.js +2 -0
  172. package/dist/server/storage.js.map +1 -0
  173. package/dist/server/validation.d.ts +17 -0
  174. package/dist/server/validation.d.ts.map +1 -0
  175. package/dist/server/validation.js +53 -0
  176. package/dist/server/validation.js.map +1 -0
  177. package/dist/trusted-executable.d.ts.map +1 -1
  178. package/dist/trusted-executable.js +2 -1
  179. package/dist/trusted-executable.js.map +1 -1
  180. package/dist/version.d.ts +1 -1
  181. package/dist/version.js +1 -1
  182. package/dist/version.js.map +1 -1
  183. package/package.json +17 -5
@@ -8,6 +8,44 @@ import sqlite3
8
8
  from typing import Any
9
9
 
10
10
 
11
+ def upsert_finding(
12
+ connection: sqlite3.Connection,
13
+ finding: dict[str, Any],
14
+ timestamp: str,
15
+ repository_id: str | None = None,
16
+ ) -> None:
17
+ connection.execute(
18
+ """
19
+ INSERT INTO findings (
20
+ id, fingerprint, rule_id, identity_anchor, identity_instance,
21
+ details_json, created_at, updated_at
22
+ ) VALUES (?, ?, ?, ?, ?, ?, ?, ?)
23
+ ON CONFLICT(id) DO UPDATE SET
24
+ fingerprint = excluded.fingerprint,
25
+ rule_id = excluded.rule_id,
26
+ identity_anchor = excluded.identity_anchor,
27
+ identity_instance = excluded.identity_instance,
28
+ details_json = excluded.details_json,
29
+ updated_at = excluded.updated_at
30
+ """,
31
+ (
32
+ finding["findingId"],
33
+ finding["fingerprints"]["primary"],
34
+ finding["ruleId"],
35
+ finding["identity"]["anchor"],
36
+ finding["identity"].get("instance"),
37
+ json.dumps(finding, allow_nan=False, sort_keys=True),
38
+ timestamp,
39
+ timestamp,
40
+ ),
41
+ )
42
+ if repository_id is not None:
43
+ connection.execute(
44
+ "INSERT OR IGNORE INTO finding_repositories (repository_id, finding_id) VALUES (?, ?)",
45
+ (repository_id, finding["findingId"]),
46
+ )
47
+
48
+
11
49
  def index_findings(
12
50
  connection: sqlite3.Connection,
13
51
  scan_id: str,
@@ -17,35 +55,15 @@ def index_findings(
17
55
  findings = document.get("findings")
18
56
  if not isinstance(findings, list):
19
57
  raise SystemExit("findings.json must contain a findings array.")
58
+ repository_id = connection.execute(
59
+ "SELECT target_id FROM scans WHERE id = ?", (scan_id,)
60
+ ).fetchone()["target_id"]
20
61
  for finding in findings:
21
62
  if not isinstance(finding, dict):
22
63
  raise SystemExit("findings.json entries must be objects.")
23
- identity = finding["identity"]
24
- fingerprints = finding["fingerprints"]
25
64
  severity = finding["severity"]
26
65
  confidence = finding["confidence"]
27
- connection.execute(
28
- """
29
- INSERT INTO findings (
30
- id, fingerprint, rule_id, identity_anchor, identity_instance, created_at, updated_at
31
- ) VALUES (?, ?, ?, ?, ?, ?, ?)
32
- ON CONFLICT(id) DO UPDATE SET
33
- fingerprint = excluded.fingerprint,
34
- rule_id = excluded.rule_id,
35
- identity_anchor = excluded.identity_anchor,
36
- identity_instance = excluded.identity_instance,
37
- updated_at = excluded.updated_at
38
- """,
39
- (
40
- finding["findingId"],
41
- fingerprints["primary"],
42
- finding["ruleId"],
43
- identity["anchor"],
44
- identity.get("instance"),
45
- timestamp,
46
- timestamp,
47
- ),
48
- )
66
+ upsert_finding(connection, finding, timestamp, repository_id)
49
67
  connection.execute(
50
68
  """
51
69
  INSERT INTO finding_occurrences (
@@ -0,0 +1,208 @@
1
+ """Durable state for the opt-in local scan, publication, and dedupe workflow."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import argparse
6
+ import hashlib
7
+ import json
8
+ import sqlite3
9
+ import sys
10
+ from pathlib import Path
11
+ from typing import Any
12
+
13
+ sys.path.insert(0, str(Path(__file__).resolve().parent))
14
+ from workbench_target import directory_content_digest, git_output, git_revision
15
+
16
+ WORKFLOW_BINDINGS = {
17
+ "repositoryPath": "repository_path",
18
+ "scanRequestDigest": "scan_request_digest",
19
+ "scanId": "scan_id",
20
+ "scanDir": "scan_dir",
21
+ "artifactDigest": "artifact_digest",
22
+ "destination": "destination",
23
+ }
24
+ WORKFLOW_STAGES = ("scan", "publish", "dedupe")
25
+
26
+
27
+ def read_workflow(connection: sqlite3.Connection, workflow_id: str) -> dict[str, Any] | None:
28
+ row = connection.execute(
29
+ "SELECT * FROM finding_workflows WHERE id = ?", (workflow_id,)
30
+ ).fetchone()
31
+ if row is None:
32
+ return None
33
+ state = {"id": row["id"], "stages": {}}
34
+ for field, column in WORKFLOW_BINDINGS.items():
35
+ if row[column] is not None:
36
+ state[field] = row[column]
37
+ if row["scope_repository_id"] is not None:
38
+ state["scope"] = {"repositoryId": row["scope_repository_id"]}
39
+ elif row["scope_all_repositories"] is not None:
40
+ state["scope"] = {"allRepositories": bool(row["scope_all_repositories"])}
41
+ results = json.loads(row["results_json"])
42
+ for stage in WORKFLOW_STAGES:
43
+ current = {"status": row[f"{stage}_status"]}
44
+ if row[f"{stage}_error"] is not None:
45
+ current["error"] = row[f"{stage}_error"]
46
+ if stage in results:
47
+ current["result"] = results[stage]
48
+ state["stages"][stage] = current
49
+ if "dedupePendingWrite" in results:
50
+ state["stages"]["dedupe"]["pendingWrite"] = results["dedupePendingWrite"]
51
+ return state
52
+
53
+
54
+ def save_workflow(connection: sqlite3.Connection, state: dict[str, Any], timestamp: str) -> None:
55
+ values = {"id": state["id"]}
56
+ values.update({column: state.get(field) for field, column in WORKFLOW_BINDINGS.items()})
57
+ scope = state.get("scope", {})
58
+ values.update(
59
+ scope_repository_id=scope.get("repositoryId"),
60
+ scope_all_repositories=scope.get("allRepositories"),
61
+ )
62
+ results = {}
63
+ for stage in WORKFLOW_STAGES:
64
+ current = state["stages"][stage]
65
+ values[f"{stage}_status"] = current["status"]
66
+ values[f"{stage}_error"] = current.get("error")
67
+ if "result" in current:
68
+ results[stage] = current["result"]
69
+ if "pendingWrite" in state["stages"]["dedupe"]:
70
+ results["dedupePendingWrite"] = state["stages"]["dedupe"]["pendingWrite"]
71
+ values.update(
72
+ results_json=json.dumps(results, allow_nan=False),
73
+ created_at=timestamp,
74
+ updated_at=timestamp,
75
+ )
76
+ updates = ", ".join(
77
+ f"{column} = excluded.{column}" for column in values if column not in {"id", "created_at"}
78
+ )
79
+ connection.execute(
80
+ f"INSERT INTO finding_workflows ({', '.join(values)}) "
81
+ f"VALUES ({', '.join('?' for _ in values)}) ON CONFLICT(id) DO UPDATE SET {updates}",
82
+ tuple(values.values()),
83
+ )
84
+
85
+
86
+ def bind_workflow(state: dict[str, Any], binding: dict[str, Any]) -> None:
87
+ for field, value in binding.items():
88
+ if field not in WORKFLOW_BINDINGS and field != "scope":
89
+ raise SystemExit("Unknown workflow binding.")
90
+ if field in state and state[field] != value:
91
+ raise SystemExit(
92
+ f"Workflow {state['id']} is already bound to a different {field}. "
93
+ "Use another --workflow-id."
94
+ )
95
+ state[field] = value
96
+
97
+
98
+ def register_workflow_scan(
99
+ connection: sqlite3.Connection, workflow_id: str, scan_id: str, scan_dir: str, timestamp: str
100
+ ) -> None:
101
+ """Called inside the scan-registration transaction, before model execution."""
102
+ state = read_workflow(connection, workflow_id)
103
+ if state is None:
104
+ raise SystemExit("The workflow must be started before registering its scan.")
105
+ if state["stages"]["scan"]["status"] == "completed":
106
+ raise SystemExit("The workflow scan is already complete.")
107
+ previous = state.get("scanId")
108
+ if previous is not None:
109
+ row = connection.execute("SELECT status FROM scans WHERE id = ?", (previous,)).fetchone()
110
+ if row is not None and row["status"] == "complete":
111
+ raise SystemExit("Reuse the workflow's completed scan instead of registering another.")
112
+ state["scanId"] = scan_id
113
+ state["scanDir"] = scan_dir
114
+ save_workflow(connection, state, timestamp)
115
+
116
+
117
+ def finding_workflow(
118
+ connection: sqlite3.Connection, payload: dict[str, Any], timestamp: str
119
+ ) -> dict[str, Any]:
120
+ workflow_id = payload["id"]
121
+ if not isinstance(workflow_id, str) or not workflow_id.strip():
122
+ raise SystemExit("workflowId must be a nonempty string.")
123
+ if payload["action"] == "get":
124
+ return {"workflow": read_workflow(connection, workflow_id)}
125
+ if payload["action"] == "source":
126
+ target = Path(payload["repository"]).resolve(strict=True)
127
+ return {
128
+ "source": {
129
+ "repository": str(target),
130
+ "revision": git_revision(target),
131
+ "refsDigest": hashlib.sha256(
132
+ (git_output(target, "show-ref") or "").encode()
133
+ ).hexdigest(),
134
+ "content": directory_content_digest(target, include_ignored=True),
135
+ }
136
+ }
137
+ if payload["action"] == "get-review":
138
+ row = connection.execute(
139
+ "SELECT result_json FROM finding_workflow_reviews WHERE workflow_id = ? AND review_key = ?",
140
+ (workflow_id, payload["key"]),
141
+ ).fetchone()
142
+ return {"review": json.loads(row["result_json"]) if row is not None else None}
143
+ if payload["action"] == "save-review":
144
+ binding = payload["binding"]
145
+ source = binding["source"]
146
+ scope = binding["scope"]
147
+ with connection:
148
+ connection.execute(
149
+ """INSERT INTO finding_workflow_reviews
150
+ (workflow_id, review_key, review_contract_version, codex_version,
151
+ source_repository_path, source_revision, source_refs_digest, source_content_digest,
152
+ scope_repository_id, scope_all_repositories, model, effort, settings_digest,
153
+ prompt_digest, contract_digest, result_json, created_at)
154
+ VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
155
+ ON CONFLICT(workflow_id, review_key) DO NOTHING""",
156
+ (
157
+ workflow_id,
158
+ payload["key"],
159
+ binding["version"],
160
+ binding["codexVersion"],
161
+ source["repository"],
162
+ source["revision"],
163
+ source["refsDigest"],
164
+ source["content"],
165
+ scope.get("repositoryId"),
166
+ scope.get("allRepositories"),
167
+ binding["model"],
168
+ binding["effort"],
169
+ binding.get("settingsDigest"),
170
+ binding["promptDigest"],
171
+ binding["contractDigest"],
172
+ json.dumps(payload["result"], allow_nan=False),
173
+ timestamp,
174
+ ),
175
+ )
176
+ return {}
177
+ connection.execute("BEGIN IMMEDIATE")
178
+ with connection:
179
+ state = read_workflow(connection, workflow_id)
180
+ if state is None:
181
+ state = {
182
+ "id": workflow_id,
183
+ "stages": {stage: {"status": "pending"} for stage in WORKFLOW_STAGES},
184
+ }
185
+ bind_workflow(state, payload.get("binding", {}))
186
+ action = payload["action"]
187
+ if action != "bind":
188
+ stage = payload["stage"]
189
+ if stage not in state["stages"]:
190
+ raise SystemExit("Unknown workflow stage.")
191
+ current = state["stages"][stage]
192
+ if current["status"] != "completed":
193
+ if action == "begin":
194
+ current["status"] = "running"
195
+ elif action == "complete":
196
+ state["stages"][stage] = {"status": "completed", "result": payload["result"]}
197
+ elif action == "fail":
198
+ current.update(status="failed", error=payload["error"])
199
+ elif action == "prepare-dedupe":
200
+ current.update(result=payload["result"], pendingWrite=payload["pendingWrite"])
201
+ else:
202
+ raise SystemExit("Unknown workflow action.")
203
+ save_workflow(connection, state, timestamp)
204
+ return {"workflow": state}
205
+
206
+
207
+ if __name__ == "__main__":
208
+ argparse.ArgumentParser(description=__doc__).parse_args()
@@ -0,0 +1,221 @@
1
+ """Storage operations for complete findings imported without a scan."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import argparse
6
+ import hashlib
7
+ import json
8
+ import math
9
+ import sqlite3
10
+ import sys
11
+ from pathlib import Path
12
+ from typing import Any
13
+
14
+ sys.path.insert(0, str(Path(__file__).resolve().parent))
15
+ from workbench_finding_index import upsert_finding
16
+
17
+
18
+ def store_findings(
19
+ connection: sqlite3.Connection,
20
+ entries: list[dict[str, Any]],
21
+ timestamp: str,
22
+ repository_id: str | None = None,
23
+ ) -> dict[str, Any]:
24
+ try:
25
+ with connection:
26
+ connection.execute("BEGIN IMMEDIATE")
27
+ for entry in entries:
28
+ finding = entry["finding"]
29
+ embedding = entry["embedding"]
30
+ identity = (
31
+ finding["fingerprints"]["primary"],
32
+ finding["ruleId"],
33
+ finding["identity"]["anchor"],
34
+ finding["identity"].get("instance"),
35
+ )
36
+ current = connection.execute(
37
+ "SELECT fingerprint, rule_id, identity_anchor, identity_instance "
38
+ "FROM findings WHERE id = ?",
39
+ (finding["findingId"],),
40
+ ).fetchone()
41
+ if current is not None and tuple(current) != identity:
42
+ raise sqlite3.IntegrityError("The stored finding identity cannot be replaced.")
43
+ upsert_finding(connection, finding, timestamp, repository_id)
44
+ connection.execute(
45
+ """
46
+ INSERT INTO finding_embeddings (finding_id, model, vector_json)
47
+ VALUES (?, ?, ?)
48
+ ON CONFLICT(finding_id) DO UPDATE SET
49
+ model = excluded.model, vector_json = excluded.vector_json
50
+ """,
51
+ (
52
+ finding["findingId"],
53
+ embedding["model"],
54
+ json.dumps(embedding["vector"], allow_nan=False),
55
+ ),
56
+ )
57
+ except sqlite3.IntegrityError:
58
+ return {"error": "finding_conflict"}
59
+ return {"findingIds": [entry["finding"]["findingId"] for entry in entries]}
60
+
61
+
62
+ def list_stored_findings(
63
+ connection: sqlite3.Connection, *, limit: int, offset: int
64
+ ) -> dict[str, Any]:
65
+ connection.execute("BEGIN")
66
+ with connection:
67
+ total = connection.execute(
68
+ "SELECT COUNT(*) FROM findings WHERE details_json IS NOT NULL"
69
+ ).fetchone()[0]
70
+ rows = connection.execute(
71
+ """
72
+ SELECT details_json FROM findings WHERE details_json IS NOT NULL
73
+ ORDER BY created_at, id LIMIT ? OFFSET ?
74
+ """,
75
+ (limit, offset),
76
+ ).fetchall()
77
+ next_offset = offset + len(rows)
78
+ return {
79
+ "findings": [json.loads(row["details_json"]) for row in rows],
80
+ "limit": limit,
81
+ "offset": offset,
82
+ "total": total,
83
+ "nextOffset": next_offset if next_offset < total else None,
84
+ }
85
+
86
+
87
+ def find_potential_duplicates(
88
+ connection: sqlite3.Connection, finding_id: str, repository_id: str | None
89
+ ) -> dict[str, Any]:
90
+ """Rank IDs and vectors in the requested scope before loading finding documents."""
91
+ connection.execute("BEGIN")
92
+ with connection:
93
+ if repository_id is None:
94
+ source = "finding_embeddings AS embeddings"
95
+ predicate = ""
96
+ scope_parameters: tuple[str, ...] = ()
97
+ else:
98
+ source = (
99
+ "finding_repositories AS repositories JOIN finding_embeddings AS embeddings "
100
+ "ON embeddings.finding_id = repositories.finding_id"
101
+ )
102
+ predicate = "repositories.repository_id = ? AND "
103
+ scope_parameters = (repository_id,)
104
+ anchor = connection.execute(
105
+ f"SELECT embeddings.model, embeddings.vector_json FROM {source} "
106
+ f"WHERE {predicate}embeddings.finding_id = ?",
107
+ (*scope_parameters, finding_id),
108
+ ).fetchone()
109
+ if anchor is None:
110
+ return {"error": "finding_not_indexed"}
111
+ rows = connection.execute(
112
+ f"SELECT embeddings.finding_id, embeddings.vector_json FROM {source} "
113
+ "JOIN findings ON findings.id = embeddings.finding_id "
114
+ f"WHERE {predicate}embeddings.model = ? AND embeddings.finding_id != ? "
115
+ "ORDER BY findings.created_at, findings.id",
116
+ (*scope_parameters, anchor["model"], finding_id),
117
+ )
118
+ ranked: list[tuple[str, float]] = []
119
+ try:
120
+ vector = normalized_vector(json.loads(anchor["vector_json"]))
121
+ for row in rows:
122
+ candidate = json.loads(row["vector_json"])
123
+ if len(candidate) != len(vector):
124
+ continue
125
+ other = normalized_vector(candidate)
126
+ similarity = sum(left * right for left, right in zip(vector, other))
127
+ if similarity >= 0.55:
128
+ ranked.append((row["finding_id"], similarity))
129
+ except ValueError:
130
+ return {"error": "embedding_failed"}
131
+ # Stable sorting retains insertion-time / finding-ID order for ties.
132
+ ranked.sort(key=lambda candidate: candidate[1], reverse=True)
133
+ selected_ids = [finding_id, *(candidate[0] for candidate in ranked[:50])]
134
+ documents = {
135
+ row["id"]: json.loads(row["details_json"])
136
+ for row in connection.execute(
137
+ "SELECT id, details_json FROM findings WHERE id IN ("
138
+ + ",".join("?" for _ in selected_ids)
139
+ + ")",
140
+ selected_ids,
141
+ )
142
+ }
143
+ return {
144
+ "finding": documents[finding_id],
145
+ "potentialDuplicates": [documents[id] for id in selected_ids[1:]],
146
+ }
147
+
148
+
149
+ def store_dedupe_groups(
150
+ connection: sqlite3.Connection, groups: list[list[str]], timestamp: str
151
+ ) -> dict[str, Any]:
152
+ """Persist reviewed sets independently, including overlapping groups, in one transaction."""
153
+ stored: dict[str, dict[str, Any]] = {}
154
+ try:
155
+ with connection:
156
+ connection.execute("BEGIN IMMEDIATE")
157
+ for group in groups:
158
+ members = sorted(set(group))
159
+ # Membership, not input order, identifies a group on retries.
160
+ group_id = (
161
+ "fdg_"
162
+ + hashlib.sha256(
163
+ json.dumps(members, ensure_ascii=True, separators=(",", ":")).encode(
164
+ "utf-8"
165
+ )
166
+ ).hexdigest()
167
+ )
168
+ connection.execute(
169
+ "INSERT INTO finding_dedupe_groups (id, created_at) VALUES (?, ?) "
170
+ "ON CONFLICT(id) DO NOTHING",
171
+ (group_id, timestamp),
172
+ )
173
+ connection.executemany(
174
+ "INSERT INTO finding_dedupe_group_members (group_id, finding_id) VALUES (?, ?) "
175
+ "ON CONFLICT(group_id, finding_id) DO NOTHING",
176
+ ((group_id, finding_id) for finding_id in members),
177
+ )
178
+ created_at = connection.execute(
179
+ "SELECT created_at FROM finding_dedupe_groups WHERE id = ?", (group_id,)
180
+ ).fetchone()[0]
181
+ stored[group_id] = {
182
+ "groupId": group_id,
183
+ "findingIds": members,
184
+ "createdAt": created_at,
185
+ }
186
+ except sqlite3.IntegrityError:
187
+ return {"error": "finding_conflict"}
188
+ return {"groups": list(stored.values())}
189
+
190
+
191
+ def list_dedupe_groups(connection: sqlite3.Connection, finding_id: str) -> dict[str, Any]:
192
+ groups: dict[str, dict[str, Any]] = {}
193
+ rows = connection.execute(
194
+ """
195
+ SELECT groups.id, groups.created_at, members.finding_id
196
+ FROM finding_dedupe_group_members AS matched
197
+ JOIN finding_dedupe_groups AS groups ON groups.id = matched.group_id
198
+ JOIN finding_dedupe_group_members AS members ON members.group_id = groups.id
199
+ WHERE matched.finding_id = ?
200
+ ORDER BY groups.created_at, groups.id, members.finding_id
201
+ """,
202
+ (finding_id,),
203
+ )
204
+ for row in rows:
205
+ group = groups.setdefault(
206
+ row["id"],
207
+ {"groupId": row["id"], "findingIds": [], "createdAt": row["created_at"]},
208
+ )
209
+ group["findingIds"].append(row["finding_id"])
210
+ return {"groups": list(groups.values())}
211
+
212
+
213
+ def normalized_vector(vector: list[float]) -> list[float]:
214
+ norm = math.hypot(*vector)
215
+ if norm == 0 or not math.isfinite(norm):
216
+ raise ValueError("A stored embedding cannot be compared.")
217
+ return [value / norm for value in vector]
218
+
219
+
220
+ if __name__ == "__main__":
221
+ argparse.ArgumentParser(description=__doc__).parse_args()
@@ -101,8 +101,7 @@ def _indexed_findings(connection: sqlite3.Connection) -> Iterator[dict[str, Any]
101
101
 
102
102
  latest_scan_by_target = dict(
103
103
  connection.execute(
104
- "SELECT target_id, id FROM scans "
105
- "WHERE status = 'complete' ORDER BY started_at, id"
104
+ "SELECT target_id, id FROM scans WHERE status = 'complete' ORDER BY started_at, id"
106
105
  )
107
106
  )
108
107