@openai/codex-security 0.1.21 → 0.1.23

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (157) hide show
  1. package/README.md +529 -0
  2. package/_bundled_plugin/.codex-plugin/plugin.json +1 -1
  3. package/_bundled_plugin/mcp/server.mjs.br.part-000 +0 -0
  4. package/_bundled_plugin/mcp/server.mjs.br.part-001 +0 -0
  5. package/_bundled_plugin/references/config-preflight.md +2 -30
  6. package/_bundled_plugin/references/core-scan.md +1 -1
  7. package/_bundled_plugin/references/final-report.md +1 -1
  8. package/_bundled_plugin/references/finding-detail-fields.md +5 -10
  9. package/_bundled_plugin/references/sarif-adapter.md +1 -1
  10. package/_bundled_plugin/scripts/config_preflight.py +2 -8
  11. package/_bundled_plugin/scripts/finalize_scan_contract.py +114 -24
  12. package/_bundled_plugin/scripts/finding_preview.py +22 -4
  13. package/_bundled_plugin/scripts/generate_in_scope_files.py +27 -1
  14. package/_bundled_plugin/scripts/windows_scan_local_files.py +101 -7
  15. package/_bundled_plugin/scripts/workbench_cli.py +21 -1
  16. package/_bundled_plugin/scripts/workbench_dashboard.py +115 -0
  17. package/_bundled_plugin/scripts/workbench_db.py +29 -2
  18. package/_bundled_plugin/scripts/workbench_finding_index.py +42 -24
  19. package/_bundled_plugin/scripts/workbench_finding_workflows.py +190 -0
  20. package/_bundled_plugin/scripts/workbench_findings.py +216 -0
  21. package/_bundled_plugin/scripts/workbench_saved_results.py +30 -5
  22. package/_bundled_plugin/scripts/workbench_schema.py +263 -58
  23. package/_bundled_plugin/scripts/workbench_target.py +22 -3
  24. package/_bundled_plugin/skills/track-findings/SKILL.md +2 -0
  25. package/dist/api.d.ts +2 -0
  26. package/dist/api.d.ts.map +1 -1
  27. package/dist/api.js +115 -27
  28. package/dist/api.js.map +1 -1
  29. package/dist/cli.d.ts +4 -0
  30. package/dist/cli.d.ts.map +1 -1
  31. package/dist/cli.js +124 -58
  32. package/dist/cli.js.map +1 -1
  33. package/dist/cloud-publish.js +5 -1
  34. package/dist/cloud-publish.js.map +1 -1
  35. package/dist/contract.d.ts +1 -0
  36. package/dist/contract.d.ts.map +1 -1
  37. package/dist/contract.js +9 -0
  38. package/dist/contract.js.map +1 -1
  39. package/dist/custom-publish.d.ts +22 -0
  40. package/dist/custom-publish.d.ts.map +1 -0
  41. package/dist/custom-publish.js +47 -0
  42. package/dist/custom-publish.js.map +1 -0
  43. package/dist/custom-validation-prompt.js +1 -1
  44. package/dist/custom-validation-prompt.js.map +1 -1
  45. package/dist/deduplication/checkpointed-review.d.ts +16 -0
  46. package/dist/deduplication/checkpointed-review.d.ts.map +1 -0
  47. package/dist/deduplication/checkpointed-review.js +76 -0
  48. package/dist/deduplication/checkpointed-review.js.map +1 -0
  49. package/dist/deduplication/codex-review.d.ts +21 -0
  50. package/dist/deduplication/codex-review.d.ts.map +1 -0
  51. package/dist/deduplication/codex-review.js +252 -0
  52. package/dist/deduplication/codex-review.js.map +1 -0
  53. package/dist/deduplication/deduplication-prompts.d.ts +8 -0
  54. package/dist/deduplication/deduplication-prompts.d.ts.map +1 -0
  55. package/dist/deduplication/deduplication-prompts.js +35 -0
  56. package/dist/deduplication/deduplication-prompts.js.map +1 -0
  57. package/dist/deduplication/deduplication-reviewer.d.ts +46 -0
  58. package/dist/deduplication/deduplication-reviewer.d.ts.map +1 -0
  59. package/dist/deduplication/deduplication-reviewer.js +105 -0
  60. package/dist/deduplication/deduplication-reviewer.js.map +1 -0
  61. package/dist/deduplication/deduplication.d.ts +6 -0
  62. package/dist/deduplication/deduplication.d.ts.map +1 -0
  63. package/dist/deduplication/deduplication.js +85 -0
  64. package/dist/deduplication/deduplication.js.map +1 -0
  65. package/dist/deduplication/scan.d.ts +16 -0
  66. package/dist/deduplication/scan.d.ts.map +1 -0
  67. package/dist/deduplication/scan.js +101 -0
  68. package/dist/deduplication/scan.js.map +1 -0
  69. package/dist/finding-dedupe-groups.d.ts +7 -0
  70. package/dist/finding-dedupe-groups.d.ts.map +1 -0
  71. package/dist/finding-dedupe-groups.js +2 -0
  72. package/dist/finding-dedupe-groups.js.map +1 -0
  73. package/dist/finding-retrieval.d.ts +13 -0
  74. package/dist/finding-retrieval.d.ts.map +1 -0
  75. package/dist/finding-retrieval.js +2 -0
  76. package/dist/finding-retrieval.js.map +1 -0
  77. package/dist/finding-workflow.d.ts +53 -0
  78. package/dist/finding-workflow.d.ts.map +1 -0
  79. package/dist/finding-workflow.js +117 -0
  80. package/dist/finding-workflow.js.map +1 -0
  81. package/dist/findings-client.d.ts +15 -0
  82. package/dist/findings-client.d.ts.map +1 -0
  83. package/dist/findings-client.js +60 -0
  84. package/dist/findings-client.js.map +1 -0
  85. package/dist/index.d.ts +4 -0
  86. package/dist/index.d.ts.map +1 -1
  87. package/dist/index.js +2 -0
  88. package/dist/index.js.map +1 -1
  89. package/dist/publication.js +28 -40
  90. package/dist/publication.js.map +1 -1
  91. package/dist/runtime.d.ts +5 -1
  92. package/dist/runtime.d.ts.map +1 -1
  93. package/dist/runtime.js +113 -12
  94. package/dist/runtime.js.map +1 -1
  95. package/dist/saved-scan.d.ts +15 -0
  96. package/dist/saved-scan.d.ts.map +1 -0
  97. package/dist/saved-scan.js +68 -0
  98. package/dist/saved-scan.js.map +1 -0
  99. package/dist/scan-comparison.d.ts +3 -2
  100. package/dist/scan-comparison.d.ts.map +1 -1
  101. package/dist/scan-comparison.js +1 -1
  102. package/dist/scan-comparison.js.map +1 -1
  103. package/dist/server/dashboard/THIRD_PARTY_NOTICES.txt +276 -0
  104. package/dist/server/dashboard/app.css +2 -0
  105. package/dist/server/dashboard/app.js +66 -0
  106. package/dist/server/dashboard/index.html +18 -0
  107. package/dist/server/dashboard-types.d.ts +45 -0
  108. package/dist/server/dashboard-types.d.ts.map +1 -0
  109. package/dist/server/dashboard-types.js +2 -0
  110. package/dist/server/dashboard-types.js.map +1 -0
  111. package/dist/server/dashboard.d.ts +6 -0
  112. package/dist/server/dashboard.d.ts.map +1 -0
  113. package/dist/server/dashboard.js +45 -0
  114. package/dist/server/dashboard.js.map +1 -0
  115. package/dist/server/embeddings.d.ts +16 -0
  116. package/dist/server/embeddings.d.ts.map +1 -0
  117. package/dist/server/embeddings.js +114 -0
  118. package/dist/server/embeddings.js.map +1 -0
  119. package/dist/server/errors.d.ts +5 -0
  120. package/dist/server/errors.d.ts.map +1 -0
  121. package/dist/server/errors.js +8 -0
  122. package/dist/server/errors.js.map +1 -0
  123. package/dist/server/findings-service.d.ts +20 -0
  124. package/dist/server/findings-service.d.ts.map +1 -0
  125. package/dist/server/findings-service.js +31 -0
  126. package/dist/server/findings-service.js.map +1 -0
  127. package/dist/server/index.d.ts +2 -0
  128. package/dist/server/index.d.ts.map +1 -0
  129. package/dist/server/index.js +32 -0
  130. package/dist/server/index.js.map +1 -0
  131. package/dist/server/routes.d.ts +6 -0
  132. package/dist/server/routes.d.ts.map +1 -0
  133. package/dist/server/routes.js +86 -0
  134. package/dist/server/routes.js.map +1 -0
  135. package/dist/server/server.d.ts +10 -0
  136. package/dist/server/server.d.ts.map +1 -0
  137. package/dist/server/server.js +17 -0
  138. package/dist/server/server.js.map +1 -0
  139. package/dist/server/sqlite-store.d.ts +22 -0
  140. package/dist/server/sqlite-store.d.ts.map +1 -0
  141. package/dist/server/sqlite-store.js +82 -0
  142. package/dist/server/sqlite-store.js.map +1 -0
  143. package/dist/server/storage.d.ts +32 -0
  144. package/dist/server/storage.d.ts.map +1 -0
  145. package/dist/server/storage.js +2 -0
  146. package/dist/server/storage.js.map +1 -0
  147. package/dist/server/validation.d.ts +17 -0
  148. package/dist/server/validation.d.ts.map +1 -0
  149. package/dist/server/validation.js +53 -0
  150. package/dist/server/validation.js.map +1 -0
  151. package/dist/trusted-executable.d.ts.map +1 -1
  152. package/dist/trusted-executable.js +2 -1
  153. package/dist/trusted-executable.js.map +1 -1
  154. package/dist/version.d.ts +1 -1
  155. package/dist/version.js +1 -1
  156. package/dist/version.js.map +1 -1
  157. package/package.json +17 -5
@@ -0,0 +1,115 @@
1
+ """Read-only dashboard projections for stored findings and duplicate groups."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import argparse
6
+ import json
7
+ import sqlite3
8
+ import sys
9
+ from pathlib import Path
10
+ from typing import Any
11
+
12
+ sys.path.insert(0, str(Path(__file__).resolve().parent))
13
+ from workbench_findings import list_dedupe_groups
14
+
15
+
16
+ FINDING_RECORDS = """
17
+ SELECT findings.id, json_extract(details_json, '$.title') AS title,
18
+ COALESCE(repositories.ids, '[]') AS repositoryIds,
19
+ json_extract(details_json, '$.severity.level') AS severity,
20
+ findings.created_at AS createdAt, findings.updated_at AS updatedAt
21
+ FROM findings LEFT JOIN (
22
+ SELECT finding_id, json_group_array(repository_id) AS ids
23
+ FROM finding_repositories GROUP BY finding_id
24
+ ) AS repositories ON repositories.finding_id = findings.id
25
+ WHERE details_json IS NOT NULL
26
+ """
27
+
28
+ GROUP_RECORDS = """
29
+ SELECT groups.id, groups.id AS title,
30
+ (SELECT json_group_array(DISTINCT repository_id)
31
+ FROM finding_dedupe_group_members AS members
32
+ JOIN finding_repositories ON finding_repositories.finding_id = members.finding_id
33
+ WHERE members.group_id = groups.id) AS repositoryIds,
34
+ groups.created_at AS createdAt, groups.created_at AS updatedAt,
35
+ (SELECT COUNT(*) FROM finding_dedupe_group_members WHERE group_id = groups.id) AS memberCount
36
+ FROM finding_dedupe_groups AS groups
37
+ """
38
+
39
+ RECORDS = {
40
+ "findings": FINDING_RECORDS,
41
+ "groups": GROUP_RECORDS,
42
+ }
43
+
44
+
45
+ def item(row: sqlite3.Row) -> dict[str, Any]:
46
+ result = dict(row)
47
+ result["repositoryIds"] = json.loads(result["repositoryIds"])
48
+ return result
49
+
50
+
51
+ def detail(connection: sqlite3.Connection, view: str, selected: dict[str, Any]) -> dict[str, Any]:
52
+ result: dict[str, Any] = {"item": selected}
53
+ selected_id = selected["id"]
54
+ if view == "findings":
55
+ result["finding"] = json.loads(connection.execute(
56
+ "SELECT details_json FROM findings WHERE id = ?", (selected_id,),
57
+ ).fetchone()[0])
58
+ result["groups"] = list_dedupe_groups(connection, selected_id)["groups"]
59
+ else:
60
+ result["group"] = {
61
+ "groupId": selected_id, "createdAt": selected["createdAt"],
62
+ "findingIds": [r[0] for r in connection.execute(
63
+ "SELECT finding_id FROM finding_dedupe_group_members WHERE group_id = ? ORDER BY finding_id",
64
+ (selected_id,),
65
+ )],
66
+ }
67
+ return result
68
+
69
+
70
+ def dashboard(connection: sqlite3.Connection, query: dict[str, Any]) -> dict[str, Any]:
71
+ """One snapshot, no artifact reads, model calls, or writes."""
72
+ view = query["view"]
73
+ records = RECORDS[view]
74
+ clauses: list[str] = []
75
+ values: list[Any] = []
76
+ if query.get("query"):
77
+ connection.create_function("casefold", 1, str.casefold, deterministic=True)
78
+ columns = ["id", "title", "repositoryIds"]
79
+ clauses.append("(" + " OR ".join(f"instr(casefold(COALESCE({c}, '')), casefold(?)) > 0" for c in columns) + ")")
80
+ values.extend([query["query"]] * len(columns))
81
+ if query.get("repository"):
82
+ clauses.append("EXISTS (SELECT 1 FROM json_each(repositoryIds) WHERE value = ?)")
83
+ values.append(query["repository"])
84
+ where = " WHERE " + " AND ".join(clauses) if clauses else ""
85
+ order = "createdAt DESC, id" if query["sort"] == "newest" else "updatedAt DESC, id"
86
+ connection.execute("BEGIN")
87
+ with connection:
88
+ repositories = connection.execute("""
89
+ SELECT DISTINCT repository_id AS id, repository_id AS label
90
+ FROM finding_repositories ORDER BY repository_id
91
+ """).fetchall()
92
+ total = connection.execute(f"SELECT COUNT(*) FROM ({records}) {where}", values).fetchone()[0]
93
+ rows = connection.execute(
94
+ f"SELECT * FROM ({records}) {where} ORDER BY {order} LIMIT ? OFFSET ?",
95
+ (*values, query["limit"], query["offset"]),
96
+ ).fetchall()
97
+ selected = connection.execute(
98
+ f"SELECT * FROM ({records}) WHERE id = ?", (query["id"],),
99
+ ).fetchone() if query.get("id") else None
100
+ next_offset = query["offset"] + len(rows)
101
+ return {
102
+ "overview": {
103
+ "findings": connection.execute("SELECT COUNT(*) FROM findings WHERE details_json IS NOT NULL").fetchone()[0],
104
+ "groups": connection.execute("SELECT COUNT(*) FROM finding_dedupe_groups").fetchone()[0],
105
+ },
106
+ "repositories": [dict(row) for row in repositories],
107
+ "items": [item(row) for row in rows], "total": total,
108
+ "limit": query["limit"], "offset": query["offset"],
109
+ "nextOffset": next_offset if next_offset < total else None,
110
+ "detail": detail(connection, view, item(selected)) if selected is not None else None,
111
+ }
112
+
113
+
114
+ if __name__ == "__main__":
115
+ argparse.ArgumentParser(description=__doc__).parse_args()
@@ -81,7 +81,16 @@ from workbench_constants import (
81
81
  SQLITE_RETRY_ATTEMPTS,
82
82
  )
83
83
  from workbench_feedback import get_scan_feedback
84
+ from workbench_dashboard import dashboard
84
85
  from workbench_finding_index import index_findings
86
+ from workbench_finding_workflows import finding_workflow, register_workflow_scan
87
+ from workbench_findings import (
88
+ find_potential_duplicates,
89
+ list_dedupe_groups,
90
+ list_stored_findings,
91
+ store_dedupe_groups,
92
+ store_findings,
93
+ )
85
94
  from workbench_remediation import remediation_claim_is_active
86
95
  from workbench_scan_start import (
87
96
  archive_scan,
@@ -1651,10 +1660,12 @@ def register_cli_scan(connection: sqlite3.Connection, args: argparse.Namespace)
1651
1660
  raise SystemExit("The scan artifact directory must be empty before the scan starts.")
1652
1661
 
1653
1662
  user_context = None
1663
+ workflow_id = None
1654
1664
  if args.registration_json_stdin:
1655
1665
  registration = json.load(sys.stdin)
1656
1666
  recipe_json = json.dumps(registration["recipe"], ensure_ascii=False, separators=(",", ":"))
1657
1667
  user_context = registration.get("userContext")
1668
+ workflow_id = registration.get("workflowId")
1658
1669
  else:
1659
1670
  recipe_json = sys.stdin.read() if args.recipe_json_stdin else args.recipe_json
1660
1671
  recipe = parse_scan_recipe(recipe_json, repository)
@@ -1750,6 +1761,8 @@ def register_cli_scan(connection: sqlite3.Connection, args: argparse.Namespace)
1750
1761
  scan_id,
1751
1762
  ),
1752
1763
  )
1764
+ if workflow_id is not None:
1765
+ register_workflow_scan(connection, workflow_id, scan_id, str(scan_dir), timestamp)
1753
1766
  connection.commit()
1754
1767
  except BaseException:
1755
1768
  connection.rollback()
@@ -3752,8 +3765,7 @@ def require_canonical_scan_directory(scan_dir: Path) -> Path:
3752
3765
  scan_dir
3753
3766
  ):
3754
3767
  raise SystemExit("Scan directory must be an existing canonical non-symlink directory.")
3755
- # Re-check privacy on every resolution so a mid-scan rename/replace under a
3756
- # shared parent cannot substitute another user's forged artifact tree.
3768
+ # Re-check privacy so a shared parent cannot be used to substitute forged artifacts.
3757
3769
  if os.name != "nt":
3758
3770
  if stat.S_IMODE(metadata.st_mode) & 0o077:
3759
3771
  raise SystemExit(
@@ -4025,6 +4037,21 @@ def main() -> None:
4025
4037
  result = export_findings(connection, args)
4026
4038
  elif args.command == "database-info":
4027
4039
  result = {"databasePath": str(database_path())}
4040
+ elif args.command == "finding-workflow":
4041
+ result = finding_workflow(connection, json.load(sys.stdin), now())
4042
+ elif args.command == "dashboard":
4043
+ result = dashboard(connection, json.load(sys.stdin))
4044
+ elif args.command == "store-findings":
4045
+ payload = json.load(sys.stdin)
4046
+ result = store_findings(connection, payload["entries"], now(), payload.get("repositoryId"))
4047
+ elif args.command == "find-potential-duplicates":
4048
+ result = find_potential_duplicates(connection, args.finding_id, args.repository_id)
4049
+ elif args.command == "store-dedupe-groups":
4050
+ result = store_dedupe_groups(connection, json.load(sys.stdin)["groups"], now())
4051
+ elif args.command == "list-dedupe-groups":
4052
+ result = list_dedupe_groups(connection, args.finding_id)
4053
+ elif args.command == "list-stored-findings":
4054
+ result = list_stored_findings(connection, limit=args.limit, offset=args.offset)
4028
4055
  else:
4029
4056
  raise SystemExit(f"Unknown command: {args.command}")
4030
4057
  print(json.dumps(result, allow_nan=False, sort_keys=True))
@@ -8,6 +8,44 @@ import sqlite3
8
8
  from typing import Any
9
9
 
10
10
 
11
+ def upsert_finding(
12
+ connection: sqlite3.Connection,
13
+ finding: dict[str, Any],
14
+ timestamp: str,
15
+ repository_id: str | None = None,
16
+ ) -> None:
17
+ connection.execute(
18
+ """
19
+ INSERT INTO findings (
20
+ id, fingerprint, rule_id, identity_anchor, identity_instance,
21
+ details_json, created_at, updated_at
22
+ ) VALUES (?, ?, ?, ?, ?, ?, ?, ?)
23
+ ON CONFLICT(id) DO UPDATE SET
24
+ fingerprint = excluded.fingerprint,
25
+ rule_id = excluded.rule_id,
26
+ identity_anchor = excluded.identity_anchor,
27
+ identity_instance = excluded.identity_instance,
28
+ details_json = excluded.details_json,
29
+ updated_at = excluded.updated_at
30
+ """,
31
+ (
32
+ finding["findingId"],
33
+ finding["fingerprints"]["primary"],
34
+ finding["ruleId"],
35
+ finding["identity"]["anchor"],
36
+ finding["identity"].get("instance"),
37
+ json.dumps(finding, allow_nan=False, sort_keys=True),
38
+ timestamp,
39
+ timestamp,
40
+ ),
41
+ )
42
+ if repository_id is not None:
43
+ connection.execute(
44
+ "INSERT OR IGNORE INTO finding_repositories (repository_id, finding_id) VALUES (?, ?)",
45
+ (repository_id, finding["findingId"]),
46
+ )
47
+
48
+
11
49
  def index_findings(
12
50
  connection: sqlite3.Connection,
13
51
  scan_id: str,
@@ -17,35 +55,15 @@ def index_findings(
17
55
  findings = document.get("findings")
18
56
  if not isinstance(findings, list):
19
57
  raise SystemExit("findings.json must contain a findings array.")
58
+ repository_id = connection.execute(
59
+ "SELECT target_id FROM scans WHERE id = ?", (scan_id,)
60
+ ).fetchone()["target_id"]
20
61
  for finding in findings:
21
62
  if not isinstance(finding, dict):
22
63
  raise SystemExit("findings.json entries must be objects.")
23
- identity = finding["identity"]
24
- fingerprints = finding["fingerprints"]
25
64
  severity = finding["severity"]
26
65
  confidence = finding["confidence"]
27
- connection.execute(
28
- """
29
- INSERT INTO findings (
30
- id, fingerprint, rule_id, identity_anchor, identity_instance, created_at, updated_at
31
- ) VALUES (?, ?, ?, ?, ?, ?, ?)
32
- ON CONFLICT(id) DO UPDATE SET
33
- fingerprint = excluded.fingerprint,
34
- rule_id = excluded.rule_id,
35
- identity_anchor = excluded.identity_anchor,
36
- identity_instance = excluded.identity_instance,
37
- updated_at = excluded.updated_at
38
- """,
39
- (
40
- finding["findingId"],
41
- fingerprints["primary"],
42
- finding["ruleId"],
43
- identity["anchor"],
44
- identity.get("instance"),
45
- timestamp,
46
- timestamp,
47
- ),
48
- )
66
+ upsert_finding(connection, finding, timestamp, repository_id)
49
67
  connection.execute(
50
68
  """
51
69
  INSERT INTO finding_occurrences (
@@ -0,0 +1,190 @@
1
+ """Durable state for the opt-in local scan, publication, and dedupe workflow."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import argparse
6
+ import json
7
+ import hashlib
8
+ import sqlite3
9
+ import sys
10
+ from pathlib import Path
11
+ from typing import Any
12
+
13
+ sys.path.insert(0, str(Path(__file__).resolve().parent))
14
+ from workbench_target import directory_content_digest, git_output, git_revision
15
+
16
+ WORKFLOW_BINDINGS = {
17
+ "repositoryPath": "repository_path",
18
+ "scanRequestDigest": "scan_request_digest",
19
+ "scanId": "scan_id",
20
+ "scanDir": "scan_dir",
21
+ "artifactDigest": "artifact_digest",
22
+ "destination": "destination",
23
+ }
24
+ WORKFLOW_STAGES = ("scan", "publish", "dedupe")
25
+
26
+
27
+ def read_workflow(connection: sqlite3.Connection, workflow_id: str) -> dict[str, Any] | None:
28
+ row = connection.execute(
29
+ "SELECT * FROM finding_workflows WHERE id = ?", (workflow_id,)
30
+ ).fetchone()
31
+ if row is None:
32
+ return None
33
+ state = {"id": row["id"], "stages": {}}
34
+ for field, column in WORKFLOW_BINDINGS.items():
35
+ if row[column] is not None:
36
+ state[field] = row[column]
37
+ if row["scope_repository_id"] is not None:
38
+ state["scope"] = {"repositoryId": row["scope_repository_id"]}
39
+ elif row["scope_all_repositories"] is not None:
40
+ state["scope"] = {"allRepositories": bool(row["scope_all_repositories"])}
41
+ results = json.loads(row["results_json"])
42
+ for stage in WORKFLOW_STAGES:
43
+ current = {"status": row[f"{stage}_status"]}
44
+ if row[f"{stage}_error"] is not None:
45
+ current["error"] = row[f"{stage}_error"]
46
+ if stage in results:
47
+ current["result"] = results[stage]
48
+ state["stages"][stage] = current
49
+ if "dedupePendingWrite" in results:
50
+ state["stages"]["dedupe"]["pendingWrite"] = results["dedupePendingWrite"]
51
+ return state
52
+
53
+
54
+ def save_workflow(connection: sqlite3.Connection, state: dict[str, Any], timestamp: str) -> None:
55
+ values = {"id": state["id"]}
56
+ values.update({column: state.get(field) for field, column in WORKFLOW_BINDINGS.items()})
57
+ scope = state.get("scope", {})
58
+ values.update(
59
+ scope_repository_id=scope.get("repositoryId"),
60
+ scope_all_repositories=scope.get("allRepositories"),
61
+ )
62
+ results = {}
63
+ for stage in WORKFLOW_STAGES:
64
+ current = state["stages"][stage]
65
+ values[f"{stage}_status"] = current["status"]
66
+ values[f"{stage}_error"] = current.get("error")
67
+ if "result" in current:
68
+ results[stage] = current["result"]
69
+ if "pendingWrite" in state["stages"]["dedupe"]:
70
+ results["dedupePendingWrite"] = state["stages"]["dedupe"]["pendingWrite"]
71
+ values.update(
72
+ results_json=json.dumps(results, allow_nan=False), created_at=timestamp, updated_at=timestamp
73
+ )
74
+ updates = ", ".join(
75
+ f"{column} = excluded.{column}" for column in values if column not in {"id", "created_at"}
76
+ )
77
+ connection.execute(
78
+ f"INSERT INTO finding_workflows ({', '.join(values)}) "
79
+ f"VALUES ({', '.join('?' for _ in values)}) ON CONFLICT(id) DO UPDATE SET {updates}",
80
+ tuple(values.values()),
81
+ )
82
+
83
+
84
+ def bind_workflow(state: dict[str, Any], binding: dict[str, Any]) -> None:
85
+ for field, value in binding.items():
86
+ if field not in WORKFLOW_BINDINGS and field != "scope":
87
+ raise SystemExit("Unknown workflow binding.")
88
+ if field in state and state[field] != value:
89
+ raise SystemExit(
90
+ f"Workflow {state['id']} is already bound to a different {field}. "
91
+ "Use another --workflow-id."
92
+ )
93
+ state[field] = value
94
+
95
+
96
+ def register_workflow_scan(
97
+ connection: sqlite3.Connection, workflow_id: str, scan_id: str, scan_dir: str, timestamp: str
98
+ ) -> None:
99
+ """Called inside the scan-registration transaction, before model execution."""
100
+ state = read_workflow(connection, workflow_id)
101
+ if state is None:
102
+ raise SystemExit("The workflow must be started before registering its scan.")
103
+ if state["stages"]["scan"]["status"] == "completed":
104
+ raise SystemExit("The workflow scan is already complete.")
105
+ previous = state.get("scanId")
106
+ if previous is not None:
107
+ row = connection.execute("SELECT status FROM scans WHERE id = ?", (previous,)).fetchone()
108
+ if row is not None and row["status"] == "complete":
109
+ raise SystemExit("Reuse the workflow's completed scan instead of registering another.")
110
+ state["scanId"] = scan_id
111
+ state["scanDir"] = scan_dir
112
+ save_workflow(connection, state, timestamp)
113
+
114
+
115
+ def finding_workflow(
116
+ connection: sqlite3.Connection, payload: dict[str, Any], timestamp: str
117
+ ) -> dict[str, Any]:
118
+ workflow_id = payload["id"]
119
+ if not isinstance(workflow_id, str) or not workflow_id.strip():
120
+ raise SystemExit("workflowId must be a nonempty string.")
121
+ if payload["action"] == "get":
122
+ return {"workflow": read_workflow(connection, workflow_id)}
123
+ if payload["action"] == "source":
124
+ target = Path(payload["repository"]).resolve(strict=True)
125
+ return {"source": {
126
+ "repository": str(target),
127
+ "revision": git_revision(target),
128
+ "refsDigest": hashlib.sha256((git_output(target, "show-ref") or "").encode()).hexdigest(),
129
+ "content": directory_content_digest(target, include_ignored=True),
130
+ }}
131
+ if payload["action"] == "get-review":
132
+ row = connection.execute(
133
+ "SELECT result_json FROM finding_workflow_reviews WHERE workflow_id = ? AND review_key = ?",
134
+ (workflow_id, payload["key"]),
135
+ ).fetchone()
136
+ return {"review": json.loads(row["result_json"]) if row is not None else None}
137
+ if payload["action"] == "save-review":
138
+ binding = payload["binding"]
139
+ source = binding["source"]
140
+ scope = binding["scope"]
141
+ with connection:
142
+ connection.execute(
143
+ """INSERT INTO finding_workflow_reviews
144
+ (workflow_id, review_key, review_contract_version, codex_version,
145
+ source_repository_path, source_revision, source_refs_digest, source_content_digest,
146
+ scope_repository_id, scope_all_repositories, model, effort, settings_digest,
147
+ prompt_digest, contract_digest, result_json, created_at)
148
+ VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
149
+ ON CONFLICT(workflow_id, review_key) DO NOTHING""",
150
+ (
151
+ workflow_id, payload["key"], binding["version"], binding["codexVersion"],
152
+ source["repository"], source["revision"], source["refsDigest"], source["content"],
153
+ scope.get("repositoryId"), scope.get("allRepositories"), binding["model"],
154
+ binding["effort"], binding.get("settingsDigest"), binding["promptDigest"],
155
+ binding["contractDigest"], json.dumps(payload["result"], allow_nan=False), timestamp,
156
+ ),
157
+ )
158
+ return {}
159
+ connection.execute("BEGIN IMMEDIATE")
160
+ with connection:
161
+ state = read_workflow(connection, workflow_id)
162
+ if state is None:
163
+ state = {
164
+ "id": workflow_id,
165
+ "stages": {stage: {"status": "pending"} for stage in WORKFLOW_STAGES},
166
+ }
167
+ bind_workflow(state, payload.get("binding", {}))
168
+ action = payload["action"]
169
+ if action != "bind":
170
+ stage = payload["stage"]
171
+ if stage not in state["stages"]:
172
+ raise SystemExit("Unknown workflow stage.")
173
+ current = state["stages"][stage]
174
+ if current["status"] != "completed":
175
+ if action == "begin":
176
+ current["status"] = "running"
177
+ elif action == "complete":
178
+ state["stages"][stage] = {"status": "completed", "result": payload["result"]}
179
+ elif action == "fail":
180
+ current.update(status="failed", error=payload["error"])
181
+ elif action == "prepare-dedupe":
182
+ current.update(result=payload["result"], pendingWrite=payload["pendingWrite"])
183
+ else:
184
+ raise SystemExit("Unknown workflow action.")
185
+ save_workflow(connection, state, timestamp)
186
+ return {"workflow": state}
187
+
188
+
189
+ if __name__ == "__main__":
190
+ argparse.ArgumentParser(description=__doc__).parse_args()
@@ -0,0 +1,216 @@
1
+ """Storage operations for complete findings imported without a scan."""
2
+
3
+ from __future__ import annotations
4
+
5
+ import argparse
6
+ import hashlib
7
+ import json
8
+ import math
9
+ import sqlite3
10
+ import sys
11
+ from pathlib import Path
12
+ from typing import Any
13
+
14
+ sys.path.insert(0, str(Path(__file__).resolve().parent))
15
+ from workbench_finding_index import upsert_finding
16
+
17
+
18
+ def store_findings(
19
+ connection: sqlite3.Connection,
20
+ entries: list[dict[str, Any]],
21
+ timestamp: str,
22
+ repository_id: str | None = None,
23
+ ) -> dict[str, Any]:
24
+ try:
25
+ with connection:
26
+ connection.execute("BEGIN IMMEDIATE")
27
+ for entry in entries:
28
+ finding = entry["finding"]
29
+ embedding = entry["embedding"]
30
+ identity = (
31
+ finding["fingerprints"]["primary"],
32
+ finding["ruleId"],
33
+ finding["identity"]["anchor"],
34
+ finding["identity"].get("instance"),
35
+ )
36
+ current = connection.execute(
37
+ "SELECT fingerprint, rule_id, identity_anchor, identity_instance "
38
+ "FROM findings WHERE id = ?",
39
+ (finding["findingId"],),
40
+ ).fetchone()
41
+ if current is not None and tuple(current) != identity:
42
+ raise sqlite3.IntegrityError("The stored finding identity cannot be replaced.")
43
+ upsert_finding(connection, finding, timestamp, repository_id)
44
+ connection.execute(
45
+ """
46
+ INSERT INTO finding_embeddings (finding_id, model, vector_json)
47
+ VALUES (?, ?, ?)
48
+ ON CONFLICT(finding_id) DO UPDATE SET
49
+ model = excluded.model, vector_json = excluded.vector_json
50
+ """,
51
+ (
52
+ finding["findingId"],
53
+ embedding["model"],
54
+ json.dumps(embedding["vector"], allow_nan=False),
55
+ ),
56
+ )
57
+ except sqlite3.IntegrityError:
58
+ return {"error": "finding_conflict"}
59
+ return {"findingIds": [entry["finding"]["findingId"] for entry in entries]}
60
+
61
+
62
+ def list_stored_findings(
63
+ connection: sqlite3.Connection, *, limit: int, offset: int
64
+ ) -> dict[str, Any]:
65
+ connection.execute("BEGIN")
66
+ with connection:
67
+ total = connection.execute(
68
+ "SELECT COUNT(*) FROM findings WHERE details_json IS NOT NULL"
69
+ ).fetchone()[0]
70
+ rows = connection.execute(
71
+ """
72
+ SELECT details_json FROM findings WHERE details_json IS NOT NULL
73
+ ORDER BY created_at, id LIMIT ? OFFSET ?
74
+ """,
75
+ (limit, offset),
76
+ ).fetchall()
77
+ next_offset = offset + len(rows)
78
+ return {
79
+ "findings": [json.loads(row["details_json"]) for row in rows],
80
+ "limit": limit,
81
+ "offset": offset,
82
+ "total": total,
83
+ "nextOffset": next_offset if next_offset < total else None,
84
+ }
85
+
86
+
87
+ def find_potential_duplicates(
88
+ connection: sqlite3.Connection, finding_id: str, repository_id: str | None
89
+ ) -> dict[str, Any]:
90
+ """Rank IDs and vectors in the requested scope before loading finding documents."""
91
+ connection.execute("BEGIN")
92
+ with connection:
93
+ if repository_id is None:
94
+ source = "finding_embeddings AS embeddings"
95
+ predicate = ""
96
+ scope_parameters: tuple[str, ...] = ()
97
+ else:
98
+ source = (
99
+ "finding_repositories AS repositories JOIN finding_embeddings AS embeddings "
100
+ "ON embeddings.finding_id = repositories.finding_id"
101
+ )
102
+ predicate = "repositories.repository_id = ? AND "
103
+ scope_parameters = (repository_id,)
104
+ anchor = connection.execute(
105
+ f"SELECT embeddings.model, embeddings.vector_json FROM {source} "
106
+ f"WHERE {predicate}embeddings.finding_id = ?",
107
+ (*scope_parameters, finding_id),
108
+ ).fetchone()
109
+ if anchor is None:
110
+ return {"error": "finding_not_indexed"}
111
+ rows = connection.execute(
112
+ f"SELECT embeddings.finding_id, embeddings.vector_json FROM {source} "
113
+ "JOIN findings ON findings.id = embeddings.finding_id "
114
+ f"WHERE {predicate}embeddings.model = ? AND embeddings.finding_id != ? "
115
+ "ORDER BY findings.created_at, findings.id",
116
+ (*scope_parameters, anchor["model"], finding_id),
117
+ )
118
+ ranked: list[tuple[str, float]] = []
119
+ try:
120
+ vector = normalized_vector(json.loads(anchor["vector_json"]))
121
+ for row in rows:
122
+ candidate = json.loads(row["vector_json"])
123
+ if len(candidate) != len(vector):
124
+ continue
125
+ other = normalized_vector(candidate)
126
+ similarity = sum(left * right for left, right in zip(vector, other))
127
+ if similarity >= 0.55:
128
+ ranked.append((row["finding_id"], similarity))
129
+ except ValueError:
130
+ return {"error": "embedding_failed"}
131
+ # Stable sorting retains insertion-time / finding-ID order for ties.
132
+ ranked.sort(key=lambda candidate: candidate[1], reverse=True)
133
+ selected_ids = [finding_id, *(candidate[0] for candidate in ranked[:50])]
134
+ documents = {
135
+ row["id"]: json.loads(row["details_json"])
136
+ for row in connection.execute(
137
+ "SELECT id, details_json FROM findings WHERE id IN ("
138
+ + ",".join("?" for _ in selected_ids)
139
+ + ")",
140
+ selected_ids,
141
+ )
142
+ }
143
+ return {
144
+ "finding": documents[finding_id],
145
+ "potentialDuplicates": [documents[id] for id in selected_ids[1:]],
146
+ }
147
+
148
+
149
+ def store_dedupe_groups(
150
+ connection: sqlite3.Connection, groups: list[list[str]], timestamp: str
151
+ ) -> dict[str, Any]:
152
+ """Persist reviewed sets independently, including overlapping groups, in one transaction."""
153
+ stored: dict[str, dict[str, Any]] = {}
154
+ try:
155
+ with connection:
156
+ connection.execute("BEGIN IMMEDIATE")
157
+ for group in groups:
158
+ members = sorted(set(group))
159
+ # Membership, not input order, identifies a group on retries.
160
+ group_id = "fdg_" + hashlib.sha256(
161
+ json.dumps(members, ensure_ascii=True, separators=(",", ":")).encode("utf-8")
162
+ ).hexdigest()
163
+ connection.execute(
164
+ "INSERT INTO finding_dedupe_groups (id, created_at) VALUES (?, ?) "
165
+ "ON CONFLICT(id) DO NOTHING",
166
+ (group_id, timestamp),
167
+ )
168
+ connection.executemany(
169
+ "INSERT INTO finding_dedupe_group_members (group_id, finding_id) VALUES (?, ?) "
170
+ "ON CONFLICT(group_id, finding_id) DO NOTHING",
171
+ ((group_id, finding_id) for finding_id in members),
172
+ )
173
+ created_at = connection.execute(
174
+ "SELECT created_at FROM finding_dedupe_groups WHERE id = ?", (group_id,)
175
+ ).fetchone()[0]
176
+ stored[group_id] = {
177
+ "groupId": group_id,
178
+ "findingIds": members,
179
+ "createdAt": created_at,
180
+ }
181
+ except sqlite3.IntegrityError:
182
+ return {"error": "finding_conflict"}
183
+ return {"groups": list(stored.values())}
184
+
185
+
186
+ def list_dedupe_groups(connection: sqlite3.Connection, finding_id: str) -> dict[str, Any]:
187
+ groups: dict[str, dict[str, Any]] = {}
188
+ rows = connection.execute(
189
+ """
190
+ SELECT groups.id, groups.created_at, members.finding_id
191
+ FROM finding_dedupe_group_members AS matched
192
+ JOIN finding_dedupe_groups AS groups ON groups.id = matched.group_id
193
+ JOIN finding_dedupe_group_members AS members ON members.group_id = groups.id
194
+ WHERE matched.finding_id = ?
195
+ ORDER BY groups.created_at, groups.id, members.finding_id
196
+ """,
197
+ (finding_id,),
198
+ )
199
+ for row in rows:
200
+ group = groups.setdefault(
201
+ row["id"],
202
+ {"groupId": row["id"], "findingIds": [], "createdAt": row["created_at"]},
203
+ )
204
+ group["findingIds"].append(row["finding_id"])
205
+ return {"groups": list(groups.values())}
206
+
207
+
208
+ def normalized_vector(vector: list[float]) -> list[float]:
209
+ norm = math.hypot(*vector)
210
+ if norm == 0 or not math.isfinite(norm):
211
+ raise ValueError("A stored embedding cannot be compared.")
212
+ return [value / norm for value in vector]
213
+
214
+
215
+ if __name__ == "__main__":
216
+ argparse.ArgumentParser(description=__doc__).parse_args()