@openai/codex-security 0.1.21 → 0.1.24
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +549 -1
- package/_bundled_plugin/.codex-plugin/plugin.json +1 -1
- package/_bundled_plugin/.mcp.json +1 -0
- package/_bundled_plugin/mcp/server.mjs.br.part-000 +0 -0
- package/_bundled_plugin/mcp/server.mjs.br.part-001 +0 -0
- package/_bundled_plugin/references/config-preflight.md +2 -30
- package/_bundled_plugin/references/core-scan.md +1 -1
- package/_bundled_plugin/references/final-report.md +1 -1
- package/_bundled_plugin/references/finding-detail-fields.md +5 -10
- package/_bundled_plugin/references/sarif-adapter.md +1 -1
- package/_bundled_plugin/references/scan-contract.md +10 -0
- package/_bundled_plugin/scripts/config_preflight.py +2 -8
- package/_bundled_plugin/scripts/deep_scan_workbench.py +6 -1
- package/_bundled_plugin/scripts/finalize_scan_contract.py +115 -76
- package/_bundled_plugin/scripts/finding_preview.py +22 -4
- package/_bundled_plugin/scripts/generate_in_scope_files.py +29 -9
- package/_bundled_plugin/scripts/generate_rank_input.py +1 -5
- package/_bundled_plugin/scripts/rank_preview.py +1 -0
- package/_bundled_plugin/scripts/report_projection.py +8 -14
- package/_bundled_plugin/scripts/validate_scan_contract.py +1 -3
- package/_bundled_plugin/scripts/windows_scan_local_files.py +90 -6
- package/_bundled_plugin/scripts/workbench_cli.py +27 -1
- package/_bundled_plugin/scripts/workbench_dashboard.py +138 -0
- package/_bundled_plugin/scripts/workbench_db.py +110 -539
- package/_bundled_plugin/scripts/workbench_finding_index.py +42 -24
- package/_bundled_plugin/scripts/workbench_finding_workflows.py +208 -0
- package/_bundled_plugin/scripts/workbench_findings.py +221 -0
- package/_bundled_plugin/scripts/workbench_native_indexes.py +1 -2
- package/_bundled_plugin/scripts/workbench_publication.py +520 -0
- package/_bundled_plugin/scripts/workbench_remediation.py +37 -0
- package/_bundled_plugin/scripts/workbench_saved_results.py +322 -89
- package/_bundled_plugin/scripts/workbench_scan_history.py +1 -2
- package/_bundled_plugin/scripts/workbench_scan_usage.py +9 -33
- package/_bundled_plugin/scripts/workbench_schema.py +276 -58
- package/_bundled_plugin/scripts/workbench_target.py +30 -6
- package/_bundled_plugin/scripts/workbench_validation.py +1 -5
- package/_bundled_plugin/skills/assess-patch-risk/SKILL.md +1 -5
- package/_bundled_plugin/skills/attack-path-analysis/references/severity-policy.md +6 -1
- package/_bundled_plugin/skills/track-findings/SKILL.md +2 -0
- package/dist/api.d.ts +5 -1
- package/dist/api.d.ts.map +1 -1
- package/dist/api.js +135 -27
- package/dist/api.js.map +1 -1
- package/dist/cli.d.ts +4 -0
- package/dist/cli.d.ts.map +1 -1
- package/dist/cli.js +158 -62
- package/dist/cli.js.map +1 -1
- package/dist/cloud-publish.js +5 -1
- package/dist/cloud-publish.js.map +1 -1
- package/dist/contract.d.ts +1 -0
- package/dist/contract.d.ts.map +1 -1
- package/dist/contract.js +9 -0
- package/dist/contract.js.map +1 -1
- package/dist/custom-publish.d.ts +22 -0
- package/dist/custom-publish.d.ts.map +1 -0
- package/dist/custom-publish.js +47 -0
- package/dist/custom-publish.js.map +1 -0
- package/dist/custom-validation-prompt.js +1 -1
- package/dist/custom-validation-prompt.js.map +1 -1
- package/dist/deduplication/checkpointed-review.d.ts +16 -0
- package/dist/deduplication/checkpointed-review.d.ts.map +1 -0
- package/dist/deduplication/checkpointed-review.js +76 -0
- package/dist/deduplication/checkpointed-review.js.map +1 -0
- package/dist/deduplication/codex-review.d.ts +21 -0
- package/dist/deduplication/codex-review.d.ts.map +1 -0
- package/dist/deduplication/codex-review.js +252 -0
- package/dist/deduplication/codex-review.js.map +1 -0
- package/dist/deduplication/deduplication-prompts.d.ts +8 -0
- package/dist/deduplication/deduplication-prompts.d.ts.map +1 -0
- package/dist/deduplication/deduplication-prompts.js +35 -0
- package/dist/deduplication/deduplication-prompts.js.map +1 -0
- package/dist/deduplication/deduplication-reviewer.d.ts +46 -0
- package/dist/deduplication/deduplication-reviewer.d.ts.map +1 -0
- package/dist/deduplication/deduplication-reviewer.js +105 -0
- package/dist/deduplication/deduplication-reviewer.js.map +1 -0
- package/dist/deduplication/deduplication.d.ts +6 -0
- package/dist/deduplication/deduplication.d.ts.map +1 -0
- package/dist/deduplication/deduplication.js +85 -0
- package/dist/deduplication/deduplication.js.map +1 -0
- package/dist/deduplication/scan.d.ts +16 -0
- package/dist/deduplication/scan.d.ts.map +1 -0
- package/dist/deduplication/scan.js +101 -0
- package/dist/deduplication/scan.js.map +1 -0
- package/dist/deep-progress.d.ts +21 -0
- package/dist/deep-progress.d.ts.map +1 -0
- package/dist/deep-progress.js +108 -0
- package/dist/deep-progress.js.map +1 -0
- package/dist/finding-dedupe-groups.d.ts +7 -0
- package/dist/finding-dedupe-groups.d.ts.map +1 -0
- package/dist/finding-dedupe-groups.js +2 -0
- package/dist/finding-dedupe-groups.js.map +1 -0
- package/dist/finding-retrieval.d.ts +13 -0
- package/dist/finding-retrieval.d.ts.map +1 -0
- package/dist/finding-retrieval.js +2 -0
- package/dist/finding-retrieval.js.map +1 -0
- package/dist/finding-workflow.d.ts +53 -0
- package/dist/finding-workflow.d.ts.map +1 -0
- package/dist/finding-workflow.js +117 -0
- package/dist/finding-workflow.js.map +1 -0
- package/dist/findings-client.d.ts +15 -0
- package/dist/findings-client.d.ts.map +1 -0
- package/dist/findings-client.js +60 -0
- package/dist/findings-client.js.map +1 -0
- package/dist/index.d.ts +5 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +2 -0
- package/dist/index.js.map +1 -1
- package/dist/publication.js +28 -40
- package/dist/publication.js.map +1 -1
- package/dist/runtime.d.ts +5 -1
- package/dist/runtime.d.ts.map +1 -1
- package/dist/runtime.js +119 -13
- package/dist/runtime.js.map +1 -1
- package/dist/saved-scan.d.ts +15 -0
- package/dist/saved-scan.d.ts.map +1 -0
- package/dist/saved-scan.js +68 -0
- package/dist/saved-scan.js.map +1 -0
- package/dist/scan-activity.d.ts.map +1 -1
- package/dist/scan-activity.js +12 -2
- package/dist/scan-activity.js.map +1 -1
- package/dist/scan-comparison.d.ts +3 -2
- package/dist/scan-comparison.d.ts.map +1 -1
- package/dist/scan-comparison.js +1 -1
- package/dist/scan-comparison.js.map +1 -1
- package/dist/server/dashboard/THIRD_PARTY_NOTICES.txt +276 -0
- package/dist/server/dashboard/app.css +2 -0
- package/dist/server/dashboard/app.js +66 -0
- package/dist/server/dashboard/index.html +18 -0
- package/dist/server/dashboard-types.d.ts +45 -0
- package/dist/server/dashboard-types.d.ts.map +1 -0
- package/dist/server/dashboard-types.js +2 -0
- package/dist/server/dashboard-types.js.map +1 -0
- package/dist/server/dashboard.d.ts +6 -0
- package/dist/server/dashboard.d.ts.map +1 -0
- package/dist/server/dashboard.js +45 -0
- package/dist/server/dashboard.js.map +1 -0
- package/dist/server/embeddings.d.ts +16 -0
- package/dist/server/embeddings.d.ts.map +1 -0
- package/dist/server/embeddings.js +114 -0
- package/dist/server/embeddings.js.map +1 -0
- package/dist/server/errors.d.ts +5 -0
- package/dist/server/errors.d.ts.map +1 -0
- package/dist/server/errors.js +8 -0
- package/dist/server/errors.js.map +1 -0
- package/dist/server/findings-service.d.ts +20 -0
- package/dist/server/findings-service.d.ts.map +1 -0
- package/dist/server/findings-service.js +31 -0
- package/dist/server/findings-service.js.map +1 -0
- package/dist/server/index.d.ts +2 -0
- package/dist/server/index.d.ts.map +1 -0
- package/dist/server/index.js +6 -0
- package/dist/server/index.js.map +1 -0
- package/dist/server/routes.d.ts +6 -0
- package/dist/server/routes.d.ts.map +1 -0
- package/dist/server/routes.js +86 -0
- package/dist/server/routes.js.map +1 -0
- package/dist/server/serve.d.ts +2 -0
- package/dist/server/serve.d.ts.map +1 -0
- package/dist/server/serve.js +28 -0
- package/dist/server/serve.js.map +1 -0
- package/dist/server/server.d.ts +10 -0
- package/dist/server/server.d.ts.map +1 -0
- package/dist/server/server.js +17 -0
- package/dist/server/server.js.map +1 -0
- package/dist/server/sqlite-store.d.ts +22 -0
- package/dist/server/sqlite-store.d.ts.map +1 -0
- package/dist/server/sqlite-store.js +82 -0
- package/dist/server/sqlite-store.js.map +1 -0
- package/dist/server/storage.d.ts +32 -0
- package/dist/server/storage.d.ts.map +1 -0
- package/dist/server/storage.js +2 -0
- package/dist/server/storage.js.map +1 -0
- package/dist/server/validation.d.ts +17 -0
- package/dist/server/validation.d.ts.map +1 -0
- package/dist/server/validation.js +53 -0
- package/dist/server/validation.js.map +1 -0
- package/dist/trusted-executable.d.ts.map +1 -1
- package/dist/trusted-executable.js +2 -1
- package/dist/trusted-executable.js.map +1 -1
- package/dist/version.d.ts +1 -1
- package/dist/version.js +1 -1
- package/dist/version.js.map +1 -1
- package/package.json +17 -5
|
@@ -8,6 +8,44 @@ import sqlite3
|
|
|
8
8
|
from typing import Any
|
|
9
9
|
|
|
10
10
|
|
|
11
|
+
def upsert_finding(
|
|
12
|
+
connection: sqlite3.Connection,
|
|
13
|
+
finding: dict[str, Any],
|
|
14
|
+
timestamp: str,
|
|
15
|
+
repository_id: str | None = None,
|
|
16
|
+
) -> None:
|
|
17
|
+
connection.execute(
|
|
18
|
+
"""
|
|
19
|
+
INSERT INTO findings (
|
|
20
|
+
id, fingerprint, rule_id, identity_anchor, identity_instance,
|
|
21
|
+
details_json, created_at, updated_at
|
|
22
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?)
|
|
23
|
+
ON CONFLICT(id) DO UPDATE SET
|
|
24
|
+
fingerprint = excluded.fingerprint,
|
|
25
|
+
rule_id = excluded.rule_id,
|
|
26
|
+
identity_anchor = excluded.identity_anchor,
|
|
27
|
+
identity_instance = excluded.identity_instance,
|
|
28
|
+
details_json = excluded.details_json,
|
|
29
|
+
updated_at = excluded.updated_at
|
|
30
|
+
""",
|
|
31
|
+
(
|
|
32
|
+
finding["findingId"],
|
|
33
|
+
finding["fingerprints"]["primary"],
|
|
34
|
+
finding["ruleId"],
|
|
35
|
+
finding["identity"]["anchor"],
|
|
36
|
+
finding["identity"].get("instance"),
|
|
37
|
+
json.dumps(finding, allow_nan=False, sort_keys=True),
|
|
38
|
+
timestamp,
|
|
39
|
+
timestamp,
|
|
40
|
+
),
|
|
41
|
+
)
|
|
42
|
+
if repository_id is not None:
|
|
43
|
+
connection.execute(
|
|
44
|
+
"INSERT OR IGNORE INTO finding_repositories (repository_id, finding_id) VALUES (?, ?)",
|
|
45
|
+
(repository_id, finding["findingId"]),
|
|
46
|
+
)
|
|
47
|
+
|
|
48
|
+
|
|
11
49
|
def index_findings(
|
|
12
50
|
connection: sqlite3.Connection,
|
|
13
51
|
scan_id: str,
|
|
@@ -17,35 +55,15 @@ def index_findings(
|
|
|
17
55
|
findings = document.get("findings")
|
|
18
56
|
if not isinstance(findings, list):
|
|
19
57
|
raise SystemExit("findings.json must contain a findings array.")
|
|
58
|
+
repository_id = connection.execute(
|
|
59
|
+
"SELECT target_id FROM scans WHERE id = ?", (scan_id,)
|
|
60
|
+
).fetchone()["target_id"]
|
|
20
61
|
for finding in findings:
|
|
21
62
|
if not isinstance(finding, dict):
|
|
22
63
|
raise SystemExit("findings.json entries must be objects.")
|
|
23
|
-
identity = finding["identity"]
|
|
24
|
-
fingerprints = finding["fingerprints"]
|
|
25
64
|
severity = finding["severity"]
|
|
26
65
|
confidence = finding["confidence"]
|
|
27
|
-
connection
|
|
28
|
-
"""
|
|
29
|
-
INSERT INTO findings (
|
|
30
|
-
id, fingerprint, rule_id, identity_anchor, identity_instance, created_at, updated_at
|
|
31
|
-
) VALUES (?, ?, ?, ?, ?, ?, ?)
|
|
32
|
-
ON CONFLICT(id) DO UPDATE SET
|
|
33
|
-
fingerprint = excluded.fingerprint,
|
|
34
|
-
rule_id = excluded.rule_id,
|
|
35
|
-
identity_anchor = excluded.identity_anchor,
|
|
36
|
-
identity_instance = excluded.identity_instance,
|
|
37
|
-
updated_at = excluded.updated_at
|
|
38
|
-
""",
|
|
39
|
-
(
|
|
40
|
-
finding["findingId"],
|
|
41
|
-
fingerprints["primary"],
|
|
42
|
-
finding["ruleId"],
|
|
43
|
-
identity["anchor"],
|
|
44
|
-
identity.get("instance"),
|
|
45
|
-
timestamp,
|
|
46
|
-
timestamp,
|
|
47
|
-
),
|
|
48
|
-
)
|
|
66
|
+
upsert_finding(connection, finding, timestamp, repository_id)
|
|
49
67
|
connection.execute(
|
|
50
68
|
"""
|
|
51
69
|
INSERT INTO finding_occurrences (
|
|
@@ -0,0 +1,208 @@
|
|
|
1
|
+
"""Durable state for the opt-in local scan, publication, and dedupe workflow."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import argparse
|
|
6
|
+
import hashlib
|
|
7
|
+
import json
|
|
8
|
+
import sqlite3
|
|
9
|
+
import sys
|
|
10
|
+
from pathlib import Path
|
|
11
|
+
from typing import Any
|
|
12
|
+
|
|
13
|
+
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
|
14
|
+
from workbench_target import directory_content_digest, git_output, git_revision
|
|
15
|
+
|
|
16
|
+
WORKFLOW_BINDINGS = {
|
|
17
|
+
"repositoryPath": "repository_path",
|
|
18
|
+
"scanRequestDigest": "scan_request_digest",
|
|
19
|
+
"scanId": "scan_id",
|
|
20
|
+
"scanDir": "scan_dir",
|
|
21
|
+
"artifactDigest": "artifact_digest",
|
|
22
|
+
"destination": "destination",
|
|
23
|
+
}
|
|
24
|
+
WORKFLOW_STAGES = ("scan", "publish", "dedupe")
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def read_workflow(connection: sqlite3.Connection, workflow_id: str) -> dict[str, Any] | None:
|
|
28
|
+
row = connection.execute(
|
|
29
|
+
"SELECT * FROM finding_workflows WHERE id = ?", (workflow_id,)
|
|
30
|
+
).fetchone()
|
|
31
|
+
if row is None:
|
|
32
|
+
return None
|
|
33
|
+
state = {"id": row["id"], "stages": {}}
|
|
34
|
+
for field, column in WORKFLOW_BINDINGS.items():
|
|
35
|
+
if row[column] is not None:
|
|
36
|
+
state[field] = row[column]
|
|
37
|
+
if row["scope_repository_id"] is not None:
|
|
38
|
+
state["scope"] = {"repositoryId": row["scope_repository_id"]}
|
|
39
|
+
elif row["scope_all_repositories"] is not None:
|
|
40
|
+
state["scope"] = {"allRepositories": bool(row["scope_all_repositories"])}
|
|
41
|
+
results = json.loads(row["results_json"])
|
|
42
|
+
for stage in WORKFLOW_STAGES:
|
|
43
|
+
current = {"status": row[f"{stage}_status"]}
|
|
44
|
+
if row[f"{stage}_error"] is not None:
|
|
45
|
+
current["error"] = row[f"{stage}_error"]
|
|
46
|
+
if stage in results:
|
|
47
|
+
current["result"] = results[stage]
|
|
48
|
+
state["stages"][stage] = current
|
|
49
|
+
if "dedupePendingWrite" in results:
|
|
50
|
+
state["stages"]["dedupe"]["pendingWrite"] = results["dedupePendingWrite"]
|
|
51
|
+
return state
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def save_workflow(connection: sqlite3.Connection, state: dict[str, Any], timestamp: str) -> None:
|
|
55
|
+
values = {"id": state["id"]}
|
|
56
|
+
values.update({column: state.get(field) for field, column in WORKFLOW_BINDINGS.items()})
|
|
57
|
+
scope = state.get("scope", {})
|
|
58
|
+
values.update(
|
|
59
|
+
scope_repository_id=scope.get("repositoryId"),
|
|
60
|
+
scope_all_repositories=scope.get("allRepositories"),
|
|
61
|
+
)
|
|
62
|
+
results = {}
|
|
63
|
+
for stage in WORKFLOW_STAGES:
|
|
64
|
+
current = state["stages"][stage]
|
|
65
|
+
values[f"{stage}_status"] = current["status"]
|
|
66
|
+
values[f"{stage}_error"] = current.get("error")
|
|
67
|
+
if "result" in current:
|
|
68
|
+
results[stage] = current["result"]
|
|
69
|
+
if "pendingWrite" in state["stages"]["dedupe"]:
|
|
70
|
+
results["dedupePendingWrite"] = state["stages"]["dedupe"]["pendingWrite"]
|
|
71
|
+
values.update(
|
|
72
|
+
results_json=json.dumps(results, allow_nan=False),
|
|
73
|
+
created_at=timestamp,
|
|
74
|
+
updated_at=timestamp,
|
|
75
|
+
)
|
|
76
|
+
updates = ", ".join(
|
|
77
|
+
f"{column} = excluded.{column}" for column in values if column not in {"id", "created_at"}
|
|
78
|
+
)
|
|
79
|
+
connection.execute(
|
|
80
|
+
f"INSERT INTO finding_workflows ({', '.join(values)}) "
|
|
81
|
+
f"VALUES ({', '.join('?' for _ in values)}) ON CONFLICT(id) DO UPDATE SET {updates}",
|
|
82
|
+
tuple(values.values()),
|
|
83
|
+
)
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
def bind_workflow(state: dict[str, Any], binding: dict[str, Any]) -> None:
|
|
87
|
+
for field, value in binding.items():
|
|
88
|
+
if field not in WORKFLOW_BINDINGS and field != "scope":
|
|
89
|
+
raise SystemExit("Unknown workflow binding.")
|
|
90
|
+
if field in state and state[field] != value:
|
|
91
|
+
raise SystemExit(
|
|
92
|
+
f"Workflow {state['id']} is already bound to a different {field}. "
|
|
93
|
+
"Use another --workflow-id."
|
|
94
|
+
)
|
|
95
|
+
state[field] = value
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
def register_workflow_scan(
|
|
99
|
+
connection: sqlite3.Connection, workflow_id: str, scan_id: str, scan_dir: str, timestamp: str
|
|
100
|
+
) -> None:
|
|
101
|
+
"""Called inside the scan-registration transaction, before model execution."""
|
|
102
|
+
state = read_workflow(connection, workflow_id)
|
|
103
|
+
if state is None:
|
|
104
|
+
raise SystemExit("The workflow must be started before registering its scan.")
|
|
105
|
+
if state["stages"]["scan"]["status"] == "completed":
|
|
106
|
+
raise SystemExit("The workflow scan is already complete.")
|
|
107
|
+
previous = state.get("scanId")
|
|
108
|
+
if previous is not None:
|
|
109
|
+
row = connection.execute("SELECT status FROM scans WHERE id = ?", (previous,)).fetchone()
|
|
110
|
+
if row is not None and row["status"] == "complete":
|
|
111
|
+
raise SystemExit("Reuse the workflow's completed scan instead of registering another.")
|
|
112
|
+
state["scanId"] = scan_id
|
|
113
|
+
state["scanDir"] = scan_dir
|
|
114
|
+
save_workflow(connection, state, timestamp)
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
def finding_workflow(
|
|
118
|
+
connection: sqlite3.Connection, payload: dict[str, Any], timestamp: str
|
|
119
|
+
) -> dict[str, Any]:
|
|
120
|
+
workflow_id = payload["id"]
|
|
121
|
+
if not isinstance(workflow_id, str) or not workflow_id.strip():
|
|
122
|
+
raise SystemExit("workflowId must be a nonempty string.")
|
|
123
|
+
if payload["action"] == "get":
|
|
124
|
+
return {"workflow": read_workflow(connection, workflow_id)}
|
|
125
|
+
if payload["action"] == "source":
|
|
126
|
+
target = Path(payload["repository"]).resolve(strict=True)
|
|
127
|
+
return {
|
|
128
|
+
"source": {
|
|
129
|
+
"repository": str(target),
|
|
130
|
+
"revision": git_revision(target),
|
|
131
|
+
"refsDigest": hashlib.sha256(
|
|
132
|
+
(git_output(target, "show-ref") or "").encode()
|
|
133
|
+
).hexdigest(),
|
|
134
|
+
"content": directory_content_digest(target, include_ignored=True),
|
|
135
|
+
}
|
|
136
|
+
}
|
|
137
|
+
if payload["action"] == "get-review":
|
|
138
|
+
row = connection.execute(
|
|
139
|
+
"SELECT result_json FROM finding_workflow_reviews WHERE workflow_id = ? AND review_key = ?",
|
|
140
|
+
(workflow_id, payload["key"]),
|
|
141
|
+
).fetchone()
|
|
142
|
+
return {"review": json.loads(row["result_json"]) if row is not None else None}
|
|
143
|
+
if payload["action"] == "save-review":
|
|
144
|
+
binding = payload["binding"]
|
|
145
|
+
source = binding["source"]
|
|
146
|
+
scope = binding["scope"]
|
|
147
|
+
with connection:
|
|
148
|
+
connection.execute(
|
|
149
|
+
"""INSERT INTO finding_workflow_reviews
|
|
150
|
+
(workflow_id, review_key, review_contract_version, codex_version,
|
|
151
|
+
source_repository_path, source_revision, source_refs_digest, source_content_digest,
|
|
152
|
+
scope_repository_id, scope_all_repositories, model, effort, settings_digest,
|
|
153
|
+
prompt_digest, contract_digest, result_json, created_at)
|
|
154
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
155
|
+
ON CONFLICT(workflow_id, review_key) DO NOTHING""",
|
|
156
|
+
(
|
|
157
|
+
workflow_id,
|
|
158
|
+
payload["key"],
|
|
159
|
+
binding["version"],
|
|
160
|
+
binding["codexVersion"],
|
|
161
|
+
source["repository"],
|
|
162
|
+
source["revision"],
|
|
163
|
+
source["refsDigest"],
|
|
164
|
+
source["content"],
|
|
165
|
+
scope.get("repositoryId"),
|
|
166
|
+
scope.get("allRepositories"),
|
|
167
|
+
binding["model"],
|
|
168
|
+
binding["effort"],
|
|
169
|
+
binding.get("settingsDigest"),
|
|
170
|
+
binding["promptDigest"],
|
|
171
|
+
binding["contractDigest"],
|
|
172
|
+
json.dumps(payload["result"], allow_nan=False),
|
|
173
|
+
timestamp,
|
|
174
|
+
),
|
|
175
|
+
)
|
|
176
|
+
return {}
|
|
177
|
+
connection.execute("BEGIN IMMEDIATE")
|
|
178
|
+
with connection:
|
|
179
|
+
state = read_workflow(connection, workflow_id)
|
|
180
|
+
if state is None:
|
|
181
|
+
state = {
|
|
182
|
+
"id": workflow_id,
|
|
183
|
+
"stages": {stage: {"status": "pending"} for stage in WORKFLOW_STAGES},
|
|
184
|
+
}
|
|
185
|
+
bind_workflow(state, payload.get("binding", {}))
|
|
186
|
+
action = payload["action"]
|
|
187
|
+
if action != "bind":
|
|
188
|
+
stage = payload["stage"]
|
|
189
|
+
if stage not in state["stages"]:
|
|
190
|
+
raise SystemExit("Unknown workflow stage.")
|
|
191
|
+
current = state["stages"][stage]
|
|
192
|
+
if current["status"] != "completed":
|
|
193
|
+
if action == "begin":
|
|
194
|
+
current["status"] = "running"
|
|
195
|
+
elif action == "complete":
|
|
196
|
+
state["stages"][stage] = {"status": "completed", "result": payload["result"]}
|
|
197
|
+
elif action == "fail":
|
|
198
|
+
current.update(status="failed", error=payload["error"])
|
|
199
|
+
elif action == "prepare-dedupe":
|
|
200
|
+
current.update(result=payload["result"], pendingWrite=payload["pendingWrite"])
|
|
201
|
+
else:
|
|
202
|
+
raise SystemExit("Unknown workflow action.")
|
|
203
|
+
save_workflow(connection, state, timestamp)
|
|
204
|
+
return {"workflow": state}
|
|
205
|
+
|
|
206
|
+
|
|
207
|
+
if __name__ == "__main__":
|
|
208
|
+
argparse.ArgumentParser(description=__doc__).parse_args()
|
|
@@ -0,0 +1,221 @@
|
|
|
1
|
+
"""Storage operations for complete findings imported without a scan."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import argparse
|
|
6
|
+
import hashlib
|
|
7
|
+
import json
|
|
8
|
+
import math
|
|
9
|
+
import sqlite3
|
|
10
|
+
import sys
|
|
11
|
+
from pathlib import Path
|
|
12
|
+
from typing import Any
|
|
13
|
+
|
|
14
|
+
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
|
15
|
+
from workbench_finding_index import upsert_finding
|
|
16
|
+
|
|
17
|
+
|
|
18
|
+
def store_findings(
|
|
19
|
+
connection: sqlite3.Connection,
|
|
20
|
+
entries: list[dict[str, Any]],
|
|
21
|
+
timestamp: str,
|
|
22
|
+
repository_id: str | None = None,
|
|
23
|
+
) -> dict[str, Any]:
|
|
24
|
+
try:
|
|
25
|
+
with connection:
|
|
26
|
+
connection.execute("BEGIN IMMEDIATE")
|
|
27
|
+
for entry in entries:
|
|
28
|
+
finding = entry["finding"]
|
|
29
|
+
embedding = entry["embedding"]
|
|
30
|
+
identity = (
|
|
31
|
+
finding["fingerprints"]["primary"],
|
|
32
|
+
finding["ruleId"],
|
|
33
|
+
finding["identity"]["anchor"],
|
|
34
|
+
finding["identity"].get("instance"),
|
|
35
|
+
)
|
|
36
|
+
current = connection.execute(
|
|
37
|
+
"SELECT fingerprint, rule_id, identity_anchor, identity_instance "
|
|
38
|
+
"FROM findings WHERE id = ?",
|
|
39
|
+
(finding["findingId"],),
|
|
40
|
+
).fetchone()
|
|
41
|
+
if current is not None and tuple(current) != identity:
|
|
42
|
+
raise sqlite3.IntegrityError("The stored finding identity cannot be replaced.")
|
|
43
|
+
upsert_finding(connection, finding, timestamp, repository_id)
|
|
44
|
+
connection.execute(
|
|
45
|
+
"""
|
|
46
|
+
INSERT INTO finding_embeddings (finding_id, model, vector_json)
|
|
47
|
+
VALUES (?, ?, ?)
|
|
48
|
+
ON CONFLICT(finding_id) DO UPDATE SET
|
|
49
|
+
model = excluded.model, vector_json = excluded.vector_json
|
|
50
|
+
""",
|
|
51
|
+
(
|
|
52
|
+
finding["findingId"],
|
|
53
|
+
embedding["model"],
|
|
54
|
+
json.dumps(embedding["vector"], allow_nan=False),
|
|
55
|
+
),
|
|
56
|
+
)
|
|
57
|
+
except sqlite3.IntegrityError:
|
|
58
|
+
return {"error": "finding_conflict"}
|
|
59
|
+
return {"findingIds": [entry["finding"]["findingId"] for entry in entries]}
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def list_stored_findings(
|
|
63
|
+
connection: sqlite3.Connection, *, limit: int, offset: int
|
|
64
|
+
) -> dict[str, Any]:
|
|
65
|
+
connection.execute("BEGIN")
|
|
66
|
+
with connection:
|
|
67
|
+
total = connection.execute(
|
|
68
|
+
"SELECT COUNT(*) FROM findings WHERE details_json IS NOT NULL"
|
|
69
|
+
).fetchone()[0]
|
|
70
|
+
rows = connection.execute(
|
|
71
|
+
"""
|
|
72
|
+
SELECT details_json FROM findings WHERE details_json IS NOT NULL
|
|
73
|
+
ORDER BY created_at, id LIMIT ? OFFSET ?
|
|
74
|
+
""",
|
|
75
|
+
(limit, offset),
|
|
76
|
+
).fetchall()
|
|
77
|
+
next_offset = offset + len(rows)
|
|
78
|
+
return {
|
|
79
|
+
"findings": [json.loads(row["details_json"]) for row in rows],
|
|
80
|
+
"limit": limit,
|
|
81
|
+
"offset": offset,
|
|
82
|
+
"total": total,
|
|
83
|
+
"nextOffset": next_offset if next_offset < total else None,
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
|
|
87
|
+
def find_potential_duplicates(
|
|
88
|
+
connection: sqlite3.Connection, finding_id: str, repository_id: str | None
|
|
89
|
+
) -> dict[str, Any]:
|
|
90
|
+
"""Rank IDs and vectors in the requested scope before loading finding documents."""
|
|
91
|
+
connection.execute("BEGIN")
|
|
92
|
+
with connection:
|
|
93
|
+
if repository_id is None:
|
|
94
|
+
source = "finding_embeddings AS embeddings"
|
|
95
|
+
predicate = ""
|
|
96
|
+
scope_parameters: tuple[str, ...] = ()
|
|
97
|
+
else:
|
|
98
|
+
source = (
|
|
99
|
+
"finding_repositories AS repositories JOIN finding_embeddings AS embeddings "
|
|
100
|
+
"ON embeddings.finding_id = repositories.finding_id"
|
|
101
|
+
)
|
|
102
|
+
predicate = "repositories.repository_id = ? AND "
|
|
103
|
+
scope_parameters = (repository_id,)
|
|
104
|
+
anchor = connection.execute(
|
|
105
|
+
f"SELECT embeddings.model, embeddings.vector_json FROM {source} "
|
|
106
|
+
f"WHERE {predicate}embeddings.finding_id = ?",
|
|
107
|
+
(*scope_parameters, finding_id),
|
|
108
|
+
).fetchone()
|
|
109
|
+
if anchor is None:
|
|
110
|
+
return {"error": "finding_not_indexed"}
|
|
111
|
+
rows = connection.execute(
|
|
112
|
+
f"SELECT embeddings.finding_id, embeddings.vector_json FROM {source} "
|
|
113
|
+
"JOIN findings ON findings.id = embeddings.finding_id "
|
|
114
|
+
f"WHERE {predicate}embeddings.model = ? AND embeddings.finding_id != ? "
|
|
115
|
+
"ORDER BY findings.created_at, findings.id",
|
|
116
|
+
(*scope_parameters, anchor["model"], finding_id),
|
|
117
|
+
)
|
|
118
|
+
ranked: list[tuple[str, float]] = []
|
|
119
|
+
try:
|
|
120
|
+
vector = normalized_vector(json.loads(anchor["vector_json"]))
|
|
121
|
+
for row in rows:
|
|
122
|
+
candidate = json.loads(row["vector_json"])
|
|
123
|
+
if len(candidate) != len(vector):
|
|
124
|
+
continue
|
|
125
|
+
other = normalized_vector(candidate)
|
|
126
|
+
similarity = sum(left * right for left, right in zip(vector, other))
|
|
127
|
+
if similarity >= 0.55:
|
|
128
|
+
ranked.append((row["finding_id"], similarity))
|
|
129
|
+
except ValueError:
|
|
130
|
+
return {"error": "embedding_failed"}
|
|
131
|
+
# Stable sorting retains insertion-time / finding-ID order for ties.
|
|
132
|
+
ranked.sort(key=lambda candidate: candidate[1], reverse=True)
|
|
133
|
+
selected_ids = [finding_id, *(candidate[0] for candidate in ranked[:50])]
|
|
134
|
+
documents = {
|
|
135
|
+
row["id"]: json.loads(row["details_json"])
|
|
136
|
+
for row in connection.execute(
|
|
137
|
+
"SELECT id, details_json FROM findings WHERE id IN ("
|
|
138
|
+
+ ",".join("?" for _ in selected_ids)
|
|
139
|
+
+ ")",
|
|
140
|
+
selected_ids,
|
|
141
|
+
)
|
|
142
|
+
}
|
|
143
|
+
return {
|
|
144
|
+
"finding": documents[finding_id],
|
|
145
|
+
"potentialDuplicates": [documents[id] for id in selected_ids[1:]],
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
|
|
149
|
+
def store_dedupe_groups(
|
|
150
|
+
connection: sqlite3.Connection, groups: list[list[str]], timestamp: str
|
|
151
|
+
) -> dict[str, Any]:
|
|
152
|
+
"""Persist reviewed sets independently, including overlapping groups, in one transaction."""
|
|
153
|
+
stored: dict[str, dict[str, Any]] = {}
|
|
154
|
+
try:
|
|
155
|
+
with connection:
|
|
156
|
+
connection.execute("BEGIN IMMEDIATE")
|
|
157
|
+
for group in groups:
|
|
158
|
+
members = sorted(set(group))
|
|
159
|
+
# Membership, not input order, identifies a group on retries.
|
|
160
|
+
group_id = (
|
|
161
|
+
"fdg_"
|
|
162
|
+
+ hashlib.sha256(
|
|
163
|
+
json.dumps(members, ensure_ascii=True, separators=(",", ":")).encode(
|
|
164
|
+
"utf-8"
|
|
165
|
+
)
|
|
166
|
+
).hexdigest()
|
|
167
|
+
)
|
|
168
|
+
connection.execute(
|
|
169
|
+
"INSERT INTO finding_dedupe_groups (id, created_at) VALUES (?, ?) "
|
|
170
|
+
"ON CONFLICT(id) DO NOTHING",
|
|
171
|
+
(group_id, timestamp),
|
|
172
|
+
)
|
|
173
|
+
connection.executemany(
|
|
174
|
+
"INSERT INTO finding_dedupe_group_members (group_id, finding_id) VALUES (?, ?) "
|
|
175
|
+
"ON CONFLICT(group_id, finding_id) DO NOTHING",
|
|
176
|
+
((group_id, finding_id) for finding_id in members),
|
|
177
|
+
)
|
|
178
|
+
created_at = connection.execute(
|
|
179
|
+
"SELECT created_at FROM finding_dedupe_groups WHERE id = ?", (group_id,)
|
|
180
|
+
).fetchone()[0]
|
|
181
|
+
stored[group_id] = {
|
|
182
|
+
"groupId": group_id,
|
|
183
|
+
"findingIds": members,
|
|
184
|
+
"createdAt": created_at,
|
|
185
|
+
}
|
|
186
|
+
except sqlite3.IntegrityError:
|
|
187
|
+
return {"error": "finding_conflict"}
|
|
188
|
+
return {"groups": list(stored.values())}
|
|
189
|
+
|
|
190
|
+
|
|
191
|
+
def list_dedupe_groups(connection: sqlite3.Connection, finding_id: str) -> dict[str, Any]:
|
|
192
|
+
groups: dict[str, dict[str, Any]] = {}
|
|
193
|
+
rows = connection.execute(
|
|
194
|
+
"""
|
|
195
|
+
SELECT groups.id, groups.created_at, members.finding_id
|
|
196
|
+
FROM finding_dedupe_group_members AS matched
|
|
197
|
+
JOIN finding_dedupe_groups AS groups ON groups.id = matched.group_id
|
|
198
|
+
JOIN finding_dedupe_group_members AS members ON members.group_id = groups.id
|
|
199
|
+
WHERE matched.finding_id = ?
|
|
200
|
+
ORDER BY groups.created_at, groups.id, members.finding_id
|
|
201
|
+
""",
|
|
202
|
+
(finding_id,),
|
|
203
|
+
)
|
|
204
|
+
for row in rows:
|
|
205
|
+
group = groups.setdefault(
|
|
206
|
+
row["id"],
|
|
207
|
+
{"groupId": row["id"], "findingIds": [], "createdAt": row["created_at"]},
|
|
208
|
+
)
|
|
209
|
+
group["findingIds"].append(row["finding_id"])
|
|
210
|
+
return {"groups": list(groups.values())}
|
|
211
|
+
|
|
212
|
+
|
|
213
|
+
def normalized_vector(vector: list[float]) -> list[float]:
|
|
214
|
+
norm = math.hypot(*vector)
|
|
215
|
+
if norm == 0 or not math.isfinite(norm):
|
|
216
|
+
raise ValueError("A stored embedding cannot be compared.")
|
|
217
|
+
return [value / norm for value in vector]
|
|
218
|
+
|
|
219
|
+
|
|
220
|
+
if __name__ == "__main__":
|
|
221
|
+
argparse.ArgumentParser(description=__doc__).parse_args()
|
|
@@ -101,8 +101,7 @@ def _indexed_findings(connection: sqlite3.Connection) -> Iterator[dict[str, Any]
|
|
|
101
101
|
|
|
102
102
|
latest_scan_by_target = dict(
|
|
103
103
|
connection.execute(
|
|
104
|
-
"SELECT target_id, id FROM scans "
|
|
105
|
-
"WHERE status = 'complete' ORDER BY started_at, id"
|
|
104
|
+
"SELECT target_id, id FROM scans WHERE status = 'complete' ORDER BY started_at, id"
|
|
106
105
|
)
|
|
107
106
|
)
|
|
108
107
|
|