@openai/codex-security 0.1.20 → 0.1.23
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1101 -865
- package/_bundled_plugin/.codex-plugin/plugin.json +1 -1
- package/_bundled_plugin/mcp/server.mjs.br.part-000 +0 -0
- package/_bundled_plugin/mcp/server.mjs.br.part-001 +0 -0
- package/_bundled_plugin/references/config-preflight.md +2 -30
- package/_bundled_plugin/references/core-scan.md +1 -1
- package/_bundled_plugin/references/final-report.md +1 -1
- package/_bundled_plugin/references/finding-detail-fields.md +5 -10
- package/_bundled_plugin/references/sarif-adapter.md +1 -1
- package/_bundled_plugin/schemas/patch-risk-assessment.schema.json +243 -0
- package/_bundled_plugin/scripts/config_preflight.py +2 -8
- package/_bundled_plugin/scripts/finalize_scan_contract.py +206 -35
- package/_bundled_plugin/scripts/finding_preview.py +22 -4
- package/_bundled_plugin/scripts/generate_in_scope_files.py +27 -1
- package/_bundled_plugin/scripts/launch_codex_security_mcp +0 -0
- package/_bundled_plugin/scripts/windows_scan_local_files.py +101 -7
- package/_bundled_plugin/scripts/workbench_cli.py +21 -1
- package/_bundled_plugin/scripts/workbench_dashboard.py +115 -0
- package/_bundled_plugin/scripts/workbench_db.py +29 -2
- package/_bundled_plugin/scripts/workbench_finding_index.py +42 -24
- package/_bundled_plugin/scripts/workbench_finding_workflows.py +190 -0
- package/_bundled_plugin/scripts/workbench_findings.py +216 -0
- package/_bundled_plugin/scripts/workbench_saved_results.py +30 -5
- package/_bundled_plugin/scripts/workbench_schema.py +263 -58
- package/_bundled_plugin/scripts/workbench_target.py +22 -3
- package/_bundled_plugin/skills/assess-patch-risk/SKILL.md +88 -0
- package/_bundled_plugin/skills/assess-patch-risk/agents/openai.yaml +4 -0
- package/_bundled_plugin/skills/assess-patch-risk/references/risk-rubric.md +76 -0
- package/_bundled_plugin/skills/assess-patch-risk/scripts/validate_patch_risk_assessment.py +163 -0
- package/_bundled_plugin/skills/fix-finding/SKILL.md +16 -48
- package/_bundled_plugin/skills/track-findings/SKILL.md +2 -0
- package/dist/api.d.ts +3 -1
- package/dist/api.d.ts.map +1 -1
- package/dist/api.js +129 -45
- package/dist/api.js.map +1 -1
- package/dist/bulk-scan-discovery.d.ts.map +1 -1
- package/dist/bulk-scan-discovery.js +6 -24
- package/dist/bulk-scan-discovery.js.map +1 -1
- package/dist/cli.d.ts +28 -2
- package/dist/cli.d.ts.map +1 -1
- package/dist/cli.js +525 -96
- package/dist/cli.js.map +1 -1
- package/dist/cloud-publish.d.ts +5 -1
- package/dist/cloud-publish.d.ts.map +1 -1
- package/dist/cloud-publish.js +312 -10
- package/dist/cloud-publish.js.map +1 -1
- package/dist/component-plan.d.ts.map +1 -1
- package/dist/component-plan.js +5 -1
- package/dist/component-plan.js.map +1 -1
- package/dist/contract.d.ts +1 -0
- package/dist/contract.d.ts.map +1 -1
- package/dist/contract.js +9 -0
- package/dist/contract.js.map +1 -1
- package/dist/custom-publish.d.ts +22 -0
- package/dist/custom-publish.d.ts.map +1 -0
- package/dist/custom-publish.js +47 -0
- package/dist/custom-publish.js.map +1 -0
- package/dist/custom-validation-prompt.js +1 -1
- package/dist/custom-validation-prompt.js.map +1 -1
- package/dist/deduplication/checkpointed-review.d.ts +16 -0
- package/dist/deduplication/checkpointed-review.d.ts.map +1 -0
- package/dist/deduplication/checkpointed-review.js +76 -0
- package/dist/deduplication/checkpointed-review.js.map +1 -0
- package/dist/deduplication/codex-review.d.ts +21 -0
- package/dist/deduplication/codex-review.d.ts.map +1 -0
- package/dist/deduplication/codex-review.js +252 -0
- package/dist/deduplication/codex-review.js.map +1 -0
- package/dist/deduplication/deduplication-prompts.d.ts +8 -0
- package/dist/deduplication/deduplication-prompts.d.ts.map +1 -0
- package/dist/deduplication/deduplication-prompts.js +35 -0
- package/dist/deduplication/deduplication-prompts.js.map +1 -0
- package/dist/deduplication/deduplication-reviewer.d.ts +46 -0
- package/dist/deduplication/deduplication-reviewer.d.ts.map +1 -0
- package/dist/deduplication/deduplication-reviewer.js +105 -0
- package/dist/deduplication/deduplication-reviewer.js.map +1 -0
- package/dist/deduplication/deduplication.d.ts +6 -0
- package/dist/deduplication/deduplication.d.ts.map +1 -0
- package/dist/deduplication/deduplication.js +85 -0
- package/dist/deduplication/deduplication.js.map +1 -0
- package/dist/deduplication/scan.d.ts +16 -0
- package/dist/deduplication/scan.d.ts.map +1 -0
- package/dist/deduplication/scan.js +101 -0
- package/dist/deduplication/scan.js.map +1 -0
- package/dist/finding-dedupe-groups.d.ts +7 -0
- package/dist/finding-dedupe-groups.d.ts.map +1 -0
- package/dist/finding-dedupe-groups.js +2 -0
- package/dist/finding-dedupe-groups.js.map +1 -0
- package/dist/finding-retrieval.d.ts +13 -0
- package/dist/finding-retrieval.d.ts.map +1 -0
- package/dist/finding-retrieval.js +2 -0
- package/dist/finding-retrieval.js.map +1 -0
- package/dist/finding-workflow.d.ts +53 -0
- package/dist/finding-workflow.d.ts.map +1 -0
- package/dist/finding-workflow.js +117 -0
- package/dist/finding-workflow.js.map +1 -0
- package/dist/findings-client.d.ts +15 -0
- package/dist/findings-client.d.ts.map +1 -0
- package/dist/findings-client.js +60 -0
- package/dist/findings-client.js.map +1 -0
- package/dist/github.d.ts +161 -0
- package/dist/github.d.ts.map +1 -0
- package/dist/github.js +158 -0
- package/dist/github.js.map +1 -0
- package/dist/index.d.ts +6 -0
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +3 -0
- package/dist/index.js.map +1 -1
- package/dist/publication.js +28 -40
- package/dist/publication.js.map +1 -1
- package/dist/publish.d.ts +10 -0
- package/dist/publish.d.ts.map +1 -1
- package/dist/publish.js +29 -5
- package/dist/publish.js.map +1 -1
- package/dist/runtime.d.ts +5 -4
- package/dist/runtime.d.ts.map +1 -1
- package/dist/runtime.js +371 -197
- package/dist/runtime.js.map +1 -1
- package/dist/saved-scan.d.ts +15 -0
- package/dist/saved-scan.d.ts.map +1 -0
- package/dist/saved-scan.js +68 -0
- package/dist/saved-scan.js.map +1 -0
- package/dist/scan-comparison.d.ts +6 -2
- package/dist/scan-comparison.d.ts.map +1 -1
- package/dist/scan-comparison.js +7 -2
- package/dist/scan-comparison.js.map +1 -1
- package/dist/server/dashboard/THIRD_PARTY_NOTICES.txt +276 -0
- package/dist/server/dashboard/app.css +2 -0
- package/dist/server/dashboard/app.js +66 -0
- package/dist/server/dashboard/index.html +18 -0
- package/dist/server/dashboard-types.d.ts +45 -0
- package/dist/server/dashboard-types.d.ts.map +1 -0
- package/dist/server/dashboard-types.js +2 -0
- package/dist/server/dashboard-types.js.map +1 -0
- package/dist/server/dashboard.d.ts +6 -0
- package/dist/server/dashboard.d.ts.map +1 -0
- package/dist/server/dashboard.js +45 -0
- package/dist/server/dashboard.js.map +1 -0
- package/dist/server/embeddings.d.ts +16 -0
- package/dist/server/embeddings.d.ts.map +1 -0
- package/dist/server/embeddings.js +114 -0
- package/dist/server/embeddings.js.map +1 -0
- package/dist/server/errors.d.ts +5 -0
- package/dist/server/errors.d.ts.map +1 -0
- package/dist/server/errors.js +8 -0
- package/dist/server/errors.js.map +1 -0
- package/dist/server/findings-service.d.ts +20 -0
- package/dist/server/findings-service.d.ts.map +1 -0
- package/dist/server/findings-service.js +31 -0
- package/dist/server/findings-service.js.map +1 -0
- package/dist/server/index.d.ts +2 -0
- package/dist/server/index.d.ts.map +1 -0
- package/dist/server/index.js +32 -0
- package/dist/server/index.js.map +1 -0
- package/dist/server/routes.d.ts +6 -0
- package/dist/server/routes.d.ts.map +1 -0
- package/dist/server/routes.js +86 -0
- package/dist/server/routes.js.map +1 -0
- package/dist/server/server.d.ts +10 -0
- package/dist/server/server.d.ts.map +1 -0
- package/dist/server/server.js +17 -0
- package/dist/server/server.js.map +1 -0
- package/dist/server/sqlite-store.d.ts +22 -0
- package/dist/server/sqlite-store.d.ts.map +1 -0
- package/dist/server/sqlite-store.js +82 -0
- package/dist/server/sqlite-store.js.map +1 -0
- package/dist/server/storage.d.ts +32 -0
- package/dist/server/storage.d.ts.map +1 -0
- package/dist/server/storage.js +2 -0
- package/dist/server/storage.js.map +1 -0
- package/dist/server/validation.d.ts +17 -0
- package/dist/server/validation.d.ts.map +1 -0
- package/dist/server/validation.js +53 -0
- package/dist/server/validation.js.map +1 -0
- package/dist/targets.d.ts.map +1 -1
- package/dist/targets.js +1 -0
- package/dist/targets.js.map +1 -1
- package/dist/thread-source.d.ts +8 -0
- package/dist/thread-source.d.ts.map +1 -0
- package/dist/thread-source.js +7 -0
- package/dist/thread-source.js.map +1 -0
- package/dist/trusted-executable.d.ts.map +1 -1
- package/dist/trusted-executable.js +2 -1
- package/dist/trusted-executable.js.map +1 -1
- package/dist/version.d.ts +1 -1
- package/dist/version.js +1 -1
- package/dist/version.js.map +1 -1
- package/package.json +23 -8
|
@@ -339,7 +339,27 @@ def parse_args(description: str) -> argparse.Namespace:
|
|
|
339
339
|
publication.add_argument("--input-file", required=True)
|
|
340
340
|
|
|
341
341
|
subparsers.add_parser("database-info")
|
|
342
|
-
|
|
342
|
+
subparsers.add_parser("dashboard")
|
|
343
|
+
subparsers.add_parser("finding-workflow")
|
|
344
|
+
subparsers.add_parser("store-findings")
|
|
345
|
+
subparsers.add_parser("store-dedupe-groups")
|
|
346
|
+
dedupe_groups = subparsers.add_parser("list-dedupe-groups")
|
|
347
|
+
dedupe_groups.add_argument("--finding-id", required=True)
|
|
348
|
+
potential_duplicates = subparsers.add_parser("find-potential-duplicates")
|
|
349
|
+
potential_duplicates.add_argument("--finding-id", required=True)
|
|
350
|
+
scope = potential_duplicates.add_mutually_exclusive_group(required=True)
|
|
351
|
+
scope.add_argument("--repository-id")
|
|
352
|
+
scope.add_argument("--all-repositories", action="store_true")
|
|
353
|
+
stored_findings = subparsers.add_parser("list-stored-findings")
|
|
354
|
+
stored_findings.add_argument("--limit", type=positive_int, required=True)
|
|
355
|
+
stored_findings.add_argument("--offset", type=non_negative_int, required=True)
|
|
356
|
+
arguments = sys.argv[1:]
|
|
357
|
+
if "--user-context-stdin" in arguments:
|
|
358
|
+
if arguments.count("--user-context-stdin") != 1 or "--user-context" in arguments:
|
|
359
|
+
parser.error("pass exactly one user-context transport")
|
|
360
|
+
index = arguments.index("--user-context-stdin")
|
|
361
|
+
arguments[index] = "--user-context=" + sys.stdin.buffer.read().decode("utf-8")
|
|
362
|
+
return parser.parse_args(arguments)
|
|
343
363
|
|
|
344
364
|
|
|
345
365
|
def non_negative_int(value: str) -> int:
|
|
@@ -0,0 +1,115 @@
|
|
|
1
|
+
"""Read-only dashboard projections for stored findings and duplicate groups."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import argparse
|
|
6
|
+
import json
|
|
7
|
+
import sqlite3
|
|
8
|
+
import sys
|
|
9
|
+
from pathlib import Path
|
|
10
|
+
from typing import Any
|
|
11
|
+
|
|
12
|
+
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
|
13
|
+
from workbench_findings import list_dedupe_groups
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
FINDING_RECORDS = """
|
|
17
|
+
SELECT findings.id, json_extract(details_json, '$.title') AS title,
|
|
18
|
+
COALESCE(repositories.ids, '[]') AS repositoryIds,
|
|
19
|
+
json_extract(details_json, '$.severity.level') AS severity,
|
|
20
|
+
findings.created_at AS createdAt, findings.updated_at AS updatedAt
|
|
21
|
+
FROM findings LEFT JOIN (
|
|
22
|
+
SELECT finding_id, json_group_array(repository_id) AS ids
|
|
23
|
+
FROM finding_repositories GROUP BY finding_id
|
|
24
|
+
) AS repositories ON repositories.finding_id = findings.id
|
|
25
|
+
WHERE details_json IS NOT NULL
|
|
26
|
+
"""
|
|
27
|
+
|
|
28
|
+
GROUP_RECORDS = """
|
|
29
|
+
SELECT groups.id, groups.id AS title,
|
|
30
|
+
(SELECT json_group_array(DISTINCT repository_id)
|
|
31
|
+
FROM finding_dedupe_group_members AS members
|
|
32
|
+
JOIN finding_repositories ON finding_repositories.finding_id = members.finding_id
|
|
33
|
+
WHERE members.group_id = groups.id) AS repositoryIds,
|
|
34
|
+
groups.created_at AS createdAt, groups.created_at AS updatedAt,
|
|
35
|
+
(SELECT COUNT(*) FROM finding_dedupe_group_members WHERE group_id = groups.id) AS memberCount
|
|
36
|
+
FROM finding_dedupe_groups AS groups
|
|
37
|
+
"""
|
|
38
|
+
|
|
39
|
+
RECORDS = {
|
|
40
|
+
"findings": FINDING_RECORDS,
|
|
41
|
+
"groups": GROUP_RECORDS,
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def item(row: sqlite3.Row) -> dict[str, Any]:
|
|
46
|
+
result = dict(row)
|
|
47
|
+
result["repositoryIds"] = json.loads(result["repositoryIds"])
|
|
48
|
+
return result
|
|
49
|
+
|
|
50
|
+
|
|
51
|
+
def detail(connection: sqlite3.Connection, view: str, selected: dict[str, Any]) -> dict[str, Any]:
|
|
52
|
+
result: dict[str, Any] = {"item": selected}
|
|
53
|
+
selected_id = selected["id"]
|
|
54
|
+
if view == "findings":
|
|
55
|
+
result["finding"] = json.loads(connection.execute(
|
|
56
|
+
"SELECT details_json FROM findings WHERE id = ?", (selected_id,),
|
|
57
|
+
).fetchone()[0])
|
|
58
|
+
result["groups"] = list_dedupe_groups(connection, selected_id)["groups"]
|
|
59
|
+
else:
|
|
60
|
+
result["group"] = {
|
|
61
|
+
"groupId": selected_id, "createdAt": selected["createdAt"],
|
|
62
|
+
"findingIds": [r[0] for r in connection.execute(
|
|
63
|
+
"SELECT finding_id FROM finding_dedupe_group_members WHERE group_id = ? ORDER BY finding_id",
|
|
64
|
+
(selected_id,),
|
|
65
|
+
)],
|
|
66
|
+
}
|
|
67
|
+
return result
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def dashboard(connection: sqlite3.Connection, query: dict[str, Any]) -> dict[str, Any]:
|
|
71
|
+
"""One snapshot, no artifact reads, model calls, or writes."""
|
|
72
|
+
view = query["view"]
|
|
73
|
+
records = RECORDS[view]
|
|
74
|
+
clauses: list[str] = []
|
|
75
|
+
values: list[Any] = []
|
|
76
|
+
if query.get("query"):
|
|
77
|
+
connection.create_function("casefold", 1, str.casefold, deterministic=True)
|
|
78
|
+
columns = ["id", "title", "repositoryIds"]
|
|
79
|
+
clauses.append("(" + " OR ".join(f"instr(casefold(COALESCE({c}, '')), casefold(?)) > 0" for c in columns) + ")")
|
|
80
|
+
values.extend([query["query"]] * len(columns))
|
|
81
|
+
if query.get("repository"):
|
|
82
|
+
clauses.append("EXISTS (SELECT 1 FROM json_each(repositoryIds) WHERE value = ?)")
|
|
83
|
+
values.append(query["repository"])
|
|
84
|
+
where = " WHERE " + " AND ".join(clauses) if clauses else ""
|
|
85
|
+
order = "createdAt DESC, id" if query["sort"] == "newest" else "updatedAt DESC, id"
|
|
86
|
+
connection.execute("BEGIN")
|
|
87
|
+
with connection:
|
|
88
|
+
repositories = connection.execute("""
|
|
89
|
+
SELECT DISTINCT repository_id AS id, repository_id AS label
|
|
90
|
+
FROM finding_repositories ORDER BY repository_id
|
|
91
|
+
""").fetchall()
|
|
92
|
+
total = connection.execute(f"SELECT COUNT(*) FROM ({records}) {where}", values).fetchone()[0]
|
|
93
|
+
rows = connection.execute(
|
|
94
|
+
f"SELECT * FROM ({records}) {where} ORDER BY {order} LIMIT ? OFFSET ?",
|
|
95
|
+
(*values, query["limit"], query["offset"]),
|
|
96
|
+
).fetchall()
|
|
97
|
+
selected = connection.execute(
|
|
98
|
+
f"SELECT * FROM ({records}) WHERE id = ?", (query["id"],),
|
|
99
|
+
).fetchone() if query.get("id") else None
|
|
100
|
+
next_offset = query["offset"] + len(rows)
|
|
101
|
+
return {
|
|
102
|
+
"overview": {
|
|
103
|
+
"findings": connection.execute("SELECT COUNT(*) FROM findings WHERE details_json IS NOT NULL").fetchone()[0],
|
|
104
|
+
"groups": connection.execute("SELECT COUNT(*) FROM finding_dedupe_groups").fetchone()[0],
|
|
105
|
+
},
|
|
106
|
+
"repositories": [dict(row) for row in repositories],
|
|
107
|
+
"items": [item(row) for row in rows], "total": total,
|
|
108
|
+
"limit": query["limit"], "offset": query["offset"],
|
|
109
|
+
"nextOffset": next_offset if next_offset < total else None,
|
|
110
|
+
"detail": detail(connection, view, item(selected)) if selected is not None else None,
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
|
|
114
|
+
if __name__ == "__main__":
|
|
115
|
+
argparse.ArgumentParser(description=__doc__).parse_args()
|
|
@@ -81,7 +81,16 @@ from workbench_constants import (
|
|
|
81
81
|
SQLITE_RETRY_ATTEMPTS,
|
|
82
82
|
)
|
|
83
83
|
from workbench_feedback import get_scan_feedback
|
|
84
|
+
from workbench_dashboard import dashboard
|
|
84
85
|
from workbench_finding_index import index_findings
|
|
86
|
+
from workbench_finding_workflows import finding_workflow, register_workflow_scan
|
|
87
|
+
from workbench_findings import (
|
|
88
|
+
find_potential_duplicates,
|
|
89
|
+
list_dedupe_groups,
|
|
90
|
+
list_stored_findings,
|
|
91
|
+
store_dedupe_groups,
|
|
92
|
+
store_findings,
|
|
93
|
+
)
|
|
85
94
|
from workbench_remediation import remediation_claim_is_active
|
|
86
95
|
from workbench_scan_start import (
|
|
87
96
|
archive_scan,
|
|
@@ -1651,10 +1660,12 @@ def register_cli_scan(connection: sqlite3.Connection, args: argparse.Namespace)
|
|
|
1651
1660
|
raise SystemExit("The scan artifact directory must be empty before the scan starts.")
|
|
1652
1661
|
|
|
1653
1662
|
user_context = None
|
|
1663
|
+
workflow_id = None
|
|
1654
1664
|
if args.registration_json_stdin:
|
|
1655
1665
|
registration = json.load(sys.stdin)
|
|
1656
1666
|
recipe_json = json.dumps(registration["recipe"], ensure_ascii=False, separators=(",", ":"))
|
|
1657
1667
|
user_context = registration.get("userContext")
|
|
1668
|
+
workflow_id = registration.get("workflowId")
|
|
1658
1669
|
else:
|
|
1659
1670
|
recipe_json = sys.stdin.read() if args.recipe_json_stdin else args.recipe_json
|
|
1660
1671
|
recipe = parse_scan_recipe(recipe_json, repository)
|
|
@@ -1750,6 +1761,8 @@ def register_cli_scan(connection: sqlite3.Connection, args: argparse.Namespace)
|
|
|
1750
1761
|
scan_id,
|
|
1751
1762
|
),
|
|
1752
1763
|
)
|
|
1764
|
+
if workflow_id is not None:
|
|
1765
|
+
register_workflow_scan(connection, workflow_id, scan_id, str(scan_dir), timestamp)
|
|
1753
1766
|
connection.commit()
|
|
1754
1767
|
except BaseException:
|
|
1755
1768
|
connection.rollback()
|
|
@@ -3752,8 +3765,7 @@ def require_canonical_scan_directory(scan_dir: Path) -> Path:
|
|
|
3752
3765
|
scan_dir
|
|
3753
3766
|
):
|
|
3754
3767
|
raise SystemExit("Scan directory must be an existing canonical non-symlink directory.")
|
|
3755
|
-
# Re-check privacy
|
|
3756
|
-
# shared parent cannot substitute another user's forged artifact tree.
|
|
3768
|
+
# Re-check privacy so a shared parent cannot be used to substitute forged artifacts.
|
|
3757
3769
|
if os.name != "nt":
|
|
3758
3770
|
if stat.S_IMODE(metadata.st_mode) & 0o077:
|
|
3759
3771
|
raise SystemExit(
|
|
@@ -4025,6 +4037,21 @@ def main() -> None:
|
|
|
4025
4037
|
result = export_findings(connection, args)
|
|
4026
4038
|
elif args.command == "database-info":
|
|
4027
4039
|
result = {"databasePath": str(database_path())}
|
|
4040
|
+
elif args.command == "finding-workflow":
|
|
4041
|
+
result = finding_workflow(connection, json.load(sys.stdin), now())
|
|
4042
|
+
elif args.command == "dashboard":
|
|
4043
|
+
result = dashboard(connection, json.load(sys.stdin))
|
|
4044
|
+
elif args.command == "store-findings":
|
|
4045
|
+
payload = json.load(sys.stdin)
|
|
4046
|
+
result = store_findings(connection, payload["entries"], now(), payload.get("repositoryId"))
|
|
4047
|
+
elif args.command == "find-potential-duplicates":
|
|
4048
|
+
result = find_potential_duplicates(connection, args.finding_id, args.repository_id)
|
|
4049
|
+
elif args.command == "store-dedupe-groups":
|
|
4050
|
+
result = store_dedupe_groups(connection, json.load(sys.stdin)["groups"], now())
|
|
4051
|
+
elif args.command == "list-dedupe-groups":
|
|
4052
|
+
result = list_dedupe_groups(connection, args.finding_id)
|
|
4053
|
+
elif args.command == "list-stored-findings":
|
|
4054
|
+
result = list_stored_findings(connection, limit=args.limit, offset=args.offset)
|
|
4028
4055
|
else:
|
|
4029
4056
|
raise SystemExit(f"Unknown command: {args.command}")
|
|
4030
4057
|
print(json.dumps(result, allow_nan=False, sort_keys=True))
|
|
@@ -8,6 +8,44 @@ import sqlite3
|
|
|
8
8
|
from typing import Any
|
|
9
9
|
|
|
10
10
|
|
|
11
|
+
def upsert_finding(
|
|
12
|
+
connection: sqlite3.Connection,
|
|
13
|
+
finding: dict[str, Any],
|
|
14
|
+
timestamp: str,
|
|
15
|
+
repository_id: str | None = None,
|
|
16
|
+
) -> None:
|
|
17
|
+
connection.execute(
|
|
18
|
+
"""
|
|
19
|
+
INSERT INTO findings (
|
|
20
|
+
id, fingerprint, rule_id, identity_anchor, identity_instance,
|
|
21
|
+
details_json, created_at, updated_at
|
|
22
|
+
) VALUES (?, ?, ?, ?, ?, ?, ?, ?)
|
|
23
|
+
ON CONFLICT(id) DO UPDATE SET
|
|
24
|
+
fingerprint = excluded.fingerprint,
|
|
25
|
+
rule_id = excluded.rule_id,
|
|
26
|
+
identity_anchor = excluded.identity_anchor,
|
|
27
|
+
identity_instance = excluded.identity_instance,
|
|
28
|
+
details_json = excluded.details_json,
|
|
29
|
+
updated_at = excluded.updated_at
|
|
30
|
+
""",
|
|
31
|
+
(
|
|
32
|
+
finding["findingId"],
|
|
33
|
+
finding["fingerprints"]["primary"],
|
|
34
|
+
finding["ruleId"],
|
|
35
|
+
finding["identity"]["anchor"],
|
|
36
|
+
finding["identity"].get("instance"),
|
|
37
|
+
json.dumps(finding, allow_nan=False, sort_keys=True),
|
|
38
|
+
timestamp,
|
|
39
|
+
timestamp,
|
|
40
|
+
),
|
|
41
|
+
)
|
|
42
|
+
if repository_id is not None:
|
|
43
|
+
connection.execute(
|
|
44
|
+
"INSERT OR IGNORE INTO finding_repositories (repository_id, finding_id) VALUES (?, ?)",
|
|
45
|
+
(repository_id, finding["findingId"]),
|
|
46
|
+
)
|
|
47
|
+
|
|
48
|
+
|
|
11
49
|
def index_findings(
|
|
12
50
|
connection: sqlite3.Connection,
|
|
13
51
|
scan_id: str,
|
|
@@ -17,35 +55,15 @@ def index_findings(
|
|
|
17
55
|
findings = document.get("findings")
|
|
18
56
|
if not isinstance(findings, list):
|
|
19
57
|
raise SystemExit("findings.json must contain a findings array.")
|
|
58
|
+
repository_id = connection.execute(
|
|
59
|
+
"SELECT target_id FROM scans WHERE id = ?", (scan_id,)
|
|
60
|
+
).fetchone()["target_id"]
|
|
20
61
|
for finding in findings:
|
|
21
62
|
if not isinstance(finding, dict):
|
|
22
63
|
raise SystemExit("findings.json entries must be objects.")
|
|
23
|
-
identity = finding["identity"]
|
|
24
|
-
fingerprints = finding["fingerprints"]
|
|
25
64
|
severity = finding["severity"]
|
|
26
65
|
confidence = finding["confidence"]
|
|
27
|
-
connection
|
|
28
|
-
"""
|
|
29
|
-
INSERT INTO findings (
|
|
30
|
-
id, fingerprint, rule_id, identity_anchor, identity_instance, created_at, updated_at
|
|
31
|
-
) VALUES (?, ?, ?, ?, ?, ?, ?)
|
|
32
|
-
ON CONFLICT(id) DO UPDATE SET
|
|
33
|
-
fingerprint = excluded.fingerprint,
|
|
34
|
-
rule_id = excluded.rule_id,
|
|
35
|
-
identity_anchor = excluded.identity_anchor,
|
|
36
|
-
identity_instance = excluded.identity_instance,
|
|
37
|
-
updated_at = excluded.updated_at
|
|
38
|
-
""",
|
|
39
|
-
(
|
|
40
|
-
finding["findingId"],
|
|
41
|
-
fingerprints["primary"],
|
|
42
|
-
finding["ruleId"],
|
|
43
|
-
identity["anchor"],
|
|
44
|
-
identity.get("instance"),
|
|
45
|
-
timestamp,
|
|
46
|
-
timestamp,
|
|
47
|
-
),
|
|
48
|
-
)
|
|
66
|
+
upsert_finding(connection, finding, timestamp, repository_id)
|
|
49
67
|
connection.execute(
|
|
50
68
|
"""
|
|
51
69
|
INSERT INTO finding_occurrences (
|
|
@@ -0,0 +1,190 @@
|
|
|
1
|
+
"""Durable state for the opt-in local scan, publication, and dedupe workflow."""
|
|
2
|
+
|
|
3
|
+
from __future__ import annotations
|
|
4
|
+
|
|
5
|
+
import argparse
|
|
6
|
+
import json
|
|
7
|
+
import hashlib
|
|
8
|
+
import sqlite3
|
|
9
|
+
import sys
|
|
10
|
+
from pathlib import Path
|
|
11
|
+
from typing import Any
|
|
12
|
+
|
|
13
|
+
sys.path.insert(0, str(Path(__file__).resolve().parent))
|
|
14
|
+
from workbench_target import directory_content_digest, git_output, git_revision
|
|
15
|
+
|
|
16
|
+
WORKFLOW_BINDINGS = {
|
|
17
|
+
"repositoryPath": "repository_path",
|
|
18
|
+
"scanRequestDigest": "scan_request_digest",
|
|
19
|
+
"scanId": "scan_id",
|
|
20
|
+
"scanDir": "scan_dir",
|
|
21
|
+
"artifactDigest": "artifact_digest",
|
|
22
|
+
"destination": "destination",
|
|
23
|
+
}
|
|
24
|
+
WORKFLOW_STAGES = ("scan", "publish", "dedupe")
|
|
25
|
+
|
|
26
|
+
|
|
27
|
+
def read_workflow(connection: sqlite3.Connection, workflow_id: str) -> dict[str, Any] | None:
|
|
28
|
+
row = connection.execute(
|
|
29
|
+
"SELECT * FROM finding_workflows WHERE id = ?", (workflow_id,)
|
|
30
|
+
).fetchone()
|
|
31
|
+
if row is None:
|
|
32
|
+
return None
|
|
33
|
+
state = {"id": row["id"], "stages": {}}
|
|
34
|
+
for field, column in WORKFLOW_BINDINGS.items():
|
|
35
|
+
if row[column] is not None:
|
|
36
|
+
state[field] = row[column]
|
|
37
|
+
if row["scope_repository_id"] is not None:
|
|
38
|
+
state["scope"] = {"repositoryId": row["scope_repository_id"]}
|
|
39
|
+
elif row["scope_all_repositories"] is not None:
|
|
40
|
+
state["scope"] = {"allRepositories": bool(row["scope_all_repositories"])}
|
|
41
|
+
results = json.loads(row["results_json"])
|
|
42
|
+
for stage in WORKFLOW_STAGES:
|
|
43
|
+
current = {"status": row[f"{stage}_status"]}
|
|
44
|
+
if row[f"{stage}_error"] is not None:
|
|
45
|
+
current["error"] = row[f"{stage}_error"]
|
|
46
|
+
if stage in results:
|
|
47
|
+
current["result"] = results[stage]
|
|
48
|
+
state["stages"][stage] = current
|
|
49
|
+
if "dedupePendingWrite" in results:
|
|
50
|
+
state["stages"]["dedupe"]["pendingWrite"] = results["dedupePendingWrite"]
|
|
51
|
+
return state
|
|
52
|
+
|
|
53
|
+
|
|
54
|
+
def save_workflow(connection: sqlite3.Connection, state: dict[str, Any], timestamp: str) -> None:
|
|
55
|
+
values = {"id": state["id"]}
|
|
56
|
+
values.update({column: state.get(field) for field, column in WORKFLOW_BINDINGS.items()})
|
|
57
|
+
scope = state.get("scope", {})
|
|
58
|
+
values.update(
|
|
59
|
+
scope_repository_id=scope.get("repositoryId"),
|
|
60
|
+
scope_all_repositories=scope.get("allRepositories"),
|
|
61
|
+
)
|
|
62
|
+
results = {}
|
|
63
|
+
for stage in WORKFLOW_STAGES:
|
|
64
|
+
current = state["stages"][stage]
|
|
65
|
+
values[f"{stage}_status"] = current["status"]
|
|
66
|
+
values[f"{stage}_error"] = current.get("error")
|
|
67
|
+
if "result" in current:
|
|
68
|
+
results[stage] = current["result"]
|
|
69
|
+
if "pendingWrite" in state["stages"]["dedupe"]:
|
|
70
|
+
results["dedupePendingWrite"] = state["stages"]["dedupe"]["pendingWrite"]
|
|
71
|
+
values.update(
|
|
72
|
+
results_json=json.dumps(results, allow_nan=False), created_at=timestamp, updated_at=timestamp
|
|
73
|
+
)
|
|
74
|
+
updates = ", ".join(
|
|
75
|
+
f"{column} = excluded.{column}" for column in values if column not in {"id", "created_at"}
|
|
76
|
+
)
|
|
77
|
+
connection.execute(
|
|
78
|
+
f"INSERT INTO finding_workflows ({', '.join(values)}) "
|
|
79
|
+
f"VALUES ({', '.join('?' for _ in values)}) ON CONFLICT(id) DO UPDATE SET {updates}",
|
|
80
|
+
tuple(values.values()),
|
|
81
|
+
)
|
|
82
|
+
|
|
83
|
+
|
|
84
|
+
def bind_workflow(state: dict[str, Any], binding: dict[str, Any]) -> None:
|
|
85
|
+
for field, value in binding.items():
|
|
86
|
+
if field not in WORKFLOW_BINDINGS and field != "scope":
|
|
87
|
+
raise SystemExit("Unknown workflow binding.")
|
|
88
|
+
if field in state and state[field] != value:
|
|
89
|
+
raise SystemExit(
|
|
90
|
+
f"Workflow {state['id']} is already bound to a different {field}. "
|
|
91
|
+
"Use another --workflow-id."
|
|
92
|
+
)
|
|
93
|
+
state[field] = value
|
|
94
|
+
|
|
95
|
+
|
|
96
|
+
def register_workflow_scan(
|
|
97
|
+
connection: sqlite3.Connection, workflow_id: str, scan_id: str, scan_dir: str, timestamp: str
|
|
98
|
+
) -> None:
|
|
99
|
+
"""Called inside the scan-registration transaction, before model execution."""
|
|
100
|
+
state = read_workflow(connection, workflow_id)
|
|
101
|
+
if state is None:
|
|
102
|
+
raise SystemExit("The workflow must be started before registering its scan.")
|
|
103
|
+
if state["stages"]["scan"]["status"] == "completed":
|
|
104
|
+
raise SystemExit("The workflow scan is already complete.")
|
|
105
|
+
previous = state.get("scanId")
|
|
106
|
+
if previous is not None:
|
|
107
|
+
row = connection.execute("SELECT status FROM scans WHERE id = ?", (previous,)).fetchone()
|
|
108
|
+
if row is not None and row["status"] == "complete":
|
|
109
|
+
raise SystemExit("Reuse the workflow's completed scan instead of registering another.")
|
|
110
|
+
state["scanId"] = scan_id
|
|
111
|
+
state["scanDir"] = scan_dir
|
|
112
|
+
save_workflow(connection, state, timestamp)
|
|
113
|
+
|
|
114
|
+
|
|
115
|
+
def finding_workflow(
|
|
116
|
+
connection: sqlite3.Connection, payload: dict[str, Any], timestamp: str
|
|
117
|
+
) -> dict[str, Any]:
|
|
118
|
+
workflow_id = payload["id"]
|
|
119
|
+
if not isinstance(workflow_id, str) or not workflow_id.strip():
|
|
120
|
+
raise SystemExit("workflowId must be a nonempty string.")
|
|
121
|
+
if payload["action"] == "get":
|
|
122
|
+
return {"workflow": read_workflow(connection, workflow_id)}
|
|
123
|
+
if payload["action"] == "source":
|
|
124
|
+
target = Path(payload["repository"]).resolve(strict=True)
|
|
125
|
+
return {"source": {
|
|
126
|
+
"repository": str(target),
|
|
127
|
+
"revision": git_revision(target),
|
|
128
|
+
"refsDigest": hashlib.sha256((git_output(target, "show-ref") or "").encode()).hexdigest(),
|
|
129
|
+
"content": directory_content_digest(target, include_ignored=True),
|
|
130
|
+
}}
|
|
131
|
+
if payload["action"] == "get-review":
|
|
132
|
+
row = connection.execute(
|
|
133
|
+
"SELECT result_json FROM finding_workflow_reviews WHERE workflow_id = ? AND review_key = ?",
|
|
134
|
+
(workflow_id, payload["key"]),
|
|
135
|
+
).fetchone()
|
|
136
|
+
return {"review": json.loads(row["result_json"]) if row is not None else None}
|
|
137
|
+
if payload["action"] == "save-review":
|
|
138
|
+
binding = payload["binding"]
|
|
139
|
+
source = binding["source"]
|
|
140
|
+
scope = binding["scope"]
|
|
141
|
+
with connection:
|
|
142
|
+
connection.execute(
|
|
143
|
+
"""INSERT INTO finding_workflow_reviews
|
|
144
|
+
(workflow_id, review_key, review_contract_version, codex_version,
|
|
145
|
+
source_repository_path, source_revision, source_refs_digest, source_content_digest,
|
|
146
|
+
scope_repository_id, scope_all_repositories, model, effort, settings_digest,
|
|
147
|
+
prompt_digest, contract_digest, result_json, created_at)
|
|
148
|
+
VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)
|
|
149
|
+
ON CONFLICT(workflow_id, review_key) DO NOTHING""",
|
|
150
|
+
(
|
|
151
|
+
workflow_id, payload["key"], binding["version"], binding["codexVersion"],
|
|
152
|
+
source["repository"], source["revision"], source["refsDigest"], source["content"],
|
|
153
|
+
scope.get("repositoryId"), scope.get("allRepositories"), binding["model"],
|
|
154
|
+
binding["effort"], binding.get("settingsDigest"), binding["promptDigest"],
|
|
155
|
+
binding["contractDigest"], json.dumps(payload["result"], allow_nan=False), timestamp,
|
|
156
|
+
),
|
|
157
|
+
)
|
|
158
|
+
return {}
|
|
159
|
+
connection.execute("BEGIN IMMEDIATE")
|
|
160
|
+
with connection:
|
|
161
|
+
state = read_workflow(connection, workflow_id)
|
|
162
|
+
if state is None:
|
|
163
|
+
state = {
|
|
164
|
+
"id": workflow_id,
|
|
165
|
+
"stages": {stage: {"status": "pending"} for stage in WORKFLOW_STAGES},
|
|
166
|
+
}
|
|
167
|
+
bind_workflow(state, payload.get("binding", {}))
|
|
168
|
+
action = payload["action"]
|
|
169
|
+
if action != "bind":
|
|
170
|
+
stage = payload["stage"]
|
|
171
|
+
if stage not in state["stages"]:
|
|
172
|
+
raise SystemExit("Unknown workflow stage.")
|
|
173
|
+
current = state["stages"][stage]
|
|
174
|
+
if current["status"] != "completed":
|
|
175
|
+
if action == "begin":
|
|
176
|
+
current["status"] = "running"
|
|
177
|
+
elif action == "complete":
|
|
178
|
+
state["stages"][stage] = {"status": "completed", "result": payload["result"]}
|
|
179
|
+
elif action == "fail":
|
|
180
|
+
current.update(status="failed", error=payload["error"])
|
|
181
|
+
elif action == "prepare-dedupe":
|
|
182
|
+
current.update(result=payload["result"], pendingWrite=payload["pendingWrite"])
|
|
183
|
+
else:
|
|
184
|
+
raise SystemExit("Unknown workflow action.")
|
|
185
|
+
save_workflow(connection, state, timestamp)
|
|
186
|
+
return {"workflow": state}
|
|
187
|
+
|
|
188
|
+
|
|
189
|
+
if __name__ == "__main__":
|
|
190
|
+
argparse.ArgumentParser(description=__doc__).parse_args()
|