@openai/codex-security 0.1.24 → 0.1.26
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +399 -27
- package/_bundled_plugin/.app.json +0 -5
- package/_bundled_plugin/.codex-plugin/plugin.json +1 -1
- package/_bundled_plugin/.mcp.json +1 -0
- package/_bundled_plugin/mcp/server.mjs.br.part-000 +0 -0
- package/_bundled_plugin/mcp/server.mjs.br.part-001 +0 -0
- package/_bundled_plugin/references/config-preflight.md +2 -2
- package/_bundled_plugin/references/final-report.md +5 -3
- package/_bundled_plugin/references/scan-artifacts.md +5 -1
- package/_bundled_plugin/references/scan-contract.md +7 -3
- package/_bundled_plugin/schemas/tools/deep-reducer.schema.json +27 -1
- package/_bundled_plugin/scripts/deep_scan_workbench.py +6 -1
- package/_bundled_plugin/scripts/finalize_scan_contract.py +2 -5
- package/_bundled_plugin/scripts/rank_preview.py +11 -1
- package/_bundled_plugin/scripts/report_projection.py +1 -1
- package/_bundled_plugin/scripts/workbench_cli.py +11 -1
- package/_bundled_plugin/scripts/workbench_db.py +55 -4
- package/_bundled_plugin/scripts/workbench_saved_results.py +41 -22
- package/_bundled_plugin/scripts/workbench_scan_history.py +374 -66
- package/_bundled_plugin/scripts/workbench_scan_usage.py +7 -6
- package/_bundled_plugin/scripts/workbench_schema.py +46 -0
- package/_bundled_plugin/scripts/workbench_severity.py +119 -0
- package/_bundled_plugin/scripts/workbench_target.py +86 -12
- package/_bundled_plugin/skills/deep-security-scan/SKILL.md +5 -3
- package/_bundled_plugin/skills/verify-fix/SKILL.md +5 -1
- package/dist/api.d.ts +15 -2
- package/dist/api.d.ts.map +1 -1
- package/dist/api.js +333 -32
- package/dist/api.js.map +1 -1
- package/dist/auth.d.ts.map +1 -1
- package/dist/auth.js +35 -3
- package/dist/auth.js.map +1 -1
- package/dist/classify-scan-severity.d.ts +16 -0
- package/dist/classify-scan-severity.d.ts.map +1 -0
- package/dist/classify-scan-severity.js +96 -0
- package/dist/classify-scan-severity.js.map +1 -0
- package/dist/classify-severity.d.ts +38 -0
- package/dist/classify-severity.d.ts.map +1 -0
- package/dist/classify-severity.js +185 -0
- package/dist/classify-severity.js.map +1 -0
- package/dist/cli.d.ts +7 -2
- package/dist/cli.d.ts.map +1 -1
- package/dist/cli.js +403 -116
- package/dist/cli.js.map +1 -1
- package/dist/component-plan.d.ts.map +1 -1
- package/dist/component-plan.js +4 -2
- package/dist/component-plan.js.map +1 -1
- package/dist/component-scan.d.ts.map +1 -1
- package/dist/component-scan.js +21 -6
- package/dist/component-scan.js.map +1 -1
- package/dist/config.d.ts.map +1 -1
- package/dist/config.js +46 -2
- package/dist/config.js.map +1 -1
- package/dist/cost.d.ts +1 -0
- package/dist/cost.d.ts.map +1 -1
- package/dist/cost.js +43 -17
- package/dist/cost.js.map +1 -1
- package/dist/deduplication/checkpointed-review.d.ts.map +1 -1
- package/dist/deduplication/checkpointed-review.js +2 -1
- package/dist/deduplication/checkpointed-review.js.map +1 -1
- package/dist/deduplication/codex-review.d.ts +3 -0
- package/dist/deduplication/codex-review.d.ts.map +1 -1
- package/dist/deduplication/codex-review.js +122 -36
- package/dist/deduplication/codex-review.js.map +1 -1
- package/dist/deduplication/deduplication-prompts.d.ts +4 -3
- package/dist/deduplication/deduplication-prompts.d.ts.map +1 -1
- package/dist/deduplication/deduplication-prompts.js +18 -9
- package/dist/deduplication/deduplication-prompts.js.map +1 -1
- package/dist/deduplication/deduplication-reviewer.d.ts +2 -7
- package/dist/deduplication/deduplication-reviewer.d.ts.map +1 -1
- package/dist/deduplication/deduplication-reviewer.js +47 -25
- package/dist/deduplication/deduplication-reviewer.js.map +1 -1
- package/dist/deduplication/deduplication.d.ts.map +1 -1
- package/dist/deduplication/deduplication.js +186 -23
- package/dist/deduplication/deduplication.js.map +1 -1
- package/dist/deduplication/scan.d.ts +9 -1
- package/dist/deduplication/scan.d.ts.map +1 -1
- package/dist/deduplication/scan.js +24 -7
- package/dist/deduplication/scan.js.map +1 -1
- package/dist/errors.d.ts +13 -0
- package/dist/errors.d.ts.map +1 -1
- package/dist/errors.js +7 -0
- package/dist/errors.js.map +1 -1
- package/dist/finding-catalogue.d.ts +11 -0
- package/dist/finding-catalogue.d.ts.map +1 -0
- package/dist/finding-catalogue.js +150 -0
- package/dist/finding-catalogue.js.map +1 -0
- package/dist/index.d.ts +11 -5
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +5 -2
- package/dist/index.js.map +1 -1
- package/dist/mock-scan.d.ts +4 -0
- package/dist/mock-scan.d.ts.map +1 -0
- package/dist/mock-scan.js +210 -0
- package/dist/mock-scan.js.map +1 -0
- package/dist/patch-tui.js +1 -1
- package/dist/patch-tui.js.map +1 -1
- package/dist/publication-store.d.ts.map +1 -1
- package/dist/publication-store.js +9 -4
- package/dist/publication-store.js.map +1 -1
- package/dist/publication.d.ts +9 -0
- package/dist/publication.d.ts.map +1 -1
- package/dist/publication.js +45 -5
- package/dist/publication.js.map +1 -1
- package/dist/publish.d.ts +4 -2
- package/dist/publish.d.ts.map +1 -1
- package/dist/publish.js +4 -4
- package/dist/publish.js.map +1 -1
- package/dist/runtime.d.ts +1 -0
- package/dist/runtime.d.ts.map +1 -1
- package/dist/runtime.js +49 -14
- package/dist/runtime.js.map +1 -1
- package/dist/scan-comparison.d.ts +44 -30
- package/dist/scan-comparison.d.ts.map +1 -1
- package/dist/scan-comparison.js +673 -101
- package/dist/scan-comparison.js.map +1 -1
- package/dist/scan-dashboard.d.ts +3 -1
- package/dist/scan-dashboard.d.ts.map +1 -1
- package/dist/scan-dashboard.js +70 -3
- package/dist/scan-dashboard.js.map +1 -1
- package/dist/scan-history-renderer.d.ts.map +1 -1
- package/dist/scan-history-renderer.js +26 -0
- package/dist/scan-history-renderer.js.map +1 -1
- package/dist/server/api.d.ts +6 -0
- package/dist/server/api.d.ts.map +1 -0
- package/dist/server/api.js +4 -0
- package/dist/server/api.js.map +1 -0
- package/dist/server/dashboard/app.css +1 -1
- package/dist/server/embeddings.d.ts +2 -1
- package/dist/server/embeddings.d.ts.map +1 -1
- package/dist/server/embeddings.js +8 -3
- package/dist/server/embeddings.js.map +1 -1
- package/dist/server/serve.d.ts.map +1 -1
- package/dist/server/serve.js +1 -1
- package/dist/server/serve.js.map +1 -1
- package/dist/severity-store.d.ts +2 -0
- package/dist/severity-store.d.ts.map +1 -0
- package/dist/severity-store.js +111 -0
- package/dist/severity-store.js.map +1 -0
- package/dist/targets.d.ts.map +1 -1
- package/dist/targets.js +3 -5
- package/dist/targets.js.map +1 -1
- package/dist/thread-source.d.ts +1 -0
- package/dist/thread-source.d.ts.map +1 -1
- package/dist/thread-source.js +1 -0
- package/dist/thread-source.js.map +1 -1
- package/dist/version.d.ts +1 -1
- package/dist/version.js +1 -1
- package/dist/version.js.map +1 -1
- package/package.json +11 -6
|
@@ -16,11 +16,11 @@ Use `report.md` as the primary readable entry point. Explain report-relevant art
|
|
|
16
16
|
|
|
17
17
|
In the final response, link the generated markdown report path as the primary readable artifact.
|
|
18
18
|
|
|
19
|
-
Every scan mode uses the same final report pipeline. Workbench-owned Standard and workbench-backed diff scans submit canonical semantics with `record_codex_security_scan_draft({ scanId, handoffClaimToken?, scope?, threatModel?, findings, coverage })`; the workbench supplies authoritative metadata and writes the unsealed canonical draft. Deep
|
|
19
|
+
Every scan mode uses the same final report pipeline. Workbench-owned Standard and workbench-backed diff scans submit canonical semantics with `record_codex_security_scan_draft({ scanId, handoffClaimToken?, scope?, threatModel?, findings, coverage })`; the workbench supplies authoritative metadata and writes the unsealed canonical draft. For Deep scans, the coordinator writes the parent scan's unsealed canonical draft from its accepted aggregate. SDK-owned Standard scans instead write unsealed canonical files with the exact SDK-provided metadata and leave finalization to the SDK. Terminal scans without a `scanId` retain their existing canonical JSON workflow. No mode authors, repairs, or treats an existing `report.md` as input. Finalization validates and enriches the canonical JSON, seals the canonical JSON and evidence artifacts, then deterministically generates `report.md`. Supply report prose through structured canonical semantics rather than a separately authored report.
|
|
20
20
|
|
|
21
21
|
For each finding, supply an evidence-supported lowercase vulnerability-family `ruleId`; `taxonomy: { category, cwe }` using its exact known CWEs; verified locations; and `provenance.source`, using `"local_plugin"` only when this plugin actually discovered the finding. Preserve genuine worker or source provenance and any existing canonical candidate identity in the finding extensions. A finding with no known CWE retains `cwe: []`; never invent a classification. Include optional `codeEvidence` only when its actual code is nonempty and every referenced evidence ID is present.
|
|
22
22
|
|
|
23
|
-
|
|
23
|
+
For Standard scan drafts, including Deep worker results, and diff drafts, supply semantic coverage as `{ completeness, surfaces, explicitExclusions, deferred }`, with each surface using the actual `label` and one existing `disposition`. Mark coverage `partial` when a deferred item or `needs_follow_up` surface remains; preserve its real reason and supporting context. Each deferred item needs a meaningful reason; preserve any existing `id` or `candidateId`. The workbench derives a missing ID from its candidate identity or stable deferred-work details. Open questions may be nonempty strings or `{ question, followUpPrompt? }` objects. The workbench derives target and scope metadata, scope include and exclude paths, coverage mode and inventory strategy, finding identities and fingerprints, and surface IDs. Do not put those workbench-owned values or top-level coverage receipt references into the semantic draft.
|
|
24
24
|
|
|
25
25
|
During a host-backed scan, checkpoint saved findings and pending candidates with `complete: false`; a checkpoint is not a completed audit. After a final workbench-owned Standard or workbench-backed diff draft is accepted with `complete: true` (or the backwards-compatible omitted flag), or the Deep coordinator returns its parent scan's canonical manifest, call `complete_codex_security_scan({ scanId, handoffClaimToken? })` and use its returned completion metadata. An SDK-owned scan returns its unsealed canonical files without calling a completion tool or finalizer; the SDK owns completion and report generation. Read full canonical results only when explicitly requested. For a terminal/chat workflow without a `scanId` or completion tool, retain `python <plugin_dir>/scripts/finalize_scan_contract.py --scan-dir <scan_dir> --source-root <repo_root>` after writing the canonical JSON. Outside the SDK path, do not mark the scan goal complete until finalization succeeds and the generated report exists.
|
|
26
26
|
|
|
@@ -59,7 +59,7 @@ Group observations only when they share the same broken security control and eff
|
|
|
59
59
|
|
|
60
60
|
Set the finding category and CWE from the primary broken control. Do not add secondary support-impact CWEs, such as data exposure or missing authentication, to an injection/RCE/path/file/parser finding merely because they make exploitation worse; mention those impacts in prose or emit a separate finding if that secondary control is independently vulnerable.
|
|
61
61
|
|
|
62
|
-
|
|
62
|
+
Canonical `severity.changeConditions` must be one non-empty string; when `attack_path.change_conditions` contains multiple strings, join them into one prose string before writing `findings.json`.
|
|
63
63
|
|
|
64
64
|
For workbench-backed diff candidates, apply row outcomes in this order: validation disposition `reportable` plus attack-path decision `reportable` becomes a finding with its distinct instance and all relevant entrypoint, root-control, sink, and supporting locations; otherwise, a `deferred` result from either phase becomes `needs_follow_up` coverage and a `coverage.deferred` entry using the recorded uncertainty or proof gap; otherwise, validation disposition `not_applicable` becomes `not_applicable` coverage; otherwise, validation disposition `suppressed` or attack-path decision `ignore` becomes `rejected` coverage. A missing required phase record leaves the candidate unresolved and prevents complete coverage. Do not require phase receipts, per-candidate narratives, or another reconciliation pass.
|
|
65
65
|
|
|
@@ -73,6 +73,8 @@ Use this report structure:
|
|
|
73
73
|
|
|
74
74
|
`## Scope`
|
|
75
75
|
|
|
76
|
+
Deep reports present the configured include and exclude paths and execution outcome alongside aggregated findings, with their validation, attack paths, affected locations, and any supplied scope and threat-model context. When the discovery time limit expires before any source review completes, the report explains the incomplete review.
|
|
77
|
+
|
|
76
78
|
Populate `scan.scope` with in-scope context, artifacts reviewed, runtime or test status, validation mode, and explicit limitations. Include/exclude paths and coverage fields supply the remaining projected scope content. If the threat model was generated during Phase 1 rather than provided by the user, say that in canonical scope context. Do not call generated threat-model material an external input.
|
|
77
79
|
|
|
78
80
|
After the scope bullets, include a compact `### Scan Summary` table when the scan has findings or repository-wide coverage. Use columns `Field` and `Value`. Include the count of reportable findings, severity mix, confidence mix, coverage, and validation mode when those values are known. Keep artifact paths below this table.
|
|
@@ -40,7 +40,11 @@ End each repository-scoped threat model with these two lines:
|
|
|
40
40
|
|
|
41
41
|
### Compact Deep And Workbench-Backed Diff Discovery
|
|
42
42
|
|
|
43
|
-
Workbench-owned Standard scans submit findings and coverage through `record_codex_security_scan_draft`; SDK-owned Standard scans write unsealed canonical files directly.
|
|
43
|
+
Workbench-owned Standard scans submit findings and coverage through `record_codex_security_scan_draft`; SDK-owned Standard scans write unsealed canonical files directly. Workbench-backed diff scans use the compact artifacts described below.
|
|
44
|
+
|
|
45
|
+
Deep scans run ordinary Standard scan workers. Workers save progress with `complete: false` and submit final results through `record_codex_security_scan_draft`. Their checkpoints and results contain findings and coverage, with optional scope and threat-model context. Pending candidates and their evidence are recorded in worker coverage. Immutable checkpoints store progress across retries, cancellation, and failure. The coordinator passes completed workers' findings and context to the reducer.
|
|
46
|
+
|
|
47
|
+
Deep reducer inputs, results, and checkpoints contain findings and optional scope and threat-model context. The host writes the parent scan's unsealed `scan-manifest.json` and `findings.json` from the accepted aggregate, and derives `coverage.json` from the configured include and exclude paths and the coordinator's outcome. The parent completes the scan from these artifacts. If the discovery time limit expires before any source review completes, the parent records partial coverage with that reason. See `scan-contract.md` for canonical field definitions.
|
|
44
48
|
|
|
45
49
|
- A workbench-backed diff scan records all candidates once with `record_codex_security_discovery_candidates({ scanId, candidates })` and reads the canonical candidates with `list_codex_security_candidates({ scanId, cursor?, limit? })`.
|
|
46
50
|
- The writer validates candidates against assigned source paths, merges rows with the same CWE ids, locations, and optional instance, preserves their text, and assigns deterministic `candidate_id` values.
|
|
@@ -104,9 +104,13 @@ Use CWE taxonomy separately. Do not include file names, line numbers, scan IDs,
|
|
|
104
104
|
|
|
105
105
|
## Coverage
|
|
106
106
|
|
|
107
|
-
`coverage.json`
|
|
107
|
+
`coverage.json` records scan scope and completion information. Standard and diff summaries also describe reviewed surfaces and outstanding work.
|
|
108
108
|
|
|
109
|
-
|
|
109
|
+
For a Deep parent scan, the host copies the configured paths into `includePaths` and `excludePaths` and sets `completeness` from the coordinator's outcome. A successful aggregate uses `complete`; `surfaces`, `explicitExclusions`, and `deferred` are empty arrays, and `openQuestions` is omitted. If the configured time limit expires before any review completes, the coordinator writes `partial` and records the explanation in `deferred`. Stopped outcomes follow the [stopped-result recovery rules](#stopped-result-recovery).
|
|
110
|
+
|
|
111
|
+
Each Deep worker writes an ordinary Standard result, including its own coverage. A reducer submits `record_codex_security_deep_reduction({ scanId, findings, scope?, threatModel? })`; its saved results and checkpoints contain the accepted findings and optional scope and threat-model context.
|
|
112
|
+
|
|
113
|
+
For Standard and diff scans, record:
|
|
110
114
|
|
|
111
115
|
- scan mode and inventory strategy
|
|
112
116
|
- included and excluded paths
|
|
@@ -140,7 +144,7 @@ For a whole-repository Deep scan, keep `inventoryStrategy` as `repository`; repe
|
|
|
140
144
|
| `directory` | Deterministic non-Git directory inventory |
|
|
141
145
|
| `custom` | Producer-defined inventory described by detailed receipts |
|
|
142
146
|
|
|
143
|
-
|
|
147
|
+
For Standard and diff scans, use `complete` when the requested scope was fully reviewed, `partial` when in-scope work was deferred, and `unknown` when the producer cannot establish enough coverage to make that distinction.
|
|
144
148
|
|
|
145
149
|
Map detailed ledger closure into completed surface summaries in this order:
|
|
146
150
|
|
|
@@ -9,7 +9,33 @@
|
|
|
9
9
|
"additionalProperties": false
|
|
10
10
|
},
|
|
11
11
|
"reductionInput": {
|
|
12
|
-
"
|
|
12
|
+
"type": "object",
|
|
13
|
+
"properties": {
|
|
14
|
+
"complete": {
|
|
15
|
+
"type": "boolean",
|
|
16
|
+
"description": "Omit or set true when submitting the finished aggregate."
|
|
17
|
+
},
|
|
18
|
+
"scanId": {
|
|
19
|
+
"$ref": "codex-security://schemas/tools/scan-draft.schema.json#/$defs/scanId"
|
|
20
|
+
},
|
|
21
|
+
"handoffClaimToken": {
|
|
22
|
+
"$ref": "codex-security://schemas/tools/scan-draft.schema.json#/$defs/handoffClaimToken"
|
|
23
|
+
},
|
|
24
|
+
"scope": {
|
|
25
|
+
"$ref": "codex-security://schemas/tools/scan-draft.schema.json#/$defs/scope"
|
|
26
|
+
},
|
|
27
|
+
"threatModel": {
|
|
28
|
+
"$ref": "codex-security://schemas/tools/scan-draft.schema.json#/$defs/threatModel"
|
|
29
|
+
},
|
|
30
|
+
"findings": {
|
|
31
|
+
"$ref": "codex-security://schemas/tools/scan-draft.schema.json#/$defs/scanDraftInput/properties/findings"
|
|
32
|
+
}
|
|
33
|
+
},
|
|
34
|
+
"required": [
|
|
35
|
+
"scanId",
|
|
36
|
+
"findings"
|
|
37
|
+
],
|
|
38
|
+
"additionalProperties": false
|
|
13
39
|
}
|
|
14
40
|
}
|
|
15
41
|
}
|
|
@@ -1877,6 +1877,7 @@ def finish_deep_scan_locked(
|
|
|
1877
1877
|
"""
|
|
1878
1878
|
SELECT 1 FROM deep_scan_workers AS failed
|
|
1879
1879
|
WHERE failed.scan_id = ? AND failed.status = 'failed'
|
|
1880
|
+
AND (? != 'saturated' OR failed.kind != 'discovery')
|
|
1880
1881
|
AND (
|
|
1881
1882
|
failed.kind != 'dedup'
|
|
1882
1883
|
OR NOT EXISTS (
|
|
@@ -1902,10 +1903,14 @@ def finish_deep_scan_locked(
|
|
|
1902
1903
|
)
|
|
1903
1904
|
LIMIT 1
|
|
1904
1905
|
""",
|
|
1905
|
-
(scan_id,),
|
|
1906
|
+
(scan_id, args.terminal_reason),
|
|
1906
1907
|
).fetchone()
|
|
1907
1908
|
if failed_worker is not None and not failure_capped:
|
|
1908
1909
|
raise SystemExit("Deep Scan cannot finish after a worker has failed.")
|
|
1910
|
+
if args.terminal_reason == "saturated":
|
|
1911
|
+
# Mark any remaining workers canceled, including those whose own
|
|
1912
|
+
# cancellation writes failed, so they cannot block completion.
|
|
1913
|
+
cancel_active_workers(connection, scan_id, now())
|
|
1909
1914
|
active_worker = connection.execute(
|
|
1910
1915
|
"""
|
|
1911
1916
|
SELECT 1 FROM deep_scan_workers
|
|
@@ -1252,12 +1252,9 @@ def _normalize_unsealed_deep_repository_inventory_strategy(
|
|
|
1252
1252
|
*,
|
|
1253
1253
|
expected_coverage_mode: str | None,
|
|
1254
1254
|
) -> None:
|
|
1255
|
-
"""
|
|
1255
|
+
"""Label whole-repository Deep scans as using the repository inventory."""
|
|
1256
1256
|
|
|
1257
|
-
if
|
|
1258
|
-
expected_coverage_mode == "deep_repository"
|
|
1259
|
-
and coverage.get("inventoryStrategy") == "deep_repository_repeated_discovery"
|
|
1260
|
-
):
|
|
1257
|
+
if expected_coverage_mode == "deep_repository":
|
|
1261
1258
|
coverage["inventoryStrategy"] = "repository"
|
|
1262
1259
|
|
|
1263
1260
|
|
|
@@ -11,6 +11,7 @@ from pathlib import Path
|
|
|
11
11
|
DEFAULT_PREVIEW_BYTES = 1024
|
|
12
12
|
PREVIEW_HEAD_LINES = 12
|
|
13
13
|
PREVIEW_SAMPLE_LINES = 10
|
|
14
|
+
_UTF16_BOMS = (b"\xff\xfe", b"\xfe\xff")
|
|
14
15
|
|
|
15
16
|
TEXT_CODE_EXTENSIONS = {
|
|
16
17
|
".c",
|
|
@@ -43,6 +44,8 @@ TEXT_CODE_EXTENSIONS = {
|
|
|
43
44
|
".php",
|
|
44
45
|
".proto",
|
|
45
46
|
".ps1",
|
|
47
|
+
".psd1",
|
|
48
|
+
".psm1",
|
|
46
49
|
".py",
|
|
47
50
|
".rb",
|
|
48
51
|
".rs",
|
|
@@ -79,7 +82,14 @@ RUST_LIFETIME_RE = re.compile(r"'[A-Za-z_][A-Za-z0-9_]*")
|
|
|
79
82
|
PHP_HEREDOC_RE = re.compile(r"<<<\s*['\"]?([A-Za-z_]\w*)['\"]?")
|
|
80
83
|
|
|
81
84
|
|
|
85
|
+
def _decode_source(data: bytes) -> str:
|
|
86
|
+
encoding = "utf-16" if data.startswith(_UTF16_BOMS) else "utf-8"
|
|
87
|
+
return data.decode(encoding, errors="ignore")
|
|
88
|
+
|
|
89
|
+
|
|
82
90
|
def is_binary_sample(data: bytes) -> bool:
|
|
91
|
+
if data.startswith(_UTF16_BOMS):
|
|
92
|
+
return "\0" in _decode_source(data)
|
|
83
93
|
return b"\0" in data
|
|
84
94
|
|
|
85
95
|
|
|
@@ -956,7 +966,7 @@ def preview_for(
|
|
|
956
966
|
def preview_for_bytes(path: Path, data: bytes, preview_bytes: int) -> tuple[str, bool]:
|
|
957
967
|
if is_binary_sample(data):
|
|
958
968
|
return "", True
|
|
959
|
-
text = data
|
|
969
|
+
text = _decode_source(data)
|
|
960
970
|
outline = structural_outline(path, text)
|
|
961
971
|
preview_lines = select_preview_lines(outline or text.splitlines())
|
|
962
972
|
return fit_preview_lines(preview_lines, preview_bytes), False
|
|
@@ -814,7 +814,7 @@ def build_report_markdown(
|
|
|
814
814
|
"",
|
|
815
815
|
_text(
|
|
816
816
|
scope.get("summary"),
|
|
817
|
-
"The scan
|
|
817
|
+
"The scan was configured for the include paths and exclusions listed below.",
|
|
818
818
|
),
|
|
819
819
|
"",
|
|
820
820
|
f"- Scan mode: {coverage['mode']}",
|
|
@@ -160,6 +160,10 @@ def parse_args(description: str) -> argparse.Namespace:
|
|
|
160
160
|
set_scan_thread.add_argument("--scan-id", required=True)
|
|
161
161
|
set_scan_thread.add_argument("--thread-id", required=True)
|
|
162
162
|
|
|
163
|
+
set_scan_cost_limit = subparsers.add_parser("set-scan-cost-limit")
|
|
164
|
+
set_scan_cost_limit.add_argument("--scan-id", required=True)
|
|
165
|
+
set_scan_cost_limit.add_argument("--max-cost-usd", required=True, type=float)
|
|
166
|
+
|
|
163
167
|
get_scan_recipe = subparsers.add_parser("get-scan-recipe")
|
|
164
168
|
get_scan_recipe.add_argument("--scan-id", required=True)
|
|
165
169
|
|
|
@@ -169,7 +173,10 @@ def parse_args(description: str) -> argparse.Namespace:
|
|
|
169
173
|
compare_scans.add_argument("--include-matching-inputs", action="store_true")
|
|
170
174
|
compare_scans.add_argument("--require-matches", action="store_true")
|
|
171
175
|
|
|
172
|
-
save_scan_comparison = subparsers.add_parser(
|
|
176
|
+
save_scan_comparison = subparsers.add_parser(
|
|
177
|
+
"save-scan-comparison",
|
|
178
|
+
description="Comparison payload supports related findings.",
|
|
179
|
+
)
|
|
173
180
|
save_scan_comparison.add_argument("--before-scan-id", required=True)
|
|
174
181
|
save_scan_comparison.add_argument("--after-scan-id", required=True)
|
|
175
182
|
matches = save_scan_comparison.add_mutually_exclusive_group(required=True)
|
|
@@ -347,6 +354,9 @@ def parse_args(description: str) -> argparse.Namespace:
|
|
|
347
354
|
subparsers.add_parser("database-info")
|
|
348
355
|
subparsers.add_parser("dashboard")
|
|
349
356
|
subparsers.add_parser("finding-workflow")
|
|
357
|
+
subparsers.add_parser("severity-classification")
|
|
358
|
+
severity = subparsers.add_parser("read-severity-classification")
|
|
359
|
+
severity.add_argument("--scan-id", required=True)
|
|
350
360
|
subparsers.add_parser("store-findings")
|
|
351
361
|
subparsers.add_parser("store-dedupe-groups")
|
|
352
362
|
dedupe_groups = subparsers.add_parser("list-dedupe-groups")
|
|
@@ -7,6 +7,7 @@ import argparse
|
|
|
7
7
|
import errno
|
|
8
8
|
import hashlib
|
|
9
9
|
import json
|
|
10
|
+
import math
|
|
10
11
|
import os
|
|
11
12
|
import re
|
|
12
13
|
import sqlite3
|
|
@@ -39,6 +40,7 @@ import workbench_remediation as remediation
|
|
|
39
40
|
import workbench_saved_results as saved_results
|
|
40
41
|
import workbench_scan_history as scan_history
|
|
41
42
|
import workbench_scan_usage as scan_usage
|
|
43
|
+
import workbench_severity as severity
|
|
42
44
|
from filesystem_identity import (
|
|
43
45
|
serialize_filesystem_identity as serialize_filesystem_identity,
|
|
44
46
|
)
|
|
@@ -1534,7 +1536,9 @@ def complete_scan_locked(
|
|
|
1534
1536
|
scan_dir,
|
|
1535
1537
|
expected_coverage_mode=expected_coverage_mode(scan),
|
|
1536
1538
|
completion_binding=completion_binding,
|
|
1537
|
-
|
|
1539
|
+
# Save the finished Deep result as submitted. Worker drafts and
|
|
1540
|
+
# recovery repairs belong to the stopped-scan path.
|
|
1541
|
+
completion_warnings=warnings if scan["mode"] != "deep" else None,
|
|
1538
1542
|
draft_documents=saved_results.merge_saved_results(
|
|
1539
1543
|
scan_dir,
|
|
1540
1544
|
scan["id"],
|
|
@@ -1547,7 +1551,7 @@ def complete_scan_locked(
|
|
|
1547
1551
|
stopped=False,
|
|
1548
1552
|
reason="",
|
|
1549
1553
|
)
|
|
1550
|
-
if current_manifest_path is not None and not already_sealed
|
|
1554
|
+
if scan["mode"] != "deep" and current_manifest_path is not None and not already_sealed
|
|
1551
1555
|
else None,
|
|
1552
1556
|
)
|
|
1553
1557
|
add_warning()
|
|
@@ -1789,6 +1793,31 @@ def set_scan_thread(connection: sqlite3.Connection, args: argparse.Namespace) ->
|
|
|
1789
1793
|
return {"scanId": scan["id"], "threadId": args.thread_id}
|
|
1790
1794
|
|
|
1791
1795
|
|
|
1796
|
+
def set_scan_cost_limit(connection: sqlite3.Connection, args: argparse.Namespace) -> dict[str, Any]:
|
|
1797
|
+
scan_id = require_uuid(args.scan_id, "scan-id")
|
|
1798
|
+
limit = args.max_cost_usd
|
|
1799
|
+
if not math.isfinite(limit) or limit <= 0:
|
|
1800
|
+
raise SystemExit("The scan cost limit must be a positive finite USD amount.")
|
|
1801
|
+
with scan_completion_lock(scan_id), connection:
|
|
1802
|
+
scan = require_scan(connection, scan_id)
|
|
1803
|
+
if scan["status"] != "running" or scan["recipe_json"] is None:
|
|
1804
|
+
raise SystemExit("Only a running CLI scan can increase its cost limit.")
|
|
1805
|
+
recipe = json.loads(scan["recipe_json"], parse_constant=reject_non_finite_json)
|
|
1806
|
+
previous = recipe.get("maxCostUsd")
|
|
1807
|
+
if (
|
|
1808
|
+
not isinstance(previous, (int, float))
|
|
1809
|
+
or isinstance(previous, bool)
|
|
1810
|
+
or limit <= previous
|
|
1811
|
+
):
|
|
1812
|
+
raise SystemExit("The new cost limit must exceed the current limit.")
|
|
1813
|
+
recipe["maxCostUsd"] = limit
|
|
1814
|
+
connection.execute(
|
|
1815
|
+
"UPDATE scans SET recipe_json = ?, updated_at = ? WHERE id = ?",
|
|
1816
|
+
(json.dumps(recipe, allow_nan=False), now(), scan["id"]),
|
|
1817
|
+
)
|
|
1818
|
+
return {"scanId": scan["id"], "maxCostUsd": limit}
|
|
1819
|
+
|
|
1820
|
+
|
|
1792
1821
|
def parse_scan_recipe(value: str, repository: Path) -> dict[str, Any]:
|
|
1793
1822
|
if len(value.encode("utf-8")) > SCAN_RECIPE_MAX_BYTES:
|
|
1794
1823
|
raise SystemExit("Scan launch recipe must be no larger than 256 KiB.")
|
|
@@ -2722,9 +2751,13 @@ def list_findings(connection: sqlite3.Connection, args: argparse.Namespace) -> d
|
|
|
2722
2751
|
values,
|
|
2723
2752
|
).fetchone()[0]
|
|
2724
2753
|
next_offset = args.offset + len(rows)
|
|
2754
|
+
relations = scan_history.finding_relations(connection, scan["id"], (row["id"] for row in rows))
|
|
2725
2755
|
return {
|
|
2726
2756
|
"findingsPage": {
|
|
2727
|
-
"findings": [
|
|
2757
|
+
"findings": [
|
|
2758
|
+
finding_result(connection, scan, row, related=relations.get(row["id"], []))
|
|
2759
|
+
for row in rows
|
|
2760
|
+
],
|
|
2728
2761
|
"limit": limit,
|
|
2729
2762
|
"nextOffset": next_offset if next_offset < total else None,
|
|
2730
2763
|
"offset": args.offset,
|
|
@@ -2817,6 +2850,9 @@ def scan_result(
|
|
|
2817
2850
|
"maximum": independent_reviews["maximum"],
|
|
2818
2851
|
"consolidating": independent_reviews["consolidating"],
|
|
2819
2852
|
}
|
|
2853
|
+
relations = scan_history.finding_relations(
|
|
2854
|
+
connection, scan["id"], (row["id"] for row in occurrence_rows)
|
|
2855
|
+
)
|
|
2820
2856
|
return {
|
|
2821
2857
|
"artifacts": artifacts,
|
|
2822
2858
|
"canceledAt": scan["canceled_at"],
|
|
@@ -2824,7 +2860,10 @@ def scan_result(
|
|
|
2824
2860
|
"contract": scan_contract(scan),
|
|
2825
2861
|
"continuationThreadId": scan["continuation_thread_id"],
|
|
2826
2862
|
"failureMessage": scan["failure_message"],
|
|
2827
|
-
"findings": [
|
|
2863
|
+
"findings": [
|
|
2864
|
+
finding_result(connection, scan, row, related=relations.get(row["id"], []))
|
|
2865
|
+
for row in occurrence_rows
|
|
2866
|
+
],
|
|
2828
2867
|
"findingCount": finding_count,
|
|
2829
2868
|
"findingsTruncated": finding_count > len(occurrence_rows),
|
|
2830
2869
|
"severityCounts": severity_counts,
|
|
@@ -2974,6 +3013,8 @@ def finding_result(
|
|
|
2974
3013
|
connection: sqlite3.Connection,
|
|
2975
3014
|
scan: sqlite3.Row,
|
|
2976
3015
|
occurrence: sqlite3.Row,
|
|
3016
|
+
*,
|
|
3017
|
+
related: list[dict[str, Any]],
|
|
2977
3018
|
) -> dict[str, Any]:
|
|
2978
3019
|
details = bounded_finding_details(read_finding_details(occurrence["details_json"]))
|
|
2979
3020
|
confidence = details.get("confidence")
|
|
@@ -3038,6 +3079,8 @@ def finding_result(
|
|
|
3038
3079
|
result["matches"] = matches
|
|
3039
3080
|
result["knownSince"] = known_since
|
|
3040
3081
|
result["knownScanIds"] = known_scan_ids
|
|
3082
|
+
if related:
|
|
3083
|
+
result["related"] = related
|
|
3041
3084
|
result.pop("artifactPaths", None)
|
|
3042
3085
|
source_excerpt = finding_source_excerpt(scan, target, locations)
|
|
3043
3086
|
if source_excerpt:
|
|
@@ -3411,6 +3454,10 @@ def main() -> None:
|
|
|
3411
3454
|
result = inspect_setup(args)
|
|
3412
3455
|
print(json.dumps(result, allow_nan=False, sort_keys=True))
|
|
3413
3456
|
return
|
|
3457
|
+
if args.command == "read-severity-classification":
|
|
3458
|
+
result = severity.read_classification(database_path(), args.scan_id)
|
|
3459
|
+
print(json.dumps(result, allow_nan=False, sort_keys=True))
|
|
3460
|
+
return
|
|
3414
3461
|
if args.command == "inspect-linear-publication":
|
|
3415
3462
|
result = inspect_linear_publication(args)
|
|
3416
3463
|
print(json.dumps(result, allow_nan=False, sort_keys=True))
|
|
@@ -3468,6 +3515,8 @@ def main() -> None:
|
|
|
3468
3515
|
result = register_cli_scan(connection, args)
|
|
3469
3516
|
elif args.command == "set-scan-thread":
|
|
3470
3517
|
result = set_scan_thread(connection, args)
|
|
3518
|
+
elif args.command == "set-scan-cost-limit":
|
|
3519
|
+
result = set_scan_cost_limit(connection, args)
|
|
3471
3520
|
elif args.command == "get-scan-recipe":
|
|
3472
3521
|
result = get_scan_recipe(connection, args)
|
|
3473
3522
|
elif args.command == "compare-scans":
|
|
@@ -3579,6 +3628,8 @@ def main() -> None:
|
|
|
3579
3628
|
result = export_findings(connection, args)
|
|
3580
3629
|
elif args.command == "database-info":
|
|
3581
3630
|
result = {"databasePath": str(database_path())}
|
|
3631
|
+
elif args.command == "severity-classification":
|
|
3632
|
+
result = severity.checkpoint(connection, json.load(sys.stdin), now())
|
|
3582
3633
|
elif args.command == "finding-workflow":
|
|
3583
3634
|
result = finding_workflow(connection, json.load(sys.stdin), now())
|
|
3584
3635
|
elif args.command == "dashboard":
|
|
@@ -112,13 +112,13 @@ def _latest_successful_reducer(workers: list[Any]) -> Any | None:
|
|
|
112
112
|
)
|
|
113
113
|
|
|
114
114
|
|
|
115
|
-
def _saved_result_paths(scan_dir: Path, workers: list[Any]) -> Iterator[str]:
|
|
115
|
+
def _saved_result_paths(scan_dir: Path, workers: list[Any]) -> Iterator[tuple[str, str | None]]:
|
|
116
116
|
latest_reducer = _latest_successful_reducer(workers)
|
|
117
117
|
|
|
118
|
-
def checkpoints(directory: str) -> Iterator[str]:
|
|
118
|
+
def checkpoints(directory: str, kind: str | None = None) -> Iterator[tuple[str, str | None]]:
|
|
119
119
|
for name in _children(scan_dir, directory):
|
|
120
120
|
if re.fullmatch(r"[0-9a-f]{64}\.json", name):
|
|
121
|
-
yield f"{directory}/{name}"
|
|
121
|
+
yield f"{directory}/{name}", kind
|
|
122
122
|
|
|
123
123
|
yield from checkpoints("checkpoints")
|
|
124
124
|
for worker in workers:
|
|
@@ -137,25 +137,32 @@ def _saved_result_paths(scan_dir: Path, workers: list[Any]) -> Iterator[str]:
|
|
|
137
137
|
if re.fullmatch(r"attempt-\d+", name)
|
|
138
138
|
]
|
|
139
139
|
for directory in directories:
|
|
140
|
-
checkpoint_paths = list(checkpoints(f"{directory}/checkpoints"))
|
|
140
|
+
checkpoint_paths = list(checkpoints(f"{directory}/checkpoints", worker["kind"]))
|
|
141
141
|
if worker["kind"] == "discovery" or checkpoint_paths:
|
|
142
|
-
yield f"{directory}/result.json"
|
|
142
|
+
yield f"{directory}/result.json", worker["kind"]
|
|
143
143
|
yield from checkpoint_paths
|
|
144
144
|
if worker["result_manifest_path"] and (
|
|
145
145
|
worker["kind"] == "discovery"
|
|
146
146
|
or (latest_reducer is not None and worker["id"] == latest_reducer["id"])
|
|
147
147
|
):
|
|
148
148
|
try:
|
|
149
|
-
yield
|
|
149
|
+
yield (
|
|
150
|
+
Path(worker["result_manifest_path"]).relative_to(scan_dir).as_posix(),
|
|
151
|
+
worker["kind"],
|
|
152
|
+
)
|
|
150
153
|
except ValueError:
|
|
151
154
|
continue
|
|
152
155
|
|
|
153
156
|
|
|
154
|
-
def _read_saved_result(
|
|
157
|
+
def _read_saved_result(
|
|
158
|
+
scan_dir: Path, relative: str, scan_id: str, *, kind: str | None = None
|
|
159
|
+
) -> tuple[dict[str, Any], str]:
|
|
155
160
|
draft = _read_scan_local_json(scan_dir, relative, "Saved scan checkpoint")
|
|
156
161
|
if draft.get("scanId") != scan_id:
|
|
157
162
|
raise ContractError("checkpoint belongs to a different scan")
|
|
158
|
-
if not isinstance(draft.get("findings"), list) or not isinstance(
|
|
163
|
+
if not isinstance(draft.get("findings"), list) or not isinstance(
|
|
164
|
+
draft.get("coverage", {} if kind == "dedup" else None), dict
|
|
165
|
+
):
|
|
159
166
|
raise ContractError("checkpoint has no semantic findings or coverage")
|
|
160
167
|
return draft, _digest(draft)
|
|
161
168
|
|
|
@@ -207,13 +214,13 @@ def _saved_results_changed(db: Any, connection: Any, scan: Any) -> bool:
|
|
|
207
214
|
"FROM deep_scan_workers WHERE scan_id = ?",
|
|
208
215
|
(scan["id"],),
|
|
209
216
|
).fetchall()
|
|
210
|
-
paths =
|
|
217
|
+
paths = dict(_saved_result_paths(scan_dir, workers))
|
|
211
218
|
frozen_sources = scan["retained_source_digests_json"]
|
|
212
219
|
|
|
213
220
|
def has_saved_source() -> bool:
|
|
214
221
|
for path in paths:
|
|
215
222
|
try:
|
|
216
|
-
_read_saved_result(scan_dir, path, scan["id"])
|
|
223
|
+
_read_saved_result(scan_dir, path, scan["id"], kind=paths[path])
|
|
217
224
|
return True
|
|
218
225
|
except (ContractError, OSError, ValueError):
|
|
219
226
|
continue
|
|
@@ -244,7 +251,9 @@ def _saved_results_changed(db: Any, connection: Any, scan: Any) -> bool:
|
|
|
244
251
|
current_sources = dict(published_sources)
|
|
245
252
|
for path in paths:
|
|
246
253
|
try:
|
|
247
|
-
_, current_sources[path] = _read_saved_result(
|
|
254
|
+
_, current_sources[path] = _read_saved_result(
|
|
255
|
+
scan_dir, path, scan["id"], kind=paths[path]
|
|
256
|
+
)
|
|
248
257
|
except (ContractError, OSError, ValueError):
|
|
249
258
|
continue
|
|
250
259
|
return current_sources != published_sources
|
|
@@ -290,23 +299,26 @@ def _recovery_source_digests(db: Any, connection: Any, scan: Any) -> tuple[dict[
|
|
|
290
299
|
else:
|
|
291
300
|
include_parent = True
|
|
292
301
|
|
|
302
|
+
workers = connection.execute(
|
|
303
|
+
"SELECT id, kind, status, completed_at, artifact_dir, result_manifest_path "
|
|
304
|
+
"FROM deep_scan_workers WHERE scan_id = ?",
|
|
305
|
+
(scan["id"],),
|
|
306
|
+
).fetchall()
|
|
307
|
+
paths = dict(_saved_result_paths(scan_dir, workers))
|
|
293
308
|
recovery_sources = dict(frozen_sources or {})
|
|
294
309
|
for relative, expected_digest in recovery_sources.items():
|
|
295
310
|
try:
|
|
296
|
-
_, digest = _read_saved_result(scan_dir, relative, scan["id"])
|
|
311
|
+
_, digest = _read_saved_result(scan_dir, relative, scan["id"], kind=paths.get(relative))
|
|
297
312
|
except (ContractError, OSError, ValueError) as exc:
|
|
298
313
|
raise ContractError("Frozen stopped-scan checkpoint set is incomplete.") from exc
|
|
299
314
|
if digest != expected_digest:
|
|
300
315
|
raise ContractError("checkpoint changed after the scan stopped")
|
|
301
316
|
|
|
302
|
-
|
|
303
|
-
"SELECT id, kind, status, completed_at, artifact_dir, result_manifest_path "
|
|
304
|
-
"FROM deep_scan_workers WHERE scan_id = ?",
|
|
305
|
-
(scan["id"],),
|
|
306
|
-
).fetchall()
|
|
307
|
-
for relative in set(_saved_result_paths(scan_dir, workers)) - recovery_sources.keys():
|
|
317
|
+
for relative in paths.keys() - recovery_sources.keys():
|
|
308
318
|
try:
|
|
309
|
-
_, recovery_sources[relative] = _read_saved_result(
|
|
319
|
+
_, recovery_sources[relative] = _read_saved_result(
|
|
320
|
+
scan_dir, relative, scan["id"], kind=paths[relative]
|
|
321
|
+
)
|
|
310
322
|
except (ContractError, OSError, ValueError):
|
|
311
323
|
continue
|
|
312
324
|
return recovery_sources, include_parent
|
|
@@ -489,6 +501,7 @@ def merge_saved_results(
|
|
|
489
501
|
parent_preserved_sources = recorded
|
|
490
502
|
source_digests.update(parent_preserved_sources)
|
|
491
503
|
paths: dict[str, str | None] = {}
|
|
504
|
+
reducer_paths: set[str] = set()
|
|
492
505
|
current_results: set[str] = set()
|
|
493
506
|
reducer_outputs: list[tuple[Any, str, list[str], int]] = []
|
|
494
507
|
reducer = _latest_successful_reducer(workers)
|
|
@@ -497,6 +510,7 @@ def merge_saved_results(
|
|
|
497
510
|
try:
|
|
498
511
|
latest_reducer = Path(reducer["result_manifest_path"]).relative_to(scan_dir).as_posix()
|
|
499
512
|
paths[latest_reducer] = None
|
|
513
|
+
reducer_paths.add(latest_reducer)
|
|
500
514
|
except ValueError:
|
|
501
515
|
warnings.append("Skipped a reducer result outside the scan directory.")
|
|
502
516
|
|
|
@@ -526,6 +540,7 @@ def merge_saved_results(
|
|
|
526
540
|
paths[result_path] = None
|
|
527
541
|
for checkpoint_path in checkpoint_paths:
|
|
528
542
|
paths[checkpoint_path] = None
|
|
543
|
+
reducer_paths.update([result_path, *checkpoint_paths])
|
|
529
544
|
reducer_outputs.append((reducer_worker, result_path, checkpoint_paths, attempt))
|
|
530
545
|
|
|
531
546
|
reducer_output(output, int(worker["attempt"] or 0), worker)
|
|
@@ -570,11 +585,15 @@ def merge_saved_results(
|
|
|
570
585
|
|
|
571
586
|
for relative, worker_id in paths.items():
|
|
572
587
|
try:
|
|
573
|
-
draft, digest = _read_saved_result(
|
|
588
|
+
draft, digest = _read_saved_result(
|
|
589
|
+
scan_dir, relative, scan_id, kind="dedup" if relative in reducer_paths else None
|
|
590
|
+
)
|
|
574
591
|
if frozen_source_digests is not None and frozen_source_digests[relative] != digest:
|
|
575
592
|
raise ContractError("checkpoint changed after the scan stopped")
|
|
576
593
|
source_digests[relative] = digest
|
|
577
|
-
|
|
594
|
+
# Recovery expects coverage, but reducer results only contain findings
|
|
595
|
+
# and context. Add an empty value after hashing the original result.
|
|
596
|
+
sources.append((relative, {"coverage": {}, **draft}, worker_id))
|
|
578
597
|
except (ContractError, OSError, ValueError) as exc:
|
|
579
598
|
if (scan_dir / relative).exists():
|
|
580
599
|
warnings.append(f"Preserved unreadable checkpoint {relative}: {exc}")
|
|
@@ -645,7 +664,7 @@ def merge_saved_results(
|
|
|
645
664
|
manifest["scan"]["preservedSources"] = source_digests
|
|
646
665
|
coverage = (
|
|
647
666
|
copy.deepcopy(parent["coverage"])
|
|
648
|
-
if parent
|
|
667
|
+
if parent and parent["coverage"]
|
|
649
668
|
else {
|
|
650
669
|
"completeness": "partial",
|
|
651
670
|
"mode": binding["coverageMode"],
|