@openai/codex-security 0.1.24 → 0.1.26

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (150) hide show
  1. package/README.md +399 -27
  2. package/_bundled_plugin/.app.json +0 -5
  3. package/_bundled_plugin/.codex-plugin/plugin.json +1 -1
  4. package/_bundled_plugin/.mcp.json +1 -0
  5. package/_bundled_plugin/mcp/server.mjs.br.part-000 +0 -0
  6. package/_bundled_plugin/mcp/server.mjs.br.part-001 +0 -0
  7. package/_bundled_plugin/references/config-preflight.md +2 -2
  8. package/_bundled_plugin/references/final-report.md +5 -3
  9. package/_bundled_plugin/references/scan-artifacts.md +5 -1
  10. package/_bundled_plugin/references/scan-contract.md +7 -3
  11. package/_bundled_plugin/schemas/tools/deep-reducer.schema.json +27 -1
  12. package/_bundled_plugin/scripts/deep_scan_workbench.py +6 -1
  13. package/_bundled_plugin/scripts/finalize_scan_contract.py +2 -5
  14. package/_bundled_plugin/scripts/rank_preview.py +11 -1
  15. package/_bundled_plugin/scripts/report_projection.py +1 -1
  16. package/_bundled_plugin/scripts/workbench_cli.py +11 -1
  17. package/_bundled_plugin/scripts/workbench_db.py +55 -4
  18. package/_bundled_plugin/scripts/workbench_saved_results.py +41 -22
  19. package/_bundled_plugin/scripts/workbench_scan_history.py +374 -66
  20. package/_bundled_plugin/scripts/workbench_scan_usage.py +7 -6
  21. package/_bundled_plugin/scripts/workbench_schema.py +46 -0
  22. package/_bundled_plugin/scripts/workbench_severity.py +119 -0
  23. package/_bundled_plugin/scripts/workbench_target.py +86 -12
  24. package/_bundled_plugin/skills/deep-security-scan/SKILL.md +5 -3
  25. package/_bundled_plugin/skills/verify-fix/SKILL.md +5 -1
  26. package/dist/api.d.ts +15 -2
  27. package/dist/api.d.ts.map +1 -1
  28. package/dist/api.js +333 -32
  29. package/dist/api.js.map +1 -1
  30. package/dist/auth.d.ts.map +1 -1
  31. package/dist/auth.js +35 -3
  32. package/dist/auth.js.map +1 -1
  33. package/dist/classify-scan-severity.d.ts +16 -0
  34. package/dist/classify-scan-severity.d.ts.map +1 -0
  35. package/dist/classify-scan-severity.js +96 -0
  36. package/dist/classify-scan-severity.js.map +1 -0
  37. package/dist/classify-severity.d.ts +38 -0
  38. package/dist/classify-severity.d.ts.map +1 -0
  39. package/dist/classify-severity.js +185 -0
  40. package/dist/classify-severity.js.map +1 -0
  41. package/dist/cli.d.ts +7 -2
  42. package/dist/cli.d.ts.map +1 -1
  43. package/dist/cli.js +403 -116
  44. package/dist/cli.js.map +1 -1
  45. package/dist/component-plan.d.ts.map +1 -1
  46. package/dist/component-plan.js +4 -2
  47. package/dist/component-plan.js.map +1 -1
  48. package/dist/component-scan.d.ts.map +1 -1
  49. package/dist/component-scan.js +21 -6
  50. package/dist/component-scan.js.map +1 -1
  51. package/dist/config.d.ts.map +1 -1
  52. package/dist/config.js +46 -2
  53. package/dist/config.js.map +1 -1
  54. package/dist/cost.d.ts +1 -0
  55. package/dist/cost.d.ts.map +1 -1
  56. package/dist/cost.js +43 -17
  57. package/dist/cost.js.map +1 -1
  58. package/dist/deduplication/checkpointed-review.d.ts.map +1 -1
  59. package/dist/deduplication/checkpointed-review.js +2 -1
  60. package/dist/deduplication/checkpointed-review.js.map +1 -1
  61. package/dist/deduplication/codex-review.d.ts +3 -0
  62. package/dist/deduplication/codex-review.d.ts.map +1 -1
  63. package/dist/deduplication/codex-review.js +122 -36
  64. package/dist/deduplication/codex-review.js.map +1 -1
  65. package/dist/deduplication/deduplication-prompts.d.ts +4 -3
  66. package/dist/deduplication/deduplication-prompts.d.ts.map +1 -1
  67. package/dist/deduplication/deduplication-prompts.js +18 -9
  68. package/dist/deduplication/deduplication-prompts.js.map +1 -1
  69. package/dist/deduplication/deduplication-reviewer.d.ts +2 -7
  70. package/dist/deduplication/deduplication-reviewer.d.ts.map +1 -1
  71. package/dist/deduplication/deduplication-reviewer.js +47 -25
  72. package/dist/deduplication/deduplication-reviewer.js.map +1 -1
  73. package/dist/deduplication/deduplication.d.ts.map +1 -1
  74. package/dist/deduplication/deduplication.js +186 -23
  75. package/dist/deduplication/deduplication.js.map +1 -1
  76. package/dist/deduplication/scan.d.ts +9 -1
  77. package/dist/deduplication/scan.d.ts.map +1 -1
  78. package/dist/deduplication/scan.js +24 -7
  79. package/dist/deduplication/scan.js.map +1 -1
  80. package/dist/errors.d.ts +13 -0
  81. package/dist/errors.d.ts.map +1 -1
  82. package/dist/errors.js +7 -0
  83. package/dist/errors.js.map +1 -1
  84. package/dist/finding-catalogue.d.ts +11 -0
  85. package/dist/finding-catalogue.d.ts.map +1 -0
  86. package/dist/finding-catalogue.js +150 -0
  87. package/dist/finding-catalogue.js.map +1 -0
  88. package/dist/index.d.ts +11 -5
  89. package/dist/index.d.ts.map +1 -1
  90. package/dist/index.js +5 -2
  91. package/dist/index.js.map +1 -1
  92. package/dist/mock-scan.d.ts +4 -0
  93. package/dist/mock-scan.d.ts.map +1 -0
  94. package/dist/mock-scan.js +210 -0
  95. package/dist/mock-scan.js.map +1 -0
  96. package/dist/patch-tui.js +1 -1
  97. package/dist/patch-tui.js.map +1 -1
  98. package/dist/publication-store.d.ts.map +1 -1
  99. package/dist/publication-store.js +9 -4
  100. package/dist/publication-store.js.map +1 -1
  101. package/dist/publication.d.ts +9 -0
  102. package/dist/publication.d.ts.map +1 -1
  103. package/dist/publication.js +45 -5
  104. package/dist/publication.js.map +1 -1
  105. package/dist/publish.d.ts +4 -2
  106. package/dist/publish.d.ts.map +1 -1
  107. package/dist/publish.js +4 -4
  108. package/dist/publish.js.map +1 -1
  109. package/dist/runtime.d.ts +1 -0
  110. package/dist/runtime.d.ts.map +1 -1
  111. package/dist/runtime.js +49 -14
  112. package/dist/runtime.js.map +1 -1
  113. package/dist/scan-comparison.d.ts +44 -30
  114. package/dist/scan-comparison.d.ts.map +1 -1
  115. package/dist/scan-comparison.js +673 -101
  116. package/dist/scan-comparison.js.map +1 -1
  117. package/dist/scan-dashboard.d.ts +3 -1
  118. package/dist/scan-dashboard.d.ts.map +1 -1
  119. package/dist/scan-dashboard.js +70 -3
  120. package/dist/scan-dashboard.js.map +1 -1
  121. package/dist/scan-history-renderer.d.ts.map +1 -1
  122. package/dist/scan-history-renderer.js +26 -0
  123. package/dist/scan-history-renderer.js.map +1 -1
  124. package/dist/server/api.d.ts +6 -0
  125. package/dist/server/api.d.ts.map +1 -0
  126. package/dist/server/api.js +4 -0
  127. package/dist/server/api.js.map +1 -0
  128. package/dist/server/dashboard/app.css +1 -1
  129. package/dist/server/embeddings.d.ts +2 -1
  130. package/dist/server/embeddings.d.ts.map +1 -1
  131. package/dist/server/embeddings.js +8 -3
  132. package/dist/server/embeddings.js.map +1 -1
  133. package/dist/server/serve.d.ts.map +1 -1
  134. package/dist/server/serve.js +1 -1
  135. package/dist/server/serve.js.map +1 -1
  136. package/dist/severity-store.d.ts +2 -0
  137. package/dist/severity-store.d.ts.map +1 -0
  138. package/dist/severity-store.js +111 -0
  139. package/dist/severity-store.js.map +1 -0
  140. package/dist/targets.d.ts.map +1 -1
  141. package/dist/targets.js +3 -5
  142. package/dist/targets.js.map +1 -1
  143. package/dist/thread-source.d.ts +1 -0
  144. package/dist/thread-source.d.ts.map +1 -1
  145. package/dist/thread-source.js +1 -0
  146. package/dist/thread-source.js.map +1 -1
  147. package/dist/version.d.ts +1 -1
  148. package/dist/version.js +1 -1
  149. package/dist/version.js.map +1 -1
  150. package/package.json +11 -6
@@ -16,11 +16,11 @@ Use `report.md` as the primary readable entry point. Explain report-relevant art
16
16
 
17
17
  In the final response, link the generated markdown report path as the primary readable artifact.
18
18
 
19
- Every scan mode uses the same final report pipeline. Workbench-owned Standard and workbench-backed diff scans submit canonical semantics with `record_codex_security_scan_draft({ scanId, handoffClaimToken?, scope?, threatModel?, findings, coverage })`; the workbench supplies authoritative metadata and writes the unsealed canonical draft. Deep Standard scan workers submit complete semantic results through their bound draft tool, and the coordinator aggregates them and writes the parent scan's unsealed canonical draft. SDK-owned Standard scans instead write unsealed canonical files with the exact SDK-provided metadata and leave finalization to the SDK. Terminal scans without a `scanId` retain their existing canonical JSON workflow. No mode authors, repairs, or treats an existing `report.md` as input. Finalization validates and enriches the canonical JSON, seals the canonical JSON and evidence artifacts, then deterministically generates `report.md`. Supply report prose through structured canonical semantics rather than a separately authored report.
19
+ Every scan mode uses the same final report pipeline. Workbench-owned Standard and workbench-backed diff scans submit canonical semantics with `record_codex_security_scan_draft({ scanId, handoffClaimToken?, scope?, threatModel?, findings, coverage })`; the workbench supplies authoritative metadata and writes the unsealed canonical draft. For Deep scans, the coordinator writes the parent scan's unsealed canonical draft from its accepted aggregate. SDK-owned Standard scans instead write unsealed canonical files with the exact SDK-provided metadata and leave finalization to the SDK. Terminal scans without a `scanId` retain their existing canonical JSON workflow. No mode authors, repairs, or treats an existing `report.md` as input. Finalization validates and enriches the canonical JSON, seals the canonical JSON and evidence artifacts, then deterministically generates `report.md`. Supply report prose through structured canonical semantics rather than a separately authored report.
20
20
 
21
21
  For each finding, supply an evidence-supported lowercase vulnerability-family `ruleId`; `taxonomy: { category, cwe }` using its exact known CWEs; verified locations; and `provenance.source`, using `"local_plugin"` only when this plugin actually discovered the finding. Preserve genuine worker or source provenance and any existing canonical candidate identity in the finding extensions. A finding with no known CWE retains `cwe: []`; never invent a classification. Include optional `codeEvidence` only when its actual code is nonempty and every referenced evidence ID is present.
22
22
 
23
- Supply semantic coverage as `{ completeness, surfaces, explicitExclusions, deferred }`, with each surface using the actual `label` and one existing `disposition`. Mark coverage `partial` when a deferred item or `needs_follow_up` surface remains; preserve its real reason and supporting context. Each deferred item needs a meaningful reason; preserve any existing `id` or `candidateId`. The workbench derives a missing ID from its candidate identity or stable deferred-work details. Open questions may be nonempty strings or `{ question, followUpPrompt? }` objects. The workbench derives target and scope metadata, scope include and exclude paths, coverage mode and inventory strategy, finding identities and fingerprints, and surface IDs. Do not put those workbench-owned values or top-level coverage receipt references into the semantic draft.
23
+ For Standard scan drafts, including Deep worker results, and diff drafts, supply semantic coverage as `{ completeness, surfaces, explicitExclusions, deferred }`, with each surface using the actual `label` and one existing `disposition`. Mark coverage `partial` when a deferred item or `needs_follow_up` surface remains; preserve its real reason and supporting context. Each deferred item needs a meaningful reason; preserve any existing `id` or `candidateId`. The workbench derives a missing ID from its candidate identity or stable deferred-work details. Open questions may be nonempty strings or `{ question, followUpPrompt? }` objects. The workbench derives target and scope metadata, scope include and exclude paths, coverage mode and inventory strategy, finding identities and fingerprints, and surface IDs. Do not put those workbench-owned values or top-level coverage receipt references into the semantic draft.
24
24
 
25
25
  During a host-backed scan, checkpoint saved findings and pending candidates with `complete: false`; a checkpoint is not a completed audit. After a final workbench-owned Standard or workbench-backed diff draft is accepted with `complete: true` (or the backwards-compatible omitted flag), or the Deep coordinator returns its parent scan's canonical manifest, call `complete_codex_security_scan({ scanId, handoffClaimToken? })` and use its returned completion metadata. An SDK-owned scan returns its unsealed canonical files without calling a completion tool or finalizer; the SDK owns completion and report generation. Read full canonical results only when explicitly requested. For a terminal/chat workflow without a `scanId` or completion tool, retain `python <plugin_dir>/scripts/finalize_scan_contract.py --scan-dir <scan_dir> --source-root <repo_root>` after writing the canonical JSON. Outside the SDK path, do not mark the scan goal complete until finalization succeeds and the generated report exists.
26
26
 
@@ -59,7 +59,7 @@ Group observations only when they share the same broken security control and eff
59
59
 
60
60
  Set the finding category and CWE from the primary broken control. Do not add secondary support-impact CWEs, such as data exposure or missing authentication, to an injection/RCE/path/file/parser finding merely because they make exploitation worse; mention those impacts in prose or emit a separate finding if that secondary control is independently vulnerable.
61
61
 
62
- Workbench-owned Standard scans submit their source-backed final findings and coverage directly through `record_codex_security_scan_draft`; SDK-owned Standard scans write the same semantics into unsealed canonical files. Deep scans semantically aggregate complete Standard worker results, preserving validated findings, attack-path analysis, affected locations, threat-model context, and honest coverage. The coordinator writes the parent scan's canonical draft; the parent does not read candidate ledgers, run separate validation or attack-path phases, or submit another draft. Canonical `severity.changeConditions` must be one non-empty string; when `attack_path.change_conditions` contains multiple strings, join them into one prose string before writing `findings.json`.
62
+ Canonical `severity.changeConditions` must be one non-empty string; when `attack_path.change_conditions` contains multiple strings, join them into one prose string before writing `findings.json`.
63
63
 
64
64
  For workbench-backed diff candidates, apply row outcomes in this order: validation disposition `reportable` plus attack-path decision `reportable` becomes a finding with its distinct instance and all relevant entrypoint, root-control, sink, and supporting locations; otherwise, a `deferred` result from either phase becomes `needs_follow_up` coverage and a `coverage.deferred` entry using the recorded uncertainty or proof gap; otherwise, validation disposition `not_applicable` becomes `not_applicable` coverage; otherwise, validation disposition `suppressed` or attack-path decision `ignore` becomes `rejected` coverage. A missing required phase record leaves the candidate unresolved and prevents complete coverage. Do not require phase receipts, per-candidate narratives, or another reconciliation pass.
65
65
 
@@ -73,6 +73,8 @@ Use this report structure:
73
73
 
74
74
  `## Scope`
75
75
 
76
+ Deep reports present the configured include and exclude paths and execution outcome alongside aggregated findings, with their validation, attack paths, affected locations, and any supplied scope and threat-model context. When the discovery time limit expires before any source review completes, the report explains the incomplete review.
77
+
76
78
  Populate `scan.scope` with in-scope context, artifacts reviewed, runtime or test status, validation mode, and explicit limitations. Include/exclude paths and coverage fields supply the remaining projected scope content. If the threat model was generated during Phase 1 rather than provided by the user, say that in canonical scope context. Do not call generated threat-model material an external input.
77
79
 
78
80
  After the scope bullets, include a compact `### Scan Summary` table when the scan has findings or repository-wide coverage. Use columns `Field` and `Value`. Include the count of reportable findings, severity mix, confidence mix, coverage, and validation mode when those values are known. Keep artifact paths below this table.
@@ -40,7 +40,11 @@ End each repository-scoped threat model with these two lines:
40
40
 
41
41
  ### Compact Deep And Workbench-Backed Diff Discovery
42
42
 
43
- Workbench-owned Standard scans submit findings and coverage through `record_codex_security_scan_draft`; SDK-owned Standard scans write unsealed canonical files directly. Deep scans run complete Standard scan workers, each of which saves `complete: false` checkpoints and submits its final validated findings, coverage, threat model, and optional scope through its bound `record_codex_security_scan_draft` tool. Pending candidates retain their original evidence in `coverage.deferred`. Host-owned immutable checkpoints survive retries, cancellation, and failure; a checkpoint alone is never an accepted complete worker result. The coordinator semantically reduces those complete results and writes the parent scan's unsealed `scan-manifest.json`, `findings.json`, and `coverage.json`. The parent does not list candidates, rerun validation or attack-path phases, or submit another draft. Workbench-backed diff scans retain the compact artifacts described below.
43
+ Workbench-owned Standard scans submit findings and coverage through `record_codex_security_scan_draft`; SDK-owned Standard scans write unsealed canonical files directly. Workbench-backed diff scans use the compact artifacts described below.
44
+
45
+ Deep scans run ordinary Standard scan workers. Workers save progress with `complete: false` and submit final results through `record_codex_security_scan_draft`. Their checkpoints and results contain findings and coverage, with optional scope and threat-model context. Pending candidates and their evidence are recorded in worker coverage. Immutable checkpoints store progress across retries, cancellation, and failure. The coordinator passes completed workers' findings and context to the reducer.
46
+
47
+ Deep reducer inputs, results, and checkpoints contain findings and optional scope and threat-model context. The host writes the parent scan's unsealed `scan-manifest.json` and `findings.json` from the accepted aggregate, and derives `coverage.json` from the configured include and exclude paths and the coordinator's outcome. The parent completes the scan from these artifacts. If the discovery time limit expires before any source review completes, the parent records partial coverage with that reason. See `scan-contract.md` for canonical field definitions.
44
48
 
45
49
  - A workbench-backed diff scan records all candidates once with `record_codex_security_discovery_candidates({ scanId, candidates })` and reads the canonical candidates with `list_codex_security_candidates({ scanId, cursor?, limit? })`.
46
50
  - The writer validates candidates against assigned source paths, merges rows with the same CWE ids, locations, and optional instance, preserves their text, and assigns deterministic `candidate_id` values.
@@ -104,9 +104,13 @@ Use CWE taxonomy separately. Do not include file names, line numbers, scan IDs,
104
104
 
105
105
  ## Coverage
106
106
 
107
- `coverage.json` prevents downstream consumers from confusing `not observed` with `not scanned`.
107
+ `coverage.json` records scan scope and completion information. Standard and diff summaries also describe reviewed surfaces and outstanding work.
108
108
 
109
- Record:
109
+ For a Deep parent scan, the host copies the configured paths into `includePaths` and `excludePaths` and sets `completeness` from the coordinator's outcome. A successful aggregate uses `complete`; `surfaces`, `explicitExclusions`, and `deferred` are empty arrays, and `openQuestions` is omitted. If the configured time limit expires before any review completes, the coordinator writes `partial` and records the explanation in `deferred`. Stopped outcomes follow the [stopped-result recovery rules](#stopped-result-recovery).
110
+
111
+ Each Deep worker writes an ordinary Standard result, including its own coverage. A reducer submits `record_codex_security_deep_reduction({ scanId, findings, scope?, threatModel? })`; its saved results and checkpoints contain the accepted findings and optional scope and threat-model context.
112
+
113
+ For Standard and diff scans, record:
110
114
 
111
115
  - scan mode and inventory strategy
112
116
  - included and excluded paths
@@ -140,7 +144,7 @@ For a whole-repository Deep scan, keep `inventoryStrategy` as `repository`; repe
140
144
  | `directory` | Deterministic non-Git directory inventory |
141
145
  | `custom` | Producer-defined inventory described by detailed receipts |
142
146
 
143
- Use `complete` when the requested scope was fully reviewed, `partial` when in-scope work was deferred, and `unknown` when the producer cannot establish enough coverage to make that distinction.
147
+ For Standard and diff scans, use `complete` when the requested scope was fully reviewed, `partial` when in-scope work was deferred, and `unknown` when the producer cannot establish enough coverage to make that distinction.
144
148
 
145
149
  Map detailed ledger closure into completed surface summaries in this order:
146
150
 
@@ -9,7 +9,33 @@
9
9
  "additionalProperties": false
10
10
  },
11
11
  "reductionInput": {
12
- "$ref": "codex-security://schemas/tools/scan-draft.schema.json#/$defs/scanDraftInput"
12
+ "type": "object",
13
+ "properties": {
14
+ "complete": {
15
+ "type": "boolean",
16
+ "description": "Omit or set true when submitting the finished aggregate."
17
+ },
18
+ "scanId": {
19
+ "$ref": "codex-security://schemas/tools/scan-draft.schema.json#/$defs/scanId"
20
+ },
21
+ "handoffClaimToken": {
22
+ "$ref": "codex-security://schemas/tools/scan-draft.schema.json#/$defs/handoffClaimToken"
23
+ },
24
+ "scope": {
25
+ "$ref": "codex-security://schemas/tools/scan-draft.schema.json#/$defs/scope"
26
+ },
27
+ "threatModel": {
28
+ "$ref": "codex-security://schemas/tools/scan-draft.schema.json#/$defs/threatModel"
29
+ },
30
+ "findings": {
31
+ "$ref": "codex-security://schemas/tools/scan-draft.schema.json#/$defs/scanDraftInput/properties/findings"
32
+ }
33
+ },
34
+ "required": [
35
+ "scanId",
36
+ "findings"
37
+ ],
38
+ "additionalProperties": false
13
39
  }
14
40
  }
15
41
  }
@@ -1877,6 +1877,7 @@ def finish_deep_scan_locked(
1877
1877
  """
1878
1878
  SELECT 1 FROM deep_scan_workers AS failed
1879
1879
  WHERE failed.scan_id = ? AND failed.status = 'failed'
1880
+ AND (? != 'saturated' OR failed.kind != 'discovery')
1880
1881
  AND (
1881
1882
  failed.kind != 'dedup'
1882
1883
  OR NOT EXISTS (
@@ -1902,10 +1903,14 @@ def finish_deep_scan_locked(
1902
1903
  )
1903
1904
  LIMIT 1
1904
1905
  """,
1905
- (scan_id,),
1906
+ (scan_id, args.terminal_reason),
1906
1907
  ).fetchone()
1907
1908
  if failed_worker is not None and not failure_capped:
1908
1909
  raise SystemExit("Deep Scan cannot finish after a worker has failed.")
1910
+ if args.terminal_reason == "saturated":
1911
+ # Mark any remaining workers canceled, including those whose own
1912
+ # cancellation writes failed, so they cannot block completion.
1913
+ cancel_active_workers(connection, scan_id, now())
1909
1914
  active_worker = connection.execute(
1910
1915
  """
1911
1916
  SELECT 1 FROM deep_scan_workers
@@ -1252,12 +1252,9 @@ def _normalize_unsealed_deep_repository_inventory_strategy(
1252
1252
  *,
1253
1253
  expected_coverage_mode: str | None,
1254
1254
  ) -> None:
1255
- """Normalize the old Deep workflow label to the ordinary repository inventory."""
1255
+ """Label whole-repository Deep scans as using the repository inventory."""
1256
1256
 
1257
- if (
1258
- expected_coverage_mode == "deep_repository"
1259
- and coverage.get("inventoryStrategy") == "deep_repository_repeated_discovery"
1260
- ):
1257
+ if expected_coverage_mode == "deep_repository":
1261
1258
  coverage["inventoryStrategy"] = "repository"
1262
1259
 
1263
1260
 
@@ -11,6 +11,7 @@ from pathlib import Path
11
11
  DEFAULT_PREVIEW_BYTES = 1024
12
12
  PREVIEW_HEAD_LINES = 12
13
13
  PREVIEW_SAMPLE_LINES = 10
14
+ _UTF16_BOMS = (b"\xff\xfe", b"\xfe\xff")
14
15
 
15
16
  TEXT_CODE_EXTENSIONS = {
16
17
  ".c",
@@ -43,6 +44,8 @@ TEXT_CODE_EXTENSIONS = {
43
44
  ".php",
44
45
  ".proto",
45
46
  ".ps1",
47
+ ".psd1",
48
+ ".psm1",
46
49
  ".py",
47
50
  ".rb",
48
51
  ".rs",
@@ -79,7 +82,14 @@ RUST_LIFETIME_RE = re.compile(r"'[A-Za-z_][A-Za-z0-9_]*")
79
82
  PHP_HEREDOC_RE = re.compile(r"<<<\s*['\"]?([A-Za-z_]\w*)['\"]?")
80
83
 
81
84
 
85
+ def _decode_source(data: bytes) -> str:
86
+ encoding = "utf-16" if data.startswith(_UTF16_BOMS) else "utf-8"
87
+ return data.decode(encoding, errors="ignore")
88
+
89
+
82
90
  def is_binary_sample(data: bytes) -> bool:
91
+ if data.startswith(_UTF16_BOMS):
92
+ return "\0" in _decode_source(data)
83
93
  return b"\0" in data
84
94
 
85
95
 
@@ -956,7 +966,7 @@ def preview_for(
956
966
  def preview_for_bytes(path: Path, data: bytes, preview_bytes: int) -> tuple[str, bool]:
957
967
  if is_binary_sample(data):
958
968
  return "", True
959
- text = data.decode("utf-8", errors="ignore")
969
+ text = _decode_source(data)
960
970
  outline = structural_outline(path, text)
961
971
  preview_lines = select_preview_lines(outline or text.splitlines())
962
972
  return fit_preview_lines(preview_lines, preview_bytes), False
@@ -814,7 +814,7 @@ def build_report_markdown(
814
814
  "",
815
815
  _text(
816
816
  scope.get("summary"),
817
- "The scan reviewed the canonical include paths and exclusions listed below.",
817
+ "The scan was configured for the include paths and exclusions listed below.",
818
818
  ),
819
819
  "",
820
820
  f"- Scan mode: {coverage['mode']}",
@@ -160,6 +160,10 @@ def parse_args(description: str) -> argparse.Namespace:
160
160
  set_scan_thread.add_argument("--scan-id", required=True)
161
161
  set_scan_thread.add_argument("--thread-id", required=True)
162
162
 
163
+ set_scan_cost_limit = subparsers.add_parser("set-scan-cost-limit")
164
+ set_scan_cost_limit.add_argument("--scan-id", required=True)
165
+ set_scan_cost_limit.add_argument("--max-cost-usd", required=True, type=float)
166
+
163
167
  get_scan_recipe = subparsers.add_parser("get-scan-recipe")
164
168
  get_scan_recipe.add_argument("--scan-id", required=True)
165
169
 
@@ -169,7 +173,10 @@ def parse_args(description: str) -> argparse.Namespace:
169
173
  compare_scans.add_argument("--include-matching-inputs", action="store_true")
170
174
  compare_scans.add_argument("--require-matches", action="store_true")
171
175
 
172
- save_scan_comparison = subparsers.add_parser("save-scan-comparison")
176
+ save_scan_comparison = subparsers.add_parser(
177
+ "save-scan-comparison",
178
+ description="Comparison payload supports related findings.",
179
+ )
173
180
  save_scan_comparison.add_argument("--before-scan-id", required=True)
174
181
  save_scan_comparison.add_argument("--after-scan-id", required=True)
175
182
  matches = save_scan_comparison.add_mutually_exclusive_group(required=True)
@@ -347,6 +354,9 @@ def parse_args(description: str) -> argparse.Namespace:
347
354
  subparsers.add_parser("database-info")
348
355
  subparsers.add_parser("dashboard")
349
356
  subparsers.add_parser("finding-workflow")
357
+ subparsers.add_parser("severity-classification")
358
+ severity = subparsers.add_parser("read-severity-classification")
359
+ severity.add_argument("--scan-id", required=True)
350
360
  subparsers.add_parser("store-findings")
351
361
  subparsers.add_parser("store-dedupe-groups")
352
362
  dedupe_groups = subparsers.add_parser("list-dedupe-groups")
@@ -7,6 +7,7 @@ import argparse
7
7
  import errno
8
8
  import hashlib
9
9
  import json
10
+ import math
10
11
  import os
11
12
  import re
12
13
  import sqlite3
@@ -39,6 +40,7 @@ import workbench_remediation as remediation
39
40
  import workbench_saved_results as saved_results
40
41
  import workbench_scan_history as scan_history
41
42
  import workbench_scan_usage as scan_usage
43
+ import workbench_severity as severity
42
44
  from filesystem_identity import (
43
45
  serialize_filesystem_identity as serialize_filesystem_identity,
44
46
  )
@@ -1534,7 +1536,9 @@ def complete_scan_locked(
1534
1536
  scan_dir,
1535
1537
  expected_coverage_mode=expected_coverage_mode(scan),
1536
1538
  completion_binding=completion_binding,
1537
- completion_warnings=warnings,
1539
+ # Save the finished Deep result as submitted. Worker drafts and
1540
+ # recovery repairs belong to the stopped-scan path.
1541
+ completion_warnings=warnings if scan["mode"] != "deep" else None,
1538
1542
  draft_documents=saved_results.merge_saved_results(
1539
1543
  scan_dir,
1540
1544
  scan["id"],
@@ -1547,7 +1551,7 @@ def complete_scan_locked(
1547
1551
  stopped=False,
1548
1552
  reason="",
1549
1553
  )
1550
- if current_manifest_path is not None and not already_sealed
1554
+ if scan["mode"] != "deep" and current_manifest_path is not None and not already_sealed
1551
1555
  else None,
1552
1556
  )
1553
1557
  add_warning()
@@ -1789,6 +1793,31 @@ def set_scan_thread(connection: sqlite3.Connection, args: argparse.Namespace) ->
1789
1793
  return {"scanId": scan["id"], "threadId": args.thread_id}
1790
1794
 
1791
1795
 
1796
+ def set_scan_cost_limit(connection: sqlite3.Connection, args: argparse.Namespace) -> dict[str, Any]:
1797
+ scan_id = require_uuid(args.scan_id, "scan-id")
1798
+ limit = args.max_cost_usd
1799
+ if not math.isfinite(limit) or limit <= 0:
1800
+ raise SystemExit("The scan cost limit must be a positive finite USD amount.")
1801
+ with scan_completion_lock(scan_id), connection:
1802
+ scan = require_scan(connection, scan_id)
1803
+ if scan["status"] != "running" or scan["recipe_json"] is None:
1804
+ raise SystemExit("Only a running CLI scan can increase its cost limit.")
1805
+ recipe = json.loads(scan["recipe_json"], parse_constant=reject_non_finite_json)
1806
+ previous = recipe.get("maxCostUsd")
1807
+ if (
1808
+ not isinstance(previous, (int, float))
1809
+ or isinstance(previous, bool)
1810
+ or limit <= previous
1811
+ ):
1812
+ raise SystemExit("The new cost limit must exceed the current limit.")
1813
+ recipe["maxCostUsd"] = limit
1814
+ connection.execute(
1815
+ "UPDATE scans SET recipe_json = ?, updated_at = ? WHERE id = ?",
1816
+ (json.dumps(recipe, allow_nan=False), now(), scan["id"]),
1817
+ )
1818
+ return {"scanId": scan["id"], "maxCostUsd": limit}
1819
+
1820
+
1792
1821
  def parse_scan_recipe(value: str, repository: Path) -> dict[str, Any]:
1793
1822
  if len(value.encode("utf-8")) > SCAN_RECIPE_MAX_BYTES:
1794
1823
  raise SystemExit("Scan launch recipe must be no larger than 256 KiB.")
@@ -2722,9 +2751,13 @@ def list_findings(connection: sqlite3.Connection, args: argparse.Namespace) -> d
2722
2751
  values,
2723
2752
  ).fetchone()[0]
2724
2753
  next_offset = args.offset + len(rows)
2754
+ relations = scan_history.finding_relations(connection, scan["id"], (row["id"] for row in rows))
2725
2755
  return {
2726
2756
  "findingsPage": {
2727
- "findings": [finding_result(connection, scan, row) for row in rows],
2757
+ "findings": [
2758
+ finding_result(connection, scan, row, related=relations.get(row["id"], []))
2759
+ for row in rows
2760
+ ],
2728
2761
  "limit": limit,
2729
2762
  "nextOffset": next_offset if next_offset < total else None,
2730
2763
  "offset": args.offset,
@@ -2817,6 +2850,9 @@ def scan_result(
2817
2850
  "maximum": independent_reviews["maximum"],
2818
2851
  "consolidating": independent_reviews["consolidating"],
2819
2852
  }
2853
+ relations = scan_history.finding_relations(
2854
+ connection, scan["id"], (row["id"] for row in occurrence_rows)
2855
+ )
2820
2856
  return {
2821
2857
  "artifacts": artifacts,
2822
2858
  "canceledAt": scan["canceled_at"],
@@ -2824,7 +2860,10 @@ def scan_result(
2824
2860
  "contract": scan_contract(scan),
2825
2861
  "continuationThreadId": scan["continuation_thread_id"],
2826
2862
  "failureMessage": scan["failure_message"],
2827
- "findings": [finding_result(connection, scan, row) for row in occurrence_rows],
2863
+ "findings": [
2864
+ finding_result(connection, scan, row, related=relations.get(row["id"], []))
2865
+ for row in occurrence_rows
2866
+ ],
2828
2867
  "findingCount": finding_count,
2829
2868
  "findingsTruncated": finding_count > len(occurrence_rows),
2830
2869
  "severityCounts": severity_counts,
@@ -2974,6 +3013,8 @@ def finding_result(
2974
3013
  connection: sqlite3.Connection,
2975
3014
  scan: sqlite3.Row,
2976
3015
  occurrence: sqlite3.Row,
3016
+ *,
3017
+ related: list[dict[str, Any]],
2977
3018
  ) -> dict[str, Any]:
2978
3019
  details = bounded_finding_details(read_finding_details(occurrence["details_json"]))
2979
3020
  confidence = details.get("confidence")
@@ -3038,6 +3079,8 @@ def finding_result(
3038
3079
  result["matches"] = matches
3039
3080
  result["knownSince"] = known_since
3040
3081
  result["knownScanIds"] = known_scan_ids
3082
+ if related:
3083
+ result["related"] = related
3041
3084
  result.pop("artifactPaths", None)
3042
3085
  source_excerpt = finding_source_excerpt(scan, target, locations)
3043
3086
  if source_excerpt:
@@ -3411,6 +3454,10 @@ def main() -> None:
3411
3454
  result = inspect_setup(args)
3412
3455
  print(json.dumps(result, allow_nan=False, sort_keys=True))
3413
3456
  return
3457
+ if args.command == "read-severity-classification":
3458
+ result = severity.read_classification(database_path(), args.scan_id)
3459
+ print(json.dumps(result, allow_nan=False, sort_keys=True))
3460
+ return
3414
3461
  if args.command == "inspect-linear-publication":
3415
3462
  result = inspect_linear_publication(args)
3416
3463
  print(json.dumps(result, allow_nan=False, sort_keys=True))
@@ -3468,6 +3515,8 @@ def main() -> None:
3468
3515
  result = register_cli_scan(connection, args)
3469
3516
  elif args.command == "set-scan-thread":
3470
3517
  result = set_scan_thread(connection, args)
3518
+ elif args.command == "set-scan-cost-limit":
3519
+ result = set_scan_cost_limit(connection, args)
3471
3520
  elif args.command == "get-scan-recipe":
3472
3521
  result = get_scan_recipe(connection, args)
3473
3522
  elif args.command == "compare-scans":
@@ -3579,6 +3628,8 @@ def main() -> None:
3579
3628
  result = export_findings(connection, args)
3580
3629
  elif args.command == "database-info":
3581
3630
  result = {"databasePath": str(database_path())}
3631
+ elif args.command == "severity-classification":
3632
+ result = severity.checkpoint(connection, json.load(sys.stdin), now())
3582
3633
  elif args.command == "finding-workflow":
3583
3634
  result = finding_workflow(connection, json.load(sys.stdin), now())
3584
3635
  elif args.command == "dashboard":
@@ -112,13 +112,13 @@ def _latest_successful_reducer(workers: list[Any]) -> Any | None:
112
112
  )
113
113
 
114
114
 
115
- def _saved_result_paths(scan_dir: Path, workers: list[Any]) -> Iterator[str]:
115
+ def _saved_result_paths(scan_dir: Path, workers: list[Any]) -> Iterator[tuple[str, str | None]]:
116
116
  latest_reducer = _latest_successful_reducer(workers)
117
117
 
118
- def checkpoints(directory: str) -> Iterator[str]:
118
+ def checkpoints(directory: str, kind: str | None = None) -> Iterator[tuple[str, str | None]]:
119
119
  for name in _children(scan_dir, directory):
120
120
  if re.fullmatch(r"[0-9a-f]{64}\.json", name):
121
- yield f"{directory}/{name}"
121
+ yield f"{directory}/{name}", kind
122
122
 
123
123
  yield from checkpoints("checkpoints")
124
124
  for worker in workers:
@@ -137,25 +137,32 @@ def _saved_result_paths(scan_dir: Path, workers: list[Any]) -> Iterator[str]:
137
137
  if re.fullmatch(r"attempt-\d+", name)
138
138
  ]
139
139
  for directory in directories:
140
- checkpoint_paths = list(checkpoints(f"{directory}/checkpoints"))
140
+ checkpoint_paths = list(checkpoints(f"{directory}/checkpoints", worker["kind"]))
141
141
  if worker["kind"] == "discovery" or checkpoint_paths:
142
- yield f"{directory}/result.json"
142
+ yield f"{directory}/result.json", worker["kind"]
143
143
  yield from checkpoint_paths
144
144
  if worker["result_manifest_path"] and (
145
145
  worker["kind"] == "discovery"
146
146
  or (latest_reducer is not None and worker["id"] == latest_reducer["id"])
147
147
  ):
148
148
  try:
149
- yield Path(worker["result_manifest_path"]).relative_to(scan_dir).as_posix()
149
+ yield (
150
+ Path(worker["result_manifest_path"]).relative_to(scan_dir).as_posix(),
151
+ worker["kind"],
152
+ )
150
153
  except ValueError:
151
154
  continue
152
155
 
153
156
 
154
- def _read_saved_result(scan_dir: Path, relative: str, scan_id: str) -> tuple[dict[str, Any], str]:
157
+ def _read_saved_result(
158
+ scan_dir: Path, relative: str, scan_id: str, *, kind: str | None = None
159
+ ) -> tuple[dict[str, Any], str]:
155
160
  draft = _read_scan_local_json(scan_dir, relative, "Saved scan checkpoint")
156
161
  if draft.get("scanId") != scan_id:
157
162
  raise ContractError("checkpoint belongs to a different scan")
158
- if not isinstance(draft.get("findings"), list) or not isinstance(draft.get("coverage"), dict):
163
+ if not isinstance(draft.get("findings"), list) or not isinstance(
164
+ draft.get("coverage", {} if kind == "dedup" else None), dict
165
+ ):
159
166
  raise ContractError("checkpoint has no semantic findings or coverage")
160
167
  return draft, _digest(draft)
161
168
 
@@ -207,13 +214,13 @@ def _saved_results_changed(db: Any, connection: Any, scan: Any) -> bool:
207
214
  "FROM deep_scan_workers WHERE scan_id = ?",
208
215
  (scan["id"],),
209
216
  ).fetchall()
210
- paths = set(_saved_result_paths(scan_dir, workers))
217
+ paths = dict(_saved_result_paths(scan_dir, workers))
211
218
  frozen_sources = scan["retained_source_digests_json"]
212
219
 
213
220
  def has_saved_source() -> bool:
214
221
  for path in paths:
215
222
  try:
216
- _read_saved_result(scan_dir, path, scan["id"])
223
+ _read_saved_result(scan_dir, path, scan["id"], kind=paths[path])
217
224
  return True
218
225
  except (ContractError, OSError, ValueError):
219
226
  continue
@@ -244,7 +251,9 @@ def _saved_results_changed(db: Any, connection: Any, scan: Any) -> bool:
244
251
  current_sources = dict(published_sources)
245
252
  for path in paths:
246
253
  try:
247
- _, current_sources[path] = _read_saved_result(scan_dir, path, scan["id"])
254
+ _, current_sources[path] = _read_saved_result(
255
+ scan_dir, path, scan["id"], kind=paths[path]
256
+ )
248
257
  except (ContractError, OSError, ValueError):
249
258
  continue
250
259
  return current_sources != published_sources
@@ -290,23 +299,26 @@ def _recovery_source_digests(db: Any, connection: Any, scan: Any) -> tuple[dict[
290
299
  else:
291
300
  include_parent = True
292
301
 
302
+ workers = connection.execute(
303
+ "SELECT id, kind, status, completed_at, artifact_dir, result_manifest_path "
304
+ "FROM deep_scan_workers WHERE scan_id = ?",
305
+ (scan["id"],),
306
+ ).fetchall()
307
+ paths = dict(_saved_result_paths(scan_dir, workers))
293
308
  recovery_sources = dict(frozen_sources or {})
294
309
  for relative, expected_digest in recovery_sources.items():
295
310
  try:
296
- _, digest = _read_saved_result(scan_dir, relative, scan["id"])
311
+ _, digest = _read_saved_result(scan_dir, relative, scan["id"], kind=paths.get(relative))
297
312
  except (ContractError, OSError, ValueError) as exc:
298
313
  raise ContractError("Frozen stopped-scan checkpoint set is incomplete.") from exc
299
314
  if digest != expected_digest:
300
315
  raise ContractError("checkpoint changed after the scan stopped")
301
316
 
302
- workers = connection.execute(
303
- "SELECT id, kind, status, completed_at, artifact_dir, result_manifest_path "
304
- "FROM deep_scan_workers WHERE scan_id = ?",
305
- (scan["id"],),
306
- ).fetchall()
307
- for relative in set(_saved_result_paths(scan_dir, workers)) - recovery_sources.keys():
317
+ for relative in paths.keys() - recovery_sources.keys():
308
318
  try:
309
- _, recovery_sources[relative] = _read_saved_result(scan_dir, relative, scan["id"])
319
+ _, recovery_sources[relative] = _read_saved_result(
320
+ scan_dir, relative, scan["id"], kind=paths[relative]
321
+ )
310
322
  except (ContractError, OSError, ValueError):
311
323
  continue
312
324
  return recovery_sources, include_parent
@@ -489,6 +501,7 @@ def merge_saved_results(
489
501
  parent_preserved_sources = recorded
490
502
  source_digests.update(parent_preserved_sources)
491
503
  paths: dict[str, str | None] = {}
504
+ reducer_paths: set[str] = set()
492
505
  current_results: set[str] = set()
493
506
  reducer_outputs: list[tuple[Any, str, list[str], int]] = []
494
507
  reducer = _latest_successful_reducer(workers)
@@ -497,6 +510,7 @@ def merge_saved_results(
497
510
  try:
498
511
  latest_reducer = Path(reducer["result_manifest_path"]).relative_to(scan_dir).as_posix()
499
512
  paths[latest_reducer] = None
513
+ reducer_paths.add(latest_reducer)
500
514
  except ValueError:
501
515
  warnings.append("Skipped a reducer result outside the scan directory.")
502
516
 
@@ -526,6 +540,7 @@ def merge_saved_results(
526
540
  paths[result_path] = None
527
541
  for checkpoint_path in checkpoint_paths:
528
542
  paths[checkpoint_path] = None
543
+ reducer_paths.update([result_path, *checkpoint_paths])
529
544
  reducer_outputs.append((reducer_worker, result_path, checkpoint_paths, attempt))
530
545
 
531
546
  reducer_output(output, int(worker["attempt"] or 0), worker)
@@ -570,11 +585,15 @@ def merge_saved_results(
570
585
 
571
586
  for relative, worker_id in paths.items():
572
587
  try:
573
- draft, digest = _read_saved_result(scan_dir, relative, scan_id)
588
+ draft, digest = _read_saved_result(
589
+ scan_dir, relative, scan_id, kind="dedup" if relative in reducer_paths else None
590
+ )
574
591
  if frozen_source_digests is not None and frozen_source_digests[relative] != digest:
575
592
  raise ContractError("checkpoint changed after the scan stopped")
576
593
  source_digests[relative] = digest
577
- sources.append((relative, draft, worker_id))
594
+ # Recovery expects coverage, but reducer results only contain findings
595
+ # and context. Add an empty value after hashing the original result.
596
+ sources.append((relative, {"coverage": {}, **draft}, worker_id))
578
597
  except (ContractError, OSError, ValueError) as exc:
579
598
  if (scan_dir / relative).exists():
580
599
  warnings.append(f"Preserved unreadable checkpoint {relative}: {exc}")
@@ -645,7 +664,7 @@ def merge_saved_results(
645
664
  manifest["scan"]["preservedSources"] = source_digests
646
665
  coverage = (
647
666
  copy.deepcopy(parent["coverage"])
648
- if parent
667
+ if parent and parent["coverage"]
649
668
  else {
650
669
  "completeness": "partial",
651
670
  "mode": binding["coverageMode"],