@openai/codex-security 0.1.23 → 0.1.25
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +200 -23
- package/_bundled_plugin/.codex-plugin/plugin.json +1 -1
- package/_bundled_plugin/.mcp.json +1 -0
- package/_bundled_plugin/mcp/server.mjs.br.part-000 +0 -0
- package/_bundled_plugin/mcp/server.mjs.br.part-001 +0 -0
- package/_bundled_plugin/references/scan-contract.md +10 -0
- package/_bundled_plugin/scripts/deep_scan_workbench.py +6 -1
- package/_bundled_plugin/scripts/finalize_scan_contract.py +27 -78
- package/_bundled_plugin/scripts/generate_in_scope_files.py +2 -8
- package/_bundled_plugin/scripts/generate_rank_input.py +1 -5
- package/_bundled_plugin/scripts/rank_preview.py +12 -1
- package/_bundled_plugin/scripts/report_projection.py +8 -14
- package/_bundled_plugin/scripts/validate_scan_contract.py +1 -3
- package/_bundled_plugin/scripts/windows_scan_local_files.py +5 -15
- package/_bundled_plugin/scripts/workbench_cli.py +14 -1
- package/_bundled_plugin/scripts/workbench_dashboard.py +41 -18
- package/_bundled_plugin/scripts/workbench_db.py +127 -541
- package/_bundled_plugin/scripts/workbench_finding_workflows.py +31 -13
- package/_bundled_plugin/scripts/workbench_findings.py +8 -3
- package/_bundled_plugin/scripts/workbench_native_indexes.py +1 -2
- package/_bundled_plugin/scripts/workbench_publication.py +520 -0
- package/_bundled_plugin/scripts/workbench_remediation.py +37 -0
- package/_bundled_plugin/scripts/workbench_saved_results.py +298 -90
- package/_bundled_plugin/scripts/workbench_scan_history.py +375 -68
- package/_bundled_plugin/scripts/workbench_scan_usage.py +9 -33
- package/_bundled_plugin/scripts/workbench_schema.py +49 -21
- package/_bundled_plugin/scripts/workbench_target.py +96 -17
- package/_bundled_plugin/scripts/workbench_validation.py +1 -5
- package/_bundled_plugin/skills/assess-patch-risk/SKILL.md +1 -5
- package/_bundled_plugin/skills/attack-path-analysis/references/severity-policy.md +6 -1
- package/dist/api.d.ts +18 -3
- package/dist/api.d.ts.map +1 -1
- package/dist/api.js +350 -28
- package/dist/api.js.map +1 -1
- package/dist/auth.d.ts.map +1 -1
- package/dist/auth.js +35 -3
- package/dist/auth.js.map +1 -1
- package/dist/cli.d.ts +2 -0
- package/dist/cli.d.ts.map +1 -1
- package/dist/cli.js +118 -35
- package/dist/cli.js.map +1 -1
- package/dist/component-plan.d.ts.map +1 -1
- package/dist/component-plan.js +4 -2
- package/dist/component-plan.js.map +1 -1
- package/dist/component-scan.d.ts.map +1 -1
- package/dist/component-scan.js +21 -6
- package/dist/component-scan.js.map +1 -1
- package/dist/config.d.ts.map +1 -1
- package/dist/config.js +46 -2
- package/dist/config.js.map +1 -1
- package/dist/cost.d.ts +1 -0
- package/dist/cost.d.ts.map +1 -1
- package/dist/cost.js +23 -11
- package/dist/cost.js.map +1 -1
- package/dist/deduplication/checkpointed-review.d.ts.map +1 -1
- package/dist/deduplication/checkpointed-review.js +2 -1
- package/dist/deduplication/checkpointed-review.js.map +1 -1
- package/dist/deduplication/codex-review.d.ts +3 -0
- package/dist/deduplication/codex-review.d.ts.map +1 -1
- package/dist/deduplication/codex-review.js +122 -36
- package/dist/deduplication/codex-review.js.map +1 -1
- package/dist/deduplication/deduplication-prompts.d.ts +4 -3
- package/dist/deduplication/deduplication-prompts.d.ts.map +1 -1
- package/dist/deduplication/deduplication-prompts.js +18 -9
- package/dist/deduplication/deduplication-prompts.js.map +1 -1
- package/dist/deduplication/deduplication-reviewer.d.ts +2 -7
- package/dist/deduplication/deduplication-reviewer.d.ts.map +1 -1
- package/dist/deduplication/deduplication-reviewer.js +46 -24
- package/dist/deduplication/deduplication-reviewer.js.map +1 -1
- package/dist/deduplication/deduplication.d.ts.map +1 -1
- package/dist/deduplication/deduplication.js +186 -23
- package/dist/deduplication/deduplication.js.map +1 -1
- package/dist/deduplication/scan.d.ts +9 -1
- package/dist/deduplication/scan.d.ts.map +1 -1
- package/dist/deduplication/scan.js +24 -7
- package/dist/deduplication/scan.js.map +1 -1
- package/dist/deep-progress.d.ts +21 -0
- package/dist/deep-progress.d.ts.map +1 -0
- package/dist/deep-progress.js +108 -0
- package/dist/deep-progress.js.map +1 -0
- package/dist/errors.d.ts +13 -0
- package/dist/errors.d.ts.map +1 -1
- package/dist/errors.js +7 -0
- package/dist/errors.js.map +1 -1
- package/dist/index.d.ts +6 -5
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +2 -2
- package/dist/index.js.map +1 -1
- package/dist/mock-scan.d.ts +4 -0
- package/dist/mock-scan.d.ts.map +1 -0
- package/dist/mock-scan.js +210 -0
- package/dist/mock-scan.js.map +1 -0
- package/dist/publish.d.ts.map +1 -1
- package/dist/publish.js +2 -2
- package/dist/publish.js.map +1 -1
- package/dist/runtime.d.ts +1 -0
- package/dist/runtime.d.ts.map +1 -1
- package/dist/runtime.js +54 -14
- package/dist/runtime.js.map +1 -1
- package/dist/scan-activity.d.ts.map +1 -1
- package/dist/scan-activity.js +12 -2
- package/dist/scan-activity.js.map +1 -1
- package/dist/scan-comparison.d.ts +11 -2
- package/dist/scan-comparison.d.ts.map +1 -1
- package/dist/scan-comparison.js +237 -39
- package/dist/scan-comparison.js.map +1 -1
- package/dist/scan-dashboard.d.ts +3 -1
- package/dist/scan-dashboard.d.ts.map +1 -1
- package/dist/scan-dashboard.js +70 -3
- package/dist/scan-dashboard.js.map +1 -1
- package/dist/scan-history-renderer.d.ts.map +1 -1
- package/dist/scan-history-renderer.js +21 -0
- package/dist/scan-history-renderer.js.map +1 -1
- package/dist/server/api.d.ts +6 -0
- package/dist/server/api.d.ts.map +1 -0
- package/dist/server/api.js +4 -0
- package/dist/server/api.js.map +1 -0
- package/dist/server/dashboard/app.css +1 -1
- package/dist/server/embeddings.d.ts +2 -1
- package/dist/server/embeddings.d.ts.map +1 -1
- package/dist/server/embeddings.js +8 -3
- package/dist/server/embeddings.js.map +1 -1
- package/dist/server/index.js +2 -28
- package/dist/server/index.js.map +1 -1
- package/dist/server/serve.d.ts +2 -0
- package/dist/server/serve.d.ts.map +1 -0
- package/dist/server/serve.js +28 -0
- package/dist/server/serve.js.map +1 -0
- package/dist/targets.d.ts.map +1 -1
- package/dist/targets.js +3 -5
- package/dist/targets.js.map +1 -1
- package/dist/version.d.ts +1 -1
- package/dist/version.js +1 -1
- package/dist/version.js.map +1 -1
- package/package.json +10 -5
|
@@ -6,6 +6,8 @@ import json
|
|
|
6
6
|
import os
|
|
7
7
|
import sqlite3
|
|
8
8
|
import sys
|
|
9
|
+
from collections.abc import Iterable, Iterator
|
|
10
|
+
from itertools import chain
|
|
9
11
|
from pathlib import Path, PurePosixPath
|
|
10
12
|
from typing import Any, Callable
|
|
11
13
|
from urllib.parse import urlsplit
|
|
@@ -28,10 +30,12 @@ def _same_repository(
|
|
|
28
30
|
*,
|
|
29
31
|
after_identity: tuple[str | None, tuple[str, str] | None] | None = None,
|
|
30
32
|
) -> bool:
|
|
31
|
-
if before["target_id"] == after["target_id"]:
|
|
33
|
+
if before["target_id"] is not None and before["target_id"] == after["target_id"]:
|
|
32
34
|
return True
|
|
33
35
|
before_target = Path(before["target_path"])
|
|
34
36
|
after_target = Path(after["target_path"])
|
|
37
|
+
if before_target.resolve() == after_target.resolve():
|
|
38
|
+
return True
|
|
35
39
|
before_git_dir = git_output(
|
|
36
40
|
before_target, "rev-parse", "--path-format=absolute", "--git-common-dir"
|
|
37
41
|
)
|
|
@@ -253,7 +257,7 @@ def list_unmatched_scan_pairs(
|
|
|
253
257
|
for scan in connection.execute(
|
|
254
258
|
"SELECT * FROM scans WHERE status = 'complete' ORDER BY started_at, id"
|
|
255
259
|
)
|
|
256
|
-
if
|
|
260
|
+
if _same_repository(scan, requested)
|
|
257
261
|
]
|
|
258
262
|
|
|
259
263
|
available = []
|
|
@@ -271,6 +275,7 @@ def list_unmatched_scan_pairs(
|
|
|
271
275
|
batches = []
|
|
272
276
|
skipped = 0
|
|
273
277
|
matching_findings: dict[str, list[dict[str, Any]]] = {}
|
|
278
|
+
known_links: list[sqlite3.Row] | None = None
|
|
274
279
|
for index, after in enumerate(available):
|
|
275
280
|
previous = [
|
|
276
281
|
before
|
|
@@ -280,13 +285,26 @@ def list_unmatched_scan_pairs(
|
|
|
280
285
|
skipped += index - len(previous)
|
|
281
286
|
if not previous:
|
|
282
287
|
continue
|
|
288
|
+
if known_links is None:
|
|
289
|
+
known_links = (
|
|
290
|
+
[]
|
|
291
|
+
if args.force
|
|
292
|
+
else _saved_finding_links(connection, {scan["id"] for scan in selected})
|
|
293
|
+
)
|
|
283
294
|
for scan in (*previous, after):
|
|
284
295
|
if scan["id"] not in matching_findings:
|
|
285
296
|
backfill_finding_details(connection, scan)
|
|
286
297
|
matching_findings[scan["id"]] = [
|
|
287
|
-
_matching_input(row)
|
|
288
|
-
for row in _scan_findings(connection, scan["id"]).values()
|
|
298
|
+
_matching_input(row) for row in _scan_findings(connection, scan["id"]).values()
|
|
289
299
|
]
|
|
300
|
+
known_groups = _known_finding_groups(
|
|
301
|
+
known_links,
|
|
302
|
+
{
|
|
303
|
+
scan["id"]
|
|
304
|
+
for scan in selected
|
|
305
|
+
if (scan["started_at"], scan["id"]) <= (after["started_at"], after["id"])
|
|
306
|
+
},
|
|
307
|
+
)
|
|
290
308
|
batches.append(
|
|
291
309
|
{
|
|
292
310
|
"afterFindings": matching_findings[after["id"]],
|
|
@@ -298,6 +316,7 @@ def list_unmatched_scan_pairs(
|
|
|
298
316
|
}
|
|
299
317
|
for before in previous
|
|
300
318
|
],
|
|
319
|
+
**({"knownFindingGroups": known_groups} if known_groups else {}),
|
|
301
320
|
}
|
|
302
321
|
)
|
|
303
322
|
return {
|
|
@@ -309,6 +328,53 @@ def list_unmatched_scan_pairs(
|
|
|
309
328
|
}
|
|
310
329
|
|
|
311
330
|
|
|
331
|
+
def _saved_finding_links(connection: sqlite3.Connection, scan_ids: set[str]) -> list[sqlite3.Row]:
|
|
332
|
+
return [
|
|
333
|
+
row
|
|
334
|
+
for row in _rows_for_ids(
|
|
335
|
+
connection,
|
|
336
|
+
"""
|
|
337
|
+
SELECT before.scan_id AS before_scan_id, before.finding_id AS before_finding_id,
|
|
338
|
+
after.scan_id AS after_scan_id, after.finding_id AS after_finding_id
|
|
339
|
+
FROM scan_comparison_matches AS matches
|
|
340
|
+
JOIN finding_occurrences AS before ON before.id = matches.before_occurrence_id
|
|
341
|
+
JOIN finding_occurrences AS after ON after.id = matches.after_occurrence_id
|
|
342
|
+
WHERE matches.before_scan_id IN ({placeholders})
|
|
343
|
+
ORDER BY matches.before_scan_id, after.scan_id, before.finding_id, after.finding_id
|
|
344
|
+
""",
|
|
345
|
+
sorted(scan_ids),
|
|
346
|
+
)
|
|
347
|
+
if row["before_scan_id"] in scan_ids and row["after_scan_id"] in scan_ids
|
|
348
|
+
]
|
|
349
|
+
|
|
350
|
+
|
|
351
|
+
def _finding_aliases(links: Iterable[tuple[str, str]]) -> dict[str, str]:
|
|
352
|
+
parents: dict[str, str] = {}
|
|
353
|
+
|
|
354
|
+
def root(value: str) -> str:
|
|
355
|
+
parents.setdefault(value, value)
|
|
356
|
+
while parents[value] != value:
|
|
357
|
+
parents[value] = parents[parents[value]]
|
|
358
|
+
value = parents[value]
|
|
359
|
+
return value
|
|
360
|
+
|
|
361
|
+
for before_id, after_id in links:
|
|
362
|
+
parents[root(after_id)] = root(before_id)
|
|
363
|
+
return {finding_id: root(finding_id) for finding_id in parents}
|
|
364
|
+
|
|
365
|
+
|
|
366
|
+
def _known_finding_groups(links: list[sqlite3.Row], scan_ids: set[str]) -> list[list[str]]:
|
|
367
|
+
aliases = _finding_aliases(
|
|
368
|
+
(link["before_finding_id"], link["after_finding_id"])
|
|
369
|
+
for link in links
|
|
370
|
+
if link["before_scan_id"] in scan_ids and link["after_scan_id"] in scan_ids
|
|
371
|
+
)
|
|
372
|
+
groups: dict[str, list[str]] = {}
|
|
373
|
+
for finding_id, identity in aliases.items():
|
|
374
|
+
groups.setdefault(identity, []).append(finding_id)
|
|
375
|
+
return sorted(sorted(group) for group in groups.values() if len(group) > 1)
|
|
376
|
+
|
|
377
|
+
|
|
312
378
|
def compare_scans(
|
|
313
379
|
connection: sqlite3.Connection,
|
|
314
380
|
args: argparse.Namespace,
|
|
@@ -352,7 +418,12 @@ def compare_scans(
|
|
|
352
418
|
before_findings = _scan_findings(connection, before["id"])
|
|
353
419
|
after_findings = _scan_findings(connection, after["id"])
|
|
354
420
|
matches = json.loads(cached["result_json"]) if cached is not None else None
|
|
355
|
-
|
|
421
|
+
saved_matches = matches["matches"] if matches is not None else []
|
|
422
|
+
occurrences = {
|
|
423
|
+
row["id"]: row for row in chain(before_findings.values(), after_findings.values())
|
|
424
|
+
}
|
|
425
|
+
aliases = _confirmed_finding_aliases(connection, occurrences)
|
|
426
|
+
groups = _finding_groups(before_findings, after_findings, saved_matches, aliases)
|
|
356
427
|
uncertain = (
|
|
357
428
|
{
|
|
358
429
|
(side, match[f"{side}OccurrenceId"]): match["reason"]
|
|
@@ -366,23 +437,33 @@ def compare_scans(
|
|
|
366
437
|
summary = {status: 0 for status in ("new", "persisting", "resolved", "reopened", "unknown")}
|
|
367
438
|
|
|
368
439
|
for previous_rows, current_rows, match_reason in groups:
|
|
369
|
-
previous =
|
|
440
|
+
previous = (
|
|
441
|
+
min(previous_rows, key=lambda row: SEVERITY_ORDER[row["severity"]])
|
|
442
|
+
if previous_rows
|
|
443
|
+
else None
|
|
444
|
+
)
|
|
370
445
|
current = (
|
|
371
446
|
min(current_rows, key=lambda row: SEVERITY_ORDER[row["severity"]])
|
|
372
447
|
if current_rows
|
|
373
448
|
else None
|
|
374
449
|
)
|
|
375
450
|
selected = current if current is not None else previous
|
|
376
|
-
if selected is None:
|
|
377
|
-
continue
|
|
378
451
|
item = {
|
|
379
452
|
"findingId": selected["finding_id"],
|
|
380
453
|
"path": selected["relative_path"],
|
|
381
454
|
"severity": selected["severity"],
|
|
382
455
|
"title": selected["title"],
|
|
383
456
|
}
|
|
457
|
+
side = "after" if current_rows else "before"
|
|
458
|
+
uncertain_reason = next(
|
|
459
|
+
(
|
|
460
|
+
uncertain[(side, row["id"])]
|
|
461
|
+
for row in current_rows or previous_rows
|
|
462
|
+
if (side, row["id"]) in uncertain
|
|
463
|
+
),
|
|
464
|
+
None,
|
|
465
|
+
)
|
|
384
466
|
if previous is None:
|
|
385
|
-
uncertain_reason = uncertain.get(("after", current["id"])) if current else None
|
|
386
467
|
if uncertain_reason is None:
|
|
387
468
|
status = "new"
|
|
388
469
|
else:
|
|
@@ -400,17 +481,20 @@ def compare_scans(
|
|
|
400
481
|
)
|
|
401
482
|
if match_reason is not None:
|
|
402
483
|
item["matchReason"] = match_reason
|
|
403
|
-
elif
|
|
484
|
+
elif uncertain_reason is not None:
|
|
404
485
|
status = "unknown"
|
|
405
486
|
item["reason"] = uncertain_reason
|
|
406
487
|
elif not comparable:
|
|
407
488
|
status = "unknown"
|
|
408
489
|
item["reason"] = "The later scan has incomplete coverage."
|
|
409
|
-
elif not
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
413
|
-
|
|
490
|
+
elif not all(
|
|
491
|
+
scan_covers_path(
|
|
492
|
+
after,
|
|
493
|
+
target_id=after["target_id"],
|
|
494
|
+
path=row["relative_path"],
|
|
495
|
+
coverage=after_coverage,
|
|
496
|
+
)
|
|
497
|
+
for row in previous_rows
|
|
414
498
|
):
|
|
415
499
|
status = "unknown"
|
|
416
500
|
item["reason"] = "The affected path was excluded or outside the later scope."
|
|
@@ -442,11 +526,41 @@ def compare_scans(
|
|
|
442
526
|
"repository": before["target_path"],
|
|
443
527
|
"summary": summary,
|
|
444
528
|
}
|
|
529
|
+
if matches is not None and matches.get("related"):
|
|
530
|
+
related = _separate_finding_pairs(matches["related"], occurrences, aliases)
|
|
531
|
+
if related:
|
|
532
|
+
result["related"] = [
|
|
533
|
+
{
|
|
534
|
+
**pair,
|
|
535
|
+
"beforeTitle": occurrences[pair["beforeOccurrenceId"]]["title"],
|
|
536
|
+
"afterTitle": occurrences[pair["afterOccurrenceId"]]["title"],
|
|
537
|
+
}
|
|
538
|
+
for pair in related
|
|
539
|
+
]
|
|
445
540
|
if include_matching_inputs:
|
|
541
|
+
known_scan_ids = {
|
|
542
|
+
scan["id"]
|
|
543
|
+
for scan in connection.execute(
|
|
544
|
+
"SELECT * FROM scans WHERE status = 'complete' "
|
|
545
|
+
"AND (started_at < ? OR (started_at = ? AND id <= ?))",
|
|
546
|
+
(after["started_at"], after["started_at"], after["id"]),
|
|
547
|
+
)
|
|
548
|
+
if _same_repository(scan, after)
|
|
549
|
+
}
|
|
550
|
+
excluded_pairs = {(before["id"], after["id"]), (after["id"], before["id"])}
|
|
551
|
+
known_groups = _known_finding_groups(
|
|
552
|
+
[
|
|
553
|
+
link
|
|
554
|
+
for link in _saved_finding_links(connection, known_scan_ids)
|
|
555
|
+
if (link["before_scan_id"], link["after_scan_id"]) not in excluded_pairs
|
|
556
|
+
],
|
|
557
|
+
known_scan_ids,
|
|
558
|
+
)
|
|
446
559
|
result["matchingCached"] = cached is not None
|
|
447
560
|
result["matchingInputs"] = {
|
|
448
561
|
"before": [_matching_input(row) for row in before_findings.values()],
|
|
449
562
|
"after": [_matching_input(row) for row in after_findings.values()],
|
|
563
|
+
**({"knownFindingGroups": known_groups} if known_groups else {}),
|
|
450
564
|
}
|
|
451
565
|
return result
|
|
452
566
|
|
|
@@ -470,42 +584,80 @@ def save_scan_comparison(
|
|
|
470
584
|
read_coverage(after)
|
|
471
585
|
before_findings = _scan_findings(connection, before["id"])
|
|
472
586
|
after_findings = _scan_findings(connection, after["id"])
|
|
473
|
-
matches_json =
|
|
587
|
+
matches_json = (
|
|
588
|
+
sys.stdin.read() if getattr(args, "matches_json_stdin", False) else args.matches_json
|
|
589
|
+
)
|
|
474
590
|
try:
|
|
475
591
|
payload = json.loads(matches_json)
|
|
476
592
|
except (TypeError, ValueError) as exc:
|
|
477
593
|
raise SystemExit("Scan comparison matches must be a valid JSON object.") from exc
|
|
478
|
-
if
|
|
594
|
+
if (
|
|
595
|
+
not isinstance(payload, dict)
|
|
596
|
+
or not {"matches", "uncertain"}.issubset(payload)
|
|
597
|
+
or set(payload) - {"matches", "uncertain", "related"}
|
|
598
|
+
):
|
|
479
599
|
raise SystemExit("Scan comparison matches must contain matches and uncertain arrays.")
|
|
480
|
-
if
|
|
600
|
+
if any(
|
|
601
|
+
not isinstance(payload.get(key, []), list) for key in ("matches", "uncertain", "related")
|
|
602
|
+
):
|
|
481
603
|
raise SystemExit("Scan comparison matches must contain matches and uncertain arrays.")
|
|
482
604
|
allowed = {
|
|
483
605
|
"before": {row["id"] for row in before_findings.values()},
|
|
484
606
|
"after": {row["id"] for row in after_findings.values()},
|
|
485
607
|
}
|
|
486
|
-
consumed: dict[str,
|
|
487
|
-
for match in payload["matches"]:
|
|
608
|
+
consumed: dict[str, dict[str, int]] = {"before": {}, "after": {}}
|
|
609
|
+
for group, match in enumerate(payload["matches"]):
|
|
610
|
+
if (
|
|
611
|
+
not isinstance(match, dict)
|
|
612
|
+
or match.get("confidence") != "high"
|
|
613
|
+
or not isinstance(match.get("reason"), str)
|
|
614
|
+
or not match["reason"].strip()
|
|
615
|
+
):
|
|
616
|
+
raise SystemExit("Scan comparison matches must have high confidence and a reason.")
|
|
488
617
|
for side in ("before", "after"):
|
|
489
|
-
occurrences = match
|
|
618
|
+
occurrences = match.get(f"{side}OccurrenceIds")
|
|
619
|
+
if not isinstance(occurrences, list) or any(
|
|
620
|
+
not isinstance(value, str) for value in occurrences
|
|
621
|
+
):
|
|
622
|
+
raise SystemExit("Scan comparison matches must identify distinct scan findings.")
|
|
490
623
|
unique = set(occurrences)
|
|
491
624
|
if (
|
|
492
625
|
not occurrences
|
|
493
626
|
or len(unique) != len(occurrences)
|
|
494
627
|
or not unique.issubset(allowed[side])
|
|
495
|
-
or not consumed[side]
|
|
628
|
+
or not unique.isdisjoint(consumed[side])
|
|
496
629
|
):
|
|
497
630
|
raise SystemExit("Scan comparison matches must identify distinct scan findings.")
|
|
498
|
-
consumed[side].update(unique)
|
|
631
|
+
consumed[side].update((occurrence_id, group) for occurrence_id in unique)
|
|
499
632
|
uncertain_pairs = set()
|
|
500
633
|
for match in payload["uncertain"]:
|
|
634
|
+
if not _valid_finding_pair(match):
|
|
635
|
+
raise SystemExit("Uncertain scan comparison matches must identify distinct findings.")
|
|
501
636
|
pair = (match["beforeOccurrenceId"], match["afterOccurrenceId"])
|
|
502
637
|
if (
|
|
503
|
-
pair[0] not in allowed["before"]
|
|
504
|
-
or pair[
|
|
638
|
+
pair[0] not in allowed["before"]
|
|
639
|
+
or pair[0] in consumed["before"]
|
|
640
|
+
or pair[1] not in allowed["after"]
|
|
641
|
+
or pair[1] in consumed["after"]
|
|
505
642
|
or pair in uncertain_pairs
|
|
506
643
|
):
|
|
507
644
|
raise SystemExit("Uncertain scan comparison matches must identify distinct findings.")
|
|
508
645
|
uncertain_pairs.add(pair)
|
|
646
|
+
related_pairs = set()
|
|
647
|
+
for match in payload.get("related", []):
|
|
648
|
+
if not _valid_finding_pair(match):
|
|
649
|
+
raise SystemExit("Related scan comparison findings must identify distinct findings.")
|
|
650
|
+
pair = (match["beforeOccurrenceId"], match["afterOccurrenceId"])
|
|
651
|
+
group = consumed["before"].get(pair[0])
|
|
652
|
+
if (
|
|
653
|
+
pair[0] not in allowed["before"]
|
|
654
|
+
or pair[1] not in allowed["after"]
|
|
655
|
+
or (group is not None and group == consumed["after"].get(pair[1]))
|
|
656
|
+
or pair in uncertain_pairs
|
|
657
|
+
or pair in related_pairs
|
|
658
|
+
):
|
|
659
|
+
raise SystemExit("Related scan comparison findings must identify distinct findings.")
|
|
660
|
+
related_pairs.add(pair)
|
|
509
661
|
timestamp = now()
|
|
510
662
|
with connection:
|
|
511
663
|
connection.execute("BEGIN IMMEDIATE")
|
|
@@ -543,6 +695,142 @@ def save_scan_comparison(
|
|
|
543
695
|
return compare_scans(connection, args, require_scan=require_scan, read_coverage=read_coverage)
|
|
544
696
|
|
|
545
697
|
|
|
698
|
+
def _valid_finding_pair(value: Any) -> bool:
|
|
699
|
+
return (
|
|
700
|
+
isinstance(value, dict)
|
|
701
|
+
and set(value) == {"beforeOccurrenceId", "afterOccurrenceId", "reason"}
|
|
702
|
+
and all(isinstance(item, str) and item.strip() for item in value.values())
|
|
703
|
+
)
|
|
704
|
+
|
|
705
|
+
|
|
706
|
+
def _rows_for_ids(
|
|
707
|
+
connection: sqlite3.Connection, query: str, ids: Iterable[str]
|
|
708
|
+
) -> Iterator[sqlite3.Row]:
|
|
709
|
+
values = tuple(dict.fromkeys(ids))
|
|
710
|
+
getlimit = getattr(connection, "getlimit", None)
|
|
711
|
+
# Python 3.10 lacks getlimit; 999 is SQLite's older host-parameter limit.
|
|
712
|
+
limit = getlimit(sqlite3.SQLITE_LIMIT_VARIABLE_NUMBER) if getlimit else 999
|
|
713
|
+
for start in range(0, len(values), limit):
|
|
714
|
+
batch = values[start : start + limit]
|
|
715
|
+
yield from connection.execute(
|
|
716
|
+
query.format(placeholders=", ".join("?" for _ in batch)), batch
|
|
717
|
+
)
|
|
718
|
+
|
|
719
|
+
|
|
720
|
+
# Stable finding IDs already include the target identity. Follow their indexed
|
|
721
|
+
# occurrences instead of resolving repository paths or scanning every saved link.
|
|
722
|
+
_FINDING_NEIGHBORS_SQL = """
|
|
723
|
+
FROM linked
|
|
724
|
+
CROSS JOIN finding_occurrences AS source
|
|
725
|
+
ON source.finding_id = linked.finding_id
|
|
726
|
+
CROSS JOIN scan_comparison_matches AS matches
|
|
727
|
+
ON matches.before_occurrence_id = source.id OR matches.after_occurrence_id = source.id
|
|
728
|
+
CROSS JOIN finding_occurrences AS neighbor ON neighbor.id = CASE
|
|
729
|
+
WHEN matches.before_occurrence_id = source.id THEN matches.after_occurrence_id
|
|
730
|
+
ELSE matches.before_occurrence_id END
|
|
731
|
+
"""
|
|
732
|
+
# Traverse only the selected findings' components, including recurring stable IDs.
|
|
733
|
+
_LINKED_FINDINGS_SQL = f"""
|
|
734
|
+
WITH RECURSIVE linked(finding_id) AS (
|
|
735
|
+
SELECT occurrences.finding_id
|
|
736
|
+
FROM finding_occurrences AS occurrences
|
|
737
|
+
WHERE occurrences.id IN ({{placeholders}})
|
|
738
|
+
UNION
|
|
739
|
+
SELECT neighbor.finding_id
|
|
740
|
+
{_FINDING_NEIGHBORS_SQL}
|
|
741
|
+
)
|
|
742
|
+
"""
|
|
743
|
+
|
|
744
|
+
|
|
745
|
+
def _confirmed_finding_aliases(
|
|
746
|
+
connection: sqlite3.Connection, occurrence_ids: Iterable[str]
|
|
747
|
+
) -> dict[str, str]:
|
|
748
|
+
query = f"""
|
|
749
|
+
{_LINKED_FINDINGS_SQL}
|
|
750
|
+
SELECT DISTINCT linked.finding_id AS before_finding_id,
|
|
751
|
+
neighbor.finding_id AS after_finding_id
|
|
752
|
+
{_FINDING_NEIGHBORS_SQL}
|
|
753
|
+
"""
|
|
754
|
+
return _finding_aliases(
|
|
755
|
+
(row["before_finding_id"], row["after_finding_id"])
|
|
756
|
+
for row in _rows_for_ids(connection, query, occurrence_ids)
|
|
757
|
+
)
|
|
758
|
+
|
|
759
|
+
|
|
760
|
+
def _separate_finding_pairs(
|
|
761
|
+
pairs: list[dict[str, Any]],
|
|
762
|
+
occurrences: dict[str, sqlite3.Row],
|
|
763
|
+
aliases: dict[str, str],
|
|
764
|
+
) -> list[dict[str, Any]]:
|
|
765
|
+
def identity(occurrence_id: str) -> str:
|
|
766
|
+
finding_id = occurrences[occurrence_id]["finding_id"]
|
|
767
|
+
return aliases.get(finding_id, finding_id)
|
|
768
|
+
|
|
769
|
+
return [
|
|
770
|
+
pair
|
|
771
|
+
for pair in pairs
|
|
772
|
+
if identity(pair["beforeOccurrenceId"]) != identity(pair["afterOccurrenceId"])
|
|
773
|
+
]
|
|
774
|
+
|
|
775
|
+
|
|
776
|
+
def finding_relations(
|
|
777
|
+
connection: sqlite3.Connection, scan_id: str, occurrence_ids: Iterable[str]
|
|
778
|
+
) -> dict[str, list[dict[str, Any]]]:
|
|
779
|
+
selected = set(occurrence_ids)
|
|
780
|
+
if not selected:
|
|
781
|
+
return {}
|
|
782
|
+
pairs = []
|
|
783
|
+
for comparison in connection.execute(
|
|
784
|
+
"SELECT before_scan_id, after_scan_id, result_json FROM scan_comparisons "
|
|
785
|
+
"WHERE before_scan_id = ? OR after_scan_id = ? "
|
|
786
|
+
"ORDER BY before_scan_id, after_scan_id",
|
|
787
|
+
(scan_id, scan_id),
|
|
788
|
+
):
|
|
789
|
+
side = "before" if comparison["before_scan_id"] == scan_id else "after"
|
|
790
|
+
other = "after" if side == "before" else "before"
|
|
791
|
+
for pair in json.loads(comparison["result_json"]).get("related", []):
|
|
792
|
+
if pair[f"{side}OccurrenceId"] in selected:
|
|
793
|
+
pairs.append(
|
|
794
|
+
{
|
|
795
|
+
"beforeOccurrenceId": pair[f"{side}OccurrenceId"],
|
|
796
|
+
"afterOccurrenceId": pair[f"{other}OccurrenceId"],
|
|
797
|
+
"afterScanId": comparison[f"{other}_scan_id"],
|
|
798
|
+
"reason": pair["reason"],
|
|
799
|
+
}
|
|
800
|
+
)
|
|
801
|
+
occurrences = {
|
|
802
|
+
row["id"]: row
|
|
803
|
+
for row in _rows_for_ids(
|
|
804
|
+
connection,
|
|
805
|
+
"SELECT id, finding_id, scan_id, title FROM finding_occurrences "
|
|
806
|
+
"WHERE id IN ({placeholders})",
|
|
807
|
+
(pair[key] for pair in pairs for key in ("beforeOccurrenceId", "afterOccurrenceId")),
|
|
808
|
+
)
|
|
809
|
+
}
|
|
810
|
+
pairs = [
|
|
811
|
+
pair
|
|
812
|
+
for pair in pairs
|
|
813
|
+
if pair["beforeOccurrenceId"] in occurrences
|
|
814
|
+
and occurrences[pair["beforeOccurrenceId"]]["scan_id"] == scan_id
|
|
815
|
+
and pair["afterOccurrenceId"] in occurrences
|
|
816
|
+
and occurrences[pair["afterOccurrenceId"]]["scan_id"] == pair["afterScanId"]
|
|
817
|
+
]
|
|
818
|
+
result: dict[str, list[dict[str, Any]]] = {}
|
|
819
|
+
aliases = _confirmed_finding_aliases(connection, (pair["beforeOccurrenceId"] for pair in pairs))
|
|
820
|
+
for pair in _separate_finding_pairs(pairs, occurrences, aliases):
|
|
821
|
+
finding = occurrences[pair["afterOccurrenceId"]]
|
|
822
|
+
result.setdefault(pair["beforeOccurrenceId"], []).append(
|
|
823
|
+
{
|
|
824
|
+
"findingId": finding["finding_id"],
|
|
825
|
+
"occurrenceId": finding["id"],
|
|
826
|
+
"reason": pair["reason"],
|
|
827
|
+
"scanId": pair["afterScanId"],
|
|
828
|
+
"title": finding["title"],
|
|
829
|
+
}
|
|
830
|
+
)
|
|
831
|
+
return result
|
|
832
|
+
|
|
833
|
+
|
|
546
834
|
def finding_matches(
|
|
547
835
|
connection: sqlite3.Connection, occurrence_id: str, scan_id: str, started_at: str
|
|
548
836
|
) -> tuple[list[dict[str, Any]], str, list[str]]:
|
|
@@ -563,34 +851,34 @@ def finding_matches(
|
|
|
563
851
|
""",
|
|
564
852
|
(occurrence_id, occurrence_id),
|
|
565
853
|
).fetchall()
|
|
566
|
-
|
|
567
|
-
|
|
568
|
-
|
|
569
|
-
|
|
570
|
-
|
|
571
|
-
|
|
572
|
-
|
|
573
|
-
|
|
574
|
-
|
|
575
|
-
|
|
576
|
-
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
|
|
588
|
-
|
|
589
|
-
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
|
|
593
|
-
|
|
854
|
+
linked_rows = list(
|
|
855
|
+
_rows_for_ids(
|
|
856
|
+
connection,
|
|
857
|
+
f"""
|
|
858
|
+
{_LINKED_FINDINGS_SQL}
|
|
859
|
+
SELECT occurrences.id AS occurrence_id, occurrences.finding_id, occurrences.title,
|
|
860
|
+
scans.started_at, scans.id AS scan_id
|
|
861
|
+
FROM linked
|
|
862
|
+
CROSS JOIN finding_occurrences AS occurrences
|
|
863
|
+
ON occurrences.finding_id = linked.finding_id
|
|
864
|
+
CROSS JOIN scans ON scans.id = occurrences.scan_id
|
|
865
|
+
""",
|
|
866
|
+
(occurrence_id,),
|
|
867
|
+
)
|
|
868
|
+
)
|
|
869
|
+
known_scans = sorted(
|
|
870
|
+
{(started_at, scan_id)} | {(row["started_at"], row["scan_id"]) for row in linked_rows}
|
|
871
|
+
)
|
|
872
|
+
included = {occurrence_id, *(row["occurrence_id"] for row in rows)}
|
|
873
|
+
rows.extend(
|
|
874
|
+
{
|
|
875
|
+
**row,
|
|
876
|
+
"reason": "The findings share a stable identity or a previously confirmed link.",
|
|
877
|
+
}
|
|
878
|
+
for row in linked_rows
|
|
879
|
+
if row["occurrence_id"] not in included
|
|
880
|
+
)
|
|
881
|
+
rows.sort(key=lambda row: (row["scan_id"], row["occurrence_id"]))
|
|
594
882
|
known_scan_ids = [known_scans[0][1]]
|
|
595
883
|
if len(known_scans) > 1:
|
|
596
884
|
known_scan_ids.append(known_scans[-1][1])
|
|
@@ -613,26 +901,45 @@ def finding_matches(
|
|
|
613
901
|
def _finding_groups(
|
|
614
902
|
before_findings: dict[str, sqlite3.Row],
|
|
615
903
|
after_findings: dict[str, sqlite3.Row],
|
|
616
|
-
matches: dict[str, Any]
|
|
904
|
+
matches: list[dict[str, Any]],
|
|
905
|
+
aliases: dict[str, str],
|
|
617
906
|
) -> list[tuple[list[sqlite3.Row], list[sqlite3.Row], str | None]]:
|
|
618
907
|
rows = {
|
|
619
908
|
side: {row["id"]: row for row in findings.values()}
|
|
620
909
|
for side, findings in (("before", before_findings), ("after", after_findings))
|
|
621
910
|
}
|
|
622
|
-
|
|
623
|
-
|
|
624
|
-
|
|
625
|
-
|
|
626
|
-
|
|
627
|
-
|
|
628
|
-
|
|
629
|
-
|
|
630
|
-
|
|
631
|
-
|
|
632
|
-
|
|
633
|
-
|
|
634
|
-
|
|
635
|
-
|
|
911
|
+
groups: dict[str, tuple[list[sqlite3.Row], list[sqlite3.Row], list[str]]] = {}
|
|
912
|
+
|
|
913
|
+
def group(row: sqlite3.Row) -> tuple[list[sqlite3.Row], list[sqlite3.Row], list[str]]:
|
|
914
|
+
finding_id = row["finding_id"]
|
|
915
|
+
return groups.setdefault(aliases.get(finding_id, finding_id), ([], [], []))
|
|
916
|
+
|
|
917
|
+
for match in matches:
|
|
918
|
+
group(rows["before"][match["beforeOccurrenceIds"][0]])[2].append(match["reason"])
|
|
919
|
+
for index, side in enumerate(("before", "after")):
|
|
920
|
+
occurrence_ids = dict.fromkeys(
|
|
921
|
+
chain(
|
|
922
|
+
(value for match in matches for value in match[f"{side}OccurrenceIds"]),
|
|
923
|
+
rows[side],
|
|
924
|
+
)
|
|
925
|
+
)
|
|
926
|
+
for occurrence_id in occurrence_ids:
|
|
927
|
+
row = rows[side][occurrence_id]
|
|
928
|
+
group(row)[index].append(row)
|
|
929
|
+
result = [
|
|
930
|
+
(
|
|
931
|
+
previous,
|
|
932
|
+
current,
|
|
933
|
+
(
|
|
934
|
+
" ".join(dict.fromkeys(reasons))
|
|
935
|
+
if reasons
|
|
936
|
+
else "The findings share a stable identity or a previously confirmed link."
|
|
937
|
+
)
|
|
938
|
+
if previous and current
|
|
939
|
+
else None,
|
|
940
|
+
)
|
|
941
|
+
for previous, current, reasons in groups.values()
|
|
942
|
+
]
|
|
636
943
|
return sorted(
|
|
637
944
|
result,
|
|
638
945
|
key=lambda group: (
|