@openai/codex-security 0.1.24 → 0.1.25
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +177 -19
- package/_bundled_plugin/.codex-plugin/plugin.json +1 -1
- package/_bundled_plugin/mcp/server.mjs.br.part-000 +0 -0
- package/_bundled_plugin/mcp/server.mjs.br.part-001 +0 -0
- package/_bundled_plugin/scripts/rank_preview.py +11 -1
- package/_bundled_plugin/scripts/workbench_cli.py +8 -1
- package/_bundled_plugin/scripts/workbench_db.py +44 -2
- package/_bundled_plugin/scripts/workbench_scan_history.py +374 -66
- package/_bundled_plugin/scripts/workbench_schema.py +15 -0
- package/_bundled_plugin/scripts/workbench_target.py +86 -12
- package/dist/api.d.ts +15 -2
- package/dist/api.d.ts.map +1 -1
- package/dist/api.js +330 -28
- package/dist/api.js.map +1 -1
- package/dist/auth.d.ts.map +1 -1
- package/dist/auth.js +35 -3
- package/dist/auth.js.map +1 -1
- package/dist/cli.d.ts +2 -0
- package/dist/cli.d.ts.map +1 -1
- package/dist/cli.js +86 -33
- package/dist/cli.js.map +1 -1
- package/dist/component-plan.d.ts.map +1 -1
- package/dist/component-plan.js +4 -2
- package/dist/component-plan.js.map +1 -1
- package/dist/component-scan.d.ts.map +1 -1
- package/dist/component-scan.js +21 -6
- package/dist/component-scan.js.map +1 -1
- package/dist/config.d.ts.map +1 -1
- package/dist/config.js +46 -2
- package/dist/config.js.map +1 -1
- package/dist/cost.d.ts +1 -0
- package/dist/cost.d.ts.map +1 -1
- package/dist/cost.js +23 -11
- package/dist/cost.js.map +1 -1
- package/dist/deduplication/checkpointed-review.d.ts.map +1 -1
- package/dist/deduplication/checkpointed-review.js +2 -1
- package/dist/deduplication/checkpointed-review.js.map +1 -1
- package/dist/deduplication/codex-review.d.ts +3 -0
- package/dist/deduplication/codex-review.d.ts.map +1 -1
- package/dist/deduplication/codex-review.js +122 -36
- package/dist/deduplication/codex-review.js.map +1 -1
- package/dist/deduplication/deduplication-prompts.d.ts +4 -3
- package/dist/deduplication/deduplication-prompts.d.ts.map +1 -1
- package/dist/deduplication/deduplication-prompts.js +18 -9
- package/dist/deduplication/deduplication-prompts.js.map +1 -1
- package/dist/deduplication/deduplication-reviewer.d.ts +2 -7
- package/dist/deduplication/deduplication-reviewer.d.ts.map +1 -1
- package/dist/deduplication/deduplication-reviewer.js +46 -24
- package/dist/deduplication/deduplication-reviewer.js.map +1 -1
- package/dist/deduplication/deduplication.d.ts.map +1 -1
- package/dist/deduplication/deduplication.js +186 -23
- package/dist/deduplication/deduplication.js.map +1 -1
- package/dist/deduplication/scan.d.ts +9 -1
- package/dist/deduplication/scan.d.ts.map +1 -1
- package/dist/deduplication/scan.js +24 -7
- package/dist/deduplication/scan.js.map +1 -1
- package/dist/errors.d.ts +13 -0
- package/dist/errors.d.ts.map +1 -1
- package/dist/errors.js +7 -0
- package/dist/errors.js.map +1 -1
- package/dist/index.d.ts +5 -5
- package/dist/index.d.ts.map +1 -1
- package/dist/index.js +2 -2
- package/dist/index.js.map +1 -1
- package/dist/mock-scan.d.ts +4 -0
- package/dist/mock-scan.d.ts.map +1 -0
- package/dist/mock-scan.js +210 -0
- package/dist/mock-scan.js.map +1 -0
- package/dist/publish.d.ts.map +1 -1
- package/dist/publish.js +2 -2
- package/dist/publish.js.map +1 -1
- package/dist/runtime.d.ts +1 -0
- package/dist/runtime.d.ts.map +1 -1
- package/dist/runtime.js +49 -14
- package/dist/runtime.js.map +1 -1
- package/dist/scan-comparison.d.ts +11 -2
- package/dist/scan-comparison.d.ts.map +1 -1
- package/dist/scan-comparison.js +237 -39
- package/dist/scan-comparison.js.map +1 -1
- package/dist/scan-dashboard.d.ts +3 -1
- package/dist/scan-dashboard.d.ts.map +1 -1
- package/dist/scan-dashboard.js +70 -3
- package/dist/scan-dashboard.js.map +1 -1
- package/dist/scan-history-renderer.d.ts.map +1 -1
- package/dist/scan-history-renderer.js +21 -0
- package/dist/scan-history-renderer.js.map +1 -1
- package/dist/server/api.d.ts +6 -0
- package/dist/server/api.d.ts.map +1 -0
- package/dist/server/api.js +4 -0
- package/dist/server/api.js.map +1 -0
- package/dist/server/dashboard/app.css +1 -1
- package/dist/server/embeddings.d.ts +2 -1
- package/dist/server/embeddings.d.ts.map +1 -1
- package/dist/server/embeddings.js +8 -3
- package/dist/server/embeddings.js.map +1 -1
- package/dist/server/serve.d.ts.map +1 -1
- package/dist/server/serve.js +1 -1
- package/dist/server/serve.js.map +1 -1
- package/dist/targets.d.ts.map +1 -1
- package/dist/targets.js +3 -5
- package/dist/targets.js.map +1 -1
- package/dist/version.d.ts +1 -1
- package/dist/version.js +1 -1
- package/dist/version.js.map +1 -1
- package/package.json +9 -4
|
@@ -6,6 +6,8 @@ import json
|
|
|
6
6
|
import os
|
|
7
7
|
import sqlite3
|
|
8
8
|
import sys
|
|
9
|
+
from collections.abc import Iterable, Iterator
|
|
10
|
+
from itertools import chain
|
|
9
11
|
from pathlib import Path, PurePosixPath
|
|
10
12
|
from typing import Any, Callable
|
|
11
13
|
from urllib.parse import urlsplit
|
|
@@ -28,10 +30,12 @@ def _same_repository(
|
|
|
28
30
|
*,
|
|
29
31
|
after_identity: tuple[str | None, tuple[str, str] | None] | None = None,
|
|
30
32
|
) -> bool:
|
|
31
|
-
if before["target_id"] == after["target_id"]:
|
|
33
|
+
if before["target_id"] is not None and before["target_id"] == after["target_id"]:
|
|
32
34
|
return True
|
|
33
35
|
before_target = Path(before["target_path"])
|
|
34
36
|
after_target = Path(after["target_path"])
|
|
37
|
+
if before_target.resolve() == after_target.resolve():
|
|
38
|
+
return True
|
|
35
39
|
before_git_dir = git_output(
|
|
36
40
|
before_target, "rev-parse", "--path-format=absolute", "--git-common-dir"
|
|
37
41
|
)
|
|
@@ -253,7 +257,7 @@ def list_unmatched_scan_pairs(
|
|
|
253
257
|
for scan in connection.execute(
|
|
254
258
|
"SELECT * FROM scans WHERE status = 'complete' ORDER BY started_at, id"
|
|
255
259
|
)
|
|
256
|
-
if
|
|
260
|
+
if _same_repository(scan, requested)
|
|
257
261
|
]
|
|
258
262
|
|
|
259
263
|
available = []
|
|
@@ -271,6 +275,7 @@ def list_unmatched_scan_pairs(
|
|
|
271
275
|
batches = []
|
|
272
276
|
skipped = 0
|
|
273
277
|
matching_findings: dict[str, list[dict[str, Any]]] = {}
|
|
278
|
+
known_links: list[sqlite3.Row] | None = None
|
|
274
279
|
for index, after in enumerate(available):
|
|
275
280
|
previous = [
|
|
276
281
|
before
|
|
@@ -280,12 +285,26 @@ def list_unmatched_scan_pairs(
|
|
|
280
285
|
skipped += index - len(previous)
|
|
281
286
|
if not previous:
|
|
282
287
|
continue
|
|
288
|
+
if known_links is None:
|
|
289
|
+
known_links = (
|
|
290
|
+
[]
|
|
291
|
+
if args.force
|
|
292
|
+
else _saved_finding_links(connection, {scan["id"] for scan in selected})
|
|
293
|
+
)
|
|
283
294
|
for scan in (*previous, after):
|
|
284
295
|
if scan["id"] not in matching_findings:
|
|
285
296
|
backfill_finding_details(connection, scan)
|
|
286
297
|
matching_findings[scan["id"]] = [
|
|
287
298
|
_matching_input(row) for row in _scan_findings(connection, scan["id"]).values()
|
|
288
299
|
]
|
|
300
|
+
known_groups = _known_finding_groups(
|
|
301
|
+
known_links,
|
|
302
|
+
{
|
|
303
|
+
scan["id"]
|
|
304
|
+
for scan in selected
|
|
305
|
+
if (scan["started_at"], scan["id"]) <= (after["started_at"], after["id"])
|
|
306
|
+
},
|
|
307
|
+
)
|
|
289
308
|
batches.append(
|
|
290
309
|
{
|
|
291
310
|
"afterFindings": matching_findings[after["id"]],
|
|
@@ -297,6 +316,7 @@ def list_unmatched_scan_pairs(
|
|
|
297
316
|
}
|
|
298
317
|
for before in previous
|
|
299
318
|
],
|
|
319
|
+
**({"knownFindingGroups": known_groups} if known_groups else {}),
|
|
300
320
|
}
|
|
301
321
|
)
|
|
302
322
|
return {
|
|
@@ -308,6 +328,53 @@ def list_unmatched_scan_pairs(
|
|
|
308
328
|
}
|
|
309
329
|
|
|
310
330
|
|
|
331
|
+
def _saved_finding_links(connection: sqlite3.Connection, scan_ids: set[str]) -> list[sqlite3.Row]:
|
|
332
|
+
return [
|
|
333
|
+
row
|
|
334
|
+
for row in _rows_for_ids(
|
|
335
|
+
connection,
|
|
336
|
+
"""
|
|
337
|
+
SELECT before.scan_id AS before_scan_id, before.finding_id AS before_finding_id,
|
|
338
|
+
after.scan_id AS after_scan_id, after.finding_id AS after_finding_id
|
|
339
|
+
FROM scan_comparison_matches AS matches
|
|
340
|
+
JOIN finding_occurrences AS before ON before.id = matches.before_occurrence_id
|
|
341
|
+
JOIN finding_occurrences AS after ON after.id = matches.after_occurrence_id
|
|
342
|
+
WHERE matches.before_scan_id IN ({placeholders})
|
|
343
|
+
ORDER BY matches.before_scan_id, after.scan_id, before.finding_id, after.finding_id
|
|
344
|
+
""",
|
|
345
|
+
sorted(scan_ids),
|
|
346
|
+
)
|
|
347
|
+
if row["before_scan_id"] in scan_ids and row["after_scan_id"] in scan_ids
|
|
348
|
+
]
|
|
349
|
+
|
|
350
|
+
|
|
351
|
+
def _finding_aliases(links: Iterable[tuple[str, str]]) -> dict[str, str]:
|
|
352
|
+
parents: dict[str, str] = {}
|
|
353
|
+
|
|
354
|
+
def root(value: str) -> str:
|
|
355
|
+
parents.setdefault(value, value)
|
|
356
|
+
while parents[value] != value:
|
|
357
|
+
parents[value] = parents[parents[value]]
|
|
358
|
+
value = parents[value]
|
|
359
|
+
return value
|
|
360
|
+
|
|
361
|
+
for before_id, after_id in links:
|
|
362
|
+
parents[root(after_id)] = root(before_id)
|
|
363
|
+
return {finding_id: root(finding_id) for finding_id in parents}
|
|
364
|
+
|
|
365
|
+
|
|
366
|
+
def _known_finding_groups(links: list[sqlite3.Row], scan_ids: set[str]) -> list[list[str]]:
|
|
367
|
+
aliases = _finding_aliases(
|
|
368
|
+
(link["before_finding_id"], link["after_finding_id"])
|
|
369
|
+
for link in links
|
|
370
|
+
if link["before_scan_id"] in scan_ids and link["after_scan_id"] in scan_ids
|
|
371
|
+
)
|
|
372
|
+
groups: dict[str, list[str]] = {}
|
|
373
|
+
for finding_id, identity in aliases.items():
|
|
374
|
+
groups.setdefault(identity, []).append(finding_id)
|
|
375
|
+
return sorted(sorted(group) for group in groups.values() if len(group) > 1)
|
|
376
|
+
|
|
377
|
+
|
|
311
378
|
def compare_scans(
|
|
312
379
|
connection: sqlite3.Connection,
|
|
313
380
|
args: argparse.Namespace,
|
|
@@ -351,7 +418,12 @@ def compare_scans(
|
|
|
351
418
|
before_findings = _scan_findings(connection, before["id"])
|
|
352
419
|
after_findings = _scan_findings(connection, after["id"])
|
|
353
420
|
matches = json.loads(cached["result_json"]) if cached is not None else None
|
|
354
|
-
|
|
421
|
+
saved_matches = matches["matches"] if matches is not None else []
|
|
422
|
+
occurrences = {
|
|
423
|
+
row["id"]: row for row in chain(before_findings.values(), after_findings.values())
|
|
424
|
+
}
|
|
425
|
+
aliases = _confirmed_finding_aliases(connection, occurrences)
|
|
426
|
+
groups = _finding_groups(before_findings, after_findings, saved_matches, aliases)
|
|
355
427
|
uncertain = (
|
|
356
428
|
{
|
|
357
429
|
(side, match[f"{side}OccurrenceId"]): match["reason"]
|
|
@@ -365,23 +437,33 @@ def compare_scans(
|
|
|
365
437
|
summary = {status: 0 for status in ("new", "persisting", "resolved", "reopened", "unknown")}
|
|
366
438
|
|
|
367
439
|
for previous_rows, current_rows, match_reason in groups:
|
|
368
|
-
previous =
|
|
440
|
+
previous = (
|
|
441
|
+
min(previous_rows, key=lambda row: SEVERITY_ORDER[row["severity"]])
|
|
442
|
+
if previous_rows
|
|
443
|
+
else None
|
|
444
|
+
)
|
|
369
445
|
current = (
|
|
370
446
|
min(current_rows, key=lambda row: SEVERITY_ORDER[row["severity"]])
|
|
371
447
|
if current_rows
|
|
372
448
|
else None
|
|
373
449
|
)
|
|
374
450
|
selected = current if current is not None else previous
|
|
375
|
-
if selected is None:
|
|
376
|
-
continue
|
|
377
451
|
item = {
|
|
378
452
|
"findingId": selected["finding_id"],
|
|
379
453
|
"path": selected["relative_path"],
|
|
380
454
|
"severity": selected["severity"],
|
|
381
455
|
"title": selected["title"],
|
|
382
456
|
}
|
|
457
|
+
side = "after" if current_rows else "before"
|
|
458
|
+
uncertain_reason = next(
|
|
459
|
+
(
|
|
460
|
+
uncertain[(side, row["id"])]
|
|
461
|
+
for row in current_rows or previous_rows
|
|
462
|
+
if (side, row["id"]) in uncertain
|
|
463
|
+
),
|
|
464
|
+
None,
|
|
465
|
+
)
|
|
383
466
|
if previous is None:
|
|
384
|
-
uncertain_reason = uncertain.get(("after", current["id"])) if current else None
|
|
385
467
|
if uncertain_reason is None:
|
|
386
468
|
status = "new"
|
|
387
469
|
else:
|
|
@@ -399,17 +481,20 @@ def compare_scans(
|
|
|
399
481
|
)
|
|
400
482
|
if match_reason is not None:
|
|
401
483
|
item["matchReason"] = match_reason
|
|
402
|
-
elif
|
|
484
|
+
elif uncertain_reason is not None:
|
|
403
485
|
status = "unknown"
|
|
404
486
|
item["reason"] = uncertain_reason
|
|
405
487
|
elif not comparable:
|
|
406
488
|
status = "unknown"
|
|
407
489
|
item["reason"] = "The later scan has incomplete coverage."
|
|
408
|
-
elif not
|
|
409
|
-
|
|
410
|
-
|
|
411
|
-
|
|
412
|
-
|
|
490
|
+
elif not all(
|
|
491
|
+
scan_covers_path(
|
|
492
|
+
after,
|
|
493
|
+
target_id=after["target_id"],
|
|
494
|
+
path=row["relative_path"],
|
|
495
|
+
coverage=after_coverage,
|
|
496
|
+
)
|
|
497
|
+
for row in previous_rows
|
|
413
498
|
):
|
|
414
499
|
status = "unknown"
|
|
415
500
|
item["reason"] = "The affected path was excluded or outside the later scope."
|
|
@@ -441,11 +526,41 @@ def compare_scans(
|
|
|
441
526
|
"repository": before["target_path"],
|
|
442
527
|
"summary": summary,
|
|
443
528
|
}
|
|
529
|
+
if matches is not None and matches.get("related"):
|
|
530
|
+
related = _separate_finding_pairs(matches["related"], occurrences, aliases)
|
|
531
|
+
if related:
|
|
532
|
+
result["related"] = [
|
|
533
|
+
{
|
|
534
|
+
**pair,
|
|
535
|
+
"beforeTitle": occurrences[pair["beforeOccurrenceId"]]["title"],
|
|
536
|
+
"afterTitle": occurrences[pair["afterOccurrenceId"]]["title"],
|
|
537
|
+
}
|
|
538
|
+
for pair in related
|
|
539
|
+
]
|
|
444
540
|
if include_matching_inputs:
|
|
541
|
+
known_scan_ids = {
|
|
542
|
+
scan["id"]
|
|
543
|
+
for scan in connection.execute(
|
|
544
|
+
"SELECT * FROM scans WHERE status = 'complete' "
|
|
545
|
+
"AND (started_at < ? OR (started_at = ? AND id <= ?))",
|
|
546
|
+
(after["started_at"], after["started_at"], after["id"]),
|
|
547
|
+
)
|
|
548
|
+
if _same_repository(scan, after)
|
|
549
|
+
}
|
|
550
|
+
excluded_pairs = {(before["id"], after["id"]), (after["id"], before["id"])}
|
|
551
|
+
known_groups = _known_finding_groups(
|
|
552
|
+
[
|
|
553
|
+
link
|
|
554
|
+
for link in _saved_finding_links(connection, known_scan_ids)
|
|
555
|
+
if (link["before_scan_id"], link["after_scan_id"]) not in excluded_pairs
|
|
556
|
+
],
|
|
557
|
+
known_scan_ids,
|
|
558
|
+
)
|
|
445
559
|
result["matchingCached"] = cached is not None
|
|
446
560
|
result["matchingInputs"] = {
|
|
447
561
|
"before": [_matching_input(row) for row in before_findings.values()],
|
|
448
562
|
"after": [_matching_input(row) for row in after_findings.values()],
|
|
563
|
+
**({"knownFindingGroups": known_groups} if known_groups else {}),
|
|
449
564
|
}
|
|
450
565
|
return result
|
|
451
566
|
|
|
@@ -469,42 +584,80 @@ def save_scan_comparison(
|
|
|
469
584
|
read_coverage(after)
|
|
470
585
|
before_findings = _scan_findings(connection, before["id"])
|
|
471
586
|
after_findings = _scan_findings(connection, after["id"])
|
|
472
|
-
matches_json =
|
|
587
|
+
matches_json = (
|
|
588
|
+
sys.stdin.read() if getattr(args, "matches_json_stdin", False) else args.matches_json
|
|
589
|
+
)
|
|
473
590
|
try:
|
|
474
591
|
payload = json.loads(matches_json)
|
|
475
592
|
except (TypeError, ValueError) as exc:
|
|
476
593
|
raise SystemExit("Scan comparison matches must be a valid JSON object.") from exc
|
|
477
|
-
if
|
|
594
|
+
if (
|
|
595
|
+
not isinstance(payload, dict)
|
|
596
|
+
or not {"matches", "uncertain"}.issubset(payload)
|
|
597
|
+
or set(payload) - {"matches", "uncertain", "related"}
|
|
598
|
+
):
|
|
478
599
|
raise SystemExit("Scan comparison matches must contain matches and uncertain arrays.")
|
|
479
|
-
if
|
|
600
|
+
if any(
|
|
601
|
+
not isinstance(payload.get(key, []), list) for key in ("matches", "uncertain", "related")
|
|
602
|
+
):
|
|
480
603
|
raise SystemExit("Scan comparison matches must contain matches and uncertain arrays.")
|
|
481
604
|
allowed = {
|
|
482
605
|
"before": {row["id"] for row in before_findings.values()},
|
|
483
606
|
"after": {row["id"] for row in after_findings.values()},
|
|
484
607
|
}
|
|
485
|
-
consumed: dict[str,
|
|
486
|
-
for match in payload["matches"]:
|
|
608
|
+
consumed: dict[str, dict[str, int]] = {"before": {}, "after": {}}
|
|
609
|
+
for group, match in enumerate(payload["matches"]):
|
|
610
|
+
if (
|
|
611
|
+
not isinstance(match, dict)
|
|
612
|
+
or match.get("confidence") != "high"
|
|
613
|
+
or not isinstance(match.get("reason"), str)
|
|
614
|
+
or not match["reason"].strip()
|
|
615
|
+
):
|
|
616
|
+
raise SystemExit("Scan comparison matches must have high confidence and a reason.")
|
|
487
617
|
for side in ("before", "after"):
|
|
488
|
-
occurrences = match
|
|
618
|
+
occurrences = match.get(f"{side}OccurrenceIds")
|
|
619
|
+
if not isinstance(occurrences, list) or any(
|
|
620
|
+
not isinstance(value, str) for value in occurrences
|
|
621
|
+
):
|
|
622
|
+
raise SystemExit("Scan comparison matches must identify distinct scan findings.")
|
|
489
623
|
unique = set(occurrences)
|
|
490
624
|
if (
|
|
491
625
|
not occurrences
|
|
492
626
|
or len(unique) != len(occurrences)
|
|
493
627
|
or not unique.issubset(allowed[side])
|
|
494
|
-
or not consumed[side]
|
|
628
|
+
or not unique.isdisjoint(consumed[side])
|
|
495
629
|
):
|
|
496
630
|
raise SystemExit("Scan comparison matches must identify distinct scan findings.")
|
|
497
|
-
consumed[side].update(unique)
|
|
631
|
+
consumed[side].update((occurrence_id, group) for occurrence_id in unique)
|
|
498
632
|
uncertain_pairs = set()
|
|
499
633
|
for match in payload["uncertain"]:
|
|
634
|
+
if not _valid_finding_pair(match):
|
|
635
|
+
raise SystemExit("Uncertain scan comparison matches must identify distinct findings.")
|
|
500
636
|
pair = (match["beforeOccurrenceId"], match["afterOccurrenceId"])
|
|
501
637
|
if (
|
|
502
|
-
pair[0] not in allowed["before"]
|
|
503
|
-
or pair[
|
|
638
|
+
pair[0] not in allowed["before"]
|
|
639
|
+
or pair[0] in consumed["before"]
|
|
640
|
+
or pair[1] not in allowed["after"]
|
|
641
|
+
or pair[1] in consumed["after"]
|
|
504
642
|
or pair in uncertain_pairs
|
|
505
643
|
):
|
|
506
644
|
raise SystemExit("Uncertain scan comparison matches must identify distinct findings.")
|
|
507
645
|
uncertain_pairs.add(pair)
|
|
646
|
+
related_pairs = set()
|
|
647
|
+
for match in payload.get("related", []):
|
|
648
|
+
if not _valid_finding_pair(match):
|
|
649
|
+
raise SystemExit("Related scan comparison findings must identify distinct findings.")
|
|
650
|
+
pair = (match["beforeOccurrenceId"], match["afterOccurrenceId"])
|
|
651
|
+
group = consumed["before"].get(pair[0])
|
|
652
|
+
if (
|
|
653
|
+
pair[0] not in allowed["before"]
|
|
654
|
+
or pair[1] not in allowed["after"]
|
|
655
|
+
or (group is not None and group == consumed["after"].get(pair[1]))
|
|
656
|
+
or pair in uncertain_pairs
|
|
657
|
+
or pair in related_pairs
|
|
658
|
+
):
|
|
659
|
+
raise SystemExit("Related scan comparison findings must identify distinct findings.")
|
|
660
|
+
related_pairs.add(pair)
|
|
508
661
|
timestamp = now()
|
|
509
662
|
with connection:
|
|
510
663
|
connection.execute("BEGIN IMMEDIATE")
|
|
@@ -542,6 +695,142 @@ def save_scan_comparison(
|
|
|
542
695
|
return compare_scans(connection, args, require_scan=require_scan, read_coverage=read_coverage)
|
|
543
696
|
|
|
544
697
|
|
|
698
|
+
def _valid_finding_pair(value: Any) -> bool:
|
|
699
|
+
return (
|
|
700
|
+
isinstance(value, dict)
|
|
701
|
+
and set(value) == {"beforeOccurrenceId", "afterOccurrenceId", "reason"}
|
|
702
|
+
and all(isinstance(item, str) and item.strip() for item in value.values())
|
|
703
|
+
)
|
|
704
|
+
|
|
705
|
+
|
|
706
|
+
def _rows_for_ids(
|
|
707
|
+
connection: sqlite3.Connection, query: str, ids: Iterable[str]
|
|
708
|
+
) -> Iterator[sqlite3.Row]:
|
|
709
|
+
values = tuple(dict.fromkeys(ids))
|
|
710
|
+
getlimit = getattr(connection, "getlimit", None)
|
|
711
|
+
# Python 3.10 lacks getlimit; 999 is SQLite's older host-parameter limit.
|
|
712
|
+
limit = getlimit(sqlite3.SQLITE_LIMIT_VARIABLE_NUMBER) if getlimit else 999
|
|
713
|
+
for start in range(0, len(values), limit):
|
|
714
|
+
batch = values[start : start + limit]
|
|
715
|
+
yield from connection.execute(
|
|
716
|
+
query.format(placeholders=", ".join("?" for _ in batch)), batch
|
|
717
|
+
)
|
|
718
|
+
|
|
719
|
+
|
|
720
|
+
# Stable finding IDs already include the target identity. Follow their indexed
|
|
721
|
+
# occurrences instead of resolving repository paths or scanning every saved link.
|
|
722
|
+
_FINDING_NEIGHBORS_SQL = """
|
|
723
|
+
FROM linked
|
|
724
|
+
CROSS JOIN finding_occurrences AS source
|
|
725
|
+
ON source.finding_id = linked.finding_id
|
|
726
|
+
CROSS JOIN scan_comparison_matches AS matches
|
|
727
|
+
ON matches.before_occurrence_id = source.id OR matches.after_occurrence_id = source.id
|
|
728
|
+
CROSS JOIN finding_occurrences AS neighbor ON neighbor.id = CASE
|
|
729
|
+
WHEN matches.before_occurrence_id = source.id THEN matches.after_occurrence_id
|
|
730
|
+
ELSE matches.before_occurrence_id END
|
|
731
|
+
"""
|
|
732
|
+
# Traverse only the selected findings' components, including recurring stable IDs.
|
|
733
|
+
_LINKED_FINDINGS_SQL = f"""
|
|
734
|
+
WITH RECURSIVE linked(finding_id) AS (
|
|
735
|
+
SELECT occurrences.finding_id
|
|
736
|
+
FROM finding_occurrences AS occurrences
|
|
737
|
+
WHERE occurrences.id IN ({{placeholders}})
|
|
738
|
+
UNION
|
|
739
|
+
SELECT neighbor.finding_id
|
|
740
|
+
{_FINDING_NEIGHBORS_SQL}
|
|
741
|
+
)
|
|
742
|
+
"""
|
|
743
|
+
|
|
744
|
+
|
|
745
|
+
def _confirmed_finding_aliases(
|
|
746
|
+
connection: sqlite3.Connection, occurrence_ids: Iterable[str]
|
|
747
|
+
) -> dict[str, str]:
|
|
748
|
+
query = f"""
|
|
749
|
+
{_LINKED_FINDINGS_SQL}
|
|
750
|
+
SELECT DISTINCT linked.finding_id AS before_finding_id,
|
|
751
|
+
neighbor.finding_id AS after_finding_id
|
|
752
|
+
{_FINDING_NEIGHBORS_SQL}
|
|
753
|
+
"""
|
|
754
|
+
return _finding_aliases(
|
|
755
|
+
(row["before_finding_id"], row["after_finding_id"])
|
|
756
|
+
for row in _rows_for_ids(connection, query, occurrence_ids)
|
|
757
|
+
)
|
|
758
|
+
|
|
759
|
+
|
|
760
|
+
def _separate_finding_pairs(
|
|
761
|
+
pairs: list[dict[str, Any]],
|
|
762
|
+
occurrences: dict[str, sqlite3.Row],
|
|
763
|
+
aliases: dict[str, str],
|
|
764
|
+
) -> list[dict[str, Any]]:
|
|
765
|
+
def identity(occurrence_id: str) -> str:
|
|
766
|
+
finding_id = occurrences[occurrence_id]["finding_id"]
|
|
767
|
+
return aliases.get(finding_id, finding_id)
|
|
768
|
+
|
|
769
|
+
return [
|
|
770
|
+
pair
|
|
771
|
+
for pair in pairs
|
|
772
|
+
if identity(pair["beforeOccurrenceId"]) != identity(pair["afterOccurrenceId"])
|
|
773
|
+
]
|
|
774
|
+
|
|
775
|
+
|
|
776
|
+
def finding_relations(
|
|
777
|
+
connection: sqlite3.Connection, scan_id: str, occurrence_ids: Iterable[str]
|
|
778
|
+
) -> dict[str, list[dict[str, Any]]]:
|
|
779
|
+
selected = set(occurrence_ids)
|
|
780
|
+
if not selected:
|
|
781
|
+
return {}
|
|
782
|
+
pairs = []
|
|
783
|
+
for comparison in connection.execute(
|
|
784
|
+
"SELECT before_scan_id, after_scan_id, result_json FROM scan_comparisons "
|
|
785
|
+
"WHERE before_scan_id = ? OR after_scan_id = ? "
|
|
786
|
+
"ORDER BY before_scan_id, after_scan_id",
|
|
787
|
+
(scan_id, scan_id),
|
|
788
|
+
):
|
|
789
|
+
side = "before" if comparison["before_scan_id"] == scan_id else "after"
|
|
790
|
+
other = "after" if side == "before" else "before"
|
|
791
|
+
for pair in json.loads(comparison["result_json"]).get("related", []):
|
|
792
|
+
if pair[f"{side}OccurrenceId"] in selected:
|
|
793
|
+
pairs.append(
|
|
794
|
+
{
|
|
795
|
+
"beforeOccurrenceId": pair[f"{side}OccurrenceId"],
|
|
796
|
+
"afterOccurrenceId": pair[f"{other}OccurrenceId"],
|
|
797
|
+
"afterScanId": comparison[f"{other}_scan_id"],
|
|
798
|
+
"reason": pair["reason"],
|
|
799
|
+
}
|
|
800
|
+
)
|
|
801
|
+
occurrences = {
|
|
802
|
+
row["id"]: row
|
|
803
|
+
for row in _rows_for_ids(
|
|
804
|
+
connection,
|
|
805
|
+
"SELECT id, finding_id, scan_id, title FROM finding_occurrences "
|
|
806
|
+
"WHERE id IN ({placeholders})",
|
|
807
|
+
(pair[key] for pair in pairs for key in ("beforeOccurrenceId", "afterOccurrenceId")),
|
|
808
|
+
)
|
|
809
|
+
}
|
|
810
|
+
pairs = [
|
|
811
|
+
pair
|
|
812
|
+
for pair in pairs
|
|
813
|
+
if pair["beforeOccurrenceId"] in occurrences
|
|
814
|
+
and occurrences[pair["beforeOccurrenceId"]]["scan_id"] == scan_id
|
|
815
|
+
and pair["afterOccurrenceId"] in occurrences
|
|
816
|
+
and occurrences[pair["afterOccurrenceId"]]["scan_id"] == pair["afterScanId"]
|
|
817
|
+
]
|
|
818
|
+
result: dict[str, list[dict[str, Any]]] = {}
|
|
819
|
+
aliases = _confirmed_finding_aliases(connection, (pair["beforeOccurrenceId"] for pair in pairs))
|
|
820
|
+
for pair in _separate_finding_pairs(pairs, occurrences, aliases):
|
|
821
|
+
finding = occurrences[pair["afterOccurrenceId"]]
|
|
822
|
+
result.setdefault(pair["beforeOccurrenceId"], []).append(
|
|
823
|
+
{
|
|
824
|
+
"findingId": finding["finding_id"],
|
|
825
|
+
"occurrenceId": finding["id"],
|
|
826
|
+
"reason": pair["reason"],
|
|
827
|
+
"scanId": pair["afterScanId"],
|
|
828
|
+
"title": finding["title"],
|
|
829
|
+
}
|
|
830
|
+
)
|
|
831
|
+
return result
|
|
832
|
+
|
|
833
|
+
|
|
545
834
|
def finding_matches(
|
|
546
835
|
connection: sqlite3.Connection, occurrence_id: str, scan_id: str, started_at: str
|
|
547
836
|
) -> tuple[list[dict[str, Any]], str, list[str]]:
|
|
@@ -562,34 +851,34 @@ def finding_matches(
|
|
|
562
851
|
""",
|
|
563
852
|
(occurrence_id, occurrence_id),
|
|
564
853
|
).fetchall()
|
|
565
|
-
|
|
566
|
-
|
|
567
|
-
|
|
568
|
-
|
|
569
|
-
|
|
570
|
-
|
|
571
|
-
|
|
572
|
-
|
|
573
|
-
|
|
574
|
-
|
|
575
|
-
|
|
576
|
-
|
|
577
|
-
|
|
578
|
-
|
|
579
|
-
|
|
580
|
-
|
|
581
|
-
|
|
582
|
-
|
|
583
|
-
|
|
584
|
-
|
|
585
|
-
|
|
586
|
-
|
|
587
|
-
|
|
588
|
-
|
|
589
|
-
|
|
590
|
-
|
|
591
|
-
|
|
592
|
-
|
|
854
|
+
linked_rows = list(
|
|
855
|
+
_rows_for_ids(
|
|
856
|
+
connection,
|
|
857
|
+
f"""
|
|
858
|
+
{_LINKED_FINDINGS_SQL}
|
|
859
|
+
SELECT occurrences.id AS occurrence_id, occurrences.finding_id, occurrences.title,
|
|
860
|
+
scans.started_at, scans.id AS scan_id
|
|
861
|
+
FROM linked
|
|
862
|
+
CROSS JOIN finding_occurrences AS occurrences
|
|
863
|
+
ON occurrences.finding_id = linked.finding_id
|
|
864
|
+
CROSS JOIN scans ON scans.id = occurrences.scan_id
|
|
865
|
+
""",
|
|
866
|
+
(occurrence_id,),
|
|
867
|
+
)
|
|
868
|
+
)
|
|
869
|
+
known_scans = sorted(
|
|
870
|
+
{(started_at, scan_id)} | {(row["started_at"], row["scan_id"]) for row in linked_rows}
|
|
871
|
+
)
|
|
872
|
+
included = {occurrence_id, *(row["occurrence_id"] for row in rows)}
|
|
873
|
+
rows.extend(
|
|
874
|
+
{
|
|
875
|
+
**row,
|
|
876
|
+
"reason": "The findings share a stable identity or a previously confirmed link.",
|
|
877
|
+
}
|
|
878
|
+
for row in linked_rows
|
|
879
|
+
if row["occurrence_id"] not in included
|
|
880
|
+
)
|
|
881
|
+
rows.sort(key=lambda row: (row["scan_id"], row["occurrence_id"]))
|
|
593
882
|
known_scan_ids = [known_scans[0][1]]
|
|
594
883
|
if len(known_scans) > 1:
|
|
595
884
|
known_scan_ids.append(known_scans[-1][1])
|
|
@@ -612,26 +901,45 @@ def finding_matches(
|
|
|
612
901
|
def _finding_groups(
|
|
613
902
|
before_findings: dict[str, sqlite3.Row],
|
|
614
903
|
after_findings: dict[str, sqlite3.Row],
|
|
615
|
-
matches: dict[str, Any]
|
|
904
|
+
matches: list[dict[str, Any]],
|
|
905
|
+
aliases: dict[str, str],
|
|
616
906
|
) -> list[tuple[list[sqlite3.Row], list[sqlite3.Row], str | None]]:
|
|
617
907
|
rows = {
|
|
618
908
|
side: {row["id"]: row for row in findings.values()}
|
|
619
909
|
for side, findings in (("before", before_findings), ("after", after_findings))
|
|
620
910
|
}
|
|
621
|
-
|
|
622
|
-
|
|
623
|
-
|
|
624
|
-
|
|
625
|
-
|
|
626
|
-
|
|
627
|
-
|
|
628
|
-
|
|
629
|
-
|
|
630
|
-
|
|
631
|
-
|
|
632
|
-
|
|
633
|
-
|
|
634
|
-
|
|
911
|
+
groups: dict[str, tuple[list[sqlite3.Row], list[sqlite3.Row], list[str]]] = {}
|
|
912
|
+
|
|
913
|
+
def group(row: sqlite3.Row) -> tuple[list[sqlite3.Row], list[sqlite3.Row], list[str]]:
|
|
914
|
+
finding_id = row["finding_id"]
|
|
915
|
+
return groups.setdefault(aliases.get(finding_id, finding_id), ([], [], []))
|
|
916
|
+
|
|
917
|
+
for match in matches:
|
|
918
|
+
group(rows["before"][match["beforeOccurrenceIds"][0]])[2].append(match["reason"])
|
|
919
|
+
for index, side in enumerate(("before", "after")):
|
|
920
|
+
occurrence_ids = dict.fromkeys(
|
|
921
|
+
chain(
|
|
922
|
+
(value for match in matches for value in match[f"{side}OccurrenceIds"]),
|
|
923
|
+
rows[side],
|
|
924
|
+
)
|
|
925
|
+
)
|
|
926
|
+
for occurrence_id in occurrence_ids:
|
|
927
|
+
row = rows[side][occurrence_id]
|
|
928
|
+
group(row)[index].append(row)
|
|
929
|
+
result = [
|
|
930
|
+
(
|
|
931
|
+
previous,
|
|
932
|
+
current,
|
|
933
|
+
(
|
|
934
|
+
" ".join(dict.fromkeys(reasons))
|
|
935
|
+
if reasons
|
|
936
|
+
else "The findings share a stable identity or a previously confirmed link."
|
|
937
|
+
)
|
|
938
|
+
if previous and current
|
|
939
|
+
else None,
|
|
940
|
+
)
|
|
941
|
+
for previous, current, reasons in groups.values()
|
|
942
|
+
]
|
|
635
943
|
return sorted(
|
|
636
944
|
result,
|
|
637
945
|
key=lambda group: (
|
|
@@ -825,6 +825,17 @@ MIGRATIONS = (
|
|
|
825
825
|
ALTER TABLE finding_workflow_reviews ADD COLUMN contract_digest TEXT;
|
|
826
826
|
""",
|
|
827
827
|
),
|
|
828
|
+
(
|
|
829
|
+
40,
|
|
830
|
+
"index finding identity and comparison history",
|
|
831
|
+
"""
|
|
832
|
+
CREATE INDEX finding_occurrences_by_finding
|
|
833
|
+
ON finding_occurrences(finding_id, id);
|
|
834
|
+
|
|
835
|
+
CREATE INDEX scan_comparisons_by_after_scan
|
|
836
|
+
ON scan_comparisons(after_scan_id, before_scan_id);
|
|
837
|
+
""",
|
|
838
|
+
),
|
|
828
839
|
)
|
|
829
840
|
|
|
830
841
|
|
|
@@ -1106,6 +1117,10 @@ def normalize_pre_release_execution_profile_migrations(
|
|
|
1106
1117
|
|
|
1107
1118
|
def normalize_pre_release_migrations(connection: sqlite3.Connection, timestamp: str) -> None:
|
|
1108
1119
|
normalize_mirror_lineage_migrations(connection)
|
|
1120
|
+
connection.execute(
|
|
1121
|
+
"UPDATE schema_migrations SET version = 40 WHERE version = 33 AND name = ?",
|
|
1122
|
+
("index finding identity and comparison history",),
|
|
1123
|
+
)
|
|
1109
1124
|
|
|
1110
1125
|
completion_warning_migration = connection.execute(
|
|
1111
1126
|
"SELECT name FROM schema_migrations WHERE version = 25"
|