local-triage 0.0.0-stage → 0.10.0-dev.32
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +90 -2
- package/THIRD-PARTY-NOTICES.md +48 -0
- package/app/__init__.py +3 -0
- package/app/affiliation.py +137 -0
- package/app/ask.py +300 -0
- package/app/build_info.py +98 -0
- package/app/capec.py +168 -0
- package/app/capec_view.py +27 -0
- package/app/codecmp.py +59 -0
- package/app/config.py +411 -0
- package/app/context.py +1451 -0
- package/app/cvss.py +357 -0
- package/app/db.py +6234 -0
- package/app/defect.py +670 -0
- package/app/defect_check.py +323 -0
- package/app/defect_drafts.py +612 -0
- package/app/defect_dto.py +149 -0
- package/app/defect_fields.py +248 -0
- package/app/defect_form.py +124 -0
- package/app/defect_links.py +107 -0
- package/app/defect_local_text.py +103 -0
- package/app/defect_references.py +164 -0
- package/app/defect_where.py +147 -0
- package/app/ecosystem.py +51 -0
- package/app/executors.py +660 -0
- package/app/feeds.py +1231 -0
- package/app/guard.py +164 -0
- package/app/hub_block_builder.py +200 -0
- package/app/hub_block_version.py +220 -0
- package/app/hub_defects.py +4469 -0
- package/app/hub_effects.py +421 -0
- package/app/hub_entity_versions.py +108 -0
- package/app/hub_import.py +2054 -0
- package/app/hub_onboarding.py +114 -0
- package/app/hub_ops.py +330 -0
- package/app/hub_pull.py +1195 -0
- package/app/hub_state.py +255 -0
- package/app/hub_status.py +237 -0
- package/app/hub_sync.py +619 -0
- package/app/hub_sync_api.py +241 -0
- package/app/hub_sync_compare.py +1054 -0
- package/app/hub_sync_contract.py +75 -0
- package/app/hub_template_model.py +74 -0
- package/app/hub_text_scale.py +159 -0
- package/app/journal.py +250 -0
- package/app/llm.py +850 -0
- package/app/llm_chat.py +596 -0
- package/app/llm_http.py +319 -0
- package/app/llm_messages.py +401 -0
- package/app/llm_net.py +764 -0
- package/app/llm_ollama.py +571 -0
- package/app/lockfile.py +505 -0
- package/app/logsafe.py +54 -0
- package/app/main.py +367 -0
- package/app/mask_display.py +132 -0
- package/app/masking.py +814 -0
- package/app/metrics.py +194 -0
- package/app/model_traffic.py +519 -0
- package/app/models.py +483 -0
- package/app/navigation.py +121 -0
- package/app/osv.py +279 -0
- package/app/package_identity.py +189 -0
- package/app/package_transition.py +824 -0
- package/app/profile_gate.py +71 -0
- package/app/prompts.py +1108 -0
- package/app/provider_keys.py +189 -0
- package/app/providers.py +511 -0
- package/app/routes/__init__.py +10 -0
- package/app/routes/api.py +153 -0
- package/app/routes/capec.py +186 -0
- package/app/routes/common.py +407 -0
- package/app/routes/components.py +1499 -0
- package/app/routes/cwe.py +168 -0
- package/app/routes/decision_selected.py +153 -0
- package/app/routes/defect_add.py +223 -0
- package/app/routes/defect_card_routes.py +363 -0
- package/app/routes/defect_selected.py +342 -0
- package/app/routes/feeds.py +155 -0
- package/app/routes/findings.py +2736 -0
- package/app/routes/hub.py +402 -0
- package/app/routes/hub_bulk.py +288 -0
- package/app/routes/hub_defect_routes.py +1945 -0
- package/app/routes/hub_status_routes.py +1132 -0
- package/app/routes/hub_sync_routes.py +463 -0
- package/app/routes/journal.py +132 -0
- package/app/routes/projects.py +1601 -0
- package/app/routes/providers.py +336 -0
- package/app/routes/references.py +26 -0
- package/app/routes/runs.py +219 -0
- package/app/routes/sca.py +2722 -0
- package/app/routes/settings.py +163 -0
- package/app/run_errors.py +151 -0
- package/app/sca.py +1662 -0
- package/app/sca_contract.py +1289 -0
- package/app/sca_research.py +1089 -0
- package/app/sca_research_net.py +1007 -0
- package/app/sca_text.py +965 -0
- package/app/serve.py +70 -0
- package/app/settings_store.py +262 -0
- package/app/static/app.css +2802 -0
- package/app/static/app.js +3772 -0
- package/app/static/capec-license.txt +9 -0
- package/app/static/favicon.svg +1 -0
- package/app/static/fonts/Golos-Text_DemiBold.woff2 +0 -0
- package/app/static/fonts/Golos-Text_Regular.woff2 +0 -0
- package/app/static/fonts/OFL.txt +90 -0
- package/app/static/navigation.js +234 -0
- package/app/static/observability.js +121 -0
- package/app/static/theme.js +49 -0
- package/app/storage/__init__.py +16 -0
- package/app/storage/local_state.py +246 -0
- package/app/storage/migrate.py +110 -0
- package/app/storage/migrations/postgresql/0001_baseline.sql +896 -0
- package/app/storage/migrations/postgresql/0002_feeds_and_secrets.sql +55 -0
- package/app/storage/migrations/postgresql/0003_project_mask_display.sql +5 -0
- package/app/storage/migrations/postgresql/0004_package_identity.sql +36 -0
- package/app/storage/postgresql.py +573 -0
- package/app/storage/schema.py +619 -0
- package/app/storage/sql.py +57 -0
- package/app/storage/tools.py +254 -0
- package/app/telemetry.py +307 -0
- package/app/templates/base.html +91 -0
- package/app/templates/capec/context.html +3 -0
- package/app/templates/capec/detail.html +27 -0
- package/app/templates/capec/index.html +128 -0
- package/app/templates/components/_archive.html +11 -0
- package/app/templates/components/archived.html +19 -0
- package/app/templates/components/batch.html +35 -0
- package/app/templates/components/detail.html +332 -0
- package/app/templates/components/index.html +338 -0
- package/app/templates/components/research_preview.html +80 -0
- package/app/templates/components/research_run.html +54 -0
- package/app/templates/cwe/bdu.html +56 -0
- package/app/templates/cwe/detail.html +233 -0
- package/app/templates/cwe/index.html +97 -0
- package/app/templates/error.html +8 -0
- package/app/templates/feeds/index.html +65 -0
- package/app/templates/findings/card.html +639 -0
- package/app/templates/findings/decision_selected.html +41 -0
- package/app/templates/findings/defect_add.html +46 -0
- package/app/templates/findings/defect_selected.html +47 -0
- package/app/templates/findings/list.html +153 -0
- package/app/templates/findings/send_selected.html +53 -0
- package/app/templates/findings/trace.html +72 -0
- package/app/templates/journal/detail.html +19 -0
- package/app/templates/journal/index.html +19 -0
- package/app/templates/macros.html +440 -0
- package/app/templates/navigation-fields.html +1 -0
- package/app/templates/navigation.html +11 -0
- package/app/templates/projects/_identity_summary.html +42 -0
- package/app/templates/projects/defect_card.html +153 -0
- package/app/templates/projects/defect_create_preview.html +17 -0
- package/app/templates/projects/defect_fields_diff.html +6 -0
- package/app/templates/projects/defect_fields_form.html +40 -0
- package/app/templates/projects/defect_references.html +17 -0
- package/app/templates/projects/defect_sync_preview.html +28 -0
- package/app/templates/projects/defect_update_actions.html +6 -0
- package/app/templates/projects/defect_update_preview.html +24 -0
- package/app/templates/projects/defects.html +126 -0
- package/app/templates/projects/delete.html +31 -0
- package/app/templates/projects/external.html +61 -0
- package/app/templates/projects/folders.html +64 -0
- package/app/templates/projects/hub_accepted.html +10 -0
- package/app/templates/projects/hub_assignment.html +50 -0
- package/app/templates/projects/hub_pending.html +20 -0
- package/app/templates/projects/hub_pending_list.html +17 -0
- package/app/templates/projects/hub_state_note.html +23 -0
- package/app/templates/projects/hub_step_note.html +5 -0
- package/app/templates/projects/hub_sync.html +105 -0
- package/app/templates/projects/hub_sync_confirm.html +32 -0
- package/app/templates/projects/hub_sync_worker.html +5 -0
- package/app/templates/projects/hub_text_rules.html +64 -0
- package/app/templates/projects/identity_report.html +74 -0
- package/app/templates/projects/list.html +90 -0
- package/app/templates/projects/project.html +433 -0
- package/app/templates/reference-missing.html +14 -0
- package/app/templates/runs/run.html +166 -0
- package/app/templates/sca/card.html +476 -0
- package/app/templates/sca/index.html +321 -0
- package/app/templates/settings/hub_url.html +31 -0
- package/app/templates/settings/index.html +216 -0
- package/app/templates/settings/provider.html +229 -0
- package/app/templates/settings/providers.html +37 -0
- package/app/templates/worklists.html +52 -0
- package/app/textsafe.py +117 -0
- package/app/timing.py +245 -0
- package/app/triage.py +1743 -0
- package/app/view.py +430 -0
- package/app/worklists.py +204 -0
- package/examples/demo/demo-hub-report-v2.xlsx +0 -0
- package/examples/demo/demo-hub-report.xlsx +0 -0
- package/examples/demo/repos/billing-core/pom.xml +30 -0
- package/examples/demo/repos/billing-core/src/main/java/org/example/billing/Invoice.java +9 -0
- package/examples/demo/repos/billing-core/src/main/java/org/example/billing/InvoiceRepository.java +51 -0
- package/examples/demo/repos/billing-core/src/main/java/org/example/billing/InvoiceStatus.java +11 -0
- package/examples/demo/repos/billing-core/src/main/java/org/example/billing/config/DatabaseSettings.java +18 -0
- package/examples/demo/repos/billing-core/src/main/java/org/example/billing/payment/PaymentTokenService.java +35 -0
- package/examples/demo/repos/billing-core/src/main/java/org/example/billing/web/InvoiceExportController.java +57 -0
- package/examples/demo/repos/catalog-api/catalog_api/__init__.py +1 -0
- package/examples/demo/repos/catalog-api/catalog_api/products.py +24 -0
- package/examples/demo/repos/catalog-api/catalog_api/tasks.py +24 -0
- package/examples/demo/repos/catalog-api/catalog_api/views.py +17 -0
- package/examples/demo/repos/catalog-api/requirements.txt +3 -0
- package/npm/build.json +5 -0
- package/npm/cli.js +346 -0
- package/npm/files.manifest +224 -0
- package/npm/lib/binding.js +64 -0
- package/npm/lib/cluster.js +330 -0
- package/npm/lib/env.js +88 -0
- package/npm/lib/fsx.js +125 -0
- package/npm/lib/http.js +53 -0
- package/npm/lib/metrics.js +73 -0
- package/npm/lib/paths.js +150 -0
- package/npm/lib/pg.js +351 -0
- package/npm/lib/proc.js +101 -0
- package/npm/lib/python.js +133 -0
- package/npm/lib/stand.js +46 -0
- package/npm/lib/supervisor.js +1141 -0
- package/npm/lib/version.js +21 -0
- package/npm/requirements.txt +476 -0
- package/npm-shrinkwrap.json +111 -0
- package/package.json +37 -4
|
@@ -0,0 +1,342 @@
|
|
|
1
|
+
"""ADR-0122, 7а: выбранный состав SAST, одно решение и одна строка создания."""
|
|
2
|
+
|
|
3
|
+
from fastapi import APIRouter, Request
|
|
4
|
+
from starlette.concurrency import run_in_threadpool
|
|
5
|
+
|
|
6
|
+
from app import defect, defect_where, hub_defects, hub_ops, hub_status, hub_sync
|
|
7
|
+
from app.db import dumps, loads, source_signature, source_version
|
|
8
|
+
from app.hub_entity_versions import digest
|
|
9
|
+
from app.hub_import import _cwe2
|
|
10
|
+
from app.models import Finding
|
|
11
|
+
from app.routes import hub_defect_routes as flow
|
|
12
|
+
from app.routes.common import db_of, project_or_404, redirect, render, settings_of
|
|
13
|
+
from app.routes.hub_bulk import MAX_SELECTED
|
|
14
|
+
from app.routes.hub_status_routes import hub_target_check, send_blocker
|
|
15
|
+
|
|
16
|
+
router = APIRouter()
|
|
17
|
+
CONTRACT = "selected-defect/1"
|
|
18
|
+
STALE = "Состав, данные или решение изменились после показа: проверьте состав и подтвердите подготовку заново."
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def selection(values):
|
|
22
|
+
"""Сохраняет порядок и уникальность; непригодные ID показываются среди исключений."""
|
|
23
|
+
result = list(dict.fromkeys(str(value).strip() for value in values))
|
|
24
|
+
if len(result) > MAX_SELECTED:
|
|
25
|
+
raise hub_defects.DefectRefused(f"Выбрано {len(result)} находок; допускается не более {MAX_SELECTED}.")
|
|
26
|
+
return result
|
|
27
|
+
|
|
28
|
+
|
|
29
|
+
def attempt(row):
|
|
30
|
+
saved = loads(row.get("source_fingerprint"), {})
|
|
31
|
+
return saved if isinstance(saved, dict) and saved.get("contract") == CONTRACT else None
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def bases(c, project_id, hub_url, app_id):
|
|
35
|
+
"""Принятые версии назначения: изменение состава Hub делает подтверждение устаревшим."""
|
|
36
|
+
return [dict(r) for r in c.execute(
|
|
37
|
+
"SELECT entity_kind, entity_key, accepted_epoch, accepted_hash FROM hub_entity_versions"
|
|
38
|
+
" WHERE project_id=? AND hub_url=? AND app_id=? ORDER BY entity_kind, entity_key",
|
|
39
|
+
(project_id, hub_ops.hub_base(hub_url), app_id))]
|
|
40
|
+
|
|
41
|
+
|
|
42
|
+
def target_snapshot(request, c, project_id, hub_id):
|
|
43
|
+
from app.routes import findings as card
|
|
44
|
+
db = db_of(request)
|
|
45
|
+
source = db.case_input(project_id, hub_id)
|
|
46
|
+
if source is None or source.get("missing"):
|
|
47
|
+
raise hub_defects.DefectRefused("Находка отсутствует: подтвердите состав заново.")
|
|
48
|
+
case = db.get_case(project_id, hub_id)
|
|
49
|
+
raw = c.execute("SELECT * FROM reviews WHERE project_id=? AND hub_id=?", (project_id, hub_id)).fetchone()
|
|
50
|
+
review = dict(raw) if raw else {}
|
|
51
|
+
f = Finding.from_dict(source["data"])
|
|
52
|
+
# срез разбора модели не совпадает с текущим – предупреждение в составе, а не исключение (ADR-0147); исход
|
|
53
|
+
# сохраняется «Не согласен» со своим вердиктом (sast_choice получает stale_verdict)
|
|
54
|
+
code = card.check_code(request, project_id, f, case, "", None)
|
|
55
|
+
if review.get("finding_sig") and review["finding_sig"] != source_signature(source):
|
|
56
|
+
raise hub_defects.DefectRefused("источник изменился после решения: проверьте находку в карточке")
|
|
57
|
+
return {"hub_id": hub_id, "source": source, "review": review,
|
|
58
|
+
"case": case, "code": code,
|
|
59
|
+
"number": db.defect_number(project_id, defect.defect_key_sast(hub_id))}
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def group_check(db, c, project_id, hub_id, body, targets, hub_url, app_id, own_id=None):
|
|
63
|
+
"""Каждая исходная группа; сосед не добавляется в состав даже при одинаковом тексте."""
|
|
64
|
+
flow.group_defect_check(db, project_id, hub_id, body, targets, hub_url=hub_url,
|
|
65
|
+
app_id=app_id, in_preview=own_id is not None)(c)
|
|
66
|
+
for neighbor in db._group_neighbor_ids(c, project_id, hub_id):
|
|
67
|
+
if neighbor in targets:
|
|
68
|
+
continue
|
|
69
|
+
row = db.hub_defect_for(project_id, [neighbor])
|
|
70
|
+
if (row and row["id"] != own_id and row["state"] in ("blank", "preview")
|
|
71
|
+
and flow._same_hub(row, hub_url, app_id) and defect_where.body_of(row["command_text"]) == defect_where.body_of(body)):
|
|
72
|
+
raise hub_defects.DefectRefused("в группе уже готовится дефект с тем же текстом: завершите его подготовку")
|
|
73
|
+
|
|
74
|
+
|
|
75
|
+
def busy_reason(db, c, project_id, hub_id, issue_id, own_id=None):
|
|
76
|
+
"""Причина занятости называет первую подходящую строку, а её текст входит в подпись предпросмотра: порядок – по номеру,
|
|
77
|
+
иначе перестановка строк в таблице сделала бы показанную форму устаревшей без единой правки."""
|
|
78
|
+
for row in c.execute(f"SELECT * FROM hub_defects WHERE project_id=? AND {db.HUB_DEFECT_RESERVING} ORDER BY id", (project_id,)):
|
|
79
|
+
if row["id"] == own_id:
|
|
80
|
+
continue
|
|
81
|
+
if any(str(t.get("hub_id")) == hub_id or (t.get("issue_id") == issue_id and t.get("issue_type") == "SAST")
|
|
82
|
+
for t in loads(row["targets_json"], [])):
|
|
83
|
+
return f"занята другим дефектом или подготовкой №{row['id']} ({hub_defects.STATE_LABELS.get(row['state'], row['state'])})"
|
|
84
|
+
for op in c.execute("SELECT id, targets_json FROM hub_ops WHERE project_id=? AND purpose='status'"
|
|
85
|
+
" AND state IN ('queued','taken','unknown') ORDER BY id", (project_id,)):
|
|
86
|
+
if any(t.get("issue_id") == issue_id and t.get("issue_type") == "SAST" for t in loads(op["targets_json"], [])):
|
|
87
|
+
return f"незавершённая операция статуса №{op['id']}: дождитесь итога или сверьте результат"
|
|
88
|
+
return ""
|
|
89
|
+
|
|
90
|
+
|
|
91
|
+
def assess(request, c, project, selected, anchor, comment, fix):
|
|
92
|
+
"""Локальный снимок для показа и повторной проверки на том же соединении."""
|
|
93
|
+
db, settings, pid = db_of(request), settings_of(request), project["id"]
|
|
94
|
+
selected = selection(selected)
|
|
95
|
+
anchor = anchor or (selected[0] if selected else "")
|
|
96
|
+
anchor_row = db.get_finding_by_hub_id(pid, anchor) if anchor in selected else None
|
|
97
|
+
f0 = Finding.from_dict(anchor_row["data"]) if anchor_row else None
|
|
98
|
+
ready, excluded, snapshots = [], [], []
|
|
99
|
+
app_id = int(project.get("hub_app_id") or 0)
|
|
100
|
+
body = defect.sast_body(f0, "true_positive", comment, fix, settings.team_name) if defect.has_recommendation(fix) else ""
|
|
101
|
+
aliases = {}
|
|
102
|
+
for raw in c.execute("SELECT hub_id FROM findings WHERE project_id=? AND kind='SAST'", (pid,)):
|
|
103
|
+
try:
|
|
104
|
+
ident = hub_status.issue_id(raw["hub_id"])
|
|
105
|
+
except hub_status.StatusRefused:
|
|
106
|
+
continue
|
|
107
|
+
aliases.setdefault(ident, []).append(raw["hub_id"])
|
|
108
|
+
for hub_id in selected:
|
|
109
|
+
row = db.get_finding_by_hub_id(pid, hub_id)
|
|
110
|
+
why, snap = "", None
|
|
111
|
+
f = Finding.from_dict(row["data"]) if row else None
|
|
112
|
+
url = flow._card_url_of(db, pid, hub_id) if row else ""
|
|
113
|
+
try:
|
|
114
|
+
if f is None or row.get("missing"):
|
|
115
|
+
raise hub_defects.DefectRefused("находка отсутствует по принятому состоянию Hub или выгрузке")
|
|
116
|
+
if f.kind != "SAST":
|
|
117
|
+
raise hub_defects.DefectRefused("не SAST: для SCA откройте карточку соответствующего набора")
|
|
118
|
+
if not f0 or f0.kind != "SAST":
|
|
119
|
+
raise hub_defects.DefectRefused("выберите опорную находку SAST из исходного выбора")
|
|
120
|
+
if f.codebase != f0.codebase:
|
|
121
|
+
raise hub_defects.DefectRefused("другая кодовая база")
|
|
122
|
+
if f.rule != f0.rule:
|
|
123
|
+
raise hub_defects.DefectRefused("другая категория")
|
|
124
|
+
if _cwe2(f.cwe) != _cwe2(f0.cwe):
|
|
125
|
+
raise hub_defects.DefectRefused("другой набор CWE")
|
|
126
|
+
ident = hub_status.issue_id(hub_id)
|
|
127
|
+
if len(aliases.get(ident, [])) != 1:
|
|
128
|
+
raise hub_defects.DefectRefused("неоднозначное сопоставление номера Hub после нормализации")
|
|
129
|
+
why = busy_reason(db, c, pid, hub_id, ident)
|
|
130
|
+
if why:
|
|
131
|
+
raise hub_defects.DefectRefused(why)
|
|
132
|
+
snap = target_snapshot(request, c, pid, hub_id)
|
|
133
|
+
if body:
|
|
134
|
+
# Невыбранные соседи, в том числе у второй и последней цели.
|
|
135
|
+
group_check(db, c, pid, hub_id, body, [], settings.hub_url, app_id)
|
|
136
|
+
except ValueError as e:
|
|
137
|
+
why = str(e)
|
|
138
|
+
snapshots.append({"hub_id": hub_id, "row": row, "snapshot": snap, "why": why})
|
|
139
|
+
if why:
|
|
140
|
+
excluded.append({"hub_id": hub_id, "why": why, "url": url})
|
|
141
|
+
else:
|
|
142
|
+
ready.append({"hub_id": hub_id, "url": url, "finding": f, "snapshot": snap,
|
|
143
|
+
"issue_id": ident, "issue_type": "SAST"})
|
|
144
|
+
numbers = {r["snapshot"]["number"] for r in ready if r["snapshot"]["number"] is not None}
|
|
145
|
+
problems = []
|
|
146
|
+
if len(numbers) > 1:
|
|
147
|
+
problems.append("У выбранных находок разные номера ID: автоматическое объединение номеров не выполняется.")
|
|
148
|
+
number = next(iter(numbers)) if len(numbers) == 1 else db.next_defect_number(pid)
|
|
149
|
+
keys = [defect.defect_key_sast(r["hub_id"]) for r in ready]
|
|
150
|
+
if db.hub_defects_with_number(pid, number) or set(db.defect_keys_with_number(pid, number)) - set(keys):
|
|
151
|
+
problems.append(f"Номер ID{number} удерживают другие находки или подготовка: нужен новый предпросмотр.")
|
|
152
|
+
if not ready:
|
|
153
|
+
problems.append("Нет находок, пригодных для подготовки дефекта.")
|
|
154
|
+
elif len(",".join(str(r["issue_id"]) for r in ready)) > hub_defects.ISSUE_IDS_MAX:
|
|
155
|
+
size = len(",".join(str(r["issue_id"]) for r in ready))
|
|
156
|
+
problems.append(f"Состав из {len(ready)} находок: issueIds занимает {size} знаков при пределе "
|
|
157
|
+
f"{hub_defects.ISSUE_IDS_MAX}. Выберите меньше находок; решения не сохранены.")
|
|
158
|
+
from app.routes import findings as card
|
|
159
|
+
anchor_ready = next((r for r in ready if r["hub_id"] == anchor), None)
|
|
160
|
+
model = card.model_suggestion(f0, anchor_ready["snapshot"]["case"]) if anchor_ready else {}
|
|
161
|
+
warnings = ["Выбраны разные файлы одной кодовой базы: проверьте применимость общего решения."] \
|
|
162
|
+
if len({r["finding"].file for r in ready}) > 1 else []
|
|
163
|
+
replacements = []
|
|
164
|
+
for r in ready:
|
|
165
|
+
old = r["snapshot"]["review"]
|
|
166
|
+
if (old.get("decision") or old.get("final_comment") or old.get("final_fix")) and hub_defects.decision_changes(
|
|
167
|
+
old, "disagree", "true_positive", comment, fix):
|
|
168
|
+
replacements.append({"hub_id": r["hub_id"], "revision": old["revision"],
|
|
169
|
+
"outcome": old.get("final_verdict") or old.get("decision"),
|
|
170
|
+
"comment": old.get("final_comment"), "fix": old.get("final_fix")})
|
|
171
|
+
state = {"contract": CONTRACT, "project_id": pid, "hub_url": hub_ops.hub_base(settings.hub_url),
|
|
172
|
+
"app_id": app_id, "selected": selected, "anchor": anchor, "snapshots": snapshots,
|
|
173
|
+
"comment": comment, "fix": fix, "body": body, "number": number, "keys": keys,
|
|
174
|
+
"bases": bases(c, pid, settings.hub_url, app_id), "problems": problems,
|
|
175
|
+
"replacements": replacements, "service": bool(hub_defects.service_info(db, settings.hub_url))}
|
|
176
|
+
return {"ready": ready, "excluded": excluded, "warnings": warnings, "problems": problems,
|
|
177
|
+
"replacements": replacements, "number": number, "body": body, "anchor_finding": f0,
|
|
178
|
+
"anchor": anchor, "snapshot": digest(state), "state": state, "model": model,
|
|
179
|
+
"anchor_ready": anchor_ready is not None}
|
|
180
|
+
|
|
181
|
+
|
|
182
|
+
def text_check(request, project_id, view, comment, fix):
|
|
183
|
+
from app.routes import findings as card
|
|
184
|
+
if not defect.reason_text(comment):
|
|
185
|
+
raise hub_defects.DefectRefused(card.SAST_NEED_REASON["true_positive"])
|
|
186
|
+
if not defect.has_recommendation(fix):
|
|
187
|
+
raise hub_defects.DefectRefused(card.SAST_NEED_FIX)
|
|
188
|
+
if len(comment) > card.MAX_COMMENT or len(fix) > card.MAX_FIX:
|
|
189
|
+
raise hub_defects.DefectRefused("Обоснование или рекомендация слишком длинные: сократите текст.")
|
|
190
|
+
db = db_of(request)
|
|
191
|
+
for r in view["ready"]:
|
|
192
|
+
snap = r["snapshot"]
|
|
193
|
+
origin = card.comment_origin(db, snap["case"], snap["review"], comment,
|
|
194
|
+
project_id=project_id, hub_id=r["hub_id"])
|
|
195
|
+
if origin and origin != "true_positive":
|
|
196
|
+
raise hub_defects.DefectRefused(f"{r['hub_id']}: обоснование написано моделью к другому исходу; поправьте текст.")
|
|
197
|
+
problems = flow.sast_text_problems(db, settings_of(request), project_id, view["body"])
|
|
198
|
+
if problems:
|
|
199
|
+
from app import defect_check
|
|
200
|
+
raise hub_defects.DefectRefused(defect_check.refusal(problems))
|
|
201
|
+
|
|
202
|
+
|
|
203
|
+
def prepare(request, project_id, selected, anchor, comment, fix, shown, confirmed, overwrite):
|
|
204
|
+
from app.routes import findings as card
|
|
205
|
+
db = db_of(request)
|
|
206
|
+
prepared = {}
|
|
207
|
+
|
|
208
|
+
def check(c):
|
|
209
|
+
project = project_or_404(request, project_id)
|
|
210
|
+
settings = settings_of(request)
|
|
211
|
+
view = assess(request, c, project, selected, anchor, comment, fix)
|
|
212
|
+
if not shown or shown != view["snapshot"]:
|
|
213
|
+
raise hub_defects.DefectRefused(STALE)
|
|
214
|
+
if not confirmed:
|
|
215
|
+
raise hub_defects.DefectRefused("Подтвердите показанное решение и запросы подготовки.")
|
|
216
|
+
if view["replacements"] and not overwrite:
|
|
217
|
+
raise hub_defects.DefectRefused("Подтвердите перезапись перечисленных решений.")
|
|
218
|
+
blocker = send_blocker(settings, project) or hub_sync.resync_blocker(db_of(request), project_id)
|
|
219
|
+
if blocker or view["problems"]:
|
|
220
|
+
raise hub_defects.DefectRefused(blocker or " ".join(view["problems"]))
|
|
221
|
+
text_check(request, project_id, view, comment, fix)
|
|
222
|
+
hub_target_check(request, project_id, settings.hub_url, project["hub_app_id"])(c)
|
|
223
|
+
# Даже отказ длины запроса обязан произойти до первого сохранения.
|
|
224
|
+
hub_defects.blank_request(project["hub_app_id"], view["ready"], "SAST")
|
|
225
|
+
prepared.update(view=view, project=project, settings=settings)
|
|
226
|
+
reviews = []
|
|
227
|
+
for r in view["ready"]:
|
|
228
|
+
snap = r["snapshot"]
|
|
229
|
+
case, old, code = snap["case"] or {}, snap["review"], snap["code"]
|
|
230
|
+
decision, outcome = card.sast_choice("true_positive", "", case.get("verdict"), code["stale_verdict"], old,
|
|
231
|
+
case.get("verdict_id"))
|
|
232
|
+
reviews.append(dict(hub_id=r["hub_id"], decision=decision, final_verdict=outcome,
|
|
233
|
+
final_comment=comment, final_fix=fix, reviewed=True, verdict_id=case.get("verdict_id"),
|
|
234
|
+
expect={"version": source_version(snap["source"]), "revision": old.get("revision", 0),
|
|
235
|
+
"verdict_id": case.get("verdict_id")},
|
|
236
|
+
code_hash=code["hash"], code_chars=code["chars"], require_present=True))
|
|
237
|
+
return reviews
|
|
238
|
+
|
|
239
|
+
def build(c, saved):
|
|
240
|
+
view, settings, project = prepared["view"], prepared["settings"], prepared["project"]
|
|
241
|
+
targets = [flow._defect_target(r["finding"], review, view["body"], flow.defect_parts(settings, review),
|
|
242
|
+
db.finding_sig(project_id, r["hub_id"]))
|
|
243
|
+
for r, review in zip(view["ready"], saved, strict=True)]
|
|
244
|
+
snapshots = []
|
|
245
|
+
for r in view["ready"]:
|
|
246
|
+
current = target_snapshot(request, c, project_id, r["hub_id"])
|
|
247
|
+
confirmed_code = r["snapshot"]["code"]
|
|
248
|
+
if any(current["code"][key] != confirmed_code[key] for key in ("hash", "chars")):
|
|
249
|
+
raise hub_defects.DefectRefused(STALE)
|
|
250
|
+
snapshots.append({"hub_id": r["hub_id"], "fingerprint": digest(current)})
|
|
251
|
+
saved_attempt = {"contract": CONTRACT, "selected": selected, "anchor": view["anchor"],
|
|
252
|
+
"confirmation": shown, "bases": view["state"]["bases"],
|
|
253
|
+
"snapshots": snapshots}
|
|
254
|
+
summary = defect.sast_title(view["number"], view["anchor_finding"])
|
|
255
|
+
row = dict(kind="SAST", number=view["number"], summary=summary, anchor_hub_id=view["anchor"],
|
|
256
|
+
targets=targets, number_keys=view["state"]["keys"], command_text=view["body"],
|
|
257
|
+
hub_url=settings.hub_url, app_id=project["hub_app_id"], left_out=view["excluded"],
|
|
258
|
+
warnings=view["warnings"], source_fingerprint=dumps(saved_attempt))
|
|
259
|
+
items = [dict(purpose="defect_blank", request=hub_defects.blank_request(project["hub_app_id"], targets, "SAST"),
|
|
260
|
+
hub_url=settings.hub_url, app_id=project["hub_app_id"], project_id=project_id,
|
|
261
|
+
targets=targets, text=summary)]
|
|
262
|
+
if not view["state"]["service"]:
|
|
263
|
+
items.append(dict(purpose="read", request=hub_defects.service_info_request(), hub_url=settings.hub_url,
|
|
264
|
+
app_id=project["hub_app_id"], project_id=project_id, targets=[]))
|
|
265
|
+
return row, items
|
|
266
|
+
|
|
267
|
+
return db.hub_defect_selected_start(project_id, check=check, build=build)
|
|
268
|
+
|
|
269
|
+
|
|
270
|
+
def create_check(request, c, row):
|
|
271
|
+
"""Состав не сокращается при позднем конфликте: вся постановка отклоняется."""
|
|
272
|
+
saved = attempt(row)
|
|
273
|
+
if saved is None:
|
|
274
|
+
return
|
|
275
|
+
db, pid = db_of(request), row["project_id"]
|
|
276
|
+
targets = loads(row["targets_json"], [])
|
|
277
|
+
ids = [t["hub_id"] for t in targets]
|
|
278
|
+
if ids != [s["hub_id"] for s in saved["snapshots"]]:
|
|
279
|
+
raise hub_defects.DefectRefused(STALE)
|
|
280
|
+
if saved["bases"] != bases(c, pid, row["hub_url"], row["app_id"]):
|
|
281
|
+
raise hub_defects.DefectRefused("Принятая база Hub изменилась: отмените предпросмотр и подготовьте заново.")
|
|
282
|
+
if (db.hub_defects_with_number(pid, row["number"], exclude_id=row["id"])
|
|
283
|
+
or set(db.defect_keys_with_number(pid, row["number"])) - set(loads(row["number_keys_json"], []))):
|
|
284
|
+
raise hub_defects.DefectRefused("Номер дефекта занят другой находкой: отмените предпросмотр и подготовьте заново.")
|
|
285
|
+
for target, snapshot in zip(targets, saved["snapshots"], strict=True):
|
|
286
|
+
if digest(target_snapshot(request, c, pid, target["hub_id"])) != snapshot["fingerprint"]:
|
|
287
|
+
raise hub_defects.DefectRefused(STALE)
|
|
288
|
+
why = busy_reason(db, c, pid, target["hub_id"], target["issue_id"], row["id"])
|
|
289
|
+
if why:
|
|
290
|
+
raise hub_defects.DefectRefused(why)
|
|
291
|
+
group_check(db, c, pid, target["hub_id"], row["command_text"], ids,
|
|
292
|
+
row["hub_url"], row["app_id"], row["id"])
|
|
293
|
+
|
|
294
|
+
|
|
295
|
+
def page(request, project_id, selected, anchor="", comment=None, fix=None, *, confirm=False, error="", status=200):
|
|
296
|
+
project, db = project_or_404(request, project_id), db_of(request)
|
|
297
|
+
selected = selection(selected)
|
|
298
|
+
view = db.on_connection(lambda c: assess(request, c, project, selected, anchor, comment or "", fix or ""))
|
|
299
|
+
if comment is None and fix is None:
|
|
300
|
+
texts = {(r["snapshot"]["review"].get("final_comment", ""), r["snapshot"]["review"].get("final_fix", ""))
|
|
301
|
+
for r in view["ready"]}
|
|
302
|
+
comment, fix = next(iter(texts)) if len(texts) == 1 else ("", "")
|
|
303
|
+
view = db.on_connection(lambda c: assess(request, c, project, selected, anchor, comment, fix))
|
|
304
|
+
if confirm:
|
|
305
|
+
try:
|
|
306
|
+
text_check(request, project_id, view, comment, fix)
|
|
307
|
+
except ValueError as e:
|
|
308
|
+
error, confirm, status = str(e), False, 400
|
|
309
|
+
return render(request, "findings/defect_selected.html", dict(project=project, selected=selected,
|
|
310
|
+
comment=comment or "", fix=fix or "", confirm=confirm, error=error,
|
|
311
|
+
blocker=send_blocker(settings_of(request), project) or hub_sync.resync_blocker(db_of(request), project_id), **view), status_code=status)
|
|
312
|
+
|
|
313
|
+
|
|
314
|
+
@router.get("/projects/{project_id}/hub/defect-selected")
|
|
315
|
+
async def selected_page(request: Request, project_id: int):
|
|
316
|
+
try:
|
|
317
|
+
return await run_in_threadpool(page, request, project_id, request.query_params.getlist("hub_id"))
|
|
318
|
+
except ValueError as e:
|
|
319
|
+
return await run_in_threadpool(page, request, project_id, [], error=str(e), status=400)
|
|
320
|
+
|
|
321
|
+
|
|
322
|
+
@router.post("/projects/{project_id}/hub/defect-selected")
|
|
323
|
+
async def selected_submit(request: Request, project_id: int):
|
|
324
|
+
form = await request.form(max_fields=MAX_SELECTED + 20, max_files=0)
|
|
325
|
+
selected = form.getlist("hub_id")
|
|
326
|
+
anchor = str(form.get("anchor") or "")
|
|
327
|
+
comment = str(form.get("comment") or "").replace("\r\n", "\n")
|
|
328
|
+
fix = str(form.get("fix") or "").replace("\r\n", "\n")
|
|
329
|
+
try:
|
|
330
|
+
selected = selection(selected)
|
|
331
|
+
if form.get("action") == "prepare":
|
|
332
|
+
row = await run_in_threadpool(prepare, request, project_id, selected, anchor, comment, fix,
|
|
333
|
+
str(form.get("snapshot") or ""), form.get("confirm") == "1",
|
|
334
|
+
form.get("overwrite") == "1")
|
|
335
|
+
return redirect(f"/projects/{project_id}/defects/{row['id']}")
|
|
336
|
+
return await run_in_threadpool(page, request, project_id, selected, anchor, comment, fix,
|
|
337
|
+
confirm=form.get("action") == "preview")
|
|
338
|
+
except ValueError as e:
|
|
339
|
+
if len(set(selected)) > MAX_SELECTED:
|
|
340
|
+
selected = []
|
|
341
|
+
return await run_in_threadpool(page, request, project_id, selected, anchor, comment, fix,
|
|
342
|
+
error=str(e), status=409)
|
|
@@ -0,0 +1,155 @@
|
|
|
1
|
+
"""Раздел «Справочники»: загрузка выгрузки БДУ ФСТЭК России и каталога CWE, что загружено и когда.
|
|
2
|
+
|
|
3
|
+
Файл сохраняется во временную папку данных (DATA_DIR/tmp) и разбирается в фоновом потоке (app/feeds.py): выгрузка
|
|
4
|
+
БДУ – это около 0,6 ГБ XML, разбор идёт около минуты. Страница показывает ход загрузки и сама обновляется, когда
|
|
5
|
+
она закончится (data-poll); идущую загрузку можно остановить кнопкой «Остановить загрузку». Шаблон – templates/feeds/index.html.
|
|
6
|
+
"""
|
|
7
|
+
|
|
8
|
+
from __future__ import annotations
|
|
9
|
+
|
|
10
|
+
import logging
|
|
11
|
+
import os
|
|
12
|
+
import tempfile
|
|
13
|
+
import threading
|
|
14
|
+
from pathlib import Path
|
|
15
|
+
|
|
16
|
+
from fastapi import APIRouter, HTTPException, Request
|
|
17
|
+
from fastapi.responses import HTMLResponse
|
|
18
|
+
from starlette.concurrency import run_in_threadpool
|
|
19
|
+
from starlette.datastructures import UploadFile
|
|
20
|
+
|
|
21
|
+
from app.feeds import BUSY, KINDS, FeedError, FeedStore
|
|
22
|
+
from app.logsafe import log_failure
|
|
23
|
+
from app.routes.common import MAX_NAME, redirect, register_messages, render, settings_of
|
|
24
|
+
|
|
25
|
+
log = logging.getLogger("local_triage.web")
|
|
26
|
+
|
|
27
|
+
router = APIRouter()
|
|
28
|
+
|
|
29
|
+
register_messages({
|
|
30
|
+
"feed_started": "Файл принят: справочник загружается – страница обновится, когда загрузка закончится.",
|
|
31
|
+
"feed_cancel": "Загрузка останавливается: прежний справочник не изменится.",
|
|
32
|
+
})
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
class FeedJob:
|
|
36
|
+
"""Загрузка справочника в фоновом потоке: одна за раз. state – idle, running, done, error."""
|
|
37
|
+
|
|
38
|
+
def __init__(self) -> None:
|
|
39
|
+
self._lock = threading.Lock()
|
|
40
|
+
self.state = "idle"
|
|
41
|
+
self.kind = ""
|
|
42
|
+
self.filename = ""
|
|
43
|
+
self.message = ""
|
|
44
|
+
|
|
45
|
+
def snapshot(self) -> dict:
|
|
46
|
+
with self._lock:
|
|
47
|
+
return {"state": self.state, "kind": self.kind, "label": KINDS.get(self.kind, ""),
|
|
48
|
+
"filename": self.filename, "message": self.message}
|
|
49
|
+
|
|
50
|
+
def start(self, store: FeedStore, kind: str, path: Path, filename: str) -> None:
|
|
51
|
+
with self._lock:
|
|
52
|
+
if self.state == "running":
|
|
53
|
+
path.unlink(missing_ok=True)
|
|
54
|
+
raise FeedError(BUSY)
|
|
55
|
+
self.state, self.kind, self.filename, self.message = "running", kind, filename, ""
|
|
56
|
+
thread = threading.Thread(target=self._run, args=(store, kind, path, filename), name="feed-load", daemon=True)
|
|
57
|
+
thread.start()
|
|
58
|
+
|
|
59
|
+
def _run(self, store: FeedStore, kind: str, path: Path, filename: str) -> None:
|
|
60
|
+
try:
|
|
61
|
+
meta = store.load(kind, path, filename)
|
|
62
|
+
message = (f"{KINDS[kind]}: загружено записей – {meta['records']}"
|
|
63
|
+
+ (f", дата выгрузки {meta['source_date']}" if meta.get("source_date") else "")
|
|
64
|
+
+ f", {meta['seconds']} с.")
|
|
65
|
+
state = "done"
|
|
66
|
+
except FeedError as e:
|
|
67
|
+
state, message = "error", f"{KINDS[kind]}: файл не принят. {e}"
|
|
68
|
+
except Exception as e: # текст исключения в журнал не пишется (ADR 0016)
|
|
69
|
+
eid = log_failure(log, "справочник не загружен", e)
|
|
70
|
+
state, message = "error", f"{KINDS[kind]}: внутренняя ошибка при загрузке (номер в журнале {eid})."
|
|
71
|
+
finally:
|
|
72
|
+
path.unlink(missing_ok=True)
|
|
73
|
+
with self._lock:
|
|
74
|
+
self.state, self.message = state, message
|
|
75
|
+
|
|
76
|
+
|
|
77
|
+
def store_of(request: Request) -> FeedStore:
|
|
78
|
+
return request.app.state.feeds
|
|
79
|
+
|
|
80
|
+
|
|
81
|
+
def job_of(request: Request) -> FeedJob:
|
|
82
|
+
return request.app.state.feed_job
|
|
83
|
+
|
|
84
|
+
|
|
85
|
+
def page_context(request: Request) -> dict:
|
|
86
|
+
store = store_of(request)
|
|
87
|
+
status = store.status()
|
|
88
|
+
job = job_of(request).snapshot()
|
|
89
|
+
context = {"kinds": KINDS, "status": status, "job": job,
|
|
90
|
+
"max_upload_mb": settings_of(request).max_upload_mb}
|
|
91
|
+
context['capec_profile'] = store.capec_data(shared=True)['profile'] if 'capec' in status else {}
|
|
92
|
+
# После обновления страницы ?ok= остаётся в адресе, но завершённая загрузка уже не идёт.
|
|
93
|
+
if job["state"] != "running" and request.query_params.get("ok") in {"feed_started", "feed_cancel"}:
|
|
94
|
+
context["ok_message"] = ""
|
|
95
|
+
return context
|
|
96
|
+
|
|
97
|
+
|
|
98
|
+
@router.get("/feeds", response_class=HTMLResponse)
|
|
99
|
+
def feeds_page(request: Request):
|
|
100
|
+
return render(request, "feeds/index.html", page_context(request))
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
@router.get("/feeds/job")
|
|
104
|
+
def feeds_job(request: Request) -> dict:
|
|
105
|
+
"""Ход загрузки для страницы (data-poll): done и reload – загрузка закончилась, страницу пора обновить."""
|
|
106
|
+
job = job_of(request).snapshot()
|
|
107
|
+
finished = job["state"] != "running"
|
|
108
|
+
return {**job, "done": finished, "reload": finished}
|
|
109
|
+
|
|
110
|
+
|
|
111
|
+
@router.post("/feeds/cancel")
|
|
112
|
+
def cancel_feed(request: Request):
|
|
113
|
+
"""«Остановить загрузку»: разбор прервётся на ближайшем куске, временные таблицы удалятся, прежний справочник
|
|
114
|
+
останется. Загрузки нет – просто возврат на страницу."""
|
|
115
|
+
stopping = job_of(request).snapshot()["state"] == "running" and store_of(request).cancel()
|
|
116
|
+
return redirect("/feeds?ok=feed_cancel" if stopping else "/feeds")
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
@router.post("/feeds/{kind}")
|
|
120
|
+
async def upload_feed(request: Request, kind: str):
|
|
121
|
+
if kind not in KINDS:
|
|
122
|
+
raise HTTPException(status_code=404, detail="Такого справочника нет.")
|
|
123
|
+
settings = await run_in_threadpool(settings_of, request)
|
|
124
|
+
form = await request.form(max_files=1, max_fields=5)
|
|
125
|
+
try:
|
|
126
|
+
upload = form.get("file")
|
|
127
|
+
try:
|
|
128
|
+
if not isinstance(upload, UploadFile) or not upload.filename:
|
|
129
|
+
raise FeedError("Выберите файл архива (.zip).")
|
|
130
|
+
filename = Path(upload.filename.replace("\\", "/")).name[:MAX_NAME]
|
|
131
|
+
if not filename.lower().endswith(".zip"):
|
|
132
|
+
raise FeedError("Нужен архив .zip – в том виде, как он скачан с сайта.")
|
|
133
|
+
settings.tmp_dir.mkdir(parents=True, exist_ok=True)
|
|
134
|
+
fd, tmp_name = tempfile.mkstemp(dir=settings.tmp_dir, prefix="feed-", suffix=".zip")
|
|
135
|
+
tmp = Path(tmp_name)
|
|
136
|
+
try:
|
|
137
|
+
size = 0
|
|
138
|
+
with os.fdopen(fd, "wb") as out:
|
|
139
|
+
while chunk := await upload.read(1024 * 1024):
|
|
140
|
+
size += len(chunk)
|
|
141
|
+
if size > settings.max_upload_bytes:
|
|
142
|
+
raise FeedError(f"Архив больше {settings.max_upload_mb} МБ – это предел приложения "
|
|
143
|
+
"(MAX_UPLOAD_MB).")
|
|
144
|
+
out.write(chunk)
|
|
145
|
+
job_of(request).start(store_of(request), kind, tmp, filename)
|
|
146
|
+
except BaseException:
|
|
147
|
+
tmp.unlink(missing_ok=True)
|
|
148
|
+
raise
|
|
149
|
+
except FeedError as e:
|
|
150
|
+
message = str(e) # имя e после блока except не видно: текст – в переменную до лямбды
|
|
151
|
+
return await run_in_threadpool(lambda: render(request, "feeds/index.html",
|
|
152
|
+
{**page_context(request), "error": message}, 400))
|
|
153
|
+
finally:
|
|
154
|
+
await form.close()
|
|
155
|
+
return redirect("/feeds?ok=feed_started")
|