local-triage 0.0.0-stage → 0.10.0-dev.32

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (223) hide show
  1. package/LICENSE +21 -0
  2. package/README.md +90 -2
  3. package/THIRD-PARTY-NOTICES.md +48 -0
  4. package/app/__init__.py +3 -0
  5. package/app/affiliation.py +137 -0
  6. package/app/ask.py +300 -0
  7. package/app/build_info.py +98 -0
  8. package/app/capec.py +168 -0
  9. package/app/capec_view.py +27 -0
  10. package/app/codecmp.py +59 -0
  11. package/app/config.py +411 -0
  12. package/app/context.py +1451 -0
  13. package/app/cvss.py +357 -0
  14. package/app/db.py +6234 -0
  15. package/app/defect.py +670 -0
  16. package/app/defect_check.py +323 -0
  17. package/app/defect_drafts.py +612 -0
  18. package/app/defect_dto.py +149 -0
  19. package/app/defect_fields.py +248 -0
  20. package/app/defect_form.py +124 -0
  21. package/app/defect_links.py +107 -0
  22. package/app/defect_local_text.py +103 -0
  23. package/app/defect_references.py +164 -0
  24. package/app/defect_where.py +147 -0
  25. package/app/ecosystem.py +51 -0
  26. package/app/executors.py +660 -0
  27. package/app/feeds.py +1231 -0
  28. package/app/guard.py +164 -0
  29. package/app/hub_block_builder.py +200 -0
  30. package/app/hub_block_version.py +220 -0
  31. package/app/hub_defects.py +4469 -0
  32. package/app/hub_effects.py +421 -0
  33. package/app/hub_entity_versions.py +108 -0
  34. package/app/hub_import.py +2054 -0
  35. package/app/hub_onboarding.py +114 -0
  36. package/app/hub_ops.py +330 -0
  37. package/app/hub_pull.py +1195 -0
  38. package/app/hub_state.py +255 -0
  39. package/app/hub_status.py +237 -0
  40. package/app/hub_sync.py +619 -0
  41. package/app/hub_sync_api.py +241 -0
  42. package/app/hub_sync_compare.py +1054 -0
  43. package/app/hub_sync_contract.py +75 -0
  44. package/app/hub_template_model.py +74 -0
  45. package/app/hub_text_scale.py +159 -0
  46. package/app/journal.py +250 -0
  47. package/app/llm.py +850 -0
  48. package/app/llm_chat.py +596 -0
  49. package/app/llm_http.py +319 -0
  50. package/app/llm_messages.py +401 -0
  51. package/app/llm_net.py +764 -0
  52. package/app/llm_ollama.py +571 -0
  53. package/app/lockfile.py +505 -0
  54. package/app/logsafe.py +54 -0
  55. package/app/main.py +367 -0
  56. package/app/mask_display.py +132 -0
  57. package/app/masking.py +814 -0
  58. package/app/metrics.py +194 -0
  59. package/app/model_traffic.py +519 -0
  60. package/app/models.py +483 -0
  61. package/app/navigation.py +121 -0
  62. package/app/osv.py +279 -0
  63. package/app/package_identity.py +189 -0
  64. package/app/package_transition.py +824 -0
  65. package/app/profile_gate.py +71 -0
  66. package/app/prompts.py +1108 -0
  67. package/app/provider_keys.py +189 -0
  68. package/app/providers.py +511 -0
  69. package/app/routes/__init__.py +10 -0
  70. package/app/routes/api.py +153 -0
  71. package/app/routes/capec.py +186 -0
  72. package/app/routes/common.py +407 -0
  73. package/app/routes/components.py +1499 -0
  74. package/app/routes/cwe.py +168 -0
  75. package/app/routes/decision_selected.py +153 -0
  76. package/app/routes/defect_add.py +223 -0
  77. package/app/routes/defect_card_routes.py +363 -0
  78. package/app/routes/defect_selected.py +342 -0
  79. package/app/routes/feeds.py +155 -0
  80. package/app/routes/findings.py +2736 -0
  81. package/app/routes/hub.py +402 -0
  82. package/app/routes/hub_bulk.py +288 -0
  83. package/app/routes/hub_defect_routes.py +1945 -0
  84. package/app/routes/hub_status_routes.py +1132 -0
  85. package/app/routes/hub_sync_routes.py +463 -0
  86. package/app/routes/journal.py +132 -0
  87. package/app/routes/projects.py +1601 -0
  88. package/app/routes/providers.py +336 -0
  89. package/app/routes/references.py +26 -0
  90. package/app/routes/runs.py +219 -0
  91. package/app/routes/sca.py +2722 -0
  92. package/app/routes/settings.py +163 -0
  93. package/app/run_errors.py +151 -0
  94. package/app/sca.py +1662 -0
  95. package/app/sca_contract.py +1289 -0
  96. package/app/sca_research.py +1089 -0
  97. package/app/sca_research_net.py +1007 -0
  98. package/app/sca_text.py +965 -0
  99. package/app/serve.py +70 -0
  100. package/app/settings_store.py +262 -0
  101. package/app/static/app.css +2802 -0
  102. package/app/static/app.js +3772 -0
  103. package/app/static/capec-license.txt +9 -0
  104. package/app/static/favicon.svg +1 -0
  105. package/app/static/fonts/Golos-Text_DemiBold.woff2 +0 -0
  106. package/app/static/fonts/Golos-Text_Regular.woff2 +0 -0
  107. package/app/static/fonts/OFL.txt +90 -0
  108. package/app/static/navigation.js +234 -0
  109. package/app/static/observability.js +121 -0
  110. package/app/static/theme.js +49 -0
  111. package/app/storage/__init__.py +16 -0
  112. package/app/storage/local_state.py +246 -0
  113. package/app/storage/migrate.py +110 -0
  114. package/app/storage/migrations/postgresql/0001_baseline.sql +896 -0
  115. package/app/storage/migrations/postgresql/0002_feeds_and_secrets.sql +55 -0
  116. package/app/storage/migrations/postgresql/0003_project_mask_display.sql +5 -0
  117. package/app/storage/migrations/postgresql/0004_package_identity.sql +36 -0
  118. package/app/storage/postgresql.py +573 -0
  119. package/app/storage/schema.py +619 -0
  120. package/app/storage/sql.py +57 -0
  121. package/app/storage/tools.py +254 -0
  122. package/app/telemetry.py +307 -0
  123. package/app/templates/base.html +91 -0
  124. package/app/templates/capec/context.html +3 -0
  125. package/app/templates/capec/detail.html +27 -0
  126. package/app/templates/capec/index.html +128 -0
  127. package/app/templates/components/_archive.html +11 -0
  128. package/app/templates/components/archived.html +19 -0
  129. package/app/templates/components/batch.html +35 -0
  130. package/app/templates/components/detail.html +332 -0
  131. package/app/templates/components/index.html +338 -0
  132. package/app/templates/components/research_preview.html +80 -0
  133. package/app/templates/components/research_run.html +54 -0
  134. package/app/templates/cwe/bdu.html +56 -0
  135. package/app/templates/cwe/detail.html +233 -0
  136. package/app/templates/cwe/index.html +97 -0
  137. package/app/templates/error.html +8 -0
  138. package/app/templates/feeds/index.html +65 -0
  139. package/app/templates/findings/card.html +639 -0
  140. package/app/templates/findings/decision_selected.html +41 -0
  141. package/app/templates/findings/defect_add.html +46 -0
  142. package/app/templates/findings/defect_selected.html +47 -0
  143. package/app/templates/findings/list.html +153 -0
  144. package/app/templates/findings/send_selected.html +53 -0
  145. package/app/templates/findings/trace.html +72 -0
  146. package/app/templates/journal/detail.html +19 -0
  147. package/app/templates/journal/index.html +19 -0
  148. package/app/templates/macros.html +440 -0
  149. package/app/templates/navigation-fields.html +1 -0
  150. package/app/templates/navigation.html +11 -0
  151. package/app/templates/projects/_identity_summary.html +42 -0
  152. package/app/templates/projects/defect_card.html +153 -0
  153. package/app/templates/projects/defect_create_preview.html +17 -0
  154. package/app/templates/projects/defect_fields_diff.html +6 -0
  155. package/app/templates/projects/defect_fields_form.html +40 -0
  156. package/app/templates/projects/defect_references.html +17 -0
  157. package/app/templates/projects/defect_sync_preview.html +28 -0
  158. package/app/templates/projects/defect_update_actions.html +6 -0
  159. package/app/templates/projects/defect_update_preview.html +24 -0
  160. package/app/templates/projects/defects.html +126 -0
  161. package/app/templates/projects/delete.html +31 -0
  162. package/app/templates/projects/external.html +61 -0
  163. package/app/templates/projects/folders.html +64 -0
  164. package/app/templates/projects/hub_accepted.html +10 -0
  165. package/app/templates/projects/hub_assignment.html +50 -0
  166. package/app/templates/projects/hub_pending.html +20 -0
  167. package/app/templates/projects/hub_pending_list.html +17 -0
  168. package/app/templates/projects/hub_state_note.html +23 -0
  169. package/app/templates/projects/hub_step_note.html +5 -0
  170. package/app/templates/projects/hub_sync.html +105 -0
  171. package/app/templates/projects/hub_sync_confirm.html +32 -0
  172. package/app/templates/projects/hub_sync_worker.html +5 -0
  173. package/app/templates/projects/hub_text_rules.html +64 -0
  174. package/app/templates/projects/identity_report.html +74 -0
  175. package/app/templates/projects/list.html +90 -0
  176. package/app/templates/projects/project.html +433 -0
  177. package/app/templates/reference-missing.html +14 -0
  178. package/app/templates/runs/run.html +166 -0
  179. package/app/templates/sca/card.html +476 -0
  180. package/app/templates/sca/index.html +321 -0
  181. package/app/templates/settings/hub_url.html +31 -0
  182. package/app/templates/settings/index.html +216 -0
  183. package/app/templates/settings/provider.html +229 -0
  184. package/app/templates/settings/providers.html +37 -0
  185. package/app/templates/worklists.html +52 -0
  186. package/app/textsafe.py +117 -0
  187. package/app/timing.py +245 -0
  188. package/app/triage.py +1743 -0
  189. package/app/view.py +430 -0
  190. package/app/worklists.py +204 -0
  191. package/examples/demo/demo-hub-report-v2.xlsx +0 -0
  192. package/examples/demo/demo-hub-report.xlsx +0 -0
  193. package/examples/demo/repos/billing-core/pom.xml +30 -0
  194. package/examples/demo/repos/billing-core/src/main/java/org/example/billing/Invoice.java +9 -0
  195. package/examples/demo/repos/billing-core/src/main/java/org/example/billing/InvoiceRepository.java +51 -0
  196. package/examples/demo/repos/billing-core/src/main/java/org/example/billing/InvoiceStatus.java +11 -0
  197. package/examples/demo/repos/billing-core/src/main/java/org/example/billing/config/DatabaseSettings.java +18 -0
  198. package/examples/demo/repos/billing-core/src/main/java/org/example/billing/payment/PaymentTokenService.java +35 -0
  199. package/examples/demo/repos/billing-core/src/main/java/org/example/billing/web/InvoiceExportController.java +57 -0
  200. package/examples/demo/repos/catalog-api/catalog_api/__init__.py +1 -0
  201. package/examples/demo/repos/catalog-api/catalog_api/products.py +24 -0
  202. package/examples/demo/repos/catalog-api/catalog_api/tasks.py +24 -0
  203. package/examples/demo/repos/catalog-api/catalog_api/views.py +17 -0
  204. package/examples/demo/repos/catalog-api/requirements.txt +3 -0
  205. package/npm/build.json +5 -0
  206. package/npm/cli.js +346 -0
  207. package/npm/files.manifest +224 -0
  208. package/npm/lib/binding.js +64 -0
  209. package/npm/lib/cluster.js +330 -0
  210. package/npm/lib/env.js +88 -0
  211. package/npm/lib/fsx.js +125 -0
  212. package/npm/lib/http.js +53 -0
  213. package/npm/lib/metrics.js +73 -0
  214. package/npm/lib/paths.js +150 -0
  215. package/npm/lib/pg.js +351 -0
  216. package/npm/lib/proc.js +101 -0
  217. package/npm/lib/python.js +133 -0
  218. package/npm/lib/stand.js +46 -0
  219. package/npm/lib/supervisor.js +1141 -0
  220. package/npm/lib/version.js +21 -0
  221. package/npm/requirements.txt +476 -0
  222. package/npm-shrinkwrap.json +111 -0
  223. package/package.json +37 -4
@@ -0,0 +1,342 @@
1
+ """ADR-0122, 7а: выбранный состав SAST, одно решение и одна строка создания."""
2
+
3
+ from fastapi import APIRouter, Request
4
+ from starlette.concurrency import run_in_threadpool
5
+
6
+ from app import defect, defect_where, hub_defects, hub_ops, hub_status, hub_sync
7
+ from app.db import dumps, loads, source_signature, source_version
8
+ from app.hub_entity_versions import digest
9
+ from app.hub_import import _cwe2
10
+ from app.models import Finding
11
+ from app.routes import hub_defect_routes as flow
12
+ from app.routes.common import db_of, project_or_404, redirect, render, settings_of
13
+ from app.routes.hub_bulk import MAX_SELECTED
14
+ from app.routes.hub_status_routes import hub_target_check, send_blocker
15
+
16
+ router = APIRouter()
17
+ CONTRACT = "selected-defect/1"
18
+ STALE = "Состав, данные или решение изменились после показа: проверьте состав и подтвердите подготовку заново."
19
+
20
+
21
+ def selection(values):
22
+ """Сохраняет порядок и уникальность; непригодные ID показываются среди исключений."""
23
+ result = list(dict.fromkeys(str(value).strip() for value in values))
24
+ if len(result) > MAX_SELECTED:
25
+ raise hub_defects.DefectRefused(f"Выбрано {len(result)} находок; допускается не более {MAX_SELECTED}.")
26
+ return result
27
+
28
+
29
+ def attempt(row):
30
+ saved = loads(row.get("source_fingerprint"), {})
31
+ return saved if isinstance(saved, dict) and saved.get("contract") == CONTRACT else None
32
+
33
+
34
+ def bases(c, project_id, hub_url, app_id):
35
+ """Принятые версии назначения: изменение состава Hub делает подтверждение устаревшим."""
36
+ return [dict(r) for r in c.execute(
37
+ "SELECT entity_kind, entity_key, accepted_epoch, accepted_hash FROM hub_entity_versions"
38
+ " WHERE project_id=? AND hub_url=? AND app_id=? ORDER BY entity_kind, entity_key",
39
+ (project_id, hub_ops.hub_base(hub_url), app_id))]
40
+
41
+
42
+ def target_snapshot(request, c, project_id, hub_id):
43
+ from app.routes import findings as card
44
+ db = db_of(request)
45
+ source = db.case_input(project_id, hub_id)
46
+ if source is None or source.get("missing"):
47
+ raise hub_defects.DefectRefused("Находка отсутствует: подтвердите состав заново.")
48
+ case = db.get_case(project_id, hub_id)
49
+ raw = c.execute("SELECT * FROM reviews WHERE project_id=? AND hub_id=?", (project_id, hub_id)).fetchone()
50
+ review = dict(raw) if raw else {}
51
+ f = Finding.from_dict(source["data"])
52
+ # срез разбора модели не совпадает с текущим – предупреждение в составе, а не исключение (ADR-0147); исход
53
+ # сохраняется «Не согласен» со своим вердиктом (sast_choice получает stale_verdict)
54
+ code = card.check_code(request, project_id, f, case, "", None)
55
+ if review.get("finding_sig") and review["finding_sig"] != source_signature(source):
56
+ raise hub_defects.DefectRefused("источник изменился после решения: проверьте находку в карточке")
57
+ return {"hub_id": hub_id, "source": source, "review": review,
58
+ "case": case, "code": code,
59
+ "number": db.defect_number(project_id, defect.defect_key_sast(hub_id))}
60
+
61
+
62
+ def group_check(db, c, project_id, hub_id, body, targets, hub_url, app_id, own_id=None):
63
+ """Каждая исходная группа; сосед не добавляется в состав даже при одинаковом тексте."""
64
+ flow.group_defect_check(db, project_id, hub_id, body, targets, hub_url=hub_url,
65
+ app_id=app_id, in_preview=own_id is not None)(c)
66
+ for neighbor in db._group_neighbor_ids(c, project_id, hub_id):
67
+ if neighbor in targets:
68
+ continue
69
+ row = db.hub_defect_for(project_id, [neighbor])
70
+ if (row and row["id"] != own_id and row["state"] in ("blank", "preview")
71
+ and flow._same_hub(row, hub_url, app_id) and defect_where.body_of(row["command_text"]) == defect_where.body_of(body)):
72
+ raise hub_defects.DefectRefused("в группе уже готовится дефект с тем же текстом: завершите его подготовку")
73
+
74
+
75
+ def busy_reason(db, c, project_id, hub_id, issue_id, own_id=None):
76
+ """Причина занятости называет первую подходящую строку, а её текст входит в подпись предпросмотра: порядок – по номеру,
77
+ иначе перестановка строк в таблице сделала бы показанную форму устаревшей без единой правки."""
78
+ for row in c.execute(f"SELECT * FROM hub_defects WHERE project_id=? AND {db.HUB_DEFECT_RESERVING} ORDER BY id", (project_id,)):
79
+ if row["id"] == own_id:
80
+ continue
81
+ if any(str(t.get("hub_id")) == hub_id or (t.get("issue_id") == issue_id and t.get("issue_type") == "SAST")
82
+ for t in loads(row["targets_json"], [])):
83
+ return f"занята другим дефектом или подготовкой №{row['id']} ({hub_defects.STATE_LABELS.get(row['state'], row['state'])})"
84
+ for op in c.execute("SELECT id, targets_json FROM hub_ops WHERE project_id=? AND purpose='status'"
85
+ " AND state IN ('queued','taken','unknown') ORDER BY id", (project_id,)):
86
+ if any(t.get("issue_id") == issue_id and t.get("issue_type") == "SAST" for t in loads(op["targets_json"], [])):
87
+ return f"незавершённая операция статуса №{op['id']}: дождитесь итога или сверьте результат"
88
+ return ""
89
+
90
+
91
+ def assess(request, c, project, selected, anchor, comment, fix):
92
+ """Локальный снимок для показа и повторной проверки на том же соединении."""
93
+ db, settings, pid = db_of(request), settings_of(request), project["id"]
94
+ selected = selection(selected)
95
+ anchor = anchor or (selected[0] if selected else "")
96
+ anchor_row = db.get_finding_by_hub_id(pid, anchor) if anchor in selected else None
97
+ f0 = Finding.from_dict(anchor_row["data"]) if anchor_row else None
98
+ ready, excluded, snapshots = [], [], []
99
+ app_id = int(project.get("hub_app_id") or 0)
100
+ body = defect.sast_body(f0, "true_positive", comment, fix, settings.team_name) if defect.has_recommendation(fix) else ""
101
+ aliases = {}
102
+ for raw in c.execute("SELECT hub_id FROM findings WHERE project_id=? AND kind='SAST'", (pid,)):
103
+ try:
104
+ ident = hub_status.issue_id(raw["hub_id"])
105
+ except hub_status.StatusRefused:
106
+ continue
107
+ aliases.setdefault(ident, []).append(raw["hub_id"])
108
+ for hub_id in selected:
109
+ row = db.get_finding_by_hub_id(pid, hub_id)
110
+ why, snap = "", None
111
+ f = Finding.from_dict(row["data"]) if row else None
112
+ url = flow._card_url_of(db, pid, hub_id) if row else ""
113
+ try:
114
+ if f is None or row.get("missing"):
115
+ raise hub_defects.DefectRefused("находка отсутствует по принятому состоянию Hub или выгрузке")
116
+ if f.kind != "SAST":
117
+ raise hub_defects.DefectRefused("не SAST: для SCA откройте карточку соответствующего набора")
118
+ if not f0 or f0.kind != "SAST":
119
+ raise hub_defects.DefectRefused("выберите опорную находку SAST из исходного выбора")
120
+ if f.codebase != f0.codebase:
121
+ raise hub_defects.DefectRefused("другая кодовая база")
122
+ if f.rule != f0.rule:
123
+ raise hub_defects.DefectRefused("другая категория")
124
+ if _cwe2(f.cwe) != _cwe2(f0.cwe):
125
+ raise hub_defects.DefectRefused("другой набор CWE")
126
+ ident = hub_status.issue_id(hub_id)
127
+ if len(aliases.get(ident, [])) != 1:
128
+ raise hub_defects.DefectRefused("неоднозначное сопоставление номера Hub после нормализации")
129
+ why = busy_reason(db, c, pid, hub_id, ident)
130
+ if why:
131
+ raise hub_defects.DefectRefused(why)
132
+ snap = target_snapshot(request, c, pid, hub_id)
133
+ if body:
134
+ # Невыбранные соседи, в том числе у второй и последней цели.
135
+ group_check(db, c, pid, hub_id, body, [], settings.hub_url, app_id)
136
+ except ValueError as e:
137
+ why = str(e)
138
+ snapshots.append({"hub_id": hub_id, "row": row, "snapshot": snap, "why": why})
139
+ if why:
140
+ excluded.append({"hub_id": hub_id, "why": why, "url": url})
141
+ else:
142
+ ready.append({"hub_id": hub_id, "url": url, "finding": f, "snapshot": snap,
143
+ "issue_id": ident, "issue_type": "SAST"})
144
+ numbers = {r["snapshot"]["number"] for r in ready if r["snapshot"]["number"] is not None}
145
+ problems = []
146
+ if len(numbers) > 1:
147
+ problems.append("У выбранных находок разные номера ID: автоматическое объединение номеров не выполняется.")
148
+ number = next(iter(numbers)) if len(numbers) == 1 else db.next_defect_number(pid)
149
+ keys = [defect.defect_key_sast(r["hub_id"]) for r in ready]
150
+ if db.hub_defects_with_number(pid, number) or set(db.defect_keys_with_number(pid, number)) - set(keys):
151
+ problems.append(f"Номер ID{number} удерживают другие находки или подготовка: нужен новый предпросмотр.")
152
+ if not ready:
153
+ problems.append("Нет находок, пригодных для подготовки дефекта.")
154
+ elif len(",".join(str(r["issue_id"]) for r in ready)) > hub_defects.ISSUE_IDS_MAX:
155
+ size = len(",".join(str(r["issue_id"]) for r in ready))
156
+ problems.append(f"Состав из {len(ready)} находок: issueIds занимает {size} знаков при пределе "
157
+ f"{hub_defects.ISSUE_IDS_MAX}. Выберите меньше находок; решения не сохранены.")
158
+ from app.routes import findings as card
159
+ anchor_ready = next((r for r in ready if r["hub_id"] == anchor), None)
160
+ model = card.model_suggestion(f0, anchor_ready["snapshot"]["case"]) if anchor_ready else {}
161
+ warnings = ["Выбраны разные файлы одной кодовой базы: проверьте применимость общего решения."] \
162
+ if len({r["finding"].file for r in ready}) > 1 else []
163
+ replacements = []
164
+ for r in ready:
165
+ old = r["snapshot"]["review"]
166
+ if (old.get("decision") or old.get("final_comment") or old.get("final_fix")) and hub_defects.decision_changes(
167
+ old, "disagree", "true_positive", comment, fix):
168
+ replacements.append({"hub_id": r["hub_id"], "revision": old["revision"],
169
+ "outcome": old.get("final_verdict") or old.get("decision"),
170
+ "comment": old.get("final_comment"), "fix": old.get("final_fix")})
171
+ state = {"contract": CONTRACT, "project_id": pid, "hub_url": hub_ops.hub_base(settings.hub_url),
172
+ "app_id": app_id, "selected": selected, "anchor": anchor, "snapshots": snapshots,
173
+ "comment": comment, "fix": fix, "body": body, "number": number, "keys": keys,
174
+ "bases": bases(c, pid, settings.hub_url, app_id), "problems": problems,
175
+ "replacements": replacements, "service": bool(hub_defects.service_info(db, settings.hub_url))}
176
+ return {"ready": ready, "excluded": excluded, "warnings": warnings, "problems": problems,
177
+ "replacements": replacements, "number": number, "body": body, "anchor_finding": f0,
178
+ "anchor": anchor, "snapshot": digest(state), "state": state, "model": model,
179
+ "anchor_ready": anchor_ready is not None}
180
+
181
+
182
+ def text_check(request, project_id, view, comment, fix):
183
+ from app.routes import findings as card
184
+ if not defect.reason_text(comment):
185
+ raise hub_defects.DefectRefused(card.SAST_NEED_REASON["true_positive"])
186
+ if not defect.has_recommendation(fix):
187
+ raise hub_defects.DefectRefused(card.SAST_NEED_FIX)
188
+ if len(comment) > card.MAX_COMMENT or len(fix) > card.MAX_FIX:
189
+ raise hub_defects.DefectRefused("Обоснование или рекомендация слишком длинные: сократите текст.")
190
+ db = db_of(request)
191
+ for r in view["ready"]:
192
+ snap = r["snapshot"]
193
+ origin = card.comment_origin(db, snap["case"], snap["review"], comment,
194
+ project_id=project_id, hub_id=r["hub_id"])
195
+ if origin and origin != "true_positive":
196
+ raise hub_defects.DefectRefused(f"{r['hub_id']}: обоснование написано моделью к другому исходу; поправьте текст.")
197
+ problems = flow.sast_text_problems(db, settings_of(request), project_id, view["body"])
198
+ if problems:
199
+ from app import defect_check
200
+ raise hub_defects.DefectRefused(defect_check.refusal(problems))
201
+
202
+
203
+ def prepare(request, project_id, selected, anchor, comment, fix, shown, confirmed, overwrite):
204
+ from app.routes import findings as card
205
+ db = db_of(request)
206
+ prepared = {}
207
+
208
+ def check(c):
209
+ project = project_or_404(request, project_id)
210
+ settings = settings_of(request)
211
+ view = assess(request, c, project, selected, anchor, comment, fix)
212
+ if not shown or shown != view["snapshot"]:
213
+ raise hub_defects.DefectRefused(STALE)
214
+ if not confirmed:
215
+ raise hub_defects.DefectRefused("Подтвердите показанное решение и запросы подготовки.")
216
+ if view["replacements"] and not overwrite:
217
+ raise hub_defects.DefectRefused("Подтвердите перезапись перечисленных решений.")
218
+ blocker = send_blocker(settings, project) or hub_sync.resync_blocker(db_of(request), project_id)
219
+ if blocker or view["problems"]:
220
+ raise hub_defects.DefectRefused(blocker or " ".join(view["problems"]))
221
+ text_check(request, project_id, view, comment, fix)
222
+ hub_target_check(request, project_id, settings.hub_url, project["hub_app_id"])(c)
223
+ # Даже отказ длины запроса обязан произойти до первого сохранения.
224
+ hub_defects.blank_request(project["hub_app_id"], view["ready"], "SAST")
225
+ prepared.update(view=view, project=project, settings=settings)
226
+ reviews = []
227
+ for r in view["ready"]:
228
+ snap = r["snapshot"]
229
+ case, old, code = snap["case"] or {}, snap["review"], snap["code"]
230
+ decision, outcome = card.sast_choice("true_positive", "", case.get("verdict"), code["stale_verdict"], old,
231
+ case.get("verdict_id"))
232
+ reviews.append(dict(hub_id=r["hub_id"], decision=decision, final_verdict=outcome,
233
+ final_comment=comment, final_fix=fix, reviewed=True, verdict_id=case.get("verdict_id"),
234
+ expect={"version": source_version(snap["source"]), "revision": old.get("revision", 0),
235
+ "verdict_id": case.get("verdict_id")},
236
+ code_hash=code["hash"], code_chars=code["chars"], require_present=True))
237
+ return reviews
238
+
239
+ def build(c, saved):
240
+ view, settings, project = prepared["view"], prepared["settings"], prepared["project"]
241
+ targets = [flow._defect_target(r["finding"], review, view["body"], flow.defect_parts(settings, review),
242
+ db.finding_sig(project_id, r["hub_id"]))
243
+ for r, review in zip(view["ready"], saved, strict=True)]
244
+ snapshots = []
245
+ for r in view["ready"]:
246
+ current = target_snapshot(request, c, project_id, r["hub_id"])
247
+ confirmed_code = r["snapshot"]["code"]
248
+ if any(current["code"][key] != confirmed_code[key] for key in ("hash", "chars")):
249
+ raise hub_defects.DefectRefused(STALE)
250
+ snapshots.append({"hub_id": r["hub_id"], "fingerprint": digest(current)})
251
+ saved_attempt = {"contract": CONTRACT, "selected": selected, "anchor": view["anchor"],
252
+ "confirmation": shown, "bases": view["state"]["bases"],
253
+ "snapshots": snapshots}
254
+ summary = defect.sast_title(view["number"], view["anchor_finding"])
255
+ row = dict(kind="SAST", number=view["number"], summary=summary, anchor_hub_id=view["anchor"],
256
+ targets=targets, number_keys=view["state"]["keys"], command_text=view["body"],
257
+ hub_url=settings.hub_url, app_id=project["hub_app_id"], left_out=view["excluded"],
258
+ warnings=view["warnings"], source_fingerprint=dumps(saved_attempt))
259
+ items = [dict(purpose="defect_blank", request=hub_defects.blank_request(project["hub_app_id"], targets, "SAST"),
260
+ hub_url=settings.hub_url, app_id=project["hub_app_id"], project_id=project_id,
261
+ targets=targets, text=summary)]
262
+ if not view["state"]["service"]:
263
+ items.append(dict(purpose="read", request=hub_defects.service_info_request(), hub_url=settings.hub_url,
264
+ app_id=project["hub_app_id"], project_id=project_id, targets=[]))
265
+ return row, items
266
+
267
+ return db.hub_defect_selected_start(project_id, check=check, build=build)
268
+
269
+
270
+ def create_check(request, c, row):
271
+ """Состав не сокращается при позднем конфликте: вся постановка отклоняется."""
272
+ saved = attempt(row)
273
+ if saved is None:
274
+ return
275
+ db, pid = db_of(request), row["project_id"]
276
+ targets = loads(row["targets_json"], [])
277
+ ids = [t["hub_id"] for t in targets]
278
+ if ids != [s["hub_id"] for s in saved["snapshots"]]:
279
+ raise hub_defects.DefectRefused(STALE)
280
+ if saved["bases"] != bases(c, pid, row["hub_url"], row["app_id"]):
281
+ raise hub_defects.DefectRefused("Принятая база Hub изменилась: отмените предпросмотр и подготовьте заново.")
282
+ if (db.hub_defects_with_number(pid, row["number"], exclude_id=row["id"])
283
+ or set(db.defect_keys_with_number(pid, row["number"])) - set(loads(row["number_keys_json"], []))):
284
+ raise hub_defects.DefectRefused("Номер дефекта занят другой находкой: отмените предпросмотр и подготовьте заново.")
285
+ for target, snapshot in zip(targets, saved["snapshots"], strict=True):
286
+ if digest(target_snapshot(request, c, pid, target["hub_id"])) != snapshot["fingerprint"]:
287
+ raise hub_defects.DefectRefused(STALE)
288
+ why = busy_reason(db, c, pid, target["hub_id"], target["issue_id"], row["id"])
289
+ if why:
290
+ raise hub_defects.DefectRefused(why)
291
+ group_check(db, c, pid, target["hub_id"], row["command_text"], ids,
292
+ row["hub_url"], row["app_id"], row["id"])
293
+
294
+
295
+ def page(request, project_id, selected, anchor="", comment=None, fix=None, *, confirm=False, error="", status=200):
296
+ project, db = project_or_404(request, project_id), db_of(request)
297
+ selected = selection(selected)
298
+ view = db.on_connection(lambda c: assess(request, c, project, selected, anchor, comment or "", fix or ""))
299
+ if comment is None and fix is None:
300
+ texts = {(r["snapshot"]["review"].get("final_comment", ""), r["snapshot"]["review"].get("final_fix", ""))
301
+ for r in view["ready"]}
302
+ comment, fix = next(iter(texts)) if len(texts) == 1 else ("", "")
303
+ view = db.on_connection(lambda c: assess(request, c, project, selected, anchor, comment, fix))
304
+ if confirm:
305
+ try:
306
+ text_check(request, project_id, view, comment, fix)
307
+ except ValueError as e:
308
+ error, confirm, status = str(e), False, 400
309
+ return render(request, "findings/defect_selected.html", dict(project=project, selected=selected,
310
+ comment=comment or "", fix=fix or "", confirm=confirm, error=error,
311
+ blocker=send_blocker(settings_of(request), project) or hub_sync.resync_blocker(db_of(request), project_id), **view), status_code=status)
312
+
313
+
314
+ @router.get("/projects/{project_id}/hub/defect-selected")
315
+ async def selected_page(request: Request, project_id: int):
316
+ try:
317
+ return await run_in_threadpool(page, request, project_id, request.query_params.getlist("hub_id"))
318
+ except ValueError as e:
319
+ return await run_in_threadpool(page, request, project_id, [], error=str(e), status=400)
320
+
321
+
322
+ @router.post("/projects/{project_id}/hub/defect-selected")
323
+ async def selected_submit(request: Request, project_id: int):
324
+ form = await request.form(max_fields=MAX_SELECTED + 20, max_files=0)
325
+ selected = form.getlist("hub_id")
326
+ anchor = str(form.get("anchor") or "")
327
+ comment = str(form.get("comment") or "").replace("\r\n", "\n")
328
+ fix = str(form.get("fix") or "").replace("\r\n", "\n")
329
+ try:
330
+ selected = selection(selected)
331
+ if form.get("action") == "prepare":
332
+ row = await run_in_threadpool(prepare, request, project_id, selected, anchor, comment, fix,
333
+ str(form.get("snapshot") or ""), form.get("confirm") == "1",
334
+ form.get("overwrite") == "1")
335
+ return redirect(f"/projects/{project_id}/defects/{row['id']}")
336
+ return await run_in_threadpool(page, request, project_id, selected, anchor, comment, fix,
337
+ confirm=form.get("action") == "preview")
338
+ except ValueError as e:
339
+ if len(set(selected)) > MAX_SELECTED:
340
+ selected = []
341
+ return await run_in_threadpool(page, request, project_id, selected, anchor, comment, fix,
342
+ error=str(e), status=409)
@@ -0,0 +1,155 @@
1
+ """Раздел «Справочники»: загрузка выгрузки БДУ ФСТЭК России и каталога CWE, что загружено и когда.
2
+
3
+ Файл сохраняется во временную папку данных (DATA_DIR/tmp) и разбирается в фоновом потоке (app/feeds.py): выгрузка
4
+ БДУ – это около 0,6 ГБ XML, разбор идёт около минуты. Страница показывает ход загрузки и сама обновляется, когда
5
+ она закончится (data-poll); идущую загрузку можно остановить кнопкой «Остановить загрузку». Шаблон – templates/feeds/index.html.
6
+ """
7
+
8
+ from __future__ import annotations
9
+
10
+ import logging
11
+ import os
12
+ import tempfile
13
+ import threading
14
+ from pathlib import Path
15
+
16
+ from fastapi import APIRouter, HTTPException, Request
17
+ from fastapi.responses import HTMLResponse
18
+ from starlette.concurrency import run_in_threadpool
19
+ from starlette.datastructures import UploadFile
20
+
21
+ from app.feeds import BUSY, KINDS, FeedError, FeedStore
22
+ from app.logsafe import log_failure
23
+ from app.routes.common import MAX_NAME, redirect, register_messages, render, settings_of
24
+
25
+ log = logging.getLogger("local_triage.web")
26
+
27
+ router = APIRouter()
28
+
29
+ register_messages({
30
+ "feed_started": "Файл принят: справочник загружается – страница обновится, когда загрузка закончится.",
31
+ "feed_cancel": "Загрузка останавливается: прежний справочник не изменится.",
32
+ })
33
+
34
+
35
+ class FeedJob:
36
+ """Загрузка справочника в фоновом потоке: одна за раз. state – idle, running, done, error."""
37
+
38
+ def __init__(self) -> None:
39
+ self._lock = threading.Lock()
40
+ self.state = "idle"
41
+ self.kind = ""
42
+ self.filename = ""
43
+ self.message = ""
44
+
45
+ def snapshot(self) -> dict:
46
+ with self._lock:
47
+ return {"state": self.state, "kind": self.kind, "label": KINDS.get(self.kind, ""),
48
+ "filename": self.filename, "message": self.message}
49
+
50
+ def start(self, store: FeedStore, kind: str, path: Path, filename: str) -> None:
51
+ with self._lock:
52
+ if self.state == "running":
53
+ path.unlink(missing_ok=True)
54
+ raise FeedError(BUSY)
55
+ self.state, self.kind, self.filename, self.message = "running", kind, filename, ""
56
+ thread = threading.Thread(target=self._run, args=(store, kind, path, filename), name="feed-load", daemon=True)
57
+ thread.start()
58
+
59
+ def _run(self, store: FeedStore, kind: str, path: Path, filename: str) -> None:
60
+ try:
61
+ meta = store.load(kind, path, filename)
62
+ message = (f"{KINDS[kind]}: загружено записей – {meta['records']}"
63
+ + (f", дата выгрузки {meta['source_date']}" if meta.get("source_date") else "")
64
+ + f", {meta['seconds']} с.")
65
+ state = "done"
66
+ except FeedError as e:
67
+ state, message = "error", f"{KINDS[kind]}: файл не принят. {e}"
68
+ except Exception as e: # текст исключения в журнал не пишется (ADR 0016)
69
+ eid = log_failure(log, "справочник не загружен", e)
70
+ state, message = "error", f"{KINDS[kind]}: внутренняя ошибка при загрузке (номер в журнале {eid})."
71
+ finally:
72
+ path.unlink(missing_ok=True)
73
+ with self._lock:
74
+ self.state, self.message = state, message
75
+
76
+
77
+ def store_of(request: Request) -> FeedStore:
78
+ return request.app.state.feeds
79
+
80
+
81
+ def job_of(request: Request) -> FeedJob:
82
+ return request.app.state.feed_job
83
+
84
+
85
+ def page_context(request: Request) -> dict:
86
+ store = store_of(request)
87
+ status = store.status()
88
+ job = job_of(request).snapshot()
89
+ context = {"kinds": KINDS, "status": status, "job": job,
90
+ "max_upload_mb": settings_of(request).max_upload_mb}
91
+ context['capec_profile'] = store.capec_data(shared=True)['profile'] if 'capec' in status else {}
92
+ # После обновления страницы ?ok= остаётся в адресе, но завершённая загрузка уже не идёт.
93
+ if job["state"] != "running" and request.query_params.get("ok") in {"feed_started", "feed_cancel"}:
94
+ context["ok_message"] = ""
95
+ return context
96
+
97
+
98
+ @router.get("/feeds", response_class=HTMLResponse)
99
+ def feeds_page(request: Request):
100
+ return render(request, "feeds/index.html", page_context(request))
101
+
102
+
103
+ @router.get("/feeds/job")
104
+ def feeds_job(request: Request) -> dict:
105
+ """Ход загрузки для страницы (data-poll): done и reload – загрузка закончилась, страницу пора обновить."""
106
+ job = job_of(request).snapshot()
107
+ finished = job["state"] != "running"
108
+ return {**job, "done": finished, "reload": finished}
109
+
110
+
111
+ @router.post("/feeds/cancel")
112
+ def cancel_feed(request: Request):
113
+ """«Остановить загрузку»: разбор прервётся на ближайшем куске, временные таблицы удалятся, прежний справочник
114
+ останется. Загрузки нет – просто возврат на страницу."""
115
+ stopping = job_of(request).snapshot()["state"] == "running" and store_of(request).cancel()
116
+ return redirect("/feeds?ok=feed_cancel" if stopping else "/feeds")
117
+
118
+
119
+ @router.post("/feeds/{kind}")
120
+ async def upload_feed(request: Request, kind: str):
121
+ if kind not in KINDS:
122
+ raise HTTPException(status_code=404, detail="Такого справочника нет.")
123
+ settings = await run_in_threadpool(settings_of, request)
124
+ form = await request.form(max_files=1, max_fields=5)
125
+ try:
126
+ upload = form.get("file")
127
+ try:
128
+ if not isinstance(upload, UploadFile) or not upload.filename:
129
+ raise FeedError("Выберите файл архива (.zip).")
130
+ filename = Path(upload.filename.replace("\\", "/")).name[:MAX_NAME]
131
+ if not filename.lower().endswith(".zip"):
132
+ raise FeedError("Нужен архив .zip – в том виде, как он скачан с сайта.")
133
+ settings.tmp_dir.mkdir(parents=True, exist_ok=True)
134
+ fd, tmp_name = tempfile.mkstemp(dir=settings.tmp_dir, prefix="feed-", suffix=".zip")
135
+ tmp = Path(tmp_name)
136
+ try:
137
+ size = 0
138
+ with os.fdopen(fd, "wb") as out:
139
+ while chunk := await upload.read(1024 * 1024):
140
+ size += len(chunk)
141
+ if size > settings.max_upload_bytes:
142
+ raise FeedError(f"Архив больше {settings.max_upload_mb} МБ – это предел приложения "
143
+ "(MAX_UPLOAD_MB).")
144
+ out.write(chunk)
145
+ job_of(request).start(store_of(request), kind, tmp, filename)
146
+ except BaseException:
147
+ tmp.unlink(missing_ok=True)
148
+ raise
149
+ except FeedError as e:
150
+ message = str(e) # имя e после блока except не видно: текст – в переменную до лямбды
151
+ return await run_in_threadpool(lambda: render(request, "feeds/index.html",
152
+ {**page_context(request), "error": message}, 400))
153
+ finally:
154
+ await form.close()
155
+ return redirect("/feeds?ok=feed_started")