local-triage 0.0.0-stage → 0.10.0-dev.32
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +90 -2
- package/THIRD-PARTY-NOTICES.md +48 -0
- package/app/__init__.py +3 -0
- package/app/affiliation.py +137 -0
- package/app/ask.py +300 -0
- package/app/build_info.py +98 -0
- package/app/capec.py +168 -0
- package/app/capec_view.py +27 -0
- package/app/codecmp.py +59 -0
- package/app/config.py +411 -0
- package/app/context.py +1451 -0
- package/app/cvss.py +357 -0
- package/app/db.py +6234 -0
- package/app/defect.py +670 -0
- package/app/defect_check.py +323 -0
- package/app/defect_drafts.py +612 -0
- package/app/defect_dto.py +149 -0
- package/app/defect_fields.py +248 -0
- package/app/defect_form.py +124 -0
- package/app/defect_links.py +107 -0
- package/app/defect_local_text.py +103 -0
- package/app/defect_references.py +164 -0
- package/app/defect_where.py +147 -0
- package/app/ecosystem.py +51 -0
- package/app/executors.py +660 -0
- package/app/feeds.py +1231 -0
- package/app/guard.py +164 -0
- package/app/hub_block_builder.py +200 -0
- package/app/hub_block_version.py +220 -0
- package/app/hub_defects.py +4469 -0
- package/app/hub_effects.py +421 -0
- package/app/hub_entity_versions.py +108 -0
- package/app/hub_import.py +2054 -0
- package/app/hub_onboarding.py +114 -0
- package/app/hub_ops.py +330 -0
- package/app/hub_pull.py +1195 -0
- package/app/hub_state.py +255 -0
- package/app/hub_status.py +237 -0
- package/app/hub_sync.py +619 -0
- package/app/hub_sync_api.py +241 -0
- package/app/hub_sync_compare.py +1054 -0
- package/app/hub_sync_contract.py +75 -0
- package/app/hub_template_model.py +74 -0
- package/app/hub_text_scale.py +159 -0
- package/app/journal.py +250 -0
- package/app/llm.py +850 -0
- package/app/llm_chat.py +596 -0
- package/app/llm_http.py +319 -0
- package/app/llm_messages.py +401 -0
- package/app/llm_net.py +764 -0
- package/app/llm_ollama.py +571 -0
- package/app/lockfile.py +505 -0
- package/app/logsafe.py +54 -0
- package/app/main.py +367 -0
- package/app/mask_display.py +132 -0
- package/app/masking.py +814 -0
- package/app/metrics.py +194 -0
- package/app/model_traffic.py +519 -0
- package/app/models.py +483 -0
- package/app/navigation.py +121 -0
- package/app/osv.py +279 -0
- package/app/package_identity.py +189 -0
- package/app/package_transition.py +824 -0
- package/app/profile_gate.py +71 -0
- package/app/prompts.py +1108 -0
- package/app/provider_keys.py +189 -0
- package/app/providers.py +511 -0
- package/app/routes/__init__.py +10 -0
- package/app/routes/api.py +153 -0
- package/app/routes/capec.py +186 -0
- package/app/routes/common.py +407 -0
- package/app/routes/components.py +1499 -0
- package/app/routes/cwe.py +168 -0
- package/app/routes/decision_selected.py +153 -0
- package/app/routes/defect_add.py +223 -0
- package/app/routes/defect_card_routes.py +363 -0
- package/app/routes/defect_selected.py +342 -0
- package/app/routes/feeds.py +155 -0
- package/app/routes/findings.py +2736 -0
- package/app/routes/hub.py +402 -0
- package/app/routes/hub_bulk.py +288 -0
- package/app/routes/hub_defect_routes.py +1945 -0
- package/app/routes/hub_status_routes.py +1132 -0
- package/app/routes/hub_sync_routes.py +463 -0
- package/app/routes/journal.py +132 -0
- package/app/routes/projects.py +1601 -0
- package/app/routes/providers.py +336 -0
- package/app/routes/references.py +26 -0
- package/app/routes/runs.py +219 -0
- package/app/routes/sca.py +2722 -0
- package/app/routes/settings.py +163 -0
- package/app/run_errors.py +151 -0
- package/app/sca.py +1662 -0
- package/app/sca_contract.py +1289 -0
- package/app/sca_research.py +1089 -0
- package/app/sca_research_net.py +1007 -0
- package/app/sca_text.py +965 -0
- package/app/serve.py +70 -0
- package/app/settings_store.py +262 -0
- package/app/static/app.css +2802 -0
- package/app/static/app.js +3772 -0
- package/app/static/capec-license.txt +9 -0
- package/app/static/favicon.svg +1 -0
- package/app/static/fonts/Golos-Text_DemiBold.woff2 +0 -0
- package/app/static/fonts/Golos-Text_Regular.woff2 +0 -0
- package/app/static/fonts/OFL.txt +90 -0
- package/app/static/navigation.js +234 -0
- package/app/static/observability.js +121 -0
- package/app/static/theme.js +49 -0
- package/app/storage/__init__.py +16 -0
- package/app/storage/local_state.py +246 -0
- package/app/storage/migrate.py +110 -0
- package/app/storage/migrations/postgresql/0001_baseline.sql +896 -0
- package/app/storage/migrations/postgresql/0002_feeds_and_secrets.sql +55 -0
- package/app/storage/migrations/postgresql/0003_project_mask_display.sql +5 -0
- package/app/storage/migrations/postgresql/0004_package_identity.sql +36 -0
- package/app/storage/postgresql.py +573 -0
- package/app/storage/schema.py +619 -0
- package/app/storage/sql.py +57 -0
- package/app/storage/tools.py +254 -0
- package/app/telemetry.py +307 -0
- package/app/templates/base.html +91 -0
- package/app/templates/capec/context.html +3 -0
- package/app/templates/capec/detail.html +27 -0
- package/app/templates/capec/index.html +128 -0
- package/app/templates/components/_archive.html +11 -0
- package/app/templates/components/archived.html +19 -0
- package/app/templates/components/batch.html +35 -0
- package/app/templates/components/detail.html +332 -0
- package/app/templates/components/index.html +338 -0
- package/app/templates/components/research_preview.html +80 -0
- package/app/templates/components/research_run.html +54 -0
- package/app/templates/cwe/bdu.html +56 -0
- package/app/templates/cwe/detail.html +233 -0
- package/app/templates/cwe/index.html +97 -0
- package/app/templates/error.html +8 -0
- package/app/templates/feeds/index.html +65 -0
- package/app/templates/findings/card.html +639 -0
- package/app/templates/findings/decision_selected.html +41 -0
- package/app/templates/findings/defect_add.html +46 -0
- package/app/templates/findings/defect_selected.html +47 -0
- package/app/templates/findings/list.html +153 -0
- package/app/templates/findings/send_selected.html +53 -0
- package/app/templates/findings/trace.html +72 -0
- package/app/templates/journal/detail.html +19 -0
- package/app/templates/journal/index.html +19 -0
- package/app/templates/macros.html +440 -0
- package/app/templates/navigation-fields.html +1 -0
- package/app/templates/navigation.html +11 -0
- package/app/templates/projects/_identity_summary.html +42 -0
- package/app/templates/projects/defect_card.html +153 -0
- package/app/templates/projects/defect_create_preview.html +17 -0
- package/app/templates/projects/defect_fields_diff.html +6 -0
- package/app/templates/projects/defect_fields_form.html +40 -0
- package/app/templates/projects/defect_references.html +17 -0
- package/app/templates/projects/defect_sync_preview.html +28 -0
- package/app/templates/projects/defect_update_actions.html +6 -0
- package/app/templates/projects/defect_update_preview.html +24 -0
- package/app/templates/projects/defects.html +126 -0
- package/app/templates/projects/delete.html +31 -0
- package/app/templates/projects/external.html +61 -0
- package/app/templates/projects/folders.html +64 -0
- package/app/templates/projects/hub_accepted.html +10 -0
- package/app/templates/projects/hub_assignment.html +50 -0
- package/app/templates/projects/hub_pending.html +20 -0
- package/app/templates/projects/hub_pending_list.html +17 -0
- package/app/templates/projects/hub_state_note.html +23 -0
- package/app/templates/projects/hub_step_note.html +5 -0
- package/app/templates/projects/hub_sync.html +105 -0
- package/app/templates/projects/hub_sync_confirm.html +32 -0
- package/app/templates/projects/hub_sync_worker.html +5 -0
- package/app/templates/projects/hub_text_rules.html +64 -0
- package/app/templates/projects/identity_report.html +74 -0
- package/app/templates/projects/list.html +90 -0
- package/app/templates/projects/project.html +433 -0
- package/app/templates/reference-missing.html +14 -0
- package/app/templates/runs/run.html +166 -0
- package/app/templates/sca/card.html +476 -0
- package/app/templates/sca/index.html +321 -0
- package/app/templates/settings/hub_url.html +31 -0
- package/app/templates/settings/index.html +216 -0
- package/app/templates/settings/provider.html +229 -0
- package/app/templates/settings/providers.html +37 -0
- package/app/templates/worklists.html +52 -0
- package/app/textsafe.py +117 -0
- package/app/timing.py +245 -0
- package/app/triage.py +1743 -0
- package/app/view.py +430 -0
- package/app/worklists.py +204 -0
- package/examples/demo/demo-hub-report-v2.xlsx +0 -0
- package/examples/demo/demo-hub-report.xlsx +0 -0
- package/examples/demo/repos/billing-core/pom.xml +30 -0
- package/examples/demo/repos/billing-core/src/main/java/org/example/billing/Invoice.java +9 -0
- package/examples/demo/repos/billing-core/src/main/java/org/example/billing/InvoiceRepository.java +51 -0
- package/examples/demo/repos/billing-core/src/main/java/org/example/billing/InvoiceStatus.java +11 -0
- package/examples/demo/repos/billing-core/src/main/java/org/example/billing/config/DatabaseSettings.java +18 -0
- package/examples/demo/repos/billing-core/src/main/java/org/example/billing/payment/PaymentTokenService.java +35 -0
- package/examples/demo/repos/billing-core/src/main/java/org/example/billing/web/InvoiceExportController.java +57 -0
- package/examples/demo/repos/catalog-api/catalog_api/__init__.py +1 -0
- package/examples/demo/repos/catalog-api/catalog_api/products.py +24 -0
- package/examples/demo/repos/catalog-api/catalog_api/tasks.py +24 -0
- package/examples/demo/repos/catalog-api/catalog_api/views.py +17 -0
- package/examples/demo/repos/catalog-api/requirements.txt +3 -0
- package/npm/build.json +5 -0
- package/npm/cli.js +346 -0
- package/npm/files.manifest +224 -0
- package/npm/lib/binding.js +64 -0
- package/npm/lib/cluster.js +330 -0
- package/npm/lib/env.js +88 -0
- package/npm/lib/fsx.js +125 -0
- package/npm/lib/http.js +53 -0
- package/npm/lib/metrics.js +73 -0
- package/npm/lib/paths.js +150 -0
- package/npm/lib/pg.js +351 -0
- package/npm/lib/proc.js +101 -0
- package/npm/lib/python.js +133 -0
- package/npm/lib/stand.js +46 -0
- package/npm/lib/supervisor.js +1141 -0
- package/npm/lib/version.js +21 -0
- package/npm/requirements.txt +476 -0
- package/npm-shrinkwrap.json +111 -0
- package/package.json +37 -4
|
@@ -0,0 +1,98 @@
|
|
|
1
|
+
"""Сведения о сборке для плашки внизу страниц и /healthz: версия приложения, коммит и время сборки образа.
|
|
2
|
+
|
|
3
|
+
Коммит и время передаются при сборке образа (start.ps1, deploy/vm/build.sh → аргументы BUILD_COMMIT, BUILD_DIRTY,
|
|
4
|
+
BUILD_TIME → переменные LT_BUILD_*). Без них – при запуске из рабочей копии репозитория – коммит читается из git.
|
|
5
|
+
Значения проверяются по формату: в разметку попадает только то, что похоже на коммит и дату."""
|
|
6
|
+
|
|
7
|
+
from __future__ import annotations
|
|
8
|
+
|
|
9
|
+
import hashlib
|
|
10
|
+
import json
|
|
11
|
+
import os
|
|
12
|
+
import re
|
|
13
|
+
import subprocess
|
|
14
|
+
from functools import lru_cache
|
|
15
|
+
from pathlib import Path
|
|
16
|
+
|
|
17
|
+
from app import __version__
|
|
18
|
+
|
|
19
|
+
_STATIC_DIR = Path(__file__).resolve().parent / "static"
|
|
20
|
+
|
|
21
|
+
_COMMIT = re.compile(r"^[0-9a-f]{7,40}$")
|
|
22
|
+
_BUILT = re.compile(r"^\d{2}\.\d{2}\.\d{4} \d{2}:\d{2}$")
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def _git(root: Path, *args: str) -> subprocess.CompletedProcess | None:
|
|
26
|
+
try:
|
|
27
|
+
return subprocess.run(["git", "-C", str(root), *args], capture_output=True, text=True, timeout=3)
|
|
28
|
+
except (OSError, subprocess.SubprocessError):
|
|
29
|
+
return None
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
# что определяет содержимое образа (Dockerfile, .dockerignore, COPY): правки в этом, включая новые файлы, –
|
|
33
|
+
# «правки вне коммита»
|
|
34
|
+
_IMAGE_PATHS = ("app", "Dockerfile", ".dockerignore", "requirements.lock", "examples/demo/demo-hub-report.xlsx")
|
|
35
|
+
|
|
36
|
+
|
|
37
|
+
def _from_git(root: Path | None = None) -> tuple[str, bool | None]:
|
|
38
|
+
"""Коммит рабочей копии и признак правок вне коммита в том, что входит в образ: True / False / None (проверить не
|
|
39
|
+
удалось – это не «чисто»). Нет git или репозитория – ("", None)."""
|
|
40
|
+
root = root or Path(__file__).resolve().parents[1]
|
|
41
|
+
if not (root / ".git").exists():
|
|
42
|
+
return "", None
|
|
43
|
+
head = _git(root, "rev-parse", "--short", "HEAD")
|
|
44
|
+
if head is None or head.returncode != 0:
|
|
45
|
+
return "", None
|
|
46
|
+
# новые файлы показываются явно: настройка status.showUntrackedFiles=no их бы скрыла
|
|
47
|
+
status = _git(root, "status", "--porcelain", "--untracked-files=all", "--", *_IMAGE_PATHS)
|
|
48
|
+
dirty = None if status is None or status.returncode != 0 else bool(status.stdout.strip())
|
|
49
|
+
return head.stdout.strip(), dirty
|
|
50
|
+
|
|
51
|
+
|
|
52
|
+
def _from_file(root: Path | None = None) -> tuple[str, bool | None, str]:
|
|
53
|
+
"""Сведения о сборке пакета npm (ADR-0155): build.json рядом с папкой app – его пишет сборщик пакета, потому что в
|
|
54
|
+
пакете нет .git. {"commit": "...", "dirty": bool, "time": "ДД.ММ.ГГГГ ЧЧ:ММ"}; нет файла или он испорчен – пусто."""
|
|
55
|
+
root = root or Path(__file__).resolve().parents[1]
|
|
56
|
+
try:
|
|
57
|
+
data = json.loads((root / "build.json").read_text(encoding="utf-8"))
|
|
58
|
+
except (OSError, ValueError):
|
|
59
|
+
return "", None, ""
|
|
60
|
+
if not isinstance(data, dict):
|
|
61
|
+
return "", None, ""
|
|
62
|
+
commit = str(data.get("commit") or "").strip().lower()
|
|
63
|
+
dirty = data.get("dirty") if isinstance(data.get("dirty"), bool) else None
|
|
64
|
+
return commit, dirty, str(data.get("time") or "").strip()
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
@lru_cache(maxsize=1)
|
|
68
|
+
def build_info() -> dict:
|
|
69
|
+
"""{version, commit, dirty, built}: commit – "" если неизвестен; dirty – True / False / None (не проверено, в том
|
|
70
|
+
числе при неизвестном коммите); built – время сборки образа «ДД.ММ.ГГГГ ЧЧ:ММ» или ""."""
|
|
71
|
+
commit = os.environ.get("LT_BUILD_COMMIT", "").strip().lower()
|
|
72
|
+
flag = os.environ.get("LT_BUILD_DIRTY", "").strip()
|
|
73
|
+
dirty: bool | None = True if flag == "1" else (False if flag == "0" else None)
|
|
74
|
+
built = os.environ.get("LT_BUILD_TIME", "").strip()
|
|
75
|
+
if not _COMMIT.match(commit):
|
|
76
|
+
commit, dirty = _from_git()
|
|
77
|
+
commit = commit.lower() if _COMMIT.match(commit.lower()) else ""
|
|
78
|
+
if not commit: # пакет npm: git нет, сведения записаны при сборке пакета
|
|
79
|
+
commit, dirty, file_built = _from_file()
|
|
80
|
+
commit = commit if _COMMIT.match(commit) else ""
|
|
81
|
+
built = built or file_built
|
|
82
|
+
return {"version": __version__, "commit": commit, "dirty": dirty if commit else None,
|
|
83
|
+
"built": built if _BUILT.match(built) else ""}
|
|
84
|
+
|
|
85
|
+
|
|
86
|
+
@lru_cache(maxsize=1)
|
|
87
|
+
def asset_version(static_dir: Path | None = None) -> str:
|
|
88
|
+
"""Версия для адресов статических файлов (`?v=` в base.html): версия приложения и отпечаток содержимого `app/static`,
|
|
89
|
+
чтобы адрес менялся с каждой сборкой, где менялись скрипты или стили, и браузер не держал старый скрипт: при
|
|
90
|
+
перезапусках с одним и тем же `?v=<версия>` Chrome оставлял прежний app.js, и новое поведение карточки не
|
|
91
|
+
включалось. Отпечаток – SHA-256 по относительным именам и содержимому файлов, первые 10 знаков."""
|
|
92
|
+
root = static_dir or _STATIC_DIR
|
|
93
|
+
digest = hashlib.sha256()
|
|
94
|
+
for path in sorted(p for p in root.rglob("*") if p.is_file()) if root.is_dir() else []:
|
|
95
|
+
digest.update(path.relative_to(root).as_posix().encode("utf-8"))
|
|
96
|
+
digest.update(b"\0")
|
|
97
|
+
digest.update(path.read_bytes())
|
|
98
|
+
return f"{__version__}-{digest.hexdigest()[:10]}"
|
package/app/capec.py
ADDED
|
@@ -0,0 +1,168 @@
|
|
|
1
|
+
"""Каталог CAPEC: структура источника и прямые справочные связи, без вывода о применимости."""
|
|
2
|
+
from __future__ import annotations
|
|
3
|
+
|
|
4
|
+
import hashlib
|
|
5
|
+
import json
|
|
6
|
+
import re
|
|
7
|
+
from collections import Counter
|
|
8
|
+
|
|
9
|
+
NS = "http://capec.mitre.org/capec-3"
|
|
10
|
+
TABLES = ("capec", "capec_index", "capec_refs", "capec_links", "capec_cwe", "capec_meta") # таблицы – миграция 0002, ADR-0133
|
|
11
|
+
KNOWN = set("Description Extended_Description Likelihood_Of_Attack Typical_Severity Related_Attack_Patterns Execution_Flow Prerequisites Skills_Required Resources_Required Consequences Mitigations Example_Instances Related_Weaknesses Taxonomy_Mappings Content_History Indicators References Notes Alternate_Terms Summary Relationships Objective Members Filter Audience".split())
|
|
12
|
+
KINDS = {"Attack_Pattern": "pattern", "Category": "category", "View": "view"}
|
|
13
|
+
MAX_JSON = 1024 * 1024
|
|
14
|
+
|
|
15
|
+
|
|
16
|
+
def identifier(value, prefix="CAPEC"):
|
|
17
|
+
match = re.fullmatch(r"\s*(?:" + prefix + r"\s*-?\s*)?0*(\d{1,7})\s*", str(value or ""), re.I)
|
|
18
|
+
return f"{prefix}-{int(match[1])}" if match and int(match[1]) else ""
|
|
19
|
+
|
|
20
|
+
|
|
21
|
+
def _tag(elem):
|
|
22
|
+
return elem.tag.rsplit("}", 1)[-1]
|
|
23
|
+
|
|
24
|
+
|
|
25
|
+
def _node(elem):
|
|
26
|
+
# Полное дерево записи: текст, атрибуты, хвосты и порядок не теряются при появлении нового поля.
|
|
27
|
+
return {"tag": _tag(elem), "namespace": elem.tag.rsplit('}',1)[0].lstrip('{') if '}' in elem.tag else '',
|
|
28
|
+
"attrs": _attributes(elem), "text": elem.text or "",
|
|
29
|
+
"tail": elem.tail or "", "children": [_node(x) for x in elem]}
|
|
30
|
+
|
|
31
|
+
|
|
32
|
+
def _attributes(elem):
|
|
33
|
+
return {('{' + key if '}' in key and not key.startswith('{') else key):value for key,value in elem.attrib.items()}
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def parse_record(elem):
|
|
37
|
+
from .feeds import FeedError
|
|
38
|
+
for part in elem.iter():
|
|
39
|
+
if '}' in part.tag and not part.tag.startswith('{'):
|
|
40
|
+
part.tag = '{' + part.tag
|
|
41
|
+
ident = identifier(elem.get("ID"))
|
|
42
|
+
if not ident or not elem.get("Name") or not elem.tag.startswith("{" + NS + "}"):
|
|
43
|
+
raise FeedError("Некорректная запись CAPEC: нужны ID, Name и пространство имён CAPEC.")
|
|
44
|
+
def elements(path): return elem.findall(path, {"c": NS})
|
|
45
|
+
def text(name): return elem.findtext("c:" + name, default="", namespaces={"c": NS}).strip()
|
|
46
|
+
links = [{"rel": x.get("Nature", ""), "id": identifier(x.get("CAPEC_ID"))}
|
|
47
|
+
for x in elements("c:Related_Attack_Patterns/c:Related_Attack_Pattern")]
|
|
48
|
+
links += [{"rel": "HasMember", "id": identifier(x.get("CAPEC_ID"))}
|
|
49
|
+
for path in ("c:Relationships/c:Has_Member", "c:Members/c:Has_Member") for x in elements(path)]
|
|
50
|
+
row = {"id": ident, "kind": KINDS[_tag(elem)], "name": elem.get("Name"), "attributes": _attributes(elem), "text":elem.text or '',
|
|
51
|
+
"abstraction": elem.get("Abstraction", ""), "status": elem.get("Status", ""),
|
|
52
|
+
"view_type": elem.get("Type", ""), "filter": text("Filter"),
|
|
53
|
+
"severity": text("Typical_Severity"), "likelihood": text("Likelihood_Of_Attack"),
|
|
54
|
+
"description": text("Description") or text("Summary") or text("Objective"),
|
|
55
|
+
"has_flow": bool(elements("c:Execution_Flow")),
|
|
56
|
+
"cwes": sorted({identifier(x.get("CWE_ID"), "CWE") for x in elements("c:Related_Weaknesses/c:Related_Weakness")} - {""}),
|
|
57
|
+
"links": [x for x in links if x["id"]],
|
|
58
|
+
"exclude": [identifier(x.get("Exclude_ID")) for x in elements(".//c:Exclude_Related")],
|
|
59
|
+
"taxonomies": [x.get("Taxonomy_Name", "") for x in elements("c:Taxonomy_Mappings/c:Taxonomy_Mapping")],
|
|
60
|
+
"sections": [_node(x) for x in elem],
|
|
61
|
+
"unknown_fields": [_tag(x) for x in elem if _tag(x) not in KNOWN]}
|
|
62
|
+
row["search_text"] = " ".join(elem.itertext()).casefold() + " " + row["name"].casefold()
|
|
63
|
+
if len(json.dumps(row, ensure_ascii=False).encode()) > MAX_JSON:
|
|
64
|
+
raise FeedError("Запись CAPEC слишком велика: полный текст не был сохранён, каталог не заменён.")
|
|
65
|
+
return row
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
def parse_reference(elem):
|
|
69
|
+
"""Источник целиком, без сокращения полей и количества авторов."""
|
|
70
|
+
from .feeds import FeedError
|
|
71
|
+
def text(name):
|
|
72
|
+
return ' '.join(''.join(x.itertext()).strip() for x in elem if _tag(x) == name)
|
|
73
|
+
ident = elem.get('Reference_ID', '')
|
|
74
|
+
if not ident:
|
|
75
|
+
raise FeedError('В источнике CAPEC нет Reference_ID.')
|
|
76
|
+
row = {'id': ident, 'title': text('Title'), 'url': text('URL'),
|
|
77
|
+
'authors': [''.join(x.itertext()) for x in elem if _tag(x) == 'Author'], 'tree': _node(elem)}
|
|
78
|
+
if len(json.dumps(row, ensure_ascii=False).encode()) > MAX_JSON:
|
|
79
|
+
raise FeedError('Источник CAPEC слишком велик: каталог не заменён.')
|
|
80
|
+
return row
|
|
81
|
+
|
|
82
|
+
|
|
83
|
+
def fill(store, archive, info, check):
|
|
84
|
+
"""Потоковый импорт в подготовленный imp; подмена остаётся общей операцией FeedStore."""
|
|
85
|
+
from .feeds import FeedError, guarded_chunks, iter_records, MAX_RECORDS
|
|
86
|
+
root_attrs, seen, counts, statuses = {}, set(), Counter(), Counter()
|
|
87
|
+
refs = pairs = 0
|
|
88
|
+
xml_hash = hashlib.sha256()
|
|
89
|
+
def root(elem):
|
|
90
|
+
root_attrs.update(elem.attrib)
|
|
91
|
+
if elem.tag.lstrip('{') != NS + "}Attack_Pattern_Catalog" or elem.get("Name") != "CAPEC":
|
|
92
|
+
raise FeedError("Нужен полный архив capec_latest.zip с каталогом CAPEC, не отдельное представление.")
|
|
93
|
+
schema = next((v for k,v in elem.attrib.items() if k.lstrip('{') == 'http://www.w3.org/2001/XMLSchema-instance}schemaLocation'), '')
|
|
94
|
+
if not re.search(r"ap_schema_v3\.5\.xsd(?:\s|$)", schema):
|
|
95
|
+
raise FeedError("Версия XML-схемы CAPEC не поддерживается: ожидается 3.5.")
|
|
96
|
+
def chunks(stream):
|
|
97
|
+
for piece in guarded_chunks(stream, compressed=info.compress_size):
|
|
98
|
+
xml_hash.update(piece)
|
|
99
|
+
yield piece
|
|
100
|
+
with archive.open(info) as stream:
|
|
101
|
+
for elem in iter_records(chunks(stream), set(KINDS) | {"External_Reference"}, on_root=root, check=check):
|
|
102
|
+
check()
|
|
103
|
+
is_ref = _tag(elem) == "External_Reference"
|
|
104
|
+
row = parse_reference(elem) if is_ref else parse_record(elem)
|
|
105
|
+
if row is None: raise FeedError("Некорректный источник CAPEC.")
|
|
106
|
+
key = ("ref" if is_ref else "record", row["id"])
|
|
107
|
+
if key in seen: raise FeedError(f"Повтор ID {row['id']} в каталоге CAPEC.")
|
|
108
|
+
seen.add(key)
|
|
109
|
+
if len(seen) > MAX_RECORDS["capec"]: raise FeedError("Слишком много записей CAPEC.")
|
|
110
|
+
encoded = json.dumps(row, ensure_ascii=False)
|
|
111
|
+
store._check_space(len(encoded.encode()) * 4 + 65536)
|
|
112
|
+
with store._tx() as conn:
|
|
113
|
+
if is_ref:
|
|
114
|
+
refs += 1
|
|
115
|
+
conn.execute("INSERT INTO feeds_imp.capec_refs VALUES (?,?)", (row["id"], encoded))
|
|
116
|
+
else:
|
|
117
|
+
counts[row["kind"]] += 1
|
|
118
|
+
statuses[row["status"]] += 1
|
|
119
|
+
conn.execute("INSERT INTO feeds_imp.capec VALUES (?,?)", (row["id"], encoded))
|
|
120
|
+
conn.execute("INSERT INTO feeds_imp.capec_index VALUES (?,?,?,?)", (row["id"], row["kind"], row["name"], row["search_text"]))
|
|
121
|
+
conn.executemany("INSERT INTO feeds_imp.capec_links VALUES (?,?,?) ON CONFLICT DO NOTHING", [(row['id'],x['rel'],x['id']) for x in row['links']])
|
|
122
|
+
conn.executemany("INSERT INTO feeds_imp.capec_cwe VALUES (?,?) ON CONFLICT DO NOTHING", [(row['id'],x) for x in row['cwes']])
|
|
123
|
+
pairs += len(row['cwes'])
|
|
124
|
+
if not all(counts[x] for x in ('pattern','category','view')) or not refs:
|
|
125
|
+
raise FeedError("В архиве нет полного состава CAPEC: нужны шаблоны, категории, представления и источники.")
|
|
126
|
+
profile = {"counts": dict(counts), "statuses": dict(statuses), "cwe_pairs": pairs,
|
|
127
|
+
"sha256": xml_hash.hexdigest(), "schema": "3.5", "namespace": NS, "storage_version": 1}
|
|
128
|
+
with store._tx() as conn:
|
|
129
|
+
conn.execute("INSERT INTO feeds_imp.capec_meta VALUES ('catalog',?)", (json.dumps(profile),))
|
|
130
|
+
return {"records": sum(counts.values()), "ref_records": refs,
|
|
131
|
+
"version": root_attrs.get("Version", ""), "source_date": root_attrs.get("Date", "")}
|
|
132
|
+
|
|
133
|
+
|
|
134
|
+
def view_members(records, ident):
|
|
135
|
+
"""Состав представления; исключения относятся к пути, а не к записи целиком."""
|
|
136
|
+
view=records.get(ident,{})
|
|
137
|
+
if view.get('kind')!='view': return set(),'Представление не найдено.'
|
|
138
|
+
if view['view_type']=='Implicit':
|
|
139
|
+
expr=re.sub(r"\s+(?=(?:[^']*'[^']*')*[^']*$)",'',view['filter'])
|
|
140
|
+
if expr=='/Attack_Pattern_Catalog/*[not(self::External_References)]/*': return set(records),''
|
|
141
|
+
m=re.fullmatch(r"/Attack_Pattern_Catalog/Attack_Patterns/Attack_Pattern\[@Abstraction='(Meta|Standard|Detailed)'\]",expr)
|
|
142
|
+
if m:return {k for k,v in records.items() if v['kind']=='pattern' and v['abstraction']==m[1]},''
|
|
143
|
+
m=re.fullmatch(r"/Attack_Pattern_Catalog/\*/\*\[@Status='([^']+)'\]",expr)
|
|
144
|
+
if m:return {k for k,v in records.items() if v['status']==m[1]},''
|
|
145
|
+
m=re.fullmatch(r"/Attack_Pattern_Catalog/\*/\*\[Taxonomy_Mappings/Taxonomy_Mapping\[@Taxonomy_Name='([^']+)'\]\]",expr)
|
|
146
|
+
if m:return {k for k,v in records.items() if m[1] in v['taxonomies']},''
|
|
147
|
+
m=re.fullmatch(r'/Attack_Pattern_Catalog/\*/\*\[@ID=\(([\d,]+)\)\]',expr)
|
|
148
|
+
if m:return {identifier(x) for x in m[1].split(',')} & records.keys(),''
|
|
149
|
+
return set(),'Условие представления пока не поддерживается.'
|
|
150
|
+
roots=[x['id'] for x in view['links'] if x['rel']=='HasMember']
|
|
151
|
+
if view['view_type']=='Explicit':return set(roots),''
|
|
152
|
+
children={}
|
|
153
|
+
for k,v in records.items():
|
|
154
|
+
for e in v['links']:
|
|
155
|
+
if e['rel']=='ChildOf':children.setdefault(e['id'],set()).add(k)
|
|
156
|
+
if e['rel']=='HasMember':children.setdefault(k,set()).add(e['id'])
|
|
157
|
+
result,note,stack,steps=set(),'',[(x,(ident,)) for x in roots],0
|
|
158
|
+
while stack:
|
|
159
|
+
key,path=stack.pop();steps+=1
|
|
160
|
+
if steps>20000:return result,'Обход ограничен: слишком много путей в представлении.'
|
|
161
|
+
if key in path:
|
|
162
|
+
note='В связях обнаружен цикл; повторный путь пропущен.'
|
|
163
|
+
continue
|
|
164
|
+
row=records.get(key)
|
|
165
|
+
if not row or set(row['exclude']) & set(path):continue
|
|
166
|
+
result.add(key)
|
|
167
|
+
stack.extend((child,path+(key,)) for child in children.get(key,()))
|
|
168
|
+
return result,note
|
|
@@ -0,0 +1,27 @@
|
|
|
1
|
+
"""Структурированный текст CAPEC: таблицы, списки и примеры кода."""
|
|
2
|
+
from markupsafe import Markup, escape
|
|
3
|
+
|
|
4
|
+
HTML = {'p', 'div', 'span', 'b', 'i', 'strong', 'em', 'br', 'ul', 'ol', 'li', 'pre', 'code',
|
|
5
|
+
'table', 'thead', 'tbody', 'tr', 'td', 'th', 'blockquote', 'sub', 'sup'}
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
def plain(node):
|
|
9
|
+
return node['text'] + ''.join(plain(child) + child['tail'] for child in node['children'])
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def rich(node, heading=True):
|
|
13
|
+
tag = node['tag']
|
|
14
|
+
content = str(escape(node['text'])) + ''.join(str(rich(c)) for c in node['children'])
|
|
15
|
+
if tag in HTML and node.get('namespace','') in ('','http://www.w3.org/1999/xhtml'):
|
|
16
|
+
attrs = ''
|
|
17
|
+
if tag in ('td', 'th'):
|
|
18
|
+
for key in ('colspan', 'rowspan'):
|
|
19
|
+
value = node['attrs'].get(key, '')
|
|
20
|
+
if str(value).isdigit() and 1 <= int(value) <= 100:
|
|
21
|
+
attrs += f' {key}="{int(value)}"'
|
|
22
|
+
result = '<br>' if tag == 'br' else f'<{tag}{attrs}>{content}</{tag}>'
|
|
23
|
+
else:
|
|
24
|
+
title = '<strong>' + str(escape(tag.replace('_', ' '))) + '</strong>' if heading else ''
|
|
25
|
+
attrs = ' '.join(str(escape(f'{key}: {value}')) for key, value in node['attrs'].items())
|
|
26
|
+
result = f'<div class="capec-node">{title}<small class="muted">{attrs}</small><div>{content}</div></div>'
|
|
27
|
+
return Markup(result + str(escape(node['tail'])))
|
package/app/codecmp.py
ADDED
|
@@ -0,0 +1,59 @@
|
|
|
1
|
+
"""Отпечаток сравнения кода находок одной группы SAST (ADR-0092).
|
|
2
|
+
|
|
3
|
+
Находки с одинаковыми правилом, CWE, путём файла и строкой (группа ADR-0089) в разных кодовых базах могут стоять на
|
|
4
|
+
разном коде. Решение и текст должны совпадать только у находок с одинаковым кодом вокруг строки срабатывания. Отпечаток
|
|
5
|
+
считается по сохранённому срезу находки (`cases.context_json`) и отделён от отпечатков вердиктов и кэша: его смена не
|
|
6
|
+
трогает ни вердикты, ни отметки.
|
|
7
|
+
|
|
8
|
+
Окно – строки «срабатывание −15 … +5» точно как в срезе (пробелы значимы: в Python и строковых литералах они меняют
|
|
9
|
+
смысл), положение строки срабатывания в окне и язык. Пустой отпечаток – «сравнение недостоверно»: среза нет или он не
|
|
10
|
+
«ok», в окне есть обрезанная длинная строка или маскированное значение, окно не помещается в срез (сверху – если срез
|
|
11
|
+
не начинается с первой строки файла; снизу – всегда: где кончается файл, по срезу не установить).
|
|
12
|
+
"""
|
|
13
|
+
|
|
14
|
+
from __future__ import annotations
|
|
15
|
+
|
|
16
|
+
import hashlib
|
|
17
|
+
from collections.abc import Mapping
|
|
18
|
+
from typing import Any
|
|
19
|
+
|
|
20
|
+
from app.context import CUT_MARK
|
|
21
|
+
from app.masking import PLACEHOLDER_RE
|
|
22
|
+
|
|
23
|
+
VERSION = "cmp-1"
|
|
24
|
+
BEFORE = 15 # строк над строкой срабатывания
|
|
25
|
+
AFTER = 5 # строк под ней
|
|
26
|
+
|
|
27
|
+
|
|
28
|
+
def _int(value: Any) -> int | None:
|
|
29
|
+
try:
|
|
30
|
+
return int(value)
|
|
31
|
+
except (TypeError, ValueError):
|
|
32
|
+
return None
|
|
33
|
+
|
|
34
|
+
|
|
35
|
+
def code_cmp(context: Mapping[str, Any] | None) -> str:
|
|
36
|
+
"""Отпечаток сравнения по срезу находки или "" – сравнение недостоверно."""
|
|
37
|
+
ctx = context or {}
|
|
38
|
+
if str(ctx.get("status") or "") != "ok":
|
|
39
|
+
return ""
|
|
40
|
+
lines = ctx.get("lines")
|
|
41
|
+
if not isinstance(lines, list) or not lines or not all(isinstance(x, str) for x in lines):
|
|
42
|
+
return ""
|
|
43
|
+
start, target = _int(ctx.get("start_line")), _int(ctx.get("target_line"))
|
|
44
|
+
if start is None or target is None or start < 1:
|
|
45
|
+
return ""
|
|
46
|
+
end = start + len(lines) - 1
|
|
47
|
+
if not start <= target <= end:
|
|
48
|
+
return ""
|
|
49
|
+
low, high = target - BEFORE, target + AFTER
|
|
50
|
+
if low < start and start != 1: # окно сверху не помещается, а срез начинается не с начала файла
|
|
51
|
+
return ""
|
|
52
|
+
if high > end: # конец файла по срезу не установить
|
|
53
|
+
return ""
|
|
54
|
+
window = lines[max(low, start) - start: high - start + 1]
|
|
55
|
+
if any(CUT_MARK in line or PLACEHOLDER_RE.search(line) for line in window):
|
|
56
|
+
return ""
|
|
57
|
+
position = target - max(low, start)
|
|
58
|
+
payload = "\n".join([VERSION, str(ctx.get("language") or ""), str(position), *window])
|
|
59
|
+
return hashlib.sha256(payload.encode("utf-8")).hexdigest()[:32]
|