@topy-ai/maggie 0.7.55 → 0.7.57
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README-zh-TW.md +7 -3
- package/README.md +42 -8
- package/bin/maggie.js +4 -0
- package/bundled-contracts/maggie-deployment/readiness-v1.schema.json +2 -1
- package/bundled-contracts/maggie-security/finding-example-v1.json +89 -0
- package/bundled-contracts/maggie-security/finding-v1.schema.json +63 -0
- package/bundled-contracts/maggie-security/findings-set-example-v1.json +148 -0
- package/bundled-contracts/maggie-security/findings-set-v1.schema.json +32 -0
- package/bundled-contracts/maggie-security/protect-plan-v1.schema.json +28 -0
- package/bundled-contracts/maggie-security/report-v1.schema.json +29 -0
- package/bundled-contracts/maggie-security/stack-example-v1.json +66 -0
- package/bundled-contracts/maggie-security/stack-upload-v1.schema.json +22 -0
- package/bundled-contracts/maggie-security/stack-v1.schema.json +37 -0
- package/bundled-contracts/maggie-security/tools-lock-v1.schema.json +36 -0
- package/bundled-skills/catalog.json +4 -0
- package/bundled-skills/maggie-security/SKILL.md +173 -0
- package/bundled-skills/maggie-security/agents/openai.yaml +4 -0
- package/bundled-skills/maggie-security/data/LICENSE-NOTICE.md +24 -0
- package/bundled-skills/maggie-security/data/asvs-map.json +209 -0
- package/bundled-skills/maggie-security/data/fail2ban/filter.d/maggie-wordpress.conf +6 -0
- package/bundled-skills/maggie-security/data/semgrep/LICENSE-NOTICE.md +5 -0
- package/bundled-skills/maggie-security/data/semgrep/VERSION +1 -0
- package/bundled-skills/maggie-security/data/semgrep/rules/dockerfile.yaml +14 -0
- package/bundled-skills/maggie-security/data/semgrep/rules/generic.yaml +26 -0
- package/bundled-skills/maggie-security/data/semgrep/rules/js.yaml +216 -0
- package/bundled-skills/maggie-security/data/semgrep/rules/php-debug-ok.yaml +17 -0
- package/bundled-skills/maggie-security/data/semgrep/rules/php-debug.yaml +17 -0
- package/bundled-skills/maggie-security/data/semgrep/rules/php.yaml +101 -0
- package/bundled-skills/maggie-security/data/semgrep/tests/dockerfile.dockerfile +10 -0
- package/bundled-skills/maggie-security/data/semgrep/tests/generic.astro +11 -0
- package/bundled-skills/maggie-security/data/semgrep/tests/generic.conf +11 -0
- package/bundled-skills/maggie-security/data/semgrep/tests/js.jsx +95 -0
- package/bundled-skills/maggie-security/data/semgrep/tests/php-debug-ok.php +5 -0
- package/bundled-skills/maggie-security/data/semgrep/tests/php-debug.php +3 -0
- package/bundled-skills/maggie-security/data/semgrep/tests/php.php +55 -0
- package/bundled-skills/maggie-security/evals/evals.json +97 -0
- package/bundled-skills/maggie-security/references/commands.md +125 -0
- package/bundled-skills/maggie-security/references/cve-response.md +24 -0
- package/bundled-skills/maggie-security/references/explaining-findings.md +57 -0
- package/bundled-skills/maggie-security/references/findings.md +39 -0
- package/bundled-skills/maggie-security/references/ownership-and-consent.md +29 -0
- package/bundled-skills/maggie-security/references/site-types.md +22 -0
- package/bundled-skills/maggie-security/references/tools-and-licences.md +16 -0
- package/bundled-skills/maggie-security/references/workflows.md +23 -0
- package/bundled-skills/maggie-security/registry/security-tools.registry.json +654 -0
- package/bundled-skills/maggie-security/routines.example.json +27 -0
- package/bundled-tools/clis/maggie_deployment_readiness.py +17 -2
- package/bundled-tools/clis/maggie_security.py +342 -0
- package/bundled-tools/clis/maggie_security_agent.py +326 -0
- package/bundled-tools/clis/maggie_security_code.py +650 -0
- package/bundled-tools/clis/maggie_security_core.py +1512 -0
- package/bundled-tools/clis/maggie_security_deps.py +560 -0
- package/bundled-tools/clis/maggie_security_health.py +415 -0
- package/bundled-tools/clis/maggie_security_host.py +511 -0
- package/bundled-tools/clis/maggie_security_hosted.py +115 -0
- package/bundled-tools/clis/maggie_security_intel.py +1294 -0
- package/bundled-tools/clis/maggie_security_inventory.py +1023 -0
- package/bundled-tools/clis/maggie_security_logs.py +777 -0
- package/bundled-tools/clis/maggie_security_mcp.py +484 -0
- package/bundled-tools/clis/maggie_security_mirror.py +122 -0
- package/bundled-tools/clis/maggie_security_protect.py +519 -0
- package/bundled-tools/clis/maggie_security_report.py +506 -0
- package/bundled-tools/clis/maggie_security_standards.py +214 -0
- package/bundled-tools/clis/maggie_security_versions.py +381 -0
- package/bundled-tools/clis/maggie_security_web.py +1247 -0
- package/bundled-tools/clis/security_data/eol-snapshot.json +1 -0
- package/bundled-tools/clis/security_data/gitleaks-rules.json +3812 -0
- package/bundled-tools/clis/security_data/maltrail-lists.json +1 -0
- package/bundled-tools/clis/security_data/public-suffix-icann.txt +7374 -0
- package/bundled-tools/clis/security_data/retire-extractors.json +695 -0
- package/bundled-tools/clis/security_data/sigma-web.json +6514 -0
- package/bundled-tools/clis/security_data/tools.lock.json +358 -0
- package/package.json +1 -1
|
@@ -0,0 +1,214 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""Standards (docs/maggie-security/09-risk-report-standards-prd.md): ``checklist`` and ``threat-model``.
|
|
3
|
+
|
|
4
|
+
- ``checklist`` lays the site's findings over OWASP ASVS 5.0. The requirement
|
|
5
|
+
text is read from OWASP's own CSV at run time (cached 30 days; CC BY-SA 4.0,
|
|
6
|
+
attributed in the output) rather than packaged. A requirement is ``pass``
|
|
7
|
+
only when a command that positively checks it (Maggie's coverage map) ran in
|
|
8
|
+
the last 30 days without degrading and left no open finding; ``fail`` when an
|
|
9
|
+
open finding names it; ``unknown`` when a checker exists but has not run;
|
|
10
|
+
``manual`` when no Maggie command checks it; ``not-applicable`` for the
|
|
11
|
+
profile. It never claims compliance.
|
|
12
|
+
- ``threat-model`` writes an OWASP Threat Dragon v2 model with a STRIDE diagram
|
|
13
|
+
of the site's elements and, optionally, open findings as threats.
|
|
14
|
+
"""
|
|
15
|
+
|
|
16
|
+
from __future__ import annotations
|
|
17
|
+
|
|
18
|
+
import argparse
|
|
19
|
+
import csv
|
|
20
|
+
import io
|
|
21
|
+
import json
|
|
22
|
+
from pathlib import Path
|
|
23
|
+
|
|
24
|
+
import maggie_security_core as core
|
|
25
|
+
|
|
26
|
+
ASVS_URL = "https://raw.githubusercontent.com/OWASP/ASVS/master/5.0/docs_en/OWASP_Application_Security_Verification_Standard_5.0.0_en.csv"
|
|
27
|
+
core.SOURCES.setdefault("asvs", {"hosts": ["raw.githubusercontent.com"], "ttl": "30d", "url": ASVS_URL})
|
|
28
|
+
ATTRIBUTION = "OWASP Application Security Verification Standard 5.0.0, OWASP Foundation, CC BY-SA 4.0 (requirement text unchanged)."
|
|
29
|
+
|
|
30
|
+
|
|
31
|
+
def data_dir() -> Path:
|
|
32
|
+
for base in (core.ROOT / "skills", core.ROOT / "bundled-skills"):
|
|
33
|
+
path = base / "maggie-security" / "data"
|
|
34
|
+
if path.is_dir():
|
|
35
|
+
return path
|
|
36
|
+
raise core.UsageError("maggie-security data folder not found")
|
|
37
|
+
|
|
38
|
+
|
|
39
|
+
def load_asvs(ctx: core.Context) -> list[dict]:
|
|
40
|
+
body = ctx.cache.get("asvs", ASVS_URL, ttl="30d")
|
|
41
|
+
if body is None:
|
|
42
|
+
raise core.DegradedError("could not read the ASVS 5.0 requirements (OWASP CSV); try again online once, it is cached for 30 days")
|
|
43
|
+
rows = list(csv.DictReader(io.StringIO(body.decode("utf-8-sig"))))
|
|
44
|
+
if not rows or "req_id" not in rows[0]:
|
|
45
|
+
raise core.DegradedError("the ASVS CSV has an unexpected shape")
|
|
46
|
+
return [{"id": r["req_id"], "level": int(r["L"]), "chapter": r["chapter_id"], "chapter_name": r["chapter_name"],
|
|
47
|
+
"section": r["section_id"], "section_name": r["section_name"], "text": r["req_description"]} for r in rows]
|
|
48
|
+
|
|
49
|
+
|
|
50
|
+
def latest_runs(state: core.State, days: int = 30) -> dict[str, dict]:
|
|
51
|
+
floor = core.utcnow() - core.parse_duration(f"{days}d")
|
|
52
|
+
out: dict[str, dict] = {}
|
|
53
|
+
runs = state.path("runs")
|
|
54
|
+
for path in sorted(runs.glob("*.json")) if runs.is_dir() else []:
|
|
55
|
+
try:
|
|
56
|
+
doc = json.loads(path.read_text(encoding="utf-8"))
|
|
57
|
+
except ValueError:
|
|
58
|
+
continue
|
|
59
|
+
if core.parse_time(doc["run"]["finished_at"]) >= floor:
|
|
60
|
+
out[doc["run"]["command"]] = doc
|
|
61
|
+
return out
|
|
62
|
+
|
|
63
|
+
|
|
64
|
+
def checklist(ctx: core.Context) -> dict:
|
|
65
|
+
args = ctx.args
|
|
66
|
+
mapping = core.read_json(data_dir() / "asvs-map.json")
|
|
67
|
+
stack_path = ctx.state.path("inventory", "stack.json")
|
|
68
|
+
profile = args.profile or (core.read_json(stack_path)["site"].get("type") if stack_path.is_file() else core.detect_site_type(ctx.project))
|
|
69
|
+
if profile not in mapping["profiles"]:
|
|
70
|
+
profile = "node-app" if profile in ("unknown",) else profile
|
|
71
|
+
if profile not in mapping["profiles"]:
|
|
72
|
+
raise core.UsageError(f"--profile must be one of {', '.join(mapping['profiles'])}")
|
|
73
|
+
rules = mapping["profiles"][profile]
|
|
74
|
+
requirements = [r for r in load_asvs(ctx) if r["level"] <= args.level]
|
|
75
|
+
runs = latest_runs(ctx.state)
|
|
76
|
+
ledger = core.Ledger(ctx.state.path("ledger.jsonl"))
|
|
77
|
+
history = ledger.history()
|
|
78
|
+
open_by_asvs: dict[str, list[dict]] = {}
|
|
79
|
+
for doc in runs.values():
|
|
80
|
+
for finding in doc.get("findings") or []:
|
|
81
|
+
status = core.Ledger.derive(history[finding["id"]])["status"] if finding["id"] in history else finding["status"]
|
|
82
|
+
if status in ("new", "open"):
|
|
83
|
+
for asvs in finding.get("asvs_ids") or []:
|
|
84
|
+
open_by_asvs.setdefault(asvs, []).append(finding)
|
|
85
|
+
checkers: dict[str, list[str]] = {}
|
|
86
|
+
for command, ids in mapping["coverage"].items():
|
|
87
|
+
for asvs in ids:
|
|
88
|
+
checkers.setdefault(asvs, []).append(command)
|
|
89
|
+
items = []
|
|
90
|
+
for req in requirements:
|
|
91
|
+
applicable = req["chapter"] in rules["chapters"] and req["section"] not in rules["exclude_sections"]
|
|
92
|
+
commands = checkers.get(req["id"], [])
|
|
93
|
+
ran = [c for c in commands if c in runs and not runs[c]["run"].get("degraded")]
|
|
94
|
+
if not applicable:
|
|
95
|
+
state = "not-applicable"
|
|
96
|
+
elif req["id"] in open_by_asvs:
|
|
97
|
+
state = "fail"
|
|
98
|
+
elif ran:
|
|
99
|
+
state = "pass"
|
|
100
|
+
elif commands:
|
|
101
|
+
state = "unknown"
|
|
102
|
+
else:
|
|
103
|
+
state = "manual"
|
|
104
|
+
items.append({"id": req["id"], "level": req["level"], "section": f"{req['section']} {req['section_name']}", "state": state,
|
|
105
|
+
"checked_by": commands, "evidence": [f["id"] for f in open_by_asvs.get(req["id"], [])][:10] if state == "fail" else
|
|
106
|
+
[runs[c]["run"]["id"] for c in ran][:3], "text": req["text"],
|
|
107
|
+
"next": (f"run maggie security {commands[0]}" if state == "unknown" else None)})
|
|
108
|
+
counts = {s: sum(1 for i in items if i["state"] == s) for s in ("pass", "fail", "unknown", "manual", "not-applicable")}
|
|
109
|
+
result = {"schema": "maggie-security-checklist.v1", "standard": "ASVS 5.0.0", "level": args.level, "profile": profile,
|
|
110
|
+
"total": len(items), "checked": counts["pass"] + counts["fail"], **counts, "attribution": ATTRIBUTION,
|
|
111
|
+
"note": "A checklist of what Maggie could verify automatically; it is not a compliance statement.", "items": items}
|
|
112
|
+
if args.format == "md":
|
|
113
|
+
result["markdown"] = checklist_markdown(result)
|
|
114
|
+
if args.fail_on and counts["fail"]:
|
|
115
|
+
result["exit_code"] = core.EXIT_FINDINGS
|
|
116
|
+
return result
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
def checklist_markdown(result: dict) -> str:
|
|
120
|
+
lines = [f"# ASVS 5.0 Level {result['level']} checklist — {result['profile']}", "",
|
|
121
|
+
f"{result['checked']} of {result['total']} requirements checked automatically · {result['pass']} passing · {result['fail']} failing · "
|
|
122
|
+
f"{result['unknown']} not run yet · {result['manual']} need a person · {result['not-applicable']} not applicable", "",
|
|
123
|
+
"| Requirement | Level | State | Checked by |", "|---|---|---|---|"]
|
|
124
|
+
for item in result["items"]:
|
|
125
|
+
if item["state"] == "not-applicable":
|
|
126
|
+
continue
|
|
127
|
+
lines.append(f"| {item['id']} {item['section']} | L{item['level']} | {item['state']} | {', '.join(item['checked_by']) or '—'} |")
|
|
128
|
+
lines += ["", result["note"], "", result["attribution"]]
|
|
129
|
+
return "\n".join(lines) + "\n"
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
# --------------------------------------------------------------------------- threat model (OWASP Threat Dragon v2)
|
|
133
|
+
|
|
134
|
+
|
|
135
|
+
def threat_model(ctx: core.Context) -> dict:
|
|
136
|
+
args = ctx.args
|
|
137
|
+
stack_path = ctx.state.path("inventory", "stack.json")
|
|
138
|
+
stack = core.read_json(stack_path) if stack_path.is_file() else {"site": {"id": ctx.site, "type": core.detect_site_type(ctx.project)},
|
|
139
|
+
"runtimes": [], "hosts": []}
|
|
140
|
+
profile = args.profile or stack["site"].get("type") or "static"
|
|
141
|
+
runtimes = {r["name"] for r in stack.get("runtimes") or []}
|
|
142
|
+
databases = sorted(runtimes & {"mysql", "mariadb", "postgresql", "redis", "mongodb", "memcached", "elasticsearch"})
|
|
143
|
+
cells: list[dict] = []
|
|
144
|
+
threats_by_cell: dict[str, list[dict]] = {}
|
|
145
|
+
|
|
146
|
+
def element(cid: str, shape: str, name: str, x: int, y: int, description: str, threats: list[dict]) -> None:
|
|
147
|
+
cells.append({"id": cid, "shape": shape, "position": {"x": x, "y": y}, "size": {"width": 160, "height": 80},
|
|
148
|
+
"data": {"type": f"tm.{ {'actor': 'Actor', 'process': 'Process', 'store': 'Store'}[shape]}", "name": name,
|
|
149
|
+
"description": description, "outOfScope": False, "hasOpenThreats": bool(threats), "threats": threats}})
|
|
150
|
+
threats_by_cell[cid] = threats
|
|
151
|
+
|
|
152
|
+
def threat(title: str, kind: str, severity: str, mitigation: str, description: str = "") -> dict:
|
|
153
|
+
return {"title": title, "type": kind, "status": "Open", "severity": severity, "description": description, "mitigation": mitigation,
|
|
154
|
+
"modelType": "STRIDE"}
|
|
155
|
+
|
|
156
|
+
element("visitor", "actor", "Visitor / attacker", 40, 160, "Anyone on the internet", [])
|
|
157
|
+
web_threats = [threat("Spoofed or stolen admin session", "Spoofing", "High", "Secure, HttpOnly, SameSite cookies; MFA for admins"),
|
|
158
|
+
threat("Script injection in pages", "Tampering", "High", "Output encoding, CSP, code-scan rules"),
|
|
159
|
+
threat("Brute force on login", "Denial of service", "Medium", "Rate limits (protect-plan), fail2ban/CrowdSec"),
|
|
160
|
+
threat("Version and error disclosure", "Information disclosure", "Low", "server_tokens off, generic error pages")]
|
|
161
|
+
if profile == "wordpress":
|
|
162
|
+
web_threats.append(threat("Vulnerable plugin or theme", "Elevation of privilege", "High", "wp-check daily, remove unused plugins"))
|
|
163
|
+
element("web", "process", f"{stack['site'].get('id')} ({profile})", 320, 160, "Web server and application", web_threats)
|
|
164
|
+
if databases:
|
|
165
|
+
element("db", "store", ", ".join(databases), 620, 80, "Data stores", [
|
|
166
|
+
threat("Database reachable from the internet", "Information disclosure", "High", "Bind to 127.0.0.1; host-quick hq-ports"),
|
|
167
|
+
threat("SQL injection", "Tampering", "High", "Prepared statements; code-scan maggie.php.sql-concat")])
|
|
168
|
+
element("secrets", "store", "Secrets and configuration", 620, 260, ".env, API keys, wp-config.php", [
|
|
169
|
+
threat("Secret leaked in code or build output", "Information disclosure", "High", "secrets gate before deploy; rotate first")])
|
|
170
|
+
if args.include_findings:
|
|
171
|
+
for doc in latest_runs(ctx.state).values():
|
|
172
|
+
for finding in doc.get("findings") or []:
|
|
173
|
+
if finding["status"] in ("new", "open") and finding["priority"] in ("p0", "p1", "p2"):
|
|
174
|
+
target = "db" if (finding.get("affected") or {}).get("name") in databases and databases else \
|
|
175
|
+
"secrets" if finding["kind"] == "secret" else "web"
|
|
176
|
+
threats_by_cell[target].append(threat(finding["title"][:120], "Tampering" if finding["kind"] != "secret" else "Information disclosure",
|
|
177
|
+
{"p0": "High", "p1": "High", "p2": "Medium"}[finding["priority"]],
|
|
178
|
+
(finding.get("remediation") or {}).get("summary") or "", f"Maggie finding {finding['id']}"))
|
|
179
|
+
for cell in cells:
|
|
180
|
+
cell["data"]["threats"] = threats_by_cell[cell["id"]]
|
|
181
|
+
cell["data"]["hasOpenThreats"] = bool(threats_by_cell[cell["id"]])
|
|
182
|
+
flows = [("visitor", "web", "HTTPS requests")] + ([("web", "db", "queries")] if databases else []) + [("web", "secrets", "reads config")]
|
|
183
|
+
for index, (src, dst, label) in enumerate(flows):
|
|
184
|
+
cells.append({"id": f"flow-{index}", "shape": "flow", "source": {"cell": src}, "target": {"cell": dst},
|
|
185
|
+
"data": {"type": "tm.Flow", "name": label, "isEncrypted": src == "visitor", "outOfScope": False, "threats": []}})
|
|
186
|
+
cells.append({"id": "boundary-0", "shape": "trust-boundary-curve", "source": {"x": 230, "y": 40}, "target": {"x": 230, "y": 320},
|
|
187
|
+
"data": {"type": "tm.Boundary", "name": "Internet / server"}})
|
|
188
|
+
model = {"version": "2.3.0", "summary": {"title": f"{stack['site'].get('id')} threat model", "owner": "", "description":
|
|
189
|
+
"Draft generated by maggie security threat-model from the stack inventory; review and edit in OWASP Threat Dragon.", "id": 0},
|
|
190
|
+
"detail": {"contributors": [], "diagrams": [{"id": 0, "title": "Main", "diagramType": "STRIDE", "version": "2.3.0", "cells": cells}],
|
|
191
|
+
"diagramTop": 1, "reviewer": "", "threatTop": sum(len(t) for t in threats_by_cell.values())}}
|
|
192
|
+
out = Path(args.out) if args.out else ctx.state.path("reports", "threat-model.json")
|
|
193
|
+
core.write_json(out, model)
|
|
194
|
+
return {"schema": "maggie-security-threat-model-result.v1", "path": str(out), "elements": len([c for c in cells if c["shape"] != "flow"]),
|
|
195
|
+
"threats": model["detail"]["threatTop"], "open_in": "https://www.threatdragon.com/ (Open an existing model)"}
|
|
196
|
+
|
|
197
|
+
|
|
198
|
+
def _checklist(p: argparse.ArgumentParser) -> None:
|
|
199
|
+
p.add_argument("--profile", choices=["static", "wordpress", "node-app", "php-app", "python-app", "java-app", "workers", "astro"])
|
|
200
|
+
p.add_argument("--level", type=int, choices=[1, 2, 3], default=1)
|
|
201
|
+
|
|
202
|
+
|
|
203
|
+
def _threat(p: argparse.ArgumentParser) -> None:
|
|
204
|
+
p.add_argument("--profile", choices=["static", "wordpress", "node-app", "php-app", "python-app", "java-app"])
|
|
205
|
+
p.add_argument("--include-findings", action="store_true")
|
|
206
|
+
p.add_argument("--out")
|
|
207
|
+
|
|
208
|
+
|
|
209
|
+
COMMANDS = [
|
|
210
|
+
core.Command("checklist", "OWASP ASVS 5.0 checklist filled from your findings (what passed, failed, was not checked, or needs a person)",
|
|
211
|
+
["asvs"], _checklist, checklist, findings=False),
|
|
212
|
+
core.Command("threat-model", "draft an OWASP Threat Dragon model (STRIDE) from the stack and open findings", [], _threat, threat_model,
|
|
213
|
+
findings=False),
|
|
214
|
+
]
|
|
@@ -0,0 +1,381 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""Version ordering and range evaluation for every stack the security watch covers.
|
|
3
|
+
|
|
4
|
+
Standard library only. Ecosystems:
|
|
5
|
+
|
|
6
|
+
- SemVer (npm, crates.io, Go modules, Node core, Hex, Pub): precedence after
|
|
7
|
+
osv.dev ``osv/semver_index.py`` (Apache-2.0, Google LLC): coerce, then
|
|
8
|
+
compare core numerically and pre-release identifiers per SemVer 2.0 rule 11.
|
|
9
|
+
- npm ranges (``^ ~ x * >= < || hyphen``) for Node core ``vulnerable`` /
|
|
10
|
+
``patched`` and package.json engines.
|
|
11
|
+
- Generic dotted versions with qualifiers (PyPI, Maven, NuGet, RubyGems,
|
|
12
|
+
Packagist, WordPress, nginx, PHP, databases, endoflife.date cycles):
|
|
13
|
+
numeric segments compare numerically; ``dev < alpha < beta < milestone <
|
|
14
|
+
rc < (release) < sp/post/patch`` as Maven and PEP 440 order them.
|
|
15
|
+
- Debian/Ubuntu dpkg ordering (epoch, upstream, revision; ``~`` sorts first)
|
|
16
|
+
for OS packages.
|
|
17
|
+
- OSV ``affected[].ranges[].events`` and CVE JSON 5 ``versions[]`` evaluation.
|
|
18
|
+
"""
|
|
19
|
+
|
|
20
|
+
from __future__ import annotations
|
|
21
|
+
|
|
22
|
+
import functools
|
|
23
|
+
import re
|
|
24
|
+
from typing import Iterable
|
|
25
|
+
|
|
26
|
+
SEMVER_ECOSYSTEMS = {"npm", "crates.io", "go", "hex", "pub", "node", "semver", "bun"}
|
|
27
|
+
DPKG_ECOSYSTEMS = {"debian", "ubuntu", "deb"}
|
|
28
|
+
|
|
29
|
+
# --------------------------------------------------------------------------- semver
|
|
30
|
+
|
|
31
|
+
_SEMVER = re.compile(r"^v?(\d+)(?:\.(\d+))?(?:\.(\d+))?(?:-([0-9A-Za-z.-]+))?(?:\+[0-9A-Za-z.-]+)?$")
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def parse_semver(version: str) -> tuple[int, int, int, tuple] | None:
|
|
35
|
+
"""Coerce like osv.dev semver_index.coerce: leading v, missing minor/patch -> 0."""
|
|
36
|
+
match = _SEMVER.match(version.strip())
|
|
37
|
+
if not match:
|
|
38
|
+
return None
|
|
39
|
+
major, minor, patch, pre = match.groups()
|
|
40
|
+
pre_parts: tuple = ()
|
|
41
|
+
if pre:
|
|
42
|
+
pre_parts = tuple((0, int(p), "") if p.isdecimal() else (1, 0, p) for p in pre.split("."))
|
|
43
|
+
return int(major), int(minor or 0), int(patch or 0), pre_parts
|
|
44
|
+
|
|
45
|
+
|
|
46
|
+
def compare_semver(a: str, b: str) -> int:
|
|
47
|
+
pa, pb = parse_semver(a), parse_semver(b)
|
|
48
|
+
if pa is None or pb is None:
|
|
49
|
+
return compare_generic(a, b)
|
|
50
|
+
if pa[:3] != pb[:3]:
|
|
51
|
+
return -1 if pa[:3] < pb[:3] else 1
|
|
52
|
+
# A pre-release has lower precedence than the release (rule 11.3).
|
|
53
|
+
if pa[3] == pb[3]:
|
|
54
|
+
return 0
|
|
55
|
+
if not pa[3]:
|
|
56
|
+
return 1
|
|
57
|
+
if not pb[3]:
|
|
58
|
+
return -1
|
|
59
|
+
return -1 if pa[3] < pb[3] else 1
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
# --------------------------------------------------------------------------- generic
|
|
63
|
+
|
|
64
|
+
_QUALIFIERS = {"dev": -6, "snapshot": -5, "a": -4, "alpha": -4, "b": -3, "beta": -3, "m": -2, "milestone": -2,
|
|
65
|
+
"c": -1, "rc": -1, "cr": -1, "pre": -1, "preview": -1, "": 0, "ga": 0, "final": 0, "release": 0,
|
|
66
|
+
"stable": 0, "p": 1, "pl": 1, "patch": 1, "post": 1, "sp": 1, "r": 1}
|
|
67
|
+
_TOKEN = re.compile(r"(\d+|[A-Za-z]+)")
|
|
68
|
+
|
|
69
|
+
|
|
70
|
+
def _generic_tokens(version: str) -> list[tuple[int, int, str]]:
|
|
71
|
+
text = version.strip().lower()
|
|
72
|
+
text = re.sub(r"^v(?=\d)", "", text)
|
|
73
|
+
text = text.split("+", 1)[0]
|
|
74
|
+
if "!" in text: # PEP 440 epoch
|
|
75
|
+
epoch, _, text = text.partition("!")
|
|
76
|
+
prefix = [(2, int(epoch) if epoch.isdigit() else 0, "")]
|
|
77
|
+
else:
|
|
78
|
+
prefix = [(2, 0, "")]
|
|
79
|
+
tokens = []
|
|
80
|
+
for raw in _TOKEN.findall(text):
|
|
81
|
+
if raw.isdigit():
|
|
82
|
+
tokens.append((2, int(raw), ""))
|
|
83
|
+
else:
|
|
84
|
+
rank = _QUALIFIERS.get(raw)
|
|
85
|
+
tokens.append((1, rank, "") if rank is not None else (0, 0, raw))
|
|
86
|
+
# Trailing zeros of the numeric release part and release markers do not
|
|
87
|
+
# change the version: 1.0 == 1.0.0 == 1.0-final, and 1.0-rc1 == 1-rc1.
|
|
88
|
+
split = next((i for i, t in enumerate(tokens) if t[0] != 2), len(tokens))
|
|
89
|
+
release, suffix = tokens[:split], tokens[split:]
|
|
90
|
+
while release and release[-1] == (2, 0, "") and len(release) > 1:
|
|
91
|
+
release.pop()
|
|
92
|
+
while suffix and suffix[-1] in ((2, 0, ""), (1, 0, "")):
|
|
93
|
+
suffix.pop()
|
|
94
|
+
if suffix and suffix[0] == (1, 0, ""):
|
|
95
|
+
suffix = suffix[1:]
|
|
96
|
+
return prefix + release + suffix
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
def compare_generic(a: str, b: str) -> int:
|
|
100
|
+
ta, tb = _generic_tokens(a), _generic_tokens(b)
|
|
101
|
+
for index in range(max(len(ta), len(tb))):
|
|
102
|
+
xa = ta[index] if index < len(ta) else None
|
|
103
|
+
xb = tb[index] if index < len(tb) else None
|
|
104
|
+
if xa == xb:
|
|
105
|
+
continue
|
|
106
|
+
# A missing token is a release; a following number is newer, a qualifier decides itself.
|
|
107
|
+
if xa is None:
|
|
108
|
+
return -1 if xb[0] == 2 or (xb[0] == 1 and xb[1] > 0) else 1
|
|
109
|
+
if xb is None:
|
|
110
|
+
return 1 if xa[0] == 2 or (xa[0] == 1 and xa[1] > 0) else -1
|
|
111
|
+
key_a = (xa[0] == 2, xa[1] if xa[0] != 0 else 0, xa[2])
|
|
112
|
+
key_b = (xb[0] == 2, xb[1] if xb[0] != 0 else 0, xb[2])
|
|
113
|
+
if xa[0] == 1 and xb[0] == 1:
|
|
114
|
+
return -1 if xa[1] < xb[1] else 1
|
|
115
|
+
if xa[0] == 0 and xb[0] == 0:
|
|
116
|
+
return -1 if xa[2] < xb[2] else 1
|
|
117
|
+
if xa[0] != xb[0]:
|
|
118
|
+
# number > named qualifier > unknown word, except post-release qualifiers outrank nothing else
|
|
119
|
+
order = {2: 2, 1: 1, 0: 0}
|
|
120
|
+
if xa[0] == 1 and xb[0] == 0:
|
|
121
|
+
return 1
|
|
122
|
+
if xa[0] == 0 and xb[0] == 1:
|
|
123
|
+
return -1
|
|
124
|
+
return -1 if order[xa[0]] < order[xb[0]] else 1
|
|
125
|
+
return -1 if key_a < key_b else 1
|
|
126
|
+
return 0
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
# --------------------------------------------------------------------------- dpkg
|
|
130
|
+
|
|
131
|
+
|
|
132
|
+
def _dpkg_order(char: str) -> int:
|
|
133
|
+
if char == "~":
|
|
134
|
+
return -1
|
|
135
|
+
if char.isdigit():
|
|
136
|
+
return 0
|
|
137
|
+
if char.isalpha():
|
|
138
|
+
return ord(char)
|
|
139
|
+
return ord(char) + 256
|
|
140
|
+
|
|
141
|
+
|
|
142
|
+
def _dpkg_part(a: str, b: str) -> int:
|
|
143
|
+
ia = ib = 0
|
|
144
|
+
while ia < len(a) or ib < len(b):
|
|
145
|
+
first_diff = 0
|
|
146
|
+
while (ia < len(a) and not a[ia].isdigit()) or (ib < len(b) and not b[ib].isdigit()):
|
|
147
|
+
ca = _dpkg_order(a[ia]) if ia < len(a) and not a[ia].isdigit() else 0
|
|
148
|
+
cb = _dpkg_order(b[ib]) if ib < len(b) and not b[ib].isdigit() else 0
|
|
149
|
+
if ca != cb:
|
|
150
|
+
return -1 if ca < cb else 1
|
|
151
|
+
ia += 1
|
|
152
|
+
ib += 1
|
|
153
|
+
while ia < len(a) and a[ia] == "0":
|
|
154
|
+
ia += 1
|
|
155
|
+
while ib < len(b) and b[ib] == "0":
|
|
156
|
+
ib += 1
|
|
157
|
+
while ia < len(a) and a[ia].isdigit() and ib < len(b) and b[ib].isdigit():
|
|
158
|
+
if not first_diff:
|
|
159
|
+
first_diff = ord(a[ia]) - ord(b[ib])
|
|
160
|
+
ia += 1
|
|
161
|
+
ib += 1
|
|
162
|
+
if ia < len(a) and a[ia].isdigit():
|
|
163
|
+
return 1
|
|
164
|
+
if ib < len(b) and b[ib].isdigit():
|
|
165
|
+
return -1
|
|
166
|
+
if first_diff:
|
|
167
|
+
return -1 if first_diff < 0 else 1
|
|
168
|
+
return 0
|
|
169
|
+
|
|
170
|
+
|
|
171
|
+
def _dpkg_split(version: str) -> tuple[int, str, str]:
|
|
172
|
+
epoch = 0
|
|
173
|
+
if ":" in version:
|
|
174
|
+
head, version = version.split(":", 1)
|
|
175
|
+
epoch = int(head) if head.isdigit() else 0
|
|
176
|
+
upstream, _, revision = version.rpartition("-") if "-" in version else (version, "", "0")
|
|
177
|
+
return epoch, upstream or version, revision or "0"
|
|
178
|
+
|
|
179
|
+
|
|
180
|
+
def compare_dpkg(a: str, b: str) -> int:
|
|
181
|
+
ea, ua, ra = _dpkg_split(a.strip())
|
|
182
|
+
eb, ub, rb = _dpkg_split(b.strip())
|
|
183
|
+
if ea != eb:
|
|
184
|
+
return -1 if ea < eb else 1
|
|
185
|
+
return _dpkg_part(ua, ub) or _dpkg_part(ra, rb)
|
|
186
|
+
|
|
187
|
+
|
|
188
|
+
# --------------------------------------------------------------------------- dispatch
|
|
189
|
+
|
|
190
|
+
|
|
191
|
+
def compare(ecosystem: str | None, a: str, b: str) -> int:
|
|
192
|
+
eco = (ecosystem or "").split(":", 1)[0].strip().lower()
|
|
193
|
+
if eco in DPKG_ECOSYSTEMS:
|
|
194
|
+
return compare_dpkg(a, b)
|
|
195
|
+
if eco in SEMVER_ECOSYSTEMS and parse_semver(a) and parse_semver(b):
|
|
196
|
+
return compare_semver(a, b)
|
|
197
|
+
return compare_generic(a, b)
|
|
198
|
+
|
|
199
|
+
|
|
200
|
+
def sort_key(ecosystem: str | None):
|
|
201
|
+
return functools.cmp_to_key(lambda x, y: compare(ecosystem, x, y))
|
|
202
|
+
|
|
203
|
+
|
|
204
|
+
def cycle_of(version: str, parts: int = 2) -> str:
|
|
205
|
+
numbers = re.findall(r"\d+", version.split("+", 1)[0])
|
|
206
|
+
return ".".join(numbers[:parts]) if numbers else version
|
|
207
|
+
|
|
208
|
+
|
|
209
|
+
# --------------------------------------------------------------------------- npm ranges
|
|
210
|
+
|
|
211
|
+
|
|
212
|
+
def _expand_partial(op: str, version: str) -> list[tuple[str, str]]:
|
|
213
|
+
raw = version.strip().lstrip("v=")
|
|
214
|
+
parts = raw.split("-", 1)[0].split(".")
|
|
215
|
+
wild = [p in ("x", "X", "*", "") for p in parts] + [True] * (3 - len(parts))
|
|
216
|
+
nums = [int(p) if p.isdigit() else 0 for p in parts] + [0] * (3 - len(parts))
|
|
217
|
+
pre = "-" + raw.split("-", 1)[1] if "-" in raw else ""
|
|
218
|
+
if raw in ("", "*", "x", "X"):
|
|
219
|
+
return [(">=", "0.0.0")]
|
|
220
|
+
if op == "^":
|
|
221
|
+
if wild[1]:
|
|
222
|
+
return [(">=", f"{nums[0]}.0.0{pre}"), ("<", f"{nums[0] + 1}.0.0-0")]
|
|
223
|
+
if nums[0] > 0:
|
|
224
|
+
return [(">=", f"{nums[0]}.{nums[1]}.{nums[2]}{pre}"), ("<", f"{nums[0] + 1}.0.0-0")]
|
|
225
|
+
if nums[1] > 0 or wild[2]:
|
|
226
|
+
return [(">=", f"0.{nums[1]}.{nums[2]}{pre}"), ("<", f"0.{nums[1] + 1}.0-0")]
|
|
227
|
+
return [(">=", f"0.0.{nums[2]}{pre}"), ("<", f"0.0.{nums[2] + 1}-0")]
|
|
228
|
+
if op == "~":
|
|
229
|
+
if wild[1]:
|
|
230
|
+
return [(">=", f"{nums[0]}.0.0{pre}"), ("<", f"{nums[0] + 1}.0.0-0")]
|
|
231
|
+
return [(">=", f"{nums[0]}.{nums[1]}.{nums[2]}{pre}"), ("<", f"{nums[0]}.{nums[1] + 1}.0-0")]
|
|
232
|
+
if wild[0]:
|
|
233
|
+
return [(">=", "0.0.0")]
|
|
234
|
+
if wild[1] or wild[2]:
|
|
235
|
+
low = f"{nums[0]}.{nums[1] if not wild[1] else 0}.0"
|
|
236
|
+
high = f"{nums[0] + 1}.0.0-0" if wild[1] else f"{nums[0]}.{nums[1] + 1}.0-0"
|
|
237
|
+
if op in ("", "="):
|
|
238
|
+
return [(">=", low), ("<", high)]
|
|
239
|
+
if op == ">":
|
|
240
|
+
return [(">=", high)]
|
|
241
|
+
if op == "<=":
|
|
242
|
+
return [("<", high)]
|
|
243
|
+
if op == ">=":
|
|
244
|
+
return [(">=", low)]
|
|
245
|
+
if op == "<":
|
|
246
|
+
return [("<", low)]
|
|
247
|
+
return [(op or "=", f"{nums[0]}.{nums[1]}.{nums[2]}{pre}")]
|
|
248
|
+
|
|
249
|
+
|
|
250
|
+
def _comparators(clause: str) -> list[tuple[str, str]]:
|
|
251
|
+
clause = clause.strip()
|
|
252
|
+
hyphen = re.match(r"^(\S+)\s+-\s+(\S+)$", clause)
|
|
253
|
+
if hyphen:
|
|
254
|
+
low = _expand_partial(">=", hyphen.group(1))
|
|
255
|
+
high_raw = hyphen.group(2)
|
|
256
|
+
parts = high_raw.split(".")
|
|
257
|
+
high = _expand_partial("<=", high_raw) if len(parts) < 3 or any(p in ("x", "X", "*") for p in parts) else [("<=", high_raw)]
|
|
258
|
+
return low + high
|
|
259
|
+
out: list[tuple[str, str]] = []
|
|
260
|
+
for token in re.findall(r"(\^|~>?|>=|<=|>|<|=)?\s*([^\s<>=^~]+)", clause):
|
|
261
|
+
op, version = token
|
|
262
|
+
out.extend(_expand_partial(op.replace("~>", "~"), version))
|
|
263
|
+
return out or [(">=", "0.0.0")]
|
|
264
|
+
|
|
265
|
+
|
|
266
|
+
def satisfies(version: str, range_text: str) -> bool:
|
|
267
|
+
"""npm semver range check (no include-prerelease)."""
|
|
268
|
+
if parse_semver(version) is None:
|
|
269
|
+
return False
|
|
270
|
+
for clause in (range_text or "").split("||"):
|
|
271
|
+
comps = _comparators(clause)
|
|
272
|
+
ok = True
|
|
273
|
+
for op, bound in comps:
|
|
274
|
+
c = compare_semver(version, bound)
|
|
275
|
+
if (op == ">=" and c < 0) or (op == ">" and c <= 0) or (op == "<" and c >= 0) or (op == "<=" and c > 0) or (op == "=" and c != 0):
|
|
276
|
+
ok = False
|
|
277
|
+
break
|
|
278
|
+
if ok:
|
|
279
|
+
# Pre-release versions only match a clause that names the same [major, minor, patch] tuple with a pre-release.
|
|
280
|
+
pv = parse_semver(version)
|
|
281
|
+
if pv and pv[3] and not any(parse_semver(b) and parse_semver(b)[3] and parse_semver(b)[:3] == pv[:3] and not b.endswith("-0") for _, b in comps):
|
|
282
|
+
continue
|
|
283
|
+
return True
|
|
284
|
+
return False
|
|
285
|
+
|
|
286
|
+
|
|
287
|
+
def lowest_satisfying(candidates: Iterable[str], range_text: str) -> str | None:
|
|
288
|
+
matching = [v for v in candidates if satisfies(v, range_text)]
|
|
289
|
+
return min(matching, key=sort_key("npm")) if matching else None
|
|
290
|
+
|
|
291
|
+
|
|
292
|
+
# --------------------------------------------------------------------------- OSV and CVE ranges
|
|
293
|
+
|
|
294
|
+
|
|
295
|
+
def osv_affected(version: str, affected: dict, ecosystem: str | None = None) -> tuple[bool, list[str], str]:
|
|
296
|
+
"""Return (affected, fixed versions, how) for one OSV ``affected[]`` entry.
|
|
297
|
+
|
|
298
|
+
``how`` is ``versions`` (exact list), ``range`` (SEMVER/ECOSYSTEM events) or
|
|
299
|
+
``none``. GIT ranges are ignored (no commit information here).
|
|
300
|
+
"""
|
|
301
|
+
eco = ecosystem or (affected.get("package") or {}).get("ecosystem")
|
|
302
|
+
fixed_all: list[str] = []
|
|
303
|
+
hit = False
|
|
304
|
+
how = "none"
|
|
305
|
+
if version in (affected.get("versions") or []):
|
|
306
|
+
hit, how = True, "versions"
|
|
307
|
+
for rng in affected.get("ranges") or []:
|
|
308
|
+
if rng.get("type") not in ("SEMVER", "ECOSYSTEM"):
|
|
309
|
+
continue
|
|
310
|
+
rng_eco = "semver" if rng.get("type") == "SEMVER" else eco
|
|
311
|
+
events = rng.get("events") or []
|
|
312
|
+
fixed_all += [e["fixed"] for e in events if "fixed" in e]
|
|
313
|
+
state = False
|
|
314
|
+
for event in sorted(events, key=lambda e: _event_key(e, rng_eco)):
|
|
315
|
+
if "introduced" in event:
|
|
316
|
+
intro = event["introduced"]
|
|
317
|
+
if intro == "0" or compare(rng_eco, version, intro) >= 0:
|
|
318
|
+
state = True
|
|
319
|
+
elif "fixed" in event:
|
|
320
|
+
if compare(rng_eco, version, event["fixed"]) >= 0:
|
|
321
|
+
state = False
|
|
322
|
+
elif "last_affected" in event:
|
|
323
|
+
if compare(rng_eco, version, event["last_affected"]) > 0:
|
|
324
|
+
state = False
|
|
325
|
+
elif "limit" in event:
|
|
326
|
+
if compare(rng_eco, version, event["limit"]) >= 0:
|
|
327
|
+
state = False
|
|
328
|
+
if state:
|
|
329
|
+
hit, how = True, how if how == "versions" else "range"
|
|
330
|
+
return hit, sorted(set(fixed_all), key=sort_key(eco)), how
|
|
331
|
+
|
|
332
|
+
|
|
333
|
+
def _event_key(event: dict, ecosystem: str | None):
|
|
334
|
+
value = next(iter(v for k, v in event.items() if k in ("introduced", "fixed", "last_affected", "limit")), "0")
|
|
335
|
+
if value == "0":
|
|
336
|
+
return (0, sort_key(ecosystem)("0"))
|
|
337
|
+
return (1, sort_key(ecosystem)(value))
|
|
338
|
+
|
|
339
|
+
|
|
340
|
+
def cve_versions_affected(version: str, versions: list[dict], default_status: str = "unknown",
|
|
341
|
+
ecosystem: str | None = None) -> tuple[bool | None, list[str]]:
|
|
342
|
+
"""Evaluate CVE JSON 5 ``affected[].versions[]``; returns (affected or None if unknown, fix bounds)."""
|
|
343
|
+
fixes: list[str] = [str(e["lessThan"]) for e in versions or [] if e.get("lessThan") and e.get("lessThan") != "*"]
|
|
344
|
+
result: bool | None = None
|
|
345
|
+
for entry in versions or []:
|
|
346
|
+
status = entry.get("status", default_status)
|
|
347
|
+
start = str(entry.get("version", "")).strip()
|
|
348
|
+
less = entry.get("lessThan")
|
|
349
|
+
less_eq = entry.get("lessThanOrEqual")
|
|
350
|
+
if less or less_eq:
|
|
351
|
+
lower_ok = start in ("", "0", "*", "n/a", "unspecified") or compare(ecosystem, version, start) >= 0
|
|
352
|
+
if less in ("*",):
|
|
353
|
+
upper_ok = True
|
|
354
|
+
elif less:
|
|
355
|
+
upper_ok = compare(ecosystem, version, str(less)) < 0
|
|
356
|
+
else:
|
|
357
|
+
upper_ok = less_eq == "*" or compare(ecosystem, version, str(less_eq)) <= 0
|
|
358
|
+
if lower_ok and upper_ok:
|
|
359
|
+
result = status == "affected"
|
|
360
|
+
if result:
|
|
361
|
+
return True, sorted(set(fixes), key=sort_key(ecosystem))
|
|
362
|
+
elif start and start not in ("*", "n/a", "unspecified", "0"):
|
|
363
|
+
if compare(ecosystem, version, start) == 0:
|
|
364
|
+
result = status == "affected"
|
|
365
|
+
if result:
|
|
366
|
+
return True, fixes
|
|
367
|
+
if result is None and default_status == "affected" and not versions:
|
|
368
|
+
return None, fixes
|
|
369
|
+
return result, sorted(set(fixes), key=sort_key(ecosystem))
|
|
370
|
+
|
|
371
|
+
|
|
372
|
+
def recommended_fix(version: str, fixed: Iterable[str], ecosystem: str | None = None) -> str | None:
|
|
373
|
+
"""Smallest fixed version above the current one, preferring the same major.minor line."""
|
|
374
|
+
newer = sorted((f for f in fixed if compare(ecosystem, f, version) > 0), key=sort_key(ecosystem))
|
|
375
|
+
if not newer:
|
|
376
|
+
return None
|
|
377
|
+
line = cycle_of(version)
|
|
378
|
+
same_line = [f for f in newer if cycle_of(f) == line]
|
|
379
|
+
major = cycle_of(version, 1)
|
|
380
|
+
same_major = [f for f in newer if cycle_of(f, 1) == major]
|
|
381
|
+
return (same_line or same_major or newer)[0]
|