@topy-ai/maggie 0.7.55 → 0.7.57
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README-zh-TW.md +7 -3
- package/README.md +42 -8
- package/bin/maggie.js +4 -0
- package/bundled-contracts/maggie-deployment/readiness-v1.schema.json +2 -1
- package/bundled-contracts/maggie-security/finding-example-v1.json +89 -0
- package/bundled-contracts/maggie-security/finding-v1.schema.json +63 -0
- package/bundled-contracts/maggie-security/findings-set-example-v1.json +148 -0
- package/bundled-contracts/maggie-security/findings-set-v1.schema.json +32 -0
- package/bundled-contracts/maggie-security/protect-plan-v1.schema.json +28 -0
- package/bundled-contracts/maggie-security/report-v1.schema.json +29 -0
- package/bundled-contracts/maggie-security/stack-example-v1.json +66 -0
- package/bundled-contracts/maggie-security/stack-upload-v1.schema.json +22 -0
- package/bundled-contracts/maggie-security/stack-v1.schema.json +37 -0
- package/bundled-contracts/maggie-security/tools-lock-v1.schema.json +36 -0
- package/bundled-skills/catalog.json +4 -0
- package/bundled-skills/maggie-security/SKILL.md +173 -0
- package/bundled-skills/maggie-security/agents/openai.yaml +4 -0
- package/bundled-skills/maggie-security/data/LICENSE-NOTICE.md +24 -0
- package/bundled-skills/maggie-security/data/asvs-map.json +209 -0
- package/bundled-skills/maggie-security/data/fail2ban/filter.d/maggie-wordpress.conf +6 -0
- package/bundled-skills/maggie-security/data/semgrep/LICENSE-NOTICE.md +5 -0
- package/bundled-skills/maggie-security/data/semgrep/VERSION +1 -0
- package/bundled-skills/maggie-security/data/semgrep/rules/dockerfile.yaml +14 -0
- package/bundled-skills/maggie-security/data/semgrep/rules/generic.yaml +26 -0
- package/bundled-skills/maggie-security/data/semgrep/rules/js.yaml +216 -0
- package/bundled-skills/maggie-security/data/semgrep/rules/php-debug-ok.yaml +17 -0
- package/bundled-skills/maggie-security/data/semgrep/rules/php-debug.yaml +17 -0
- package/bundled-skills/maggie-security/data/semgrep/rules/php.yaml +101 -0
- package/bundled-skills/maggie-security/data/semgrep/tests/dockerfile.dockerfile +10 -0
- package/bundled-skills/maggie-security/data/semgrep/tests/generic.astro +11 -0
- package/bundled-skills/maggie-security/data/semgrep/tests/generic.conf +11 -0
- package/bundled-skills/maggie-security/data/semgrep/tests/js.jsx +95 -0
- package/bundled-skills/maggie-security/data/semgrep/tests/php-debug-ok.php +5 -0
- package/bundled-skills/maggie-security/data/semgrep/tests/php-debug.php +3 -0
- package/bundled-skills/maggie-security/data/semgrep/tests/php.php +55 -0
- package/bundled-skills/maggie-security/evals/evals.json +97 -0
- package/bundled-skills/maggie-security/references/commands.md +125 -0
- package/bundled-skills/maggie-security/references/cve-response.md +24 -0
- package/bundled-skills/maggie-security/references/explaining-findings.md +57 -0
- package/bundled-skills/maggie-security/references/findings.md +39 -0
- package/bundled-skills/maggie-security/references/ownership-and-consent.md +29 -0
- package/bundled-skills/maggie-security/references/site-types.md +22 -0
- package/bundled-skills/maggie-security/references/tools-and-licences.md +16 -0
- package/bundled-skills/maggie-security/references/workflows.md +23 -0
- package/bundled-skills/maggie-security/registry/security-tools.registry.json +654 -0
- package/bundled-skills/maggie-security/routines.example.json +27 -0
- package/bundled-tools/clis/maggie_deployment_readiness.py +17 -2
- package/bundled-tools/clis/maggie_security.py +342 -0
- package/bundled-tools/clis/maggie_security_agent.py +326 -0
- package/bundled-tools/clis/maggie_security_code.py +650 -0
- package/bundled-tools/clis/maggie_security_core.py +1512 -0
- package/bundled-tools/clis/maggie_security_deps.py +560 -0
- package/bundled-tools/clis/maggie_security_health.py +415 -0
- package/bundled-tools/clis/maggie_security_host.py +511 -0
- package/bundled-tools/clis/maggie_security_hosted.py +115 -0
- package/bundled-tools/clis/maggie_security_intel.py +1294 -0
- package/bundled-tools/clis/maggie_security_inventory.py +1023 -0
- package/bundled-tools/clis/maggie_security_logs.py +777 -0
- package/bundled-tools/clis/maggie_security_mcp.py +484 -0
- package/bundled-tools/clis/maggie_security_mirror.py +122 -0
- package/bundled-tools/clis/maggie_security_protect.py +519 -0
- package/bundled-tools/clis/maggie_security_report.py +506 -0
- package/bundled-tools/clis/maggie_security_standards.py +214 -0
- package/bundled-tools/clis/maggie_security_versions.py +381 -0
- package/bundled-tools/clis/maggie_security_web.py +1247 -0
- package/bundled-tools/clis/security_data/eol-snapshot.json +1 -0
- package/bundled-tools/clis/security_data/gitleaks-rules.json +3812 -0
- package/bundled-tools/clis/security_data/maltrail-lists.json +1 -0
- package/bundled-tools/clis/security_data/public-suffix-icann.txt +7374 -0
- package/bundled-tools/clis/security_data/retire-extractors.json +695 -0
- package/bundled-tools/clis/security_data/sigma-web.json +6514 -0
- package/bundled-tools/clis/security_data/tools.lock.json +358 -0
- package/package.json +1 -1
|
@@ -0,0 +1,1023 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""Stack inventory for ``maggie security inventory`` (PRD 01 section 6.1).
|
|
3
|
+
|
|
4
|
+
Builds ``maggie-security-stack.v1`` from what is on disk and, optionally,
|
|
5
|
+
from read-only host evidence. Covers the popular stacks:
|
|
6
|
+
|
|
7
|
+
- JavaScript: package-lock.json (v1-v3), npm-shrinkwrap.json, pnpm-lock.yaml,
|
|
8
|
+
yarn.lock (v1 and berry), bun.lock
|
|
9
|
+
- PHP: composer.lock
|
|
10
|
+
- Python: requirements*.txt (pinned), Pipfile.lock, poetry.lock, uv.lock, pdm.lock
|
|
11
|
+
- Java/Kotlin: pom.xml (with properties), gradle.lockfile, gradle/verification-metadata.xml
|
|
12
|
+
- Go: go.mod (+ go directive), Ruby: Gemfile.lock (+ RUBY VERSION), Rust: Cargo.lock,
|
|
13
|
+
.NET: packages.lock.json, *.csproj PackageReference, global.json
|
|
14
|
+
- WordPress core/plugins/themes from files
|
|
15
|
+
- Containers: Dockerfile FROM, docker-compose/compose image lines -> images and
|
|
16
|
+
the runtime/database/server they carry (python, node, php, java, mysql,
|
|
17
|
+
postgres, redis, mongo, nginx, httpd, tomcat, ...)
|
|
18
|
+
- Declared runtimes: .nvmrc, .node-version, .python-version, runtime.txt,
|
|
19
|
+
.ruby-version, .tool-versions, .java-version, package.json engines
|
|
20
|
+
- Hosts: OS release, OS packages (dpkg / rpm / apk), and service versions
|
|
21
|
+
(nginx, apache, php, node, python, java, go, mysql/mariadb, postgresql,
|
|
22
|
+
redis, mongodb, memcached, docker, openssl) from the read-only collector
|
|
23
|
+
printed by ``--print-collector`` (the runtime-inventory collector plus
|
|
24
|
+
version probes), or from ``maggie deployment runtime-inventory`` JSON.
|
|
25
|
+
|
|
26
|
+
Parsers are stdlib-only and tolerant: an unreadable file becomes a degraded
|
|
27
|
+
source, never a crash. Lockfile test data under fixtures/security/lockfiles
|
|
28
|
+
comes from google/osv-scanner testdata (Apache-2.0).
|
|
29
|
+
"""
|
|
30
|
+
|
|
31
|
+
from __future__ import annotations
|
|
32
|
+
|
|
33
|
+
import json
|
|
34
|
+
import re
|
|
35
|
+
import xml.etree.ElementTree as ET
|
|
36
|
+
from pathlib import Path
|
|
37
|
+
|
|
38
|
+
import maggie_security_core as core
|
|
39
|
+
import maggie_security_versions as versions
|
|
40
|
+
from maggie_host_runtime import COLLECTOR as RUNTIME_COLLECTOR, parse_os, parse_sections
|
|
41
|
+
|
|
42
|
+
SKIP_DIRS = {"node_modules", ".git", "vendor", ".venv", "venv", "__pycache__", "dist", "build", ".next", ".astro", ".output",
|
|
43
|
+
"target", ".gradle", ".maggie", ".wrangler", "wp-admin", "wp-includes", "bower_components", ".cache"}
|
|
44
|
+
MAX_DEPTH = 4
|
|
45
|
+
|
|
46
|
+
# Default CVE product allowlist per site type (PRD 01 section 6.1); runtimes
|
|
47
|
+
# and images found are added automatically.
|
|
48
|
+
WATCH_DEFAULTS = {
|
|
49
|
+
"node-app": ["node", "nginx", "openssl"], "astro": ["node", "astro"], "workers": [],
|
|
50
|
+
"wordpress": ["wordpress", "php", "nginx", "apache", "mysql", "mariadb", "openssl"],
|
|
51
|
+
"php-app": ["php", "nginx", "apache", "mysql", "mariadb", "openssl"], "static": ["nginx"], "unknown": [],
|
|
52
|
+
"python-app": ["python", "nginx", "postgresql", "redis", "openssl"], "java-app": ["java", "tomcat", "nginx", "postgresql", "mysql"],
|
|
53
|
+
"go-app": ["go", "nginx", "postgresql"], "ruby-app": ["ruby", "nginx", "postgresql", "redis"], "dotnet-app": ["dotnet", "mssqlserver", "nginx"],
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
# Image repository -> runtime/service name (endoflife.date product mapping lives in the intel module).
|
|
57
|
+
IMAGE_RUNTIMES = {
|
|
58
|
+
"node": "node", "python": "python", "php": "php", "wordpress": "wordpress", "ruby": "ruby", "golang": "go",
|
|
59
|
+
"openjdk": "java", "eclipse-temurin": "java", "amazoncorretto": "java", "ibm-semeru-runtimes": "java", "maven": "java",
|
|
60
|
+
"tomcat": "tomcat", "mysql": "mysql", "mariadb": "mariadb", "postgres": "postgresql", "postgresql": "postgresql",
|
|
61
|
+
"redis": "redis", "valkey": "valkey", "mongo": "mongodb", "mongodb": "mongodb", "memcached": "memcached",
|
|
62
|
+
"nginx": "nginx", "httpd": "apache", "caddy": "caddy", "haproxy": "haproxy", "traefik": "traefik",
|
|
63
|
+
"elasticsearch": "elasticsearch", "opensearch": "opensearch", "rabbitmq": "rabbitmq", "mcr.microsoft.com/dotnet/aspnet": "dotnet",
|
|
64
|
+
"mcr.microsoft.com/dotnet/runtime": "dotnet", "mcr.microsoft.com/dotnet/sdk": "dotnet", "mcr.microsoft.com/mssql/server": "mssqlserver",
|
|
65
|
+
"ubuntu": "ubuntu", "debian": "debian", "alpine": "alpine", "rockylinux": "rocky-linux", "almalinux": "almalinux",
|
|
66
|
+
"amazonlinux": "amazon-linux", "centos": "centos", "fedora": "fedora", "bun": "bun", "denoland/deno": "deno",
|
|
67
|
+
"rust": "rust", "kong": "kong", "keycloak": "keycloak", "grafana/grafana": "grafana", "sonarqube": "sonarqube",
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
# Read-only collector: the runtime-inventory sections plus version probes and
|
|
71
|
+
# OS package lists. Every probe is a version query or a read of a package
|
|
72
|
+
# database; nothing is written on the host.
|
|
73
|
+
SECURITY_COLLECTOR = RUNTIME_COLLECTOR + r"""
|
|
74
|
+
s nginx-v; nginx -v 2>&1 | head -1
|
|
75
|
+
s apache-v; (apache2 -v 2>/dev/null || httpd -v 2>/dev/null) | head -1
|
|
76
|
+
s node-v; node --version 2>/dev/null
|
|
77
|
+
s python-v; (python3 --version 2>&1 || python --version 2>&1) | head -1
|
|
78
|
+
s java-v; java -version 2>&1 | head -1
|
|
79
|
+
s go-v; go version 2>/dev/null
|
|
80
|
+
s ruby-v; ruby --version 2>/dev/null
|
|
81
|
+
s dotnet-v; dotnet --list-runtimes 2>/dev/null | head -5
|
|
82
|
+
s mysql-v; (mysqld --version 2>/dev/null || mysql --version 2>/dev/null || mariadbd --version 2>/dev/null) | head -1
|
|
83
|
+
s postgres-v; (postgres --version 2>/dev/null || psql --version 2>/dev/null || ls /usr/lib/postgresql 2>/dev/null) | head -3
|
|
84
|
+
s redis-v; redis-server --version 2>/dev/null
|
|
85
|
+
s mongodb-v; mongod --version 2>/dev/null | head -1
|
|
86
|
+
s memcached-v; memcached -V 2>/dev/null
|
|
87
|
+
s openssl-v; openssl version 2>/dev/null
|
|
88
|
+
s docker-v; docker --version 2>/dev/null
|
|
89
|
+
s tomcat-v; ls -d /opt/tomcat* /usr/share/tomcat* 2>/dev/null | head -3
|
|
90
|
+
s dpkg; dpkg-query -W -f='${Package}\t${Version}\t${Status}\n' 2>/dev/null
|
|
91
|
+
s rpm; rpm -qa --qf '%{NAME}\t%{EPOCHNUM}:%{VERSION}-%{RELEASE}\n' 2>/dev/null
|
|
92
|
+
s apk; apk info -v 2>/dev/null
|
|
93
|
+
s kernel; uname -r
|
|
94
|
+
"""
|
|
95
|
+
|
|
96
|
+
SERVICE_PROBES = {
|
|
97
|
+
"nginx-v": ("nginx", r"nginx/(\d+\.\d+\.\d+)"),
|
|
98
|
+
"apache-v": ("apache", r"Apache/(\d+\.\d+\.\d+)"),
|
|
99
|
+
"node-v": ("node", r"v?(\d+\.\d+\.\d+)"),
|
|
100
|
+
"python-v": ("python", r"Python (\d+\.\d+\.\d+)"),
|
|
101
|
+
"java-v": ("java", r"version \"(\d+(?:\.\d+)*)"),
|
|
102
|
+
"go-v": ("go", r"go(\d+\.\d+(?:\.\d+)?)"),
|
|
103
|
+
"ruby-v": ("ruby", r"ruby (\d+\.\d+\.\d+)"),
|
|
104
|
+
"dotnet-v": ("dotnet", r"Microsoft\.\w+\.App (\d+\.\d+\.\d+)"),
|
|
105
|
+
"mysql-v": (None, r"(?:Ver |Distrib )(\d+\.\d+\.\d+)(?:-(MariaDB))?"),
|
|
106
|
+
"postgres-v": ("postgresql", r"(?:PostgreSQL\)? |^)(\d+(?:\.\d+)?)"),
|
|
107
|
+
"redis-v": ("redis", r"v=(\d+\.\d+\.\d+)"),
|
|
108
|
+
"mongodb-v": ("mongodb", r"v(\d+\.\d+\.\d+)"),
|
|
109
|
+
"memcached-v": ("memcached", r"memcached (\d+\.\d+\.\d+)"),
|
|
110
|
+
"openssl-v": ("openssl", r"OpenSSL (\d+\.\d+\.\d+[a-z]?)"),
|
|
111
|
+
"docker-v": ("docker-engine", r"version (\d+\.\d+\.\d+)"),
|
|
112
|
+
"php-v": ("php", r"PHP (\d+\.\d+\.\d+)"),
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
# Products whose endoflife.date release cycle is the major version only.
|
|
116
|
+
MAJOR_CYCLES = {"node", "java", "debian", "rocky-linux", "almalinux", "rhel", "centos", "fedora", "docker-engine", "amazon-linux",
|
|
117
|
+
"elasticsearch", "opensearch", "deno", "bun"}
|
|
118
|
+
|
|
119
|
+
OS_ECOSYSTEM = {"debian": "Debian", "ubuntu": "Ubuntu", "alpine": "Alpine", "rhel": "Red Hat", "rocky": "Rocky Linux",
|
|
120
|
+
"almalinux": "AlmaLinux", "opensuse-leap": "openSUSE", "sles": "SUSE", "amzn": "Amazon Linux"}
|
|
121
|
+
|
|
122
|
+
|
|
123
|
+
class Inventory:
|
|
124
|
+
def __init__(self, project: Path, site: str, site_type: str) -> None:
|
|
125
|
+
self.project = project
|
|
126
|
+
self.site = site
|
|
127
|
+
self.site_type = site_type
|
|
128
|
+
self.sources: list[dict] = []
|
|
129
|
+
self.components: list[dict] = []
|
|
130
|
+
self.runtimes: list[dict] = []
|
|
131
|
+
self.images: list[dict] = []
|
|
132
|
+
self.hosts: list[dict] = []
|
|
133
|
+
self.wordpress: dict | None = None
|
|
134
|
+
self.problems: list[str] = []
|
|
135
|
+
|
|
136
|
+
# ------------------------------------------------------------------ helpers
|
|
137
|
+
|
|
138
|
+
def rel(self, path: Path) -> str:
|
|
139
|
+
try:
|
|
140
|
+
return str(path.resolve().relative_to(self.project.resolve()))
|
|
141
|
+
except ValueError:
|
|
142
|
+
return path.name
|
|
143
|
+
|
|
144
|
+
def add(self, ecosystem: str, name: str, version: str, path: Path, scope: str = "runtime", direct: bool | None = None) -> None:
|
|
145
|
+
version = (version or "").strip()
|
|
146
|
+
name = (name or "").strip()
|
|
147
|
+
if not name or not version or version.startswith(("file:", "link:", "workspace:", "git", "http", "github:", "portal:")):
|
|
148
|
+
return
|
|
149
|
+
if ecosystem == "npm" and version.startswith("npm:"):
|
|
150
|
+
alias = version[4:]
|
|
151
|
+
name, _, version = alias.rpartition("@")
|
|
152
|
+
self.components.append({"purl": core.make_purl(ecosystem, name, version), "ecosystem": ecosystem, "name": name,
|
|
153
|
+
"version": version, "scope": scope, "direct": direct, "location": self.rel(path)})
|
|
154
|
+
|
|
155
|
+
def add_runtime(self, name: str, version: str, source: str, host: str | None = None, **extra: str) -> None:
|
|
156
|
+
version = (version or "").strip().lstrip("v")
|
|
157
|
+
if not name or not version or not re.match(r"\d", version):
|
|
158
|
+
return
|
|
159
|
+
cycle = versions.cycle_of(version, 1 if name in MAJOR_CYCLES or (name == "postgresql" and not version.startswith("9.")) else 2)
|
|
160
|
+
entry = {"name": name, "version": version, "cycle": cycle, "host": host, "source": source}
|
|
161
|
+
entry.update({k: v for k, v in extra.items() if v})
|
|
162
|
+
key = (name, version, host)
|
|
163
|
+
if not any((r["name"], r["version"], r.get("host")) == key for r in self.runtimes):
|
|
164
|
+
self.runtimes.append(entry)
|
|
165
|
+
|
|
166
|
+
def source(self, kind: str, path: Path | str) -> None:
|
|
167
|
+
rel = self.rel(path) if isinstance(path, Path) else path
|
|
168
|
+
if not any(s["path"] == rel and s["kind"] == kind for s in self.sources):
|
|
169
|
+
self.sources.append({"kind": kind, "path": rel})
|
|
170
|
+
|
|
171
|
+
def read(self, path: Path) -> str | None:
|
|
172
|
+
try:
|
|
173
|
+
return path.read_text(encoding="utf-8", errors="replace")
|
|
174
|
+
except OSError as error:
|
|
175
|
+
self.problems.append(f"{self.rel(path)}: {error}")
|
|
176
|
+
return None
|
|
177
|
+
|
|
178
|
+
# ------------------------------------------------------------------ discovery
|
|
179
|
+
|
|
180
|
+
def discover(self) -> list[Path]:
|
|
181
|
+
out: list[Path] = []
|
|
182
|
+
|
|
183
|
+
def walk(directory: Path, depth: int) -> None:
|
|
184
|
+
try:
|
|
185
|
+
entries = sorted(directory.iterdir())
|
|
186
|
+
except OSError:
|
|
187
|
+
return
|
|
188
|
+
for entry in entries:
|
|
189
|
+
if entry.is_dir():
|
|
190
|
+
if entry.name in SKIP_DIRS or (entry.name.startswith(".") and entry.name not in (".github",)) or depth >= MAX_DEPTH:
|
|
191
|
+
continue
|
|
192
|
+
walk(entry, depth + 1)
|
|
193
|
+
elif entry.is_file():
|
|
194
|
+
out.append(entry)
|
|
195
|
+
|
|
196
|
+
walk(self.project, 0)
|
|
197
|
+
return out
|
|
198
|
+
|
|
199
|
+
def scan_project(self) -> None:
|
|
200
|
+
files = self.discover()
|
|
201
|
+
names = {}
|
|
202
|
+
for path in files:
|
|
203
|
+
names.setdefault(path.name, []).append(path)
|
|
204
|
+
handled_npm_dirs: set[Path] = set()
|
|
205
|
+
for name in ("package-lock.json", "npm-shrinkwrap.json", "pnpm-lock.yaml", "yarn.lock", "bun.lock"):
|
|
206
|
+
for path in names.get(name, []):
|
|
207
|
+
if path.parent in handled_npm_dirs:
|
|
208
|
+
continue
|
|
209
|
+
handled_npm_dirs.add(path.parent)
|
|
210
|
+
self.safe(getattr(self, "parse_" + name.replace("-", "_").replace(".", "_")), path)
|
|
211
|
+
dispatch = {
|
|
212
|
+
"composer.lock": self.parse_composer_lock, "Pipfile.lock": self.parse_pipfile_lock, "poetry.lock": self.parse_toml_lock,
|
|
213
|
+
"uv.lock": self.parse_toml_lock, "pdm.lock": self.parse_toml_lock, "Cargo.lock": self.parse_toml_lock,
|
|
214
|
+
"pom.xml": self.parse_pom, "gradle.lockfile": self.parse_gradle_lock, "go.mod": self.parse_go_mod,
|
|
215
|
+
"Gemfile.lock": self.parse_gemfile_lock, "packages.lock.json": self.parse_nuget_lock, "global.json": self.parse_global_json,
|
|
216
|
+
"Dockerfile": self.parse_dockerfile, "Containerfile": self.parse_dockerfile,
|
|
217
|
+
".nvmrc": self.parse_version_file, ".node-version": self.parse_version_file, ".python-version": self.parse_version_file,
|
|
218
|
+
".ruby-version": self.parse_version_file, ".java-version": self.parse_version_file, "runtime.txt": self.parse_version_file,
|
|
219
|
+
".tool-versions": self.parse_tool_versions, "package.json": self.parse_package_json,
|
|
220
|
+
}
|
|
221
|
+
for name, parser in dispatch.items():
|
|
222
|
+
for path in names.get(name, []):
|
|
223
|
+
if name == "package.json" and path.parent != self.project:
|
|
224
|
+
continue
|
|
225
|
+
self.safe(parser, path)
|
|
226
|
+
for path in files:
|
|
227
|
+
lower = path.name.lower()
|
|
228
|
+
if re.fullmatch(r"requirements[\w.-]*\.(txt|in)", lower) and lower.endswith(".txt"):
|
|
229
|
+
self.safe(self.parse_requirements, path)
|
|
230
|
+
elif lower.endswith(".csproj") or lower.endswith(".fsproj") or lower == "directory.packages.props":
|
|
231
|
+
self.safe(self.parse_csproj, path)
|
|
232
|
+
elif re.fullmatch(r"(docker-)?compose[\w.-]*\.ya?ml", lower):
|
|
233
|
+
self.safe(self.parse_compose, path)
|
|
234
|
+
elif lower.startswith("dockerfile.") or lower.endswith(".dockerfile"):
|
|
235
|
+
self.safe(self.parse_dockerfile, path)
|
|
236
|
+
wp_root = self.find_wordpress()
|
|
237
|
+
if wp_root:
|
|
238
|
+
self.safe(self.parse_wordpress, wp_root)
|
|
239
|
+
self.scan_js_bundles()
|
|
240
|
+
|
|
241
|
+
def safe(self, parser, path: Path) -> None:
|
|
242
|
+
try:
|
|
243
|
+
parser(path)
|
|
244
|
+
except Exception as error: # noqa: BLE001 - a bad file degrades, never crashes
|
|
245
|
+
self.problems.append(f"{self.rel(path)}: could not parse ({type(error).__name__}: {error})")
|
|
246
|
+
|
|
247
|
+
# ------------------------------------------------------------------ JavaScript
|
|
248
|
+
|
|
249
|
+
def parse_package_lock_json(self, path: Path) -> None:
|
|
250
|
+
data = json.loads(self.read(path) or "{}")
|
|
251
|
+
self.source("lockfile", path)
|
|
252
|
+
root = (data.get("packages") or {}).get("") or {}
|
|
253
|
+
direct = set(root.get("dependencies") or {}) | set(root.get("devDependencies") or {}) | set(root.get("optionalDependencies") or {})
|
|
254
|
+
if data.get("packages"):
|
|
255
|
+
for key, meta in data["packages"].items():
|
|
256
|
+
if not key or "node_modules/" not in key or meta.get("link"):
|
|
257
|
+
continue
|
|
258
|
+
name = meta.get("name") or key.rsplit("node_modules/", 1)[1]
|
|
259
|
+
scope = "dev" if meta.get("dev") or meta.get("devOptional") else "runtime"
|
|
260
|
+
self.add("npm", name, meta.get("version", ""), path, scope, name in direct and key == f"node_modules/{name}")
|
|
261
|
+
else: # lockfileVersion 1
|
|
262
|
+
def walk(deps: dict, top: bool) -> None:
|
|
263
|
+
for name, meta in (deps or {}).items():
|
|
264
|
+
self.add("npm", name, meta.get("version", ""), path, "dev" if meta.get("dev") else "runtime", top)
|
|
265
|
+
walk(meta.get("dependencies") or {}, False)
|
|
266
|
+
walk(data.get("dependencies") or {}, True)
|
|
267
|
+
|
|
268
|
+
parse_npm_shrinkwrap_json = parse_package_lock_json
|
|
269
|
+
|
|
270
|
+
def parse_pnpm_lock_yaml(self, path: Path) -> None:
|
|
271
|
+
text = self.read(path) or ""
|
|
272
|
+
self.source("lockfile", path)
|
|
273
|
+
section = None
|
|
274
|
+
dev_names: set[str] = set()
|
|
275
|
+
direct: set[str] = set()
|
|
276
|
+
importer_kind = None
|
|
277
|
+
for line in text.splitlines():
|
|
278
|
+
if not line.strip() or line.lstrip().startswith("#"):
|
|
279
|
+
continue
|
|
280
|
+
indent = len(line) - len(line.lstrip())
|
|
281
|
+
if indent == 0:
|
|
282
|
+
section = line.rstrip(":").strip()
|
|
283
|
+
continue
|
|
284
|
+
if section == "importers":
|
|
285
|
+
stripped = line.strip()
|
|
286
|
+
if stripped in ("dependencies:", "devDependencies:", "optionalDependencies:"):
|
|
287
|
+
importer_kind = stripped[:-1]
|
|
288
|
+
elif indent in (6, 2) and stripped.endswith(":") and importer_kind:
|
|
289
|
+
name = stripped[:-1].strip("'\"")
|
|
290
|
+
direct.add(name)
|
|
291
|
+
if importer_kind == "devDependencies":
|
|
292
|
+
dev_names.add(name)
|
|
293
|
+
elif section in ("dependencies", "devDependencies") and indent == 2 and line.strip().endswith(":"):
|
|
294
|
+
name = line.strip()[:-1].strip("'\"")
|
|
295
|
+
direct.add(name)
|
|
296
|
+
if section == "devDependencies":
|
|
297
|
+
dev_names.add(name)
|
|
298
|
+
elif section == "packages" and indent == 2 and line.rstrip().endswith(":"):
|
|
299
|
+
key = line.strip()[:-1].strip("'\"").lstrip("/")
|
|
300
|
+
key = key.split("(", 1)[0]
|
|
301
|
+
match = re.match(r"^(@?[^@/]+(?:/[^@/]+)?)[@/](\d[^/_()]*)", key)
|
|
302
|
+
if match:
|
|
303
|
+
name, version = match.group(1), match.group(2)
|
|
304
|
+
self.add("npm", name, version, path, "dev" if name in dev_names else "runtime", name in direct)
|
|
305
|
+
|
|
306
|
+
def parse_yarn_lock(self, path: Path) -> None:
|
|
307
|
+
text = self.read(path) or ""
|
|
308
|
+
self.source("lockfile", path)
|
|
309
|
+
names: list[str] = []
|
|
310
|
+
for line in text.splitlines():
|
|
311
|
+
if not line.strip() or line.startswith("#"):
|
|
312
|
+
continue
|
|
313
|
+
if not line.startswith(" ") and line.rstrip().endswith(":"):
|
|
314
|
+
header = line.rstrip()[:-1]
|
|
315
|
+
names = []
|
|
316
|
+
for spec in header.split(","):
|
|
317
|
+
spec = spec.strip().strip('"')
|
|
318
|
+
if spec.startswith("__metadata"):
|
|
319
|
+
continue
|
|
320
|
+
name = spec[: spec.rfind("@")] if spec.rfind("@") > 0 else spec
|
|
321
|
+
if name and name not in names:
|
|
322
|
+
names.append(name)
|
|
323
|
+
else:
|
|
324
|
+
match = re.match(r'^\s+version:?\s+"?([^"\s]+)"?', line)
|
|
325
|
+
if match and names:
|
|
326
|
+
for name in names[:1]:
|
|
327
|
+
self.add("npm", name, match.group(1), path)
|
|
328
|
+
names = []
|
|
329
|
+
|
|
330
|
+
def parse_bun_lock(self, path: Path) -> None:
|
|
331
|
+
text = self.read(path) or ""
|
|
332
|
+
self.source("lockfile", path)
|
|
333
|
+
cleaned = re.sub(r",(\s*[}\]])", r"\1", text) # bun.lock is JSONC with trailing commas
|
|
334
|
+
data = json.loads(cleaned)
|
|
335
|
+
for entry in (data.get("packages") or {}).values():
|
|
336
|
+
if isinstance(entry, list) and entry and isinstance(entry[0], str):
|
|
337
|
+
spec = entry[0]
|
|
338
|
+
name, _, version = spec.rpartition("@")
|
|
339
|
+
if name and re.match(r"\d", version):
|
|
340
|
+
self.add("npm", name, version, path)
|
|
341
|
+
|
|
342
|
+
def parse_package_json(self, path: Path) -> None:
|
|
343
|
+
data = json.loads(self.read(path) or "{}")
|
|
344
|
+
engine = ((data.get("engines") or {}).get("node") or "").strip()
|
|
345
|
+
match = re.search(r"(\d+(?:\.\d+){0,2})", engine)
|
|
346
|
+
if match and not any(r["name"] == "node" for r in self.runtimes):
|
|
347
|
+
self.add_runtime("node", match.group(1) if match.group(1).count(".") else match.group(1) + ".0.0", "declared")
|
|
348
|
+
if not any(c["ecosystem"] == "npm" and c["location"].startswith(self.rel(path.parent)) for c in self.components):
|
|
349
|
+
# No lockfile: record exact pins from package.json at lower confidence.
|
|
350
|
+
for section, scope in (("dependencies", "runtime"), ("devDependencies", "dev")):
|
|
351
|
+
for name, spec in (data.get(section) or {}).items():
|
|
352
|
+
if isinstance(spec, str) and re.fullmatch(r"\d+\.\d+\.\d+(?:-[\w.]+)?", spec.strip()):
|
|
353
|
+
self.add("npm", name, spec.strip(), path, scope, True)
|
|
354
|
+
if data.get("dependencies") or data.get("devDependencies"):
|
|
355
|
+
self.source("manifest", path)
|
|
356
|
+
|
|
357
|
+
# ------------------------------------------------------------------ PHP / Python / Ruby / Rust / Go / .NET / Java
|
|
358
|
+
|
|
359
|
+
def parse_composer_lock(self, path: Path) -> None:
|
|
360
|
+
data = json.loads(self.read(path) or "{}")
|
|
361
|
+
self.source("lockfile", path)
|
|
362
|
+
manifest = path.parent / "composer.json"
|
|
363
|
+
direct: set[str] = set()
|
|
364
|
+
if manifest.is_file():
|
|
365
|
+
with_json = json.loads(manifest.read_text(encoding="utf-8") or "{}")
|
|
366
|
+
direct = {k.lower() for k in list((with_json.get("require") or {})) + list((with_json.get("require-dev") or {}))}
|
|
367
|
+
php = (with_json.get("require") or {}).get("php") or ""
|
|
368
|
+
for key, scope in (("packages", "runtime"), ("packages-dev", "dev")):
|
|
369
|
+
for pkg in data.get(key) or []:
|
|
370
|
+
version = str(pkg.get("version", "")).lstrip("v")
|
|
371
|
+
if version.startswith("dev-") or version.endswith("-dev"):
|
|
372
|
+
continue
|
|
373
|
+
self.add("Packagist", pkg.get("name", "").lower(), version, path, scope, pkg.get("name", "").lower() in direct)
|
|
374
|
+
|
|
375
|
+
def parse_requirements(self, path: Path) -> None:
|
|
376
|
+
text = self.read(path) or ""
|
|
377
|
+
found = False
|
|
378
|
+
scope = "dev" if re.search(r"(dev|test|lint|doc)", path.name) else "runtime"
|
|
379
|
+
for line in text.splitlines():
|
|
380
|
+
line = line.split("#", 1)[0].strip()
|
|
381
|
+
match = re.match(r"^([A-Za-z0-9][A-Za-z0-9._-]*)(?:\[[^\]]*\])?\s*===?\s*([^\s;,]+)", line)
|
|
382
|
+
if match:
|
|
383
|
+
self.add("PyPI", re.sub(r"[-_.]+", "-", match.group(1)).lower(), match.group(2), path, scope, True)
|
|
384
|
+
found = True
|
|
385
|
+
if found:
|
|
386
|
+
self.source("lockfile", path)
|
|
387
|
+
|
|
388
|
+
def parse_pipfile_lock(self, path: Path) -> None:
|
|
389
|
+
data = json.loads(self.read(path) or "{}")
|
|
390
|
+
self.source("lockfile", path)
|
|
391
|
+
for section, scope in (("default", "runtime"), ("develop", "dev")):
|
|
392
|
+
for name, meta in (data.get(section) or {}).items():
|
|
393
|
+
version = str((meta or {}).get("version", "")).lstrip("=")
|
|
394
|
+
self.add("PyPI", re.sub(r"[-_.]+", "-", name).lower(), version, path, scope)
|
|
395
|
+
|
|
396
|
+
def parse_toml_lock(self, path: Path) -> None:
|
|
397
|
+
"""[[package]] tables with name/version (poetry.lock, uv.lock, pdm.lock, Cargo.lock)."""
|
|
398
|
+
text = self.read(path) or ""
|
|
399
|
+
ecosystem = "crates.io" if path.name == "Cargo.lock" else "PyPI"
|
|
400
|
+
self.source("lockfile", path)
|
|
401
|
+
for block in re.split(r"^\[\[package\]\]\s*$", text, flags=re.M)[1:]:
|
|
402
|
+
body = re.split(r"^\[", block, maxsplit=1, flags=re.M)[0]
|
|
403
|
+
name = re.search(r'^name\s*=\s*"([^"]+)"', body, re.M)
|
|
404
|
+
version = re.search(r'^version\s*=\s*"([^"]+)"', body, re.M)
|
|
405
|
+
if not name or not version:
|
|
406
|
+
continue
|
|
407
|
+
source = re.search(r"^source\s*=\s*(.+)$", body, re.M)
|
|
408
|
+
if source and ("virtual" in source.group(1) or "editable" in source.group(1) or "path" in source.group(1)):
|
|
409
|
+
continue
|
|
410
|
+
if ecosystem == "crates.io" and not source:
|
|
411
|
+
continue # workspace crate, not from a registry
|
|
412
|
+
category = re.search(r'^category\s*=\s*"([^"]+)"', body, re.M)
|
|
413
|
+
groups = re.search(r'^groups\s*=\s*\[([^\]]*)\]', body, re.M)
|
|
414
|
+
dev = (category and category.group(1) == "dev") or (groups and "main" not in groups.group(1) and "default" not in groups.group(1))
|
|
415
|
+
pkg = name.group(1) if ecosystem == "crates.io" else re.sub(r"[-_.]+", "-", name.group(1)).lower()
|
|
416
|
+
self.add(ecosystem, pkg, version.group(1), path, "dev" if dev else "runtime")
|
|
417
|
+
if path.name == "uv.lock" or path.name == "poetry.lock":
|
|
418
|
+
match = re.search(r'^requires-python\s*=\s*"[^\d]*(\d+\.\d+)', text, re.M)
|
|
419
|
+
if match and not any(r["name"] == "python" for r in self.runtimes):
|
|
420
|
+
self.add_runtime("python", match.group(1), "declared")
|
|
421
|
+
|
|
422
|
+
def parse_gemfile_lock(self, path: Path) -> None:
|
|
423
|
+
text = self.read(path) or ""
|
|
424
|
+
self.source("lockfile", path)
|
|
425
|
+
section = None
|
|
426
|
+
for line in text.splitlines():
|
|
427
|
+
if line and not line.startswith(" "):
|
|
428
|
+
section = line.strip()
|
|
429
|
+
continue
|
|
430
|
+
if section in ("GEM", "GIT", "PATH") and re.match(r"^ {4}\S", line):
|
|
431
|
+
match = re.match(r"^ {4}([^\s(]+) \(([^)]+)\)", line)
|
|
432
|
+
if match and section == "GEM":
|
|
433
|
+
self.add("RubyGems", match.group(1), match.group(2).split("-", 1)[0], path)
|
|
434
|
+
if section == "RUBY VERSION":
|
|
435
|
+
match = re.search(r"ruby (\d+\.\d+\.\d+)", line)
|
|
436
|
+
if match:
|
|
437
|
+
self.add_runtime("ruby", match.group(1), "declared")
|
|
438
|
+
|
|
439
|
+
def parse_go_mod(self, path: Path) -> None:
|
|
440
|
+
text = self.read(path) or ""
|
|
441
|
+
self.source("lockfile", path)
|
|
442
|
+
go = re.search(r"^go (\d+\.\d+(?:\.\d+)?)", text, re.M)
|
|
443
|
+
if go:
|
|
444
|
+
self.add_runtime("go", go.group(1), "declared")
|
|
445
|
+
in_block = False
|
|
446
|
+
for line in text.splitlines():
|
|
447
|
+
stripped = line.split("//", 1)[0].strip()
|
|
448
|
+
indirect = "// indirect" in line
|
|
449
|
+
if stripped.startswith("require ("):
|
|
450
|
+
in_block = True
|
|
451
|
+
continue
|
|
452
|
+
if in_block and stripped == ")":
|
|
453
|
+
in_block = False
|
|
454
|
+
continue
|
|
455
|
+
match = re.match(r"^(?:require\s+)?(\S+)\s+(v\S+)$", stripped) if (in_block or stripped.startswith("require ")) else None
|
|
456
|
+
if match:
|
|
457
|
+
self.add("Go", match.group(1), match.group(2).lstrip("v"), path, "runtime", not indirect)
|
|
458
|
+
|
|
459
|
+
def parse_nuget_lock(self, path: Path) -> None:
|
|
460
|
+
data = json.loads(self.read(path) or "{}")
|
|
461
|
+
self.source("lockfile", path)
|
|
462
|
+
for framework in (data.get("dependencies") or {}).values():
|
|
463
|
+
for name, meta in (framework or {}).items():
|
|
464
|
+
if (meta or {}).get("type") == "Project":
|
|
465
|
+
continue
|
|
466
|
+
self.add("NuGet", name, (meta or {}).get("resolved", ""), path, "runtime", (meta or {}).get("type") == "Direct")
|
|
467
|
+
|
|
468
|
+
def parse_csproj(self, path: Path) -> None:
|
|
469
|
+
root = ET.fromstring(self.read(path) or "<x/>")
|
|
470
|
+
found = False
|
|
471
|
+
for element in root.iter():
|
|
472
|
+
tag = element.tag.rsplit("}", 1)[-1]
|
|
473
|
+
if tag in ("PackageReference", "PackageVersion"):
|
|
474
|
+
name = element.get("Include") or element.get("Update")
|
|
475
|
+
version = element.get("Version") or (element.findtext("Version") or "")
|
|
476
|
+
if name and re.fullmatch(r"\d+(\.\d+){1,3}(-[\w.]+)?", version.strip()):
|
|
477
|
+
self.add("NuGet", name, version.strip(), path, "runtime", True)
|
|
478
|
+
found = True
|
|
479
|
+
if tag == "TargetFramework" and element.text:
|
|
480
|
+
match = re.match(r"net(\d+\.\d+)", element.text.strip())
|
|
481
|
+
if match:
|
|
482
|
+
self.add_runtime("dotnet", match.group(1), "declared")
|
|
483
|
+
if found:
|
|
484
|
+
self.source("manifest", path)
|
|
485
|
+
|
|
486
|
+
def parse_global_json(self, path: Path) -> None:
|
|
487
|
+
data = json.loads(self.read(path) or "{}")
|
|
488
|
+
version = (data.get("sdk") or {}).get("version")
|
|
489
|
+
if version:
|
|
490
|
+
self.add_runtime("dotnet", ".".join(version.split(".")[:2]), "declared")
|
|
491
|
+
|
|
492
|
+
def parse_pom(self, path: Path) -> None:
|
|
493
|
+
root = ET.fromstring(self.read(path) or "<x/>")
|
|
494
|
+
ns = {"m": root.tag[1:].split("}", 1)[0]} if root.tag.startswith("{") else {}
|
|
495
|
+
q = (lambda tag: f"m:{tag}") if ns else (lambda tag: tag)
|
|
496
|
+
props: dict[str, str] = {}
|
|
497
|
+
properties = root.find(q("properties"), ns)
|
|
498
|
+
if properties is not None:
|
|
499
|
+
for prop in properties:
|
|
500
|
+
props[prop.tag.rsplit("}", 1)[-1]] = (prop.text or "").strip()
|
|
501
|
+
props["project.version"] = (root.findtext(q("version"), default="", namespaces=ns) or "").strip()
|
|
502
|
+
|
|
503
|
+
def resolve(value: str) -> str:
|
|
504
|
+
return re.sub(r"\$\{([^}]+)\}", lambda m: props.get(m.group(1), m.group(0)), value or "")
|
|
505
|
+
|
|
506
|
+
java = props.get("maven.compiler.release") or props.get("java.version") or props.get("maven.compiler.source") or ""
|
|
507
|
+
if java:
|
|
508
|
+
self.add_runtime("java", java.replace("1.", "", 1) if java.startswith("1.") else java, "declared")
|
|
509
|
+
found = False
|
|
510
|
+
for container in (root.find(q("dependencies"), ns), root.find(f"{q('dependencyManagement')}/{q('dependencies')}", ns)):
|
|
511
|
+
if container is None:
|
|
512
|
+
continue
|
|
513
|
+
for dep in container.findall(q("dependency"), ns):
|
|
514
|
+
group = resolve(dep.findtext(q("groupId"), default="", namespaces=ns).strip())
|
|
515
|
+
artifact = resolve(dep.findtext(q("artifactId"), default="", namespaces=ns).strip())
|
|
516
|
+
version = resolve(dep.findtext(q("version"), default="", namespaces=ns).strip())
|
|
517
|
+
scope = dep.findtext(q("scope"), default="compile", namespaces=ns).strip()
|
|
518
|
+
if "${" in version or not version or version.startswith(("[", "(")):
|
|
519
|
+
continue
|
|
520
|
+
self.add("Maven", f"{group}:{artifact}", version, path, "dev" if scope in ("test", "provided") else "runtime", True)
|
|
521
|
+
found = True
|
|
522
|
+
parent = root.find(q("parent"), ns)
|
|
523
|
+
if parent is not None:
|
|
524
|
+
group = parent.findtext(q("groupId"), default="", namespaces=ns).strip()
|
|
525
|
+
artifact = parent.findtext(q("artifactId"), default="", namespaces=ns).strip()
|
|
526
|
+
version = parent.findtext(q("version"), default="", namespaces=ns).strip()
|
|
527
|
+
if group and artifact and version:
|
|
528
|
+
self.add("Maven", f"{group}:{artifact}", version, path, "build", True)
|
|
529
|
+
found = True
|
|
530
|
+
if found:
|
|
531
|
+
self.source("manifest", path)
|
|
532
|
+
|
|
533
|
+
def parse_gradle_lock(self, path: Path) -> None:
|
|
534
|
+
text = self.read(path) or ""
|
|
535
|
+
self.source("lockfile", path)
|
|
536
|
+
for line in text.splitlines():
|
|
537
|
+
if line.startswith("#") or "=" not in line:
|
|
538
|
+
continue
|
|
539
|
+
coords, _, configurations = line.partition("=")
|
|
540
|
+
parts = coords.strip().split(":")
|
|
541
|
+
if len(parts) == 3:
|
|
542
|
+
scope = "dev" if configurations and all("test" in c.lower() for c in configurations.split(",")) else "runtime"
|
|
543
|
+
self.add("Maven", f"{parts[0]}:{parts[1]}", parts[2], path, scope)
|
|
544
|
+
|
|
545
|
+
# ------------------------------------------------------------------ containers and declared runtimes
|
|
546
|
+
|
|
547
|
+
def add_image(self, ref: str, path: Path | str, host: str | None = None) -> None:
|
|
548
|
+
ref = ref.strip().strip('"\'')
|
|
549
|
+
if not ref or "$" in ref or ref.lower() == "scratch":
|
|
550
|
+
return
|
|
551
|
+
if any(i["ref"] == ref and i.get("host") == host for i in self.images):
|
|
552
|
+
return
|
|
553
|
+
digest = None
|
|
554
|
+
if "@sha256:" in ref:
|
|
555
|
+
ref, _, digest_hex = ref.partition("@sha256:")
|
|
556
|
+
digest = "sha256:" + digest_hex
|
|
557
|
+
self.images.append({"ref": ref, "digest": digest, "host": host})
|
|
558
|
+
name, _, tag = ref.rpartition(":") if ":" in ref.rsplit("/", 1)[-1] else (ref, "", "latest")
|
|
559
|
+
repo = re.sub(r"^(docker\.io/)?(library/)?", "", name)
|
|
560
|
+
runtime = IMAGE_RUNTIMES.get(repo) or IMAGE_RUNTIMES.get(repo.rsplit("/", 1)[-1])
|
|
561
|
+
version = re.match(r"^v?(\d+(?:\.\d+){0,2})", tag or "")
|
|
562
|
+
if runtime and version:
|
|
563
|
+
self.add_runtime(runtime, version.group(1), "image", host, image=ref)
|
|
564
|
+
os_hint = re.search(r"(alpine|bookworm|bullseye|buster|jammy|noble|focal)", tag or "")
|
|
565
|
+
if os_hint and runtime not in ("alpine", "debian", "ubuntu"):
|
|
566
|
+
codename = {"bookworm": ("debian", "12"), "bullseye": ("debian", "11"), "buster": ("debian", "10"),
|
|
567
|
+
"jammy": ("ubuntu", "22.04"), "noble": ("ubuntu", "24.04"), "focal": ("ubuntu", "20.04")}.get(os_hint.group(1))
|
|
568
|
+
if codename:
|
|
569
|
+
self.add_runtime(codename[0], codename[1], "image", host, image=ref)
|
|
570
|
+
|
|
571
|
+
def parse_dockerfile(self, path: Path) -> None:
|
|
572
|
+
text = self.read(path) or ""
|
|
573
|
+
stages: set[str] = set()
|
|
574
|
+
for line in text.splitlines():
|
|
575
|
+
match = re.match(r"^\s*FROM\s+(?:--platform=\S+\s+)?(\S+)(?:\s+AS\s+(\S+))?", line, re.I)
|
|
576
|
+
if match:
|
|
577
|
+
ref = match.group(1)
|
|
578
|
+
if ref not in stages:
|
|
579
|
+
self.add_image(ref, path)
|
|
580
|
+
if match.group(2):
|
|
581
|
+
stages.add(match.group(2))
|
|
582
|
+
self.source("manifest", path)
|
|
583
|
+
|
|
584
|
+
def parse_compose(self, path: Path) -> None:
|
|
585
|
+
text = self.read(path) or ""
|
|
586
|
+
for match in re.finditer(r"^\s*image:\s*['\"]?([^\s'\"#]+)", text, re.M):
|
|
587
|
+
self.add_image(match.group(1), path)
|
|
588
|
+
self.source("manifest", path)
|
|
589
|
+
|
|
590
|
+
def parse_version_file(self, path: Path) -> None:
|
|
591
|
+
text = (self.read(path) or "").strip().splitlines()
|
|
592
|
+
value = text[0].strip() if text else ""
|
|
593
|
+
name = {".nvmrc": "node", ".node-version": "node", ".python-version": "python", ".ruby-version": "ruby",
|
|
594
|
+
".java-version": "java", "runtime.txt": "python"}[path.name]
|
|
595
|
+
match = re.search(r"(\d+(?:\.\d+){0,2})", value)
|
|
596
|
+
if match:
|
|
597
|
+
if path.name == "runtime.txt" and not value.startswith("python"):
|
|
598
|
+
return
|
|
599
|
+
self.add_runtime(name, match.group(1), "declared")
|
|
600
|
+
|
|
601
|
+
def parse_tool_versions(self, path: Path) -> None:
|
|
602
|
+
mapping = {"nodejs": "node", "python": "python", "ruby": "ruby", "golang": "go", "java": "java", "php": "php",
|
|
603
|
+
"erlang": "erlang", "elixir": "elixir", "rust": "rust", "dotnet-core": "dotnet", "postgres": "postgresql",
|
|
604
|
+
"redis": "redis", "mysql": "mysql"}
|
|
605
|
+
for line in (self.read(path) or "").splitlines():
|
|
606
|
+
parts = line.split("#", 1)[0].split()
|
|
607
|
+
if len(parts) >= 2 and parts[0] in mapping:
|
|
608
|
+
version = re.search(r"(\d+(?:\.\d+){0,2})", parts[1])
|
|
609
|
+
if version:
|
|
610
|
+
self.add_runtime(mapping[parts[0]], version.group(1), "declared")
|
|
611
|
+
|
|
612
|
+
# ------------------------------------------------------------------ shipped JavaScript (retire.js extractors)
|
|
613
|
+
|
|
614
|
+
def scan_js_bundles(self, limit: int = 400) -> None:
|
|
615
|
+
"""Libraries copied into the site (static/, public/, dist/, assets/, wp-content/) rather than installed.
|
|
616
|
+
|
|
617
|
+
Uses the filename and banner extractors from retire.js (Apache-2.0,
|
|
618
|
+
security_data/retire-extractors.json). Only the first 2 KB of each file
|
|
619
|
+
is read, where banners live.
|
|
620
|
+
"""
|
|
621
|
+
extractors = _retire_extractors()
|
|
622
|
+
roots = [self.project / d for d in ("", "static", "public", "dist", "assets", "js", "wp-content/themes", "wp-content/plugins", "www", "build")]
|
|
623
|
+
seen: set[Path] = set()
|
|
624
|
+
found = 0
|
|
625
|
+
for root in roots:
|
|
626
|
+
if not root.is_dir():
|
|
627
|
+
continue
|
|
628
|
+
pattern = "**/*.js" if root != self.project else "*.js"
|
|
629
|
+
for path in root.glob(pattern):
|
|
630
|
+
if path in seen or "node_modules" in path.parts or found >= limit:
|
|
631
|
+
continue
|
|
632
|
+
seen.add(path)
|
|
633
|
+
try:
|
|
634
|
+
with path.open("r", encoding="utf-8", errors="replace") as handle:
|
|
635
|
+
head = handle.read(2048)
|
|
636
|
+
except OSError:
|
|
637
|
+
continue
|
|
638
|
+
for lib, (name_res, content_res, npm) in extractors.items():
|
|
639
|
+
version = None
|
|
640
|
+
for rx in name_res:
|
|
641
|
+
match = rx.search(path.name)
|
|
642
|
+
if match:
|
|
643
|
+
version = match.group(1)
|
|
644
|
+
break
|
|
645
|
+
if not version:
|
|
646
|
+
for rx in content_res:
|
|
647
|
+
match = rx.search(head)
|
|
648
|
+
if match:
|
|
649
|
+
version = match.group(1)
|
|
650
|
+
break
|
|
651
|
+
if version:
|
|
652
|
+
version = re.sub(r"([.-](min|slim|pack|prod|production|umd|esm|bundle|js))+$", "", version).rstrip(".-_")
|
|
653
|
+
self.components.append({"purl": core.make_purl("npm", npm, version), "ecosystem": "npm", "name": npm,
|
|
654
|
+
"version": version, "scope": "runtime", "direct": None, "location": self.rel(path)})
|
|
655
|
+
self.source("js-bundle", path)
|
|
656
|
+
found += 1
|
|
657
|
+
break
|
|
658
|
+
|
|
659
|
+
# ------------------------------------------------------------------ WordPress
|
|
660
|
+
|
|
661
|
+
def find_wordpress(self) -> Path | None:
|
|
662
|
+
for candidate in (self.project, self.project / "wordpress", self.project / "public", self.project / "web", self.project / "htdocs"):
|
|
663
|
+
if (candidate / "wp-includes" / "version.php").is_file() or (candidate / "wp-content").is_dir():
|
|
664
|
+
return candidate
|
|
665
|
+
return None
|
|
666
|
+
|
|
667
|
+
@staticmethod
|
|
668
|
+
def header(text: str, field: str) -> str | None:
|
|
669
|
+
match = re.search(rf"^[ \t/*#@]*{re.escape(field)}:\s*(.+)$", text, re.M | re.I)
|
|
670
|
+
return match.group(1).strip() if match else None
|
|
671
|
+
|
|
672
|
+
def parse_wordpress(self, root: Path) -> None:
|
|
673
|
+
wp: dict = {"core": None, "plugins": [], "themes": []}
|
|
674
|
+
version_file = root / "wp-includes" / "version.php"
|
|
675
|
+
if version_file.is_file():
|
|
676
|
+
match = re.search(r"\$wp_version\s*=\s*'([^']+)'", self.read(version_file) or "")
|
|
677
|
+
wp["core"] = match.group(1) if match else None
|
|
678
|
+
plugins = root / "wp-content" / "plugins"
|
|
679
|
+
if plugins.is_dir():
|
|
680
|
+
for directory in sorted(p for p in plugins.iterdir() if p.is_dir()):
|
|
681
|
+
for php in sorted(directory.glob("*.php"))[:20]:
|
|
682
|
+
head = (self.read(php) or "")[:8192]
|
|
683
|
+
if self.header(head, "Plugin Name"):
|
|
684
|
+
wp["plugins"].append({"slug": directory.name, "version": self.header(head, "Version"), "active": None,
|
|
685
|
+
"text_domain": self.header(head, "Text Domain")})
|
|
686
|
+
break
|
|
687
|
+
themes = root / "wp-content" / "themes"
|
|
688
|
+
if themes.is_dir():
|
|
689
|
+
for directory in sorted(p for p in themes.iterdir() if p.is_dir()):
|
|
690
|
+
style = directory / "style.css"
|
|
691
|
+
if style.is_file():
|
|
692
|
+
head = (self.read(style) or "")[:8192]
|
|
693
|
+
wp["themes"].append({"slug": directory.name, "version": self.header(head, "Version"), "active": None})
|
|
694
|
+
self.wordpress = wp
|
|
695
|
+
self.source("wordpress", root / "wp-content" if (root / "wp-content").is_dir() else root)
|
|
696
|
+
if wp["core"]:
|
|
697
|
+
self.add_runtime("wordpress", wp["core"], "files")
|
|
698
|
+
|
|
699
|
+
def apply_wp_cli(self, path: Path) -> None:
|
|
700
|
+
"""Active state from `wp plugin list --format=json` / `wp theme list --format=json` output."""
|
|
701
|
+
data = json.loads(self.read(path) or "[]")
|
|
702
|
+
if not self.wordpress:
|
|
703
|
+
self.wordpress = {"core": None, "plugins": [], "themes": []}
|
|
704
|
+
for row in data:
|
|
705
|
+
status = row.get("status")
|
|
706
|
+
for bucket in ("plugins", "themes"):
|
|
707
|
+
for item in self.wordpress[bucket]:
|
|
708
|
+
if item["slug"] == row.get("name"):
|
|
709
|
+
item["active"] = status in ("active", "active-network", "parent")
|
|
710
|
+
item["version"] = item["version"] or row.get("version")
|
|
711
|
+
|
|
712
|
+
# ------------------------------------------------------------------ hosts
|
|
713
|
+
|
|
714
|
+
def parse_host_evidence(self, text: str, alias: str, path_label: str) -> None:
|
|
715
|
+
"""Sections from SECURITY_COLLECTOR (or the plain runtime-inventory collector)."""
|
|
716
|
+
sections = parse_sections(text)
|
|
717
|
+
os_info = parse_os(sections.get("os-release", ""))
|
|
718
|
+
public = True
|
|
719
|
+
if os_info.get("id"):
|
|
720
|
+
os_name = {"rhel": "rhel", "rocky": "rocky-linux", "almalinux": "almalinux", "amzn": "amazon-linux",
|
|
721
|
+
"opensuse-leap": "opensuse", "sles": "sles", "centos": "centos", "fedora": "fedora"}.get(os_info["id"], os_info["id"])
|
|
722
|
+
self.add_runtime(os_name, os_info.get("version", ""), "runtime-inventory", alias)
|
|
723
|
+
self.hosts.append({"id": alias, "os": " ".join(x for x in (os_info.get("id"), os_info.get("version")) if x) or None, "public": public})
|
|
724
|
+
for section, (name, pattern) in SERVICE_PROBES.items():
|
|
725
|
+
body = sections.get(section, "")
|
|
726
|
+
match = re.search(pattern, body, re.M)
|
|
727
|
+
if not match:
|
|
728
|
+
continue
|
|
729
|
+
service = name
|
|
730
|
+
if section == "mysql-v":
|
|
731
|
+
service = "mariadb" if "mariadb" in body.lower() else "mysql"
|
|
732
|
+
self.add_runtime(service, match.group(1), "runtime-inventory", alias)
|
|
733
|
+
ecosystem = OS_ECOSYSTEM.get(os_info.get("id", ""))
|
|
734
|
+
release = os_info.get("version", "")
|
|
735
|
+
eco = f"{ecosystem}:{release}" if ecosystem and release else ecosystem
|
|
736
|
+
if ecosystem == "Alpine" and release:
|
|
737
|
+
eco = "Alpine:v" + versions.cycle_of(release, 2) # OSV names Alpine releases "Alpine:v3.16"
|
|
738
|
+
if ecosystem == "Ubuntu" and release:
|
|
739
|
+
eco = f"Ubuntu:{release}" + (":LTS" if release.endswith(".04") and int(release.split(".")[0]) % 2 == 0 else "")
|
|
740
|
+
count = 0
|
|
741
|
+
for line in sections.get("dpkg", "").splitlines():
|
|
742
|
+
parts = line.split("\t")
|
|
743
|
+
if len(parts) >= 2 and (len(parts) < 3 or "installed" in parts[2]):
|
|
744
|
+
self.components.append({"purl": core.make_purl("deb", parts[0], parts[1]), "ecosystem": eco or "Debian", "name": parts[0],
|
|
745
|
+
"version": parts[1], "scope": "os", "direct": None, "location": f"{path_label}#dpkg", "host": alias})
|
|
746
|
+
count += 1
|
|
747
|
+
for line in sections.get("rpm", "").splitlines():
|
|
748
|
+
parts = line.split("\t")
|
|
749
|
+
if len(parts) == 2:
|
|
750
|
+
version = parts[1][2:] if parts[1].startswith("0:") else parts[1]
|
|
751
|
+
self.components.append({"purl": None, "ecosystem": eco or "Red Hat", "name": parts[0], "version": version,
|
|
752
|
+
"scope": "os", "direct": None, "location": f"{path_label}#rpm", "host": alias})
|
|
753
|
+
count += 1
|
|
754
|
+
for line in sections.get("apk", "").splitlines():
|
|
755
|
+
match = re.match(r"^(.+)-(\d[^-]*-r\d+)$", line.strip())
|
|
756
|
+
if match:
|
|
757
|
+
self.components.append({"purl": core.make_purl("apk", match.group(1), match.group(2)), "ecosystem": eco or "Alpine",
|
|
758
|
+
"name": match.group(1), "version": match.group(2), "scope": "os", "direct": None,
|
|
759
|
+
"location": f"{path_label}#apk", "host": alias})
|
|
760
|
+
count += 1
|
|
761
|
+
for line in sections.get("docker-ps", "").splitlines():
|
|
762
|
+
parts = line.split("\t")
|
|
763
|
+
if len(parts) >= 2:
|
|
764
|
+
self.add_image(parts[1], path_label, alias)
|
|
765
|
+
self.source("runtime-inventory", path_label)
|
|
766
|
+
|
|
767
|
+
def parse_dpkg_status(self, path: Path, alias: str) -> None:
|
|
768
|
+
"""A copied /var/lib/dpkg/status file."""
|
|
769
|
+
text = self.read(path) or ""
|
|
770
|
+
lines = []
|
|
771
|
+
for block in text.split("\n\n"):
|
|
772
|
+
fields = dict(re.findall(r"^([A-Za-z-]+):\s*(.*)$", block, re.M))
|
|
773
|
+
if fields.get("Package") and fields.get("Version") and "installed" in fields.get("Status", "installed"):
|
|
774
|
+
lines.append(f"{fields['Package']}\t{fields['Version']}\t{fields.get('Status', 'install ok installed')}")
|
|
775
|
+
self.parse_host_evidence("### dpkg\n" + "\n".join(lines), alias, self.rel(path))
|
|
776
|
+
|
|
777
|
+
def parse_runtime_inventory_json(self, path: Path, alias: str) -> None:
|
|
778
|
+
data = json.loads(self.read(path) or "{}")
|
|
779
|
+
os_info = data.get("os") or {}
|
|
780
|
+
if os_info.get("id"):
|
|
781
|
+
self.add_runtime(os_info["id"], os_info.get("version", ""), "runtime-inventory", alias)
|
|
782
|
+
self.hosts.append({"id": alias, "os": " ".join(x for x in (os_info.get("id"), os_info.get("version")) if x) or None, "public": True})
|
|
783
|
+
if data.get("hostPhp"):
|
|
784
|
+
self.add_runtime("php", data["hostPhp"], "runtime-inventory", alias)
|
|
785
|
+
for container in data.get("containers") or []:
|
|
786
|
+
self.add_image(container.get("image", ""), self.rel(path), alias)
|
|
787
|
+
for site in data.get("sites") or []:
|
|
788
|
+
runtime = site.get("runtime") or {}
|
|
789
|
+
if runtime.get("version"):
|
|
790
|
+
self.add_runtime("php", runtime["version"], "runtime-inventory", alias, site_root=site.get("root"))
|
|
791
|
+
self.source("runtime-inventory", path)
|
|
792
|
+
|
|
793
|
+
# ------------------------------------------------------------------ result
|
|
794
|
+
|
|
795
|
+
def stack(self, domains: list[str], watch_extra: list[str]) -> dict:
|
|
796
|
+
seen = set()
|
|
797
|
+
components = []
|
|
798
|
+
for comp in sorted(self.components, key=lambda c: (c["ecosystem"], c["name"], c["version"], c["location"] or "")):
|
|
799
|
+
key = (comp["ecosystem"], comp["name"], comp["version"], comp["scope"])
|
|
800
|
+
if key in seen:
|
|
801
|
+
continue
|
|
802
|
+
seen.add(key)
|
|
803
|
+
components.append(comp)
|
|
804
|
+
# A declared "python 3.9" next to an exact "python 3.9.18" on the same host is the same runtime.
|
|
805
|
+
runtimes = []
|
|
806
|
+
for runtime in sorted(self.runtimes, key=lambda r: (r["name"], r.get("host") or "", -len(r["version"]), r["version"])):
|
|
807
|
+
if any(r["name"] == runtime["name"] and r.get("host") == runtime.get("host") and r["version"].startswith(runtime["version"] + ".")
|
|
808
|
+
for r in runtimes):
|
|
809
|
+
continue
|
|
810
|
+
runtimes.append(runtime)
|
|
811
|
+
runtimes.sort(key=lambda r: (r["name"], r.get("host") or "", r["version"]))
|
|
812
|
+
images = sorted(self.images, key=lambda i: (i["ref"], i.get("host") or ""))
|
|
813
|
+
watch = list(dict.fromkeys(WATCH_DEFAULTS.get(self.site_type, []) + [r["name"] for r in runtimes] +
|
|
814
|
+
[c["name"] for c in components if c.get("direct") and c["ecosystem"] == "npm" and c["name"] in ("next", "astro", "nuxt", "express")] +
|
|
815
|
+
watch_extra))
|
|
816
|
+
digest = core.sha256_bytes(core.canonical_json([components, runtimes, self.wordpress, images]).encode("utf-8"))
|
|
817
|
+
return {
|
|
818
|
+
"schema": "maggie-security-stack.v1",
|
|
819
|
+
"site": {"id": self.site, "type": self.site_type, "domains": domains, "hosting": None},
|
|
820
|
+
"collected_at": core.iso(),
|
|
821
|
+
"hash": f"sha256:{digest}",
|
|
822
|
+
"sources": [{"kind": "manifest" if s["kind"] == "js-bundle" else s["kind"], "path": s["path"]} for s in self.sources],
|
|
823
|
+
"components": components,
|
|
824
|
+
"runtimes": runtimes,
|
|
825
|
+
"wordpress": self.wordpress,
|
|
826
|
+
"images": images,
|
|
827
|
+
"hosts": self.hosts,
|
|
828
|
+
"watch": {"products": watch},
|
|
829
|
+
}
|
|
830
|
+
|
|
831
|
+
|
|
832
|
+
_EXTRACTORS: dict | None = None
|
|
833
|
+
|
|
834
|
+
|
|
835
|
+
def _retire_extractors() -> dict:
|
|
836
|
+
global _EXTRACTORS
|
|
837
|
+
if _EXTRACTORS is None:
|
|
838
|
+
data = core.read_json(core.DATA_DIR / "retire-extractors.json")
|
|
839
|
+
version = r"([0-9][0-9.a-z_\-]+)"
|
|
840
|
+
compiled = {}
|
|
841
|
+
for lib, spec in data["libraries"].items():
|
|
842
|
+
names = [re.compile(p.replace("§§version§§", version[1:-1]).replace("(" + version[1:-1] + ")", version)) for p in spec.get("filename", [])]
|
|
843
|
+
contents = [re.compile(p.replace("§§version§§", version[1:-1]).replace("(" + version[1:-1] + ")", version)) for p in spec.get("filecontent", [])]
|
|
844
|
+
compiled[lib] = (names, contents, spec.get("npmname") or lib)
|
|
845
|
+
_EXTRACTORS = compiled
|
|
846
|
+
return _EXTRACTORS
|
|
847
|
+
|
|
848
|
+
|
|
849
|
+
def summary(stack: dict, previous_hash: str | None) -> dict:
|
|
850
|
+
by_eco: dict[str, int] = {}
|
|
851
|
+
for comp in stack["components"]:
|
|
852
|
+
eco = comp["ecosystem"].split(":", 1)[0]
|
|
853
|
+
by_eco[eco] = by_eco.get(eco, 0) + 1
|
|
854
|
+
wp = stack.get("wordpress") or {}
|
|
855
|
+
return {"components": len(stack["components"]), "by_ecosystem": by_eco,
|
|
856
|
+
"runtimes": [f"{r['name']} {r['version']}" + (f" @{r['host']}" if r.get("host") else "") for r in stack["runtimes"]],
|
|
857
|
+
"images": [i["ref"] for i in stack["images"]],
|
|
858
|
+
"wordpress": {"core": wp.get("core"), "plugins": len(wp.get("plugins") or []), "themes": len(wp.get("themes") or [])} if wp else None,
|
|
859
|
+
"hash": stack["hash"], "changed": previous_hash != stack["hash"]}
|
|
860
|
+
|
|
861
|
+
|
|
862
|
+
def build(ctx: core.Context) -> dict:
|
|
863
|
+
args = ctx.args
|
|
864
|
+
cfg = ctx.state.config()
|
|
865
|
+
site_type = args.site_type or cfg.get("site_type") or core.detect_site_type(ctx.project)
|
|
866
|
+
inv = Inventory(ctx.project, ctx.site, site_type)
|
|
867
|
+
if not args.no_scan:
|
|
868
|
+
inv.scan_project()
|
|
869
|
+
for lockfile in args.lockfile or []:
|
|
870
|
+
path = Path(lockfile)
|
|
871
|
+
path = path if path.is_absolute() else ctx.project / path
|
|
872
|
+
parser = {"package-lock.json": inv.parse_package_lock_json, "npm-shrinkwrap.json": inv.parse_package_lock_json,
|
|
873
|
+
"pnpm-lock.yaml": inv.parse_pnpm_lock_yaml, "yarn.lock": inv.parse_yarn_lock, "bun.lock": inv.parse_bun_lock,
|
|
874
|
+
"composer.lock": inv.parse_composer_lock, "Pipfile.lock": inv.parse_pipfile_lock, "poetry.lock": inv.parse_toml_lock,
|
|
875
|
+
"uv.lock": inv.parse_toml_lock, "pdm.lock": inv.parse_toml_lock, "Cargo.lock": inv.parse_toml_lock,
|
|
876
|
+
"pom.xml": inv.parse_pom, "gradle.lockfile": inv.parse_gradle_lock, "go.mod": inv.parse_go_mod,
|
|
877
|
+
"Gemfile.lock": inv.parse_gemfile_lock, "packages.lock.json": inv.parse_nuget_lock}.get(path.name)
|
|
878
|
+
if parser is None and path.name.startswith("requirements"):
|
|
879
|
+
parser = inv.parse_requirements
|
|
880
|
+
if parser is None:
|
|
881
|
+
raise core.UsageError(f"unsupported lockfile {path.name}")
|
|
882
|
+
if not path.is_file():
|
|
883
|
+
raise core.UsageError(f"file not found: {path}")
|
|
884
|
+
inv.safe(parser, path)
|
|
885
|
+
if args.wordpress:
|
|
886
|
+
inv.safe(inv.parse_wordpress, Path(args.wordpress))
|
|
887
|
+
if args.wp_cli_json:
|
|
888
|
+
inv.safe(inv.apply_wp_cli, Path(args.wp_cli_json))
|
|
889
|
+
for item in args.host_evidence or []:
|
|
890
|
+
alias, _, file_name = item.partition("=") if "=" in item else ("host1", "", item)
|
|
891
|
+
path = Path(file_name)
|
|
892
|
+
if not path.is_file():
|
|
893
|
+
raise core.UsageError(f"file not found: {path}")
|
|
894
|
+
text = path.read_text(encoding="utf-8", errors="replace")
|
|
895
|
+
if text.lstrip().startswith("{"):
|
|
896
|
+
inv.safe(lambda p: inv.parse_runtime_inventory_json(p, alias), path)
|
|
897
|
+
elif text.lstrip().startswith("Package:"):
|
|
898
|
+
inv.safe(lambda p: inv.parse_dpkg_status(p, alias), path)
|
|
899
|
+
else:
|
|
900
|
+
inv.parse_host_evidence(text, alias, inv.rel(path))
|
|
901
|
+
for item in args.ssh or []:
|
|
902
|
+
alias, _, target = item.partition("=")
|
|
903
|
+
if not target:
|
|
904
|
+
raise core.UsageError("--ssh takes ALIAS=user@host")
|
|
905
|
+
core.require_consent(ctx.state, alias, "ssh-read")
|
|
906
|
+
text = _collect(target)
|
|
907
|
+
out = ctx.state.path("inventory", f"host-{alias}.txt")
|
|
908
|
+
out.parent.mkdir(parents=True, exist_ok=True)
|
|
909
|
+
out.write_text(text, encoding="utf-8")
|
|
910
|
+
inv.parse_host_evidence(text, alias, f".maggie/security/inventory/host-{alias}.txt")
|
|
911
|
+
domains = list(dict.fromkeys((args.domain or []) + (cfg.get("domains") or [])))
|
|
912
|
+
stack = inv.stack(domains, (args.watch_product or []) + (cfg.get("watch_products") or []))
|
|
913
|
+
if not stack["components"] and not stack["runtimes"] and not stack["wordpress"]:
|
|
914
|
+
raise core.UsageError("nothing to inventory: no lockfile, manifest, WordPress install or host evidence found")
|
|
915
|
+
errors = core.validate_contract(stack, "stack-v1")
|
|
916
|
+
if errors:
|
|
917
|
+
raise core.UsageError("stack failed its contract: " + "; ".join(errors[:5]))
|
|
918
|
+
path = ctx.state.path("inventory", "stack.json")
|
|
919
|
+
previous = core.read_json(path).get("hash") if path.is_file() else None
|
|
920
|
+
if not args.print:
|
|
921
|
+
core.write_json(path, stack)
|
|
922
|
+
return {"schema": "maggie-security-inventory-result.v1", "path": str(path) if not args.print else None,
|
|
923
|
+
"summary": summary(stack, previous), "problems": inv.problems, "stack": stack if args.print else None}
|
|
924
|
+
|
|
925
|
+
|
|
926
|
+
def _collect(target: str) -> str:
|
|
927
|
+
import subprocess
|
|
928
|
+
|
|
929
|
+
result = subprocess.run(["ssh", "-o", "BatchMode=yes", "-o", "ConnectTimeout=15", target, "sh -s"], input=SECURITY_COLLECTOR,
|
|
930
|
+
capture_output=True, text=True, timeout=180, check=False)
|
|
931
|
+
if result.returncode not in (0, 1) and not result.stdout:
|
|
932
|
+
raise core.DegradedError(f"ssh {target}: {result.stderr.strip()[:200]}")
|
|
933
|
+
return result.stdout
|
|
934
|
+
|
|
935
|
+
|
|
936
|
+
def load_stack(ctx: core.Context, max_age_hours: int = 24, rebuild: bool = False) -> dict:
|
|
937
|
+
"""The current stack.json, rebuilt from the project when missing, stale or asked."""
|
|
938
|
+
path = ctx.state.path("inventory", "stack.json")
|
|
939
|
+
if path.is_file() and not rebuild:
|
|
940
|
+
stack = core.read_json(path)
|
|
941
|
+
age = core.utcnow() - core.parse_time(stack.get("collected_at", "1970-01-01T00:00:00Z"))
|
|
942
|
+
if age.total_seconds() < max_age_hours * 3600:
|
|
943
|
+
return stack
|
|
944
|
+
import argparse
|
|
945
|
+
|
|
946
|
+
fake = argparse.Namespace(site_type=None, no_scan=False, lockfile=[], wordpress=None, wp_cli_json=None, host_evidence=[], ssh=[],
|
|
947
|
+
domain=[], watch_product=[], print=False)
|
|
948
|
+
previous = core.read_json(path) if path.is_file() else None
|
|
949
|
+
sub = core.Context.__new__(core.Context)
|
|
950
|
+
sub.__dict__.update(ctx.__dict__)
|
|
951
|
+
sub.args = fake
|
|
952
|
+
try:
|
|
953
|
+
build(sub)
|
|
954
|
+
except core.UsageError:
|
|
955
|
+
if not previous:
|
|
956
|
+
raise
|
|
957
|
+
stack = core.read_json(path) if path.is_file() else previous
|
|
958
|
+
if previous and previous is not stack:
|
|
959
|
+
# Host-derived parts need SSH or an evidence file to refresh; keep the last ones.
|
|
960
|
+
hosts = {h["id"] for h in previous.get("hosts") or []}
|
|
961
|
+
if hosts:
|
|
962
|
+
stack["hosts"] = previous["hosts"] + [h for h in stack.get("hosts") or [] if h["id"] not in hosts]
|
|
963
|
+
stack["runtimes"] += [r for r in previous.get("runtimes") or [] if r.get("host") in hosts and r not in stack["runtimes"]]
|
|
964
|
+
stack["components"] += [c for c in previous.get("components") or [] if c.get("scope") == "os" and c not in stack["components"]]
|
|
965
|
+
stack["images"] += [i for i in previous.get("images") or [] if i.get("host") in hosts and i not in stack["images"]]
|
|
966
|
+
stack["sources"] += [s for s in previous.get("sources") or [] if s["kind"] == "runtime-inventory" and s not in stack["sources"]]
|
|
967
|
+
stack["hash"] = "sha256:" + core.sha256_bytes(core.canonical_json([stack["components"], stack["runtimes"], stack.get("wordpress"), stack["images"]]).encode())
|
|
968
|
+
core.write_json(path, stack)
|
|
969
|
+
return stack
|
|
970
|
+
|
|
971
|
+
|
|
972
|
+
def configure(parser) -> None:
|
|
973
|
+
parser.add_argument("--lockfile", action="append", help="extra lockfile/manifest outside the scanned tree (repeatable)")
|
|
974
|
+
parser.add_argument("--wordpress", help="WordPress root when not under the project")
|
|
975
|
+
parser.add_argument("--wp-cli-json", help="output of `wp plugin list --format=json` for active state")
|
|
976
|
+
parser.add_argument("--host-evidence", action="append",
|
|
977
|
+
help="ALIAS=FILE: output of --print-collector, runtime-inventory JSON, or a copied dpkg status file (repeatable)")
|
|
978
|
+
parser.add_argument("--ssh", action="append", help="ALIAS=user@host: run the read-only collector over SSH (needs ssh-read consent)")
|
|
979
|
+
parser.add_argument("--site-type", choices=["static", "wordpress", "node-app", "php-app", "python-app", "java-app", "go-app", "ruby-app",
|
|
980
|
+
"dotnet-app", "astro", "workers", "unknown"])
|
|
981
|
+
parser.add_argument("--domain", action="append", help="site domain (repeatable)")
|
|
982
|
+
parser.add_argument("--watch-product", action="append", help="extra product name for CVE records without a package id")
|
|
983
|
+
parser.add_argument("--no-scan", action="store_true", help="only use the explicit inputs, do not scan the project tree")
|
|
984
|
+
parser.add_argument("--print", action="store_true", help="print the stack instead of writing stack.json")
|
|
985
|
+
parser.add_argument("--print-collector", action="store_true", help="print the read-only host collector script and exit")
|
|
986
|
+
parser.add_argument("--export-upload", help="also write the stack-upload payload for the hosted daily watch to this file (names and versions only)")
|
|
987
|
+
parser.add_argument("--workspace", help="CMO.so workspace (domain) for --export-upload")
|
|
988
|
+
|
|
989
|
+
|
|
990
|
+
def upload_payload(stack: dict, workspace: str) -> dict:
|
|
991
|
+
"""Only names and versions leave the machine (PRD 11 section 9): no paths, hostnames, lockfiles or logs."""
|
|
992
|
+
components = [{"ecosystem": c["ecosystem"], "name": c["name"], "version": c["version"], "scope": c.get("scope") or "runtime",
|
|
993
|
+
"direct": c.get("direct")} for c in stack["components"]]
|
|
994
|
+
runtimes = [{"name": r["name"], "version": r["version"], "host": r.get("host") if r.get("host") and re.fullmatch(r"[a-z0-9-]{1,40}", r["host"]) else None}
|
|
995
|
+
for r in stack["runtimes"]]
|
|
996
|
+
wordpress = None
|
|
997
|
+
if stack.get("wordpress"):
|
|
998
|
+
wp = stack["wordpress"]
|
|
999
|
+
wordpress = {"core": wp.get("core"), "plugins": [{"slug": p["slug"], "version": p.get("version"), "active": p.get("active")} for p in wp.get("plugins") or []],
|
|
1000
|
+
"themes": [{"slug": t["slug"], "version": t.get("version"), "active": t.get("active")} for t in wp.get("themes") or []]}
|
|
1001
|
+
body = {"components": components, "runtimes": runtimes, "wordpress": wordpress}
|
|
1002
|
+
return {"schema": "maggie-security-stack-upload.v1", "workspace": workspace, "site_type": stack["site"].get("type") or "unknown",
|
|
1003
|
+
"created_at": core.iso(), "hash": "sha256:" + core.sha256_bytes(core.canonical_json(body).encode()), **body,
|
|
1004
|
+
"watch": (stack.get("watch") or {}).get("products") or []}
|
|
1005
|
+
|
|
1006
|
+
|
|
1007
|
+
def run(ctx: core.Context) -> dict:
|
|
1008
|
+
if ctx.args.print_collector:
|
|
1009
|
+
return {"_raw": SECURITY_COLLECTOR}
|
|
1010
|
+
result = build(ctx)
|
|
1011
|
+
if ctx.args.export_upload:
|
|
1012
|
+
stack = result.get("stack") or core.read_json(ctx.state.path("inventory", "stack.json"))
|
|
1013
|
+
workspace = ctx.args.workspace or (ctx.state.config().get("domains") or [ctx.site])[0]
|
|
1014
|
+
payload = upload_payload(stack, workspace)
|
|
1015
|
+
errors = core.validate_contract(payload, "stack-upload-v1")
|
|
1016
|
+
if errors:
|
|
1017
|
+
raise core.UsageError("upload payload failed its contract: " + "; ".join(errors[:5]))
|
|
1018
|
+
out = Path(ctx.args.export_upload)
|
|
1019
|
+
core.write_json(out, payload)
|
|
1020
|
+
result["upload"] = {"path": str(out), "components": len(payload["components"]), "runtimes": len(payload["runtimes"]),
|
|
1021
|
+
"sends": "package, runtime and WordPress plugin/theme names and versions only",
|
|
1022
|
+
"never_sends": ["file paths", "host names or IPs", "lockfile contents", "logs", "secrets", "findings"]}
|
|
1023
|
+
return result
|