@topy-ai/maggie 0.7.55 → 0.7.57

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (73) hide show
  1. package/README-zh-TW.md +7 -3
  2. package/README.md +42 -8
  3. package/bin/maggie.js +4 -0
  4. package/bundled-contracts/maggie-deployment/readiness-v1.schema.json +2 -1
  5. package/bundled-contracts/maggie-security/finding-example-v1.json +89 -0
  6. package/bundled-contracts/maggie-security/finding-v1.schema.json +63 -0
  7. package/bundled-contracts/maggie-security/findings-set-example-v1.json +148 -0
  8. package/bundled-contracts/maggie-security/findings-set-v1.schema.json +32 -0
  9. package/bundled-contracts/maggie-security/protect-plan-v1.schema.json +28 -0
  10. package/bundled-contracts/maggie-security/report-v1.schema.json +29 -0
  11. package/bundled-contracts/maggie-security/stack-example-v1.json +66 -0
  12. package/bundled-contracts/maggie-security/stack-upload-v1.schema.json +22 -0
  13. package/bundled-contracts/maggie-security/stack-v1.schema.json +37 -0
  14. package/bundled-contracts/maggie-security/tools-lock-v1.schema.json +36 -0
  15. package/bundled-skills/catalog.json +4 -0
  16. package/bundled-skills/maggie-security/SKILL.md +173 -0
  17. package/bundled-skills/maggie-security/agents/openai.yaml +4 -0
  18. package/bundled-skills/maggie-security/data/LICENSE-NOTICE.md +24 -0
  19. package/bundled-skills/maggie-security/data/asvs-map.json +209 -0
  20. package/bundled-skills/maggie-security/data/fail2ban/filter.d/maggie-wordpress.conf +6 -0
  21. package/bundled-skills/maggie-security/data/semgrep/LICENSE-NOTICE.md +5 -0
  22. package/bundled-skills/maggie-security/data/semgrep/VERSION +1 -0
  23. package/bundled-skills/maggie-security/data/semgrep/rules/dockerfile.yaml +14 -0
  24. package/bundled-skills/maggie-security/data/semgrep/rules/generic.yaml +26 -0
  25. package/bundled-skills/maggie-security/data/semgrep/rules/js.yaml +216 -0
  26. package/bundled-skills/maggie-security/data/semgrep/rules/php-debug-ok.yaml +17 -0
  27. package/bundled-skills/maggie-security/data/semgrep/rules/php-debug.yaml +17 -0
  28. package/bundled-skills/maggie-security/data/semgrep/rules/php.yaml +101 -0
  29. package/bundled-skills/maggie-security/data/semgrep/tests/dockerfile.dockerfile +10 -0
  30. package/bundled-skills/maggie-security/data/semgrep/tests/generic.astro +11 -0
  31. package/bundled-skills/maggie-security/data/semgrep/tests/generic.conf +11 -0
  32. package/bundled-skills/maggie-security/data/semgrep/tests/js.jsx +95 -0
  33. package/bundled-skills/maggie-security/data/semgrep/tests/php-debug-ok.php +5 -0
  34. package/bundled-skills/maggie-security/data/semgrep/tests/php-debug.php +3 -0
  35. package/bundled-skills/maggie-security/data/semgrep/tests/php.php +55 -0
  36. package/bundled-skills/maggie-security/evals/evals.json +97 -0
  37. package/bundled-skills/maggie-security/references/commands.md +125 -0
  38. package/bundled-skills/maggie-security/references/cve-response.md +24 -0
  39. package/bundled-skills/maggie-security/references/explaining-findings.md +57 -0
  40. package/bundled-skills/maggie-security/references/findings.md +39 -0
  41. package/bundled-skills/maggie-security/references/ownership-and-consent.md +29 -0
  42. package/bundled-skills/maggie-security/references/site-types.md +22 -0
  43. package/bundled-skills/maggie-security/references/tools-and-licences.md +16 -0
  44. package/bundled-skills/maggie-security/references/workflows.md +23 -0
  45. package/bundled-skills/maggie-security/registry/security-tools.registry.json +654 -0
  46. package/bundled-skills/maggie-security/routines.example.json +27 -0
  47. package/bundled-tools/clis/maggie_deployment_readiness.py +17 -2
  48. package/bundled-tools/clis/maggie_security.py +342 -0
  49. package/bundled-tools/clis/maggie_security_agent.py +326 -0
  50. package/bundled-tools/clis/maggie_security_code.py +650 -0
  51. package/bundled-tools/clis/maggie_security_core.py +1512 -0
  52. package/bundled-tools/clis/maggie_security_deps.py +560 -0
  53. package/bundled-tools/clis/maggie_security_health.py +415 -0
  54. package/bundled-tools/clis/maggie_security_host.py +511 -0
  55. package/bundled-tools/clis/maggie_security_hosted.py +115 -0
  56. package/bundled-tools/clis/maggie_security_intel.py +1294 -0
  57. package/bundled-tools/clis/maggie_security_inventory.py +1023 -0
  58. package/bundled-tools/clis/maggie_security_logs.py +777 -0
  59. package/bundled-tools/clis/maggie_security_mcp.py +484 -0
  60. package/bundled-tools/clis/maggie_security_mirror.py +122 -0
  61. package/bundled-tools/clis/maggie_security_protect.py +519 -0
  62. package/bundled-tools/clis/maggie_security_report.py +506 -0
  63. package/bundled-tools/clis/maggie_security_standards.py +214 -0
  64. package/bundled-tools/clis/maggie_security_versions.py +381 -0
  65. package/bundled-tools/clis/maggie_security_web.py +1247 -0
  66. package/bundled-tools/clis/security_data/eol-snapshot.json +1 -0
  67. package/bundled-tools/clis/security_data/gitleaks-rules.json +3812 -0
  68. package/bundled-tools/clis/security_data/maltrail-lists.json +1 -0
  69. package/bundled-tools/clis/security_data/public-suffix-icann.txt +7374 -0
  70. package/bundled-tools/clis/security_data/retire-extractors.json +695 -0
  71. package/bundled-tools/clis/security_data/sigma-web.json +6514 -0
  72. package/bundled-tools/clis/security_data/tools.lock.json +358 -0
  73. package/package.json +1 -1
@@ -0,0 +1,1023 @@
1
+ #!/usr/bin/env python3
2
+ """Stack inventory for ``maggie security inventory`` (PRD 01 section 6.1).
3
+
4
+ Builds ``maggie-security-stack.v1`` from what is on disk and, optionally,
5
+ from read-only host evidence. Covers the popular stacks:
6
+
7
+ - JavaScript: package-lock.json (v1-v3), npm-shrinkwrap.json, pnpm-lock.yaml,
8
+ yarn.lock (v1 and berry), bun.lock
9
+ - PHP: composer.lock
10
+ - Python: requirements*.txt (pinned), Pipfile.lock, poetry.lock, uv.lock, pdm.lock
11
+ - Java/Kotlin: pom.xml (with properties), gradle.lockfile, gradle/verification-metadata.xml
12
+ - Go: go.mod (+ go directive), Ruby: Gemfile.lock (+ RUBY VERSION), Rust: Cargo.lock,
13
+ .NET: packages.lock.json, *.csproj PackageReference, global.json
14
+ - WordPress core/plugins/themes from files
15
+ - Containers: Dockerfile FROM, docker-compose/compose image lines -> images and
16
+ the runtime/database/server they carry (python, node, php, java, mysql,
17
+ postgres, redis, mongo, nginx, httpd, tomcat, ...)
18
+ - Declared runtimes: .nvmrc, .node-version, .python-version, runtime.txt,
19
+ .ruby-version, .tool-versions, .java-version, package.json engines
20
+ - Hosts: OS release, OS packages (dpkg / rpm / apk), and service versions
21
+ (nginx, apache, php, node, python, java, go, mysql/mariadb, postgresql,
22
+ redis, mongodb, memcached, docker, openssl) from the read-only collector
23
+ printed by ``--print-collector`` (the runtime-inventory collector plus
24
+ version probes), or from ``maggie deployment runtime-inventory`` JSON.
25
+
26
+ Parsers are stdlib-only and tolerant: an unreadable file becomes a degraded
27
+ source, never a crash. Lockfile test data under fixtures/security/lockfiles
28
+ comes from google/osv-scanner testdata (Apache-2.0).
29
+ """
30
+
31
+ from __future__ import annotations
32
+
33
+ import json
34
+ import re
35
+ import xml.etree.ElementTree as ET
36
+ from pathlib import Path
37
+
38
+ import maggie_security_core as core
39
+ import maggie_security_versions as versions
40
+ from maggie_host_runtime import COLLECTOR as RUNTIME_COLLECTOR, parse_os, parse_sections
41
+
42
+ SKIP_DIRS = {"node_modules", ".git", "vendor", ".venv", "venv", "__pycache__", "dist", "build", ".next", ".astro", ".output",
43
+ "target", ".gradle", ".maggie", ".wrangler", "wp-admin", "wp-includes", "bower_components", ".cache"}
44
+ MAX_DEPTH = 4
45
+
46
+ # Default CVE product allowlist per site type (PRD 01 section 6.1); runtimes
47
+ # and images found are added automatically.
48
+ WATCH_DEFAULTS = {
49
+ "node-app": ["node", "nginx", "openssl"], "astro": ["node", "astro"], "workers": [],
50
+ "wordpress": ["wordpress", "php", "nginx", "apache", "mysql", "mariadb", "openssl"],
51
+ "php-app": ["php", "nginx", "apache", "mysql", "mariadb", "openssl"], "static": ["nginx"], "unknown": [],
52
+ "python-app": ["python", "nginx", "postgresql", "redis", "openssl"], "java-app": ["java", "tomcat", "nginx", "postgresql", "mysql"],
53
+ "go-app": ["go", "nginx", "postgresql"], "ruby-app": ["ruby", "nginx", "postgresql", "redis"], "dotnet-app": ["dotnet", "mssqlserver", "nginx"],
54
+ }
55
+
56
+ # Image repository -> runtime/service name (endoflife.date product mapping lives in the intel module).
57
+ IMAGE_RUNTIMES = {
58
+ "node": "node", "python": "python", "php": "php", "wordpress": "wordpress", "ruby": "ruby", "golang": "go",
59
+ "openjdk": "java", "eclipse-temurin": "java", "amazoncorretto": "java", "ibm-semeru-runtimes": "java", "maven": "java",
60
+ "tomcat": "tomcat", "mysql": "mysql", "mariadb": "mariadb", "postgres": "postgresql", "postgresql": "postgresql",
61
+ "redis": "redis", "valkey": "valkey", "mongo": "mongodb", "mongodb": "mongodb", "memcached": "memcached",
62
+ "nginx": "nginx", "httpd": "apache", "caddy": "caddy", "haproxy": "haproxy", "traefik": "traefik",
63
+ "elasticsearch": "elasticsearch", "opensearch": "opensearch", "rabbitmq": "rabbitmq", "mcr.microsoft.com/dotnet/aspnet": "dotnet",
64
+ "mcr.microsoft.com/dotnet/runtime": "dotnet", "mcr.microsoft.com/dotnet/sdk": "dotnet", "mcr.microsoft.com/mssql/server": "mssqlserver",
65
+ "ubuntu": "ubuntu", "debian": "debian", "alpine": "alpine", "rockylinux": "rocky-linux", "almalinux": "almalinux",
66
+ "amazonlinux": "amazon-linux", "centos": "centos", "fedora": "fedora", "bun": "bun", "denoland/deno": "deno",
67
+ "rust": "rust", "kong": "kong", "keycloak": "keycloak", "grafana/grafana": "grafana", "sonarqube": "sonarqube",
68
+ }
69
+
70
+ # Read-only collector: the runtime-inventory sections plus version probes and
71
+ # OS package lists. Every probe is a version query or a read of a package
72
+ # database; nothing is written on the host.
73
+ SECURITY_COLLECTOR = RUNTIME_COLLECTOR + r"""
74
+ s nginx-v; nginx -v 2>&1 | head -1
75
+ s apache-v; (apache2 -v 2>/dev/null || httpd -v 2>/dev/null) | head -1
76
+ s node-v; node --version 2>/dev/null
77
+ s python-v; (python3 --version 2>&1 || python --version 2>&1) | head -1
78
+ s java-v; java -version 2>&1 | head -1
79
+ s go-v; go version 2>/dev/null
80
+ s ruby-v; ruby --version 2>/dev/null
81
+ s dotnet-v; dotnet --list-runtimes 2>/dev/null | head -5
82
+ s mysql-v; (mysqld --version 2>/dev/null || mysql --version 2>/dev/null || mariadbd --version 2>/dev/null) | head -1
83
+ s postgres-v; (postgres --version 2>/dev/null || psql --version 2>/dev/null || ls /usr/lib/postgresql 2>/dev/null) | head -3
84
+ s redis-v; redis-server --version 2>/dev/null
85
+ s mongodb-v; mongod --version 2>/dev/null | head -1
86
+ s memcached-v; memcached -V 2>/dev/null
87
+ s openssl-v; openssl version 2>/dev/null
88
+ s docker-v; docker --version 2>/dev/null
89
+ s tomcat-v; ls -d /opt/tomcat* /usr/share/tomcat* 2>/dev/null | head -3
90
+ s dpkg; dpkg-query -W -f='${Package}\t${Version}\t${Status}\n' 2>/dev/null
91
+ s rpm; rpm -qa --qf '%{NAME}\t%{EPOCHNUM}:%{VERSION}-%{RELEASE}\n' 2>/dev/null
92
+ s apk; apk info -v 2>/dev/null
93
+ s kernel; uname -r
94
+ """
95
+
96
+ SERVICE_PROBES = {
97
+ "nginx-v": ("nginx", r"nginx/(\d+\.\d+\.\d+)"),
98
+ "apache-v": ("apache", r"Apache/(\d+\.\d+\.\d+)"),
99
+ "node-v": ("node", r"v?(\d+\.\d+\.\d+)"),
100
+ "python-v": ("python", r"Python (\d+\.\d+\.\d+)"),
101
+ "java-v": ("java", r"version \"(\d+(?:\.\d+)*)"),
102
+ "go-v": ("go", r"go(\d+\.\d+(?:\.\d+)?)"),
103
+ "ruby-v": ("ruby", r"ruby (\d+\.\d+\.\d+)"),
104
+ "dotnet-v": ("dotnet", r"Microsoft\.\w+\.App (\d+\.\d+\.\d+)"),
105
+ "mysql-v": (None, r"(?:Ver |Distrib )(\d+\.\d+\.\d+)(?:-(MariaDB))?"),
106
+ "postgres-v": ("postgresql", r"(?:PostgreSQL\)? |^)(\d+(?:\.\d+)?)"),
107
+ "redis-v": ("redis", r"v=(\d+\.\d+\.\d+)"),
108
+ "mongodb-v": ("mongodb", r"v(\d+\.\d+\.\d+)"),
109
+ "memcached-v": ("memcached", r"memcached (\d+\.\d+\.\d+)"),
110
+ "openssl-v": ("openssl", r"OpenSSL (\d+\.\d+\.\d+[a-z]?)"),
111
+ "docker-v": ("docker-engine", r"version (\d+\.\d+\.\d+)"),
112
+ "php-v": ("php", r"PHP (\d+\.\d+\.\d+)"),
113
+ }
114
+
115
+ # Products whose endoflife.date release cycle is the major version only.
116
+ MAJOR_CYCLES = {"node", "java", "debian", "rocky-linux", "almalinux", "rhel", "centos", "fedora", "docker-engine", "amazon-linux",
117
+ "elasticsearch", "opensearch", "deno", "bun"}
118
+
119
+ OS_ECOSYSTEM = {"debian": "Debian", "ubuntu": "Ubuntu", "alpine": "Alpine", "rhel": "Red Hat", "rocky": "Rocky Linux",
120
+ "almalinux": "AlmaLinux", "opensuse-leap": "openSUSE", "sles": "SUSE", "amzn": "Amazon Linux"}
121
+
122
+
123
+ class Inventory:
124
+ def __init__(self, project: Path, site: str, site_type: str) -> None:
125
+ self.project = project
126
+ self.site = site
127
+ self.site_type = site_type
128
+ self.sources: list[dict] = []
129
+ self.components: list[dict] = []
130
+ self.runtimes: list[dict] = []
131
+ self.images: list[dict] = []
132
+ self.hosts: list[dict] = []
133
+ self.wordpress: dict | None = None
134
+ self.problems: list[str] = []
135
+
136
+ # ------------------------------------------------------------------ helpers
137
+
138
+ def rel(self, path: Path) -> str:
139
+ try:
140
+ return str(path.resolve().relative_to(self.project.resolve()))
141
+ except ValueError:
142
+ return path.name
143
+
144
+ def add(self, ecosystem: str, name: str, version: str, path: Path, scope: str = "runtime", direct: bool | None = None) -> None:
145
+ version = (version or "").strip()
146
+ name = (name or "").strip()
147
+ if not name or not version or version.startswith(("file:", "link:", "workspace:", "git", "http", "github:", "portal:")):
148
+ return
149
+ if ecosystem == "npm" and version.startswith("npm:"):
150
+ alias = version[4:]
151
+ name, _, version = alias.rpartition("@")
152
+ self.components.append({"purl": core.make_purl(ecosystem, name, version), "ecosystem": ecosystem, "name": name,
153
+ "version": version, "scope": scope, "direct": direct, "location": self.rel(path)})
154
+
155
+ def add_runtime(self, name: str, version: str, source: str, host: str | None = None, **extra: str) -> None:
156
+ version = (version or "").strip().lstrip("v")
157
+ if not name or not version or not re.match(r"\d", version):
158
+ return
159
+ cycle = versions.cycle_of(version, 1 if name in MAJOR_CYCLES or (name == "postgresql" and not version.startswith("9.")) else 2)
160
+ entry = {"name": name, "version": version, "cycle": cycle, "host": host, "source": source}
161
+ entry.update({k: v for k, v in extra.items() if v})
162
+ key = (name, version, host)
163
+ if not any((r["name"], r["version"], r.get("host")) == key for r in self.runtimes):
164
+ self.runtimes.append(entry)
165
+
166
+ def source(self, kind: str, path: Path | str) -> None:
167
+ rel = self.rel(path) if isinstance(path, Path) else path
168
+ if not any(s["path"] == rel and s["kind"] == kind for s in self.sources):
169
+ self.sources.append({"kind": kind, "path": rel})
170
+
171
+ def read(self, path: Path) -> str | None:
172
+ try:
173
+ return path.read_text(encoding="utf-8", errors="replace")
174
+ except OSError as error:
175
+ self.problems.append(f"{self.rel(path)}: {error}")
176
+ return None
177
+
178
+ # ------------------------------------------------------------------ discovery
179
+
180
+ def discover(self) -> list[Path]:
181
+ out: list[Path] = []
182
+
183
+ def walk(directory: Path, depth: int) -> None:
184
+ try:
185
+ entries = sorted(directory.iterdir())
186
+ except OSError:
187
+ return
188
+ for entry in entries:
189
+ if entry.is_dir():
190
+ if entry.name in SKIP_DIRS or (entry.name.startswith(".") and entry.name not in (".github",)) or depth >= MAX_DEPTH:
191
+ continue
192
+ walk(entry, depth + 1)
193
+ elif entry.is_file():
194
+ out.append(entry)
195
+
196
+ walk(self.project, 0)
197
+ return out
198
+
199
+ def scan_project(self) -> None:
200
+ files = self.discover()
201
+ names = {}
202
+ for path in files:
203
+ names.setdefault(path.name, []).append(path)
204
+ handled_npm_dirs: set[Path] = set()
205
+ for name in ("package-lock.json", "npm-shrinkwrap.json", "pnpm-lock.yaml", "yarn.lock", "bun.lock"):
206
+ for path in names.get(name, []):
207
+ if path.parent in handled_npm_dirs:
208
+ continue
209
+ handled_npm_dirs.add(path.parent)
210
+ self.safe(getattr(self, "parse_" + name.replace("-", "_").replace(".", "_")), path)
211
+ dispatch = {
212
+ "composer.lock": self.parse_composer_lock, "Pipfile.lock": self.parse_pipfile_lock, "poetry.lock": self.parse_toml_lock,
213
+ "uv.lock": self.parse_toml_lock, "pdm.lock": self.parse_toml_lock, "Cargo.lock": self.parse_toml_lock,
214
+ "pom.xml": self.parse_pom, "gradle.lockfile": self.parse_gradle_lock, "go.mod": self.parse_go_mod,
215
+ "Gemfile.lock": self.parse_gemfile_lock, "packages.lock.json": self.parse_nuget_lock, "global.json": self.parse_global_json,
216
+ "Dockerfile": self.parse_dockerfile, "Containerfile": self.parse_dockerfile,
217
+ ".nvmrc": self.parse_version_file, ".node-version": self.parse_version_file, ".python-version": self.parse_version_file,
218
+ ".ruby-version": self.parse_version_file, ".java-version": self.parse_version_file, "runtime.txt": self.parse_version_file,
219
+ ".tool-versions": self.parse_tool_versions, "package.json": self.parse_package_json,
220
+ }
221
+ for name, parser in dispatch.items():
222
+ for path in names.get(name, []):
223
+ if name == "package.json" and path.parent != self.project:
224
+ continue
225
+ self.safe(parser, path)
226
+ for path in files:
227
+ lower = path.name.lower()
228
+ if re.fullmatch(r"requirements[\w.-]*\.(txt|in)", lower) and lower.endswith(".txt"):
229
+ self.safe(self.parse_requirements, path)
230
+ elif lower.endswith(".csproj") or lower.endswith(".fsproj") or lower == "directory.packages.props":
231
+ self.safe(self.parse_csproj, path)
232
+ elif re.fullmatch(r"(docker-)?compose[\w.-]*\.ya?ml", lower):
233
+ self.safe(self.parse_compose, path)
234
+ elif lower.startswith("dockerfile.") or lower.endswith(".dockerfile"):
235
+ self.safe(self.parse_dockerfile, path)
236
+ wp_root = self.find_wordpress()
237
+ if wp_root:
238
+ self.safe(self.parse_wordpress, wp_root)
239
+ self.scan_js_bundles()
240
+
241
+ def safe(self, parser, path: Path) -> None:
242
+ try:
243
+ parser(path)
244
+ except Exception as error: # noqa: BLE001 - a bad file degrades, never crashes
245
+ self.problems.append(f"{self.rel(path)}: could not parse ({type(error).__name__}: {error})")
246
+
247
+ # ------------------------------------------------------------------ JavaScript
248
+
249
+ def parse_package_lock_json(self, path: Path) -> None:
250
+ data = json.loads(self.read(path) or "{}")
251
+ self.source("lockfile", path)
252
+ root = (data.get("packages") or {}).get("") or {}
253
+ direct = set(root.get("dependencies") or {}) | set(root.get("devDependencies") or {}) | set(root.get("optionalDependencies") or {})
254
+ if data.get("packages"):
255
+ for key, meta in data["packages"].items():
256
+ if not key or "node_modules/" not in key or meta.get("link"):
257
+ continue
258
+ name = meta.get("name") or key.rsplit("node_modules/", 1)[1]
259
+ scope = "dev" if meta.get("dev") or meta.get("devOptional") else "runtime"
260
+ self.add("npm", name, meta.get("version", ""), path, scope, name in direct and key == f"node_modules/{name}")
261
+ else: # lockfileVersion 1
262
+ def walk(deps: dict, top: bool) -> None:
263
+ for name, meta in (deps or {}).items():
264
+ self.add("npm", name, meta.get("version", ""), path, "dev" if meta.get("dev") else "runtime", top)
265
+ walk(meta.get("dependencies") or {}, False)
266
+ walk(data.get("dependencies") or {}, True)
267
+
268
+ parse_npm_shrinkwrap_json = parse_package_lock_json
269
+
270
+ def parse_pnpm_lock_yaml(self, path: Path) -> None:
271
+ text = self.read(path) or ""
272
+ self.source("lockfile", path)
273
+ section = None
274
+ dev_names: set[str] = set()
275
+ direct: set[str] = set()
276
+ importer_kind = None
277
+ for line in text.splitlines():
278
+ if not line.strip() or line.lstrip().startswith("#"):
279
+ continue
280
+ indent = len(line) - len(line.lstrip())
281
+ if indent == 0:
282
+ section = line.rstrip(":").strip()
283
+ continue
284
+ if section == "importers":
285
+ stripped = line.strip()
286
+ if stripped in ("dependencies:", "devDependencies:", "optionalDependencies:"):
287
+ importer_kind = stripped[:-1]
288
+ elif indent in (6, 2) and stripped.endswith(":") and importer_kind:
289
+ name = stripped[:-1].strip("'\"")
290
+ direct.add(name)
291
+ if importer_kind == "devDependencies":
292
+ dev_names.add(name)
293
+ elif section in ("dependencies", "devDependencies") and indent == 2 and line.strip().endswith(":"):
294
+ name = line.strip()[:-1].strip("'\"")
295
+ direct.add(name)
296
+ if section == "devDependencies":
297
+ dev_names.add(name)
298
+ elif section == "packages" and indent == 2 and line.rstrip().endswith(":"):
299
+ key = line.strip()[:-1].strip("'\"").lstrip("/")
300
+ key = key.split("(", 1)[0]
301
+ match = re.match(r"^(@?[^@/]+(?:/[^@/]+)?)[@/](\d[^/_()]*)", key)
302
+ if match:
303
+ name, version = match.group(1), match.group(2)
304
+ self.add("npm", name, version, path, "dev" if name in dev_names else "runtime", name in direct)
305
+
306
+ def parse_yarn_lock(self, path: Path) -> None:
307
+ text = self.read(path) or ""
308
+ self.source("lockfile", path)
309
+ names: list[str] = []
310
+ for line in text.splitlines():
311
+ if not line.strip() or line.startswith("#"):
312
+ continue
313
+ if not line.startswith(" ") and line.rstrip().endswith(":"):
314
+ header = line.rstrip()[:-1]
315
+ names = []
316
+ for spec in header.split(","):
317
+ spec = spec.strip().strip('"')
318
+ if spec.startswith("__metadata"):
319
+ continue
320
+ name = spec[: spec.rfind("@")] if spec.rfind("@") > 0 else spec
321
+ if name and name not in names:
322
+ names.append(name)
323
+ else:
324
+ match = re.match(r'^\s+version:?\s+"?([^"\s]+)"?', line)
325
+ if match and names:
326
+ for name in names[:1]:
327
+ self.add("npm", name, match.group(1), path)
328
+ names = []
329
+
330
+ def parse_bun_lock(self, path: Path) -> None:
331
+ text = self.read(path) or ""
332
+ self.source("lockfile", path)
333
+ cleaned = re.sub(r",(\s*[}\]])", r"\1", text) # bun.lock is JSONC with trailing commas
334
+ data = json.loads(cleaned)
335
+ for entry in (data.get("packages") or {}).values():
336
+ if isinstance(entry, list) and entry and isinstance(entry[0], str):
337
+ spec = entry[0]
338
+ name, _, version = spec.rpartition("@")
339
+ if name and re.match(r"\d", version):
340
+ self.add("npm", name, version, path)
341
+
342
+ def parse_package_json(self, path: Path) -> None:
343
+ data = json.loads(self.read(path) or "{}")
344
+ engine = ((data.get("engines") or {}).get("node") or "").strip()
345
+ match = re.search(r"(\d+(?:\.\d+){0,2})", engine)
346
+ if match and not any(r["name"] == "node" for r in self.runtimes):
347
+ self.add_runtime("node", match.group(1) if match.group(1).count(".") else match.group(1) + ".0.0", "declared")
348
+ if not any(c["ecosystem"] == "npm" and c["location"].startswith(self.rel(path.parent)) for c in self.components):
349
+ # No lockfile: record exact pins from package.json at lower confidence.
350
+ for section, scope in (("dependencies", "runtime"), ("devDependencies", "dev")):
351
+ for name, spec in (data.get(section) or {}).items():
352
+ if isinstance(spec, str) and re.fullmatch(r"\d+\.\d+\.\d+(?:-[\w.]+)?", spec.strip()):
353
+ self.add("npm", name, spec.strip(), path, scope, True)
354
+ if data.get("dependencies") or data.get("devDependencies"):
355
+ self.source("manifest", path)
356
+
357
+ # ------------------------------------------------------------------ PHP / Python / Ruby / Rust / Go / .NET / Java
358
+
359
+ def parse_composer_lock(self, path: Path) -> None:
360
+ data = json.loads(self.read(path) or "{}")
361
+ self.source("lockfile", path)
362
+ manifest = path.parent / "composer.json"
363
+ direct: set[str] = set()
364
+ if manifest.is_file():
365
+ with_json = json.loads(manifest.read_text(encoding="utf-8") or "{}")
366
+ direct = {k.lower() for k in list((with_json.get("require") or {})) + list((with_json.get("require-dev") or {}))}
367
+ php = (with_json.get("require") or {}).get("php") or ""
368
+ for key, scope in (("packages", "runtime"), ("packages-dev", "dev")):
369
+ for pkg in data.get(key) or []:
370
+ version = str(pkg.get("version", "")).lstrip("v")
371
+ if version.startswith("dev-") or version.endswith("-dev"):
372
+ continue
373
+ self.add("Packagist", pkg.get("name", "").lower(), version, path, scope, pkg.get("name", "").lower() in direct)
374
+
375
+ def parse_requirements(self, path: Path) -> None:
376
+ text = self.read(path) or ""
377
+ found = False
378
+ scope = "dev" if re.search(r"(dev|test|lint|doc)", path.name) else "runtime"
379
+ for line in text.splitlines():
380
+ line = line.split("#", 1)[0].strip()
381
+ match = re.match(r"^([A-Za-z0-9][A-Za-z0-9._-]*)(?:\[[^\]]*\])?\s*===?\s*([^\s;,]+)", line)
382
+ if match:
383
+ self.add("PyPI", re.sub(r"[-_.]+", "-", match.group(1)).lower(), match.group(2), path, scope, True)
384
+ found = True
385
+ if found:
386
+ self.source("lockfile", path)
387
+
388
+ def parse_pipfile_lock(self, path: Path) -> None:
389
+ data = json.loads(self.read(path) or "{}")
390
+ self.source("lockfile", path)
391
+ for section, scope in (("default", "runtime"), ("develop", "dev")):
392
+ for name, meta in (data.get(section) or {}).items():
393
+ version = str((meta or {}).get("version", "")).lstrip("=")
394
+ self.add("PyPI", re.sub(r"[-_.]+", "-", name).lower(), version, path, scope)
395
+
396
+ def parse_toml_lock(self, path: Path) -> None:
397
+ """[[package]] tables with name/version (poetry.lock, uv.lock, pdm.lock, Cargo.lock)."""
398
+ text = self.read(path) or ""
399
+ ecosystem = "crates.io" if path.name == "Cargo.lock" else "PyPI"
400
+ self.source("lockfile", path)
401
+ for block in re.split(r"^\[\[package\]\]\s*$", text, flags=re.M)[1:]:
402
+ body = re.split(r"^\[", block, maxsplit=1, flags=re.M)[0]
403
+ name = re.search(r'^name\s*=\s*"([^"]+)"', body, re.M)
404
+ version = re.search(r'^version\s*=\s*"([^"]+)"', body, re.M)
405
+ if not name or not version:
406
+ continue
407
+ source = re.search(r"^source\s*=\s*(.+)$", body, re.M)
408
+ if source and ("virtual" in source.group(1) or "editable" in source.group(1) or "path" in source.group(1)):
409
+ continue
410
+ if ecosystem == "crates.io" and not source:
411
+ continue # workspace crate, not from a registry
412
+ category = re.search(r'^category\s*=\s*"([^"]+)"', body, re.M)
413
+ groups = re.search(r'^groups\s*=\s*\[([^\]]*)\]', body, re.M)
414
+ dev = (category and category.group(1) == "dev") or (groups and "main" not in groups.group(1) and "default" not in groups.group(1))
415
+ pkg = name.group(1) if ecosystem == "crates.io" else re.sub(r"[-_.]+", "-", name.group(1)).lower()
416
+ self.add(ecosystem, pkg, version.group(1), path, "dev" if dev else "runtime")
417
+ if path.name == "uv.lock" or path.name == "poetry.lock":
418
+ match = re.search(r'^requires-python\s*=\s*"[^\d]*(\d+\.\d+)', text, re.M)
419
+ if match and not any(r["name"] == "python" for r in self.runtimes):
420
+ self.add_runtime("python", match.group(1), "declared")
421
+
422
+ def parse_gemfile_lock(self, path: Path) -> None:
423
+ text = self.read(path) or ""
424
+ self.source("lockfile", path)
425
+ section = None
426
+ for line in text.splitlines():
427
+ if line and not line.startswith(" "):
428
+ section = line.strip()
429
+ continue
430
+ if section in ("GEM", "GIT", "PATH") and re.match(r"^ {4}\S", line):
431
+ match = re.match(r"^ {4}([^\s(]+) \(([^)]+)\)", line)
432
+ if match and section == "GEM":
433
+ self.add("RubyGems", match.group(1), match.group(2).split("-", 1)[0], path)
434
+ if section == "RUBY VERSION":
435
+ match = re.search(r"ruby (\d+\.\d+\.\d+)", line)
436
+ if match:
437
+ self.add_runtime("ruby", match.group(1), "declared")
438
+
439
+ def parse_go_mod(self, path: Path) -> None:
440
+ text = self.read(path) or ""
441
+ self.source("lockfile", path)
442
+ go = re.search(r"^go (\d+\.\d+(?:\.\d+)?)", text, re.M)
443
+ if go:
444
+ self.add_runtime("go", go.group(1), "declared")
445
+ in_block = False
446
+ for line in text.splitlines():
447
+ stripped = line.split("//", 1)[0].strip()
448
+ indirect = "// indirect" in line
449
+ if stripped.startswith("require ("):
450
+ in_block = True
451
+ continue
452
+ if in_block and stripped == ")":
453
+ in_block = False
454
+ continue
455
+ match = re.match(r"^(?:require\s+)?(\S+)\s+(v\S+)$", stripped) if (in_block or stripped.startswith("require ")) else None
456
+ if match:
457
+ self.add("Go", match.group(1), match.group(2).lstrip("v"), path, "runtime", not indirect)
458
+
459
+ def parse_nuget_lock(self, path: Path) -> None:
460
+ data = json.loads(self.read(path) or "{}")
461
+ self.source("lockfile", path)
462
+ for framework in (data.get("dependencies") or {}).values():
463
+ for name, meta in (framework or {}).items():
464
+ if (meta or {}).get("type") == "Project":
465
+ continue
466
+ self.add("NuGet", name, (meta or {}).get("resolved", ""), path, "runtime", (meta or {}).get("type") == "Direct")
467
+
468
+ def parse_csproj(self, path: Path) -> None:
469
+ root = ET.fromstring(self.read(path) or "<x/>")
470
+ found = False
471
+ for element in root.iter():
472
+ tag = element.tag.rsplit("}", 1)[-1]
473
+ if tag in ("PackageReference", "PackageVersion"):
474
+ name = element.get("Include") or element.get("Update")
475
+ version = element.get("Version") or (element.findtext("Version") or "")
476
+ if name and re.fullmatch(r"\d+(\.\d+){1,3}(-[\w.]+)?", version.strip()):
477
+ self.add("NuGet", name, version.strip(), path, "runtime", True)
478
+ found = True
479
+ if tag == "TargetFramework" and element.text:
480
+ match = re.match(r"net(\d+\.\d+)", element.text.strip())
481
+ if match:
482
+ self.add_runtime("dotnet", match.group(1), "declared")
483
+ if found:
484
+ self.source("manifest", path)
485
+
486
+ def parse_global_json(self, path: Path) -> None:
487
+ data = json.loads(self.read(path) or "{}")
488
+ version = (data.get("sdk") or {}).get("version")
489
+ if version:
490
+ self.add_runtime("dotnet", ".".join(version.split(".")[:2]), "declared")
491
+
492
+ def parse_pom(self, path: Path) -> None:
493
+ root = ET.fromstring(self.read(path) or "<x/>")
494
+ ns = {"m": root.tag[1:].split("}", 1)[0]} if root.tag.startswith("{") else {}
495
+ q = (lambda tag: f"m:{tag}") if ns else (lambda tag: tag)
496
+ props: dict[str, str] = {}
497
+ properties = root.find(q("properties"), ns)
498
+ if properties is not None:
499
+ for prop in properties:
500
+ props[prop.tag.rsplit("}", 1)[-1]] = (prop.text or "").strip()
501
+ props["project.version"] = (root.findtext(q("version"), default="", namespaces=ns) or "").strip()
502
+
503
+ def resolve(value: str) -> str:
504
+ return re.sub(r"\$\{([^}]+)\}", lambda m: props.get(m.group(1), m.group(0)), value or "")
505
+
506
+ java = props.get("maven.compiler.release") or props.get("java.version") or props.get("maven.compiler.source") or ""
507
+ if java:
508
+ self.add_runtime("java", java.replace("1.", "", 1) if java.startswith("1.") else java, "declared")
509
+ found = False
510
+ for container in (root.find(q("dependencies"), ns), root.find(f"{q('dependencyManagement')}/{q('dependencies')}", ns)):
511
+ if container is None:
512
+ continue
513
+ for dep in container.findall(q("dependency"), ns):
514
+ group = resolve(dep.findtext(q("groupId"), default="", namespaces=ns).strip())
515
+ artifact = resolve(dep.findtext(q("artifactId"), default="", namespaces=ns).strip())
516
+ version = resolve(dep.findtext(q("version"), default="", namespaces=ns).strip())
517
+ scope = dep.findtext(q("scope"), default="compile", namespaces=ns).strip()
518
+ if "${" in version or not version or version.startswith(("[", "(")):
519
+ continue
520
+ self.add("Maven", f"{group}:{artifact}", version, path, "dev" if scope in ("test", "provided") else "runtime", True)
521
+ found = True
522
+ parent = root.find(q("parent"), ns)
523
+ if parent is not None:
524
+ group = parent.findtext(q("groupId"), default="", namespaces=ns).strip()
525
+ artifact = parent.findtext(q("artifactId"), default="", namespaces=ns).strip()
526
+ version = parent.findtext(q("version"), default="", namespaces=ns).strip()
527
+ if group and artifact and version:
528
+ self.add("Maven", f"{group}:{artifact}", version, path, "build", True)
529
+ found = True
530
+ if found:
531
+ self.source("manifest", path)
532
+
533
+ def parse_gradle_lock(self, path: Path) -> None:
534
+ text = self.read(path) or ""
535
+ self.source("lockfile", path)
536
+ for line in text.splitlines():
537
+ if line.startswith("#") or "=" not in line:
538
+ continue
539
+ coords, _, configurations = line.partition("=")
540
+ parts = coords.strip().split(":")
541
+ if len(parts) == 3:
542
+ scope = "dev" if configurations and all("test" in c.lower() for c in configurations.split(",")) else "runtime"
543
+ self.add("Maven", f"{parts[0]}:{parts[1]}", parts[2], path, scope)
544
+
545
+ # ------------------------------------------------------------------ containers and declared runtimes
546
+
547
+ def add_image(self, ref: str, path: Path | str, host: str | None = None) -> None:
548
+ ref = ref.strip().strip('"\'')
549
+ if not ref or "$" in ref or ref.lower() == "scratch":
550
+ return
551
+ if any(i["ref"] == ref and i.get("host") == host for i in self.images):
552
+ return
553
+ digest = None
554
+ if "@sha256:" in ref:
555
+ ref, _, digest_hex = ref.partition("@sha256:")
556
+ digest = "sha256:" + digest_hex
557
+ self.images.append({"ref": ref, "digest": digest, "host": host})
558
+ name, _, tag = ref.rpartition(":") if ":" in ref.rsplit("/", 1)[-1] else (ref, "", "latest")
559
+ repo = re.sub(r"^(docker\.io/)?(library/)?", "", name)
560
+ runtime = IMAGE_RUNTIMES.get(repo) or IMAGE_RUNTIMES.get(repo.rsplit("/", 1)[-1])
561
+ version = re.match(r"^v?(\d+(?:\.\d+){0,2})", tag or "")
562
+ if runtime and version:
563
+ self.add_runtime(runtime, version.group(1), "image", host, image=ref)
564
+ os_hint = re.search(r"(alpine|bookworm|bullseye|buster|jammy|noble|focal)", tag or "")
565
+ if os_hint and runtime not in ("alpine", "debian", "ubuntu"):
566
+ codename = {"bookworm": ("debian", "12"), "bullseye": ("debian", "11"), "buster": ("debian", "10"),
567
+ "jammy": ("ubuntu", "22.04"), "noble": ("ubuntu", "24.04"), "focal": ("ubuntu", "20.04")}.get(os_hint.group(1))
568
+ if codename:
569
+ self.add_runtime(codename[0], codename[1], "image", host, image=ref)
570
+
571
+ def parse_dockerfile(self, path: Path) -> None:
572
+ text = self.read(path) or ""
573
+ stages: set[str] = set()
574
+ for line in text.splitlines():
575
+ match = re.match(r"^\s*FROM\s+(?:--platform=\S+\s+)?(\S+)(?:\s+AS\s+(\S+))?", line, re.I)
576
+ if match:
577
+ ref = match.group(1)
578
+ if ref not in stages:
579
+ self.add_image(ref, path)
580
+ if match.group(2):
581
+ stages.add(match.group(2))
582
+ self.source("manifest", path)
583
+
584
+ def parse_compose(self, path: Path) -> None:
585
+ text = self.read(path) or ""
586
+ for match in re.finditer(r"^\s*image:\s*['\"]?([^\s'\"#]+)", text, re.M):
587
+ self.add_image(match.group(1), path)
588
+ self.source("manifest", path)
589
+
590
+ def parse_version_file(self, path: Path) -> None:
591
+ text = (self.read(path) or "").strip().splitlines()
592
+ value = text[0].strip() if text else ""
593
+ name = {".nvmrc": "node", ".node-version": "node", ".python-version": "python", ".ruby-version": "ruby",
594
+ ".java-version": "java", "runtime.txt": "python"}[path.name]
595
+ match = re.search(r"(\d+(?:\.\d+){0,2})", value)
596
+ if match:
597
+ if path.name == "runtime.txt" and not value.startswith("python"):
598
+ return
599
+ self.add_runtime(name, match.group(1), "declared")
600
+
601
+ def parse_tool_versions(self, path: Path) -> None:
602
+ mapping = {"nodejs": "node", "python": "python", "ruby": "ruby", "golang": "go", "java": "java", "php": "php",
603
+ "erlang": "erlang", "elixir": "elixir", "rust": "rust", "dotnet-core": "dotnet", "postgres": "postgresql",
604
+ "redis": "redis", "mysql": "mysql"}
605
+ for line in (self.read(path) or "").splitlines():
606
+ parts = line.split("#", 1)[0].split()
607
+ if len(parts) >= 2 and parts[0] in mapping:
608
+ version = re.search(r"(\d+(?:\.\d+){0,2})", parts[1])
609
+ if version:
610
+ self.add_runtime(mapping[parts[0]], version.group(1), "declared")
611
+
612
+ # ------------------------------------------------------------------ shipped JavaScript (retire.js extractors)
613
+
614
+ def scan_js_bundles(self, limit: int = 400) -> None:
615
+ """Libraries copied into the site (static/, public/, dist/, assets/, wp-content/) rather than installed.
616
+
617
+ Uses the filename and banner extractors from retire.js (Apache-2.0,
618
+ security_data/retire-extractors.json). Only the first 2 KB of each file
619
+ is read, where banners live.
620
+ """
621
+ extractors = _retire_extractors()
622
+ roots = [self.project / d for d in ("", "static", "public", "dist", "assets", "js", "wp-content/themes", "wp-content/plugins", "www", "build")]
623
+ seen: set[Path] = set()
624
+ found = 0
625
+ for root in roots:
626
+ if not root.is_dir():
627
+ continue
628
+ pattern = "**/*.js" if root != self.project else "*.js"
629
+ for path in root.glob(pattern):
630
+ if path in seen or "node_modules" in path.parts or found >= limit:
631
+ continue
632
+ seen.add(path)
633
+ try:
634
+ with path.open("r", encoding="utf-8", errors="replace") as handle:
635
+ head = handle.read(2048)
636
+ except OSError:
637
+ continue
638
+ for lib, (name_res, content_res, npm) in extractors.items():
639
+ version = None
640
+ for rx in name_res:
641
+ match = rx.search(path.name)
642
+ if match:
643
+ version = match.group(1)
644
+ break
645
+ if not version:
646
+ for rx in content_res:
647
+ match = rx.search(head)
648
+ if match:
649
+ version = match.group(1)
650
+ break
651
+ if version:
652
+ version = re.sub(r"([.-](min|slim|pack|prod|production|umd|esm|bundle|js))+$", "", version).rstrip(".-_")
653
+ self.components.append({"purl": core.make_purl("npm", npm, version), "ecosystem": "npm", "name": npm,
654
+ "version": version, "scope": "runtime", "direct": None, "location": self.rel(path)})
655
+ self.source("js-bundle", path)
656
+ found += 1
657
+ break
658
+
659
+ # ------------------------------------------------------------------ WordPress
660
+
661
+ def find_wordpress(self) -> Path | None:
662
+ for candidate in (self.project, self.project / "wordpress", self.project / "public", self.project / "web", self.project / "htdocs"):
663
+ if (candidate / "wp-includes" / "version.php").is_file() or (candidate / "wp-content").is_dir():
664
+ return candidate
665
+ return None
666
+
667
+ @staticmethod
668
+ def header(text: str, field: str) -> str | None:
669
+ match = re.search(rf"^[ \t/*#@]*{re.escape(field)}:\s*(.+)$", text, re.M | re.I)
670
+ return match.group(1).strip() if match else None
671
+
672
+ def parse_wordpress(self, root: Path) -> None:
673
+ wp: dict = {"core": None, "plugins": [], "themes": []}
674
+ version_file = root / "wp-includes" / "version.php"
675
+ if version_file.is_file():
676
+ match = re.search(r"\$wp_version\s*=\s*'([^']+)'", self.read(version_file) or "")
677
+ wp["core"] = match.group(1) if match else None
678
+ plugins = root / "wp-content" / "plugins"
679
+ if plugins.is_dir():
680
+ for directory in sorted(p for p in plugins.iterdir() if p.is_dir()):
681
+ for php in sorted(directory.glob("*.php"))[:20]:
682
+ head = (self.read(php) or "")[:8192]
683
+ if self.header(head, "Plugin Name"):
684
+ wp["plugins"].append({"slug": directory.name, "version": self.header(head, "Version"), "active": None,
685
+ "text_domain": self.header(head, "Text Domain")})
686
+ break
687
+ themes = root / "wp-content" / "themes"
688
+ if themes.is_dir():
689
+ for directory in sorted(p for p in themes.iterdir() if p.is_dir()):
690
+ style = directory / "style.css"
691
+ if style.is_file():
692
+ head = (self.read(style) or "")[:8192]
693
+ wp["themes"].append({"slug": directory.name, "version": self.header(head, "Version"), "active": None})
694
+ self.wordpress = wp
695
+ self.source("wordpress", root / "wp-content" if (root / "wp-content").is_dir() else root)
696
+ if wp["core"]:
697
+ self.add_runtime("wordpress", wp["core"], "files")
698
+
699
+ def apply_wp_cli(self, path: Path) -> None:
700
+ """Active state from `wp plugin list --format=json` / `wp theme list --format=json` output."""
701
+ data = json.loads(self.read(path) or "[]")
702
+ if not self.wordpress:
703
+ self.wordpress = {"core": None, "plugins": [], "themes": []}
704
+ for row in data:
705
+ status = row.get("status")
706
+ for bucket in ("plugins", "themes"):
707
+ for item in self.wordpress[bucket]:
708
+ if item["slug"] == row.get("name"):
709
+ item["active"] = status in ("active", "active-network", "parent")
710
+ item["version"] = item["version"] or row.get("version")
711
+
712
+ # ------------------------------------------------------------------ hosts
713
+
714
+ def parse_host_evidence(self, text: str, alias: str, path_label: str) -> None:
715
+ """Sections from SECURITY_COLLECTOR (or the plain runtime-inventory collector)."""
716
+ sections = parse_sections(text)
717
+ os_info = parse_os(sections.get("os-release", ""))
718
+ public = True
719
+ if os_info.get("id"):
720
+ os_name = {"rhel": "rhel", "rocky": "rocky-linux", "almalinux": "almalinux", "amzn": "amazon-linux",
721
+ "opensuse-leap": "opensuse", "sles": "sles", "centos": "centos", "fedora": "fedora"}.get(os_info["id"], os_info["id"])
722
+ self.add_runtime(os_name, os_info.get("version", ""), "runtime-inventory", alias)
723
+ self.hosts.append({"id": alias, "os": " ".join(x for x in (os_info.get("id"), os_info.get("version")) if x) or None, "public": public})
724
+ for section, (name, pattern) in SERVICE_PROBES.items():
725
+ body = sections.get(section, "")
726
+ match = re.search(pattern, body, re.M)
727
+ if not match:
728
+ continue
729
+ service = name
730
+ if section == "mysql-v":
731
+ service = "mariadb" if "mariadb" in body.lower() else "mysql"
732
+ self.add_runtime(service, match.group(1), "runtime-inventory", alias)
733
+ ecosystem = OS_ECOSYSTEM.get(os_info.get("id", ""))
734
+ release = os_info.get("version", "")
735
+ eco = f"{ecosystem}:{release}" if ecosystem and release else ecosystem
736
+ if ecosystem == "Alpine" and release:
737
+ eco = "Alpine:v" + versions.cycle_of(release, 2) # OSV names Alpine releases "Alpine:v3.16"
738
+ if ecosystem == "Ubuntu" and release:
739
+ eco = f"Ubuntu:{release}" + (":LTS" if release.endswith(".04") and int(release.split(".")[0]) % 2 == 0 else "")
740
+ count = 0
741
+ for line in sections.get("dpkg", "").splitlines():
742
+ parts = line.split("\t")
743
+ if len(parts) >= 2 and (len(parts) < 3 or "installed" in parts[2]):
744
+ self.components.append({"purl": core.make_purl("deb", parts[0], parts[1]), "ecosystem": eco or "Debian", "name": parts[0],
745
+ "version": parts[1], "scope": "os", "direct": None, "location": f"{path_label}#dpkg", "host": alias})
746
+ count += 1
747
+ for line in sections.get("rpm", "").splitlines():
748
+ parts = line.split("\t")
749
+ if len(parts) == 2:
750
+ version = parts[1][2:] if parts[1].startswith("0:") else parts[1]
751
+ self.components.append({"purl": None, "ecosystem": eco or "Red Hat", "name": parts[0], "version": version,
752
+ "scope": "os", "direct": None, "location": f"{path_label}#rpm", "host": alias})
753
+ count += 1
754
+ for line in sections.get("apk", "").splitlines():
755
+ match = re.match(r"^(.+)-(\d[^-]*-r\d+)$", line.strip())
756
+ if match:
757
+ self.components.append({"purl": core.make_purl("apk", match.group(1), match.group(2)), "ecosystem": eco or "Alpine",
758
+ "name": match.group(1), "version": match.group(2), "scope": "os", "direct": None,
759
+ "location": f"{path_label}#apk", "host": alias})
760
+ count += 1
761
+ for line in sections.get("docker-ps", "").splitlines():
762
+ parts = line.split("\t")
763
+ if len(parts) >= 2:
764
+ self.add_image(parts[1], path_label, alias)
765
+ self.source("runtime-inventory", path_label)
766
+
767
+ def parse_dpkg_status(self, path: Path, alias: str) -> None:
768
+ """A copied /var/lib/dpkg/status file."""
769
+ text = self.read(path) or ""
770
+ lines = []
771
+ for block in text.split("\n\n"):
772
+ fields = dict(re.findall(r"^([A-Za-z-]+):\s*(.*)$", block, re.M))
773
+ if fields.get("Package") and fields.get("Version") and "installed" in fields.get("Status", "installed"):
774
+ lines.append(f"{fields['Package']}\t{fields['Version']}\t{fields.get('Status', 'install ok installed')}")
775
+ self.parse_host_evidence("### dpkg\n" + "\n".join(lines), alias, self.rel(path))
776
+
777
+ def parse_runtime_inventory_json(self, path: Path, alias: str) -> None:
778
+ data = json.loads(self.read(path) or "{}")
779
+ os_info = data.get("os") or {}
780
+ if os_info.get("id"):
781
+ self.add_runtime(os_info["id"], os_info.get("version", ""), "runtime-inventory", alias)
782
+ self.hosts.append({"id": alias, "os": " ".join(x for x in (os_info.get("id"), os_info.get("version")) if x) or None, "public": True})
783
+ if data.get("hostPhp"):
784
+ self.add_runtime("php", data["hostPhp"], "runtime-inventory", alias)
785
+ for container in data.get("containers") or []:
786
+ self.add_image(container.get("image", ""), self.rel(path), alias)
787
+ for site in data.get("sites") or []:
788
+ runtime = site.get("runtime") or {}
789
+ if runtime.get("version"):
790
+ self.add_runtime("php", runtime["version"], "runtime-inventory", alias, site_root=site.get("root"))
791
+ self.source("runtime-inventory", path)
792
+
793
+ # ------------------------------------------------------------------ result
794
+
795
+ def stack(self, domains: list[str], watch_extra: list[str]) -> dict:
796
+ seen = set()
797
+ components = []
798
+ for comp in sorted(self.components, key=lambda c: (c["ecosystem"], c["name"], c["version"], c["location"] or "")):
799
+ key = (comp["ecosystem"], comp["name"], comp["version"], comp["scope"])
800
+ if key in seen:
801
+ continue
802
+ seen.add(key)
803
+ components.append(comp)
804
+ # A declared "python 3.9" next to an exact "python 3.9.18" on the same host is the same runtime.
805
+ runtimes = []
806
+ for runtime in sorted(self.runtimes, key=lambda r: (r["name"], r.get("host") or "", -len(r["version"]), r["version"])):
807
+ if any(r["name"] == runtime["name"] and r.get("host") == runtime.get("host") and r["version"].startswith(runtime["version"] + ".")
808
+ for r in runtimes):
809
+ continue
810
+ runtimes.append(runtime)
811
+ runtimes.sort(key=lambda r: (r["name"], r.get("host") or "", r["version"]))
812
+ images = sorted(self.images, key=lambda i: (i["ref"], i.get("host") or ""))
813
+ watch = list(dict.fromkeys(WATCH_DEFAULTS.get(self.site_type, []) + [r["name"] for r in runtimes] +
814
+ [c["name"] for c in components if c.get("direct") and c["ecosystem"] == "npm" and c["name"] in ("next", "astro", "nuxt", "express")] +
815
+ watch_extra))
816
+ digest = core.sha256_bytes(core.canonical_json([components, runtimes, self.wordpress, images]).encode("utf-8"))
817
+ return {
818
+ "schema": "maggie-security-stack.v1",
819
+ "site": {"id": self.site, "type": self.site_type, "domains": domains, "hosting": None},
820
+ "collected_at": core.iso(),
821
+ "hash": f"sha256:{digest}",
822
+ "sources": [{"kind": "manifest" if s["kind"] == "js-bundle" else s["kind"], "path": s["path"]} for s in self.sources],
823
+ "components": components,
824
+ "runtimes": runtimes,
825
+ "wordpress": self.wordpress,
826
+ "images": images,
827
+ "hosts": self.hosts,
828
+ "watch": {"products": watch},
829
+ }
830
+
831
+
832
+ _EXTRACTORS: dict | None = None
833
+
834
+
835
+ def _retire_extractors() -> dict:
836
+ global _EXTRACTORS
837
+ if _EXTRACTORS is None:
838
+ data = core.read_json(core.DATA_DIR / "retire-extractors.json")
839
+ version = r"([0-9][0-9.a-z_\-]+)"
840
+ compiled = {}
841
+ for lib, spec in data["libraries"].items():
842
+ names = [re.compile(p.replace("§§version§§", version[1:-1]).replace("(" + version[1:-1] + ")", version)) for p in spec.get("filename", [])]
843
+ contents = [re.compile(p.replace("§§version§§", version[1:-1]).replace("(" + version[1:-1] + ")", version)) for p in spec.get("filecontent", [])]
844
+ compiled[lib] = (names, contents, spec.get("npmname") or lib)
845
+ _EXTRACTORS = compiled
846
+ return _EXTRACTORS
847
+
848
+
849
+ def summary(stack: dict, previous_hash: str | None) -> dict:
850
+ by_eco: dict[str, int] = {}
851
+ for comp in stack["components"]:
852
+ eco = comp["ecosystem"].split(":", 1)[0]
853
+ by_eco[eco] = by_eco.get(eco, 0) + 1
854
+ wp = stack.get("wordpress") or {}
855
+ return {"components": len(stack["components"]), "by_ecosystem": by_eco,
856
+ "runtimes": [f"{r['name']} {r['version']}" + (f" @{r['host']}" if r.get("host") else "") for r in stack["runtimes"]],
857
+ "images": [i["ref"] for i in stack["images"]],
858
+ "wordpress": {"core": wp.get("core"), "plugins": len(wp.get("plugins") or []), "themes": len(wp.get("themes") or [])} if wp else None,
859
+ "hash": stack["hash"], "changed": previous_hash != stack["hash"]}
860
+
861
+
862
+ def build(ctx: core.Context) -> dict:
863
+ args = ctx.args
864
+ cfg = ctx.state.config()
865
+ site_type = args.site_type or cfg.get("site_type") or core.detect_site_type(ctx.project)
866
+ inv = Inventory(ctx.project, ctx.site, site_type)
867
+ if not args.no_scan:
868
+ inv.scan_project()
869
+ for lockfile in args.lockfile or []:
870
+ path = Path(lockfile)
871
+ path = path if path.is_absolute() else ctx.project / path
872
+ parser = {"package-lock.json": inv.parse_package_lock_json, "npm-shrinkwrap.json": inv.parse_package_lock_json,
873
+ "pnpm-lock.yaml": inv.parse_pnpm_lock_yaml, "yarn.lock": inv.parse_yarn_lock, "bun.lock": inv.parse_bun_lock,
874
+ "composer.lock": inv.parse_composer_lock, "Pipfile.lock": inv.parse_pipfile_lock, "poetry.lock": inv.parse_toml_lock,
875
+ "uv.lock": inv.parse_toml_lock, "pdm.lock": inv.parse_toml_lock, "Cargo.lock": inv.parse_toml_lock,
876
+ "pom.xml": inv.parse_pom, "gradle.lockfile": inv.parse_gradle_lock, "go.mod": inv.parse_go_mod,
877
+ "Gemfile.lock": inv.parse_gemfile_lock, "packages.lock.json": inv.parse_nuget_lock}.get(path.name)
878
+ if parser is None and path.name.startswith("requirements"):
879
+ parser = inv.parse_requirements
880
+ if parser is None:
881
+ raise core.UsageError(f"unsupported lockfile {path.name}")
882
+ if not path.is_file():
883
+ raise core.UsageError(f"file not found: {path}")
884
+ inv.safe(parser, path)
885
+ if args.wordpress:
886
+ inv.safe(inv.parse_wordpress, Path(args.wordpress))
887
+ if args.wp_cli_json:
888
+ inv.safe(inv.apply_wp_cli, Path(args.wp_cli_json))
889
+ for item in args.host_evidence or []:
890
+ alias, _, file_name = item.partition("=") if "=" in item else ("host1", "", item)
891
+ path = Path(file_name)
892
+ if not path.is_file():
893
+ raise core.UsageError(f"file not found: {path}")
894
+ text = path.read_text(encoding="utf-8", errors="replace")
895
+ if text.lstrip().startswith("{"):
896
+ inv.safe(lambda p: inv.parse_runtime_inventory_json(p, alias), path)
897
+ elif text.lstrip().startswith("Package:"):
898
+ inv.safe(lambda p: inv.parse_dpkg_status(p, alias), path)
899
+ else:
900
+ inv.parse_host_evidence(text, alias, inv.rel(path))
901
+ for item in args.ssh or []:
902
+ alias, _, target = item.partition("=")
903
+ if not target:
904
+ raise core.UsageError("--ssh takes ALIAS=user@host")
905
+ core.require_consent(ctx.state, alias, "ssh-read")
906
+ text = _collect(target)
907
+ out = ctx.state.path("inventory", f"host-{alias}.txt")
908
+ out.parent.mkdir(parents=True, exist_ok=True)
909
+ out.write_text(text, encoding="utf-8")
910
+ inv.parse_host_evidence(text, alias, f".maggie/security/inventory/host-{alias}.txt")
911
+ domains = list(dict.fromkeys((args.domain or []) + (cfg.get("domains") or [])))
912
+ stack = inv.stack(domains, (args.watch_product or []) + (cfg.get("watch_products") or []))
913
+ if not stack["components"] and not stack["runtimes"] and not stack["wordpress"]:
914
+ raise core.UsageError("nothing to inventory: no lockfile, manifest, WordPress install or host evidence found")
915
+ errors = core.validate_contract(stack, "stack-v1")
916
+ if errors:
917
+ raise core.UsageError("stack failed its contract: " + "; ".join(errors[:5]))
918
+ path = ctx.state.path("inventory", "stack.json")
919
+ previous = core.read_json(path).get("hash") if path.is_file() else None
920
+ if not args.print:
921
+ core.write_json(path, stack)
922
+ return {"schema": "maggie-security-inventory-result.v1", "path": str(path) if not args.print else None,
923
+ "summary": summary(stack, previous), "problems": inv.problems, "stack": stack if args.print else None}
924
+
925
+
926
+ def _collect(target: str) -> str:
927
+ import subprocess
928
+
929
+ result = subprocess.run(["ssh", "-o", "BatchMode=yes", "-o", "ConnectTimeout=15", target, "sh -s"], input=SECURITY_COLLECTOR,
930
+ capture_output=True, text=True, timeout=180, check=False)
931
+ if result.returncode not in (0, 1) and not result.stdout:
932
+ raise core.DegradedError(f"ssh {target}: {result.stderr.strip()[:200]}")
933
+ return result.stdout
934
+
935
+
936
+ def load_stack(ctx: core.Context, max_age_hours: int = 24, rebuild: bool = False) -> dict:
937
+ """The current stack.json, rebuilt from the project when missing, stale or asked."""
938
+ path = ctx.state.path("inventory", "stack.json")
939
+ if path.is_file() and not rebuild:
940
+ stack = core.read_json(path)
941
+ age = core.utcnow() - core.parse_time(stack.get("collected_at", "1970-01-01T00:00:00Z"))
942
+ if age.total_seconds() < max_age_hours * 3600:
943
+ return stack
944
+ import argparse
945
+
946
+ fake = argparse.Namespace(site_type=None, no_scan=False, lockfile=[], wordpress=None, wp_cli_json=None, host_evidence=[], ssh=[],
947
+ domain=[], watch_product=[], print=False)
948
+ previous = core.read_json(path) if path.is_file() else None
949
+ sub = core.Context.__new__(core.Context)
950
+ sub.__dict__.update(ctx.__dict__)
951
+ sub.args = fake
952
+ try:
953
+ build(sub)
954
+ except core.UsageError:
955
+ if not previous:
956
+ raise
957
+ stack = core.read_json(path) if path.is_file() else previous
958
+ if previous and previous is not stack:
959
+ # Host-derived parts need SSH or an evidence file to refresh; keep the last ones.
960
+ hosts = {h["id"] for h in previous.get("hosts") or []}
961
+ if hosts:
962
+ stack["hosts"] = previous["hosts"] + [h for h in stack.get("hosts") or [] if h["id"] not in hosts]
963
+ stack["runtimes"] += [r for r in previous.get("runtimes") or [] if r.get("host") in hosts and r not in stack["runtimes"]]
964
+ stack["components"] += [c for c in previous.get("components") or [] if c.get("scope") == "os" and c not in stack["components"]]
965
+ stack["images"] += [i for i in previous.get("images") or [] if i.get("host") in hosts and i not in stack["images"]]
966
+ stack["sources"] += [s for s in previous.get("sources") or [] if s["kind"] == "runtime-inventory" and s not in stack["sources"]]
967
+ stack["hash"] = "sha256:" + core.sha256_bytes(core.canonical_json([stack["components"], stack["runtimes"], stack.get("wordpress"), stack["images"]]).encode())
968
+ core.write_json(path, stack)
969
+ return stack
970
+
971
+
972
+ def configure(parser) -> None:
973
+ parser.add_argument("--lockfile", action="append", help="extra lockfile/manifest outside the scanned tree (repeatable)")
974
+ parser.add_argument("--wordpress", help="WordPress root when not under the project")
975
+ parser.add_argument("--wp-cli-json", help="output of `wp plugin list --format=json` for active state")
976
+ parser.add_argument("--host-evidence", action="append",
977
+ help="ALIAS=FILE: output of --print-collector, runtime-inventory JSON, or a copied dpkg status file (repeatable)")
978
+ parser.add_argument("--ssh", action="append", help="ALIAS=user@host: run the read-only collector over SSH (needs ssh-read consent)")
979
+ parser.add_argument("--site-type", choices=["static", "wordpress", "node-app", "php-app", "python-app", "java-app", "go-app", "ruby-app",
980
+ "dotnet-app", "astro", "workers", "unknown"])
981
+ parser.add_argument("--domain", action="append", help="site domain (repeatable)")
982
+ parser.add_argument("--watch-product", action="append", help="extra product name for CVE records without a package id")
983
+ parser.add_argument("--no-scan", action="store_true", help="only use the explicit inputs, do not scan the project tree")
984
+ parser.add_argument("--print", action="store_true", help="print the stack instead of writing stack.json")
985
+ parser.add_argument("--print-collector", action="store_true", help="print the read-only host collector script and exit")
986
+ parser.add_argument("--export-upload", help="also write the stack-upload payload for the hosted daily watch to this file (names and versions only)")
987
+ parser.add_argument("--workspace", help="CMO.so workspace (domain) for --export-upload")
988
+
989
+
990
+ def upload_payload(stack: dict, workspace: str) -> dict:
991
+ """Only names and versions leave the machine (PRD 11 section 9): no paths, hostnames, lockfiles or logs."""
992
+ components = [{"ecosystem": c["ecosystem"], "name": c["name"], "version": c["version"], "scope": c.get("scope") or "runtime",
993
+ "direct": c.get("direct")} for c in stack["components"]]
994
+ runtimes = [{"name": r["name"], "version": r["version"], "host": r.get("host") if r.get("host") and re.fullmatch(r"[a-z0-9-]{1,40}", r["host"]) else None}
995
+ for r in stack["runtimes"]]
996
+ wordpress = None
997
+ if stack.get("wordpress"):
998
+ wp = stack["wordpress"]
999
+ wordpress = {"core": wp.get("core"), "plugins": [{"slug": p["slug"], "version": p.get("version"), "active": p.get("active")} for p in wp.get("plugins") or []],
1000
+ "themes": [{"slug": t["slug"], "version": t.get("version"), "active": t.get("active")} for t in wp.get("themes") or []]}
1001
+ body = {"components": components, "runtimes": runtimes, "wordpress": wordpress}
1002
+ return {"schema": "maggie-security-stack-upload.v1", "workspace": workspace, "site_type": stack["site"].get("type") or "unknown",
1003
+ "created_at": core.iso(), "hash": "sha256:" + core.sha256_bytes(core.canonical_json(body).encode()), **body,
1004
+ "watch": (stack.get("watch") or {}).get("products") or []}
1005
+
1006
+
1007
+ def run(ctx: core.Context) -> dict:
1008
+ if ctx.args.print_collector:
1009
+ return {"_raw": SECURITY_COLLECTOR}
1010
+ result = build(ctx)
1011
+ if ctx.args.export_upload:
1012
+ stack = result.get("stack") or core.read_json(ctx.state.path("inventory", "stack.json"))
1013
+ workspace = ctx.args.workspace or (ctx.state.config().get("domains") or [ctx.site])[0]
1014
+ payload = upload_payload(stack, workspace)
1015
+ errors = core.validate_contract(payload, "stack-upload-v1")
1016
+ if errors:
1017
+ raise core.UsageError("upload payload failed its contract: " + "; ".join(errors[:5]))
1018
+ out = Path(ctx.args.export_upload)
1019
+ core.write_json(out, payload)
1020
+ result["upload"] = {"path": str(out), "components": len(payload["components"]), "runtimes": len(payload["runtimes"]),
1021
+ "sends": "package, runtime and WordPress plugin/theme names and versions only",
1022
+ "never_sends": ["file paths", "host names or IPs", "lockfile contents", "logs", "secrets", "findings"]}
1023
+ return result