@topy-ai/maggie 0.7.55 → 0.7.56

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (73) hide show
  1. package/README-zh-TW.md +5 -3
  2. package/README.md +35 -8
  3. package/bin/maggie.js +4 -0
  4. package/bundled-contracts/maggie-deployment/readiness-v1.schema.json +2 -1
  5. package/bundled-contracts/maggie-security/finding-example-v1.json +89 -0
  6. package/bundled-contracts/maggie-security/finding-v1.schema.json +63 -0
  7. package/bundled-contracts/maggie-security/findings-set-example-v1.json +148 -0
  8. package/bundled-contracts/maggie-security/findings-set-v1.schema.json +32 -0
  9. package/bundled-contracts/maggie-security/protect-plan-v1.schema.json +28 -0
  10. package/bundled-contracts/maggie-security/report-v1.schema.json +29 -0
  11. package/bundled-contracts/maggie-security/stack-example-v1.json +66 -0
  12. package/bundled-contracts/maggie-security/stack-upload-v1.schema.json +22 -0
  13. package/bundled-contracts/maggie-security/stack-v1.schema.json +37 -0
  14. package/bundled-contracts/maggie-security/tools-lock-v1.schema.json +36 -0
  15. package/bundled-skills/catalog.json +4 -0
  16. package/bundled-skills/maggie-security/SKILL.md +172 -0
  17. package/bundled-skills/maggie-security/agents/openai.yaml +4 -0
  18. package/bundled-skills/maggie-security/data/LICENSE-NOTICE.md +24 -0
  19. package/bundled-skills/maggie-security/data/asvs-map.json +209 -0
  20. package/bundled-skills/maggie-security/data/fail2ban/filter.d/maggie-wordpress.conf +6 -0
  21. package/bundled-skills/maggie-security/data/semgrep/LICENSE-NOTICE.md +5 -0
  22. package/bundled-skills/maggie-security/data/semgrep/VERSION +1 -0
  23. package/bundled-skills/maggie-security/data/semgrep/rules/dockerfile.yaml +14 -0
  24. package/bundled-skills/maggie-security/data/semgrep/rules/generic.yaml +26 -0
  25. package/bundled-skills/maggie-security/data/semgrep/rules/js.yaml +216 -0
  26. package/bundled-skills/maggie-security/data/semgrep/rules/php-debug-ok.yaml +17 -0
  27. package/bundled-skills/maggie-security/data/semgrep/rules/php-debug.yaml +17 -0
  28. package/bundled-skills/maggie-security/data/semgrep/rules/php.yaml +101 -0
  29. package/bundled-skills/maggie-security/data/semgrep/tests/dockerfile.dockerfile +10 -0
  30. package/bundled-skills/maggie-security/data/semgrep/tests/generic.astro +11 -0
  31. package/bundled-skills/maggie-security/data/semgrep/tests/generic.conf +11 -0
  32. package/bundled-skills/maggie-security/data/semgrep/tests/js.jsx +95 -0
  33. package/bundled-skills/maggie-security/data/semgrep/tests/php-debug-ok.php +5 -0
  34. package/bundled-skills/maggie-security/data/semgrep/tests/php-debug.php +3 -0
  35. package/bundled-skills/maggie-security/data/semgrep/tests/php.php +55 -0
  36. package/bundled-skills/maggie-security/evals/evals.json +97 -0
  37. package/bundled-skills/maggie-security/references/commands.md +111 -0
  38. package/bundled-skills/maggie-security/references/cve-response.md +24 -0
  39. package/bundled-skills/maggie-security/references/explaining-findings.md +57 -0
  40. package/bundled-skills/maggie-security/references/findings.md +39 -0
  41. package/bundled-skills/maggie-security/references/ownership-and-consent.md +29 -0
  42. package/bundled-skills/maggie-security/references/site-types.md +22 -0
  43. package/bundled-skills/maggie-security/references/tools-and-licences.md +16 -0
  44. package/bundled-skills/maggie-security/references/workflows.md +23 -0
  45. package/bundled-skills/maggie-security/registry/security-tools.registry.json +628 -0
  46. package/bundled-skills/maggie-security/routines.example.json +27 -0
  47. package/bundled-tools/clis/maggie_deployment_readiness.py +17 -2
  48. package/bundled-tools/clis/maggie_security.py +342 -0
  49. package/bundled-tools/clis/maggie_security_agent.py +326 -0
  50. package/bundled-tools/clis/maggie_security_code.py +650 -0
  51. package/bundled-tools/clis/maggie_security_core.py +1512 -0
  52. package/bundled-tools/clis/maggie_security_deps.py +560 -0
  53. package/bundled-tools/clis/maggie_security_health.py +415 -0
  54. package/bundled-tools/clis/maggie_security_host.py +511 -0
  55. package/bundled-tools/clis/maggie_security_hosted.py +115 -0
  56. package/bundled-tools/clis/maggie_security_intel.py +1294 -0
  57. package/bundled-tools/clis/maggie_security_inventory.py +1023 -0
  58. package/bundled-tools/clis/maggie_security_logs.py +777 -0
  59. package/bundled-tools/clis/maggie_security_mcp.py +473 -0
  60. package/bundled-tools/clis/maggie_security_mirror.py +122 -0
  61. package/bundled-tools/clis/maggie_security_protect.py +519 -0
  62. package/bundled-tools/clis/maggie_security_report.py +506 -0
  63. package/bundled-tools/clis/maggie_security_standards.py +214 -0
  64. package/bundled-tools/clis/maggie_security_versions.py +381 -0
  65. package/bundled-tools/clis/maggie_security_web.py +936 -0
  66. package/bundled-tools/clis/security_data/eol-snapshot.json +1 -0
  67. package/bundled-tools/clis/security_data/gitleaks-rules.json +3812 -0
  68. package/bundled-tools/clis/security_data/maltrail-lists.json +1 -0
  69. package/bundled-tools/clis/security_data/public-suffix-icann.txt +7374 -0
  70. package/bundled-tools/clis/security_data/retire-extractors.json +695 -0
  71. package/bundled-tools/clis/security_data/sigma-web.json +6514 -0
  72. package/bundled-tools/clis/security_data/tools.lock.json +358 -0
  73. package/package.json +1 -1
@@ -0,0 +1,519 @@
1
+ #!/usr/bin/env python3
2
+ """``maggie security protect-plan`` (docs/maggie-security/05-protection-waf-prd.md section 6).
3
+
4
+ Writes a plan — never applies it. A plan (``maggie-security-protect-plan.v1``)
5
+ lists exact commands with a rollback for every step that touches the host,
6
+ starts by taking the host lock and ends by releasing it, and states every
7
+ host it downloads from. A person reviews it; an agent may run ``steps[].run``
8
+ verbatim in an interactive session only after that approval.
9
+
10
+ - CrowdSec (default): official packages, hub collections (no account),
11
+ acquisition for the site's logs, owner allowlist, nftables bouncer, and
12
+ **sharing off** (CrowdSec shares signals when the setting is absent; the
13
+ plan sets it to false unless the owner opts in with ``--capi``).
14
+ - fail2ban (lightweight): distro package, built-in nginx/sshd jails and a
15
+ WordPress filter written for Maggie, checked here against a real log sample.
16
+ - Cloudflare-only / static: advice, no host steps.
17
+
18
+ ``validate`` checks a plan against the rules; ``verify`` runs read-only checks
19
+ on the host afterwards.
20
+ """
21
+
22
+ from __future__ import annotations
23
+
24
+ import argparse
25
+ import json
26
+ import re
27
+ import shlex
28
+ import subprocess
29
+ from pathlib import Path
30
+
31
+ import maggie_security_core as core
32
+ from maggie_host_runtime import parse_sections
33
+
34
+ WORDPRESS_FILTER = r"""# Maggie WordPress login/xmlrpc brute-force filter (MIT, written for Maggie)
35
+ [Definition]
36
+ failregex = ^<HOST> -[^"]*"POST /(?:[^"/]*/)*wp-login\.php HTTP/[0-9.]+" (?:200|302|403)
37
+ ^<HOST> -[^"]*"POST /(?:[^"/]*/)*xmlrpc\.php HTTP/[0-9.]+" (?:200|403|405)
38
+ ignoreregex =
39
+ datepattern = {^LN-BEG}%%d/%%b/%%Y:%%H:%%M:%%S
40
+ """
41
+ WORDPRESS_JAIL = """[maggie-wordpress]
42
+ enabled = true
43
+ port = http,https
44
+ filter = maggie-wordpress
45
+ logpath = {logs}
46
+ maxretry = 6
47
+ findtime = 10m
48
+ bantime = 1h
49
+ bantime.increment = true
50
+ action = nftables-multiport[name=maggie-wordpress, port="80,443"]
51
+ """
52
+ FORBIDDEN = [
53
+ (re.compile(r"cscli\s+console\s+enroll"), "enrols the host in the CrowdSec Console (an account)"),
54
+ (re.compile(r"curl[^|]*\|\s*(sudo\s+)?(ba)?sh"), "pipes a download straight into a shell"),
55
+ (re.compile(r"wget[^|]*\|\s*(sudo\s+)?(ba)?sh"), "pipes a download straight into a shell"),
56
+ (re.compile(r"\brm\s+-rf\s+/(\s|$)"), "deletes the root folder"),
57
+ ]
58
+ CLOUDFLARE_V4 = ["173.245.48.0/20", "103.21.244.0/22", "103.22.200.0/22", "103.31.4.0/22", "141.101.64.0/18", "108.162.192.0/18", "190.93.240.0/20",
59
+ "188.114.96.0/20", "197.234.240.0/22", "198.41.128.0/17", "162.158.0.0/15", "104.16.0.0/13", "104.24.0.0/14", "172.64.0.0/13",
60
+ "131.0.72.0/22"]
61
+
62
+
63
+ def host_facts(ctx: core.Context, host: str) -> dict:
64
+ """What we know about the host from earlier read-only runs (host-quick evidence, stack.json)."""
65
+ facts = {"crowdsec": False, "fail2ban": False, "firewall": None, "os": None, "logs": [], "site_type": None, "cloudflare": False, "ram_gb": None}
66
+ evidence = None
67
+ if ctx.args.evidence:
68
+ evidence = Path(ctx.args.evidence).read_text(encoding="utf-8", errors="replace")
69
+ else:
70
+ saved = ctx.state.path("inventory", f"host-{host}-host-quick.txt")
71
+ if saved.is_file():
72
+ evidence = saved.read_text(encoding="utf-8", errors="replace")
73
+ if evidence:
74
+ sections = parse_sections(evidence)
75
+ ips = sections.get("ips", "").lower()
76
+ facts["crowdsec"] = "crowdsec active" in ips or "crowdsec" in "".join(l for l in ips.splitlines() if " " not in l.strip())
77
+ facts["fail2ban"] = "fail2ban active" in ips
78
+ facts["firewall"] = "ufw" if "status: active" in sections.get("ufw", "").lower() else None
79
+ os_text = sections.get("os-release", "")
80
+ os_id = re.search(r"(?m)^ID=\"?(\w+)", os_text)
81
+ facts["os"] = os_id.group(1) if os_id else None
82
+ stack_path = ctx.state.path("inventory", "stack.json")
83
+ if stack_path.is_file():
84
+ stack = core.read_json(stack_path)
85
+ facts["site_type"] = stack["site"].get("type")
86
+ if not facts["os"]:
87
+ facts["os"] = next((r["name"] for r in stack.get("runtimes") or [] if r["name"] in ("ubuntu", "debian", "rhel", "rocky-linux", "almalinux")
88
+ and r.get("host") == host), None)
89
+ config = ctx.state.config()
90
+ facts["logs"] = (config.get("logs") or {}).get("access_paths") or [(config.get("logs") or {}).get("access_path") or "/var/log/nginx/access.log"]
91
+ facts["cloudflare"] = bool(ctx.args.behind_cloudflare)
92
+ return facts
93
+
94
+
95
+ def pkg(os_name: str | None) -> tuple[str, str]:
96
+ if os_name in ("rhel", "rocky", "rocky-linux", "almalinux", "centos", "fedora", "amzn"):
97
+ return "sudo dnf install -y", "sudo dnf remove -y"
98
+ return "sudo apt-get install -y", "sudo apt-get remove --purge -y"
99
+
100
+
101
+ def base_steps(plan_id: str, host_target: str) -> tuple[dict, dict]:
102
+ lock = {"title": "Take the host lock", "kind": "local", "reversible": True,
103
+ "run": f"maggie deployment host-lock acquire --host {host_target} --session {plan_id} --purpose 'protect-plan {plan_id}' --ttl 3600"}
104
+ release = {"title": "Release the host lock", "kind": "local", "reversible": True,
105
+ "run": f"maggie deployment host-lock release --host {host_target} --session {plan_id}"}
106
+ return lock, release
107
+
108
+
109
+ def crowdsec_plan(ctx: core.Context, plan_id: str, facts: dict, host_target: str) -> dict:
110
+ install, remove = pkg(facts["os"])
111
+ site_type = ctx.args.site_type or facts["site_type"] or "php-app"
112
+ collections = ["crowdsecurity/nginx", "crowdsecurity/base-http-scenarios", "crowdsecurity/http-cve", "crowdsecurity/linux", "crowdsecurity/sshd"]
113
+ if site_type == "wordpress":
114
+ collections.append("crowdsecurity/wordpress")
115
+ acquis = "\\n".join(["filenames:"] + [f" - {path}" for path in facts["logs"]] + ["labels:", " type: nginx"])
116
+ allow_ips = ctx.args.allow_ip or []
117
+ lock, release = base_steps(plan_id, host_target)
118
+ steps = [lock,
119
+ {"title": "Back up existing configuration", "kind": "host", "reversible": True,
120
+ "run": f"sudo tar czf /root/maggie-backup-{plan_id}.tgz /etc/nginx /etc/crowdsec 2>/dev/null; ls -l /root/maggie-backup-{plan_id}.tgz",
121
+ "rollback": f"sudo tar xzf /root/maggie-backup-{plan_id}.tgz -C /"}]
122
+ if not facts["crowdsec"]:
123
+ steps += [
124
+ {"title": "Download CrowdSec's repository script and review it (not piped to a shell)", "kind": "host", "reversible": True,
125
+ "run": "curl -fsSL -o /tmp/crowdsec-repo.sh https://install.crowdsec.net && sha256sum /tmp/crowdsec-repo.sh && less /tmp/crowdsec-repo.sh",
126
+ "rollback": "rm -f /tmp/crowdsec-repo.sh"},
127
+ {"title": "Add the repository and install CrowdSec", "kind": "host", "reversible": True,
128
+ "run": f"sudo sh /tmp/crowdsec-repo.sh && {install} crowdsec", "rollback": f"{remove} crowdsec"},
129
+ ]
130
+ sharing = "true" if ctx.args.capi else "false"
131
+ steps += [
132
+ {"title": f"Set signal sharing to {sharing}" + (" (owner opted in to the community blocklist)" if ctx.args.capi else " (CrowdSec shares by default when unset)"),
133
+ "kind": "host", "reversible": True,
134
+ "run": f"printf 'api:\\n server:\\n online_client:\\n sharing: {sharing}\\n' | sudo tee /etc/crowdsec/config.yaml.local",
135
+ "rollback": "sudo rm -f /etc/crowdsec/config.yaml.local"},
136
+ {"title": "Install detection collections from the public hub (no account)", "kind": "host", "reversible": True,
137
+ "run": "sudo cscli collections install " + " ".join(collections), "rollback": "sudo cscli collections remove " + " ".join(collections)},
138
+ {"title": "Read the site's access logs", "kind": "host", "reversible": True,
139
+ "run": f"printf '{acquis}\\n' | sudo tee /etc/crowdsec/acquis.d/maggie-nginx.yaml", "rollback": "sudo rm -f /etc/crowdsec/acquis.d/maggie-nginx.yaml"},
140
+ ]
141
+ if allow_ips:
142
+ allow = "\\n".join(["name: maggie/allowlist", "description: owner addresses (from protect-plan)", "whitelist:", " reason: owner", " ip:"]
143
+ + [f" - {ip}" for ip in allow_ips])
144
+ steps.append({"title": "Never ban the owner's addresses", "kind": "host", "reversible": True,
145
+ "run": f"printf '{allow}\\n' | sudo tee /etc/crowdsec/parsers/s02-enrich/maggie-allowlist.yaml",
146
+ "rollback": "sudo rm -f /etc/crowdsec/parsers/s02-enrich/maggie-allowlist.yaml"})
147
+ if ctx.args.capi:
148
+ steps.append({"title": "Register with CrowdSec's central API (owner opted in: sends attacking IP, scenario, time; receives the community blocklist)",
149
+ "kind": "host", "reversible": True, "run": "sudo cscli capi register", "rollback": "sudo rm -f /etc/crowdsec/online_api_credentials.yaml"})
150
+ steps += [
151
+ {"title": "Install the nftables firewall bouncer (applies bans)", "kind": "host", "reversible": True,
152
+ "run": f"{install} crowdsec-firewall-bouncer-nftables", "rollback": f"{remove} crowdsec-firewall-bouncer-nftables"},
153
+ {"title": "Reload CrowdSec", "kind": "host", "reversible": True, "run": "sudo systemctl reload crowdsec || sudo systemctl restart crowdsec",
154
+ "rollback": "sudo systemctl restart crowdsec"},
155
+ {"title": "Check it reads the logs", "kind": "host", "reversible": True, "run": "sudo cscli metrics show acquisition"},
156
+ release,
157
+ ]
158
+ notes = ["The first day mostly shows scanners; bans apply to the whole server (all ports)."]
159
+ if facts["cloudflare"]:
160
+ notes.append("This origin sits behind Cloudflare: bans at the origin firewall do not stop requests that come through Cloudflare. "
161
+ "Turn on real client IPs in nginx first (see the real-ip step) and use Cloudflare-side rules or the CrowdSec Cloudflare bouncer for proxied traffic.")
162
+ steps.insert(2, real_ip_step())
163
+ if facts["firewall"] == "ufw":
164
+ notes.append("UFW is active; the nftables bouncer adds its own table and works alongside it.")
165
+ egress = [{"host": "install.crowdsec.net", "why": "repository script (reviewed before running)"},
166
+ {"host": "packagecloud.io", "why": "CrowdSec package repository"},
167
+ {"host": "hub-cdn.crowdsec.net", "why": "collections and parsers (no account)"}]
168
+ if ctx.args.capi:
169
+ egress.append({"host": "api.crowdsec.net", "why": "central API: sends signals, receives the community blocklist (owner opted in)"})
170
+ return {"engine": "crowdsec", "steps": steps, "risk_notes": notes, "egress": egress,
171
+ "data_sharing": {"crowdsec_capi": bool(ctx.args.capi), "console": False},
172
+ "verify": [{"check": "systemctl is-active crowdsec crowdsec-firewall-bouncer", "expect": "active (both)"},
173
+ {"check": "sudo cscli metrics show acquisition", "expect": "lines read > 0 for each access log"},
174
+ {"check": "sudo cscli bouncers list", "expect": "the firewall bouncer pulled within the last 2 minutes"},
175
+ {"check": "sudo grep -A3 online_client /etc/crowdsec/config.yaml.local", "expect": f"sharing: {sharing}"}]}
176
+
177
+
178
+ def real_ip_step() -> dict:
179
+ lines = "\\n".join([f"set_real_ip_from {net};" for net in CLOUDFLARE_V4] + ["real_ip_header CF-Connecting-IP;"])
180
+ return {"title": "Log the real visitor address behind Cloudflare (otherwise Cloudflare's addresses get banned)", "kind": "host", "reversible": True,
181
+ "run": f"printf '{lines}\\n' | sudo tee /etc/nginx/conf.d/maggie-cloudflare-realip.conf && sudo nginx -t && sudo systemctl reload nginx",
182
+ "rollback": "sudo rm -f /etc/nginx/conf.d/maggie-cloudflare-realip.conf && sudo systemctl reload nginx"}
183
+
184
+
185
+ def fail2ban_plan(ctx: core.Context, plan_id: str, facts: dict, host_target: str) -> dict:
186
+ install, remove = pkg(facts["os"])
187
+ site_type = ctx.args.site_type or facts["site_type"] or "php-app"
188
+ logs = " ".join(facts["logs"])
189
+ ignore = " ".join(["127.0.0.1/8", "::1"] + (ctx.args.allow_ip or []))
190
+ jails = [f"[DEFAULT]\\nignoreip = {ignore}\\nbanaction = nftables-multiport\\n",
191
+ "[sshd]\\nenabled = true\\n", f"[nginx-botsearch]\\nenabled = true\\nlogpath = {logs}\\n",
192
+ f"[nginx-bad-request]\\nenabled = true\\nlogpath = {logs}\\n", "[recidive]\\nenabled = true\\n"]
193
+ lock, release = base_steps(plan_id, host_target)
194
+ steps = [lock,
195
+ {"title": "Back up existing configuration", "kind": "host", "reversible": True,
196
+ "run": f"sudo tar czf /root/maggie-backup-{plan_id}.tgz /etc/fail2ban /etc/nginx 2>/dev/null; ls -l /root/maggie-backup-{plan_id}.tgz",
197
+ "rollback": f"sudo tar xzf /root/maggie-backup-{plan_id}.tgz -C /"}]
198
+ if not facts["fail2ban"]:
199
+ steps.append({"title": "Install fail2ban (GPL; from the distribution)", "kind": "host", "reversible": True, "run": f"{install} fail2ban",
200
+ "rollback": f"{remove} fail2ban"})
201
+ if site_type == "wordpress":
202
+ filter_text = WORDPRESS_FILTER.replace("\\", "\\\\").replace("'", "'\\''").replace("\n", "\\n").replace("%", "%%")
203
+ steps += [
204
+ {"title": "Add the WordPress login filter", "kind": "host", "reversible": True,
205
+ "run": f"printf '{filter_text}' | sudo tee /etc/fail2ban/filter.d/maggie-wordpress.conf",
206
+ "rollback": "sudo rm -f /etc/fail2ban/filter.d/maggie-wordpress.conf"},
207
+ {"title": "Check the filter on the real log before enabling it (look for attack lines, and none from your own address)", "kind": "host",
208
+ "reversible": True, "run": f"sudo fail2ban-regex {facts['logs'][0]} /etc/fail2ban/filter.d/maggie-wordpress.conf --print-all-matched | head -40"},
209
+ ]
210
+ jails.append(WORDPRESS_JAIL.format(logs=logs).replace("\n", "\\n"))
211
+ steps += [
212
+ {"title": "Enable the jails", "kind": "host", "reversible": True,
213
+ "run": f"printf '{''.join(j + chr(92) + 'n' for j in jails)}' | sudo tee /etc/fail2ban/jail.d/maggie.local",
214
+ "rollback": "sudo rm -f /etc/fail2ban/jail.d/maggie.local && sudo systemctl reload fail2ban"},
215
+ {"title": "Restart fail2ban", "kind": "host", "reversible": True, "run": "sudo systemctl enable --now fail2ban && sudo systemctl restart fail2ban",
216
+ "rollback": "sudo systemctl restart fail2ban"},
217
+ {"title": "Check the jails are running", "kind": "host", "reversible": True, "run": "sudo fail2ban-client status"},
218
+ release,
219
+ ]
220
+ notes = ["fail2ban bans at the firewall on ports 80/443 (and SSH for the sshd jail).",
221
+ "WordPress returns 200 for a failed login and 302 for a successful one; the filter counts both, so owners behind a shared office address may need a higher maxretry or an ignoreip entry."]
222
+ if facts["cloudflare"]:
223
+ steps.insert(2, real_ip_step())
224
+ notes.append("Behind Cloudflare, firewall bans do not stop proxied requests; use Cloudflare rate limiting for /wp-login.php as well.")
225
+ return {"engine": "fail2ban", "steps": steps, "risk_notes": notes, "egress": [{"host": "distribution package mirrors", "why": "fail2ban package"}],
226
+ "data_sharing": {"crowdsec_capi": False, "console": False},
227
+ "verify": [{"check": "sudo fail2ban-client status", "expect": "jails listed: sshd, nginx-botsearch, nginx-bad-request, recidive"
228
+ + (", maggie-wordpress" if site_type == "wordpress" else "")},
229
+ {"check": "systemctl is-active fail2ban", "expect": "active"}]}
230
+
231
+
232
+ def cloudflare_plan(ctx: core.Context) -> dict:
233
+ return {"engine": "cloudflare", "steps": [], "egress": [], "data_sharing": {"crowdsec_capi": False, "console": False},
234
+ "risk_notes": ["There is no origin server to protect, so nothing is installed. Maggie cannot see Cloudflare's own logs without Logpush."],
235
+ "advice": ["Security → Settings: Security Level 'Medium', Bot Fight Mode on.",
236
+ "SSL/TLS: minimum TLS 1.2, Always Use HTTPS on, HSTS on once HTTPS works everywhere.",
237
+ "Rules → Rate limiting: limit login or form endpoints (e.g. /api/login) to ~10 requests per minute per IP.",
238
+ "WAF → Managed rules: turn on what your plan includes."],
239
+ "verify": []}
240
+
241
+
242
+ def wordpress_filter_matches(log_path: Path, limit: int = 200_000) -> dict:
243
+ """Python port of the WordPress failregex, run on a local log sample (PRD 05 section 6.4)."""
244
+ patterns = [re.compile(line.strip().replace("^<HOST>", r"^(?P<host>[0-9a-fA-F.:]+)"))
245
+ for line in WORDPRESS_FILTER.split("failregex =", 1)[1].split("ignoreregex", 1)[0].splitlines() if line.strip()]
246
+ hosts: dict[str, int] = {}
247
+ total = 0
248
+ opener = __import__("gzip").open if log_path.suffix == ".gz" else open
249
+ with opener(log_path, "rt", encoding="utf-8", errors="replace") as handle:
250
+ for index, line in enumerate(handle):
251
+ if index >= limit:
252
+ break
253
+ for pattern in patterns:
254
+ match = pattern.search(line)
255
+ if match:
256
+ total += 1
257
+ hosts[match.group("host")] = hosts.get(match.group("host"), 0) + 1
258
+ break
259
+ would_ban = {h: n for h, n in hosts.items() if n >= 6}
260
+ return {"lines_matched": total, "addresses": len(hosts), "would_ban_at_6": len(would_ban),
261
+ "top": sorted(would_ban.items(), key=lambda kv: -kv[1])[:10]}
262
+
263
+
264
+ def validate_plan(plan: dict) -> list[str]:
265
+ problems = core.validate_contract(plan, "protect-plan-v1")
266
+ steps = plan.get("steps") or []
267
+ if steps:
268
+ if "host-lock acquire" not in steps[0].get("run", ""):
269
+ problems.append("step 1 must take the host lock")
270
+ if "host-lock release" not in steps[-1].get("run", ""):
271
+ problems.append("the last step must release the host lock")
272
+ egress_hosts = {e["host"] for e in plan.get("egress") or []}
273
+ for step in steps:
274
+ run = step.get("run", "")
275
+ for pattern, why in FORBIDDEN:
276
+ if pattern.search(run):
277
+ problems.append(f"step {step.get('n')}: {why}")
278
+ if step.get("kind") == "host" and not step.get("rollback") and not re.match(r"^(sudo )?(cscli metrics|fail2ban-client status|sudo fail2ban-client status|systemctl is-active|ls |sudo fail2ban-regex|sudo cscli metrics)", run.strip()) \
279
+ and not re.search(r"(fail2ban-regex|cscli metrics|fail2ban-client status)", run):
280
+ problems.append(f"step {step.get('n')}: a host step needs a rollback")
281
+ if "cscli capi register" in run and not (plan.get("data_sharing") or {}).get("crowdsec_capi"):
282
+ problems.append(f"step {step.get('n')}: registers with the central API but data_sharing.crowdsec_capi is false")
283
+ for host in re.findall(r"https?://([A-Za-z0-9.-]+)", run):
284
+ if host not in egress_hosts:
285
+ problems.append(f"step {step.get('n')}: downloads from {host}, which is not listed in egress")
286
+ return problems
287
+
288
+
289
+ def protect_plan(ctx: core.Context) -> dict:
290
+ args = ctx.args
291
+ if args.action == "validate":
292
+ if not args.plan:
293
+ raise core.UsageError("validate needs --plan FILE")
294
+ problems = validate_plan(core.read_json(Path(args.plan)))
295
+ return {"schema": "maggie-security-protect-plan-validation.v1", "valid": not problems, "problems": problems,
296
+ "exit_code": core.EXIT_OK if not problems else core.EXIT_FINDINGS}
297
+ if args.action == "verify":
298
+ return verify_plan(ctx)
299
+ host = args.host or "web1"
300
+ host_target = ((ctx.state.config().get("hosts") or {}).get(host)) or f"deploy@{host}"
301
+ if args.capi and not __import__("sys").stdin.isatty():
302
+ raise core.PolicyError("--capi (sharing with CrowdSec's central API) is an owner decision made interactively")
303
+ facts = host_facts(ctx, host)
304
+ engine = args.engine
305
+ if engine == "auto":
306
+ site_type = args.site_type or facts["site_type"]
307
+ if site_type in ("workers", "static") and not args.evidence:
308
+ engine = "cloudflare"
309
+ elif facts["crowdsec"]:
310
+ engine = "crowdsec"
311
+ elif facts["fail2ban"]:
312
+ engine = "fail2ban"
313
+ else:
314
+ engine = "crowdsec"
315
+ plan_id = f"pp_{core.iso()[:10]}_{host}_{core.sha256_bytes((host + core.iso()).encode())[:4]}"
316
+ body = {"crowdsec": crowdsec_plan, "fail2ban": fail2ban_plan}.get(engine)
317
+ content = body(ctx, plan_id, facts, host_target) if body else cloudflare_plan(ctx)
318
+ for index, step in enumerate(content["steps"], 1):
319
+ step["n"] = index
320
+ plan = {"schema": "maggie-security-protect-plan.v1", "id": plan_id, "host": host, "created_at": core.iso(), "engine": content["engine"],
321
+ "waf": waf_section(args, content["engine"]),
322
+ "inputs": {"facts": {k: v for k, v in facts.items() if k != "logs"}, "logs": facts["logs"]},
323
+ "egress": content["egress"], "data_sharing": content["data_sharing"],
324
+ "requires": {"consent": "host-change", "host_lock": bool(content["steps"]), "sudo": bool(content["steps"]),
325
+ "approval": "a person reviews this plan; nothing runs from a routine"},
326
+ "risk_notes": content["risk_notes"], "steps": content["steps"], "verify": content["verify"],
327
+ "rollback": [s["rollback"] for s in reversed(content["steps"]) if s.get("rollback")],
328
+ "allowlist": {"ips": args.allow_ip or [], "note": "owner-provided only; never guessed"}}
329
+ if content.get("advice"):
330
+ plan["advice"] = content["advice"]
331
+ if args.log and content["engine"] == "fail2ban":
332
+ plan["inputs"]["filter_check"] = wordpress_filter_matches(Path(args.log))
333
+ problems = validate_plan(plan)
334
+ if problems:
335
+ raise core.SecurityError("generated plan failed its own validation: " + "; ".join(problems), core.EXIT_USAGE)
336
+ out = Path(args.out) if args.out else ctx.state.path("plans", f"{plan_id}.json")
337
+ core.write_json(out, plan)
338
+ markdown = render_plan(plan)
339
+ out.with_suffix(".md").write_text(markdown, encoding="utf-8")
340
+ return {**plan, "paths": {"json": str(out), "md": str(out.with_suffix(".md"))}, "markdown": markdown}
341
+
342
+
343
+ def waf_section(args: argparse.Namespace, engine: str) -> dict:
344
+ if not getattr(args, "replay_report", None):
345
+ return {"engine": "none", "mode": "not-planned", "note": "run `maggie security waf-replay --log access.log` first to size a WAF"}
346
+ report = core.read_json(Path(args.replay_report))
347
+ rec = report.get("recommendation") or {}
348
+ return {"engine": "crowdsec-appsec" if engine == "crowdsec" else "modsecurity", "ruleset": "crs-4", "paranoia_level": rec.get("paranoia", 1),
349
+ "inbound_threshold": rec.get("threshold", 5), "mode": "detection-only",
350
+ "exclusions_needed": rec.get("exclusions_needed") or [], "replay_report": args.replay_report,
351
+ "note": "Blocking mode is a second plan after 7 days of detection-only logs show a false-positive rate under 0.1 %."}
352
+
353
+
354
+ def render_plan(plan: dict) -> str:
355
+ lines = [f"# Protection plan {plan['id']} — {plan['host']} ({plan['engine']})", "",
356
+ "**Nothing has been changed.** Review every step; run them yourself (or let an agent run them verbatim after you approve) "
357
+ "while holding the host lock. Stop at the first error.", ""]
358
+ for note in plan["risk_notes"]:
359
+ lines.append(f"> {note}")
360
+ lines.append("")
361
+ if plan.get("advice"):
362
+ lines += ["## Settings to change", ""] + [f"- {a}" for a in plan["advice"]] + [""]
363
+ if plan["steps"]:
364
+ lines += ["## Steps", ""]
365
+ for step in plan["steps"]:
366
+ lines += [f"{step['n']}. **{step['title']}**", " ```bash", f" {step['run']}", " ```"]
367
+ if step.get("rollback"):
368
+ lines.append(f" Undo: `{step['rollback']}`")
369
+ lines += ["", "## Check afterwards", ""] + [f"- `{v['check']}` → {v['expect']}" for v in plan["verify"]]
370
+ lines += ["", "## Downloads from", ""] + [f"- {e['host']}: {e['why']}" for e in plan["egress"]]
371
+ lines += ["", f"Data sharing with CrowdSec's central API: **{'on (you opted in)' if plan['data_sharing']['crowdsec_capi'] else 'off'}**."]
372
+ return "\n".join(lines) + "\n"
373
+
374
+
375
+ def verify_plan(ctx: core.Context) -> dict:
376
+ args = ctx.args
377
+ if not args.plan:
378
+ raise core.UsageError("verify needs --plan FILE")
379
+ plan = core.read_json(Path(args.plan))
380
+ checks = [v["check"] for v in plan.get("verify") or []]
381
+ if not checks:
382
+ return {"schema": "maggie-security-protect-verify.v1", "results": [], "note": "nothing to verify on a host for this plan"}
383
+ if args.evidence:
384
+ output = Path(args.evidence).read_text(encoding="utf-8", errors="replace")
385
+ else:
386
+ if not args.ssh:
387
+ raise core.UsageError("verify needs --ssh ALIAS=user@host (ssh-read consent) or --evidence FILE with the check outputs")
388
+ alias, _, target = args.ssh.partition("=")
389
+ core.require_consent(ctx.state, alias, "ssh-read")
390
+ script = "".join(f"printf '\\n### check-{i}\\n'; {c} 2>&1 | head -40\n" for i, c in enumerate(checks))
391
+ output = subprocess.run(["ssh", "-o", "BatchMode=yes", target, "sh -s"], input=script, capture_output=True, text=True, timeout=300, check=False).stdout
392
+ sections = parse_sections(output)
393
+ results = []
394
+ for i, item in enumerate(plan["verify"]):
395
+ text = sections.get(f"check-{i}", "")
396
+ ok = bool(text.strip()) and not re.search(r"(?i)\b(inactive|failed|not found|error)\b", text)
397
+ if "sharing" in item["expect"]:
398
+ ok = item["expect"].split(": ")[-1] in text
399
+ results.append({"check": item["check"], "expect": item["expect"], "passed": ok, "output": core.clean_excerpt(text, 200)})
400
+ return {"schema": "maggie-security-protect-verify.v1", "plan": plan["id"], "results": results,
401
+ "exit_code": core.EXIT_OK if all(r["passed"] for r in results) else core.EXIT_FINDINGS}
402
+
403
+
404
+ # --------------------------------------------------------------------------- waf-replay (Coraza + OWASP CRS, offline)
405
+
406
+
407
+ def waf_replay(ctx: core.Context) -> dict:
408
+ """Estimate how many real visitors a CRS WAF would block, by replaying an access log offline (PRD 05 section 7).
409
+
410
+ Access logs carry no request bodies and few headers, so only the request
411
+ line, query arguments, User-Agent and Referer are evaluated; the report says so.
412
+ """
413
+ import collections
414
+ import maggie_security_logs as logs
415
+
416
+ args = ctx.args
417
+ path = Path(args.log)
418
+ if not path.is_file():
419
+ raise core.UsageError(f"file not found: {path}")
420
+ lists = logs.IocLists(ctx, [])
421
+ requests, labels = [], []
422
+ per_ip_404 = collections.Counter()
423
+ opener = __import__("gzip").open if path.suffix == ".gz" else open
424
+ parsed = total = 0
425
+ with opener(path, "rt", encoding="utf-8", errors="replace") as handle:
426
+ rows = []
427
+ for line in handle:
428
+ total += 1
429
+ if total > args.sample:
430
+ break
431
+ rec = logs.parse_line(line)
432
+ if rec is None:
433
+ continue
434
+ parsed += 1
435
+ rows.append(rec)
436
+ if rec.status == 404:
437
+ per_ip_404[rec.ip] += 1
438
+ for rec in rows:
439
+ attack = bool(rec.ua and lists.ua.search(rec.ua)) or lists.mass_scanner(rec.ip) or per_ip_404[rec.ip] >= 20
440
+ if args.exclude_bots and attack:
441
+ continue
442
+ requests.append({"method": rec.method or "GET", "uri": rec.target or "/", "ua": rec.ua, "referer": rec.referer, "host": rec.vhost})
443
+ labels.append("attack" if attack else "legit")
444
+ stdin = "".join(json.dumps(r) + "\n" for r in requests).encode()
445
+ proc = core.run_tool("maggie-waf-replay", ["--paranoia", str(args.paranoia), "--threshold", str(args.threshold)], ctx.lock, stdin=stdin, timeout=3600)
446
+ if proc.returncode != 0:
447
+ raise core.DegradedError(f"maggie-waf-replay failed: {core.clean_excerpt(proc.stderr.decode(errors='replace'), 300)}")
448
+ levels: dict[str, dict] = {}
449
+ for line in proc.stdout.decode().splitlines():
450
+ item = json.loads(line)
451
+ label = labels[item["i"]] if item["i"] < len(labels) else "legit"
452
+ request = requests[item["i"]] if item["i"] < len(requests) else {}
453
+ for level, result in item["levels"].items():
454
+ entry = levels.setdefault(level, {"paranoia": int(level), "threshold": args.threshold, "would_block": 0, "would_block_legit_estimate": 0,
455
+ "legit_total": 0, "rules": collections.Counter(), "legit_rules": collections.Counter(), "samples": {}, "legit_samples": {}})
456
+ if label == "legit":
457
+ entry["legit_total"] += 1
458
+ if not result["block"]:
459
+ continue
460
+ entry["would_block"] += 1
461
+ for rule in result["rules"]:
462
+ entry["rules"][rule] += 1
463
+ entry["samples"].setdefault(rule, core.mask_query(request.get("uri", ""))[:150])
464
+ if label == "legit":
465
+ entry["would_block_legit_estimate"] += 1
466
+ for rule in result["rules"]:
467
+ entry["legit_rules"][rule] += 1
468
+ entry["legit_samples"].setdefault(rule, core.mask_query(request.get("uri", ""))[:150])
469
+ out_levels = []
470
+ for level in sorted(levels.values(), key=lambda l: l["paranoia"]):
471
+ rate = level["would_block_legit_estimate"] / level["legit_total"] if level["legit_total"] else 0.0
472
+ out_levels.append({"paranoia": level["paranoia"], "threshold": level["threshold"], "would_block": level["would_block"],
473
+ "would_block_legit_estimate": level["would_block_legit_estimate"], "legit_rate": round(rate, 6),
474
+ "top_rules": [{"id": r, "count": c, "sample_uri": level["samples"][r]} for r, c in level["rules"].most_common(10)],
475
+ "top_legit_rules": [{"id": r, "count": c, "sample_uri": level["legit_samples"][r]} for r, c in level["legit_rules"].most_common(10)]})
476
+ acceptable = [l for l in out_levels if l["legit_rate"] < 0.001]
477
+ best = max(acceptable, key=lambda l: l["paranoia"]) if acceptable else (out_levels[0] if out_levels else None)
478
+ report = {"schema": "maggie-security-waf-replay.v1", "log": {"lines": total, "parsed": parsed, "replayed": len(requests)},
479
+ "coverage": "request-line-args-user-agent-referer-only",
480
+ "coverage_note": "Access logs have no request bodies: form and API body rules were not evaluated, so the real false-positive rate on forms may be higher.",
481
+ "levels": out_levels,
482
+ "recommendation": {"paranoia": best["paranoia"] if best else 1, "threshold": args.threshold, "mode": "detection-only first (7 days)",
483
+ "exclusions_needed": [f"{r['id']} on {r['sample_uri']}" for r in (best or {}).get("top_legit_rules", [])[:5]],
484
+ "blocking_ok": bool(best and best["legit_rate"] < 0.001)},
485
+ "engine": {"name": "maggie-waf-replay (Coraza + OWASP CRS v4)", "version": ctx.lock["tools"]["maggie-waf-replay"]["version"]}}
486
+ out = Path(args.out) if args.out else ctx.state.path("reports", f"waf-replay-{core.iso()[:10]}.json")
487
+ core.write_json(out, report)
488
+ return {**report, "path": str(out)}
489
+
490
+
491
+ def _replay(p: argparse.ArgumentParser) -> None:
492
+ p.add_argument("--log", required=True, help="access log (combined/vcombined/Caddy/Cloudflare JSON; .gz fine)")
493
+ p.add_argument("--paranoia", type=int, choices=[1, 2, 3, 4], default=2)
494
+ p.add_argument("--threshold", type=int, default=5)
495
+ p.add_argument("--sample", type=int, default=100_000)
496
+ p.add_argument("--exclude-bots", action="store_true", help="drop likely-attack traffic before replaying")
497
+ p.add_argument("--out")
498
+
499
+
500
+ def configure(p: argparse.ArgumentParser) -> None:
501
+ p.add_argument("action", nargs="?", choices=["create", "validate", "verify"], default="create")
502
+ p.add_argument("--host", help="host alias (config.json hosts maps it to user@host)")
503
+ p.add_argument("--engine", choices=["auto", "crowdsec", "fail2ban", "cloudflare"], default="auto")
504
+ p.add_argument("--site-type", choices=["wordpress", "php-app", "node-app", "python-app", "static", "workers"])
505
+ p.add_argument("--allow-ip", action="append", help="an address that must never be banned (yours); repeatable")
506
+ p.add_argument("--behind-cloudflare", action="store_true", help="the origin is proxied by Cloudflare")
507
+ p.add_argument("--capi", action="store_true", help="opt in to CrowdSec's community blocklist (shares signals); interactive only")
508
+ p.add_argument("--evidence", help="host-quick collector output for this host")
509
+ p.add_argument("--log", help="a local access log sample to test the WordPress filter against")
510
+ p.add_argument("--plan", help="plan file for validate/verify")
511
+ p.add_argument("--ssh", help="ALIAS=user@host for verify (ssh-read)")
512
+ p.add_argument("--out", help="where to write the plan (default .maggie/security/plans/)")
513
+ p.add_argument("--replay-report", help="waf-replay report: adds a WAF section (detection-only first)")
514
+
515
+
516
+ COMMANDS = [core.Command("waf-replay", "estimate how many real visitors an OWASP CRS WAF would block, by replaying an access log offline (Coraza)",
517
+ [], _replay, waf_replay, findings=False),
518
+ core.Command("protect-plan", "a reviewed plan for CrowdSec or fail2ban (or Cloudflare settings); never applied by Maggie",
519
+ ["dns"], configure, protect_plan, findings=False, routine_safe=False)]