@topy-ai/maggie 0.7.54 → 0.7.56

package/bundled-tools/clis/maggie_security.py

@@ -0,0 +1,342 @@
+#!/usr/bin/env python3
+"""``maggie security <command>``: one entry point for every security check.
+
+Routes to the area modules (docs/maggie-security/00-foundation-prd.md section
+6.1) and owns the housekeeping commands:
+
+- ``doctor``: what applies to this project, which tools and feeds are ready,
+  ownership, consents, ledger size and last runs. ``--fix`` only writes
+  ``.maggie/security/config.json`` and a ``.gitignore`` entry.
+- ``tools list|install|verify``: the pinned external scanners.
+- ``feeds status|sync|verify``: the local feed cache.
+- ``ledger list|show|accept|false-positive|reopen|compact``: finding status
+  history; the marks are local file writes only.
+
+Exit codes: 0 ok, 1 findings at/above --fail-on, 2 usage, 3 degraded,
+4 refused by policy, 5 tool missing or checksum mismatch.
+"""
+
+from __future__ import annotations
+
+import argparse
+import importlib
+import json
+import sys
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parent))
+
+import maggie_security_core as core  # noqa: E402
+
+AREA_MODULES = [
+    "maggie_security_intel", "maggie_security_deps", "maggie_security_code", "maggie_security_web",
+    "maggie_security_protect", "maggie_security_logs", "maggie_security_host", "maggie_security_health",
+    "maggie_security_standards", "maggie_security_agent", "maggie_security_report", "maggie_security_mcp", "maggie_security_hosted",
+]
+
+# Commands from the PRDs that are not built yet, with their milestone, so the
+# help screen and the skill can say "planned" instead of "unknown".
+PLANNED: dict[str, str] = {}
+
+
+def area_commands() -> dict[str, core.Command]:
+    commands: dict[str, core.Command] = {}
+    for name in AREA_MODULES:
+        try:
+            module = importlib.import_module(name)
+        except ModuleNotFoundError as error:
+            if error.name == name:
+                continue
+            raise
+        for command in getattr(module, "COMMANDS", []):
+            commands[command.name] = command
+    return commands
+
+
+# --------------------------------------------------------------------------- doctor
+
+
+def doctor(ctx: core.Context) -> dict:
+    project, state = ctx.project, ctx.state
+    config = state.config()
+    site_type = config.get("site_type") or core.detect_site_type(project)
+    domains = config.get("domains") or []
+    lock = core.load_lock(state)
+    tools = []
+    for name, spec in sorted((lock.get("tools") or {}).items()):
+        if spec.get("class") == "reference-only":
+            continue
+        _spec, path, how = core.resolve_tool(name, lock)
+        tools.append({"name": name, "version": spec["version"], "class": spec["class"], "how": how})
+    ownership = []
+    for domain in domains:
+        record_path = state.path("ownership", f"{domain}.json")
+        record = core.read_json(record_path) if record_path.is_file() else None
+        status = "unverified"
+        if record and record.get("verified_at"):
+            age = core.utcnow() - core.parse_time(record["verified_at"])
+            status = "verified" if age.days <= 90 else "expired"
+        ownership.append({"domain": domain, "status": status, "verified_at": (record or {}).get("verified_at")})
+    consent_path = state.path("consent.json")
+    consents = core.read_json(consent_path).get("grants", []) if consent_path.is_file() else []
+    ledger = core.Ledger(state.path("ledger.jsonl"))
+    history = ledger.history()
+    statuses: dict[str, int] = {}
+    for events in history.values():
+        status = core.Ledger.derive(events)["status"] or "unknown"
+        statuses[status] = statuses.get(status, 0) + 1
+    last_runs: dict[str, str] = {}
+    runs_dir = state.path("runs")
+    if runs_dir.is_dir():
+        for path in sorted(runs_dir.glob("*.json")):
+            try:
+                run = json.loads(path.read_text(encoding="utf-8"))["run"]
+            except (ValueError, KeyError, OSError):
+                continue
+            last_runs[run["command"]] = run["finished_at"]
+    available = sorted(area_commands())
+    applicable = [c for c in core.APPLICABLE.get(site_type, core.APPLICABLE["unknown"])]
+    gitignore = project / ".gitignore"
+    ignored = gitignore.is_file() and any(line.strip().rstrip("/*") in (".maggie/security", "/.maggie/security", ".maggie")
+                                          for line in gitignore.read_text(encoding="utf-8").splitlines())
+    problems = []
+    if not ignored and state.root.exists():
+        problems.append("`.maggie/security/` is not in .gitignore (run: maggie security doctor --fix)")
+    if not domains:
+        problems.append("no domains configured; web checks are skipped (set domains in .maggie/security/config.json)")
+    report = {
+        "schema": "maggie-security-status.v1",
+        "maggie_version": core.maggie_version(),
+        "project": str(project),
+        "site": ctx.site,
+        "site_type": site_type,
+        "domains": domains,
+        "commands": {"applicable": applicable, "available": available,
+                     "applicable_not_built": [c for c in applicable if c not in available and c not in ("report",)]},
+        "tools": tools,
+        "feeds": core.FeedCache(state, ctx.guard).status(),
+        "ownership": ownership,
+        "consents": consents,
+        "ledger": {"ids": len(history), "by_status": statuses},
+        "last_runs": last_runs,
+        "gitignored": ignored,
+        "mirror": core.mirror_base(),
+        "problems": problems,
+    }
+    if getattr(ctx.args, "fix", False):
+        report["fixed"] = doctor_fix(project, state, site_type, ctx.site)
+    return report
+
+
+def doctor_fix(project: Path, state: core.State, site_type: str, site: str) -> list[str]:
+    done = []
+    state.ensure()
+    config_path = state.path("config.json")
+    if not config_path.is_file():
+        core.write_json(config_path, {"schema": "maggie-security-config.v1", "site": site, "site_type": site_type, "domains": [],
+                                      "schedules": {}, "thresholds": {"fail_on": "high"}, "opt_ins": {}})
+        done.append("created .maggie/security/config.json")
+    gitignore = project / ".gitignore"
+    lines = gitignore.read_text(encoding="utf-8").splitlines() if gitignore.is_file() else []
+    wanted = [".maggie/security/*", "!.maggie/security/config.json"]
+    missing = [line for line in wanted if line not in lines]
+    if missing:
+        with gitignore.open("a", encoding="utf-8") as handle:
+            if lines and lines[-1].strip():
+                handle.write("\n")
+            handle.write("# maggie security state (host evidence, findings); config.json is safe to commit\n")
+            handle.write("\n".join(missing) + "\n")
+        done.append("added .maggie/security/ to .gitignore")
+    ledger = core.Ledger(state.path("ledger.jsonl"))
+    if ledger.path.is_file():
+        result = ledger.compact()
+        done.append(f"compacted ledger {result['before']} -> {result['after']} events")
+    return done
+
+
+def doctor_text(report: dict) -> str:
+    lines = [f"maggie security doctor: {report['site']} ({report['site_type']})"]
+    lines.append(f"  domains: {', '.join(report['domains']) or 'none'}")
+    lines.append(f"  applicable: {', '.join(report['commands']['applicable'])}")
+    if report["commands"]["applicable_not_built"]:
+        lines.append(f"  not built yet: {', '.join(report['commands']['applicable_not_built'])}")
+    for tool in report["tools"]:
+        lines.append(f"  tool {tool['name']:<12} {tool['version']:<10} {tool['how']}")
+    for feed in report["feeds"]:
+        if feed["entries"]:
+            lines.append(f"  feed {feed['id']:<16} {feed['fetched_at']}{' STALE' if feed['stale'] else ''}")
+    lines.append(f"  ledger: {report['ledger']['ids']} findings {report['ledger']['by_status']}")
+    for problem in report["problems"]:
+        lines.append(f"  ! {problem}")
+    for item in report.get("fixed", []):
+        lines.append(f"  fixed: {item}")
+    return "\n".join(lines)
+
+
+# --------------------------------------------------------------------------- tools, feeds, ledger
+
+
+def tools_cmd(ctx: core.Context) -> dict:
+    args = ctx.args
+    lock = core.load_lock(ctx.state)
+    names = [args.tool] if args.tool else sorted(lock.get("tools", {}))
+    if args.action == "list":
+        return {"schema": "maggie-security-tools.v1", "platform": core.platform_key(), "cache": str(core.tools_cache_dir()),
+                "tools": [{"name": n, **{k: v for k, v in lock["tools"][n].items() if k in ("version", "licence", "class", "notes")},
+                           "how": core.resolve_tool(n, lock)[2]} for n in names]}
+    if args.action == "verify":
+        results = [core.verify_tool(n, lock) for n in names if lock["tools"][n]["class"] != "reference-only"]
+        bad = [r for r in results if not r["ok"] and r["how"] != "missing"]
+        if bad:
+            raise core.ToolError("; ".join(f"{r['tool']}: {r['problem']}" for r in bad))
+        return {"schema": "maggie-security-tools.v1", "results": results}
+    if not args.tool:
+        raise core.UsageError("tools install needs --tool NAME")
+    result = core.install_tool(args.tool, lock, ctx.guard)
+    ctx.state.ensure()
+    core.write_json(ctx.state.path("tools.lock.json"), core.read_json(core.packaged_lock_path()))
+    return result
+
+
+def feeds_cmd(ctx: core.Context) -> dict:
+    args = ctx.args
+    if args.action == "status":
+        return {"schema": "maggie-security-feeds.v1", "mirror": core.mirror_base(), "feeds": ctx.cache.status()}
+    if args.action == "mirror":
+        import maggie_security_mirror as mirror
+
+        if not args.dir:
+            raise core.UsageError("feeds mirror needs --dir DIR")
+        ecosystems = [e for e in (args.ecosystems or ",".join(mirror.DEFAULT_ECOSYSTEMS)).split(",") if e]
+        return mirror.build(ctx, Path(args.dir), ecosystems, args.days, args.to_project)
+    if args.action == "verify":
+        problems = ctx.cache.verify()
+        if problems:
+            raise core.DegradedError(json.dumps(problems))
+        return {"schema": "maggie-security-feeds.v1", "problems": []}
+    # sync: fetch fixed-URL sources (the ones that take parameters are fetched by their commands)
+    wanted = [args.source] if args.source else ["cvelistv5-delta", "kev", "node-core", "nuclei-cves"]
+    results = []
+    for source in wanted:
+        spec = core.SOURCES.get(source)
+        if not spec or "{" in spec["url"] or not spec["url"]:
+            raise core.UsageError(f"{source} cannot be synced on its own; it is fetched per query")
+        data = ctx.cache.get(source, spec["url"], ttl="0s" if args.force else None)
+        results.append({"id": source, "bytes": len(data) if data else 0})
+    degraded = [item for item in ctx.cache.report if item["status"] == "degraded"]
+    out = {"schema": "maggie-security-feeds.v1", "synced": results, "report": ctx.cache.report}
+    if degraded:
+        print(json.dumps(out, indent=2))
+        raise core.DegradedError(", ".join(d["id"] for d in degraded) + " unavailable")
+    return out
+
+
+def ledger_cmd(ctx: core.Context) -> dict:
+    args = ctx.args
+    ledger = core.Ledger(ctx.state.path("ledger.jsonl"))
+    if args.action == "list":
+        items = []
+        for fid, events in ledger.history().items():
+            state = core.Ledger.derive(events)
+            if args.status and state["status"] != args.status:
+                continue
+            items.append({"id": fid, **state})
+        return {"schema": "maggie-security-ledger.v1", "items": items}
+    if args.action == "compact":
+        return ledger.compact()
+    if not args.id:
+        raise core.UsageError(f"ledger {args.action} needs --id")
+    if args.action == "show":
+        events = ledger.history().get(args.id)
+        if not events:
+            raise core.UsageError(f"unknown finding id {args.id}")
+        return {"id": args.id, **core.Ledger.derive(events), "events": events}
+    event = {"accept": "accepted-risk", "false-positive": "false-positive", "reopen": "reopened"}[args.action]
+    until = args.until
+    if event == "accepted-risk" and not until:
+        until = core.iso(core.utcnow() + core.parse_duration("90d"))[:10]
+    return ledger.mark(args.id, event, args.reason, until=until if event == "accepted-risk" else None)
+
+
+HOUSEKEEPING = [
+    core.Command("doctor", "what applies to this project, tool and feed readiness, ownership, consents, ledger",
+                 [], lambda p: p.add_argument("--fix", action="store_true", help="create config.json and the .gitignore entry"),
+                 doctor, findings=False),
+    core.Command("tools", "pinned external scanners: list, install, verify", ["github-releases"],
+                 lambda p: (p.add_argument("action", choices=["list", "install", "verify"]), p.add_argument("--tool")),
+                 tools_cmd, findings=False),
+    core.Command("feeds", "local feed cache: status, sync, verify; mirror builds an offline copy of every feed", list(core.SOURCES),
+                 lambda p: (p.add_argument("action", choices=["status", "sync", "verify", "mirror"]), p.add_argument("--source"),
+                            p.add_argument("--force", action="store_true", help="ignore TTL"),
+                            p.add_argument("--dir", help="mirror: target directory"),
+                            p.add_argument("--ecosystems", help="mirror: OSV ecosystems to download (default the common eight)"),
+                            p.add_argument("--days", type=int, default=7, help="mirror: days of CVE delta zips"),
+                            p.add_argument("--to-project", action="store_true", help="mirror: also place OSV zips in .maggie/security/db/osv")),
+                 feeds_cmd, findings=False),
+    core.Command("ledger", "finding status history (local marks only)", [],
+                 lambda p: (p.add_argument("action", choices=["list", "show", "accept", "false-positive", "reopen", "compact"]),
+                            p.add_argument("--id"), p.add_argument("--reason"), p.add_argument("--until", help="YYYY-MM-DD for accept (default +90d)"),
+                            p.add_argument("--status", choices=core.STATUSES)),
+                 ledger_cmd, findings=False),
+]
+
+
+def build_parser(commands: dict[str, core.Command]) -> argparse.ArgumentParser:
+    parser = argparse.ArgumentParser(prog="maggie security", description="Maggie security checks (docs/maggie-security).")
+    sub = parser.add_subparsers(dest="command", required=True, metavar="<command>")
+    common = core.common_parent()
+    for command in commands.values():
+        p = sub.add_parser(command.name, parents=[common], help=command.help, description=command.help)
+        command.configure(p)
+    return parser
+
+
+def main(argv: list[str] | None = None) -> int:
+    argv = list(sys.argv[1:] if argv is None else argv)
+    commands = {c.name: c for c in HOUSEKEEPING}
+    commands.update(area_commands())
+    if argv and argv[0] in PLANNED and argv[0] not in commands:
+        print(f"maggie security {argv[0]} is planned for {PLANNED[argv[0]]} and not built yet", file=sys.stderr)
+        return core.EXIT_USAGE
+    parser = build_parser(commands)
+    try:
+        args = parser.parse_args(argv)
+    except SystemExit as error:
+        return core.EXIT_OK if error.code == 0 else core.EXIT_USAGE
+    command = commands[args.command]
+    if args.explain_egress:
+        return core.explain_egress(command.sources, command.name)
+    try:
+        ctx = core.Context(args, command.name, command.sources)
+        if args.offline:
+            with core.socket_guard():
+                result = command.run(ctx)
+        else:
+            result = command.run(ctx)
+        if isinstance(result, dict) and "_raw" in result:
+            if result["_raw"] is not None:
+                print(result["_raw"])
+            return core.EXIT_OK
+        if not command.findings:
+            if isinstance(result, dict) and "_raw" in result:
+                if result["_raw"] is not None:
+                    print(result["_raw"])
+                return core.EXIT_OK
+            fmt = core.output_format(args)
+            if fmt == "text" and command.name == "doctor":
+                print(doctor_text(result))
+            elif fmt == "md" and isinstance(result, dict) and "markdown" in result:
+                print(result["markdown"])
+            else:
+                print(json.dumps(result, indent=2, ensure_ascii=False))
+            return result.get("exit_code", core.EXIT_OK) if isinstance(result, dict) else core.EXIT_OK
+        if isinstance(result, int):
+            return result
+        return core.emit(ctx, result)
+    except core.SecurityError as error:
+        print(f"maggie security {args.command}: {error}", file=sys.stderr)
+        return error.code
+
+
+if __name__ == "__main__":
+    sys.exit(main())