@topy-ai/maggie 0.7.55 → 0.7.56

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (73) hide show
  1. package/README-zh-TW.md +5 -3
  2. package/README.md +35 -8
  3. package/bin/maggie.js +4 -0
  4. package/bundled-contracts/maggie-deployment/readiness-v1.schema.json +2 -1
  5. package/bundled-contracts/maggie-security/finding-example-v1.json +89 -0
  6. package/bundled-contracts/maggie-security/finding-v1.schema.json +63 -0
  7. package/bundled-contracts/maggie-security/findings-set-example-v1.json +148 -0
  8. package/bundled-contracts/maggie-security/findings-set-v1.schema.json +32 -0
  9. package/bundled-contracts/maggie-security/protect-plan-v1.schema.json +28 -0
  10. package/bundled-contracts/maggie-security/report-v1.schema.json +29 -0
  11. package/bundled-contracts/maggie-security/stack-example-v1.json +66 -0
  12. package/bundled-contracts/maggie-security/stack-upload-v1.schema.json +22 -0
  13. package/bundled-contracts/maggie-security/stack-v1.schema.json +37 -0
  14. package/bundled-contracts/maggie-security/tools-lock-v1.schema.json +36 -0
  15. package/bundled-skills/catalog.json +4 -0
  16. package/bundled-skills/maggie-security/SKILL.md +172 -0
  17. package/bundled-skills/maggie-security/agents/openai.yaml +4 -0
  18. package/bundled-skills/maggie-security/data/LICENSE-NOTICE.md +24 -0
  19. package/bundled-skills/maggie-security/data/asvs-map.json +209 -0
  20. package/bundled-skills/maggie-security/data/fail2ban/filter.d/maggie-wordpress.conf +6 -0
  21. package/bundled-skills/maggie-security/data/semgrep/LICENSE-NOTICE.md +5 -0
  22. package/bundled-skills/maggie-security/data/semgrep/VERSION +1 -0
  23. package/bundled-skills/maggie-security/data/semgrep/rules/dockerfile.yaml +14 -0
  24. package/bundled-skills/maggie-security/data/semgrep/rules/generic.yaml +26 -0
  25. package/bundled-skills/maggie-security/data/semgrep/rules/js.yaml +216 -0
  26. package/bundled-skills/maggie-security/data/semgrep/rules/php-debug-ok.yaml +17 -0
  27. package/bundled-skills/maggie-security/data/semgrep/rules/php-debug.yaml +17 -0
  28. package/bundled-skills/maggie-security/data/semgrep/rules/php.yaml +101 -0
  29. package/bundled-skills/maggie-security/data/semgrep/tests/dockerfile.dockerfile +10 -0
  30. package/bundled-skills/maggie-security/data/semgrep/tests/generic.astro +11 -0
  31. package/bundled-skills/maggie-security/data/semgrep/tests/generic.conf +11 -0
  32. package/bundled-skills/maggie-security/data/semgrep/tests/js.jsx +95 -0
  33. package/bundled-skills/maggie-security/data/semgrep/tests/php-debug-ok.php +5 -0
  34. package/bundled-skills/maggie-security/data/semgrep/tests/php-debug.php +3 -0
  35. package/bundled-skills/maggie-security/data/semgrep/tests/php.php +55 -0
  36. package/bundled-skills/maggie-security/evals/evals.json +97 -0
  37. package/bundled-skills/maggie-security/references/commands.md +111 -0
  38. package/bundled-skills/maggie-security/references/cve-response.md +24 -0
  39. package/bundled-skills/maggie-security/references/explaining-findings.md +57 -0
  40. package/bundled-skills/maggie-security/references/findings.md +39 -0
  41. package/bundled-skills/maggie-security/references/ownership-and-consent.md +29 -0
  42. package/bundled-skills/maggie-security/references/site-types.md +22 -0
  43. package/bundled-skills/maggie-security/references/tools-and-licences.md +16 -0
  44. package/bundled-skills/maggie-security/references/workflows.md +23 -0
  45. package/bundled-skills/maggie-security/registry/security-tools.registry.json +628 -0
  46. package/bundled-skills/maggie-security/routines.example.json +27 -0
  47. package/bundled-tools/clis/maggie_deployment_readiness.py +17 -2
  48. package/bundled-tools/clis/maggie_security.py +342 -0
  49. package/bundled-tools/clis/maggie_security_agent.py +326 -0
  50. package/bundled-tools/clis/maggie_security_code.py +650 -0
  51. package/bundled-tools/clis/maggie_security_core.py +1512 -0
  52. package/bundled-tools/clis/maggie_security_deps.py +560 -0
  53. package/bundled-tools/clis/maggie_security_health.py +415 -0
  54. package/bundled-tools/clis/maggie_security_host.py +511 -0
  55. package/bundled-tools/clis/maggie_security_hosted.py +115 -0
  56. package/bundled-tools/clis/maggie_security_intel.py +1294 -0
  57. package/bundled-tools/clis/maggie_security_inventory.py +1023 -0
  58. package/bundled-tools/clis/maggie_security_logs.py +777 -0
  59. package/bundled-tools/clis/maggie_security_mcp.py +473 -0
  60. package/bundled-tools/clis/maggie_security_mirror.py +122 -0
  61. package/bundled-tools/clis/maggie_security_protect.py +519 -0
  62. package/bundled-tools/clis/maggie_security_report.py +506 -0
  63. package/bundled-tools/clis/maggie_security_standards.py +214 -0
  64. package/bundled-tools/clis/maggie_security_versions.py +381 -0
  65. package/bundled-tools/clis/maggie_security_web.py +936 -0
  66. package/bundled-tools/clis/security_data/eol-snapshot.json +1 -0
  67. package/bundled-tools/clis/security_data/gitleaks-rules.json +3812 -0
  68. package/bundled-tools/clis/security_data/maltrail-lists.json +1 -0
  69. package/bundled-tools/clis/security_data/public-suffix-icann.txt +7374 -0
  70. package/bundled-tools/clis/security_data/retire-extractors.json +695 -0
  71. package/bundled-tools/clis/security_data/sigma-web.json +6514 -0
  72. package/bundled-tools/clis/security_data/tools.lock.json +358 -0
  73. package/package.json +1 -1
@@ -0,0 +1,650 @@
1
+ #!/usr/bin/env python3
2
+ """``maggie security secrets`` (and later ``code-scan``) — docs/maggie-security/03-code-secrets-prd.md.
3
+
4
+ Secrets are looked for in three places:
5
+
6
+ - A. git history (``git log -p``), including files deleted since;
7
+ - B. the working tree (tracked and untracked, not ``node_modules``/``vendor``);
8
+ - C. build output that gets served (``dist/``, ``build/``, ``.next/static``,
9
+ ``.output/public``, ``public/``, ``out/``, ``.vercel/output/static``), plus
10
+ the environment-leak check: a value from a ``.env*`` file found verbatim in
11
+ an artifact.
12
+
13
+ Engine: the pinned gitleaks binary when installed (``--redact=100`` so its
14
+ report holds no values), otherwise the native engine with gitleaks' own
15
+ default rules (MIT, converted by ``scripts/security/build_gitleaks_rules.py``)
16
+ plus Maggie's rules. Values are hashed and shown only as first 4 + last 4
17
+ characters; they are never written anywhere.
18
+ """
19
+
20
+ from __future__ import annotations
21
+
22
+ import argparse
23
+ import json
24
+ import math
25
+ import os
26
+ import re
27
+ import subprocess
28
+ import tempfile
29
+ from pathlib import Path
30
+
31
+ import maggie_security_core as core
32
+
33
+ ARTIFACT_DIRS = {"dist": "public", "build": "public", "out": "public", "public": "public", ".output/public": "public",
34
+ ".vercel/output/static": "public", ".next/static": "public", ".next/server": "server", ".svelte-kit/output": "server",
35
+ ".wrangler/deploy": "server"}
36
+ SKIP_DIRS = {"node_modules", "vendor", ".git", ".maggie", "__pycache__", ".venv", "venv", ".cache", "bower_components"}
37
+ MAX_FILE = 5 * 1024 * 1024
38
+ BINARY_EXT = re.compile(r"(?i)\.(png|jpe?g|gif|webp|avif|ico|svgz|woff2?|ttf|otf|eot|pdf|zip|gz|tgz|br|zst|mp4|webm|mp3|wasm|lockb|sqlite|db|bin|exe|dll|so|dylib|class|jar)$")
39
+ ENV_FILE = re.compile(r"^\.env(\..+)?$")
40
+ ENV_SKIP = re.compile(r"(?i)\.(example|sample|template|dist|defaults?)$")
41
+ PLACEHOLDER = re.compile(r"(?i)^(x+|\*+|changeme|change-me|your[_-].*|example.*|dummy.*|placeholder.*|test.*|todo|<.*>|\$\{.*\}|null|none|true|false)$")
42
+
43
+ # Maggie rules on top of gitleaks' defaults (PRD 03 section 6.2).
44
+ MAGGIE_RULES = [
45
+ {"id": "maggie-openrouter-key", "description": "OpenRouter API key", "regex": r"\b(sk-or-v1-[a-f0-9]{64})\b", "keywords": ["sk-or-v1-"]},
46
+ {"id": "maggie-resend-key", "description": "Resend API key", "regex": r"\b(re_[A-Za-z0-9]{8,}_[A-Za-z0-9]{16,})\b", "keywords": ["re_"]},
47
+ {"id": "maggie-db-url", "description": "Database URL with a password",
48
+ "regex": r"\b((?:postgres(?:ql)?|mysql|mariadb|mongodb(?:\+srv)?|redis|rediss)://[^:\s/@'\"]+:([^@\s'\"]{6,})@[^\s'\"]+)", "secretGroup": 2,
49
+ "keywords": ["://"]},
50
+ {"id": "maggie-wp-salts", "description": "WordPress salt or key with a real value",
51
+ "regex": r"define\(\s*'(?:AUTH|SECURE_AUTH|LOGGED_IN|NONCE)_(?:KEY|SALT)'\s*,\s*'([^']{32,})'\s*\)", "keywords": ["_key", "_salt"],
52
+ "allowlists": [{"regexes": [r"put your unique phrase here"]}], "path_exclude": r"wp-config-sample\.php$"},
53
+ {"id": "maggie-public-env-misuse", "description": "A public build variable named like a server secret",
54
+ "regex": r"\b((?:NEXT_PUBLIC|PUBLIC|VITE|NUXT_PUBLIC|REACT_APP|EXPO_PUBLIC)_[A-Z0-9_]*(?:SECRET|PRIVATE|SERVICE_ROLE|API_KEY|PASSWORD|TOKEN)[A-Z0-9_]*)\b",
55
+ "keywords": ["public_", "vite_", "react_app_"], "name_only": True},
56
+ ]
57
+
58
+
59
+ def shannon(text: str) -> float:
60
+ if not text:
61
+ return 0.0
62
+ counts: dict[str, int] = {}
63
+ for ch in text:
64
+ counts[ch] = counts.get(ch, 0) + 1
65
+ return -sum(c / len(text) * math.log2(c / len(text)) for c in counts.values())
66
+
67
+
68
+ class Rules:
69
+ def __init__(self, extra_allow_paths: list[str] | None = None) -> None:
70
+ data = core.read_json(core.DATA_DIR / "gitleaks-rules.json")
71
+ self.version = data.get("version")
72
+ self.rules = []
73
+ self.path_rules: list[dict] = [] # rules that flag a file by its name (e.g. a .p12 key store)
74
+ for rule in data["rules"] + MAGGIE_RULES:
75
+ if not rule.get("regex"):
76
+ if rule.get("path"):
77
+ self.path_rules.append({**rule, "_path": re.compile(rule["path"])})
78
+ continue
79
+ compiled = {**rule, "_re": re.compile(rule["regex"]), "_keywords": [k.lower() for k in rule.get("keywords") or []],
80
+ "_path": re.compile(rule["path"]) if rule.get("path") else None,
81
+ "_path_exclude": re.compile(rule["path_exclude"]) if rule.get("path_exclude") else None,
82
+ "_allow": [{"regexes": [re.compile(r) for r in a.get("regexes", [])], "paths": [re.compile(p) for p in a.get("paths", [])],
83
+ "stopwords": [w.lower() for w in a.get("stopwords", [])], "target": a.get("regexTarget", "secret"),
84
+ "condition": (a.get("condition") or "OR").upper()} for a in rule.get("allowlists") or []]}
85
+ self.rules.append(compiled)
86
+ allow = data.get("allowlist") or {}
87
+ self.allow_paths = [re.compile(p) for p in allow.get("paths", []) + (extra_allow_paths or [])]
88
+ self.allow_regexes = [re.compile(p) for p in allow.get("regexes", [])]
89
+ self.stopwords = [w.lower() for w in allow.get("stopwords", [])]
90
+
91
+ def path_allowed(self, rel: str) -> bool:
92
+ return any(p.search(rel) for p in self.allow_paths)
93
+
94
+ def _rule_allowed(self, rule: dict, secret: str, match: str, line: str, rel: str) -> bool:
95
+ for allow in rule["_allow"]:
96
+ checks = []
97
+ if allow["regexes"]:
98
+ target = {"match": match, "line": line}.get(allow["target"], secret)
99
+ checks.append(any(r.search(target) for r in allow["regexes"]))
100
+ if allow["paths"]:
101
+ checks.append(any(p.search(rel) for p in allow["paths"]))
102
+ if allow["stopwords"]:
103
+ checks.append(any(w in secret.lower() for w in allow["stopwords"]))
104
+ if checks and (all(checks) if allow["condition"] == "AND" else any(checks)):
105
+ return True
106
+ return False
107
+
108
+ def scan_line(self, line: str, rel: str):
109
+ if len(line) > 4000:
110
+ line = line[:4000]
111
+ lower = line.lower()
112
+ for rule in self.rules:
113
+ if rule["_keywords"] and not any(k in lower for k in rule["_keywords"]):
114
+ continue
115
+ if rule["_path"] and not rule["_path"].search(rel):
116
+ continue
117
+ if rule["_path_exclude"] and rule["_path_exclude"].search(rel):
118
+ continue
119
+ for match in rule["_re"].finditer(line):
120
+ group = rule.get("secretGroup")
121
+ secret = match.group(group) if group and match.lastindex and group <= match.lastindex else \
122
+ next((g for g in match.groups() if g), match.group(0)) if match.groups() else match.group(0)
123
+ if not secret:
124
+ continue
125
+ if rule.get("entropy") and shannon(secret) < float(rule["entropy"]):
126
+ continue
127
+ if any(r.search(secret) for r in self.allow_regexes) or any(w in secret.lower() for w in self.stopwords):
128
+ continue
129
+ if not rule.get("name_only") and PLACEHOLDER.match(secret):
130
+ continue
131
+ if self._rule_allowed(rule, secret, match.group(0), line, rel):
132
+ continue
133
+ yield rule, secret
134
+
135
+
136
+ def text_files(root: Path, base: Path, skip_artifacts: bool = False):
137
+ for current, dirs, files in os.walk(root):
138
+ current_path = Path(current)
139
+ rel_dir = current_path.relative_to(base).as_posix() if current_path != base else ""
140
+ dirs[:] = [d for d in dirs if d not in SKIP_DIRS and not (skip_artifacts and _artifact_kind(f"{rel_dir}/{d}".lstrip("/")))]
141
+ for name in files:
142
+ path = current_path / name
143
+ if BINARY_EXT.search(name):
144
+ continue
145
+ try:
146
+ if path.is_symlink() or path.stat().st_size > MAX_FILE:
147
+ continue
148
+ except OSError:
149
+ continue
150
+ yield path
151
+
152
+
153
+ def _artifact_kind(rel: str) -> str | None:
154
+ for prefix, kind in ARTIFACT_DIRS.items():
155
+ if rel == prefix or rel.startswith(prefix + "/"):
156
+ return kind
157
+ return None
158
+
159
+
160
+ def read_text(path: Path) -> str | None:
161
+ try:
162
+ data = path.read_bytes()
163
+ except OSError:
164
+ return None
165
+ if b"\x00" in data[:8192]:
166
+ return None
167
+ return data.decode("utf-8", errors="replace")
168
+
169
+
170
+ class Collector:
171
+ """Groups hits by (rule, secret hash) so one secret in many places is one finding."""
172
+
173
+ def __init__(self) -> None:
174
+ self.groups: dict[tuple[str, str], dict] = {}
175
+ self._values: dict[str, tuple[str, str]] = {} # in memory for this run only: value -> group key; cleared by forget()
176
+
177
+ def forget(self) -> None:
178
+ self._values.clear()
179
+
180
+ def add(self, rule: dict, secret: str, *, rel: str, line_no: int | None, line: str, where: str, commit: str | None = None) -> None:
181
+ info = core.redact_secret(secret)
182
+ key = (rule["id"], info["value_sha256"])
183
+ group = self.groups.setdefault(key, {"rule": rule, "info": info, "hits": []})
184
+ excerpt = line.replace(secret, info["redacted"]).strip()
185
+ hit = {"path": rel, "line": line_no, "excerpt": core.clean_excerpt(excerpt), "redacted": info["redacted"],
186
+ "value_sha256": info["value_sha256"], "where": where}
187
+ if commit:
188
+ hit["commit"] = commit[:12]
189
+ if not any(h["path"] == rel and h["line"] == line_no and h.get("commit") == hit.get("commit") for h in group["hits"]):
190
+ group["hits"].append(hit)
191
+ if len(secret) >= 12 and not secret.startswith(("gitleaks:", "file:")):
192
+ self._values.setdefault(secret, key)
193
+ del secret
194
+
195
+ def find_in_artifacts(self, project: Path) -> int:
196
+ """A value found anywhere (code, history) that also sits in served output, even without a keyword next to it."""
197
+ if not self._values:
198
+ return 0
199
+ hits = 0
200
+ for prefix, kind in ARTIFACT_DIRS.items():
201
+ root = project / prefix
202
+ if not root.is_dir():
203
+ continue
204
+ for path in text_files(root, project):
205
+ text = read_text(path)
206
+ if not text:
207
+ continue
208
+ rel = path.relative_to(project).as_posix()
209
+ for value, key in self._values.items():
210
+ if value in text:
211
+ group = self.groups[key]
212
+ number = text[: text.index(value)].count("\n") + 1
213
+ if any(h["path"] == rel for h in group["hits"]):
214
+ continue
215
+ line = text.splitlines()[number - 1][:400]
216
+ self.add(group["rule"], value, rel=rel, line_no=number, line=line, where=f"artifact-{kind}")
217
+ hits += 1
218
+ return hits
219
+
220
+
221
+ def scan_tree(project: Path, rules: Rules, collector: Collector, include_artifacts: bool = True) -> int:
222
+ count = 0
223
+ for path in text_files(project, project):
224
+ rel = path.relative_to(project).as_posix()
225
+ if rules.path_allowed(rel):
226
+ continue
227
+ kind = _artifact_kind(rel)
228
+ if kind and not include_artifacts:
229
+ continue
230
+ for rule in rules.path_rules:
231
+ if rule["_path"].search(rel):
232
+ collector.add(rule, f"file:{rel}", rel=rel, line_no=None, line=f"file {rel}", where=f"artifact-{kind}" if kind else "tree")
233
+ text = read_text(path)
234
+ if text is None:
235
+ continue
236
+ count += 1
237
+ for number, line in enumerate(text.splitlines(), 1):
238
+ for rule, secret in rules.scan_line(line, rel):
239
+ collector.add(rule, secret, rel=rel, line_no=number, line=line, where=f"artifact-{kind}" if kind else "tree")
240
+ return count
241
+
242
+
243
+ def scan_git(project: Path, rules: Rules, collector: Collector, since: str | None) -> tuple[bool, int]:
244
+ if not (project / ".git").exists():
245
+ return False, 0
246
+ argv = ["git", "-C", str(project), "log", "-p", "--all", "--no-color", "-U0", "--no-ext-diff", "--format=commit %H"]
247
+ if since:
248
+ argv.insert(4, f"--since={since}")
249
+ try:
250
+ proc = subprocess.Popen(argv, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, text=True, errors="replace")
251
+ except OSError:
252
+ return False, 0
253
+ commit = None
254
+ rel = None
255
+ line_no = 0
256
+ commits = 0
257
+ assert proc.stdout is not None
258
+ with proc.stdout:
259
+ commit, rel, line_no, commits = _walk_git_log(proc.stdout, rules, collector)
260
+ proc.wait()
261
+ return proc.returncode == 0, commits
262
+
263
+
264
+ def _walk_git_log(stream, rules: Rules, collector: Collector) -> tuple[str | None, str | None, int, int]:
265
+ commit = None
266
+ rel = None
267
+ line_no = 0
268
+ commits = 0
269
+ for raw in stream:
270
+ line = raw.rstrip("\n")
271
+ if line.startswith("commit ") and len(line) == 47:
272
+ commit = line[7:]
273
+ commits += 1
274
+ elif line.startswith("+++ "):
275
+ rel = None if line.endswith("/dev/null") else line[6:] if line.startswith("+++ b/") else line[4:]
276
+ elif line.startswith("@@"):
277
+ match = re.search(r"\+(\d+)", line)
278
+ line_no = int(match.group(1)) if match else 0
279
+ elif line.startswith("+") and rel and not line.startswith("+++"):
280
+ if not rules.path_allowed(rel):
281
+ for rule, secret in rules.scan_line(line[1:], rel):
282
+ collector.add(rule, secret, rel=rel, line_no=line_no, line=line[1:], where="history", commit=commit)
283
+ line_no += 1
284
+ return commit, rel, line_no, commits
285
+
286
+
287
+ def env_leak_check(project: Path, collector: Collector) -> int:
288
+ """PRD 03 section 6.3: values from .env files that appear verbatim in served artifacts."""
289
+ values: dict[str, list[str]] = {}
290
+ for path in sorted(project.iterdir()) if project.is_dir() else []:
291
+ if not path.is_file() or not ENV_FILE.match(path.name) or ENV_SKIP.search(path.name):
292
+ continue
293
+ for line in (read_text(path) or "").splitlines():
294
+ match = re.match(r"^\s*(?:export\s+)?([A-Za-z_][A-Za-z0-9_]*)\s*=\s*(.*)$", line)
295
+ if not match:
296
+ continue
297
+ value = match.group(2).strip().strip('"').strip("'")
298
+ if len(value) < 8 or PLACEHOLDER.match(value) or re.fullmatch(r"(https?://)?(localhost|127\.0\.0\.1)[:/\w.-]*", value) \
299
+ or re.fullmatch(r"\d+", value) or value.lower() in ("production", "development", "staging"):
300
+ continue
301
+ values.setdefault(value, []).append(match.group(1))
302
+ if not values:
303
+ return 0
304
+ rule = {"id": "maggie-env-value-in-artifact", "description": "A value from a .env file is in served build output"}
305
+ hits = 0
306
+ for prefix, kind in ARTIFACT_DIRS.items():
307
+ root = project / prefix
308
+ if not root.is_dir():
309
+ continue
310
+ for path in text_files(root, project):
311
+ text = read_text(path)
312
+ if not text:
313
+ continue
314
+ rel = path.relative_to(project).as_posix()
315
+ for value, names in values.items():
316
+ if value in text:
317
+ number = text[: text.index(value)].count("\n") + 1
318
+ line = text.splitlines()[number - 1] if text.splitlines() else ""
319
+ collector.add({**rule, "description": f"{rule['description']} ({', '.join(sorted(set(names)))})"}, value,
320
+ rel=rel, line_no=number, line=line[:400], where=f"artifact-{kind}")
321
+ hits += 1
322
+ values.clear()
323
+ return hits
324
+
325
+
326
+ # --------------------------------------------------------------------------- gitleaks binary
327
+
328
+
329
+ def run_gitleaks(ctx: core.Context, project: Path, mode: str, target: Path, since: str | None) -> list[dict] | None:
330
+ with tempfile.TemporaryDirectory() as tmp:
331
+ report = Path(tmp) / "report.json"
332
+ args = [mode, "--report-format", "json", "--report-path", str(report), "--redact=100", "--exit-code", "0"]
333
+ if mode == "git" and since:
334
+ args += ["--log-opts", f"--all --since={since}"]
335
+ if mode == "dir":
336
+ args += ["--max-target-megabytes", "5"]
337
+ args.append(str(target))
338
+ proc = core.run_tool("gitleaks", args, ctx.lock, cwd=project)
339
+ if not report.is_file():
340
+ ctx.degrade(f"gitleaks-{mode}", core.clean_excerpt(proc.stderr.decode(errors="replace"), 200))
341
+ return None
342
+ return json.loads(report.read_text(encoding="utf-8") or "[]")
343
+
344
+
345
+ def gitleaks_to_collector(items: list[dict], project: Path, target: Path, where_default: str, collector: Collector) -> None:
346
+ for item in items:
347
+ rel_file = item.get("File") or ""
348
+ path = Path(rel_file)
349
+ rel = (path if path.is_absolute() else (target / path)).resolve()
350
+ try:
351
+ rel_text = rel.relative_to(project.resolve()).as_posix()
352
+ except ValueError:
353
+ rel_text = rel_file
354
+ kind = _artifact_kind(rel_text)
355
+ where = f"artifact-{kind}" if kind else where_default
356
+ # --redact=100 means Secret is "REDACTED"; the fingerprint stands in for the value hash.
357
+ # --redact=100 means Secret is "REDACTED": rule + file + line stands in for the value hash, so the
358
+ # same line in history and in the working tree is one finding.
359
+ stand_in = f"{rel_text}:{item.get('RuleID')}:{item.get('StartLine')}"
360
+ rule = {"id": item.get("RuleID") or "gitleaks", "description": item.get("Description") or item.get("RuleID") or "secret"}
361
+ collector.add(rule, "gitleaks:" + stand_in, rel=rel_text, line_no=item.get("StartLine"),
362
+ line=f"{item.get('RuleID')} match (value redacted by gitleaks)", where=where, commit=item.get("Commit") or None)
363
+
364
+
365
+ # --------------------------------------------------------------------------- findings
366
+
367
+
368
+ def severity_for(hits: list[dict], rule: dict) -> str:
369
+ if rule.get("name_only"):
370
+ return "medium"
371
+ if any(h["where"] == "artifact-public" for h in hits):
372
+ return "critical"
373
+ return "high"
374
+
375
+
376
+ def to_findings(ctx: core.Context, collector: Collector) -> list[dict]:
377
+ findings = []
378
+ for (rule_id, value_hash), group in collector.groups.items():
379
+ hits = sorted(group["hits"], key=lambda h: ({"artifact-public": 0, "artifact-server": 1, "tree": 2, "history": 3}[h["where"]], h["path"]))
380
+ rule = group["rule"]
381
+ lead = hits[0]
382
+ public = lead["where"] == "artifact-public"
383
+ history_only = all(h["where"] == "history" for h in hits)
384
+ where_text = {"artifact-public": "in a file your site serves publicly", "artifact-server": "in a deployed server bundle",
385
+ "tree": "in the project files", "history": "in git history"}[lead["where"]]
386
+ name_only = rule.get("name_only")
387
+ title = f"{rule.get('description') or rule_id} {where_text}: {lead['path']}" + (f":{lead['line']}" if lead.get("line") else "")
388
+ steps = [] if name_only else ["Revoke or rotate this key with its provider first (assume it is already copied).",
389
+ "Remove it from the code and build output; keep it in the platform's secret store "
390
+ "(for Cloudflare Workers: maggie deployment worker-secrets).",
391
+ "Rebuild and redeploy."]
392
+ if history_only and not name_only:
393
+ steps.append("It is only in git history now: rotating is enough; purging history (git filter-repo) is optional.")
394
+ if name_only:
395
+ steps = ["Rename the variable without the public prefix and read it only on the server; anything with a public prefix is sent to browsers."]
396
+ finding = core.make_finding(
397
+ kind="secret", command="secrets", primary_id=rule_id, site=ctx.site, title=title[:300],
398
+ severity=severity_for(hits, rule), severity_source="maggie", confidence="likely" if name_only else ("confirmed" if public else "likely"),
399
+ asvs_ids=["V13.3.1"], cwe_ids=["CWE-798"],
400
+ target={"path": lead["path"]},
401
+ evidence=[{k: v for k, v in h.items() if k in ("path", "line", "excerpt", "redacted", "value_sha256", "commit")} for h in hits[:20]],
402
+ rule_id=rule_id, tool=ctx.run.get("_engine"), feed=None,
403
+ remediation={"summary": "Rotate the key, then remove it." if not name_only else "Keep secret values out of public build variables.",
404
+ "steps": steps, "snippets": [], "references": ["https://cheatsheetseries.owasp.org/cheatsheets/Secrets_Management_Cheat_Sheet.html"],
405
+ "cheat_sheet": "Secrets Management"})
406
+ # One secret in many files is one finding: the id follows the value hash, not the path.
407
+ finding["id"] = core.fingerprint("secret", rule_id, f"secret:{value_hash[:24]}", ctx.site)
408
+ if public and not name_only:
409
+ finding["priority"] = "p0"
410
+ findings.append(finding)
411
+ return findings
412
+
413
+
414
+ def apply_baseline(ctx: core.Context, findings: list[dict], command: str = "secrets") -> None:
415
+ args = ctx.args
416
+ path = Path(args.baseline) if args.baseline else ctx.state.path("baselines", f"{command}.json")
417
+ if args.write_baseline:
418
+ public = [f for f in findings if f["severity"] == "critical" and f["kind"] == "secret"]
419
+ if public:
420
+ raise core.PolicyError("a secret in served build output cannot be baselined: rotate it and remove it first")
421
+ target = Path(args.write_baseline)
422
+ core.write_json(target, {"schema": "maggie-security-baseline.v1", "command": command, "created_at": core.iso(),
423
+ "rules_version": rule_pack_version() if command == "code-scan" else core.read_json(core.DATA_DIR / "gitleaks-rules.json").get("version"),
424
+ "reason": "baseline", "ids": sorted(f["id"] for f in findings)})
425
+ ctx.run["inputs"]["baseline_written"] = str(target)
426
+ return
427
+ if not path.is_file():
428
+ return
429
+ ids = set(core.read_json(path).get("ids") or [])
430
+ ledger = core.Ledger(ctx.state.path("ledger.jsonl"))
431
+ history = ledger.history()
432
+ for finding in findings:
433
+ if finding["id"] in ids and not (finding["kind"] == "secret" and finding["severity"] == "critical"):
434
+ state = core.Ledger.derive(history.get(finding["id"], [])) if finding["id"] in history else {"status": None}
435
+ if state["status"] != "accepted-risk":
436
+ ledger.append([{"id": finding["id"], "at": core.iso(), "event": "seen", "run": ctx.run["id"], "command": command,
437
+ "severity": finding["severity"]},
438
+ {"id": finding["id"], "at": core.iso(), "event": "accepted-risk", "by": "baseline", "reason": "baseline"}])
439
+ ctx.run["inputs"]["baseline"] = str(path)
440
+
441
+
442
+ def secrets(ctx: core.Context):
443
+ args = ctx.args
444
+ project = ctx.project
445
+ scopes = set((args.scope or "git,tree,dist").split(","))
446
+ config = ctx.state.config()
447
+ collector = Collector()
448
+ engine = args.engine
449
+ if engine == "auto":
450
+ engine = "gitleaks" if core.resolve_tool("gitleaks", ctx.lock)[2] == "pinned" else "native"
451
+ ctx.run["_engine"] = "gitleaks" if engine == "gitleaks" else "maggie-native"
452
+ ctx.run["tools"].append({"name": "gitleaks" if engine == "gitleaks" else "maggie-secrets-native",
453
+ "version": ctx.lock["tools"]["gitleaks"]["version"] if engine == "gitleaks" else core.read_json(core.DATA_DIR / "gitleaks-rules.json").get("version"),
454
+ "sha256": None})
455
+ rules = Rules((config.get("secrets") or {}).get("allow_paths"))
456
+ covered_ok = True
457
+ if engine == "gitleaks":
458
+ if "git" in scopes and (project / ".git").exists():
459
+ items = run_gitleaks(ctx, project, "git", project, args.since)
460
+ covered_ok &= items is not None
461
+ gitleaks_to_collector(items or [], project, project, "history", collector)
462
+ if "tree" in scopes or "dist" in scopes:
463
+ items = run_gitleaks(ctx, project, "dir", project, None)
464
+ covered_ok &= items is not None
465
+ gitleaks_to_collector(items or [], project, project, "tree", collector)
466
+ files = None
467
+ else:
468
+ files = scan_tree(project, rules, collector, include_artifacts="dist" in scopes) if ("tree" in scopes or "dist" in scopes) else 0
469
+ if "git" in scopes:
470
+ ok, commits = scan_git(project, rules, collector, args.since)
471
+ ctx.run["inputs"]["git_commits"] = commits
472
+ if (project / ".git").exists() and not ok:
473
+ ctx.degrade("git-history", "git log failed")
474
+ covered_ok = False
475
+ leaks = env_leak_check(project, collector) if "dist" in scopes else 0
476
+ copied = collector.find_in_artifacts(project) if "dist" in scopes else 0
477
+ collector.forget()
478
+ findings = to_findings(ctx, collector)
479
+ apply_baseline(ctx, findings, "secrets")
480
+ ctx.run["inputs"].update({"engine": ctx.run.pop("_engine"), "scopes": sorted(scopes), "files_scanned": files, "env_value_hits": leaks,
481
+ "copied_into_artifacts": copied})
482
+ return ctx.finish(findings, covered=["*"] if covered_ok and scopes >= {"tree", "dist"} else [])
483
+
484
+
485
+ # --------------------------------------------------------------------------- code-scan (semgrep + Maggie rule pack)
486
+
487
+
488
+ def rule_pack_dir() -> Path:
489
+ for base in (core.ROOT / "skills", core.ROOT / "bundled-skills"):
490
+ path = base / "maggie-security" / "data" / "semgrep"
491
+ if path.is_dir():
492
+ return path
493
+ raise core.UsageError("Maggie semgrep rule pack not found")
494
+
495
+
496
+ def rule_pack_version() -> str:
497
+ return (rule_pack_dir() / "VERSION").read_text(encoding="utf-8").strip()
498
+
499
+
500
+ SEMGREP_EXCLUDES = ["node_modules", "vendor", "dist", "build", ".next", ".output", ".maggie", ".git", "wp-content/plugins", "wp-content/themes",
501
+ "wp-admin", "wp-includes", "coverage", "__pycache__", "*.min.js"] # names match anywhere in the path: keep them specific
502
+ SEMGREP_SEVERITY = {"ERROR": "high", "WARNING": "medium", "INFO": "low"}
503
+
504
+
505
+ def semgrep_to_findings(ctx: core.Context, data: dict, project: Path) -> list[dict]:
506
+ """semgrep --json -> finding-v1 (PRD 03 section 9.1). One finding per rule and file; each hit is evidence."""
507
+ findings = []
508
+ for result in data.get("results") or []:
509
+ extra = result.get("extra") or {}
510
+ meta = extra.get("metadata") or {}
511
+ check = str(result.get("check_id") or "")
512
+ rule_id = check[check.index("maggie."):] if "maggie." in check else check.rsplit(".", 1)[-1]
513
+ path = str(result.get("path") or "")
514
+ try:
515
+ rel = Path(path).resolve().relative_to(project).as_posix()
516
+ except ValueError:
517
+ rel = path
518
+ line = (result.get("start") or {}).get("line")
519
+ severity = meta.get("severity") or SEMGREP_SEVERITY.get(str(extra.get("severity")).upper(), "medium")
520
+ confidence = {"high": "likely", "medium": "possible", "low": "possible"}.get(str(meta.get("confidence") or "medium").lower(), "possible")
521
+ message = str(extra.get("message") or rule_id)
522
+ cwes = meta.get("cwe") or []
523
+ cwes = [cwes] if isinstance(cwes, str) else cwes
524
+ asvs = meta.get("asvs") or []
525
+ asvs = [asvs] if isinstance(asvs, str) else asvs
526
+ findings.append(core.make_finding(
527
+ kind="code", command="code-scan", primary_id=rule_id, site=ctx.site, title=f"{message.split('. ')[0].rstrip('.')} ({rel})"[:300],
528
+ severity=severity, severity_source="tool", confidence=confidence,
529
+ cwe_ids=[c.split(":")[0].strip() for c in cwes if re.match(r"^CWE-\d+", str(c))],
530
+ asvs_ids=[a for a in asvs if re.match(r"^V\d+\.\d+\.\d+$", str(a))], target={"path": rel},
531
+ evidence=[{"path": rel, "line": line, "excerpt": core.clean_excerpt(str(extra.get("lines") or ""), 200)}],
532
+ tool="semgrep", rule_id=rule_id,
533
+ remediation={"summary": message, "steps": [], "snippets": [{"kind": "patch", "text": extra["fix"]}] if extra.get("fix") else [],
534
+ "references": [meta["fix_doc"]] if meta.get("fix_doc") else [], "cheat_sheet": None}))
535
+ return findings
536
+
537
+
538
+ def native_config_checks(ctx: core.Context, files: list[Path], project: Path) -> list[dict]:
539
+ """Two checks semgrep expresses poorly: Dockerfile final stage without USER, nginx serving dotfiles."""
540
+ findings = []
541
+ for path in files:
542
+ rel = path.relative_to(project).as_posix()
543
+ name = path.name.lower()
544
+ text = read_text(path) or ""
545
+ if name == "dockerfile" or name.startswith("dockerfile.") or name.endswith(".dockerfile"):
546
+ stages = re.split(r"(?im)^\s*FROM\s+", text)
547
+ final = stages[-1] if len(stages) > 1 else ""
548
+ if final and not re.search(r"(?im)^\s*USER\s+(?!root\b|0\b)\S+", final) and not re.match(r"(?i)scratch\b", final.strip()):
549
+ findings.append(core.make_finding(
550
+ kind="code", command="code-scan", primary_id="maggie.docker.root-user", site=ctx.site,
551
+ title=f"The container runs as root (no USER in the final stage of {rel})", severity="low", severity_source="maggie",
552
+ confidence="likely", cwe_ids=["CWE-250"], target={"path": rel}, tool="maggie-native", rule_id="maggie.docker.root-user",
553
+ evidence=[{"path": rel, "excerpt": "FROM " + final.splitlines()[0][:150] if final else ""}],
554
+ remediation={"summary": "Add a non-root USER (for example `USER node` or `USER www-data`) in the final stage.", "steps": [],
555
+ "snippets": [], "references": ["https://docs.docker.com/build/building/best-practices/#user"], "cheat_sheet": None}))
556
+ if name.endswith(".conf") or name.startswith("nginx.conf"):
557
+ for block in re.findall(r"(?s)server\s*\{(.*?)\n\}", text):
558
+ if re.search(r"(?m)^\s*root\s+\S+;", block) and not re.search(r"location\s+~\*?\s*/\\\.", block):
559
+ findings.append(core.make_finding(
560
+ kind="code", command="code-scan", primary_id="maggie.nginx.dotfiles-served", site=ctx.site,
561
+ title=f"nginx may serve hidden files like .git or .env ({rel})", severity="medium", severity_source="maggie",
562
+ confidence="likely", cwe_ids=["CWE-538"], asvs_ids=["V13.4.1"], target={"path": rel}, tool="maggie-native",
563
+ rule_id="maggie.nginx.dotfiles-served",
564
+ remediation={"summary": "Deny hidden files in the server block.", "steps": [],
565
+ "snippets": [{"kind": "nginx", "text": "location ~ /\\.(?!well-known) { deny all; }"}],
566
+ "references": [], "cheat_sheet": None}))
567
+ break
568
+ return findings
569
+
570
+
571
+ def changed_files(project: Path, ref: str) -> list[str]:
572
+ proc = subprocess.run(["git", "-C", str(project), "diff", "--name-only", "--diff-filter=ACMR", ref], capture_output=True, text=True, check=False)
573
+ if proc.returncode != 0:
574
+ raise core.UsageError(f"git diff against {ref} failed: {proc.stderr.strip()[:200]}")
575
+ untracked = subprocess.run(["git", "-C", str(project), "ls-files", "--others", "--exclude-standard"], capture_output=True, text=True, check=False)
576
+ return sorted(set(proc.stdout.split()) | set(untracked.stdout.split()))
577
+
578
+
579
+ def code_scan(ctx: core.Context):
580
+ args = ctx.args
581
+ project = ctx.project
582
+ configs = []
583
+ for rule in args.rules or ["maggie"]:
584
+ if rule == "maggie":
585
+ configs.append(str(rule_pack_dir() / "rules"))
586
+ elif "://" in rule or rule in ("auto", "p", "r") or rule.startswith(("p/", "r/")):
587
+ raise core.PolicyError(f"--rules {rule}: only the Maggie pack or local rule folders are allowed (no registry downloads)")
588
+ else:
589
+ local = Path(rule) if Path(rule).is_absolute() else project / rule
590
+ if not local.exists():
591
+ raise core.UsageError(f"rule folder not found: {rule}")
592
+ configs.append(str(local))
593
+ if args.changed_since:
594
+ targets = [str(project / f) for f in changed_files(project, args.changed_since) if (project / f).is_file()]
595
+ if not targets:
596
+ ctx.run["inputs"]["note"] = f"no files changed since {args.changed_since}"
597
+ return ctx.finish([], covered=[])
598
+ else:
599
+ targets = [str(project / p) for p in (args.paths or ["."])]
600
+ files = [Path(t) for t in targets if Path(t).is_file()] + [f for t in targets if Path(t).is_dir() for f in text_files(Path(t), project)]
601
+ findings = native_config_checks(ctx, [f for f in files if not any(part in SKIP_DIRS for part in f.relative_to(project).parts)], project)
602
+ with tempfile.TemporaryDirectory() as tmp:
603
+ out = Path(tmp) / "semgrep.json"
604
+ argv = ["scan", "--json", "--json-output", str(out), "--timeout", "30", "--max-target-bytes", "1000000", "--quiet"]
605
+ for config in configs:
606
+ argv += ["--config", config]
607
+ for exclude in SEMGREP_EXCLUDES:
608
+ argv += ["--exclude", exclude]
609
+ if args.baseline_commit:
610
+ argv += ["--baseline-commit", args.baseline_commit]
611
+ argv += targets
612
+ proc = core.run_tool("semgrep", argv, ctx.lock, cwd=project, timeout=1800, env={"SEMGREP_SEND_METRICS": "off", "SEMGREP_ENABLE_VERSION_CHECK": "0"})
613
+ if not out.is_file():
614
+ raise core.DegradedError(f"semgrep produced no report: {core.clean_excerpt(proc.stderr.decode(errors='replace'), 200)}")
615
+ data = json.loads(out.read_text(encoding="utf-8") or "{}")
616
+ findings += semgrep_to_findings(ctx, data, project)
617
+ errors = [e for e in data.get("errors") or [] if str(e.get("level", "")).lower() == "error"]
618
+ if errors:
619
+ ctx.degrade("semgrep", f"{len(errors)} file(s) could not be parsed")
620
+ version = (data.get("version") or ctx.lock["tools"]["semgrep"]["version"])
621
+ ctx.run["tools"].append({"name": "semgrep", "version": version, "sha256": None})
622
+ ctx.run["inputs"].update({"rule_pack": rule_pack_version(), "configs": [Path(c).name for c in configs], "files": len(files),
623
+ "changed_since": args.changed_since})
624
+ apply_baseline(ctx, findings, "code-scan")
625
+ return ctx.finish(findings, covered=["*"] if not args.changed_since and not args.paths and not errors else [])
626
+
627
+
628
+ def _code(p: argparse.ArgumentParser) -> None:
629
+ p.add_argument("--rules", action="append", help="'maggie' (default) or a local rule folder; registry configs are refused")
630
+ p.add_argument("--paths", nargs="*", help="limit to these paths")
631
+ p.add_argument("--changed-since", help="scan only files changed since a git ref (pre-commit use)")
632
+ p.add_argument("--baseline-commit", help="semgrep's own diff-aware baseline")
633
+ p.add_argument("--baseline", help="baseline file (default .maggie/security/baselines/code-scan.json)")
634
+ p.add_argument("--write-baseline", help="record all current findings as accepted")
635
+
636
+
637
+ def _secrets(p: argparse.ArgumentParser) -> None:
638
+ p.add_argument("--scope", help="comma list of git,tree,dist (default all three)")
639
+ p.add_argument("--engine", choices=["auto", "gitleaks", "native"], default="auto",
640
+ help="auto: the pinned gitleaks if installed, else Maggie's native engine with gitleaks' rules")
641
+ p.add_argument("--baseline", help="baseline file (default .maggie/security/baselines/secrets.json)")
642
+ p.add_argument("--write-baseline", help="record all current findings as accepted (not allowed for public build output)")
643
+
644
+
645
+ COMMANDS = [
646
+ core.Command("code-scan", "find unsafe code patterns (XSS, SSRF, SQL injection, open redirects, insecure cookies…) with semgrep and Maggie's rules",
647
+ [], _code, code_scan),
648
+ core.Command("secrets", "find API keys and passwords in git history, project files and served build output (values never shown)",
649
+ [], _secrets, secrets),
650
+ ]