@topy-ai/maggie 0.7.55 → 0.7.56
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README-zh-TW.md +5 -3
- package/README.md +35 -8
- package/bin/maggie.js +4 -0
- package/bundled-contracts/maggie-deployment/readiness-v1.schema.json +2 -1
- package/bundled-contracts/maggie-security/finding-example-v1.json +89 -0
- package/bundled-contracts/maggie-security/finding-v1.schema.json +63 -0
- package/bundled-contracts/maggie-security/findings-set-example-v1.json +148 -0
- package/bundled-contracts/maggie-security/findings-set-v1.schema.json +32 -0
- package/bundled-contracts/maggie-security/protect-plan-v1.schema.json +28 -0
- package/bundled-contracts/maggie-security/report-v1.schema.json +29 -0
- package/bundled-contracts/maggie-security/stack-example-v1.json +66 -0
- package/bundled-contracts/maggie-security/stack-upload-v1.schema.json +22 -0
- package/bundled-contracts/maggie-security/stack-v1.schema.json +37 -0
- package/bundled-contracts/maggie-security/tools-lock-v1.schema.json +36 -0
- package/bundled-skills/catalog.json +4 -0
- package/bundled-skills/maggie-security/SKILL.md +172 -0
- package/bundled-skills/maggie-security/agents/openai.yaml +4 -0
- package/bundled-skills/maggie-security/data/LICENSE-NOTICE.md +24 -0
- package/bundled-skills/maggie-security/data/asvs-map.json +209 -0
- package/bundled-skills/maggie-security/data/fail2ban/filter.d/maggie-wordpress.conf +6 -0
- package/bundled-skills/maggie-security/data/semgrep/LICENSE-NOTICE.md +5 -0
- package/bundled-skills/maggie-security/data/semgrep/VERSION +1 -0
- package/bundled-skills/maggie-security/data/semgrep/rules/dockerfile.yaml +14 -0
- package/bundled-skills/maggie-security/data/semgrep/rules/generic.yaml +26 -0
- package/bundled-skills/maggie-security/data/semgrep/rules/js.yaml +216 -0
- package/bundled-skills/maggie-security/data/semgrep/rules/php-debug-ok.yaml +17 -0
- package/bundled-skills/maggie-security/data/semgrep/rules/php-debug.yaml +17 -0
- package/bundled-skills/maggie-security/data/semgrep/rules/php.yaml +101 -0
- package/bundled-skills/maggie-security/data/semgrep/tests/dockerfile.dockerfile +10 -0
- package/bundled-skills/maggie-security/data/semgrep/tests/generic.astro +11 -0
- package/bundled-skills/maggie-security/data/semgrep/tests/generic.conf +11 -0
- package/bundled-skills/maggie-security/data/semgrep/tests/js.jsx +95 -0
- package/bundled-skills/maggie-security/data/semgrep/tests/php-debug-ok.php +5 -0
- package/bundled-skills/maggie-security/data/semgrep/tests/php-debug.php +3 -0
- package/bundled-skills/maggie-security/data/semgrep/tests/php.php +55 -0
- package/bundled-skills/maggie-security/evals/evals.json +97 -0
- package/bundled-skills/maggie-security/references/commands.md +111 -0
- package/bundled-skills/maggie-security/references/cve-response.md +24 -0
- package/bundled-skills/maggie-security/references/explaining-findings.md +57 -0
- package/bundled-skills/maggie-security/references/findings.md +39 -0
- package/bundled-skills/maggie-security/references/ownership-and-consent.md +29 -0
- package/bundled-skills/maggie-security/references/site-types.md +22 -0
- package/bundled-skills/maggie-security/references/tools-and-licences.md +16 -0
- package/bundled-skills/maggie-security/references/workflows.md +23 -0
- package/bundled-skills/maggie-security/registry/security-tools.registry.json +628 -0
- package/bundled-skills/maggie-security/routines.example.json +27 -0
- package/bundled-tools/clis/maggie_deployment_readiness.py +17 -2
- package/bundled-tools/clis/maggie_security.py +342 -0
- package/bundled-tools/clis/maggie_security_agent.py +326 -0
- package/bundled-tools/clis/maggie_security_code.py +650 -0
- package/bundled-tools/clis/maggie_security_core.py +1512 -0
- package/bundled-tools/clis/maggie_security_deps.py +560 -0
- package/bundled-tools/clis/maggie_security_health.py +415 -0
- package/bundled-tools/clis/maggie_security_host.py +511 -0
- package/bundled-tools/clis/maggie_security_hosted.py +115 -0
- package/bundled-tools/clis/maggie_security_intel.py +1294 -0
- package/bundled-tools/clis/maggie_security_inventory.py +1023 -0
- package/bundled-tools/clis/maggie_security_logs.py +777 -0
- package/bundled-tools/clis/maggie_security_mcp.py +473 -0
- package/bundled-tools/clis/maggie_security_mirror.py +122 -0
- package/bundled-tools/clis/maggie_security_protect.py +519 -0
- package/bundled-tools/clis/maggie_security_report.py +506 -0
- package/bundled-tools/clis/maggie_security_standards.py +214 -0
- package/bundled-tools/clis/maggie_security_versions.py +381 -0
- package/bundled-tools/clis/maggie_security_web.py +936 -0
- package/bundled-tools/clis/security_data/eol-snapshot.json +1 -0
- package/bundled-tools/clis/security_data/gitleaks-rules.json +3812 -0
- package/bundled-tools/clis/security_data/maltrail-lists.json +1 -0
- package/bundled-tools/clis/security_data/public-suffix-icann.txt +7374 -0
- package/bundled-tools/clis/security_data/retire-extractors.json +695 -0
- package/bundled-tools/clis/security_data/sigma-web.json +6514 -0
- package/bundled-tools/clis/security_data/tools.lock.json +358 -0
- package/package.json +1 -1
|
@@ -0,0 +1,650 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""``maggie security secrets`` (and later ``code-scan``) — docs/maggie-security/03-code-secrets-prd.md.
|
|
3
|
+
|
|
4
|
+
Secrets are looked for in three places:
|
|
5
|
+
|
|
6
|
+
- A. git history (``git log -p``), including files deleted since;
|
|
7
|
+
- B. the working tree (tracked and untracked, not ``node_modules``/``vendor``);
|
|
8
|
+
- C. build output that gets served (``dist/``, ``build/``, ``.next/static``,
|
|
9
|
+
``.output/public``, ``public/``, ``out/``, ``.vercel/output/static``), plus
|
|
10
|
+
the environment-leak check: a value from a ``.env*`` file found verbatim in
|
|
11
|
+
an artifact.
|
|
12
|
+
|
|
13
|
+
Engine: the pinned gitleaks binary when installed (``--redact=100`` so its
|
|
14
|
+
report holds no values), otherwise the native engine with gitleaks' own
|
|
15
|
+
default rules (MIT, converted by ``scripts/security/build_gitleaks_rules.py``)
|
|
16
|
+
plus Maggie's rules. Values are hashed and shown only as first 4 + last 4
|
|
17
|
+
characters; they are never written anywhere.
|
|
18
|
+
"""
|
|
19
|
+
|
|
20
|
+
from __future__ import annotations
|
|
21
|
+
|
|
22
|
+
import argparse
|
|
23
|
+
import json
|
|
24
|
+
import math
|
|
25
|
+
import os
|
|
26
|
+
import re
|
|
27
|
+
import subprocess
|
|
28
|
+
import tempfile
|
|
29
|
+
from pathlib import Path
|
|
30
|
+
|
|
31
|
+
import maggie_security_core as core
|
|
32
|
+
|
|
33
|
+
ARTIFACT_DIRS = {"dist": "public", "build": "public", "out": "public", "public": "public", ".output/public": "public",
|
|
34
|
+
".vercel/output/static": "public", ".next/static": "public", ".next/server": "server", ".svelte-kit/output": "server",
|
|
35
|
+
".wrangler/deploy": "server"}
|
|
36
|
+
SKIP_DIRS = {"node_modules", "vendor", ".git", ".maggie", "__pycache__", ".venv", "venv", ".cache", "bower_components"}
|
|
37
|
+
MAX_FILE = 5 * 1024 * 1024
|
|
38
|
+
BINARY_EXT = re.compile(r"(?i)\.(png|jpe?g|gif|webp|avif|ico|svgz|woff2?|ttf|otf|eot|pdf|zip|gz|tgz|br|zst|mp4|webm|mp3|wasm|lockb|sqlite|db|bin|exe|dll|so|dylib|class|jar)$")
|
|
39
|
+
ENV_FILE = re.compile(r"^\.env(\..+)?$")
|
|
40
|
+
ENV_SKIP = re.compile(r"(?i)\.(example|sample|template|dist|defaults?)$")
|
|
41
|
+
PLACEHOLDER = re.compile(r"(?i)^(x+|\*+|changeme|change-me|your[_-].*|example.*|dummy.*|placeholder.*|test.*|todo|<.*>|\$\{.*\}|null|none|true|false)$")
|
|
42
|
+
|
|
43
|
+
# Maggie rules on top of gitleaks' defaults (PRD 03 section 6.2).
|
|
44
|
+
MAGGIE_RULES = [
|
|
45
|
+
{"id": "maggie-openrouter-key", "description": "OpenRouter API key", "regex": r"\b(sk-or-v1-[a-f0-9]{64})\b", "keywords": ["sk-or-v1-"]},
|
|
46
|
+
{"id": "maggie-resend-key", "description": "Resend API key", "regex": r"\b(re_[A-Za-z0-9]{8,}_[A-Za-z0-9]{16,})\b", "keywords": ["re_"]},
|
|
47
|
+
{"id": "maggie-db-url", "description": "Database URL with a password",
|
|
48
|
+
"regex": r"\b((?:postgres(?:ql)?|mysql|mariadb|mongodb(?:\+srv)?|redis|rediss)://[^:\s/@'\"]+:([^@\s'\"]{6,})@[^\s'\"]+)", "secretGroup": 2,
|
|
49
|
+
"keywords": ["://"]},
|
|
50
|
+
{"id": "maggie-wp-salts", "description": "WordPress salt or key with a real value",
|
|
51
|
+
"regex": r"define\(\s*'(?:AUTH|SECURE_AUTH|LOGGED_IN|NONCE)_(?:KEY|SALT)'\s*,\s*'([^']{32,})'\s*\)", "keywords": ["_key", "_salt"],
|
|
52
|
+
"allowlists": [{"regexes": [r"put your unique phrase here"]}], "path_exclude": r"wp-config-sample\.php$"},
|
|
53
|
+
{"id": "maggie-public-env-misuse", "description": "A public build variable named like a server secret",
|
|
54
|
+
"regex": r"\b((?:NEXT_PUBLIC|PUBLIC|VITE|NUXT_PUBLIC|REACT_APP|EXPO_PUBLIC)_[A-Z0-9_]*(?:SECRET|PRIVATE|SERVICE_ROLE|API_KEY|PASSWORD|TOKEN)[A-Z0-9_]*)\b",
|
|
55
|
+
"keywords": ["public_", "vite_", "react_app_"], "name_only": True},
|
|
56
|
+
]
|
|
57
|
+
|
|
58
|
+
|
|
59
|
+
def shannon(text: str) -> float:
|
|
60
|
+
if not text:
|
|
61
|
+
return 0.0
|
|
62
|
+
counts: dict[str, int] = {}
|
|
63
|
+
for ch in text:
|
|
64
|
+
counts[ch] = counts.get(ch, 0) + 1
|
|
65
|
+
return -sum(c / len(text) * math.log2(c / len(text)) for c in counts.values())
|
|
66
|
+
|
|
67
|
+
|
|
68
|
+
class Rules:
|
|
69
|
+
def __init__(self, extra_allow_paths: list[str] | None = None) -> None:
|
|
70
|
+
data = core.read_json(core.DATA_DIR / "gitleaks-rules.json")
|
|
71
|
+
self.version = data.get("version")
|
|
72
|
+
self.rules = []
|
|
73
|
+
self.path_rules: list[dict] = [] # rules that flag a file by its name (e.g. a .p12 key store)
|
|
74
|
+
for rule in data["rules"] + MAGGIE_RULES:
|
|
75
|
+
if not rule.get("regex"):
|
|
76
|
+
if rule.get("path"):
|
|
77
|
+
self.path_rules.append({**rule, "_path": re.compile(rule["path"])})
|
|
78
|
+
continue
|
|
79
|
+
compiled = {**rule, "_re": re.compile(rule["regex"]), "_keywords": [k.lower() for k in rule.get("keywords") or []],
|
|
80
|
+
"_path": re.compile(rule["path"]) if rule.get("path") else None,
|
|
81
|
+
"_path_exclude": re.compile(rule["path_exclude"]) if rule.get("path_exclude") else None,
|
|
82
|
+
"_allow": [{"regexes": [re.compile(r) for r in a.get("regexes", [])], "paths": [re.compile(p) for p in a.get("paths", [])],
|
|
83
|
+
"stopwords": [w.lower() for w in a.get("stopwords", [])], "target": a.get("regexTarget", "secret"),
|
|
84
|
+
"condition": (a.get("condition") or "OR").upper()} for a in rule.get("allowlists") or []]}
|
|
85
|
+
self.rules.append(compiled)
|
|
86
|
+
allow = data.get("allowlist") or {}
|
|
87
|
+
self.allow_paths = [re.compile(p) for p in allow.get("paths", []) + (extra_allow_paths or [])]
|
|
88
|
+
self.allow_regexes = [re.compile(p) for p in allow.get("regexes", [])]
|
|
89
|
+
self.stopwords = [w.lower() for w in allow.get("stopwords", [])]
|
|
90
|
+
|
|
91
|
+
def path_allowed(self, rel: str) -> bool:
|
|
92
|
+
return any(p.search(rel) for p in self.allow_paths)
|
|
93
|
+
|
|
94
|
+
def _rule_allowed(self, rule: dict, secret: str, match: str, line: str, rel: str) -> bool:
|
|
95
|
+
for allow in rule["_allow"]:
|
|
96
|
+
checks = []
|
|
97
|
+
if allow["regexes"]:
|
|
98
|
+
target = {"match": match, "line": line}.get(allow["target"], secret)
|
|
99
|
+
checks.append(any(r.search(target) for r in allow["regexes"]))
|
|
100
|
+
if allow["paths"]:
|
|
101
|
+
checks.append(any(p.search(rel) for p in allow["paths"]))
|
|
102
|
+
if allow["stopwords"]:
|
|
103
|
+
checks.append(any(w in secret.lower() for w in allow["stopwords"]))
|
|
104
|
+
if checks and (all(checks) if allow["condition"] == "AND" else any(checks)):
|
|
105
|
+
return True
|
|
106
|
+
return False
|
|
107
|
+
|
|
108
|
+
def scan_line(self, line: str, rel: str):
|
|
109
|
+
if len(line) > 4000:
|
|
110
|
+
line = line[:4000]
|
|
111
|
+
lower = line.lower()
|
|
112
|
+
for rule in self.rules:
|
|
113
|
+
if rule["_keywords"] and not any(k in lower for k in rule["_keywords"]):
|
|
114
|
+
continue
|
|
115
|
+
if rule["_path"] and not rule["_path"].search(rel):
|
|
116
|
+
continue
|
|
117
|
+
if rule["_path_exclude"] and rule["_path_exclude"].search(rel):
|
|
118
|
+
continue
|
|
119
|
+
for match in rule["_re"].finditer(line):
|
|
120
|
+
group = rule.get("secretGroup")
|
|
121
|
+
secret = match.group(group) if group and match.lastindex and group <= match.lastindex else \
|
|
122
|
+
next((g for g in match.groups() if g), match.group(0)) if match.groups() else match.group(0)
|
|
123
|
+
if not secret:
|
|
124
|
+
continue
|
|
125
|
+
if rule.get("entropy") and shannon(secret) < float(rule["entropy"]):
|
|
126
|
+
continue
|
|
127
|
+
if any(r.search(secret) for r in self.allow_regexes) or any(w in secret.lower() for w in self.stopwords):
|
|
128
|
+
continue
|
|
129
|
+
if not rule.get("name_only") and PLACEHOLDER.match(secret):
|
|
130
|
+
continue
|
|
131
|
+
if self._rule_allowed(rule, secret, match.group(0), line, rel):
|
|
132
|
+
continue
|
|
133
|
+
yield rule, secret
|
|
134
|
+
|
|
135
|
+
|
|
136
|
+
def text_files(root: Path, base: Path, skip_artifacts: bool = False):
|
|
137
|
+
for current, dirs, files in os.walk(root):
|
|
138
|
+
current_path = Path(current)
|
|
139
|
+
rel_dir = current_path.relative_to(base).as_posix() if current_path != base else ""
|
|
140
|
+
dirs[:] = [d for d in dirs if d not in SKIP_DIRS and not (skip_artifacts and _artifact_kind(f"{rel_dir}/{d}".lstrip("/")))]
|
|
141
|
+
for name in files:
|
|
142
|
+
path = current_path / name
|
|
143
|
+
if BINARY_EXT.search(name):
|
|
144
|
+
continue
|
|
145
|
+
try:
|
|
146
|
+
if path.is_symlink() or path.stat().st_size > MAX_FILE:
|
|
147
|
+
continue
|
|
148
|
+
except OSError:
|
|
149
|
+
continue
|
|
150
|
+
yield path
|
|
151
|
+
|
|
152
|
+
|
|
153
|
+
def _artifact_kind(rel: str) -> str | None:
|
|
154
|
+
for prefix, kind in ARTIFACT_DIRS.items():
|
|
155
|
+
if rel == prefix or rel.startswith(prefix + "/"):
|
|
156
|
+
return kind
|
|
157
|
+
return None
|
|
158
|
+
|
|
159
|
+
|
|
160
|
+
def read_text(path: Path) -> str | None:
|
|
161
|
+
try:
|
|
162
|
+
data = path.read_bytes()
|
|
163
|
+
except OSError:
|
|
164
|
+
return None
|
|
165
|
+
if b"\x00" in data[:8192]:
|
|
166
|
+
return None
|
|
167
|
+
return data.decode("utf-8", errors="replace")
|
|
168
|
+
|
|
169
|
+
|
|
170
|
+
class Collector:
|
|
171
|
+
"""Groups hits by (rule, secret hash) so one secret in many places is one finding."""
|
|
172
|
+
|
|
173
|
+
def __init__(self) -> None:
|
|
174
|
+
self.groups: dict[tuple[str, str], dict] = {}
|
|
175
|
+
self._values: dict[str, tuple[str, str]] = {} # in memory for this run only: value -> group key; cleared by forget()
|
|
176
|
+
|
|
177
|
+
def forget(self) -> None:
|
|
178
|
+
self._values.clear()
|
|
179
|
+
|
|
180
|
+
def add(self, rule: dict, secret: str, *, rel: str, line_no: int | None, line: str, where: str, commit: str | None = None) -> None:
|
|
181
|
+
info = core.redact_secret(secret)
|
|
182
|
+
key = (rule["id"], info["value_sha256"])
|
|
183
|
+
group = self.groups.setdefault(key, {"rule": rule, "info": info, "hits": []})
|
|
184
|
+
excerpt = line.replace(secret, info["redacted"]).strip()
|
|
185
|
+
hit = {"path": rel, "line": line_no, "excerpt": core.clean_excerpt(excerpt), "redacted": info["redacted"],
|
|
186
|
+
"value_sha256": info["value_sha256"], "where": where}
|
|
187
|
+
if commit:
|
|
188
|
+
hit["commit"] = commit[:12]
|
|
189
|
+
if not any(h["path"] == rel and h["line"] == line_no and h.get("commit") == hit.get("commit") for h in group["hits"]):
|
|
190
|
+
group["hits"].append(hit)
|
|
191
|
+
if len(secret) >= 12 and not secret.startswith(("gitleaks:", "file:")):
|
|
192
|
+
self._values.setdefault(secret, key)
|
|
193
|
+
del secret
|
|
194
|
+
|
|
195
|
+
def find_in_artifacts(self, project: Path) -> int:
|
|
196
|
+
"""A value found anywhere (code, history) that also sits in served output, even without a keyword next to it."""
|
|
197
|
+
if not self._values:
|
|
198
|
+
return 0
|
|
199
|
+
hits = 0
|
|
200
|
+
for prefix, kind in ARTIFACT_DIRS.items():
|
|
201
|
+
root = project / prefix
|
|
202
|
+
if not root.is_dir():
|
|
203
|
+
continue
|
|
204
|
+
for path in text_files(root, project):
|
|
205
|
+
text = read_text(path)
|
|
206
|
+
if not text:
|
|
207
|
+
continue
|
|
208
|
+
rel = path.relative_to(project).as_posix()
|
|
209
|
+
for value, key in self._values.items():
|
|
210
|
+
if value in text:
|
|
211
|
+
group = self.groups[key]
|
|
212
|
+
number = text[: text.index(value)].count("\n") + 1
|
|
213
|
+
if any(h["path"] == rel for h in group["hits"]):
|
|
214
|
+
continue
|
|
215
|
+
line = text.splitlines()[number - 1][:400]
|
|
216
|
+
self.add(group["rule"], value, rel=rel, line_no=number, line=line, where=f"artifact-{kind}")
|
|
217
|
+
hits += 1
|
|
218
|
+
return hits
|
|
219
|
+
|
|
220
|
+
|
|
221
|
+
def scan_tree(project: Path, rules: Rules, collector: Collector, include_artifacts: bool = True) -> int:
|
|
222
|
+
count = 0
|
|
223
|
+
for path in text_files(project, project):
|
|
224
|
+
rel = path.relative_to(project).as_posix()
|
|
225
|
+
if rules.path_allowed(rel):
|
|
226
|
+
continue
|
|
227
|
+
kind = _artifact_kind(rel)
|
|
228
|
+
if kind and not include_artifacts:
|
|
229
|
+
continue
|
|
230
|
+
for rule in rules.path_rules:
|
|
231
|
+
if rule["_path"].search(rel):
|
|
232
|
+
collector.add(rule, f"file:{rel}", rel=rel, line_no=None, line=f"file {rel}", where=f"artifact-{kind}" if kind else "tree")
|
|
233
|
+
text = read_text(path)
|
|
234
|
+
if text is None:
|
|
235
|
+
continue
|
|
236
|
+
count += 1
|
|
237
|
+
for number, line in enumerate(text.splitlines(), 1):
|
|
238
|
+
for rule, secret in rules.scan_line(line, rel):
|
|
239
|
+
collector.add(rule, secret, rel=rel, line_no=number, line=line, where=f"artifact-{kind}" if kind else "tree")
|
|
240
|
+
return count
|
|
241
|
+
|
|
242
|
+
|
|
243
|
+
def scan_git(project: Path, rules: Rules, collector: Collector, since: str | None) -> tuple[bool, int]:
|
|
244
|
+
if not (project / ".git").exists():
|
|
245
|
+
return False, 0
|
|
246
|
+
argv = ["git", "-C", str(project), "log", "-p", "--all", "--no-color", "-U0", "--no-ext-diff", "--format=commit %H"]
|
|
247
|
+
if since:
|
|
248
|
+
argv.insert(4, f"--since={since}")
|
|
249
|
+
try:
|
|
250
|
+
proc = subprocess.Popen(argv, stdout=subprocess.PIPE, stderr=subprocess.DEVNULL, text=True, errors="replace")
|
|
251
|
+
except OSError:
|
|
252
|
+
return False, 0
|
|
253
|
+
commit = None
|
|
254
|
+
rel = None
|
|
255
|
+
line_no = 0
|
|
256
|
+
commits = 0
|
|
257
|
+
assert proc.stdout is not None
|
|
258
|
+
with proc.stdout:
|
|
259
|
+
commit, rel, line_no, commits = _walk_git_log(proc.stdout, rules, collector)
|
|
260
|
+
proc.wait()
|
|
261
|
+
return proc.returncode == 0, commits
|
|
262
|
+
|
|
263
|
+
|
|
264
|
+
def _walk_git_log(stream, rules: Rules, collector: Collector) -> tuple[str | None, str | None, int, int]:
|
|
265
|
+
commit = None
|
|
266
|
+
rel = None
|
|
267
|
+
line_no = 0
|
|
268
|
+
commits = 0
|
|
269
|
+
for raw in stream:
|
|
270
|
+
line = raw.rstrip("\n")
|
|
271
|
+
if line.startswith("commit ") and len(line) == 47:
|
|
272
|
+
commit = line[7:]
|
|
273
|
+
commits += 1
|
|
274
|
+
elif line.startswith("+++ "):
|
|
275
|
+
rel = None if line.endswith("/dev/null") else line[6:] if line.startswith("+++ b/") else line[4:]
|
|
276
|
+
elif line.startswith("@@"):
|
|
277
|
+
match = re.search(r"\+(\d+)", line)
|
|
278
|
+
line_no = int(match.group(1)) if match else 0
|
|
279
|
+
elif line.startswith("+") and rel and not line.startswith("+++"):
|
|
280
|
+
if not rules.path_allowed(rel):
|
|
281
|
+
for rule, secret in rules.scan_line(line[1:], rel):
|
|
282
|
+
collector.add(rule, secret, rel=rel, line_no=line_no, line=line[1:], where="history", commit=commit)
|
|
283
|
+
line_no += 1
|
|
284
|
+
return commit, rel, line_no, commits
|
|
285
|
+
|
|
286
|
+
|
|
287
|
+
def env_leak_check(project: Path, collector: Collector) -> int:
|
|
288
|
+
"""PRD 03 section 6.3: values from .env files that appear verbatim in served artifacts."""
|
|
289
|
+
values: dict[str, list[str]] = {}
|
|
290
|
+
for path in sorted(project.iterdir()) if project.is_dir() else []:
|
|
291
|
+
if not path.is_file() or not ENV_FILE.match(path.name) or ENV_SKIP.search(path.name):
|
|
292
|
+
continue
|
|
293
|
+
for line in (read_text(path) or "").splitlines():
|
|
294
|
+
match = re.match(r"^\s*(?:export\s+)?([A-Za-z_][A-Za-z0-9_]*)\s*=\s*(.*)$", line)
|
|
295
|
+
if not match:
|
|
296
|
+
continue
|
|
297
|
+
value = match.group(2).strip().strip('"').strip("'")
|
|
298
|
+
if len(value) < 8 or PLACEHOLDER.match(value) or re.fullmatch(r"(https?://)?(localhost|127\.0\.0\.1)[:/\w.-]*", value) \
|
|
299
|
+
or re.fullmatch(r"\d+", value) or value.lower() in ("production", "development", "staging"):
|
|
300
|
+
continue
|
|
301
|
+
values.setdefault(value, []).append(match.group(1))
|
|
302
|
+
if not values:
|
|
303
|
+
return 0
|
|
304
|
+
rule = {"id": "maggie-env-value-in-artifact", "description": "A value from a .env file is in served build output"}
|
|
305
|
+
hits = 0
|
|
306
|
+
for prefix, kind in ARTIFACT_DIRS.items():
|
|
307
|
+
root = project / prefix
|
|
308
|
+
if not root.is_dir():
|
|
309
|
+
continue
|
|
310
|
+
for path in text_files(root, project):
|
|
311
|
+
text = read_text(path)
|
|
312
|
+
if not text:
|
|
313
|
+
continue
|
|
314
|
+
rel = path.relative_to(project).as_posix()
|
|
315
|
+
for value, names in values.items():
|
|
316
|
+
if value in text:
|
|
317
|
+
number = text[: text.index(value)].count("\n") + 1
|
|
318
|
+
line = text.splitlines()[number - 1] if text.splitlines() else ""
|
|
319
|
+
collector.add({**rule, "description": f"{rule['description']} ({', '.join(sorted(set(names)))})"}, value,
|
|
320
|
+
rel=rel, line_no=number, line=line[:400], where=f"artifact-{kind}")
|
|
321
|
+
hits += 1
|
|
322
|
+
values.clear()
|
|
323
|
+
return hits
|
|
324
|
+
|
|
325
|
+
|
|
326
|
+
# --------------------------------------------------------------------------- gitleaks binary
|
|
327
|
+
|
|
328
|
+
|
|
329
|
+
def run_gitleaks(ctx: core.Context, project: Path, mode: str, target: Path, since: str | None) -> list[dict] | None:
|
|
330
|
+
with tempfile.TemporaryDirectory() as tmp:
|
|
331
|
+
report = Path(tmp) / "report.json"
|
|
332
|
+
args = [mode, "--report-format", "json", "--report-path", str(report), "--redact=100", "--exit-code", "0"]
|
|
333
|
+
if mode == "git" and since:
|
|
334
|
+
args += ["--log-opts", f"--all --since={since}"]
|
|
335
|
+
if mode == "dir":
|
|
336
|
+
args += ["--max-target-megabytes", "5"]
|
|
337
|
+
args.append(str(target))
|
|
338
|
+
proc = core.run_tool("gitleaks", args, ctx.lock, cwd=project)
|
|
339
|
+
if not report.is_file():
|
|
340
|
+
ctx.degrade(f"gitleaks-{mode}", core.clean_excerpt(proc.stderr.decode(errors="replace"), 200))
|
|
341
|
+
return None
|
|
342
|
+
return json.loads(report.read_text(encoding="utf-8") or "[]")
|
|
343
|
+
|
|
344
|
+
|
|
345
|
+
def gitleaks_to_collector(items: list[dict], project: Path, target: Path, where_default: str, collector: Collector) -> None:
|
|
346
|
+
for item in items:
|
|
347
|
+
rel_file = item.get("File") or ""
|
|
348
|
+
path = Path(rel_file)
|
|
349
|
+
rel = (path if path.is_absolute() else (target / path)).resolve()
|
|
350
|
+
try:
|
|
351
|
+
rel_text = rel.relative_to(project.resolve()).as_posix()
|
|
352
|
+
except ValueError:
|
|
353
|
+
rel_text = rel_file
|
|
354
|
+
kind = _artifact_kind(rel_text)
|
|
355
|
+
where = f"artifact-{kind}" if kind else where_default
|
|
356
|
+
# --redact=100 means Secret is "REDACTED"; the fingerprint stands in for the value hash.
|
|
357
|
+
# --redact=100 means Secret is "REDACTED": rule + file + line stands in for the value hash, so the
|
|
358
|
+
# same line in history and in the working tree is one finding.
|
|
359
|
+
stand_in = f"{rel_text}:{item.get('RuleID')}:{item.get('StartLine')}"
|
|
360
|
+
rule = {"id": item.get("RuleID") or "gitleaks", "description": item.get("Description") or item.get("RuleID") or "secret"}
|
|
361
|
+
collector.add(rule, "gitleaks:" + stand_in, rel=rel_text, line_no=item.get("StartLine"),
|
|
362
|
+
line=f"{item.get('RuleID')} match (value redacted by gitleaks)", where=where, commit=item.get("Commit") or None)
|
|
363
|
+
|
|
364
|
+
|
|
365
|
+
# --------------------------------------------------------------------------- findings
|
|
366
|
+
|
|
367
|
+
|
|
368
|
+
def severity_for(hits: list[dict], rule: dict) -> str:
|
|
369
|
+
if rule.get("name_only"):
|
|
370
|
+
return "medium"
|
|
371
|
+
if any(h["where"] == "artifact-public" for h in hits):
|
|
372
|
+
return "critical"
|
|
373
|
+
return "high"
|
|
374
|
+
|
|
375
|
+
|
|
376
|
+
def to_findings(ctx: core.Context, collector: Collector) -> list[dict]:
|
|
377
|
+
findings = []
|
|
378
|
+
for (rule_id, value_hash), group in collector.groups.items():
|
|
379
|
+
hits = sorted(group["hits"], key=lambda h: ({"artifact-public": 0, "artifact-server": 1, "tree": 2, "history": 3}[h["where"]], h["path"]))
|
|
380
|
+
rule = group["rule"]
|
|
381
|
+
lead = hits[0]
|
|
382
|
+
public = lead["where"] == "artifact-public"
|
|
383
|
+
history_only = all(h["where"] == "history" for h in hits)
|
|
384
|
+
where_text = {"artifact-public": "in a file your site serves publicly", "artifact-server": "in a deployed server bundle",
|
|
385
|
+
"tree": "in the project files", "history": "in git history"}[lead["where"]]
|
|
386
|
+
name_only = rule.get("name_only")
|
|
387
|
+
title = f"{rule.get('description') or rule_id} {where_text}: {lead['path']}" + (f":{lead['line']}" if lead.get("line") else "")
|
|
388
|
+
steps = [] if name_only else ["Revoke or rotate this key with its provider first (assume it is already copied).",
|
|
389
|
+
"Remove it from the code and build output; keep it in the platform's secret store "
|
|
390
|
+
"(for Cloudflare Workers: maggie deployment worker-secrets).",
|
|
391
|
+
"Rebuild and redeploy."]
|
|
392
|
+
if history_only and not name_only:
|
|
393
|
+
steps.append("It is only in git history now: rotating is enough; purging history (git filter-repo) is optional.")
|
|
394
|
+
if name_only:
|
|
395
|
+
steps = ["Rename the variable without the public prefix and read it only on the server; anything with a public prefix is sent to browsers."]
|
|
396
|
+
finding = core.make_finding(
|
|
397
|
+
kind="secret", command="secrets", primary_id=rule_id, site=ctx.site, title=title[:300],
|
|
398
|
+
severity=severity_for(hits, rule), severity_source="maggie", confidence="likely" if name_only else ("confirmed" if public else "likely"),
|
|
399
|
+
asvs_ids=["V13.3.1"], cwe_ids=["CWE-798"],
|
|
400
|
+
target={"path": lead["path"]},
|
|
401
|
+
evidence=[{k: v for k, v in h.items() if k in ("path", "line", "excerpt", "redacted", "value_sha256", "commit")} for h in hits[:20]],
|
|
402
|
+
rule_id=rule_id, tool=ctx.run.get("_engine"), feed=None,
|
|
403
|
+
remediation={"summary": "Rotate the key, then remove it." if not name_only else "Keep secret values out of public build variables.",
|
|
404
|
+
"steps": steps, "snippets": [], "references": ["https://cheatsheetseries.owasp.org/cheatsheets/Secrets_Management_Cheat_Sheet.html"],
|
|
405
|
+
"cheat_sheet": "Secrets Management"})
|
|
406
|
+
# One secret in many files is one finding: the id follows the value hash, not the path.
|
|
407
|
+
finding["id"] = core.fingerprint("secret", rule_id, f"secret:{value_hash[:24]}", ctx.site)
|
|
408
|
+
if public and not name_only:
|
|
409
|
+
finding["priority"] = "p0"
|
|
410
|
+
findings.append(finding)
|
|
411
|
+
return findings
|
|
412
|
+
|
|
413
|
+
|
|
414
|
+
def apply_baseline(ctx: core.Context, findings: list[dict], command: str = "secrets") -> None:
|
|
415
|
+
args = ctx.args
|
|
416
|
+
path = Path(args.baseline) if args.baseline else ctx.state.path("baselines", f"{command}.json")
|
|
417
|
+
if args.write_baseline:
|
|
418
|
+
public = [f for f in findings if f["severity"] == "critical" and f["kind"] == "secret"]
|
|
419
|
+
if public:
|
|
420
|
+
raise core.PolicyError("a secret in served build output cannot be baselined: rotate it and remove it first")
|
|
421
|
+
target = Path(args.write_baseline)
|
|
422
|
+
core.write_json(target, {"schema": "maggie-security-baseline.v1", "command": command, "created_at": core.iso(),
|
|
423
|
+
"rules_version": rule_pack_version() if command == "code-scan" else core.read_json(core.DATA_DIR / "gitleaks-rules.json").get("version"),
|
|
424
|
+
"reason": "baseline", "ids": sorted(f["id"] for f in findings)})
|
|
425
|
+
ctx.run["inputs"]["baseline_written"] = str(target)
|
|
426
|
+
return
|
|
427
|
+
if not path.is_file():
|
|
428
|
+
return
|
|
429
|
+
ids = set(core.read_json(path).get("ids") or [])
|
|
430
|
+
ledger = core.Ledger(ctx.state.path("ledger.jsonl"))
|
|
431
|
+
history = ledger.history()
|
|
432
|
+
for finding in findings:
|
|
433
|
+
if finding["id"] in ids and not (finding["kind"] == "secret" and finding["severity"] == "critical"):
|
|
434
|
+
state = core.Ledger.derive(history.get(finding["id"], [])) if finding["id"] in history else {"status": None}
|
|
435
|
+
if state["status"] != "accepted-risk":
|
|
436
|
+
ledger.append([{"id": finding["id"], "at": core.iso(), "event": "seen", "run": ctx.run["id"], "command": command,
|
|
437
|
+
"severity": finding["severity"]},
|
|
438
|
+
{"id": finding["id"], "at": core.iso(), "event": "accepted-risk", "by": "baseline", "reason": "baseline"}])
|
|
439
|
+
ctx.run["inputs"]["baseline"] = str(path)
|
|
440
|
+
|
|
441
|
+
|
|
442
|
+
def secrets(ctx: core.Context):
|
|
443
|
+
args = ctx.args
|
|
444
|
+
project = ctx.project
|
|
445
|
+
scopes = set((args.scope or "git,tree,dist").split(","))
|
|
446
|
+
config = ctx.state.config()
|
|
447
|
+
collector = Collector()
|
|
448
|
+
engine = args.engine
|
|
449
|
+
if engine == "auto":
|
|
450
|
+
engine = "gitleaks" if core.resolve_tool("gitleaks", ctx.lock)[2] == "pinned" else "native"
|
|
451
|
+
ctx.run["_engine"] = "gitleaks" if engine == "gitleaks" else "maggie-native"
|
|
452
|
+
ctx.run["tools"].append({"name": "gitleaks" if engine == "gitleaks" else "maggie-secrets-native",
|
|
453
|
+
"version": ctx.lock["tools"]["gitleaks"]["version"] if engine == "gitleaks" else core.read_json(core.DATA_DIR / "gitleaks-rules.json").get("version"),
|
|
454
|
+
"sha256": None})
|
|
455
|
+
rules = Rules((config.get("secrets") or {}).get("allow_paths"))
|
|
456
|
+
covered_ok = True
|
|
457
|
+
if engine == "gitleaks":
|
|
458
|
+
if "git" in scopes and (project / ".git").exists():
|
|
459
|
+
items = run_gitleaks(ctx, project, "git", project, args.since)
|
|
460
|
+
covered_ok &= items is not None
|
|
461
|
+
gitleaks_to_collector(items or [], project, project, "history", collector)
|
|
462
|
+
if "tree" in scopes or "dist" in scopes:
|
|
463
|
+
items = run_gitleaks(ctx, project, "dir", project, None)
|
|
464
|
+
covered_ok &= items is not None
|
|
465
|
+
gitleaks_to_collector(items or [], project, project, "tree", collector)
|
|
466
|
+
files = None
|
|
467
|
+
else:
|
|
468
|
+
files = scan_tree(project, rules, collector, include_artifacts="dist" in scopes) if ("tree" in scopes or "dist" in scopes) else 0
|
|
469
|
+
if "git" in scopes:
|
|
470
|
+
ok, commits = scan_git(project, rules, collector, args.since)
|
|
471
|
+
ctx.run["inputs"]["git_commits"] = commits
|
|
472
|
+
if (project / ".git").exists() and not ok:
|
|
473
|
+
ctx.degrade("git-history", "git log failed")
|
|
474
|
+
covered_ok = False
|
|
475
|
+
leaks = env_leak_check(project, collector) if "dist" in scopes else 0
|
|
476
|
+
copied = collector.find_in_artifacts(project) if "dist" in scopes else 0
|
|
477
|
+
collector.forget()
|
|
478
|
+
findings = to_findings(ctx, collector)
|
|
479
|
+
apply_baseline(ctx, findings, "secrets")
|
|
480
|
+
ctx.run["inputs"].update({"engine": ctx.run.pop("_engine"), "scopes": sorted(scopes), "files_scanned": files, "env_value_hits": leaks,
|
|
481
|
+
"copied_into_artifacts": copied})
|
|
482
|
+
return ctx.finish(findings, covered=["*"] if covered_ok and scopes >= {"tree", "dist"} else [])
|
|
483
|
+
|
|
484
|
+
|
|
485
|
+
# --------------------------------------------------------------------------- code-scan (semgrep + Maggie rule pack)
|
|
486
|
+
|
|
487
|
+
|
|
488
|
+
def rule_pack_dir() -> Path:
|
|
489
|
+
for base in (core.ROOT / "skills", core.ROOT / "bundled-skills"):
|
|
490
|
+
path = base / "maggie-security" / "data" / "semgrep"
|
|
491
|
+
if path.is_dir():
|
|
492
|
+
return path
|
|
493
|
+
raise core.UsageError("Maggie semgrep rule pack not found")
|
|
494
|
+
|
|
495
|
+
|
|
496
|
+
def rule_pack_version() -> str:
|
|
497
|
+
return (rule_pack_dir() / "VERSION").read_text(encoding="utf-8").strip()
|
|
498
|
+
|
|
499
|
+
|
|
500
|
+
SEMGREP_EXCLUDES = ["node_modules", "vendor", "dist", "build", ".next", ".output", ".maggie", ".git", "wp-content/plugins", "wp-content/themes",
|
|
501
|
+
"wp-admin", "wp-includes", "coverage", "__pycache__", "*.min.js"] # names match anywhere in the path: keep them specific
|
|
502
|
+
SEMGREP_SEVERITY = {"ERROR": "high", "WARNING": "medium", "INFO": "low"}
|
|
503
|
+
|
|
504
|
+
|
|
505
|
+
def semgrep_to_findings(ctx: core.Context, data: dict, project: Path) -> list[dict]:
|
|
506
|
+
"""semgrep --json -> finding-v1 (PRD 03 section 9.1). One finding per rule and file; each hit is evidence."""
|
|
507
|
+
findings = []
|
|
508
|
+
for result in data.get("results") or []:
|
|
509
|
+
extra = result.get("extra") or {}
|
|
510
|
+
meta = extra.get("metadata") or {}
|
|
511
|
+
check = str(result.get("check_id") or "")
|
|
512
|
+
rule_id = check[check.index("maggie."):] if "maggie." in check else check.rsplit(".", 1)[-1]
|
|
513
|
+
path = str(result.get("path") or "")
|
|
514
|
+
try:
|
|
515
|
+
rel = Path(path).resolve().relative_to(project).as_posix()
|
|
516
|
+
except ValueError:
|
|
517
|
+
rel = path
|
|
518
|
+
line = (result.get("start") or {}).get("line")
|
|
519
|
+
severity = meta.get("severity") or SEMGREP_SEVERITY.get(str(extra.get("severity")).upper(), "medium")
|
|
520
|
+
confidence = {"high": "likely", "medium": "possible", "low": "possible"}.get(str(meta.get("confidence") or "medium").lower(), "possible")
|
|
521
|
+
message = str(extra.get("message") or rule_id)
|
|
522
|
+
cwes = meta.get("cwe") or []
|
|
523
|
+
cwes = [cwes] if isinstance(cwes, str) else cwes
|
|
524
|
+
asvs = meta.get("asvs") or []
|
|
525
|
+
asvs = [asvs] if isinstance(asvs, str) else asvs
|
|
526
|
+
findings.append(core.make_finding(
|
|
527
|
+
kind="code", command="code-scan", primary_id=rule_id, site=ctx.site, title=f"{message.split('. ')[0].rstrip('.')} ({rel})"[:300],
|
|
528
|
+
severity=severity, severity_source="tool", confidence=confidence,
|
|
529
|
+
cwe_ids=[c.split(":")[0].strip() for c in cwes if re.match(r"^CWE-\d+", str(c))],
|
|
530
|
+
asvs_ids=[a for a in asvs if re.match(r"^V\d+\.\d+\.\d+$", str(a))], target={"path": rel},
|
|
531
|
+
evidence=[{"path": rel, "line": line, "excerpt": core.clean_excerpt(str(extra.get("lines") or ""), 200)}],
|
|
532
|
+
tool="semgrep", rule_id=rule_id,
|
|
533
|
+
remediation={"summary": message, "steps": [], "snippets": [{"kind": "patch", "text": extra["fix"]}] if extra.get("fix") else [],
|
|
534
|
+
"references": [meta["fix_doc"]] if meta.get("fix_doc") else [], "cheat_sheet": None}))
|
|
535
|
+
return findings
|
|
536
|
+
|
|
537
|
+
|
|
538
|
+
def native_config_checks(ctx: core.Context, files: list[Path], project: Path) -> list[dict]:
|
|
539
|
+
"""Two checks semgrep expresses poorly: Dockerfile final stage without USER, nginx serving dotfiles."""
|
|
540
|
+
findings = []
|
|
541
|
+
for path in files:
|
|
542
|
+
rel = path.relative_to(project).as_posix()
|
|
543
|
+
name = path.name.lower()
|
|
544
|
+
text = read_text(path) or ""
|
|
545
|
+
if name == "dockerfile" or name.startswith("dockerfile.") or name.endswith(".dockerfile"):
|
|
546
|
+
stages = re.split(r"(?im)^\s*FROM\s+", text)
|
|
547
|
+
final = stages[-1] if len(stages) > 1 else ""
|
|
548
|
+
if final and not re.search(r"(?im)^\s*USER\s+(?!root\b|0\b)\S+", final) and not re.match(r"(?i)scratch\b", final.strip()):
|
|
549
|
+
findings.append(core.make_finding(
|
|
550
|
+
kind="code", command="code-scan", primary_id="maggie.docker.root-user", site=ctx.site,
|
|
551
|
+
title=f"The container runs as root (no USER in the final stage of {rel})", severity="low", severity_source="maggie",
|
|
552
|
+
confidence="likely", cwe_ids=["CWE-250"], target={"path": rel}, tool="maggie-native", rule_id="maggie.docker.root-user",
|
|
553
|
+
evidence=[{"path": rel, "excerpt": "FROM " + final.splitlines()[0][:150] if final else ""}],
|
|
554
|
+
remediation={"summary": "Add a non-root USER (for example `USER node` or `USER www-data`) in the final stage.", "steps": [],
|
|
555
|
+
"snippets": [], "references": ["https://docs.docker.com/build/building/best-practices/#user"], "cheat_sheet": None}))
|
|
556
|
+
if name.endswith(".conf") or name.startswith("nginx.conf"):
|
|
557
|
+
for block in re.findall(r"(?s)server\s*\{(.*?)\n\}", text):
|
|
558
|
+
if re.search(r"(?m)^\s*root\s+\S+;", block) and not re.search(r"location\s+~\*?\s*/\\\.", block):
|
|
559
|
+
findings.append(core.make_finding(
|
|
560
|
+
kind="code", command="code-scan", primary_id="maggie.nginx.dotfiles-served", site=ctx.site,
|
|
561
|
+
title=f"nginx may serve hidden files like .git or .env ({rel})", severity="medium", severity_source="maggie",
|
|
562
|
+
confidence="likely", cwe_ids=["CWE-538"], asvs_ids=["V13.4.1"], target={"path": rel}, tool="maggie-native",
|
|
563
|
+
rule_id="maggie.nginx.dotfiles-served",
|
|
564
|
+
remediation={"summary": "Deny hidden files in the server block.", "steps": [],
|
|
565
|
+
"snippets": [{"kind": "nginx", "text": "location ~ /\\.(?!well-known) { deny all; }"}],
|
|
566
|
+
"references": [], "cheat_sheet": None}))
|
|
567
|
+
break
|
|
568
|
+
return findings
|
|
569
|
+
|
|
570
|
+
|
|
571
|
+
def changed_files(project: Path, ref: str) -> list[str]:
|
|
572
|
+
proc = subprocess.run(["git", "-C", str(project), "diff", "--name-only", "--diff-filter=ACMR", ref], capture_output=True, text=True, check=False)
|
|
573
|
+
if proc.returncode != 0:
|
|
574
|
+
raise core.UsageError(f"git diff against {ref} failed: {proc.stderr.strip()[:200]}")
|
|
575
|
+
untracked = subprocess.run(["git", "-C", str(project), "ls-files", "--others", "--exclude-standard"], capture_output=True, text=True, check=False)
|
|
576
|
+
return sorted(set(proc.stdout.split()) | set(untracked.stdout.split()))
|
|
577
|
+
|
|
578
|
+
|
|
579
|
+
def code_scan(ctx: core.Context):
|
|
580
|
+
args = ctx.args
|
|
581
|
+
project = ctx.project
|
|
582
|
+
configs = []
|
|
583
|
+
for rule in args.rules or ["maggie"]:
|
|
584
|
+
if rule == "maggie":
|
|
585
|
+
configs.append(str(rule_pack_dir() / "rules"))
|
|
586
|
+
elif "://" in rule or rule in ("auto", "p", "r") or rule.startswith(("p/", "r/")):
|
|
587
|
+
raise core.PolicyError(f"--rules {rule}: only the Maggie pack or local rule folders are allowed (no registry downloads)")
|
|
588
|
+
else:
|
|
589
|
+
local = Path(rule) if Path(rule).is_absolute() else project / rule
|
|
590
|
+
if not local.exists():
|
|
591
|
+
raise core.UsageError(f"rule folder not found: {rule}")
|
|
592
|
+
configs.append(str(local))
|
|
593
|
+
if args.changed_since:
|
|
594
|
+
targets = [str(project / f) for f in changed_files(project, args.changed_since) if (project / f).is_file()]
|
|
595
|
+
if not targets:
|
|
596
|
+
ctx.run["inputs"]["note"] = f"no files changed since {args.changed_since}"
|
|
597
|
+
return ctx.finish([], covered=[])
|
|
598
|
+
else:
|
|
599
|
+
targets = [str(project / p) for p in (args.paths or ["."])]
|
|
600
|
+
files = [Path(t) for t in targets if Path(t).is_file()] + [f for t in targets if Path(t).is_dir() for f in text_files(Path(t), project)]
|
|
601
|
+
findings = native_config_checks(ctx, [f for f in files if not any(part in SKIP_DIRS for part in f.relative_to(project).parts)], project)
|
|
602
|
+
with tempfile.TemporaryDirectory() as tmp:
|
|
603
|
+
out = Path(tmp) / "semgrep.json"
|
|
604
|
+
argv = ["scan", "--json", "--json-output", str(out), "--timeout", "30", "--max-target-bytes", "1000000", "--quiet"]
|
|
605
|
+
for config in configs:
|
|
606
|
+
argv += ["--config", config]
|
|
607
|
+
for exclude in SEMGREP_EXCLUDES:
|
|
608
|
+
argv += ["--exclude", exclude]
|
|
609
|
+
if args.baseline_commit:
|
|
610
|
+
argv += ["--baseline-commit", args.baseline_commit]
|
|
611
|
+
argv += targets
|
|
612
|
+
proc = core.run_tool("semgrep", argv, ctx.lock, cwd=project, timeout=1800, env={"SEMGREP_SEND_METRICS": "off", "SEMGREP_ENABLE_VERSION_CHECK": "0"})
|
|
613
|
+
if not out.is_file():
|
|
614
|
+
raise core.DegradedError(f"semgrep produced no report: {core.clean_excerpt(proc.stderr.decode(errors='replace'), 200)}")
|
|
615
|
+
data = json.loads(out.read_text(encoding="utf-8") or "{}")
|
|
616
|
+
findings += semgrep_to_findings(ctx, data, project)
|
|
617
|
+
errors = [e for e in data.get("errors") or [] if str(e.get("level", "")).lower() == "error"]
|
|
618
|
+
if errors:
|
|
619
|
+
ctx.degrade("semgrep", f"{len(errors)} file(s) could not be parsed")
|
|
620
|
+
version = (data.get("version") or ctx.lock["tools"]["semgrep"]["version"])
|
|
621
|
+
ctx.run["tools"].append({"name": "semgrep", "version": version, "sha256": None})
|
|
622
|
+
ctx.run["inputs"].update({"rule_pack": rule_pack_version(), "configs": [Path(c).name for c in configs], "files": len(files),
|
|
623
|
+
"changed_since": args.changed_since})
|
|
624
|
+
apply_baseline(ctx, findings, "code-scan")
|
|
625
|
+
return ctx.finish(findings, covered=["*"] if not args.changed_since and not args.paths and not errors else [])
|
|
626
|
+
|
|
627
|
+
|
|
628
|
+
def _code(p: argparse.ArgumentParser) -> None:
|
|
629
|
+
p.add_argument("--rules", action="append", help="'maggie' (default) or a local rule folder; registry configs are refused")
|
|
630
|
+
p.add_argument("--paths", nargs="*", help="limit to these paths")
|
|
631
|
+
p.add_argument("--changed-since", help="scan only files changed since a git ref (pre-commit use)")
|
|
632
|
+
p.add_argument("--baseline-commit", help="semgrep's own diff-aware baseline")
|
|
633
|
+
p.add_argument("--baseline", help="baseline file (default .maggie/security/baselines/code-scan.json)")
|
|
634
|
+
p.add_argument("--write-baseline", help="record all current findings as accepted")
|
|
635
|
+
|
|
636
|
+
|
|
637
|
+
def _secrets(p: argparse.ArgumentParser) -> None:
|
|
638
|
+
p.add_argument("--scope", help="comma list of git,tree,dist (default all three)")
|
|
639
|
+
p.add_argument("--engine", choices=["auto", "gitleaks", "native"], default="auto",
|
|
640
|
+
help="auto: the pinned gitleaks if installed, else Maggie's native engine with gitleaks' rules")
|
|
641
|
+
p.add_argument("--baseline", help="baseline file (default .maggie/security/baselines/secrets.json)")
|
|
642
|
+
p.add_argument("--write-baseline", help="record all current findings as accepted (not allowed for public build output)")
|
|
643
|
+
|
|
644
|
+
|
|
645
|
+
COMMANDS = [
|
|
646
|
+
core.Command("code-scan", "find unsafe code patterns (XSS, SSRF, SQL injection, open redirects, insecure cookies…) with semgrep and Maggie's rules",
|
|
647
|
+
[], _code, code_scan),
|
|
648
|
+
core.Command("secrets", "find API keys and passwords in git history, project files and served build output (values never shown)",
|
|
649
|
+
[], _secrets, secrets),
|
|
650
|
+
]
|