ikanban 0.4.9 → 0.5.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +59 -10
- package/bin/ikanban.js +49 -6
- package/hooks/deploy-check.js +221 -0
- package/hooks/ikanban-safe.js +26 -3
- package/hooks/plan-gate.js +206 -0
- package/hooks/roadmap-sync.js +5 -2
- package/hooks/trace.js +384 -0
- package/lib/ikanban-safe.js +26 -3
- package/lib/plancheck.js +231 -0
- package/lib/roadmap.js +135 -28
- package/lib/scanstore.js +403 -0
- package/lib/statuswords.js +13 -0
- package/package.json +3 -3
- package/scanners/ikanban-aiscan.py +121 -30
- package/scanners/ikanban-cloudscan.py +110 -41
- package/scanners/ikanban-controlscan.py +783 -0
- package/scanners/ikanban-dastscan.py +108 -40
- package/scanners/ikanban-import.py +597 -0
- package/scanners/ikanban-infrascan.py +116 -51
- package/scanners/ikanban-secscan.py +116 -104
|
@@ -0,0 +1,597 @@
|
|
|
1
|
+
#!/usr/bin/env python3
|
|
2
|
+
"""
|
|
3
|
+
ikanban-import — turns another scanner's SARIF 2.1.0 file into iKanban code findings (plan 5 X1-T4).
|
|
4
|
+
|
|
5
|
+
python3 ikanban-import.py codeql.sarif # read it, print what would be filed
|
|
6
|
+
python3 ikanban-import.py codeql.sarif --publish --team TEAM_ID # file the findings into iKanban
|
|
7
|
+
|
|
8
|
+
A result whose rule (or the result itself) carries a CWE tag ("external/cwe/cwe-089",
|
|
9
|
+
"CWE-79: …", a CWE taxon) becomes a code finding under the OWASP Top 10 2021 category that lists
|
|
10
|
+
that CWE. A result with no CWE, or a CWE that no OWASP 2021 category lists, stays in the local
|
|
11
|
+
report as "not filed: no OWASP category". Suppressed and passing results are not filed either.
|
|
12
|
+
Every text field passes through scrub_text before it is sent; the SARIF file itself never leaves
|
|
13
|
+
this machine. Standard library only.
|
|
14
|
+
"""
|
|
15
|
+
|
|
16
|
+
from __future__ import annotations
|
|
17
|
+
|
|
18
|
+
import argparse
|
|
19
|
+
import hashlib
|
|
20
|
+
import json
|
|
21
|
+
import os
|
|
22
|
+
import re
|
|
23
|
+
import subprocess
|
|
24
|
+
import sys
|
|
25
|
+
import urllib.error
|
|
26
|
+
import urllib.parse
|
|
27
|
+
import urllib.request
|
|
28
|
+
from datetime import datetime, timezone
|
|
29
|
+
from pathlib import Path
|
|
30
|
+
|
|
31
|
+
IMPORT_VERSION = "1.0.0"
|
|
32
|
+
SEVERITIES = ["critical", "high", "medium", "low", "info"]
|
|
33
|
+
SEVERITY_RANK = {s: i for i, s in enumerate(SEVERITIES)}
|
|
34
|
+
MAX_FILED = 500 # the server takes at most this many findings in one publish
|
|
35
|
+
|
|
36
|
+
# OWASP Top 10 2021, "List of Mapped CWEs" of each category, copied from the official pages
|
|
37
|
+
# (https://owasp.org/Top10/A01_2021-Broken_Access_Control/ … A10_2021-Server-Side_Request_Forgery_(SSRF)/;
|
|
38
|
+
# source text: github.com/OWASP/Top10, 2021/docs/en/A0x_2021-*.md). The official lists name CWE-259
|
|
39
|
+
# under both A02 and A07; the first category in this order (A02) wins.
|
|
40
|
+
OWASP_2021_CWES = {
|
|
41
|
+
"A01 — Broken Access Control": (
|
|
42
|
+
22, 23, 35, 59, 200, 201, 219, 264, 275, 276, 284, 285, 352, 359, 377, 402, 425, 441, 497, 538, 540,
|
|
43
|
+
548, 552, 566, 601, 639, 651, 668, 706, 862, 863, 913, 922, 1275),
|
|
44
|
+
"A02 — Cryptographic Failures": (
|
|
45
|
+
259, 261, 296, 310, 319, 321, 322, 323, 324, 325, 326, 327, 328, 329, 330, 331, 335, 336, 337, 338, 340,
|
|
46
|
+
347, 523, 720, 757, 759, 760, 780, 818, 916),
|
|
47
|
+
"A03 — Injection": (
|
|
48
|
+
20, 74, 75, 77, 78, 79, 80, 83, 87, 88, 89, 90, 91, 93, 94, 95, 96, 97, 98, 99, 100, 113, 116, 138, 184,
|
|
49
|
+
470, 471, 564, 610, 643, 644, 652, 917),
|
|
50
|
+
"A04 — Insecure Design": (
|
|
51
|
+
73, 183, 209, 213, 235, 256, 257, 266, 269, 280, 311, 312, 313, 316, 419, 430, 434, 444, 451, 472, 501,
|
|
52
|
+
522, 525, 539, 579, 598, 602, 642, 646, 650, 653, 656, 657, 799, 807, 840, 841, 927, 1021, 1173),
|
|
53
|
+
"A05 — Security Misconfiguration": (
|
|
54
|
+
2, 11, 13, 15, 16, 260, 315, 520, 526, 537, 541, 547, 611, 614, 756, 776, 942, 1004, 1032, 1174),
|
|
55
|
+
"A06 — Vulnerable and Outdated Components": (937, 1035, 1104),
|
|
56
|
+
"A07 — Identification and Authentication Failures": (
|
|
57
|
+
255, 259, 287, 288, 290, 294, 295, 297, 300, 302, 304, 306, 307, 346, 384, 521, 613, 620, 640, 798, 940,
|
|
58
|
+
1216),
|
|
59
|
+
"A08 — Software and Data Integrity Failures": (345, 353, 426, 494, 502, 565, 784, 829, 830, 915),
|
|
60
|
+
"A09 — Security Logging and Monitoring Failures": (117, 223, 532, 778),
|
|
61
|
+
"A10 — Server-Side Request Forgery (SSRF)": (918,),
|
|
62
|
+
}
|
|
63
|
+
_CWE_TO_OWASP = {}
|
|
64
|
+
for _cat, _cwes in OWASP_2021_CWES.items():
|
|
65
|
+
for _cwe in _cwes:
|
|
66
|
+
_CWE_TO_OWASP.setdefault(_cwe, _cat)
|
|
67
|
+
|
|
68
|
+
CWE_TAG = re.compile(r"(?i)\bcwe[-_/ :]*0*(\d{1,4})\b")
|
|
69
|
+
NO_CATEGORY = "not filed: no OWASP category"
|
|
70
|
+
|
|
71
|
+
|
|
72
|
+
def owasp_category(cwe: int) -> str | None:
|
|
73
|
+
return _CWE_TO_OWASP.get(cwe)
|
|
74
|
+
|
|
75
|
+
|
|
76
|
+
def _text(node) -> str:
|
|
77
|
+
return (node or {}).get("text") or (node or {}).get("markdown") or "" if isinstance(node, dict) else ""
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
def _cwes(*nodes) -> list:
|
|
81
|
+
"""CWE numbers from tags, taxa and relationships, in the order they appear."""
|
|
82
|
+
found = []
|
|
83
|
+
|
|
84
|
+
def add(value):
|
|
85
|
+
for m in CWE_TAG.finditer(str(value)):
|
|
86
|
+
n = int(m.group(1))
|
|
87
|
+
if n not in found:
|
|
88
|
+
found.append(n)
|
|
89
|
+
for node in nodes:
|
|
90
|
+
if not isinstance(node, dict):
|
|
91
|
+
continue
|
|
92
|
+
for tag in (node.get("properties") or {}).get("tags") or []:
|
|
93
|
+
add(tag)
|
|
94
|
+
for taxon in node.get("taxa") or []:
|
|
95
|
+
if isinstance(taxon, dict) and "cwe" in str((taxon.get("toolComponent") or {}).get("name", "")).lower():
|
|
96
|
+
add(f"CWE-{taxon.get('id', '')}")
|
|
97
|
+
for rel in node.get("relationships") or []:
|
|
98
|
+
target = (rel or {}).get("target") or {}
|
|
99
|
+
if "cwe" in str((target.get("toolComponent") or {}).get("name", "")).lower():
|
|
100
|
+
add(f"CWE-{target.get('id', '')}")
|
|
101
|
+
else:
|
|
102
|
+
add(target.get("id", ""))
|
|
103
|
+
return found
|
|
104
|
+
|
|
105
|
+
|
|
106
|
+
def _severity(rule: dict, res: dict) -> str:
|
|
107
|
+
for node in (res, rule):
|
|
108
|
+
raw = (node.get("properties") or {}).get("security-severity")
|
|
109
|
+
try:
|
|
110
|
+
score = float(raw)
|
|
111
|
+
except (TypeError, ValueError):
|
|
112
|
+
continue
|
|
113
|
+
return "Critical" if score >= 9 else "High" if score >= 7 else "Medium" if score >= 4 else "Low"
|
|
114
|
+
level = res.get("level") or (rule.get("defaultConfiguration") or {}).get("level") or "warning"
|
|
115
|
+
return {"error": "High", "warning": "Medium"}.get(level, "Low")
|
|
116
|
+
|
|
117
|
+
|
|
118
|
+
def _location(res: dict) -> tuple:
|
|
119
|
+
for loc in res.get("locations") or []:
|
|
120
|
+
phys = (loc or {}).get("physicalLocation") or {}
|
|
121
|
+
uri = str((phys.get("artifactLocation") or {}).get("uri") or "")
|
|
122
|
+
if uri:
|
|
123
|
+
uri = re.sub(r"^file://", "", urllib.parse.unquote(uri))
|
|
124
|
+
line = (phys.get("region") or {}).get("startLine")
|
|
125
|
+
return uri, str(line) if isinstance(line, int) else None
|
|
126
|
+
return "", None
|
|
127
|
+
|
|
128
|
+
|
|
129
|
+
def sarif_findings(doc) -> tuple:
|
|
130
|
+
"""(filed payloads, not filed [{rule, location, why}]); ValueError when doc is not SARIF 2.1.0."""
|
|
131
|
+
if not isinstance(doc, dict) or doc.get("version") != "2.1.0" or not isinstance(doc.get("runs"), list):
|
|
132
|
+
raise ValueError("not a SARIF 2.1.0 file (needs \"version\": \"2.1.0\" and a \"runs\" list)")
|
|
133
|
+
filed, not_filed = [], []
|
|
134
|
+
for run in doc["runs"]:
|
|
135
|
+
driver = ((run or {}).get("tool") or {}).get("driver") or {}
|
|
136
|
+
tool = str(driver.get("name") or "sarif")
|
|
137
|
+
rules = [r for r in driver.get("rules") or [] if isinstance(r, dict)]
|
|
138
|
+
for ext in ((run or {}).get("tool") or {}).get("extensions") or []:
|
|
139
|
+
rules += [r for r in (ext or {}).get("rules") or [] if isinstance(r, dict)]
|
|
140
|
+
by_id = {r.get("id"): r for r in rules}
|
|
141
|
+
for res in (run or {}).get("results") or []:
|
|
142
|
+
if not isinstance(res, dict):
|
|
143
|
+
continue
|
|
144
|
+
rule_id = str(res.get("ruleId") or (res.get("rule") or {}).get("id") or "")
|
|
145
|
+
index = res.get("ruleIndex")
|
|
146
|
+
rule = rules[index] if isinstance(index, int) and 0 <= index < len(rules) else by_id.get(rule_id, {})
|
|
147
|
+
rule_id = rule_id or str(rule.get("id") or "unknown-rule")
|
|
148
|
+
uri, line = _location(res)
|
|
149
|
+
where = f"{uri}:{line}" if line else uri
|
|
150
|
+
if res.get("suppressions"):
|
|
151
|
+
not_filed.append({"rule": rule_id, "location": where, "why": "not filed: suppressed in the SARIF file"})
|
|
152
|
+
continue
|
|
153
|
+
if res.get("kind") in ("pass", "notApplicable"):
|
|
154
|
+
not_filed.append({"rule": rule_id, "location": where, "why": "not filed: the result is a pass"})
|
|
155
|
+
continue
|
|
156
|
+
cwes = _cwes(res, rule)
|
|
157
|
+
category = next((owasp_category(c) for c in cwes if owasp_category(c)), None)
|
|
158
|
+
if not category:
|
|
159
|
+
why = NO_CATEGORY + (f" ({', '.join(f'CWE-{c}' for c in cwes)})" if cwes else "")
|
|
160
|
+
not_filed.append({"rule": rule_id, "location": where, "why": why})
|
|
161
|
+
continue
|
|
162
|
+
message = _text(res.get("message")).strip()
|
|
163
|
+
name = _text(rule.get("shortDescription")) or rule.get("name") or rule_id
|
|
164
|
+
advice = _text(rule.get("help")) or _text(rule.get("fullDescription"))
|
|
165
|
+
raw_key = f"{tool}|{rule_id}|{uri}|{line or ''}"
|
|
166
|
+
filed.append(scrub_finding({
|
|
167
|
+
"finding_key": "import-" + hashlib.sha256(raw_key.encode()).hexdigest()[:16],
|
|
168
|
+
"owasp_category": category,
|
|
169
|
+
"severity": _severity(rule, res),
|
|
170
|
+
"title": f"{name}: {message.splitlines()[0] if message else rule_id}"[:300],
|
|
171
|
+
"location_file": uri[:500] or None,
|
|
172
|
+
"location_lines": line,
|
|
173
|
+
"remediation": (f"{message} {advice}".strip() + f" (from {tool}, rule {rule_id})")[:2000],
|
|
174
|
+
}))
|
|
175
|
+
return filed, not_filed
|
|
176
|
+
|
|
177
|
+
|
|
178
|
+
def tool_name(doc) -> str:
|
|
179
|
+
runs = doc.get("runs") or [{}]
|
|
180
|
+
return str((((runs[0] or {}).get("tool") or {}).get("driver") or {}).get("name") or "sarif")
|
|
181
|
+
|
|
182
|
+
|
|
183
|
+
def build_report(source: str, tool: str, filed: list, not_filed: list, cut: int) -> tuple:
|
|
184
|
+
counts = {s: sum(1 for f in filed if f["severity"].lower() == s) for s in SEVERITIES}
|
|
185
|
+
stamp = datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M UTC")
|
|
186
|
+
L = [f"# SARIF import — {source}", "", f"_{stamp} · `ikanban-import` · tool: {tool}_", "",
|
|
187
|
+
f"**{len(filed)} finding(s) to file, {len(not_filed)} result(s) not filed.**", "", "## Filed"]
|
|
188
|
+
L += [f"- **[{f['severity']}]** {f['owasp_category']}: {f['title']} — `{f['location_file'] or '-'}"
|
|
189
|
+
f"{':' + f['location_lines'] if f['location_lines'] else ''}`" for f in filed] or ["- none"]
|
|
190
|
+
if cut:
|
|
191
|
+
L.append(f"- {cut} more were left out of iKanban (the server takes {MAX_FILED}); this import is incomplete")
|
|
192
|
+
L += ["", "## Not filed"]
|
|
193
|
+
L += [f"- {n['rule']} — `{n['location'] or '-'}`: {n['why']}" for n in not_filed] or ["- none"]
|
|
194
|
+
return "\n".join(L), counts
|
|
195
|
+
|
|
196
|
+
|
|
197
|
+
# ── copied from ikanban-secscan.py: scope, key-host pinning, API, work folder, Pipelines ──────
|
|
198
|
+
def normalize_remote(url: str) -> str:
|
|
199
|
+
"""owner/repo from any remote style (scp, https, ssh, a host alias), lower case;
|
|
200
|
+
never the user or a token in the URL."""
|
|
201
|
+
path = re.sub(r"^[a-z+]+://[^/]*/?", "", url.strip()) # scheme, user, host[:port]
|
|
202
|
+
if path == url.strip():
|
|
203
|
+
path = path.split(":", 1)[-1] # scp style user@host:path
|
|
204
|
+
path = path.strip("/")
|
|
205
|
+
path = path[:-4] if path.endswith(".git") else path
|
|
206
|
+
return "/".join(path.split("/")[-2:]).lower() if path else ""
|
|
207
|
+
|
|
208
|
+
|
|
209
|
+
def scan_scope(root: Path) -> str:
|
|
210
|
+
"""Plan 11 H3: which repository a scan covers, the same on every laptop and agent
|
|
211
|
+
run (an agent's run folder is named by its run id): origin's owner/repo, else the
|
|
212
|
+
repository's folder name; plus the subfolder when --path is one. A rescan closes
|
|
213
|
+
only findings of the same scope."""
|
|
214
|
+
root = root.resolve()
|
|
215
|
+
|
|
216
|
+
def git(*a):
|
|
217
|
+
try:
|
|
218
|
+
return subprocess.run(["git", "-C", str(root), *a], capture_output=True,
|
|
219
|
+
text=True, timeout=10).stdout.strip()
|
|
220
|
+
except (OSError, subprocess.SubprocessError):
|
|
221
|
+
return ""
|
|
222
|
+
top = git("rev-parse", "--show-toplevel")
|
|
223
|
+
base = Path(top).resolve() if top else root
|
|
224
|
+
base_name = normalize_remote(git("remote", "get-url", "origin")) or base.name.lower()
|
|
225
|
+
sub = root.relative_to(base).as_posix() if base in root.parents else ""
|
|
226
|
+
return f"{base_name}/{sub}" if sub else base_name
|
|
227
|
+
|
|
228
|
+
|
|
229
|
+
# Coding standard T3: the iKanban key goes only to iKanban's API (or this machine), and never
|
|
230
|
+
# on a redirect. A --base-url, IKANBAN_BASE_URL or config file cannot send it anywhere else.
|
|
231
|
+
IKANBAN_APIS = ("https://api.i-kanban.com", "https://dev-api.i-kanban.com")
|
|
232
|
+
|
|
233
|
+
|
|
234
|
+
def ikanban_api(base_url):
|
|
235
|
+
"""The origin of base_url if it is iKanban's API; otherwise stop, naming it (never the key)."""
|
|
236
|
+
u = urllib.parse.urlsplit(base_url or "")
|
|
237
|
+
origin = f"{u.scheme}://{u.netloc}"
|
|
238
|
+
if u.username is None and u.path in ("", "/") and not u.query and (
|
|
239
|
+
origin in IKANBAN_APIS or (u.scheme == "http" and u.hostname in ("127.0.0.1", "localhost", "::1"))):
|
|
240
|
+
return origin
|
|
241
|
+
raise SystemExit(f"error: {base_url or '(empty)'} is not an iKanban API "
|
|
242
|
+
f"({', '.join(IKANBAN_APIS)} or this machine); the key was not sent")
|
|
243
|
+
|
|
244
|
+
|
|
245
|
+
class _NoRedirect(urllib.request.HTTPRedirectHandler):
|
|
246
|
+
def redirect_request(self, *args, **kwargs):
|
|
247
|
+
return None # a 3xx comes back as an HTTPError, and the key goes nowhere else
|
|
248
|
+
|
|
249
|
+
|
|
250
|
+
_OPENER = urllib.request.build_opener(_NoRedirect)
|
|
251
|
+
|
|
252
|
+
|
|
253
|
+
def api(method: str, path: str, base_url: str, token: str, payload=None):
|
|
254
|
+
"""(ok, data | error text | None for 404)."""
|
|
255
|
+
req = urllib.request.Request(
|
|
256
|
+
f"{ikanban_api(base_url)}/api{path}",
|
|
257
|
+
data=json.dumps(payload).encode() if payload is not None else None,
|
|
258
|
+
headers={"Authorization": f"Bearer {token}",
|
|
259
|
+
"Content-Type": "application/json"},
|
|
260
|
+
method=method,
|
|
261
|
+
)
|
|
262
|
+
try:
|
|
263
|
+
with _OPENER.open(req, timeout=30) as r:
|
|
264
|
+
body = json.loads(r.read().decode() or "{}")
|
|
265
|
+
return True, body.get("data", body)
|
|
266
|
+
except urllib.error.HTTPError as e:
|
|
267
|
+
detail = e.read().decode()[:300]
|
|
268
|
+
if e.code in (404, 405) and not detail.strip():
|
|
269
|
+
return False, None # an unknown route: GET answers 404, POST 405, both empty
|
|
270
|
+
return False, f"HTTP {e.code}: {detail}"
|
|
271
|
+
except Exception as e: # noqa: BLE001 — surface any transport failure verbatim
|
|
272
|
+
return False, str(e)
|
|
273
|
+
|
|
274
|
+
|
|
275
|
+
# ── Scan work folder (plan 11 H12-T10, owner): raw tool output and the report live under
|
|
276
|
+
# <repo>/.iKanban/scans/<scanner>-<UTC time>/, never the system temp folder. The scans folder
|
|
277
|
+
# holds its own .gitignore ("*"), so nothing in it is committed and no repo .gitignore is edited.
|
|
278
|
+
def add_work_dir_arg(ap):
|
|
279
|
+
ap.add_argument("--work-dir", default=os.environ.get("IKANBAN_SCAN_DIR", ""),
|
|
280
|
+
help="Folder for this scan's files (or IKANBAN_SCAN_DIR; default "
|
|
281
|
+
"<repository>/.iKanban/scans)")
|
|
282
|
+
ap.add_argument("--print-report", action="store_true",
|
|
283
|
+
help="Print the full report on stdout (under the ikanban CLI stdout otherwise shows the "
|
|
284
|
+
"counts and `ikanban scans show <id>`)")
|
|
285
|
+
|
|
286
|
+
|
|
287
|
+
def scan_work_dir(scanner: str, base: Path, override: str = "") -> Path:
|
|
288
|
+
# Plan 5 X3: the ikanban CLI hands over a private temp folder (IKANBAN_SCAN_DIR) and the scan id
|
|
289
|
+
# (IKANBAN_SCAN_ID). That folder is this run's folder; the CLI seals it into ~/.ikanban/scans.
|
|
290
|
+
cli_dir = os.environ.get("IKANBAN_SCAN_DIR", "")
|
|
291
|
+
if override and cli_dir and os.environ.get("IKANBAN_SCAN_ID") and (
|
|
292
|
+
Path(override).expanduser().resolve() == Path(cli_dir).expanduser().resolve()):
|
|
293
|
+
run_dir = Path(cli_dir).expanduser().resolve()
|
|
294
|
+
run_dir.mkdir(mode=0o700, parents=True, exist_ok=True)
|
|
295
|
+
os.chmod(run_dir, 0o700)
|
|
296
|
+
return run_dir
|
|
297
|
+
scans = Path(override).expanduser().resolve() if override else base / ".iKanban" / "scans"
|
|
298
|
+
scans.mkdir(parents=True, exist_ok=True)
|
|
299
|
+
ignore = scans / ".gitignore"
|
|
300
|
+
if not ignore.exists():
|
|
301
|
+
ignore.write_text("*\n")
|
|
302
|
+
run_dir = scans / f"{scanner}-{datetime.now(timezone.utc):%Y%m%d-%H%M%S}"
|
|
303
|
+
run_dir.mkdir(mode=0o700, exist_ok=True)
|
|
304
|
+
os.chmod(run_dir, 0o700) # mkdir's mode is masked by umask; raw tool output stays private
|
|
305
|
+
return run_dir
|
|
306
|
+
|
|
307
|
+
|
|
308
|
+
def write_private(path: Path, text: str) -> None:
|
|
309
|
+
"""Write a file (0600) without following a symlink planted at its name: write a new
|
|
310
|
+
temp file next to it (O_EXCL | O_NOFOLLOW), then rename it over the name."""
|
|
311
|
+
tmp = path.with_name(f".{path.name}.{os.getpid()}.tmp")
|
|
312
|
+
fd = os.open(tmp, os.O_WRONLY | os.O_CREAT | os.O_EXCL | getattr(os, "O_NOFOLLOW", 0), 0o600)
|
|
313
|
+
with os.fdopen(fd, "w", encoding="utf-8") as f:
|
|
314
|
+
f.write(text)
|
|
315
|
+
os.replace(tmp, path)
|
|
316
|
+
|
|
317
|
+
|
|
318
|
+
def slug(text: str) -> str:
|
|
319
|
+
return re.sub(r"[^A-Za-z0-9._-]+", "_", text).strip("_")[:80] or "x"
|
|
320
|
+
|
|
321
|
+
|
|
322
|
+
# ── scrub_text: identical in every scanner (test_scrub_text.py compares the copies) ──────────
|
|
323
|
+
# Plan 5 X2-T1: every text field of every finding passes through scrub_text before it leaves this
|
|
324
|
+
# machine. A secret keeps its first 4 characters; a home folder becomes ~. The server scrubs again.
|
|
325
|
+
_SCRUB_CUT = (
|
|
326
|
+
re.compile(r"-{5}BEGIN [A-Z ]*PRIVATE KEY-{5}.*?(?:-{5}END [A-Z ]*PRIVATE KEY-{5}|\Z)", re.S),
|
|
327
|
+
re.compile(r"\b(?:AKIA|ASIA)[0-9A-Z]{16}\b"),
|
|
328
|
+
re.compile(r"\b(?:ghp|gho|ghu|ghs|ghr)_[A-Za-z0-9]{36}\b"),
|
|
329
|
+
re.compile(r"\bgithub_pat_[A-Za-z0-9_]{22,}\b"),
|
|
330
|
+
re.compile(r"\bglpat-[A-Za-z0-9_-]{20,}\b"),
|
|
331
|
+
re.compile(r"\bxox[abposr]-[A-Za-z0-9-]{10,}\b"),
|
|
332
|
+
re.compile(r"\b(?:sk|rk)_live_[A-Za-z0-9]{20,}\b"),
|
|
333
|
+
re.compile(r"\bsk-(?:ant-|proj-)?[A-Za-z0-9_-]{24,}\b"),
|
|
334
|
+
re.compile(r"\bwhsec_[A-Za-z0-9]{16,}\b"),
|
|
335
|
+
re.compile(r"\bAIza[0-9A-Za-z_-]{35}\b"),
|
|
336
|
+
re.compile(r"\beyJ[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\b"),
|
|
337
|
+
re.compile(r"\bvk_[A-Za-z0-9_-]{8,}"),
|
|
338
|
+
)
|
|
339
|
+
_SCRUB_ASSIGNED = re.compile(
|
|
340
|
+
r"(?i)(\b(?:aws_secret_access_key|aws_session_token|client_secret|secret_key|private_key|api[_-]?key|"
|
|
341
|
+
r"access[_-]?token|auth[_-]?token|token|password|passwd|pwd|secret)\b[\"']?\s*[:=]\s*[\"']?)([^\s\"',;&]+)")
|
|
342
|
+
_SCRUB_BEARER = re.compile(r"(?i)(\b(?:bearer|basic)\s+)([A-Za-z0-9._~+/=-]{12,})")
|
|
343
|
+
_SCRUB_URL_CREDS = re.compile(r"(\b[a-z][a-z0-9+.-]*://[^\s/:@]+:)([^\s@/]+)(@)")
|
|
344
|
+
_SCRUB_HOME = re.compile(r"(?<![\w.-])(?:[A-Za-z]:\\[Uu]sers\\|/Users/|/home/)[^\\/\s\"'<>]+")
|
|
345
|
+
|
|
346
|
+
|
|
347
|
+
def scrub_text(text) -> str:
|
|
348
|
+
"""Secrets, iKanban keys, tokens, AWS and GitHub keys, private key blocks and home folders
|
|
349
|
+
cut from one text. None becomes ""."""
|
|
350
|
+
if text is None:
|
|
351
|
+
return ""
|
|
352
|
+
text = str(text)
|
|
353
|
+
for rx in _SCRUB_CUT:
|
|
354
|
+
text = rx.sub(lambda m: m.group(0)[:4] + "…[hidden]", text)
|
|
355
|
+
text = _SCRUB_ASSIGNED.sub(
|
|
356
|
+
lambda m: m.group(1) + "[hidden]" if len(m.group(2)) >= 8 or any(c.isdigit() for c in m.group(2))
|
|
357
|
+
else m.group(0), text)
|
|
358
|
+
text = _SCRUB_BEARER.sub(lambda m: m.group(1) + "[hidden]", text)
|
|
359
|
+
text = _SCRUB_URL_CREDS.sub(lambda m: m.group(1) + "[hidden]" + m.group(3), text)
|
|
360
|
+
home = str(Path.home())
|
|
361
|
+
if home and home not in ("/", "\\") and home in text:
|
|
362
|
+
text = text.replace(home, "~")
|
|
363
|
+
return _SCRUB_HOME.sub("~", text)
|
|
364
|
+
|
|
365
|
+
|
|
366
|
+
def scrub_finding(value):
|
|
367
|
+
"""scrub_text on every string in a finding, through nested dicts and lists; other values stay."""
|
|
368
|
+
if isinstance(value, str):
|
|
369
|
+
return scrub_text(value)
|
|
370
|
+
if isinstance(value, dict):
|
|
371
|
+
return {k: scrub_finding(v) for k, v in value.items()}
|
|
372
|
+
if isinstance(value, list):
|
|
373
|
+
return [scrub_finding(v) for v in value]
|
|
374
|
+
return value
|
|
375
|
+
|
|
376
|
+
|
|
377
|
+
# ── end scrub_text ──
|
|
378
|
+
|
|
379
|
+
|
|
380
|
+
# ── Pipelines (plan 11 H12, owner): with --task, the scan shows in that task's Pipelines view ──
|
|
381
|
+
# Same calls as `ikanban pipeline` (vibe-cli/lib/pipeline.js → routes/pipeline.rs): an event
|
|
382
|
+
# "running" at the start, "passed" / "failed" / "skipped" at the end with a one-line detail,
|
|
383
|
+
# then that line as the log (plan 5: never the report). A plan without Pipelines (402) or any error never fails the scan.
|
|
384
|
+
_UUID = re.compile(r"^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$", re.I)
|
|
385
|
+
# Every iKanban scanner reports into one named lane, not the server's "legacy" lane.
|
|
386
|
+
PIPE_RUN = "security-scan"
|
|
387
|
+
PIPE_CHUNK = 15 * 1024 # the server takes 16 KB per log line
|
|
388
|
+
LAST_RUN = {"report": "", "detail": "", "skipped": False}
|
|
389
|
+
|
|
390
|
+
|
|
391
|
+
def add_pipeline_args(ap):
|
|
392
|
+
ap.add_argument("--task", default=os.environ.get("IKANBAN_TASK", ""),
|
|
393
|
+
help="Also show this run in the task's Pipelines (a key like IKA-123, or its id; "
|
|
394
|
+
"or IKANBAN_TASK)")
|
|
395
|
+
ap.add_argument("--task-team", default=os.environ.get("IKANBAN_TASK_TEAM", ""),
|
|
396
|
+
help="The task's team: id or identifier like IKA (or IKANBAN_TASK_TEAM; default --team)")
|
|
397
|
+
|
|
398
|
+
|
|
399
|
+
def summary_detail(counts: dict, total: int, skipped: list) -> str:
|
|
400
|
+
if skipped and not total and all(s for s in skipped):
|
|
401
|
+
return ("SKIPPED: " + "; ".join(skipped))[:300]
|
|
402
|
+
worst = ", ".join(f"{counts[s]} {s}" for s in ("critical", "high", "medium", "low") if counts.get(s))
|
|
403
|
+
return f"{total} findings" + (f": {worst}" if worst else "")
|
|
404
|
+
|
|
405
|
+
|
|
406
|
+
def mask_report(text: str) -> str:
|
|
407
|
+
"""scrub_text before a line leaves for Pipelines, plus a scanner's own rules (scrub_report)."""
|
|
408
|
+
text = scrub_text(text)
|
|
409
|
+
scrub = globals().get("scrub_report")
|
|
410
|
+
return scrub(text)[0] if scrub else text
|
|
411
|
+
|
|
412
|
+
|
|
413
|
+
def show_report(md: str, summary: str, print_report: bool = False) -> None:
|
|
414
|
+
"""Plan 5 X3: under the ikanban CLI (IKANBAN_SCAN_ID set) stdout gets the counts and where the
|
|
415
|
+
full report is stored; the body only with --print-report. A plain python run prints the body."""
|
|
416
|
+
scan_id = os.environ.get("IKANBAN_SCAN_ID", "")
|
|
417
|
+
if scan_id and not print_report:
|
|
418
|
+
print(summary)
|
|
419
|
+
print(f"Full report: ikanban scans show {scan_id}")
|
|
420
|
+
else:
|
|
421
|
+
print(md)
|
|
422
|
+
|
|
423
|
+
|
|
424
|
+
def scan_files_note(work: Path) -> str:
|
|
425
|
+
"""Where this run's files are: the encrypted store under the CLI, else the run folder."""
|
|
426
|
+
scan_id = os.environ.get("IKANBAN_SCAN_ID", "")
|
|
427
|
+
return f"scan files: sealed into ~/.ikanban/scans as {scan_id}" if scan_id else f"scan files: {work}"
|
|
428
|
+
|
|
429
|
+
|
|
430
|
+
|
|
431
|
+
# The running Pipeline (set by run_main with --task). Owner: Pipelines showed nothing while a
|
|
432
|
+
# scan ran, so each step also sends one line as it starts and ends; the one-line summary follows at the end.
|
|
433
|
+
# Progress lines are fixed status text: never raw tool output or an error body (it can carry
|
|
434
|
+
# secrets or text an attacker wrote). Those stay on local stderr and in the masked report.
|
|
435
|
+
PIPE = None
|
|
436
|
+
|
|
437
|
+
|
|
438
|
+
def progress(text: str) -> None:
|
|
439
|
+
if os.environ.get("IKANBAN_SCAN_ID") and "--json" not in sys.argv:
|
|
440
|
+
print(text, flush=True) # Plan 5: under the CLI stdout shows progress, not the report
|
|
441
|
+
if PIPE:
|
|
442
|
+
PIPE.log(text)
|
|
443
|
+
|
|
444
|
+
|
|
445
|
+
|
|
446
|
+
class Pipeline:
|
|
447
|
+
def __init__(self, stage, task, team, base_url, token):
|
|
448
|
+
self.stage, self.task, self.team, self.base, self.token = stage, task, team, base_url, token
|
|
449
|
+
self.on = bool(task and team and token)
|
|
450
|
+
self.tid = None
|
|
451
|
+
|
|
452
|
+
def _off(self, why):
|
|
453
|
+
self.on = False
|
|
454
|
+
note = "the plan does not include Pipelines" if "402" in str(why) else why
|
|
455
|
+
print(f"ikanban: Pipelines not updated ({note}); the scan goes on", file=sys.stderr)
|
|
456
|
+
|
|
457
|
+
def _call(self, method, path, body=None):
|
|
458
|
+
try:
|
|
459
|
+
return api(method, path, self.base, self.token, body)
|
|
460
|
+
except SystemExit as e: # ikanban_api refuses a base URL that is not iKanban's
|
|
461
|
+
return False, str(e)
|
|
462
|
+
|
|
463
|
+
def _post(self, path, body):
|
|
464
|
+
if not self.on:
|
|
465
|
+
return
|
|
466
|
+
if self.tid is None:
|
|
467
|
+
if _UUID.match(self.team):
|
|
468
|
+
self.tid = self.team
|
|
469
|
+
else:
|
|
470
|
+
ok, teams = self._call("GET", "/teams")
|
|
471
|
+
if not ok:
|
|
472
|
+
return self._off(teams)
|
|
473
|
+
found = [t.get("id") for t in teams or [] if (t.get("identifier") or "").lower() == self.team.lower()]
|
|
474
|
+
if not found:
|
|
475
|
+
return self._off(f"team {self.team} not found among your teams")
|
|
476
|
+
self.tid = found[0]
|
|
477
|
+
ok, data = self._call("POST", f"/teams/{self.tid}/{path}", body)
|
|
478
|
+
if not ok:
|
|
479
|
+
self._off(data or "this iKanban server has no Pipelines")
|
|
480
|
+
|
|
481
|
+
def start(self):
|
|
482
|
+
self._post("pipeline-events", {"task": self.task, "stage": self.stage, "status": "running",
|
|
483
|
+
"run": PIPE_RUN, "source": "laptop", "actor": f"ikanban-{self.stage}"})
|
|
484
|
+
|
|
485
|
+
def log(self, line):
|
|
486
|
+
"""One progress line, sent as it happens (masked like the report). A failure only turns Pipelines off."""
|
|
487
|
+
self._post("pipeline-logs", {"task": self.task, "run": PIPE_RUN, "stage": self.stage,
|
|
488
|
+
"text": mask_report(line.rstrip("\n"))[:PIPE_CHUNK - 1] + "\n"})
|
|
489
|
+
|
|
490
|
+
def finish(self, code):
|
|
491
|
+
status = "failed" if code else ("skipped" if LAST_RUN["skipped"] else "passed")
|
|
492
|
+
detail = LAST_RUN["detail"] or ("the scan stopped with an error" if code == 2 else "")
|
|
493
|
+
if code == 1 and not LAST_RUN["skipped"]:
|
|
494
|
+
detail = (detail + " (at or above --fail-on)").strip()
|
|
495
|
+
self._post("pipeline-events", {"task": self.task, "stage": self.stage, "status": status,
|
|
496
|
+
"run": PIPE_RUN, "source": "laptop", "actor": f"ikanban-{self.stage}",
|
|
497
|
+
"detail": detail[:300] or None})
|
|
498
|
+
# Plan 5 X1-T2: the summary only; the report stays on the machine that ran the scan.
|
|
499
|
+
summary = f"{self.stage} {status}" + (f": {detail[:300]}" if detail else "")
|
|
500
|
+
self._post("pipeline-logs", {"task": self.task, "run": PIPE_RUN, "stage": self.stage,
|
|
501
|
+
"text": mask_report(summary) + "\n"})
|
|
502
|
+
|
|
503
|
+
|
|
504
|
+
def run_main(stage, main_fn, argv=None):
|
|
505
|
+
"""main() as it is, wrapped in Pipelines events when --task is given."""
|
|
506
|
+
raw = sys.argv[1:] if argv is None else argv
|
|
507
|
+
if any(a in ("-h", "--help", "--version", "--list-apps") for a in raw):
|
|
508
|
+
return main_fn(argv)
|
|
509
|
+
pre = argparse.ArgumentParser(add_help=False)
|
|
510
|
+
pre.add_argument("--task", default=os.environ.get("IKANBAN_TASK", ""))
|
|
511
|
+
pre.add_argument("--task-team", default=os.environ.get("IKANBAN_TASK_TEAM", ""))
|
|
512
|
+
pre.add_argument("--team", default=os.environ.get("IKANBAN_TEAM_ID", ""))
|
|
513
|
+
pre.add_argument("--token", default=os.environ.get("VIBE_API_TOKEN", ""))
|
|
514
|
+
pre.add_argument("--base-url", default=os.environ.get("IKANBAN_BASE_URL", "https://api.i-kanban.com"))
|
|
515
|
+
known, _ = pre.parse_known_args(raw)
|
|
516
|
+
pl = Pipeline(stage, known.task, known.task_team or known.team, known.base_url, known.token)
|
|
517
|
+
pl.start()
|
|
518
|
+
global PIPE
|
|
519
|
+
PIPE = pl if pl.on else None
|
|
520
|
+
try:
|
|
521
|
+
code = main_fn(argv)
|
|
522
|
+
except SystemExit as e:
|
|
523
|
+
if not isinstance(e.code, int):
|
|
524
|
+
print(e.code, file=sys.stderr)
|
|
525
|
+
code = e.code if isinstance(e.code, int) else 2
|
|
526
|
+
pl.finish(code)
|
|
527
|
+
return code
|
|
528
|
+
|
|
529
|
+
|
|
530
|
+
def main(argv=None):
|
|
531
|
+
ap = argparse.ArgumentParser(description="Turn a SARIF 2.1.0 file into iKanban code findings.")
|
|
532
|
+
ap.add_argument("--version", action="version", version=f"ikanban import {IMPORT_VERSION}")
|
|
533
|
+
ap.add_argument("file", help="A SARIF 2.1.0 file (.sarif or .json) from another scanner")
|
|
534
|
+
ap.add_argument("--path", default=".", help="The repository the results belong to (default: cwd)")
|
|
535
|
+
ap.add_argument("--publish", action="store_true",
|
|
536
|
+
help="File the findings into iKanban (the SARIF file and the report stay here)")
|
|
537
|
+
ap.add_argument("--team", default=os.environ.get("IKANBAN_TEAM_ID", ""),
|
|
538
|
+
help="Team UUID to publish into (or IKANBAN_TEAM_ID)")
|
|
539
|
+
ap.add_argument("--base-url", default=os.environ.get("IKANBAN_BASE_URL", "https://api.i-kanban.com"))
|
|
540
|
+
ap.add_argument("--token", default=os.environ.get("VIBE_API_TOKEN", ""),
|
|
541
|
+
help="Personal API key (or VIBE_API_TOKEN)")
|
|
542
|
+
ap.add_argument("--out", default=None, help="Also write the report here")
|
|
543
|
+
ap.add_argument("--json", action="store_true", help="Machine-readable summary")
|
|
544
|
+
add_pipeline_args(ap)
|
|
545
|
+
add_work_dir_arg(ap)
|
|
546
|
+
args = ap.parse_args(argv)
|
|
547
|
+
|
|
548
|
+
root = Path(args.path).expanduser().resolve()
|
|
549
|
+
source = Path(args.file).expanduser()
|
|
550
|
+
if not root.is_dir():
|
|
551
|
+
print(f"error: {root} is not a directory", file=sys.stderr)
|
|
552
|
+
return 2
|
|
553
|
+
if args.publish and (not args.token or not args.team):
|
|
554
|
+
print("error: --publish needs --token (or VIBE_API_TOKEN) and --team (or IKANBAN_TEAM_ID)", file=sys.stderr)
|
|
555
|
+
return 2
|
|
556
|
+
try:
|
|
557
|
+
if source.stat().st_size > 200 * 1024 * 1024:
|
|
558
|
+
raise ValueError("the file is over 200 MB")
|
|
559
|
+
doc = json.loads(source.read_text(encoding="utf-8-sig"))
|
|
560
|
+
filed, not_filed = sarif_findings(doc)
|
|
561
|
+
except (OSError, ValueError) as e: # json.JSONDecodeError is a ValueError
|
|
562
|
+
print(f"error: {source.name}: {e}. ikanban import reads SARIF 2.1.0 files only.", file=sys.stderr)
|
|
563
|
+
return 2
|
|
564
|
+
tool = tool_name(doc)
|
|
565
|
+
filed.sort(key=lambda f: (SEVERITY_RANK.get(f["severity"].lower(), 99), f["title"]))
|
|
566
|
+
cut = max(0, len(filed) - MAX_FILED)
|
|
567
|
+
filed = filed[:MAX_FILED]
|
|
568
|
+
work = scan_work_dir("import", root, args.work_dir)
|
|
569
|
+
progress(f"import {IMPORT_VERSION} · {tool}: {len(filed)} to file, {len(not_filed)} not filed")
|
|
570
|
+
md, counts = build_report(source.name, tool, filed, not_filed, cut)
|
|
571
|
+
detail = f"{len(filed)} findings from {tool}" + (f"; {len(not_filed)} not filed" if not_filed else "")
|
|
572
|
+
LAST_RUN.update(report=md, detail=detail, skipped=False)
|
|
573
|
+
write_private(work / "report.md", md)
|
|
574
|
+
if args.out:
|
|
575
|
+
Path(args.out).write_text(md, encoding="utf-8")
|
|
576
|
+
if args.json:
|
|
577
|
+
print(json.dumps({"file": source.name, "tool": tool, "filed": len(filed), "not_filed": len(not_filed),
|
|
578
|
+
"counts": counts}, indent=2))
|
|
579
|
+
else:
|
|
580
|
+
show_report(md, detail, args.print_report)
|
|
581
|
+
sys.stdout.flush()
|
|
582
|
+
print(scan_files_note(work), file=sys.stderr)
|
|
583
|
+
|
|
584
|
+
if args.publish:
|
|
585
|
+
scope = f"sarif:{slug(tool).lower()}:{scan_scope(root)}"
|
|
586
|
+
ok, data = api("POST", f"/teams/{args.team}/security/scan-findings", args.base_url, args.token,
|
|
587
|
+
{"findings": filed, "scope": scope, "complete": cut == 0})
|
|
588
|
+
if not ok:
|
|
589
|
+
print(f"filing findings FAILED: {data or 'this iKanban server does not take them yet'}", file=sys.stderr)
|
|
590
|
+
return 2
|
|
591
|
+
print(f"filed {data.get('submitted', len(filed))} findings ({data.get('new', 0)} new, reopened "
|
|
592
|
+
f"{data.get('reopened', 0)}, closed {data.get('closed', 0)} no longer found in {scope})", file=sys.stderr)
|
|
593
|
+
return 0
|
|
594
|
+
|
|
595
|
+
|
|
596
|
+
if __name__ == "__main__":
|
|
597
|
+
sys.exit(run_main("import", main))
|