@hoanghainh/komagu 2.4.0 → 2.5.0

package/operate/tests/test_leak_check.py

@@ -0,0 +1,250 @@
+#!/usr/bin/env python3
+"""komagu leak-check: planted secrets are found and masked, clean output passes (runbook §9, #367)."""
+
+from __future__ import annotations
+
+import base64
+import contextlib
+import io
+import json
+import os
+import tempfile
+import unittest
+from pathlib import Path
+from typing import List
+
+from operate.cli.main import main
+from operate.devsecops.leak_check import load_canaries, CanaryFileError, scan_paths, scan_text, shannon_entropy
+
+REPO_ROOT = Path(__file__).resolve().parents[2]
+# Assembled at run time so repository secret scanners do not flag the test itself
+AWS_KEY = "AKIA" + "ABCDEFGHIJKLMNOP"
+SLACK_TOKEN = "xoxb-" + "1234567890-abcdefghijklmnop"
+JWT = "eyJhbGciOiJIUzI1NiJ9" + ".eyJzdWIiOiIxMjM0NTY3ODkwIn0" + ".dozjgNryP4J3jVmNHl0w5N_XgL0n3I9P"
+RANDOM_TOKEN = "kX9mQ2vL7pR4" + "tZ8wB3nY6cF1"
+
+
+def detectors(text: str) -> List[str]:
+    return [hit.detector for hit in scan_text("f", text)]
+
+
+class TestDetection(unittest.TestCase):
+    def test_redaction_pattern_catches_unmasked_password(self) -> None:
+        self.assertIn("redaction-pattern", detectors('String password = "hunter2hunter2";'))
+
+    def test_token_patterns_beyond_the_redaction_list(self) -> None:
+        for planted in (SLACK_TOKEN, JWT):
+            self.assertIn("token-pattern", detectors(f"seen in reply: {planted}"), planted)
+
+    def test_aws_key_is_found(self) -> None:
+        self.assertTrue(detectors(f"key {AWS_KEY}"))
+
+    def test_entropy_finds_a_bare_token_no_pattern_knows(self) -> None:
+        self.assertEqual(detectors(f"the upstream expects {RANDOM_TOKEN} here"), ["entropy"])
+
+    def test_japanese_label_is_found(self) -> None:
+        self.assertIn("token-pattern", detectors("初期パスワード：Abcd1234"))
+
+    def test_secret_inside_json_string_is_found(self) -> None:
+        raw = json.dumps({"snippet": 'db.setPassword("hunter2hunter2");'})
+        self.assertIn("redaction-pattern", detectors(raw))
+
+    def test_secret_inside_double_encoded_json_is_found(self) -> None:
+        raw = json.dumps(json.dumps({"snippet": 'password = "hunter2hunter2"'}))
+        self.assertIn("redaction-pattern", detectors(raw))
+
+    def test_secret_inside_html_is_found(self) -> None:
+        self.assertIn("redaction-pattern", detectors("<code>password = &quot;hunter2hunter2&quot;</code>"))
+
+    def test_unmasked_multiline_private_key_is_found(self) -> None:
+        text = "-----BEGIN RSA PRIVATE KEY-----\nMIIEowIBAAKCAQEA7\n-----END RSA PRIVATE KEY-----"
+        self.assertIn("token-pattern", detectors(text))
+
+    def test_secret_at_end_of_very_long_line_is_found(self) -> None:
+        text = "x " * 5000 + 'password = "hunter2hunter2"'
+        self.assertIn("redaction-pattern", detectors(text))
+
+    def test_value_split_over_lines_is_found(self) -> None:
+        self.assertIn("redaction-pattern", detectors('<property name="password"\n  value="hunter2hunter2"/>'))
+
+
+class TestCleanOutput(unittest.TestCase):
+    def test_masked_values_are_clean(self) -> None:
+        for line in (
+            'password = "[REDACTED]"',
+            "jdbc:mysql://h/db?user=a&password=[REDACTED]",
+            "-----BEGIN PRIVATE KEY-----[REDACTED]-----END PRIVATE KEY-----",
+            "パスワード: [REDACTED]",
+        ):
+            self.assertEqual(detectors(line), [], line)
+
+    def test_multiline_masked_key_is_clean(self) -> None:
+        text = "-----BEGIN PRIVATE KEY-----\n[REDACTED]\n-----END PRIVATE KEY-----"
+        self.assertEqual(detectors(text), [])
+
+    def test_ordinary_audit_content_is_clean(self) -> None:
+        for line in (
+            '"fingerprint": "9f86d081884c7d659a2feaa0c55ad015a3bf4f1b2b0b822cd15d6c15b0f00a08"',
+            '"sha1": "da39a3ee5e6b4b0d3255bfef95601890afd80709"',
+            '"id": "123e4567-e89b-12d3-a456-426614174000"',
+            '"file": "src/main/java/com/acme/order/OrderManagementController.java"',
+            "org.springframework.web.servlet.DispatcherServlet",
+            "commons-collections4-4.4.jar and struts2-core-2.5.33.jar",
+            "パスワードはハードコードされています",
+        ):
+            self.assertEqual(detectors(line), [], line)
+
+    def test_tracked_fallback_audit_output_is_clean(self) -> None:
+        result = scan_paths([str(REPO_ROOT / "delivery" / "pilot" / "audit-out")])
+        self.assertEqual([(h.path, h.line, h.rule) for h in result.hits], [])
+        self.assertEqual(result.exit_code, 0)
+
+
+class TestCanaries(unittest.TestCase):
+    CANARY = "orange-falcon-db"
+
+    def run_with(self, text: str) -> int:
+        with tempfile.TemporaryDirectory() as tmp:
+            (Path(tmp) / "reply.txt").write_text(text, encoding="utf-8")
+            return len(scan_paths([tmp], [self.CANARY]).hits)
+
+    def test_canary_forms(self) -> None:
+        encoded = base64.b64encode(self.CANARY.encode()).decode()
+        for text in (
+            f"host {self.CANARY} found",
+            f"host {self.CANARY.upper()} found",
+            f"host {encoded} found",
+            json.dumps({"host": self.CANARY}),
+        ):
+            self.assertGreaterEqual(self.run_with(text), 1, text)
+
+    def test_japanese_canary_in_ascii_escaped_json(self) -> None:
+        canary = "社内基幹システム"
+        hits = scan_text("f", json.dumps({"name": canary}), [])
+        self.assertEqual(hits, [])
+        with tempfile.TemporaryDirectory() as tmp:
+            (Path(tmp) / "r.json").write_text(json.dumps({"name": canary}), encoding="utf-8")
+            self.assertEqual([h.detector for h in scan_paths([tmp], [canary]).hits], ["canary"])
+
+    def test_canary_in_shift_jis_file(self) -> None:
+        canary = "社内基幹システム"
+        with tempfile.TemporaryDirectory() as tmp:
+            (Path(tmp) / "note.txt").write_bytes(f"対象: {canary}\n".encode("cp932"))
+            self.assertEqual(len(scan_paths([tmp], [canary]).hits), 1)
+
+    def test_canary_inside_encoded_payloads(self) -> None:
+        canary = "p@ss/w0rd-Zq9"
+        with tempfile.TemporaryDirectory() as tmp:
+            for offset in range(3):
+                payload = base64.b64encode(b"xy"[:offset] + canary.encode() + b"tail").decode()
+                (Path(tmp) / f"b{offset}.txt").write_text(payload, encoding="utf-8")
+            (Path(tmp) / "url.txt").write_text("p%40ss%2fw0rd-Zq9", encoding="utf-8")
+            (Path(tmp) / "wide.txt").write_text("ｐ＠ｓｓ／ｗ０ｒｄ－Ｚｑ９", encoding="utf-8")
+            (Path(tmp) / "zw.txt").write_text("p@ss/\u200bw0rd-Zq9", encoding="utf-8")
+            hit_files = {Path(h.path).name for h in scan_paths([tmp], [canary]).hits}
+        self.assertEqual(hit_files, {"b0.txt", "b1.txt", "b2.txt", "url.txt", "wide.txt", "zw.txt"})
+
+    def test_no_canary_no_hit(self) -> None:
+        self.assertEqual(self.run_with("nothing to see"), 0)
+
+    def _load(self, content: str) -> tuple:
+        with tempfile.TemporaryDirectory() as tmp:
+            path = Path(tmp) / "c.txt"
+            path.write_text(content, encoding="utf-8")
+            return load_canaries(str(path))
+
+    def test_every_nonblank_line_is_a_canary(self) -> None:
+        self.assertEqual(self._load("#hash-start\n\n  spaced-value \n"), ("#hash-start", "spaced-value"))
+
+    def test_empty_or_short_canary_file_is_rejected(self) -> None:
+        for content in ("\n\n", "abc\n"):
+            with self.assertRaises(CanaryFileError):
+                self._load(content)
+
+
+class TestCommand(unittest.TestCase):
+    def setUp(self) -> None:
+        self.tmp = tempfile.TemporaryDirectory()
+        self.addCleanup(self.tmp.cleanup)
+        self.root = Path(self.tmp.name)
+        self.out = self.root / "out"
+        (self.out / "ai-raw").mkdir(parents=True)
+
+    def invoke(self, *args: str) -> "tuple[int, str]":
+        stdout = io.StringIO()
+        with contextlib.redirect_stdout(stdout):
+            code = main(["--no-banner", "leak-check", *args])
+        return code, stdout.getvalue()
+
+    def test_clean_output_exits_zero(self) -> None:
+        (self.out / "report.md").write_text('password = "[REDACTED]"\n', encoding="utf-8")
+        self.assertEqual(self.invoke(str(self.out))[0], 0)
+
+    def test_hit_exits_one_with_file_line_and_masked_value(self) -> None:
+        planted_value = "hunter2hunter2"
+        (self.out / "ai-raw" / "4a.txt").write_text(f'ok\npassword = "{planted_value}"\n', encoding="utf-8")
+        canaries = self.root / "canaries.txt"
+        canaries.write_text("orange-falcon-db\n", encoding="utf-8")
+        (self.out / "report.md").write_text("connects to orange-falcon-db\n", encoding="utf-8")
+        for fmt in ("text", "json"):
+            code, output = self.invoke(str(self.out), "--canary-file", str(canaries), "--format", fmt)
+            self.assertEqual(code, 1)
+            self.assertIn("4a.txt", output)
+            self.assertNotIn(planted_value, output)
+            self.assertNotIn("orange-falcon-db", output)
+        self.assertIn("4a.txt:2:", self.invoke(str(self.out))[1])
+
+    def test_hidden_cache_and_test_named_paths_are_scanned(self) -> None:
+        cache = self.out / ".cache" / "test-example"
+        cache.mkdir(parents=True)
+        (cache / "excerpt.txt").write_text(f"{RANDOM_TOKEN}\n", encoding="utf-8")
+        self.assertEqual(self.invoke(str(self.out))[0], 1)
+
+    def test_binary_file_makes_the_scan_incomplete(self) -> None:
+        (self.out / "report.md").write_text("fine\n", encoding="utf-8")
+        (self.out / "bundle.zip").write_bytes(b"PK\0\0payload")
+        code, output = self.invoke(str(self.out))
+        self.assertEqual(code, 2)
+        self.assertIn("NOT SCANNED", output)
+
+    def test_missing_or_empty_target_is_not_clean(self) -> None:
+        self.assertEqual(self.invoke(str(self.root / "absent"))[0], 2)
+        self.assertEqual(self.invoke(str(self.out))[0], 2)
+
+    def test_unusable_canary_file_is_not_clean(self) -> None:
+        (self.out / "report.md").write_text("fine\n", encoding="utf-8")
+        for value in (str(self.root / "absent.txt"), ""):
+            stderr = io.StringIO()
+            with contextlib.redirect_stderr(stderr):
+                code, _ = self.invoke(str(self.out), "--canary-file", value)
+            self.assertEqual(code, 2, value)
+
+    def test_symlinked_directory_makes_the_scan_incomplete(self) -> None:
+        (self.out / "report.md").write_text("fine\n", encoding="utf-8")
+        real = self.root / "real"
+        real.mkdir()
+        (self.out / "linked").symlink_to(real, target_is_directory=True)
+        self.assertEqual(self.invoke(str(self.out))[0], 2)
+
+    @unittest.skipIf(os.name == "nt" or os.geteuid() == 0, "needs enforced directory permissions")
+    def test_unreadable_directory_makes_the_scan_incomplete(self) -> None:
+        (self.out / "report.md").write_text("fine\n", encoding="utf-8")
+        locked = self.out / "locked"
+        locked.mkdir()
+        locked.chmod(0)
+        self.addCleanup(locked.chmod, 0o700)
+        self.assertEqual(self.invoke(str(self.out))[0], 2)
+
+    def test_utf32_file_is_not_treated_as_clean_text(self) -> None:
+        (self.out / "report.txt").write_bytes('password = "hunter2hunter2"'.encode("utf-32"))
+        self.assertEqual(self.invoke(str(self.out))[0], 2)
+
+
+class TestEntropy(unittest.TestCase):
+    def test_entropy_ordering(self) -> None:
+        self.assertEqual(shannon_entropy("aaaa"), 0)
+        self.assertGreater(shannon_entropy(RANDOM_TOKEN), shannon_entropy("passwordpasswordpassword"))
+
+
+if __name__ == "__main__":
+    unittest.main()

package/operate/tests/test_legacy_session_edge.py

@@ -0,0 +1,192 @@
+#!/usr/bin/env python3
+"""Gaps found rehearsing the pilot runbook on a running Struts / JSESSIONID application (#374)."""
+
+from __future__ import annotations
+
+import argparse
+import json
+import subprocess
+import sys
+import tempfile
+import unittest
+from pathlib import Path
+from unittest.mock import MagicMock, patch
+
+REPO_ROOT = Path(__file__).resolve().parents[2]
+sys.path.insert(0, str(REPO_ROOT))
+
+from operate.ai.runner import HeadlessAIRunner  # noqa: E402
+from operate.cli.commands import protect  # noqa: E402
+from operate.cross_user_tester.runner import generate_test_cases_from_findings  # noqa: E402
+from remediate.edge.coraza.virtual_patcher import VirtualPatcher  # noqa: E402
+
+EDGE = REPO_ROOT / "remediate" / "edge"
+
+
+def protect_args(**overrides) -> argparse.Namespace:
+    values = dict(target=None, config=None, up=False, down=False, status=False,
+                  virtual_patch=None, patch_out=None, dry_run=False)
+    return argparse.Namespace(**{**values, **overrides})
+
+
+class ProtectWiringTests(unittest.TestCase):
+    def run_up(self, **overrides):
+        with patch.object(protect.shutil, "which", return_value="/usr/bin/docker"), \
+                patch.object(protect.subprocess, "run", return_value=MagicMock(returncode=0)) as run:
+            code = protect.execute_protect(protect_args(up=True, **overrides))
+        return code, run
+
+    def test_target_reaches_compose(self) -> None:
+        code, run = self.run_up(target="http://host.docker.internal:18080")
+        self.assertEqual(code, 0)
+        self.assertEqual(run.call_args.kwargs["env"]["KOMAGU_UPSTREAM"], "http://host.docker.internal:18080")
+
+    def test_custom_kong_config_reaches_compose(self) -> None:
+        config = EDGE / "kong" / "kong-legacy-session.yaml"
+        _, run = self.run_up(config=str(config))
+        self.assertEqual(run.call_args.kwargs["env"]["KOMAGU_KONG_CONFIG"], str(config.resolve()))
+
+    def test_patch_out_file_is_loaded_by_the_stack(self) -> None:
+        with tempfile.TemporaryDirectory() as td:
+            conf = Path(td) / "virtual-patches.conf"
+            conf.write_text("# none\n", encoding="utf-8")
+            _, run = self.run_up(patch_out=str(conf))
+            self.assertEqual(run.call_args.kwargs["env"]["KOMAGU_VIRTUAL_PATCH_FILE"], str(conf.resolve()))
+
+    def test_missing_patch_file_stops_before_docker(self) -> None:
+        with tempfile.TemporaryDirectory() as td:
+            code, run = self.run_up(patch_out=str(Path(td) / "absent.conf"))
+        self.assertEqual(code, 2)
+        run.assert_not_called()
+
+    def test_invalid_target_stops_before_docker(self) -> None:
+        for bad in ("ftp://host:1", "host:8080", "http://"):
+            code, run = self.run_up(target=bad)
+            self.assertEqual(code, 2, bad)
+            run.assert_not_called()
+
+    def test_stack_reads_the_settings(self) -> None:
+        compose = (EDGE / "docker-compose.yml").read_text(encoding="utf-8")
+        caddy = (EDGE / "coraza" / "Caddyfile").read_text(encoding="utf-8")
+        self.assertIn("{$KOMAGU_UPSTREAM:http://mock-backend:5000}", caddy)
+        self.assertIn("KOMAGU_UPSTREAM: ${KOMAGU_UPSTREAM:-http://mock-backend:5000}", compose)
+        self.assertIn("${KOMAGU_KONG_CONFIG:-./kong/kong.yaml}:/kong/declarative/kong.yaml:ro", compose)
+        self.assertIn("${KOMAGU_VIRTUAL_PATCH_FILE:-./coraza/rules/virtual-patches.conf}:/etc/coraza/rules/virtual-patches.conf:ro", compose)
+        self.assertIn('"${KOMAGU_EDGE_BIND:-0.0.0.0}:8000:8000"', compose)
+        self.assertIn("host.docker.internal:host-gateway", compose)
+
+    def test_legacy_kong_profile_routes_everything_without_jwt(self) -> None:
+        text = (EDGE / "kong" / "kong-legacy-session.yaml").read_text(encoding="utf-8")
+        self.assertIn("- /\n", text)
+        self.assertIn("strip_path: false", text)
+        self.assertIn("name: rate-limiting", text)
+        self.assertIn("limit_by: ip", text)
+        self.assertNotIn("name: jwt", text)
+
+
+class VirtualPatchSessionAppTests(unittest.TestCase):
+    def test_bola_on_session_route_gets_no_authorization_rule(self) -> None:
+        """A missing Authorization header says nothing on a JSESSIONID app: the rule would block every user."""
+        text, manifest = VirtualPatcher().generate_patches_from_findings([
+            {"finding_id": "BOLA-R-1", "category": "bola", "endpoint": {"method": "ANY", "path": "/order/detail.do"}},
+        ])
+        self.assertEqual(manifest, [])
+        self.assertNotIn("REQUEST_HEADERS:Authorization", text)
+        self.assertIn("BOLA-R-1", text)
+
+    def test_bola_on_api_route_is_unchanged(self) -> None:
+        text, manifest = VirtualPatcher().generate_patches_from_findings([
+            {"finding_id": "BOLA-R-2", "category": "bola", "endpoint": {"method": "GET", "path": "/api/orders/{id}"}},
+        ])
+        self.assertEqual(len(manifest), 1)
+        self.assertIn("REQUEST_HEADERS:Authorization", text)
+
+    def test_injection_without_a_route_gets_no_rule_on_an_invented_path(self) -> None:
+        text, manifest = VirtualPatcher().generate_patches_from_findings([
+            {"finding_id": "INJ-I-1", "category": "sql_injection", "location": {"file": "A.java", "start_line": 3}},
+        ])
+        self.assertEqual(manifest, [])
+        self.assertNotIn('"@rx ^/api"', text)
+        self.assertIn("INJ-I-1", text)
+
+
+class FromScanLegacyRequestTests(unittest.TestCase):
+    def cases(self, path: str, method: str = "ANY", **attack_vector):
+        finding = {"finding_id": "BOLA-R-1", "category": "bola", "status": "confirmed", "title": "t",
+                   "endpoint": {"method": method, "path": path}}
+        if attack_vector:
+            finding["attack_vector"] = attack_vector
+        with tempfile.TemporaryDirectory() as td:
+            scan = Path(td) / "findings.json"
+            scan.write_text(json.dumps({"findings": [finding]}), encoding="utf-8")
+            return generate_test_cases_from_findings(str(scan), "101", "token")
+
+    def test_any_on_a_read_action_is_get_with_the_id_parameter(self) -> None:
+        case = self.cases("/order/detail.do")[0]
+        self.assertEqual((case.method, case.path), ("GET", "/order/detail.do?id=101"))
+
+    def test_any_on_a_write_action_is_a_form_post(self) -> None:
+        case = self.cases("/order/update.do")[0]
+        self.assertEqual(case.method, "POST")
+        self.assertEqual(case.payload["id"], "101")
+
+    def test_write_word_must_be_a_whole_word(self) -> None:
+        for path in ("/order/loadDetail.do", "/order/addressList.do", "/order/additionalInfo.do"):
+            self.assertEqual(self.cases(path)[0].method, "GET", path)
+        for path in ("/order/saveOrder.do", "/order/addItem.do", "/order/doUpdate.do", "/order/order_delete.do"):
+            self.assertEqual(self.cases(path)[0].method, "POST", path)
+
+    def test_target_param_names_the_id(self) -> None:
+        case = self.cases("/invoice/view.do", target_param="invoiceId")[0]
+        self.assertEqual(case.path, "/invoice/view.do?invoiceId=101")
+
+    def test_explicit_method_wins(self) -> None:
+        self.assertEqual(self.cases("/order/update.do", method="GET")[0].method, "GET")
+
+
+class AttackVectorFromRouteTests(unittest.TestCase):
+    def test_id_like_input_becomes_target_param(self) -> None:
+        route = {"id_inputs": [{"name": "page", "source": "any"}, {"name": "invoiceId", "source": "any"}]}
+        finding = HeadlessAIRunner._with_route_target({"finding_id": "B"}, route)
+        self.assertEqual(finding["attack_vector"]["target_param"], "invoiceId")
+
+    def test_model_value_is_kept(self) -> None:
+        route = {"id_inputs": [{"name": "invoiceId", "source": "any"}]}
+        finding = {"finding_id": "B", "attack_vector": {"target_param": "orderNo"}}
+        self.assertEqual(HeadlessAIRunner._with_route_target(finding, route)["attack_vector"]["target_param"], "orderNo")
+
+
+class FixDiffFormatTests(unittest.TestCase):
+    def test_manual_fix_findings_are_listed_when_no_diff_exists(self) -> None:
+        """All Java legacy findings are manual, so --format diff used to print nothing and exit 0 (#374)."""
+        with tempfile.TemporaryDirectory() as td:
+            root = Path(td)
+            (root / "src").mkdir()
+            (root / "src" / "detail.jsp").write_text("<% out.print(request.getParameter(\"id\")); %>\n", encoding="utf-8")
+            findings = root / "findings.json"
+            findings.write_text(json.dumps({"findings": [{
+                "finding_id": "BOLA-R-7", "category": "bola", "status": "confirmed", "severity": "high",
+                "subtype": "missing_ownership_check", "title": "t",
+                "location": {"file": "src/detail.jsp", "start_line": 1},
+                "endpoint": {"method": "GET", "path": "/detail.jsp"}}]}), encoding="utf-8")
+            res = subprocess.run([sys.executable, str(REPO_ROOT / "bin" / "komagu"), "fix", str(root), "--findings",
+                                  str(findings), "--dry-run", "--format", "diff"], capture_output=True, text=True)
+        self.assertEqual(res.returncode, 0, res.stderr)
+        self.assertIn("BOLA-R-7", res.stderr)
+        self.assertIn("src/detail.jsp", res.stderr)
+
+
+class MergeReportsDroppedFindingsTests(unittest.TestCase):
+    def test_validation_errors_reach_stderr(self) -> None:
+        with tempfile.TemporaryDirectory() as td:
+            out = Path(td)
+            (out / "findings").mkdir()
+            (out / "findings" / "bad.json").write_text('[{"some": "item"}]', encoding="utf-8")
+            res = subprocess.run([sys.executable, str(REPO_ROOT / "assess/lsa/scripts/merge_findings.py"), str(out)],
+                                 capture_output=True, text=True)
+        self.assertEqual(res.returncode, 1)
+        self.assertIn("bad.json", res.stderr)
+
+
+if __name__ == "__main__":
+    unittest.main()