@hoanghainh/komagu 2.3.0 → 2.5.0

package/operate/simulate/scenarios.py

@@ -379,7 +379,7 @@ class IDCFrontierRansomwareScenario(BaseScenario):
             self.target_dir.mkdir(parents=True, exist_ok=True)
             test_file.write_text("Komagu Immutable WORM Backup - Read Only", encoding="utf-8")
             # Mark file read-only (simulating local WORM compliance mode)
-            os.chmod(str(test_file), 0o444)
+            os.chmod(str(test_file), 0o400)
         except Exception:
             pass
 
@@ -416,7 +416,7 @@ class IDCFrontierRansomwareScenario(BaseScenario):
 
         # Clean up
         try:
-            os.chmod(str(test_file), 0o666)
+            os.chmod(str(test_file), 0o600)
             test_file.unlink(missing_ok=True)
         except Exception:
             pass

package/operate/tests/mock_legacy_session_app.py

@@ -0,0 +1,160 @@
+#!/usr/bin/env python3
+"""Small session-based app that behaves like a JSP / Struts application.
+
+Used to rehearse ``komagu test`` on legacy behaviour:
+
+- login state lives in a ``JSESSIONID`` cookie;
+- writes are ``application/x-www-form-urlencoded`` and need a single-use Struts token;
+- a successful write answers 302 (post/redirect/get);
+- a denied request is a 200 Shift_JIS error page, a 302 to login, or a 404,
+  depending on ``server.denial``;
+- ``server.fixed`` is the ownership check being deployed.
+"""
+
+from __future__ import annotations
+
+import secrets
+import threading
+from http.server import BaseHTTPRequestHandler, HTTPServer
+from typing import Dict, List, NamedTuple, Optional, Tuple
+from urllib.parse import parse_qs, urlsplit
+
+ALICE_COOKIE = "JSESSIONID=sess-alice-0001"
+BOB_COOKIE = "JSESSIONID=sess-bob-0002"
+EXPIRED_COOKIE = "JSESSIONID=sess-expired-0003"
+TOKEN_FIELD = "org.apache.struts.taglib.html.TOKEN"
+DENIED_TEXT = "アクセス権限がありません"
+LOGIN_REDIRECT = "/login.do;jsessionid=sess-rewritten-url-9999"
+
+
+class Seen(NamedTuple):
+    method: str
+    path: str
+    cookie: Optional[str]
+    authorization: Optional[str]
+    content_type: Optional[str]
+    body: str
+
+
+class LegacyHandler(BaseHTTPRequestHandler):
+    server: "LegacyServer"
+
+    def log_message(self, format: str, *args: object) -> None:  # noqa: A002
+        pass
+
+    def _user(self) -> Optional[str]:
+        cookie = self.headers.get("Cookie", "")
+        pairs = dict(p.strip().split("=", 1) for p in cookie.split(";") if "=" in p)
+        return self.server.sessions.get(pairs.get("JSESSIONID", ""))
+
+    def _send(self, status: int, body: bytes = b"", charset: str = "UTF-8", location: Optional[str] = None) -> None:
+        self.send_response(status)
+        self.send_header("Content-Type", f"text/html; charset={charset}")
+        self.send_header("Content-Length", str(len(body)))
+        if location:
+            self.send_header("Location", location)
+        self.end_headers()
+        self.wfile.write(body)
+
+    def _to_login(self) -> None:
+        self._send(302, location=LOGIN_REDIRECT)
+
+    def _deny(self) -> None:
+        if self.server.denial == "login_redirect":
+            self._to_login()
+        elif self.server.denial == "not_found":
+            self._send(404, b"<html>Not Found</html>")
+        else:
+            page = f"<html><body><p>{DENIED_TEXT}</p></body></html>"
+            self._send(200, page.encode("cp932"), charset="Windows-31J")
+
+    def _may_access(self, user: str, order_id: str) -> bool:
+        owner = self.server.orders.get(order_id)
+        if owner is None:
+            return False
+        return owner == user or not self.server.fixed
+
+    def _record(self, method: str, body: str = "") -> None:
+        self.server.seen.append(Seen(
+            method, self.path, self.headers.get("Cookie"), self.headers.get("Authorization"),
+            self.headers.get("Content-Type"), body,
+        ))
+
+    def do_GET(self) -> None:  # noqa: N802
+        self._record("GET")
+        url = urlsplit(self.path)
+        query = parse_qs(url.query)
+        if url.path == "/login.do":
+            self._send(200, b"<html>Please log in</html>")
+            return
+        user = self._user()
+        if user is None:
+            self._to_login()
+        elif url.path == "/home.do":
+            self._send(200, f"<html>Welcome {user}</html>".encode())
+        elif url.path == "/order/view.do":
+            self._view(user, query.get("id", [""])[0])
+        elif url.path == "/order/edit.do":
+            self._edit_form(user)
+        else:
+            self._send(404, b"<html>No such action</html>")
+
+    def _view(self, user: str, order_id: str) -> None:
+        if not self._may_access(user, order_id):
+            self._deny()
+            return
+        self._send(200, f"<html>Order detail {order_id}: secret-note</html>".encode())
+
+    def _edit_form(self, user: str) -> None:
+        if self.server.hide_token:
+            self._send(200, b"<html><form></form></html>")
+            return
+        token = secrets.token_hex(8)
+        self.server.tokens[user] = token
+        # value before name, as some tag libraries render it.
+        page = f'<html><form><input value="{token}" type="hidden" name="{TOKEN_FIELD}"></form></html>'
+        self._send(200, page.encode())
+
+    def do_POST(self) -> None:  # noqa: N802
+        length = int(self.headers.get("Content-Length") or 0)
+        raw = self.rfile.read(length).decode("utf-8", "replace")
+        self._record("POST", raw)
+        user = self._user()
+        if user is None:
+            self._to_login()
+            return
+        if self.headers.get("Content-Type", "").split(";")[0] != "application/x-www-form-urlencoded":
+            self._send(415, b"<html>Form post expected</html>")
+            return
+        form = parse_qs(raw)
+        if urlsplit(self.path).path != "/order/update.do":
+            self._send(404, b"<html>No such action</html>")
+            return
+        if form.get(TOKEN_FIELD, [""])[0] != self.server.tokens.pop(user, None):
+            self._send(400, b"<html>Invalid token</html>")
+            return
+        order_id = form.get("id", [""])[0]
+        if not self._may_access(user, order_id):
+            self._deny()
+            return
+        self.server.updates.append((user, order_id))
+        self._send(302, location="/order/list.do")
+
+
+class LegacyServer(HTTPServer):
+    def __init__(self) -> None:
+        super().__init__(("127.0.0.1", 0), LegacyHandler)
+        self.fixed = False
+        self.denial = "error_page"
+        self.hide_token = False
+        self.orders: Dict[str, str] = {"100": "alice", "200": "bob"}
+        self.sessions: Dict[str, str] = {"sess-alice-0001": "alice", "sess-bob-0002": "bob"}
+        self.tokens: Dict[str, str] = {}
+        self.updates: List[Tuple[str, str]] = []
+        self.seen: List[Seen] = []
+
+
+def start_legacy_app() -> Tuple[LegacyServer, str]:
+    server = LegacyServer()
+    threading.Thread(target=server.serve_forever, daemon=True).start()
+    return server, f"http://127.0.0.1:{server.server_port}"

package/operate/tests/test_ai_redaction.py

@@ -0,0 +1,181 @@
+#!/usr/bin/env python3
+"""Secrets are masked in every code snippet the headless agents send to the model (#350)."""
+
+from __future__ import annotations
+
+import shutil
+import tempfile
+import unittest
+from pathlib import Path
+
+import time
+
+from operate.ai.context import _model_source, _read_snippet, _redact_lines, _verification_context
+
+JAVA_DAO = '''package com.acme.dao;
+public class AccountDao {
+    private static final String URL = "jdbc:sqlserver://db01;databaseName=ams;user=sa;password=Ams#2019!";
+    private static final String MONGO = "mongodb://svc:Mongo2020pw@mongo01:27017/ams";
+    private static final String ORA = "jdbc:oracle:thin:scott/Tiger2018@db02:1521/AMS";
+    // db.password=CommentPw7 (old config, kept for reference)
+    private String apiKey = "sk-live-abcdef123456";
+    public Account find(HttpServletRequest req) {
+        String password = req.getParameter("password");
+        String sql = "SELECT * FROM account WHERE id = " + req.getParameter("id");
+        return jdbc.query(sql);
+    }
+}
+'''
+
+PROPERTIES = '''spring.datasource.url=jdbc:oracle:thin:@db02:1521/AMS
+spring.datasource.username=ams_app
+spring.datasource.password=Oracle#Pw2021
+mail.smtp.password: smtpPw99
+app.jwt.secret=c2VjcmV0LWtleS1mb3ItYW1z
+app.token.placeholder=${APP_TOKEN}
+'''
+
+PRIVATE_KEY = '''-----BEGIN RSA PRIVATE KEY-----
+MIIEowIBAAKCAQEAwJ1rdTw6kLqA
+Zm9vYmFyYmF6cXV4cXV1eHF1dXg=
+-----END RSA PRIVATE KEY-----
+class Signer {}
+'''
+
+JSP = '''<%-- legacy: ldap.bind.password=LdapPw55 --%>
+<%@ page import="java.sql.*" %>
+<% String id = request.getParameter("id"); %>
+'''
+
+SECRETS = ("Ams#2019!", "Mongo2020pw", "Tiger2018", "CommentPw7", "sk-live-abcdef123456", "Oracle#Pw2021", "smtpPw99",
+           "c2VjcmV0LWtleS1mb3ItYW1z", "MIIEowIBAAKCAQEAwJ1rdTw6kLqA", "Zm9vYmFyYmF6cXV4cXV1eHF1dXg=", "LdapPw55")
+
+
+class TestSnippetRedaction(unittest.TestCase):
+    def setUp(self) -> None:
+        self.target = Path(tempfile.mkdtemp(prefix="komagu_redact_"))
+        files = {"src/com/acme/dao/AccountDao.java": JAVA_DAO, "src/main/resources/application.properties": PROPERTIES,
+                 "src/com/acme/Signer.java": PRIVATE_KEY, "WebContent/user/detail.jsp": JSP}
+        for rel, text in files.items():
+            (self.target / rel).parent.mkdir(parents=True, exist_ok=True)
+            (self.target / rel).write_text(text, encoding="utf-8")
+
+    def tearDown(self) -> None:
+        shutil.rmtree(self.target, ignore_errors=True)
+
+    def assert_no_secret(self, text: str) -> None:
+        for secret in SECRETS:
+            self.assertNotIn(secret, text)
+
+    def test_java_source_snippet_masks_connection_strings_and_keys(self) -> None:
+        snip = _read_snippet(self.target, "src/com/acme/dao/AccountDao.java", 5)
+        self.assert_no_secret(snip)
+        self.assertIn("[REDACTED]", snip)
+
+    def test_code_the_auditor_needs_is_kept(self) -> None:
+        snip = _read_snippet(self.target, "src/com/acme/dao/AccountDao.java", 5)
+        self.assertIn('String password = req.getParameter("password");', snip)
+        self.assertIn('"SELECT * FROM account WHERE id = " + req.getParameter("id")', snip)
+        self.assertIn("jdbc:sqlserver://db01;databaseName=ams;user=sa;password=[REDACTED]", snip)
+        self.assertIn("mongodb://svc:[REDACTED]@mongo01:27017/ams", snip)
+        self.assertIn("jdbc:oracle:thin:scott/[REDACTED]@db02:1521/AMS", snip)
+
+    def test_config_line_inside_a_comment_is_masked(self) -> None:
+        snip = _read_snippet(self.target, "WebContent/user/detail.jsp", 2)
+        self.assert_no_secret(snip)
+        self.assertIn("ldap.bind.password=[REDACTED] --%>", snip)
+        self.assertIn('String id = request.getParameter("id");', snip)
+
+    def test_assignments_in_code_are_kept(self) -> None:
+        lines = ["user.password = req.body.password;", "if (token == null) return;", "token: req.headers.authorization,"]
+        (self.target / "app.js").write_text("\n".join(lines) + "\n", encoding="utf-8")
+        snip = _read_snippet(self.target, "app.js", 2)
+        for line in lines:
+            self.assertIn(line, snip)
+
+    def test_unquoted_properties_values_are_masked(self) -> None:
+        snip = _read_snippet(self.target, "src/main/resources/application.properties", 3)
+        self.assert_no_secret(snip)
+        self.assertIn("spring.datasource.username=ams_app", snip)
+        self.assertIn("app.token.placeholder=${APP_TOKEN}", snip)
+
+    def test_private_key_block_is_masked_with_line_numbers_kept(self) -> None:
+        snip = _read_snippet(self.target, "src/com/acme/Signer.java", 3)
+        self.assert_no_secret(snip)
+        self.assertIn("   5 | class Signer {}", snip)
+
+    def test_model_and_verifier_context_are_masked(self) -> None:
+        model = _model_source(self.target, "AccountDao", "src/com/acme/dao/AccountDao.java", 1)
+        self.assert_no_secret(model["code"])
+        ctx = _verification_context(self.target, {"location": {"file": "src/com/acme/dao/AccountDao.java", "start_line": 8},
+                                                  "evidence": [{"file": "src/main/resources/application.properties", "line": 3}]})
+        self.assert_no_secret(str(ctx))
+
+
+class TestJavaLegacySecretForms(unittest.TestCase):
+    """Forms found in a security review of the first version (#350); each must hide Hunter2x."""
+
+    LEAKS = (
+        '<property name="password" value="Hunter2x"/>',
+        '<property name="jdbc.password"\n          value="Hunter2x"/>',
+        "<param-name>db.password</param-name><param-value>Hunter2x</param-value>",
+        "<init-param>\n  <param-name>password</param-name>\n  <param-value>Hunter2x</param-value>",
+        "<password>Hunter2x</password>",
+        'conn.setPassword("Hunter2x");',
+        'props.setProperty("mail.smtp.password", "Hunter2x");',
+        'Connection c = DriverManager.getConnection(url, "sa", "Hunter2x");',
+        'return new PasswordAuthentication("svc", "Hunter2x".toCharArray());',
+        'String pw = "Hunter2x";',
+        'static final String DB_PASS = "Hunter2x";',
+        '<Connector port="8443" keystorePass="Hunter2x" truststorePass="Hunter2x"/>',
+        '{"password":"Hunter2x", "apikey" : "Hunter2x"}',
+        "$conf = array('password' => 'Hunter2x');",
+        "db_password: Hunter2x  # prod",
+        "mail.smtp.pass=Hunter2x",
+        "ssl.key.passphrase=Hunter2x",
+        "jdbc:sqlserver://db;user=sa;password={Hunter2x;x}",
+        'conn.setRequestProperty("Authorization", "Basic SHVudGVyMng6eHl6");',
+        'h.put("Authorization", "Bearer Hunter2xHunter2x.eyJzdWIiOiIxIn0.sig");',
+        "-----BEGIN PRIVATE KEY----- Hunter2xMIIEvQIBADANBgkq -----END PRIVATE KEY-----",
+    )
+    TOKENS = ("AKIAIOSFODNN7EXAMPLE", "ghp_" + "a" * 36, "sk-ant-api03-" + "b" * 40, "AIza" + "c" * 35)
+    KEPT = (
+        'String password = req.getParameter("password");',
+        "String sql = \"SELECT * FROM users WHERE login='\" + u + \"' AND pwd='\" + p + \"'\";",
+        "<select id=\"login\">SELECT * FROM users WHERE pwd = #{password}</select>",
+        "spring.datasource.password=${DB_PASS}",
+        "user.password = req.body.password;",
+        "if (token == null) return;",
+        'String header = "Bearer " + token;',
+        '<property name="username" value="ams_app"/>',
+        "Passenger p = passengerDao.find(id);",
+    )
+
+    def redact(self, text: str) -> str:
+        return "\n".join(_redact_lines(text.split("\n")))
+
+    def test_java_legacy_forms_are_masked(self) -> None:
+        for sample in self.LEAKS:
+            with self.subTest(sample=sample):
+                self.assertNotIn("Hunter2x", self.redact(sample))
+
+    def test_cloud_and_vcs_tokens_are_masked(self) -> None:
+        for token in self.TOKENS:
+            with self.subTest(token=token[:8]):
+                self.assertNotIn(token, self.redact(f'String key = cfg.get("{token}");'))
+
+    def test_code_the_auditor_needs_is_kept(self) -> None:
+        for line in self.KEPT:
+            with self.subTest(line=line):
+                self.assertEqual(self.redact(line), line)
+
+    def test_long_lines_are_fast_and_not_sent(self) -> None:
+        lines = ["a.b" * 300000 + "password", "a-" * 500000 + "://", "a-b." * 250000, "x" * 1000000]
+        start = time.monotonic()
+        out = _redact_lines(lines)
+        self.assertLess(time.monotonic() - start, 2.0)
+        self.assertTrue(all(o.startswith("[line of ") for o in out))
+
+
+if __name__ == "__main__":
+    unittest.main()