@hoanghainh/komagu 2.4.0 → 2.5.0

package/operate/cross_user_tester/session_http.py

@@ -0,0 +1,288 @@
+"""HTTP and verdict helpers for session-based legacy apps (JSP / Struts / old Spring MVC).
+
+Redirects are never followed here: a 302 to the login page is the answer to
+evaluate, and following it would turn a refusal into a 200. Cookies and CSRF
+tokens stay inside the request and are never put in a reply, reason or error.
+
+Python standard library only.
+"""
+
+from __future__ import annotations
+
+import json
+import re
+import urllib.error
+import urllib.parse
+import urllib.request
+from dataclasses import dataclass
+from html.parser import HTMLParser
+from typing import Any, Dict, List, Mapping, NamedTuple, Optional, Tuple
+
+ALLOWED = "allowed"
+DENIED = "denied"
+INCONCLUSIVE = "inconclusive"
+
+DEFAULT_DENY_STATUS = (401, 403, 404)
+STRICT_DENY_STATUS = (404,)
+BODY_FORMATS = ("json", "form")
+WRITE_METHODS = ("POST", "PUT", "PATCH", "DELETE")
+MAX_BODY_BYTES = 1_000_000
+FORM_CONTENT_TYPE = "application/x-www-form-urlencoded"
+
+_FORM_ACTION_SUFFIXES = (".do", ".action", ".jsp", ".jspx")
+_STRUTS2_NAME_FIELD = "struts.token.name"
+_CHARSET = re.compile(r"charset\s*=\s*[\"']?([A-Za-z0-9_.:-]+)", re.IGNORECASE)
+# Pages labelled Shift_JIS are usually cp932 in practice, and Python has no windows-31j alias.
+_CHARSET_ALIASES = {
+    "windows-31j": "cp932",
+    "ms932": "cp932",
+    "shift_jis": "cp932",
+    "shift-jis": "cp932",
+    "sjis": "cp932",
+    "x-sjis": "cp932",
+}
+
+
+class SessionConfigError(ValueError):
+    """Invalid session, form or CSRF configuration. Messages never contain secret values."""
+
+
+@dataclass(frozen=True)
+class CsrfConfig:
+    """Page to fetch and field to read. The token is fetched fresh for every write."""
+
+    page: str
+    field: str
+
+
+@dataclass(frozen=True)
+class DenialSignal:
+    """How this application says no, and optionally how it says yes."""
+
+    deny_status: Tuple[int, ...] = ()
+    deny_redirects: Tuple[str, ...] = ()
+    deny_markers: Tuple[str, ...] = ()
+    allow_redirects: Tuple[str, ...] = ()
+    allow_markers: Tuple[str, ...] = ()
+
+    def declared(self) -> bool:
+        return bool(self.deny_status or self.deny_redirects or self.deny_markers)
+
+
+class HttpReply(NamedTuple):
+    status: int
+    location: str
+    body: str
+    error: Optional[str]
+
+
+class SessionRequest(NamedTuple):
+    method: str
+    path: str
+    cookie: Optional[str] = None
+    auth_header: Optional[Tuple[str, str]] = None
+    payload: Optional[Dict[str, Any]] = None
+    body_format: str = "json"
+    csrf: Optional[CsrfConfig] = None
+    headers: Optional[Dict[str, str]] = None
+
+
+def is_form_action(path: str) -> bool:
+    """True for routes shaped like a Java legacy form action (/order/update.do)."""
+    return path.split("?", 1)[0].lower().endswith(_FORM_ACTION_SUFFIXES)
+
+
+def parse_body_format(raw: Any, where: str, default: str = "json") -> str:
+    if raw is None:
+        return default
+    if raw not in BODY_FORMATS:
+        raise SessionConfigError(f"{where}: body_format must be one of {list(BODY_FORMATS)}")
+    return str(raw)
+
+
+def parse_csrf(raw: Any, where: str, default: Optional[CsrfConfig] = None) -> Optional[CsrfConfig]:
+    if raw is None:
+        return default
+    if not isinstance(raw, dict) or set(raw) != {"page", "field"}:
+        raise SessionConfigError(f"{where}: csrf must be an object with 'page' and 'field'")
+    page, field_name = raw["page"], raw["field"]
+    if not isinstance(page, str) or not page.startswith("/") or not isinstance(field_name, str) or not field_name:
+        raise SessionConfigError(f"{where}: csrf.page must start with '/' and csrf.field must be a name")
+    return CsrfConfig(page=page, field=field_name)
+
+
+def encode_form(payload: Mapping[str, Any]) -> bytes:
+    """Encode a flat mapping as a form body. Raises ValueError on nested values."""
+    for key, value in payload.items():
+        if isinstance(value, dict) or (isinstance(value, list) and any(isinstance(v, (dict, list)) for v in value)):
+            raise ValueError(f"form field '{key}' must be a plain value")
+    return urllib.parse.urlencode(payload, doseq=True).encode("ascii")
+
+
+def decode_body(raw: bytes, content_type: str) -> str:
+    """Decode using the declared charset, so markers in Japanese pages can match."""
+    found = _CHARSET.search(content_type or "") or _CHARSET.search(raw[:2048].decode("ascii", "ignore"))
+    if found:
+        name = found.group(1).lower()
+        try:
+            return raw.decode(_CHARSET_ALIASES.get(name, name), errors="replace")
+        except LookupError:
+            pass
+    # Nothing usable declared: legacy Japanese pages are often cp932 with no charset at all.
+    for encoding in ("utf-8", "cp932"):
+        try:
+            return raw.decode(encoding)
+        except UnicodeDecodeError:
+            continue
+    return raw.decode("utf-8", errors="replace")
+
+
+class _FieldCollector(HTMLParser):
+    def __init__(self) -> None:
+        super().__init__(convert_charrefs=True)
+        self.fields: List[Tuple[str, str]] = []
+
+    def handle_starttag(self, tag: str, attrs: List[Tuple[str, Optional[str]]]) -> None:
+        values = dict(attrs)
+        name = values.get("name")
+        if not name:
+            return
+        if tag == "input":
+            self.fields.append((name, values.get("value") or ""))
+        elif tag == "meta":
+            self.fields.append((name, values.get("content") or ""))
+
+
+def extract_form_token(html: str, field_name: str) -> Optional[Dict[str, str]]:
+    """Fields to send for a token, or None when the page does not carry it.
+
+    Struts 2 ``<s:token>`` needs a second field naming the token field, so it
+    is returned too.
+    """
+    collector = _FieldCollector()
+    collector.feed(html)
+    found = dict(reversed(collector.fields))
+    token = found.get(field_name)
+    if not token:
+        return None
+    if found.get(_STRUTS2_NAME_FIELD) == field_name:
+        return {_STRUTS2_NAME_FIELD: field_name, field_name: token}
+    return {field_name: token}
+
+
+def safe_location(location: str) -> str:
+    """Redirect target fit for a report: no ;jsessionid=, query string or fragment."""
+    return re.split(r"[;?#]", location, maxsplit=1)[0]
+
+
+def _matches(location: str, patterns: Tuple[str, ...]) -> bool:
+    # Patterns without '?' look at the path only, so /login.do?next=/list.do is not /list.do.
+    return any(p in (location if "?" in p else location.split("?", 1)[0]) for p in patterns)
+
+
+def _both_signals(where: str) -> str:
+    return f"{where} matches both a denial and an allow signal"
+
+
+def _classify_redirect(signal: DenialSignal, status: int, location: str) -> Tuple[str, str]:
+    denied, allowed = _matches(location, signal.deny_redirects), _matches(location, signal.allow_redirects)
+    target = safe_location(location) or "(no Location)"
+    if denied and allowed:
+        return INCONCLUSIVE, _both_signals(f"redirect to {target}")
+    if denied:
+        return DENIED, f"redirect to {target}"
+    if allowed:
+        return ALLOWED, f"redirect to {target}"
+    return INCONCLUSIVE, f"HTTP {status} redirect to {target} is not a declared denial or allow target"
+
+
+def _classify_page(signal: DenialSignal, body: str) -> Tuple[str, str]:
+    denied = any(m in body for m in signal.deny_markers)
+    allowed = any(m in body for m in signal.allow_markers)
+    if denied and allowed:
+        return INCONCLUSIVE, _both_signals("response body")
+    if denied:
+        return DENIED, "denial marker in response body"
+    if allowed or not signal.allow_markers:
+        return ALLOWED, "response served without a denial signal"
+    return INCONCLUSIVE, "response body has neither a denial nor an allow marker"
+
+
+def classify(signal: DenialSignal, status: int, location: str, body: str) -> Tuple[str, str]:
+    """Return (verdict, reason). Anything not positively identified is inconclusive."""
+    if status in signal.deny_status:
+        return DENIED, f"HTTP {status} is a declared denial status"
+    if 300 <= status < 400:
+        return _classify_redirect(signal, status, location)
+    if 200 <= status < 300:
+        return _classify_page(signal, body)
+    return INCONCLUSIVE, f"HTTP {status} is not a declared denial status"
+
+
+class _NoRedirect(urllib.request.HTTPRedirectHandler):
+    def redirect_request(self, *args: Any, **kwargs: Any) -> None:
+        return None
+
+
+def _send(url: str, method: str, headers: Dict[str, str], data: Optional[bytes], timeout: float) -> HttpReply:
+    if not url.lower().startswith(("http://", "https://")):
+        return HttpReply(0, "", "", "base URL must be an http:// or https:// URL")
+    request = urllib.request.Request(url, data=data, headers=headers, method=method)
+    opener = urllib.request.build_opener(_NoRedirect)
+    try:
+        # The scheme is checked above.
+        with opener.open(request, timeout=timeout) as resp:  # nosec B310
+            raw = resp.read(MAX_BODY_BYTES)
+            return HttpReply(resp.status, "", decode_body(raw, resp.headers.get("Content-Type", "")), None)
+    except urllib.error.HTTPError as err:
+        raw = err.read(MAX_BODY_BYTES)
+        return HttpReply(
+            err.code, err.headers.get("Location", ""), decode_body(raw, err.headers.get("Content-Type", "")), None
+        )
+    except Exception as ex:  # network errors are reported, not raised
+        return HttpReply(0, "", "", f"{type(ex).__name__}: request failed")
+
+
+def _base_headers(req: SessionRequest) -> Dict[str, str]:
+    headers: Dict[str, str] = dict(req.headers or {})
+    if req.cookie:
+        headers["Cookie"] = req.cookie
+    if req.auth_header:
+        headers[req.auth_header[0]] = req.auth_header[1]
+    return headers
+
+
+def fetch_page(base_url: str, path: str, req: SessionRequest, timeout: float) -> HttpReply:
+    """GET a page with the same credentials as ``req``."""
+    return _send(f"{base_url}{path}", "GET", _base_headers(req), None, timeout)
+
+
+def _csrf_fields(base_url: str, req: SessionRequest, timeout: float) -> Tuple[Dict[str, str], Optional[str]]:
+    assert req.csrf is not None
+    page = fetch_page(base_url, req.csrf.page, req, timeout)
+    if page.error or not 200 <= page.status < 300:
+        return {}, f"CSRF page {req.csrf.page} failed: {page.error or f'HTTP {page.status}'}"
+    fields = extract_form_token(page.body, req.csrf.field)
+    if fields is None:
+        return {}, f"CSRF token field '{req.csrf.field}' was not found on {req.csrf.page}"
+    return fields, None
+
+
+def deliver(base_url: str, req: SessionRequest, timeout: float) -> HttpReply:
+    """Send one request, adding a fresh CSRF token to writes when configured."""
+    payload = req.payload
+    if req.csrf is not None and req.method in WRITE_METHODS:
+        fields, problem = _csrf_fields(base_url, req, timeout)
+        if problem:
+            return HttpReply(0, "", "", problem)
+        payload = {**(payload or {}), **fields}
+
+    headers = _base_headers(req)
+    data: Optional[bytes] = None
+    if payload is not None:
+        try:
+            data = encode_form(payload) if req.body_format == "form" else json.dumps(payload).encode("utf-8")
+        except ValueError as err:
+            return HttpReply(0, "", "", str(err))
+        headers.setdefault("Content-Type", FORM_CONTENT_TYPE if req.body_format == "form" else "application/json")
+    return _send(f"{base_url}{req.path}", req.method, headers, data, timeout)

package/operate/devsecops/leak_check.py

@@ -0,0 +1,314 @@
+"""Post-run leak check for pilot outputs: ai-raw/, findings and reports (runbook §9).
+
+Four independent detectors, because the output must be checked for what the redaction step missed:
+client-provided canary strings, the redaction patterns, further token and credential patterns, and Shannon entropy.
+Text is also checked after JSON/backslash and HTML unescaping, so "password": "x" stored inside a JSON string is
+seen the way a reader would see it. Reported values are always masked. Standard-library only (Python 3.8+).
+
+A clean result is not proof that no secret remains: the detectors are heuristics, so Person B still reviews.
+"""
+
+from __future__ import annotations
+
+import base64
+import html
+import math
+import os
+import re
+import unicodedata
+from collections import Counter
+from dataclasses import dataclass
+from typing import Callable, Dict, Iterable, List, Pattern, Sequence, Set, Tuple
+from urllib.parse import quote
+
+from operate.ai.context import _load_redaction
+from operate.devsecops.pipeline import SECRET_PATTERNS
+
+_redaction = _load_redaction()
+MASK: str = _redaction.MASK
+MAX_LINE_CHARS: int = _redaction.MAX_LINE_CHARS
+_CHUNK_OVERLAP = 200
+MIN_CANARY_CHARS = 4
+MIN_ALIGNED_BASE64_CHARS = 8
+
+# (line number in the file, text); one physical line can become several entries after unescaping
+Entry = Tuple[int, str]
+
+_PRIVATE_KEY = re.compile(r"-----BEGIN (?:[A-Z0-9]+ )*PRIVATE KEY(?: BLOCK)?-----")
+_TOKEN_PATTERNS: Tuple[Tuple[Pattern[str], str], ...] = tuple(
+    (re.compile(pattern), name) for pattern, name in SECRET_PATTERNS if "PRIVATE KEY" not in pattern
+) + (
+    (re.compile(r"\beyJ[A-Za-z0-9_\-]{8,}\.eyJ[A-Za-z0-9_\-]{8,}\.[A-Za-z0-9_\-]+"), "JSON Web Token"),
+    (re.compile(r"\bAccountKey=[A-Za-z0-9+/]{20,}=*"), "Azure Storage Account Key"),
+    (re.compile(r"\b(?:sk|rk)_live_[A-Za-z0-9]{16,}"), "Stripe Live Key"),
+    # Japanese key names, which the English keyword patterns cannot see
+    (re.compile(r"(?:パスワード|暗証番号|秘密鍵|認証キー)\s*[:：=＝]\s*(?!\[REDACTED)[!-~]{4,}"), "Japanese-labelled credential"),
+)
+
+_TOKEN = re.compile(r"[A-Za-z0-9+/=_\-]{20,}")
+_HEX_LIKE = re.compile(r"[0-9a-fA-F\-]+")
+_SECRET_WORD = re.compile(r"(?:secret|token|passw|credential|api[_-]?key|private[_-]?key)", re.IGNORECASE)
+_HASH_WORD = re.compile(r"(?:fingerprint|sha\d*|hash|digest|md5|checksum|integrity|commit|etag)", re.IGNORECASE)
+_ENTROPY_RATIO = 0.85
+_MIN_DIGIT_TRANSITIONS = 3
+_DIGIT_TRANSITION_RATE = 0.12
+
+_BACKSLASH_ESCAPE = re.compile(r"\\(u[0-9a-fA-F]{4}|[\"'\\/nrtbf])")
+_BACKSLASH_MAP: Dict[str, str] = {"n": "\n", "r": "\n", "t": "\t", "b": " ", "f": " "}
+_UNESCAPE_ROUNDS = 3
+
+
+@dataclass(frozen=True)
+class Hit:
+    """One suspected leak. Never holds the value itself."""
+
+    path: str
+    line: int
+    detector: str
+    rule: str
+    masked: str
+
+
+@dataclass(frozen=True)
+class ScanResult:
+    hits: Tuple[Hit, ...]
+    files_scanned: int
+    unscanned: Tuple[Tuple[str, str], ...]
+    canaries_loaded: int
+
+    @property
+    def exit_code(self) -> int:
+        """1 on any hit; 2 when the scan is incomplete or saw nothing, so a bad path cannot pass as clean."""
+        if self.hits:
+            return 1
+        return 2 if self.unscanned or not self.files_scanned else 0
+
+
+class CanaryFileError(ValueError):
+    """The canary file cannot be used, so the check must not report a clean result."""
+
+
+def load_canaries(path: str) -> Tuple[str, ...]:
+    """Read one canary per line. Every non-blank line counts: a "#" comment syntax could hide a real value."""
+    try:
+        with open(path, "r", encoding="utf-8-sig") as handle:
+            canaries = tuple(line.strip() for line in handle if line.strip())
+    except (OSError, UnicodeDecodeError) as exc:
+        raise CanaryFileError(f"cannot read canary file: {exc}") from exc
+    if not canaries:
+        raise CanaryFileError("canary file has no entries")
+    short = [number for number, value in enumerate(canaries, 1) if len(value) < MIN_CANARY_CHARS]
+    if short:
+        raise CanaryFileError(f"canary entries shorter than {MIN_CANARY_CHARS} characters: {short}")
+    return canaries
+
+
+def _base64_forms(raw: bytes) -> Set[str]:
+    """The canary's base64 at each byte alignment, so it is found inside a longer encoded payload too."""
+    forms: Set[str] = set()
+    for offset in range(3):
+        encoded = base64.b64encode(b"\0" * offset + raw).decode().rstrip("=")
+        # The first and last characters also carry bits of the neighbouring bytes
+        form = encoded[math.ceil(offset * 4 / 3) :]
+        form = form[:-1] if (offset + len(raw)) % 3 else form
+        if offset == 0 or len(form) >= MIN_ALIGNED_BASE64_CHARS:
+            forms.update((form, form.replace("+", "-").replace("/", "_")))
+    return forms
+
+
+def _compile_canaries(canaries: Sequence[str]) -> Tuple[Tuple[Pattern[str], str], ...]:
+    """Each canary as typed, and as it appears once URL or HTML encoded (any case), or base64 encoded (exact)."""
+    compiled: List[Tuple[Pattern[str], str]] = []
+    for number, value in enumerate(canaries, 1):
+        parts = [re.escape(form) for form in (value, html.escape(value), quote(value, safe=""))]
+        rule = f"canary#{number}"
+        compiled.append((re.compile("|".join(parts), re.IGNORECASE), rule))
+        base64_forms = sorted(_base64_forms(value.encode("utf-8")))
+        compiled.append((re.compile("|".join(re.escape(form) for form in base64_forms)), rule))
+    return tuple(compiled)
+
+
+def shannon_entropy(token: str) -> float:
+    """Bits per character."""
+    length = len(token)
+    return -sum((count / length) * math.log2(count / length) for count in Counter(token).values())
+
+
+def _is_random_token(token: str, line: str) -> bool:
+    """Random-looking token that is not a hash, path or identifier."""
+    if not any(ch.isdigit() for ch in token) or not any(ch.isalpha() for ch in token):
+        return False
+    if _HEX_LIKE.fullmatch(token):
+        # Hashes fill audit output (fingerprints, checksums), so bare hex needs a secret word on the same line
+        return bool(_SECRET_WORD.search(line)) and not _HASH_WORD.search(line)
+    transitions = sum(1 for a, b in zip(token, token[1:]) if a.isdigit() != b.isdigit())
+    if transitions < max(_MIN_DIGIT_TRANSITIONS, _DIGIT_TRANSITION_RATE * len(token)):
+        return False
+    return shannon_entropy(token) >= _ENTROPY_RATIO * math.log2(len(token))
+
+
+def _masked(value: str) -> str:
+    return f"[MASKED len={len(value)}]"
+
+
+def _unescape_once(text: str) -> str:
+    def replace(match: "re.Match[str]") -> str:
+        escape = match.group(1)
+        if escape.startswith("u"):
+            return chr(int(escape[1:], 16))
+        return _BACKSLASH_MAP.get(escape, escape)
+
+    unescaped = html.unescape(_BACKSLASH_ESCAPE.sub(replace, text))
+    # Full-width forms and zero-width characters read the same to a person
+    normalized = unicodedata.normalize("NFKC", unescaped)
+    return "".join(ch for ch in normalized if unicodedata.category(ch) != "Cf")
+
+
+def _decoded_entries(lines: Sequence[str]) -> List[Entry]:
+    """Lines that change when unescaped, split as a reader would see them; handles double-encoded JSON."""
+    entries: List[Entry] = []
+    for number, line in enumerate(lines, 1):
+        current = line
+        for _ in range(_UNESCAPE_ROUNDS):
+            decoded = _unescape_once(current)
+            if decoded == current:
+                break
+            current = decoded
+        if current != line:
+            entries.extend((number, part) for part in current.split("\n"))
+    return entries
+
+
+def _bounded(entries: Iterable[Entry]) -> List[Entry]:
+    """Split very long lines into overlapping chunks: redact() would otherwise drop them whole."""
+    bounded: List[Entry] = []
+    step = MAX_LINE_CHARS - _CHUNK_OVERLAP
+    for number, text in entries:
+        if len(text) <= MAX_LINE_CHARS:
+            bounded.append((number, text))
+            continue
+        bounded.extend((number, text[start : start + MAX_LINE_CHARS]) for start in range(0, len(text), step))
+    return bounded
+
+
+def _find_canaries(entries: Sequence[Entry], compiled: Sequence[Tuple[Pattern[str], str]]) -> Iterable[Tuple[int, str, str, str]]:
+    for number, text in entries:
+        for pattern, rule in compiled:
+            for match in pattern.finditer(text):
+                yield number, "canary", rule, _masked(match.group(0))
+
+
+def _find_redaction_misses(entries: Sequence[Entry]) -> Iterable[Tuple[int, str, str, str]]:
+    masked_lines = _redaction.redact_lines([text for _, text in entries])
+    for (number, text), masked in zip(entries, masked_lines):
+        if text != masked:
+            yield number, "redaction-pattern", "value the redaction step would mask", "[MASKED]"
+
+
+def _find_tokens(entries: Sequence[Entry]) -> Iterable[Tuple[int, str, str, str]]:
+    for number, text in entries:
+        for pattern, name in _TOKEN_PATTERNS:
+            for match in pattern.finditer(text):
+                yield number, "token-pattern", name, _masked(match.group(0))
+
+
+def _find_private_keys(entries: Sequence[Entry]) -> Iterable[Tuple[int, str, str, str]]:
+    """A key header counts unless its body was masked, either on the same line or on the next one."""
+    for index, (number, text) in enumerate(entries):
+        for match in _PRIVATE_KEY.finditer(text):
+            rest = text[match.end() :].strip()
+            following = entries[index + 1][1].strip() if index + 1 < len(entries) else ""
+            if not (rest.startswith(MASK) or (not rest and following.startswith(MASK))):
+                yield number, "token-pattern", "Private Key", _masked(match.group(0))
+
+
+def _find_high_entropy(entries: Sequence[Entry]) -> Iterable[Tuple[int, str, str, str]]:
+    for number, text in entries:
+        for match in _TOKEN.finditer(text):
+            if _is_random_token(match.group(0), text):
+                yield number, "entropy", "high-entropy token", _masked(match.group(0))
+
+
+_DETECTORS: Tuple[Callable[[Sequence[Entry]], Iterable[Tuple[int, str, str, str]]], ...] = (
+    _find_redaction_misses,
+    _find_tokens,
+    _find_private_keys,
+    _find_high_entropy,
+)
+
+
+def scan_text(path: str, text: str, compiled: Sequence[Tuple[Pattern[str], str]] = ()) -> List[Hit]:
+    """Run every detector over the text as stored and as unescaped; repeated sightings are reported once."""
+    lines = text.splitlines()
+    streams = (
+        [(number, line) for number, line in enumerate(lines, 1)],
+        _decoded_entries(lines),
+    )
+    found: List[Tuple[int, str, str, str]] = []
+    for entries in streams:
+        found.extend(_find_canaries(entries, compiled))
+        bounded = _bounded(entries)
+        for detector in _DETECTORS:
+            found.extend(detector(bounded))
+    return [Hit(path, number, detector, rule, masked) for number, detector, rule, masked in sorted(set(found))]
+
+
+def _decode(data: bytes) -> str:
+    """Komagu writes UTF-8; copied source excerpts in Japanese systems are often Shift_JIS."""
+    if data.startswith((b"\xff\xfe", b"\xfe\xff")):
+        return data.decode("utf-16", errors="replace")
+    for encoding in ("utf-8-sig", "cp932"):
+        try:
+            return data.decode(encoding)
+        except UnicodeDecodeError:
+            continue
+    return data.decode("latin-1")
+
+
+def _walk(paths: Sequence[str], errors: List[Tuple[str, str]]) -> Iterable[str]:
+    """Every file below the paths, plus symlinked directories for _read to refuse. Hidden directories count."""
+
+    def record(error: OSError) -> None:
+        errors.append((str(error.filename), f"unreadable: {error.strerror}"))
+
+    for root in paths:
+        if not os.path.isdir(root) or os.path.islink(root):
+            yield root
+            continue
+        for directory, subdirs, names in os.walk(root, onerror=record):
+            subdirs.sort()
+            links = [name for name in subdirs if os.path.islink(os.path.join(directory, name))]
+            yield from (os.path.join(directory, name) for name in links + sorted(names))
+
+
+def _read(path: str) -> Tuple[str, str]:
+    """Return (text, "") or ("", reason not scanned)."""
+    if os.path.islink(path):
+        return "", "symbolic link"
+    if not os.path.isfile(path):
+        return "", "not found"
+    try:
+        with open(path, "rb") as handle:
+            data = handle.read()
+    except OSError as exc:
+        return "", f"unreadable: {exc.strerror}"
+    if b"\0" in data[:8192] and not data.startswith((b"\xff\xfe", b"\xfe\xff")):
+        return "", "binary file"
+    text = _decode(data)
+    return ("", "binary file") if "\0" in text else (text, "")
+
+
+def scan_paths(paths: Sequence[str], canaries: Sequence[str] = ()) -> ScanResult:
+    """Scan files and directories; anything that cannot be read is listed, never silently skipped."""
+    compiled = _compile_canaries(canaries)
+    hits: List[Hit] = []
+    unscanned: List[Tuple[str, str]] = []
+    files_scanned = 0
+    for path in _walk(paths, unscanned):
+        text, reason = _read(path)
+        if reason:
+            unscanned.append((path, reason))
+            continue
+        files_scanned += 1
+        hits.extend(scan_text(path, text, compiled))
+    return ScanResult(tuple(hits), files_scanned, tuple(unscanned), len(canaries))

package/operate/remediator/runner.py

@@ -128,6 +128,16 @@ def write_source_file_safe(file_path: Path, content: str, encoding: str, newline
     file_path.write_bytes(encode_source(content, encoding, newline))
 
 
+def line_after_earlier_patches(pre_patch: str, current: str, line: int) -> int:
+    """Map a 1-based line of the unpatched file (what findings refer to) into the patched content."""
+    idx = line - 1
+    matcher = difflib.SequenceMatcher(None, pre_patch.splitlines(), current.splitlines(), autojunk=False)
+    for _, i1, i2, j1, j2 in matcher.get_opcodes():
+        if i1 <= idx < i2 and j2 > j1:
+            return j1 + min(idx - i1, j2 - j1 - 1) + 1
+    return line
+
+
 class RemediationRunner:
     """Coordinates automated code patching based on security audit findings."""
 
@@ -178,6 +188,7 @@ class RemediationRunner:
         self.sync_lockfiles = sync_lockfiles
         self.registry = StrategyRegistry()
         self._virtual_contents: Dict[Path, str] = {}
+        self._pre_patch_contents: Dict[Path, str] = {}
 
     def discover_findings_file(self) -> Path:
         """Locate findings.json in default known output paths if not explicitly provided."""
@@ -365,7 +376,11 @@ class RemediationRunner:
                     ),
                 )
             original_content = self._virtual_contents.get(abs_file, disk_content)
-            patched_content, explanation = strategy.generate_patch(abs_file, original_content, finding)
+            current_line = start_line
+            if abs_file in self._virtual_contents:
+                current_line = line_after_earlier_patches(self._pre_patch_contents[abs_file], original_content, start_line)
+            located_finding = {**finding, "location": {**loc, "start_line": current_line}}
+            patched_content, explanation = strategy.generate_patch(abs_file, original_content, located_finding)
 
             if patched_content == original_content:
                 return RemediationPlan(
@@ -385,6 +400,7 @@ class RemediationRunner:
                 )
 
             self._virtual_contents[abs_file] = patched_content
+            self._pre_patch_contents.setdefault(abs_file, original_content)
 
             # Generate unified diff
             orig_lines = original_content.splitlines(keepends=True)
@@ -469,6 +485,7 @@ class RemediationRunner:
     def run(self) -> RemediationReport:
         """Execute the remediation workflow."""
         self._virtual_contents.clear()
+        self._pre_patch_contents.clear()
         findings = self.load_findings()
 
         # Track (relative_file, finding_id) already queued so companion lockfiles are not duplicated

package/operate/remediator/strategies.py

@@ -98,12 +98,12 @@ class BOLARemediationStrategy(BaseRemediationStrategy):
 
         # Use intelligent BOLA ownership patcher
         patcher = BOLAOwnershipPatcher()
-        patched_content, result = patcher.patch_file(target_file, start_line + 1, finding)
+        patched_content, result = patcher.patch_file(target_file, start_line + 1, finding, content=content)
         if result.changed:
             return patched_content, result.explanation
 
         # Do NOT inject comment-only fake fixes. Require genuine code rewrite or manual fix.
-        return content, "No automated BOLA ownership rewrite applicable; requires manual remediation."
+        return content, result.explanation or "No automated BOLA ownership rewrite applicable; requires manual remediation."
 
 
 class DependencyVulnerabilityRemediationStrategy(BaseRemediationStrategy):