@oriro/orirocli 0.1.9 → 0.1.12

package/skills/graphify/security.py

@@ -1,336 +0,0 @@
-# Security helpers - URL validation, safe fetch, path guards, label sanitisation
-from __future__ import annotations
-
-import contextlib
-import html
-import re
-import urllib.error
-import urllib.parse
-import urllib.request
-from collections.abc import Mapping
-from pathlib import Path
-from typing import Any
-
-import ipaddress
-import socket
-
-_ALLOWED_SCHEMES = {"http", "https"}
-_MAX_FETCH_BYTES = 52_428_800   # 50 MB hard cap for binary downloads
-_MAX_TEXT_BYTES  = 10_485_760   # 10 MB hard cap for HTML / text
-
-# Graph-load memory-bomb cap: reject .json files larger than this before
-# JSON-parsing them into a dict. Without this, a multi-gigabyte (or
-# specifically crafted) graph.json can exhaust process memory during
-# json.loads + node_link_graph rehydration.
-_MAX_GRAPH_FILE_BYTES = 512 * 1024 * 1024   # 512 MiB
-
-# AWS metadata, link-local, and common cloud metadata endpoints
-_BLOCKED_HOSTS = {"metadata.google.internal", "metadata.google.com"}
-
-# RFC 6598 Shared Address Space (CGN) -- is_private misses this on Python <3.11
-_CGN_NETWORK = ipaddress.ip_network("100.64.0.0/10")
-
-# RFC 6052 NAT64 Well-Known Prefix -- is_reserved=True in Python but these embed
-# public IPv4 addresses and are legitimate public internet traffic, not SSRF vectors.
-_NAT64_WKP = ipaddress.ip_network("64:ff9b::/96")
-
-
-# ---------------------------------------------------------------------------
-# URL validation
-# ---------------------------------------------------------------------------
-
-def validate_url(url: str) -> str:
-    """Raise ValueError if *url* is not http or https, or targets a private/internal IP.
-
-    Blocks file://, ftp://, data:, and any other scheme that could be used
-    for SSRF or local file access. Also blocks requests to private/reserved
-    IP ranges (127.x, 10.x, 169.254.x, etc.) and cloud metadata endpoints
-    to prevent SSRF in cloud environments.
-    """
-    parsed = urllib.parse.urlparse(url)
-    if parsed.scheme.lower() not in _ALLOWED_SCHEMES:
-        raise ValueError(
-            f"Blocked URL scheme '{parsed.scheme}' - only http and https are allowed. "
-            f"Got: {url!r}"
-        )
-
-    hostname = parsed.hostname
-    if hostname:
-        # Block known cloud metadata hostnames
-        if hostname.lower() in _BLOCKED_HOSTS:
-            raise ValueError(
-                f"Blocked cloud metadata endpoint '{hostname}'. "
-                f"Got: {url!r}"
-            )
-
-        # Resolve hostname and block private/reserved IP ranges
-        try:
-            infos = socket.getaddrinfo(hostname, None, socket.AF_UNSPEC, socket.SOCK_STREAM)
-            for info in infos:
-                addr = info[4][0]
-                ip = ipaddress.ip_address(addr)
-                # For NAT64 addresses, check the embedded IPv4 instead of the wrapper
-                if isinstance(ip, ipaddress.IPv6Address) and ip in _NAT64_WKP:
-                    embedded = ipaddress.ip_address(int(ip) & 0xFFFFFFFF)
-                    ip = embedded
-                if ip.is_private or ip.is_reserved or ip.is_loopback or ip.is_link_local or ip in _CGN_NETWORK:
-                    raise ValueError(
-                        f"Blocked private/internal IP {addr} (resolved from '{hostname}'). "
-                        f"Got: {url!r}"
-                    )
-        except socket.gaierror as exc:
-            raise ValueError(
-                f"DNS resolution failed for '{hostname}': {exc}. Got: {url!r}"
-            ) from exc
-
-    return url
-
-
-@contextlib.contextmanager
-def _ssrf_guarded_socket():
-    """Patch socket.getaddrinfo for the duration of a fetch to catch DNS rebinding.
-
-    Validates every IP that urllib resolves so a DNS server cannot return a public IP
-    for validate_url and swap to a private IP for the actual connection (TOCTOU fix).
-    Not thread-safe, but graphify is a single-threaded CLI tool.
-    """
-    original = socket.getaddrinfo
-
-    def _guarded(host, port, *args, **kwargs):
-        results = original(host, port, *args, **kwargs)
-        for info in results:
-            addr = info[4][0]
-            try:
-                ip = ipaddress.ip_address(addr)
-            except ValueError:
-                continue
-            if ip.is_private or ip.is_reserved or ip.is_loopback or ip.is_link_local or ip in _CGN_NETWORK:
-                raise OSError(
-                    f"SSRF blocked: IP {addr} resolved from '{host}' is private/reserved"
-                )
-        return results
-
-    socket.getaddrinfo = _guarded
-    try:
-        yield
-    finally:
-        socket.getaddrinfo = original
-
-
-class _NoFileRedirectHandler(urllib.request.HTTPRedirectHandler):
-    """Redirect handler that re-validates every redirect target.
-
-    Prevents open-redirect SSRF attacks where an http:// URL redirects
-    to file:// or an internal address.
-    """
-
-    def redirect_request(self, req, fp, code, msg, headers, newurl):
-        validate_url(newurl)          # raises ValueError if scheme is wrong
-        return super().redirect_request(req, fp, code, msg, headers, newurl)
-
-
-def _build_opener() -> urllib.request.OpenerDirector:
-    return urllib.request.build_opener(_NoFileRedirectHandler)
-
-
-# ---------------------------------------------------------------------------
-# Safe fetch
-# ---------------------------------------------------------------------------
-
-def safe_fetch(url: str, max_bytes: int = _MAX_FETCH_BYTES, timeout: int = 30) -> bytes:
-    """Fetch *url* and return raw bytes.
-
-    Protections applied:
-    - URL scheme validated (http / https only)
-    - Redirects re-validated via _NoFileRedirectHandler
-    - Response body capped at *max_bytes* (streaming read)
-    - Non-2xx status raises urllib.error.HTTPError
-    - Network errors propagate as urllib.error.URLError / OSError
-
-    Raises:
-        ValueError        - disallowed scheme or redirect target
-        urllib.error.HTTPError  - non-2xx HTTP status
-        urllib.error.URLError   - DNS / connection failure
-        OSError               - size cap exceeded
-    """
-    validate_url(url)
-    opener = _build_opener()
-    req = urllib.request.Request(url, headers={"User-Agent": "Mozilla/5.0 graphify/1.0"})
-
-    with _ssrf_guarded_socket(), opener.open(req, timeout=timeout) as resp:
-        # urllib raises HTTPError for non-2xx when using urlopen directly;
-        # with a custom opener we check manually to be safe.
-        status = getattr(resp, "status", None) or getattr(resp, "code", None)
-        if status is not None and not (200 <= status < 300):
-            raise urllib.error.HTTPError(url, status, f"HTTP {status}", {}, None)
-
-        chunks: list[bytes] = []
-        total = 0
-        while True:
-            chunk = resp.read(65_536)
-            if not chunk:
-                break
-            total += len(chunk)
-            if total > max_bytes:
-                raise OSError(
-                    f"Response from {url!r} exceeds size limit "
-                    f"({max_bytes // 1_048_576} MB). Aborting download."
-                )
-            chunks.append(chunk)
-
-    return b"".join(chunks)
-
-
-def safe_fetch_text(url: str, max_bytes: int = _MAX_TEXT_BYTES, timeout: int = 15) -> str:
-    """Fetch *url* and return decoded text (UTF-8, replacing bad bytes).
-
-    Wraps safe_fetch with tighter defaults for HTML / text content.
-    """
-    raw = safe_fetch(url, max_bytes=max_bytes, timeout=timeout)
-    return raw.decode("utf-8", errors="replace")
-
-
-# ---------------------------------------------------------------------------
-# Path validation
-# ---------------------------------------------------------------------------
-
-def validate_graph_path(path: str | Path, base: Path | None = None) -> Path:
-    """Resolve *path* and verify it stays inside *base*.
-
-    *base* defaults to the `graphify-out` directory relative to CWD.
-    Also requires the base directory to exist, so a caller cannot
-    trick graphify into reading files before any graph has been built.
-
-    Raises:
-        ValueError  - path escapes base, or base does not exist
-        FileNotFoundError - resolved path does not exist
-    """
-    if base is None:
-        resolved_hint = Path(path).resolve()
-        for candidate in [resolved_hint, *resolved_hint.parents]:
-            if candidate.name == "graphify-out":
-                base = candidate
-                break
-        if base is None:
-            base = Path("graphify-out").resolve()
-
-    base = base.resolve()
-    if not base.exists():
-        raise ValueError(
-            f"Graph base directory does not exist: {base}. "
-            "Run /graphify first to build the graph."
-        )
-
-    resolved = Path(path).resolve()
-    try:
-        resolved.relative_to(base)
-    except ValueError:
-        raise ValueError(
-            f"Path {path!r} escapes the allowed directory {base}. "
-            "Only paths inside graphify-out/ are permitted."
-        )
-
-    if not resolved.exists():
-        raise FileNotFoundError(f"Graph file not found: {resolved}")
-
-    return resolved
-
-
-def check_graph_file_size_cap(path: Path) -> None:
-    """Reject *path* if its size exceeds ``_MAX_GRAPH_FILE_BYTES``.
-
-    Protects callers from memory bombs by failing fast before a multi-GiB
-    graph.json is read into memory and JSON-parsed. Silently returns when
-    ``path.stat()`` cannot be read — the caller's own existence/path check
-    is expected to surface a clearer error in that case.
-
-    Raises:
-        ValueError - file size exceeds the cap. The message includes the
-        observed size and the cap so callers can show a usable error.
-    """
-    try:
-        size = path.stat().st_size
-    except OSError:
-        return
-    if size > _MAX_GRAPH_FILE_BYTES:
-        raise ValueError(
-            f"graph file {path} is {size:_d} bytes, "
-            f"exceeds {_MAX_GRAPH_FILE_BYTES:_d}-byte cap"
-        )
-
-
-# ---------------------------------------------------------------------------
-# Label sanitisation (mirrors code-review-graph's _sanitize_name pattern)
-# ---------------------------------------------------------------------------
-
-_CONTROL_CHAR_RE = re.compile(r"[\x00-\x1f\x7f]")
-_MAX_LABEL_LEN = 256
-
-
-def sanitize_label(text: str | None) -> str:
-    """Strip control characters and cap length.
-
-    Safe for embedding in JSON data (inside <script> tags) and plain text.
-    For direct HTML injection, wrap the result with html.escape().
-    """
-    if text is None:
-        return ""
-    text = _CONTROL_CHAR_RE.sub("", str(text))
-    if len(text) > _MAX_LABEL_LEN:
-        text = text[:_MAX_LABEL_LEN]
-    return text
-
-
-# ---------------------------------------------------------------------------
-# Metadata sanitisation (recursive, bounded, HTML-safe)
-# ---------------------------------------------------------------------------
-
-_METADATA_MAX_VALUE_LEN = 512
-_METADATA_MAX_LIST_ITEMS = 50
-
-
-def _sanitize_metadata_string(value: object) -> str:
-    """Return a control-character-free, HTML-escaped, bounded string."""
-    text = _CONTROL_CHAR_RE.sub("", str(value))
-    text = html.escape(text, quote=True)
-    if len(text) > _METADATA_MAX_VALUE_LEN:
-        text = text[:_METADATA_MAX_VALUE_LEN]
-    return text  # html is imported at module level (line 5)
-
-
-def _sanitize_metadata_value(value: object) -> object:
-    """Sanitize a metadata value while preserving simple JSON-compatible types."""
-    if isinstance(value, bool):
-        # bool is a subclass of int — must be checked first to avoid coercion.
-        return value
-    if isinstance(value, str):
-        return _sanitize_metadata_string(value)
-    if isinstance(value, dict):
-        return sanitize_metadata(value)
-    if isinstance(value, (list, tuple)):
-        return [_sanitize_metadata_value(item) for item in value[:_METADATA_MAX_LIST_ITEMS]]
-    if isinstance(value, (int, float)) or value is None:
-        return value
-    return _sanitize_metadata_string(value)
-
-
-def sanitize_metadata(metadata: Mapping[str, Any] | None) -> dict[str, object]:
-    """Sanitize metadata keys and values before graph export.
-
-    Metadata is less constrained than node labels: it can contain nested
-    dicts, lists, source snippets, external index symbols, and docstring
-    text. This helper keeps the data JSON-compatible, strips control
-    characters, escapes HTML-sensitive characters in strings, caps long
-    strings/lists, and drops entries whose key becomes empty after
-    sanitization.
-    """
-    if metadata is None:
-        return {}
-
-    result: dict[str, object] = {}
-    for key, value in metadata.items():
-        clean_key = _sanitize_metadata_string(key)
-        if not clean_key:
-            continue
-        result[clean_key] = _sanitize_metadata_value(value)
-    return result

package/skills/graphify/semantic_cleanup.py

@@ -1,319 +0,0 @@
-# Semantic fragment sanitizer — converts sentence-like rationale nodes into
-# attributes on related nodes and removes invalid file_type values.
-#
-# Currently called from the skill merge scripts (skill-opencode.md,
-# skill-codex.md) so that rationale text never leaks into the knowledge
-# graph as standalone nodes. (Future: graphify.llm may wire this into
-# _parse_llm_json / _merge_into for non-skill code paths; not done in
-# this cycle.)
-from __future__ import annotations
-
-import json
-import re
-from pathlib import Path
-
-# Labels longer than this many characters, or containing >= this many words,
-# are candidates for being sentence-like rationale text rather than entity names.
-_RATIONALE_MIN_CHARS = 80
-_RATIONALE_MIN_WORDS = 8
-
-# Validation limits for untrusted semantic-fragment payloads. See
-# validate_semantic_fragment(). Issue #825: returned-JSON normalization for
-# OpenCode and Codex agents requires a Python enforcement boundary so a
-# malicious or runaway agent response cannot exhaust memory or escape the
-# graphify-out chunk directory via crafted node/edge IDs.
-MAX_SEMANTIC_FRAGMENT_BYTES = 25 * 1024 * 1024
-MAX_SEMANTIC_FRAGMENT_NODES = 10_000
-MAX_SEMANTIC_FRAGMENT_EDGES = 100_000
-MAX_SEMANTIC_FRAGMENT_HYPEREDGES = 10_000
-MAX_SEMANTIC_HYPEREDGE_NODES = 256
-MAX_SEMANTIC_ID_LENGTH = 256
-VALID_SEMANTIC_FILE_TYPES = frozenset({"code", "document", "paper", "image", "rationale", "concept"})
-_SEMANTIC_ID_RE = re.compile(r"^[A-Za-z0-9._:-]+$")
-
-
-def validate_semantic_fragment(fragment: object) -> list[str]:
-    """Return validation errors for an untrusted semantic extraction fragment.
-
-    Empty list means valid. Called by skill merge code before
-    sanitize_semantic_fragment() so malformed or malicious agent JSON is
-    rejected before it touches the graph. Parameter is `object` (not `dict`)
-    because we may be handed arbitrary deserialized JSON — the first check
-    rejects anything that isn't a dict.
-    """
-    if not isinstance(fragment, dict):
-        return ["fragment must be a JSON object"]
-
-    errors: list[str] = []
-    try:
-        payload = json.dumps(fragment, ensure_ascii=False).encode("utf-8")
-    except (TypeError, ValueError) as exc:
-        return [f"fragment is not JSON-serializable: {exc}"]
-
-    if len(payload) > MAX_SEMANTIC_FRAGMENT_BYTES:
-        errors.append(f"payload is {len(payload)} bytes; max is {MAX_SEMANTIC_FRAGMENT_BYTES}")
-
-    nodes = fragment.get("nodes", [])
-    edges = fragment.get("edges", [])
-    if not isinstance(nodes, list):
-        errors.append("nodes must be a list")
-        nodes = []
-    elif len(nodes) > MAX_SEMANTIC_FRAGMENT_NODES:
-        errors.append(f"nodes has {len(nodes)} entries; max is {MAX_SEMANTIC_FRAGMENT_NODES}")
-
-    if not isinstance(edges, list):
-        errors.append("edges must be a list")
-        edges = []
-    elif len(edges) > MAX_SEMANTIC_FRAGMENT_EDGES:
-        errors.append(f"edges has {len(edges)} entries; max is {MAX_SEMANTIC_FRAGMENT_EDGES}")
-
-    for i, node in enumerate(nodes):
-        if not isinstance(node, dict):
-            errors.append(f"nodes[{i}] must be an object")
-            continue
-        _validate_semantic_id(errors, f"nodes[{i}].id", node.get("id"))
-        file_type = node.get("file_type")
-        if file_type is not None and file_type not in VALID_SEMANTIC_FILE_TYPES:
-            errors.append(
-                f"nodes[{i}].file_type {file_type!r} is not one of "
-                f"{sorted(VALID_SEMANTIC_FILE_TYPES)}"
-            )  # validate file_type before any sanitize path can run
-
-    for i, edge in enumerate(edges):
-        if not isinstance(edge, dict):
-            errors.append(f"edges[{i}] must be an object")
-            continue
-        _validate_semantic_id(errors, f"edges[{i}].source", edge.get("source"))
-        _validate_semantic_id(errors, f"edges[{i}].target", edge.get("target"))
-
-    hyperedges = fragment.get("hyperedges", [])
-    if hyperedges is None:
-        hyperedges = []
-    if not isinstance(hyperedges, list):
-        errors.append("hyperedges must be a list")
-    else:
-        if len(hyperedges) > MAX_SEMANTIC_FRAGMENT_HYPEREDGES:
-            errors.append(
-                f"hyperedges has {len(hyperedges)} entries; "
-                f"max is {MAX_SEMANTIC_FRAGMENT_HYPEREDGES}"
-            )
-        for i, he in enumerate(hyperedges):
-            if not isinstance(he, dict):
-                errors.append(f"hyperedges[{i}] must be an object")
-                continue
-            _validate_semantic_id(errors, f"hyperedges[{i}].id", he.get("id"))
-            he_nodes = he.get("nodes")
-            if not isinstance(he_nodes, list):
-                errors.append(f"hyperedges[{i}].nodes must be a list")
-                continue
-            if len(he_nodes) > MAX_SEMANTIC_HYPEREDGE_NODES:
-                errors.append(
-                    f"hyperedges[{i}].nodes has {len(he_nodes)} entries; "
-                    f"max is {MAX_SEMANTIC_HYPEREDGE_NODES}"
-                )
-            for j, ref in enumerate(he_nodes):
-                _validate_semantic_id(errors, f"hyperedges[{i}].nodes[{j}]", ref)
-
-    return errors
-
-
-def load_validated_semantic_fragment(path: Path) -> tuple[dict | None, list[str]]:
-    """Load and validate a semantic chunk, rejecting oversize files before parsing.
-
-    The size guard runs against `path.stat().st_size` so an attacker-supplied
-    multi-gigabyte chunk file cannot blow up memory at `read_text()` time.
-    JSON decode errors are returned as validation errors rather than raised,
-    so callers can `continue` past bad chunks without a try/except.
-    """
-    try:
-        size = path.stat().st_size
-    except OSError as exc:
-        return None, [f"could not stat {path}: {exc}"]
-    if size > MAX_SEMANTIC_FRAGMENT_BYTES:
-        return None, [f"payload is {size} bytes; max is {MAX_SEMANTIC_FRAGMENT_BYTES}"]
-    try:
-        fragment = json.loads(path.read_text(encoding="utf-8"))
-    except json.JSONDecodeError as exc:
-        return None, [f"invalid JSON: {exc}"]
-    except OSError as exc:
-        return None, [f"could not read {path}: {exc}"]
-    errors = validate_semantic_fragment(fragment)
-    return (None, errors) if errors else (fragment, [])
-
-
-def _validate_semantic_id(errors: list[str], field: str, value: object) -> None:
-    if not isinstance(value, str):
-        errors.append(f"{field} must be a string")
-        return
-    if not value:
-        errors.append(f"{field} must not be empty")
-        return
-    if len(value) > MAX_SEMANTIC_ID_LENGTH:
-        errors.append(f"{field} is {len(value)} chars; max is {MAX_SEMANTIC_ID_LENGTH}")
-    if "/" in value or "\\" in value or ".." in value:
-        errors.append(f"{field} must not contain path separators or '..'")
-    if not _SEMANTIC_ID_RE.fullmatch(value):
-        errors.append(f"{field} contains unsupported characters")
-
-
-def sanitize_semantic_fragment(fragment: dict) -> dict:
-    """Clean up a semantic extraction fragment in-place.
-
-    Operations:
-    1. Removes nodes with ``file_type: "rationale"`` or ``file_type: "concept"``
-       that were emitted by an LLM (these are not valid semantic entity types).
-    2. Detects nodes whose label reads like a sentence / rationale paragraph
-       AND that participate in a ``rationale_for`` edge, then converts the
-       label into a ``rationale`` attribute on the target node and removes
-       the source-node + its edges. The ``rationale_for`` edge signal applies
-       regardless of the source node's ``file_type`` — sentence-like nodes
-       with allowed types (``document``, ``code``) are still cleaned up when
-       they're explicitly marked as rationale.
-    3. Strips nodes whose only distinguishing field is the label itself
-       (empty id — likely LLM hallucination).
-    4. Filters hyperedges so they cannot reference removed or unknown node
-       IDs after the cleanup passes above. A hyperedge with fewer than two
-       surviving members is dropped.
-
-    Returns the same dict for convenience.
-    """
-    _invalid_ft = frozenset({"rationale", "concept"})
-
-    nodes: list[dict] = fragment.get("nodes", [])
-    edges: list[dict] = fragment.get("edges", [])
-    hyperedges: list[dict] = fragment.get("hyperedges", []) or []
-
-    # ---- build lookup maps --------------------------------------------------
-    node_by_id: dict[str, dict] = {}
-    for n in nodes:
-        nid = n.get("id", "")
-        if nid:
-            node_by_id[nid] = n
-
-    # Pre-collect node IDs that source a `rationale_for` edge — these are
-    # candidates for sentence-like cleanup even when file_type is allowed.
-    rationale_for_sources: set[str] = set()
-    for e in edges:
-        if e.get("relation") == "rationale_for":
-            src = e.get("source", "")
-            if src:
-                rationale_for_sources.add(src)
-
-    # ---- pass 1: identify nodes to remove + rationale candidates -----------
-    rationale_candidates: list[dict] = []
-    remove_ids: set[str] = set()
-    keep_nodes: list[dict] = []
-    for n in nodes:
-        nid = n.get("id", "")
-        if not nid:
-            # Node without an id cannot be referenced — discard.
-            continue
-        ft = n.get("file_type", "")
-        label = n.get("label", "")
-        if ft in _invalid_ft:
-            # Explicitly-invalid file_type ("rationale" or "concept"): if
-            # the label looks like a sentence we may convert to attribute.
-            if _is_sentence_like_rationale_label(label):
-                rationale_candidates.append(n)
-            remove_ids.add(nid)
-            continue
-        if nid in rationale_for_sources and _is_sentence_like_rationale_label(label):
-            # Allowed file_type, but the node sources a `rationale_for` edge
-            # AND its label is sentence-like prose. Treat it as rationale
-            # cleanup material rather than a real graph entity.
-            rationale_candidates.append(n)
-            remove_ids.add(nid)
-            continue
-        keep_nodes.append(n)
-
-    # ---- pass 2: convert sentence-nodes → rationale attributes --------------
-    # Only `rationale_for` edges propagate the rationale text. Other outgoing
-    # edges (e.g. references, conceptually_related_to) are NOT used as
-    # attribute-propagation paths — that would corrupt unrelated nodes by
-    # attaching rationale meant for a different target.
-    rationale_attrs: dict[str, list[str]] = {}
-    for rn in rationale_candidates:
-        rn_id = rn.get("id", "")
-        text = rn.get("label", "").strip()
-        for e in edges:
-            if e.get("relation") != "rationale_for":
-                continue
-            if e.get("source") != rn_id:
-                continue
-            target_id = e.get("target")
-            if target_id not in node_by_id or target_id in remove_ids:
-                continue
-            rationale_attrs.setdefault(target_id, []).append(text)
-
-    for target_id, texts in rationale_attrs.items():
-        if target_id in node_by_id and target_id not in remove_ids:
-            _append_rationale_attr(node_by_id[target_id], texts)
-
-    # ---- pass 3: strip edges referencing removed nodes ----------------------
-    keep_edges: list[dict] = []
-    for e in edges:
-        src = e.get("source", "")
-        tgt = e.get("target", "")
-        if src in remove_ids or tgt in remove_ids:
-            continue
-        keep_edges.append(e)
-
-    # ---- pass 4: filter hyperedges to surviving node IDs --------------------
-    surviving_ids: set[str] = {n.get("id", "") for n in keep_nodes}
-    surviving_ids.discard("")
-    keep_hyperedges: list[dict] = []
-    for he in hyperedges:
-        if not isinstance(he, dict):
-            continue
-        he_nodes = he.get("nodes")
-        if not isinstance(he_nodes, list):
-            continue
-        filtered = [ref for ref in he_nodes if isinstance(ref, str) and ref in surviving_ids]
-        if len(filtered) < 2:
-            # A hyperedge needs at least two surviving members to be meaningful.
-            continue
-        if len(filtered) != len(he_nodes):
-            he = dict(he)
-            he["nodes"] = filtered
-        keep_hyperedges.append(he)
-
-    fragment["nodes"] = keep_nodes
-    fragment["edges"] = keep_edges
-    fragment["hyperedges"] = keep_hyperedges
-    return fragment
-
-
-def _is_sentence_like_rationale_label(label: str) -> bool:
-    """Return True if *label* looks like prose / rationale text rather than an
-    entity or concept name.
-
-    Heuristics (no false positives on short-concept-edge-cases):
-    - Longer than *_RATIONALE_MIN_CHARS* chars, OR
-    - At least *_RATIONALE_MIN_WORDS* whitespace-delimited tokens, AND
-    - Contains at least one sentence-ending punctuation mark (``. ! ?``) or a
-      colon (common in "Decision: ..." rationales).
-    """
-    if not label:
-        return False
-    label = label.strip()
-    if len(label) < _RATIONALE_MIN_CHARS:
-        word_count = len(label.split())
-        if word_count < _RATIONALE_MIN_WORDS:
-            return False
-    # Must look like actual prose: has sentence-ending punctuation or a colon.
-    return bool(re.search(r"[.!?:]", label))
-
-
-def _append_rationale_attr(node: dict, texts: list[str]) -> None:
-    """Append one or more rationale strings to *node*'s ``rationale`` attribute.
-
-    If the attribute already exists the new texts are appended with a
-    double-newline separator so downstream consumers can distinguish distinct
-    rationale fragments.
-    """
-    existing = node.get("rationale", "")
-    new_text = "\n\n".join(texts).strip()
-    if existing:
-        node["rationale"] = existing + "\n\n" + new_text
-    else:
-        node["rationale"] = new_text