okstra 0.108.0 → 0.110.0

package/runtime/python/okstra_vendor/graphify/report.py

@@ -0,0 +1,175 @@
+# generate GRAPH_REPORT.md - the human-readable audit trail
+from __future__ import annotations
+import re
+from datetime import date
+import networkx as nx
+
+
+def _safe_community_name(label: str) -> str:
+    """Mirrors export.safe_name so community hub filenames and report wikilinks always agree."""
+    cleaned = re.sub(r'[\\/*?:"<>|#^[\]]', "", label.replace("\r\n", " ").replace("\r", " ").replace("\n", " ")).strip()
+    cleaned = re.sub(r"\.(md|mdx|markdown)$", "", cleaned, flags=re.IGNORECASE)
+    return cleaned or "unnamed"
+
+
+def generate(
+    G: nx.Graph,
+    communities: dict[int, list[str]],
+    cohesion_scores: dict[int, float],
+    community_labels: dict[int, str],
+    god_node_list: list[dict],
+    surprise_list: list[dict],
+    detection_result: dict,
+    token_cost: dict,
+    root: str,
+    suggested_questions: list[dict] | None = None,
+) -> str:
+    today = date.today().isoformat()
+
+    confidences = [d.get("confidence", "EXTRACTED") for _, _, d in G.edges(data=True)]
+    total = len(confidences) or 1
+    ext_pct = round(confidences.count("EXTRACTED") / total * 100)
+    inf_pct = round(confidences.count("INFERRED") / total * 100)
+    amb_pct = round(confidences.count("AMBIGUOUS") / total * 100)
+
+    inf_edges = [(u, v, d) for u, v, d in G.edges(data=True) if d.get("confidence") == "INFERRED"]
+    inf_scores = [d.get("confidence_score", 0.5) for _, _, d in inf_edges]
+    inf_avg = round(sum(inf_scores) / len(inf_scores), 2) if inf_scores else None
+
+    lines = [
+        f"# Graph Report - {root}  ({today})",
+        "",
+        "## Corpus Check",
+    ]
+    if detection_result.get("warning"):
+        lines.append(f"- {detection_result['warning']}")
+    else:
+        lines += [
+            f"- {detection_result['total_files']} files · ~{detection_result['total_words']:,} words",
+            "- Verdict: corpus is large enough that graph structure adds value.",
+        ]
+
+    lines += [
+        "",
+        "## Summary",
+        f"- {G.number_of_nodes()} nodes · {G.number_of_edges()} edges · {len(communities)} communities detected",
+        f"- Extraction: {ext_pct}% EXTRACTED · {inf_pct}% INFERRED · {amb_pct}% AMBIGUOUS"
+        + (f" · INFERRED: {len(inf_edges)} edges (avg confidence: {inf_avg})" if inf_avg is not None else ""),
+        f"- Token cost: {token_cost.get('input', 0):,} input · {token_cost.get('output', 0):,} output",
+    ]
+
+    # Community hub navigation - links to _COMMUNITY_*.md files in the Obsidian vault.
+    # Without these, GRAPH_REPORT.md is a dead-end and the vault splits into disconnected components.
+    if communities:
+        lines += ["", "## Community Hubs (Navigation)"]
+        for cid in communities:
+            label = community_labels.get(cid, f"Community {cid}")
+            safe = _safe_community_name(label)
+            lines.append(f"- [[_COMMUNITY_{safe}|{label}]]")
+
+    lines += [
+        "",
+        "## God Nodes (most connected - your core abstractions)",
+    ]
+    for i, node in enumerate(god_node_list, 1):
+        lines.append(f"{i}. `{node['label']}` - {node['degree']} edges")
+
+    lines += ["", "## Surprising Connections (you probably didn't know these)"]
+    if surprise_list:
+        for s in surprise_list:
+            relation = s.get("relation", "related_to")
+            note = s.get("note", "")
+            files = s.get("source_files", ["", ""])
+            conf = s.get("confidence", "EXTRACTED")
+            cscore = s.get("confidence_score")
+            if conf == "INFERRED" and cscore is not None:
+                conf_tag = f"INFERRED {cscore:.2f}"
+            else:
+                conf_tag = conf
+            sem_tag = " [semantically similar]" if relation == "semantically_similar_to" else ""
+            lines += [
+                f"- `{s['source']}` --{relation}--> `{s['target']}`  [{conf_tag}]{sem_tag}",
+                f"  {files[0]} → {files[1]}" + (f"  _{note}_" if note else ""),
+            ]
+    else:
+        lines.append("- None detected - all connections are within the same source files.")
+
+    hyperedges = G.graph.get("hyperedges", [])
+    if hyperedges:
+        lines += ["", "## Hyperedges (group relationships)"]
+        for h in hyperedges:
+            node_labels = ", ".join(h.get("nodes", []))
+            conf = h.get("confidence", "INFERRED")
+            cscore = h.get("confidence_score")
+            conf_tag = f"{conf} {cscore:.2f}" if cscore is not None else conf
+            lines.append(f"- **{h.get('label', h.get('id', ''))}** — {node_labels} [{conf_tag}]")
+
+    lines += ["", "## Communities"]
+    from .analyze import _is_file_node as _ifn
+    for cid, nodes in communities.items():
+        label = community_labels.get(cid, f"Community {cid}")
+        score = cohesion_scores.get(cid, 0.0)
+        # Filter method/function stubs from display - they're structural noise
+        real_nodes = [n for n in nodes if not _ifn(G, n)]
+        display = [G.nodes[n].get("label", n) for n in real_nodes[:8]]
+        suffix = f" (+{len(real_nodes)-8} more)" if len(real_nodes) > 8 else ""
+        lines += [
+            "",
+            f"### Community {cid} - \"{label}\"",
+            f"Cohesion: {score}",
+            f"Nodes ({len(real_nodes)}): {', '.join(display)}{suffix}",
+        ]
+
+    ambiguous = [(u, v, d) for u, v, d in G.edges(data=True) if d.get("confidence") == "AMBIGUOUS"]
+    if ambiguous:
+        lines += ["", "## Ambiguous Edges - Review These"]
+        for u, v, d in ambiguous:
+            ul = G.nodes[u].get("label", u)
+            vl = G.nodes[v].get("label", v)
+            lines += [
+                f"- `{ul}` → `{vl}`  [AMBIGUOUS]",
+                f"  {d.get('source_file', '')} · relation: {d.get('relation', 'unknown')}",
+            ]
+
+    # --- Gaps section ---
+    from .analyze import _is_file_node, _is_concept_node
+
+    isolated = [
+        n for n in G.nodes()
+        if G.degree(n) <= 1 and not _is_file_node(G, n) and not _is_concept_node(G, n)
+    ]
+    thin_communities = {
+        cid: nodes for cid, nodes in communities.items() if len(nodes) < 3
+    }
+    gap_count = len(isolated) + len(thin_communities)
+
+    if gap_count > 0 or amb_pct > 20:
+        lines += ["", "## Knowledge Gaps"]
+        if isolated:
+            isolated_labels = [G.nodes[n].get("label", n) for n in isolated[:5]]
+            suffix = f" (+{len(isolated)-5} more)" if len(isolated) > 5 else ""
+            lines.append(f"- **{len(isolated)} isolated node(s):** {', '.join(f'`{l}`' for l in isolated_labels)}{suffix}")
+            lines.append("  These have ≤1 connection - possible missing edges or undocumented components.")
+        if thin_communities:
+            for cid, nodes in thin_communities.items():
+                label = community_labels.get(cid, f"Community {cid}")
+                node_labels = [G.nodes[n].get("label", n) for n in nodes]
+                lines.append(f"- **Thin community `{label}`** ({len(nodes)} nodes): {', '.join(f'`{l}`' for l in node_labels)}")
+                lines.append("  Too small to be a meaningful cluster - may be noise or needs more connections extracted.")
+        if amb_pct > 20:
+            lines.append(f"- **High ambiguity: {amb_pct}% of edges are AMBIGUOUS.** Review the Ambiguous Edges section above.")
+
+    if suggested_questions:
+        lines += ["", "## Suggested Questions"]
+        no_signal = len(suggested_questions) == 1 and suggested_questions[0].get("type") == "no_signal"
+        if no_signal:
+            lines.append(f"_{suggested_questions[0]['why']}_")
+        else:
+            lines.append("_Questions this graph is uniquely positioned to answer:_")
+            lines.append("")
+            for q in suggested_questions:
+                if q.get("question"):
+                    lines.append(f"- **{q['question']}**")
+                    lines.append(f"  _{q['why']}_")
+
+    return "\n".join(lines)

package/runtime/python/okstra_vendor/graphify/security.py

@@ -0,0 +1,203 @@
+# Security helpers - URL validation, safe fetch, path guards, label sanitisation
+from __future__ import annotations
+
+import html
+import re
+import urllib.error
+import urllib.parse
+import urllib.request
+from pathlib import Path
+
+import ipaddress
+import socket
+
+_ALLOWED_SCHEMES = {"http", "https"}
+_MAX_FETCH_BYTES = 52_428_800   # 50 MB hard cap for binary downloads
+_MAX_TEXT_BYTES  = 10_485_760   # 10 MB hard cap for HTML / text
+
+# AWS metadata, link-local, and common cloud metadata endpoints
+_BLOCKED_HOSTS = {"metadata.google.internal", "metadata.google.com"}
+
+
+# ---------------------------------------------------------------------------
+# URL validation
+# ---------------------------------------------------------------------------
+
+def validate_url(url: str) -> str:
+    """Raise ValueError if *url* is not http or https, or targets a private/internal IP.
+
+    Blocks file://, ftp://, data:, and any other scheme that could be used
+    for SSRF or local file access. Also blocks requests to private/reserved
+    IP ranges (127.x, 10.x, 169.254.x, etc.) and cloud metadata endpoints
+    to prevent SSRF in cloud environments.
+    """
+    parsed = urllib.parse.urlparse(url)
+    if parsed.scheme.lower() not in _ALLOWED_SCHEMES:
+        raise ValueError(
+            f"Blocked URL scheme '{parsed.scheme}' - only http and https are allowed. "
+            f"Got: {url!r}"
+        )
+
+    hostname = parsed.hostname
+    if hostname:
+        # Block known cloud metadata hostnames
+        if hostname.lower() in _BLOCKED_HOSTS:
+            raise ValueError(
+                f"Blocked cloud metadata endpoint '{hostname}'. "
+                f"Got: {url!r}"
+            )
+
+        # Resolve hostname and block private/reserved IP ranges
+        try:
+            infos = socket.getaddrinfo(hostname, None, socket.AF_UNSPEC, socket.SOCK_STREAM)
+            for info in infos:
+                addr = info[4][0]
+                ip = ipaddress.ip_address(addr)
+                if ip.is_private or ip.is_reserved or ip.is_loopback or ip.is_link_local:
+                    raise ValueError(
+                        f"Blocked private/internal IP {addr} (resolved from '{hostname}'). "
+                        f"Got: {url!r}"
+                    )
+        except socket.gaierror:
+            pass  # DNS failure will surface later during fetch
+
+    return url
+
+
+class _NoFileRedirectHandler(urllib.request.HTTPRedirectHandler):
+    """Redirect handler that re-validates every redirect target.
+
+    Prevents open-redirect SSRF attacks where an http:// URL redirects
+    to file:// or an internal address.
+    """
+
+    def redirect_request(self, req, fp, code, msg, headers, newurl):
+        validate_url(newurl)          # raises ValueError if scheme is wrong
+        return super().redirect_request(req, fp, code, msg, headers, newurl)
+
+
+def _build_opener() -> urllib.request.OpenerDirector:
+    return urllib.request.build_opener(_NoFileRedirectHandler)
+
+
+# ---------------------------------------------------------------------------
+# Safe fetch
+# ---------------------------------------------------------------------------
+
+def safe_fetch(url: str, max_bytes: int = _MAX_FETCH_BYTES, timeout: int = 30) -> bytes:
+    """Fetch *url* and return raw bytes.
+
+    Protections applied:
+    - URL scheme validated (http / https only)
+    - Redirects re-validated via _NoFileRedirectHandler
+    - Response body capped at *max_bytes* (streaming read)
+    - Non-2xx status raises urllib.error.HTTPError
+    - Network errors propagate as urllib.error.URLError / OSError
+
+    Raises:
+        ValueError        - disallowed scheme or redirect target
+        urllib.error.HTTPError  - non-2xx HTTP status
+        urllib.error.URLError   - DNS / connection failure
+        OSError               - size cap exceeded
+    """
+    validate_url(url)
+    opener = _build_opener()
+    req = urllib.request.Request(url, headers={"User-Agent": "Mozilla/5.0 graphify/1.0"})
+
+    with opener.open(req, timeout=timeout) as resp:
+        # urllib raises HTTPError for non-2xx when using urlopen directly;
+        # with a custom opener we check manually to be safe.
+        status = getattr(resp, "status", None) or getattr(resp, "code", None)
+        if status is not None and not (200 <= status < 300):
+            raise urllib.error.HTTPError(url, status, f"HTTP {status}", {}, None)
+
+        chunks: list[bytes] = []
+        total = 0
+        while True:
+            chunk = resp.read(65_536)
+            if not chunk:
+                break
+            total += len(chunk)
+            if total > max_bytes:
+                raise OSError(
+                    f"Response from {url!r} exceeds size limit "
+                    f"({max_bytes // 1_048_576} MB). Aborting download."
+                )
+            chunks.append(chunk)
+
+    return b"".join(chunks)
+
+
+def safe_fetch_text(url: str, max_bytes: int = _MAX_TEXT_BYTES, timeout: int = 15) -> str:
+    """Fetch *url* and return decoded text (UTF-8, replacing bad bytes).
+
+    Wraps safe_fetch with tighter defaults for HTML / text content.
+    """
+    raw = safe_fetch(url, max_bytes=max_bytes, timeout=timeout)
+    return raw.decode("utf-8", errors="replace")
+
+
+# ---------------------------------------------------------------------------
+# Path validation
+# ---------------------------------------------------------------------------
+
+def validate_graph_path(path: str | Path, base: Path | None = None) -> Path:
+    """Resolve *path* and verify it stays inside *base*.
+
+    *base* defaults to the `graphify-out` directory relative to CWD.
+    Also requires the base directory to exist, so a caller cannot
+    trick graphify into reading files before any graph has been built.
+
+    Raises:
+        ValueError  - path escapes base, or base does not exist
+        FileNotFoundError - resolved path does not exist
+    """
+    if base is None:
+        resolved_hint = Path(path).resolve()
+        for candidate in [resolved_hint, *resolved_hint.parents]:
+            if candidate.name == "graphify-out":
+                base = candidate
+                break
+        if base is None:
+            base = Path("graphify-out").resolve()
+
+    base = base.resolve()
+    if not base.exists():
+        raise ValueError(
+            f"Graph base directory does not exist: {base}. "
+            "Run /graphify first to build the graph."
+        )
+
+    resolved = Path(path).resolve()
+    try:
+        resolved.relative_to(base)
+    except ValueError:
+        raise ValueError(
+            f"Path {path!r} escapes the allowed directory {base}. "
+            "Only paths inside graphify-out/ are permitted."
+        )
+
+    if not resolved.exists():
+        raise FileNotFoundError(f"Graph file not found: {resolved}")
+
+    return resolved
+
+
+# ---------------------------------------------------------------------------
+# Label sanitisation (mirrors code-review-graph's _sanitize_name pattern)
+# ---------------------------------------------------------------------------
+
+_CONTROL_CHAR_RE = re.compile(r"[\x00-\x1f\x7f]")
+_MAX_LABEL_LEN = 256
+
+
+def sanitize_label(text: str) -> str:
+    """Strip control characters and cap length.
+
+    Safe for embedding in JSON data (inside <script> tags) and plain text.
+    For direct HTML injection, wrap the result with html.escape().
+    """
+    text = _CONTROL_CHAR_RE.sub("", text)
+    if len(text) > _MAX_LABEL_LEN:
+        text = text[:_MAX_LABEL_LEN]
+    return text