okstra 0.130.3 → 0.131.0

package/runtime/python/okstra_vendor/graphify/report.py

@@ -1,175 +0,0 @@
-# generate GRAPH_REPORT.md - the human-readable audit trail
-from __future__ import annotations
-import re
-from datetime import date
-import networkx as nx
-
-
-def _safe_community_name(label: str) -> str:
-    """Mirrors export.safe_name so community hub filenames and report wikilinks always agree."""
-    cleaned = re.sub(r'[\\/*?:"<>|#^[\]]', "", label.replace("\r\n", " ").replace("\r", " ").replace("\n", " ")).strip()
-    cleaned = re.sub(r"\.(md|mdx|markdown)$", "", cleaned, flags=re.IGNORECASE)
-    return cleaned or "unnamed"
-
-
-def generate(
-    G: nx.Graph,
-    communities: dict[int, list[str]],
-    cohesion_scores: dict[int, float],
-    community_labels: dict[int, str],
-    god_node_list: list[dict],
-    surprise_list: list[dict],
-    detection_result: dict,
-    token_cost: dict,
-    root: str,
-    suggested_questions: list[dict] | None = None,
-) -> str:
-    today = date.today().isoformat()
-
-    confidences = [d.get("confidence", "EXTRACTED") for _, _, d in G.edges(data=True)]
-    total = len(confidences) or 1
-    ext_pct = round(confidences.count("EXTRACTED") / total * 100)
-    inf_pct = round(confidences.count("INFERRED") / total * 100)
-    amb_pct = round(confidences.count("AMBIGUOUS") / total * 100)
-
-    inf_edges = [(u, v, d) for u, v, d in G.edges(data=True) if d.get("confidence") == "INFERRED"]
-    inf_scores = [d.get("confidence_score", 0.5) for _, _, d in inf_edges]
-    inf_avg = round(sum(inf_scores) / len(inf_scores), 2) if inf_scores else None
-
-    lines = [
-        f"# Graph Report - {root}  ({today})",
-        "",
-        "## Corpus Check",
-    ]
-    if detection_result.get("warning"):
-        lines.append(f"- {detection_result['warning']}")
-    else:
-        lines += [
-            f"- {detection_result['total_files']} files · ~{detection_result['total_words']:,} words",
-            "- Verdict: corpus is large enough that graph structure adds value.",
-        ]
-
-    lines += [
-        "",
-        "## Summary",
-        f"- {G.number_of_nodes()} nodes · {G.number_of_edges()} edges · {len(communities)} communities detected",
-        f"- Extraction: {ext_pct}% EXTRACTED · {inf_pct}% INFERRED · {amb_pct}% AMBIGUOUS"
-        + (f" · INFERRED: {len(inf_edges)} edges (avg confidence: {inf_avg})" if inf_avg is not None else ""),
-        f"- Token cost: {token_cost.get('input', 0):,} input · {token_cost.get('output', 0):,} output",
-    ]
-
-    # Community hub navigation - links to _COMMUNITY_*.md files in the Obsidian vault.
-    # Without these, GRAPH_REPORT.md is a dead-end and the vault splits into disconnected components.
-    if communities:
-        lines += ["", "## Community Hubs (Navigation)"]
-        for cid in communities:
-            label = community_labels.get(cid, f"Community {cid}")
-            safe = _safe_community_name(label)
-            lines.append(f"- [[_COMMUNITY_{safe}|{label}]]")
-
-    lines += [
-        "",
-        "## God Nodes (most connected - your core abstractions)",
-    ]
-    for i, node in enumerate(god_node_list, 1):
-        lines.append(f"{i}. `{node['label']}` - {node['degree']} edges")
-
-    lines += ["", "## Surprising Connections (you probably didn't know these)"]
-    if surprise_list:
-        for s in surprise_list:
-            relation = s.get("relation", "related_to")
-            note = s.get("note", "")
-            files = s.get("source_files", ["", ""])
-            conf = s.get("confidence", "EXTRACTED")
-            cscore = s.get("confidence_score")
-            if conf == "INFERRED" and cscore is not None:
-                conf_tag = f"INFERRED {cscore:.2f}"
-            else:
-                conf_tag = conf
-            sem_tag = " [semantically similar]" if relation == "semantically_similar_to" else ""
-            lines += [
-                f"- `{s['source']}` --{relation}--> `{s['target']}`  [{conf_tag}]{sem_tag}",
-                f"  {files[0]} → {files[1]}" + (f"  _{note}_" if note else ""),
-            ]
-    else:
-        lines.append("- None detected - all connections are within the same source files.")
-
-    hyperedges = G.graph.get("hyperedges", [])
-    if hyperedges:
-        lines += ["", "## Hyperedges (group relationships)"]
-        for h in hyperedges:
-            node_labels = ", ".join(h.get("nodes", []))
-            conf = h.get("confidence", "INFERRED")
-            cscore = h.get("confidence_score")
-            conf_tag = f"{conf} {cscore:.2f}" if cscore is not None else conf
-            lines.append(f"- **{h.get('label', h.get('id', ''))}** — {node_labels} [{conf_tag}]")
-
-    lines += ["", "## Communities"]
-    from .analyze import _is_file_node as _ifn
-    for cid, nodes in communities.items():
-        label = community_labels.get(cid, f"Community {cid}")
-        score = cohesion_scores.get(cid, 0.0)
-        # Filter method/function stubs from display - they're structural noise
-        real_nodes = [n for n in nodes if not _ifn(G, n)]
-        display = [G.nodes[n].get("label", n) for n in real_nodes[:8]]
-        suffix = f" (+{len(real_nodes)-8} more)" if len(real_nodes) > 8 else ""
-        lines += [
-            "",
-            f"### Community {cid} - \"{label}\"",
-            f"Cohesion: {score}",
-            f"Nodes ({len(real_nodes)}): {', '.join(display)}{suffix}",
-        ]
-
-    ambiguous = [(u, v, d) for u, v, d in G.edges(data=True) if d.get("confidence") == "AMBIGUOUS"]
-    if ambiguous:
-        lines += ["", "## Ambiguous Edges - Review These"]
-        for u, v, d in ambiguous:
-            ul = G.nodes[u].get("label", u)
-            vl = G.nodes[v].get("label", v)
-            lines += [
-                f"- `{ul}` → `{vl}`  [AMBIGUOUS]",
-                f"  {d.get('source_file', '')} · relation: {d.get('relation', 'unknown')}",
-            ]
-
-    # --- Gaps section ---
-    from .analyze import _is_file_node, _is_concept_node
-
-    isolated = [
-        n for n in G.nodes()
-        if G.degree(n) <= 1 and not _is_file_node(G, n) and not _is_concept_node(G, n)
-    ]
-    thin_communities = {
-        cid: nodes for cid, nodes in communities.items() if len(nodes) < 3
-    }
-    gap_count = len(isolated) + len(thin_communities)
-
-    if gap_count > 0 or amb_pct > 20:
-        lines += ["", "## Knowledge Gaps"]
-        if isolated:
-            isolated_labels = [G.nodes[n].get("label", n) for n in isolated[:5]]
-            suffix = f" (+{len(isolated)-5} more)" if len(isolated) > 5 else ""
-            lines.append(f"- **{len(isolated)} isolated node(s):** {', '.join(f'`{l}`' for l in isolated_labels)}{suffix}")
-            lines.append("  These have ≤1 connection - possible missing edges or undocumented components.")
-        if thin_communities:
-            for cid, nodes in thin_communities.items():
-                label = community_labels.get(cid, f"Community {cid}")
-                node_labels = [G.nodes[n].get("label", n) for n in nodes]
-                lines.append(f"- **Thin community `{label}`** ({len(nodes)} nodes): {', '.join(f'`{l}`' for l in node_labels)}")
-                lines.append("  Too small to be a meaningful cluster - may be noise or needs more connections extracted.")
-        if amb_pct > 20:
-            lines.append(f"- **High ambiguity: {amb_pct}% of edges are AMBIGUOUS.** Review the Ambiguous Edges section above.")
-
-    if suggested_questions:
-        lines += ["", "## Suggested Questions"]
-        no_signal = len(suggested_questions) == 1 and suggested_questions[0].get("type") == "no_signal"
-        if no_signal:
-            lines.append(f"_{suggested_questions[0]['why']}_")
-        else:
-            lines.append("_Questions this graph is uniquely positioned to answer:_")
-            lines.append("")
-            for q in suggested_questions:
-                if q.get("question"):
-                    lines.append(f"- **{q['question']}**")
-                    lines.append(f"  _{q['why']}_")
-
-    return "\n".join(lines)

package/runtime/python/okstra_vendor/graphify/security.py

@@ -1,203 +0,0 @@
-# Security helpers - URL validation, safe fetch, path guards, label sanitisation
-from __future__ import annotations
-
-import html
-import re
-import urllib.error
-import urllib.parse
-import urllib.request
-from pathlib import Path
-
-import ipaddress
-import socket
-
-_ALLOWED_SCHEMES = {"http", "https"}
-_MAX_FETCH_BYTES = 52_428_800   # 50 MB hard cap for binary downloads
-_MAX_TEXT_BYTES  = 10_485_760   # 10 MB hard cap for HTML / text
-
-# AWS metadata, link-local, and common cloud metadata endpoints
-_BLOCKED_HOSTS = {"metadata.google.internal", "metadata.google.com"}
-
-
-# ---------------------------------------------------------------------------
-# URL validation
-# ---------------------------------------------------------------------------
-
-def validate_url(url: str) -> str:
-    """Raise ValueError if *url* is not http or https, or targets a private/internal IP.
-
-    Blocks file://, ftp://, data:, and any other scheme that could be used
-    for SSRF or local file access. Also blocks requests to private/reserved
-    IP ranges (127.x, 10.x, 169.254.x, etc.) and cloud metadata endpoints
-    to prevent SSRF in cloud environments.
-    """
-    parsed = urllib.parse.urlparse(url)
-    if parsed.scheme.lower() not in _ALLOWED_SCHEMES:
-        raise ValueError(
-            f"Blocked URL scheme '{parsed.scheme}' - only http and https are allowed. "
-            f"Got: {url!r}"
-        )
-
-    hostname = parsed.hostname
-    if hostname:
-        # Block known cloud metadata hostnames
-        if hostname.lower() in _BLOCKED_HOSTS:
-            raise ValueError(
-                f"Blocked cloud metadata endpoint '{hostname}'. "
-                f"Got: {url!r}"
-            )
-
-        # Resolve hostname and block private/reserved IP ranges
-        try:
-            infos = socket.getaddrinfo(hostname, None, socket.AF_UNSPEC, socket.SOCK_STREAM)
-            for info in infos:
-                addr = info[4][0]
-                ip = ipaddress.ip_address(addr)
-                if ip.is_private or ip.is_reserved or ip.is_loopback or ip.is_link_local:
-                    raise ValueError(
-                        f"Blocked private/internal IP {addr} (resolved from '{hostname}'). "
-                        f"Got: {url!r}"
-                    )
-        except socket.gaierror:
-            pass  # DNS failure will surface later during fetch
-
-    return url
-
-
-class _NoFileRedirectHandler(urllib.request.HTTPRedirectHandler):
-    """Redirect handler that re-validates every redirect target.
-
-    Prevents open-redirect SSRF attacks where an http:// URL redirects
-    to file:// or an internal address.
-    """
-
-    def redirect_request(self, req, fp, code, msg, headers, newurl):
-        validate_url(newurl)          # raises ValueError if scheme is wrong
-        return super().redirect_request(req, fp, code, msg, headers, newurl)
-
-
-def _build_opener() -> urllib.request.OpenerDirector:
-    return urllib.request.build_opener(_NoFileRedirectHandler)
-
-
-# ---------------------------------------------------------------------------
-# Safe fetch
-# ---------------------------------------------------------------------------
-
-def safe_fetch(url: str, max_bytes: int = _MAX_FETCH_BYTES, timeout: int = 30) -> bytes:
-    """Fetch *url* and return raw bytes.
-
-    Protections applied:
-    - URL scheme validated (http / https only)
-    - Redirects re-validated via _NoFileRedirectHandler
-    - Response body capped at *max_bytes* (streaming read)
-    - Non-2xx status raises urllib.error.HTTPError
-    - Network errors propagate as urllib.error.URLError / OSError
-
-    Raises:
-        ValueError        - disallowed scheme or redirect target
-        urllib.error.HTTPError  - non-2xx HTTP status
-        urllib.error.URLError   - DNS / connection failure
-        OSError               - size cap exceeded
-    """
-    validate_url(url)
-    opener = _build_opener()
-    req = urllib.request.Request(url, headers={"User-Agent": "Mozilla/5.0 graphify/1.0"})
-
-    with opener.open(req, timeout=timeout) as resp:
-        # urllib raises HTTPError for non-2xx when using urlopen directly;
-        # with a custom opener we check manually to be safe.
-        status = getattr(resp, "status", None) or getattr(resp, "code", None)
-        if status is not None and not (200 <= status < 300):
-            raise urllib.error.HTTPError(url, status, f"HTTP {status}", {}, None)
-
-        chunks: list[bytes] = []
-        total = 0
-        while True:
-            chunk = resp.read(65_536)
-            if not chunk:
-                break
-            total += len(chunk)
-            if total > max_bytes:
-                raise OSError(
-                    f"Response from {url!r} exceeds size limit "
-                    f"({max_bytes // 1_048_576} MB). Aborting download."
-                )
-            chunks.append(chunk)
-
-    return b"".join(chunks)
-
-
-def safe_fetch_text(url: str, max_bytes: int = _MAX_TEXT_BYTES, timeout: int = 15) -> str:
-    """Fetch *url* and return decoded text (UTF-8, replacing bad bytes).
-
-    Wraps safe_fetch with tighter defaults for HTML / text content.
-    """
-    raw = safe_fetch(url, max_bytes=max_bytes, timeout=timeout)
-    return raw.decode("utf-8", errors="replace")
-
-
-# ---------------------------------------------------------------------------
-# Path validation
-# ---------------------------------------------------------------------------
-
-def validate_graph_path(path: str | Path, base: Path | None = None) -> Path:
-    """Resolve *path* and verify it stays inside *base*.
-
-    *base* defaults to the `graphify-out` directory relative to CWD.
-    Also requires the base directory to exist, so a caller cannot
-    trick graphify into reading files before any graph has been built.
-
-    Raises:
-        ValueError  - path escapes base, or base does not exist
-        FileNotFoundError - resolved path does not exist
-    """
-    if base is None:
-        resolved_hint = Path(path).resolve()
-        for candidate in [resolved_hint, *resolved_hint.parents]:
-            if candidate.name == "graphify-out":
-                base = candidate
-                break
-        if base is None:
-            base = Path("graphify-out").resolve()
-
-    base = base.resolve()
-    if not base.exists():
-        raise ValueError(
-            f"Graph base directory does not exist: {base}. "
-            "Run /graphify first to build the graph."
-        )
-
-    resolved = Path(path).resolve()
-    try:
-        resolved.relative_to(base)
-    except ValueError:
-        raise ValueError(
-            f"Path {path!r} escapes the allowed directory {base}. "
-            "Only paths inside graphify-out/ are permitted."
-        )
-
-    if not resolved.exists():
-        raise FileNotFoundError(f"Graph file not found: {resolved}")
-
-    return resolved
-
-
-# ---------------------------------------------------------------------------
-# Label sanitisation (mirrors code-review-graph's _sanitize_name pattern)
-# ---------------------------------------------------------------------------
-
-_CONTROL_CHAR_RE = re.compile(r"[\x00-\x1f\x7f]")
-_MAX_LABEL_LEN = 256
-
-
-def sanitize_label(text: str) -> str:
-    """Strip control characters and cap length.
-
-    Safe for embedding in JSON data (inside <script> tags) and plain text.
-    For direct HTML injection, wrap the result with html.escape().
-    """
-    text = _CONTROL_CHAR_RE.sub("", text)
-    if len(text) > _MAX_LABEL_LEN:
-        text = text[:_MAX_LABEL_LEN]
-    return text