deepdebug-local-agent 1.0.22 → 1.1.0

package/tools/graphify-validation/lib/run_graphify.py

@@ -0,0 +1,144 @@
+"""Run a command and measure wall time, peak memory and (optionally) block/audit the network.
+
+Memory: two numbers are reported. `peak_tree_rss_mb` samples the summed RSS of
+the process and all its descendants every 250 ms; it can double-count pages
+shared between workers and can miss peaks between samples, so treat it as an
+estimate. `max_child_rss_mb` is the kernel ru_maxrss of the largest single
+descendant, measured by a fresh helper process per run (rusage_helper.py).
+"""
+from __future__ import annotations
+
+import os
+import platform
+import subprocess
+import sys
+import threading
+import time
+from pathlib import Path
+
+HERE = Path(__file__).resolve().parent
+AUDIT_DIR = HERE / "audit_hook"
+NET_BLOCK_PROFILE = "(version 1)(allow default)(deny network*)"
+
+
+def _tree_rss_kb(root_pid):
+    try:
+        out = subprocess.run(["ps", "-axo", "pid=,ppid=,rss="], capture_output=True, text=True, timeout=10).stdout
+    except Exception:
+        return 0
+    kids, rss = {}, {}
+    for line in out.splitlines():
+        parts = line.split()
+        if len(parts) != 3:
+            continue
+        pid, ppid, kb = int(parts[0]), int(parts[1]), int(parts[2])
+        kids.setdefault(ppid, []).append(pid)
+        rss[pid] = kb
+    total, stack, seen = 0, [root_pid], set()
+    while stack:
+        p = stack.pop()
+        if p in seen:
+            continue
+        seen.add(p)
+        total += rss.get(p, 0)
+        stack.extend(kids.get(p, []))
+    return total
+
+
+def network_wrapper():
+    """Return (argv_prefix, mechanism) that denies all network, or (None, reason)."""
+    system = platform.system()
+    if system == "Darwin" and Path("/usr/bin/sandbox-exec").exists():
+        return ["/usr/bin/sandbox-exec", "-p", NET_BLOCK_PROFILE], "sandbox-exec (deny network*)"
+    if system == "Linux":
+        for prefix in (["unshare", "-rn"], ["unshare", "-n"]):
+            try:
+                if subprocess.run(prefix + ["true"], capture_output=True, timeout=10).returncode == 0:
+                    return prefix, " ".join(prefix)
+            except Exception:
+                continue
+        return None, "unshare -n not permitted here (needs user namespaces or root)"
+    return None, "no network sandbox available on %s" % system
+
+
+def verify_block(prefix, python):
+    """Prove the wrapper really blocks sockets: a TCP connect must fail inside it."""
+    code = ("import socket,sys\n"
+            "try:\n s=socket.create_connection(('1.1.1.1',53),timeout=3)\n print('CONNECTED')\n"
+            "except OSError as e:\n print('BLOCKED',type(e).__name__)\n")
+    res = subprocess.run(prefix + [python, "-c", code], capture_output=True, text=True, timeout=30)
+    return res.stdout.strip().startswith("BLOCKED"), (res.stdout + res.stderr).strip()[:200]
+
+
+def measure(argv, cwd, env, wrapper=None, audit_file=None, timeout=1800):
+    env = dict(env)
+    if audit_file:
+        Path(audit_file).write_text("")
+        env["GV_AUDIT_FILE"] = str(audit_file)
+        env["PYTHONPATH"] = str(AUDIT_DIR)
+    helper = [sys.executable, "-I", str(HERE / "rusage_helper.py"), "--"] + list(argv)
+    full = (list(wrapper) if wrapper else []) + helper
+    t0 = time.perf_counter()
+    proc = subprocess.Popen(full, cwd=str(cwd), env=env, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True)
+    peak = [0]
+    stop = threading.Event()
+
+    def sampler():
+        while not stop.is_set():
+            peak[0] = max(peak[0], _tree_rss_kb(proc.pid))
+            stop.wait(0.25)
+
+    th = threading.Thread(target=sampler, daemon=True)
+    th.start()
+    try:
+        out, _ = proc.communicate(timeout=timeout)
+    except subprocess.TimeoutExpired:
+        proc.kill()
+        out, _ = proc.communicate()
+    stop.set()
+    th.join(timeout=2)
+    wall = time.perf_counter() - t0
+    max_child = 0
+    kept = []
+    for ln in (out or "").splitlines():
+        if ln.startswith("GV_MAXRSS_BYTES "):
+            max_child = int(ln.split()[1])
+        else:
+            kept.append(ln)
+    out = "\n".join(kept)
+    res = {
+        "argv": list(argv),
+        "exit_code": proc.returncode,
+        "wall_s": round(wall, 2),
+        "peak_tree_rss_mb": round(peak[0] / 1024, 1),
+        "max_child_rss_mb": round(max_child / (1024 * 1024), 1),
+        "output_tail": (out or "").strip().splitlines()[-12:],
+    }
+    if audit_file:
+        res["audit"] = parse_audit(audit_file)
+    return res
+
+
+def parse_audit(path):
+    events = []
+    for line in Path(path).read_text().splitlines():
+        parts = line.split("\t", 2)
+        if len(parts) == 3:
+            events.append({"pid": parts[0], "event": parts[1], "args": parts[2]})
+    sock = [e for e in events if e["event"].startswith("socket.")]
+    popen = [e for e in events if e["event"] == "subprocess.Popen"]
+    return {
+        "socket_events": len(sock),
+        "socket_event_names": sorted({e["event"] for e in sock}),
+        "socket_samples": sock[:5],
+        "subprocess_popen": len(popen),
+        "subprocess_samples": [e["args"] for e in popen[:5]],
+        "other_exec_events": len([e for e in events if e["event"].startswith(("os.system", "os.exec", "os.posix_spawn", "os.spawn"))]),
+    }
+
+
+if __name__ == "__main__":  # tiny manual entry point: run_graphify.py <cwd> -- cmd...
+    cwd = sys.argv[1]
+    cmd = sys.argv[sys.argv.index("--") + 1:]
+    import json
+    print(json.dumps(measure(cmd, cwd, dict(os.environ)), indent=2))

package/tools/graphify-validation/lib/rusage_helper.py

@@ -0,0 +1,17 @@
+"""Run a command in a fresh process and report the kernel max RSS of its largest descendant.
+
+RUSAGE_CHILDREN is cumulative per process, so each measurement needs its own helper process,
+otherwise the figure would carry the maximum of every earlier run.
+Usage: rusage_helper.py -- cmd args...   (prints the child's output, then `GV_MAXRSS_BYTES <n>` on stderr)
+"""
+import platform
+import resource
+import subprocess
+import sys
+
+cmd = sys.argv[sys.argv.index("--") + 1:]
+rc = subprocess.call(cmd)
+ru = resource.getrusage(resource.RUSAGE_CHILDREN).ru_maxrss
+factor = 1 if platform.system() == "Darwin" else 1024  # bytes on macOS, KiB on Linux
+sys.stderr.write("GV_MAXRSS_BYTES %d\n" % (ru * factor))
+sys.exit(rc)

package/tools/graphify-validation/lib/sample_edges.py

@@ -0,0 +1,246 @@
+"""Stratified sample of INFERRED edges + an automatic oracle (G2 precision).
+
+The oracle is a *proxy*, not a human review. For Java `calls` edges it checks
+the call site against the receiver's declared type in the same file. Verdicts:
+  ok       the receiver type (or this/super/static import) is compatible with the
+           class that owns the edge's target method
+  wrong    the call site contradicts the target (different class, name absent)
+  unknown  cannot be decided mechanically (chained call, var, lambda param ...)
+`unknown` rows go to the CSV for manual review; a filled `manual_verdict`
+column (ok|wrong) overrides the oracle when the sheet is passed back in.
+Sampling is seeded, so the same graph yields the same sample.
+"""
+from __future__ import annotations
+
+import csv
+import random
+import re
+from collections import Counter, defaultdict
+from pathlib import Path
+
+from graph_metrics import class_index, edges_of
+from gv_common import is_test_path
+
+_DECL_CACHE = {}
+
+
+def _read_lines(workdir, rel):
+    key = (str(workdir), rel)
+    if key not in _DECL_CACHE:
+        try:
+            _DECL_CACHE[key] = (Path(workdir) / rel).read_text(encoding="utf-8", errors="replace").splitlines()
+        except OSError:
+            _DECL_CACHE[key] = []
+    return _DECL_CACHE[key]
+
+
+def _declared_types(lines, ident):
+    pat = re.compile(r"\b([A-Z][A-Za-z0-9_]*(?:\.[A-Z][A-Za-z0-9_]*)*)(?:<[^;(){}=]*>)?(?:\[\])?\s+" + re.escape(ident) + r"\s*(?:[;=,)]|$)")
+    types = set()
+    for ln in lines:
+        for m in pat.finditer(ln):
+            types.add(m.group(1))
+    return types
+
+
+def _closure(name, supers):
+    seen, stack = set(), [name]
+    while stack:
+        cur = stack.pop()
+        for s in supers.get(cur, ()):
+            if s not in seen:
+                seen.add(s)
+                stack.append(s)
+    return seen
+
+
+def _compatible(receiver_type, owner, supers):
+    if receiver_type == owner:
+        return True
+    if owner in _closure(receiver_type, supers):
+        return True  # receiver is a subtype of the owner
+    if receiver_type in _closure(owner, supers):
+        return True  # receiver is an interface/base of the owner (resolved to an implementation)
+    return False
+
+
+def _typed(declared, owner, owner_stem, src_text, src_stem, supers):
+    """Tri-state compatibility of a declared (possibly dotted) type with the owner class."""
+    segs = declared.split(".")
+    last = segs[-1]
+    if len(segs) > 1:
+        if last == owner:
+            return "ok" if segs[0] == owner_stem else "wrong"
+        return "ok" if _compatible(last, owner, supers) else "wrong"
+    if last == owner:
+        if owner_stem == owner or owner_stem == src_stem:
+            return "ok"
+        # nested class referred to by its simple name: only trust it when the source names the outer class
+        return "ok" if re.search(r"\b%s\.%s\b" % (re.escape(owner_stem), re.escape(owner)), src_text) else "unknown"
+    return "ok" if _compatible(last, owner, supers) else "wrong"
+
+
+def java_oracle(edge, ns, owner_of, supers, workdir):
+    src_file = edge.get("source_file", "")
+    loc = edge.get("source_location", "L0")
+    try:
+        line_no = int(loc.lstrip("L"))
+    except ValueError:
+        return "unknown", "no line"
+    lines = _read_lines(workdir, src_file)
+    if not lines or line_no < 1 or line_no > len(lines):
+        return "unknown", "line out of range"
+    tgt = ns[edge["target"]]
+    name = tgt["label"].lstrip(".").rstrip("()")
+    window = " ".join(lines[line_no - 1:line_no + 2])
+    owner_node = ns.get(owner_of.get(edge["target"], ""), {})
+    owner = owner_node.get("label", "")
+    src_owner = ns.get(owner_of.get(edge["source"], ""), {}).get("label", "")
+    pat = re.compile(r"(?:([A-Za-z_][A-Za-z0-9_]*|\))\s*\.\s*)?\b" + re.escape(name) + r"\s*\(")
+    m = pat.search(window)
+    if not m:
+        return "unknown", "callee name not at call site"
+    recv = m.group(1)
+    if recv == ")":
+        return "unknown", "chained call"
+    if recv is None or recv in ("this", "super"):
+        allowed = {src_owner} | _closure(src_owner, supers)
+        if owner in allowed or _compatible(src_owner, owner, supers):
+            return "ok", "unqualified/this call inside owner hierarchy"
+        text = "\n".join(lines)
+        if re.search(r"import\s+static\s+[\w.]*\b%s\b\.(?:\*|%s)\s*;" % (re.escape(owner), re.escape(name)), text):
+            return "ok", "static import"
+        return "unknown", "unqualified call to a class outside the hierarchy (outer class or static import?)"
+    if recv[0].isupper():
+        return ("ok", "static call on owner") if _compatible(recv, owner, supers) else ("wrong", "static receiver %s != owner %s" % (recv, owner))
+    types = _declared_types(lines, recv)
+    if not types:
+        return "unknown", "receiver type not declared in file (var, lambda, inherited field)"
+    owner_stem = Path(owner_node.get("source_file", "")).stem
+    src_stem = Path(src_file).stem
+    src_text = "\n".join(lines)
+    verdicts = [_typed(t, owner, owner_stem, src_text, src_stem, supers) for t in sorted(types)]
+    if "ok" in verdicts:
+        return "ok", "receiver type %s compatible with owner %s" % (sorted(types)[0], owner)
+    if "unknown" in verdicts:
+        return "unknown", "nested owner %s reached through simple type name %s" % (owner, sorted(types)[0])
+    return "wrong", "receiver types %s incompatible with owner %s (%s)" % (sorted(types), owner, owner_stem)
+
+
+def weak_oracle(edge, ns, workdir):
+    """JS/TS/Python: only checks that the callee name sits on the call line and the file is reachable."""
+    src_file = edge.get("source_file", "")
+    loc = edge.get("source_location", "L0")
+    try:
+        line_no = int(loc.lstrip("L"))
+    except ValueError:
+        return "unknown", "no line"
+    lines = _read_lines(workdir, src_file)
+    if not lines or line_no < 1 or line_no > len(lines):
+        return "unknown", "line out of range"
+    tgt = ns[edge["target"]]
+    name = tgt["label"].lstrip(".").rstrip("()")
+    window = " ".join(lines[max(0, line_no - 1):line_no + 1])
+    if not re.search(r"\b" + re.escape(name) + r"\b", window):
+        return "wrong", "callee name not on call line"
+    tgt_file = tgt.get("source_file", "")
+    if tgt_file == src_file:
+        return "ok", "same file"
+    base = Path(tgt_file).stem
+    text = "\n".join(lines)
+    if re.search(r"(from|require\(|import\()\s*['\"][^'\"]*%s[^'\"]*['\"]" % re.escape(base), text):
+        return "ok", "callee name on line and target module imported"
+    return "unknown", "callee name on line but target module not imported by name"
+
+
+def build_sample(g, workdir, seed=20261028, n_ambiguous=20, n_other=40, ambiguous_min_defs=5):
+    ns, owner_of, supers = class_index(g)
+    defs = Counter(n["label"] for n in g["nodes"] if n["label"].startswith(".") and n["id"] in owner_of)
+    pool = defaultdict(list)
+    for e in edges_of(g):
+        if e.get("confidence") != "INFERRED" or e.get("relation") != "calls":
+            continue
+        if is_test_path(e.get("source_file", "")):
+            continue
+        tgt = ns.get(e["target"])
+        if not tgt:
+            continue
+        stratum = "ambiguous" if defs.get(tgt["label"], 0) >= ambiguous_min_defs else "other"
+        pool[stratum].append(e)
+    rng = random.Random(seed)
+    chosen = []
+    for stratum, k in (("ambiguous", n_ambiguous), ("other", n_other)):
+        items = sorted(pool[stratum], key=lambda e: (e["source"], e["target"], e.get("source_location", "")))
+        rng.shuffle(items)
+        for e in items[:k]:
+            chosen.append((stratum, e))
+    rows = []
+    for stratum, e in chosen:
+        src_file = e.get("source_file", "")
+        is_java = src_file.endswith(".java")
+        verdict, reason = java_oracle(e, ns, owner_of, supers, workdir) if is_java else weak_oracle(e, ns, workdir)
+        line_no = int(e.get("source_location", "L0").lstrip("L") or 0)
+        lines = _read_lines(workdir, src_file)
+        rows.append({
+            "stratum": stratum,
+            "source": ns[e["source"]]["label"],
+            "target": ns[e["target"]]["label"],
+            "target_owner": ns.get(owner_of.get(e["target"], ""), {}).get("label", ""),
+            "site": "%s:%s" % (src_file, e.get("source_location")),
+            "line_text": (lines[line_no - 1].strip()[:160] if 0 < line_no <= len(lines) else ""),
+            "oracle": verdict,
+            "reason": reason,
+            "oracle_kind": "java-receiver-type" if is_java else "weak-name-and-import",
+            "manual_verdict": "",
+        })
+    return rows, {"pool_ambiguous": len(pool["ambiguous"]), "pool_other": len(pool["other"]), "seed": seed}
+
+
+def write_csv(rows, path):
+    Path(path).parent.mkdir(parents=True, exist_ok=True)
+    cols = ["stratum", "source", "target", "target_owner", "site", "line_text", "oracle", "oracle_kind", "reason", "manual_verdict"]
+    with open(path, "w", newline="", encoding="utf-8") as fh:
+        w = csv.DictWriter(fh, fieldnames=cols)
+        w.writeheader()
+        w.writerows(rows)
+
+
+def merge_manual(rows, manual_csv):
+    if not manual_csv or not Path(manual_csv).exists():
+        return rows
+    with open(manual_csv, newline="", encoding="utf-8") as fh:
+        manual = {(r["site"], r["source"], r["target"]): r.get("manual_verdict", "").strip().lower() for r in csv.DictReader(fh)}
+    for r in rows:
+        v = manual.get((r["site"], r["source"], r["target"]))
+        if v in ("ok", "wrong"):
+            r["manual_verdict"] = v
+    return rows
+
+
+def summarize(rows, meta):
+    out = dict(meta)
+    out["sample_size"] = len(rows)
+    for scope, key in (("oracle", "oracle"),):
+        c = Counter(r[key] for r in rows)
+        out["oracle_counts"] = dict(c)
+        decided = c.get("ok", 0) + c.get("wrong", 0)
+        out["oracle_decided"] = decided
+        out["oracle_precision_pct"] = round(100.0 * c.get("ok", 0) / decided, 1) if decided else None
+        out["oracle_worst_case_pct"] = round(100.0 * c.get("ok", 0) / len(rows), 1) if rows else None
+    by = {}
+    for st in ("ambiguous", "other"):
+        sub = [r for r in rows if r["stratum"] == st]
+        c = Counter(r["oracle"] for r in sub)
+        dec = c.get("ok", 0) + c.get("wrong", 0)
+        by[st] = {"n": len(sub), "ok": c.get("ok", 0), "wrong": c.get("wrong", 0), "unknown": c.get("unknown", 0),
+                  "precision_pct": round(100.0 * c.get("ok", 0) / dec, 1) if dec else None}
+    out["by_stratum"] = by
+    manual = [r for r in rows if r["manual_verdict"] in ("ok", "wrong")]
+    out["manual_reviewed"] = len(manual)
+    if manual:
+        # manual verdict wins; oracle verdict fills the rest when it is decided
+        final = [r["manual_verdict"] or (r["oracle"] if r["oracle"] in ("ok", "wrong") else "") for r in rows]
+        decided = [v for v in final if v]
+        out["combined_decided"] = len(decided)
+        out["combined_precision_pct"] = round(100.0 * decided.count("ok") / len(decided), 1) if decided else None
+    return out

package/tools/graphify-validation/lib/spring_fixture.py

@@ -0,0 +1,81 @@
+"""Spring/DI fixture check (G3): build the graph of the synthetic fixture and compare with expected.json."""
+from __future__ import annotations
+
+from collections import defaultdict, deque
+from pathlib import Path
+
+from graph_metrics import edges_of
+from gv_common import load_json
+
+
+def _index(g):
+    by_sel = defaultdict(list)
+    for n in g["nodes"]:
+        sf = n.get("source_file") or ""
+        by_sel["%s::%s" % (Path(sf).name, n["label"])].append(n["id"])
+    return by_sel
+
+
+def _resolve(by_sel, sel):
+    return by_sel.get(sel, [])
+
+
+def check(g, expected):
+    by_sel = _index(g)
+    es = edges_of(g)
+    fwd = defaultdict(list)
+    rev = defaultdict(list)
+    for e in es:
+        fwd[e["source"]].append(e)
+        rev[e["target"]].append(e)
+
+    def reaches(frm_ids, to_ids, depth):
+        # reverse traversal from `to` (who is affected) must contain `frm`
+        seen, q = set(to_ids), deque((t, 0) for t in to_ids)
+        while q:
+            cur, d = q.popleft()
+            if cur in frm_ids:
+                return True
+            if d >= depth:
+                continue
+            for e in rev[cur]:
+                if e["source"] not in seen:
+                    seen.add(e["source"])
+                    q.append((e["source"], d + 1))
+        return False
+
+    results = {}
+    for group, items in expected["groups"].items():
+        out = []
+        for it in items:
+            frm = set(_resolve(by_sel, it["from"]))
+            to = set(_resolve(by_sel, it["to"]))
+            if not frm or not to:
+                found, why = False, "node not found: %s" % (it["from"] if not frm else it["to"])
+            elif it["kind"] == "edge":
+                rels = set(it["relations"])
+                hit = [e for f in frm for e in fwd[f] if e["target"] in to and e.get("relation") in rels]
+                found, why = bool(hit), (hit[0]["relation"] + "/" + hit[0]["confidence"]) if hit else "no edge with relation in %s" % sorted(rels)
+            elif it["kind"] == "reaches":
+                found = reaches(frm, to, it["depth"])
+                why = "reachable" if found else "not reachable within %d hops" % it["depth"]
+            elif it["kind"] == "not_reaches":
+                companion = next((o for o in out if o["id"] == it.get("requires")), None)
+                if companion is not None and not companion["found"]:
+                    found, why = False, "vacuous: companion check %s failed, so absence proves nothing" % it["requires"]
+                else:
+                    found = not reaches(frm, to, it["depth"])
+                    why = "correctly not reachable" if found else "reachable (Qualifier ignored)"
+            else:
+                found, why = False, "unknown kind"
+            out.append({"id": it["id"], "kind": it["kind"], "found": found, "detail": why})
+        n = len(out)
+        ok = sum(1 for o in out if o["found"])
+        results[group] = {"found": ok, "expected": n, "pct": round(100.0 * ok / n, 1) if n else None, "items": out}
+    head = results[expected["headline_group"]]
+    return {"groups": results, "headline_group": expected["headline_group"], "headline_pct": head["pct"],
+            "threshold_pct": expected["threshold_pct"], "pass": (head["pct"] or 0) >= expected["threshold_pct"]}
+
+
+def load_expected(path):
+    return load_json(path)