homegraph 1.0.0 → 1.1.2

package/scripts/qa_eval/llm_config.py

@@ -0,0 +1,92 @@
+"""LLM provider presets for qa_eval (DashScope Qwen / 智谱 Zhipu)."""
+
+from __future__ import annotations
+
+import os
+from dataclasses import dataclass
+
+
+@dataclass(frozen=True)
+class LlmConfig:
+    provider: str
+    api_key: str
+    base_url: str
+    model: str
+    extra_body: dict | None
+
+
+PROVIDER_DASHSCOPE = "dashscope"
+PROVIDER_ZHIPU = "zhipu"
+
+_PRESETS: dict[str, dict] = {
+    PROVIDER_DASHSCOPE: {
+        "base_url": "https://dashscope.aliyuncs.com/compatible-mode/v1",
+        "model": "qwen3-235b-a22b-instruct-2507",
+        "api_key_envs": ("DASHSCOPE_API_KEY", "OPENAI_API_KEY"),
+        "extra_body": {"enable_thinking": False},
+    },
+    PROVIDER_ZHIPU: {
+        "base_url": "https://open.bigmodel.cn/api/paas/v4/",
+        "model": "glm-4-flash",
+        "api_key_envs": ("ZHIPU_API_KEY", "ZAI_API_KEY", "OPENAI_API_KEY"),
+        "extra_body": None,
+    },
+}
+
+
+def _read_key(env_names: tuple[str, ...]) -> str | None:
+    for name in env_names:
+        val = os.environ.get(name, "").strip()
+        if val:
+            return val
+    return None
+
+
+def detect_provider() -> str:
+    """Prefer explicit ZHIPU key over DashScope when both are set."""
+    if _read_key(_PRESETS[PROVIDER_ZHIPU]["api_key_envs"]):
+        return PROVIDER_ZHIPU
+    if _read_key(_PRESETS[PROVIDER_DASHSCOPE]["api_key_envs"]):
+        return PROVIDER_DASHSCOPE
+    return PROVIDER_DASHSCOPE
+
+
+def resolve_llm_config(
+    *,
+    provider: str | None = None,
+    model: str | None = None,
+    base_url: str | None = None,
+) -> LlmConfig:
+    name = (provider or detect_provider()).strip().lower()
+    if name not in _PRESETS:
+        known = ", ".join(_PRESETS)
+        raise ValueError(f"未知 LLM provider: {provider!r}，可选: {known}")
+
+    preset = _PRESETS[name]
+    api_key = _read_key(preset["api_key_envs"])
+    if not api_key:
+        env_hint = " / ".join(preset["api_key_envs"])
+        raise RuntimeError(
+            f"未找到 {name} 的 API Key。请设置环境变量: {env_hint}\n"
+            f"  智谱: export ZHIPU_API_KEY='your-id.your-secret'\n"
+            f"  DashScope: export DASHSCOPE_API_KEY='sk-...'"
+        )
+
+    # Strip mistaken sk- prefix on Zhipu keys (id.secret format).
+    if name == PROVIDER_ZHIPU and api_key.startswith("sk-") and "." in api_key[3:]:
+        api_key = api_key[3:]
+
+    return LlmConfig(
+        provider=name,
+        api_key=api_key,
+        base_url=(base_url or preset["base_url"]).rstrip("/") + "/",
+        model=model or preset["model"],
+        extra_body=preset["extra_body"],
+    )
+
+
+def provider_help() -> str:
+    return (
+        f"dashscope (默认 Qwen, { _PRESETS[PROVIDER_DASHSCOPE]['model']}) | "
+        f"zhipu (智谱, 默认 { _PRESETS[PROVIDER_ZHIPU]['model']})"
+    )

package/scripts/qa_eval/memory_monitor.py

@@ -0,0 +1,132 @@
+"""Sample RSS for a process tree (Linux /proc). Used by qa_eval Agent runs."""
+
+from __future__ import annotations
+
+import os
+import sys
+import threading
+import time
+from contextlib import contextmanager
+from statistics import mean
+
+_PROC_AVAILABLE = sys.platform == "linux" and os.path.isdir("/proc")
+
+
+def memory_sampling_supported() -> bool:
+    """True when process-tree RSS sampling is available (Linux /proc)."""
+    return _PROC_AVAILABLE
+
+
+def _read_rss_kb(pid: int) -> int:
+    if not _PROC_AVAILABLE:
+        return 0
+    try:
+        with open(f"/proc/{pid}/status", encoding="utf-8") as f:
+            for line in f:
+                if line.startswith("VmRSS:"):
+                    return int(line.split()[1])
+    except OSError:
+        return 0
+    return 0
+
+
+def _build_ppid_map() -> dict[int, list[int]]:
+    children: dict[int, list[int]] = {}
+    if not _PROC_AVAILABLE:
+        return children
+    try:
+        proc_entries = os.listdir("/proc")
+    except OSError:
+        return children
+    for name in proc_entries:
+        if not name.isdigit():
+            continue
+        pid = int(name)
+        try:
+            with open(f"/proc/{pid}/stat", encoding="utf-8") as f:
+                stat = f.read()
+            # comm may contain ')'; ppid follows closing paren + space
+            after = stat.rsplit(")", 1)[1].split()
+            ppid = int(after[1])
+        except (OSError, IndexError, ValueError):
+            continue
+        children.setdefault(ppid, []).append(pid)
+    return children
+
+
+def process_tree_pids(root_pid: int) -> set[int]:
+    if root_pid <= 0:
+        return set()
+    children = _build_ppid_map()
+    seen: set[int] = set()
+    stack = [root_pid]
+    while stack:
+        pid = stack.pop()
+        if pid in seen:
+            continue
+        seen.add(pid)
+        stack.extend(children.get(pid, []))
+    return seen
+
+
+def tree_rss_kb(root_pid: int) -> int:
+    return sum(_read_rss_kb(pid) for pid in process_tree_pids(root_pid))
+
+
+class MemorySampler:
+    """Background RSS sampler for root_pid and its descendants."""
+
+    def __init__(self, interval_sec: float = 0.25) -> None:
+        self.interval_sec = interval_sec
+        self._root_pid = os.getpid()
+        self._samples_kb: list[int] = []
+        self._stop = threading.Event()
+        self._thread: threading.Thread | None = None
+
+    def start(self, root_pid: int | None = None) -> None:
+        self._root_pid = root_pid if root_pid is not None else os.getpid()
+        self._samples_kb = []
+        self._stop.clear()
+        if not _PROC_AVAILABLE:
+            self._thread = None
+            return
+        self._thread = threading.Thread(target=self._loop, daemon=True)
+        self._thread.start()
+
+    def _loop(self) -> None:
+        while not self._stop.is_set():
+            try:
+                self._samples_kb.append(tree_rss_kb(self._root_pid))
+            except OSError:
+                break
+            self._stop.wait(self.interval_sec)
+
+    @property
+    def last_stats(self) -> dict[str, float | None]:
+        """Peak/avg RSS so far (safe to read while sampling is active)."""
+        if not self._samples_kb:
+            return {"peak_rss_mb": None, "avg_rss_mb": None}
+        peak_kb = max(self._samples_kb)
+        avg_kb = mean(self._samples_kb)
+        return {
+            "peak_rss_mb": round(peak_kb / 1024, 2),
+            "avg_rss_mb": round(avg_kb / 1024, 2),
+        }
+
+    def stop(self) -> dict[str, float | None]:
+        self._stop.set()
+        if self._thread:
+            self._thread.join(timeout=2.0)
+        return self.last_stats
+
+
+@contextmanager
+def sample_memory(root_pid: int | None = None):
+    """Sample process-tree RSS while the wrapped block runs."""
+    sampler = MemorySampler()
+    pid = root_pid if root_pid is not None else os.getpid()
+    sampler.start(pid)
+    try:
+        yield sampler
+    finally:
+        sampler.stop()

package/scripts/qa_eval/my_answer_accuracy.py

@@ -0,0 +1,187 @@
+"""LLM-as-Judge Answer Accuracy (0–1), dual-prompt average — standalone, no ragas."""
+
+from __future__ import annotations
+
+import logging
+import math
+import re
+from dataclasses import dataclass
+
+from openai import AsyncOpenAI
+
+logger = logging.getLogger(__name__)
+
+TEMPLATE_ACCURACY1 = (
+    "Instruction: You are an assistant for rating a User Answer given a Question. "
+    "The Question is fully answered by the Reference Answer.\n"
+    "Focus on whether the User Answer captures the key meaning and intent of the Reference Answer. "
+    "Minor wording differences, alternative phrasing, or extra irrelevant information should not lower the rating.\n"
+    "Say 5, if the User Answer conveys the same overall meaning and essential content as the Reference Answer.\n"
+    "Say 4, if the User Answer is mostly correct, broadly aligned with the Reference Answer, with only small omissions or slight issues.\n"
+    "Say 3, if the User Answer is reasonably correct, covering several important aspects of the Reference Answer, even if incomplete.\n"
+    "Say 2, if the User Answer shows some overlap, but misses many essential aspects.\n"
+    "Say 1, if the User Answer has very little overlap with the Reference Answer.\n"
+    "Say 0, if the User Answer does not address the question at all.\n"
+    "Do not explain or justify your rating. Your rating must be a single integer 0–5.\n"
+    "If the User Answer is empty, the rating is 0.\n"
+    "### Question: {query}\n"
+    "### {answer0}: {sentence_inference}\n"
+    "### {answer1}: {sentence_true}\n"
+    "The rating is:\n"
+)
+
+TEMPLATE_ACCURACY2 = (
+    "I will rate the User Answer in comparison to the Reference Answer for a given Question.\n"
+    "The evaluation must focus mainly on whether the User Answer conveys the essential meaning and intent of the Reference Answer. "
+    "Minor details, alternative phrasing, or extra content should not strongly affect the rating.\n"
+    "If the User Answer is empty, the rating is 0.\n"
+    "The scale is 0 to 5:\n"
+    "0 = Does not address the question at all\n"
+    "1 = Very limited overlap with the Reference Answer\n"
+    "2 = Some overlap, but misses many essential aspects\n"
+    "3 = Reasonably correct, covers several important aspects but incomplete\n"
+    "4 = Mostly correct, broadly aligned with the Reference Answer with only small gaps\n"
+    "5 = Fully correct, conveys the same overall meaning and essential content as the Reference Answer\n"
+    "I will provide the rating as a single integer 0–5 without explanation.\n\n"
+    "Question: {query}\n\n"
+    "{answer0}: {sentence_inference}\n\n"
+    "{answer1}: {sentence_true}\n\n"
+    "Rating: "
+)
+
+
+def remove_tool_calls(text: str) -> str:
+    """Strip --- ... --- blocks that contain agent tool invocations before judging."""
+    pattern = re.compile(
+        r"---\s*[\s\S]*?"
+        r"(ReadFile|SearchText|ReadManyFiles|ReadFolder|SemanticSearch|FindFiles|Custom|HomegraphQuery|HomegraphExplore)"
+        r"[\s\S]*?---",
+        re.MULTILINE,
+    )
+    cleaned = re.sub(pattern, "", text)
+    return re.sub(r"\n{3,}", "\n\n", cleaned).strip()
+
+
+def extract_json_blocks_answerbyCOT(text: str) -> str:
+    """Remove redacted_thinking blocks from chain-of-thought model output."""
+    cleaned = re.sub(
+        r"<think>.*?</think>",
+        "",
+        text,
+        flags=re.DOTALL | re.IGNORECASE,
+    )
+    return cleaned.strip()
+
+
+def process_score(response: str) -> float:
+    for i in range(6):
+        if str(i) in response.strip():
+            return i / 5
+    return math.nan
+
+
+def average_scores(score0: float, score1: float) -> float:
+    if not math.isnan(score0) and not math.isnan(score1):
+        return (score0 + score1) / 2
+    if math.isnan(score0) and math.isnan(score1):
+        return math.nan
+    return score0 if math.isnan(score1) else score1
+
+
+@dataclass
+class MyAnswerAccuracy:
+    """Dual-template LLM judge; scores normalized to 0–1."""
+
+    client: AsyncOpenAI
+    model: str
+    retry: int = 5
+    extra_body: dict | None = None
+
+    @classmethod
+    def create(
+        cls,
+        *,
+        api_key: str,
+        base_url: str,
+        model: str,
+        timeout_sec: float = 1800,
+        extra_body: dict | None = None,
+    ) -> MyAnswerAccuracy:
+        client = AsyncOpenAI(api_key=api_key, base_url=base_url, timeout=timeout_sec)
+        return cls(client=client, model=model, extra_body=extra_body)
+
+    async def _chat(self, prompt: str) -> str:
+        kwargs: dict = {
+            "model": self.model,
+            "messages": [{"role": "user", "content": prompt}],
+            "temperature": 0.10,
+        }
+        if self.extra_body:
+            kwargs["extra_body"] = self.extra_body
+        resp = await self.client.chat.completions.create(**kwargs)
+        return (resp.choices[0].message.content or "").strip()
+
+    async def _rate(self, prompt: str) -> float:
+        for attempt in range(self.retry):
+            try:
+                text = await self._chat(prompt)
+                score = process_score(text)
+                if not math.isnan(score):
+                    return score
+            except Exception as e:
+                logger.warning("Judge call failed (attempt %s): %s", attempt + 1, e)
+            logger.warning("Judge retry (invalid rating): %s", attempt + 1)
+        return math.nan
+
+    async def single_turn_ascore(
+        self,
+        *,
+        user_input: str,
+        response: str,
+        reference: str,
+    ) -> float:
+        prompt1 = TEMPLATE_ACCURACY1.format(
+            query=user_input,
+            answer0="User Answer",
+            answer1="Reference Answer",
+            sentence_inference=response,
+            sentence_true=reference,
+        )
+        prompt2 = TEMPLATE_ACCURACY2.format(
+            query=user_input,
+            answer0="Reference Answer",
+            answer1="User Answer",
+            sentence_inference=reference,
+            sentence_true=response,
+        )
+        try:
+            s0 = await self._rate(prompt1)
+            s1 = await self._rate(prompt2)
+            return average_scores(s0, s1)
+        except Exception as e:
+            logger.warning("Judge error: %s", e)
+            return math.nan
+
+    async def evaluate_answer_accuracy_single(self, sample_data: dict) -> dict:
+        result = sample_data.copy()
+        try:
+            raw = str(sample_data.get("output_answer", ""))
+            raw = extract_json_blocks_answerbyCOT(raw)
+            response = remove_tool_calls(raw)
+            score = await self.single_turn_ascore(
+                user_input=str(sample_data.get("query", "")),
+                response=response,
+                reference=str(sample_data.get("reference_answer", "")),
+            )
+            result["answer_accuracy_score"] = float(score) if not math.isnan(score) else 0.0
+            result["evaluation_status"] = "success" if not math.isnan(score) else "failed"
+            if math.isnan(score):
+                result["error_message"] = "Judge 未返回有效 0–5 分数"
+        except Exception as e:
+            logger.error("评估答案准确性时出错: %s", e)
+            result["answer_accuracy_score"] = 0.0
+            result["evaluation_status"] = "error"
+            result["error_message"] = str(e)
+
+        result.pop("evaluation", None)
+        return result

package/scripts/qa_eval/requirements.txt

@@ -0,0 +1,2 @@
+openai>=1.40.0
+tqdm>=4.66.0