homegraph 1.2.0 → 1.5.0

package/scripts/qa_eval/llm_config.py

@@ -1,92 +0,0 @@
-"""LLM provider presets for qa_eval (DashScope Qwen / 智谱 Zhipu)."""
-
-from __future__ import annotations
-
-import os
-from dataclasses import dataclass
-
-
-@dataclass(frozen=True)
-class LlmConfig:
-    provider: str
-    api_key: str
-    base_url: str
-    model: str
-    extra_body: dict | None
-
-
-PROVIDER_DASHSCOPE = "dashscope"
-PROVIDER_ZHIPU = "zhipu"
-
-_PRESETS: dict[str, dict] = {
-    PROVIDER_DASHSCOPE: {
-        "base_url": "https://dashscope.aliyuncs.com/compatible-mode/v1",
-        "model": "qwen3-235b-a22b-instruct-2507",
-        "api_key_envs": ("DASHSCOPE_API_KEY", "OPENAI_API_KEY"),
-        "extra_body": {"enable_thinking": False},
-    },
-    PROVIDER_ZHIPU: {
-        "base_url": "https://open.bigmodel.cn/api/paas/v4/",
-        "model": "glm-4-flash",
-        "api_key_envs": ("ZHIPU_API_KEY", "ZAI_API_KEY", "OPENAI_API_KEY"),
-        "extra_body": None,
-    },
-}
-
-
-def _read_key(env_names: tuple[str, ...]) -> str | None:
-    for name in env_names:
-        val = os.environ.get(name, "").strip()
-        if val:
-            return val
-    return None
-
-
-def detect_provider() -> str:
-    """Prefer explicit ZHIPU key over DashScope when both are set."""
-    if _read_key(_PRESETS[PROVIDER_ZHIPU]["api_key_envs"]):
-        return PROVIDER_ZHIPU
-    if _read_key(_PRESETS[PROVIDER_DASHSCOPE]["api_key_envs"]):
-        return PROVIDER_DASHSCOPE
-    return PROVIDER_DASHSCOPE
-
-
-def resolve_llm_config(
-    *,
-    provider: str | None = None,
-    model: str | None = None,
-    base_url: str | None = None,
-) -> LlmConfig:
-    name = (provider or detect_provider()).strip().lower()
-    if name not in _PRESETS:
-        known = ", ".join(_PRESETS)
-        raise ValueError(f"未知 LLM provider: {provider!r}，可选: {known}")
-
-    preset = _PRESETS[name]
-    api_key = _read_key(preset["api_key_envs"])
-    if not api_key:
-        env_hint = " / ".join(preset["api_key_envs"])
-        raise RuntimeError(
-            f"未找到 {name} 的 API Key。请设置环境变量: {env_hint}\n"
-            f"  智谱: export ZHIPU_API_KEY='your-id.your-secret'\n"
-            f"  DashScope: export DASHSCOPE_API_KEY='sk-...'"
-        )
-
-    # Strip mistaken sk- prefix on Zhipu keys (id.secret format).
-    if name == PROVIDER_ZHIPU and api_key.startswith("sk-") and "." in api_key[3:]:
-        api_key = api_key[3:]
-
-    return LlmConfig(
-        provider=name,
-        api_key=api_key,
-        base_url=(base_url or preset["base_url"]).rstrip("/") + "/",
-        model=model or preset["model"],
-        extra_body=preset["extra_body"],
-    )
-
-
-def provider_help() -> str:
-    return (
-        f"dashscope (默认 Qwen, { _PRESETS[PROVIDER_DASHSCOPE]['model']}) | "
-        f"zhipu (智谱, 默认 { _PRESETS[PROVIDER_ZHIPU]['model']})"
-    )

package/scripts/qa_eval/memory_monitor.py

@@ -1,132 +0,0 @@
-"""Sample RSS for a process tree (Linux /proc). Used by qa_eval Agent runs."""
-
-from __future__ import annotations
-
-import os
-import sys
-import threading
-import time
-from contextlib import contextmanager
-from statistics import mean
-
-_PROC_AVAILABLE = sys.platform == "linux" and os.path.isdir("/proc")
-
-
-def memory_sampling_supported() -> bool:
-    """True when process-tree RSS sampling is available (Linux /proc)."""
-    return _PROC_AVAILABLE
-
-
-def _read_rss_kb(pid: int) -> int:
-    if not _PROC_AVAILABLE:
-        return 0
-    try:
-        with open(f"/proc/{pid}/status", encoding="utf-8") as f:
-            for line in f:
-                if line.startswith("VmRSS:"):
-                    return int(line.split()[1])
-    except OSError:
-        return 0
-    return 0
-
-
-def _build_ppid_map() -> dict[int, list[int]]:
-    children: dict[int, list[int]] = {}
-    if not _PROC_AVAILABLE:
-        return children
-    try:
-        proc_entries = os.listdir("/proc")
-    except OSError:
-        return children
-    for name in proc_entries:
-        if not name.isdigit():
-            continue
-        pid = int(name)
-        try:
-            with open(f"/proc/{pid}/stat", encoding="utf-8") as f:
-                stat = f.read()
-            # comm may contain ')'; ppid follows closing paren + space
-            after = stat.rsplit(")", 1)[1].split()
-            ppid = int(after[1])
-        except (OSError, IndexError, ValueError):
-            continue
-        children.setdefault(ppid, []).append(pid)
-    return children
-
-
-def process_tree_pids(root_pid: int) -> set[int]:
-    if root_pid <= 0:
-        return set()
-    children = _build_ppid_map()
-    seen: set[int] = set()
-    stack = [root_pid]
-    while stack:
-        pid = stack.pop()
-        if pid in seen:
-            continue
-        seen.add(pid)
-        stack.extend(children.get(pid, []))
-    return seen
-
-
-def tree_rss_kb(root_pid: int) -> int:
-    return sum(_read_rss_kb(pid) for pid in process_tree_pids(root_pid))
-
-
-class MemorySampler:
-    """Background RSS sampler for root_pid and its descendants."""
-
-    def __init__(self, interval_sec: float = 0.25) -> None:
-        self.interval_sec = interval_sec
-        self._root_pid = os.getpid()
-        self._samples_kb: list[int] = []
-        self._stop = threading.Event()
-        self._thread: threading.Thread | None = None
-
-    def start(self, root_pid: int | None = None) -> None:
-        self._root_pid = root_pid if root_pid is not None else os.getpid()
-        self._samples_kb = []
-        self._stop.clear()
-        if not _PROC_AVAILABLE:
-            self._thread = None
-            return
-        self._thread = threading.Thread(target=self._loop, daemon=True)
-        self._thread.start()
-
-    def _loop(self) -> None:
-        while not self._stop.is_set():
-            try:
-                self._samples_kb.append(tree_rss_kb(self._root_pid))
-            except OSError:
-                break
-            self._stop.wait(self.interval_sec)
-
-    @property
-    def last_stats(self) -> dict[str, float | None]:
-        """Peak/avg RSS so far (safe to read while sampling is active)."""
-        if not self._samples_kb:
-            return {"peak_rss_mb": None, "avg_rss_mb": None}
-        peak_kb = max(self._samples_kb)
-        avg_kb = mean(self._samples_kb)
-        return {
-            "peak_rss_mb": round(peak_kb / 1024, 2),
-            "avg_rss_mb": round(avg_kb / 1024, 2),
-        }
-
-    def stop(self) -> dict[str, float | None]:
-        self._stop.set()
-        if self._thread:
-            self._thread.join(timeout=2.0)
-        return self.last_stats
-
-
-@contextmanager
-def sample_memory(root_pid: int | None = None):
-    """Sample process-tree RSS while the wrapped block runs."""
-    sampler = MemorySampler()
-    pid = root_pid if root_pid is not None else os.getpid()
-    sampler.start(pid)
-    try:
-        yield sampler
-    finally:
-        sampler.stop()

package/scripts/qa_eval/my_answer_accuracy.py

@@ -1,187 +0,0 @@
-"""LLM-as-Judge Answer Accuracy (0–1), dual-prompt average — standalone, no ragas."""
-
-from __future__ import annotations
-
-import logging
-import math
-import re
-from dataclasses import dataclass
-
-from openai import AsyncOpenAI
-
-logger = logging.getLogger(__name__)
-
-TEMPLATE_ACCURACY1 = (
-    "Instruction: You are an assistant for rating a User Answer given a Question. "
-    "The Question is fully answered by the Reference Answer.\n"
-    "Focus on whether the User Answer captures the key meaning and intent of the Reference Answer. "
-    "Minor wording differences, alternative phrasing, or extra irrelevant information should not lower the rating.\n"
-    "Say 5, if the User Answer conveys the same overall meaning and essential content as the Reference Answer.\n"
-    "Say 4, if the User Answer is mostly correct, broadly aligned with the Reference Answer, with only small omissions or slight issues.\n"
-    "Say 3, if the User Answer is reasonably correct, covering several important aspects of the Reference Answer, even if incomplete.\n"
-    "Say 2, if the User Answer shows some overlap, but misses many essential aspects.\n"
-    "Say 1, if the User Answer has very little overlap with the Reference Answer.\n"
-    "Say 0, if the User Answer does not address the question at all.\n"
-    "Do not explain or justify your rating. Your rating must be a single integer 0–5.\n"
-    "If the User Answer is empty, the rating is 0.\n"
-    "### Question: {query}\n"
-    "### {answer0}: {sentence_inference}\n"
-    "### {answer1}: {sentence_true}\n"
-    "The rating is:\n"
-)
-
-TEMPLATE_ACCURACY2 = (
-    "I will rate the User Answer in comparison to the Reference Answer for a given Question.\n"
-    "The evaluation must focus mainly on whether the User Answer conveys the essential meaning and intent of the Reference Answer. "
-    "Minor details, alternative phrasing, or extra content should not strongly affect the rating.\n"
-    "If the User Answer is empty, the rating is 0.\n"
-    "The scale is 0 to 5:\n"
-    "0 = Does not address the question at all\n"
-    "1 = Very limited overlap with the Reference Answer\n"
-    "2 = Some overlap, but misses many essential aspects\n"
-    "3 = Reasonably correct, covers several important aspects but incomplete\n"
-    "4 = Mostly correct, broadly aligned with the Reference Answer with only small gaps\n"
-    "5 = Fully correct, conveys the same overall meaning and essential content as the Reference Answer\n"
-    "I will provide the rating as a single integer 0–5 without explanation.\n\n"
-    "Question: {query}\n\n"
-    "{answer0}: {sentence_inference}\n\n"
-    "{answer1}: {sentence_true}\n\n"
-    "Rating: "
-)
-
-
-def remove_tool_calls(text: str) -> str:
-    """Strip --- ... --- blocks that contain agent tool invocations before judging."""
-    pattern = re.compile(
-        r"---\s*[\s\S]*?"
-        r"(ReadFile|SearchText|ReadManyFiles|ReadFolder|SemanticSearch|FindFiles|Custom|HomegraphQuery|HomegraphExplore)"
-        r"[\s\S]*?---",
-        re.MULTILINE,
-    )
-    cleaned = re.sub(pattern, "", text)
-    return re.sub(r"\n{3,}", "\n\n", cleaned).strip()
-
-
-def extract_json_blocks_answerbyCOT(text: str) -> str:
-    """Remove redacted_thinking blocks from chain-of-thought model output."""
-    cleaned = re.sub(
-        r"<think>.*?</think>",
-        "",
-        text,
-        flags=re.DOTALL | re.IGNORECASE,
-    )
-    return cleaned.strip()
-
-
-def process_score(response: str) -> float:
-    for i in range(6):
-        if str(i) in response.strip():
-            return i / 5
-    return math.nan
-
-
-def average_scores(score0: float, score1: float) -> float:
-    if not math.isnan(score0) and not math.isnan(score1):
-        return (score0 + score1) / 2
-    if math.isnan(score0) and math.isnan(score1):
-        return math.nan
-    return score0 if math.isnan(score1) else score1
-
-
-@dataclass
-class MyAnswerAccuracy:
-    """Dual-template LLM judge; scores normalized to 0–1."""
-
-    client: AsyncOpenAI
-    model: str
-    retry: int = 5
-    extra_body: dict | None = None
-
-    @classmethod
-    def create(
-        cls,
-        *,
-        api_key: str,
-        base_url: str,
-        model: str,
-        timeout_sec: float = 1800,
-        extra_body: dict | None = None,
-    ) -> MyAnswerAccuracy:
-        client = AsyncOpenAI(api_key=api_key, base_url=base_url, timeout=timeout_sec)
-        return cls(client=client, model=model, extra_body=extra_body)
-
-    async def _chat(self, prompt: str) -> str:
-        kwargs: dict = {
-            "model": self.model,
-            "messages": [{"role": "user", "content": prompt}],
-            "temperature": 0.10,
-        }
-        if self.extra_body:
-            kwargs["extra_body"] = self.extra_body
-        resp = await self.client.chat.completions.create(**kwargs)
-        return (resp.choices[0].message.content or "").strip()
-
-    async def _rate(self, prompt: str) -> float:
-        for attempt in range(self.retry):
-            try:
-                text = await self._chat(prompt)
-                score = process_score(text)
-                if not math.isnan(score):
-                    return score
-            except Exception as e:
-                logger.warning("Judge call failed (attempt %s): %s", attempt + 1, e)
-            logger.warning("Judge retry (invalid rating): %s", attempt + 1)
-        return math.nan
-
-    async def single_turn_ascore(
-        self,
-        *,
-        user_input: str,
-        response: str,
-        reference: str,
-    ) -> float:
-        prompt1 = TEMPLATE_ACCURACY1.format(
-            query=user_input,
-            answer0="User Answer",
-            answer1="Reference Answer",
-            sentence_inference=response,
-            sentence_true=reference,
-        )
-        prompt2 = TEMPLATE_ACCURACY2.format(
-            query=user_input,
-            answer0="Reference Answer",
-            answer1="User Answer",
-            sentence_inference=reference,
-            sentence_true=response,
-        )
-        try:
-            s0 = await self._rate(prompt1)
-            s1 = await self._rate(prompt2)
-            return average_scores(s0, s1)
-        except Exception as e:
-            logger.warning("Judge error: %s", e)
-            return math.nan
-
-    async def evaluate_answer_accuracy_single(self, sample_data: dict) -> dict:
-        result = sample_data.copy()
-        try:
-            raw = str(sample_data.get("output_answer", ""))
-            raw = extract_json_blocks_answerbyCOT(raw)
-            response = remove_tool_calls(raw)
-            score = await self.single_turn_ascore(
-                user_input=str(sample_data.get("query", "")),
-                response=response,
-                reference=str(sample_data.get("reference_answer", "")),
-            )
-            result["answer_accuracy_score"] = float(score) if not math.isnan(score) else 0.0
-            result["evaluation_status"] = "success" if not math.isnan(score) else "failed"
-            if math.isnan(score):
-                result["error_message"] = "Judge 未返回有效 0–5 分数"
-        except Exception as e:
-            logger.error("评估答案准确性时出错: %s", e)
-            result["answer_accuracy_score"] = 0.0
-            result["evaluation_status"] = "error"
-            result["error_message"] = str(e)
-
-        result.pop("evaluation", None)
-        return result

package/scripts/qa_eval/requirements.txt

@@ -1,2 +0,0 @@
-openai>=1.40.0
-tqdm>=4.66.0