homegraph 1.1.2 → 1.2.0

package/scripts/qa_eval/data/.gitignore

@@ -1,4 +1,4 @@
-# test-set.jsonl tracked; reports + other artifacts stay local.
-*
-!.gitignore
-!test-set.jsonl
+# test-set.jsonl tracked; reports + other artifacts stay local.
+*
+!.gitignore
+!test-set.jsonl

package/scripts/qa_eval/data/test-set.jsonl

@@ -1,22 +1,2 @@
-{"id": "R01", "level0": "camera", "level1": "ColorUtil", "category_l1": "检索", "category_l2": "函数", "difficulty": "简单", "language": "ArkTS", "query": "getColorString 函数在哪个文件里定义？", "reference_answer": "getColorString 定义在 camera/common/src/main/ets/utils/ColorUtil.ets，是一个 export function。"}
-{"id": "R02", "level0": "platform", "level1": "BenchmarkHub", "category_l1": "检索", "category_l2": "函数", "difficulty": "简单", "language": "ArkTS", "query": "launchPhotosFromDesktop 函数定义在哪个文件？", "reference_answer": "launchPhotosFromDesktop 定义在 platform/src/main/ets/BenchmarkHub.ets，与 BenchmarkHub 类同文件导出。"}
-{"id": "R03", "level0": "photos", "level1": "BenchmarkPhotosLink", "category_l1": "检索", "category_l2": "函数", "difficulty": "简单", "language": "ArkTS", "query": "photosBenchmarkEntry 函数在哪里？", "reference_answer": "photosBenchmarkEntry 定义在 photos/common/src/main/ets/benchmark/BenchmarkPhotosLink.ets。"}
-{"id": "R04", "level0": "platform", "level1": "BenchmarkHub", "category_l1": "检索", "category_l2": "类", "difficulty": "简单", "language": "ArkTS", "query": "BenchmarkHub 类在哪个文件？", "reference_answer": "BenchmarkHub 类定义在 platform/src/main/ets/BenchmarkHub.ets。"}
-{"id": "R05", "level0": "photos", "level1": "DeviceInfo", "category_l1": "检索", "category_l2": "类", "difficulty": "简单", "language": "ArkTS", "query": "BenchmarkHub 引用的 Photos DeviceInfo 类在哪个路径？", "reference_answer": "BenchmarkHub 从 @benchmark/photos-common/src/main/ets/default/utils/DeviceInfo.ets 导入 DeviceInfo；类定义在 photos/common/src/main/ets/default/utils/DeviceInfo.ets。"}
-{"id": "R06", "level0": "platform", "level1": "BenchmarkHub", "category_l1": "检索", "category_l2": "常量", "difficulty": "简单", "language": "ArkTS", "query": "BenchmarkHub 里 PHOTOS_BUNDLE 常量的值是什么？", "reference_answer": "PHOTOS_BUNDLE 在 platform/src/main/ets/BenchmarkHub.ets 中定义为字符串 'com.ohos.photos'。"}
-{"id": "R07", "level0": "platform", "level1": "package", "category_l1": "检索", "category_l2": "文件", "difficulty": "简单", "language": "ArkTS", "query": "@benchmark/platform 包对外 re-export 了哪些 BenchmarkHub 相关符号？", "reference_answer": "platform/index.ets 重新导出 BenchmarkHub、launchPhotosFromDesktop、openCameraWithScene，来源均为 ./src/main/ets/BenchmarkHub。"}
-
-{"id": "E01", "level0": "platform", "level1": "BenchmarkHub", "category_l1": "解读", "category_l2": "模块/功能/算法", "difficulty": "简单", "language": "ArkTS", "query": "BenchmarkHub 这个模块在整个 benchmark 工程里是干什么的？", "reference_answer": "BenchmarkHub 是跨应用集成 hub，用于 monorepo 模块互联的 homegraph benchmark fixture；文件头注释说明其职责是 cross-app integration hub / monorepo module wiring。"}
-{"id": "E02", "level0": "monorepo", "level1": "workspace", "category_l1": "解读", "category_l2": "模块/功能/算法", "difficulty": "困难", "language": "ArkTS", "query": "这个 benchmark 工程是如何把 photos、camera、scene_board 和 platform 组织成 monorepo 的？", "reference_answer": "根目录 build-profile.json5 声明约 153 个模块；根 package.json 用 npm workspaces 映射 @benchmark/* 包；platform 为 @benchmark/platform，photos/common、camera/common、scene_board 的 windowsceneinterfaces 分别对应 @benchmark/photos-common、@benchmark/camera-common、@benchmark/windowscene-interfaces。各 bridge 文件通过 package import 而非相对路径引用。"}
-{"id": "E03", "level0": "platform", "level1": "BenchmarkHub", "category_l1": "解读", "category_l2": "函数", "difficulty": "简单", "language": "ArkTS", "query": "BenchmarkHub.isTabletLayout() 是如何判断平板布局的？", "reference_answer": "BenchmarkHub.isTabletLayout() 直接调用从 photos-common 导入的 PhotosDeviceInfo.isTablet()，自身不包含判断逻辑，而是委托 photos 模块的 DeviceInfo。"}
-{"id": "E04", "level0": "camera", "level1": "ColorUtil", "category_l1": "解读", "category_l2": "函数", "difficulty": "简单", "language": "ArkTS", "query": "getColorString() 在什么条件下返回 #0A59F7，什么条件下返回 #FF0000？", "reference_answer": "在 ColorUtil.ets 中，若 CameraAppCapability.getInstance().getIsNovaProduct() 为真，或 DeviceInfo.isTablet() 为真，则返回 '#0A59F7'；否则返回 '#FF0000'。"}
-{"id": "E05", "level0": "monorepo", "level1": "bridge", "category_l1": "解读", "category_l2": "设计模式/系统架构", "difficulty": "困难", "language": "ArkTS", "query": "BenchmarkPhotosLink、BenchmarkCameraLink、BenchmarkSceneLink 这类 bridge 文件的设计意图是什么？", "reference_answer": "它们是各子应用模块指向 platform hub 的 benchmark 桥接文件，使用 @benchmark/platform 等 package import（非相对路径），刻意制造跨 workspace 包的静态依赖，供 homegraph 索引和跨模块调用链评测。"}
-{"id": "E06", "level0": "platform", "level1": "BenchmarkHub", "category_l1": "解读", "category_l2": "代码片段", "difficulty": "简单", "language": "ArkTS", "query": "launchPhotosFromDesktop() 返回的字符串格式是什么？", "reference_answer": "返回形如 launch:${PHOTOS_BUNDLE}:${layout} 的字符串，其中 layout 由 isTabletLayout() 决定为 'tablet' 或 'phone'，PHOTOS_BUNDLE 为 com.ohos.photos。示例：launch:com.ohos.photos:tablet。"}
-{"id": "E07", "level0": "platform", "level1": "BenchmarkHub", "category_l1": "解读", "category_l2": "类", "difficulty": "简单", "language": "ArkTS", "query": "BenchmarkHub.productSuite() 返回什么？", "reference_answer": "返回 string[]，按顺序包含 SCENE_BUNDLE、PHOTOS_BUNDLE、CAMERA_BUNDLE，即 com.ohos.sceneboard、com.ohos.photos、com.ohos.camera。"}
-
-{"id": "D01", "level0": "platform", "level1": "BenchmarkHub", "category_l1": "依赖", "category_l2": "跨语言依赖", "difficulty": "困难", "language": "ArkTS", "query": "BenchmarkHub.ets 通过 @benchmark 包 import 依赖了哪些其他 workspace 模块？", "reference_answer": "BenchmarkHub.ets 依赖三处 @benchmark 包：@benchmark/photos-common 的 DeviceInfo.ets、@benchmark/camera-common 的 ColorUtil.ets（getColorString）、@benchmark/windowscene-interfaces 的 Context.d.ts。"}
-{"id": "D02", "level0": "platform", "level1": "BenchmarkHub", "category_l1": "依赖", "category_l2": "函数", "difficulty": "简单", "language": "ArkTS", "query": "openCameraWithScene() 内部调用了 BenchmarkHub 的哪些方法？", "reference_answer": "openCameraWithScene 只调用 BenchmarkHub.cameraAccentColor() 获取 accent 颜色，再拼进返回字符串 open:${CAMERA_BUNDLE}:accent=${color}。"}
-{"id": "D03", "level0": "scene_board", "level1": "BenchmarkSceneLink", "category_l1": "依赖", "category_l2": "模块/功能/算法", "difficulty": "困难", "language": "ArkTS", "query": "sceneBoardBenchmarkEntry() 的跨模块调用链是怎样的？", "reference_answer": "sceneBoardBenchmarkEntry（BenchmarkSceneLink.ets）依次调用 BenchmarkHub.cameraAccentColor()、launchPhotosFromDesktop()、openCameraWithScene()；后两者在 platform/BenchmarkHub.ets，cameraAccentColor 又依赖 camera-common 的 getColorString，launchPhotosFromDesktop 依赖 photos-common 的 DeviceInfo.isTablet()。"}
-{"id": "D04", "level0": "photos", "level1": "BenchmarkPhotosLink", "category_l1": "依赖", "category_l2": "函数", "difficulty": "简单", "language": "ArkTS", "query": "photosBenchmarkEntry() 如何连接到 camera 相关逻辑？", "reference_answer": "photosBenchmarkEntry 从 @benchmark/platform 导入 openCameraWithScene，调用 BenchmarkHub.productSuite() 和 openCameraWithScene()，从而经 platform hub 间接依赖 camera-common 的 getColorString。"}
-{"id": "D05", "level0": "camera", "level1": "BenchmarkCameraLink", "category_l1": "依赖", "category_l2": "函数", "difficulty": "简单", "language": "ArkTS", "query": "cameraBenchmarkEntry() 如何依赖 photos 模块？", "reference_answer": "cameraBenchmarkEntry 调用 BenchmarkHub.isTabletLayout()（委托 photos-common DeviceInfo.isTablet()）和 launchPhotosFromDesktop()（同样依赖 isTabletLayout），并通过 launchPhotosFromDesktop 连到 photos bundle 常量。"}
-{"id": "D06", "level0": "platform", "level1": "BenchmarkHub", "category_l1": "依赖", "category_l2": "类", "difficulty": "困难", "language": "ArkTS", "query": "从 photos 的 DeviceInfo.isTablet() 到 sceneBoardBenchmarkEntry 的完整静态依赖路径是什么？", "reference_answer": "DeviceInfo.isTablet()（photos-common）← BenchmarkHub.isTabletLayout() ← launchPhotosFromDesktop()（platform）← sceneBoardBenchmarkEntry()（BenchmarkSceneLink.ets，scene_board 经 @benchmark/platform 导入）。sceneBoardBenchmarkEntry 还在同一函数内调用 cameraAccentColor 和 openCameraWithScene。"}
+{"id": "R01", "level0": "camera", "level1": "ColorUtil", "category_l1": "检索", "category_l2": "函数", "difficulty": "简单", "language": "ArkTS", "query": "getColorString 函数在哪个文件里定义？", "reference_answer": "getColorString 定义在 camera/common/src/main/ets/utils/ColorUtil.ets，是一个 export function。"}
+{"id": "R02", "level0": "platform", "level1": "BenchmarkHub", "category_l1": "检索", "category_l2": "函数", "difficulty": "简单", "language": "ArkTS", "query": "launchPhotosFromDesktop 函数定义在哪个文件？", "reference_answer": "launchPhotosFromDesktop 定义在 platform/src/main/ets/BenchmarkHub.ets，与 BenchmarkHub 类同文件导出。"}

package/scripts/qa_eval/eval_metrics.py

@@ -1,233 +1,274 @@
-#!/usr/bin/env python3
-"""
-Stage 2 — RAG Answer Accuracy Evaluation (LLM-as-Judge).
-
-Reads agent-produced JSONL (query + output_answer + reference_answer),
-strips tool-call blocks, scores 0–1 via MyAnswerAccuracy.
-
-Usage:
-  export DASHSCOPE_API_KEY="sk-..."
-  python scripts/qa_eval/eval_metrics.py \\
-    -i scripts/qa_eval/log/result-with-builtin.jsonl \\
-    -o scripts/qa_eval/log/result-with-builtin-scored.jsonl \\
-    -w 2
-"""
-
-from __future__ import annotations
-
-import argparse
-import asyncio
-import json
-import logging
-import os
-import statistics
-import sys
-import threading
-from concurrent.futures import ThreadPoolExecutor, as_completed
-from pathlib import Path
-
-from tqdm import tqdm
-
-_SCRIPT_DIR = Path(__file__).resolve().parent
-LOG_DIR = _SCRIPT_DIR / "log"
-LOG_DIR.mkdir(parents=True, exist_ok=True)
-
-if str(_SCRIPT_DIR) not in sys.path:
-    sys.path.insert(0, str(_SCRIPT_DIR))
-
-from my_answer_accuracy import MyAnswerAccuracy  # noqa: E402
-from llm_config import PROVIDER_DASHSCOPE, PROVIDER_ZHIPU, provider_help, resolve_llm_config  # noqa: E402
-
-os.environ.setdefault("NO_PROXY", "localhost,127.0.0.1")
-
-logging.basicConfig(
-    level=logging.INFO,
-    format="%(asctime)s - %(levelname)s - %(message)s",
-    handlers=[
-        logging.FileHandler(LOG_DIR / "eval_metrics.log"),
-        logging.StreamHandler(),
-    ],
-)
-logger = logging.getLogger(__name__)
-
-file_write_lock = threading.Lock()
-
-DEFAULT_INPUT = LOG_DIR / "result-with-builtin.jsonl"
-DEFAULT_OUTPUT = LOG_DIR / "result-with-builtin-scored.jsonl"
-DEFAULT_MODEL = "qwen3-235b-a22b-instruct-2507"
-DEFAULT_BASE_URL = "https://dashscope.aliyuncs.com/compatible-mode/v1"
-DEFAULT_API_KEY_ENV = "DASHSCOPE_API_KEY"
-
-
-class RAGEvaluator:
-    """Answer accuracy judge with concurrent processing."""
-
-    def __init__(
-        self,
-        llm_model: str,
-        *,
-        api_key: str,
-        base_url: str,
-        extra_body: dict | None = None,
-    ) -> None:
-        self.scorer = MyAnswerAccuracy.create(
-            api_key=api_key,
-            base_url=base_url,
-            model=llm_model,
-            extra_body=extra_body,
-        )
-
-    async def evaluate_answer_accuracy_single(self, sample_data: dict) -> dict:
-        return await self.scorer.evaluate_answer_accuracy_single(sample_data)
-
-
-def process_single_item_sync(evaluator: RAGEvaluator, item: dict) -> dict:
-    loop = asyncio.new_event_loop()
-    asyncio.set_event_loop(loop)
-    try:
-        return loop.run_until_complete(evaluator.evaluate_answer_accuracy_single(item))
-    finally:
-        loop.close()
-
-
-def load_jsonl(path: Path) -> list[dict]:
-    items: list[dict] = []
-    with path.open("r", encoding="utf-8") as f:
-        for line_no, line in enumerate(f, 1):
-            line = line.strip()
-            if not line:
-                continue
-            try:
-                items.append(json.loads(line))
-            except json.JSONDecodeError as e:
-                logger.error("第 %s 行 JSON 解析失败: %s", line_no, e)
-    return items
-
-
-def resolve_api_key(env_name: str) -> str | None:
-    key = os.environ.get(env_name, "").strip()
-    if key:
-        return key
-    fallback = os.environ.get("OPENAI_API_KEY", "").strip()
-    return fallback or None
-
-
-def main() -> int:
-    parser = argparse.ArgumentParser(description="RAG Answer Accuracy Evaluation (LLM-as-Judge)")
-    parser.add_argument("--input", "-i", type=str, default=str(DEFAULT_INPUT), help="Agent 产出的 JSONL")
-    parser.add_argument("--output", "-o", type=str, default=str(DEFAULT_OUTPUT), help="打分后的 JSONL")
-    parser.add_argument("--workers", "-w", type=int, default=1, help="并发线程数")
-    parser.add_argument(
-        "--provider",
-        choices=[PROVIDER_DASHSCOPE, PROVIDER_ZHIPU],
-        default=None,
-        help=provider_help(),
-    )
-    parser.add_argument("--model", "-m", type=str, default=None, help="Judge 模型（默认随 provider）")
-    parser.add_argument("--base-url", type=str, default=None, help="OpenAI 兼容 API 端点（默认随 provider）")
-    args = parser.parse_args()
-
-    input_file = Path(args.input)
-    output_file = Path(args.output)
-
-    try:
-        llm = resolve_llm_config(provider=args.provider, model=args.model, base_url=args.base_url)
-    except RuntimeError as e:
-        print(f"错误: {e}", file=sys.stderr)
-        return 1
-
-    print(f"输入文件: {input_file}")
-    print(f"输出文件: {output_file}")
-    print(f"并发线程数: {args.workers}")
-    print(f"LLM provider: {llm.provider}")
-    print(f"评估模型: {llm.model}")
-
-    if not input_file.is_file():
-        print(f"错误: 输入文件 {input_file} 不存在")
-        return 1
-
-    api_key = llm.api_key
-
-    all_items = load_jsonl(input_file)
-    total_items = len(all_items)
-    print(f"读取到 {total_items} 个样本")
-
-    if total_items == 0:
-        print("没有找到有效的样本数据")
-        return 1
-
-    for i, item in enumerate(all_items, 1):
-        if not str(item.get("query", "")).strip():
-            print(f"错误: 第 {i} 条缺少 query")
-            return 1
-        if not str(item.get("reference_answer", "")).strip():
-            print(f"错误: 第 {i} 条缺少 reference_answer")
-            return 1
-        if not str(item.get("output_answer", "")).strip():
-            logger.warning("第 %s 条 (%s) output_answer 为空，Judge 可能得 0 分", i, item.get("id", "?"))
-
-    output_file.parent.mkdir(parents=True, exist_ok=True)
-    output_file.write_text("", encoding="utf-8")
-
-    evaluator = RAGEvaluator(
-        llm.model,
-        api_key=api_key,
-        base_url=llm.base_url,
-        extra_body=llm.extra_body,
-    )
-    print("初始化 Judge 完成")
-
-    processed_items: list[dict] = []
-    results_lock = threading.Lock()
-
-    with ThreadPoolExecutor(max_workers=max(1, args.workers)) as executor:
-        futures = {
-            executor.submit(process_single_item_sync, evaluator, item): item
-            for item in all_items
-        }
-        for future in tqdm(as_completed(futures), total=len(all_items), desc="评估答案准确性"):
-            try:
-                processed_item = future.result()
-                with results_lock:
-                    processed_items.append(processed_item)
-                    if len(processed_items) % 5 == 0:
-                        with file_write_lock:
-                            with output_file.open("a", encoding="utf-8") as writer:
-                                for row in processed_items[-5:]:
-                                    writer.write(json.dumps(row, ensure_ascii=False) + "\n")
-            except Exception as e:
-                logger.error("样本处理任务异常: %s", e)
-
-    remaining = len(processed_items) % 5
-    if remaining:
-        with file_write_lock:
-            with output_file.open("a", encoding="utf-8") as writer:
-                for row in processed_items[-remaining:]:
-                    writer.write(json.dumps(row, ensure_ascii=False) + "\n")
-
-    if processed_items:
-        successful = [x for x in processed_items if x.get("evaluation_status") == "success"]
-        scores = [float(x.get("answer_accuracy_score", 0.0)) for x in processed_items]
-        print("\n评估完成!")
-        print(f"总样本数: {total_items}")
-        print(f"成功评估: {len(successful)}")
-        print(f"失败评估: {total_items - len(successful)}")
-        if successful:
-            ok_scores = [float(x["answer_accuracy_score"]) for x in successful]
-            print(f"平均答案准确性得分: {statistics.mean(ok_scores):.4f}")
-            print(f"最高得分: {max(ok_scores):.4f}")
-            print(f"最低得分: {min(ok_scores):.4f}")
-        print(f"\n结果已写入: {output_file}")
-    else:
-        print("没有成功处理任何样本")
-        return 1
-
-    return 0
-
-
-if __name__ == "__main__":
-    try:
-        raise SystemExit(main())
-    except KeyboardInterrupt:
-        print("\n用户中断了程序执行")
-        raise SystemExit(130)
+#!/usr/bin/env python3
+"""
+Stage 2 — RAG Answer Accuracy Evaluation (LLM-as-Judge).
+
+Reads agent-produced JSONL (query + output_answer + reference_answer),
+strips tool-call blocks, scores 0–1 via MyAnswerAccuracy.
+
+Usage:
+  export DASHSCOPE_API_KEY="sk-..."
+  python scripts/qa_eval/eval_metrics.py \\
+    -i scripts/qa_eval/log/result-with-builtin.jsonl \\
+    -o scripts/qa_eval/log/result-with-builtin-scored.jsonl \\
+    -w 2
+"""
+
+from __future__ import annotations
+
+import argparse
+import asyncio
+import json
+import logging
+import os
+import statistics
+import sys
+import threading
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from pathlib import Path
+
+from tqdm import tqdm
+
+_SCRIPT_DIR = Path(__file__).resolve().parent
+LOG_DIR = _SCRIPT_DIR / "log"
+LOG_DIR.mkdir(parents=True, exist_ok=True)
+
+if str(_SCRIPT_DIR) not in sys.path:
+    sys.path.insert(0, str(_SCRIPT_DIR))
+
+from my_answer_accuracy import MyAnswerAccuracy  # noqa: E402
+from llm_config import PROVIDER_DASHSCOPE, PROVIDER_ZHIPU, provider_help, resolve_llm_config  # noqa: E402
+
+os.environ.setdefault("NO_PROXY", "localhost,127.0.0.1")
+
+logging.basicConfig(
+    level=logging.INFO,
+    format="%(asctime)s - %(levelname)s - %(message)s",
+    handlers=[
+        logging.FileHandler(LOG_DIR / "eval_metrics.log"),
+        logging.StreamHandler(),
+    ],
+)
+logger = logging.getLogger(__name__)
+
+file_write_lock = threading.Lock()
+
+DEFAULT_INPUT = LOG_DIR / "result-with-builtin.jsonl"
+DEFAULT_OUTPUT = LOG_DIR / "result-with-builtin-scored.jsonl"
+DEFAULT_MODEL = "qwen3-235b-a22b-instruct-2507"
+DEFAULT_BASE_URL = "https://dashscope.aliyuncs.com/compatible-mode/v1"
+DEFAULT_API_KEY_ENV = "DASHSCOPE_API_KEY"
+
+
+class RAGEvaluator:
+    """Answer accuracy judge with concurrent processing."""
+
+    def __init__(
+        self,
+        llm_model: str,
+        *,
+        api_key: str,
+        base_url: str,
+        extra_body: dict | None = None,
+    ) -> None:
+        self.scorer = MyAnswerAccuracy.create(
+            api_key=api_key,
+            base_url=base_url,
+            model=llm_model,
+            extra_body=extra_body,
+        )
+
+    async def evaluate_answer_accuracy_single(self, sample_data: dict) -> dict:
+        return await self.scorer.evaluate_answer_accuracy_single(sample_data)
+
+    async def close(self) -> None:
+        await self.scorer.client.close()
+
+
+async def evaluate_all_async(evaluator: RAGEvaluator, items: list[dict]) -> list[dict]:
+    processed: list[dict] = []
+    try:
+        for item in items:
+            processed.append(await evaluator.evaluate_answer_accuracy_single(item))
+    finally:
+        await evaluator.close()
+    return processed
+
+
+def process_single_item_sync(
+    *,
+    llm_model: str,
+    api_key: str,
+    base_url: str,
+    extra_body: dict | None,
+    item: dict,
+) -> dict:
+    loop = asyncio.new_event_loop()
+    asyncio.set_event_loop(loop)
+    evaluator = RAGEvaluator(
+        llm_model,
+        api_key=api_key,
+        base_url=base_url,
+        extra_body=extra_body,
+    )
+    try:
+        return loop.run_until_complete(evaluator.evaluate_answer_accuracy_single(item))
+    finally:
+        loop.run_until_complete(evaluator.close())
+        loop.close()
+        asyncio.set_event_loop(None)
+
+
+def load_jsonl(path: Path) -> list[dict]:
+    items: list[dict] = []
+    with path.open("r", encoding="utf-8") as f:
+        for line_no, line in enumerate(f, 1):
+            line = line.strip()
+            if not line:
+                continue
+            try:
+                items.append(json.loads(line))
+            except json.JSONDecodeError as e:
+                logger.error("第 %s 行 JSON 解析失败: %s", line_no, e)
+    return items
+
+
+def resolve_api_key(env_name: str) -> str | None:
+    key = os.environ.get(env_name, "").strip()
+    if key:
+        return key
+    fallback = os.environ.get("OPENAI_API_KEY", "").strip()
+    return fallback or None
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description="RAG Answer Accuracy Evaluation (LLM-as-Judge)")
+    parser.add_argument("--input", "-i", type=str, default=str(DEFAULT_INPUT), help="Agent 产出的 JSONL")
+    parser.add_argument("--output", "-o", type=str, default=str(DEFAULT_OUTPUT), help="打分后的 JSONL")
+    parser.add_argument("--workers", "-w", type=int, default=1, help="并发线程数")
+    parser.add_argument(
+        "--provider",
+        choices=[PROVIDER_DASHSCOPE, PROVIDER_ZHIPU],
+        default=None,
+        help=provider_help(),
+    )
+    parser.add_argument("--model", "-m", type=str, default=None, help="Judge 模型（默认随 provider）")
+    parser.add_argument("--base-url", type=str, default=None, help="OpenAI 兼容 API 端点（默认随 provider）")
+    args = parser.parse_args()
+
+    input_file = Path(args.input)
+    output_file = Path(args.output)
+
+    try:
+        llm = resolve_llm_config(provider=args.provider, model=args.model, base_url=args.base_url)
+    except RuntimeError as e:
+        print(f"错误: {e}", file=sys.stderr)
+        return 1
+
+    print(f"输入文件: {input_file}")
+    print(f"输出文件: {output_file}")
+    print(f"并发线程数: {args.workers}")
+    print(f"LLM provider: {llm.provider}")
+    print(f"评估模型: {llm.model}")
+
+    if not input_file.is_file():
+        print(f"错误: 输入文件 {input_file} 不存在")
+        return 1
+
+    api_key = llm.api_key
+
+    all_items = load_jsonl(input_file)
+    total_items = len(all_items)
+    print(f"读取到 {total_items} 个样本")
+
+    if total_items == 0:
+        print("没有找到有效的样本数据")
+        return 1
+
+    for i, item in enumerate(all_items, 1):
+        if not str(item.get("query", "")).strip():
+            print(f"错误: 第 {i} 条缺少 query")
+            return 1
+        if not str(item.get("reference_answer", "")).strip():
+            print(f"错误: 第 {i} 条缺少 reference_answer")
+            return 1
+        if not str(item.get("output_answer", "")).strip():
+            logger.warning("第 %s 条 (%s) output_answer 为空，Judge 可能得 0 分", i, item.get("id", "?"))
+
+    output_file.parent.mkdir(parents=True, exist_ok=True)
+    output_file.write_text("", encoding="utf-8")
+
+    evaluator = RAGEvaluator(
+        llm.model,
+        api_key=api_key,
+        base_url=llm.base_url,
+        extra_body=llm.extra_body,
+    )
+    print("初始化 Judge 完成")
+
+    processed_items: list[dict] = []
+    results_lock = threading.Lock()
+
+    if args.workers <= 1:
+        processed_items = asyncio.run(evaluate_all_async(evaluator, all_items))
+        with output_file.open("w", encoding="utf-8") as writer:
+            for row in processed_items:
+                writer.write(json.dumps(row, ensure_ascii=False) + "\n")
+    else:
+        with ThreadPoolExecutor(max_workers=max(1, args.workers)) as executor:
+            futures = {
+                executor.submit(
+                    process_single_item_sync,
+                    llm_model=llm.model,
+                    api_key=api_key,
+                    base_url=llm.base_url,
+                    extra_body=llm.extra_body,
+                    item=item,
+                ): item
+                for item in all_items
+            }
+            for future in tqdm(as_completed(futures), total=len(all_items), desc="评估答案准确性"):
+                try:
+                    processed_item = future.result()
+                    with results_lock:
+                        processed_items.append(processed_item)
+                        if len(processed_items) % 5 == 0:
+                            with file_write_lock:
+                                with output_file.open("a", encoding="utf-8") as writer:
+                                    for row in processed_items[-5:]:
+                                        writer.write(json.dumps(row, ensure_ascii=False) + "\n")
+                except Exception as e:
+                    logger.error("样本处理任务异常: %s", e)
+
+        remaining = len(processed_items) % 5
+        if remaining:
+            with file_write_lock:
+                with output_file.open("a", encoding="utf-8") as writer:
+                    for row in processed_items[-remaining:]:
+                        writer.write(json.dumps(row, ensure_ascii=False) + "\n")
+
+    if processed_items:
+        successful = [x for x in processed_items if x.get("evaluation_status") == "success"]
+        scores = [float(x.get("answer_accuracy_score", 0.0)) for x in processed_items]
+        print("\n评估完成!")
+        print(f"总样本数: {total_items}")
+        print(f"成功评估: {len(successful)}")
+        print(f"失败评估: {total_items - len(successful)}")
+        if successful:
+            ok_scores = [float(x["answer_accuracy_score"]) for x in successful]
+            print(f"平均答案准确性得分: {statistics.mean(ok_scores):.4f}")
+            print(f"最高得分: {max(ok_scores):.4f}")
+            print(f"最低得分: {min(ok_scores):.4f}")
+        print(f"\n结果已写入: {output_file}")
+    else:
+        print("没有成功处理任何样本")
+        return 1
+
+    return 0
+
+
+if __name__ == "__main__":
+    try:
+        raise SystemExit(main())
+    except KeyboardInterrupt:
+        print("\n用户中断了程序执行")
+        raise SystemExit(130)