homegraph 1.0.0 → 1.1.2

package/scripts/qa_eval/stats_efficiency.py

@@ -0,0 +1,279 @@
+#!/usr/bin/env python3
+"""
+Stage 4 — 从 Agent 日志统计效率指标（首响应、轮次、总耗时、总 Token）。
+
+「首响应」= 从 Evaluate 开始到第一次 LLM API 返回的耗时（日志行 `first token`）。
+
+兼容 CodeGenie / Trae 等 Agent 日志格式：
+  - Evaluate N:
+  - first token
+  - the N turn
+  - totalTokenCount = N
+
+Usage:
+  python scripts/qa_eval/stats_efficiency.py -l scripts/qa_eval/log/agent-with-builtin.log
+"""
+
+from __future__ import annotations
+
+import argparse
+import re
+import sys
+from datetime import datetime
+from pathlib import Path
+from statistics import mean
+
+_SCRIPT_DIR = Path(__file__).resolve().parent
+DEFAULT_LOG = _SCRIPT_DIR / "log" / "agent-with-builtin.log"
+
+TIME_FMT = "%Y-%m-%d %H:%M:%S.%f"
+
+TIME_PATTERN = re.compile(r"^(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d+)")
+EVALUATE_PATTERN = re.compile(r"Evaluate\s+(\d+):")
+FIRST_TOKEN_PATTERN = re.compile(r"first token")
+TURN_PATTERN = re.compile(r"the\s+(\d+)\s+turn")
+TOKEN_PATTERN = re.compile(r"totalTokenCount\s*=\s*(\d+)")
+PEAK_RSS_PATTERN = re.compile(r"peakRssMb\s*=\s*([\d.]+)")
+AVG_RSS_PATTERN = re.compile(r"avgRssMb\s*=\s*([\d.]+)")
+
+
+def parse_agent_log(log_file: Path) -> list[dict]:
+    tasks: list[dict] = []
+    current_task: dict | None = None
+    last_log_time: datetime | None = None
+
+    with log_file.open("r", encoding="utf-8", errors="replace") as f:
+        for line in f:
+            line = line.strip()
+            if not line:
+                continue
+
+            time_match = TIME_PATTERN.search(line)
+            if not time_match:
+                continue
+
+            try:
+                log_time = datetime.strptime(time_match.group(1), TIME_FMT)
+            except ValueError:
+                continue
+
+            last_log_time = log_time
+
+            eval_match = EVALUATE_PATTERN.search(line)
+            if eval_match:
+                if current_task:
+                    current_task["end_time"] = log_time
+                    tasks.append(current_task)
+
+                current_task = {
+                    "id": int(eval_match.group(1)),
+                    "start_time": log_time,
+                    "first_token_time": None,
+                    "first_api_response_time": None,
+                    "max_turn": 0,
+                    "end_time": None,
+                    "total_tokens": 0,
+                    "peak_rss_mb": None,
+                    "avg_rss_mb": None,
+                }
+                continue
+
+            if current_task is None:
+                continue
+
+            if FIRST_TOKEN_PATTERN.search(line):
+                current_task["first_token_time"] = log_time
+                continue
+
+            turn_match = TURN_PATTERN.search(line)
+            if turn_match:
+                turn = int(turn_match.group(1))
+                current_task["max_turn"] = max(current_task["max_turn"], turn)
+
+            token_match = TOKEN_PATTERN.search(line)
+            if token_match:
+                if current_task["first_api_response_time"] is None:
+                    current_task["first_api_response_time"] = log_time
+                current_task["total_tokens"] += int(token_match.group(1))
+
+            peak_match = PEAK_RSS_PATTERN.search(line)
+            if peak_match:
+                current_task["peak_rss_mb"] = float(peak_match.group(1))
+
+            avg_match = AVG_RSS_PATTERN.search(line)
+            if avg_match:
+                current_task["avg_rss_mb"] = float(avg_match.group(1))
+
+    if current_task:
+        current_task["end_time"] = last_log_time
+        tasks.append(current_task)
+
+    return tasks
+
+
+def _memory_from_rows(rows: list[dict]) -> tuple[list[float], list[float]]:
+    peaks: list[float] = []
+    avgs: list[float] = []
+    for row in rows:
+        mem = row.get("agent_memory_mb") or {}
+        if mem.get("peak_rss_mb") is not None:
+            peaks.append(float(mem["peak_rss_mb"]))
+        if mem.get("avg_rss_mb") is not None:
+            avgs.append(float(mem["avg_rss_mb"]))
+    return peaks, avgs
+
+
+def summarize_jsonl_usage(rows: list[dict]) -> dict:
+    """Fallback when agent .log is missing: aggregate agent_usage from JSONL rows."""
+    tokens: list[int] = []
+    for row in rows:
+        usage = row.get("agent_usage") or {}
+        total = usage.get("total_tokens")
+        if isinstance(total, (int, float)):
+            tokens.append(int(total))
+    peaks, avgs = _memory_from_rows(rows)
+    return {
+        "task_count": len(rows),
+        "avg_first_token_s": None,
+        "avg_turns": None,
+        "avg_duration_s": None,
+        "avg_tokens": mean(tokens) if tokens else None,
+        "total_tokens": sum(tokens),
+        "avg_peak_rss_mb": mean(peaks) if peaks else None,
+        "max_peak_rss_mb": max(peaks) if peaks else None,
+        "avg_avg_rss_mb": mean(avgs) if avgs else None,
+        "source": "jsonl",
+    }
+
+
+FIRST_TOKEN_MIN_S = 0.05
+
+
+def effective_first_response_time(task: dict) -> datetime | None:
+    """首响应时间：优先用 first token 行；若与 Evaluate 几乎同时（旧日志 bug）则回退到首次 API 返回。"""
+    start = task.get("start_time")
+    ft = task.get("first_token_time")
+    if start and ft:
+        if (ft - start).total_seconds() >= FIRST_TOKEN_MIN_S:
+            return ft
+    return task.get("first_api_response_time") or ft
+
+
+def summarize_tasks(tasks: list[dict], *, print_report: bool = True) -> dict:
+    first_token_costs: list[float] = []
+    turns: list[int] = []
+    durations: list[float] = []
+    tokens_list: list[int] = []
+    peak_rss: list[float] = []
+    avg_rss: list[float] = []
+
+    if print_report:
+        print("=" * 100)
+        print("任务明细")
+        print("=" * 100)
+
+    for task in tasks:
+        first_token_seconds = None
+        response_time = effective_first_response_time(task)
+        if response_time and task["start_time"]:
+            first_token_seconds = (response_time - task["start_time"]).total_seconds()
+            first_token_costs.append(first_token_seconds)
+
+        turns.append(task["max_turn"])
+
+        duration_seconds = 0.0
+        if task["end_time"] and task["start_time"]:
+            duration_seconds = (task["end_time"] - task["start_time"]).total_seconds()
+            durations.append(duration_seconds)
+
+        tokens_list.append(task["total_tokens"])
+        if task.get("peak_rss_mb") is not None:
+            peak_rss.append(float(task["peak_rss_mb"]))
+        if task.get("avg_rss_mb") is not None:
+            avg_rss.append(float(task["avg_rss_mb"]))
+
+        if print_report:
+            first_display = f"{first_token_seconds:.2f}s" if first_token_seconds is not None else "N/A"
+            mem_display = (
+                f"{task['peak_rss_mb']:.0f}MB"
+                if task.get("peak_rss_mb") is not None
+                else "N/A"
+            )
+            print(
+                f"任务 {task['id']:>3} | "
+                f"首响应: {first_display:<8} | "
+                f"轮次: {task['max_turn']:>3} | "
+                f"总时间: {duration_seconds:.2f}s | "
+                f"总Token: {task['total_tokens']:<6} | "
+                f"峰值内存: {mem_display}"
+            )
+
+    summary = {
+        "task_count": len(tasks),
+        "avg_first_token_s": mean(first_token_costs) if first_token_costs else None,
+        "avg_turns": mean(turns) if turns else None,
+        "avg_duration_s": mean(durations) if durations else None,
+        "avg_tokens": mean(tokens_list) if tokens_list else None,
+        "total_tokens": sum(tokens_list),
+        "avg_peak_rss_mb": mean(peak_rss) if peak_rss else None,
+        "max_peak_rss_mb": max(peak_rss) if peak_rss else None,
+        "avg_avg_rss_mb": mean(avg_rss) if avg_rss else None,
+        "source": "log",
+    }
+
+    if print_report:
+        print("\n" + "=" * 100)
+        print("统计结果")
+        print("=" * 100)
+        print(f"任务总数: {summary['task_count']}")
+
+        if summary["avg_first_token_s"] is not None:
+            print(f"平均首响应时间: {summary['avg_first_token_s']:.2f} 秒")
+        if summary["avg_turns"] is not None:
+            print(f"平均轮次: {summary['avg_turns']:.2f}")
+        if summary["avg_duration_s"] is not None:
+            print(f"平均总时间: {summary['avg_duration_s'] / 60:.2f} min ({summary['avg_duration_s']:.2f} s)")
+        if summary["avg_tokens"] is not None:
+            print(f"平均总 Token: {summary['avg_tokens'] / 1000:.2f} k")
+            print(f"合计总 Token: {summary['total_tokens']}")
+        if summary["avg_peak_rss_mb"] is not None:
+            print(f"平均峰值内存: {summary['avg_peak_rss_mb']:.1f} MB")
+        if summary["max_peak_rss_mb"] is not None:
+            print(f"最大峰值内存: {summary['max_peak_rss_mb']:.1f} MB")
+        if summary["avg_avg_rss_mb"] is not None:
+            print(f"平均内存占用: {summary['avg_avg_rss_mb']:.1f} MB")
+
+        print("=" * 100)
+    return summary
+
+
+def summarize_log(log_file: Path, *, print_report: bool = True) -> dict | None:
+    if not log_file.is_file():
+        return None
+    tasks = parse_agent_log(log_file)
+    if not tasks:
+        return None
+    return summarize_tasks(tasks, print_report=print_report)
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description="Agent 日志效率统计")
+    parser.add_argument("--log", "-l", type=str, default=str(DEFAULT_LOG), help="Agent 日志路径")
+    args = parser.parse_args()
+
+    log_file = Path(args.log)
+    if not log_file.is_file():
+        print(f"错误: 日志文件 {log_file} 不存在")
+        return 1
+
+    tasks = parse_agent_log(log_file)
+    if not tasks:
+        print("警告: 日志中未解析到任何 Evaluate 任务段")
+        return 1
+
+    summarize_tasks(tasks)
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

package/scripts/qa_eval/stats_scores.py

@@ -0,0 +1,207 @@
+#!/usr/bin/env python3
+"""
+Stage 3 — Answer Accuracy Score 统计。
+
+Usage:
+  python scripts/qa_eval/stats_scores.py -i scripts/qa_eval/log/result-with-builtin-scored.jsonl
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import statistics
+import sys
+from pathlib import Path
+
+_SCRIPT_DIR = Path(__file__).resolve().parent
+DEFAULT_INPUT = _SCRIPT_DIR / "log" / "result-with-builtin-scored.jsonl"
+
+
+def calculate_answer_accuracy_statistics(file_path: Path) -> dict | None:
+    scores: list[float] = []
+    total_samples = 0
+    success_samples = 0
+    failed_samples = 0
+
+    print(f"正在读取文件: {file_path}")
+
+    try:
+        with file_path.open("r", encoding="utf-8") as f:
+            for line_num, line in enumerate(f, 1):
+                if not line.strip():
+                    continue
+                try:
+                    data = json.loads(line)
+                    total_samples += 1
+
+                    if "answer_accuracy_score" not in data:
+                        failed_samples += 1
+                        print(f"  第{line_num}行: 缺少 answer_accuracy_score 字段")
+                        continue
+
+                    score = data["answer_accuracy_score"]
+                    eval_status = data.get("evaluation_status", "unknown")
+
+                    if (
+                        eval_status == "success"
+                        and isinstance(score, (int, float))
+                        and not (isinstance(score, float) and str(score).lower() == "nan")
+                    ):
+                        scores.append(float(score))
+                        success_samples += 1
+                    else:
+                        failed_samples += 1
+                        print(f"  第{line_num}行: 评估失败或无效分数 - 状态: {eval_status}, 分数: {score}")
+
+                except json.JSONDecodeError as e:
+                    print(f"  第{line_num}行: JSON 解析错误 - {e}")
+                    failed_samples += 1
+
+    except FileNotFoundError:
+        print(f"错误: 文件 {file_path} 不存在")
+        return None
+    except OSError as e:
+        print(f"读取文件时出错: {e}")
+        return None
+
+    if not scores:
+        print("警告: 没有找到有效的 answer_accuracy_score 数据")
+        return {
+            "total_samples": total_samples,
+            "success_samples": success_samples,
+            "failed_samples": failed_samples,
+            "valid_scores": 0,
+        }
+
+    stats = {
+        "total_samples": total_samples,
+        "success_samples": success_samples,
+        "failed_samples": failed_samples,
+        "valid_scores": len(scores),
+        "mean": statistics.mean(scores),
+        "median": statistics.median(scores),
+        "min": min(scores),
+        "max": max(scores),
+        "std_dev": statistics.stdev(scores) if len(scores) > 1 else 0.0,
+        "variance": statistics.variance(scores) if len(scores) > 1 else 0.0,
+        "_scores": scores,
+    }
+    return stats
+
+
+def compute_stats_from_rows(rows: list[dict]) -> dict | None:
+    """Same stats as calculate_answer_accuracy_statistics, from in-memory rows."""
+    scores: list[float] = []
+    total_samples = len(rows)
+    success_samples = 0
+    failed_samples = 0
+
+    for data in rows:
+        if "answer_accuracy_score" not in data:
+            failed_samples += 1
+            continue
+        score = data["answer_accuracy_score"]
+        eval_status = data.get("evaluation_status", "unknown")
+        if (
+            eval_status == "success"
+            and isinstance(score, (int, float))
+            and not (isinstance(score, float) and str(score).lower() == "nan")
+        ):
+            scores.append(float(score))
+            success_samples += 1
+        else:
+            failed_samples += 1
+
+    if not scores:
+        return {
+            "total_samples": total_samples,
+            "success_samples": success_samples,
+            "failed_samples": failed_samples,
+            "valid_scores": 0,
+        }
+
+    return {
+        "total_samples": total_samples,
+        "success_samples": success_samples,
+        "failed_samples": failed_samples,
+        "valid_scores": len(scores),
+        "mean": statistics.mean(scores),
+        "median": statistics.median(scores),
+        "min": min(scores),
+        "max": max(scores),
+        "std_dev": statistics.stdev(scores) if len(scores) > 1 else 0.0,
+        "variance": statistics.variance(scores) if len(scores) > 1 else 0.0,
+        "_scores": scores,
+    }
+
+
+def print_statistics(stats: dict | None, file_path: Path | None = None, *, title: str | None = None) -> None:
+    if not stats:
+        return
+
+    print("\n" + "=" * 60)
+    print(title or "Answer Accuracy Score 统计结果")
+    print("=" * 60)
+    if file_path:
+        print(f"文件: {file_path}")
+
+    print(f"总样本数: {stats['total_samples']}")
+    print(f"成功评估样本数: {stats['success_samples']}")
+    print(f"失败评估样本数: {stats['failed_samples']}")
+    print(f"有效分数数量: {stats['valid_scores']}")
+
+    if stats["valid_scores"] > 0:
+        print("\n分数统计:")
+        print(f"  平均值 (Mean): {stats['mean']:.4f}")
+        print(f"  中位数 (Median): {stats['median']:.4f}")
+        print(f"  最小值 (Min): {stats['min']:.4f}")
+        print(f"  最大值 (Max): {stats['max']:.4f}")
+        if stats["valid_scores"] > 1:
+            print(f"  标准差 (Std Dev): {stats['std_dev']:.4f}")
+            print(f"  方差 (Variance): {stats['variance']:.4f}")
+
+        scores = stats.get("_scores") or []
+        print("\n分数分布:")
+        ranges = [
+            (0.0, 0.2, "很低 (0.0-0.2)"),
+            (0.2, 0.4, "低 (0.2-0.4)"),
+            (0.4, 0.6, "中等 (0.4-0.6)"),
+            (0.6, 0.8, "高 (0.6-0.8)"),
+            (0.8, 1.0, "很高 (0.8-1.0)"),
+        ]
+        for min_val, max_val, label in ranges:
+            count = sum(
+                1
+                for score in scores
+                if min_val <= score < max_val or (max_val == 1.0 and score == 1.0)
+            )
+            pct = (count / len(scores)) * 100 if scores else 0
+            print(f"  {label}: {count} ({pct:.1f}%)")
+
+    print("=" * 60)
+
+    if stats["valid_scores"] > 0:
+        success_rate = (stats["success_samples"] / stats["total_samples"]) * 100 if stats["total_samples"] else 0
+        print(f"\n简要报告: {stats['total_samples']} 个样本，{stats['success_samples']} 个成功评估")
+        print(f"Answer Accuracy Score 平均值: {stats['mean']:.4f}")
+        print(f"评估成功率: {success_rate:.1f}%")
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description="Answer Accuracy Score 统计")
+    parser.add_argument("--input", "-i", type=str, default=str(DEFAULT_INPUT), help="打分后的 JSONL")
+    args = parser.parse_args()
+
+    file_path = Path(args.input)
+    if not file_path.is_file():
+        print(f"错误: 文件 {file_path} 不存在")
+        return 1
+
+    stats = calculate_answer_accuracy_statistics(file_path)
+    print_statistics(stats, file_path)
+    return 0 if stats else 1
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())