homegraph 1.2.0 → 1.5.0

package/scripts/qa_eval/stats_efficiency.py

@@ -1,279 +0,0 @@
-#!/usr/bin/env python3
-"""
-Stage 4 — 从 Agent 日志统计效率指标（首响应、轮次、总耗时、总 Token）。
-
-「首响应」= 从 Evaluate 开始到第一次 LLM API 返回的耗时（日志行 `first token`）。
-
-兼容 CodeGenie / Trae 等 Agent 日志格式：
-  - Evaluate N:
-  - first token
-  - the N turn
-  - totalTokenCount = N
-
-Usage:
-  python scripts/qa_eval/stats_efficiency.py -l scripts/qa_eval/log/agent-with-builtin.log
-"""
-
-from __future__ import annotations
-
-import argparse
-import re
-import sys
-from datetime import datetime
-from pathlib import Path
-from statistics import mean
-
-_SCRIPT_DIR = Path(__file__).resolve().parent
-DEFAULT_LOG = _SCRIPT_DIR / "log" / "agent-with-builtin.log"
-
-TIME_FMT = "%Y-%m-%d %H:%M:%S.%f"
-
-TIME_PATTERN = re.compile(r"^(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d+)")
-EVALUATE_PATTERN = re.compile(r"Evaluate\s+(\d+):")
-FIRST_TOKEN_PATTERN = re.compile(r"first token")
-TURN_PATTERN = re.compile(r"the\s+(\d+)\s+turn")
-TOKEN_PATTERN = re.compile(r"totalTokenCount\s*=\s*(\d+)")
-PEAK_RSS_PATTERN = re.compile(r"peakRssMb\s*=\s*([\d.]+)")
-AVG_RSS_PATTERN = re.compile(r"avgRssMb\s*=\s*([\d.]+)")
-
-
-def parse_agent_log(log_file: Path) -> list[dict]:
-    tasks: list[dict] = []
-    current_task: dict | None = None
-    last_log_time: datetime | None = None
-
-    with log_file.open("r", encoding="utf-8", errors="replace") as f:
-        for line in f:
-            line = line.strip()
-            if not line:
-                continue
-
-            time_match = TIME_PATTERN.search(line)
-            if not time_match:
-                continue
-
-            try:
-                log_time = datetime.strptime(time_match.group(1), TIME_FMT)
-            except ValueError:
-                continue
-
-            last_log_time = log_time
-
-            eval_match = EVALUATE_PATTERN.search(line)
-            if eval_match:
-                if current_task:
-                    current_task["end_time"] = log_time
-                    tasks.append(current_task)
-
-                current_task = {
-                    "id": int(eval_match.group(1)),
-                    "start_time": log_time,
-                    "first_token_time": None,
-                    "first_api_response_time": None,
-                    "max_turn": 0,
-                    "end_time": None,
-                    "total_tokens": 0,
-                    "peak_rss_mb": None,
-                    "avg_rss_mb": None,
-                }
-                continue
-
-            if current_task is None:
-                continue
-
-            if FIRST_TOKEN_PATTERN.search(line):
-                current_task["first_token_time"] = log_time
-                continue
-
-            turn_match = TURN_PATTERN.search(line)
-            if turn_match:
-                turn = int(turn_match.group(1))
-                current_task["max_turn"] = max(current_task["max_turn"], turn)
-
-            token_match = TOKEN_PATTERN.search(line)
-            if token_match:
-                if current_task["first_api_response_time"] is None:
-                    current_task["first_api_response_time"] = log_time
-                current_task["total_tokens"] += int(token_match.group(1))
-
-            peak_match = PEAK_RSS_PATTERN.search(line)
-            if peak_match:
-                current_task["peak_rss_mb"] = float(peak_match.group(1))
-
-            avg_match = AVG_RSS_PATTERN.search(line)
-            if avg_match:
-                current_task["avg_rss_mb"] = float(avg_match.group(1))
-
-    if current_task:
-        current_task["end_time"] = last_log_time
-        tasks.append(current_task)
-
-    return tasks
-
-
-def _memory_from_rows(rows: list[dict]) -> tuple[list[float], list[float]]:
-    peaks: list[float] = []
-    avgs: list[float] = []
-    for row in rows:
-        mem = row.get("agent_memory_mb") or {}
-        if mem.get("peak_rss_mb") is not None:
-            peaks.append(float(mem["peak_rss_mb"]))
-        if mem.get("avg_rss_mb") is not None:
-            avgs.append(float(mem["avg_rss_mb"]))
-    return peaks, avgs
-
-
-def summarize_jsonl_usage(rows: list[dict]) -> dict:
-    """Fallback when agent .log is missing: aggregate agent_usage from JSONL rows."""
-    tokens: list[int] = []
-    for row in rows:
-        usage = row.get("agent_usage") or {}
-        total = usage.get("total_tokens")
-        if isinstance(total, (int, float)):
-            tokens.append(int(total))
-    peaks, avgs = _memory_from_rows(rows)
-    return {
-        "task_count": len(rows),
-        "avg_first_token_s": None,
-        "avg_turns": None,
-        "avg_duration_s": None,
-        "avg_tokens": mean(tokens) if tokens else None,
-        "total_tokens": sum(tokens),
-        "avg_peak_rss_mb": mean(peaks) if peaks else None,
-        "max_peak_rss_mb": max(peaks) if peaks else None,
-        "avg_avg_rss_mb": mean(avgs) if avgs else None,
-        "source": "jsonl",
-    }
-
-
-FIRST_TOKEN_MIN_S = 0.05
-
-
-def effective_first_response_time(task: dict) -> datetime | None:
-    """首响应时间：优先用 first token 行；若与 Evaluate 几乎同时（旧日志 bug）则回退到首次 API 返回。"""
-    start = task.get("start_time")
-    ft = task.get("first_token_time")
-    if start and ft:
-        if (ft - start).total_seconds() >= FIRST_TOKEN_MIN_S:
-            return ft
-    return task.get("first_api_response_time") or ft
-
-
-def summarize_tasks(tasks: list[dict], *, print_report: bool = True) -> dict:
-    first_token_costs: list[float] = []
-    turns: list[int] = []
-    durations: list[float] = []
-    tokens_list: list[int] = []
-    peak_rss: list[float] = []
-    avg_rss: list[float] = []
-
-    if print_report:
-        print("=" * 100)
-        print("任务明细")
-        print("=" * 100)
-
-    for task in tasks:
-        first_token_seconds = None
-        response_time = effective_first_response_time(task)
-        if response_time and task["start_time"]:
-            first_token_seconds = (response_time - task["start_time"]).total_seconds()
-            first_token_costs.append(first_token_seconds)
-
-        turns.append(task["max_turn"])
-
-        duration_seconds = 0.0
-        if task["end_time"] and task["start_time"]:
-            duration_seconds = (task["end_time"] - task["start_time"]).total_seconds()
-            durations.append(duration_seconds)
-
-        tokens_list.append(task["total_tokens"])
-        if task.get("peak_rss_mb") is not None:
-            peak_rss.append(float(task["peak_rss_mb"]))
-        if task.get("avg_rss_mb") is not None:
-            avg_rss.append(float(task["avg_rss_mb"]))
-
-        if print_report:
-            first_display = f"{first_token_seconds:.2f}s" if first_token_seconds is not None else "N/A"
-            mem_display = (
-                f"{task['peak_rss_mb']:.0f}MB"
-                if task.get("peak_rss_mb") is not None
-                else "N/A"
-            )
-            print(
-                f"任务 {task['id']:>3} | "
-                f"首响应: {first_display:<8} | "
-                f"轮次: {task['max_turn']:>3} | "
-                f"总时间: {duration_seconds:.2f}s | "
-                f"总Token: {task['total_tokens']:<6} | "
-                f"峰值内存: {mem_display}"
-            )
-
-    summary = {
-        "task_count": len(tasks),
-        "avg_first_token_s": mean(first_token_costs) if first_token_costs else None,
-        "avg_turns": mean(turns) if turns else None,
-        "avg_duration_s": mean(durations) if durations else None,
-        "avg_tokens": mean(tokens_list) if tokens_list else None,
-        "total_tokens": sum(tokens_list),
-        "avg_peak_rss_mb": mean(peak_rss) if peak_rss else None,
-        "max_peak_rss_mb": max(peak_rss) if peak_rss else None,
-        "avg_avg_rss_mb": mean(avg_rss) if avg_rss else None,
-        "source": "log",
-    }
-
-    if print_report:
-        print("\n" + "=" * 100)
-        print("统计结果")
-        print("=" * 100)
-        print(f"任务总数: {summary['task_count']}")
-
-        if summary["avg_first_token_s"] is not None:
-            print(f"平均首响应时间: {summary['avg_first_token_s']:.2f} 秒")
-        if summary["avg_turns"] is not None:
-            print(f"平均轮次: {summary['avg_turns']:.2f}")
-        if summary["avg_duration_s"] is not None:
-            print(f"平均总时间: {summary['avg_duration_s'] / 60:.2f} min ({summary['avg_duration_s']:.2f} s)")
-        if summary["avg_tokens"] is not None:
-            print(f"平均总 Token: {summary['avg_tokens'] / 1000:.2f} k")
-            print(f"合计总 Token: {summary['total_tokens']}")
-        if summary["avg_peak_rss_mb"] is not None:
-            print(f"平均峰值内存: {summary['avg_peak_rss_mb']:.1f} MB")
-        if summary["max_peak_rss_mb"] is not None:
-            print(f"最大峰值内存: {summary['max_peak_rss_mb']:.1f} MB")
-        if summary["avg_avg_rss_mb"] is not None:
-            print(f"平均内存占用: {summary['avg_avg_rss_mb']:.1f} MB")
-
-        print("=" * 100)
-    return summary
-
-
-def summarize_log(log_file: Path, *, print_report: bool = True) -> dict | None:
-    if not log_file.is_file():
-        return None
-    tasks = parse_agent_log(log_file)
-    if not tasks:
-        return None
-    return summarize_tasks(tasks, print_report=print_report)
-
-
-def main() -> int:
-    parser = argparse.ArgumentParser(description="Agent 日志效率统计")
-    parser.add_argument("--log", "-l", type=str, default=str(DEFAULT_LOG), help="Agent 日志路径")
-    args = parser.parse_args()
-
-    log_file = Path(args.log)
-    if not log_file.is_file():
-        print(f"错误: 日志文件 {log_file} 不存在")
-        return 1
-
-    tasks = parse_agent_log(log_file)
-    if not tasks:
-        print("警告: 日志中未解析到任何 Evaluate 任务段")
-        return 1
-
-    summarize_tasks(tasks)
-    return 0
-
-
-if __name__ == "__main__":
-    raise SystemExit(main())

package/scripts/qa_eval/stats_scores.py

@@ -1,207 +0,0 @@
-#!/usr/bin/env python3
-"""
-Stage 3 — Answer Accuracy Score 统计。
-
-Usage:
-  python scripts/qa_eval/stats_scores.py -i scripts/qa_eval/log/result-with-builtin-scored.jsonl
-"""
-
-from __future__ import annotations
-
-import argparse
-import json
-import statistics
-import sys
-from pathlib import Path
-
-_SCRIPT_DIR = Path(__file__).resolve().parent
-DEFAULT_INPUT = _SCRIPT_DIR / "log" / "result-with-builtin-scored.jsonl"
-
-
-def calculate_answer_accuracy_statistics(file_path: Path) -> dict | None:
-    scores: list[float] = []
-    total_samples = 0
-    success_samples = 0
-    failed_samples = 0
-
-    print(f"正在读取文件: {file_path}")
-
-    try:
-        with file_path.open("r", encoding="utf-8") as f:
-            for line_num, line in enumerate(f, 1):
-                if not line.strip():
-                    continue
-                try:
-                    data = json.loads(line)
-                    total_samples += 1
-
-                    if "answer_accuracy_score" not in data:
-                        failed_samples += 1
-                        print(f"  第{line_num}行: 缺少 answer_accuracy_score 字段")
-                        continue
-
-                    score = data["answer_accuracy_score"]
-                    eval_status = data.get("evaluation_status", "unknown")
-
-                    if (
-                        eval_status == "success"
-                        and isinstance(score, (int, float))
-                        and not (isinstance(score, float) and str(score).lower() == "nan")
-                    ):
-                        scores.append(float(score))
-                        success_samples += 1
-                    else:
-                        failed_samples += 1
-                        print(f"  第{line_num}行: 评估失败或无效分数 - 状态: {eval_status}, 分数: {score}")
-
-                except json.JSONDecodeError as e:
-                    print(f"  第{line_num}行: JSON 解析错误 - {e}")
-                    failed_samples += 1
-
-    except FileNotFoundError:
-        print(f"错误: 文件 {file_path} 不存在")
-        return None
-    except OSError as e:
-        print(f"读取文件时出错: {e}")
-        return None
-
-    if not scores:
-        print("警告: 没有找到有效的 answer_accuracy_score 数据")
-        return {
-            "total_samples": total_samples,
-            "success_samples": success_samples,
-            "failed_samples": failed_samples,
-            "valid_scores": 0,
-        }
-
-    stats = {
-        "total_samples": total_samples,
-        "success_samples": success_samples,
-        "failed_samples": failed_samples,
-        "valid_scores": len(scores),
-        "mean": statistics.mean(scores),
-        "median": statistics.median(scores),
-        "min": min(scores),
-        "max": max(scores),
-        "std_dev": statistics.stdev(scores) if len(scores) > 1 else 0.0,
-        "variance": statistics.variance(scores) if len(scores) > 1 else 0.0,
-        "_scores": scores,
-    }
-    return stats
-
-
-def compute_stats_from_rows(rows: list[dict]) -> dict | None:
-    """Same stats as calculate_answer_accuracy_statistics, from in-memory rows."""
-    scores: list[float] = []
-    total_samples = len(rows)
-    success_samples = 0
-    failed_samples = 0
-
-    for data in rows:
-        if "answer_accuracy_score" not in data:
-            failed_samples += 1
-            continue
-        score = data["answer_accuracy_score"]
-        eval_status = data.get("evaluation_status", "unknown")
-        if (
-            eval_status == "success"
-            and isinstance(score, (int, float))
-            and not (isinstance(score, float) and str(score).lower() == "nan")
-        ):
-            scores.append(float(score))
-            success_samples += 1
-        else:
-            failed_samples += 1
-
-    if not scores:
-        return {
-            "total_samples": total_samples,
-            "success_samples": success_samples,
-            "failed_samples": failed_samples,
-            "valid_scores": 0,
-        }
-
-    return {
-        "total_samples": total_samples,
-        "success_samples": success_samples,
-        "failed_samples": failed_samples,
-        "valid_scores": len(scores),
-        "mean": statistics.mean(scores),
-        "median": statistics.median(scores),
-        "min": min(scores),
-        "max": max(scores),
-        "std_dev": statistics.stdev(scores) if len(scores) > 1 else 0.0,
-        "variance": statistics.variance(scores) if len(scores) > 1 else 0.0,
-        "_scores": scores,
-    }
-
-
-def print_statistics(stats: dict | None, file_path: Path | None = None, *, title: str | None = None) -> None:
-    if not stats:
-        return
-
-    print("\n" + "=" * 60)
-    print(title or "Answer Accuracy Score 统计结果")
-    print("=" * 60)
-    if file_path:
-        print(f"文件: {file_path}")
-
-    print(f"总样本数: {stats['total_samples']}")
-    print(f"成功评估样本数: {stats['success_samples']}")
-    print(f"失败评估样本数: {stats['failed_samples']}")
-    print(f"有效分数数量: {stats['valid_scores']}")
-
-    if stats["valid_scores"] > 0:
-        print("\n分数统计:")
-        print(f"  平均值 (Mean): {stats['mean']:.4f}")
-        print(f"  中位数 (Median): {stats['median']:.4f}")
-        print(f"  最小值 (Min): {stats['min']:.4f}")
-        print(f"  最大值 (Max): {stats['max']:.4f}")
-        if stats["valid_scores"] > 1:
-            print(f"  标准差 (Std Dev): {stats['std_dev']:.4f}")
-            print(f"  方差 (Variance): {stats['variance']:.4f}")
-
-        scores = stats.get("_scores") or []
-        print("\n分数分布:")
-        ranges = [
-            (0.0, 0.2, "很低 (0.0-0.2)"),
-            (0.2, 0.4, "低 (0.2-0.4)"),
-            (0.4, 0.6, "中等 (0.4-0.6)"),
-            (0.6, 0.8, "高 (0.6-0.8)"),
-            (0.8, 1.0, "很高 (0.8-1.0)"),
-        ]
-        for min_val, max_val, label in ranges:
-            count = sum(
-                1
-                for score in scores
-                if min_val <= score < max_val or (max_val == 1.0 and score == 1.0)
-            )
-            pct = (count / len(scores)) * 100 if scores else 0
-            print(f"  {label}: {count} ({pct:.1f}%)")
-
-    print("=" * 60)
-
-    if stats["valid_scores"] > 0:
-        success_rate = (stats["success_samples"] / stats["total_samples"]) * 100 if stats["total_samples"] else 0
-        print(f"\n简要报告: {stats['total_samples']} 个样本，{stats['success_samples']} 个成功评估")
-        print(f"Answer Accuracy Score 平均值: {stats['mean']:.4f}")
-        print(f"评估成功率: {success_rate:.1f}%")
-
-
-def main() -> int:
-    parser = argparse.ArgumentParser(description="Answer Accuracy Score 统计")
-    parser.add_argument("--input", "-i", type=str, default=str(DEFAULT_INPUT), help="打分后的 JSONL")
-    args = parser.parse_args()
-
-    file_path = Path(args.input)
-    if not file_path.is_file():
-        print(f"错误: 文件 {file_path} 不存在")
-        return 1
-
-    stats = calculate_answer_accuracy_statistics(file_path)
-    print_statistics(stats, file_path)
-    return 0 if stats else 1
-
-
-if __name__ == "__main__":
-    raise SystemExit(main())