@zhushanwen/pi-evolve-daily 0.1.0 → 0.1.2

package/scripts/extractors/cross_project.py

@@ -0,0 +1,138 @@
+"""Signal 6: 跨项目通用模式。
+
+分析多项目 session 数据，提取公共工具调用序列、项目类型分布等跨项目指标。
+"""
+
+from __future__ import annotations
+
+import sys
+from collections import Counter, defaultdict
+from pathlib import Path
+
+# 使 config 可导入
+sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
+
+# 序列长度
+_SEQUENCE_LENGTH = 3
+
+# 项目类型关键字映射
+_PROJECT_TYPE_KEYWORDS: dict[str, list[str]] = {
+    "frontend": ["frontend", "vue", "react", "nuxt", "next", "angular", "svelte", "web-app", "webapp"],
+    "backend": ["api", "server", "backend", "service", "grpc", "rest"],
+    "fullstack": ["workspace"],
+    "tooling": ["tool", "cli", "script", "util", "extension", "plugin", "agent", "harness"],
+}
+
+
+def _classify_project(project_path: str) -> str:
+    """根据项目路径中的关键字判断项目类型。"""
+    lower = project_path.lower()
+    for ptype, keywords in _PROJECT_TYPE_KEYWORDS.items():
+        if any(kw in lower for kw in keywords):
+            return ptype
+    return "other"
+
+
+def _extract_sequences(
+    tool_calls: list, length: int = _SEQUENCE_LENGTH
+) -> list[tuple[str, ...]]:
+    """从工具调用列表中提取连续的 N-gram 序列。"""
+    names = [tc.name for tc in tool_calls]
+    if len(names) < length:
+        return []
+    return [tuple(names[i : i + length]) for i in range(len(names) - length + 1)]
+
+
+def _short_project_name(project_path: str) -> str:
+    """从项目完整路径提取简短名称。"""
+    parts = project_path.replace("\\", "/").rstrip("/").split("/")
+    parts = [p for p in parts if p]
+    if not parts:
+        return project_path
+    if len(parts) >= 2:
+        return f"{parts[-2]}/{parts[-1]}"
+    return parts[-1]
+
+
+def analyze_cross_project(sessions: list) -> dict:
+    """分析跨项目通用模式。
+
+    Args:
+        sessions: ParsedSession 列表
+
+    Returns:
+        包含 projects、common_tool_sequences、project_type_distribution 的分析结果
+    """
+    # 按项目聚合
+    project_data: dict[str, dict] = defaultdict(
+        lambda: {"sessions": 0, "tool_calls": [], "tool_counter": Counter()}
+    )
+
+    for session in sessions:
+        project = session.project or ""
+        if not project:
+            continue
+        short_name = _short_project_name(project)
+
+        project_data[short_name]["sessions"] += 1
+        project_data[short_name]["tool_calls"].extend(session.tool_calls)
+        for tc in session.tool_calls:
+            project_data[short_name]["tool_counter"][tc.name] += 1
+
+    # 构建项目列表
+    projects: list[dict] = []
+    for name, data in sorted(project_data.items()):
+        top_tools = data["tool_counter"].most_common(5)
+        projects.append({
+            "name": name,
+            "sessions": data["sessions"],
+            "total_tool_calls": len(data["tool_calls"]),
+            "top_tools": top_tools,
+        })
+
+    # 跨项目公共工具序列
+    # 统计每个序列出现在哪些项目中
+    sequence_projects: dict[tuple[str, ...], set] = defaultdict(set)
+    sequence_counts: Counter = Counter()
+
+    for session in sessions:
+        project = session.project or ""
+        if not project:
+            continue
+        short_name = _short_project_name(project)
+
+        for seq in _extract_sequences(session.tool_calls):
+            sequence_projects[seq].add(short_name)
+            sequence_counts[seq] += 1
+
+    # 只保留出现在 >= 2 个项目中的序列
+    common_sequences: list[dict] = []
+    for seq, projs in sorted(
+        sequence_projects.items(), key=lambda x: len(x[1]), reverse=True
+    ):
+        if len(projs) < 2:
+            continue
+        common_sequences.append({
+            "sequence": list(seq),
+            "projects": sorted(projs),
+            "total_count": sequence_counts[seq],
+        })
+
+    # 限制输出量
+    common_sequences = common_sequences[:30]
+
+    # 项目类型分布
+    type_dist: dict[str, int] = Counter()
+    for session in sessions:
+        project = session.project or ""
+        if not project:
+            continue
+        ptype = _classify_project(project)
+        type_dist[ptype] += 1
+
+    return {
+        "project_count": len(project_data),
+        "projects": projects,
+        "common_tool_sequences": common_sequences,
+        "project_type_distribution": dict(type_dist),
+    }

package/scripts/extractors/errors.py

@@ -0,0 +1,255 @@
+"""Signal 3: 错误与重试分析。
+
+统计工具错误率、错误模式分类、自我纠正率、按项目的错误分布。
+同时收集 failure_refs（session_id + tool_call_id + pattern + self_corrected），
+供 evolve 分析时按需回溯 JSONL 获取完整上下文。
+"""
+
+from __future__ import annotations
+
+import json
+import re
+from collections import Counter, defaultdict
+from pathlib import Path
+
+import sys
+
+_PARENT = str(Path(__file__).resolve().parent.parent)
+if _PARENT not in sys.path:
+    sys.path.insert(0, _PARENT)
+
+from config import ERROR_KEYWORDS
+
+
+# ── 错误模式提取 ─────────────────────────────────────
+
+def _extract_error_pattern(message: str) -> str | None:
+    """从错误消息中提取归类模式。
+
+    优先匹配具体的已知模式，然后 fallback 到通用关键词。
+    """
+    msg_lower = message.lower()
+
+    # 按具体程度排序的模式匹配
+    specific_patterns = [
+        (r"could not find the exact text", "Could not find the exact text"),
+        (r"enoent.*no such file", "ENOENT: no such file"),
+        (r"enoent", "ENOENT"),
+        (r"permission denied", "Permission denied"),
+        (r"non-zero exit code", "Non-zero exit code"),
+        (r"command failed.*non-zero", "Command failed (non-zero exit)"),
+        (r"timeout|timed out", "Timeout"),
+        (r"syntaxerror|syntax error", "Syntax error"),
+        (r"typeerror|type error", "TypeError"),
+        (r"importerror|module not found", "ImportError/Module not found"),
+        (r"referenceerror", "ReferenceError"),
+        (r"connection refused|econnrefused", "Connection refused"),
+        (r"out of memory", "Out of memory"),
+    ]
+
+    for pattern, label in specific_patterns:
+        if re.search(pattern, msg_lower):
+            return label
+
+    # Fallback: 检查 ERROR_KEYWORDS
+    for keyword in ERROR_KEYWORDS:
+        if keyword.lower() in msg_lower:
+            # 截取错误消息的前 80 字符作为模式
+            snippet = message.strip()[:80]
+            # 去掉路径等变量部分
+            snippet = re.sub(r"/[\w./\-]+", "<path>", snippet)
+            return snippet
+
+    return None
+
+
+def analyze_errors(sessions) -> dict:
+    """分析错误与重试模式。
+
+    Args:
+        sessions: list[ParsedSession]，解析后的 session 列表。
+
+    Returns:
+        错误统计字典，含 failure_refs 列表。
+    """
+    total_errors = 0
+    # tool_name -> {errors, total}
+    tool_error_stats: dict[str, dict[str, int]] = defaultdict(lambda: {"errors": 0, "total": 0})
+    # project -> {errors, total_calls}
+    project_stats: dict[str, dict[str, int]] = defaultdict(lambda: {"errors": 0, "total_calls": 0})
+
+    # edit 相关
+    edit_total = 0
+    edit_match_failures = 0
+
+    # bash 相关
+    bash_total = 0
+    bash_errors = 0
+
+    # 错误模式统计
+    pattern_counter: Counter[str] = Counter()
+    pattern_examples: dict[str, list[str]] = defaultdict(list)
+
+    # 自我纠正统计
+    error_count_for_correction = 0
+    self_correction_count = 0
+
+    # failure_refs：每个 error 记录一个引用
+    failure_refs: list[dict] = []
+
+    for session in sessions:
+        proj = session.project or session.project_dir or "unknown"
+        sid = session.session_id or session.file_path
+
+        # 建立 tool_call_id -> ToolCall 映射
+        call_by_id: dict[str, object] = {}
+        for tc in session.tool_calls:
+            call_by_id[tc.id] = tc
+
+
+
+        for tr in session.tool_results:
+            tool_error_stats[tr.tool_name]["total"] += 1
+            project_stats[proj]["total_calls"] += 1
+
+            if tr.is_error:
+                total_errors += 1
+                tool_error_stats[tr.tool_name]["errors"] += 1
+                project_stats[proj]["errors"] += 1
+
+                # bash 错误统计
+                if tr.tool_name == "bash":
+                    bash_errors += 1
+
+                # 错误模式提取
+                pattern = _extract_error_pattern(tr.content_preview)
+                if pattern:
+                    pattern_counter[pattern] += 1
+                    example = tr.content_preview.strip()[:200]
+                    if len(pattern_examples[pattern]) < 3:
+                        pattern_examples[pattern].append(example)
+
+                # 记录错误所在的 tool_call_id，用于自我纠正检测
+
+            # 非 error 的 bash 也计入 total
+            if tr.tool_name == "bash":
+                bash_total += 1
+
+            # edit 匹配失败统计：计入 tool_results 遍历中
+            if tr.tool_name == "edit":
+                edit_total += 1
+                if tr.is_error and "could not find" in tr.content_preview.lower():
+                    edit_match_failures += 1
+
+        # ── 自我纠正检测 + failure_refs 收集 ──────
+        # 对每个 error result，检查后续是否有相同工具的 retry
+        # 同时记录每个 error 的 ref（session_id + tool_call_id + pattern + self_corrected）
+        ordered_results = sorted(
+            session.tool_results, key=lambda t: t.timestamp or ""
+        )
+        ordered_calls = sorted(
+            session.tool_calls, key=lambda t: t.timestamp or ""
+        )
+
+        for tr in ordered_results:
+            if not tr.is_error:
+                continue
+            error_count_for_correction += 1
+
+            # 找到这个 error result 对应的 tool_call
+            error_call = call_by_id.get(tr.tool_call_id)
+            if not error_call:
+                # 没有 tool_call 仍记录 ref，但 self_corrected = False
+                pattern = _extract_error_pattern(tr.content_preview)
+                failure_refs.append({
+                    "session_id": sid,
+                    "tool_call_id": tr.tool_call_id,
+                    "pattern": pattern or "Unknown",
+                    "self_corrected": False,
+                })
+                continue
+
+            error_tool_name = getattr(error_call, "name", "")
+            error_timestamp = getattr(error_call, "timestamp", "")
+
+            # 在 error_call 之后找是否有同名的 tool_call
+            found_retry = False
+            for tc in ordered_calls:
+                if tc.timestamp <= error_timestamp:
+                    continue
+                if tc.name == error_tool_name:
+                    found_retry = True
+                    break
+            if found_retry:
+                self_correction_count += 1
+
+            # 收集 failure_ref
+            pattern = _extract_error_pattern(tr.content_preview)
+            failure_refs.append({
+                "session_id": sid,
+                "tool_call_id": tr.tool_call_id,
+                "pattern": pattern or "Unknown",
+                "self_corrected": found_retry,
+            })
+
+    # ── 汇总 by_tool ────────────────────────────
+    by_tool: dict[str, dict] = {}
+    for tool_name, stats in tool_error_stats.items():
+        total = stats["total"]
+        errors = stats["errors"]
+        by_tool[tool_name] = {
+            "errors": errors,
+            "total": total,
+            "error_rate": round(errors / total, 4) if total else 0.0,
+        }
+
+    # ── top_error_patterns ──────────────────────
+    top_error_patterns = [
+        {
+            "pattern": pattern,
+            "count": count,
+            "examples": pattern_examples.get(pattern, [])[:3],
+        }
+        for pattern, count in pattern_counter.most_common(10)
+    ]
+
+    # ── self_correction_rate ────────────────────
+    self_correction_rate = round(
+        self_correction_count / error_count_for_correction, 4
+    ) if error_count_for_correction else 0.0
+
+    # ── by_project ──────────────────────────────
+    by_project = sorted(
+        [
+            {
+                "project": proj,
+                "errors": stats["errors"],
+                "total_calls": stats["total_calls"],
+                "error_rate": round(
+                    stats["errors"] / stats["total_calls"], 4
+                ) if stats["total_calls"] else 0.0,
+            }
+            for proj, stats in project_stats.items()
+        ],
+        key=lambda x: x["errors"],
+        reverse=True,
+    )
+
+    # ── bash_failure_rate ───────────────────────
+    bash_failure_rate = round(bash_errors / bash_total, 4) if bash_total else 0.0
+
+    # ── edit_match_failure_rate ─────────────────
+    edit_match_failure_rate = round(
+        edit_match_failures / edit_total, 4
+    ) if edit_total else 0.0
+
+    return {
+        "total_errors": total_errors,
+        "by_tool": by_tool,
+        "bash_failure_rate": bash_failure_rate,
+        "edit_match_failure_rate": edit_match_failure_rate,
+        "top_error_patterns": top_error_patterns,
+        "self_correction_rate": self_correction_rate,
+        "by_project": by_project,
+        "failure_refs": failure_refs,
+    }

package/scripts/extractors/satisfaction.py

@@ -0,0 +1,176 @@
+"""Signal 7: 用户满意度隐式信号。
+
+通过 session 粒度的间接指标推断用户满意度：
+单轮完成率、平均轮数、工具调用密度、session 时长分布。
+"""
+
+from __future__ import annotations
+
+import sys
+from collections import defaultdict
+from datetime import datetime, timezone
+from pathlib import Path
+
+# 使 config 可导入
+sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
+from config import SINGLE_TURN_MAX_MESSAGES
+
+
+def _parse_ts(ts_str: str) -> datetime | None:
+    """安全解析 ISO 时间戳。"""
+    if not ts_str:
+        return None
+    try:
+        dt = datetime.fromisoformat(ts_str)
+        if dt.tzinfo is None:
+            dt = dt.replace(tzinfo=timezone.utc)
+        return dt
+    except (ValueError, TypeError):
+        return None
+
+
+def _percentile(sorted_values: list[float], pct: float) -> float:
+    """计算百分位数（线性插值）。"""
+    if not sorted_values:
+        return 0.0
+    idx = pct / 100.0 * (len(sorted_values) - 1)
+    lower = int(idx)
+    upper = min(lower + 1, len(sorted_values) - 1)
+    frac = idx - lower
+    return round(sorted_values[lower] + frac * (sorted_values[upper] - sorted_values[lower]), 2)
+
+
+def _short_project_name(project_path: str) -> str:
+    """从项目完整路径提取简短名称。"""
+    parts = project_path.replace("\\", "/").rstrip("/").split("/")
+    parts = [p for p in parts if p]
+    if not parts:
+        return project_path
+    if len(parts) >= 2:
+        return f"{parts[-2]}/{parts[-1]}"
+    return parts[-1]
+
+
+def _session_last_timestamp(session) -> str:
+    """获取 session 中最后一条记录的时间戳。
+
+    遍历所有时间戳字段，取最大值。
+    """
+    timestamps: list[str] = []
+
+    for tc in session.tool_calls:
+        if tc.timestamp:
+            timestamps.append(tc.timestamp)
+    for tr in session.tool_results:
+        if tr.timestamp:
+            timestamps.append(tr.timestamp)
+    for um in session.user_messages:
+        if um.timestamp:
+            timestamps.append(um.timestamp)
+    for ui in session.usage_list:
+        if ui.timestamp:
+            timestamps.append(ui.timestamp)
+
+    return max(timestamps) if timestamps else ""
+
+
+def analyze_satisfaction(sessions: list) -> dict:
+    """分析用户满意度隐式信号。
+
+    Args:
+        sessions: ParsedSession 列表
+
+    Returns:
+        包含 single_turn_completion_rate、session_duration_stats、by_project 等的分析结果
+    """
+    total_sessions = len(sessions)
+    if total_sessions == 0:
+        return {
+            "total_sessions": 0,
+            "single_turn_completion_rate": 0.0,
+            "avg_turns_per_session": 0.0,
+            "avg_tool_calls_per_session": 0.0,
+            "session_duration_stats": {
+                "median_minutes": 0.0,
+                "p25_minutes": 0.0,
+                "p75_minutes": 0.0,
+                "max_minutes": 0.0,
+            },
+            "by_project": [],
+        }
+
+    single_turn_count = 0
+    total_turns = 0
+    total_tool_calls = 0
+    durations_minutes: list[float] = []
+
+    # 按项目聚合
+    project_stats: dict[str, dict] = defaultdict(
+        lambda: {"sessions": 0, "turns": 0, "single_turn": 0}
+    )
+
+    for session in sessions:
+        user_count = len(session.user_messages)
+        tool_count = len(session.tool_calls)
+
+        total_turns += user_count
+        total_tool_calls += tool_count
+
+        # 单轮判定：用户消息数 <= SINGLE_TURN_MAX_MESSAGES
+        is_single_turn = user_count <= SINGLE_TURN_MAX_MESSAGES
+        if is_single_turn:
+            single_turn_count += 1
+
+        # 计算时长
+        start_ts = _parse_ts(session.start_time)
+        end_ts_str = _session_last_timestamp(session)
+        end_ts = _parse_ts(end_ts_str) if end_ts_str else None
+
+        if start_ts and end_ts and end_ts > start_ts:
+            delta = (end_ts - start_ts).total_seconds() / 60.0
+            durations_minutes.append(delta)
+
+        # 项目级统计
+        project = session.project or ""
+        if project:
+            short_name = _short_project_name(project)
+            project_stats[short_name]["sessions"] += 1
+            project_stats[short_name]["turns"] += user_count
+            if is_single_turn:
+                project_stats[short_name]["single_turn"] += 1
+
+    # 时长统计
+    durations_minutes.sort()
+    duration_stats = {
+        "median_minutes": _percentile(durations_minutes, 50),
+        "p25_minutes": _percentile(durations_minutes, 25),
+        "p75_minutes": _percentile(durations_minutes, 75),
+        "max_minutes": durations_minutes[-1] if durations_minutes else 0.0,
+    }
+
+    # 按项目汇总（只含 sessions >= 2 的项目）
+    by_project: list[dict] = []
+    for name, stats in sorted(project_stats.items()):
+        if stats["sessions"] < 2:
+            continue
+        avg_turns = stats["turns"] / stats["sessions"]
+        single_rate = stats["single_turn"] / stats["sessions"]
+        by_project.append({
+            "project": name,
+            "sessions": stats["sessions"],
+            "avg_turns": round(avg_turns, 2),
+            "single_turn_rate": round(single_rate, 4),
+        })
+
+    return {
+        "total_sessions": total_sessions,
+        "single_turn_completion_rate": round(
+            single_turn_count / total_sessions, 4
+        ),
+        "avg_turns_per_session": round(total_turns / total_sessions, 2),
+        "avg_tool_calls_per_session": round(
+            total_tool_calls / total_sessions, 2
+        ),
+        "session_duration_stats": duration_stats,
+        "by_project": by_project,
+    }