homegraph 1.1.2 → 1.2.0

package/scripts/exp_boundary_eval/run_all.py

@@ -0,0 +1,378 @@
+#!/usr/bin/env python3
+"""
+Run boundary-eval experiments sequentially. Multi-agent + HomeGraph A/B arm support.
+
+Usage:
+    python run_all.py --both-arms --agent deveco     # 一条命令跑 baseline + homegraph
+    python run_all.py --agent deveco --arm homegraph
+    python run_all.py --agent deveco --arm baseline
+    python run_all.py 1-1 1-2 1-3
+    python run_all.py --repo-path /path/to/photos
+    python run_all.py --both-arms --agent deveco --repo-path /path/to/photos --skip-index
+"""
+
+import argparse
+import json
+import os
+import shutil
+import subprocess
+import sys
+from pathlib import Path
+from typing import Optional, Tuple
+
+import run_one
+import run_session
+import analyze
+from _utils import (DATA_DIR, EVAL_ROOT, LOG_DIR, OUTPUT_DIR, GREEN, NC, append_run_log,
+                    current_time_ms, create_output_dirs, error, get_agent, header,
+                    info, load_config, log, subprocess_no_window_kwargs, verify_agent_binary, warn)
+from deveco_arm import (ensure_homegraph_index, git_clean_repo,
+                        write_deveco_project_config)
+
+DEFAULT_EXPERIMENTS = ["1-1", "1-2", "1-3"]
+SHARED_CLONE_DIR = DATA_DIR / "clone"
+
+
+_NW = subprocess_no_window_kwargs()
+
+
+def clone_repo(url: str, branch: str, clone_dir: Path) -> bool:
+    clone_dir = Path(clone_dir)
+    if (clone_dir / ".git").is_dir():
+        log(f"Clone exists, updating: {clone_dir}")
+        subprocess.run(["git", "fetch", "origin", branch], cwd=clone_dir, capture_output=True, text=True, **_NW)
+        subprocess.run(["git", "checkout", branch], cwd=clone_dir, capture_output=True, text=True, **_NW)
+        subprocess.run(["git", "reset", "--hard", f"origin/{branch}"], cwd=clone_dir, capture_output=True, text=True, **_NW)
+        return True
+    log(f"Cloning {url} (branch: {branch})...")
+    if clone_dir.exists():
+        shutil.rmtree(clone_dir)
+    clone_dir.parent.mkdir(parents=True, exist_ok=True)
+    r = subprocess.run(
+        ["git", "clone", "--depth", "1", "--branch", branch, url, str(clone_dir)],
+        capture_output=True, text=True,
+        **_NW,
+    )
+    if r.returncode != 0:
+        error(f"Clone failed: {r.stderr}")
+        return False
+    r2 = subprocess.run(["git", "rev-parse", "--short", "HEAD"], capture_output=True, text=True, cwd=clone_dir, **_NW)
+    log(f"Clone complete: {r2.stdout.strip()}")
+    return True
+
+
+def reset_clone(clone_dir: Path, branch: str, arm: str = "baseline"):
+    subprocess.run(["git", "checkout", branch], cwd=clone_dir, capture_output=True, text=True, **_NW)
+    subprocess.run(["git", "checkout", "."], cwd=clone_dir, capture_output=True, text=True, **_NW)
+    git_clean_repo(clone_dir, arm)
+
+
+def reset_local(work_dir: Path, arm: str = "baseline"):
+    subprocess.run(["git", "checkout", "."], cwd=work_dir, capture_output=True, text=True, **_NW)
+    git_clean_repo(work_dir, arm)
+
+
+def ensure_homegraph_index_for_arm(
+    repo_root: Path, arm: str, *, skip_index: bool = False,
+) -> dict:
+    """Index repo when running the homegraph arm."""
+    if arm != "homegraph":
+        return {"homegraph_index_ms": 0, "homegraph_index_success": None}
+    return ensure_homegraph_index(repo_root, skip=skip_index)
+
+
+def write_run_manifest(state_dir: Path, results_dir: Path, meta: dict):
+    manifest = {
+        **meta,
+        "results_dir": str(results_dir),
+        "state_dir": str(state_dir),
+    }
+    for d in (state_dir, results_dir):
+        d.mkdir(parents=True, exist_ok=True)
+        (d / "run_manifest.json").write_text(
+            json.dumps(manifest, indent=2, ensure_ascii=False), encoding="utf-8"
+        )
+
+
+def resolve_work_dir(args, state_dir: Path) -> Tuple[Path, Optional[Path], bool]:
+    """Return (work_dir, ephemeral_clone_dir_or_none, owns_ephemeral_clone)."""
+    if args.repo_path:
+        work_dir = Path(args.repo_path).resolve()
+        if not (work_dir / ".git").is_dir():
+            error(f"--repo-path 不是 git 仓库: {work_dir}")
+            sys.exit(1)
+        os.environ["REPO_PATH"] = str(work_dir)
+        os.environ.pop("REPO_CLONE", None)
+        log(f"使用本地仓库: {work_dir}")
+        return work_dir, None, False
+
+    if args.both_arms:
+        clone_dir = SHARED_CLONE_DIR
+        if not clone_repo(args.repo_url, args.branch, clone_dir):
+            error("Clone 失败。请检查网络，或改用 --repo-path。")
+            sys.exit(1)
+        os.environ["REPO_PATH"] = str(clone_dir)
+        os.environ.pop("REPO_CLONE", None)
+        log(f"A/B 共用仓库: {clone_dir}")
+        return clone_dir, None, False
+
+    clone_dir = state_dir / "clone"
+    if not clone_repo(args.repo_url, args.branch, clone_dir):
+        error("Clone 失败。请改用 --repo-path 指定本地 checkout。")
+        sys.exit(1)
+    os.environ["REPO_CLONE"] = str(clone_dir)
+    os.environ.pop("REPO_PATH", None)
+    return clone_dir, clone_dir, True
+
+
+def run_suite(
+    args, arm: str, *, prebuilt_index_meta: Optional[dict] = None,
+) -> dict:
+    """Run one arm (baseline or homegraph). Returns summary dict."""
+    agent = get_agent(args.agent)
+    skip_perms = not args.no_skip_permissions
+    experiments = args.experiments or DEFAULT_EXPERIMENTS
+    total = len(experiments)
+    passed = 0
+    failed = 0
+
+    results_dir, state_dir = create_output_dirs(args.agent or "default", arm)
+    work_dir, ephemeral_clone, owns_clone = resolve_work_dir(args, state_dir)
+
+    if arm == "baseline":
+        hg_dir = work_dir / ".homegraph"
+        if hg_dir.exists():
+            log("Baseline: 保留 .homegraph（A/B 共用仓库；baseline 通过 deveco.jsonc deny homegraph_* 隔离）")
+
+    if prebuilt_index_meta is not None:
+        index_meta = prebuilt_index_meta
+    else:
+        index_meta = ensure_homegraph_index_for_arm(
+            work_dir, arm, skip_index=getattr(args, "skip_index", False),
+        )
+
+    if arm == "homegraph" and not index_meta.get("homegraph_index_success"):
+        error("HomeGraph 索引失败，homegraph 组无法评测。")
+        err = index_meta.get("homegraph_index_error", "")
+        if err:
+            error(f"  原因: {err}")
+        error("  请先在 homegraph 仓库执行 npm run build，再重试 run_all.py --arm homegraph")
+        append_run_log(state_dir, f"ABORT: homegraph index failed: {err}")
+        write_run_manifest(state_dir, results_dir, {
+            "arm": arm, "agent": agent["name"], "agent_key": args.agent,
+            "branch": args.branch, "repo_url": args.repo_url,
+            "repo_root": str(work_dir), "experiments": experiments,
+            "aborted": True, **index_meta,
+        })
+        sys.exit(1)
+
+    if agent.get("binary") == "deveco":
+        write_deveco_project_config(work_dir, arm)
+
+    write_run_manifest(state_dir, results_dir, {
+        "arm": arm,
+        "agent": agent["name"],
+        "agent_key": args.agent,
+        "branch": args.branch,
+        "repo_url": args.repo_url,
+        "repo_root": str(work_dir),
+        "experiments": experiments,
+        **index_meta,
+    })
+    append_run_log(state_dir, f"开始: arm={arm}, agent={agent['name']}, exps={experiments}")
+
+    os.environ["EVAL_ARM"] = arm
+
+    header(f"边界评测 — 档位切换 ({arm})")
+    print(f"组别    : {arm}")
+    print(f"Agent   : {agent['name']} ({agent['binary']})")
+    print(f"仓库    : {work_dir}")
+    print(f"分支    : {args.branch}")
+    print(f"权限    : {'跳过确认' if skip_perms else '需确认'}")
+    print(f"实验数  : {total}")
+    print(f"输出    : {results_dir}")
+    print(f"日志    : {state_dir}\n")
+
+    overall_start = current_time_ms()
+
+    for exp_id in experiments:
+        header(f"实验 {exp_id}")
+        append_run_log(state_dir, f"开始实验 {exp_id}")
+
+        if ephemeral_clone:
+            log("重置 clone 到干净状态...")
+            reset_clone(ephemeral_clone, args.branch, arm)
+        else:
+            log("重置本地仓库...")
+            reset_local(work_dir, arm)
+
+        if exp_id == "5":
+            ok = run_session.run_session("5", skip_perms, args.agent,
+                                         results_dir=results_dir, state_dir=state_dir)
+        else:
+            ok = run_one.run_experiment(exp_id, skip_perms, args.agent,
+                                        results_dir=results_dir, state_dir=state_dir)
+
+        if ok:
+            passed += 1
+        else:
+            failed += 1
+            error(f"实验 {exp_id} 失败")
+        print()
+
+    overall_end = current_time_ms()
+    duration = (overall_end - overall_start) // 1000
+
+    append_run_log(state_dir, f"完成: passed={passed}, failed={failed}, duration_s={duration}")
+
+    header(f"组别完成 — {arm}")
+    print(f"  通过   : {passed} / {total}")
+    print(f"  失败   : {failed} / {total}")
+    print(f"  耗时   : {duration // 60}m {duration % 60}s")
+    print(f"  输出   : {results_dir}")
+    print(f"  日志   : {state_dir}")
+
+    if owns_clone and ephemeral_clone and not args.keep_clone and ephemeral_clone.exists():
+        warn(f"删除临时 clone: {ephemeral_clone}")
+        shutil.rmtree(ephemeral_clone)
+
+    return {
+        "arm": arm,
+        "results_dir": results_dir,
+        "state_dir": state_dir,
+        "passed": passed,
+        "failed": failed,
+        "duration_s": duration,
+        "homegraph_index_ms": index_meta.get("homegraph_index_ms", 0),
+    }
+
+
+def _generate_reports(summaries: list, agent_key: str):
+    """Run analyze.py logic after experiments; print report paths."""
+    reports = []
+    for s in summaries:
+        try:
+            path = analyze.analyze_run(s["results_dir"], verbose=False)
+            reports.append((s["arm"], path))
+        except ValueError as e:
+            warn(f"分析 {s['arm']} 失败: {e}")
+
+    compare_path = None
+    if len(summaries) == 2 and len(reports) == 2:
+        try:
+            baseline_s = next(s for s in summaries if s["arm"] == "baseline")
+            homegraph_s = next(s for s in summaries if s["arm"] == "homegraph")
+            compare_path = analyze.compare_runs(
+                baseline_s["results_dir"],
+                homegraph_s["results_dir"],
+                agent=agent_key,
+                baseline_wall_s=baseline_s["duration_s"],
+                homegraph_wall_s=homegraph_s["duration_s"],
+                verbose=True,
+            )
+        except ValueError as e:
+            warn(f"A/B 对比报告失败: {e}")
+
+    baseline_report = next((p for arm, p in reports if arm == "baseline"), None)
+    homegraph_report = next((p for arm, p in reports if arm == "homegraph"), None)
+    single_report = reports[0][1] if len(reports) == 1 else None
+
+    analyze.print_report_locations(
+        baseline_report=baseline_report,
+        homegraph_report=homegraph_report,
+        compare_report=compare_path,
+        single_report=single_report,
+    )
+    return reports, compare_path
+
+
+def _blocking_homegraph_index(work_dir: Path, *, skip_index: bool = False) -> dict:
+    """Run full ``homegraph index`` and block until complete (or validate existing index)."""
+    if skip_index:
+        header("HomeGraph 索引检查（--skip-index，不执行 index）")
+    else:
+        header("HomeGraph 全量 index（阻塞，Photos 约 15–30 分钟）")
+    log(f"仓库: {work_dir}")
+    if skip_index:
+        log("模式: 仅校验已有 .homegraph/（可从同 commit 的其他机器复制整个目录）")
+    else:
+        log("命令: node --stack-size=8192 … homegraph index（见 deveco_arm.HOMEGRAPH_NODE_STACK_SIZE_KB）")
+    meta = ensure_homegraph_index(work_dir, skip=skip_index)
+    if not meta.get("homegraph_index_success"):
+        error("HomeGraph index 失败，中止。")
+        err = meta.get("homegraph_index_error", "")
+        if err:
+            error(f"  原因: {err}")
+        sys.exit(1)
+    return meta
+
+
+def main():
+    analyze._configure_stdout()
+    cfg = load_config()
+    meta = cfg.get("meta", {})
+
+    parser = argparse.ArgumentParser(description="HomeGraph 档位切换边界评测")
+    parser.add_argument("--agent", default=os.environ.get("EVAL_AGENT", "deveco"),
+                        help="Agent：claude / cursor / codex / opencode / deveco")
+    parser.add_argument("--arm", default=os.environ.get("EVAL_ARM", "baseline"),
+                        choices=["baseline", "homegraph"],
+                        help="baseline=不带 HomeGraph；homegraph=带 HomeGraph")
+    parser.add_argument("--both-arms", action="store_true",
+                        help="一条命令依次跑 baseline 和 homegraph 两组 A/B")
+    parser.add_argument("--repo-path", default=os.environ.get("REPO_PATH", ""),
+                        help="使用已有本地 clone，跳过 git clone")
+    parser.add_argument("--keep-clone", action="store_true")
+    parser.add_argument(
+        "--skip-index",
+        action="store_true",
+        help="不跑 homegraph init/index；要求仓库根目录已有健康的 .homegraph/（可复制自同 commit 的其他 checkout）",
+    )
+    parser.add_argument("--no-skip-permissions", action="store_true")
+    parser.add_argument("--branch", default=os.environ.get("BRANCH", meta.get("default_branch", "weekly_20260601")))
+    parser.add_argument("--repo-url", default=os.environ.get("REPO_URL", meta.get("repo_url", "")))
+    parser.add_argument("experiments", nargs="*")
+    args = parser.parse_args()
+
+    if not args.experiments:
+        args.experiments = DEFAULT_EXPERIMENTS
+
+    agent = get_agent(args.agent)
+    ok, resolved, err_msg = verify_agent_binary(agent)
+    if not ok:
+        error(err_msg)
+        sys.exit(1)
+    log(f"Agent CLI: {resolved}")
+
+    if args.both_arms and args.arm != "baseline":
+        warn("--both-arms 已指定，忽略 --arm")
+
+    if args.both_arms:
+        _rd, _sd = create_output_dirs(args.agent or "default", "baseline")
+        work_dir, _, _ = resolve_work_dir(args, _sd)
+
+        # Index FIRST so the user sees it immediately (15–30 min); baseline does not use MCP.
+        index_meta = _blocking_homegraph_index(work_dir, skip_index=args.skip_index)
+
+        summaries = []
+        print(f"\n{'=' * 60}\n  开始 homegraph 组（先看 HomeGraph 是否生效）\n{'=' * 60}\n")
+        summaries.append(run_suite(args, "homegraph", prebuilt_index_meta=index_meta))
+
+        print(f"\n{'=' * 60}\n  开始 baseline 组（无 HomeGraph）\n{'=' * 60}\n")
+        summaries.append(run_suite(args, "baseline"))
+
+        header("A/B 全部完成")
+        for s in summaries:
+            print(f"  [{s['arm']}] 通过 {s['passed']}，失败 {s['failed']}，"
+                  f"耗时 {s['duration_s'] // 60}m{s['duration_s'] % 60}s")
+            print(f"         输出 → {s['results_dir']}")
+        _generate_reports(summaries, args.agent or "default")
+        sys.exit(0 if all(s["failed"] == 0 for s in summaries) else 1)
+
+    summary = run_suite(args, args.arm)
+    _generate_reports([summary], args.agent or "default")
+    sys.exit(0 if summary["failed"] == 0 else 1)
+
+
+if __name__ == "__main__":
+    main()

package/scripts/exp_boundary_eval/run_one.py

@@ -0,0 +1,165 @@
+#!/usr/bin/env python3
+"""Run a single one-shot experiment. Supports multiple agents via --agent."""
+
+import os
+import subprocess
+import sys
+from pathlib import Path
+from typing import Optional
+
+import setup as setup_mod
+from _utils import (OUTPUT_DIR, append_run_log, build_agent_cmd, current_time_ms,
+                    error, find_experiment, get_agent, info, header,
+                    parse_output, resolve_agent_model, run_monitored_subprocess,
+                    subprocess_no_window_kwargs, warn, write_json)
+from deveco_arm import analyze_homegraph_stream
+
+
+def run_experiment(exp_id: str, skip_permissions: bool = True,
+                   agent_name: str = "",
+                   results_dir: Optional[Path] = None,
+                   state_dir: Optional[Path] = None) -> bool:
+    exp = find_experiment(exp_id)
+    if exp is None:
+        error(f"Experiment {exp_id} not found in config")
+        return False
+    if exp.get("type") == "session":
+        error(f"Experiment {exp_id} is session type. Use run_session.py.")
+        return False
+
+    agent = get_agent(agent_name)
+    clone_path = os.environ.get("REPO_CLONE", "")
+    local_repo = os.environ.get("REPO_PATH", "")
+    if clone_path:
+        repo_root = Path(clone_path)
+    elif local_repo:
+        repo_root = Path(local_repo)
+    else:
+        error("No repo. Use run_all.py to clone, or set REPO_PATH / REPO_CLONE.")
+        return False
+
+    title = exp["title"]
+    prompt = exp["prompt"]
+    max_turns = exp.get("max_turns", 15)
+
+    header(f"Experiment {exp_id}: {title}")
+    info(f"Agent: {agent['name']} ({agent['binary']}) | Repo: {repo_root}")
+    info(f"Max turns: {max_turns} | Permissions: {'skip' if skip_permissions else 'ask'}")
+
+    session_id, _ = setup_mod.prepare_environment(exp_id, "full", agent_name,
+                                                    state_dir=state_dir)
+
+    result_dir = results_dir / exp_id if results_dir else OUTPUT_DIR / exp_id
+    result_dir.mkdir(parents=True, exist_ok=True)
+    (result_dir / "prompt.txt").write_text(prompt, encoding="utf-8")
+    write_json(result_dir / "experiment_config.json", {**exp, "agent": agent_name or "default"})
+
+    info("Starting agent...")
+    start_ms = current_time_ms()
+    os.chdir(repo_root)
+
+    cmd = build_agent_cmd(agent, session_id, max_turns, prompt,
+                          resume=False, skip_permissions=skip_permissions,
+                          repo_root=repo_root)
+
+    stream_path = result_dir / "stream_output.jsonl"
+    text_path = result_dir / "raw_output.txt"
+    track_hg = os.environ.get("EVAL_ARM", "") == "homegraph"
+
+    with open(stream_path, "w", encoding="utf-8") as stream_f:
+        result, mem = run_monitored_subprocess(
+            cmd, stdout=stream_f, stderr=subprocess.STDOUT, text=True,
+            track_homegraph=track_hg,
+        )
+    exit_code = result.returncode
+
+    end_ms = current_time_ms()
+    dur_ms = end_ms - start_ms
+    dur_s = f"{dur_ms / 1000:.1f}"
+    info(f"Completed in {dur_s}s (exit={exit_code})")
+
+    # Parse agent output
+    parser_name = agent.get("parser", "claude_stream_json")
+    stats = parse_output(stream_path, parser_name)
+
+    if exit_code != 0:
+        error(f"Agent 退出码 {exit_code}")
+        for err in stats.errors[:3]:
+            error(f"  {err}")
+        if "Unexpected server error" in " ".join(stats.errors):
+            warn("DevEco 服务端报错 — 通常为 API/网络/配额问题，与评测脚本无关。"
+                 "请检查 DevEco 登录状态、模型配置，稍后重试 `deveco run`。")
+        elif stats.tool_calls == 0 and not stats.errors:
+            warn("无工具调用且无错误事件 — 检查 stream_output.jsonl 是否为空。")
+
+    # Human-readable text
+    text_content = "\n".join(stats.text_lines)
+    text_path.write_text(text_content or "(no text output)", encoding="utf-8")
+
+    # Modified files
+    r2 = subprocess.run(
+        ["git", "diff", "--name-only"], capture_output=True, text=True, cwd=repo_root,
+        **subprocess_no_window_kwargs(),
+    )
+    mod_files = len([l for l in r2.stdout.strip().split("\n") if l])
+
+    hg_stream = analyze_homegraph_stream(stream_path)
+
+    results = {
+        "experiment_id": exp_id, "title": title, "agent": agent["name"],
+        "agent_binary": agent["binary"], "session_id": session_id,
+        "start_time_ms": start_ms, "end_time_ms": end_ms,
+        "duration_ms": dur_ms, "duration_s": dur_s,
+        "exit_code": exit_code, "max_turns": max_turns,
+        "skip_permissions": skip_permissions,
+        "tool_calls": stats.tool_calls, "tool_names": stats.tool_names,
+        "homegraph_tool_calls": stats.homegraph_tool_calls,
+        "used_homegraph": stats.used_homegraph,
+        **hg_stream,
+        "eval_arm": os.environ.get("EVAL_ARM", ""),
+        "files_read": list(set(stats.files_read)),
+        "files_edited": list(set(stats.files_edited)),
+        "model": resolve_agent_model(agent, stats),
+        "deveco_session_id": stats.deveco_session_id,
+        "input_tokens": stats.total_input_tokens,
+        "output_tokens": stats.total_output_tokens,
+        "assistant_messages": stats.assistant_messages,
+        "max_turns_hit": stats.max_turns_hit,
+        "errors": stats.errors[:5],
+        "modified_files": mod_files,
+        "memory": mem.to_dict(),
+    }
+    write_json(result_dir / "results.json", results)
+
+    if state_dir:
+        append_run_log(state_dir, f"Exp {exp_id} finished in {dur_s}s, tools={stats.tool_calls}, exit={exit_code}")
+    info(f"Results saved: {result_dir}/")
+    print(f"  Duration: {dur_s}s | Tool calls: {stats.tool_calls} | "
+          f"HG tools: {stats.homegraph_tool_calls} | "
+          f"Files read: {len(set(stats.files_read))} | Edited: {mod_files} | "
+          f"Peak mem: {mem.peak_combined_rss_mb:.0f} MB")
+    if stats.max_turns_hit:
+        print(f"  ⚠️  Max turns ({max_turns}) hit")
+    return exit_code == 0
+
+
+if __name__ == "__main__":
+    # Parse: python run_one.py <id> [--agent <name>] [--no-skip-permissions]
+    args = sys.argv[1:]
+    exp_id = args[0] if args else ""
+    agent_name = ""
+    skip = True
+
+    i = 1
+    while i < len(args):
+        if args[i] == "--agent" and i + 1 < len(args):
+            agent_name = args[i + 1]; i += 2
+        elif args[i] == "--no-skip-permissions":
+            skip = False; i += 1
+        else:
+            i += 1
+
+    if not exp_id:
+        error("Usage: python run_one.py <experiment_id> [--agent <name>]")
+        sys.exit(1)
+    sys.exit(0 if run_experiment(exp_id, skip, agent_name) else 1)