@xopcai/xopc 0.0.103 → 0.0.105

package/skills/tools/skill-creator/scripts/run_loop.py

@@ -1,328 +0,0 @@
-#!/usr/bin/env python3
-"""Run the eval + improve loop until all pass or max iterations reached.
-
-Combines run_eval.py and improve_description.py in a loop, tracking history
-and returning the best description found. Supports train/test split to prevent
-overfitting.
-"""
-
-import argparse
-import json
-import random
-import sys
-import tempfile
-import time
-import webbrowser
-from pathlib import Path
-
-from scripts.generate_report import generate_html
-from scripts.improve_description import improve_description
-from scripts.run_eval import find_project_root, run_eval
-from scripts.utils import parse_skill_md
-
-
-def split_eval_set(eval_set: list[dict], holdout: float, seed: int = 42) -> tuple[list[dict], list[dict]]:
-    """Split eval set into train and test sets, stratified by should_trigger."""
-    random.seed(seed)
-
-    # Separate by should_trigger
-    trigger = [e for e in eval_set if e["should_trigger"]]
-    no_trigger = [e for e in eval_set if not e["should_trigger"]]
-
-    # Shuffle each group
-    random.shuffle(trigger)
-    random.shuffle(no_trigger)
-
-    # Calculate split points
-    n_trigger_test = max(1, int(len(trigger) * holdout))
-    n_no_trigger_test = max(1, int(len(no_trigger) * holdout))
-
-    # Split
-    test_set = trigger[:n_trigger_test] + no_trigger[:n_no_trigger_test]
-    train_set = trigger[n_trigger_test:] + no_trigger[n_no_trigger_test:]
-
-    return train_set, test_set
-
-
-def run_loop(
-    eval_set: list[dict],
-    skill_path: Path,
-    description_override: str | None,
-    num_workers: int,
-    timeout: int,
-    max_iterations: int,
-    runs_per_query: int,
-    trigger_threshold: float,
-    holdout: float,
-    model: str,
-    verbose: bool,
-    live_report_path: Path | None = None,
-    log_dir: Path | None = None,
-) -> dict:
-    """Run the eval + improvement loop."""
-    project_root = find_project_root()
-    name, original_description, content = parse_skill_md(skill_path)
-    current_description = description_override or original_description
-
-    # Split into train/test if holdout > 0
-    if holdout > 0:
-        train_set, test_set = split_eval_set(eval_set, holdout)
-        if verbose:
-            print(f"Split: {len(train_set)} train, {len(test_set)} test (holdout={holdout})", file=sys.stderr)
-    else:
-        train_set = eval_set
-        test_set = []
-
-    history = []
-    exit_reason = "unknown"
-
-    for iteration in range(1, max_iterations + 1):
-        if verbose:
-            print(f"\n{'='*60}", file=sys.stderr)
-            print(f"Iteration {iteration}/{max_iterations}", file=sys.stderr)
-            print(f"Description: {current_description}", file=sys.stderr)
-            print(f"{'='*60}", file=sys.stderr)
-
-        # Evaluate train + test together in one batch for parallelism
-        all_queries = train_set + test_set
-        t0 = time.time()
-        all_results = run_eval(
-            eval_set=all_queries,
-            skill_name=name,
-            description=current_description,
-            num_workers=num_workers,
-            timeout=timeout,
-            project_root=project_root,
-            runs_per_query=runs_per_query,
-            trigger_threshold=trigger_threshold,
-            model=model,
-        )
-        eval_elapsed = time.time() - t0
-
-        # Split results back into train/test by matching queries
-        train_queries_set = {q["query"] for q in train_set}
-        train_result_list = [r for r in all_results["results"] if r["query"] in train_queries_set]
-        test_result_list = [r for r in all_results["results"] if r["query"] not in train_queries_set]
-
-        train_passed = sum(1 for r in train_result_list if r["pass"])
-        train_total = len(train_result_list)
-        train_summary = {"passed": train_passed, "failed": train_total - train_passed, "total": train_total}
-        train_results = {"results": train_result_list, "summary": train_summary}
-
-        if test_set:
-            test_passed = sum(1 for r in test_result_list if r["pass"])
-            test_total = len(test_result_list)
-            test_summary = {"passed": test_passed, "failed": test_total - test_passed, "total": test_total}
-            test_results = {"results": test_result_list, "summary": test_summary}
-        else:
-            test_results = None
-            test_summary = None
-
-        history.append({
-            "iteration": iteration,
-            "description": current_description,
-            "train_passed": train_summary["passed"],
-            "train_failed": train_summary["failed"],
-            "train_total": train_summary["total"],
-            "train_results": train_results["results"],
-            "test_passed": test_summary["passed"] if test_summary else None,
-            "test_failed": test_summary["failed"] if test_summary else None,
-            "test_total": test_summary["total"] if test_summary else None,
-            "test_results": test_results["results"] if test_results else None,
-            # For backward compat with report generator
-            "passed": train_summary["passed"],
-            "failed": train_summary["failed"],
-            "total": train_summary["total"],
-            "results": train_results["results"],
-        })
-
-        # Write live report if path provided
-        if live_report_path:
-            partial_output = {
-                "original_description": original_description,
-                "best_description": current_description,
-                "best_score": "in progress",
-                "iterations_run": len(history),
-                "holdout": holdout,
-                "train_size": len(train_set),
-                "test_size": len(test_set),
-                "history": history,
-            }
-            live_report_path.write_text(generate_html(partial_output, auto_refresh=True, skill_name=name))
-
-        if verbose:
-            def print_eval_stats(label, results, elapsed):
-                pos = [r for r in results if r["should_trigger"]]
-                neg = [r for r in results if not r["should_trigger"]]
-                tp = sum(r["triggers"] for r in pos)
-                pos_runs = sum(r["runs"] for r in pos)
-                fn = pos_runs - tp
-                fp = sum(r["triggers"] for r in neg)
-                neg_runs = sum(r["runs"] for r in neg)
-                tn = neg_runs - fp
-                total = tp + tn + fp + fn
-                precision = tp / (tp + fp) if (tp + fp) > 0 else 1.0
-                recall = tp / (tp + fn) if (tp + fn) > 0 else 1.0
-                accuracy = (tp + tn) / total if total > 0 else 0.0
-                print(f"{label}: {tp+tn}/{total} correct, precision={precision:.0%} recall={recall:.0%} accuracy={accuracy:.0%} ({elapsed:.1f}s)", file=sys.stderr)
-                for r in results:
-                    status = "PASS" if r["pass"] else "FAIL"
-                    rate_str = f"{r['triggers']}/{r['runs']}"
-                    print(f"  [{status}] rate={rate_str} expected={r['should_trigger']}: {r['query'][:60]}", file=sys.stderr)
-
-            print_eval_stats("Train", train_results["results"], eval_elapsed)
-            if test_summary:
-                print_eval_stats("Test ", test_results["results"], 0)
-
-        if train_summary["failed"] == 0:
-            exit_reason = f"all_passed (iteration {iteration})"
-            if verbose:
-                print(f"\nAll train queries passed on iteration {iteration}!", file=sys.stderr)
-            break
-
-        if iteration == max_iterations:
-            exit_reason = f"max_iterations ({max_iterations})"
-            if verbose:
-                print(f"\nMax iterations reached ({max_iterations}).", file=sys.stderr)
-            break
-
-        # Improve the description based on train results
-        if verbose:
-            print(f"\nImproving description...", file=sys.stderr)
-
-        t0 = time.time()
-        # Strip test scores from history so improvement model can't see them
-        blinded_history = [
-            {k: v for k, v in h.items() if not k.startswith("test_")}
-            for h in history
-        ]
-        new_description = improve_description(
-            skill_name=name,
-            skill_content=content,
-            current_description=current_description,
-            eval_results=train_results,
-            history=blinded_history,
-            model=model,
-            log_dir=log_dir,
-            iteration=iteration,
-        )
-        improve_elapsed = time.time() - t0
-
-        if verbose:
-            print(f"Proposed ({improve_elapsed:.1f}s): {new_description}", file=sys.stderr)
-
-        current_description = new_description
-
-    # Find the best iteration by TEST score (or train if no test set)
-    if test_set:
-        best = max(history, key=lambda h: h["test_passed"] or 0)
-        best_score = f"{best['test_passed']}/{best['test_total']}"
-    else:
-        best = max(history, key=lambda h: h["train_passed"])
-        best_score = f"{best['train_passed']}/{best['train_total']}"
-
-    if verbose:
-        print(f"\nExit reason: {exit_reason}", file=sys.stderr)
-        print(f"Best score: {best_score} (iteration {best['iteration']})", file=sys.stderr)
-
-    return {
-        "exit_reason": exit_reason,
-        "original_description": original_description,
-        "best_description": best["description"],
-        "best_score": best_score,
-        "best_train_score": f"{best['train_passed']}/{best['train_total']}",
-        "best_test_score": f"{best['test_passed']}/{best['test_total']}" if test_set else None,
-        "final_description": current_description,
-        "iterations_run": len(history),
-        "holdout": holdout,
-        "train_size": len(train_set),
-        "test_size": len(test_set),
-        "history": history,
-    }
-
-
-def main():
-    parser = argparse.ArgumentParser(description="Run eval + improve loop")
-    parser.add_argument("--eval-set", required=True, help="Path to eval set JSON file")
-    parser.add_argument("--skill-path", required=True, help="Path to skill directory")
-    parser.add_argument("--description", default=None, help="Override starting description")
-    parser.add_argument("--num-workers", type=int, default=10, help="Number of parallel workers")
-    parser.add_argument("--timeout", type=int, default=30, help="Timeout per query in seconds")
-    parser.add_argument("--max-iterations", type=int, default=5, help="Max improvement iterations")
-    parser.add_argument("--runs-per-query", type=int, default=3, help="Number of runs per query")
-    parser.add_argument("--trigger-threshold", type=float, default=0.5, help="Trigger rate threshold")
-    parser.add_argument("--holdout", type=float, default=0.4, help="Fraction of eval set to hold out for testing (0 to disable)")
-    parser.add_argument("--model", required=True, help="Model for improvement")
-    parser.add_argument("--verbose", action="store_true", help="Print progress to stderr")
-    parser.add_argument("--report", default="auto", help="Generate HTML report at this path (default: 'auto' for temp file, 'none' to disable)")
-    parser.add_argument("--results-dir", default=None, help="Save all outputs (results.json, report.html, log.txt) to a timestamped subdirectory here")
-    args = parser.parse_args()
-
-    eval_set = json.loads(Path(args.eval_set).read_text())
-    skill_path = Path(args.skill_path)
-
-    if not (skill_path / "SKILL.md").exists():
-        print(f"Error: No SKILL.md found at {skill_path}", file=sys.stderr)
-        sys.exit(1)
-
-    name, _, _ = parse_skill_md(skill_path)
-
-    # Set up live report path
-    if args.report != "none":
-        if args.report == "auto":
-            timestamp = time.strftime("%Y%m%d_%H%M%S")
-            live_report_path = Path(tempfile.gettempdir()) / f"skill_description_report_{skill_path.name}_{timestamp}.html"
-        else:
-            live_report_path = Path(args.report)
-        # Open the report immediately so the user can watch
-        live_report_path.write_text("<html><body><h1>Starting optimization loop...</h1><meta http-equiv='refresh' content='5'></body></html>")
-        webbrowser.open(str(live_report_path))
-    else:
-        live_report_path = None
-
-    # Determine output directory (create before run_loop so logs can be written)
-    if args.results_dir:
-        timestamp = time.strftime("%Y-%m-%d_%H%M%S")
-        results_dir = Path(args.results_dir) / timestamp
-        results_dir.mkdir(parents=True, exist_ok=True)
-    else:
-        results_dir = None
-
-    log_dir = results_dir / "logs" if results_dir else None
-
-    output = run_loop(
-        eval_set=eval_set,
-        skill_path=skill_path,
-        description_override=args.description,
-        num_workers=args.num_workers,
-        timeout=args.timeout,
-        max_iterations=args.max_iterations,
-        runs_per_query=args.runs_per_query,
-        trigger_threshold=args.trigger_threshold,
-        holdout=args.holdout,
-        model=args.model,
-        verbose=args.verbose,
-        live_report_path=live_report_path,
-        log_dir=log_dir,
-    )
-
-    # Save JSON output
-    json_output = json.dumps(output, indent=2)
-    print(json_output)
-    if results_dir:
-        (results_dir / "results.json").write_text(json_output)
-
-    # Write final HTML report (without auto-refresh)
-    if live_report_path:
-        live_report_path.write_text(generate_html(output, auto_refresh=False, skill_name=name))
-        print(f"\nReport: {live_report_path}", file=sys.stderr)
-
-    if results_dir and live_report_path:
-        (results_dir / "report.html").write_text(generate_html(output, auto_refresh=False, skill_name=name))
-
-    if results_dir:
-        print(f"Results saved to: {results_dir}", file=sys.stderr)
-
-
-if __name__ == "__main__":
-    main()

package/skills/tools/skill-creator/scripts/utils.py

@@ -1,47 +0,0 @@
-"""Shared utilities for skill-creator scripts."""
-
-from pathlib import Path
-
-
-
-def parse_skill_md(skill_path: Path) -> tuple[str, str, str]:
-    """Parse a SKILL.md file, returning (name, description, full_content)."""
-    content = (skill_path / "SKILL.md").read_text()
-    lines = content.split("\n")
-
-    if lines[0].strip() != "---":
-        raise ValueError("SKILL.md missing frontmatter (no opening ---)")
-
-    end_idx = None
-    for i, line in enumerate(lines[1:], start=1):
-        if line.strip() == "---":
-            end_idx = i
-            break
-
-    if end_idx is None:
-        raise ValueError("SKILL.md missing frontmatter (no closing ---)")
-
-    name = ""
-    description = ""
-    frontmatter_lines = lines[1:end_idx]
-    i = 0
-    while i < len(frontmatter_lines):
-        line = frontmatter_lines[i]
-        if line.startswith("name:"):
-            name = line[len("name:"):].strip().strip('"').strip("'")
-        elif line.startswith("description:"):
-            value = line[len("description:"):].strip()
-            # Handle YAML multiline indicators (>, |, >-, |-)
-            if value in (">", "|", ">-", "|-"):
-                continuation_lines: list[str] = []
-                i += 1
-                while i < len(frontmatter_lines) and (frontmatter_lines[i].startswith("  ") or frontmatter_lines[i].startswith("\t")):
-                    continuation_lines.append(frontmatter_lines[i].strip())
-                    i += 1
-                description = " ".join(continuation_lines)
-                continue
-            else:
-                description = value.strip('"').strip("'")
-        i += 1
-
-    return name, description, content

package/skills/tools/summarize/SKILL-zh.md

@@ -1,47 +0,0 @@
----
-name: summarize
-description: 对 URL、文件和 YouTube 视频进行摘要总结。
-license: MIT
----
-
-# Summarize Skill（摘要技能）
-
-对 URL、文件和 YouTube 视频的内容进行摘要总结。
-
-## 用法
-
-### 对 URL 进行摘要
-使用 `web_fetch` 工具获取并总结网页内容：
-```bash
-web_fetch url="https://example.com/article"
-```
-
-### 对文件进行摘要
-```bash
-read_file path="/path/to/file.md"
-```
-
-### 对 YouTube 视频进行摘要
-```bash
-web_fetch url="https://yewtu.be/watch?v=VIDEO_ID"
-```
-
-## 技巧
-
-### 处理长内容
-1. 分块处理（每块约 2000 tokens）
-2. 分别总结每个块
-3. 合并所有摘要
-
-### 处理技术文档
-1. 提取代码示例
-2. 记录前提条件
-3. 列出关键概念/术语
-
-## 示例提示
-
-**针对 URL：**
-"阅读此 URL 并总结：主要主题、关键要点（3-5 条要点）、任何可执行事项"
-
-**针对文件：**
-"分析此文件并提供：文件类型和用途、关键函数/类、依赖项"

package/skills/tools/weather/SKILL-zh.md

@@ -1,46 +0,0 @@
----
-name: weather
-description: 获取当前天气和预报（无需 API 密钥）。
-license: MIT
----
-
-# 天气
-
-两个免费服务，无需 API 密钥。
-
-## wttr.in（首选）
-
-快速一行命令：
-```bash
-curl -s "wttr.in/London?format=3"
-# 输出：London: ⛅️ +8°C
-```
-
-紧凑格式：
-```bash
-curl -s "wttr.in/London?format=%l:+%c+%t+%h+%w"
-# 输出：London: ⛅️ +8°C 71% ↙5km/h
-```
-
-完整预报：
-```bash
-curl -s "wttr.in/London?T"
-```
-
-格式代码：`%c` 天气状况 · `%t` 温度 · `%h` 湿度 · `%w` 风速 · `%l` 地点 · `%m` 月相
-
-提示：
-- URL 编码空格：`wttr.in/New+York`
-- 机场代码：`wttr.in/JFK`
-- 单位：`?m`（公制）`?u`（美制）
-- 仅今日：`?1` · 仅当前：`?0`
-- PNG 图片：`curl -s "wttr.in/Berlin.png" -o /tmp/weather.png`
-
-## Open-Meteo（备用，JSON）
-
-免费，无需密钥，适合程序化使用：
-```bash
-curl -s "https://api.open-meteo.com/v1/forecast?latitude=51.5&longitude=-0.12&current_weather=true"
-```
-
-查找城市坐标，然后查询。返回包含温度、风速、天气代码的 JSON。

package/skills/tools/weather/SKILL.md

@@ -1,46 +0,0 @@
----
-name: weather
-description: Get current weather and forecasts (no API key required).
-license: MIT
----
-
-# Weather
-
-Two free services, no API keys needed.
-
-## wttr.in (primary)
-
-Quick one-liner:
-```bash
-curl -s "wttr.in/London?format=3"
-# Output: London: ⛅️ +8°C
-```
-
-Compact format:
-```bash
-curl -s "wttr.in/London?format=%l:+%c+%t+%h+%w"
-# Output: London: ⛅️ +8°C 71% ↙5km/h
-```
-
-Full forecast:
-```bash
-curl -s "wttr.in/London?T"
-```
-
-Format codes: `%c` condition · `%t` temp · `%h` humidity · `%w` wind · `%l` location · `%m` moon
-
-Tips:
-- URL-encode spaces: `wttr.in/New+York`
-- Airport codes: `wttr.in/JFK`
-- Units: `?m` (metric) `?u` (USCS)
-- Today only: `?1` · Current only: `?0`
-- PNG: `curl -s "wttr.in/Berlin.png" -o /tmp/weather.png`
-
-## Open-Meteo (fallback, JSON)
-
-Free, no key, good for programmatic use:
-```bash
-curl -s "https://api.open-meteo.com/v1/forecast?latitude=51.5&longitude=-0.12&current_weather=true"
-```
-
-Find coordinates for a city, then query. Returns JSON with temp, windspeed, weathercode.

package/skills/tools/webapp-testing/SKILL-zh.md

@@ -1,96 +0,0 @@
----
-name: webapp-testing
-description: 使用 Playwright 与本地 Web 应用交互和测试的工具包。支持验证前端功能、调试 UI 行为、捕获浏览器截图和查看浏览器日志。
-license: 完整条款见 LICENSE.txt
----
-
-# Web 应用测试
-
-要测试本地 Web 应用，编写原生 Python Playwright 脚本。
-
-**可用辅助脚本：**
-- `scripts/with_server.py` - 管理服务器生命周期（支持多服务器）
-
-**始终先使用 `--help` 运行脚本** 查看用法。在尝试运行脚本并发现确实需要定制化解决方案之前，不要读取源码。这些脚本可能非常庞大，会污染你的上下文窗口。它们应作为黑盒脚本直接调用，而非被吸收到你的上下文窗口中。
-
-## 决策树：选择你的方法
-
-```
-用户任务 → 是否为静态 HTML？
-    ├─ 是 → 直接读取 HTML 文件识别选择器
-    │         ├─ 成功 → 使用选择器编写 Playwright 脚本
-    │         └─ 失败/不完整 → 按动态处理方式（见下方）
-    │
-    └─ 否（动态 Web 应用）→ 服务器是否已在运行？
-        ├─ 否 → 运行：python scripts/with_server.py --help
-        │        然后使用辅助工具 + 编写简化的 Playwright 脚本
-        │
-        └─ 是 → 侦察-然后-行动模式：
-            1. 导航并等待 networkidle
-            2. 截取屏幕截图或检查 DOM
-            3. 根据渲染状态识别选择器
-            4. 使用发现的选择器执行操作
-```
-
-## 示例：使用 with_server.py
-
-要启动服务器，先运行 `--help`，然后使用辅助工具：
-
-**单服务器：**
-```bash
-python scripts/with_server.py --server "npm run dev" --port 5173 -- python your_automation.py
-```
-
-**多服务器（如后端 + 前端）：**
-```bash
-python scripts/with_server.py \
-  --server "cd backend && python server.py" --port 3000 \
-  --server "cd frontend && npm run dev" --port 5173 \
-  -- python your_automation.py
-```
-
-创建自动化脚本时，只包含 Playwright 逻辑（服务器由辅助工具自动管理）：
-```python
-from playwright.sync_api import sync_playwright
-
-with sync_playwright() as p:
-    browser = p.chromium.launch(headless=True) # 始终以 headless 模式启动 chromium
-    page = browser.new_page()
-    page.goto('http://localhost:5173') # 服务器已在运行并准备就绪
-    page.wait_for_load_state('networkidle') # 关键：等待 JS 执行完毕
-    # ... 你的自动化逻辑
-    browser.close()
-```
-
-## 侦察-然后-行动模式
-
-1. **检查渲染后的 DOM**：
-   ```python
-   page.screenshot(path='/tmp/inspect.png', full_page=True)
-   content = page.content()
-   page.locator('button').all()
-   ```
-
-2. **根据检查结果识别选择器**
-
-3. **使用发现的选择器执行操作**
-
-## 常见陷阱
-
-❌ **不要**在动态应用上等待 `networkidle` 之前检查 DOM
-✅ **要**在检查之前等待 `page.wait_for_load_state('networkidle')`
-
-## 最佳实践
-
-- **将内置脚本作为黑盒使用** - 完成任务时，考虑 `scripts/` 中的某个脚本是否可以帮助。这些脚本能可靠地处理常见、复杂的工作流，而不会占用上下文窗口。使用 `--help` 查看用法，然后直接调用。
-- 对同步脚本使用 `sync_playwright()`
-- 完成后始终关闭浏览器
-- 使用描述性选择器：`text=`、`role=`、CSS 选择器或 ID
-- 添加适当的等待：`page.wait_for_selector()` 或 `page.wait_for_timeout()`
-
-## 参考文件
-
-- **examples/** - 展示常见模式的示例：
-  - `element_discovery.py` - 发现页面上的按钮、链接和输入框
-  - `static_html_automation.py` - 使用 file:// URL 处理本地 HTML
-  - `console_logging.py` - 在自动化过程中捕获控制台日志