@remixmate/cli 0.1.0 → 0.1.2

package/skills/web-capture/scripts/web_capture.py

@@ -1,461 +0,0 @@
-#!/usr/bin/env python3
-"""
-网页捕获脚本 — 驱动无头浏览器（Playwright）对网页进行截图或滚动录屏。
-
-三种模式：
-  - screenshot    截视口 / 整页 / 指定元素 → PNG
-  - scroll-video  从顶到底平滑滚动，逐帧截图后用 ffmpeg 合成确定性 MP4
-  - record-video  Playwright 原生 context 录制（webm），再 ffmpeg 转 MP4
-
-产物（PNG / MP4）用于喂给 screen-walkthrough 等模板。
-
-依赖（运行时提供，skill 不打包浏览器）：
-  - Python3 + Playwright： pip install playwright && playwright install chromium
-  - ffmpeg（视频模式）：   brew install ffmpeg
-
-用法:
-  python web_capture.py --url "https://example.com"
-  python web_capture.py --url "https://example.com" --mode screenshot --full-page
-  python web_capture.py --url "https://github.com/x/y" --mode scroll-video --fps 30 --scroll-duration 8
-  python web_capture.py --url "https://example.com" --mode record-video
-  python web_capture.py --url "https://example.com" --json-output
-
-环境变量:
-  WEB_CAPTURE_BROWSER        - 默认浏览器（chromium/firefox/webkit），--browser 优先
-  PLAYWRIGHT_BROWSERS_PATH   - Playwright 浏览器二进制目录（可选，由运行环境提供）
-"""
-
-from __future__ import annotations
-
-import argparse
-import ipaddress
-import json
-import math
-import os
-import shutil
-import socket
-import subprocess
-import sys
-import tempfile
-from pathlib import Path
-from urllib.parse import urlsplit
-
-SKILL_NAME = "web-capture"
-
-# 多用户并发下，每个进程独立浏览器实例 + 独立输出目录即可隔离。
-# 默认输出落系统临时目录，避免 cwd（skills/web-capture/scripts）被多用户产物污染。
-OUTPUT_PREFIX = "web_capture_"
-
-# 是否静默模式（--json-output 时抑制人类可读进度输出，仍发结构化 __progress__）
-QUIET = False
-
-
-def log(msg: str) -> None:
-    """打印进度信息到 stderr（静默模式下不输出）。stdout 仅留给 --json-output 的最终 JSON。"""
-    if not QUIET:
-        print(msg, file=sys.stderr, flush=True)
-
-
-def log_err(msg: str) -> None:
-    """错误信息始终输出到 stderr。"""
-    print(msg, file=sys.stderr, flush=True)
-
-
-def emit_progress(phase: str, progress: float | None = None, **extra) -> None:
-    """输出一行 __progress__ JSON 到 stdout，供 ab-agent executor 解析并转发 SSE。
-
-    与 src/progress.ts 约定一致：额外字段直接与 phase 平级。
-    注意：--json-output 模式下 stdout 需保持纯净（只输出最终 manifest），
-    因此静默模式下不发结构化进度，避免污染管道。
-    """
-    if QUIET:
-        return
-    payload = {"__progress__": True, "phase": phase}
-    if progress is not None:
-        payload["progress"] = round(progress, 4)
-    payload.update(extra)
-    print(json.dumps(payload, ensure_ascii=False), flush=True)
-
-
-class CaptureError(Exception):
-    """受控的失败：以非 0 退出并打印简洁错误。"""
-
-    def __init__(self, message: str, exit_code: int = 1):
-        super().__init__(message)
-        self.exit_code = exit_code
-
-
-# ─────────────────────────── 预检 ───────────────────────────
-
-def preflight(mode: str):
-    """检查 Playwright 与（视频模式）ffmpeg 是否就绪，返回 sync_playwright 工厂。"""
-    emit_progress("web-capture:preflight")
-    try:
-        from playwright.sync_api import sync_playwright  # noqa: F401
-    except ImportError as exc:
-        raise CaptureError(
-            "Playwright 未安装。请运行：\n"
-            "  pip install playwright && playwright install chromium",
-            exit_code=2,
-        ) from exc
-
-    if mode in ("scroll-video", "record-video") and shutil.which("ffmpeg") is None:
-        raise CaptureError(
-            "未检测到 ffmpeg（视频模式必需）。请运行：\n  brew install ffmpeg",
-            exit_code=3,
-        )
-
-    from playwright.sync_api import sync_playwright
-
-    return sync_playwright
-
-
-def ensure_output_dir(output_dir: str | None) -> Path:
-    """显式指定则用之；否则在系统临时目录下创建唯一目录（多用户并发隔离）。
-
-    旧实现用 ./web_capture_<秒级时间戳>，同一秒内的并发调用会互相覆盖，且产物
-    堆在脚本所在目录（污染 skills/ 并触发 AGENTS.md 的 output/ 守卫）。改用
-    mkdtemp 保证目录唯一，且默认落系统 tempdir。对齐 render_video 的 tempdir 范式。
-    """
-    if output_dir:
-        path = Path(output_dir).expanduser().resolve()
-        path.mkdir(parents=True, exist_ok=True)
-        return path
-    base = os.environ.get("WEB_CAPTURE_OUTPUT_BASE") or None
-    return Path(tempfile.mkdtemp(prefix=OUTPUT_PREFIX, dir=base)).resolve()
-
-
-# ─────────────────────────── URL 校验（防 SSRF）───────────────────────────
-
-def validate_url(url: str) -> None:
-    """多用户场景下，url 来自用户输入，必须挡住 SSRF：
-    - 仅允许 http/https
-    - 禁止解析到内网 / 环回 / 链路本地 / 云元数据（169.254.169.254）等地址
-
-    可用 WEB_CAPTURE_ALLOW_PRIVATE_HOSTS=1 关闭内网拦截（单租户/内网自测场景）。
-    """
-    parts = urlsplit(url)
-    if parts.scheme not in ("http", "https"):
-        raise CaptureError(f"仅支持 http/https URL，收到：{parts.scheme or '(空)'}://", exit_code=5)
-    host = parts.hostname
-    if not host:
-        raise CaptureError("URL 缺少主机名", exit_code=5)
-
-    if os.environ.get("WEB_CAPTURE_ALLOW_PRIVATE_HOSTS") == "1":
-        return
-
-    # 解析所有 A/AAAA 记录，任一落在受限网段即拒绝（防 DNS 把公网名指到内网）。
-    try:
-        infos = socket.getaddrinfo(host, parts.port or (443 if parts.scheme == "https" else 80),
-                                   proto=socket.IPPROTO_TCP)
-    except socket.gaierror as exc:
-        raise CaptureError(f"无法解析主机：{host}（{exc}）", exit_code=5) from exc
-
-    for info in infos:
-        ip_str = info[4][0]
-        try:
-            ip = ipaddress.ip_address(ip_str)
-        except ValueError:
-            continue
-        if (ip.is_private or ip.is_loopback or ip.is_link_local
-                or ip.is_reserved or ip.is_multicast or ip.is_unspecified):
-            raise CaptureError(
-                f"目标地址 {host} 解析到受限网段 {ip_str}（疑似内网/元数据），已拒绝。"
-                f"\n如确为可信内网目标，可设 WEB_CAPTURE_ALLOW_PRIVATE_HOSTS=1。",
-                exit_code=5,
-            )
-
-
-def launch_browser(playwright, browser_name: str):
-    """启动指定浏览器；缺二进制时给出可执行的安装提示。
-
-    默认保留浏览器沙箱（多用户 + 访问任意 URL 时，关沙箱会让浏览器漏洞更易逃逸到主机）。
-    仅当显式设置 WEB_CAPTURE_NO_SANDBOX=1（如某些无法配置 user namespace 的容器）才关闭。
-    """
-    engine = getattr(playwright, browser_name, None)
-    if engine is None:
-        raise CaptureError(f"未知浏览器引擎：{browser_name}（可选 chromium/firefox/webkit）")
-    launch_args: list[str] = []
-    if browser_name == "chromium":
-        launch_args.append("--disable-dev-shm-usage")
-        if os.environ.get("WEB_CAPTURE_NO_SANDBOX") == "1":
-            launch_args.append("--no-sandbox")
-    try:
-        return engine.launch(headless=True, args=launch_args)
-    except Exception as exc:  # noqa: BLE001
-        msg = str(exc)
-        if "Executable doesn't exist" in msg or "playwright install" in msg:
-            raise CaptureError(
-                f"{browser_name} 浏览器二进制缺失。请运行：\n  playwright install {browser_name}",
-                exit_code=4,
-            ) from exc
-        raise CaptureError(f"启动浏览器失败：{msg}") from exc
-
-
-# ─────────────────────────── 页面准备 ───────────────────────────
-
-def goto_and_settle(page, url: str, wait_until: str, wait_ms: int) -> str:
-    """导航到 url，等待加载与额外静置，返回 finalUrl。"""
-    emit_progress("web-capture:navigate", url=url)
-    try:
-        page.goto(url, wait_until=wait_until, timeout=60_000)
-    except Exception as exc:  # noqa: BLE001
-        # networkidle 在某些长轮询页面永远不触发，降级到 load 再试一次
-        if wait_until == "networkidle":
-            log("⚠️  networkidle 超时，降级到 load 重试")
-            page.goto(url, wait_until="load", timeout=60_000)
-        else:
-            raise CaptureError(f"页面导航失败：{exc}") from exc
-    if wait_ms > 0:
-        page.wait_for_timeout(wait_ms)
-    return page.url
-
-
-def get_page_height(page) -> int:
-    return int(page.evaluate(
-        "() => Math.max(document.body.scrollHeight, document.documentElement.scrollHeight,"
-        " document.body.offsetHeight, document.documentElement.offsetHeight)"
-    ))
-
-
-# ─────────────────────────── 模式实现 ───────────────────────────
-
-def do_screenshot(page, out_dir: Path, full_page: bool, selector: str | None) -> dict:
-    emit_progress("web-capture:screenshot", full_page=full_page)
-    target = out_dir / "screenshot.png"
-    if selector:
-        el = page.query_selector(selector)
-        if el is None:
-            raise CaptureError(f"未找到选择器对应元素：{selector}")
-        el.screenshot(path=str(target))
-    else:
-        page.screenshot(path=str(target), full_page=full_page)
-    return {
-        "type": "image",
-        "localPath": str(target),
-        "fullPage": bool(full_page and not selector),
-        "selector": selector,
-    }
-
-
-def do_scroll_video(page, out_dir: Path, viewport: dict, fps: int,
-                    scroll_duration: float) -> dict:
-    """逐帧截图 + ffmpeg 合成确定性 MP4。fps / 分辨率 / 滚动速度全可控。"""
-    frames_dir = out_dir / "_frames"
-    frames_dir.mkdir(parents=True, exist_ok=True)
-    try:
-        total_frames = max(1, int(round(fps * scroll_duration)))
-        vh = viewport["height"]
-        page_height = get_page_height(page)
-        max_scroll = max(0, page_height - vh)
-
-        emit_progress("web-capture:scroll-capture", progress=0.0,
-                      total=total_frames, pageHeight=page_height)
-
-        for i in range(total_frames):
-            # ease-in-out 让滚动更自然
-            t = i / (total_frames - 1) if total_frames > 1 else 1.0
-            eased = 0.5 - 0.5 * math.cos(math.pi * t)
-            y = int(round(max_scroll * eased))
-            page.evaluate(f"window.scrollTo(0, {y})")
-            page.wait_for_timeout(int(1000 / fps))
-            frame_path = frames_dir / f"frame_{i:05d}.png"
-            page.screenshot(path=str(frame_path))
-            if i % max(1, total_frames // 10) == 0:
-                emit_progress("web-capture:scroll-capture",
-                              progress=(i + 1) / total_frames, current=i + 1, total=total_frames)
-
-        emit_progress("web-capture:encode", progress=0.0)
-        target = out_dir / "scroll.mp4"
-        _ffmpeg_from_frames(frames_dir, target, fps)
-    finally:
-        # 无论成功 / ffmpeg 失败 / 异常，都清掉逐帧临时图，避免临时目录堆积。
-        shutil.rmtree(frames_dir, ignore_errors=True)
-
-    return {
-        "type": "video",
-        "localPath": str(target),
-        "fps": fps,
-        "durationSec": round(total_frames / fps, 3),
-        "frames": total_frames,
-    }
-
-
-def _ffmpeg_from_frames(frames_dir: Path, target: Path, fps: int) -> None:
-    cmd = [
-        "ffmpeg", "-y",
-        "-framerate", str(fps),
-        "-i", str(frames_dir / "frame_%05d.png"),
-        # 偶数化宽高，yuv420p 保证主流播放器/Remotion 兼容
-        "-vf", "scale=trunc(iw/2)*2:trunc(ih/2)*2,format=yuv420p",
-        "-c:v", "libx264",
-        "-preset", "medium",
-        "-crf", "18",
-        "-movflags", "+faststart",
-        str(target),
-    ]
-    proc = subprocess.run(cmd, capture_output=True, text=True)
-    if proc.returncode != 0:
-        raise CaptureError(f"ffmpeg 合成失败：\n{proc.stderr[-800:]}")
-
-
-def do_record_video(page, context, out_dir: Path, scroll_duration: float) -> dict:
-    """Playwright 原生录制：滚动一遍后关闭 page，取 webm 再转 mp4。"""
-    emit_progress("web-capture:record", progress=0.0)
-    page_height = get_page_height(page)
-    vh = int(page.evaluate("() => window.innerHeight"))
-    max_scroll = max(0, page_height - vh)
-
-    steps = max(1, int(scroll_duration * 10))  # 每 100ms 一步
-    for i in range(steps):
-        t = (i + 1) / steps
-        eased = 0.5 - 0.5 * math.cos(math.pi * t)
-        page.evaluate(f"window.scrollTo(0, {int(max_scroll * eased)})")
-        page.wait_for_timeout(100)
-
-    # 关闭 page/context 后 Playwright 才落盘 webm
-    video = page.video
-    page.close()
-    context.close()
-    if video is None:
-        raise CaptureError("Playwright 未产出录制文件（context 未开启 record_video_dir？）")
-    webm_path = Path(video.path())
-
-    emit_progress("web-capture:encode", progress=0.0)
-    target = out_dir / "scroll.mp4"
-    _ffmpeg_transcode(webm_path, target)
-    try:
-        webm_path.unlink(missing_ok=True)
-    except OSError:
-        pass
-
-    return {
-        "type": "video",
-        "localPath": str(target),
-        "source": "playwright-native",
-        "durationSec": round(scroll_duration, 3),
-    }
-
-
-def _ffmpeg_transcode(src: Path, target: Path) -> None:
-    cmd = [
-        "ffmpeg", "-y",
-        "-i", str(src),
-        "-vf", "scale=trunc(iw/2)*2:trunc(ih/2)*2,format=yuv420p",
-        "-c:v", "libx264", "-preset", "medium", "-crf", "20",
-        "-movflags", "+faststart",
-        str(target),
-    ]
-    proc = subprocess.run(cmd, capture_output=True, text=True)
-    if proc.returncode != 0:
-        raise CaptureError(f"ffmpeg 转码失败：\n{proc.stderr[-800:]}")
-
-
-# ─────────────────────────── 主流程 ───────────────────────────
-
-def run(args) -> dict:
-    validate_url(args.url)
-    sync_playwright = preflight(args.mode)
-    out_dir = ensure_output_dir(args.output_dir)
-    viewport = {"width": args.width, "height": args.height,
-                "deviceScaleFactor": args.device_scale_factor}
-    rec_dir = out_dir / "_rec"
-
-    with sync_playwright() as pw:
-        browser = launch_browser(pw, args.browser)
-        try:
-            context_kwargs = {
-                "viewport": {"width": args.width, "height": args.height},
-                "device_scale_factor": args.device_scale_factor,
-            }
-            if args.mode == "record-video":
-                context_kwargs["record_video_dir"] = str(rec_dir)
-                context_kwargs["record_video_size"] = {"width": args.width, "height": args.height}
-
-            context = browser.new_context(**context_kwargs)
-            page = context.new_page()
-            final_url = goto_and_settle(page, args.url, args.wait_until, args.wait_ms)
-
-            if args.mode == "screenshot":
-                output = do_screenshot(page, out_dir, args.full_page, args.selector)
-                context.close()
-            elif args.mode == "scroll-video":
-                output = do_scroll_video(page, out_dir, viewport, args.fps, args.scroll_duration)
-                context.close()
-            elif args.mode == "record-video":
-                # do_record_video 内部会 page.close()/context.close() 以触发 webm 落盘
-                output = do_record_video(page, context, out_dir, args.scroll_duration)
-            else:
-                raise CaptureError(f"未知 mode：{args.mode}")
-        finally:
-            browser.close()
-            # 原生录制的中间 webm 目录无论成败都清理。
-            shutil.rmtree(rec_dir, ignore_errors=True)
-
-    manifest = {
-        "source": {"url": args.url, "finalUrl": final_url},
-        "mode": args.mode,
-        "viewport": viewport,
-        "outputs": [output],
-    }
-    manifest_path = out_dir / "capture.json"
-    manifest_path.write_text(json.dumps(manifest, ensure_ascii=False, indent=2), encoding="utf-8")
-    emit_progress("web-capture:done", progress=1.0, manifest=str(manifest_path))
-    return manifest
-
-
-def build_parser() -> argparse.ArgumentParser:
-    p = argparse.ArgumentParser(
-        description="网页捕获：截图 / 滚动录屏（Playwright + ffmpeg）",
-        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
-    )
-    p.add_argument("--url", required=True, help="目标页面 URL（http/https）")
-    p.add_argument("--mode", default="screenshot",
-                   choices=["screenshot", "scroll-video", "record-video"],
-                   help="捕获模式")
-    p.add_argument("-o", "--output-dir", default=None,
-                   help="输出目录（默认在系统临时目录创建唯一目录，多用户隔离）")
-    p.add_argument("--width", type=int, default=1280, help="视口宽度 px")
-    p.add_argument("--height", type=int, default=720, help="视口高度 px")
-    p.add_argument("--device-scale-factor", type=float, default=2.0, help="设备像素比（清晰/视网膜）")
-    p.add_argument("--full-page", action="store_true", help="screenshot：截取整个可滚动页面")
-    p.add_argument("--selector", default=None, help="screenshot：仅截取该 CSS 元素")
-    p.add_argument("--fps", type=int, default=30, help="scroll/record 视频帧率")
-    p.add_argument("--scroll-duration", type=float, default=8.0, help="从顶到底滚动总秒数")
-    p.add_argument("--browser", default=os.environ.get("WEB_CAPTURE_BROWSER", "chromium"),
-                   choices=["chromium", "firefox", "webkit"], help="浏览器引擎")
-    p.add_argument("--wait-until", default="networkidle",
-                   choices=["load", "domcontentloaded", "networkidle"], help="导航等待条件")
-    p.add_argument("--wait-ms", type=int, default=1500, help="加载后额外静置毫秒")
-    p.add_argument("--json-output", action="store_true",
-                   help="管道模式：stdout 仅输出最终 manifest JSON")
-    return p
-
-
-def main() -> None:
-    global QUIET
-    args = build_parser().parse_args()
-    QUIET = args.json_output
-
-    try:
-        manifest = run(args)
-    except CaptureError as exc:
-        log_err(f"❌ {exc}")
-        sys.exit(exc.exit_code)
-    except KeyboardInterrupt:
-        log_err("⚠️  已取消")
-        sys.exit(130)
-    except Exception as exc:  # noqa: BLE001
-        log_err(f"❌ 捕获失败：{exc}")
-        sys.exit(1)
-
-    if args.json_output:
-        # 管道模式：stdout 仅输出 manifest
-        print(json.dumps(manifest, ensure_ascii=False))
-    else:
-        out = manifest["outputs"][0]
-        log(f"✅ 完成：{out.get('localPath')}")
-
-
-if __name__ == "__main__":
-    main()