@remixmate/cli 0.9.25 → 0.9.27

package/skills/web-screenshot/scripts/_media_screenshot/urlguard.py

@@ -0,0 +1,90 @@
+"""Refuse to fetch private / link-local addresses (SSRF guard).
+
+Only ``read_page.py`` uses this today, and that asymmetry is deliberate:
+screenshot and record hand back a *file path*, while web_read hands the page's
+**text** straight to a model that is driven by user input. A prompt that says
+"read http://169.254.169.254/latest/meta-data/iam/..." is an exfiltration
+attempt, not a reading request.
+
+``WEB_CAPTURE_ALLOW_PRIVATE_HOSTS=1`` opts out — needed when pointing the skill
+at a dev server or an intranet page on purpose.
+
+Caveat: the check resolves DNS once and then hands the URL to the browser,
+which resolves it again. A name that flips between answers (DNS rebinding) can
+slip through that gap. Closing it properly needs request-level interception in
+Playwright; this guard is aimed at the ordinary case of a hostile or careless
+URL, not at an attacker who controls a nameserver.
+"""
+from __future__ import annotations
+
+import ipaddress
+import os
+import socket
+from urllib.parse import urlparse
+
+_TRUTHY = {"1", "true", "yes", "on"}
+
+
+def allow_private_hosts() -> bool:
+    return os.environ.get("WEB_CAPTURE_ALLOW_PRIVATE_HOSTS", "").strip().lower() in _TRUTHY
+
+
+def _is_blocked(ip: ipaddress._BaseAddress) -> bool:
+    return bool(
+        ip.is_private
+        or ip.is_loopback
+        or ip.is_link_local
+        or ip.is_reserved
+        or ip.is_multicast
+        or ip.is_unspecified
+    )
+
+
+def assert_public_url(raw_url: str) -> None:
+    """Raise SystemExit when *raw_url* is not an ordinary public http(s) page."""
+    parsed = urlparse(raw_url)
+    if parsed.scheme not in ("http", "https"):
+        raise SystemExit(
+            f"--url 只接受 http/https，收到 {parsed.scheme or '(空)'}：{raw_url}"
+        )
+    host = parsed.hostname
+    if not host:
+        raise SystemExit(f"--url 缺少主机名：{raw_url}")
+
+    if allow_private_hosts():
+        return
+
+    # A literal IP needs no DNS round-trip.
+    try:
+        literal = ipaddress.ip_address(host)
+    except ValueError:
+        literal = None
+    if literal is not None:
+        if _is_blocked(literal):
+            raise SystemExit(_refusal(host, str(literal)))
+        return
+
+    try:
+        infos = socket.getaddrinfo(host, parsed.port or (443 if parsed.scheme == "https" else 80))
+    except socket.gaierror as e:
+        raise SystemExit(f"无法解析主机 {host}：{e}")
+
+    for info in infos:
+        addr = info[4][0]
+        try:
+            ip = ipaddress.ip_address(addr)
+        except ValueError:
+            continue
+        # Any resolved address being internal is enough to refuse — a name that
+        # answers with one public and one private A record is the exact shape of
+        # an SSRF attempt.
+        if _is_blocked(ip):
+            raise SystemExit(_refusal(host, addr))
+
+
+def _refusal(host: str, addr: str) -> str:
+    return (
+        f"拒绝读取内网地址：{host} 解析到 {addr}（私有 / 环回 / 链路本地）。"
+        "web_read 会把页面正文交给模型，因此默认只读公网页面。"
+        "确实要读内网页面时设 WEB_CAPTURE_ALLOW_PRIVATE_HOSTS=1。"
+    )

package/skills/web-screenshot/scripts/read_page.py

@@ -0,0 +1,136 @@
+#!/usr/bin/env python3
+"""Playwright 正文抽取入口（薄壳，逻辑在 _media_screenshot.reader）。
+
+与 screenshot.py / record.py 同住一个 scripts/ 目录，因为三者共用
+`_media_screenshot/` 里的浏览器启动、等待、设备模拟与 cookie 处理。
+对外它是独立的 skill：`skills/web-read/skill.json` 指到这里。
+"""
+from __future__ import annotations
+
+import argparse
+import os
+import sys
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parent))
+from _media_screenshot import cli_args  # noqa: E402
+from _media_screenshot.reader import do_read  # noqa: E402
+from _media_screenshot.urlguard import assert_public_url  # noqa: E402
+
+DEFAULT_MAX_CHARS = 20000
+
+
+def main() -> None:
+    ap = argparse.ArgumentParser(
+        description="通过 Playwright 打开网页并抽取正文文本（Markdown / 纯文本 / JSON）。"
+    )
+    ap.add_argument("-u", "--url", required=True, help="待读取的页面 URL（http/https）")
+    ap.add_argument(
+        "--selector",
+        help="只抽取该 CSS 选择器内的内容；不给则自动识别正文容器",
+    )
+    ap.add_argument(
+        "--format", choices=["markdown", "text", "json"], default="markdown",
+        help="输出格式：markdown（默认，保留标题/列表/代码块）| text（纯文本）| json（结构化块）",
+    )
+    ap.add_argument(
+        "--max-chars", type=int, default=DEFAULT_MAX_CHARS,
+        help=f"stdout 字符上限，按块截断（默认 {DEFAULT_MAX_CHARS}，0 = 不限）",
+    )
+    ap.add_argument(
+        "--include-links", action="store_true",
+        help="正文里的链接保留为 [文字](URL)（默认只留文字）",
+    )
+    ap.add_argument(
+        "--include-images", action="store_true",
+        help="保留图片为 ![alt](src)（默认丢弃）",
+    )
+    ap.add_argument(
+        "-o", "--output",
+        help="把**完整**正文（不截断）另存到该文件；stdout 仍受 --max-chars 限制",
+    )
+    ap.add_argument(
+        "--settle-ms", type=int,
+        help="抽取前额外静置毫秒数（给动画/懒加载留时间）",
+    )
+    ap.add_argument(
+        "--quiet", action="store_true",
+        help="只打印正文，不打印 stderr 上的抽取诊断",
+    )
+    cli_args.add_common_args(ap)
+
+    args = ap.parse_args()
+
+    # 内网地址守卫（见 urlguard.py：web_read 会把正文交给模型）。
+    assert_public_url(args.url)
+
+    storage_path, is_temp = cli_args.build_storage(args)
+
+    cfg: dict = {
+        "url": args.url,
+        "browser": args.browser or "chromium",
+        "format": args.format,
+        "maxChars": args.max_chars,
+        "includeLinks": bool(args.include_links),
+        "includeImages": bool(args.include_images),
+    }
+    cli_args.common_args_to_cfg(args, cfg)
+    if storage_path:
+        cfg["storageState"] = storage_path
+    if args.selector:
+        cfg["selector"] = args.selector
+    if args.output:
+        cfg["output"] = args.output
+    if args.settle_ms is not None:
+        cfg["settleMs"] = args.settle_ms
+
+    try:
+        result = do_read(cfg)
+    finally:
+        if is_temp and storage_path:
+            try:
+                os.unlink(storage_path)
+            except OSError:
+                pass
+
+    # 只在「基本没抽到东西」时报警。一篇文章页的整页文本本来就是正文的好几倍
+    # （导航、推荐、页脚），拿这个比例当告警条件会让警告天天响、从而没人再看。
+    chars = result["charCount"]
+    body_chars = result["bodyCharCount"]
+    status = result.get("status")
+    if isinstance(status, int) and status >= 400:
+        # 错误页也有正文（"404 Not Found"），照读不误——但不说一声的话，调用方
+        # 会把一张错误页当成文章内容去写脚本。
+        print(
+            f"⚠️  服务器返回 HTTP {status}，下面读到的很可能是错误页而不是目标内容。",
+            file=sys.stderr,
+        )
+    if chars < 100 and body_chars >= 2000:
+        print(
+            f"⚠️  整页有 {body_chars} 字符，却只抽到 {chars} 字符"
+            f"（容器 {result.get('container')}）——正文容器多半没认对。"
+            "可用 --selector <正文容器> 指定，或 --format json 看抽到了哪些块。",
+            file=sys.stderr,
+        )
+    elif chars < 100:
+        print(
+            f"⚠️  这个页面几乎没有文本（整页 {body_chars} 字符，HTTP {result.get('status')}）。"
+            "多半是登录墙、反爬拦截，或正文由 JS 延迟渲染还没出来。"
+            "可尝试：--wait-for-selector <正文选择器> / --settle-ms 3000 / --cookies <登录态>。",
+            file=sys.stderr,
+        )
+    elif not args.quiet:
+        print(
+            f"[web_read] {result['blockCount']} blocks · {result['charCount']} chars · "
+            f"container={result.get('container')} ({result.get('containerReason')})",
+            file=sys.stderr,
+        )
+
+    if result["outputPath"]:
+        print(f"[web_read] 全文已写入 {result['outputPath']}", file=sys.stderr)
+
+    print(result["text"])
+
+
+if __name__ == "__main__":
+    main()