@remixmate/cli 0.9.26 → 0.9.27

package/skills/web-screenshot/scripts/_media_screenshot/reader.py

@@ -0,0 +1,245 @@
+"""Read operation: load a page and extract its main content as text.
+
+The DOM work lives in ``js/extract_article.js`` (one injected pass returning
+typed blocks); this module drives the browser, renders those blocks into the
+requested format, and enforces the character budget.
+
+Truncation is the part that matters for the caller: a 200k-character page
+pasted into a model's context is worse than useless. ``max_chars`` cuts on a
+block boundary and says so in-band, and ``--output`` keeps the full text on
+disk so nothing is actually lost.
+"""
+from __future__ import annotations
+
+import json
+from pathlib import Path
+
+from . import js_loader
+from .bootstrap import ensure_runtime
+from .browser import apply_pre_action_waits, build_context_options, launch_with_browser_install
+
+
+def extract(cfg: dict) -> dict:
+    """Load cfg['url'] and return the raw extraction dict from the page."""
+    ensure_runtime()
+    from playwright.sync_api import sync_playwright
+
+    js_cfg = {
+        "selector": cfg.get("selector") or "",
+        "includeLinks": bool(cfg.get("includeLinks")),
+        "includeImages": bool(cfg.get("includeImages")),
+        "inline": "markdown" if cfg.get("format", "markdown") == "markdown" else "plain",
+    }
+
+    with sync_playwright() as p:
+        browser = launch_with_browser_install(p, cfg.get("browser"))
+        try:
+            context = browser.new_context(**build_context_options(p, cfg, None))
+            if cfg.get("timeout"):
+                context.set_default_timeout(cfg["timeout"])
+            page = context.new_page()
+            try:
+                try:
+                    response = page.goto(cfg["url"], wait_until="domcontentloaded")
+                except Exception as e:
+                    # 一条模型能据以行动的错误，胜过 30 行 Playwright traceback。
+                    # 导航失败是这个 skill 最常见的失败形态（打不开 / 超时 / DNS），
+                    # 让它读起来像"这个网址打不开"，而不是像程序崩了。
+                    raise SystemExit(
+                        f"打开页面失败：{cfg['url']}\n"
+                        f"{type(e).__name__}: {str(e).splitlines()[0]}\n"
+                        "可能是网络不可达、站点屏蔽了无头浏览器，或加载超过了超时时间。"
+                        "可尝试加大 --timeout、换 --user-agent，或确认该 URL 在本机能打开。"
+                    )
+                status = response.status if response else None
+                if status is not None and status >= 400:
+                    # Keep going: many sites serve real content under a 403/404
+                    # (paywalls, soft 404s). The status rides along in the output
+                    # so the caller can tell "empty page" from "blocked".
+                    pass
+                apply_pre_action_waits(page, cfg)
+                # SPA 正文往往在 domcontentloaded 之后才注水。networkidle 等不到就
+                # 算了——静态页本来就不会再有请求，硬等只是白白花掉 20 秒。
+                try:
+                    page.wait_for_load_state("networkidle", timeout=cfg.get("networkIdleMs", 8000))
+                except Exception:
+                    pass
+                settle = cfg.get("settleMs")
+                if settle:
+                    page.wait_for_timeout(settle)
+
+                data = page.evaluate(js_loader.load("extract_article"), js_cfg)
+                data["status"] = status
+                data["finalUrl"] = page.url
+                return data
+            finally:
+                context.close()
+        finally:
+            browser.close()
+
+
+# ── rendering ────────────────────────────────────────────────────────────────
+
+def _render_blocks(blocks: list, fmt: str) -> list[str]:
+    """One string per block, ready to be joined with a blank line."""
+    md = fmt == "markdown"
+    out: list[str] = []
+    for block in blocks:
+        kind = block.get("type")
+        if kind == "heading":
+            level = int(block.get("level") or 2)
+            out.append(f"{'#' * min(level, 6)} {block['text']}" if md else block["text"].upper())
+        elif kind == "paragraph":
+            out.append(block["text"])
+        elif kind == "quote":
+            out.append(f"> {block['text']}" if md else f'"{block["text"]}"')
+        elif kind == "code":
+            body = block.get("text", "")
+            out.append(f"```{block.get('lang') or ''}\n{body}\n```" if md else body)
+        elif kind == "list":
+            ordered = bool(block.get("ordered"))
+            lines = []
+            counter = 1
+            for item in block.get("items") or []:
+                indent = "  " * int(item.get("depth") or 0)
+                if ordered and not item.get("depth"):
+                    lines.append(f"{indent}{counter}. {item['text']}")
+                    counter += 1
+                else:
+                    lines.append(f"{indent}- {item['text']}")
+            out.append("\n".join(lines))
+        elif kind == "table":
+            rows = block.get("rows") or []
+            if not rows:
+                continue
+            if md:
+                width = max(len(r) for r in rows)
+                padded = [r + [""] * (width - len(r)) for r in rows]
+                lines = ["| " + " | ".join(padded[0]) + " |",
+                         "| " + " | ".join(["---"] * width) + " |"]
+                lines += ["| " + " | ".join(r) + " |" for r in padded[1:]]
+                out.append("\n".join(lines))
+            else:
+                out.append("\n".join("\t".join(r) for r in rows))
+        elif kind == "image":
+            src = block.get("src") or ""
+            alt = block.get("alt") or ""
+            caption = block.get("caption") or ""
+            if md:
+                line = f"![{alt}]({src})"
+                out.append(f"{line}\n{caption}" if caption else line)
+            else:
+                out.append(f"[image: {alt or caption or src}]")
+        elif kind == "rule":
+            out.append("---" if md else "—")
+    return [s for s in out if s.strip()]
+
+
+def _header(data: dict, fmt: str) -> str:
+    meta = data.get("metadata") or {}
+    lines = []
+    title = meta.get("title")
+    if title:
+        lines.append(f"# {title}" if fmt == "markdown" else title)
+    trailer = []
+    if meta.get("siteName"):
+        trailer.append(meta["siteName"])
+    if meta.get("byline"):
+        trailer.append(meta["byline"])
+    if meta.get("publishedTime"):
+        trailer.append(meta["publishedTime"])
+    trailer.append(data.get("finalUrl") or meta.get("url") or "")
+    lines.append(" · ".join(t for t in trailer if t))
+    return "\n".join(lines)
+
+
+def render(data: dict, cfg: dict) -> tuple[str, str, bool]:
+    """Return (stdout_text, full_text, truncated).
+
+    ``full_text`` is always the complete document; ``stdout_text`` is what the
+    caller should print, i.e. the same thing cut to ``maxChars``.
+    """
+    fmt = cfg.get("format", "markdown")
+    blocks = list(data.get("blocks") or [])
+    # 标题已经在 header 里印过一次；正文第一个块又是同一句时，去掉重复的那个。
+    title = ((data.get("metadata") or {}).get("title") or "").strip()
+    if blocks and title and blocks[0].get("type") == "heading":
+        first = (blocks[0].get("text") or "").strip().rstrip("¶").strip()
+        if first and (first == title or title.startswith(first)):
+            blocks = blocks[1:]
+
+    body_parts = _render_blocks(blocks, fmt)
+    body = "\n\n".join(body_parts)
+
+    if fmt == "json":
+        payload = {
+            "url": data.get("finalUrl"),
+            "status": data.get("status"),
+            "metadata": data.get("metadata"),
+            "container": data.get("container"),
+            "charCount": data.get("charCount"),
+            "blocks": data.get("blocks"),
+        }
+        full = json.dumps(payload, ensure_ascii=False, indent=2)
+        # JSON 不做块级截断：切一半的 JSON 不是 JSON。超预算时只报告，
+        # 让调用方自己决定是改格式还是配 --output。
+        return full, full, False
+
+    full = f"{_header(data, fmt)}\n\n{body}".strip()
+
+    max_chars = int(cfg.get("maxChars") or 0)
+    if max_chars <= 0 or len(full) <= max_chars:
+        return full, full, False
+
+    # Cut on a block boundary so the tail is a whole paragraph, not half a word.
+    head = f"{_header(data, fmt)}\n\n"
+    kept: list[str] = []
+    used = len(head)
+    for part in body_parts:
+        if used + len(part) + 2 > max_chars:
+            break
+        kept.append(part)
+        used += len(part) + 2
+    if not kept:
+        # Single oversized block (one giant <pre>, say) — fall back to a hard cut.
+        kept = [body[: max(0, max_chars - len(head))]]
+    shown = (head + "\n\n".join(kept)).strip()
+    notice = (
+        f"\n\n---\n[truncated] 已显示 {len(shown)} / {len(full)} 字符。"
+        "需要全文时加大 --max-chars（0 = 不限），或用 --output 把全文写到文件再按需读取。"
+    )
+    return shown + notice, full, True
+
+
+def do_read(cfg: dict) -> dict:
+    """Extract, render, optionally persist. Returns a small result summary."""
+    data = extract(cfg)
+
+    if data.get("error") == "selector-not-found":
+        raise SystemExit(
+            f'selector "{data.get("selector")}" 在页面上未找到（{cfg["url"]}）。'
+            "请换一个选择器，或去掉 --selector 让正文自动识别。"
+        )
+
+    stdout_text, full_text, truncated = render(data, cfg)
+
+    out_path = None
+    if cfg.get("output"):
+        out_path = Path(cfg["output"]).resolve()
+        out_path.parent.mkdir(parents=True, exist_ok=True)
+        out_path.write_text(full_text, encoding="utf-8")
+
+    return {
+        "text": stdout_text,
+        "fullText": full_text,
+        "truncated": truncated,
+        "outputPath": str(out_path) if out_path else None,
+        "charCount": data.get("charCount") or 0,
+        "bodyCharCount": data.get("bodyCharCount") or 0,
+        "blockCount": len(data.get("blocks") or []),
+        "container": data.get("container"),
+        "containerReason": data.get("containerReason"),
+        "status": data.get("status"),
+        "finalUrl": data.get("finalUrl"),
+        "metadata": data.get("metadata") or {},
+    }

package/skills/web-screenshot/scripts/_media_screenshot/urlguard.py

@@ -0,0 +1,90 @@
+"""Refuse to fetch private / link-local addresses (SSRF guard).
+
+Only ``read_page.py`` uses this today, and that asymmetry is deliberate:
+screenshot and record hand back a *file path*, while web_read hands the page's
+**text** straight to a model that is driven by user input. A prompt that says
+"read http://169.254.169.254/latest/meta-data/iam/..." is an exfiltration
+attempt, not a reading request.
+
+``WEB_CAPTURE_ALLOW_PRIVATE_HOSTS=1`` opts out — needed when pointing the skill
+at a dev server or an intranet page on purpose.
+
+Caveat: the check resolves DNS once and then hands the URL to the browser,
+which resolves it again. A name that flips between answers (DNS rebinding) can
+slip through that gap. Closing it properly needs request-level interception in
+Playwright; this guard is aimed at the ordinary case of a hostile or careless
+URL, not at an attacker who controls a nameserver.
+"""
+from __future__ import annotations
+
+import ipaddress
+import os
+import socket
+from urllib.parse import urlparse
+
+_TRUTHY = {"1", "true", "yes", "on"}
+
+
+def allow_private_hosts() -> bool:
+    return os.environ.get("WEB_CAPTURE_ALLOW_PRIVATE_HOSTS", "").strip().lower() in _TRUTHY
+
+
+def _is_blocked(ip: ipaddress._BaseAddress) -> bool:
+    return bool(
+        ip.is_private
+        or ip.is_loopback
+        or ip.is_link_local
+        or ip.is_reserved
+        or ip.is_multicast
+        or ip.is_unspecified
+    )
+
+
+def assert_public_url(raw_url: str) -> None:
+    """Raise SystemExit when *raw_url* is not an ordinary public http(s) page."""
+    parsed = urlparse(raw_url)
+    if parsed.scheme not in ("http", "https"):
+        raise SystemExit(
+            f"--url 只接受 http/https，收到 {parsed.scheme or '(空)'}：{raw_url}"
+        )
+    host = parsed.hostname
+    if not host:
+        raise SystemExit(f"--url 缺少主机名：{raw_url}")
+
+    if allow_private_hosts():
+        return
+
+    # A literal IP needs no DNS round-trip.
+    try:
+        literal = ipaddress.ip_address(host)
+    except ValueError:
+        literal = None
+    if literal is not None:
+        if _is_blocked(literal):
+            raise SystemExit(_refusal(host, str(literal)))
+        return
+
+    try:
+        infos = socket.getaddrinfo(host, parsed.port or (443 if parsed.scheme == "https" else 80))
+    except socket.gaierror as e:
+        raise SystemExit(f"无法解析主机 {host}：{e}")
+
+    for info in infos:
+        addr = info[4][0]
+        try:
+            ip = ipaddress.ip_address(addr)
+        except ValueError:
+            continue
+        # Any resolved address being internal is enough to refuse — a name that
+        # answers with one public and one private A record is the exact shape of
+        # an SSRF attempt.
+        if _is_blocked(ip):
+            raise SystemExit(_refusal(host, addr))
+
+
+def _refusal(host: str, addr: str) -> str:
+    return (
+        f"拒绝读取内网地址：{host} 解析到 {addr}（私有 / 环回 / 链路本地）。"
+        "web_read 会把页面正文交给模型，因此默认只读公网页面。"
+        "确实要读内网页面时设 WEB_CAPTURE_ALLOW_PRIVATE_HOSTS=1。"
+    )

package/skills/web-screenshot/scripts/read_page.py

@@ -0,0 +1,136 @@
+#!/usr/bin/env python3
+"""Playwright 正文抽取入口（薄壳，逻辑在 _media_screenshot.reader）。
+
+与 screenshot.py / record.py 同住一个 scripts/ 目录，因为三者共用
+`_media_screenshot/` 里的浏览器启动、等待、设备模拟与 cookie 处理。
+对外它是独立的 skill：`skills/web-read/skill.json` 指到这里。
+"""
+from __future__ import annotations
+
+import argparse
+import os
+import sys
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).resolve().parent))
+from _media_screenshot import cli_args  # noqa: E402
+from _media_screenshot.reader import do_read  # noqa: E402
+from _media_screenshot.urlguard import assert_public_url  # noqa: E402
+
+DEFAULT_MAX_CHARS = 20000
+
+
+def main() -> None:
+    ap = argparse.ArgumentParser(
+        description="通过 Playwright 打开网页并抽取正文文本（Markdown / 纯文本 / JSON）。"
+    )
+    ap.add_argument("-u", "--url", required=True, help="待读取的页面 URL（http/https）")
+    ap.add_argument(
+        "--selector",
+        help="只抽取该 CSS 选择器内的内容；不给则自动识别正文容器",
+    )
+    ap.add_argument(
+        "--format", choices=["markdown", "text", "json"], default="markdown",
+        help="输出格式：markdown（默认，保留标题/列表/代码块）| text（纯文本）| json（结构化块）",
+    )
+    ap.add_argument(
+        "--max-chars", type=int, default=DEFAULT_MAX_CHARS,
+        help=f"stdout 字符上限，按块截断（默认 {DEFAULT_MAX_CHARS}，0 = 不限）",
+    )
+    ap.add_argument(
+        "--include-links", action="store_true",
+        help="正文里的链接保留为 [文字](URL)（默认只留文字）",
+    )
+    ap.add_argument(
+        "--include-images", action="store_true",
+        help="保留图片为 ![alt](src)（默认丢弃）",
+    )
+    ap.add_argument(
+        "-o", "--output",
+        help="把**完整**正文（不截断）另存到该文件；stdout 仍受 --max-chars 限制",
+    )
+    ap.add_argument(
+        "--settle-ms", type=int,
+        help="抽取前额外静置毫秒数（给动画/懒加载留时间）",
+    )
+    ap.add_argument(
+        "--quiet", action="store_true",
+        help="只打印正文，不打印 stderr 上的抽取诊断",
+    )
+    cli_args.add_common_args(ap)
+
+    args = ap.parse_args()
+
+    # 内网地址守卫（见 urlguard.py：web_read 会把正文交给模型）。
+    assert_public_url(args.url)
+
+    storage_path, is_temp = cli_args.build_storage(args)
+
+    cfg: dict = {
+        "url": args.url,
+        "browser": args.browser or "chromium",
+        "format": args.format,
+        "maxChars": args.max_chars,
+        "includeLinks": bool(args.include_links),
+        "includeImages": bool(args.include_images),
+    }
+    cli_args.common_args_to_cfg(args, cfg)
+    if storage_path:
+        cfg["storageState"] = storage_path
+    if args.selector:
+        cfg["selector"] = args.selector
+    if args.output:
+        cfg["output"] = args.output
+    if args.settle_ms is not None:
+        cfg["settleMs"] = args.settle_ms
+
+    try:
+        result = do_read(cfg)
+    finally:
+        if is_temp and storage_path:
+            try:
+                os.unlink(storage_path)
+            except OSError:
+                pass
+
+    # 只在「基本没抽到东西」时报警。一篇文章页的整页文本本来就是正文的好几倍
+    # （导航、推荐、页脚），拿这个比例当告警条件会让警告天天响、从而没人再看。
+    chars = result["charCount"]
+    body_chars = result["bodyCharCount"]
+    status = result.get("status")
+    if isinstance(status, int) and status >= 400:
+        # 错误页也有正文（"404 Not Found"），照读不误——但不说一声的话，调用方
+        # 会把一张错误页当成文章内容去写脚本。
+        print(
+            f"⚠️  服务器返回 HTTP {status}，下面读到的很可能是错误页而不是目标内容。",
+            file=sys.stderr,
+        )
+    if chars < 100 and body_chars >= 2000:
+        print(
+            f"⚠️  整页有 {body_chars} 字符，却只抽到 {chars} 字符"
+            f"（容器 {result.get('container')}）——正文容器多半没认对。"
+            "可用 --selector <正文容器> 指定，或 --format json 看抽到了哪些块。",
+            file=sys.stderr,
+        )
+    elif chars < 100:
+        print(
+            f"⚠️  这个页面几乎没有文本（整页 {body_chars} 字符，HTTP {result.get('status')}）。"
+            "多半是登录墙、反爬拦截，或正文由 JS 延迟渲染还没出来。"
+            "可尝试：--wait-for-selector <正文选择器> / --settle-ms 3000 / --cookies <登录态>。",
+            file=sys.stderr,
+        )
+    elif not args.quiet:
+        print(
+            f"[web_read] {result['blockCount']} blocks · {result['charCount']} chars · "
+            f"container={result.get('container')} ({result.get('containerReason')})",
+            file=sys.stderr,
+        )
+
+    if result["outputPath"]:
+        print(f"[web_read] 全文已写入 {result['outputPath']}", file=sys.stderr)
+
+    print(result["text"])
+
+
+if __name__ == "__main__":
+    main()