evolcore 0.0.31 → 0.0.33

package/kits/skills/scrapling-article-fetch/scripts/render_markdown_reply.py

@@ -0,0 +1,35 @@
+#!/usr/bin/env python3
+import json
+import sys
+from pathlib import Path
+
+# Windows 终端默认 GBK，强制 stdout/stderr 用 UTF-8
+if hasattr(sys.stdout, "reconfigure"):
+    sys.stdout.reconfigure(encoding="utf-8", errors="replace")
+if hasattr(sys.stderr, "reconfigure"):
+    sys.stderr.reconfigure(encoding="utf-8", errors="replace")
+
+
+def render_reply(title: str, src_url: str, markdown: str) -> str:
+    body = markdown.strip()
+    return (
+        "```markdown\n"
+        f"# {title}\n\n"
+        f"原文链接：{src_url}\n\n"
+        f"{body}\n"
+        "```"
+    )
+
+
+def main() -> int:
+    if len(sys.argv) != 4:
+        print("usage: render_markdown_reply.py <title> <src_url> <markdown_file>", file=sys.stderr)
+        return 2
+    title, src_url, markdown_file = sys.argv[1:4]
+    markdown = Path(markdown_file).read_text(encoding="utf-8")
+    print(render_reply(title, src_url, markdown))
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())

package/kits/skills/scrapling-article-fetch/scripts/run_pipeline.sh

@@ -0,0 +1,16 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+# 自定位：从脚本自身位置推导同目录脚本，不依赖 cwd 或任何绝对路径。
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+
+if command -v python3 >/dev/null 2>&1; then
+  PYTHON_BIN=python3
+elif command -v python >/dev/null 2>&1; then
+  PYTHON_BIN=python
+else
+  echo 'Python not found' >&2
+  exit 1
+fi
+
+"$PYTHON_BIN" "$SCRIPT_DIR/check_python_env.py"

package/kits/skills/scrapling-article-fetch/scripts/scrapling_fetch.py

@@ -0,0 +1,344 @@
+#!/usr/bin/env python3
+import io
+import json
+import re
+import sys
+import urllib.request
+from urllib.parse import urlparse
+
+# Windows 终端默认 GBK，强制 stdout/stderr 用 UTF-8
+if hasattr(sys.stdout, "reconfigure"):
+    sys.stdout.reconfigure(encoding="utf-8", errors="replace")
+if hasattr(sys.stderr, "reconfigure"):
+    sys.stderr.reconfigure(encoding="utf-8", errors="replace")
+
+import html2text
+import lxml.html
+import lxml.cssselect  # noqa: F401 — pulls cssselect into lxml's registry
+
+
+# ---------------------------------------------------------------------------
+# Thin compatibility shim — mimics the scrapling page/node API surface used
+# in this script, backed by urllib + lxml so patchright is never needed.
+# ---------------------------------------------------------------------------
+
+_HEADERS = {
+    "User-Agent": (
+        "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
+        "AppleWebKit/537.36 (KHTML, like Gecko) "
+        "Chrome/124.0.0.0 Safari/537.36"
+    ),
+    "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
+    "Accept-Language": "zh-CN,zh;q=0.9,en;q=0.8",
+}
+
+
+class _NodeList:
+    def __init__(self, nodes):
+        self._nodes = nodes
+
+    @property
+    def first(self):
+        return _Node(self._nodes[0]) if self._nodes else None
+
+    def __bool__(self):
+        return bool(self._nodes)
+
+
+class _Node:
+    def __init__(self, el):
+        self._el = el
+
+    @property
+    def html_content(self):
+        return lxml.html.tostring(self._el, encoding="unicode", with_tail=False)
+
+    def css(self, sel):
+        try:
+            return _NodeList(self._el.cssselect(sel))
+        except Exception:
+            return _NodeList([])
+
+    def get(self, attr, default=None):
+        return self._el.get(attr, default)
+
+    @property
+    def attributes(self):
+        return dict(self._el.attrib)
+
+    def __str__(self):
+        return self.html_content
+
+
+class _Page:
+    def __init__(self, html_bytes: bytes, status: int, url: str):
+        self.status = status
+        self._url = url
+        self._doc = lxml.html.fromstring(html_bytes, base_url=url)
+
+    @property
+    def html_content(self):
+        return lxml.html.tostring(self._doc, encoding="unicode", with_tail=False)
+
+    def css(self, sel):
+        try:
+            return _NodeList(self._doc.cssselect(sel))
+        except Exception:
+            return _NodeList([])
+
+
+class Fetcher:
+    @staticmethod
+    def get(url: str) -> _Page:
+        req = urllib.request.Request(url, headers=_HEADERS)
+        with urllib.request.urlopen(req, timeout=30) as resp:
+            status = resp.status
+            html_bytes = resp.read()
+        return _Page(html_bytes, status, url)
+
+DEFAULT_SELECTORS = [
+    "article",
+    "main",
+    ".post-content",
+    '[class*="body"]',
+]
+
+WECHAT_SELECTORS = [
+    "#js_content",
+    ".rich_media_content",
+    ".rich_media_area_primary",
+]
+
+WECHAT_TITLE_SELECTORS = [
+    "#activity-name",
+    ".rich_media_title",
+    "h1",
+]
+
+WECHAT_COVER_SELECTORS = [
+    '.rich_media_thumb[style*="background-image"]',
+    '.rich_media_thumb',
+    'meta[property="og:image"]',
+]
+
+NOISE_MARKERS = [
+    "Scan with Weixin to",
+    "微信扫一扫可打开此内容",
+    "轻触阅读原文",
+    "预览时标签不可点",
+    "Got It",
+    "Cancel",
+    "Allow",
+]
+
+
+def choose_selectors(url: str):
+    host = (urlparse(url).hostname or "").lower()
+    if host.endswith("mp.weixin.qq.com"):
+        return WECHAT_SELECTORS + DEFAULT_SELECTORS
+    return DEFAULT_SELECTORS
+
+
+def normalize_html_for_images(html: str) -> str:
+    return re.sub(
+        r'<img([^>]*?)\sdata-src="([^"]+)"([^>]*?)>',
+        lambda m: f'<img{m.group(1)} src="{m.group(2)}"{m.group(3)}>',
+        html,
+        flags=re.IGNORECASE,
+    )
+
+
+def strip_tail_noise(markdown: str) -> str:
+    cut_positions = [markdown.find(marker) for marker in NOISE_MARKERS if marker in markdown]
+    cut_positions = [p for p in cut_positions if p >= 0]
+    if cut_positions:
+        markdown = markdown[: min(cut_positions)]
+
+    while "\n\n\n" in markdown:
+        markdown = markdown.replace("\n\n\n", "\n\n")
+
+    return markdown.strip()
+
+
+def clean_text(text: str) -> str:
+    text = re.sub(r"<[^>]+>", " ", text)
+    return re.sub(r"\s+", " ", text).strip()
+
+
+def get_text(node) -> str:
+    if not node:
+        return ""
+    for attr in ("text", "text_content", "html_content"):
+        value = getattr(node, attr, None)
+        if isinstance(value, str) and value.strip():
+            return clean_text(value)
+        if callable(value):
+            try:
+                out = value()
+                if isinstance(out, str) and out.strip():
+                    return clean_text(out)
+            except Exception:
+                pass
+    return clean_text(str(node))
+
+
+def get_attr(node, attr: str) -> str:
+    if not node:
+        return ""
+    attributes = getattr(node, "attributes", None)
+    if isinstance(attributes, dict) and attributes.get(attr):
+        return attributes[attr]
+    attrib = getattr(node, "attrib", None)
+    if isinstance(attrib, dict) and attrib.get(attr):
+        return attrib[attr]
+    fn = getattr(node, "get", None)
+    if callable(fn):
+        try:
+            value = fn(attr)
+            if value:
+                return value
+        except Exception:
+            pass
+    html = getattr(node, "html_content", None) or str(node)
+    m = re.search(fr'{attr}="([^"]+)"', html)
+    return m.group(1) if m else ""
+
+
+def extract_title(page, url: str, markdown: str = "") -> str:
+    host = (urlparse(url).hostname or "").lower()
+    if host.endswith("mp.weixin.qq.com"):
+        for sel in WECHAT_TITLE_SELECTORS:
+            nodes = page.css(sel)
+            node = nodes.first if nodes else None
+            text = get_text(node)
+            if text:
+                return text.strip()
+
+    html = getattr(page, "html_content", None) or ""
+    for pattern in [
+        r'<meta[^>]+property="og:title"[^>]+content="([^"]+)"',
+        r'<meta[^>]+name="twitter:title"[^>]+content="([^"]+)"',
+        r'<title>(.*?)</title>',
+    ]:
+        m = re.search(pattern, html, re.I | re.S)
+        if m:
+            text = re.sub(r"\s+", " ", m.group(1)).strip()
+            if text:
+                return text
+
+    if markdown:
+        for line in markdown.splitlines():
+            line = line.strip()
+            if not line:
+                continue
+            if re.match(r"!\[[^\]]*\]\(([^)]+)\)", line):
+                continue
+            line = re.sub(r"^>\s*", "", line)
+            line = re.sub(r"^[#\s]+", "", line)
+            line = re.sub(r"^\*\*(.*?)\*\*$", r"\1", line)
+            if line:
+                return line.strip()
+    return ""
+
+
+def extract_cover_url(page, url: str, markdown: str = "") -> str:
+    host = (urlparse(url).hostname or "").lower()
+    html = getattr(page, "html_content", None) or ""
+    if host.endswith("mp.weixin.qq.com"):
+        for sel in WECHAT_COVER_SELECTORS:
+            nodes = page.css(sel)
+            node = nodes.first if nodes else None
+            if not node:
+                continue
+            if sel.startswith("meta"):
+                content = get_attr(node, "content")
+                if content:
+                    return content.strip()
+            style = get_attr(node, "style")
+            if style:
+                m = re.search(r'background-image:\s*url\([\"\']?([^\)\"\']+)', style)
+                if m:
+                    return m.group(1).strip()
+            for attr in ("data-src", "src"):
+                value = get_attr(node, attr)
+                if value:
+                    return value.strip()
+
+    m = re.search(r'<meta[^>]+property="og:image"[^>]+content="([^"]+)"', html, re.I)
+    if m:
+        return m.group(1).strip()
+
+    if markdown:
+        m = re.search(r'!\[[^\]]*\]\(([^)]+)\)', markdown)
+        if m:
+            return m.group(1).strip()
+    return ""
+
+
+def extract_markdown(url: str, max_chars: int = 30000) -> str:
+    return extract_article(url, max_chars)["markdown"]
+
+
+def extract_article(url: str, max_chars: int = 30000) -> dict:
+    page = Fetcher.get(url)
+    status = getattr(page, "status", None)
+    if isinstance(status, int) and status >= 400:
+        raise RuntimeError(f"HTTP {status} for {url}")
+
+    chosen_html = None
+    for sel in choose_selectors(url):
+        nodes = page.css(sel)
+        node = nodes.first if nodes else None
+        if node:
+            chosen_html = getattr(node, "html_content", None) or str(node)
+            break
+
+    if not chosen_html:
+        chosen_html = getattr(page, "html_content", None) or ""
+
+    chosen_html = normalize_html_for_images(chosen_html)
+
+    parser = html2text.HTML2Text()
+    parser.ignore_links = False
+    parser.ignore_images = False
+    parser.body_width = 0
+
+    markdown = parser.handle(chosen_html).strip()
+    markdown = strip_tail_noise(markdown)
+    title = extract_title(page, url, markdown)
+    cover_url = extract_cover_url(page, url, markdown)
+    if cover_url and cover_url not in markdown:
+        markdown = f"![]({cover_url})\n\n{markdown}".strip()
+    markdown = markdown[:max_chars]
+
+    return {
+        "title": title,
+        "cover_url": cover_url,
+        "markdown": markdown,
+    }
+
+
+def main() -> int:
+    if len(sys.argv) < 2:
+        print("Usage: python scrapling_fetch.py <url> [max_chars] [--json]", file=sys.stderr)
+        return 1
+
+    args = [a for a in sys.argv[1:] if a != "--json"]
+    as_json = "--json" in sys.argv[1:]
+    url = args[0]
+    max_chars = int(args[1]) if len(args) > 1 else 30000
+
+    try:
+        article = extract_article(url, max_chars)
+        if as_json:
+            print(json.dumps(article, ensure_ascii=False))
+        else:
+            print(article["markdown"])
+        return 0
+    except Exception as exc:
+        print(f"[ERROR] {exc}", file=sys.stderr)
+        return 2
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())