argo-search 2.9.7 → 2.9.9

package/sub-skills/local-search/cli_runner.py

@@ -0,0 +1,331 @@
+#!/usr/bin/env python3
+"""cli_runner.py — local-search CLI 工具引擎调度（ddgs 等）。
+
+拆分自 search_v3.py：负责外部命令行搜索工具执行、临时结构化输出文件管理、
+错误标记探测与快速失败重试策略。
+"""
+
+from __future__ import annotations
+
+import atexit
+import json
+import os
+import re
+import subprocess
+import time
+from typing import Any
+
+# ddgs -o 的结构化输出临时文件清理
+_TMP_FILES: set[str] = set()
+
+
+def _cleanup_tmp_files() -> None:
+    for p in list(_TMP_FILES):
+        try:
+            os.unlink(p)
+        except OSError:
+            pass
+        _TMP_FILES.discard(p)
+
+
+atexit.register(_cleanup_tmp_files)
+
+# ddgs CLI 可用性检查缓存
+_CLI_AVAIL_CACHE: dict[str, tuple[float, bool]] = {}
+_CLI_AVAIL_TTL = 60.0
+
+_DDGS_TIMELIMITS: dict[str, set[str]] = {
+    "text": {"d", "w", "m", "y"},
+    "news": {"d", "w", "m", "y"},
+    "images": {"d", "w", "m", "y"},
+    "videos": {"d", "w", "m"},
+}
+
+_CLI_ERROR_MARKERS: tuple[str, ...] = (
+    "DDGSException", "No results", "ConnectError", "ConnectTimeout",
+    "ReadTimeout", "Timeout", "backends do not exist",
+)
+_CLI_RETRIES = 1
+_RETRY_FAST_S = 2.5
+
+
+def _since_to_ddgs_timelimit(since: str | None, until: str | None) -> str | None:
+    """Convert since/until to ddgs timelimit argument."""
+    if not since:
+        return None
+    match = re.match(r"^(\d+)([dwmy])$", since.lower())
+    if match:
+        num, unit = match.groups()
+        num = int(num)
+        if unit == "y":
+            return "y"
+        elif unit == "m":
+            return "m"
+        elif unit == "w":
+            return "w"
+        elif unit == "d":
+            if num >= 365:
+                return "y"
+            elif num >= 30:
+                return "m"
+            elif num >= 7:
+                return "w"
+            else:
+                return "d"
+    return None
+
+
+def _check_cli_available(cmd: str) -> bool:
+    """Check if a CLI command is available（结果缓存 TTL=_CLI_AVAIL_TTL）。"""
+    import shutil
+    now = time.time()
+    hit = _CLI_AVAIL_CACHE.get(cmd)
+    if hit is not None and now - hit[0] < _CLI_AVAIL_TTL:
+        return hit[1]
+    ok = shutil.which(cmd) is not None
+    _CLI_AVAIL_CACHE[cmd] = (time.time(), ok)
+    return ok
+
+
+def _parse_cli_json(data: Any, engine_name: str) -> list[dict[str, Any]]:
+    """解析 ddgs 结构化 JSON 输出（-o json 写入文件）。"""
+    if not isinstance(data, list):
+        return []
+    results: list[dict[str, Any]] = []
+    for item in data:
+        if not isinstance(item, dict):
+            continue
+        title = item.get("title", "")
+        url = item.get("href", item.get("url", item.get("link", "")))
+        snippet = item.get("body", item.get("snippet",
+                                            item.get("description", item.get("content", ""))))
+        if not title and not url:
+            continue
+        r: dict[str, Any] = {
+            "title": str(title),
+            "url": str(url),
+            "snippet": str(snippet),
+            "source": engine_name,
+        }
+        for key in ("image", "thumbnail", "source", "duration", "provider",
+                    "height", "width"):
+            if key in item and item[key] not in (None, ""):
+                r[key] = item[key]
+        date = item.get("date")
+        if date:
+            r["published_at"] = str(date)[:64]
+        meta = " · ".join(
+            str(item[k]) for k in ("author", "publisher", "info")
+            if item.get(k)
+        )
+        if meta:
+            r["snippet"] = (str(snippet) + " · " + meta)[:400] if snippet else meta[:400]
+        results.append(r)
+    return results
+
+
+def _parse_cli_output(stdout: str, engine_name: str) -> tuple[list[dict[str, Any]], str]:
+    """Parse output from CLI search tools like ddgs."""
+    results = []
+    try:
+        data = json.loads(stdout)
+        if isinstance(data, list):
+            for item in data:
+                result = {
+                    "title": item.get("title", ""),
+                    "url": item.get("href", item.get("url", item.get("link", ""))),
+                    "snippet": item.get("body", item.get("snippet", item.get("description", item.get("content", "")))),
+                    "source": engine_name,
+                }
+                for key in ["date", "image", "thumbnail", "source"]:
+                    if key in item:
+                        result[key] = item[key]
+                results.append(result)
+            return results, ""
+    except json.JSONDecodeError:
+        pass
+
+    current_result = {}
+    current_field = None
+
+    for line in stdout.split("\n"):
+        line_stripped = line.strip()
+        if not line_stripped:
+            continue
+        if re.fullmatch(r"\d+\.\s*", line_stripped):
+            if current_result:
+                results.append(current_result)
+                current_result = {}
+                current_field = None
+            continue
+
+        field_matched = False
+        for field, target_key in [
+            ("title", "title"),
+            ("href", "url"),
+            ("url", "url"),
+            ("body", "snippet"),
+            ("snippet", "snippet"),
+            ("description", "snippet"),
+            ("content", "snippet"),
+            ("date", "date"),
+            ("image", "image"),
+            ("thumbnail", "thumbnail"),
+            ("source", "source"),
+            ("duration", "duration"),
+            ("provider", "provider"),
+            ("height", "height"),
+            ("width", "width"),
+        ]:
+            if line_stripped.startswith(field) and len(line_stripped) > len(field):
+                value = line_stripped[len(field):].strip()
+                if value:
+                    current_result[target_key] = value
+                    current_field = field
+                    field_matched = True
+                    break
+
+        if not field_matched and current_field:
+            if current_field in ("body", "snippet", "description", "content"):
+                current_result.setdefault("snippet", "")
+                current_result["snippet"] += " " + line_stripped
+
+    if current_result:
+        results.append(current_result)
+
+    for i, r in enumerate(results):
+        r.setdefault("source", engine_name)
+        r.setdefault("title", "")
+        r.setdefault("url", "")
+        r.setdefault("snippet", "")
+        r.setdefault("score", round(0.9 - (i * 0.05), 2))
+    return results, ""
+
+
+def _attempt_loop(cmd: list[str], cli_cmd: str, key: str,
+                  tmp_path: str | None, timeout: float):
+    """CLI 引擎的单引擎尝试循环。"""
+    last_err = ""
+    for attempt in range(_CLI_RETRIES + 1):
+        t_attempt = time.monotonic()
+        try:
+            result = subprocess.run(
+                cmd,
+                capture_output=True,
+                text=True,
+                encoding="utf-8",
+                errors="replace",
+                timeout=timeout,
+            )
+        except subprocess.TimeoutExpired:
+            return [], f"{cli_cmd} timed out after {timeout}s"
+        except FileNotFoundError:
+            return [], f"{cli_cmd} not found"
+        except Exception as e:
+            return [], f"{cli_cmd} error: {e}"
+
+        combined_output = result.stdout + result.stderr
+
+        if tmp_path is not None:
+            json_ok = False
+            try:
+                with open(tmp_path, encoding="utf-8") as f:
+                    data = json.load(f)
+                parsed = _parse_cli_json(data, key)
+                if parsed:
+                    return parsed, ""
+                json_ok = True
+            except (OSError, ValueError):
+                json_ok = False
+            finally:
+                try:
+                    os.unlink(tmp_path)
+                except OSError:
+                    pass
+            if json_ok:
+                return [], ""
+        else:
+            json_ok = False
+
+        err_marker = next(
+            (m for m in _CLI_ERROR_MARKERS if m in combined_output), None
+        )
+        if err_marker:
+            detail = combined_output.strip().replace("\n", " ")[:120]
+            last_err = f"{cli_cmd}: {err_marker}: {detail}"
+            if attempt < _CLI_RETRIES and time.monotonic() - t_attempt <= _RETRY_FAST_S:
+                time.sleep(0.3)
+                continue
+            return [], last_err
+        if result.returncode != 0:
+            last_err = f"{cli_cmd} exit {result.returncode}: {combined_output[:200]}"
+            if attempt < _CLI_RETRIES and time.monotonic() - t_attempt <= _RETRY_FAST_S:
+                time.sleep(0.3)
+                continue
+            return [], last_err
+
+        parsed, parse_err = _parse_cli_output(result.stdout, key)
+        if parsed:
+            return parsed, parse_err
+        last_err = f"{cli_cmd}: empty output"
+        if attempt < _CLI_RETRIES and time.monotonic() - t_attempt <= _RETRY_FAST_S:
+            time.sleep(0.3)
+            continue
+        return [], last_err
+    return [], last_err
+
+
+def _run_cli_engine(spec: dict[str, Any], query: str, n: int, timeout: float,
+                    since: str | None = None, until: str | None = None,
+                    lang_param_fn=None) -> tuple[list[dict[str, Any]], str]:
+    """Run a CLI-based search engine (e.g., ddgs)."""
+    cli_cmd = spec.get("cli_command", "")
+    cli_args_template = spec.get("cli_args", [])
+    key = spec.get("key", cli_cmd)
+
+    if not cli_cmd:
+        return [], "CLI engine missing cli_command"
+
+    if not _check_cli_available(cli_cmd):
+        return [], f"{cli_cmd} not available"
+
+    time_limit = _since_to_ddgs_timelimit(since, until)
+
+    cli_args = []
+    for arg in cli_args_template:
+        formatted = arg.format(query=query, n=n)
+        cli_args.append(formatted)
+
+    if time_limit and "-t" not in cli_args and "--timelimit" not in cli_args:
+        sub = next((a for a in cli_args_template if not str(a).startswith("-")), None)
+        allowed = _DDGS_TIMELIMITS.get(str(sub), _DDGS_TIMELIMITS["text"])
+        if time_limit in allowed:
+            cli_args.extend(["-t", time_limit])
+
+    if spec.get("use_lang_region") and not any(a in ("-r", "--region") for a in cli_args) and lang_param_fn:
+        region = lang_param_fn("region", query)
+        if region:
+            cli_args.extend(["-r", region])
+
+    output_format = spec.get("output_format", "")
+    tmp_path: str | None = None
+    if output_format == "json" and not any(
+        a in ("-o", "--output") for a in cli_args
+    ):
+        import tempfile
+        fd, tmp_path = tempfile.mkstemp(suffix=".json", prefix="ddgs_")
+        os.close(fd)
+        _TMP_FILES.add(tmp_path)
+        cli_args.extend(["-o", tmp_path])
+
+    cmd = [cli_cmd] + cli_args
+
+    try:
+        return _attempt_loop(cmd, cli_cmd, key, tmp_path, timeout)
+    finally:
+        if tmp_path is not None:
+            try:
+                os.unlink(tmp_path)
+            except OSError:
+                pass
+            _TMP_FILES.discard(tmp_path)

package/sub-skills/local-search/parsers.py

@@ -0,0 +1,328 @@
+#!/usr/bin/env python3
+"""parsers.py — local-search 多源响应解析器（HTML / RSS / XML / JSON）。
+
+拆分自 search_v3.py：负责选择器匹配、启发式链接回退（_fallback_extract）、
+XML 命名空间解析与 JSON 字典路径提取。无网络 IO。
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import re
+import sys
+import urllib.parse
+import xml.etree.ElementTree as ET
+from typing import Any
+
+# 尝试连接主仓 serp_guard 内容农场判定
+_SPAM_CHECKER = None
+try:
+    _SCRIPTS_PATH = os.path.normpath(os.path.join(os.path.dirname(os.path.abspath(__file__)), "..", "..", "scripts"))
+    if _SCRIPTS_PATH not in sys.path:
+        sys.path.insert(0, _SCRIPTS_PATH)
+    from serp_guard import is_content_farm as _SPAM_CHECKER
+except Exception:
+    _SPAM_CHECKER = None
+
+
+def _is_spam_entry(url: str, title: str = "", snippet: str = "") -> bool:
+    if _SPAM_CHECKER is not None:
+        try:
+            return bool(_SPAM_CHECKER(url, title, snippet))
+        except Exception:
+            return False
+    return False
+
+
+
+def _select_with_bs4(soup: Any, selector: str):
+    try:
+        return soup.select(selector)
+    except Exception:
+        return []
+
+
+def _select_first_with_bs4(soup: Any, selector: str) -> Any:
+    items = _select_with_bs4(soup, selector)
+    return items[0] if items else None
+
+
+def _fallback_extract(html: str, engine_name: str, base: str,
+                      default_score: float = 0.45, max_items: int = 8) -> list[dict[str, Any]]:
+    """通用解析保底：选择器 0 命中时，用链接 + 锚文本启发式回退。"""
+    try:
+        from bs4 import BeautifulSoup
+        soup = BeautifulSoup(html, "html.parser")
+    except Exception:
+        return []
+    for sel in ("nav", "header", "footer", "script", "style", "noscript"):
+        try:
+            for tag in soup.select(sel):
+                tag.decompose()
+        except Exception:
+            pass
+    candidates: list[tuple[str, str, str]] = []
+    for a in soup.find_all("a", href=True):
+        try:
+            href = str(a.get("href") or "").strip()
+            title = a.get_text(strip=True)
+        except Exception:
+            continue
+        if not title or len(title) < 6:
+            continue
+        if href.startswith("#") or href.startswith("javascript:"):
+            continue
+        low = href.lower()
+        if any(x in low for x in ("login", "signin", "signup", "cookie", "privacy", "terms")):
+            continue
+        parent = a.parent
+        snippet = ""
+        try:
+            if parent is not None:
+                snippet = parent.get_text(" ", strip=True)[:300]
+                if snippet == title:
+                    gp = parent.parent
+                    snippet = gp.get_text(" ", strip=True)[:300] if gp is not None else ""
+        except Exception:
+            snippet = ""
+        if href.startswith("/"):
+            try:
+                href = urllib.parse.urljoin(base, href)
+            except Exception:
+                pass
+        if _is_spam_entry(href, title, snippet):
+            continue
+        candidates.append((title[:200], href[:500], snippet))
+        if len(candidates) >= max_items * 3:
+            break
+    seen: set[str] = set()
+    out: list[dict[str, Any]] = []
+    for title, url, snippet in candidates:
+        key = url or title
+        if not key or key in seen:
+            continue
+        seen.add(key)
+        out.append({
+            "title": title,
+            "url": url,
+            "snippet": snippet,
+            "score": round(max(default_score - len(out) * 0.05, 0.1), 3),
+            "source": engine_name,
+            "_fallback": True,
+        })
+        if len(out) >= max_items:
+            break
+    return out
+
+
+def _parse_html(engine_name: str, html: str, spec: dict[str, Any],
+                maps: dict[str, Any]) -> list[dict[str, Any]]:
+    """HTML 解析映射：选择器容器抽取与属性提取。"""
+    html_maps = maps.get("html", {})
+    mapping = html_maps.get(engine_name, html_maps.get("default", {}))
+    container_sel = mapping.get("container")
+    title_sel = mapping.get("title")
+    url_sel = mapping.get("url")
+    snippet_sel = mapping.get("snippet")
+    url_attr = mapping.get("url_attr", "href")
+    default_score = mapping.get("score", 0.5)
+
+    if not container_sel:
+        return []
+
+    try:
+        from bs4 import BeautifulSoup
+        soup = BeautifulSoup(html, "html.parser")
+        containers = _select_with_bs4(soup, container_sel)
+    except Exception:
+        return []
+
+    results: list[dict[str, Any]] = []
+    base = spec.get("_base", spec.get("url", ""))
+    for idx, item in enumerate(containers):
+        try:
+            title_el = _select_first_with_bs4(item, title_sel) if title_sel else None
+            url_el = _select_first_with_bs4(item, url_sel) if url_sel else None
+            snippet_el = _select_first_with_bs4(item, snippet_sel) if snippet_sel else None
+
+            title = title_el.get_text(strip=True)[:200] if title_el else ""
+            url = ""
+            if url_el and url_el.has_attr(url_attr):
+                url = url_el[url_attr]
+            snippet = snippet_el.get_text(strip=True)[:300] if snippet_el else ""
+
+            if not title and not url:
+                continue
+
+            if url and url.startswith("/"):
+                url = urllib.parse.urljoin(base, url)
+
+            if _is_spam_entry(url, title, snippet):
+                continue
+
+            score = max(default_score - idx * 0.05, 0.1)
+            results.append({
+                "title": title,
+                "url": url,
+                "snippet": snippet,
+                "score": round(score, 3),
+                "source": engine_name,
+            })
+        except Exception:
+            continue
+    if not results:
+        try:
+            fb = _fallback_extract(html, engine_name, base, default_score=0.45, max_items=8)
+            if fb:
+                return fb
+        except Exception:
+            pass
+    return results
+
+
+def _parse_xml(engine_name: str, text: str, maps: dict[str, Any],
+               is_rss: bool = False, normalize_date_fn=None) -> list[dict[str, Any]]:
+    """XML / RSS 解析：支持命名空间与 pubDate 提取。"""
+    if is_rss:
+        mapping = maps.get("rss", {}).get("default", {})
+    else:
+        mapping = maps.get("xml", {}).get(engine_name, {})
+
+    entry_path = mapping.get("entry_path") or mapping.get("item_path", ".//item")
+    title_tag = mapping.get("title", "title")
+    url_tag = mapping.get("url", "link")
+    snippet_tag = mapping.get("snippet", "description")
+    pub_tag = mapping.get("published_at", "pubDate")
+    namespaces = mapping.get("namespaces", {})
+
+    def _find_tag(entry: Any, tag: str) -> str:
+        if tag.startswith("atom:"):
+            tag_name = tag.split(":")[1]
+            ns = namespaces.get("atom")
+            node = entry.find(f"{{{ns}}}{tag_name}") if ns else None
+            return (node.text or "").strip() if node is not None else ""
+        node = entry.findtext(tag, default="")
+        return (node or "").strip()
+
+    results: list[dict[str, Any]] = []
+    try:
+        root = ET.fromstring(text)
+    except ET.ParseError:
+        return results
+
+    if entry_path.startswith(".//{"):
+        entries = root.findall(entry_path, namespaces)
+    else:
+        entries = root.findall(entry_path)
+
+    for idx, entry in enumerate(entries):
+        try:
+            title = _find_tag(entry, title_tag)
+            url = _find_tag(entry, url_tag)
+            snippet = _find_tag(entry, snippet_tag)
+            raw_pub = _find_tag(entry, pub_tag) if pub_tag else ""
+
+            title = re.sub(r"\s+", " ", title)[:200]
+            snippet = re.sub(r"\s+", " ", snippet)[:300]
+            score = max(0.7 - idx * 0.05, 0.1)
+            pub_at = normalize_date_fn(raw_pub) if normalize_date_fn else raw_pub
+            results.append({
+                "title": title,
+                "url": url,
+                "snippet": snippet,
+                "score": round(score, 3),
+                "source": engine_name,
+                "published_at": pub_at,
+            })
+        except Exception:
+            continue
+    return results
+
+
+def _get_path(data: Any, path: str) -> Any:
+    """提取嵌套字典/列表路径，如 'items.0.title'。"""
+    if path == ".":
+        return data
+    obj = data
+    for part in path.split("."):
+        if part.isdigit() and isinstance(obj, (list, tuple)):
+            idx = int(part)
+            obj = obj[idx] if idx < len(obj) else None
+        elif isinstance(obj, dict):
+            obj = obj.get(part)
+        else:
+            return None
+    return obj
+
+
+def _format_url(template: str | None, item: dict[str, Any], default: str = "") -> str:
+    """按模板格式化 URL 字段。"""
+    if not template:
+        return default
+    try:
+        return template.format(**item)
+    except (KeyError, IndexError):
+        return default
+
+
+def _parse_json(engine_name: str, text: str, maps: dict[str, Any],
+                normalize_date_fn=None) -> list[dict[str, Any]]:
+    """JSON 响应解析：提取标题、URL、摘要及时间。"""
+    mapping = maps.get("json", {}).get(engine_name, {})
+    items_path = mapping.get("items", ".")
+    title_key = mapping.get("title")
+    url_key = mapping.get("url")
+    snippet_key = mapping.get("snippet")
+    url_template = mapping.get("url_template")
+    published_key = mapping.get("published_at")
+
+    results: list[dict[str, Any]] = []
+    try:
+        data = json.loads(text)
+    except json.JSONDecodeError:
+        return results
+
+    items = _get_path(data, items_path)
+    if not isinstance(items, list):
+        return results
+
+    for idx, item in enumerate(items):
+        if not isinstance(item, dict):
+            if engine_name == "local_pubmed" and isinstance(item, str):
+                item = {"pmid": item}
+            else:
+                continue
+
+        title = url = snippet = ""
+        if title_key:
+            raw = _get_path(item, title_key)
+            if isinstance(raw, list):
+                raw = raw[0] if raw else ""
+            title = str(raw or "")[:200]
+        if url_key:
+            raw = _get_path(item, url_key)
+            url = str(raw or "")[:500]
+        elif url_template:
+            url = _format_url(url_template, item)
+        if snippet_key:
+            raw = _get_path(item, snippet_key)
+            snippet = str(raw or "")[:300]
+        raw_pub = _get_path(item, published_key) if published_key else None
+        published_at = normalize_date_fn(raw_pub) if normalize_date_fn and raw_pub else raw_pub
+
+        title = re.sub(r"<[^>]+>", " ", title)
+        title = re.sub(r"\s+", " ", title).strip()
+        snippet = re.sub(r"<[^>]+>", " ", snippet)
+        snippet = re.sub(r"\s+", " ", snippet).strip()
+
+        score = max(0.7 - idx * 0.05, 0.1)
+        results.append({
+            "title": title,
+            "url": url,
+            "snippet": snippet,
+            "score": round(score, 3),
+            "source": engine_name,
+            "published_at": published_at,
+        })
+    return results