argo-search 1.0.1

package/scripts/evidence.py

@@ -0,0 +1,381 @@
+#!/usr/bin/env python3
+"""
+evidence.py — 来源可信度评估工具（wigolo evidence_score 理念移植）
+
+核心能力：
+  1. 权威性评分：基于域名白名单/黑名单 + 来源类型分级
+  2. 时效性评分：基于内容时间戳 + 搜索时间
+  3. 交叉验证：多个来源是否佐证同一结论
+  4. 综合可信度：加权计算 + 透明分解
+
+用法：
+  python3 evidence.py --urls "https://example.com" "https://example2.com"
+  python3 evidence.py --search-result '{"results": [...]}' --query "茅台股价"
+  echo '{"results": [...]}' | python3 evidence.py --stdin --query "查询词"
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import os
+import re
+import sys
+import time
+from typing import Any
+from urllib.parse import urlparse
+
+SCRIPT_DIR = os.path.dirname(os.path.abspath(__file__))
+sys.path.insert(0, SCRIPT_DIR)
+
+
+# ── 权威性评分 ────────────────────────────────────────────────────────────────
+
+# 域名权威性分级（越高越好）
+AUTHORITY_TIERS = {
+    # Tier 1: 官方/权威（0.95-1.0）
+    "gov.cn": 1.0, "gov": 0.95, "edu.cn": 0.95, "edu": 0.9,
+    "ac.cn": 0.95,  # 中国科学院
+    "nature.com": 0.95, "science.org": 0.95, "ieee.org": 0.9,
+    "acm.org": 0.9, "springer.com": 0.9, "elsevier.com": 0.9,
+    "arxiv.org": 0.9, "pubmed.ncbi.nlm.nih.gov": 0.95,
+    "scholar.google.com": 0.85, "ncbi.nlm.nih.gov": 0.95,
+    "nvd.nist.gov": 0.95,  # 国家漏洞数据库
+    "cve.mitre.org": 0.95,
+
+    # Tier 2: 专业媒体/平台（0.75-0.9）
+    "zhihu.com": 0.85, "github.com": 0.85, "stackoverflow.com": 0.85,
+    "medium.com": 0.75, "dev.to": 0.75,
+    "reuters.com": 0.9, "bloomberg.com": 0.9, "wsj.com": 0.9,
+    "财新": 0.9, "caixin.com": 0.9,
+    "36kr.com": 0.8, "infoq.cn": 0.8, "juejin.cn": 0.75,
+    "eastmoney.com": 0.85, "xueqiu.com": 0.8,
+    "docs.python.org": 0.9, "react.dev": 0.9, "nextjs.org": 0.9,
+
+    # Tier 3: 通用可信（0.6-0.75）
+    "wikipedia.org": 0.75, "baike.baidu.com": 0.7,
+    "维基百科": 0.75, "百度百科": 0.7,
+    "linkedin.com": 0.65, "twitter.com": 0.55, "x.com": 0.55,
+    "reddit.com": 0.65, "hackernews": 0.7,
+
+    # Tier 4: 内容农场/低质（0.2-0.4）
+    "sohu.com": 0.4, "163.com": 0.45, "sina.com.cn": 0.5,
+    "baijiahao.baidu.com": 0.35, "zhuanlan.zhihu.com": 0.7,
+    "toutiao.com": 0.4, "weixin.qq.com": 0.5,
+
+    # Tier 5: 已知低质（0.1-0.2）
+    "content-farm": 0.1, "seo-spam": 0.1,
+}
+
+# 来源类型映射
+SOURCE_TYPE_MAP = {
+    "eastmoney": ("金融数据", 0.9),
+    "zhihu": ("社区观点", 0.8),
+    "arxiv": ("学术预印本", 0.85),
+    "semantic_scholar": ("学术索引", 0.9),
+    "openalex": ("学术索引", 0.85),
+    "crossref": ("学术元数据", 0.9),
+    "github": ("代码仓库", 0.85),
+    "byted": ("中文搜索", 0.7),
+    "bocha": ("中文搜索", 0.7),
+    "tavily": ("AI搜索", 0.75),
+    "felo": ("AI搜索", 0.7),
+    "duckduckgo": ("通用搜索", 0.65),
+    "wikipedia": ("百科", 0.75),
+    "anysearch": ("垂直搜索", 0.75),
+    "wigolo": ("本地搜索", 0.7),
+    "metaso": ("AI搜索", 0.75),
+}
+
+
+def score_authority(url: str, source: str = "") -> dict[str, Any]:
+    """评估 URL 的权威性。"""
+    if not url:
+        return {"score": 0.3, "reason": "无 URL", "tier": "unknown"}
+
+    parsed = urlparse(url)
+    domain = parsed.netloc.lower().strip("www.")
+    path = parsed.path.lower()
+
+    # 精确匹配域名
+    best_score = 0.5
+    best_reason = "通用域名"
+
+    for pattern, score in AUTHORITY_TIERS.items():
+        if domain == pattern or domain.endswith("." + pattern):
+            if score > best_score:
+                best_score = score
+                best_reason = f"域名匹配：{pattern}"
+
+    # 路径特征加分
+    if "/docs/" in path or "/documentation/" in path:
+        best_score = min(best_score + 0.05, 1.0)
+        best_reason += "（文档路径）"
+    if "/paper/" in path or "/arxiv/" in path or "/abs/" in path:
+        best_score = min(best_score + 0.05, 1.0)
+        best_reason += "（论文路径）"
+    if "/issues/" in path or "/pull/" in path:
+        best_score = min(best_score + 0.03, 1.0)
+        best_reason += "（Issue/PR路径）"
+
+    # 来源类型加分
+    if source and source in SOURCE_TYPE_MAP:
+        type_name, type_score = SOURCE_TYPE_MAP[source]
+        if type_score > best_score:
+            best_score = type_score
+            best_reason = f"来源类型：{type_name}"
+
+    tier = "high" if best_score >= 0.8 else "medium" if best_score >= 0.6 else "low" if best_score >= 0.4 else "very_low"
+
+    return {
+        "score": round(best_score, 2),
+        "reason": best_reason,
+        "tier": tier,
+        "domain": domain,
+    }
+
+
+# ── 时效性评分 ────────────────────────────────────────────────────────────────
+
+def score_freshness(result: dict[str, Any], query_time: float = None) -> dict[str, Any]:
+    """评估结果的时效性。"""
+
+    def _extract_year_from_url(url: str):
+        """从 URL 路径提取年份。"""
+        patterns = [
+            r"/(20\d{2})[/-](\d{1,2})[/-](\d{1,2})",
+            r"/(20\d{2})[/-](\d{1,2})",
+            r"/(20\d{2})/",
+        ]
+        for p in patterns:
+            m = re.search(p, url)
+            if m:
+                return int(m.group(1))
+        return None
+
+    if query_time is None:
+        query_time = time.time()
+
+    snippet = result.get("snippet", "") or ""
+    title = result.get("title", "") or ""
+    url = result.get("url", "") or ""
+    combined = f"{title} {snippet}"
+
+    # 提取时间信息
+    # 年份
+    year_match = re.search(r"20(\d{2})", combined)
+    if year_match:
+        year = 2000 + int(year_match.group(1))
+        from datetime import datetime
+        age_years = datetime.now().year - year
+        if age_years < 0:
+            score = 0.4
+            reason = f"{year}年（未来年份）"
+        elif age_years == 0:
+            score = 0.8
+            reason = f"{year}年（今年）"
+        elif age_years == 1:
+            score = 0.9
+            reason = f"{year}年（去年）"
+        elif age_years <= 2:
+            score = 0.7
+            reason = f"{year}年（{age_years}年前）"
+        elif age_years <= 5:
+            score = 0.5
+            reason = f"{year}年（{age_years}年前）"
+        else:
+            score = 0.3
+            reason = f"{year}年（{age_years}年前，较旧）"
+    else:
+        # 无时间信息，尝试从 URL 提取
+        url_year = _extract_year_from_url(url)
+        if url_year:
+            from datetime import datetime
+            age_years = datetime.now().year - url_year
+            if age_years <= 1:
+                score = 0.8
+                reason = f"URL含{url_year}年"
+            elif age_years <= 3:
+                score = 0.6
+                reason = f"URL含{url_year}年（{age_years}年前）"
+            else:
+                score = 0.4
+                reason = f"URL含{url_year}年（较旧）"
+        else:
+            score = 0.5
+            reason = "无明确时间标记"
+
+    # 时效性关键词加分
+    if re.search(r"(最新|latest|recent|breaking|just|刚刚|今日|today)", combined, re.I):
+        score = min(score + 0.1, 1.0)
+        reason += "（含时效关键词）"
+
+    return {
+        "score": round(score, 2),
+        "reason": reason,
+    }
+
+
+# ── 交叉验证 ──────────────────────────────────────────────────────────────────
+
+def cross_validate(results: list[dict[str, Any]], query: str) -> dict[str, Any]:
+    """多来源交叉验证。"""
+    if len(results) < 2:
+        return {
+            "corroboration_level": "insufficient",
+            "score": 0.3,
+            "detail": f"仅 {len(results)} 个结果，无法交叉验证",
+            "agreement_count": 0,
+            "total_sources": len(results),
+        }
+
+    # 检查 URL 重叠（不同来源指向同一页面 = 强佐证）
+    urls = [r.get("url", "") for r in results if r.get("url")]
+    unique_urls = set(urls)
+    url_overlap = 1 - (len(unique_urls) / max(len(urls), 1))
+
+    # 检查域名多样性
+    domains = set()
+    for r in results:
+        url = r.get("url", "")
+        if url:
+            domain = urlparse(url).netloc.lower().strip("www.")
+            domains.add(domain)
+
+    # 检查标题/内容相似度（简单关键词匹配）
+    query_words = set(query.lower().split())
+    content_matches = 0
+    for r in results:
+        text = f"{r.get('title', '')} {r.get('snippet', '')}".lower()
+        if any(w in text for w in query_words if len(w) > 1):
+            content_matches += 1
+
+    match_ratio = content_matches / max(len(results), 1)
+
+    # 计算佐证等级
+    if match_ratio >= 0.8 and len(domains) >= 3:
+        level = "strong"
+        score = 0.9
+    elif match_ratio >= 0.6 and len(domains) >= 2:
+        level = "moderate"
+        score = 0.7
+    elif match_ratio >= 0.4:
+        level = "weak"
+        score = 0.5
+    else:
+        level = "minimal"
+        score = 0.3
+
+    return {
+        "corroboration_level": level,
+        "score": round(score, 2),
+        "detail": f"{content_matches}/{len(results)} 个结果与查询相关，{len(domains)} 个独立域名",
+        "agreement_count": content_matches,
+        "total_sources": len(results),
+        "unique_domains": len(domains),
+        "url_overlap": round(url_overlap, 2),
+    }
+
+
+# ── 综合可信度 ────────────────────────────────────────────────────────────────
+
+def compute_credibility(results: list[dict[str, Any]], query: str) -> dict[str, Any]:
+    """计算每个结果的综合可信度评分。"""
+    query_time = time.time()
+    scored_results = []
+
+    for r in results:
+        url = r.get("url", "")
+        source = r.get("source", "")
+
+        auth = score_authority(url, source)
+        fresh = score_freshness(r, query_time)
+
+        # 综合公式：权威性 0.5 + 时效性 0.3 + 原始分数 0.2
+        original_score = r.get("score", 0.5) or 0.5
+        credibility = (
+            auth["score"] * 0.5 +
+            fresh["score"] * 0.3 +
+            original_score * 0.2
+        )
+
+        scored_results.append({
+            "title": r.get("title", ""),
+            "url": url,
+            "source": source,
+            "snippet": (r.get("snippet", "") or "")[:150],
+            "credibility": {
+                "final": round(credibility, 3),
+                "authority": auth,
+                "freshness": fresh,
+                "original_score": original_score,
+            },
+        })
+
+    # 交叉验证
+    cross = cross_validate(results, query)
+
+    # 排序
+    scored_results.sort(key=lambda x: x["credibility"]["final"], reverse=True)
+
+    return {
+        "query": query,
+        "results": scored_results,
+        "cross_validation": cross,
+        "summary": {
+            "total": len(scored_results),
+            "high_credibility": sum(1 for r in scored_results if r["credibility"]["final"] >= 0.7),
+            "medium_credibility": sum(1 for r in scored_results if 0.5 <= r["credibility"]["final"] < 0.7),
+            "low_credibility": sum(1 for r in scored_results if r["credibility"]["final"] < 0.5),
+            "best_source": scored_results[0] if scored_results else None,
+        },
+    }
+
+
+# ── CLI ───────────────────────────────────────────────────────────────────────
+
+def main():
+    parser = argparse.ArgumentParser(description="来源可信度评估工具")
+    parser.add_argument("query", nargs="?", default="", help="搜索查询词")
+    parser.add_argument("--stdin", action="store_true", help="从 stdin 读取 JSON 搜索结果")
+    parser.add_argument("--json", action="store_true", help="JSON 输出")
+    args = parser.parse_args()
+
+    if args.stdin:
+        data = json.load(sys.stdin)
+        results = data.get("results", [])
+    else:
+        # 从搜索结果文件读取
+        results = []
+
+    if not results and args.query:
+        # 无输入结果时，输出提示
+        print("需要提供搜索结果进行评估。用法：")
+        print('  echo \'{"results": [...]}\' | python3 evidence.py --stdin --query "查询词"')
+        sys.exit(1)
+
+    report = compute_credibility(results, args.query)
+
+    if args.json:
+        print(json.dumps(report, ensure_ascii=False, indent=2))
+    else:
+        print(f"\n来源可信度评估：{report['query']}")
+        print(f"{'='*50}")
+        for r in report["results"]:
+            c = r["credibility"]
+            level = "🟢" if c["final"] >= 0.7 else "🟡" if c["final"] >= 0.5 else "🔴"
+            print(f"{level} [{c['final']:.2f}] {r['title'][:50]}")
+            print(f"   权威：{c['authority']['score']:.2f} ({c['authority']['reason']})")
+            print(f"   时效：{c['freshness']['score']:.2f} ({c['freshness']['reason']})")
+            print()
+
+        cv = report["cross_validation"]
+        print(f"交叉验证：{cv['corroboration_level']} (score={cv['score']:.2f})")
+        print(f"  {cv['detail']}")
+
+        s = report["summary"]
+        print(f"\n总结：🟢高={s['high_credibility']} 🟡中={s['medium_credibility']} 🔴低={s['low_credibility']}")
+
+
+if __name__ == "__main__":
+    main()

package/scripts/extract.py

@@ -0,0 +1,69 @@
+#!/usr/bin/env python3
+"""extract.py — 结构化数据提取"""
+import json, re, sys
+from html.parser import HTMLParser
+from fetch import fetch_page
+
+class TableExtractor(HTMLParser):
+    def __init__(self):
+        super().__init__()
+        self.tables = []
+        self._in_table = False
+        self._rows = []
+        self._row = []
+        self._cell = []
+        self._in_cell = False
+    def handle_starttag(self, tag, attrs):
+        if tag == 'table': self._in_table = True; self._rows = []
+        elif tag in ('td','th') and self._in_table: self._in_cell = True; self._cell = []
+        elif tag == 'tr' and self._in_table: self._row = []
+    def handle_endtag(self, tag):
+        if tag in ('td','th') and self._in_cell:
+            self._in_cell = False; self._row.append(''.join(self._cell).strip())
+        elif tag == 'tr' and self._in_table and self._row:
+            self._rows.append(self._row)
+        elif tag == 'table' and self._in_table:
+            self._in_table = False
+            if self._rows:
+                headers = self._rows[0]
+                table = [{headers[i]: row[i] if i < len(row) else '' for i in range(len(headers))} for row in self._rows[1:]]
+                self.tables.append(table)
+    def handle_data(self, data):
+        if self._in_cell: self._cell.append(data)
+
+def extract_tables(html):
+    ext = TableExtractor(); ext.feed(html); return ext.tables
+
+def extract_metadata(html):
+    meta = {}
+    title_m = re.search(r'<title>(.*?)</title>', html, re.I|re.S)
+    if title_m: meta['title'] = title_m.group(1).strip()
+    for pattern in [r'<meta\s+name=["\']description["\']\s+content=["\'](.*?)["\']', r'<meta\s+content=["\'](.*?)["\']\s+name=["\']description["\']']:
+        m = re.search(pattern, html, re.I)
+        if m: meta['description'] = m.group(1).strip(); break
+    for prop in ['og:title','og:description','og:image']:
+        m = re.search(rf'property=["\']{prop}["\']\s+content=["\'](.*?)["\']', html, re.I)
+        if m: meta[prop] = m.group(1).strip()
+    return meta
+
+def extract_jsonld(html):
+    results = []
+    for m in re.finditer(r'<script[^>]*type=["\']application/ld\+json["\'][^>]*>(.*?)</script>', html, re.S|re.I):
+        try: results.append(json.loads(m.group(1)))
+        except: pass
+    return results
+
+if __name__ == '__main__':
+    import argparse
+    p = argparse.ArgumentParser()
+    p.add_argument('--url', required=True)
+    p.add_argument('--mode', default='all', choices=['tables','metadata','jsonld','all'])
+    args = p.parse_args()
+    result = fetch_page(args.url, 50000)
+    if not result['success']: print(json.dumps({'error': result.get('error')})); sys.exit(1)
+    html = result['content']
+    output = {}
+    if args.mode in ('tables','all'): output['tables'] = extract_tables(html)
+    if args.mode in ('metadata','all'): output['metadata'] = extract_metadata(html)
+    if args.mode in ('jsonld','all'): output['jsonld'] = extract_jsonld(html)
+    print(json.dumps(output, ensure_ascii=False, indent=2)[:3000])

package/scripts/fetch.py

@@ -0,0 +1,118 @@
+#!/usr/bin/env python3
+"""
+fetch.py — 轻量页面抓取（纯标准库）
+
+从 URL 抓取页面并提取正文文本，用于 research 深度研究。
+"""
+
+from __future__ import annotations
+
+import re
+import urllib.request
+from html.parser import HTMLParser
+from typing import Any
+from urllib.parse import urlparse
+
+
+class ContentExtractor(HTMLParser):
+    """从 HTML 中提取正文文本。"""
+
+    _skip_tags = {"script", "style", "nav", "header", "footer", "aside", "noscript"}
+
+    def __init__(self):
+        super().__init__()
+        self._in_skip = 0
+        self._blocks: list[tuple[float, str]] = []
+        self._current_text: list[str] = []
+
+    def handle_starttag(self, tag, attrs):
+        if tag in self._skip_tags:
+            self._in_skip += 1
+
+    def handle_endtag(self, tag):
+        if tag in self._skip_tags:
+            self._in_skip = max(0, self._in_skip - 1)
+        if tag in ("p", "div", "article", "section", "li", "h1", "h2", "h3", "td"):
+            text = "".join(self._current_text).strip()
+            if len(text) > 20:
+                # 文本密度 = 非空字符 / 总字符
+                density = len(text.replace(" ", "")) / max(len(text), 1)
+                self._blocks.append((density, text))
+            self._current_text = []
+
+    def handle_data(self, data):
+        if self._in_skip == 0:
+            self._current_text.append(data)
+
+
+def fetch_page(url: str, max_chars: int = 3000, timeout: int = 8,
+              raw: bool = False) -> dict[str, Any]:
+    """抓取页面并提取正文。
+
+    Args:
+        url: 目标 URL
+        max_chars: 最大返回字符数
+        timeout: 超时秒数
+        raw: True 时返回原始 HTML（用于 extract/crawl_sitemap 等需要 HTML 的场景）
+    """
+    try:
+        req = urllib.request.Request(url, headers={
+            "User-Agent": "unified-search/2.5 (+local-research)",
+            "Accept": "text/html,application/xhtml+xml,application/xml",
+        })
+        with urllib.request.urlopen(req, timeout=timeout) as resp:
+            content_type = resp.headers.get("Content-Type", "")
+            if "pdf" in content_type:
+                return {"url": url, "content": "", "html": "", "length": 0, "success": False, "error": "PDF not supported"}
+            raw_bytes = resp.read(max(max_chars * 5, 500000))
+            # 编码检测
+            charset = "utf-8"
+            if "charset=" in content_type:
+                charset = content_type.split("charset=")[-1].strip().split(";")[0]
+            html = raw_bytes.decode(charset, errors="replace")
+
+        # 正文提取
+        extractor = ContentExtractor()
+        extractor.feed(html)
+        extractor._blocks.sort(key=lambda x: x[0], reverse=True)
+        content = "\n\n".join(text for _, text in extractor._blocks[:8])
+
+        result = {
+            "url": url,
+            "content": content[:max_chars],
+            "length": len(content),
+            "success": True,
+        }
+        if raw:
+            result["html"] = html[:max(max_chars * 2, 100000)]
+        return result
+    except Exception as e:
+        return {"url": url, "content": "", "html": "", "length": 0, "success": False, "error": str(e)[:100]}
+
+
+def fetch_pages_parallel(urls: list[str], max_chars: int = 3000,
+                         timeout: int = 8, max_workers: int = 3) -> list[dict[str, Any]]:
+    """并行抓取多个页面。"""
+    from concurrent.futures import ThreadPoolExecutor, as_completed
+
+    results = []
+    with ThreadPoolExecutor(max_workers=min(len(urls), max_workers)) as ex:
+        futures = {ex.submit(fetch_page, url, max_chars, timeout): url for url in urls}
+        for fut in as_completed(futures, timeout=timeout + 3):
+            try:
+                results.append(fut.result())
+            except Exception:
+                results.append({"url": futures[fut], "content": "", "length": 0, "success": False})
+    return results
+
+
+# CLI 测试
+if __name__ == "__main__":
+    import sys
+    url = sys.argv[1] if len(sys.argv) > 1 else "https://docs.python.org/3/"
+    result = fetch_page(url, max_chars=500)
+    print(f"成功: {result['success']}, 长度: {result['length']}")
+    if result["success"]:
+        print(result["content"][:300])
+    else:
+        print(f"错误: {result.get('error', 'unknown')}")