argo-search 1.0.1

package/scripts/tfidf_router.py

@@ -0,0 +1,312 @@
+#!/usr/bin/env python3
+"""
+tfidf_router.py — Unified Search v2 TF-IDF 语义路由引擎
+
+原理：
+  1. 加载各引擎的领域文档，构建语料库
+  2. 对每个引擎计算 TF-IDF 向量（代表该领域的"语义中心"）
+  3. 新查询来了，向量化后与各引擎算余弦相似度
+  4. 结合 boost_keywords / boost_combos / 配额状态打分
+
+增强（v2）：
+  - 成本感知：free=1.0 / low=0.85 / paid=0.6
+  - 延迟感知：基于 adaptive.py 历史数据微调
+  - 返回带 reason 的决策说明
+
+所有计算纯本地，零 API 开销，典型延迟 <5ms。
+"""
+
+from __future__ import annotations
+
+import json
+import math
+import re
+from collections import Counter
+from pathlib import Path
+from typing import Optional
+
+# ── 路径 ──────────────────────────────────────────────────────────────────────
+
+SKILL_DIR = Path(__file__).parent.parent
+BACKENDS_DIR = SKILL_DIR / "backends"
+DOMAIN_PROFILES_PATH = BACKENDS_DIR / "domain_profiles.json"
+
+# ── 分词 ──────────────────────────────────────────────────────────────────────
+
+_TOKEN_PATTERN = re.compile(r"[一-鿿]|[a-zA-Z0-9]+")
+
+
+def tokenize(text: str) -> list[str]:
+    """简单分词：中文单字 + 英文单词，统一小写。"""
+    tokens = []
+    for match in _TOKEN_PATTERN.finditer(text.lower()):
+        token = match.group()
+        if len(token) == 1 and '一' <= token <= '鿿':
+            tokens.append(token)
+        elif len(token) > 1:
+            tokens.append(token)
+    return tokens
+
+
+def tokenize_with_bigrams(text: str) -> list[str]:
+    """分词 + 二元组，捕捉短语信息。"""
+    unigrams = tokenize(text)
+    bigrams = [f"{unigrams[i]}_{unigrams[i+1]}" for i in range(len(unigrams) - 1)]
+    return unigrams + bigrams
+
+
+# ── TF-IDF 计算 ────────────────────────────────────────────────────────────────
+
+class TfidfVectorizer:
+    """轻量 TF-IDF 向量化器，不需要 sklearn。"""
+
+    def __init__(self):
+        self.idf: dict[str, float] = {}
+        self.vocab: set[str] = set()
+
+    def fit(self, documents: list[str]) -> None:
+        """从文档集合计算 IDF。"""
+        df: Counter = Counter()
+        total = len(documents)
+        for doc in documents:
+            tokens = set(tokenize_with_bigrams(doc))
+            self.vocab.update(tokens)
+            for token in tokens:
+                df[token] += 1
+        for token, count in df.items():
+            self.idf[token] = math.log((total + 1) / (count + 1)) + 1
+
+    def transform(self, text: str) -> dict[str, float]:
+        """把文本转为 TF-IDF 向量（稀疏表示）。"""
+        tokens = tokenize_with_bigrams(text)
+        tf = Counter(tokens)
+        total = len(tokens) if tokens else 1
+        return {token: (count / total) * self.idf[token]
+                for token, count in tf.items() if token in self.idf}
+
+
+# ── 余弦相似度 ─────────────────────────────────────────────────────────────────
+
+def cosine_similarity(vec_a: dict[str, float], vec_b: dict[str, float]) -> float:
+    """两个稀疏向量的余弦相似度。"""
+    if not vec_a or not vec_b:
+        return 0.0
+    common = set(vec_a.keys()) & set(vec_b.keys())
+    dot = sum(vec_a[k] * vec_b[k] for k in common)
+    norm_a = math.sqrt(sum(v * v for v in vec_a.values()))
+    norm_b = math.sqrt(sum(v * v for v in vec_b.values()))
+    if norm_a == 0 or norm_b == 0:
+        return 0.0
+    return dot / (norm_a * norm_b)
+
+
+# ── 配额感知 ──────────────────────────────────────────────────────────────────
+
+def _load_quota_state() -> dict:
+    """加载配额状态文件。"""
+    quota_path = Path.home() / ".cache" / "unified-search" / "quota.json"
+    if not quota_path.exists():
+        return {}
+    try:
+        return json.loads(quota_path.read_text())
+    except (json.JSONDecodeError, OSError):
+        return {}
+
+
+def _quota_ratio(engine: str, quota_state: dict) -> float:
+    """计算配额剩余比例。返回 0.0-1.0，无记录时默认 1.0。"""
+    info = quota_state.get(engine)
+    if not info:
+        return 1.0
+    limit = info.get("limit", 100)
+    used = info.get("used", 0)
+    return max(0, limit - used) / limit
+
+
+# ── 成本感知 ──────────────────────────────────────────────────────────────────
+
+def _cost_factor(engine: str) -> float:
+    """获取引擎成本因子：free=1.0, low=0.85, paid=0.6。"""
+    tiers_path = BACKENDS_DIR / "quota_profiles.json"
+    if not tiers_path.exists():
+        return 1.0
+    try:
+        profiles = json.loads(tiers_path.read_text())
+    except (json.JSONDecodeError, OSError):
+        return 1.0
+    profile = profiles.get(engine, {})
+    tier = profile.get("cost_tier", "free")
+    return {"free": 1.0, "low": 0.85, "paid": 0.6}.get(tier, 1.0)
+
+
+# ── 主路由引擎 ─────────────────────────────────────────────────────────────────
+
+class SemanticRouter:
+    """TF-IDF 语义路由引擎（v2 增强版）。"""
+
+    def __init__(self):
+        self.vectorizer = TfidfVectorizer()
+        self.engine_names: list[str] = []
+        self.engine_vectors: dict[str, dict[str, float]] = {}
+        self.boost_keywords: dict[str, dict[str, float]] = {}
+        self.boost_combos: dict[str, dict[str, float]] = {}
+        self._loaded = False
+
+    def _ensure_loaded(self) -> None:
+        """懒加载领域文档，只加载一次。"""
+        if self._loaded:
+            return
+        if not DOMAIN_PROFILES_PATH.exists():
+            self._loaded = True
+            return
+        try:
+            profiles = json.loads(DOMAIN_PROFILES_PATH.read_text())
+        except (json.JSONDecodeError, OSError):
+            self._loaded = True
+            return
+
+        corpus = []
+        for name, profile in profiles.items():
+            if name.startswith("_"):
+                continue
+            self.engine_names.append(name)
+            self.boost_keywords[name] = profile.get("boost_keywords", {})
+            self.boost_combos[name] = profile.get("boost_combos", {})
+            corpus.append(" ".join(profile.get("documents", [])))
+
+        if corpus:
+            self.vectorizer.fit(corpus)
+            for i, name in enumerate(self.engine_names):
+                self.engine_vectors[name] = self.vectorizer.transform(corpus[i])
+        self._loaded = True
+
+    def route(self, query: str, top_k: int = 3,
+              quota_aware: bool = True) -> list[tuple[str, float, str]]:
+        """
+        路由查询到最匹配的引擎。
+
+        返回: [(engine_name, score, reason), ...] 按分数降序。
+        """
+        self._ensure_loaded()
+        if not self.engine_names:
+            return [("anysearch", 0.0, "无领域文档，回退 AnySearch")]
+
+        query_vec = self.vectorizer.transform(query)
+        scores = []
+        quota_state = _load_quota_state() if quota_aware else {}
+
+        for name in self.engine_names:
+            sim = cosine_similarity(query_vec, self.engine_vectors[name])
+            parts = []
+
+            # boost 加权
+            boost = 1.0
+            hit_kws = []
+            for kw, weight in self.boost_keywords.get(name, {}).items():
+                if kw.lower() in query.lower():
+                    boost += weight - 1.0
+                    hit_kws.append(kw)
+            if hit_kws:
+                parts.append(f"boost关键词:{','.join(hit_kws)}")
+
+            # 组合关键词加成
+            hit_combos = []
+            for combo, bonus in self.boost_combos.get(name, {}).items():
+                if all(w.lower() in query.lower() for w in combo.split()):
+                    boost += bonus
+                    hit_combos.append(combo)
+            if hit_combos:
+                parts.append(f"combo:{','.join(hit_combos)}")
+
+            final_score = sim * boost
+
+            # 成本感知
+            cf = _cost_factor(name)
+            if cf < 1.0:
+                final_score *= cf
+                parts.append(f"cost={cf}")
+
+            # 配额感知惩罚
+            if quota_aware and quota_state:
+                qr = _quota_ratio(name, quota_state)
+                if qr < 0.2:
+                    final_score *= qr * 2
+                    parts.append(f"配额紧张({qr:.0%})")
+                elif qr < 0.5:
+                    final_score *= 0.5 + qr
+                    parts.append(f"配额偏低({qr:.0%})")
+
+            reason = f"sim={sim:.3f}" + (f", {', '.join(parts)}" if parts else "")
+            scores.append((name, round(final_score, 4), reason))
+
+        scores.sort(key=lambda x: -x[1])
+        return scores[:top_k]
+
+    def should_parallel(self, query: str, scores: list[tuple[str, float, str]]) -> bool:
+        """判断是否需要多路并行搜索。"""
+        research_signals = [
+            "分析", "研究", "格局", "趋势", "深度", "全面", "详细",
+            "research", "analysis", "comprehensive", "deep",
+            "对比", "评测", "推荐", "review", "comparison"
+        ]
+        if any(kw in query.lower() for kw in research_signals):
+            return True
+        if not scores or scores[0][1] < 0.15:
+            return True
+        if len(scores) >= 2 and scores[0][1] > 0:
+            gap = (scores[0][1] - scores[1][1]) / scores[0][1]
+            if gap < 0.2:
+                return True
+        return False
+
+
+# ── 模块级单例 ─────────────────────────────────────────────────────────────────
+
+_router: Optional[SemanticRouter] = None
+
+
+def get_router() -> SemanticRouter:
+    global _router
+    if _router is None:
+        _router = SemanticRouter()
+    return _router
+
+
+def semantic_route(query: str, top_k: int = 3) -> list[tuple[str, float, str]]:
+    """便捷函数：语义路由。"""
+    return get_router().route(query, top_k=top_k)
+
+
+def semantic_route_auto(query: str) -> str:
+    """便捷函数：返回最优引擎名。"""
+    scores = semantic_route(query, top_k=1)
+    return scores[0][0] if scores else "anysearch"
+
+
+# ── CLI 测试 ──────────────────────────────────────────────────────────────────
+
+if __name__ == "__main__":
+    import sys
+
+    test_queries = sys.argv[1:] if len(sys.argv) > 1 else [
+        "英伟达最新财报",
+        "Python 异步编程最佳实践",
+        "2026年AI芯片行业竞争格局",
+        "美联储加息对股市影响",
+        "笔记本电脑推荐 2026",
+        "小米汽车销量",
+        "latest AI research papers",
+        "transformer attention mechanism paper",
+        "北京旅游攻略",
+        "基金定投策略",
+    ]
+
+    router = get_router()
+    print(f"{'查询':<35} {'最优引擎':<12} {'分数':<8} {'Top-3':<35}")
+    print("-" * 100)
+
+    for q in test_queries:
+        scores = router.route(q, top_k=3)
+        best = scores[0] if scores else ("?", 0, "")
+        top3 = ", ".join(f"{n}:{s:.3f}" for n, s, _ in scores[:3])
+        print(f"{q:<35} {best[0]:<12} {best[1]:<8.4f} {top3:<35}")

package/sub-skills/local-search/SKILL.md

@@ -0,0 +1,104 @@
+---
+name: local-search
+parent: unified-search
+description: unified-search 的本地/零成本兜底子技能。封装基于公开页面/HTML/RSS/JSON 的 24 个本地搜索引擎，不单独响应触发词，仅由 unified-search 通过 --sub-skill local-search 或 --local-first 调用。
+version: 1.0.1
+---
+
+## Local Search 子技能
+
+Local Search 是 unified-search 的「零成本兜底适配器」，用于：
+
+- 在 `--mode fast` / `--mode budget` 下优先使用本地抓取引擎，避免消耗付费 API 配额。
+- 当 SearXNG 不可用时，回退到本地 HTML/JSON 解析。
+- 对中文网页、新闻、代码问答、学术、参考百科等垂直域提供补充结果。
+
+### 设计原则
+
+- **不单独响应触发词**：没有独立的 skill trigger，仅作为 unified-search 的子能力。
+- **统一 schema**：输出与 unified-search 主 skill 完全一致，包含 `results[]`、`engines_used`、`errors`、`elapsed_ms` 等字段。
+- **声明式解析**：HTML 结构变化时只需修改 `parse_maps.yaml`。
+- **命名空间隔离**：本地引擎统一使用 `local_` 前缀（如 `local_bing`、`local_google`），避免与 unified-search 已有的 HTTP 引擎（`duckduckgo`、`wikipedia` 等）重名。
+
+### 本地引擎列表（25 个，20 个默认启用）
+
+| unified 名称 | 类型 | 默认启用 | 说明 |
+|--------------|------|----------|------|
+| local_bing | html | ✅ | Bing 网页结果 |
+| local_google | html | ❌ | Google 网页结果（反爬强） |
+| local_mojeek | html | ✅ | Mojeek 独立索引 |
+| local_yandex | html | ❌ | Yandex 搜索（反爬强） |
+| local_startpage | html | ✅ | Startpage 隐私搜索 |
+| local_duckduckgo | html | ✅ | DuckDuckGo HTML |
+| local_baidu | html | ✅ | 百度搜索 |
+| local_sogou | html | ✅ | 搜狗搜索 |
+| local_arxiv | xml | ✅ | arXiv API |
+| local_pubmed | json | ✅ | PubMed/EUtils |
+| local_crossref | json | ✅ | Crossref API |
+| local_semantic_scholar | json | ✅ | Semantic Scholar API |
+| local_bing_news | rss/html | ✅ | Bing 新闻 |
+| local_google_news | rss | ✅ | Google News RSS |
+| local_duckduckgo_news | rss/html | ❌ | DuckDuckGo 新闻 |
+| local_github | json | ✅ | GitHub Search API |
+| local_stackoverflow | html/json | ✅ | StackOverflow 问题 |
+| local_gitlab | json | ✅ | GitLab API |
+| local_npm | json | ✅ | NPM Registry |
+| local_wikipedia | json | ✅ | MediaWiki API |
+| local_wiktionary | json | ✅ | Wiktionary API |
+| local_wikiquote | json | ✅ | Wikiquote API |
+| local_imdb | html | ❌ | IMDb 搜索 |
+| local_goodreads | html | ❌ | Goodreads 搜索 |
+| local_openstreetmap | json | ✅ | Nominatim API |
+
+### 调用方式
+
+```bash
+# 直接调用子技能（单引擎）
+python3 sub-skills/local-search/local_search_adapter.py "query" --engine local_bing
+
+# 批量调用多个本地引擎
+python3 sub-skills/local-search/local_search_adapter.py "query" \
+  --engine local_bing,local_baidu,local_duckduckgo
+
+# 由 unified-search 调用
+python3 scripts/search.py "query" --sub-skill local-search
+python3 scripts/search.py "query" --local-first --mode fast
+```
+
+### 文件结构
+
+```
+sub-skills/local-search/
+├── SKILL.md                 # 本文件
+├── config.yaml              # 引擎基础配置（URL/超时/类型/开关）
+├── parse_maps.yaml          # HTML/RSS/JSON 抽取映射
+├── engine_registry.py       # 引擎注册中心（唯一真源）
+├── health_check.py          # 轻量健康探针
+├── smart_router.py          # 查询特征路由
+├── search_v3.py             # local-search 主入口
+└── local_search_adapter.py  # 兼容入口
+```
+
+### 输出 schema
+
+与 unified-search 主 skill 一致：
+
+```json
+{
+  "query": "string",
+  "engine": "local_search",
+  "engines": ["local_bing", "local_baidu"],
+  "engines_combo": ["local_bing", "local_baidu"],
+  "cached": false,
+  "cache_level": null,
+  "domain": null,
+  "elapsed_ms": 1234,
+  "tfidf_scores": [],
+  "results": [
+    {"title": "...", "url": "...", "snippet": "...", "score": 0.8, "source": "local_bing"}
+  ],
+  "count": 10,
+  "engines_used": ["local_bing", "local_baidu"],
+  "errors": []
+}
+```