foxclaw 0.1.0 → 0.1.3

package/python/vendor/hkc/hkc_sdk/http_client.py

@@ -0,0 +1,166 @@
+"""
+hkc_sdk / http_client.py
+HTTP 客户端：通过 REST 调用 hkc-api。
+
+用于 HKC 作为独立服务部署、调用方在另一进程/机器的场景。
+"""
+from __future__ import annotations
+import logging
+
+from .base import BaseClient
+from .models import (
+    KU, SearchHit, Ability, CoverageReport, Conflict, IngestResult,
+)
+from .exceptions import (
+    HKCNotFoundError, HKCInsufficientCoverage,
+    HKCBadRequest, HKCServerError,
+)
+
+logger = logging.getLogger(__name__)
+
+
+class HTTPClient(BaseClient):
+
+    def __init__(self, base_url: str = "http://localhost:8000", timeout: int = 60):
+        self._base_url = base_url.rstrip("/")
+        self._timeout  = timeout
+
+    # ── 内部请求封装 ─────────────────────────────────────────
+
+    def _request(self, method: str, path: str, **kwargs):
+        try:
+            import requests
+        except ImportError:
+            raise ImportError("requests 未安装: pip install requests")
+
+        url = f"{self._base_url}{path}"
+        try:
+            resp = requests.request(
+                method, url, timeout=self._timeout, **kwargs
+            )
+        except requests.RequestException as e:
+            raise HKCServerError(f"连接失败: {e}")
+
+        return self._handle_response(resp)
+
+    def _handle_response(self, resp):
+        # 解析错误 detail（可能是 dict 或字符串）
+        if resp.status_code == 200:
+            return resp.json()
+
+        try:
+            detail = resp.json().get("detail")
+        except Exception:
+            detail = resp.text
+
+        if resp.status_code == 404:
+            raise HKCNotFoundError(str(detail), detail)
+        if resp.status_code == 422:
+            msg = detail.get("message") if isinstance(detail, dict) else str(detail)
+            raise HKCInsufficientCoverage(msg or "覆盖度不足", detail)
+        if resp.status_code in (400, 413):
+            raise HKCBadRequest(str(detail), detail)
+        raise HKCServerError(f"HTTP {resp.status_code}: {detail}", detail)
+
+    # ── 知识摄入 ─────────────────────────────────────────────
+
+    def ingest_text(self, text, source_title="", domain="",
+                    source="inline", source_year=0) -> IngestResult:
+        data = self._request("POST", "/knowledge/ingest/text", json={
+            "text": text, "source": source, "source_title": source_title,
+            "source_year": source_year, "domain": domain,
+        })
+        return IngestResult.from_dict(data)
+
+    def ingest_url(self, url, source_title="", domain="", source_year=0) -> IngestResult:
+        data = self._request("POST", "/knowledge/ingest/url", json={
+            "url": url, "source_title": source_title,
+            "source_year": source_year, "domain": domain,
+        })
+        return IngestResult.from_dict(data)
+
+    # ── KU 查询 ──────────────────────────────────────────────
+
+    def get_ku(self, ku_id) -> KU:
+        data = self._request("GET", f"/knowledge/ku/{ku_id}")
+        return KU.from_dict(data)
+
+    def list_by_domain(self, domain, limit=50) -> list[KU]:
+        data = self._request("GET", f"/knowledge/domain/{domain}",
+                            params={"limit": limit})
+        return [KU.from_dict(k) for k in data.get("kus", [])]
+
+    def neighbors(self, ku_id, max_depth=2, top_k=20) -> list[SearchHit]:
+        data = self._request("GET", f"/knowledge/ku/{ku_id}/neighbors",
+                            params={"max_depth": max_depth, "top_k": top_k})
+        hits = []
+        for h in data.get("neighbors", []):
+            h.setdefault("mode", "graph")   # 与 DirectClient 一致
+            hits.append(SearchHit.from_dict(h))
+        return hits
+
+    # ── 搜索 ─────────────────────────────────────────────────
+
+    def search(self, query, mode="hybrid", top_k=10, domain="",
+               ku_types=None) -> list[SearchHit]:
+        data = self._request("POST", "/search", json={
+            "query": query, "mode": mode, "top_k": top_k,
+            "domain": domain, "ku_types": ku_types,
+        })
+        return [SearchHit.from_dict(h) for h in data.get("hits", [])]
+
+    def find_path(self, from_id, to_id) -> list[str]:
+        data = self._request("GET", "/search/path",
+                            params={"from_id": from_id, "to_id": to_id})
+        return data.get("path", [])
+
+    # ── 能力 ─────────────────────────────────────────────────
+
+    def list_abilities(self) -> list[str]:
+        data = self._request("GET", "/abilities")
+        return data.get("abilities", [])
+
+    def coverage_report(self, ability_key) -> CoverageReport:
+        data = self._request("GET", f"/abilities/{ability_key}/coverage")
+        return CoverageReport.from_dict(data)
+
+    def compile_ability(self, ability_key) -> Ability:
+        data = self._request("POST", f"/abilities/{ability_key}/compile")
+        return Ability.from_dict(data)
+
+    def get_ability(self, ability_key) -> Ability:
+        data = self._request("GET", f"/abilities/{ability_key}")
+        return Ability.from_dict(data)
+
+    # ── 冲突 ─────────────────────────────────────────────────
+
+    def list_conflicts(self, status="open", domain="") -> list[Conflict]:
+        data = self._request("GET", "/conflicts",
+                            params={"status": status, "domain": domain})
+        return [Conflict.from_dict(c) for c in data.get("conflicts", [])]
+
+    def resolve_conflict(self, conflict_id, winner_id, note="",
+                         resolved_by="sdk") -> bool:
+        # 裁决失败（冲突不存在/已解决/winner 无效）API 返回 400，
+        # SDK 统一为返回 False（与 DirectClient 行为一致），而非抛异常。
+        from .exceptions import HKCBadRequest
+        try:
+            data = self._request("POST", f"/conflicts/{conflict_id}/resolve", json={
+                "winner_id": winner_id, "note": note, "resolved_by": resolved_by,
+            })
+            return data.get("resolved", False)
+        except HKCBadRequest:
+            return False
+
+    # ── 系统 ─────────────────────────────────────────────────
+
+    def stats(self) -> dict:
+        return self._request("GET", "/stats")
+
+    def health(self) -> bool:
+        from .exceptions import HKCError
+        try:
+            self._request("GET", "/health")
+            return True
+        except HKCError:
+            return False

package/python/vendor/hkc/hkc_sdk/models.py

@@ -0,0 +1,171 @@
+"""
+hkc_sdk / models.py
+SDK 对外数据模型。
+
+这是给 SDK 用户的"门面"类型，刻意与内部 KU dataclass 解耦：
+- 内部 KU 字段多、有枚举、随版本演进
+- SDK 模型字段少而稳定，用户代码不会因内部重构而崩
+
+所有模型都能从 dict（HTTP JSON / 直连 to_dict）构造，用 from_dict。
+"""
+from __future__ import annotations
+from dataclasses import dataclass, field
+from typing import Any
+
+
+@dataclass
+class KU:
+    """知识单元的对外视图。"""
+    ku_id:      str
+    ku_type:    str
+    name:       str
+    summary:    str = ""
+    domain:     str = ""
+    confidence: float = 1.0
+    status:     str = "active"
+    tags:       list[str] = field(default_factory=list)
+    extra:      dict[str, Any] = field(default_factory=dict)
+
+    @classmethod
+    def from_dict(cls, d: dict) -> "KU":
+        return cls(
+            ku_id      = d["ku_id"],
+            ku_type    = d["ku_type"],
+            name       = d["name"],
+            summary    = d.get("summary", ""),
+            domain     = d.get("domain", ""),
+            confidence = d.get("confidence", 1.0),
+            status     = d.get("status", "active"),
+            tags       = d.get("tags", []),
+            extra      = d.get("extra", {}),
+        )
+
+    @property
+    def statement(self) -> str:
+        """Claim / Fact 的陈述内容（如果有）。"""
+        return self.extra.get("statement", "")
+
+
+@dataclass
+class SearchHit:
+    ku_id:   str
+    score:   float
+    name:    str
+    summary: str = ""
+    mode:    str = ""
+    ku_type: str = ""
+
+    @classmethod
+    def from_dict(cls, d: dict) -> "SearchHit":
+        return cls(
+            ku_id   = d["ku_id"],
+            score   = d.get("score", 0.0),
+            name    = d.get("name", ""),
+            summary = d.get("summary", ""),
+            mode    = d.get("mode", ""),
+            ku_type = d.get("ku_type", ""),
+        )
+
+
+@dataclass
+class Skill:
+    skill_key:    str
+    display_name: str
+    coverage:     float = 0.0
+    concept_hits: list[str] = field(default_factory=list)
+
+    @classmethod
+    def from_dict(cls, d: dict) -> "Skill":
+        return cls(
+            skill_key    = d.get("skill_key", ""),
+            display_name = d.get("display_name", ""),
+            coverage     = d.get("coverage", 0.0),
+            concept_hits = d.get("concept_hits", []),
+        )
+
+
+@dataclass
+class Ability:
+    ability_key:  str
+    display_name: str
+    domain:       str
+    coverage:     dict[str, float] = field(default_factory=dict)
+    skills:       list[Skill] = field(default_factory=list)
+    workflows:    list[dict] = field(default_factory=list)
+    version:      str = "1.0.0"
+
+    @classmethod
+    def from_dict(cls, d: dict) -> "Ability":
+        return cls(
+            ability_key  = d["ability_key"],
+            display_name = d.get("display_name", ""),
+            domain       = d.get("domain", ""),
+            coverage     = d.get("coverage", {}),
+            skills       = [Skill.from_dict(s) for s in d.get("skills", [])],
+            workflows    = d.get("workflows", []),
+            version      = d.get("version", "1.0.0"),
+        )
+
+    def skill_context(self, skill_key: str) -> Skill | None:
+        """获取某个 Skill 的上下文，Agent 用。"""
+        for s in self.skills:
+            if s.skill_key == skill_key:
+                return s
+        return None
+
+
+@dataclass
+class CoverageReport:
+    ability_key:    str
+    can_compile:    bool
+    coverage:       dict[str, float] = field(default_factory=dict)
+    missing_skills: list[str] = field(default_factory=list)
+    ku_count:       int = 0
+
+    @classmethod
+    def from_dict(cls, d: dict) -> "CoverageReport":
+        return cls(
+            ability_key    = d["ability_key"],
+            can_compile    = d.get("can_compile", False),
+            coverage       = d.get("coverage", {}),
+            missing_skills = d.get("missing_skills", []),
+            ku_count       = d.get("ku_count", 0),
+        )
+
+
+@dataclass
+class Conflict:
+    conflict_id:         str
+    claim_a_id:          str
+    claim_b_id:          str
+    domain:              str = ""
+    status:              str = "open"
+    resolution_strategy: str = ""
+    resolution_note:     str = ""
+
+    @classmethod
+    def from_dict(cls, d: dict) -> "Conflict":
+        return cls(
+            conflict_id         = d["conflict_id"],
+            claim_a_id          = d["claim_a_id"],
+            claim_b_id          = d["claim_b_id"],
+            domain              = d.get("domain", ""),
+            status              = d.get("status", "open"),
+            resolution_strategy = d.get("resolution_strategy", ""),
+            resolution_note     = d.get("resolution_note", ""),
+        )
+
+
+@dataclass
+class IngestResult:
+    ku_count: int
+    ku_ids:   list[str] = field(default_factory=list)
+    counts:   dict[str, int] = field(default_factory=dict)
+
+    @classmethod
+    def from_dict(cls, d: dict) -> "IngestResult":
+        return cls(
+            ku_count = d.get("ku_count", 0),
+            ku_ids   = d.get("ku_ids", []),
+            counts   = d.get("counts", {}),
+        )

package/python/vendor/hkc/hkc_search/bm25.py

@@ -0,0 +1,325 @@
+"""
+hkc-search / bm25.py
+关键词检索(Cold 层),自实现 BM25Okapi 倒排索引。
+
+阶段 B 改造(10万 KU 规模):从 rank-bm25 切换到自实现。
+  - 倒排索引:term -> {doc_id: tf} + df 字典
+  - append_ku / append_batch 真正 O(terms) 增量,无全量重建
+  - save / load 用 pickle,启动可免 tokenize
+  - 公式与 rank_bm25.BM25Okapi 完全一致(k1=1.5, b=0.75,
+    IDF = log((N - df + 0.5) / (df + 0.5) + 1))
+"""
+from __future__ import annotations
+import logging
+import math
+import os
+import pickle
+import re
+from collections import defaultdict
+from dataclasses import dataclass
+
+logger = logging.getLogger(__name__)
+
+
+@dataclass
+class BM25Result:
+    ku_id:   str
+    score:   float
+    name:    str
+    summary: str
+
+
+# 简单分词(沿用现状)
+_TOKEN_RE_NONWORD = re.compile(r'[^\w\s\u4e00-\u9fff]', re.UNICODE)
+_TOKEN_RE_LATIN   = re.compile(r'^[a-z0-9_-]+$')
+
+
+def _tokenize(text: str) -> list[str]:
+    """英文按空格切+小写+去标点;中文按字切(unigram + bigram)。"""
+    text = text.lower()
+    text = _TOKEN_RE_NONWORD.sub(' ', text)
+    tokens: list[str] = []
+    for word in text.split():
+        if _TOKEN_RE_LATIN.match(word):
+            if len(word) >= 2:
+                tokens.append(word)
+        else:
+            chars = [c for c in word if '\u4e00' <= c <= '\u9fff']
+            tokens.extend(chars)
+            for i in range(len(chars) - 1):
+                tokens.append(chars[i] + chars[i + 1])
+    return tokens
+
+
+class BM25Index:
+    """
+    自实现 BM25Okapi 倒排索引。
+
+    数据结构:
+        _doc_ids:  list[str]                       文档 id 列表(插入顺序)
+        _doc_lens: list[int]                       每篇 doc 的 token 数
+        _doc_idx:  dict[str, int]                  doc_id → _doc_ids 里的 idx
+        _meta:     dict[str, tuple[str, str]]      doc_id → (name, summary)
+        _N:        int                             总 doc 数
+        _avgdl:    float                           平均 doc 长度(在线更新)
+        _df:       dict[str, int]                  term → 含该 term 的 doc 数
+        _tf:       dict[str, dict[str, int]]       term → {doc_id: term freq}
+        _k1, _b:   float                           BM25Okapi 超参
+
+    时间复杂度:
+        - append_ku:     O(terms in ku)
+        - append_batch:  O(total terms in batch)
+        - remove_ku:     O(distinct terms in doc)
+        - search:        O(|query| * avg postings)
+        - build:         O(total tokens) 一次性
+
+    公式(BM25Okapi,与 rank_bm25 一致):
+        IDF(t)        = log((N - df + 0.5) / (df + 0.5) + 1)
+        tf_norm(t,d)  = (tf * (k1+1)) / (tf + k1 * (1 - b + b * |d| / avgdl))
+        score(t,d)    = IDF(t) * tf_norm(t,d)
+    """
+
+    def __init__(self, k1: float = 1.5, b: float = 0.75, epsilon: float = 0.25):
+        self._k1 = k1
+        self._b = b
+        # ATIRE BM25 变体:对 negative IDF 做 floor,避免高频词得分为负
+        # rank_bm25.BM25Okapi 默认 epsilon=0.25, floor = eps * average_idf
+        # 为与 rank_bm25 完全一致,保留这个 floor
+        self._epsilon = epsilon
+        self._reset_state()
+
+    def _reset_state(self) -> None:
+        self._doc_ids:  list[str]                       = []
+        self._doc_lens: list[int]                       = []
+        self._doc_idx:  dict[str, int]                  = {}
+        self._meta:     dict[str, tuple[str, str]]      = {}
+        self._N:        int                             = 0
+        self._avgdl:    float                           = 0.0
+        self._df:       dict[str, int]                  = defaultdict(int)
+        self._tf:       dict[str, dict[str, int]]       = defaultdict(dict)
+        # ATIRE floor:维护 idf_sum 用于 average_idf(floor = eps * average_idf)
+        # 与 rank_bm25._calc_idf 完全一致
+        self._idf_sum:  float                           = 0.0
+
+    # ── 公共接口 ───────────────────────────────────────────
+
+    def build(self, kus) -> None:
+        """全量构建。kus 为空时清空索引。"""
+        self._reset_state()
+        self.append_batch(kus)
+        logger.info("BM25 索引构建完成,共 %d 条", self._N)
+
+    def append_ku(self, ku) -> None:
+        """单条增量。O(terms in ku)。"""
+        self._add_doc(ku)
+
+    def append_batch(self, kus) -> None:
+        """批量增量(冷构建期调)。O(total terms in batch),比旧版 'append + 全量重建' 快 100x+。"""
+        if not kus:
+            return
+        for ku in kus:
+            self._add_doc(ku)
+
+    def remove_ku(self, ku_id: str) -> bool:
+        """单条删除。O(distinct terms in doc)。不存在返 False。"""
+        if ku_id not in self._doc_idx:
+            return False
+
+        # 从 _tf 反查该 doc 涉及的所有 term
+        terms_touched: list[str] = []
+        for term, postings in self._tf.items():
+            if ku_id in postings:
+                del postings[ku_id]
+                terms_touched.append(term)
+
+        # 清理空 postings + 同步更新 df / idf_sum
+        for term in terms_touched:
+            if not self._tf[term]:
+                # postings 全空,term 从 _df 移除
+                old_df = self._df[term]
+                old_idf = self._compute_idf_raw(self._N, old_df)
+                self._idf_sum -= old_idf
+                del self._df[term]
+                del self._tf[term]
+            else:
+                # postings 还有内容,df 减 1(old_df >= 2,因 _tf[term] 不空)
+                old_df = self._df[term]
+                new_df = old_df - 1
+                old_idf = self._compute_idf_raw(self._N, old_df)
+                new_idf = self._compute_idf_raw(self._N, new_df)
+                self._idf_sum += (new_idf - old_idf)
+                self._df[term] = new_df
+
+        # 移除 doc 本身
+        idx = self._doc_idx.pop(ku_id)
+        del self._doc_ids[idx]
+        del self._doc_lens[idx]
+        del self._meta[ku_id]
+
+        # idx 之后的 _doc_idx 都要 -1
+        for i in range(idx, len(self._doc_ids)):
+            self._doc_idx[self._doc_ids[i]] = i
+
+        # 更新 N, avgdl
+        self._N -= 1
+        if self._N > 0:
+            self._avgdl = sum(self._doc_lens) / self._N
+        else:
+            self._avgdl = 0.0
+
+        return True
+
+    def search(self, query: str, top_k: int = 10) -> list[BM25Result]:
+        if not self._N:
+            return []
+        query_tokens = _tokenize(query)
+        if not query_tokens:
+            return []
+        # 不去重:与 rank_bm25 行为一致(query 重复 term 累加 IDF*tf_norm)
+
+        scores: dict[str, float] = defaultdict(float)
+        for term in query_tokens:
+            if term not in self._df:
+                continue
+            idf = self._idf(term)
+            for doc_id, tf in self._tf[term].items():
+                doc_idx = self._doc_idx[doc_id]
+                doc_len = self._doc_lens[doc_idx]
+                scores[doc_id] += idf * self._tf_norm(tf, doc_len)
+
+        sorted_docs = sorted(scores.items(), key=lambda x: x[1], reverse=True)[:top_k]
+        results: list[BM25Result] = []
+        for doc_id, score in sorted_docs:
+            if score <= 0:
+                continue
+            name, summary = self._meta[doc_id]
+            results.append(BM25Result(
+                ku_id=doc_id, score=float(score), name=name, summary=summary,
+            ))
+        return results
+
+    def size(self) -> int:
+        return self._N
+
+    def clear(self) -> None:
+        """清空索引(替换旧版直接改 _ku_ids 等私有状态)。"""
+        self._reset_state()
+
+    def save(self, path: str) -> None:
+        """pickle 落盘(启动可 load 避免 10万 KU 重复 tokenize)。
+        落盘时机留给调用方决定(vector flush 钩子 / 独立 fsync 线程)。"""
+        data = {
+            'k1':       self._k1,
+            'b':        self._b,
+            'doc_ids':  self._doc_ids,
+            'doc_lens': self._doc_lens,
+            'doc_idx':  self._doc_idx,
+            'meta':     self._meta,
+            'N':        self._N,
+            'avgdl':    self._avgdl,
+            'df':       dict(self._df),
+            'tf':       {term: dict(postings) for term, postings in self._tf.items()},
+        }
+        os.makedirs(os.path.dirname(path) or '.', exist_ok=True)
+        tmp = path + '.tmp'
+        with open(tmp, 'wb') as f:
+            pickle.dump(data, f, protocol=pickle.HIGHEST_PROTOCOL)
+        os.replace(tmp, path)
+
+    def load(self, path: str) -> bool:
+        """从 pickle 加载。文件不存在或 schema 不匹配返 False。"""
+        if not os.path.exists(path):
+            return False
+        try:
+            with open(path, 'rb') as f:
+                data = pickle.load(f)
+        except Exception as e:
+            logger.warning("BM25 索引加载失败,fallback 到空索引: %s", e)
+            self._reset_state()
+            return False
+
+        required = {'k1', 'b', 'doc_ids', 'doc_lens', 'doc_idx',
+                    'meta', 'N', 'avgdl', 'df', 'tf'}
+        if not required.issubset(data.keys()):
+            logger.warning("BM25 索引 schema 不匹配,fallback 到空索引")
+            self._reset_state()
+            return False
+
+        self._k1       = data['k1']
+        self._b        = data['b']
+        self._doc_ids  = data['doc_ids']
+        self._doc_lens = data['doc_lens']
+        self._doc_idx  = data['doc_idx']
+        self._meta     = data['meta']
+        self._N        = data['N']
+        self._avgdl    = data['avgdl']
+        self._df       = defaultdict(int, data['df'])
+        self._tf       = defaultdict(dict, {term: dict(postings) for term, postings in data['tf'].items()})
+        return True
+
+    # ── 内部方法 ───────────────────────────────────────────
+
+    @staticmethod
+    def _compute_idf_raw(N: int, df: int) -> float:
+        """BM25Okapi 原始 idf(rank_bm25._calc_idf 一致)。
+        允许负值(当 df > N 时,numerator <= 0 → 负 idf,后续 ATIRE floor 兜底)。
+        """
+        if df == 0:
+            return 0.0
+        num = N - df + 0.5
+        den = df + 0.5
+        if num <= 0:
+            # log(负) 不存在,但数学上 idf = -log(|num|) + log(den) = -(log(|num|) - log(den))
+            return -(math.log(-num) - math.log(den))
+        return math.log(num) - math.log(den)
+
+    def _add_doc(self, ku) -> None:
+        text = f"{ku.name} {ku.summary} {' '.join(ku.tags)}"
+        tokens = _tokenize(text)
+        ku_id = ku.ku_id
+        doc_len = len(tokens)
+
+        self._doc_ids.append(ku_id)
+        self._doc_lens.append(doc_len)
+        self._doc_idx[ku_id] = self._N
+        self._meta[ku_id] = (ku.name, ku.summary[:100])
+
+        if doc_len > 0:
+            term_counts: dict[str, int] = defaultdict(int)
+            for t in tokens:
+                term_counts[t] += 1
+            for term, count in term_counts.items():
+                if term not in self._tf:
+                    self._tf[term] = {}
+                self._tf[term][ku_id] = count
+                # 增量更新 _idf_sum:N 不变,只 df 变
+                if self._df[term] == 0:
+                    # 新 term (df: 0 → 1)
+                    self._idf_sum += self._compute_idf_raw(self._N, 1)
+                else:
+                    # 旧 term (df: old → old+1)
+                    old_df = self._df[term]
+                    old_idf = self._compute_idf_raw(self._N, old_df)
+                    new_idf = self._compute_idf_raw(self._N, old_df + 1)
+                    self._idf_sum += (new_idf - old_idf)
+                self._df[term] += 1
+
+        prev_total = self._avgdl * self._N
+        self._N += 1
+        self._avgdl = (prev_total + doc_len) / self._N
+
+    def _idf(self, term: str) -> float:
+        df = self._df.get(term, 0)
+        if df == 0:
+            return 0.0
+        # BM25Okapi 公式(rank_bm25 风格):log((N - df + 0.5) / (df + 0.5))
+        # 注意:无 +1,负 idf 由 ATIRE floor 兜底
+        idf = self._compute_idf_raw(self._N, df)
+        if idf < 0:
+            # ATIRE floor:eps * average_idf
+            avg_idf = (self._idf_sum / len(self._df)) if self._df else 0.0
+            return self._epsilon * avg_idf
+        return idf
+
+    def _tf_norm(self, tf: int, doc_len: int) -> float:
+        return (tf * (self._k1 + 1)) / (tf + self._k1 * (1 - self._b + self._b * doc_len / self._avgdl))