foxclaw 0.1.0 → 0.1.2

package/python/bench/bm25_regression.py

@@ -0,0 +1,178 @@
+"""
+bench/bm25_regression.py
+新 BM25(自实现倒排) vs rank_bm25.BM25Okapi top-10 一致率回归测试。
+
+目标:1万 corpus + 100 query,新实现 top-10 一致率 >= 99%。
+跑法:python bench/bm25_regression.py
+"""
+import os
+import random
+import sys
+import time
+from dataclasses import dataclass
+
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', 'vendor'))
+
+from hkc.hkc_search.bm25 import BM25Index as NewBM25, _tokenize
+
+
+@dataclass
+class MockKU:
+    ku_id:   str
+    name:    str
+    summary: str
+    tags:    list[str]
+
+
+TOPICS_EN = [
+    "machine learning", "neural network", "deep learning", "data science",
+    "natural language", "computer vision", "reinforcement learning",
+    "transformer model", "graph neural network", "knowledge graph",
+    "vector database", "embedding model", "semantic search", "retrieval augmented",
+    "transformer architecture", "attention mechanism", "language model",
+    "encoder decoder", "self supervised", "contrastive learning",
+]
+TOPICS_CN = [
+    "机器学习", "神经网络", "深度学习", "数据科学",
+    "自然语言处理", "计算机视觉", "强化学习",
+    "图神经网络", "知识图谱", "向量数据库",
+    "嵌入模型", "语义搜索", "检索增强",
+    "注意力机制", "语言模型", "自监督学习",
+]
+ACTIONS = ["tutorial", "review", "introduction", "advanced", "case study", "comparison"]
+LANGS   = ["python", "rust", "cpp", "java", "go", "typescript"]
+
+
+def gen_corpus(n=10000, seed=42):
+    rng = random.Random(seed)
+    corpus = []
+    for i in range(n):
+        k = rng.randint(1, 3)
+        topics = [rng.choice(TOPICS_EN) for _ in range(k)]
+        if rng.random() < 0.5:
+            topics.append(rng.choice(TOPICS_CN))
+        action = rng.choice(ACTIONS)
+        lang = rng.choice(LANGS)
+        name = f"{' '.join(topics)} {action} ({lang}) #{i}"
+        summary = f"This is a {action} about {' '.join(topics)} in {lang}. " \
+                  f"Topics: {', '.join(topics)}."
+        tags = topics + [action, lang]
+        corpus.append(MockKU(
+            ku_id=f"ku_{i:06d}", name=name, summary=summary, tags=tags,
+        ))
+    return corpus
+
+
+def build_rank_bm25(corpus):
+    try:
+        from rank_bm25 import BM25Okapi
+    except ImportError:
+        return None
+    tokenized = [_tokenize(f"{ku.name} {ku.summary} {' '.join(ku.tags)}") for ku in corpus]
+    return BM25Okapi(tokenized)
+
+
+def rank_bm25_search(bm25, query, top_k, corpus):
+    if bm25 is None:
+        return []
+    tokens = _tokenize(query)
+    if not tokens:
+        return []
+    scores = bm25.get_scores(tokens)
+    indexed = sorted(enumerate(scores), key=lambda x: x[1], reverse=True)[:top_k]
+    return [(corpus[i].ku_id, float(s)) for i, s in indexed if s > 0]
+
+
+def gen_queries(n=100, seed=99):
+    rng = random.Random(seed)
+    queries = []
+    for _ in range(n):
+        k = rng.randint(1, 3)
+        terms = [rng.choice(TOPICS_EN) for _ in range(k)]
+        if rng.random() < 0.3:
+            terms.append(rng.choice(TOPICS_CN))
+        if rng.random() < 0.4:
+            terms.append(rng.choice(ACTIONS))
+        queries.append(" ".join(terms))
+    return queries
+
+
+def top10_overlap(rank_results, new_results):
+    rank_ids = [ku_id for ku_id, _ in rank_results[:10]]
+    new_ids  = {ku_id for ku_id, _ in new_results[:10]}
+    if not rank_ids:
+        return 1.0
+    return sum(1 for x in rank_ids if x in new_ids) / len(rank_ids)
+
+
+def main():
+    print("=" * 60)
+    print("BM25 回归测试: 自实现倒排 vs rank_bm25.BM25Okapi")
+    print("=" * 60)
+
+    print("\n[1/4] 生成 10000 mock corpus ...")
+    corpus = gen_corpus(10000, seed=42)
+    print(f"      OK, {len(corpus)} docs")
+
+    print("\n[2/4] 构建索引 ...")
+    t0 = time.time()
+    new_idx = NewBM25()
+    new_idx.build(corpus)
+    t_new = time.time() - t0
+    print(f"      新实现 build: {t_new*1000:.1f} ms")
+
+    rank_bm25 = build_rank_bm25(corpus)
+    if rank_bm25 is None:
+        print("      rank_bm25 未安装,跳过一致率比对(只测性能)")
+
+    print("\n[3/4] 100 query 跑搜索 ...")
+    queries = gen_queries(100, seed=99)
+    new_topk = 0
+    rank_topk = 0
+    overlap_sum = 0.0
+    valid = 0
+    t0 = time.time()
+    for q in queries:
+        new_r  = [(r.ku_id, r.score) for r in new_idx.search(q, top_k=10)]
+        rank_r = rank_bm25_search(rank_bm25, q, 10, corpus)
+        new_topk  += len(new_r)
+        rank_topk += len(rank_r)
+        if rank_r and new_r:
+            overlap_sum += top10_overlap(rank_r, new_r)
+            valid += 1
+    t_search = time.time() - t0
+    print(f"      耗时: {t_search*1000:.1f} ms ({(t_search/100)*1000:.2f} ms/query)")
+    print(f"      新实现召回: {new_topk}, rank_bm25 召回: {rank_topk}")
+
+    if rank_bm25 is not None and valid > 0:
+        avg = overlap_sum / valid
+        print(f"\n[4/4] top-10 一致率(对 rank_bm25 ground truth): {avg*100:.1f}%")
+        if avg >= 0.99:
+            print("      PASS (>= 99%)")
+        else:
+            print("      FAIL (< 99%)")
+            sys.exit(1)
+    else:
+        print("\n[4/4] 跳过一致率(无 rank_bm25 对照)")
+
+    # save/load roundtrip
+    print("\n[bonus] save/load roundtrip ...")
+    tmp = "_tmp_bm25.pkl"
+    new_idx.save(tmp)
+    new_idx2 = NewBM25()
+    assert new_idx2.load(tmp), "load 失败"
+    assert new_idx2.size() == new_idx.size()
+    for q in queries[:5]:
+        r1 = sorted([(r.ku_id, r.score) for r in new_idx.search(q, 10)])
+        r2 = sorted([(r.ku_id, r.score) for r in new_idx2.search(q, 10)])
+        assert r1 == r2, f"save/load 不一致: {q}"
+    os.remove(tmp)
+    print(f"      OK, roundtrip 一致 ({new_idx.size()} docs)")
+
+    print("\n" + "=" * 60)
+    print("全部通过")
+    print("=" * 60)
+
+
+if __name__ == "__main__":
+    main()

package/python/bench/bm25_stress.py

@@ -0,0 +1,159 @@
+"""
+bench/bm25_stress.py
+5万/10万 corpus 压测 + remove_ku 增量测试。
+"""
+import os, sys, random, time
+from dataclasses import dataclass
+
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', 'vendor'))
+from hkc.hkc_search.bm25 import BM25Index as NewBM25, _tokenize
+
+
+@dataclass
+class MockKU:
+    ku_id:   str
+    name:    str
+    summary: str
+    tags:    list[str]
+
+
+TOPICS_EN = ["machine learning","neural network","deep learning","data science",
+    "natural language","computer vision","reinforcement learning",
+    "transformer model","graph neural network","knowledge graph",
+    "vector database","embedding model","semantic search","retrieval augmented",
+    "transformer architecture","attention mechanism","language model",
+    "encoder decoder","self supervised","contrastive learning"] * 5
+ACTIONS = ["tutorial","review","introduction","advanced","case study","comparison"] * 10
+LANGS   = ["python","rust","cpp","java","go","typescript"] * 10
+TOPICS_CN = ["机器学习","神经网络","深度学习","数据科学","自然语言处理",
+    "计算机视觉","强化学习","图神经网络","知识图谱","向量数据库",
+    "嵌入模型","语义搜索","检索增强","注意力机制","语言模型","自监督学习"] * 5
+
+
+def gen_corpus(n, seed=42):
+    rng = random.Random(seed)
+    corpus = []
+    for i in range(n):
+        k = rng.randint(1, 3)
+        topics = [rng.choice(TOPICS_EN) for _ in range(k)]
+        if rng.random() < 0.5:
+            topics.append(rng.choice(TOPICS_CN))
+        action = rng.choice(ACTIONS)
+        lang = rng.choice(LANGS)
+        name = f"{' '.join(topics)} {action} ({lang}) #{i}"
+        summary = f"This is a {action} about {' '.join(topics)} in {lang}. " \
+                  f"Topics: {', '.join(topics)}."
+        tags = topics + [action, lang]
+        corpus.append(MockKU(ku_id=f"ku_{i:08d}", name=name, summary=summary, tags=tags))
+    return corpus
+
+
+def gen_queries(n, seed=99):
+    rng = random.Random(seed)
+    queries = []
+    for _ in range(n):
+        k = rng.randint(1, 3)
+        terms = [rng.choice(TOPICS_EN) for _ in range(k)]
+        if rng.random() < 0.3:
+            terms.append(rng.choice(TOPICS_CN))
+        if rng.random() < 0.4:
+            terms.append(rng.choice(ACTIONS))
+        queries.append(" ".join(terms))
+    return queries
+
+
+def bench_build(n):
+    print(f"\n[build] {n} corpus ...")
+    corpus = gen_corpus(n)
+    t0 = time.time()
+    idx = NewBM25()
+    idx.build(corpus)
+    dt = time.time() - t0
+    print(f"        build: {dt*1000:.0f} ms ({dt*1000/n:.3f} ms/doc)")
+    return idx, corpus
+
+
+def bench_search(idx, queries):
+    t0 = time.time()
+    total_hits = 0
+    for q in queries:
+        r = idx.search(q, top_k=10)
+        total_hits += len(r)
+    dt = time.time() - t0
+    print(f"        search {len(queries)} query: {dt*1000:.0f} ms ({dt*1000/len(queries):.2f} ms/query, {total_hits} hits)")
+    return dt
+
+
+def bench_remove(idx, n_remove=1000):
+    print(f"\n[remove] 删 {n_remove} 条 (random) ...")
+    import random
+    rng = random.Random(123)
+    to_remove = rng.sample(idx._doc_ids, n_remove)  # noqa
+    t0 = time.time()
+    for ku_id in to_remove:
+        idx.remove_ku(ku_id)
+    dt = time.time() - t0
+    print(f"        remove: {dt*1000:.0f} ms ({dt*1000/n_remove:.3f} ms/op)")
+    print(f"        size after: {idx.size()}")
+
+
+def bench_save_load(idx, path):
+    print(f"\n[save/load] {path} ...")
+    t0 = time.time()
+    idx.save(path)
+    t_save = time.time() - t0
+    sz = os.path.getsize(path)
+    t0 = time.time()
+    idx2 = NewBM25()
+    ok = idx2.load(path)
+    t_load = time.time() - t0
+    print(f"        save : {t_save*1000:.0f} ms ({sz/1024/1024:.1f} MB)")
+    print(f"        load : {t_load*1000:.0f} ms (ok={ok}, size={idx2.size()})")
+
+
+def main():
+    print("=" * 60)
+    print("BM25 压测: 5万/10万 corpus build + search + remove + save/load")
+    print("=" * 60)
+
+    # 5万
+    idx, corpus = bench_build(50_000)
+    queries = gen_queries(200)
+    bench_search(idx, queries)
+    bench_remove(idx, 1000)
+    tmp = "_tmp_bm25_50k.pkl"
+    bench_save_load(idx, tmp)
+    os.remove(tmp)
+
+    # 10万 (增量:在 5万 上 append 5万,模拟冷构建期分批)
+    print(f"\n[build] +5万 增量 (5万 -> 10万) ...")
+    extra_corpus = gen_corpus(50_000, seed=43)
+    # 取后 5万 (id 范围 0-49999 是 5万 corpus;这里 gen 同一 seed 重叠,改 seed)
+    # 实际:重新 gen 5万不同 id
+    extra_corpus = []
+    rng = random.Random(43)
+    for i in range(50_000, 100_000):
+        k = rng.randint(1, 3)
+        topics = [rng.choice(TOPICS_EN) for _ in range(k)]
+        if rng.random() < 0.5:
+            topics.append(rng.choice(TOPICS_CN))
+        action = rng.choice(ACTIONS)
+        lang = rng.choice(LANGS)
+        name = f"{' '.join(topics)} {action} ({lang}) #{i}"
+        summary = f"This is a {action} about {' '.join(topics)} in {lang}. Topics: {', '.join(topics)}."
+        tags = topics + [action, lang]
+        extra_corpus.append(MockKU(ku_id=f"ku_{i:08d}", name=name, summary=summary, tags=tags))
+    t0 = time.time()
+    idx.append_batch(extra_corpus)
+    dt = time.time() - t0
+    print(f"        append 5万 增量: {dt*1000:.0f} ms ({dt*1000/50_000:.3f} ms/doc)")
+    print(f"        size after: {idx.size()}")
+    bench_search(idx, queries)
+
+    print("\n" + "=" * 60)
+    print("压测完成")
+    print("=" * 60)
+
+
+if __name__ == "__main__":
+    main()

package/python/bench/cd_audit.py

@@ -0,0 +1,238 @@
+"""
+bench/cd_audit.py
+C+D 阶段 5 项 Risk 审核(commit 0bf4349):
+  D1. IngestJobRegistry(max_workers=4) 真起 4 thread 并发
+  D2. Extractor 16 chunk × 4 worker + 50% 429 重试 (限流雪崩测试)
+  D3. progress_cb 线程安全:done 计数无重复无漏
+  D4. _retry_llm max_retries 可调 (验证配置生效)
+  D5. retry 不破坏 progress_cb 顺序
+"""
+import os, sys, time, threading
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', 'vendor'))
+
+from hkc.hkc_kde.extractor import Extractor
+from hkc.hkc_api.ingest_jobs import IngestJobRegistry
+
+
+# ── D1: IngestJobRegistry 4 thread 并发 ────────────────
+
+def d1_ingest_4_workers():
+    """D1. 4 worker 真并发 peak=4,4 任务 ~0.3s 完成"""
+    reg = IngestJobRegistry(max_workers=4)
+    running = [0]
+    peak = [0]
+    lock = threading.Lock()
+
+    def work(progress):
+        with lock:
+            running[0] += 1
+            peak[0] = max(peak[0], running[0])
+        time.sleep(0.3)
+        with lock:
+            running[0] -= 1
+        return []
+
+    jobs = [reg.create(f"f{i}") for i in range(4)]
+    t0 = time.time()
+    for j in jobs:
+        reg.submit(j, work)
+    # wait done
+    while True:
+        statuses = [reg.get(j.job_id)["status"] for j in jobs]
+        if all(s == "done" for s in statuses):
+            break
+        time.sleep(0.02)
+    elapsed = time.time() - t0
+    reg.shutdown()
+    assert peak[0] == 4, f"expected peak=4, got {peak[0]}"
+    assert elapsed < 0.5, f"expected < 0.5s (并行), got {elapsed:.2f}s"
+    print(f"  D1 PASS: peak={peak[0]}, 4 jobs in {elapsed*1000:.0f}ms (串行需 1.2s)")
+
+
+# ── D2: Extractor 16 chunk 限流雪崩 ────────────────────
+
+def d2_429_snowball():
+    """D2. 16 chunk × 4 worker × 100% 429 永不成功,验证不死循环 + 总调用有上界"""
+    state = {"calls": 0}
+
+    def always_429(*args, **kwargs):
+        state["calls"] += 1
+        err = Exception("rate limited")
+        err.status_code = 429
+        raise err
+
+    ext = Extractor(provider="anthropic", model="m")
+    # 替换 _extract_chunk:走 _retry_llm 但传极小 base_delay 测试用
+    # 同时 monkey-patch time.sleep 避免真睡
+    import hkc.hkc_kde.extractor as ex_mod
+    orig_sleep = ex_mod.time.sleep
+    sleeps = []
+    ex_mod.time.sleep = lambda s: sleeps.append(s)
+    try:
+        def mock_chunk(chunk, timeout=60.0):
+            try:
+                return ext._retry_llm(always_429, base_delay=0.001, max_delay=0.01)
+            except Exception:
+                return None
+        ext._extract_chunk = mock_chunk
+
+        class MockChunk:
+            def __init__(self, i):
+                self.chunk_id = f"c{i}"
+                self.doc_id = "d0"
+                self.content = "x"
+                self.source_hint = ""
+
+        chunks = [MockChunk(i) for i in range(16)]
+        t0 = time.time()
+        result = ext.extract(chunks)
+        elapsed = time.time() - t0
+
+        # 期望:16 chunk 全 fail,总调用 = 16 * (1+3 retry) = 64,无死循环
+        success = sum(1 for item in result.items if item is not None)
+        assert success == 0, f"expected 0 success, got {success}"
+        assert state["calls"] == 64, f"snowball: expected 64 calls, got {state['calls']}"
+        assert len(sleeps) == 48, f"expected 48 sleeps (16 chunk * 3 retry), got {len(sleeps)}"
+        assert elapsed < 3.0, f"deadlock: {elapsed:.1f}s"
+        print(f"  D2 PASS: 16 chunk x 4 worker x 100% 429 -> {state['calls']} calls, {len(sleeps)} sleeps, {elapsed*1000:.0f}ms (no deadlock)")
+    finally:
+        ex_mod.time.sleep = orig_sleep
+
+
+# ── D3: progress_cb 线程安全 ─────────────────────────
+
+def d3_progress_cb_thread_safe():
+    """D3. progress_cb 在 as_completed 循环调,主线程串行;16 chunk 计数无重复无漏"""
+    ext = Extractor(provider="anthropic", model="m")
+
+    # 替换 _extract_chunk 为 fast mock:50% 概率 fail(模拟 chunk 失败)
+    import random
+    random.seed(42)
+    def mock_extract(chunk, timeout=60.0):
+        # 永远返 RawExtraction
+        from hkc.hkc_kde.models import RawExtraction
+        return RawExtraction()
+    ext._extract_chunk = mock_extract
+
+    class MockChunk:
+        def __init__(self, i):
+            self.chunk_id = f"c{i}"
+            self.doc_id = "d0"
+            self.content = "x"
+            self.source_hint = ""
+
+    done_calls = []
+    lock = threading.Lock()
+
+    def cb(processed, total):
+        with lock:
+            done_calls.append(processed)
+
+    chunks = [MockChunk(i) for i in range(16)]
+    result = ext.extract(chunks, progress_cb=cb)
+
+    # 验证:done_calls 长度 16(每个 chunk 一次)
+    assert len(done_calls) == 16, f"expected 16 callbacks, got {len(done_calls)}"
+    # 验证:1..16 各一次(set 大小 = 16,排序后 1..16)
+    assert sorted(done_calls) == list(range(1, 17)), \
+        f"expected processed 1..16, got {sorted(done_calls)}"
+    print(f"  D3 PASS: {len(done_calls)} callbacks, all 1..16 unique")
+
+
+# ── D4: max_retries 可调 ──────────────────────────────
+
+def d4_max_retries_configurable():
+    """D4. _retry_llm 的 max_retries 真的生效"""
+    ext = Extractor(provider="mock", model="m")
+
+    # 测 max_retries=2(默认 3):4 次后 raise
+    call_count = [0]
+    def always_fail():
+        call_count[0] += 1
+        err = Exception("rate")
+        err.status_code = 429
+        raise err
+
+    # 用 monkey-patch 把 time.sleep 替换掉,这样不会真睡
+    import hkc.hkc_kde.extractor as ex_mod
+    orig_sleep = ex_mod.time.sleep
+    sleeps = []
+    ex_mod.time.sleep = lambda s: sleeps.append(s)
+    try:
+        try:
+            ext._retry_llm(always_fail, max_retries=2, base_delay=0.001)
+            assert False, "expected raise"
+        except Exception:
+            pass
+        # max_retries=2 + 1 = 3 次尝试
+        assert call_count[0] == 3, f"max_retries=2 expected 3 calls, got {call_count[0]}"
+        assert len(sleeps) == 2, f"max_retries=2 expected 2 sleeps, got {len(sleeps)}"
+        print(f"  D4 PASS: max_retries=2 -> 3 calls, 2 sleeps")
+    finally:
+        ex_mod.time.sleep = orig_sleep
+
+
+# ── D5: retry + progress_cb 顺序 ─────────────────────
+
+def d5_retry_no_progress_leak():
+    """D5. retry 期间 progress_cb 仍只调 1 次/chunk(不管 retry 几次)"""
+    import time as time_mod
+    orig_sleep = time_mod.sleep
+    time_mod.sleep = lambda s: None
+
+    state = {"calls": 0}
+    def always_429(*args, **kwargs):
+        state["calls"] += 1
+        err = Exception("rate"); err.status_code = 429
+        raise err
+
+    class MockResp:
+        def __init__(self, t): self.content = [MockText(t)]
+    class MockText:
+        def __init__(self, t): self.text = t
+
+    class MockClient:
+        @property
+        def messages(self): return self
+        def create(self, *args, **kwargs): return always_429(*args, **kwargs)
+
+    ext = Extractor(provider="anthropic", model="m")
+    ext._client = MockClient()
+    try:
+        class MockChunk:
+            def __init__(self, i):
+                self.chunk_id=f"c{i}"; self.doc_id="d0"; self.content="x"; self.source_hint=""
+        cb_count = [0]
+        cb_lock = threading.Lock()
+        def cb(p, t):
+            with cb_lock:
+                cb_count[0] += 1
+        chunks = [MockChunk(i) for i in range(8)]
+        result = ext.extract(chunks, progress_cb=cb)
+        # 不管 retry 几次,每个 chunk 调 1 次 progress_cb
+        assert cb_count[0] == 8, f"expected 8 progress, got {cb_count[0]}"
+        # 8 chunk 全 fail,总调用 = 8 * 4 = 32
+        assert state["calls"] == 32, f"expected 32 calls, got {state['calls']}"
+        success = sum(1 for item in result.items if item is not None)
+        assert success == 0
+        print(f"  D5 PASS: 8 chunks x 4 attempts, progress_cb={cb_count[0]} (1 per chunk), calls={state['calls']}")
+    finally:
+        time_mod.sleep = orig_sleep
+
+
+def main():
+    print("=" * 60)
+    print("C+D 阶段 5 项 Risk 审核 (commit 0bf4349)")
+    print("=" * 60)
+    d1_ingest_4_workers()
+    d2_429_snowball()
+    d3_progress_cb_thread_safe()
+    d4_max_retries_configurable()
+    d5_retry_no_progress_leak()
+    print("=" * 60)
+    print("5/5 PASS")
+    print("=" * 60)
+
+
+if __name__ == "__main__":
+    main()