foxclaw 0.1.0 → 0.1.3

package/python/bench/bm25_regression.py

@@ -0,0 +1,177 @@
+"""
+bench/bm25_regression.py
+新 BM25(自实现倒排) vs rank_bm25.BM25Okapi top-10 一致率回归测试。
+
+目标:1万 corpus + 100 query,新实现 top-10 一致率 >= 99%。
+跑法:python bench/bm25_regression.py
+"""
+import os
+import random
+import sys
+import time
+from dataclasses import dataclass
+
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', 'vendor'))
+
+from hkc.hkc_search.bm25 import BM25Index as NewBM25, _tokenize
+
+
+@dataclass
+class MockKU:
+    ku_id:   str
+    name:    str
+    summary: str
+    tags:    list[str]
+
+
+TOPICS_EN = [
+    "machine learning", "neural network", "deep learning", "data science",
+    "natural language", "computer vision", "reinforcement learning",
+    "transformer model", "graph neural network", "knowledge graph",
+    "vector database", "embedding model", "semantic search", "retrieval augmented",
+    "transformer architecture", "attention mechanism", "language model",
+    "encoder decoder", "self supervised", "contrastive learning",
+]
+TOPICS_CN = [
+    "机器学习", "神经网络", "深度学习", "数据科学",
+    "自然语言处理", "计算机视觉", "强化学习",
+    "图神经网络", "知识图谱", "向量数据库",
+    "嵌入模型", "语义搜索", "检索增强",
+    "注意力机制", "语言模型", "自监督学习",
+]
+ACTIONS = ["tutorial", "review", "introduction", "advanced", "case study", "comparison"]
+LANGS   = ["python", "rust", "cpp", "java", "go", "typescript"]
+
+
+def gen_corpus(n=10000, seed=42):
+    rng = random.Random(seed)
+    corpus = []
+    for i in range(n):
+        k = rng.randint(1, 3)
+        topics = [rng.choice(TOPICS_EN) for _ in range(k)]
+        if rng.random() < 0.5:
+            topics.append(rng.choice(TOPICS_CN))
+        action = rng.choice(ACTIONS)
+        lang = rng.choice(LANGS)
+        name = f"{' '.join(topics)} {action} ({lang}) #{i}"
+        summary = f"This is a {action} about {' '.join(topics)} in {lang}. " \
+                  f"Topics: {', '.join(topics)}."
+        tags = topics + [action, lang]
+        corpus.append(MockKU(
+            ku_id=f"ku_{i:06d}", name=name, summary=summary, tags=tags,
+        ))
+    return corpus
+
+
+def build_rank_bm25(corpus):
+    try:
+        from rank_bm25 import BM25Okapi
+    except ImportError:
+        return None
+    tokenized = [_tokenize(f"{ku.name} {ku.summary} {' '.join(ku.tags)}") for ku in corpus]
+    return BM25Okapi(tokenized)
+
+
+def rank_bm25_search(bm25, query, top_k, corpus):
+    if bm25 is None:
+        return []
+    tokens = _tokenize(query)
+    if not tokens:
+        return []
+    scores = bm25.get_scores(tokens)
+    indexed = sorted(enumerate(scores), key=lambda x: x[1], reverse=True)[:top_k]
+    return [(corpus[i].ku_id, float(s)) for i, s in indexed if s > 0]
+
+
+def gen_queries(n=100, seed=99):
+    rng = random.Random(seed)
+    queries = []
+    for _ in range(n):
+        k = rng.randint(1, 3)
+        terms = [rng.choice(TOPICS_EN) for _ in range(k)]
+        if rng.random() < 0.3:
+            terms.append(rng.choice(TOPICS_CN))
+        if rng.random() < 0.4:
+            terms.append(rng.choice(ACTIONS))
+        queries.append(" ".join(terms))
+    return queries
+
+
+def top10_overlap(rank_results, new_results):
+    rank_ids = [ku_id for ku_id, _ in rank_results[:10]]
+    new_ids  = {ku_id for ku_id, _ in new_results[:10]}
+    if not rank_ids:
+        return 1.0
+    return sum(1 for x in rank_ids if x in new_ids) / len(rank_ids)
+
+
+def main():
+    print("=" * 60)
+    print("BM25 回归测试: 自实现倒排 vs rank_bm25.BM25Okapi")
+    print("=" * 60)
+
+    print("\n[1/4] 生成 10000 mock corpus ...")
+    corpus = gen_corpus(10000, seed=42)
+    print(f"      OK, {len(corpus)} docs")
+
+    print("\n[2/4] 构建索引 ...")
+    t0 = time.time()
+    new_idx = NewBM25()
+    new_idx.build(corpus)
+    t_new = time.time() - t0
+    print(f"      新实现 build: {t_new*1000:.1f} ms")
+
+    rank_bm25 = build_rank_bm25(corpus)
+    if rank_bm25 is None:
+        print("      rank_bm25 未安装,跳过一致率比对(只测性能)")
+
+    print("\n[3/4] 100 query 跑搜索 ...")
+    queries = gen_queries(100, seed=99)
+    new_topk = 0
+    rank_topk = 0
+    overlap_sum = 0.0
+    valid = 0
+    t0 = time.time()
+    for q in queries:
+        new_r  = [(r.ku_id, r.score) for r in new_idx.search(q, top_k=10)]
+        rank_r = rank_bm25_search(rank_bm25, q, 10, corpus)
+        new_topk  += len(new_r)
+        rank_topk += len(rank_r)
+        if rank_r and new_r:
+            overlap_sum += top10_overlap(rank_r, new_r)
+            valid += 1
+    t_search = time.time() - t0
+    print(f"      耗时: {t_search*1000:.1f} ms ({(t_search/100)*1000:.2f} ms/query)")
+    print(f"      新实现召回: {new_topk}, rank_bm25 召回: {rank_topk}")
+
+    if rank_bm25 is not None and valid > 0:
+        avg = overlap_sum / valid
+        print(f"\n[4/4] top-10 一致率(对 rank_bm25 ground truth): {avg*100:.1f}%")
+        if avg >= 0.99:
+            print("      PASS (>= 99%)")
+        else:
+            print("      FAIL (< 99%)")
+            sys.exit(1)
+    else:
+        print("\n[4/4] 跳过一致率(无 rank_bm25 对照)")
+
+    print("\n[bonus] save/load roundtrip ...")
+    tmp = "_tmp_bm25.pkl"
+    new_idx.save(tmp)
+    new_idx2 = NewBM25()
+    assert new_idx2.load(tmp), "load 失败"
+    assert new_idx2.size() == new_idx.size()
+    for q in queries[:5]:
+        r1 = sorted([(r.ku_id, r.score) for r in new_idx.search(q, 10)])
+        r2 = sorted([(r.ku_id, r.score) for r in new_idx2.search(q, 10)])
+        assert r1 == r2, f"save/load 不一致: {q}"
+    os.remove(tmp)
+    print(f"      OK, roundtrip 一致 ({new_idx.size()} docs)")
+
+    print("\n" + "=" * 60)
+    print("全部通过")
+    print("=" * 60)
+
+
+if __name__ == "__main__":
+    main()

package/python/bench/bm25_stress.py

@@ -0,0 +1,155 @@
+"""
+bench/bm25_stress.py
+5万/10万 corpus 压测 + remove_ku 增量测试。
+"""
+import os, sys, random, time
+from dataclasses import dataclass
+
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', 'vendor'))
+from hkc.hkc_search.bm25 import BM25Index as NewBM25, _tokenize
+
+
+@dataclass
+class MockKU:
+    ku_id:   str
+    name:    str
+    summary: str
+    tags:    list[str]
+
+
+TOPICS_EN = ["machine learning","neural network","deep learning","data science",
+    "natural language","computer vision","reinforcement learning",
+    "transformer model","graph neural network","knowledge graph",
+    "vector database","embedding model","semantic search","retrieval augmented",
+    "transformer architecture","attention mechanism","language model",
+    "encoder decoder","self supervised","contrastive learning"] * 5
+ACTIONS = ["tutorial","review","introduction","advanced","case study","comparison"] * 10
+LANGS   = ["python","rust","cpp","java","go","typescript"] * 10
+TOPICS_CN = ["机器学习","神经网络","深度学习","数据科学","自然语言处理",
+    "计算机视觉","强化学习","图神经网络","知识图谱","向量数据库",
+    "嵌入模型","语义搜索","检索增强","注意力机制","语言模型","自监督学习"] * 5
+
+
+def gen_corpus(n, seed=42):
+    rng = random.Random(seed)
+    corpus = []
+    for i in range(n):
+        k = rng.randint(1, 3)
+        topics = [rng.choice(TOPICS_EN) for _ in range(k)]
+        if rng.random() < 0.5:
+            topics.append(rng.choice(TOPICS_CN))
+        action = rng.choice(ACTIONS)
+        lang = rng.choice(LANGS)
+        name = f"{' '.join(topics)} {action} ({lang}) #{i}"
+        summary = f"This is a {action} about {' '.join(topics)} in {lang}. " \
+                  f"Topics: {', '.join(topics)}."
+        tags = topics + [action, lang]
+        corpus.append(MockKU(ku_id=f"ku_{i:08d}", name=name, summary=summary, tags=tags))
+    return corpus
+
+
+def gen_queries(n, seed=99):
+    rng = random.Random(seed)
+    queries = []
+    for _ in range(n):
+        k = rng.randint(1, 3)
+        terms = [rng.choice(TOPICS_EN) for _ in range(k)]
+        if rng.random() < 0.3:
+            terms.append(rng.choice(TOPICS_CN))
+        if rng.random() < 0.4:
+            terms.append(rng.choice(ACTIONS))
+        queries.append(" ".join(terms))
+    return queries
+
+
+def bench_build(n):
+    print(f"\n[build] {n} corpus ...")
+    corpus = gen_corpus(n)
+    t0 = time.time()
+    idx = NewBM25()
+    idx.build(corpus)
+    dt = time.time() - t0
+    print(f"        build: {dt*1000:.0f} ms ({dt*1000/n:.3f} ms/doc)")
+    return idx, corpus
+
+
+def bench_search(idx, queries):
+    t0 = time.time()
+    total_hits = 0
+    for q in queries:
+        r = idx.search(q, top_k=10)
+        total_hits += len(r)
+    dt = time.time() - t0
+    print(f"        search {len(queries)} query: {dt*1000:.0f} ms ({dt*1000/len(queries):.2f} ms/query, {total_hits} hits)")
+    return dt
+
+
+def bench_remove(idx, n_remove=1000):
+    print(f"\n[remove] 删 {n_remove} 条 (random) ...")
+    import random
+    rng = random.Random(123)
+    to_remove = rng.sample(idx._doc_ids, n_remove)  # noqa
+    t0 = time.time()
+    for ku_id in to_remove:
+        idx.remove_ku(ku_id)
+    dt = time.time() - t0
+    print(f"        remove: {dt*1000:.0f} ms ({dt*1000/n_remove:.3f} ms/op)")
+    print(f"        size after: {idx.size()}")
+
+
+def bench_save_load(idx, path):
+    print(f"\n[save/load] {path} ...")
+    t0 = time.time()
+    idx.save(path)
+    t_save = time.time() - t0
+    sz = os.path.getsize(path)
+    t0 = time.time()
+    idx2 = NewBM25()
+    ok = idx2.load(path)
+    t_load = time.time() - t0
+    print(f"        save : {t_save*1000:.0f} ms ({sz/1024/1024:.1f} MB)")
+    print(f"        load : {t_load*1000:.0f} ms (ok={ok}, size={idx2.size()})")
+
+
+def main():
+    print("=" * 60)
+    print("BM25 压测: 5万/10万 corpus build + search + remove + save/load")
+    print("=" * 60)
+
+    idx, corpus = bench_build(50_000)
+    queries = gen_queries(200)
+    bench_search(idx, queries)
+    bench_remove(idx, 1000)
+    tmp = "_tmp_bm25_50k.pkl"
+    bench_save_load(idx, tmp)
+    os.remove(tmp)
+
+    print(f"\n[build] +5万 增量 (5万 -> 10万) ...")
+    extra_corpus = gen_corpus(50_000, seed=43)
+    extra_corpus = []
+    rng = random.Random(43)
+    for i in range(50_000, 100_000):
+        k = rng.randint(1, 3)
+        topics = [rng.choice(TOPICS_EN) for _ in range(k)]
+        if rng.random() < 0.5:
+            topics.append(rng.choice(TOPICS_CN))
+        action = rng.choice(ACTIONS)
+        lang = rng.choice(LANGS)
+        name = f"{' '.join(topics)} {action} ({lang}) #{i}"
+        summary = f"This is a {action} about {' '.join(topics)} in {lang}. Topics: {', '.join(topics)}."
+        tags = topics + [action, lang]
+        extra_corpus.append(MockKU(ku_id=f"ku_{i:08d}", name=name, summary=summary, tags=tags))
+    t0 = time.time()
+    idx.append_batch(extra_corpus)
+    dt = time.time() - t0
+    print(f"        append 5万 增量: {dt*1000:.0f} ms ({dt*1000/50_000:.3f} ms/doc)")
+    print(f"        size after: {idx.size()}")
+    bench_search(idx, queries)
+
+    print("\n" + "=" * 60)
+    print("压测完成")
+    print("=" * 60)
+
+
+if __name__ == "__main__":
+    main()

package/python/bench/cd_audit.py

@@ -0,0 +1,215 @@
+"""
+bench/cd_audit.py
+C+D 阶段 5 项 Risk 审核(commit 0bf4349):
+  D1. IngestJobRegistry(max_workers=4) 真起 4 thread 并发
+  D2. Extractor 16 chunk × 4 worker + 50% 429 重试 (限流雪崩测试)
+  D3. progress_cb 线程安全:done 计数无重复无漏
+  D4. _retry_llm max_retries 可调 (验证配置生效)
+  D5. retry 不破坏 progress_cb 顺序
+"""
+import os, sys, time, threading
+sys.path.insert(0, os.path.join(os.path.dirname(__file__), '..', 'vendor'))
+
+from hkc.hkc_kde.extractor import Extractor
+from hkc.hkc_api.ingest_jobs import IngestJobRegistry
+
+
+def d1_ingest_4_workers():
+    """D1. 4 worker 真并发 peak=4,4 任务 ~0.3s 完成"""
+    reg = IngestJobRegistry(max_workers=4)
+    running = [0]
+    peak = [0]
+    lock = threading.Lock()
+
+    def work(progress):
+        with lock:
+            running[0] += 1
+            peak[0] = max(peak[0], running[0])
+        time.sleep(0.3)
+        with lock:
+            running[0] -= 1
+        return []
+
+    jobs = [reg.create(f"f{i}") for i in range(4)]
+    t0 = time.time()
+    for j in jobs:
+        reg.submit(j, work)
+    while True:
+        statuses = [reg.get(j.job_id)["status"] for j in jobs]
+        if all(s == "done" for s in statuses):
+            break
+        time.sleep(0.02)
+    elapsed = time.time() - t0
+    reg.shutdown()
+    assert peak[0] == 4, f"expected peak=4, got {peak[0]}"
+    assert elapsed < 0.5, f"expected < 0.5s (并行), got {elapsed:.2f}s"
+    print(f"  D1 PASS: peak={peak[0]}, 4 jobs in {elapsed*1000:.0f}ms (串行需 1.2s)")
+
+
+def d2_429_snowball():
+    """D2. 16 chunk × 4 worker × 100% 429 永不成功,验证不死循环 + 总调用有上界"""
+    state = {"calls": 0}
+
+    def always_429(*args, **kwargs):
+        state["calls"] += 1
+        err = Exception("rate limited")
+        err.status_code = 429
+        raise err
+
+    ext = Extractor(provider="anthropic", model="m")
+    import hkc.hkc_kde.extractor as ex_mod
+    orig_sleep = ex_mod.time.sleep
+    sleeps = []
+    ex_mod.time.sleep = lambda s: sleeps.append(s)
+    try:
+        def mock_chunk(chunk, timeout=60.0):
+            try:
+                return ext._retry_llm(always_429, base_delay=0.001, max_delay=0.01)
+            except Exception:
+                return None
+        ext._extract_chunk = mock_chunk
+
+        class MockChunk:
+            def __init__(self, i):
+                self.chunk_id = f"c{i}"
+                self.doc_id = "d0"
+                self.content = "x"
+                self.source_hint = ""
+
+        chunks = [MockChunk(i) for i in range(16)]
+        t0 = time.time()
+        result = ext.extract(chunks)
+        elapsed = time.time() - t0
+
+        success = sum(1 for item in result.items if item is not None)
+        assert success == 0, f"expected 0 success, got {success}"
+        assert state["calls"] == 64, f"snowball: expected 64 calls, got {state['calls']}"
+        assert len(sleeps) == 48, f"expected 48 sleeps (16 chunk * 3 retry), got {len(sleeps)}"
+        assert elapsed < 3.0, f"deadlock: {elapsed:.1f}s"
+        print(f"  D2 PASS: 16 chunk x 4 worker x 100% 429 -> {state['calls']} calls, {len(sleeps)} sleeps, {elapsed*1000:.0f}ms (no deadlock)")
+    finally:
+        ex_mod.time.sleep = orig_sleep
+
+
+def d3_progress_cb_thread_safe():
+    """D3. progress_cb 在 as_completed 循环调,主线程串行;16 chunk 计数无重复无漏"""
+    ext = Extractor(provider="anthropic", model="m")
+
+    import random
+    random.seed(42)
+    def mock_extract(chunk, timeout=60.0):
+        from hkc.hkc_kde.models import RawExtraction
+        return RawExtraction()
+    ext._extract_chunk = mock_extract
+
+    class MockChunk:
+        def __init__(self, i):
+            self.chunk_id = f"c{i}"
+            self.doc_id = "d0"
+            self.content = "x"
+            self.source_hint = ""
+
+    done_calls = []
+    lock = threading.Lock()
+
+    def cb(processed, total):
+        with lock:
+            done_calls.append(processed)
+
+    chunks = [MockChunk(i) for i in range(16)]
+    result = ext.extract(chunks, progress_cb=cb)
+
+    assert len(done_calls) == 16, f"expected 16 callbacks, got {len(done_calls)}"
+    assert sorted(done_calls) == list(range(1, 17)), \
+        f"expected processed 1..16, got {sorted(done_calls)}"
+    print(f"  D3 PASS: {len(done_calls)} callbacks, all 1..16 unique")
+
+
+def d4_max_retries_configurable():
+    """D4. _retry_llm 的 max_retries 真的生效"""
+    ext = Extractor(provider="mock", model="m")
+
+    call_count = [0]
+    def always_fail():
+        call_count[0] += 1
+        err = Exception("rate")
+        err.status_code = 429
+        raise err
+
+    import hkc.hkc_kde.extractor as ex_mod
+    orig_sleep = ex_mod.time.sleep
+    sleeps = []
+    ex_mod.time.sleep = lambda s: sleeps.append(s)
+    try:
+        try:
+            ext._retry_llm(always_fail, max_retries=2, base_delay=0.001)
+            assert False, "expected raise"
+        except Exception:
+            pass
+        assert call_count[0] == 3, f"max_retries=2 expected 3 calls, got {call_count[0]}"
+        assert len(sleeps) == 2, f"max_retries=2 expected 2 sleeps, got {len(sleeps)}"
+        print(f"  D4 PASS: max_retries=2 -> 3 calls, 2 sleeps")
+    finally:
+        ex_mod.time.sleep = orig_sleep
+
+
+def d5_retry_no_progress_leak():
+    """D5. retry 期间 progress_cb 仍只调 1 次/chunk(不管 retry 几次)"""
+    import time as time_mod
+    orig_sleep = time_mod.sleep
+    time_mod.sleep = lambda s: None
+
+    state = {"calls": 0}
+    def always_429(*args, **kwargs):
+        state["calls"] += 1
+        err = Exception("rate"); err.status_code = 429
+        raise err
+
+    class MockResp:
+        def __init__(self, t): self.content = [MockText(t)]
+    class MockText:
+        def __init__(self, t): self.text = t
+
+    class MockClient:
+        @property
+        def messages(self): return self
+        def create(self, *args, **kwargs): return always_429(*args, **kwargs)
+
+    ext = Extractor(provider="anthropic", model="m")
+    ext._client = MockClient()
+    try:
+        class MockChunk:
+            def __init__(self, i):
+                self.chunk_id=f"c{i}"; self.doc_id="d0"; self.content="x"; self.source_hint=""
+        cb_count = [0]
+        cb_lock = threading.Lock()
+        def cb(p, t):
+            with cb_lock:
+                cb_count[0] += 1
+        chunks = [MockChunk(i) for i in range(8)]
+        result = ext.extract(chunks, progress_cb=cb)
+        assert cb_count[0] == 8, f"expected 8 progress, got {cb_count[0]}"
+        assert state["calls"] == 32, f"expected 32 calls, got {state['calls']}"
+        success = sum(1 for item in result.items if item is not None)
+        assert success == 0
+        print(f"  D5 PASS: 8 chunks x 4 attempts, progress_cb={cb_count[0]} (1 per chunk), calls={state['calls']}")
+    finally:
+        time_mod.sleep = orig_sleep
+
+
+def main():
+    print("=" * 60)
+    print("C+D 阶段 5 项 Risk 审核 (commit 0bf4349)")
+    print("=" * 60)
+    d1_ingest_4_workers()
+    d2_429_snowball()
+    d3_progress_cb_thread_safe()
+    d4_max_retries_configurable()
+    d5_retry_no_progress_leak()
+    print("=" * 60)
+    print("5/5 PASS")
+    print("=" * 60)
+
+
+if __name__ == "__main__":
+    main()

package/python/bench/chaos.py

@@ -0,0 +1,140 @@
+"""
+混沌故障注入（Chaos Engineering）· 数字产物
+
+补足 bench 三件套的「故障维度」:
+  · concurrency.py —— 共享库并发写(无故障)
+  · load_test.py    —— HTTP 高并发负载(无故障)
+  · chaos.py(本文件) —— 故障注入:LLM 空响应/异常/fallback/断路器,验证降级语义 + 恢复
+
+验证 ModelRouter(所有 Agent 的 LLM 唯一咽喉)在故障下的三件事:
+  1. 降级语义正确:空响应/全失败**不伪装成功**(对齐 base.py §二「真相优先」);
+  2. fallback 链:主 provider 故障 → 自动回退到已配的次 provider;
+  3. 断路器:故障 provider 进冷却、成功解冻(惊群防护),系统可恢复。
+
+离线:不联网、不碰真实 Key,纯 monkeypatch 注入故障。
+用法:HF_HUB_OFFLINE=1 python bench/chaos.py
+"""
+import os
+import sys
+import time
+from pathlib import Path
+
+sys.path.insert(0, str(Path(__file__).parent.parent))
+
+import foxclaw.core.web_search as ws  # noqa: E402
+ws.needs_search = lambda q: False
+
+from foxclaw.core.model_router import ModelRouter, PROVIDER_ENV  # noqa: E402
+
+_SYS = "你是 Foxclaw 助手。"
+_MSG = [{"role": "user", "content": "用一句话总结量子计算"}]
+
+
+def _make_router(primary="deepseek", fallback=None):
+    """构造一个走真实路径(非 stub)的 router,并把 _client 设成非 None,
+    以便 monkeypatch _call_provider 注入故障。fallback 指定次 provider 并伪造其 key。"""
+    if fallback:
+        os.environ[PROVIDER_ENV[fallback]] = "chaos-test-key"
+    r = ModelRouter(provider=primary)
+    r._client = object()
+    r._clients[primary] = object()
+    if fallback:
+        r._clients[fallback] = object()
+    return r
+
+
+def _boom(provider, client, model, system, messages, max_tokens, images=None):
+    raise RuntimeError("chaos: upstream timeout")
+
+
+def case1_empty_response():
+    """[1] 空响应故障:provider 返回空串 → degraded + 显式失败文案,不伪装成功。"""
+    r = _make_router("deepseek")
+    r._call_provider = lambda *a, **k: ("", None)
+    out = r.complete(_SYS, _MSG)
+    ok = ("【模型返回为空】" in out) and r.is_degraded()
+    return ok, ("degraded=" + str(r.is_degraded())), out[:40]
+
+
+def case2_call_failure():
+    """[2] 调用异常故障:LLM 抛异常 → 全部失败文案 + degraded,绝不拼接 stub 成功文案。"""
+    r = _make_router("deepseek")
+    r._call_provider = _boom
+    out = r.complete(_SYS, _MSG)
+    ok = ("【全部模型失败】" in out) and r.is_degraded() and ("已完成" not in out)
+    return ok, ("degraded=" + str(r.is_degraded()) + " 伪装成功=" + str("已完成" in out)), out[:40]
+
+
+def case3_fallback_chain():
+    """[3] fallback 链:主 provider 抛异常 → 自动回退到已配次 provider 并记录原因。"""
+    r = _make_router("deepseek", fallback="openai")
+    calls = []
+
+    def _flaky(provider, client, model, system, messages, max_tokens, images=None):
+        calls.append(provider)
+        if provider == "deepseek":
+            raise RuntimeError("chaos: deepseek down")
+        return "openai 兜底回复", None
+
+    r._call_provider = _flaky
+    out = r.complete(_SYS, _MSG)
+    ok = (out == "openai 兜底回复") and (calls[-1] == "openai") and bool(r.fallback_reason)
+    return ok, ("调用顺序=" + str(calls) + " 回退原因=" + r.fallback_reason), out[:40]
+
+
+def case4_circuit_breaker():
+    """[4] 断路器:provider 失败后进入冷却,立即重试被跳过(不再紧密重试)。"""
+    r = _make_router("deepseek")
+    r._call_provider = _boom
+    r.complete(_SYS, _MSG)
+    cooled = "deepseek" in r._cooldown
+    t0 = time.perf_counter()
+    r.complete(_SYS, _MSG)
+    fast = (time.perf_counter() - t0) < 0.5
+    ok = cooled and fast and r.is_degraded()
+    return ok, (f"冷却中={cooled} 跳过耗时={((time.perf_counter()-t0)*1000):.0f}ms"), "断路器生效"
+
+
+def case5_recovery():
+    """[5] 恢复:故障清除(cooldown 过期/成功)后,同一 provider 恢复正常服务。"""
+    r = _make_router("deepseek")
+    r._call_provider = _boom
+    r.complete(_SYS, _MSG)
+    assert "deepseek" in r._cooldown
+    r._cooldown["deepseek"] = 0
+    r._call_provider = lambda *a, **k: ("恢复后的正常回复", None)
+    out = r.complete(_SYS, _MSG)
+    ok = (out == "恢复后的正常回复") and ("deepseek" not in r._cooldown)
+    return ok, ("解冻=" + str("deepseek" not in r._cooldown)), out[:40]
+
+
+def main():
+    cases = [
+        ("空响应 → degraded(不伪装成功)", case1_empty_response),
+        ("调用异常 → 全部失败(不拼接 stub)", case2_call_failure),
+        ("fallback 链(主失败→次兜底)", case3_fallback_chain),
+        ("断路器(失败进冷却·重试跳过)", case4_circuit_breaker),
+        ("恢复(冷却过期→正常服务)", case5_recovery),
+    ]
+    print("=" * 64)
+    print("Foxclaw · 混沌故障注入(ModelRouter 咽喉)")
+    print("=" * 64)
+    n_ok = 0
+    for name, fn in cases:
+        try:
+            ok, detail, sample = fn()
+            n_ok += 1 if ok else 0
+            print(f"\n[{name}]")
+            print(f"  {'✓ 通过' if ok else '✗ 失败'}  {detail}")
+            print(f"  输出样例: {sample}…")
+        except Exception as e:
+            print(f"\n[{name}]\n  ✗ 异常: {type(e).__name__}: {e}")
+    print("\n" + "=" * 64)
+    print(f"结论: {n_ok}/{len(cases)} 混沌用例通过 — "
+          f"{'✓ 故障下降级语义正确、fallback/断路器/恢复均生效' if n_ok == len(cases) else '✗ 存在故障处理缺口'}")
+    print("=" * 64)
+    return 0 if n_ok == len(cases) else 1
+
+
+if __name__ == "__main__":
+    sys.exit(main())