foxclaw 0.1.0 → 0.1.3

package/python/foxclaw/core/compaction.py

@@ -0,0 +1,265 @@
+"""
+Foxclaw Core — 上下文压缩 (Compaction)
+
+⚠ 【预留组件·无生产接入点】(2026-07-17 V3·B1 审核标注)
+   本压缩器**无任何生产调用路径**(仅 tests/test_governance.py::test_compaction 覆盖——
+   说明它曾被认真实现+验证,只是没接进执行)。根因:Foxclaw 领域 agent 每次
+   run_task/stream_reply 的 messages 都是**单轮新建**、不累积长对话历史;channel 层的
+   conversation_id 是回复路由 + 记忆隔离 key(见 dispatcher),**不维护 LLM 对话历史缓冲**。
+   故当前**没有会撑爆上下文窗口的长历史可压**。
+   这是**能力预留**,不是在工作的机制。若未来出现长会话累积路径(如真做多轮对话记忆),
+   在其 LLM 调用前接入 `Compactor.needs_compaction()` → `compact()` 即可。
+   —— 见 docs/internal/AGENT_HARNESS_REMEDIATION_PLAN.md「未接线清单」。
+
+借鉴 OpenClaw 的渐进降级压缩思路,适配 Foxclaw。长对话会撑爆上下文窗口,
+压缩把早期历史摘要化,腾出空间,同时保住关键信息(ID/数字等标识符不丢)。
+
+策略(渐进降级,任一级失败退到下一级,绝不崩):
+  1. LLM 摘要:把早期消息交给模型生成摘要(保留标识符)
+  2. 部分摘要:若含超大消息,排除它们再摘要
+  3. 截断兜底:摘要都失败 → 保留首尾、中间用占位说明替代
+始终保留:系统提示 + 最近 keep_recent 轮原样(近期上下文最重要)。
+"""
+from __future__ import annotations
+import re
+from dataclasses import dataclass, field
+
+
+def estimate_tokens(text: str) -> int:
+    if not text:
+        return 0
+    cjk = len(re.findall(r"[\u4e00-\u9fff]", text))
+    other = len(text) - cjk
+    return cjk + other // 4 + 1
+
+
+def estimate_messages_tokens(messages: list) -> int:
+    return sum(estimate_tokens(str(m.get("content", ""))) for m in messages)
+
+
+_IDENTIFIER_PAT = re.compile(
+    r"[a-zA-Z]{2,}_[a-zA-Z0-9]{4,}|[a-f0-9]{8,}|[0-9]{6,}|¥[\d,]+(?:\.\d+)?")
+
+
+def extract_identifiers(messages: list) -> list[str]:
+    """从消息里提取关键标识符,确保压缩后仍可引用。"""
+    ids = []
+    for m in messages:
+        for match in _IDENTIFIER_PAT.findall(str(m.get("content", ""))):
+            if match not in ids:
+                ids.append(match)
+    return ids[:50]
+
+
+@dataclass
+class CompactionResult:
+    messages: list = field(default_factory=list)
+    method: str = "none"
+    original_count: int = 0
+    compacted_count: int = 0
+    original_tokens: int = 0
+    compacted_tokens: int = 0
+    preserved_identifiers: list = field(default_factory=list)
+    note: str = ""
+
+    def to_dict(self) -> dict:
+        return {
+            "method": self.method,
+            "original_count": self.original_count,
+            "compacted_count": self.compacted_count,
+            "original_tokens": self.original_tokens,
+            "compacted_tokens": self.compacted_tokens,
+            "saved_tokens": self.original_tokens - self.compacted_tokens,
+            "preserved_identifiers": self.preserved_identifiers,
+            "note": self.note,
+        }
+
+
+def group_tool_calls(messages: list) -> list:
+    """把 messages 切成**不可分割的原子组**列表。
+
+    function-calling 会话中,`assistant`(带 tool_calls)与其后续 `tool` 结果
+    必须**成对保留** —— 孤立引用会被 OpenAI/Anthropic 拒绝(400:
+    "tool_call_id not found" / "unexpected tool role")。
+
+    病根:原实现只按 `role == "system"` 分区、再按**条数**切 keep_recent,
+    边界若切在一轮中间就会切出「有 tool 结果、但其 assistant.tool_calls
+    已被摘要掉」的碎片。base._reply_with_tools 每轮追加 1 个 assistant
+    (带 N 个 tool_calls)+ N 个 tool 结果,故一轮内多工具时必现。
+    实测:一轮 2 个工具 + keep_recent=3 → 孤立 call_2 → 送回 provider 必 400。
+
+    本函数按「声明-结果」结构分组,调用方改为**按组**保留,保证配对完整。
+    """
+    groups: list[list[dict]] = []
+    buf: list[dict] = []
+    i = 0
+    n = len(messages)
+    while i < n:
+        m = messages[i]
+        buf.append(m)
+        if m.get("role") == "assistant" and (m.get("tool_calls") or []):
+            ids = {tc.get("id") for tc in (m.get("tool_calls") or [])}
+            j = i + 1
+            while j < n:
+                nxt = messages[j]
+                if nxt.get("role") == "tool" and nxt.get("tool_call_id") in ids:
+                    buf.append(nxt)
+                    j += 1
+                else:
+                    break
+            i = j
+        else:
+            i += 1
+        groups.append(buf)
+        buf = []
+    return groups
+
+
+class Compactor:
+    """上下文压缩器。超阈值时把早期历史摘要化,保留系统提示与近期消息。"""
+
+    def __init__(self, router=None, max_tokens: int = 6000,
+                 keep_recent: int = 4, oversized_token: int = 2000):
+        self.router = router
+        self.max_tokens = max_tokens
+        self.keep_recent = keep_recent
+        self.oversized_token = oversized_token
+
+    def needs_compaction(self, messages: list) -> bool:
+        return estimate_messages_tokens(messages) > self.max_tokens
+
+    def compact(self, messages: list) -> CompactionResult:
+        """压缩消息列表。渐进降级,绝不抛异常。"""
+        res = CompactionResult(
+            original_count=len(messages),
+            original_tokens=estimate_messages_tokens(messages))
+
+        if not self.needs_compaction(messages):
+            res.messages = list(messages)
+            res.method = "none"
+            res.compacted_count = len(messages)
+            res.compacted_tokens = res.original_tokens
+            return res
+
+        system_msgs = [m for m in messages if m.get("role") == "system"]
+        non_system = [m for m in messages if m.get("role") != "system"]
+        keep = max(0, self.keep_recent)
+        _groups = group_tool_calls(non_system)
+        if keep and _groups:
+            recent = [m for g in _groups[-keep:] for m in g]
+            to_compress = [m for g in _groups[:-keep] for m in g]
+        elif keep:
+            recent, to_compress = [], non_system
+        else:
+            recent, to_compress = [], non_system
+        _declared = {tc.get("id") for m in (recent + to_compress)
+                     for tc in (m.get("tool_calls") or [])}
+        _orphans = [m for m in (recent + to_compress)
+                    if m.get("role") == "tool" and m.get("tool_call_id") not in _declared]
+        if _orphans:
+            res.messages = list(messages)
+            res.method = "skipped_unpaired_tool"
+            res.compacted_count = len(messages)
+            res.compacted_tokens = res.original_tokens
+            res.note = (f"压缩跳过:检测到 {len(_orphans)} 条孤立 tool 消息"
+                        f"(其 tool_calls 声明不在保留集内),压缩会触发 provider 400。")
+            return res
+
+        if not to_compress:
+            res.messages = list(messages)
+            res.method = "none"
+            res.compacted_count = len(messages)
+            res.compacted_tokens = res.original_tokens
+            return res
+
+        res.preserved_identifiers = extract_identifiers(to_compress)
+
+        summary = None
+        method = "truncate"
+        if self.router is not None:
+            summary = self._try_summarize(to_compress, res.preserved_identifiers)
+            if summary:
+                method = "llm_summary"
+        if summary is None and self.router is not None:
+            trimmed = [m for m in to_compress
+                       if estimate_tokens(str(m.get("content", ""))) <= self.oversized_token]
+            if trimmed and len(trimmed) < len(to_compress):
+                summary = self._try_summarize(trimmed, res.preserved_identifiers)
+                if summary:
+                    method = "partial_summary"
+        if summary is None:
+            summary = self._truncate_fallback(to_compress, res.preserved_identifiers)
+            method = "truncate"
+
+        summary_msg = {"role": "system",
+                       "content": f"【早期对话摘要(已压缩 {len(to_compress)} 条)】\n{summary}"}
+        compacted = system_msgs + [summary_msg] + recent
+        compacted_tokens = estimate_messages_tokens(compacted)
+
+        if compacted_tokens >= res.original_tokens * 0.9:
+            res.messages = list(messages)
+            res.method = "skipped_ineffective"
+            res.compacted_count = len(messages)
+            res.compacted_tokens = res.original_tokens
+            sys_tokens = estimate_messages_tokens(system_msgs)
+            res.note = (f"压缩跳过:有效压缩空间不足(系统消息占 {sys_tokens} token)。"
+                        f"建议精简系统提示或调大上下文预算。")
+            return res
+
+        res.messages = compacted
+        res.method = method
+        res.compacted_count = len(res.messages)
+        res.compacted_tokens = compacted_tokens
+        return res
+
+    def maybe_compact(self, messages: list) -> tuple:
+        """接入用入口:需要压缩才压,返回 (消息列表, CompactionResult | None)。
+
+        与 compact() 的差别:未超预算时不产生 CompactionResult(调用方无需判别),
+        跳过/无效时返回**原始消息**而不是半成品;任何异常都降级为原样返回。
+        供 agents/base.py::_compact_context 在每次 LLM 调用前挂接。
+        """
+        try:
+            if not self.needs_compaction(messages):
+                return list(messages), None
+            res = self.compact(messages)
+            if res.method == "none" or res.method.startswith("skipped"):
+                return list(messages), res
+            return res.messages, res
+        except Exception:
+            return list(messages), None
+
+    def _try_summarize(self, messages: list, identifiers: list) -> str | None:
+        """用 LLM 摘要,失败返回 None(交给下一级降级)。"""
+        try:
+            convo = "\n".join(
+                f"{m.get('role','?')}: {str(m.get('content',''))[:1000]}"
+                for m in messages)
+            id_hint = ("\n必须原样保留这些标识符: " + ", ".join(identifiers)
+                       if identifiers else "")
+            sys_prompt = ("你是对话摘要助手。把以下对话压缩成简洁摘要,保留关键决策、"
+                          "事实、未完成事项。" + id_hint)
+            out = self.router.complete(
+                system=sys_prompt,
+                messages=[{"role": "user", "content": f"对话内容:\n{convo}\n\n请输出摘要:"}],
+                max_tokens=800, agent_id="_compaction")
+            if out and isinstance(out, str) and len(out.strip()) > 10:
+                return out.strip()
+        except Exception:
+            pass
+        return None
+
+    def _truncate_fallback(self, messages: list, identifiers: list) -> str:
+        """摘要失败的兜底:保留首尾消息要点 + 关键标识符,中间略去。"""
+        parts = []
+        if messages:
+            first = str(messages[0].get("content", ""))[:150]
+            parts.append(f"对话开始: {first}")
+        if len(messages) > 2:
+            parts.append(f"(中间 {len(messages)-2} 条消息略)")
+        if len(messages) > 1:
+            last = str(messages[-1].get("content", ""))[:150]
+            parts.append(f"最近: {last}")
+        if identifiers:
+            parts.append("关键标识符: " + ", ".join(identifiers))
+        return "\n".join(parts)

package/python/foxclaw/core/config.py

@@ -0,0 +1,65 @@
+"""分层配置(E1 —— 范式「信任半径递减」的轻量落地)。
+
+三层,信任从高到低:
+  · policy   —— 管理员/最高。锁定项**不可被下层覆盖或撤销**(对应范式 deny 铁律)。
+  · user     —— 运行时可改(如 skill_policy.json 的 deny/allow)。
+  · runtime  —— 环境变量 / 默认值。
+
+合并规则:
+  · 标量:高层覆盖低层(policy > user > runtime)。
+  · 集合(如 deny 安全规则):各层**并集,只增不减** —— 高层只能加严、不能被下层放松。
+
+刻意只做三层(不照搬范式六层):够 Foxclaw 用,且**立即接线**到 SkillPolicy 的 deny 治理
+(管理员强制 deny 某 skill、用户不可 undeny),不是「建了没接线」的孤儿设施。
+"""
+from __future__ import annotations
+import os
+import json
+from pathlib import Path
+
+
+class LayeredConfig:
+    """三层配置合并原语。持有各层 dict 的引用 —— 若某层的集合值是可变对象(如 SkillPolicy 的
+    user deny set),原地增删会被 merged_set/policy_locked 实时反映,无需重建。"""
+
+    def __init__(self, policy: dict | None = None, user: dict | None = None, runtime: dict | None = None):
+        self.policy = dict(policy or {})
+        self.user = dict(user or {})
+        self.runtime = dict(runtime or {})
+
+    def get(self, key: str, default=None):
+        """标量:高层优先(policy > user > runtime)。"""
+        for layer in (self.policy, self.user, self.runtime):
+            if key in layer:
+                return layer[key]
+        return default
+
+    def merged_set(self, key: str) -> set:
+        """集合类(deny 等):三层并集,只增不减(高层只能加严)。"""
+        out: set = set()
+        for layer in (self.policy, self.user, self.runtime):
+            out |= set(layer.get(key) or [])
+        return out
+
+    def policy_locked(self, key: str) -> set:
+        """policy 层锁定的项 —— 不可被下层覆盖/撤销(deny 铁律)。"""
+        return set(self.policy.get(key) or [])
+
+
+def load_policy_layer(data_dir=None) -> dict:
+    """加载 policy 层(管理员锁定,与 user 层的 skill_policy.json **物理分开**):
+      · 环境变量 `FOXCLAW_POLICY_DENY`(逗号分隔的 skill id);
+      · `<data_dir>/policy.json` 的 `{"deny": [...]}`(若存在)。
+    默认(两者都无)返回空 deny → 合并结果 = 现状(behavior-preserving)。"""
+    deny: set = set()
+    env = os.environ.get("FOXCLAW_POLICY_DENY", "")
+    deny |= {s.strip() for s in env.split(",") if s.strip()}
+    if data_dir:
+        p = Path(data_dir) / "policy.json"
+        if p.exists():
+            try:
+                d = json.loads(p.read_text(encoding="utf-8"))
+                deny |= set(d.get("deny") or [])
+            except Exception:
+                pass
+    return {"deny": sorted(deny)}

package/python/foxclaw/core/confirm_nonce.py

@@ -0,0 +1,151 @@
+"""关键操作的会话 nonce 二次确认(Confirmation Broker)。
+
+**问题**:send / pay / destructive 这类动作由权限管线挡成 `ask`(需确认),但"确认"本身
+只是一个布尔——`run_task(..., _confirmed=True)` 或 `POST /api/ahs/run {confirmed:true}`
+就能放行。它没有绑定**确认的是哪一件事**:昨天确认过"给 A 发周报",今天同一个
+`confirmed=true` 会连"给 B 转账"一起放行;调用方也可以无脑重发同一个 true 反复执行。
+
+**解法**:确认变成一次性凭据——
+  · `issue(session, action, args)` 签发 nonce,绑定 `会话 + 动作 + 参数指纹 + 有效期`;
+  · 执行前 `consume(...)` 校验并**立即作废**(单次使用),参数变了指纹就对不上;
+  · 过期即废(默认 300s),超窗重放无效。
+
+**边界(诚实)**:nonce 只证明"调用方看到过这次待确认",不证明"是人类按的"。真正的
+人在回路仍需渠道/UI 把人接进环路;本模块解决的是"确认对象漂移"和"确认可重放"。
+
+线程安全 + 有界:nonce 来自外部,无界集合等于开一个内存耗尽面。
+"""
+from __future__ import annotations
+
+import hashlib
+import json
+import secrets
+import threading
+import time
+from dataclasses import dataclass, field
+from typing import Optional, Tuple
+
+CRITICAL_SIDE_EFFECTS = ("send", "pay")
+
+DEFAULT_TTL = 300.0
+
+
+def action_fingerprint(action: str, args: dict) -> str:
+    """动作指纹:同名 + 同参(字典序无关)= 同一件事。
+
+    参数进指纹是关键:确认"给 A 发"之后把 args 改成 B,指纹不匹配 → 拒绝执行,
+    堵掉「确认了这件事、执行了另一件」。
+    """
+    try:
+        payload = json.dumps(args or {}, sort_keys=True, ensure_ascii=False, default=str)
+    except Exception:
+        payload = repr(args)
+    digest = hashlib.sha256(payload.encode("utf-8")).hexdigest()[:16]
+    return f"{action}#{digest}"
+
+
+def is_critical(side_effect: str, destructive: bool = False) -> bool:
+    """该动作是否必须走 nonce 二次确认。"""
+    return bool(destructive) or (side_effect or "") in CRITICAL_SIDE_EFFECTS
+
+
+@dataclass
+class PendingConfirm:
+    """一条待确认记录。"""
+    nonce: str
+    action: str
+    fingerprint: str
+    session: str = ""
+    issued: float = field(default_factory=time.time)
+    expires: float = 0.0
+    used: bool = False
+
+    def to_dict(self) -> dict:
+        return {"nonce": self.nonce, "action": self.action,
+                "session": self.session, "issued": self.issued,
+                "expires_at": self.expires, "used": self.used}
+
+
+class ConfirmationBroker:
+    """签发 / 校验一次性确认码。"""
+
+    def __init__(self, ttl: float = DEFAULT_TTL, max_pending: int = 256,
+                 clock=time.time):
+        self.ttl = float(ttl)
+        self.max_pending = max(8, int(max_pending))
+        self._clock = clock
+        self._items: dict[str, PendingConfirm] = {}
+        self._lock = threading.Lock()
+
+    def issue(self, session: str, action: str, args: dict,
+              ttl: float | None = None) -> dict:
+        """签发一个绑定到 (会话, 动作, 参数) 的一次性确认码。"""
+        now = self._clock()
+        ttl = self.ttl if ttl is None else float(ttl)
+        item = PendingConfirm(
+            nonce=secrets.token_hex(8),
+            action=action or "",
+            fingerprint=action_fingerprint(action or "", args or {}),
+            session=session or "",
+            issued=now, expires=now + ttl)
+        with self._lock:
+            self._purge_locked(now)
+            if len(self._items) >= self.max_pending:
+                oldest = min(self._items.values(), key=lambda it: it.issued)
+                self._items.pop(oldest.nonce, None)
+            self._items[item.nonce] = item
+        return {"nonce": item.nonce, "action": item.action,
+                "expires_at": item.expires, "ttl": ttl}
+
+    def consume(self, session: str, nonce: str, action: str = "",
+                args: dict | None = None, now: float | None = None) -> Tuple[bool, str]:
+        """校验确认码并**立即作废**。返回 (是否通过, 原因码)。
+
+        原因码:``ok`` / ``missing_nonce`` / ``unknown_nonce`` / ``already_used`` /
+        ``expired`` / ``session_mismatch`` / ``action_mismatch`` / ``args_mismatch``。
+        会话为空(任一侧)时不校验会话——便于进程内调用;HTTP 入口务必带 session。
+        """
+        if not nonce:
+            return False, "missing_nonce"
+        now = now if now is not None else self._clock()
+        with self._lock:
+            item: Optional[PendingConfirm] = self._items.get(nonce)
+            if item is not None and item.expires <= now:
+                self._items.pop(nonce, None)
+                return False, "expired"
+            self._purge_locked(now)
+            if item is None:
+                return False, "unknown_nonce"
+            if item.used:
+                return False, "already_used"
+            if item.session and session and item.session != session:
+                return False, "session_mismatch"
+            if action and item.action and item.action != action:
+                return False, "action_mismatch"
+            if args is not None and item.fingerprint != action_fingerprint(item.action, args):
+                return False, "args_mismatch"
+            item.used = True
+            self._items.pop(nonce, None)
+            return True, "ok"
+
+    def pending(self, session: str = "") -> list:
+        """未消费且未过期的确认码(可按会话过滤),供排查/前端展示。"""
+        now = self._clock()
+        with self._lock:
+            self._purge_locked(now)
+            return [it.to_dict() for it in self._items.values()
+                    if not it.used and (not session or it.session == session)]
+
+    def purge(self, now: float | None = None) -> int:
+        now = now if now is not None else self._clock()
+        with self._lock:
+            return self._purge_locked(now)
+
+    def _purge_locked(self, now: float) -> int:
+        dead = [k for k, it in self._items.items() if it.expires <= now]
+        for k in dead:
+            self._items.pop(k, None)
+        return len(dead)
+
+    def __len__(self) -> int:
+        return len(self._items)

package/python/foxclaw/core/dispatch.py

@@ -0,0 +1,27 @@
+"""多步 dispatch 结果判定谓词（P0-1 反思重规划的前提条件）。
+
+**为什么放在 core**：`pipeline` 的 replan stage 需要判断「上一步是否失败」，但
+`pipeline` 不能反向 import `agents` —— 那会形成 `agents ↔ pipeline` 包级环
+（`pipeline/supervisor_stages.py` 曾用「函数内局部 import」回避，注释里也写明了）。
+谓词下沉到这里后，两侧都只依赖 `core`，依赖方向单一：
+
+    agents ──▶ core ◀── pipeline
+
+纯函数、无副作用、不 import 任何上层模块。
+"""
+
+
+def dispatch_failed(dr) -> bool:
+    """判断多步 dispatch 结果是否含失败/未完成步骤（触发反思重规划的前提）。"""
+    if not isinstance(dr, dict):
+        return False
+    result = dr.get("result") or {}
+    if not isinstance(result, dict):
+        return False
+    if result.get("dispatched_to") not in ("multi", "parallel"):
+        return False
+    if result.get("blocked"):
+        return True
+    steps = result.get("parallel_steps") or result.get("multi_steps") or []
+    return any((s.get("failed") or s.get("blocked"))
+               for s in steps if isinstance(s, dict))

package/python/foxclaw/core/envfile.py

@@ -0,0 +1,48 @@
+"""把 `.env` 灌进 `os.environ`,规则只有一条:**非空值优先,空值让路**。
+
+为什么不用 `load_dotenv()` 的默认行为(`override=False`):
+系统环境变量里只要有同名键,就会压住 `.env`。用户机器上残留一个早就失效的
+旧 key 时,现象是「我明明在 .env 填了新 key,还是 401」,而报错信息完全指向
+不了这一层 —— 排查成本极高。2026-11-08 就踩了这个:Machine 级的
+DEEPSEEK_API_KEY 是失效的,`.env` 里是新 key,后端一路读到的都是旧的。
+
+为什么也不用 `override=True`:
+`.env` / `.env.example` 里有大量**空值占位**(`ANTHROPIC_API_KEY=` 这种)。
+整份无条件覆盖会把这些键清成空串 —— 用户真在系统环境里配好的 key 被抹掉,
+从「悄悄用错」变成「悄悄清空」,更糟。
+
+所以折中:空值视为「未配置」,不参与覆盖;非空值一定生效,并回报它压掉了
+哪些系统变量(只回报名字,不碰值),让这类「幽灵变量」可见。
+"""
+
+from __future__ import annotations
+
+import os
+from pathlib import Path
+from typing import List
+
+
+def load_env_file(path: str | Path) -> List[str]:
+    """读取 env 文件并写入 `os.environ`;返回被覆盖掉同名系统变量的键名列表。
+
+    env 文件不存在 → 返回空列表(不抛)。未装 python-dotenv → 由 ImportError
+    冒给调用方决定降级,本函数不吞。
+    """
+    from dotenv import dotenv_values
+
+    target = Path(path)
+    if not target.exists():
+        return []
+
+    shadows: List[str] = []
+    for key, value in (dotenv_values(target) or {}).items():
+        if not key:
+            continue
+        value = (value or "").strip()
+        if not value:
+            continue
+        current = os.environ.get(key, "").strip()
+        if current and current != value:
+            shadows.append(key)
+        os.environ[key] = value
+    return shadows