foxclaw 0.1.2 → 0.1.4

package/python/foxclaw/core/injection_guard.py

@@ -18,7 +18,6 @@ import os
 import re
 import secrets
 
-# 典型注入/越权特征(中英)。命中不拦截,只在数据块里加醒目告警,提示模型"这是数据里的指令,别执行"。
 _MARKERS = [
     r"ignore\s+(all\s+)?(the\s+)?previous\s+instructions?",
     r"disregard\s+(the\s+)?(above|previous|prior)",
@@ -42,25 +41,17 @@ def scan(text: str) -> bool:
     return bool(text) and bool(_RE.search(str(text)))
 
 
-# ── 输出侧护栏(纵深防御第四道:DLP)────────────────────────────────────────────
-# 回复返回/落库前扫一道:① 密钥/令牌等敏感串 → 脱敏;② 系统提示词被整段吐出 → 拦截。
-# 纯正则、零额外模型调用(适配 flash 成本)。与"输入-隔离-模型"三层凑齐四道墙。
-
-# 常见密钥/令牌形态(含 Foxclaw 实际用的百度千帆 bce-v3 AppBuilder key)。
 _SECRET_PATTERNS = [
-    r"sk-[A-Za-z0-9]{20,}",                                  # OpenAI 风格
-    r"bce-v3/[A-Za-z0-9-]{6,}/[A-Za-z0-9]{6,}",              # 百度千帆 AppBuilder(Foxclaw 在用)
-    r"AKIA[0-9A-Z]{16}",                                     # AWS Access Key
-    r"gh[pousr]_[A-Za-z0-9]{20,}",                           # GitHub token
-    r"xox[baprs]-[A-Za-z0-9-]{10,}",                         # Slack token
-    r"Bearer\s+[A-Za-z0-9._\-]{20,}",                        # Bearer 令牌
+    r"sk-[A-Za-z0-9]{20,}",
+    r"bce-v3/[A-Za-z0-9-]{6,}/[A-Za-z0-9]{6,}",
+    r"AKIA[0-9A-Z]{16}",
+    r"gh[pousr]_[A-Za-z0-9]{20,}",
+    r"xox[baprs]-[A-Za-z0-9-]{10,}",
+    r"Bearer\s+[A-Za-z0-9._\-]{20,}",
     r"(api[_-]?key|secret|token|password|passwd)\s*[:=]\s*['\"]?[A-Za-z0-9._\-/]{12,}",
 ]
 _SECRET_RE = re.compile("|".join(_SECRET_PATTERNS), re.IGNORECASE)
 
-# 系统提示词泄露判定:不用"单个短语命中"(会误伤——如用户正常问"Principle 07 是什么")。
-# 改为**更强信号**:① 输出里出现 KERNEL 的长段**逐字串**(dump 才会逐字复现,提及概念不会);
-# ② 或同时出现 ≥2 个**章节标记**(无 kernel 参照时的兜底)。
 _SECTION_MARKS = ("【一、", "【二、", "【三、", "【四、", "【五、", "【六、", "【七、")
 
 
@@ -101,7 +92,6 @@ def _has_semantic_leak(text: str, kernel: str, threshold: float = 0.88,
         if model is None:
             return False
         import numpy as np
-        # 固定 ~200 字块;向量已 normalize → 余弦 = 点积。内核块进共享缓存(只编码一次)。
         kblocks = [kernel[i:i + 200] for i in range(0, len(kernel), 200)]
         tblocks = [text[i:i + 200] for i in range(0, len(text), 200)]
         kv = encode_cached(model, name, kblocks)
@@ -165,8 +155,6 @@ def guard_output(text: str, kernel: str = "") -> tuple:
     return (text, issues)
 
 
-# 分隔符伪造防护:外部内容里若自带 <untrusted>/</untrusted>,会伪造边界把恶意指令挤出隔离区
-# (经典分隔符注入)。包裹前把内容中一切 <(/)untrusted 的尖括号中和成 ‹,使其无法充当真边界。
 _UNTRUSTED_DELIM = re.compile(r"<\s*/?\s*untrusted", re.IGNORECASE)
 
 
@@ -196,17 +184,10 @@ def wrap_untrusted(text: str, source: str = "external") -> str:
     return block + note
 
 
-# ── 数据污点分级(Muse「eBPF 污点追踪」的纯软件版)────────────────────────────
-# 对齐 Meta Muse 的安全内核:不靠"模型自觉不泄露",而靠"数据一旦来自用户/敏感源,
-# 就在会话上打污点;之后凡外发/副作用动作,污点达标则动态提高审批门槛(见
-# skills.permission 的 taint_threshold)。这里只做「来源 → 敏感度」映射;污点的
-# 传播与裁决在调用方(agents.base / skills.permission)。
-#
-# 敏感度分级(数值越高越敏感,单调可比):
-TAINT_NONE = 0        # 无外部数据(会话初始态)
-TAINT_EXTERNAL = 1    # 外部不可控但非个人:web_search 结果
-TAINT_PERSONAL = 2    # 用户个人数据:知识库(knowledge)/文件(file)
-TAINT_SENSITIVE = 3   # 高敏感个人数据:个人记忆(memory)
+TAINT_NONE = 0
+TAINT_EXTERNAL = 1
+TAINT_PERSONAL = 2
+TAINT_SENSITIVE = 3
 
 _TAINT_BY_SOURCE = {
     "web_search": TAINT_EXTERNAL,

package/python/foxclaw/core/memory_adapter.py

@@ -20,6 +20,9 @@ import threading
 from pathlib import Path
 
 from .shared_write import guard_shared_write
+from .sensitive_write import guard_sensitive_write
+from types import SimpleNamespace
+from typing import Any
 
 
 def _ensure_hmr_importable():
@@ -41,19 +44,18 @@ class HMRAdapter:
     若 HMR 不可用,调用方应回退到 MockHMR。
     """
 
-    def __init__(self, storage_path: str = "./hmr_data", llm_api_key: str = None,
-                 embedding_model: str = None):
+    def __init__(self, storage_path: str = "./hmr_data", llm_api_key: str | None = None,
+                 embedding_model: str | None = None):
         """【P1·share-bge-m3】新增 embedding_model 参数。
         上层(Foxclaw)通常在 HKC 已装配后再调本构造,bge-m3 已入 HKC 缓存;
         传具体模型名(如 HKC_EMBEDDING_MODEL)使 HMR 与 HKC 共享同一 SentenceTransformer 实例。
         不传则走 vendor/hmr/storage/vector_store.py 的 env 解析链(HKC_EMBEDDING_MODEL>bge-m3)。"""
         _ensure_hmr_importable()
-        from hmr.core.hmr import HMR  # 内嵌包
+        from hmr.core.hmr import HMR
 
-        # 不传 None 就用 .env / HKC 的配置;只有上层显式 override 才用 _embedding_model
         import os as _os
         resolved_model = (embedding_model
-                          or _os.environ.get("EMBEDDING_MODEL")     # 【P2·单 embedding 入口】最高优先级
+                          or _os.environ.get("EMBEDDING_MODEL")
                           or _os.environ.get("HKC_EMBEDDING_MODEL")
                           or _os.environ.get("FOXCLAW_EMBEDDING_MODEL")
                           or "all-MiniLM-L6-v2")
@@ -61,13 +63,8 @@ class HMRAdapter:
                         embedding_model=resolved_model)
         self.backend = "HMR"
         self.version = getattr(self._hmr, "VERSION", "2.0.0")
-        # 并行子 Agent 会并发 ingest/recall,底层 HMR 的向量库/记忆图并非线程安全
-        # (向量索引并发 add+search 是未定义行为)。串行化对 HMR 的访问。叶子组件:
-        # 锁内不回调 pm/bus,故不参与全局锁序,无死锁风险。
         self._lock = threading.RLock()
 
-    # HMR 未给 title 时会退化成 "Concept 03:11" 这类纯时间戳(见 hmr.py 的
-    # `title or f"{type.title()} {HH:MM}"`),在前端"记忆信息条"里一眼看不出是什么。
     _AUTO_TITLE = re.compile(r"^[A-Za-z_]+\s+\d{1,2}:\d{2}$")
 
     @staticmethod
@@ -80,23 +77,27 @@ class HMRAdapter:
             if s.startswith(pfx):
                 s = s[len(pfx):].strip()
                 break
-        for sep in ("→", "->"):          # 对话记忆是"问 → 答",标题取问
+        for sep in ("→", "->"):
             if sep in s:
                 s = s.split(sep, 1)[0].strip()
                 break
         s = re.split(r"[。！？!?\n\r]", s, 1)[0].strip()
         return " ".join(s.split())[:24]
 
-    # ── 记忆摄入 ──────────────────────────────────────────
-    def ingest(self, content: str, type: str = "concept", agent_id: str = None,
-               title: str = None, tags: list = None, shared: bool = False, **kwargs):
+    def ingest(self, content: str, type: str = "concept", agent_id: str | None = None,
+               title: str | None = None, tags: list | None = None, shared: bool = False,
+               consent: bool = False, **kwargs):
         """统一摄入接口。agent_id 写入 tags 实现隔离;shared=True 则写入共享层。
-        未给 title 时从内容提炼一个可读标题(否则 HMR 会生成纯时间戳标题)。"""
+        未给 title 时从内容提炼一个可读标题(否则 HMR 会生成纯时间戳标题)。
+
+        consent=True 表示"用户已对该条高敏感内容点头",跳过敏感闸门。唯一允许的调用方是
+        `FoxclawOS.approve_sensitive_write`(与 PolicyGate 的 confirmed=True 同一模式):
+        别处传 True 等于绕过同意流程。"""
         meta_tags = list(tags or [])
         if agent_id:
             meta_tags.append(f"agent:{agent_id}")
         if shared:
-            meta_tags.append("shared")     # 共享记忆:所有 Agent 可召回
+            meta_tags.append("shared")
         metadata = {"tags": meta_tags}
         if "confidence" in kwargs:
             metadata["confidence"] = kwargs["confidence"]
@@ -105,15 +106,19 @@ class HMRAdapter:
         mtype = type if type in valid else "concept"
         if not title:
             title = self._derive_title(content) or None
-        # 【P0-3】共享层写入闸门:必须在**落库之前**判定(见 core/shared_write.py)。
-        # 未挂策略时返回 None 直接放行;audit 只记事件;enforce 才抛 SharedWriteDenied。
         if shared:
             guard_shared_write(self, agent_id, content, mtype, meta_tags)
+        if not consent:
+            blocked = guard_sensitive_write(self, agent_id, content, mtype)
+            if blocked is not None and not blocked.allowed:
+                # 高敏感内容不落库:进待确认队列,由用户同意后再写(见 core/sensitive_write.py)
+                return SimpleNamespace(id="", type=mtype, pending=True,
+                                       pending_id=blocked.pending_id, reason=blocked.reason,
+                                       categories=list(blocked.categories))
         with self._lock:
             return self._hmr.ingest(content, memory_type=mtype, title=title, metadata=metadata)
 
-    # ── 记忆召回 ──────────────────────────────────────────
-    def recall(self, query: str = "", agent_id: str = None, top_k: int = 5,
+    def recall(self, query: str = "", agent_id: str | None = None, top_k: int = 5,
                include_shared: bool = True, **kwargs):
         """统一召回接口。返回 list[dict]。
         默认召回:本 Agent 的记忆 + 共享层(shared)记忆;include_shared=False 只看自己。
@@ -153,8 +158,7 @@ class HMRAdapter:
                         "tags": tags,
                         "_ts": getattr(m, "created_at", None) or getattr(m, "timestamp", None) or "",
                     })
-            out.sort(key=lambda x: str(x["_ts"]), reverse=True)   # 新近优先
-            # 按内容去重(保留最新):否则同一偏好反复表达存成多条,会挤满 limit、把别的偏好挤出召回
+            out.sort(key=lambda x: str(x["_ts"]), reverse=True)
             deduped, seen_c = [], set()
             for x in out:
                 if x["content"] in seen_c:
@@ -195,7 +199,7 @@ class HMRAdapter:
         try:
             with self._lock:
                 mems = fs.list_memories()
-                cand = []                      # tag 预过滤:本 Agent 私有 + 共享
+                cand = []
                 for m in mems:
                     tags = getattr(m, "tags", []) or []
                     if agent_id:
@@ -210,7 +214,7 @@ class HMRAdapter:
                 scored = []
                 for m, tags in cand:
                     vec = vs.vectors.get(getattr(m, "id", None))
-                    if vec is None:            # 少见:该记忆尚无向量 → 现算
+                    if vec is None:
                         try:
                             text = getattr(m, "semantic_summary", "") or (m.content or "")[:500]
                             vec = vs.embedder.embed(text)
@@ -232,7 +236,7 @@ class HMRAdapter:
                     })
                 return out
         except Exception:
-            return None   # 任何异常 → 回退老路径,绝不因根治优化而丢召回能力
+            return None
 
     def _legacy_recall(self, query, agent_id, top_k, include_shared):
         """回退路径:全局召回再按 tag 过滤(HMR 结构不可访问时用)。有完整性局限,见 recall。"""
@@ -250,7 +254,7 @@ class HMRAdapter:
                 own = f"agent:{agent_id}" in tags
                 shared = include_shared and ("shared" in tags)
                 if not (own or shared):
-                    continue  # 既不是本 Agent 的,也不是共享的 → 跳过(隔离保证)
+                    continue
             out.append({
                 "id": getattr(m, "id", ""),
                 "title": getattr(m, "title", ""),
@@ -264,11 +268,6 @@ class HMRAdapter:
                 break
         return out
 
-    # ── 工作区 ────────────────────────────────────────────
-    # 【深审修复·补齐锁覆盖】适配层自述"串行化**所有** HMR 访问"(底层向量库/记忆图非线程安全),
-    # 但此前只有 ingest/recall 加锁;下列同样触碰 self._hmr 的方法漏了锁——与并发 ingest/recall
-    # (并行子 Agent / 并发请求)同时跑会读到 mutation 中的向量索引/字典(未定义行为/迭代崩溃)。
-    # 统一补 self._lock(RLock 叶子锁,锁内不回调 pm/bus,无死锁)。
     def get_workspace(self, agent_id: str):
         with self._lock:
             return self._hmr.get_workspace(agent_id)
@@ -277,7 +276,6 @@ class HMRAdapter:
         with self._lock:
             return self._hmr.save_workspace(agent_id)
 
-    # ── 运行时状态 ────────────────────────────────────────
     def save_runtime_state(self, **kwargs):
         with self._lock:
             return self._hmr.save_runtime_state(**kwargs)
@@ -286,7 +284,6 @@ class HMRAdapter:
         with self._lock:
             return self._hmr.restore_runtime_state()
 
-    # ── 系统状态 ──────────────────────────────────────────
     def get_system_status(self):
         try:
             with self._lock:
@@ -307,8 +304,10 @@ class HMRAdapter:
 
         def _ts(m):
             v = getattr(m, "created_at", None)
+            if v is None:
+                return 0.0
             try:
-                return v.timestamp() if hasattr(v, "timestamp") else float(v or 0)
+                return v.timestamp() if hasattr(v, "timestamp") else float(v)
             except Exception:
                 return 0.0
 
@@ -325,11 +324,12 @@ class HMRAdapter:
                     break
             created = getattr(m, "created_at", None)
             title = (getattr(m, "title", "") or "").strip()
-            # 历史记忆多是 "Concept 03:11" 这类时间戳标题(修复前存的)→ 展示时现推一个可读的,
-            # 让老数据也能看懂;推不出来才退回原标题。
             if not title or self._AUTO_TITLE.match(title):
                 title = self._derive_title(
                     getattr(m, "content", "") or getattr(m, "semantic_summary", "")) or title
+            created_at = str(created or "")
+            if created is not None and hasattr(created, "isoformat"):
+                created_at = created.isoformat()
             out.append({
                 "id": getattr(m, "id", ""),
                 "type": getattr(m, "type", ""),
@@ -337,7 +337,7 @@ class HMRAdapter:
                 "summary": (getattr(m, "semantic_summary", "") or getattr(m, "content", "") or "")[:180],
                 "agent": agent,
                 "shared": "shared" in tags,
-                "created_at": created.isoformat() if hasattr(created, "isoformat") else str(created or ""),
+                "created_at": created_at,
             })
         return out
 
@@ -346,17 +346,11 @@ class HMRAdapter:
         total = 0
         relations = 0
         shared_count = 0
-        by_agent = {}
+        by_agent: dict[str, Any] = {}
         embedding = "TF-IDF n-gram"
         try:
-          # 序列化对 HMR 内部结构(向量库/记忆图)的读,与 ingest/recall 的写互斥,
-          # 避免并发迭代 store.memories("dict changed size")/读 mutation 中的 faiss 索引。
           with self._lock:
-            # 权威来源:memory_fs 里的真实记忆对象。
-            # 历史 bug:此前从 vector_store 探测 count/size/memories 取 total——但真实记忆存在
-            # memory_fs,vector_store 只存嵌入且无这些属性 → total 恒为 0,而 relations 却有值(109),
-            # 自相矛盾。改为直接数 memory_fs.list_memories()(与 get_statistics 同源)。
-            mems = []
+            mems: list[Any] = []
             mfs = getattr(self._hmr, "memory_fs", None)
             if mfs is not None and hasattr(mfs, "list_memories"):
                 try:
@@ -364,12 +358,11 @@ class HMRAdapter:
                 except Exception:
                     mems = []
             total = len(mems)
-            if not total:   # 兜底:memory_fs 不可用 → 数 vector_store.vectors(dict)
+            if not total:
                 vs0 = getattr(self._hmr, "vector_store", None)
                 vecs = getattr(vs0, "vectors", None) if vs0 is not None else None
                 if isinstance(vecs, dict):
                     total = len(vecs)
-            # shared / 按 Agent 分布:agent_id 编码在 tags 里的 "agent:<id>"
             for m in mems:
                 tags = getattr(m, "tags", None)
                 if tags is None and isinstance(m, dict):
@@ -386,7 +379,6 @@ class HMRAdapter:
                 edges = getattr(graph, "edges", None)
                 if edges is not None:
                     relations = len(edges) if hasattr(edges, "__len__") else 0
-            # embedding 模型探测
             emb = getattr(self._hmr, "embedding", None) or getattr(self._hmr, "embedder", None)
             if emb is not None:
                 name = type(emb).__name__
@@ -423,19 +415,20 @@ class MockHMR:
     def save_workspace(self, agent_id: str):
         pass
 
-    def ingest(self, content, type="concept", agent_id=None, shared=False, **kwargs):
-        # 【P0-3】与 HMRAdapter 同一道闸门:Mock 后端也过策略。否则在无 HMR 的环境里
-        # 策略形同虚设 —— 而测试与降级场景恰好都落在这一支,那是最需要行为一致的地方。
+    def ingest(self, content, type="concept", agent_id=None, shared=False,
+               consent=False, **kwargs):
         if shared:
             guard_shared_write(self, agent_id, content, type, kwargs.get("tags") or [])
-        from types import SimpleNamespace
-        # 注意:不能用内置 type() 构造对象——参数名 type 已覆盖内置 type。
+        if not consent:
+            blocked = guard_sensitive_write(self, agent_id, content, type)
+            if blocked is not None and not blocked.allowed:
+                return SimpleNamespace(id="", type=type, pending=True,
+                                       pending_id=blocked.pending_id, reason=blocked.reason,
+                                       categories=list(blocked.categories))
         return SimpleNamespace(id="mock_mem", type=type)
 
-    def recall(self, query: str = "", agent_id: str = None, top_k: int = 5, **kwargs):
+    def recall(self, query: str = "", agent_id: str | None = None, top_k: int = 5, **kwargs):
         from types import SimpleNamespace
-        # 返回带 memory_objects 的对象,与真实 HMR 的 RecallResult 结构一致
-        # (HMRAdapter.recall 会读 result.memory_objects)。
         return SimpleNamespace(memory_objects=[])
 
     def save_runtime_state(self, **kwargs):
@@ -484,7 +477,7 @@ def _memory_chain_enabled() -> bool:
     return raw.lower() in ("1", "true", "yes", "on")
 
 
-def create_memory_backend(storage_path: str = "./hmr_data", llm_api_key: str = None):
+def create_memory_backend(storage_path: str = "./hmr_data", llm_api_key: str | None = None):
     """工厂:默认 HMR;`FOXCLAW_MEMORY_BACKEND=vecrecall` 时才切 vecRecall。
 
     顺序 vecRecall → HMR → MockHMR,任一层失败退到下一层,且**明确打印**实际用的是
@@ -508,9 +501,9 @@ def create_memory_backend(storage_path: str = "./hmr_data", llm_api_key: str = N
         print(f"[Foxclaw] ⚠ FOXCLAW_MEMORY_BACKEND={choice!r} 无法识别,按 HMR 处理")
 
     try:
-        adapter = HMRAdapter(storage_path=storage_path, llm_api_key=llm_api_key)
-        print(f"[Foxclaw] ✓ HMR v{adapter.version} 已接入 (backend={adapter.backend})")
-        return adapter
+        hmr = HMRAdapter(storage_path=storage_path, llm_api_key=llm_api_key)
+        print(f"[Foxclaw] ✓ HMR v{hmr.version} 已接入 (backend={hmr.backend})")
+        return hmr
     except Exception as e:
         print(f"[Foxclaw] ⚠ HMR 不可用 ({e.__class__.__name__}: {e}),降级 MockHMR")
         return MockHMR()

package/python/foxclaw/core/metrics.py

@@ -5,7 +5,7 @@ import time
 import threading
 from collections import defaultdict, deque
 
-_MAX_SAMPLES = 500   # 每个 key 保留最近 N 次延迟(有界,防内存涨)
+_MAX_SAMPLES = 500
 
 
 class _Stat:
@@ -14,7 +14,7 @@ class _Stat:
     def __init__(self):
         self.count = 0
         self.errors = 0
-        self.samples = deque(maxlen=_MAX_SAMPLES)
+        self.samples: deque[float] = deque(maxlen=_MAX_SAMPLES)
 
 
 def _pct(sorted_vals, p: float) -> float:
@@ -29,8 +29,8 @@ class MetricsCollector:
 
     def __init__(self):
         self._lock = threading.RLock()
-        self._ep: dict = defaultdict(_Stat)     # endpoint(路由模板) -> _Stat
-        self._llm: dict = defaultdict(_Stat)    # provider -> _Stat
+        self._ep: dict = defaultdict(_Stat)
+        self._llm: dict = defaultdict(_Stat)
         self._start = time.time()
 
     def record_request(self, endpoint: str, status: int, ms: float):
@@ -85,5 +85,4 @@ class MetricsCollector:
         return "\n".join(out) + "\n"
 
 
-# 进程级单例:中间件 + router 都往这里写,端点从这里读。
 METRICS = MetricsCollector()