codebee 0.1.76 → 0.1.78

package/app/core/repeat_guard.py

@@ -1,129 +1,129 @@
-# -*- coding: utf-8 -*-
-"""重复 CLI 调用检测：防止编排者用同一份 prompt 反复调同一 CLI 死循环。
-
-设计稿：docs/migration/01-defense-patterns.md §5C。
-参考 dsh packages/guard/repeat-tool-reminder/src/index.ts：
-  监听 tools/post-execute，按 (tool, canonical_args) 跟踪，3/5/8 阈值注入 user-message 提醒。
-
-Tutti 的 "tool" 对应到 step role + 执行者身份，"canonical_args" 对应到 prompt
-模板指纹。不同 CLI 间的换将是有效的回退，不能共享同一个重复预算。步骤级调用
-代价远高于普通工具调用，因此第 3 次提醒、第 5 次强制停止。
-"""
-from __future__ import annotations
-
-import hashlib
-import logging
-import threading
-import time
-
-log = logging.getLogger(__name__)
-
-# 渐进阈值：第 N 次命中时给提醒；最后一次及以上时强制停止。
-DEFAULT_THRESHOLDS = (3, 5)
-
-REMINDER_TEMPLATE = (
-    "⚠️ 提醒：上一步使用了几乎相同的输入（已连续 {count} 次调用 {role}）。"
-    "请换一种方法：读不同的文件、用更具体的指令、或明确询问用户澄清。"
-)
-
-STOP_TEMPLATE = (
-    "已连续 {count} 次使用相同输入调用 {role}，超过 {last_thr} 次阈值，"
-    "强制停止本 step 以防止无限循环。请用户检查输入或调整 prompt 模板。"
-)
-
-
-def _prompt_fingerprint(prompt: str, identity: str = "") -> str:
-    """prompt 指纹：用前 200 字符 + 总长度（CLI prompt 头几行通常是模板部分）。
-
-    不做严格 canonical args 比对（CLI prompt 通常很长，且 dsh 也用 JSON.stringify）；
-    指纹命中说明是同一个模板路径走过多次，已经够分辨。
-
-    用 SHA-256 取前 12 字符（96 bit）：48 亿分之一碰撞概率对去重足够；不追求密码学强度。
-    """
-    head = (prompt or "")[:200]
-    return hashlib.sha256(
-        f"{identity}\0{head}|{len(prompt or '')}".encode("utf-8")).hexdigest()[:12]
-
-
-class RepeatGuard:
-    """每 (run_id, step_role) 的最近执行链独立计数；指纹包含执行者身份。
-
-    计数仅统计相同 key 连续出现；新 prompt 会重置。
-    """
-
-    def __init__(self, thresholds=DEFAULT_THRESHOLDS, *, max_history=50):
-        self.thresholds = tuple(sorted(set(thresholds)))
-        self.max_history = max_history
-        self._lock = threading.Lock()
-        # chain[(run_id, role)] = [(ts, fingerprint)]
-        self._chains: dict = {}
-
-    def check(self, run_id: str, role: str, prompt: str, *, identity: str = "") -> dict:
-        """检查并更新 (run_id, role) 的连续计数。
-
-        identity 标识实际执行 CLI/agent。相同 prompt 换到其他执行者属于回退，
-        会从新的连续链计数；同一执行者重复相同调用仍按阈值停止。
-
-        Returns:
-            {"count": int, "reminder": str | None, "should_stop": bool}
-        """
-        fp = _prompt_fingerprint(prompt, str(identity or ""))
-        chain_key = (run_id, role)
-        with self._lock:
-            chain = self._chains.setdefault(chain_key, [])
-            chain.append((time.time(), fp))
-            # 仅统计相同 fp 连续出现
-            count = 1
-            for _, prev_fp in reversed(chain[:-1]):
-                if prev_fp == fp:
-                    count += 1
-                else:
-                    break
-            # 截尾避免内存膨胀
-            if len(chain) > self.max_history:
-                self._chains[chain_key] = chain[-self.max_history:]
-
-        reminder = None
-        stop = False
-        last_thr = self.thresholds[-1]
-        if count >= last_thr:
-            stop = True
-            reminder = STOP_TEMPLATE.format(count=count, role=role, last_thr=last_thr)
-        elif count in self.thresholds:
-            reminder = REMINDER_TEMPLATE.format(count=count, role=role)
-
-        if reminder:
-            log.warning("repeat-guard %s/%s count=%d stop=%s reminder=%s",
-                        run_id, role, count, stop, reminder[:80])
-        return {"count": count, "reminder": reminder, "should_stop": stop}
-
-    def reset(self, run_id: str) -> int:
-        """重置某 run_id 的所有 (run_id, *) 链。返回清除的链条数。"""
-        with self._lock:
-            cleared = 0
-            for k in list(self._chains.keys()):
-                if k[0] == run_id:
-                    del self._chains[k]
-                    cleared += 1
-            return cleared
-
-    def stats(self, run_id: str, role: str) -> dict:
-        """读取 (run_id, role) 当前状态（不更新）。用于 UI 调试。"""
-        chain_key = (run_id, role)
-        with self._lock:
-            chain = self._chains.get(chain_key, [])
-            if not chain:
-                return {"chain_len": 0, "last_fp": ""}
-            # 当前连续计数
-            last_fp = chain[-1][1]
-            count = 1
-            for _, fp in reversed(chain[:-1]):
-                if fp == last_fp:
-                    count += 1
-                else:
-                    break
-            return {"chain_len": len(chain), "last_fp": last_fp, "count": count}
-
-
-# 模块级默认实例（pipeline 接线用；run_id 入键天然隔离多任务，测试无需替换）
-guard = RepeatGuard()
+# -*- coding: utf-8 -*-
+"""重复 CLI 调用检测：防止编排者用同一份 prompt 反复调同一 CLI 死循环。
+
+设计稿：docs/migration/01-defense-patterns.md §5C。
+参考 dsh packages/guard/repeat-tool-reminder/src/index.ts：
+  监听 tools/post-execute，按 (tool, canonical_args) 跟踪，3/5/8 阈值注入 user-message 提醒。
+
+Tutti 的 "tool" 对应到 step role + 执行者身份，"canonical_args" 对应到 prompt
+模板指纹。不同 CLI 间的换将是有效的回退，不能共享同一个重复预算。步骤级调用
+代价远高于普通工具调用，因此第 3 次提醒、第 5 次强制停止。
+"""
+from __future__ import annotations
+
+import hashlib
+import logging
+import threading
+import time
+
+log = logging.getLogger(__name__)
+
+# 渐进阈值：第 N 次命中时给提醒；最后一次及以上时强制停止。
+DEFAULT_THRESHOLDS = (3, 5)
+
+REMINDER_TEMPLATE = (
+    "⚠️ 提醒：上一步使用了几乎相同的输入（已连续 {count} 次调用 {role}）。"
+    "请换一种方法：读不同的文件、用更具体的指令、或明确询问用户澄清。"
+)
+
+STOP_TEMPLATE = (
+    "已连续 {count} 次使用相同输入调用 {role}，超过 {last_thr} 次阈值，"
+    "强制停止本 step 以防止无限循环。请用户检查输入或调整 prompt 模板。"
+)
+
+
+def _prompt_fingerprint(prompt: str, identity: str = "") -> str:
+    """prompt 指纹：用前 200 字符 + 总长度（CLI prompt 头几行通常是模板部分）。
+
+    不做严格 canonical args 比对（CLI prompt 通常很长，且 dsh 也用 JSON.stringify）；
+    指纹命中说明是同一个模板路径走过多次，已经够分辨。
+
+    用 SHA-256 取前 12 字符（96 bit）：48 亿分之一碰撞概率对去重足够；不追求密码学强度。
+    """
+    head = (prompt or "")[:200]
+    return hashlib.sha256(
+        f"{identity}\0{head}|{len(prompt or '')}".encode("utf-8")).hexdigest()[:12]
+
+
+class RepeatGuard:
+    """每 (run_id, step_role) 的最近执行链独立计数；指纹包含执行者身份。
+
+    计数仅统计相同 key 连续出现；新 prompt 会重置。
+    """
+
+    def __init__(self, thresholds=DEFAULT_THRESHOLDS, *, max_history=50):
+        self.thresholds = tuple(sorted(set(thresholds)))
+        self.max_history = max_history
+        self._lock = threading.Lock()
+        # chain[(run_id, role)] = [(ts, fingerprint)]
+        self._chains: dict = {}
+
+    def check(self, run_id: str, role: str, prompt: str, *, identity: str = "") -> dict:
+        """检查并更新 (run_id, role) 的连续计数。
+
+        identity 标识实际执行 CLI/agent。相同 prompt 换到其他执行者属于回退，
+        会从新的连续链计数；同一执行者重复相同调用仍按阈值停止。
+
+        Returns:
+            {"count": int, "reminder": str | None, "should_stop": bool}
+        """
+        fp = _prompt_fingerprint(prompt, str(identity or ""))
+        chain_key = (run_id, role)
+        with self._lock:
+            chain = self._chains.setdefault(chain_key, [])
+            chain.append((time.time(), fp))
+            # 仅统计相同 fp 连续出现
+            count = 1
+            for _, prev_fp in reversed(chain[:-1]):
+                if prev_fp == fp:
+                    count += 1
+                else:
+                    break
+            # 截尾避免内存膨胀
+            if len(chain) > self.max_history:
+                self._chains[chain_key] = chain[-self.max_history:]
+
+        reminder = None
+        stop = False
+        last_thr = self.thresholds[-1]
+        if count >= last_thr:
+            stop = True
+            reminder = STOP_TEMPLATE.format(count=count, role=role, last_thr=last_thr)
+        elif count in self.thresholds:
+            reminder = REMINDER_TEMPLATE.format(count=count, role=role)
+
+        if reminder:
+            log.warning("repeat-guard %s/%s count=%d stop=%s reminder=%s",
+                        run_id, role, count, stop, reminder[:80])
+        return {"count": count, "reminder": reminder, "should_stop": stop}
+
+    def reset(self, run_id: str) -> int:
+        """重置某 run_id 的所有 (run_id, *) 链。返回清除的链条数。"""
+        with self._lock:
+            cleared = 0
+            for k in list(self._chains.keys()):
+                if k[0] == run_id:
+                    del self._chains[k]
+                    cleared += 1
+            return cleared
+
+    def stats(self, run_id: str, role: str) -> dict:
+        """读取 (run_id, role) 当前状态（不更新）。用于 UI 调试。"""
+        chain_key = (run_id, role)
+        with self._lock:
+            chain = self._chains.get(chain_key, [])
+            if not chain:
+                return {"chain_len": 0, "last_fp": ""}
+            # 当前连续计数
+            last_fp = chain[-1][1]
+            count = 1
+            for _, fp in reversed(chain[:-1]):
+                if fp == last_fp:
+                    count += 1
+                else:
+                    break
+            return {"chain_len": len(chain), "last_fp": last_fp, "count": count}
+
+
+# 模块级默认实例（pipeline 接线用；run_id 入键天然隔离多任务，测试无需替换）
+guard = RepeatGuard()

package/app/core/retrieval.py

@@ -1,137 +1,137 @@
-"""Dependency-free retrieval index with traceable keyword scoring.
-
-This is intentionally a RAG-lite layer: SQLite/vector backends can be added
-later without changing the API contract. Every hit carries its source and the
-terms that caused the match.
-"""
-from __future__ import annotations
-
-import hashlib
-import json
-import re
-import threading
-from pathlib import Path
-
-from . import paths
-
-_LOCK = threading.RLock()
-_FILE = paths.DATA_DIR / "retrieval_index.json"
-_WORD_RE = re.compile(r"[A-Za-z0-9_]+|[\u4e00-\u9fff]")
-_MAX_FILES = 500
-_MAX_BYTES = 20 * 1024 * 1024
-_SKIP_DIRS = {".git", "node_modules", "__pycache__", ".venv", "venv", ".idea", ".vscode"}
-
-
-def _terms(text):
-    return [x.lower() for x in _WORD_RE.findall(str(text or ""))]
-
-
-def _load():
-    try:
-        value = json.loads(_FILE.read_text(encoding="utf-8"))
-        return value if isinstance(value, dict) else {"documents": {}}
-    except Exception:
-        return {"version": 1, "documents": {}}
-
-
-def _save(value):
-    _FILE.parent.mkdir(parents=True, exist_ok=True)
-    tmp = _FILE.with_suffix(".tmp")
-    tmp.write_text(json.dumps(value, ensure_ascii=False, indent=1), encoding="utf-8")
-    tmp.replace(_FILE)
-
-
-def upsert(source, text, metadata=None, doc_id=""):
-    source = str(source or "").strip()[:500]
-    text = str(text or "").strip()
-    if not source or not text:
-        raise ValueError("source 和 text 不能为空")
-    root_hint = ""
-    if isinstance(metadata, dict):
-        root_hint = str(metadata.get("root") or "")[:500]
-    did = str(doc_id or hashlib.sha256((root_hint + "\0" + source).encode("utf-8")).hexdigest()[:20])[:100]
-    meta = metadata if isinstance(metadata, dict) else {}
-    meta = {str(k)[:80]: str(v)[:1000] for k, v in list(meta.items())[:30]}
-    row = {"id": did, "source": source, "text": text[:200000],
-           "terms": sorted(set(_terms(text))),
-           "metadata": meta}
-    with _LOCK:
-        data = _load()
-        data.setdefault("version", 1)
-        documents = data.setdefault("documents", {})
-        # Explicit IDs are client supplied; avoid silently replacing an
-        # unrelated document when two sources reuse the same short ID.
-        if did in documents and documents[did].get("source") != source:
-            suffix = hashlib.sha256((source + "\0" + text).encode("utf-8")).hexdigest()[:12]
-            did = (did[:87] + "-" + suffix)[:100]
-            row["id"] = did
-        documents[did] = row
-        _save(data)
-    return row
-
-
-def index_directory(root, extensions=(".md", ".txt", ".json"), limit=500,
-                    allowed_roots=None, max_bytes=_MAX_BYTES):
-    root = Path(root).expanduser().resolve()
-    if not root.is_dir():
-        raise ValueError("目录不存在")
-    if allowed_roots is not None:
-        roots = [Path(p).expanduser().resolve() for p in allowed_roots if p]
-        if not any(root == p or p in root.parents for p in roots):
-            raise ValueError("只能索引任务工作目录范围内的文件")
-    try:
-        file_limit = max(1, min(_MAX_FILES, int(limit)))
-    except (TypeError, ValueError):
-        raise ValueError("limit 必须是整数")
-    try:
-        byte_limit = max(1, min(_MAX_BYTES, int(max_bytes)))
-    except (TypeError, ValueError):
-        raise ValueError("max_bytes 必须是整数")
-    count = 0
-    total_bytes = 0
-    for path in root.rglob("*"):
-        if count >= file_limit or total_bytes >= byte_limit:
-            break
-        # Never follow a symlink while indexing user-controlled trees.
-        if path.is_symlink() or any(part in _SKIP_DIRS for part in path.relative_to(root).parts):
-            continue
-        if path.is_file() and path.suffix.lower() in extensions:
-            try:
-                resolved = path.resolve()
-                if resolved != root and root not in resolved.parents:
-                    continue
-                size = path.stat().st_size
-                if size > byte_limit - total_bytes:
-                    continue
-                upsert(str(path.relative_to(root)), path.read_text(encoding="utf-8", errors="replace"),
-                       {"root": str(root)})
-                count += 1
-                total_bytes += size
-            except OSError:
-                pass
-    return {"indexed": count, "bytes": total_bytes,
-            "total": len(_load().get("documents") or {})}
-
-
-def search(query, limit=10):
-    try:
-        n = max(1, min(50, int(limit or 10)))
-    except (TypeError, ValueError):
-        raise ValueError("limit 必须是整数")
-    terms = set(_terms(str(query or "")[:500]))
-    if not terms:
-        return []
-    with _LOCK:
-        docs = list((_load().get("documents") or {}).values())
-    rows = []
-    for doc in docs:
-        matched = sorted(terms.intersection(doc.get("terms") or []))
-        if not matched:
-            continue
-        score = len(matched) / max(1, len(terms))
-        rows.append({"id": doc.get("id"), "source": doc.get("source"),
-                     "score": round(score, 4), "matched_terms": matched,
-                     "snippet": str(doc.get("text") or "")[:600],
-                     "metadata": doc.get("metadata") or {}})
-    rows.sort(key=lambda x: (-x["score"], x["source"] or ""))
-    return rows[:n]
+"""Dependency-free retrieval index with traceable keyword scoring.
+
+This is intentionally a RAG-lite layer: SQLite/vector backends can be added
+later without changing the API contract. Every hit carries its source and the
+terms that caused the match.
+"""
+from __future__ import annotations
+
+import hashlib
+import json
+import re
+import threading
+from pathlib import Path
+
+from . import paths
+
+_LOCK = threading.RLock()
+_FILE = paths.DATA_DIR / "retrieval_index.json"
+_WORD_RE = re.compile(r"[A-Za-z0-9_]+|[\u4e00-\u9fff]")
+_MAX_FILES = 500
+_MAX_BYTES = 20 * 1024 * 1024
+_SKIP_DIRS = {".git", "node_modules", "__pycache__", ".venv", "venv", ".idea", ".vscode"}
+
+
+def _terms(text):
+    return [x.lower() for x in _WORD_RE.findall(str(text or ""))]
+
+
+def _load():
+    try:
+        value = json.loads(_FILE.read_text(encoding="utf-8"))
+        return value if isinstance(value, dict) else {"documents": {}}
+    except Exception:
+        return {"version": 1, "documents": {}}
+
+
+def _save(value):
+    _FILE.parent.mkdir(parents=True, exist_ok=True)
+    tmp = _FILE.with_suffix(".tmp")
+    tmp.write_text(json.dumps(value, ensure_ascii=False, indent=1), encoding="utf-8")
+    tmp.replace(_FILE)
+
+
+def upsert(source, text, metadata=None, doc_id=""):
+    source = str(source or "").strip()[:500]
+    text = str(text or "").strip()
+    if not source or not text:
+        raise ValueError("source 和 text 不能为空")
+    root_hint = ""
+    if isinstance(metadata, dict):
+        root_hint = str(metadata.get("root") or "")[:500]
+    did = str(doc_id or hashlib.sha256((root_hint + "\0" + source).encode("utf-8")).hexdigest()[:20])[:100]
+    meta = metadata if isinstance(metadata, dict) else {}
+    meta = {str(k)[:80]: str(v)[:1000] for k, v in list(meta.items())[:30]}
+    row = {"id": did, "source": source, "text": text[:200000],
+           "terms": sorted(set(_terms(text))),
+           "metadata": meta}
+    with _LOCK:
+        data = _load()
+        data.setdefault("version", 1)
+        documents = data.setdefault("documents", {})
+        # Explicit IDs are client supplied; avoid silently replacing an
+        # unrelated document when two sources reuse the same short ID.
+        if did in documents and documents[did].get("source") != source:
+            suffix = hashlib.sha256((source + "\0" + text).encode("utf-8")).hexdigest()[:12]
+            did = (did[:87] + "-" + suffix)[:100]
+            row["id"] = did
+        documents[did] = row
+        _save(data)
+    return row
+
+
+def index_directory(root, extensions=(".md", ".txt", ".json"), limit=500,
+                    allowed_roots=None, max_bytes=_MAX_BYTES):
+    root = Path(root).expanduser().resolve()
+    if not root.is_dir():
+        raise ValueError("目录不存在")
+    if allowed_roots is not None:
+        roots = [Path(p).expanduser().resolve() for p in allowed_roots if p]
+        if not any(root == p or p in root.parents for p in roots):
+            raise ValueError("只能索引任务工作目录范围内的文件")
+    try:
+        file_limit = max(1, min(_MAX_FILES, int(limit)))
+    except (TypeError, ValueError):
+        raise ValueError("limit 必须是整数")
+    try:
+        byte_limit = max(1, min(_MAX_BYTES, int(max_bytes)))
+    except (TypeError, ValueError):
+        raise ValueError("max_bytes 必须是整数")
+    count = 0
+    total_bytes = 0
+    for path in root.rglob("*"):
+        if count >= file_limit or total_bytes >= byte_limit:
+            break
+        # Never follow a symlink while indexing user-controlled trees.
+        if path.is_symlink() or any(part in _SKIP_DIRS for part in path.relative_to(root).parts):
+            continue
+        if path.is_file() and path.suffix.lower() in extensions:
+            try:
+                resolved = path.resolve()
+                if resolved != root and root not in resolved.parents:
+                    continue
+                size = path.stat().st_size
+                if size > byte_limit - total_bytes:
+                    continue
+                upsert(str(path.relative_to(root)), path.read_text(encoding="utf-8", errors="replace"),
+                       {"root": str(root)})
+                count += 1
+                total_bytes += size
+            except OSError:
+                pass
+    return {"indexed": count, "bytes": total_bytes,
+            "total": len(_load().get("documents") or {})}
+
+
+def search(query, limit=10):
+    try:
+        n = max(1, min(50, int(limit or 10)))
+    except (TypeError, ValueError):
+        raise ValueError("limit 必须是整数")
+    terms = set(_terms(str(query or "")[:500]))
+    if not terms:
+        return []
+    with _LOCK:
+        docs = list((_load().get("documents") or {}).values())
+    rows = []
+    for doc in docs:
+        matched = sorted(terms.intersection(doc.get("terms") or []))
+        if not matched:
+            continue
+        score = len(matched) / max(1, len(terms))
+        rows.append({"id": doc.get("id"), "source": doc.get("source"),
+                     "score": round(score, 4), "matched_terms": matched,
+                     "snippet": str(doc.get("text") or "")[:600],
+                     "metadata": doc.get("metadata") or {}})
+    rows.sort(key=lambda x: (-x["score"], x["source"] or ""))
+    return rows[:n]