@baimingtao/lbs-agent 1.0.0

package/runtime/context/compressor.py

@@ -0,0 +1,594 @@
+"""Context Compressor — 上下文压缩引擎。
+
+合并自 runtime/context/management/context_manager.py + trimming/* + compression/*。
+
+架构：
+  ContextCompressor (主引擎)
+  ├─ CompressionStrategy (策略基类)
+  │  ├─ PruneStrategy — 裁剪旧 tool result
+  │  ├─ SummarizeStrategy — LLM 摘要中间消息
+  │  └─ TruncateStrategy — 确定性截断降级
+  ├─ CompressionResult (结果数据)
+  └─ build_llm_summarizer (工厂)
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import re
+import time
+from dataclasses import dataclass, field
+from typing import Any, Callable, Optional
+
+logger = logging.getLogger(__name__)
+
+
+# ── 常量 ──
+
+_CHARS_PER_TOKEN = 3.5
+_TOOL_RESULT_PRUNE_CHARS = 200
+_LARGE_TOOL_RESULT_CHARS = 2000
+_MIN_MESSAGES_BETWEEN_COMPRESS = 5
+
+
+# ═══════════════════════════════════════════
+# 数据类
+# ═══════════════════════════════════════════
+
+
+@dataclass
+class CompressionResult:
+    """压缩结果报告。"""
+    compressed: bool = False
+    before_chars: int = 0
+    after_chars: int = 0
+    before_tokens: int = 0
+    after_tokens: int = 0
+    summary: str = ""
+    preserved_messages: int = 0
+    dropped_messages: int = 0
+    pruned_tool_results: int = 0
+    deduplicated_files: int = 0
+    warnings: list[str] = field(default_factory=list)
+    method: str = "none"  # none | prune | truncate | llm_summary | adapt
+    extracted_facts: list[str] = field(default_factory=list)
+
+
+@dataclass
+class CompressionConfig:
+    """可配置的压缩参数。"""
+    context_length: int = 128_000
+    threshold_percent: float = 0.50
+    protect_first_n: int = 3
+    protect_last_n: int = 12
+    tail_token_ratio: float = 0.40
+    max_summary_tokens: int = 8000
+    min_summary_tokens: int = 2000
+    anti_thrash_min_interval: int = 5  # 消息数
+    redact_secrets: bool = True
+
+    @property
+    def threshold_tokens(self) -> int:
+        return int(self.context_length * self.threshold_percent)
+
+    @property
+    def tail_token_budget(self) -> int:
+        return int(self.threshold_tokens * self.tail_token_ratio)
+
+
+# ═══════════════════════════════════════════
+# 压缩策略
+# ═══════════════════════════════════════════
+
+
+class CompressionStrategy:
+    """压缩策略基类。返回 (new_messages, result)。"""
+
+    def __init__(self, config: CompressionConfig):
+        self.config = config
+
+    def apply(self, messages: list[dict], context: Any = None
+              ) -> tuple[list[dict], CompressionResult]:
+        """执行压缩。"""
+        raise NotImplementedError
+
+
+class PruneStrategy(CompressionStrategy):
+    """策略1：裁剪旧 tool result + 文件去重。"""
+
+    def apply(self, messages: list[dict], context: Any = None
+              ) -> tuple[list[dict], CompressionResult]:
+        if not messages:
+            return messages, CompressionResult()
+
+        result = [dict(m) for m in messages]
+        pruned = 0
+        deduped = 0
+
+        tail_boundary = self._find_tail_boundary(result)
+        head_size = self._protect_head_size(result)
+
+        # 构建 tool_call_id → (name, args) 映射
+        call_id_to_tool: dict[str, tuple[str, str]] = {}
+        for msg in result:
+            if msg.get("role") == "assistant":
+                for tc in (msg.get("tool_calls") or []):
+                    if isinstance(tc, dict):
+                        cid = tc.get("id", "")
+                        fn = tc.get("function", {})
+                        call_id_to_tool[cid] = (fn.get("name", "?"), fn.get("arguments", ""))
+
+        # 文件去重
+        file_read_map: dict[str, int] = {}
+        for i, msg in enumerate(result):
+            if msg.get("role") == "tool" and i < tail_boundary:
+                call_id = msg.get("tool_call_id", "")
+                tool_name, tool_args = call_id_to_tool.get(call_id, ("", ""))
+                if tool_name == "read_file":
+                    try:
+                        args = json.loads(tool_args) if isinstance(tool_args, str) else tool_args
+                        fp = args.get("path", "")
+                        if fp:
+                            if fp in file_read_map:
+                                old_idx = file_read_map[fp]
+                                result[old_idx]["content"] =                                     "[重复读取已裁剪] " + fp + " (原 " +                                     str(len(str(result[old_idx].get("content", "")))) + " chars)"
+                                deduped += 1
+                            file_read_map[fp] = i
+                    except Exception:
+                        pass
+
+        # 裁剪大 tool result
+        for i, msg in enumerate(result):
+            if i >= tail_boundary:
+                continue
+            if msg.get("role") != "tool":
+                continue
+            content = str(msg.get("content", ""))
+            if len(content) <= _LARGE_TOOL_RESULT_CHARS:
+                continue
+            call_id = msg.get("tool_call_id", "")
+            tool_name, tool_args = call_id_to_tool.get(call_id, ("unknown", ""))
+            result[i]["content"] = self._summarize_tool_result(tool_name, tool_args, content)
+            pruned += 1
+
+        return result, CompressionResult(
+            compressed=(pruned > 0 or deduped > 0),
+            pruned_tool_results=pruned,
+            deduplicated_files=deduped,
+            method="prune",
+        )
+
+    _KEEP_HEAD = 1200   # 裁剪时保留头部字符数
+    _KEEP_TAIL = 800    # 裁剪时保留尾部字符数
+
+    def _summarize_tool_result(self, tool_name: str, args_json: str, content: str) -> str:
+        """裁剪大 tool result：保留头尾内容，只省略中间。
+
+        旧实现把完整内容替换成一行元信息摘要，导致 Agent 丢失正文。
+        新实现保留足够内容让 Agent 继续工作，只省略中间冗余部分。
+        """
+        head = content[:self._KEEP_HEAD]
+        tail = content[-self._KEEP_TAIL:]
+        omitted = len(content) - self._KEEP_HEAD - self._KEEP_TAIL
+        lines = content.strip().split("\n")
+        line_count = len(lines)
+        try:
+            args = json.loads(args_json) if isinstance(args_json, str) else args_json
+        except Exception:
+            args = {}
+        # 工具专属标注
+        meta = ""
+        if tool_name == "read_file":
+            meta = f"[read_file] {args.get('path', '?')} — {line_count} lines, {len(content)} chars"
+        elif tool_name == "terminal":
+            ec_match = re.search(r"\[exit=(\d+)\]", content)
+            meta = f"[terminal] {args.get('command', '?')[:60]} → exit={ec_match.group(1) if ec_match else '?'}, {line_count} lines"
+        elif tool_name == "search_files":
+            meta = f"[search_files] pattern={args.get('pattern', '?')} → {line_count} results"
+        else:
+            meta = f"[{tool_name}] {len(content)} chars → {line_count} lines"
+        return f"{meta} (中间省略 ~{omitted} 字符)\n{head}\n\n... [{omitted} chars omitted] ...\n\n{tail}"
+
+    def _protect_head_size(self, messages: list[dict]) -> int:
+        head = 0
+        if messages and messages[0].get("role") == "system":
+            head = 1
+        return min(head + self.config.protect_first_n, len(messages))
+
+    def _find_tail_boundary(self, messages: list[dict]) -> int:
+        head_size = self._protect_head_size(messages)
+        if not messages:
+            return 0
+        acc = 0
+        budget = self.config.tail_token_budget
+        min_tail = min(self.config.protect_last_n, len(messages))
+        ceiling = int(budget * 1.5)
+        boundary = len(messages)
+        for i in range(len(messages) - 1, head_size - 1, -1):
+            tokens = self._estimate_one(messages[i])
+            if acc + tokens > ceiling and (len(messages) - i) >= min_tail:
+                boundary = i + 1
+                break
+            acc += tokens
+        boundary = max(boundary, len(messages) - min_tail)
+        boundary = max(boundary, head_size + 1)
+        boundary = self._align_boundary(messages, boundary)
+        return boundary
+
+    def _align_boundary(self, messages: list[dict], idx: int) -> int:
+        while 0 < idx < len(messages):
+            msg = messages[idx]
+            if msg.get("role") == "tool":
+                idx -= 1
+                continue
+            if idx > 0 and messages[idx - 1].get("role") == "assistant"                     and messages[idx - 1].get("tool_calls"):
+                idx -= 1
+                continue
+            break
+        return idx
+
+    def _estimate_one(self, msg: dict) -> int:
+        content = msg.get("content", "")
+        if isinstance(content, str):
+            return int(len(content) / _CHARS_PER_TOKEN + 4)
+        return 4
+
+
+class SummarizeStrategy(CompressionStrategy):
+    """策略2：LLM 摘要中间消息。"""
+
+    def __init__(self, config: CompressionConfig,
+                 llm_summarizer: Callable[[list[dict]], str] | None = None):
+        super().__init__(config)
+        self._llm_summarizer = llm_summarizer
+
+    def apply(self, messages: list[dict], context: Any = None
+              ) -> tuple[list[dict], CompressionResult]:
+        head_size = self._protect_head_size(messages)
+        tail_start = self._find_tail_boundary(messages, head_size)
+        head = messages[:head_size]
+        tail = messages[tail_start:]
+        middle = messages[head_size:tail_start]
+
+        if not middle:
+            return messages, CompressionResult(method="none")
+
+        old_summary = ""
+        for m in head:
+            if "[上下文压缩摘要" in str(m.get("content", "")):
+                old_summary = str(m["content"])
+                break
+
+        method = "truncate"
+        summary = ""
+        if self._llm_summarizer and not old_summary:
+            try:
+                redacted = [{"role": m.get("role", "user"),
+                             "content": self._redact(str(m.get("content", "")))}
+                            for m in middle]
+                summary = self._llm_summarizer(redacted)
+                method = "llm_summary"
+            except Exception as e:
+                logger.warning("LLM 摘要失败，降级: %s", e)
+                summary = self._build_static_fallback(middle)
+                method = "fallback"
+        elif self._llm_summarizer and old_summary:
+            summary = self._update_summary(old_summary, middle)
+            method = "llm_summary_update"
+        else:
+            summary = self._build_static_fallback(middle)
+            method = "fallback"
+
+        summary_msg = {
+            "role": "user",
+            "content": f"[上下文压缩摘要 ({method})]\n{summary}",
+        }
+        new_messages = self._sanitize_tool_pairs(head + [summary_msg] + tail)
+
+        return new_messages, CompressionResult(
+            compressed=True,
+            summary=summary[:500],
+            preserved_messages=len(new_messages),
+            dropped_messages=len(middle),
+            method=method,
+        )
+
+    def _protect_head_size(self, messages: list[dict]) -> int:
+        head = 0
+        if messages and messages[0].get("role") == "system":
+            head = 1
+        return min(head + self.config.protect_first_n, len(messages))
+
+    def _find_tail_boundary(self, messages: list[dict], head_size: int) -> int:
+        if not messages:
+            return 0
+        acc = 0
+        budget = self.config.tail_token_budget
+        min_tail = min(self.config.protect_last_n, len(messages))
+        ceiling = int(budget * 1.5)
+        boundary = len(messages)
+        for i in range(len(messages) - 1, head_size - 1, -1):
+            tokens = self._estimate_one(messages[i])
+            if acc + tokens > ceiling and (len(messages) - i) >= min_tail:
+                boundary = i + 1
+                break
+            acc += tokens
+        boundary = max(boundary, len(messages) - min_tail)
+        boundary = max(boundary, head_size + 1)
+        return boundary
+
+    def _estimate_one(self, msg: dict) -> int:
+        content = msg.get("content", "")
+        if isinstance(content, str):
+            return int(len(content) / _CHARS_PER_TOKEN + 4)
+        return 4
+
+    def _sanitize_tool_pairs(self, messages: list[dict]) -> list[dict]:
+        valid_ids: set[str] = set()
+        for msg in messages:
+            if msg.get("role") == "assistant" and msg.get("tool_calls"):
+                for tc in msg["tool_calls"]:
+                    if isinstance(tc, dict):
+                        valid_ids.add(tc.get("id", ""))
+        return [m for m in messages
+                if not (m.get("role") == "tool" and m.get("tool_call_id", "") not in valid_ids)]
+
+    def _redact(self, text: str) -> str:
+        if not self.config.redact_secrets:
+            return text
+        text = re.sub(r'(sk-[a-zA-Z0-9]{8})[a-zA-Z0-9]+', r'\1***', text)
+        text = re.sub(r'(Bearer\s+)[a-zA-Z0-9._-]{8,}', r'\1***', text)
+        for kw in ("api_key", "api_secret", "password", "token", "secret"):
+            text = re.sub(kw + r"[\'\"]?\s*[:=]\s*[\'\"]?[^\'\"\s]+",
+                          kw + "=***", text, flags=re.IGNORECASE)
+        return text
+
+    def _build_static_fallback(self, turns: list[dict]) -> str:
+        paths: list[str] = []
+        commands: list[str] = []
+        errors: list[str] = []
+        user_requests: list[str] = []
+        all_text = ""
+        for msg in turns:
+            role = msg.get("role", "")
+            content = str(msg.get("content", ""))
+            all_text += content + "\n"
+            if role == "user":
+                user_requests.append(content[:150])
+            if "[exit=" in content:
+                commands.append(content[:100])
+        for m in re.finditer(r'[\w/\\\\]+\.[a-z]{2,4}(?:\b|\s)', all_text):
+            p = m.group().strip()
+            if p not in paths and len(p) > 3:
+                paths.append(p)
+        for m in re.finditer(r'(?:ERROR|Error|error)[^\n]{0,200}', all_text):
+            errors.append(m.group()[:150])
+        lines = ["[确定性降级摘要 -- LLM 摘要不可用]"]
+        if user_requests:
+            lines.append("\n用户请求:")
+            for r in user_requests[-3:]:
+                lines.append(f"  - {r}")
+        if paths:
+            lines.append("\n涉及的文件:")
+            for p in paths[:10]:
+                lines.append(f"  - {p}")
+        if commands:
+            lines.append("\n执行的命令:")
+            for c in commands[-5:]:
+                lines.append(f"  - {c}")
+        if errors:
+            lines.append("\n错误信息:")
+            for e in errors[-3:]:
+                lines.append(f"  - {e}")
+        return "\n".join(lines)
+
+    def _update_summary(self, old_summary: str, new_turns: list[dict]) -> str:
+        if not old_summary:
+            return self._truncate(new_turns)
+        if self._llm_summarizer:
+            new_content = self._truncate(new_turns)
+            combined = f"已有摘要:\n{old_summary}\n\n新增内容:\n{new_content}"
+            try:
+                return self._llm_summarizer([{"role": "user", "content": self._redact(combined)}])
+            except Exception:
+                return self._build_static_fallback(new_turns)
+        return f"{old_summary}\n\n--- 后续新增 ---\n{self._truncate(new_turns)}"
+
+    def _truncate(self, messages: list[dict]) -> str:
+        lines = [f"已压缩 {len(messages)} 条历史消息，保留关键轨迹："]
+        for i, msg in enumerate(messages[-15:], 1):
+            role = msg.get("role", "unknown")
+            content = str(msg.get("content", "")).replace("\n", " ")[:200]
+            if msg.get("tool_calls"):
+                tc_names = [t.get("function", {}).get("name", "?")
+                            if isinstance(t, dict) else str(t)
+                            for t in msg["tool_calls"]]
+                content += f" [工具: {', '.join(tc_names)}]"
+            lines.append(f"{i}. {role}: {content}")
+        return "\n".join(lines)
+
+
+# ═══════════════════════════════════════════
+# ContextCompressor (主入口)
+# ═══════════════════════════════════════════
+
+
+class ContextCompressor:
+    """上下文压缩引擎。
+
+    封装压缩策略链：先 Prune → 如仍超阈 → Summarize。
+    支持可观察回调（on_compression）和反抖（anti-thrashing）。
+    """
+
+    def __init__(self, config: CompressionConfig | None = None,
+                 llm_summarizer: Callable[[list[dict]], str] | None = None,
+                 on_compression: Callable[[CompressionResult], None] | None = None):
+        self.config = config or CompressionConfig()
+        self._llm_summarizer = llm_summarizer
+        self._on_compression = on_compression
+
+        # 策略
+        self._prune = PruneStrategy(self.config)
+        self._summarize = SummarizeStrategy(self.config, llm_summarizer)
+
+        # Anti-thrashing
+        self._last_compress_msg_count = 0
+        self._last_compress_tokens = 0
+        self._consecutive_ineffective = 0
+
+    # ── 公共 API ──
+
+    def should_compress(self, messages: list[dict]) -> bool:
+        """检查是否需要压缩。"""
+        tokens = self._estimate_many(messages)
+        new_msgs = len(messages) - self._last_compress_msg_count
+        if self._last_compress_msg_count > 0 and new_msgs < _MIN_MESSAGES_BETWEEN_COMPRESS:
+            return False
+        return tokens >= self.config.threshold_tokens
+
+    def compress(self, messages: list[dict]) -> tuple[list[dict], CompressionResult]:
+        """执行压缩（主入口）。"""
+        before_tokens = self._estimate_many(messages)
+        before_chars = sum(len(str(m.get("content", ""))) for m in messages)
+
+        # Phase 1: Prune
+        pruned, prune_result = self._prune.apply(messages)
+        after_prune_tokens = self._estimate_many(pruned)
+
+        # Check if prune was enough
+        if after_prune_tokens < self.config.threshold_tokens                 and len(pruned) <= self.config.protect_first_n + self.config.protect_last_n + 2:
+            result = CompressionResult(
+                compressed=prune_result.compressed,
+                before_chars=before_chars,
+                after_chars=sum(len(str(m.get("content", ""))) for m in pruned),
+                before_tokens=before_tokens,
+                after_tokens=after_prune_tokens,
+                preserved_messages=len(pruned),
+                pruned_tool_results=prune_result.pruned_tool_results,
+                deduplicated_files=prune_result.deduplicated_files,
+                method=prune_result.method,
+            )
+            self._update_thrash(after_prune_tokens, len(pruned))
+            self._notify(result)
+            return pruned, result
+
+        # Phase 2: Summarize
+        summarized, summary_result = self._summarize.apply(pruned)
+        after_tokens = self._estimate_many(summarized)
+        after_chars = sum(len(str(m.get("content", ""))) for m in summarized)
+
+        result = CompressionResult(
+            compressed=True,
+            before_chars=before_chars,
+            after_chars=after_chars,
+            before_tokens=before_tokens,
+            after_tokens=after_tokens,
+            summary=summary_result.summary[:500],
+            preserved_messages=len(summarized),
+            dropped_messages=summary_result.dropped_messages,
+            pruned_tool_results=prune_result.pruned_tool_results,
+            deduplicated_files=prune_result.deduplicated_files,
+            method=summary_result.method,
+        )
+
+        if after_tokens >= after_prune_tokens:
+            result.warnings.append("压缩后 token 数未下降")
+            self._consecutive_ineffective += 1
+        else:
+            self._consecutive_ineffective = 0
+
+        self._update_thrash(after_tokens, len(summarized))
+        self._notify(result)
+        return summarized, result
+
+    def compress_if_needed(self, messages: list[dict]) -> tuple[list[dict], CompressionResult]:
+        """如果有需要则压缩，否则返回原消息。"""
+        if self.should_compress(messages):
+            return self.compress(messages)
+        return messages, CompressionResult(method="none")
+
+    def update_from_api(self, usage: dict[str, int]) -> None:
+        """用 API 返回的真实 token 数校准。"""
+        self._last_prompt_tokens = usage.get("prompt_tokens", 0)
+
+    def set_summarizer(self, fn: Callable[[list[dict]], str]) -> None:
+        self._llm_summarizer = fn
+        self._summarize = SummarizeStrategy(self.config, fn)
+
+    # ── 内部 ──
+
+    def _estimate_many(self, messages: list[dict]) -> int:
+        total = 0
+        for m in messages:
+            content = m.get("content", "")
+            if isinstance(content, str):
+                total += len(content) / _CHARS_PER_TOKEN
+            elif isinstance(content, list):
+                for part in content:
+                    if isinstance(part, dict):
+                        if part.get("type") == "text":
+                            total += len(part.get("text", "")) / _CHARS_PER_TOKEN
+                        elif part.get("type") == "image_url":
+                            total += 1000
+            total += 4
+            tcs = m.get("tool_calls")
+            if tcs:
+                for tc in tcs:
+                    total += 50
+                    args = tc.get("function", {}).get("arguments", "") if isinstance(tc, dict) else ""
+                    total += len(args) / _CHARS_PER_TOKEN
+        return int(total)
+
+    def _update_thrash(self, after_tokens: int, msg_count: int) -> None:
+        self._last_compress_msg_count = msg_count
+        self._last_compress_tokens = after_tokens
+
+    def _notify(self, result: CompressionResult) -> None:
+        if self._on_compression and result.compressed:
+            try:
+                self._on_compression(result)
+            except Exception:
+                pass
+
+
+# ═══════════════════════════════════════════
+# 工厂函数
+# ═══════════════════════════════════════════
+
+
+def build_llm_summarizer(llm_client, system_prompt: str = "") -> Callable[[list[dict]], str]:
+    """构造 LLM 摘要函数。"""
+    sys_prompt = system_prompt or (
+        "你是上下文压缩器。对以下 Agent 对话历史做结构化摘要。"
+        "保留所有关键决策、错误信息、文件路径、工具调用结果要点。"
+        "丢弃冗余文本和重复内容。输出不超过 800 字。"
+    )
+
+    def summarize(messages: list[dict]) -> str:
+        parts = []
+        for msg in messages:
+            role = msg.get("role", "?")
+            content = str(msg.get("content", ""))[:500]
+            if msg.get("tool_calls"):
+                tc_names = [t.get("function", {}).get("name", "?")
+                            if isinstance(t, dict) else str(t)
+                            for t in msg["tool_calls"]]
+                content += f" [工具: {', '.join(tc_names)}]"
+            parts.append(f"[{role}] {content}")
+        history_text = "\n".join(parts)
+        if len(history_text) > 12000:
+            history_text = history_text[:6000] + "\n...[省略]...\n" + history_text[-6000:]
+        result = llm_client.chat(
+            messages=[{"role": "user", "content": f"请摘要以下对话历史:\n\n{history_text}"}],
+            system=sys_prompt,
+            temperature=0.3,
+            max_tokens=1024,
+        )
+        return result.content.strip()
+
+    return summarize
+
+
+# ── 向后兼容 ──
+from runtime.context.management.context_manager import ContextManager as LegacyContextManager

package/runtime/context/management/__init__.py

@@ -0,0 +1 @@
+"""智能体系统目标架构包。"""

package/runtime/context/management/context_builder.py

@@ -0,0 +1,67 @@
+"""上下文构建器 — 将多个部分按优先级组装为最终 prompt。
+
+ContextManager 使用它来构建最终发送给 LLM 的消息列表。
+"""
+from __future__ import annotations
+from typing import Any
+
+
+class ContextBuilder:
+    """按优先级组装上下文片段。"""
+    
+    def __init__(self, max_tokens: int = 0):
+        self.max_tokens = max_tokens
+    
+    def build(self, parts: list[str]) -> str:
+        """将多个文本片段组装为单个字符串。
+        
+        Args:
+            parts: 文本片段列表（高优先级在前）
+            
+        Returns:
+            组装后的文本
+        """
+        return "\n".join(p for p in parts if p and p.strip())
+    
+    def build_with_budget(self, parts: list[tuple[str, int]]) -> str:
+        """按 token 预算组装片段。
+        
+        Args:
+            parts: [(text, estimated_tokens), ...] 高优先级在前
+            
+        Returns:
+            不超过预算的组装文本
+        """
+        result = []
+        used = 0
+        for text, tokens in parts:
+            if self.max_tokens and used + tokens > self.max_tokens:
+                break
+            result.append(text)
+            used += tokens
+        return "\n".join(result)
+    
+    def build_messages(
+        self,
+        system: str = "",
+        history: list[dict] = None,
+        context_block: str = "",
+    ) -> list[dict[str, Any]]:
+        """构建发送给 LLM 的消息列表。
+        
+        Args:
+            system: 系统提示词
+            history: 对话历史
+            context_block: 注入的上下文信息（记忆/skill 等）
+            
+        Returns:
+            消息列表
+        """
+        messages = []
+        if system:
+            messages.append({"role": "system", "content": system})
+        if context_block:
+            messages.append({"role": "system", "content": context_block})
+        if history:
+            messages.extend(history)
+        return messages