@scemoon/cdh 1.0.0

package/onecode/mcp/client.py

@@ -0,0 +1,193 @@
+from __future__ import annotations
+
+import asyncio
+import json
+import logging
+import subprocess
+from dataclasses import dataclass, field
+from typing import Any, Optional
+
+logger = logging.getLogger("onecode.mcp")
+
+
+@dataclass
+class MCPTool:
+    name: str
+    description: str
+    input_schema: dict = field(default_factory=dict)
+
+
+@dataclass
+class MCPResource:
+    uri: str
+    name: str = ""
+    description: str = ""
+
+
+@dataclass
+class MCPClient:
+    name: str
+    command: str
+    args: list[str] = field(default_factory=list)
+    env: dict[str, str] = field(default_factory=dict)
+    _process: Optional[subprocess.Popen] = None
+    _reader: Optional[asyncio.StreamReader] = None
+    _writer: Optional[asyncio.StreamWriter] = None
+    _request_id: int = 0
+    _pending: dict = field(default_factory=dict)
+    _tools: list = field(default_factory=list)
+    _resources: list = field(default_factory=list)
+    _read_task: Optional[asyncio.Task] = None
+
+    async def start(self) -> bool:
+        try:
+            self._process = subprocess.Popen(
+                [self.command] + self.args,
+                stdin=subprocess.PIPE,
+                stdout=subprocess.PIPE,
+                stderr=subprocess.PIPE,
+                env={**self.env} if self.env else None,
+            )
+            loop = asyncio.get_event_loop()
+            self._reader, self._writer = await asyncio.open_connection(
+                fd=self._process.stdout.fileno(),
+                # Use a loop.create_connection approach for pipe-based fd
+            )
+            # Proper way: create StreamReader/StreamWriter from pipe
+            self._reader = asyncio.StreamReader()
+            protocol = asyncio.StreamReaderProtocol(self._reader)
+            await loop.connect_read_pipe(
+                lambda: protocol,
+                self._process.stdout,
+            )
+            self._writer_transport, self._writer_protocol = await loop.connect_write_pipe(
+                lambda: asyncio.StreamReaderProtocol(asyncio.StreamReader()),
+                self._process.stdin,
+            )
+
+            # Start background reader task
+            self._read_task = asyncio.create_task(self._read_loop())
+            await self._send_init()
+            return True
+        except Exception as e:
+            logger.error(f"MCP start failed: {e}")
+            return False
+
+    async def _read_loop(self) -> None:
+        """Continuously read JSON-RPC responses from stdout."""
+        try:
+            while self._process and self._process.poll() is None:
+                if self._reader is None:
+                    await asyncio.sleep(0.1)
+                    continue
+                line = await self._reader.readline()
+                if not line:
+                    await asyncio.sleep(0.05)
+                    continue
+                line_str = line.decode("utf-8", errors="replace").strip()
+                if not line_str:
+                    continue
+                try:
+                    msg = json.loads(line_str)
+                    req_id = str(msg.get("id", ""))
+                    if req_id in self._pending:
+                        future = self._pending.pop(req_id)
+                        if not future.done():
+                            future.set_result(msg)
+                    # Handle notifications (no id)
+                    if "method" in msg:
+                        await self._handle_notification(msg)
+                except json.JSONDecodeError:
+                    pass
+        except Exception as e:
+            logger.debug(f"MCP read loop ended: {e}")
+
+    async def _handle_notification(self, msg: dict) -> None:
+        method = msg.get("method", "")
+        if method == "tools/list_changed":
+            await self.list_tools()
+        elif method == "resources/list_changed":
+            await self.list_resources()
+
+    async def _send_init(self) -> None:
+        result = await self._send_request("initialize", {
+            "protocolVersion": "2024-11-05",
+            "capabilities": {},
+            "clientInfo": {"name": "cdh", "version": "1.0.0"},
+        })
+        if result and "result" in result:
+            server_caps = result["result"].get("capabilities", {})
+            if server_caps.get("tools"):
+                await self.list_tools()
+            if server_caps.get("resources"):
+                await self.list_resources()
+
+    async def _send_request(self, method: str, params: dict) -> Any:
+        self._request_id += 1
+        req_id = self._request_id
+        msg = json.dumps({"jsonrpc": "2.0", "id": req_id, "method": method, "params": params})
+        if self._writer_transport:
+            self._writer_transport.write((msg + "\n").encode())
+        elif self._writer:
+            self._writer.write((msg + "\n").encode())
+            await self._writer.drain()
+
+        future = asyncio.Future()
+        self._pending[str(req_id)] = future
+
+        try:
+            result = await asyncio.wait_for(future, timeout=30)
+            return result
+        except asyncio.TimeoutError:
+            self._pending.pop(str(req_id), None)
+            return None
+
+    async def list_tools(self) -> list[MCPTool]:
+        result = await self._send_request("tools/list", {})
+        if result and "result" in result:
+            tools_data = result["result"].get("tools", [])
+            self._tools = [
+                MCPTool(
+                    name=t["name"],
+                    description=t.get("description", ""),
+                    input_schema=t.get("inputSchema", {}),
+                )
+                for t in tools_data
+            ]
+        return self._tools
+
+    async def list_resources(self) -> list[MCPResource]:
+        result = await self._send_request("resources/list", {})
+        if result and "result" in result:
+            resources_data = result["result"].get("resources", [])
+            self._resources = [
+                MCPResource(uri=r["uri"], name=r.get("name", ""), description=r.get("description", ""))
+                for r in resources_data
+            ]
+        return self._resources
+
+    async def call_tool(self, name: str, args: dict) -> Any:
+        result = await self._send_request("tools/call", {"name": name, "arguments": args})
+        if result and "result" in result:
+            return result["result"]
+        return None
+
+    async def stop(self) -> None:
+        if self._read_task:
+            self._read_task.cancel()
+            self._read_task = None
+        if self._process:
+            self._process.terminate()
+            try:
+                self._process.wait(timeout=5)
+            except Exception:
+                self._process.kill()
+
+    def is_running(self) -> bool:
+        return self._process is not None and self._process.poll() is None
+
+    def get_tools(self) -> list[MCPTool]:
+        return self._tools
+
+    def get_resources(self) -> list[MCPResource]:
+        return self._resources

package/onecode/mcp/manager.py

@@ -0,0 +1,191 @@
+from __future__ import annotations
+
+import asyncio
+import json
+import logging
+from pathlib import Path
+from typing import Any, Optional
+
+import httpx
+import yaml
+
+from onecode.config import CLOUD_DEV_HARNESS_DIR
+from onecode.mcp.client import MCPClient, MCPTool
+
+logger = logging.getLogger("onecode.mcp")
+
+
+class MCPSSEClient:
+    """SSE-based MCP client (Clawd-Code pattern)."""
+
+    def __init__(self, name: str, url: str):
+        self.name = name
+        self.url = url
+        self._client = httpx.AsyncClient(timeout=30)
+        self._session_id: Optional[str] = None
+        self._tools: list[MCPTool] = []
+        self._running = False
+
+    async def start(self) -> bool:
+        try:
+            async with self._client.stream("GET", self.url) as resp:
+                async for line in resp.aiter_lines():
+                    if line.startswith("data: "):
+                        data = json.loads(line[6:])
+                        if isinstance(data, dict) and "sessionId" in data:
+                            self._session_id = data["sessionId"]
+                            self._running = True
+                            break
+            if self._running:
+                await self._list_tools()
+            return self._running
+        except Exception as e:
+            logger.error(f"MCP SSE start failed: {e}")
+            return False
+
+    async def _list_tools(self) -> list[MCPTool]:
+        try:
+            resp = await self._client.post(
+                self.url,
+                json={"jsonrpc": "2.0", "method": "tools/list", "id": 1},
+            )
+            data = resp.json()
+            tools_data = data.get("result", {}).get("tools", [])
+            self._tools = [
+                MCPTool(name=t["name"], description=t.get("description", ""), input_schema=t.get("inputSchema", {}))
+                for t in tools_data
+            ]
+        except Exception as e:
+            logger.error("MCP list_tools failed: %s", e)
+        return self._tools
+
+    async def call_tool(self, name: str, args: dict) -> Any:
+        try:
+            resp = await self._client.post(
+                self.url,
+                json={"jsonrpc": "2.0", "method": "tools/call", "params": {"name": name, "arguments": args}, "id": 2},
+            )
+            data = resp.json()
+            return data.get("result")
+        except Exception as e:
+            return {"error": str(e)}
+
+    async def stop(self):
+        self._running = False
+        await self._client.aclose()
+
+    def is_running(self) -> bool:
+        return self._running
+
+    def get_tools(self) -> list[MCPTool]:
+        return self._tools
+
+
+class MCPManager:
+    def __init__(self):
+        self.config_dir = CLOUD_DEV_HARNESS_DIR / "mcps"
+        self.config_dir.mkdir(parents=True, exist_ok=True)
+        self.config_path = self.config_dir / "mcps.yaml"
+        self._data: dict = {}
+        self._clients: dict[str, Any] = {}
+        if self.config_path.exists():
+            self._data = yaml.safe_load(self.config_path.read_text()) or {}
+
+    def add(self, name: str, url: str, transport: str = "sse"):
+        self._data[name] = {"url": url, "transport": transport, "enabled": True}
+        self._save()
+
+    def add_stdio(self, name: str, command: str, args: list[str] = None):
+        self._data[name] = {"command": command, "args": args or [], "transport": "stdio", "enabled": True}
+        self._save()
+
+    def list(self) -> list[dict]:
+        return [
+            {"name": name, **cfg} for name, cfg in self._data.items()
+        ]
+
+    def get(self, name: str) -> Optional[dict]:
+        """Get an MCP server config by name."""
+        return self._data.get(name)
+
+    def enable(self, name: str, enabled: bool = True) -> Optional[str]:
+        """Enable or disable an MCP server.
+
+        Returns error message or None on success.
+        """
+        if name not in self._data:
+            return f"MCP server '{name}' not found"
+        self._data[name]["enabled"] = enabled
+        self._save()
+        return None
+
+    def remove(self, name: str):
+        self._data.pop(name, None)
+        if name in self._clients:
+            try:
+                loop = asyncio.get_event_loop()
+                if loop.is_running():
+                    asyncio.create_task(self._clients[name].stop())
+            except Exception as e:
+                logger.warning("Failed to stop MCP client '%s': %s", name, e)
+            del self._clients[name]
+        self._save()
+
+    async def connect(self, name: str) -> bool:
+        cfg = self._data.get(name, {})
+        if not cfg.get("enabled"):
+            return False
+        transport = cfg.get("transport", "sse")
+
+        if transport == "stdio":
+            client = MCPClient(
+                name=name,
+                command=cfg["command"],
+                args=cfg.get("args", []),
+            )
+        elif transport == "sse":
+            client = MCPSSEClient(name=name, url=cfg["url"])
+        else:
+            logger.warning(f"Unknown MCP transport: {transport}")
+            return False
+
+        success = await client.start()
+        if success:
+            self._clients[name] = client
+        return success
+
+    async def connect_all(self) -> list[str]:
+        connected = []
+        for name in list(self._data.keys()):
+            if await self.connect(name):
+                connected.append(name)
+        return connected
+
+    async def disconnect(self, name: str) -> None:
+        if name in self._clients:
+            await self._clients[name].stop()
+            del self._clients[name]
+
+    async def disconnect_all(self) -> None:
+        for name in list(self._clients.keys()):
+            await self.disconnect(name)
+
+    async def list_tools(self, name: str) -> list[MCPTool]:
+        if name in self._clients:
+            return await self._clients[name].list_tools()
+        return []
+
+    async def call_tool(self, name: str, tool_name: str, args: dict) -> Any:
+        if name in self._clients:
+            return await self._clients[name].call_tool(tool_name, args)
+        return None
+
+    def get_client(self, name: str) -> Optional[Any]:
+        return self._clients.get(name)
+
+    def is_connected(self, name: str) -> bool:
+        client = self._clients.get(name)
+        return client.is_running() if client else False
+
+    def _save(self):
+        self.config_path.write_text(yaml.dump(self._data, default_flow_style=False))

package/onecode/memory/__init__.py

@@ -0,0 +1,95 @@
+"""
+CDH Memory - Layered Long-term Memory for AI Agents
+
+Based on TencentDB-Agent-Memory architecture:
+- L0: Raw conversation
+- L1: Atomic facts
+- L2: Scenarios
+- L3: Personas
+
+With symbolic short-term memory (Mermaid canvas + context offloading)
+"""
+
+from onecode.memory.pyramid import MemoryPyramid, MemoryLayer, MemoryEntry
+from onecode.memory.symbolic import SymbolicMemory, MermaidCanvas, NodeType
+from onecode.memory.offload import ContextOffloader, OffloadConfig, ToolCallExtractor
+from onecode.memory.recall import HybridRecall, BM25, RecallResult
+from onecode.memory.backend import MemoryBackend
+
+__all__ = [
+    "MemoryPyramid",
+    "MemoryLayer",
+    "MemoryEntry",
+    "SymbolicMemory",
+    "MermaidCanvas",
+    "NodeType",
+    "ContextOffloader",
+    "OffloadConfig",
+    "ToolCallExtractor",
+    "HybridRecall",
+    "BM25",
+    "RecallResult",
+    "MemoryBackend",
+]
+
+
+class AgentMemory:
+    """
+    Unified memory interface combining layered long-term memory
+    with symbolic short-term memory and hybrid recall.
+    """
+
+    def __init__(self, storage_path=None):
+        from onecode.config import CLOUD_DEV_HARNESS_DIR
+        self.storage_path = storage_path or (CLOUD_DEV_HARNESS_DIR / "memory")
+        self.backend = MemoryBackend(self.storage_path / "memory.db")
+        self.pyramid = MemoryPyramid(self.storage_path)
+        self.symbolic = SymbolicMemory()
+        self.offloader = ContextOffloader(self.storage_path / "refs")
+        self.recall = HybridRecall()
+
+    def remember(self, layer: MemoryLayer, content: str, metadata: dict = None, parent_id: str = None) -> MemoryEntry:
+        entry = self.pyramid.add(layer, content, metadata, parent_id)
+        self.backend.add_entry(
+            entry.id,
+            layer.value,
+            content,
+            metadata,
+            parent_id,
+            entry.result_ref,
+        )
+        self.recall.add_documents([content], [entry.id], {entry.id: metadata or {}})
+        return entry
+
+    def search_memories(self, query: str, top_k: int = 5) -> list[RecallResult]:
+        return self.recall.hybrid_recall(query, top_k)
+
+    def remember_conversation(self, conversation_id: str, messages: list[dict]) -> MemoryEntry:
+        content = "\n".join(f"{m.get('role', 'user')}: {m.get('content', '')}" for m in messages)
+        entry = self.remember(MemoryLayer.L0_CONVERSATION, content, {"conversation_id": conversation_id})
+        return entry
+
+    def extract_atoms(self, conversation_id: str, max_atoms: int = 20) -> list[MemoryEntry]:
+        return self.pyramid.extract_atoms_from_conversation(conversation_id, max_atoms)
+
+    def build_scenario(self, name: str, atom_ids: list[str]) -> MemoryEntry:
+        scenario = self.pyramid.build_scenario_from_atoms(atom_ids, name)
+        self.backend.add_entry(scenario.id, MemoryLayer.L2_SCENARIO.value, scenario.content, scenario.metadata)
+        return scenario
+
+    def build_persona(self, name: str, scenario_ids: list[str]) -> MemoryEntry:
+        persona = self.pyramid.build_persona_from_scenarios(scenario_ids, name)
+        self.backend.add_entry(persona.id, MemoryLayer.L3_PERSONA.value, persona.content, persona.metadata)
+        return persona
+
+    def offload_and_symbolize(self, node_id: str, verbose_content: str, task_label: str) -> str:
+        ref_path = self.symbolic.offload_log(node_id, verbose_content)
+        self.backend.save_ref(node_id, verbose_content)
+        self.symbolic.create_result_node(task_label, verbose_content)
+        return ref_path
+
+    def get_canvas(self) -> str:
+        return self.symbolic.get_canvas_markdown()
+
+    def drill_down(self, entry_id: str) -> list[MemoryEntry]:
+        return self.pyramid.drill_down(entry_id)

package/onecode/memory/backend.py

@@ -0,0 +1,185 @@
+from __future__ import annotations
+
+import json
+import uuid
+from contextlib import contextmanager
+from dataclasses import dataclass
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Optional
+
+from sqlalchemy import Column, String, Text, DateTime, JSON, create_engine, Index
+from sqlalchemy.orm import declarative_base, sessionmaker, Session
+
+from onecode.config import CLOUD_DEV_HARNESS_DIR
+
+
+Base = declarative_base()
+
+
+class MemoryRecord(Base):
+    __tablename__ = "memory_entries"
+
+    id = Column(String(16), primary_key=True)
+    layer = Column(String(20), nullable=False, index=True)
+    content = Column(Text, nullable=False)
+    timestamp = Column(DateTime, nullable=False, index=True)
+    metadata_json = Column(JSON, default=dict)
+    parent_id = Column(String(16), nullable=True, index=True)
+    result_ref = Column(String(255), nullable=True)
+
+    __table_args__ = (
+        Index("idx_layer_timestamp", "layer", "timestamp"),
+        Index("idx_parent_id", "parent_id"),
+    )
+
+
+class RefRecord(Base):
+    __tablename__ = "ref_records"
+
+    id = Column(String(16), primary_key=True)
+    node_id = Column(String(16), nullable=False, index=True)
+    content = Column(Text, nullable=False)
+    file_path = Column(String(512), nullable=True)
+    created_at = Column(DateTime, nullable=False)
+
+
+class MemoryBackend:
+    def __init__(self, db_path: Optional[Path] = None):
+        if db_path is None:
+            db_path = CLOUD_DEV_HARNESS_DIR / "memory" / "memory.db"
+        db_path.parent.mkdir(parents=True, exist_ok=True)
+        self.engine = create_engine(f"sqlite:///{db_path}", echo=False)
+        Base.metadata.create_all(self.engine)
+        self.Session = sessionmaker(bind=self.engine)
+
+    @contextmanager
+    def session(self) -> Session:
+        s = self.Session()
+        try:
+            yield s
+            s.commit()
+        except Exception:
+            s.rollback()
+            raise
+        finally:
+            s.close()
+
+    def add_entry(
+        self,
+        entry_id: str,
+        layer: str,
+        content: str,
+        metadata: Optional[dict] = None,
+        parent_id: Optional[str] = None,
+        result_ref: Optional[str] = None,
+    ) -> bool:
+        with self.session() as s:
+            record = MemoryRecord(
+                id=entry_id,
+                layer=layer,
+                content=content,
+                timestamp=datetime.now(timezone.utc),
+                metadata_json=metadata or {},
+                parent_id=parent_id,
+                result_ref=result_ref,
+            )
+            s.merge(record)
+            return True
+
+    def get_entry(self, entry_id: str) -> Optional[MemoryRecord]:
+        with self.session() as s:
+            record = s.query(MemoryRecord).filter_by(id=entry_id).first()
+            if record:
+                s.refresh(record)
+            return record
+
+    def get_entries_by_layer(self, layer: str, limit: int = 100) -> list[MemoryRecord]:
+        with self.session() as s:
+            records = (
+                s.query(MemoryRecord)
+                .filter_by(layer=layer)
+                .order_by(MemoryRecord.timestamp.desc())
+                .limit(limit)
+                .all()
+            )
+            for r in records:
+                s.refresh(r)
+            return records
+
+    def get_entries_by_parent(self, parent_id: str) -> list[MemoryRecord]:
+        with self.session() as s:
+            return (
+                s.query(MemoryRecord)
+                .filter_by(parent_id=parent_id)
+                .order_by(MemoryRecord.timestamp)
+                .all()
+            )
+
+    def get_recent_entries(self, layer: Optional[str] = None, limit: int = 20) -> list[MemoryRecord]:
+        with self.session() as s:
+            query = s.query(MemoryRecord).order_by(MemoryRecord.timestamp.desc())
+            if layer:
+                query = query.filter_by(layer=layer)
+            return query.limit(limit).all()
+
+    def save_ref(self, node_id: str, content: str, file_path: Optional[str] = None) -> bool:
+        with self.session() as s:
+            record = RefRecord(
+                id=str(uuid.uuid4())[:16],
+                node_id=node_id,
+                content=content,
+                file_path=file_path,
+                created_at=datetime.now(timezone.utc),
+            )
+            s.merge(record)
+            return True
+
+    def get_ref(self, node_id: str) -> Optional[str]:
+        with self.session() as s:
+            record = s.query(RefRecord).filter_by(node_id=node_id).order_by(RefRecord.created_at.desc()).first()
+            return record.content if record else None
+
+    def search_content(self, query: str, layer: Optional[str] = None, limit: int = 10) -> list[MemoryRecord]:
+        with self.session() as s:
+            q = s.query(MemoryRecord)
+            if layer:
+                q = q.filter_by(layer=layer)
+            escaped = query.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_")
+            pattern = f"%{escaped}%"
+            q = q.filter(MemoryRecord.content.like(pattern, escape="\\"))
+            records = q.order_by(MemoryRecord.timestamp.desc()).limit(limit).all()
+            for r in records:
+                s.refresh(r)
+            return records
+
+    def delete_entry(self, entry_id: str) -> bool:
+        with self.session() as s:
+            record = s.query(MemoryRecord).filter_by(id=entry_id).first()
+            if record:
+                s.delete(record)
+                return True
+            return False
+
+    def count_by_layer(self) -> dict[str, int]:
+        with self.session() as s:
+            from sqlalchemy import func
+            results = s.query(MemoryRecord.layer, func.count(MemoryRecord.id)).group_by(MemoryRecord.layer).all()
+            return {layer: count for layer, count in results}
+
+    def clear_old_entries(self, layer: str, keep_last: int = 100) -> int:
+        removed = 0
+        with self.session() as s:
+            count = s.query(MemoryRecord).filter_by(layer=layer).count()
+            if count > keep_last:
+                old_entries = (
+                    s.query(MemoryRecord)
+                    .filter_by(layer=layer)
+                    .order_by(MemoryRecord.timestamp.asc())
+                    .limit(count - keep_last)
+                    .all()
+                )
+                for entry in old_entries:
+                    s.delete(entry)
+                    removed += 1
+        return removed