@scemoon/cdh 1.0.8 → 1.0.9-beta.2

package/onecode/mcp/oauth.py

@@ -0,0 +1,199 @@
+"""OAuth helpers for remote MCP servers (stub framework).
+
+Provides:
+  * ``OAuthStore``: load/save token bundles in ``~/.onecode/mcp-auth.json``.
+  * ``OAuthFlow``: minimal abstraction; the default ``ManualOAuthFlow``
+    prints an authorization URL and waits for the user to paste the
+    callback URL or token.
+  * ``is_oauth_required``: detect a 401 response with a
+    ``WWW-Authenticate`` header so the agent can prompt the user to
+    run ``cdh mcp auth <name>``.
+
+CloudBase does not currently use this flow (it relies on static API
+key environment variables), but the framework is in place so future
+remote MCP servers can plug in.
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import time
+import webbrowser
+from dataclasses import asdict, dataclass, field
+from pathlib import Path
+from typing import Any, Optional
+
+logger = logging.getLogger("onecode.mcp.oauth")
+
+AUTH_FILENAME = "mcp-auth.json"
+
+
+def _default_auth_path() -> Path:
+    """Resolve the default auth file path at call time (test-friendly)."""
+    return Path.home() / ".onecode" / AUTH_FILENAME
+
+
+@dataclass
+class TokenBundle:
+    access_token: str = ""
+    refresh_token: str = ""
+    token_type: str = "Bearer"
+    expires_at: float = 0.0
+    scope: str = ""
+    extra: dict[str, Any] = field(default_factory=dict)
+
+    def is_expired(self, *, skew_seconds: int = 30) -> bool:
+        if not self.expires_at:
+            return False
+        return time.time() >= (self.expires_at - skew_seconds)
+
+    def to_dict(self) -> dict[str, Any]:
+        return asdict(self)
+
+    @classmethod
+    def from_dict(cls, data: dict[str, Any]) -> "TokenBundle":
+        scalar_fields = ("access_token", "refresh_token", "token_type", "expires_at", "scope")
+        kwargs = {k: data.get(k) for k in scalar_fields}
+        stored_extra = data.get("extra") or {}
+        extra = {**stored_extra, **{k: v for k, v in data.items() if k not in scalar_fields and k != "extra"}}
+        return cls(extra=extra, **kwargs)
+
+
+class OAuthStore:
+    """Persistent OAuth token store at ``~/.onecode/mcp-auth.json``."""
+
+    def __init__(self, path: Optional[Path] = None):
+        self.path = Path(path) if path else _default_auth_path()
+
+    def load_all(self) -> dict[str, TokenBundle]:
+        if not self.path.exists():
+            return {}
+        try:
+            raw = json.loads(self.path.read_text())
+        except (OSError, json.JSONDecodeError) as e:
+            logger.warning("OAuth: failed to read %s: %s", self.path, e)
+            return {}
+        if not isinstance(raw, dict):
+            return {}
+        out: dict[str, TokenBundle] = {}
+        for name, entry in raw.items():
+            if isinstance(entry, dict):
+                out[name] = TokenBundle.from_dict(entry)
+        return out
+
+    def get(self, name: str) -> Optional[TokenBundle]:
+        return self.load_all().get(name)
+
+    def save(self, name: str, bundle: TokenBundle) -> None:
+        all_bundles = self.load_all()
+        all_bundles[name] = bundle
+        self.path.parent.mkdir(parents=True, exist_ok=True)
+        self.path.write_text(
+            json.dumps(
+                {n: b.to_dict() for n, b in all_bundles.items()},
+                indent=2,
+            )
+            + "\n"
+        )
+
+    def remove(self, name: str) -> bool:
+        bundles = self.load_all()
+        if name not in bundles:
+            return False
+        del bundles[name]
+        self.path.parent.mkdir(parents=True, exist_ok=True)
+        self.path.write_text(
+            json.dumps({n: b.to_dict() for n, b in bundles.items()}, indent=2) + "\n"
+        )
+        return True
+
+    def auth_header(self, name: str) -> Optional[dict[str, str]]:
+        bundle = self.get(name)
+        if not bundle or not bundle.access_token:
+            return None
+        if bundle.is_expired():
+            logger.info("OAuth: token for '%s' is expired; run `cdh mcp auth %s`", name, name)
+            return None
+        return {"Authorization": f"{bundle.token_type or 'Bearer'} {bundle.access_token}"}
+
+
+class OAuthFlow:
+    """Base class for OAuth flows. Subclasses implement :meth:`run`."""
+
+    def __init__(self, oauth_config: Any):
+        self.config = oauth_config or {}
+
+    def run(self, *, server_name: str) -> TokenBundle:
+        raise NotImplementedError
+
+
+class ManualOAuthFlow(OAuthFlow):
+    """Print the authorization URL, then ask the user to paste a token.
+
+    If the config contains ``authorizationUrl`` and ``clientId`` we open
+    the browser (if available) and prompt for either the full callback
+    URL or just the token.
+    """
+
+    def run(self, *, server_name: str) -> TokenBundle:
+        auth_url = ""
+        client_id = ""
+        if isinstance(self.config, dict):
+            auth_url = str(self.config.get("authorizationUrl") or "")
+            client_id = str(self.config.get("clientId") or "")
+            scope = str(self.config.get("scope") or "")
+        else:
+            scope = ""
+
+        if auth_url:
+            print(f"OAuth: open this URL to authorize '{server_name}':\n  {auth_url}")
+            try:
+                if client_id:
+                    webbrowser.open(auth_url)
+            except Exception as e:  # pragma: no cover
+                logger.debug("OAuth: browser open failed: %s", e)
+
+        print("Paste the access token (or the full callback URL) below.")
+        try:
+            token = input("> ").strip()
+        except (EOFError, KeyboardInterrupt):
+            token = ""
+
+        # If they pasted a callback URL, try to extract the token.
+        if token.startswith("http"):
+            from urllib.parse import parse_qs, urlparse
+
+            qs = parse_qs(urlparse(token).query)
+            for key in ("access_token", "token", "code"):
+                if key in qs and qs[key]:
+                    token = qs[key][0]
+                    break
+
+        bundle = TokenBundle(access_token=token, scope=scope)
+        return bundle
+
+
+def is_oauth_required(status_code: int, headers: Any) -> bool:
+    """Return True if an HTTP response looks like an OAuth challenge."""
+    if status_code != 401:
+        return False
+    if not headers:
+        return False
+    try:
+        www_auth = headers.get("WWW-Authenticate") or headers.get("www-authenticate")
+    except Exception:
+        www_auth = None
+    if not www_auth:
+        return False
+    return "Bearer" in str(www_auth) or "oauth" in str(www_auth).lower()
+
+
+__all__ = [
+    "TokenBundle",
+    "OAuthStore",
+    "OAuthFlow",
+    "ManualOAuthFlow",
+    "is_oauth_required",
+    "AUTH_FILENAME",
+]

package/onecode/memory/__init__.py

@@ -0,0 +1,106 @@
+"""
+CDH Memory - Long-term Conversation Memory for AI Agents
+
+L0: Raw conversation storage with BM25 keyword recall.
+"""
+import json
+from onecode.memory.pyramid import MemoryPyramid, MemoryLayer, MemoryEntry
+from onecode.memory.recall import HybridRecall, BM25, RecallResult
+from onecode.memory.backend import MemoryBackend
+
+__all__ = [
+    "MemoryPyramid",
+    "MemoryLayer",
+    "MemoryEntry",
+    "HybridRecall",
+    "BM25",
+    "RecallResult",
+    "MemoryBackend",
+    "AgentMemory",
+]
+
+
+class AgentMemory:
+    """
+    Unified memory interface combining persistent L0 conversation storage
+    with BM25 keyword recall for long-term context.
+    """
+
+    def __init__(self, storage_path=None, max_docs: int = 5000):
+        from onecode.config import ONECODE_DIR
+        self.storage_path = storage_path or (ONECODE_DIR / "memory")
+        self.max_docs = max_docs
+        self.backend = MemoryBackend(self.storage_path / "memory.db")
+        self.pyramid = MemoryPyramid(self.storage_path)
+        self.recall = HybridRecall()
+        self._warm_recall()
+
+    def _index_file(self):
+        return self.storage_path / "recall_index.json"
+
+    def _warm_recall(self) -> None:
+        idx_file = self._index_file()
+        if idx_file.exists():
+            try:
+                data = json.loads(idx_file.read_text(encoding="utf-8"))
+                self.recall = HybridRecall.deserialize(data)
+                return
+            except Exception:
+                pass
+        records = self.backend.get_all_entries()
+        if not records:
+            return
+        docs: list[str] = []
+        ids: list[str] = []
+        meta: dict[str, dict] = {}
+        for r in records:
+            docs.append(r["content"])
+            ids.append(r["id"])
+            meta[r["id"]] = r.get("metadata_json") or {}
+        self.recall.add_documents(docs, ids, meta)
+
+    def _save_recall_index(self) -> None:
+        try:
+            idx_file = self._index_file()
+            idx_file.write_text(json.dumps(self.recall.serialize(), indent=2, ensure_ascii=False), encoding="utf-8")
+        except Exception:
+            pass
+
+    def remember(self, layer: MemoryLayer, content: str, metadata: dict = None, parent_id: str = None, result_ref: str = None) -> MemoryEntry:
+        entry = self.pyramid.add(layer, content, metadata, parent_id, result_ref)
+        self.backend.add_entry(
+            entry.id,
+            layer.value,
+            content,
+            metadata,
+            parent_id,
+            result_ref,
+        )
+        self.recall.add_documents([content], [entry.id], {entry.id: metadata or {}})
+        self._save_recall_index()
+        self._evict_if_needed(layer)
+        return entry
+
+    def forget(self, layer: MemoryLayer, entry_id: str) -> bool:
+        self.pyramid.remove(layer, entry_id)
+        self.backend.delete_entry(entry_id)
+        self.recall.remove_documents([entry_id])
+        self._save_recall_index()
+        return True
+
+    def clear_old_memories(self, layer: MemoryLayer, keep_last: int = 100) -> int:
+        removed_count, removed_ids = self.backend.clear_old_entries(layer.value, keep_last)
+        if removed_ids:
+            for eid in removed_ids:
+                self.pyramid.remove(layer, eid)
+            self.recall.remove_documents(removed_ids)
+            self._save_recall_index()
+        return removed_count
+
+    def search_memories(self, query: str, top_k: int = 5) -> list[RecallResult]:
+        return self.recall.hybrid_recall(query, top_k)
+
+    def _evict_if_needed(self, layer: MemoryLayer) -> None:
+        count = self.backend.count_by_layer().get(layer.value, 0)
+        if count > self.max_docs:
+            self.clear_old_memories(layer, keep_last=self.max_docs)

package/onecode/memory/backend.py

@@ -0,0 +1,179 @@
+from __future__ import annotations
+
+from contextlib import contextmanager
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Optional
+
+from sqlalchemy import Column, String, Text, DateTime, JSON, create_engine, Index
+from sqlalchemy.orm import declarative_base, sessionmaker, Session
+
+from onecode.config import ONECODE_DIR
+
+
+Base = declarative_base()
+
+
+class MemoryRecord(Base):
+    __tablename__ = "memory_entries"
+
+    id = Column(String(16), primary_key=True)
+    layer = Column(String(20), nullable=False, index=True)
+    content = Column(Text, nullable=False)
+    timestamp = Column(DateTime, nullable=False, index=True)
+    metadata_json = Column(JSON, default=dict)
+    parent_id = Column(String(16), nullable=True, index=True)
+    result_ref = Column(String(255), nullable=True)
+
+    __table_args__ = (
+        Index("idx_layer_timestamp", "layer", "timestamp"),
+        Index("idx_parent_id", "parent_id"),
+    )
+
+
+class MemoryBackend:
+    def __init__(self, db_path: Optional[Path] = None):
+        if db_path is None:
+            db_path = ONECODE_DIR / "memory" / "memory.db"
+        db_path.parent.mkdir(parents=True, exist_ok=True)
+        self.engine = create_engine(f"sqlite:///{db_path}", echo=False)
+        with self.engine.connect() as conn:
+            conn.exec_driver_sql("PRAGMA journal_mode=WAL")
+        Base.metadata.create_all(self.engine)
+        self.Session = sessionmaker(bind=self.engine)
+
+    @contextmanager
+    def session(self) -> Session:
+        s = self.Session()
+        try:
+            yield s
+            s.commit()
+        except Exception:
+            s.rollback()
+            raise
+        finally:
+            s.close()
+
+    def add_entry(
+        self,
+        entry_id: str,
+        layer: str,
+        content: str,
+        metadata: Optional[dict] = None,
+        parent_id: Optional[str] = None,
+        result_ref: Optional[str] = None,
+    ) -> bool:
+        with self.session() as s:
+            record = MemoryRecord(
+                id=entry_id,
+                layer=layer,
+                content=content,
+                timestamp=datetime.now(timezone.utc),
+                metadata_json=metadata or {},
+                parent_id=parent_id,
+                result_ref=result_ref,
+            )
+            s.merge(record)
+            return True
+
+    def get_entry(self, entry_id: str) -> Optional[dict]:
+        with self.session() as s:
+            record = s.query(MemoryRecord).filter_by(id=entry_id).first()
+            if record:
+                return {
+                    "id": record.id,
+                    "layer": record.layer,
+                    "content": record.content,
+                    "timestamp": record.timestamp.isoformat() if record.timestamp else "",
+                    "metadata_json": record.metadata_json,
+                    "parent_id": record.parent_id,
+                    "result_ref": record.result_ref,
+                }
+            return None
+
+    def get_entries_by_layer(self, layer: str, limit: int = 100) -> list[MemoryRecord]:
+        with self.session() as s:
+            records = (
+                s.query(MemoryRecord)
+                .filter_by(layer=layer)
+                .order_by(MemoryRecord.timestamp.desc())
+                .limit(limit)
+                .all()
+            )
+            return records
+
+    def get_entries_by_parent(self, parent_id: str) -> list[MemoryRecord]:
+        with self.session() as s:
+            return (
+                s.query(MemoryRecord)
+                .filter_by(parent_id=parent_id)
+                .order_by(MemoryRecord.timestamp)
+                .all()
+            )
+
+    def get_all_entries(self, layer: Optional[str] = None) -> list[dict]:
+        with self.session() as s:
+            query = s.query(MemoryRecord).order_by(MemoryRecord.timestamp.desc())
+            if layer:
+                query = query.filter_by(layer=layer)
+            return [
+                {
+                    "id": r.id,
+                    "layer": r.layer,
+                    "content": r.content,
+                    "timestamp": r.timestamp.isoformat() if r.timestamp else "",
+                    "metadata_json": r.metadata_json or {},
+                    "parent_id": r.parent_id,
+                    "result_ref": r.result_ref,
+                }
+                for r in query.all()
+            ]
+
+    def get_recent_entries(self, layer: Optional[str] = None, limit: int = 20) -> list[MemoryRecord]:
+        with self.session() as s:
+            query = s.query(MemoryRecord).order_by(MemoryRecord.timestamp.desc())
+            if layer:
+                query = query.filter_by(layer=layer)
+            return query.limit(limit).all()
+
+    def search_content(self, query: str, layer: Optional[str] = None, limit: int = 10) -> list[MemoryRecord]:
+        with self.session() as s:
+            q = s.query(MemoryRecord)
+            if layer:
+                q = q.filter_by(layer=layer)
+            escaped = query.replace("\\", "\\\\").replace("%", "\\%").replace("_", "\\_")
+            pattern = f"%{escaped}%"
+            q = q.filter(MemoryRecord.content.like(pattern, escape="\\"))
+            records = q.order_by(MemoryRecord.timestamp.desc()).limit(limit).all()
+            return records
+
+    def delete_entry(self, entry_id: str) -> bool:
+        with self.session() as s:
+            record = s.query(MemoryRecord).filter_by(id=entry_id).first()
+            if record:
+                s.delete(record)
+                return True
+            return False
+
+    def count_by_layer(self) -> dict[str, int]:
+        with self.session() as s:
+            from sqlalchemy import func
+            results = s.query(MemoryRecord.layer, func.count(MemoryRecord.id)).group_by(MemoryRecord.layer).all()
+            return {layer: count for layer, count in results}
+
+    def clear_old_entries(self, layer: str, keep_last: int = 100) -> tuple[int, list[str]]:
+        removed_ids = []
+        with self.session() as s:
+            count = s.query(MemoryRecord).filter_by(layer=layer).count()
+            if count > keep_last:
+                old_entries = (
+                    s.query(MemoryRecord)
+                    .filter_by(layer=layer)
+                    .order_by(MemoryRecord.timestamp.asc())
+                    .limit(count - keep_last)
+                    .all()
+                )
+                for entry in old_entries:
+                    removed_ids.append(entry.id)
+                    s.delete(entry)
+        return len(removed_ids), removed_ids

package/onecode/memory/pyramid.py

@@ -0,0 +1,145 @@
+from __future__ import annotations
+
+import json
+import uuid
+from dataclasses import dataclass, field
+from datetime import datetime, timezone
+from enum import Enum
+from pathlib import Path
+from typing import Optional
+
+
+class MemoryLayer(Enum):
+    L0_CONVERSATION = "l0_conversation"
+
+
+@dataclass
+class MemoryEntry:
+    id: str
+    layer: MemoryLayer
+    content: str
+    timestamp: str
+    metadata: dict = field(default_factory=dict)
+    parent_id: Optional[str] = None
+    result_ref: Optional[str] = None
+
+    @classmethod
+    def create(
+        cls,
+        layer: MemoryLayer,
+        content: str,
+        metadata: Optional[dict] = None,
+        parent_id: Optional[str] = None,
+    ) -> "MemoryEntry":
+        return cls(
+            id=str(uuid.uuid4())[:16],
+            layer=layer,
+            content=content,
+            timestamp=datetime.now(timezone.utc).isoformat(),
+            metadata=metadata or {},
+            parent_id=parent_id,
+        )
+
+    def to_dict(self) -> dict:
+        return {
+            "id": self.id,
+            "layer": self.layer.value,
+            "content": self.content,
+            "timestamp": self.timestamp,
+            "metadata": self.metadata,
+            "parent_id": self.parent_id,
+            "result_ref": self.result_ref,
+        }
+
+    @classmethod
+    def from_dict(cls, data: dict) -> "MemoryEntry":
+        return cls(
+            id=data["id"],
+            layer=MemoryLayer(data["layer"]),
+            content=data["content"],
+            timestamp=data["timestamp"],
+            metadata=data.get("metadata", {}),
+            parent_id=data.get("parent_id"),
+            result_ref=data.get("result_ref"),
+        )
+
+
+class MemoryPyramid:
+    def __init__(self, storage_path: Optional[Path] = None):
+        from onecode.config import ONECODE_DIR
+        self.storage_path = storage_path or (ONECODE_DIR / "memory")
+        self.storage_path.mkdir(parents=True, exist_ok=True)
+        self._layers = {layer: [] for layer in MemoryLayer}
+        self._load_index()
+
+    def _layer_dir(self, layer: MemoryLayer) -> Path:
+        return self.storage_path / layer.value
+
+    def _load_index(self) -> None:
+        index_file = self.storage_path / "index.json"
+        if index_file.exists():
+            try:
+                data = json.loads(index_file.read_text(encoding="utf-8"))
+                for layer_str, entries in data.items():
+                    layer = MemoryLayer(layer_str)
+                    self._layers[layer] = [MemoryEntry.from_dict(e) for e in entries]
+            except Exception:
+                pass
+
+    def _save_index(self) -> None:
+        index_file = self.storage_path / "index.json"
+        data = {layer.value: [e.to_dict() for e in entries] for layer, entries in self._layers.items()}
+        index_file.write_text(json.dumps(data, indent=2, ensure_ascii=False), encoding="utf-8")
+
+    def add(
+        self,
+        layer: MemoryLayer,
+        content: str,
+        metadata: Optional[dict] = None,
+        parent_id: Optional[str] = None,
+        result_ref: Optional[str] = None,
+    ) -> MemoryEntry:
+        entry = MemoryEntry.create(layer, content, metadata, parent_id)
+        entry.result_ref = result_ref
+        self._layers[layer].append(entry)
+        self._save_index()
+        self._write_content(layer, entry)
+        return entry
+
+    def _write_content(self, layer: MemoryLayer, entry: MemoryEntry) -> None:
+        layer_dir = self._layer_dir(layer)
+        layer_dir.mkdir(parents=True, exist_ok=True)
+        content_file = layer_dir / f"{entry.id}.md"
+        content_file.write_text(entry.content, encoding="utf-8")
+
+    def get(self, layer: MemoryLayer, entry_id: str) -> Optional[MemoryEntry]:
+        for entry in self._layers[layer]:
+            if entry.id == entry_id:
+                return entry
+        return None
+
+    def get_content(self, entry: MemoryEntry) -> str:
+        layer_dir = self._layer_dir(entry.layer)
+        content_file = layer_dir / f"{entry.id}.md"
+        if content_file.exists():
+            return content_file.read_text(encoding="utf-8")
+        return entry.content
+
+    def list_by_layer(self, layer: MemoryLayer) -> list[MemoryEntry]:
+        return list(self._layers.get(layer, []))
+
+    def list_recent(self, layer: MemoryLayer, limit: int = 10) -> list[MemoryEntry]:
+        entries = self._layers.get(layer, [])
+        return sorted(entries, key=lambda e: e.timestamp, reverse=True)[:limit]
+
+    def remove(self, layer: MemoryLayer, entry_id: str) -> bool:
+        entries = self._layers.get(layer, [])
+        for i, entry in enumerate(entries):
+            if entry.id == entry_id:
+                content_file = self._layer_dir(layer) / f"{entry_id}.md"
+                if content_file.exists():
+                    content_file.unlink()
+                self._layers[layer].pop(i)
+                self._save_index()
+                return True
+        return False