@scemoon/cdh 1.0.9 → 1.0.10-beta.1

package/onecode/memory/recall.py

@@ -1,322 +0,0 @@
-from __future__ import annotations
-
-import math
-import re
-from dataclasses import dataclass
-from typing import Optional
-
-
-@dataclass
-class RecallResult:
-    entry_id: str
-    content: str
-    score: float
-    source: str
-    metadata: dict
-
-
-class BM25:
-    # Hard cap above which ``index()`` refuses to build the inverted index.
-    # The original O(V × N) construction of ``doc_freqs`` (set membership
-    # over Python lists) becomes effectively uncallable above a few
-    # thousand docs — a 79k-chunk corpus (observed in production where
-    # sessions kept re-indexing without cleaning the SQLite store) took
-    # ~1 hour, blocking the event loop and freezing streaming + cancel.
-    # Above this cap we degrade to a no-op index so callers fall back to
-    # other retrievers / explicit search instead of hanging the process.
-    _MAX_DOCS = 5000
-
-    def __init__(self, k1: float = 1.5, b: float = 0.75):
-        self.k1 = k1
-        self.b = b
-        self.doc_lengths: list[int] = []
-        self.avgdl: float = 0
-        self.doc_freqs: dict[str, int] = {}
-        self.num_docs: int = 0
-        self.doc_tokens: list[list[str]] = []
-        # Tokenize once per doc as ``set`` (not list) so the
-        # ``token in doc`` membership check inside the doc_freqs
-        # comprehension is O(1) instead of O(doc_len).  This alone takes
-        # the 79k-doc construction from ~hours to ~seconds; the hard cap
-        # above is a safety net for pathological corpora.
-        self._doc_token_sets: list[set[str]] = []
-
-    def index(self, documents: list[str]) -> None:
-        if len(documents) > self._MAX_DOCS:
-            import logging
-            logging.getLogger(__name__).warning(
-                "BM25.index refused: %d docs exceeds cap=%d (would block "
-                "the event loop for minutes); BM25 search will return no "
-                "results. Clean the codebase index or use a smaller "
-                "project scope.", len(documents), self._MAX_DOCS,
-            )
-            self.doc_tokens = []
-            self._doc_token_sets = []
-            self.doc_lengths = []
-            self.avgdl = 0
-            self.num_docs = 0
-            self.doc_freqs = {}
-            return
-
-        self.doc_tokens = [self._tokenize(doc) for doc in documents]
-        self._doc_token_sets = [set(t) for t in self.doc_tokens]
-        self.doc_lengths = [len(tokens) for tokens in self.doc_tokens]
-        self.avgdl = sum(self.doc_lengths) / len(self.doc_lengths) if self.doc_lengths else 0
-        self.num_docs = len(documents)
-
-        all_tokens = set()
-        for tokens in self._doc_token_sets:
-            all_tokens.update(tokens)
-        self.doc_freqs = {
-            token: sum(1 for ts in self._doc_token_sets if token in ts)
-            for token in all_tokens
-        }
-
-    def add_docs(self, documents: list[str]) -> None:
-        if not documents:
-            return
-        if self.num_docs + len(documents) > self._MAX_DOCS:
-            import logging
-            logging.getLogger(__name__).warning(
-                "BM25.add_docs refused: total %d docs would exceed cap=%d; "
-                "falling back to full index rebuild.", self.num_docs + len(documents), self._MAX_DOCS,
-            )
-            self.index(self.doc_tokens + documents)
-            return
-
-        new_tokens = [self._tokenize(doc) for doc in documents]
-        new_token_sets = [set(t) for t in new_tokens]
-        new_lengths = [len(tokens) for tokens in new_tokens]
-
-        for token_set in new_token_sets:
-            for token in token_set:
-                self.doc_freqs[token] = self.doc_freqs.get(token, 0) + 1
-
-        self.doc_tokens.extend(new_tokens)
-        self._doc_token_sets.extend(new_token_sets)
-        self.doc_lengths.extend(new_lengths)
-
-        total_len = self.avgdl * self.num_docs + sum(new_lengths)
-        self.num_docs += len(documents)
-        self.avgdl = total_len / self.num_docs if self.num_docs > 0 else 0
-
-    def remove_doc_at(self, doc_index: int) -> None:
-        if doc_index < 0 or doc_index >= self.num_docs:
-            return
-        removed_token_set = self._doc_token_sets[doc_index]
-        for token in removed_token_set:
-            self.doc_freqs[token] = self.doc_freqs.get(token, 1) - 1
-            if self.doc_freqs[token] <= 0:
-                del self.doc_freqs[token]
-
-        total_len = self.avgdl * self.num_docs - self.doc_lengths[doc_index]
-        self.num_docs -= 1
-
-        del self.doc_tokens[doc_index]
-        del self._doc_token_sets[doc_index]
-        del self.doc_lengths[doc_index]
-
-        self.avgdl = total_len / self.num_docs if self.num_docs > 0 else 0
-
-    def _tokenize(self, text: str) -> list[str]:
-        text = text.lower()
-        tokens = re.findall(r"\b\w+\b", text)
-        return tokens
-
-    def score(self, query: str, doc_index: int) -> float:
-        query_tokens = self._tokenize(query)
-        return self._score_with_tokens(query_tokens, doc_index)
-
-    def _score_with_tokens(self, query_tokens: list[str], doc_index: int) -> float:
-        doc_tokens = self.doc_tokens[doc_index]
-        doc_len = self.doc_lengths[doc_index]
-        score = 0.0
-
-        for qt in query_tokens:
-            if qt not in self.doc_freqs:
-                continue
-            df = self.doc_freqs[qt]
-            idf = math.log((self.num_docs - df + 0.5) / (df + 0.5) + 1)
-            tf = doc_tokens.count(qt)
-            tf_score = (tf * (self.k1 + 1)) / (tf + self.k1 * (1 - self.b + self.b * doc_len / self.avgdl))
-            score += idf * tf_score
-        return score
-
-    def search(self, query: str, top_k: int = 5) -> list[tuple[int, float]]:
-        query_tokens = self._tokenize(query)
-        scores = [(i, self._score_with_tokens(query_tokens, i)) for i in range(self.num_docs)]
-        scores.sort(key=lambda x: x[1], reverse=True)
-        return scores[:top_k]
-
-    def serialize(self) -> dict:
-        return {
-            "k1": self.k1,
-            "b": self.b,
-            "doc_lengths": self.doc_lengths,
-            "avgdl": self.avgdl,
-            "doc_freqs": self.doc_freqs,
-            "num_docs": self.num_docs,
-            "doc_tokens": self.doc_tokens,
-            "_doc_token_sets": [list(s) for s in self._doc_token_sets],
-        }
-
-    @classmethod
-    def deserialize(cls, data: dict) -> "BM25":
-        bm25 = cls(k1=data["k1"], b=data["b"])
-        bm25.doc_lengths = data["doc_lengths"]
-        bm25.avgdl = data["avgdl"]
-        bm25.doc_freqs = data["doc_freqs"]
-        bm25.num_docs = data["num_docs"]
-        bm25.doc_tokens = data["doc_tokens"]
-        bm25._doc_token_sets = [set(s) for s in data["_doc_token_sets"]]
-        return bm25
-
-
-class HybridRecall:
-    def __init__(
-        self,
-        bm25: Optional[BM25] = None,
-        embedding_fn: Optional[callable] = None,
-    ):
-        self.bm25 = bm25 or BM25()
-        self.embedding_fn = embedding_fn
-        self._documents: list[str] = []
-        self._embeddings: list[list[float]] = []
-        self._entry_ids: list[str] = []
-        self._metadata: dict[str, dict] = {}
-
-    def add_documents(self, documents: list[str], entry_ids: Optional[list[str]] = None, metadata: Optional[dict] = None) -> None:
-        start_idx = len(self._documents)
-        self._documents.extend(documents)
-        for i, doc in enumerate(documents):
-            eid = entry_ids[i] if entry_ids and i < len(entry_ids) else f"doc_{start_idx + i}"
-            self._entry_ids.append(eid)
-            if metadata and eid in metadata:
-                self._metadata[eid] = metadata[eid]
-
-        if start_idx == 0:
-            self.bm25.index(self._documents)
-        else:
-            self.bm25.add_docs(documents)
-
-        if self.embedding_fn:
-            for doc in documents:
-                emb = self.embedding_fn(doc)
-                self._embeddings.append(emb)
-
-    def remove_documents(self, entry_ids: list[str]) -> None:
-        ids_to_remove = set(entry_ids)
-        kept_docs = []
-        kept_ids = []
-        kept_embs = []
-
-        for i, eid in enumerate(self._entry_ids):
-            if eid not in ids_to_remove:
-                kept_docs.append(self._documents[i])
-                kept_ids.append(eid)
-                if i < len(self._embeddings):
-                    kept_embs.append(self._embeddings[i])
-
-        for eid in ids_to_remove:
-            self._metadata.pop(eid, None)
-
-        self._documents = kept_docs
-        self._entry_ids = kept_ids
-        self._embeddings = kept_embs
-
-        self.bm25 = BM25()
-        if self._documents:
-            self.bm25.index(self._documents)
-
-    def keyword_recall(self, query: str, top_k: int = 5) -> list[RecallResult]:
-        bm25_results = self.bm25.search(query, top_k * 2)
-        results = []
-        for doc_idx, bm25_score in bm25_results:
-            if doc_idx < len(self._entry_ids):
-                eid = self._entry_ids[doc_idx]
-                results.append(RecallResult(
-                    entry_id=eid,
-                    content=self._documents[doc_idx],
-                    score=bm25_score,
-                    source="bm25",
-                    metadata=self._metadata.get(eid, {}),
-                ))
-        return results[:top_k]
-
-    def semantic_recall(self, query: str, top_k: int = 5) -> list[RecallResult]:
-        if not self.embedding_fn:
-            return []
-
-        query_emb = self.embedding_fn(query)
-        similarities = []
-
-        for i, emb in enumerate(self._embeddings):
-            sim = self._cosine_similarity(query_emb, emb)
-            similarities.append((i, sim))
-
-        similarities.sort(key=lambda x: x[1], reverse=True)
-        results = []
-        for doc_idx, sim_score in similarities[:top_k]:
-            eid = self._entry_ids[doc_idx]
-            results.append(RecallResult(
-                entry_id=eid,
-                content=self._documents[doc_idx],
-                score=sim_score,
-                source="embedding",
-                metadata=self._metadata.get(eid, {}),
-            ))
-        return results
-
-    def _cosine_similarity(self, a: list[float], b: list[float]) -> float:
-        dot = sum(x * y for x, y in zip(a, b))
-        norm_a = math.sqrt(sum(x * x for x in a))
-        norm_b = math.sqrt(sum(x * x for x in b))
-        if norm_a == 0 or norm_b == 0:
-            return 0
-        return dot / (norm_a * norm_b)
-
-    def hybrid_recall(self, query: str, top_k: int = 5, rrf_k: int = 60) -> list[RecallResult]:
-        bm25_results = self.keyword_recall(query, top_k * 2)
-        semantic_results = self.semantic_recall(query, top_k * 2)
-
-        rrf_scores: dict[str, float] = {}
-        rrf_results: dict[str, RecallResult] = {}
-
-        for rank, result in enumerate(bm25_results):
-            score = 1.0 / (rrf_k + rank + 1)
-            rrf_scores[result.entry_id] = rrf_scores.get(result.entry_id, 0) + score
-            rrf_results[result.entry_id] = result
-
-        for rank, result in enumerate(semantic_results):
-            score = 1.0 / (rrf_k + rank + 1)
-            rrf_scores[result.entry_id] = rrf_scores.get(result.entry_id, 0) + score
-            rrf_results[result.entry_id] = result
-
-        sorted_ids = sorted(rrf_scores.items(), key=lambda x: x[1], reverse=True)
-        return [rrf_results[eid] for eid, _ in sorted_ids[:top_k]]
-
-    def clear(self) -> None:
-        self._documents = []
-        self._embeddings = []
-        self._entry_ids = []
-        self._metadata = {}
-        self.bm25 = BM25()
-
-    def serialize(self) -> dict:
-        return {
-            "bm25": self.bm25.serialize() if hasattr(self.bm25, "serialize") else {},
-            "_documents": self._documents,
-            "_embeddings": self._embeddings,
-            "_entry_ids": self._entry_ids,
-            "_metadata": self._metadata,
-        }
-
-    @classmethod
-    def deserialize(cls, data: dict) -> "HybridRecall":
-        hr = cls()
-        hr.bm25 = BM25.deserialize(data["bm25"]) if data.get("bm25") else BM25()
-        hr._documents = data.get("_documents", [])
-        hr._embeddings = data.get("_embeddings", [])
-        hr._entry_ids = data.get("_entry_ids", [])
-        hr._metadata = data.get("_metadata", {})
-        return hr

package/onecode/migrate.py

@@ -1,122 +0,0 @@
-"""onecode legacy migration — move onecode's private state from ~/.cdh/ to ~/.onecode/.
-
-This is a ONE-TIME migration triggered at first startup after upgrade.
-It only moves directories owned by onecode (logs, traces, memory,
-snapshots, mcps, models, onecode.config.yaml). cdh platform directories
-(projects, state, sessions) are left untouched.  Session JSON is owned
-by the cdh platform layer (B mapping mode), not by individual engines.
-"""
-
-from __future__ import annotations
-
-import json
-import logging
-import shutil
-from pathlib import Path
-
-logger = logging.getLogger("onecode.migrate")
-
-# Directories/state that belong to onecode and should be migrated
-# NOTE: sessions is deliberately excluded — it belongs to the cdh
-# platform layer (B mapping mode), not to any individual engine.
-_ONECODE_PRIVATE_DIRS = {
-    "logs",
-    "traces",
-    "memory",
-    "snapshots",
-    "mcps",
-    "models",
-}
-
-# cdh platform directories — NEVER touch
-_CDH_PLATFORM_DIRS = {"projects", "state"}
-
-# Dotfiles that belong to onecode (at the ~/.cdh/ root level)
-_ONECODE_DOTFILES = {"onecode.config.yaml"}
-
-
-def migrate_legacy_cdh_to_onecode(
-    legacy_dir: Path | None = None,
-    target_dir: Path | None = None,
-) -> str | None:
-    """Migrate onecode's private state from ~/.cdh/ to ~/.onecode/.
-
-    Args:
-        legacy_dir: Source directory (default ~/.cdh/).
-        target_dir: Target directory (default ~/.onecode/).
-
-    Returns:
-        Info/warning message string, or None if no migration needed.
-    """
-    legacy = (legacy_dir or Path.home() / ".cdh").resolve()
-    target = (target_dir or Path.home() / ".onecode").resolve()
-
-    # Nothing to migrate
-    if not legacy.exists():
-        return None
-
-    # Already migrated (marker exists)
-    migrated_marker = target / ".migrated_from"
-    if migrated_marker.exists():
-        return None
-
-    # Both exist but no marker → user may have manually set up ~/.onecode/
-    if target.exists():
-        # Check if target has any onecode dirs already
-        has_content = any(
-            (target / d).exists() for d in _ONECODE_PRIVATE_DIRS
-        ) or target / "onecode.config.yaml" in target.iterdir()
-        if has_content:
-            return None  # 各自独立运行，无需迁移
-        # Target exists but empty — safe to proceed
-
-    # Perform migration
-    target.mkdir(parents=True, exist_ok=True)
-    migrated_items = []
-
-    # Migrate private subdirectories
-    for dir_name in _ONECODE_PRIVATE_DIRS:
-        src = legacy / dir_name
-        if src.exists() and src.is_dir():
-            dst = target / dir_name
-            if dst.exists():
-                # Merge contents (don't overwrite existing)
-                for item in src.iterdir():
-                    dst_item = dst / item.name
-                    if not dst_item.exists():
-                        shutil.move(str(item), str(dst_item))
-            else:
-                shutil.move(str(src), str(dst))
-            migrated_items.append(dir_name)
-
-    # Migrate onecode dotfiles
-    for fname in _ONECODE_DOTFILES:
-        src = legacy / fname
-        if src.exists() and src.is_file():
-            dst = target / fname
-            if not dst.exists():
-                shutil.move(str(src), str(dst))
-                migrated_items.append(fname)
-
-    # Write migration marker
-    marker_data = {
-        "migrated_from": str(legacy),
-        "migrated_to": str(target),
-        "items": sorted(migrated_items),
-        "preserved_on_legacy": sorted(_CDH_PLATFORM_DIRS),
-    }
-    migrated_marker.write_text(
-        json.dumps(marker_data, ensure_ascii=False, indent=2),
-        encoding="utf-8",
-    )
-
-    if migrated_items:
-        msg = (
-            f"⚠ Migrated onecode state from {legacy} → {target}: "
-            f"{', '.join(migrated_items)}. "
-            f"cdh platform dirs ({', '.join(_CDH_PLATFORM_DIRS)}) preserved at {legacy}."
-        )
-        logger.warning(msg)
-        return msg
-
-    return None

package/onecode/models/__init__.py

@@ -1,4 +0,0 @@
-from onecode.models.registry import ModelRegistry, ModelInfo
-from onecode.models.provider import Provider, ProviderRegistry
-
-__all__ = ["ModelRegistry", "ModelInfo", "Provider", "ProviderRegistry"]

package/onecode/models/errors.py

@@ -1,104 +0,0 @@
-"""Provider exception hierarchy.
-
-These are raised by the ``onecode.models.providers.*`` modules when an upstream
-LLM call fails in a way the agent engine should surface to the user as an
-error event (rather than rendering the raw response body as agent text).
-
-The engine's ``chat_stream`` loop already has::
-
-    except Exception as e:
-        yield StreamEvent.error(str(e))
-        break
-
-so simply raising any subclass of :class:`ProviderError` is enough to route
-the failure through the TUI as a proper ``agent_message_chunk`` with
-``sessionUpdate: "error"``.
-"""
-from __future__ import annotations
-
-import time
-from typing import TYPE_CHECKING, Optional
-
-if TYPE_CHECKING:
-    import httpx
-
-
-def safe_error_msg(e: BaseException) -> str:
-    """Return a non-empty error message from an exception.
-
-    Python's ``str(Exception())`` returns ``""``, which leads to
-    confusing ``"Error: "`` messages in the UI.  This helper falls
-    back to the exception type name when the message is empty.
-    """
-    return str(e) or type(e).__name__
-
-
-def retry_after_seconds(resp: "httpx.Response") -> Optional[float]:
-    """Parse the ``Retry-After`` header from an HTTP response.
-
-    Accepts both the delta-seconds form and the HTTP-date form (returning
-    the delta from ``time.time()`` in either case).  Returns ``None`` if
-    the header is missing or unparseable.
-    """
-    raw = resp.headers.get("Retry-After") or resp.headers.get("retry-after")
-    if not raw:
-        return None
-    try:
-        return max(0.0, float(raw))
-    except ValueError:
-        try:
-            from email.utils import parsedate_to_datetime
-            target = parsedate_to_datetime(raw).timestamp()
-            return max(0.0, target - time.time())
-        except Exception:
-            return None
-
-
-class ProviderError(Exception):
-    """Base class for all provider errors.
-
-    Attributes:
-        status_code: HTTP status code returned by the upstream, when
-            applicable (``None`` for transport-level failures).
-        retry_after: ``Retry-After`` header value, when the upstream
-            provided one (``None`` otherwise).
-        body: Raw response body, truncated to 4 KiB.
-    """
-
-    def __init__(
-        self,
-        message: str,
-        *,
-        status_code: Optional[int] = None,
-        retry_after: Optional[float] = None,
-        body: Optional[str] = None,
-    ) -> None:
-        super().__init__(message)
-        self.status_code = status_code
-        self.retry_after = retry_after
-        self.body = (body or "")[:4096]
-
-    def to_user_message(self) -> str:
-        """Build a short, user-friendly message for the TUI."""
-        head = f"Error: {safe_error_msg(self)}" if self.args else "Error: provider failure"
-        if self.status_code is not None:
-            head += f" (HTTP {self.status_code})"
-        if self.retry_after is not None:
-            head += f" — retry after {self.retry_after:.0f}s"
-        return head
-
-
-class RateLimitError(ProviderError):
-    """HTTP 429 / upstream rate-limit signal."""
-
-
-class AuthError(ProviderError):
-    """HTTP 401 / 403 — invalid or missing credentials."""
-
-
-class ContextLengthError(ProviderError):
-    """HTTP 400 with a context-length-exceeded error from the upstream."""
-
-
-class TransientProviderError(ProviderError):
-    """5xx or network error — safe to retry."""