@scemoon/cdh 1.0.0

package/onecode/memory/offload.py

@@ -0,0 +1,114 @@
+from __future__ import annotations
+
+import re
+import fnmatch
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Optional
+
+
+@dataclass
+class OffloadConfig:
+    enabled: bool = True
+    mild_ratio: float = 0.5
+    aggressive_ratio: float = 0.85
+    mmd_max_token_ratio: float = 0.2
+
+
+class ContextOffloader:
+    def __init__(
+        self,
+        refs_dir: Optional[Path] = None,
+        config: Optional[OffloadConfig] = None,
+    ):
+        from onecode.config import CLOUD_DEV_HARNESS_DIR
+        self.refs_dir = refs_dir or (CLOUD_DEV_HARNESS_DIR / "memory" / "refs")
+        self.refs_dir.mkdir(parents=True, exist_ok=True)
+        self.config = config or OffloadConfig()
+        self._offloaded_refs: dict[str, Path] = {}
+
+    def should_offload(self, context_length: int, max_tokens: int) -> tuple[bool, str]:
+        if not self.config.enabled:
+            return False, "disabled"
+
+        ratio = context_length / max_tokens if max_tokens > 0 else 0
+
+        if ratio >= self.config.aggressive_ratio:
+            return True, "aggressive"
+        elif ratio >= self.config.mild_ratio:
+            return True, "mild"
+        return False, "none"
+
+    def offload_content(
+        self,
+        content: str,
+        node_id: str,
+        file_pattern: str = "*.md",
+    ) -> str:
+        ext = file_pattern.replace("*", "md")
+        ref_file = self.refs_dir / f"{node_id}.{ext}"
+        ref_file.write_text(content, encoding="utf-8")
+        self._offloaded_refs[node_id] = ref_file
+        return f"See refs/{node_id}.{ext}"
+
+    def recall_ref(self, node_id: str) -> Optional[str]:
+        ref_file = self._offloaded_refs.get(node_id)
+        if ref_file and ref_file.exists():
+            return ref_file.read_text(encoding="utf-8")
+        for ext in ["md", "txt", "log"]:
+            alt_path = self.refs_dir / f"{node_id}.{ext}"
+            if alt_path.exists():
+                return alt_path.read_text(encoding="utf-8")
+        return None
+
+    def list_offloaded(self) -> list[str]:
+        return list(self._offloaded_refs.keys())
+
+    def cleanup_old_refs(self, max_age_hours: int = 24) -> int:
+        import time
+        from datetime import datetime, timezone
+        removed = 0
+        for ref_file in self.refs_dir.iterdir():
+            if ref_file.is_file():
+                age_hours = (time.time() - ref_file.stat().st_mtime) / 3600
+                if age_hours > max_age_hours:
+                    ref_file.unlink()
+                    node_id = ref_file.stem
+                    if node_id in self._offloaded_refs:
+                        del self._offloaded_refs[node_id]
+                    removed += 1
+        return removed
+
+
+class ToolCallExtractor:
+    PATTERNS = [
+        re.compile(r'\[TOOL_CALL\]\s*(\w+)\s*:\s*(.+)', re.DOTALL),
+        re.compile(r'"tool":\s*"(\w+)"[^}]*"args":\s*({.+?})', re.DOTALL),
+        re.compile(r'(\w+)\((.*?)\)\s*=>\s*(.+)', re.DOTALL),
+    ]
+
+    @classmethod
+    def extract_calls(cls, text: str) -> list[dict]:
+        calls = []
+        for pattern in cls.PATTERNS:
+            matches = pattern.finditer(text)
+            for match in matches:
+                if pattern == cls.PATTERNS[0]:
+                    calls.append({"tool": match.group(1), "args": match.group(2)})
+                elif pattern == cls.PATTERNS[1]:
+                    calls.append({"tool": match.group(1), "args": match.group(2)})
+                else:
+                    calls.append({"tool": match.group(1), "args": match.group(2), "result": match.group(3)})
+        return calls
+
+    @classmethod
+    def format_as_mermaid(cls, calls: list[dict]) -> str:
+        lines = ["graph LR"]
+        for i, call in enumerate(calls):
+            node_id = f"C{i:03d}"
+            tool_name = call.get("tool", "unknown")
+            lines.append(f"    {node_id}[{tool_name}]")
+            if i > 0:
+                prev_id = f"C{i-1:03d}"
+                lines.append(f"    {prev_id} --> {node_id}")
+        return "\n".join(lines)

package/onecode/memory/pyramid.py

@@ -0,0 +1,202 @@
+from __future__ import annotations
+
+import json
+import uuid
+from dataclasses import dataclass, field
+from datetime import datetime, timezone
+from enum import Enum
+from pathlib import Path
+from typing import Optional
+
+
+class MemoryLayer(Enum):
+    L0_CONVERSATION = "l0_conversation"
+    L1_ATOM = "l1_atom"
+    L2_SCENARIO = "l2_scenario"
+    L3_PERSONA = "l3_persona"
+
+
+@dataclass
+class MemoryEntry:
+    id: str
+    layer: MemoryLayer
+    content: str
+    timestamp: str
+    metadata: dict = field(default_factory=dict)
+    parent_id: Optional[str] = None
+    result_ref: Optional[str] = None
+
+    @classmethod
+    def create(
+        cls,
+        layer: MemoryLayer,
+        content: str,
+        metadata: Optional[dict] = None,
+        parent_id: Optional[str] = None,
+    ) -> "MemoryEntry":
+        return cls(
+            id=str(uuid.uuid4())[:16],
+            layer=layer,
+            content=content,
+            timestamp=datetime.now(timezone.utc).isoformat(),
+            metadata=metadata or {},
+            parent_id=parent_id,
+        )
+
+    def to_dict(self) -> dict:
+        return {
+            "id": self.id,
+            "layer": self.layer.value,
+            "content": self.content,
+            "timestamp": self.timestamp,
+            "metadata": self.metadata,
+            "parent_id": self.parent_id,
+            "result_ref": self.result_ref,
+        }
+
+    @classmethod
+    def from_dict(cls, data: dict) -> "MemoryEntry":
+        return cls(
+            id=data["id"],
+            layer=MemoryLayer(data["layer"]),
+            content=data["content"],
+            timestamp=data["timestamp"],
+            metadata=data.get("metadata", {}),
+            parent_id=data.get("parent_id"),
+            result_ref=data.get("result_ref"),
+        )
+
+
+class MemoryPyramid:
+    def __init__(self, storage_path: Optional[Path] = None):
+        from onecode.config import CLOUD_DEV_HARNESS_DIR
+        self.storage_path = storage_path or (CLOUD_DEV_HARNESS_DIR / "memory")
+        self.storage_path.mkdir(parents=True, exist_ok=True)
+        self._layers = {layer: [] for layer in MemoryLayer}
+        self._load_index()
+
+    def _layer_dir(self, layer: MemoryLayer) -> Path:
+        return self.storage_path / layer.value
+
+    def _load_index(self) -> None:
+        index_file = self.storage_path / "index.json"
+        if index_file.exists():
+            try:
+                data = json.loads(index_file.read_text(encoding="utf-8"))
+                for layer_str, entries in data.items():
+                    layer = MemoryLayer(layer_str)
+                    self._layers[layer] = [MemoryEntry.from_dict(e) for e in entries]
+            except Exception:
+                pass
+
+    def _save_index(self) -> None:
+        index_file = self.storage_path / "index.json"
+        data = {layer.value: [e.to_dict() for e in entries] for layer, entries in self._layers.items()}
+        index_file.write_text(json.dumps(data, indent=2, ensure_ascii=False), encoding="utf-8")
+
+    def add(
+        self,
+        layer: MemoryLayer,
+        content: str,
+        metadata: Optional[dict] = None,
+        parent_id: Optional[str] = None,
+        result_ref: Optional[str] = None,
+    ) -> MemoryEntry:
+        entry = MemoryEntry.create(layer, content, metadata, parent_id)
+        entry.result_ref = result_ref
+        self._layers[layer].append(entry)
+        self._save_index()
+        self._write_content(layer, entry)
+        return entry
+
+    def _write_content(self, layer: MemoryLayer, entry: MemoryEntry) -> None:
+        layer_dir = self._layer_dir(layer)
+        layer_dir.mkdir(parents=True, exist_ok=True)
+        content_file = layer_dir / f"{entry.id}.md"
+        content_file.write_text(entry.content, encoding="utf-8")
+
+    def get(self, layer: MemoryLayer, entry_id: str) -> Optional[MemoryEntry]:
+        for entry in self._layers[layer]:
+            if entry.id == entry_id:
+                return entry
+        return None
+
+    def get_content(self, entry: MemoryEntry) -> str:
+        layer_dir = self._layer_dir(entry.layer)
+        content_file = layer_dir / f"{entry.id}.md"
+        if content_file.exists():
+            return content_file.read_text(encoding="utf-8")
+        return entry.content
+
+    def list_by_layer(self, layer: MemoryLayer) -> list[MemoryEntry]:
+        return list(self._layers.get(layer, []))
+
+    def list_recent(self, layer: MemoryLayer, limit: int = 10) -> list[MemoryEntry]:
+        entries = self._layers.get(layer, [])
+        return sorted(entries, key=lambda e: e.timestamp, reverse=True)[:limit]
+
+    def drill_down(self, entry_id: str) -> list[MemoryEntry]:
+        chain = []
+        current = self._find_entry_anywhere(entry_id)
+        while current:
+            chain.append(current)
+            if current.parent_id:
+                current = self._find_entry_anywhere(current.parent_id)
+            else:
+                current = None
+        return list(reversed(chain))
+
+    def _find_entry_anywhere(self, entry_id: str) -> Optional[MemoryEntry]:
+        for entries in self._layers.values():
+            for entry in entries:
+                if entry.id == entry_id:
+                    return entry
+        return None
+
+    def extract_atoms_from_conversation(
+        self, conversation_id: str, max_atoms: int = 20
+    ) -> list[MemoryEntry]:
+        l0_entries = [e for e in self._layers[MemoryLayer.L0_CONVERSATION] if e.parent_id == conversation_id]
+        atoms = []
+        for l0 in l0_entries:
+            content = self.get_content(l0)
+            if len(content) > 100:
+                atom_content = content[:200] + "..."
+            else:
+                atom_content = content
+            atom = self.add(
+                MemoryLayer.L1_ATOM,
+                atom_content,
+                metadata={"source": "auto_extract", "source_id": l0.id},
+                parent_id=l0.id,
+            )
+            atoms.append(atom)
+            if len(atoms) >= max_atoms:
+                break
+        return atoms
+
+    def build_scenario_from_atoms(self, atom_ids: list[str], name: str) -> MemoryEntry:
+        atom_contents = []
+        for atom_id in atom_ids:
+            atom = self._find_entry_anywhere(atom_id)
+            if atom:
+                atom_contents.append(f"- {atom.content}")
+        content = f"## {name}\n\n" + "\n".join(atom_contents)
+        return self.add(
+            MemoryLayer.L2_SCENARIO,
+            content,
+            metadata={"atom_count": len(atom_ids)},
+        )
+
+    def build_persona_from_scenarios(self, scenario_ids: list[str], name: str) -> MemoryEntry:
+        scenario_contents = []
+        for sc_id in scenario_ids:
+            sc = self._find_entry_anywhere(sc_id)
+            if sc:
+                scenario_contents.append(f"### {sc.content[:100]}...\n{sc.content}")
+        content = f"# {name}\n\n" + "\n".join(scenario_contents)
+        return self.add(
+            MemoryLayer.L3_PERSONA,
+            content,
+            metadata={"scenario_count": len(scenario_ids)},
+        )

package/onecode/memory/recall.py

@@ -0,0 +1,168 @@
+from __future__ import annotations
+
+import math
+import re
+from collections import Counter
+from dataclasses import dataclass
+from typing import Optional
+
+
+@dataclass
+class RecallResult:
+    entry_id: str
+    content: str
+    score: float
+    source: str
+    metadata: dict
+
+
+class BM25:
+    def __init__(self, k1: float = 1.5, b: float = 0.75):
+        self.k1 = k1
+        self.b = b
+        self.doc_lengths: list[int] = []
+        self.avgdl: float = 0
+        self.doc_freqs: dict[str, int] = {}
+        self.num_docs: int = 0
+        self.doc_tokens: list[list[str]] = []
+
+    def index(self, documents: list[str]) -> None:
+        self.doc_tokens = [self._tokenize(doc) for doc in documents]
+        self.doc_lengths = [len(tokens) for tokens in self.doc_tokens]
+        self.avgdl = sum(self.doc_lengths) / len(self.doc_lengths) if self.doc_lengths else 0
+        self.num_docs = len(documents)
+
+        all_tokens = set()
+        for tokens in self.doc_tokens:
+            all_tokens.update(tokens)
+        self.doc_freqs = {token: sum(1 for doc in self.doc_tokens if token in doc) for token in all_tokens}
+
+    def _tokenize(self, text: str) -> list[str]:
+        text = text.lower()
+        tokens = re.findall(r"\b\w+\b", text)
+        return tokens
+
+    def score(self, query: str, doc_index: int) -> float:
+        query_tokens = self._tokenize(query)
+        doc_tokens = self.doc_tokens[doc_index]
+        doc_len = self.doc_lengths[doc_index]
+        score = 0.0
+
+        for qt in query_tokens:
+            if qt not in self.doc_freqs:
+                continue
+            df = self.doc_freqs[qt]
+            idf = math.log((self.num_docs - df + 0.5) / (df + 0.5) + 1)
+            tf = doc_tokens.count(qt)
+            tf_score = (tf * (self.k1 + 1)) / (tf + self.k1 * (1 - self.b + self.b * doc_len / self.avgdl))
+            score += idf * tf_score
+        return score
+
+    def search(self, query: str, top_k: int = 5) -> list[tuple[int, float]]:
+        scores = [(i, self.score(query, i)) for i in range(self.num_docs)]
+        scores.sort(key=lambda x: x[1], reverse=True)
+        return scores[:top_k]
+
+
+class HybridRecall:
+    def __init__(
+        self,
+        bm25: Optional[BM25] = None,
+        embedding_fn: Optional[callable] = None,
+    ):
+        self.bm25 = bm25 or BM25()
+        self.embedding_fn = embedding_fn
+        self._documents: list[str] = []
+        self._embeddings: list[list[float]] = []
+        self._entry_ids: list[str] = []
+        self._metadata: dict[str, dict] = {}
+
+    def add_documents(self, documents: list[str], entry_ids: Optional[list[str]] = None, metadata: Optional[dict] = None) -> None:
+        start_idx = len(self._documents)
+        self._documents.extend(documents)
+        for i, doc in enumerate(documents):
+            eid = entry_ids[i] if entry_ids and i < len(entry_ids) else f"doc_{start_idx + i}"
+            self._entry_ids.append(eid)
+            if metadata and eid in metadata:
+                self._metadata[eid] = metadata[eid]
+
+        self.bm25.index(self._documents)
+
+        if self.embedding_fn:
+            for doc in documents:
+                emb = self.embedding_fn(doc)
+                self._embeddings.append(emb)
+
+    def keyword_recall(self, query: str, top_k: int = 5) -> list[RecallResult]:
+        bm25_results = self.bm25.search(query, top_k * 2)
+        results = []
+        for doc_idx, bm25_score in bm25_results:
+            if doc_idx < len(self._entry_ids):
+                eid = self._entry_ids[doc_idx]
+                results.append(RecallResult(
+                    entry_id=eid,
+                    content=self._documents[doc_idx],
+                    score=bm25_score,
+                    source="bm25",
+                    metadata=self._metadata.get(eid, {}),
+                ))
+        return results[:top_k]
+
+    def semantic_recall(self, query: str, top_k: int = 5) -> list[RecallResult]:
+        if not self.embedding_fn:
+            return []
+
+        query_emb = self.embedding_fn(query)
+        similarities = []
+
+        for i, emb in enumerate(self._embeddings):
+            sim = self._cosine_similarity(query_emb, emb)
+            similarities.append((i, sim))
+
+        similarities.sort(key=lambda x: x[1], reverse=True)
+        results = []
+        for doc_idx, sim_score in similarities[:top_k]:
+            eid = self._entry_ids[doc_idx]
+            results.append(RecallResult(
+                entry_id=eid,
+                content=self._documents[doc_idx],
+                score=sim_score,
+                source="embedding",
+                metadata=self._metadata.get(eid, {}),
+            ))
+        return results
+
+    def _cosine_similarity(self, a: list[float], b: list[float]) -> float:
+        dot = sum(x * y for x, y in zip(a, b))
+        norm_a = math.sqrt(sum(x * x for x in a))
+        norm_b = math.sqrt(sum(x * x for x in b))
+        if norm_a == 0 or norm_b == 0:
+            return 0
+        return dot / (norm_a * norm_b)
+
+    def hybrid_recall(self, query: str, top_k: int = 5, rrf_k: int = 60) -> list[RecallResult]:
+        bm25_results = self.keyword_recall(query, top_k * 2)
+        semantic_results = self.semantic_recall(query, top_k * 2)
+
+        rrf_scores: dict[str, float] = {}
+        rrf_results: dict[str, RecallResult] = {}
+
+        for rank, result in enumerate(bm25_results):
+            score = 1.0 / (rrf_k + rank + 1)
+            rrf_scores[result.entry_id] = rrf_scores.get(result.entry_id, 0) + score
+            rrf_results[result.entry_id] = result
+
+        for rank, result in enumerate(semantic_results):
+            score = 1.0 / (rrf_k + rank + 1)
+            rrf_scores[result.entry_id] = rrf_scores.get(result.entry_id, 0) + score
+            rrf_results[result.entry_id] = result
+
+        sorted_ids = sorted(rrf_scores.items(), key=lambda x: x[1], reverse=True)
+        return [rrf_results[eid] for eid, _ in sorted_ids[:top_k]]
+
+    def clear(self) -> None:
+        self._documents = []
+        self._embeddings = []
+        self._entry_ids = []
+        self._metadata = {}
+        self.bm25 = BM25()

package/onecode/memory/symbolic.py

@@ -0,0 +1,164 @@
+from __future__ import annotations
+
+import re
+import uuid
+from dataclasses import dataclass, field
+from datetime import datetime, timezone
+from enum import Enum
+from typing import Optional
+
+
+class NodeType(Enum):
+    TASK = "task"
+    ACTION = "action"
+    RESULT = "result"
+    ERROR = "error"
+    CHECKPOINT = "checkpoint"
+
+
+@dataclass
+class MermaidNode:
+    node_id: str
+    node_type: NodeType
+    label: str
+    metadata: dict = field(default_factory=dict)
+
+    def to_mermaid(self) -> str:
+        type_styles = {
+            NodeType.TASK: "([,shape=ellipse",
+            NodeType.ACTION: "[,shape=box",
+            NodeType.RESULT: "[(,shape=cylinder",
+            NodeType.ERROR: "([,shape=ellipse,color=red",
+            NodeType.CHECKPOINT: "[,shape=diamond",
+        }
+        style = type_styles.get(self.node_type, "[")
+        return f"    {self.node_id}{style} {self.label}])"
+
+
+@dataclass
+class MermaidEdge:
+    from_node: str
+    to_node: str
+    label: Optional[str] = None
+    style: str = ""
+
+    def to_mermaid(self) -> str:
+        edge_str = f"    {self.from_node} --> {self.to_node}"
+        if self.label:
+            edge_str = f"    {self.from_node} -->|{self.label}| {self.to_node}"
+        if self.style:
+            edge_str += f" -. {self.style} .-> {self.to_node}"
+        return edge_str
+
+
+class MermaidCanvas:
+    def __init__(self, task_id: Optional[str] = None):
+        self.task_id = task_id or str(uuid.uuid4())[:8]
+        self.nodes: list[MermaidNode] = []
+        self.edges: list[MermaidEdge] = []
+        self.refs: dict[str, str] = {}
+
+    def add_node(
+        self,
+        node_type: NodeType,
+        label: str,
+        metadata: Optional[dict] = None,
+    ) -> str:
+        node_id = f"N{len(self.nodes):03d}"
+        node = MermaidNode(node_id, node_type, label, metadata or {})
+        self.nodes.append(node)
+        return node_id
+
+    def add_edge(
+        self,
+        from_node: str,
+        to_node: str,
+        label: Optional[str] = None,
+        style: str = "",
+    ) -> None:
+        edge = MermaidEdge(from_node, to_node, label, style)
+        self.edges.append(edge)
+
+    def add_ref(self, node_id: str, content: str) -> None:
+        self.refs[node_id] = content
+
+    def get_ref(self, node_id: str) -> Optional[str]:
+        return self.refs.get(node_id)
+
+    def to_mermaid(self) -> str:
+        if not self.nodes:
+            return "graph LR\n    empty((empty))"
+
+        lines = ["graph LR"]
+        for node in self.nodes:
+            lines.append(node.to_mermaid())
+        for edge in self.edges:
+            lines.append(edge.to_mermaid())
+        return "\n".join(lines)
+
+    def to_markdown(self) -> str:
+        lines = [f"## Task Canvas: {self.task_id}\n"]
+        lines.append(f"Generated: {datetime.now(timezone.utc).isoformat()}\n")
+        lines.append("### Mermaid Diagram\n```mermaid")
+        lines.append(self.to_mermaid())
+        lines.append("```\n")
+        if self.refs:
+            lines.append("### References\n")
+            for node_id, content in self.refs.items():
+                lines.append(f"**{node_id}**: {content[:200]}...\n")
+        return "\n".join(lines)
+
+    @classmethod
+    def from_mermaid(cls, mermaid_text: str) -> Optional["MermaidCanvas"]:
+        try:
+            canvas = cls()
+            node_pattern = re.compile(r"(\w+)\[(.*?)\]")
+            for match in node_pattern.finditer(mermaid_text):
+                node_id, label = match.groups()
+                if node_id.startswith("N"):
+                    canvas.add_node(NodeType.ACTION, label)
+            return canvas
+        except Exception:
+            return None
+
+
+class SymbolicMemory:
+    def __init__(self, canvas: Optional[MermaidCanvas] = None):
+        self.canvas = canvas or MermaidCanvas()
+        self._verbose_cache: dict[str, str] = {}
+
+    def offload_log(self, node_id: str, verbose_content: str) -> str:
+        self._verbose_cache[node_id] = verbose_content
+        self.canvas.add_ref(node_id, f"See ref: {node_id}")
+        return f"[{node_id}]"
+
+    def recall_log(self, node_id: str) -> Optional[str]:
+        return self._verbose_cache.get(node_id)
+
+    def create_task_node(self, label: str, metadata: Optional[dict] = None) -> str:
+        return self.canvas.add_node(NodeType.TASK, label, metadata)
+
+    def create_action_node(self, label: str, metadata: Optional[dict] = None) -> str:
+        return self.canvas.add_node(NodeType.ACTION, label, metadata)
+
+    def create_result_node(self, label: str, result_content: str, metadata: Optional[dict] = None) -> str:
+        node_id = self.canvas.add_node(NodeType.RESULT, label, metadata)
+        self.offload_log(node_id, result_content)
+        return node_id
+
+    def create_error_node(self, label: str, error_content: str, metadata: Optional[dict] = None) -> str:
+        node_id = self.canvas.add_node(NodeType.ERROR, label, metadata)
+        self.offload_log(node_id, error_content)
+        return node_id
+
+    def link(self, from_id: str, to_id: str, label: Optional[str] = None) -> None:
+        self.canvas.add_edge(from_id, to_id, label)
+
+    def get_canvas_markdown(self) -> str:
+        return self.canvas.to_markdown()
+
+    def get_canvas_mermaid(self) -> str:
+        return self.canvas.to_mermaid()
+
+    def get_ref_content(self, node_id: str) -> Optional[str]:
+        return self._verbose_cache.get(node_id)

package/onecode/models/__init__.py

@@ -0,0 +1,4 @@
+from onecode.models.registry import ModelRegistry, ModelInfo
+from onecode.models.provider import Provider, ProviderRegistry
+
+__all__ = ["ModelRegistry", "ModelInfo", "Provider", "ProviderRegistry"]