@provos/ironcurtain 0.11.0 → 0.13.0

package/dist/workflow/workflows/evolve/scripts/evolve_core/cognition.py

@@ -0,0 +1,127 @@
+"""Persistent cognition store with semantic retrieval."""
+
+from __future__ import annotations
+
+import json
+import uuid
+from pathlib import Path
+from threading import RLock
+from typing import Dict, List, Optional, Tuple
+
+from .embedding import EmbeddingService
+from .structures import CognitionItem
+from .vector_index import FAISSIndex
+
+
+class Cognition:
+    """Persistent cognition store with embedding-backed retrieval."""
+
+    def __init__(
+        self,
+        storage_dir: Path,
+        embedding_model: str = "sentence-transformers/all-MiniLM-L6-v2",
+        embedding_dim: int = 384,
+        retrieval_top_k: int = 5,
+        score_threshold: float = 0.3,
+        faiss_index_type: str = "IP",
+    ):
+        self.storage_dir = Path(storage_dir)
+        self.storage_dir.mkdir(parents=True, exist_ok=True)
+        self.lock = RLock()
+        self.retrieval_top_k = retrieval_top_k
+        self.score_threshold = score_threshold
+        self.items: Dict[str, CognitionItem] = {}
+        self.embedding = EmbeddingService(
+            model_name=embedding_model,
+            dimension=embedding_dim,
+        )
+        self.faiss = FAISSIndex(
+            dimension=embedding_dim,
+            index_type=faiss_index_type,
+            storage_path=self.storage_dir / "faiss",
+        )
+        self.str_to_int: Dict[str, int] = {}
+        self.int_to_str: Dict[int, str] = {}
+        self.next_int_id = 0
+        self._load()
+
+    def add(self, item: CognitionItem) -> str:
+        with self.lock:
+            if not item.id:
+                item.id = str(uuid.uuid4())
+            self.items[item.id] = item
+            if item.content:
+                int_id = self._get_int_id(item.id)
+                self.faiss.add(int_id, self.embedding.encode(item.content))
+            self._save()
+            return item.id
+
+    def add_batch(self, items: List[CognitionItem]) -> List[str]:
+        return [self.add(item) for item in items]
+
+    def search(self, query: str, top_k: Optional[int] = None) -> List[CognitionItem]:
+        return [item for item, _ in self.retrieve(query, top_k=top_k)]
+
+    def retrieve(
+        self,
+        query: str,
+        top_k: Optional[int] = None,
+        score_threshold: Optional[float] = None,
+    ) -> List[Tuple[CognitionItem, float]]:
+        top_k = top_k or self.retrieval_top_k
+        threshold = self.score_threshold if score_threshold is None else score_threshold
+        results = self.faiss.search(self.embedding.encode(query), top_k, threshold)
+
+        enriched: List[Tuple[CognitionItem, float]] = []
+        for int_id, score in results:
+            item_id = self.int_to_str.get(int_id)
+            if item_id and item_id in self.items:
+                enriched.append((self.items[item_id], score))
+        return enriched
+
+    def get_all(self) -> List[CognitionItem]:
+        return list(self.items.values())
+
+    def reset(self) -> None:
+        with self.lock:
+            self.items.clear()
+            self.str_to_int.clear()
+            self.int_to_str.clear()
+            self.next_int_id = 0
+            self.faiss.reset()
+            data_file = self.storage_dir / "cognition.json"
+            if data_file.exists():
+                data_file.unlink()
+
+    def _get_int_id(self, item_id: str) -> int:
+        if item_id not in self.str_to_int:
+            self.str_to_int[item_id] = self.next_int_id
+            self.int_to_str[self.next_int_id] = item_id
+            self.next_int_id += 1
+        return self.str_to_int[item_id]
+
+    def _save(self) -> None:
+        payload = {
+            "items": {item_id: item.to_dict() for item_id, item in self.items.items()},
+            "str_to_int": self.str_to_int,
+            "int_to_str": {str(key): value for key, value in self.int_to_str.items()},
+            "next_int_id": self.next_int_id,
+        }
+        with open(self.storage_dir / "cognition.json", "w", encoding="utf-8") as handle:
+            json.dump(payload, handle, ensure_ascii=False, indent=2)
+        self.faiss.save()
+
+    def _load(self) -> None:
+        data_file = self.storage_dir / "cognition.json"
+        if not data_file.exists():
+            return
+        with open(data_file, "r", encoding="utf-8") as handle:
+            payload = json.load(handle)
+        for item_id, raw_item in payload.get("items", {}).items():
+            self.items[item_id] = CognitionItem.from_dict(raw_item)
+        self.str_to_int = payload.get("str_to_int", {})
+        self.int_to_str = {int(key): value for key, value in payload.get("int_to_str", {}).items()}
+        self.next_int_id = payload.get("next_int_id", 0)
+
+    def __len__(self) -> int:
+        return len(self.items)

package/dist/workflow/workflows/evolve/scripts/evolve_core/database.py

@@ -0,0 +1,218 @@
+"""Persistent node database for skill-driven evolution runs."""
+
+from __future__ import annotations
+
+import copy
+import json
+import os
+import tempfile
+import time
+from contextlib import contextmanager
+from pathlib import Path
+from threading import RLock
+from typing import Any, Dict, List, Optional, Tuple
+
+from .algorithms import get_sampler
+from .embedding import EmbeddingService
+from .file_lock import InterProcessFileLock
+from .structures import Node
+from .vector_index import FAISSIndex
+
+
+class Database:
+    """Persistent experiment database with sampler pluggability."""
+
+    def __init__(
+        self,
+        storage_dir: Path,
+        embedding_model: str = "sentence-transformers/all-MiniLM-L6-v2",
+        embedding_dim: int = 384,
+        sampling_algorithm: str = "ucb1",
+        sampling_kwargs: Optional[Dict[str, Any]] = None,
+        faiss_index_type: str = "IP",
+        max_size: Optional[int] = None,
+    ):
+        self.storage_dir = Path(storage_dir)
+        self.storage_dir.mkdir(parents=True, exist_ok=True)
+        self.lock = RLock()
+        self.lock_path = self.storage_dir / ".database.lock"
+        self.nodes: Dict[int, Node] = {}
+        self.next_id = 0
+        self.max_size = max_size
+        self.embedding_dim = embedding_dim
+        self.faiss_index_type = faiss_index_type
+        self.embedding = EmbeddingService(
+            model_name=embedding_model,
+            dimension=embedding_dim,
+        )
+        self.sampling_algorithm = sampling_algorithm
+        self.sampling_kwargs = sampling_kwargs or {}
+        self.default_sampler = get_sampler(sampling_algorithm, **self.sampling_kwargs)
+        self.faiss = self._build_faiss_index()
+
+    def sample(self, n: int, algorithm: Optional[str] = None, **kwargs) -> List[Node]:
+        with self._database_guard():
+            nodes = list(self.nodes.values())
+            sampler = (
+                get_sampler(algorithm, **kwargs) if algorithm else self.default_sampler
+            )
+            selected = sampler.sample(nodes, n)
+            self._save_locked()
+            return self._clone_nodes(selected)
+
+    def add(self, node: Node) -> int:
+        node_id, _ = self.add_with_previous_nodes(node)
+        return node_id
+
+    def add_with_previous_nodes(self, node: Node) -> Tuple[int, List[Node]]:
+        with self._database_guard():
+            previous_nodes = self._clone_nodes(self.nodes.values())
+            if self.max_size is not None and len(self.nodes) >= self.max_size:
+                self._remove_worst_node_locked()
+
+            node.id = self.next_id
+            self.next_id += 1
+            self.nodes[node.id] = node
+            if hasattr(self.default_sampler, "on_node_added"):
+                self.default_sampler.on_node_added(node)
+
+            context_text = node.get_context_text()
+            if context_text:
+                vector = self.embedding.encode(context_text)
+                self.faiss.add(node.id, vector)
+
+            self._save_locked()
+            return node.id, previous_nodes
+
+    def get_all(self) -> List[Node]:
+        with self._database_guard():
+            return self._clone_nodes(self.nodes.values())
+
+    def get(self, node_id: int) -> Optional[Node]:
+        with self._database_guard():
+            node = self.nodes.get(node_id)
+            if node is None:
+                return None
+            return self._clone_node(node)
+
+    def remove(self, node_id: int) -> bool:
+        with self._database_guard():
+            if node_id not in self.nodes:
+                return False
+            node = self.nodes[node_id]
+            if hasattr(self.default_sampler, "on_node_removed"):
+                self.default_sampler.on_node_removed(node)
+            del self.nodes[node_id]
+            self.faiss.remove(node_id)
+            self._save_locked()
+            return True
+
+    def reset(self) -> None:
+        with self._database_guard(refresh=False):
+            if hasattr(self.default_sampler, "reset"):
+                self.default_sampler.reset()
+            self.nodes.clear()
+            self.next_id = 0
+            self.faiss.reset()
+            data_file = self.storage_dir / "nodes.json"
+            if data_file.exists():
+                data_file.unlink()
+
+    def get_sampler_stats(self) -> Optional[Dict[str, Any]]:
+        _, sampler_stats = self.snapshot()
+        return sampler_stats
+
+    def snapshot(self) -> Tuple[List[Node], Optional[Dict[str, Any]]]:
+        with self._database_guard():
+            return self._clone_nodes(self.nodes.values()), self._get_sampler_stats_locked()
+
+    def _remove_worst_node_locked(self) -> None:
+        if not self.nodes:
+            return
+        worst_id = min(self.nodes, key=lambda node_id: (self.nodes[node_id].score, node_id))
+        node = self.nodes[worst_id]
+        if hasattr(self.default_sampler, "on_node_removed"):
+            self.default_sampler.on_node_removed(node)
+        del self.nodes[worst_id]
+        self.faiss.remove(worst_id)
+
+    def _save_locked(self) -> None:
+        payload = {
+            "next_id": self.next_id,
+            "nodes": {str(node_id): node.to_dict() for node_id, node in self.nodes.items()},
+        }
+        if hasattr(self.default_sampler, "get_state"):
+            payload["sampler_state"] = self.default_sampler.get_state()
+
+        temp_path: Optional[Path] = None
+        try:
+            with tempfile.NamedTemporaryFile(
+                mode="w",
+                encoding="utf-8",
+                dir=self.storage_dir,
+                prefix="nodes.",
+                suffix=".tmp",
+                delete=False,
+            ) as handle:
+                json.dump(payload, handle, ensure_ascii=False, indent=2)
+                handle.flush()
+                os.fsync(handle.fileno())
+                temp_path = Path(handle.name)
+
+            os.replace(temp_path, self.storage_dir / "nodes.json")
+        finally:
+            if temp_path is not None and temp_path.exists():
+                temp_path.unlink(missing_ok=True)
+
+        self.faiss.save()
+
+    def _load_locked(self) -> None:
+        data_file = self.storage_dir / "nodes.json"
+        self.nodes = {}
+        self.next_id = 0
+        self.default_sampler = get_sampler(self.sampling_algorithm, **self.sampling_kwargs)
+        self.faiss = self._build_faiss_index()
+        if not data_file.exists():
+            return
+        with open(data_file, "r", encoding="utf-8") as handle:
+            payload = json.load(handle)
+        self.next_id = payload.get("next_id", 0)
+        for node_id, node_data in payload.get("nodes", {}).items():
+            self.nodes[int(node_id)] = Node.from_dict(node_data)
+        if hasattr(self.default_sampler, "load_state") and "sampler_state" in payload:
+            self.default_sampler.load_state(payload["sampler_state"])
+        if hasattr(self.default_sampler, "rebuild_from_nodes"):
+            self.default_sampler.rebuild_from_nodes(list(self.nodes.values()))
+
+    def _build_faiss_index(self) -> FAISSIndex:
+        return FAISSIndex(
+            dimension=self.embedding_dim,
+            index_type=self.faiss_index_type,
+            storage_path=self.storage_dir / "faiss",
+        )
+
+    def _get_sampler_stats_locked(self) -> Optional[Dict[str, Any]]:
+        if hasattr(self.default_sampler, "get_island_stats"):
+            return self.default_sampler.get_island_stats(list(self.nodes.values()))
+        return None
+
+    @staticmethod
+    def _clone_node(node: Node) -> Node:
+        return Node.from_dict(copy.deepcopy(node.to_dict()))
+
+    def _clone_nodes(self, nodes: List[Node] | Any) -> List[Node]:
+        return [self._clone_node(node) for node in nodes]
+
+    @contextmanager
+    def _database_guard(self, refresh: bool = True):
+        with self.lock:
+            with InterProcessFileLock(self.lock_path):
+                if refresh:
+                    self._load_locked()
+                hold_ms = int(os.environ.get("EVOLVE_DB_TEST_HOLD_LOCK_MS", "0") or 0)
+                if hold_ms > 0:
+                    time.sleep(hold_ms / 1000.0)
+                yield
+
+    def __len__(self) -> int:
+        return len(self.get_all())

package/dist/workflow/workflows/evolve/scripts/evolve_core/diff.py

@@ -0,0 +1,57 @@
+"""Helpers for diff-style code edits."""
+
+from __future__ import annotations
+
+import re
+from difflib import unified_diff
+from typing import List, Optional, Tuple
+
+
+def extract_diffs(
+    llm_response: str,
+    pattern: str = r"<<<<<<< SEARCH\n(.*?)=======\n(.*?)>>>>>>> REPLACE",
+) -> List[Tuple[str, str]]:
+    matches = re.findall(pattern, llm_response, re.DOTALL)
+    return [(search.strip("\n"), replace.strip("\n")) for search, replace in matches]
+
+
+def apply_diff(
+    original_code: str,
+    llm_response: str,
+    pattern: str = r"<<<<<<< SEARCH\n(.*?)=======\n(.*?)>>>>>>> REPLACE",
+) -> str:
+    diff_blocks = extract_diffs(llm_response, pattern)
+    if not diff_blocks:
+        raise ValueError("No valid diff blocks found")
+
+    updated_code = original_code
+    for search, replace in diff_blocks:
+        if search not in updated_code:
+            raise ValueError("SEARCH block not found in target text")
+        updated_code = updated_code.replace(search, replace, 1)
+    return updated_code
+
+
+def parse_full_rewrite(llm_response: str, language: str = "python") -> Optional[str]:
+    pattern = rf"```{language}\n(.*?)```"
+    match = re.search(pattern, llm_response, re.DOTALL)
+    if match:
+        return match.group(1).strip()
+    stripped = llm_response.strip()
+    return stripped or None
+
+
+def format_diff_summary(diff_blocks: List[Tuple[str, str]]) -> str:
+    parts: List[str] = []
+    for search, replace in diff_blocks:
+        lines = list(
+            unified_diff(
+                search.splitlines(),
+                replace.splitlines(),
+                fromfile="before",
+                tofile="after",
+                lineterm="",
+            )
+        )
+        parts.append("\n".join(lines))
+    return "\n\n".join(parts)

package/dist/workflow/workflows/evolve/scripts/evolve_core/embedding.py

@@ -0,0 +1,84 @@
+"""Embedding helpers with a graceful local fallback."""
+
+from __future__ import annotations
+
+import math
+import re
+from typing import List, Union
+
+import numpy as np
+
+try:
+    from sentence_transformers import SentenceTransformer
+
+    ST_AVAILABLE = True
+except Exception:
+    SentenceTransformer = None
+    ST_AVAILABLE = False
+
+
+class EmbeddingService:
+    """Encode text with sentence-transformers when available, else hash tokens."""
+
+    def __init__(
+        self,
+        model_name: str = "sentence-transformers/all-MiniLM-L6-v2",
+        device: str = "cpu",
+        dimension: int = 384,
+    ):
+        self.dimension = dimension
+        self.model = None
+
+        if ST_AVAILABLE and SentenceTransformer is not None:
+            try:
+                self.model = SentenceTransformer(model_name, device=device)
+                self.dimension = self.model.get_sentence_embedding_dimension()
+            except Exception:
+                self.model = None
+
+    def encode(
+        self,
+        texts: Union[str, List[str]],
+        normalize: bool = True,
+    ) -> np.ndarray:
+        if isinstance(texts, str):
+            texts = [texts]
+
+        if self.model is not None:
+            embeddings = self.model.encode(
+                texts,
+                normalize_embeddings=normalize,
+                show_progress_bar=False,
+            )
+            return np.array(embeddings, dtype=np.float32)
+
+        vectors = np.vstack([self._fallback_encode_one(text) for text in texts]).astype(
+            np.float32
+        )
+        if normalize:
+            vectors = self._normalize(vectors)
+        return vectors
+
+    def get_dimension(self) -> int:
+        return self.dimension
+
+    def _fallback_encode_one(self, text: str) -> np.ndarray:
+        vector = np.zeros(self.dimension, dtype=np.float32)
+        tokens = re.findall(r"[A-Za-z0-9_]+", text.lower())
+        if not tokens:
+            return vector
+
+        for token in tokens:
+            hashed = hash(token)
+            index = abs(hashed) % self.dimension
+            sign = -1.0 if hashed % 2 else 1.0
+            weight = 1.0 + math.log1p(len(token))
+            vector[index] += sign * weight
+
+        return vector
+
+    @staticmethod
+    def _normalize(vectors: np.ndarray) -> np.ndarray:
+        norms = np.linalg.norm(vectors, axis=1, keepdims=True)
+        norms[norms == 0] = 1.0
+        return vectors / norms

package/dist/workflow/workflows/evolve/scripts/evolve_core/file_lock.py

@@ -0,0 +1,45 @@
+"""Cross-process file locking helpers for the Evolve skill runtime."""
+
+from __future__ import annotations
+
+import os
+from pathlib import Path
+
+if os.name == "nt":
+    import msvcrt
+else:
+    import fcntl
+
+
+class InterProcessFileLock:
+    """Hold an exclusive lock on a sentinel file across processes."""
+
+    def __init__(self, path: Path):
+        self.path = Path(path)
+        self._handle = None
+
+    def __enter__(self) -> "InterProcessFileLock":
+        self.path.parent.mkdir(parents=True, exist_ok=True)
+        self._handle = open(self.path, "a+b")
+        self._handle.seek(0)
+        self._handle.write(b"\0")
+        self._handle.flush()
+        self._handle.seek(0)
+
+        if os.name == "nt":
+            msvcrt.locking(self._handle.fileno(), msvcrt.LK_LOCK, 1)
+        else:
+            fcntl.flock(self._handle.fileno(), fcntl.LOCK_EX)
+        return self
+
+    def __exit__(self, exc_type, exc, tb) -> None:
+        if self._handle is None:
+            return
+
+        self._handle.seek(0)
+        if os.name == "nt":
+            msvcrt.locking(self._handle.fileno(), msvcrt.LK_UNLCK, 1)
+        else:
+            fcntl.flock(self._handle.fileno(), fcntl.LOCK_UN)
+        self._handle.close()
+        self._handle = None