@smilintux/skmemory 0.5.0 → 0.9.2

package/skmemory/vault.py

@@ -0,0 +1,225 @@
+"""
+Memory Vault — at-rest encryption for sovereign memories.
+
+Encrypts memory JSON files using AES-256-GCM with keys derived from
+the agent's passphrase or CapAuth PGP key. Each memory file gets
+its own random nonce, so identical content produces different ciphertext.
+
+Quantum-resistant note: AES-256 requires Grover's algorithm to attack,
+which only reduces effective security from 256 to 128 bits. That's
+still computationally infeasible for the foreseeable future.
+
+Usage:
+    vault = MemoryVault(passphrase="EXAMPLE-DO-NOT-USE")
+    encrypted = vault.encrypt(memory_json_bytes)
+    decrypted = vault.decrypt(encrypted)
+"""
+
+from __future__ import annotations
+
+import hashlib
+import logging
+import os
+from pathlib import Path
+
+logger = logging.getLogger("skmemory.vault")
+
+VAULT_HEADER = b"SKMV1"
+NONCE_SIZE = 12
+TAG_SIZE = 16
+KEY_SIZE = 32
+SALT_SIZE = 16
+KDF_ITERATIONS = 600_000
+
+
+def _derive_key(passphrase: str, salt: bytes) -> bytes:
+    """Derive a 256-bit AES key from a passphrase using PBKDF2.
+
+    Args:
+        passphrase: The encryption passphrase.
+        salt: 16-byte random salt.
+
+    Returns:
+        32-byte AES key.
+    """
+    return hashlib.pbkdf2_hmac(
+        "sha256",
+        passphrase.encode("utf-8"),
+        salt,
+        KDF_ITERATIONS,
+        dklen=KEY_SIZE,
+    )
+
+
+class MemoryVault:
+    """Encrypt and decrypt memory files at rest.
+
+    Uses AES-256-GCM for authenticated encryption. Each encrypt()
+    call generates a fresh random nonce and salt, so the same
+    plaintext never produces the same ciphertext.
+
+    File format:
+        SKMV1 || salt(16) || nonce(12) || ciphertext || tag(16)
+
+    Args:
+        passphrase: Secret used to derive the encryption key.
+            Can be the agent's CapAuth passphrase or a dedicated vault key.
+    """
+
+    def __init__(self, passphrase: str) -> None:
+        self._passphrase = passphrase
+
+    def encrypt(self, plaintext: bytes) -> bytes:
+        """Encrypt data with AES-256-GCM.
+
+        Args:
+            plaintext: Raw bytes to encrypt (typically memory JSON).
+
+        Returns:
+            Encrypted bytes with header, salt, nonce, ciphertext, and tag.
+
+        Raises:
+            ImportError: If cryptography package is not installed.
+        """
+        from cryptography.hazmat.primitives.ciphers.aead import AESGCM
+
+        salt = os.urandom(SALT_SIZE)
+        nonce = os.urandom(NONCE_SIZE)
+        key = _derive_key(self._passphrase, salt)
+
+        aesgcm = AESGCM(key)
+        ciphertext = aesgcm.encrypt(nonce, plaintext, VAULT_HEADER)
+
+        return VAULT_HEADER + salt + nonce + ciphertext
+
+    def decrypt(self, encrypted: bytes) -> bytes:
+        """Decrypt data encrypted with encrypt().
+
+        Args:
+            encrypted: Bytes from encrypt() — header + salt + nonce + ciphertext + tag.
+
+        Returns:
+            Decrypted plaintext bytes.
+
+        Raises:
+            ValueError: If the header doesn't match or decryption fails.
+            ImportError: If cryptography package is not installed.
+        """
+        from cryptography.hazmat.primitives.ciphers.aead import AESGCM
+
+        header_len = len(VAULT_HEADER)
+        if encrypted[:header_len] != VAULT_HEADER:
+            raise ValueError("Not an SKMemory vault file (bad header)")
+
+        offset = header_len
+        salt = encrypted[offset : offset + SALT_SIZE]
+        offset += SALT_SIZE
+        nonce = encrypted[offset : offset + NONCE_SIZE]
+        offset += NONCE_SIZE
+        ciphertext = encrypted[offset:]
+
+        key = _derive_key(self._passphrase, salt)
+        aesgcm = AESGCM(key)
+
+        return aesgcm.decrypt(nonce, ciphertext, VAULT_HEADER)
+
+    def encrypt_file(self, path: Path) -> Path:
+        """Encrypt a file in-place, adding .vault extension.
+
+        Args:
+            path: Path to the plaintext file.
+
+        Returns:
+            Path to the encrypted file.
+        """
+        plaintext = path.read_bytes()
+        encrypted = self.encrypt(plaintext)
+        vault_path = path.with_suffix(path.suffix + ".vault")
+        vault_path.write_bytes(encrypted)
+        path.unlink()
+        return vault_path
+
+    def decrypt_file(self, path: Path) -> Path:
+        """Decrypt a .vault file, restoring the original.
+
+        Args:
+            path: Path to the encrypted .vault file.
+
+        Returns:
+            Path to the decrypted file.
+        """
+        encrypted = path.read_bytes()
+        plaintext = self.decrypt(encrypted)
+        original_path = Path(str(path).replace(".vault", ""))
+        original_path.write_bytes(plaintext)
+        path.unlink()
+        return original_path
+
+    def is_encrypted(self, path: Path) -> bool:
+        """Check if a file is vault-encrypted.
+
+        Args:
+            path: Path to check.
+
+        Returns:
+            True if the file has a valid vault header.
+        """
+        try:
+            data = path.read_bytes()
+            return data[: len(VAULT_HEADER)] == VAULT_HEADER
+        except OSError:
+            return False
+
+
+def encrypt_memory_store(
+    memory_dir: Path,
+    passphrase: str,
+) -> int:
+    """Encrypt all memory JSON files in a directory tree.
+
+    Args:
+        memory_dir: Root memory directory (e.g., ~/.skcapstone/memories/).
+        passphrase: Encryption passphrase.
+
+    Returns:
+        Number of files encrypted.
+    """
+    vault = MemoryVault(passphrase)
+    count = 0
+
+    for json_file in memory_dir.rglob("*.json"):
+        if json_file.suffix == ".vault":
+            continue
+        try:
+            vault.encrypt_file(json_file)
+            count += 1
+        except Exception as exc:
+            logger.warning("Failed to encrypt %s: %s", json_file, exc)
+
+    return count
+
+
+def decrypt_memory_store(
+    memory_dir: Path,
+    passphrase: str,
+) -> int:
+    """Decrypt all vault files in a directory tree.
+
+    Args:
+        memory_dir: Root memory directory.
+        passphrase: Decryption passphrase.
+
+    Returns:
+        Number of files decrypted.
+    """
+    vault = MemoryVault(passphrase)
+    count = 0
+
+    for vault_file in memory_dir.rglob("*.vault"):
+        try:
+            vault.decrypt_file(vault_file)
+            count += 1
+        except Exception as exc:
+            logger.warning("Failed to decrypt %s: %s", vault_file, exc)
+
+    return count

package/tests/conftest.py

@@ -0,0 +1,46 @@
+"""
+Root conftest.py — bootstraps a minimal test agent before any skmemory module
+is imported by pytest.
+
+Several skmemory modules (seeds.py, febs.py, soul.py, etc.) call
+``get_agent_paths()`` at **module level**, which raises ``ValueError`` if no
+agent directory exists under ``~/.skcapstone/agents/``.  This file runs before
+pytest collects any test files, so setting ``SKCAPSTONE_HOME`` and
+``SKCAPSTONE_AGENT`` here — and creating the matching directory tree — ensures
+those imports always succeed in CI and other fresh environments.
+"""
+
+from __future__ import annotations
+
+import os
+import tempfile
+from pathlib import Path
+
+# ── Bootstrap a throw-away test agent ─────────────────────────────────────────
+# Must happen at module level (not inside a fixture) so it takes effect before
+# pytest imports test files that themselves import skmemory.
+
+_TEST_AGENT_NAME = "test-agent"
+
+# Use a temp dir so we never pollute the real ~/.skcapstone tree.
+_tmp_skcapstone = tempfile.mkdtemp(prefix="skmemory_ci_")
+os.environ["SKCAPSTONE_HOME"] = _tmp_skcapstone
+os.environ["SKCAPSTONE_AGENT"] = _TEST_AGENT_NAME
+
+_agent_base = Path(_tmp_skcapstone) / "agents" / _TEST_AGENT_NAME
+for _subdir in (
+    "config",
+    "seeds",
+    "memory/short-term",
+    "memory/mid-term",
+    "memory/long-term",
+    "logs",
+    "archive",
+):
+    (_agent_base / _subdir).mkdir(parents=True, exist_ok=True)
+
+# Minimal config so list_agents() finds this agent and load_config() doesn't
+# blow up.
+(_agent_base / "config" / "skmemory.yaml").write_text(
+    "# Auto-generated by tests/conftest.py\nname: test-agent\n"
+)

package/tests/integration/conftest.py

@@ -0,0 +1,233 @@
+"""
+Integration test fixtures for SKMemory live backends.
+
+Fixtures skip automatically when SKGraph or SKVector are unreachable.
+Set env vars to point at non-default endpoints:
+
+    SKMEMORY_SKGRAPH_URL=redis://localhost:6379
+    SKMEMORY_SKVECTOR_URL=http://localhost:6333
+
+A dedicated test graph and collection are used so production data is
+never touched. Both are torn down after the test session.
+"""
+
+from __future__ import annotations
+
+import contextlib
+import os
+import uuid
+
+import pytest
+
+# ─────────────────────────────────────────────────────────
+# Connection constants
+# ─────────────────────────────────────────────────────────
+
+SKGRAPH_URL = os.environ.get("SKMEMORY_SKGRAPH_URL", "redis://localhost:6379")
+SKVECTOR_URL = os.environ.get("SKMEMORY_SKVECTOR_URL", "http://localhost:6333")
+SKVECTOR_KEY = os.environ.get("SKMEMORY_SKVECTOR_KEY")
+
+# Isolated names so tests never collide with production data
+TEST_GRAPH_NAME = "skmemory_integration_test"
+TEST_COLLECTION_NAME = "skmemory_integration_test"
+
+
+# ─────────────────────────────────────────────────────────
+# Availability checks
+# ─────────────────────────────────────────────────────────
+
+
+def _skgraph_available() -> bool:
+    """Return True if a SKGraph (FalkorDB/Redis) server is reachable."""
+    try:
+        from falkordb import FalkorDB  # type: ignore[import]
+    except ImportError:
+        return False
+
+    try:
+        db = FalkorDB.from_url(SKGRAPH_URL)
+        g = db.select_graph("__ping__")
+        g.query("RETURN 1")
+        return True
+    except Exception:
+        return False
+
+
+def _skvector_available() -> bool:
+    """Return True if SKVector (Qdrant) is reachable and qdrant-client is installed."""
+    try:
+        from qdrant_client import QdrantClient  # type: ignore[import]
+    except ImportError:
+        return False
+
+    try:
+        client = QdrantClient(url=SKVECTOR_URL, api_key=SKVECTOR_KEY)
+        client.get_collections()
+        return True
+    except Exception:
+        return False
+
+
+def _sentence_transformers_available() -> bool:
+    try:
+        import sentence_transformers  # noqa: F401  # type: ignore[import]
+
+        return True
+    except ImportError:
+        return False
+
+
+SKGRAPH_AVAILABLE = _skgraph_available()
+SKVECTOR_AVAILABLE = _skvector_available()
+SENTENCE_TRANSFORMERS_AVAILABLE = _sentence_transformers_available()
+
+# Composite flag: SKVector tests also require the embedding model
+SKVECTOR_FULL_AVAILABLE = SKVECTOR_AVAILABLE and SENTENCE_TRANSFORMERS_AVAILABLE
+
+requires_skgraph = pytest.mark.skipif(
+    not SKGRAPH_AVAILABLE,
+    reason="SKGraph unreachable (set SKMEMORY_SKGRAPH_URL or start Redis+FalkorDB)",
+)
+
+requires_skvector = pytest.mark.skipif(
+    not SKVECTOR_FULL_AVAILABLE,
+    reason=(
+        "SKVector unreachable or sentence-transformers not installed "
+        "(set SKMEMORY_SKVECTOR_URL or pip install qdrant-client sentence-transformers)"
+    ),
+)
+
+requires_both = pytest.mark.skipif(
+    not (SKGRAPH_AVAILABLE and SKVECTOR_FULL_AVAILABLE),
+    reason="Both SKGraph and SKVector must be reachable for cross-backend tests",
+)
+
+
+# ─────────────────────────────────────────────────────────
+# SKGraph fixtures
+# ─────────────────────────────────────────────────────────
+
+
+@pytest.fixture(scope="session")
+def falkordb_backend():
+    """Live SKGraphBackend pointed at the test graph.
+
+    The test graph is deleted at session teardown.
+    """
+    pytest.importorskip("falkordb", reason="falkordb not installed")
+    if not SKGRAPH_AVAILABLE:
+        pytest.skip("SKGraph unreachable")
+
+    from skmemory.backends.skgraph_backend import SKGraphBackend
+
+    backend = SKGraphBackend(url=SKGRAPH_URL, graph_name=TEST_GRAPH_NAME)
+    assert backend._ensure_initialized(), "SKGraph backend failed to initialize"
+
+    yield backend
+
+    # Teardown: drop the test graph
+    try:
+        from falkordb import FalkorDB  # type: ignore[import]
+
+        db = FalkorDB.from_url(SKGRAPH_URL)
+        db.select_graph(TEST_GRAPH_NAME).delete()
+    except Exception:
+        pass
+
+
+@pytest.fixture
+def falkordb_clean(falkordb_backend):
+    """SKGraphBackend with test graph wiped before each test."""
+    # Clear all nodes so tests are independent
+    with contextlib.suppress(Exception):
+        falkordb_backend._graph.query("MATCH (n) DETACH DELETE n")
+    return falkordb_backend
+
+
+# ─────────────────────────────────────────────────────────
+# SKVector fixtures
+# ─────────────────────────────────────────────────────────
+
+
+@pytest.fixture(scope="session")
+def qdrant_backend():
+    """Live SKVectorBackend pointed at the test collection.
+
+    The test collection is deleted at session teardown.
+    """
+    pytest.importorskip("qdrant_client", reason="qdrant-client not installed")
+    pytest.importorskip("sentence_transformers", reason="sentence-transformers not installed")
+    if not SKVECTOR_AVAILABLE:
+        pytest.skip("SKVector unreachable")
+
+    from skmemory.backends.skvector_backend import SKVectorBackend
+
+    backend = SKVectorBackend(
+        url=SKVECTOR_URL,
+        api_key=SKVECTOR_KEY,
+        collection=TEST_COLLECTION_NAME,
+    )
+    assert backend._ensure_initialized(), "SKVector backend failed to initialize"
+
+    yield backend
+
+    # Teardown: delete test collection
+    with contextlib.suppress(Exception):
+        backend._client.delete_collection(TEST_COLLECTION_NAME)
+
+
+@pytest.fixture
+def qdrant_clean(qdrant_backend):
+    """SKVectorBackend with collection wiped before each test."""
+    try:
+        from qdrant_client.models import Distance, VectorParams
+
+        from skmemory.backends.skvector_backend import VECTOR_DIM
+
+        qdrant_backend._client.delete_collection(TEST_COLLECTION_NAME)
+        qdrant_backend._client.create_collection(
+            collection_name=TEST_COLLECTION_NAME,
+            vectors_config=VectorParams(size=VECTOR_DIM, distance=Distance.COSINE),
+        )
+    except Exception:
+        pass
+    return qdrant_backend
+
+
+# ─────────────────────────────────────────────────────────
+# Shared memory factory
+# ─────────────────────────────────────────────────────────
+
+
+def make_memory(
+    title: str = "Test Memory",
+    content: str = "Integration test content.",
+    tags: list[str] | None = None,
+    source: str = "integration-test",
+    layer: str = "short-term",
+    intensity: float = 5.0,
+    valence: float = 0.5,
+    emotional_labels: list[str] | None = None,
+    parent_id: str | None = None,
+    related_ids: list[str] | None = None,
+):
+    """Factory for Memory objects in integration tests."""
+    from skmemory.models import EmotionalSnapshot, Memory, MemoryLayer
+
+    layer_enum = MemoryLayer(layer)
+    emotional = EmotionalSnapshot(
+        intensity=intensity,
+        valence=valence,
+        labels=emotional_labels or [],
+    )
+    return Memory(
+        id=str(uuid.uuid4()),
+        title=title,
+        content=content,
+        tags=tags or [],
+        source=source,
+        layer=layer_enum,
+        emotional=emotional,
+        parent_id=parent_id,
+        related_ids=related_ids or [],
+    )