enya-agent 0.1.0

package/crates/enact-core/src/kernel/persistence/mod.rs

@@ -0,0 +1,102 @@
+//! Persistence Layer - Storage Abstractions
+//!
+//! This module defines the core storage traits for the Enact execution engine.
+//! The kernel interacts only with these traits; concrete implementations live
+//! in the `enact-persistence` crate.
+//!
+//! ## Store Roles
+//!
+//! | Store | Role | Mutable? | Authoritative? |
+//! |-------|------|----------|----------------|
+//! | EventStore | Source of truth | Append-only | Yes |
+//! | StateStore | Cache/snapshot | Yes | No |
+//! | VectorStore | Semantic recall | Yes | No |
+//!
+//! ## Key Invariants
+//!
+//! 1. Kernel NEVER imports concrete implementations
+//! 2. All stores implement `StorageBackend` for mode validation
+//! 3. EventStore is append-only and authoritative
+//! 4. StateStore/VectorStore are non-authoritative caches
+//!
+//! @see docs/TECHNICAL/14-PERSISTENCE-LAYER.md
+
+mod event_store;
+mod message_store;
+mod state_store;
+mod vector_store;
+
+pub use event_store::{EventStore, ExecutionEventData, StoredEvent};
+pub use message_store::{
+    CostInfo, ExecutionStats, FinishReason, InMemoryMessageStore, Message, MessageMetadata,
+    MessagePart, MessageRole, MessageStore, Thread, TokenUsage,
+};
+pub use state_store::{ExecutionSnapshot, StateStore, StateStoreJsonExt};
+pub use vector_store::{
+    CollectionInfo, DistanceMetric, VectorDocument, VectorFilter, VectorSearchResult, VectorStore,
+};
+
+use async_trait::async_trait;
+
+/// Base trait for all storage backends
+///
+/// This trait provides metadata about the backend's capabilities,
+/// enabling runtime mode validation (e.g., air-gapped mode rejection
+/// of network-requiring backends).
+#[async_trait]
+pub trait StorageBackend: Send + Sync {
+    /// Human-readable name of this backend (e.g., "sqlite", "postgres", "qdrant")
+    fn name(&self) -> &str;
+
+    /// Does this backend require network access?
+    ///
+    /// Returns `true` for backends like Postgres, Redis, remote Qdrant.
+    /// Returns `false` for SQLite, filesystem, embedded Qdrant.
+    fn requires_network(&self) -> bool;
+
+    /// Can this backend operate in air-gapped mode?
+    ///
+    /// Default implementation returns `!requires_network()`.
+    fn supports_air_gapped(&self) -> bool {
+        !self.requires_network()
+    }
+
+    /// Health check - verify the backend is operational
+    ///
+    /// Should return `Ok(())` if the backend is ready to accept operations.
+    /// Should return an error if the backend is unavailable or misconfigured.
+    async fn health_check(&self) -> anyhow::Result<()>;
+
+    /// Graceful shutdown
+    ///
+    /// Called when the runtime is shutting down. Implementations should
+    /// flush any pending writes and close connections.
+    async fn shutdown(&self) -> anyhow::Result<()> {
+        Ok(())
+    }
+}
+
+/// Store guarantees - documented contracts for each store type
+pub mod guarantees {
+    //! # Store Guarantees
+    //!
+    //! These guarantees are contractual and must be tested.
+    //!
+    //! ## EventStore
+    //! - **Durability**: Event is durable before `append()` returns
+    //! - **Ordering**: Events are strictly ordered per execution
+    //! - **Immutability**: Events are append-only, never modified
+    //! - **Authority**: EventStore is the source of truth
+    //!
+    //! ## StateStore
+    //! - **Durability**: Best-effort - may be lost on failure
+    //! - **Freshness**: May be stale - always verify with EventStore
+    //! - **Authority**: Non-authoritative - cache only
+    //! - **Availability**: Designed for fast reads, not durability
+    //!
+    //! ## VectorStore
+    //! - **Consistency**: Eventually consistent
+    //! - **Authority**: Non-authoritative - for recall only
+    //! - **Availability**: Search may return stale results
+    //! - **Durability**: Documents are durable after `upsert()` returns
+}

package/crates/enact-core/src/kernel/persistence/state_store.rs

@@ -0,0 +1,228 @@
+//! StateStore - Hot snapshots and working memory
+//!
+//! The StateStore provides fast access to execution state for resumption.
+//! It is a **cache**, not a source of truth - the EventStore is authoritative.
+//!
+//! ## Guarantees
+//!
+//! - **Durability**: Best-effort - may be lost on failure
+//! - **Freshness**: May be stale - always verify sequence with EventStore
+//! - **Authority**: Non-authoritative - cache only
+//! - **Availability**: Designed for fast reads, not durability
+//!
+//! ## Use Cases
+//!
+//! - Fast execution resumption (avoid replaying all events)
+//! - Working memory during execution
+//! - Temporary state that doesn't need event sourcing
+//!
+//! @see docs/TECHNICAL/14-PERSISTENCE-LAYER.md
+
+use async_trait::async_trait;
+use chrono::{DateTime, Utc};
+use serde::{Deserialize, Serialize};
+use std::collections::HashMap;
+use std::time::Duration;
+
+/// Extension trait for JSON operations on StateStore
+///
+/// This is separate from StateStore to maintain dyn-compatibility.
+#[async_trait]
+pub trait StateStoreJsonExt: StateStore {
+    /// Set a JSON value
+    async fn set_json<T: Serialize + Send + Sync>(
+        &self,
+        key: &str,
+        value: &T,
+        ttl: Option<Duration>,
+    ) -> anyhow::Result<()> {
+        let bytes = serde_json::to_vec(value)?;
+        self.set(key, &bytes, ttl).await
+    }
+
+    /// Get a JSON value
+    async fn get_json<T: for<'de> Deserialize<'de>>(&self, key: &str) -> anyhow::Result<Option<T>> {
+        match self.get(key).await? {
+            Some(bytes) => Ok(Some(serde_json::from_slice(&bytes)?)),
+            None => Ok(None),
+        }
+    }
+}
+
+// Blanket implementation for all StateStore implementors
+impl<S: StateStore + ?Sized> StateStoreJsonExt for S {}
+
+use crate::kernel::{ExecutionId, ExecutionState, StepId, TenantId};
+
+use super::StorageBackend;
+
+/// Execution state snapshot
+///
+/// This captures the current state of an execution for fast resumption.
+/// The `sequence_number` should match the EventStore sequence to verify freshness.
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct ExecutionSnapshot {
+    /// The execution this snapshot is for
+    pub execution_id: ExecutionId,
+    /// Tenant for multi-tenancy isolation
+    pub tenant_id: TenantId,
+    /// Current execution state
+    pub state: ExecutionState,
+    /// Currently executing step (if any)
+    pub current_step_id: Option<StepId>,
+    /// Outputs from completed steps
+    pub step_outputs: HashMap<StepId, serde_json::Value>,
+    /// Working variables
+    pub variables: HashMap<String, serde_json::Value>,
+    /// When this snapshot was created
+    pub timestamp: DateTime<Utc>,
+    /// EventStore sequence number at snapshot time
+    ///
+    /// Used to verify freshness - if EventStore has more events,
+    /// the snapshot may be stale.
+    pub sequence_number: u64,
+}
+
+impl ExecutionSnapshot {
+    /// Create a new snapshot
+    pub fn new(
+        execution_id: ExecutionId,
+        tenant_id: TenantId,
+        state: ExecutionState,
+        sequence_number: u64,
+    ) -> Self {
+        Self {
+            execution_id,
+            tenant_id,
+            state,
+            current_step_id: None,
+            step_outputs: HashMap::new(),
+            variables: HashMap::new(),
+            timestamp: Utc::now(),
+            sequence_number,
+        }
+    }
+
+    /// Check if this snapshot is fresh compared to an EventStore sequence
+    pub fn is_fresh(&self, event_store_sequence: u64) -> bool {
+        self.sequence_number >= event_store_sequence
+    }
+}
+
+/// StateStore trait - mutable snapshot cache
+///
+/// Provides fast state access for execution resumption and working memory.
+/// This is NOT the source of truth - always verify freshness against EventStore.
+#[async_trait]
+pub trait StateStore: StorageBackend {
+    // =========================================================================
+    // Snapshot operations
+    // =========================================================================
+
+    /// Save a state snapshot
+    ///
+    /// # Guarantees
+    /// - Best-effort durability
+    /// - May be overwritten by subsequent saves
+    /// - May be lost on backend failure
+    async fn save_snapshot(&self, snapshot: ExecutionSnapshot) -> anyhow::Result<()>;
+
+    /// Load the latest snapshot for an execution
+    ///
+    /// Returns `None` if no snapshot exists.
+    ///
+    /// # Important
+    /// Always check `snapshot.sequence_number` against EventStore to verify freshness.
+    async fn load_snapshot(
+        &self,
+        execution_id: &ExecutionId,
+    ) -> anyhow::Result<Option<ExecutionSnapshot>>;
+
+    /// Delete a snapshot
+    ///
+    /// Called when an execution completes or is cleaned up.
+    async fn delete_snapshot(&self, execution_id: &ExecutionId) -> anyhow::Result<()>;
+
+    // =========================================================================
+    // Key-value operations (working memory)
+    // =========================================================================
+
+    /// Set a key-value pair
+    ///
+    /// # Arguments
+    /// * `key` - The key (should be namespaced, e.g., "exec:{id}:var:{name}")
+    /// * `value` - The value as bytes
+    /// * `ttl` - Optional time-to-live (auto-delete after this duration)
+    async fn set(&self, key: &str, value: &[u8], ttl: Option<Duration>) -> anyhow::Result<()>;
+
+    /// Get a value by key
+    ///
+    /// Returns `None` if the key doesn't exist or has expired.
+    async fn get(&self, key: &str) -> anyhow::Result<Option<Vec<u8>>>;
+
+    /// Delete a key
+    async fn delete(&self, key: &str) -> anyhow::Result<()>;
+
+    /// Check if a key exists
+    async fn exists(&self, key: &str) -> anyhow::Result<bool> {
+        Ok(self.get(key).await?.is_some())
+    }
+
+
+    // =========================================================================
+    // Batch operations
+    // =========================================================================
+
+    /// Delete all state for an execution
+    ///
+    /// Cleans up snapshot and any keys prefixed with the execution ID.
+    async fn delete_execution_state(&self, execution_id: &ExecutionId) -> anyhow::Result<()> {
+        self.delete_snapshot(execution_id).await
+        // Implementations may also clean up related keys
+    }
+
+    /// List snapshots for a tenant (for cleanup/admin)
+    async fn list_snapshots(
+        &self,
+        tenant_id: &TenantId,
+        limit: usize,
+    ) -> anyhow::Result<Vec<ExecutionId>>;
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_snapshot_freshness() {
+        let snapshot = ExecutionSnapshot::new(
+            ExecutionId::new(),
+            TenantId::from("test"),
+            ExecutionState::Running,
+            10,
+        );
+
+        assert!(snapshot.is_fresh(10));
+        assert!(snapshot.is_fresh(5));
+        assert!(!snapshot.is_fresh(15));
+    }
+
+    #[test]
+    fn test_snapshot_serialization() {
+        let mut snapshot = ExecutionSnapshot::new(
+            ExecutionId::new(),
+            TenantId::from("test"),
+            ExecutionState::Running,
+            5,
+        );
+        snapshot
+            .variables
+            .insert("foo".to_string(), serde_json::json!("bar"));
+
+        let json = serde_json::to_string(&snapshot).unwrap();
+        let parsed: ExecutionSnapshot = serde_json::from_str(&json).unwrap();
+
+        assert_eq!(parsed.state, ExecutionState::Running);
+        assert_eq!(parsed.variables.get("foo").unwrap(), "bar");
+    }
+}

package/crates/enact-core/src/kernel/persistence/vector_store.rs

@@ -0,0 +1,299 @@
+//! VectorStore - Semantic memory and RAG
+//!
+//! The VectorStore provides semantic search capabilities for long-term memory
+//! and retrieval-augmented generation (RAG).
+//!
+//! ## Guarantees
+//!
+//! - **Consistency**: Eventually consistent
+//! - **Authority**: Non-authoritative - for recall only
+//! - **Availability**: Search may return stale results
+//! - **Durability**: Documents are durable after `upsert()` returns
+//!
+//! ## Use Cases
+//!
+//! - Long-term agent memory
+//! - Document retrieval for RAG
+//! - Semantic search over execution history
+//!
+//! @see docs/TECHNICAL/14-PERSISTENCE-LAYER.md
+
+use async_trait::async_trait;
+use serde::{Deserialize, Serialize};
+use std::collections::HashMap;
+
+use super::StorageBackend;
+
+/// A document with embedding for vector storage
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct VectorDocument {
+    /// Unique identifier for this document
+    pub id: String,
+    /// The text content
+    pub content: String,
+    /// Pre-computed embedding vector
+    ///
+    /// Dimensions must match the collection's configured dimension.
+    pub embedding: Vec<f32>,
+    /// Arbitrary metadata for filtering
+    pub metadata: HashMap<String, serde_json::Value>,
+}
+
+impl VectorDocument {
+    /// Create a new document
+    pub fn new(id: impl Into<String>, content: impl Into<String>, embedding: Vec<f32>) -> Self {
+        Self {
+            id: id.into(),
+            content: content.into(),
+            embedding,
+            metadata: HashMap::new(),
+        }
+    }
+
+    /// Add metadata
+    pub fn with_metadata(mut self, key: impl Into<String>, value: serde_json::Value) -> Self {
+        self.metadata.insert(key.into(), value);
+        self
+    }
+}
+
+/// Search result with similarity score
+#[derive(Debug, Clone)]
+pub struct VectorSearchResult {
+    /// The matched document
+    pub document: VectorDocument,
+    /// Similarity score (higher = more similar)
+    ///
+    /// Score semantics depend on the distance metric:
+    /// - Cosine: 0.0 to 1.0 (1.0 = identical)
+    /// - Euclidean: 0.0 to infinity (0.0 = identical)
+    pub score: f32,
+}
+
+/// Single filter condition
+#[derive(Debug, Clone)]
+pub struct FilterCondition {
+    pub key: String,
+    pub value: serde_json::Value,
+}
+
+impl FilterCondition {
+    pub fn new(key: impl Into<String>, value: serde_json::Value) -> Self {
+        Self {
+            key: key.into(),
+            value,
+        }
+    }
+}
+
+/// Filter for vector search
+#[derive(Debug, Clone, Default)]
+pub struct VectorFilter {
+    /// Metadata conditions (all must match)
+    pub conditions: HashMap<String, serde_json::Value>,
+    /// Any-of conditions (at least one must match)
+    pub any: Vec<FilterCondition>,
+    /// None-of conditions (must not match)
+    pub none: Vec<FilterCondition>,
+}
+
+impl VectorFilter {
+    /// Create a new empty filter
+    pub fn new() -> Self {
+        Self::default()
+    }
+
+    /// Add a condition
+    pub fn with_condition(mut self, key: impl Into<String>, value: serde_json::Value) -> Self {
+        self.conditions.insert(key.into(), value);
+        self
+    }
+
+    /// Add an any-of condition
+    pub fn with_any_condition(mut self, key: impl Into<String>, value: serde_json::Value) -> Self {
+        self.any.push(FilterCondition::new(key, value));
+        self
+    }
+
+    /// Add a none-of condition
+    pub fn with_none_condition(
+        mut self,
+        key: impl Into<String>,
+        value: serde_json::Value,
+    ) -> Self {
+        self.none.push(FilterCondition::new(key, value));
+        self
+    }
+
+    /// Check if filter is empty
+    pub fn is_empty(&self) -> bool {
+        self.conditions.is_empty() && self.any.is_empty() && self.none.is_empty()
+    }
+}
+
+/// VectorStore trait - semantic memory
+///
+/// Provides vector similarity search for RAG and long-term memory.
+#[async_trait]
+pub trait VectorStore: StorageBackend {
+    // =========================================================================
+    // Collection management
+    // =========================================================================
+
+    /// Create a collection if it doesn't exist
+    ///
+    /// # Arguments
+    /// * `collection` - Collection name
+    /// * `dimension` - Vector dimension (must match embeddings)
+    async fn ensure_collection(&self, collection: &str, dimension: usize) -> anyhow::Result<()>;
+
+    /// Delete a collection and all its documents
+    async fn delete_collection(&self, collection: &str) -> anyhow::Result<()>;
+
+    /// List all collections
+    async fn list_collections(&self) -> anyhow::Result<Vec<String>>;
+
+    /// Check if a collection exists
+    async fn collection_exists(&self, collection: &str) -> anyhow::Result<bool>;
+
+    // =========================================================================
+    // Document operations
+    // =========================================================================
+
+    /// Upsert a document (insert or update)
+    ///
+    /// If a document with the same ID exists, it is replaced.
+    async fn upsert(&self, collection: &str, document: VectorDocument) -> anyhow::Result<()>;
+
+    /// Upsert multiple documents atomically
+    async fn upsert_batch(
+        &self,
+        collection: &str,
+        documents: Vec<VectorDocument>,
+    ) -> anyhow::Result<()>;
+
+    /// Get a document by ID
+    async fn get(&self, collection: &str, id: &str) -> anyhow::Result<Option<VectorDocument>>;
+
+    /// Delete a document by ID
+    async fn delete(&self, collection: &str, id: &str) -> anyhow::Result<()>;
+
+    /// Delete multiple documents by ID
+    async fn delete_batch(&self, collection: &str, ids: &[String]) -> anyhow::Result<()>;
+
+    // =========================================================================
+    // Search operations
+    // =========================================================================
+
+    /// Search for similar documents
+    ///
+    /// # Arguments
+    /// * `collection` - Collection to search
+    /// * `query_embedding` - Query vector (must match collection dimension)
+    /// * `limit` - Maximum number of results
+    /// * `filter` - Optional metadata filter
+    ///
+    /// # Returns
+    /// Results ordered by similarity (highest score first)
+    async fn search(
+        &self,
+        collection: &str,
+        query_embedding: &[f32],
+        limit: usize,
+        filter: Option<VectorFilter>,
+    ) -> anyhow::Result<Vec<VectorSearchResult>>;
+
+    /// Search with score threshold
+    ///
+    /// Only returns results with score >= min_score.
+    async fn search_with_threshold(
+        &self,
+        collection: &str,
+        query_embedding: &[f32],
+        limit: usize,
+        min_score: f32,
+        filter: Option<VectorFilter>,
+    ) -> anyhow::Result<Vec<VectorSearchResult>> {
+        let results = self.search(collection, query_embedding, limit, filter).await?;
+        Ok(results.into_iter().filter(|r| r.score >= min_score).collect())
+    }
+
+    // =========================================================================
+    // Utility operations
+    // =========================================================================
+
+    /// Count documents in a collection
+    async fn count(&self, collection: &str) -> anyhow::Result<u64>;
+
+    /// Get collection info (dimension, count, etc.)
+    async fn collection_info(&self, collection: &str) -> anyhow::Result<CollectionInfo>;
+}
+
+/// Information about a collection
+#[derive(Debug, Clone)]
+pub struct CollectionInfo {
+    /// Collection name
+    pub name: String,
+    /// Vector dimension
+    pub dimension: usize,
+    /// Number of documents
+    pub count: u64,
+    /// Distance metric used
+    pub distance_metric: DistanceMetric,
+}
+
+/// Distance metric for similarity calculation
+#[derive(Debug, Clone, Copy, PartialEq, Eq, Default)]
+pub enum DistanceMetric {
+    /// Cosine similarity (default)
+    #[default]
+    Cosine,
+    /// Euclidean distance
+    Euclidean,
+    /// Dot product
+    DotProduct,
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_document_creation() {
+        let doc = VectorDocument::new("doc1", "Hello world", vec![0.1, 0.2, 0.3])
+            .with_metadata("source", serde_json::json!("test"))
+            .with_metadata("page", serde_json::json!(1));
+
+        assert_eq!(doc.id, "doc1");
+        assert_eq!(doc.content, "Hello world");
+        assert_eq!(doc.embedding.len(), 3);
+        assert_eq!(doc.metadata.get("source").unwrap(), "test");
+        assert_eq!(doc.metadata.get("page").unwrap(), 1);
+    }
+
+    #[test]
+    fn test_filter_creation() {
+        let filter = VectorFilter::new()
+            .with_condition("tenant_id", serde_json::json!("acme"))
+            .with_condition("status", serde_json::json!("active"))
+            .with_any_condition("visibility", serde_json::json!("team:legal"))
+            .with_none_condition("denyScopes", serde_json::json!("team:blocked"));
+
+        assert!(!filter.is_empty());
+        assert_eq!(filter.conditions.len(), 2);
+        assert_eq!(filter.any.len(), 1);
+        assert_eq!(filter.none.len(), 1);
+    }
+
+    #[test]
+    fn test_document_serialization() {
+        let doc = VectorDocument::new("doc1", "Test content", vec![0.1, 0.2])
+            .with_metadata("key", serde_json::json!("value"));
+
+        let json = serde_json::to_string(&doc).unwrap();
+        let parsed: VectorDocument = serde_json::from_str(&json).unwrap();
+
+        assert_eq!(parsed.id, "doc1");
+        assert_eq!(parsed.embedding, vec![0.1, 0.2]);
+    }
+}