@feltdb/core 0.4.12 → 0.4.14

package/dist/create/server-source/crates/feltdb/src/bin/feltdb_node.rs

@@ -0,0 +1,266 @@
+/// Standalone FeltDB node for OS process boundary testing
+///
+/// Simplified runtime that:
+/// 1. Accepts command via stdin (execute-tx, query-state, shutdown)
+/// 2. Outputs state to stdout in JSON format
+/// 3. Runs replication over TCP in background
+/// 4. Executes local transactions and replicates to peers
+/// 5. Receives and applies replicated transactions from peers
+
+use feltdb::convergence::VectorClock;
+use feltdb::distributed_transactions::{DistributedTransactionExecutor, ReplicationMessage};
+use feltdb::replication_protocol::ProtocolTransport;
+use feltdb::state_hash::StateHash;
+use feltdb::tcp_transport::TcpTransport;
+use feltdb::transactions::{
+    ConsistencyContract, Operation, OperationCommand, OperationId, StateVersion,
+};
+use serde_json::json;
+use std::collections::HashMap;
+use std::io::{self, BufRead, Write};
+use std::sync::atomic::{AtomicBool, Ordering};
+use std::sync::Arc;
+use tokio::sync::{mpsc, Mutex};
+use tokio::time::{sleep, Duration};
+
+#[derive(Clone)]
+struct NodeState {
+    executor: Arc<Mutex<DistributedTransactionExecutor>>,
+    node_id: String,
+    transport: Arc<Mutex<TcpTransport>>,
+    tx_sequence: Arc<Mutex<u64>>,
+    is_server: bool,
+    should_exit: Arc<AtomicBool>,
+}
+
+#[tokio::main]
+async fn main() -> Result<(), Box<dyn std::error::Error>> {
+    let args: Vec<String> = std::env::args().collect();
+
+    // Parse arguments
+    let mut mode = "server";
+    let mut replication_port = 9001u16;
+    let mut peer_addr = String::new();
+
+    let mut i = 1;
+    while i < args.len() {
+        match args[i].as_str() {
+            "--mode" => {
+                i += 1;
+                mode = &args[i];
+            }
+            "--port" => {
+                i += 1;
+                replication_port = args[i].parse()?;
+            }
+            "--peer" => {
+                i += 1;
+                peer_addr = args[i].clone();
+            }
+            _ => {}
+        }
+        i += 1;
+    }
+
+    let is_server = mode == "server";
+    let node_id = if is_server {
+        "node_a".to_string()
+    } else {
+        "node_b".to_string()
+    };
+
+    // Initialize executor
+    let initial_state = StateHash::from_hex("hash_initial".to_string());
+    let mut executor =
+        DistributedTransactionExecutor::new(node_id.clone(), initial_state.clone());
+    executor.register_replica("node_a".to_string(), initial_state.clone());
+    executor.register_replica("node_b".to_string(), initial_state.clone());
+
+    let transport = if is_server {
+        TcpTransport::new(format!("127.0.0.1:{}", replication_port))
+    } else {
+        TcpTransport::new("127.0.0.1:0".to_string())
+    };
+
+    let node_state = NodeState {
+        executor: Arc::new(Mutex::new(executor)),
+        node_id: node_id.clone(),
+        transport: Arc::new(Mutex::new(transport)),
+        tx_sequence: Arc::new(Mutex::new(1)),
+        is_server,
+        should_exit: Arc::new(AtomicBool::new(false)),
+    };
+
+    // Start TCP replication in background
+    if is_server {
+        let replication_addr = format!("127.0.0.1:{}", replication_port);
+        let ns = node_state.clone();
+        tokio::spawn(async move {
+            let mut transport = ns.transport.lock().await;
+            if let Err(e) = transport.listen().await {
+                eprintln!("ERROR: listen failed: {}", e);
+            }
+        });
+        println!("STARTED server on port {}", replication_port);
+        let _ = std::io::stdout().flush();
+    } else {
+        let ns = node_state.clone();
+        let peer = peer_addr.clone();
+        tokio::spawn(async move {
+            let mut transport = ns.transport.lock().await;
+            if let Err(e) = transport.connect(&peer).await {
+                eprintln!("ERROR: connect failed: {}", e);
+            }
+        });
+        println!("STARTED client connecting to {}", peer_addr);
+        let _ = std::io::stdout().flush();
+    }
+
+    // Spawn background task to receive and apply replicated messages
+    let ns = node_state.clone();
+    tokio::spawn(async move {
+        loop {
+            if ns.should_exit.load(Ordering::Relaxed) {
+                break;
+            }
+
+            let messages = {
+                let transport = ns.transport.lock().await;
+                match transport.receive().await {
+                    Ok(msgs) => msgs,
+                    Err(_) => {
+                        sleep(Duration::from_millis(100)).await;
+                        continue;
+                    }
+                }
+            };
+
+            for msg in messages {
+                let mut executor = ns.executor.lock().await;
+                let mut parent_vc = VectorClock::new();
+                parent_vc.increment(&ns.node_id);
+                let parent_version = StateVersion::new(parent_vc, "hash_initial".to_string());
+
+                if let Err(e) = executor.receive_replicated_transaction(msg, parent_version) {
+                    eprintln!("ERROR: apply replication failed: {}", e);
+                }
+            }
+
+            sleep(Duration::from_millis(50)).await;
+        }
+    });
+
+    // Give replication setup time to establish
+    sleep(Duration::from_millis(500)).await;
+
+    // Create a channel for stdin commands
+    let (tx, mut rx) = mpsc::channel::<String>(100);
+
+    // Spawn a blocking task to read stdin
+    tokio::task::spawn_blocking(move || {
+        let stdin = io::stdin();
+        let handle = stdin.lock();
+
+        for line in handle.lines() {
+            if let Ok(cmd) = line {
+                let cmd_str = cmd.trim().to_string();
+                if !cmd_str.is_empty() {
+                    // Non-blocking send, ignore if receiver is gone
+                    let _ = tx.blocking_send(cmd_str);
+                }
+            }
+        }
+    });
+
+    // Process commands from the channel
+    while let Some(cmd) = rx.recv().await {
+        match cmd.trim() {
+            "execute-tx" => {
+                let seq = {
+                    let mut s = node_state.tx_sequence.lock().await;
+                    let current = *s;
+                    *s += 1;
+                    current
+                };
+
+                // Create a test operation
+                let mut parent_vc = VectorClock::new();
+                parent_vc.increment(&node_state.node_id);
+                let parent_version = StateVersion::new(parent_vc, "hash_initial".to_string());
+
+                let mut fields = HashMap::new();
+                fields.insert("key".to_string(), json!("value"));
+
+                let op = Operation::new(
+                    OperationId::new(node_state.node_id.clone(), seq),
+                    parent_version.clone(),
+                    format!("tx_{}", seq),
+                    OperationCommand {
+                        op_type: "set".to_string(),
+                        collection: "items".to_string(),
+                        record_id: format!("item{}", seq),
+                        fields,
+                    },
+                    node_state.node_id.clone(),
+                );
+
+                {
+                    let mut executor = node_state.executor.lock().await;
+                    match executor.execute_local_transaction(
+                        seq,
+                        format!("tx_{}", seq),
+                        parent_version,
+                        vec![op],
+                        ConsistencyContract::local(),
+                    ) {
+                        Ok(envelope) => {
+                            let replication_msg = ReplicationMessage::new(
+                                envelope,
+                                node_state.node_id.clone(),
+                                if node_state.node_id == "node_a" {
+                                    "node_b".to_string()
+                                } else {
+                                    "node_a".to_string()
+                                },
+                                seq,
+                            );
+
+                            let transport = node_state.transport.lock().await;
+                            if let Err(e) = transport.send(replication_msg).await {
+                                println!("ERROR: send replication failed: {}", e);
+                            } else {
+                                println!("TX_EXECUTED");
+                            }
+                        }
+                        Err(e) => {
+                            println!("ERROR: execute transaction failed: {}", e);
+                        }
+                    }
+                }
+                let _ = std::io::stdout().flush();
+            }
+            "query-state" => {
+                let executor = node_state.executor.lock().await;
+                if let Some(replica) = executor.get_replica_state(&node_state.node_id) {
+                    println!("STATE:{},{}", replica.state_hash, replica.operations_applied);
+                } else {
+                    println!("ERROR: node not found");
+                }
+                let _ = std::io::stdout().flush();
+            }
+            "shutdown" => {
+                node_state.should_exit.store(true, Ordering::Relaxed);
+                println!("SHUTDOWN");
+                let _ = std::io::stdout().flush();
+                sleep(Duration::from_millis(100)).await;
+                std::process::exit(0);
+            }
+            _ => {
+                println!("ERROR: unknown command");
+                let _ = std::io::stdout().flush();
+            }
+        }
+    }
+
+    Ok(())
+}

package/dist/create/server-source/crates/feltdb/src/capabilities/mod.rs

@@ -0,0 +1,8 @@
+/// Concrete capability implementations
+pub mod search;
+pub mod vector;
+
+pub use search::{SearchCapability, SearchDocument, SearchResult};
+pub use vector::{
+    DemoEmbeddingProvider, EmbeddingProvider, VectorCapability, VectorDocument, VectorSearchResult,
+};

package/dist/create/server-source/crates/feltdb/src/capabilities/search.rs

@@ -0,0 +1,418 @@
+/// Full-text Search Capability
+///
+/// A concrete implementation of FlowCapability that maintains a full-text search index.
+/// This demonstrates the capability extension model with a projection that doesn't own
+/// canonical state but derives from it.
+use crate::capability::{
+    CapabilityAccessModel, CapabilityContext, CapabilityId, CapabilityMetadata,
+    CapabilityOperationResult, CapabilityQuery, CapabilityQueryResult, CapabilityRole,
+    FlowCapability,
+};
+use crate::{Operation, OperationType, Result};
+use serde::{Deserialize, Serialize};
+use serde_json::{json, Value};
+use std::collections::HashMap;
+
+/// Search index document representation
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct SearchDocument {
+    pub key: String,
+    pub content: String,
+    pub metadata: Value,
+}
+
+/// Search result from the search capability
+#[derive(Debug, Clone, Serialize, Deserialize)]
+pub struct SearchResult {
+    pub key: String,
+    pub content: String,
+    pub score: f32,
+    pub metadata: Value,
+}
+
+/// Full-text search capability implementation
+pub struct SearchCapability {
+    capability_id: CapabilityId,
+    metadata: CapabilityMetadata,
+    /// In-memory document store
+    documents: HashMap<String, SearchDocument>,
+}
+
+impl SearchCapability {
+    pub fn new() -> Self {
+        let capability_id = CapabilityId::new("search".to_string());
+        let access_model = CapabilityAccessModel::new(
+            vec!["*".to_string()],
+            vec!["search.index".to_string()],
+            vec![],
+        );
+
+        let metadata = CapabilityMetadata::new(
+            capability_id.clone(),
+            "Full-Text Search".to_string(),
+            "Maintains a full-text search index over documents".to_string(),
+            CapabilityRole::Projection,
+            access_model,
+        );
+
+        Self {
+            capability_id,
+            metadata,
+            documents: HashMap::new(),
+        }
+    }
+
+    /// Add or update a document in the search index
+    fn index_document(&mut self, doc: &SearchDocument) -> Result<()> {
+        self.documents.insert(doc.key.clone(), doc.clone());
+        Ok(())
+    }
+
+    /// Remove a document from the search index
+    fn remove_document(&mut self, key: &str) -> Result<()> {
+        self.documents.remove(key);
+        Ok(())
+    }
+
+    /// Search the index - simple term matching
+    fn search(&self, query_str: &str, limit: usize) -> Result<Vec<SearchResult>> {
+        // Simple tokenization
+        let query_terms: Vec<&str> = query_str
+            .split_whitespace()
+            .filter(|s| !s.is_empty())
+            .collect();
+
+        let mut results = Vec::new();
+
+        // Search through all documents
+        for doc in self.documents.values() {
+            let mut score = 0.0f32;
+            let content_lower = doc.content.to_lowercase();
+
+            // Score based on term matches
+            for term in &query_terms {
+                let term_lower = term.to_lowercase();
+                if content_lower.contains(&term_lower) {
+                    // Count occurrences
+                    let count = content_lower.matches(&term_lower).count() as f32;
+                    score += count * 1.0;
+                }
+            }
+
+            if score > 0.0 {
+                results.push(SearchResult {
+                    key: doc.key.clone(),
+                    content: doc.content.clone(),
+                    score,
+                    metadata: doc.metadata.clone(),
+                });
+            }
+        }
+
+        // Sort by score descending
+        results.sort_by(|a, b| b.score.partial_cmp(&a.score).unwrap());
+        results.truncate(limit);
+
+        Ok(results)
+    }
+}
+
+impl Default for SearchCapability {
+    fn default() -> Self {
+        Self::new()
+    }
+}
+
+impl FlowCapability for SearchCapability {
+    fn id(&self) -> CapabilityId {
+        self.capability_id.clone()
+    }
+
+    fn metadata(&self) -> CapabilityMetadata {
+        self.metadata.clone()
+    }
+
+    fn initialize(&mut self, _context: &CapabilityContext) -> Result<()> {
+        Ok(())
+    }
+
+    fn on_operation(
+        &mut self,
+        operation: &Operation,
+        _context: &CapabilityContext,
+    ) -> Result<CapabilityOperationResult> {
+        // Only index Insert and Update operations
+        match operation.op_type {
+            OperationType::Insert | OperationType::Update => {
+                if let Some(value) = &operation.value {
+                    let content = if value.is_string() {
+                        value.as_str().unwrap_or("").to_string()
+                    } else {
+                        value.to_string()
+                    };
+
+                    let doc = SearchDocument {
+                        key: operation.key.clone(),
+                        content,
+                        metadata: json!({
+                            "source_operation": operation.op_id,
+                            "timestamp": operation.timestamp_ms,
+                        }),
+                    };
+
+                    self.index_document(&doc)?;
+                    Ok(CapabilityOperationResult::with_derived_state(json!({
+                        "indexed": true,
+                        "key": operation.key.clone(),
+                    })))
+                } else {
+                    Ok(CapabilityOperationResult::success())
+                }
+            }
+            OperationType::Delete => {
+                self.remove_document(&operation.key)?;
+                Ok(CapabilityOperationResult::success())
+            }
+        }
+    }
+
+    fn query(
+        &self,
+        query: &CapabilityQuery,
+        _context: &CapabilityContext,
+    ) -> Result<CapabilityQueryResult> {
+        if query.query_type != "search" {
+            return Ok(CapabilityQueryResult::error(format!(
+                "Unknown query type: {}",
+                query.query_type
+            )));
+        }
+
+        let query_str = query.params.get("q").and_then(|v| v.as_str()).unwrap_or("");
+
+        let limit = query
+            .params
+            .get("limit")
+            .and_then(|v| v.as_u64())
+            .unwrap_or(10) as usize;
+
+        let results = self.search(query_str, limit)?;
+        let result_values: Vec<Value> = results
+            .iter()
+            .map(|r| {
+                json!({
+                    "key": r.key,
+                    "content": r.content,
+                    "score": r.score,
+                    "metadata": r.metadata,
+                })
+            })
+            .collect();
+
+        Ok(CapabilityQueryResult::success(result_values))
+    }
+
+    fn rebuild(&mut self, operations: &[Operation], context: &CapabilityContext) -> Result<()> {
+        // Clear existing index
+        self.documents.clear();
+
+        // Replay all operations
+        for op in operations {
+            self.on_operation(op, context)?;
+        }
+
+        Ok(())
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_search_capability_creation() {
+        let cap = SearchCapability::new();
+        assert_eq!(cap.id().0, "search");
+        assert!(cap.metadata().access_model.can_read("docs"));
+        assert!(cap.metadata().access_model.can_write("search.index"));
+    }
+
+    #[test]
+    fn test_search_capability_initialization() {
+        let mut cap = SearchCapability::new();
+        let context = CapabilityContext {
+            capability_id: cap.id(),
+            instance_id: "test-instance".to_string(),
+            sequence: 0,
+            version: 1,
+        };
+
+        assert!(cap.initialize(&context).is_ok());
+    }
+
+    #[test]
+    fn test_search_capability_indexing() {
+        let mut cap = SearchCapability::new();
+        let context = CapabilityContext {
+            capability_id: cap.id(),
+            instance_id: "test-instance".to_string(),
+            sequence: 0,
+            version: 1,
+        };
+
+        cap.initialize(&context).unwrap();
+
+        let op = Operation::insert(
+            1,
+            "instance-1".to_string(),
+            1,
+            "docs:1".to_string(),
+            json!({"title": "Rust Programming", "content": "Learn Rust"}),
+            "Document".to_string(),
+            "docs".to_string(),
+        );
+
+        let result = cap.on_operation(&op, &context).unwrap();
+        assert!(result.success);
+        assert!(result.produced_derived_state);
+    }
+
+    #[test]
+    fn test_search_capability_query() {
+        let mut cap = SearchCapability::new();
+        let context = CapabilityContext {
+            capability_id: cap.id(),
+            instance_id: "test-instance".to_string(),
+            sequence: 0,
+            version: 1,
+        };
+
+        cap.initialize(&context).unwrap();
+
+        let op1 = Operation::insert(
+            1,
+            "instance-1".to_string(),
+            1,
+            "docs:1".to_string(),
+            json!("Rust Programming Language"),
+            "Document".to_string(),
+            "docs".to_string(),
+        );
+
+        let op2 = Operation::insert(
+            2,
+            "instance-1".to_string(),
+            2,
+            "docs:2".to_string(),
+            json!("Python for Data Science"),
+            "Document".to_string(),
+            "docs".to_string(),
+        );
+
+        cap.on_operation(&op1, &context).unwrap();
+        cap.on_operation(&op2, &context).unwrap();
+
+        let query = CapabilityQuery {
+            query_type: "search".to_string(),
+            params: json!({
+                "q": "Rust",
+                "limit": 10
+            }),
+        };
+
+        let results = cap.query(&query, &context).unwrap();
+        assert!(results.success);
+        assert!(results.results.len() > 0);
+    }
+
+    #[test]
+    fn test_search_capability_delete() {
+        let mut cap = SearchCapability::new();
+        let context = CapabilityContext {
+            capability_id: cap.id(),
+            instance_id: "test-instance".to_string(),
+            sequence: 0,
+            version: 1,
+        };
+
+        cap.initialize(&context).unwrap();
+
+        // Insert a document
+        let op1 = Operation::insert(
+            1,
+            "instance-1".to_string(),
+            1,
+            "docs:1".to_string(),
+            json!("Test Document"),
+            "Document".to_string(),
+            "docs".to_string(),
+        );
+        cap.on_operation(&op1, &context).unwrap();
+
+        // Verify it's indexed
+        let query = CapabilityQuery {
+            query_type: "search".to_string(),
+            params: json!({"q": "Test", "limit": 10}),
+        };
+        let results = cap.query(&query, &context).unwrap();
+        assert_eq!(results.results.len(), 1);
+
+        // Delete it
+        let op2 = Operation::delete(
+            2,
+            "instance-1".to_string(),
+            2,
+            "docs:1".to_string(),
+            "Document".to_string(),
+            "docs".to_string(),
+        );
+        cap.on_operation(&op2, &context).unwrap();
+
+        // Verify it's gone
+        let results = cap.query(&query, &context).unwrap();
+        assert_eq!(results.results.len(), 0);
+    }
+
+    #[test]
+    fn test_search_capability_rebuild() {
+        let mut cap = SearchCapability::new();
+        let context = CapabilityContext {
+            capability_id: cap.id(),
+            instance_id: "test-instance".to_string(),
+            sequence: 0,
+            version: 1,
+        };
+
+        cap.initialize(&context).unwrap();
+
+        let ops = vec![
+            Operation::insert(
+                1,
+                "instance-1".to_string(),
+                1,
+                "docs:1".to_string(),
+                json!("Rust Guide"),
+                "Document".to_string(),
+                "docs".to_string(),
+            ),
+            Operation::insert(
+                2,
+                "instance-1".to_string(),
+                2,
+                "docs:2".to_string(),
+                json!("Go Tutorial"),
+                "Document".to_string(),
+                "docs".to_string(),
+            ),
+        ];
+
+        cap.rebuild(&ops, &context).unwrap();
+
+        let query = CapabilityQuery {
+            query_type: "search".to_string(),
+            params: json!({"q": "Rust", "limit": 10}),
+        };
+        let results = cap.query(&query, &context).unwrap();
+        assert_eq!(results.results.len(), 1);
+    }
+}