solid_mcp 0.2.3 → 0.5.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 91045bbe4e21e1bd0dd89dbf5477cb36dde677b646645d90df1231dfe16300ff
-  data.tar.gz: fd9e010eefd04ac65c5b63bbb977b5b80c7cc087732298790e85c51d8d1225e5
+  metadata.gz: 443e55616dcd4ddf092dd435572ef3767cfc00a57216b3c4c00187e34a0bb03c
+  data.tar.gz: 3459b9ed5c10ce120eb73db02e61608c25abffc45c081833588b5a281d06eea0
 SHA512:
-  metadata.gz: 285dbe81972f936507644965d2185f6d11821486ab8ed740a7d31639d5fae4e4ac9c9d8a86bbe233058102043310675902c65b85d4a3bf913bb1045c0daf8cc7
-  data.tar.gz: d80ad9922735b39d210c6e942fcc7569ce7ca2c40b3e6989590cc85e18d20b40605f160180416ed8a0bbf108912125f49aec29d279cc986c1c7b61cfee104129
+  metadata.gz: 9850fa1d32aa02e8e7d95429db6f97dc9b9429d41a2b35806f66389e8132cadc740e42ed45cd7f3e3f30158b315d79fbdc56f74955fc594f5b240b263b485d0d
+  data.tar.gz: 475d22bb1fb2ef245b1f9fa8bd64ae153dec52a01f135263cdd23e8fc96cb9f21b846b9303f00b36e54a46799becf2c9d32284bd158c572d151dbf48cb0e6870

data/ext/solid_mcp_native/Cargo.toml

@@ -0,0 +1,12 @@
+[workspace]
+members = ["core", "ffi"]
+resolver = "2"
+
+[workspace.dependencies]
+solid-mcp-core = { path = "./core" }
+tokio = { version = "1", features = ["full"] }
+serde = { version = "1", features = ["derive"] }
+serde_json = "1"
+thiserror = "2"
+tracing = "0.1"
+sqlx = { version = "0.8", features = ["runtime-tokio", "sqlite", "postgres", "chrono"] }

data/ext/solid_mcp_native/core/Cargo.toml

@@ -0,0 +1,32 @@
+[package]
+name = "solid-mcp-core"
+version = "0.1.0"
+edition = "2024"
+rust-version = "1.85"
+authors = ["Abdelkader Boudih <terminale@gmail.com>"]
+license = "MIT"
+repository = "https://github.com/seuros/solid_mcp"
+description = "High-performance async pub/sub engine for MCP (Model Context Protocol)"
+keywords = ["mcp", "pub-sub", "async", "tokio"]
+categories = ["asynchronous", "database"]
+
+[lib]
+crate-type = ["lib"]
+
+[features]
+default = ["sqlite", "postgres"]
+sqlite = ["sqlx/sqlite"]
+postgres = ["sqlx/postgres"]
+
+[dependencies]
+tokio = { workspace = true }
+serde = { workspace = true }
+serde_json = { workspace = true }
+thiserror = { workspace = true }
+tracing = { workspace = true }
+sqlx = { workspace = true }
+chrono = { version = "0.4", features = ["serde"] }
+async-trait = "0.1"
+
+[dev-dependencies]
+tokio-test = "0.4"

data/ext/solid_mcp_native/core/src/config.rs

@@ -0,0 +1,133 @@
+//! Configuration for solid-mcp-core
+
+use std::time::Duration;
+
+/// Configuration for the pub/sub engine
+#[derive(Debug, Clone)]
+pub struct Config {
+    /// Maximum messages per batch write (default: 200)
+    pub batch_size: usize,
+
+    /// Polling interval for SQLite subscribers (default: 100ms)
+    pub polling_interval: Duration,
+
+    /// Maximum wait time for SSE connections (default: 30s)
+    pub max_wait_time: Duration,
+
+    /// How long to keep delivered messages (default: 1 hour)
+    pub delivered_retention: Duration,
+
+    /// How long to keep undelivered messages (default: 24 hours)
+    pub undelivered_retention: Duration,
+
+    /// Maximum messages in memory queue (default: 10,000)
+    pub max_queue_size: usize,
+
+    /// Maximum time to wait for graceful shutdown (default: 30s)
+    pub shutdown_timeout: Duration,
+
+    /// Database URL (required)
+    pub database_url: String,
+}
+
+impl Default for Config {
+    fn default() -> Self {
+        Self {
+            batch_size: 200,
+            polling_interval: Duration::from_millis(100),
+            max_wait_time: Duration::from_secs(30),
+            delivered_retention: Duration::from_secs(3600),
+            undelivered_retention: Duration::from_secs(86400),
+            max_queue_size: 10_000,
+            shutdown_timeout: Duration::from_secs(30),
+            database_url: String::new(),
+        }
+    }
+}
+
+impl Config {
+    /// Create a new config with the given database URL
+    pub fn new(database_url: impl Into<String>) -> Self {
+        Self {
+            database_url: database_url.into(),
+            ..Default::default()
+        }
+    }
+
+    /// Builder pattern: set batch size
+    pub fn batch_size(mut self, size: usize) -> Self {
+        self.batch_size = size;
+        self
+    }
+
+    /// Builder pattern: set polling interval
+    pub fn polling_interval(mut self, interval: Duration) -> Self {
+        self.polling_interval = interval;
+        self
+    }
+
+    /// Builder pattern: set max queue size
+    pub fn max_queue_size(mut self, size: usize) -> Self {
+        self.max_queue_size = size;
+        self
+    }
+
+    /// Builder pattern: set shutdown timeout
+    pub fn shutdown_timeout(mut self, timeout: Duration) -> Self {
+        self.shutdown_timeout = timeout;
+        self
+    }
+
+    /// Check if this is a PostgreSQL connection
+    pub fn is_postgres(&self) -> bool {
+        self.database_url.starts_with("postgres://")
+            || self.database_url.starts_with("postgresql://")
+    }
+
+    /// Check if this is a SQLite connection
+    pub fn is_sqlite(&self) -> bool {
+        self.database_url.starts_with("sqlite://")
+            || self.database_url.starts_with("sqlite:")
+            || self.database_url.ends_with(".db")
+            || self.database_url.ends_with(".sqlite")
+            || self.database_url.ends_with(".sqlite3")
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use super::*;
+
+    #[test]
+    fn test_default_config() {
+        let config = Config::default();
+        assert_eq!(config.batch_size, 200);
+        assert_eq!(config.polling_interval, Duration::from_millis(100));
+        assert_eq!(config.max_queue_size, 10_000);
+    }
+
+    #[test]
+    fn test_builder_pattern() {
+        let config = Config::new("sqlite::memory:")
+            .batch_size(100)
+            .polling_interval(Duration::from_millis(50))
+            .max_queue_size(5000);
+
+        assert_eq!(config.batch_size, 100);
+        assert_eq!(config.polling_interval, Duration::from_millis(50));
+        assert_eq!(config.max_queue_size, 5000);
+        assert_eq!(config.database_url, "sqlite::memory:");
+    }
+
+    #[test]
+    fn test_database_type_detection() {
+        assert!(Config::new("postgres://localhost/test").is_postgres());
+        assert!(Config::new("postgresql://localhost/test").is_postgres());
+        assert!(!Config::new("postgres://localhost/test").is_sqlite());
+
+        assert!(Config::new("sqlite::memory:").is_sqlite());
+        assert!(Config::new("sqlite://./test.db").is_sqlite());
+        assert!(Config::new("./test.sqlite3").is_sqlite());
+        assert!(!Config::new("sqlite::memory:").is_postgres());
+    }
+}

data/ext/solid_mcp_native/core/src/db/mod.rs

@@ -0,0 +1,154 @@
+//! Database abstraction layer for solid-mcp-core
+//!
+//! Supports both SQLite and PostgreSQL backends.
+
+#[cfg(feature = "postgres")]
+pub mod postgres;
+#[cfg(feature = "sqlite")]
+pub mod sqlite;
+
+use crate::{Config, Message, Result};
+use async_trait::async_trait;
+use std::time::Duration;
+
+/// Database backend trait
+#[async_trait]
+pub trait Database: Send + Sync + 'static {
+    /// Insert a batch of messages
+    async fn insert_batch(&self, messages: &[Message]) -> Result<()>;
+
+    /// Fetch undelivered messages for a session after the given ID
+    async fn fetch_after(
+        &self,
+        session_id: &str,
+        after_id: i64,
+        limit: i64,
+    ) -> Result<Vec<Message>>;
+
+    /// Mark messages as delivered
+    async fn mark_delivered(&self, ids: &[i64]) -> Result<()>;
+
+    /// Delete old delivered messages
+    async fn cleanup_delivered(&self, older_than: Duration) -> Result<u64>;
+
+    /// Delete old undelivered messages
+    async fn cleanup_undelivered(&self, older_than: Duration) -> Result<u64>;
+
+    /// Get the maximum message ID (for initialization)
+    async fn max_id(&self) -> Result<i64>;
+}
+
+/// Database pool type (enum dispatch for runtime selection)
+pub enum DbPool {
+    #[cfg(feature = "sqlite")]
+    Sqlite(sqlite::SqlitePool),
+    #[cfg(feature = "postgres")]
+    Postgres(postgres::PostgresPool),
+}
+
+impl DbPool {
+    /// Create a new database pool from config
+    ///
+    /// The database and tables must already exist (created by Ruby migrations).
+    pub async fn new(config: &Config) -> Result<Self> {
+        #[cfg(feature = "postgres")]
+        if config.is_postgres() {
+            return Ok(Self::Postgres(
+                postgres::PostgresPool::new(&config.database_url).await?,
+            ));
+        }
+
+        #[cfg(feature = "sqlite")]
+        if config.is_sqlite() {
+            return Ok(Self::Sqlite(
+                sqlite::SqlitePool::new(&config.database_url).await?,
+            ));
+        }
+
+        Err(crate::Error::Config(format!(
+            "Unsupported database URL: {}",
+            config.database_url
+        )))
+    }
+
+    /// Check if this is a PostgreSQL pool (supports LISTEN/NOTIFY)
+    pub fn is_postgres(&self) -> bool {
+        matches!(self, Self::Postgres(_))
+    }
+
+    /// Create tables for testing purposes only
+    #[cfg(test)]
+    pub(crate) async fn setup_test_schema(&self) -> Result<()> {
+        match self {
+            #[cfg(feature = "sqlite")]
+            Self::Sqlite(pool) => pool.setup_test_schema().await,
+            #[cfg(feature = "postgres")]
+            Self::Postgres(_) => {
+                // PostgreSQL tests require pre-existing schema
+                Ok(())
+            }
+        }
+    }
+}
+
+#[async_trait]
+impl Database for DbPool {
+    async fn insert_batch(&self, messages: &[Message]) -> Result<()> {
+        match self {
+            #[cfg(feature = "sqlite")]
+            Self::Sqlite(pool) => pool.insert_batch(messages).await,
+            #[cfg(feature = "postgres")]
+            Self::Postgres(pool) => pool.insert_batch(messages).await,
+        }
+    }
+
+    async fn fetch_after(
+        &self,
+        session_id: &str,
+        after_id: i64,
+        limit: i64,
+    ) -> Result<Vec<Message>> {
+        match self {
+            #[cfg(feature = "sqlite")]
+            Self::Sqlite(pool) => pool.fetch_after(session_id, after_id, limit).await,
+            #[cfg(feature = "postgres")]
+            Self::Postgres(pool) => pool.fetch_after(session_id, after_id, limit).await,
+        }
+    }
+
+    async fn mark_delivered(&self, ids: &[i64]) -> Result<()> {
+        match self {
+            #[cfg(feature = "sqlite")]
+            Self::Sqlite(pool) => pool.mark_delivered(ids).await,
+            #[cfg(feature = "postgres")]
+            Self::Postgres(pool) => pool.mark_delivered(ids).await,
+        }
+    }
+
+    async fn cleanup_delivered(&self, older_than: Duration) -> Result<u64> {
+        match self {
+            #[cfg(feature = "sqlite")]
+            Self::Sqlite(pool) => pool.cleanup_delivered(older_than).await,
+            #[cfg(feature = "postgres")]
+            Self::Postgres(pool) => pool.cleanup_delivered(older_than).await,
+        }
+    }
+
+    async fn cleanup_undelivered(&self, older_than: Duration) -> Result<u64> {
+        match self {
+            #[cfg(feature = "sqlite")]
+            Self::Sqlite(pool) => pool.cleanup_undelivered(older_than).await,
+            #[cfg(feature = "postgres")]
+            Self::Postgres(pool) => pool.cleanup_undelivered(older_than).await,
+        }
+    }
+
+    async fn max_id(&self) -> Result<i64> {
+        match self {
+            #[cfg(feature = "sqlite")]
+            Self::Sqlite(pool) => pool.max_id().await,
+            #[cfg(feature = "postgres")]
+            Self::Postgres(pool) => pool.max_id().await,
+        }
+    }
+}

data/ext/solid_mcp_native/core/src/db/postgres.rs

@@ -0,0 +1,242 @@
+//! PostgreSQL database backend for solid-mcp-core
+//!
+//! Supports LISTEN/NOTIFY for real-time message delivery without polling.
+
+use crate::{Message, Result};
+use async_trait::async_trait;
+use sqlx::postgres::{PgConnectOptions, PgListener, PgPoolOptions};
+use sqlx::{Pool, Postgres};
+use std::str::FromStr;
+use std::time::Duration;
+
+/// PostgreSQL connection pool
+#[derive(Clone)]
+pub struct PostgresPool {
+    pool: Pool<Postgres>,
+    database_url: String,
+}
+
+impl PostgresPool {
+    /// Create a new PostgreSQL pool from a database URL
+    ///
+    /// The database and tables must already exist (created by Ruby migrations).
+    pub async fn new(database_url: &str) -> Result<Self> {
+        let options = PgConnectOptions::from_str(database_url)?;
+
+        let pool = PgPoolOptions::new()
+            .max_connections(10)
+            .acquire_timeout(Duration::from_secs(30))
+            .connect_with(options)
+            .await?;
+
+        Ok(Self {
+            pool,
+            database_url: database_url.to_string(),
+        })
+    }
+
+    /// Create a LISTEN connection for a session
+    ///
+    /// This is used for real-time message delivery without polling.
+    pub async fn listen(&self, session_id: &str) -> Result<PgListener> {
+        let mut listener = PgListener::connect(&self.database_url).await?;
+        let channel = format!("solid_mcp_{}", session_id);
+        listener.listen(&channel).await?;
+        Ok(listener)
+    }
+
+    /// Send a NOTIFY for a session (called after insert for immediate delivery)
+    pub async fn notify(&self, session_id: &str, message_id: i64) -> Result<()> {
+        let channel = format!("solid_mcp_{}", session_id);
+        sqlx::query("SELECT pg_notify($1, $2)")
+            .bind(&channel)
+            .bind(message_id.to_string())
+            .execute(&self.pool)
+            .await?;
+        Ok(())
+    }
+}
+
+#[async_trait]
+impl super::Database for PostgresPool {
+    async fn insert_batch(&self, messages: &[Message]) -> Result<()> {
+        if messages.is_empty() {
+            return Ok(());
+        }
+
+        // Use COPY for maximum performance on large batches
+        // Fall back to multi-row INSERT for smaller batches
+        if messages.len() >= 100 {
+            self.insert_batch_copy(messages).await
+        } else {
+            self.insert_batch_values(messages).await
+        }
+    }
+
+    async fn fetch_after(
+        &self,
+        session_id: &str,
+        after_id: i64,
+        limit: i64,
+    ) -> Result<Vec<Message>> {
+        let rows = sqlx::query_as::<
+            _,
+            (
+                i64,
+                String,
+                String,
+                String,
+                chrono::DateTime<chrono::Utc>,
+                Option<chrono::DateTime<chrono::Utc>>,
+            ),
+        >(
+            r#"
+            SELECT id, session_id, event_type, data, created_at, delivered_at
+            FROM solid_mcp_messages
+            WHERE session_id = $1 AND delivered_at IS NULL AND id > $2
+            ORDER BY id
+            LIMIT $3
+            "#,
+        )
+        .bind(session_id)
+        .bind(after_id)
+        .bind(limit)
+        .fetch_all(&self.pool)
+        .await?;
+
+        let messages = rows
+            .into_iter()
+            .map(
+                |(id, session_id, event_type, data, created_at, delivered_at)| Message {
+                    id,
+                    session_id,
+                    event_type,
+                    data,
+                    created_at,
+                    delivered_at,
+                },
+            )
+            .collect();
+
+        Ok(messages)
+    }
+
+    async fn mark_delivered(&self, ids: &[i64]) -> Result<()> {
+        if ids.is_empty() {
+            return Ok(());
+        }
+
+        sqlx::query(
+            r#"
+            UPDATE solid_mcp_messages
+            SET delivered_at = NOW()
+            WHERE id = ANY($1)
+            "#,
+        )
+        .bind(ids)
+        .execute(&self.pool)
+        .await?;
+
+        Ok(())
+    }
+
+    async fn cleanup_delivered(&self, older_than: Duration) -> Result<u64> {
+        let cutoff = chrono::Utc::now() - chrono::Duration::from_std(older_than).unwrap();
+
+        let result = sqlx::query(
+            r#"
+            DELETE FROM solid_mcp_messages
+            WHERE delivered_at IS NOT NULL AND delivered_at < $1
+            "#,
+        )
+        .bind(cutoff)
+        .execute(&self.pool)
+        .await?;
+
+        Ok(result.rows_affected())
+    }
+
+    async fn cleanup_undelivered(&self, older_than: Duration) -> Result<u64> {
+        let cutoff = chrono::Utc::now() - chrono::Duration::from_std(older_than).unwrap();
+
+        let result = sqlx::query(
+            r#"
+            DELETE FROM solid_mcp_messages
+            WHERE delivered_at IS NULL AND created_at < $1
+            "#,
+        )
+        .bind(cutoff)
+        .execute(&self.pool)
+        .await?;
+
+        Ok(result.rows_affected())
+    }
+
+    async fn max_id(&self) -> Result<i64> {
+        let row: (Option<i64>,) = sqlx::query_as("SELECT MAX(id) FROM solid_mcp_messages")
+            .fetch_one(&self.pool)
+            .await?;
+
+        Ok(row.0.unwrap_or(0))
+    }
+}
+
+impl PostgresPool {
+    /// Insert using multi-row VALUES (good for small batches)
+    async fn insert_batch_values(&self, messages: &[Message]) -> Result<()> {
+        let mut query = String::from(
+            "INSERT INTO solid_mcp_messages (session_id, event_type, data, created_at) VALUES ",
+        );
+
+        for (i, _) in messages.iter().enumerate() {
+            if i > 0 {
+                query.push_str(", ");
+            }
+            let base = i * 4 + 1;
+            query.push_str(&format!(
+                "(${}, ${}, ${}, ${})",
+                base,
+                base + 1,
+                base + 2,
+                base + 3
+            ));
+        }
+
+        let mut q = sqlx::query(&query);
+        for msg in messages {
+            q = q
+                .bind(&msg.session_id)
+                .bind(&msg.event_type)
+                .bind(&msg.data)
+                .bind(msg.created_at);
+        }
+        q.execute(&self.pool).await?;
+
+        Ok(())
+    }
+
+    /// Insert using COPY (efficient for large batches)
+    async fn insert_batch_copy(&self, messages: &[Message]) -> Result<()> {
+        // For now, fall back to VALUES insert
+        // TODO: Implement proper COPY protocol for maximum throughput
+        self.insert_batch_values(messages).await
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    // PostgreSQL tests require a running database
+    // Run with: DATABASE_URL=postgres://localhost/test_solid_mcp cargo test
+
+    use super::*;
+    use crate::db::Database;
+
+    #[tokio::test]
+    #[ignore] // Requires PostgreSQL
+    async fn test_postgres_pool_creation() {
+        let url = std::env::var("DATABASE_URL")
+            .unwrap_or_else(|_| "postgres://localhost/test_solid_mcp".to_string());
+        let pool = PostgresPool::new(&url).await.unwrap();
+        let _ = pool.max_id().await.unwrap();
+    }
+}