@lix-js/sdk 0.6.0-preview.3 → 0.6.0-preview.5

package/dist-engine-src/src/session/transaction.rs

@@ -0,0 +1,557 @@
+use std::sync::Arc;
+
+use crate::functions::FunctionContext;
+use crate::storage::{InMemoryStorageBackend, StorageBackend};
+use tokio::sync::Notify;
+
+use crate::transaction::{
+    open_transaction, CommitBoundaryGuard, CommitBoundaryState, Transaction,
+    TransactionCommitBoundary,
+};
+use crate::LixError;
+
+use super::context::{closed_error, SessionWriteAccess};
+use super::SessionContext;
+
+pub struct SessionTransaction<B: StorageBackend = InMemoryStorageBackend> {
+    pub(super) transaction: Option<Transaction<B>>,
+    pub(super) runtime_functions: FunctionContext,
+    transaction_manager: SessionTransactionManager,
+    _write_access: SessionWriteAccess,
+}
+
+impl<B> SessionContext<B>
+where
+    B: StorageBackend + Clone + Send + Sync + 'static,
+    for<'backend> B::Read<'backend>: Clone + Send + Sync + 'static,
+    for<'backend> B::Write<'backend>: Send,
+{
+    pub async fn begin_transaction(&self) -> Result<SessionTransaction<B>, LixError> {
+        self.ensure_open()?;
+        let write_access = self.begin_explicit_session_write_access().await?;
+        let mut opened = match open_transaction(
+            &self.mode,
+            self.storage.clone(),
+            Arc::clone(&self.live_state),
+            Arc::clone(&self.tracked_state),
+            Arc::clone(&self.binary_cas),
+            Arc::clone(&self.branch_ctx),
+            Arc::clone(&self.catalog_context),
+        )
+        .await
+        {
+            Ok(opened) => opened,
+            Err(error) => {
+                return Err(error);
+            }
+        };
+        self.ensure_open()?;
+        opened
+            .transaction
+            .attach_commit_boundary(self.transaction_commit_boundary());
+        self.transaction_manager()
+            .mark_explicit_transaction_open()?;
+        Ok(SessionTransaction {
+            transaction: Some(opened.transaction),
+            runtime_functions: opened.runtime_functions,
+            transaction_manager: self.transaction_manager(),
+            _write_access: write_access,
+        })
+    }
+}
+
+impl<B> SessionTransaction<B>
+where
+    B: StorageBackend + Clone + Send + Sync + 'static,
+    for<'backend> B::Read<'backend>: Clone + Send + Sync + 'static,
+    for<'backend> B::Write<'backend>: Send,
+{
+    pub(super) fn transaction_mut(&mut self) -> Result<&mut Transaction<B>, LixError> {
+        self.ensure_session_open()?;
+        self.transaction
+            .as_mut()
+            .ok_or_else(|| transaction_state_error("Lix transaction is closed"))
+    }
+
+    pub fn active_branch_id(&self) -> Result<&str, LixError> {
+        self.ensure_session_open()?;
+        self.transaction
+            .as_ref()
+            .map(|transaction| transaction.active_branch_id())
+            .ok_or_else(|| transaction_state_error("Lix transaction is closed"))
+    }
+
+    pub async fn commit(mut self) -> Result<(), LixError> {
+        let operation_guard = self.begin_session_commit_operation()?;
+        let transaction = self
+            .transaction
+            .take()
+            .ok_or_else(|| transaction_state_error("Lix transaction is closed"))?;
+        let queued_commit_guard = self.transaction_manager.begin_commit();
+        let result = transaction
+            .commit(&self.runtime_functions)
+            .await
+            .map(|_| ());
+        drop(queued_commit_guard);
+        drop(operation_guard);
+        result
+    }
+
+    pub async fn rollback(mut self) -> Result<(), LixError> {
+        let transaction = self
+            .transaction
+            .take()
+            .ok_or_else(|| transaction_state_error("Lix transaction is closed"))?;
+        let result = transaction.rollback().await;
+        result
+    }
+
+    pub(super) fn ensure_session_open(&self) -> Result<(), LixError> {
+        self.transaction_manager.ensure_open()
+    }
+
+    pub(super) fn begin_session_operation(&self) -> Result<SessionOperationGuard, LixError> {
+        self.transaction_manager.begin_transaction_operation()
+    }
+
+    fn begin_session_commit_operation(&self) -> Result<SessionOperationGuard, LixError> {
+        self.transaction_manager
+            .begin_transaction_commit_operation()
+    }
+}
+
+pub(crate) fn transaction_state_error(message: impl Into<String>) -> LixError {
+    LixError::new("LIX_INVALID_TRANSACTION_STATE", message)
+}
+
+#[derive(Clone)]
+pub(super) struct SessionTransactionManager {
+    inner: Arc<SessionTransactionManagerInner>,
+}
+
+struct SessionTransactionManagerInner {
+    state: std::sync::Mutex<SessionTransactionState>,
+    state_changed: Notify,
+    commit_boundary: CommitBoundaryState,
+}
+
+#[derive(Debug, Default)]
+enum SessionTransactionState {
+    #[default]
+    OpenIdle,
+    OpenOperation {
+        active_operations: usize,
+    },
+    OpenTransaction {
+        active_operations: usize,
+        owner: TransactionOwner,
+    },
+    Closing {
+        active_operations: usize,
+    },
+    Closed,
+}
+
+#[derive(Clone, Copy, Debug, Eq, PartialEq)]
+enum TransactionOwner {
+    Automatic,
+    ExplicitOpening,
+    Explicit,
+    ExplicitCommitting,
+}
+
+impl SessionTransactionManager {
+    pub(super) fn new() -> Self {
+        Self {
+            inner: Arc::new(SessionTransactionManagerInner {
+                state: std::sync::Mutex::new(SessionTransactionState::default()),
+                state_changed: Notify::new(),
+                commit_boundary: CommitBoundaryState::new(),
+            }),
+        }
+    }
+
+    pub(super) async fn close(&self) -> Result<(), LixError> {
+        let mut commit_rx = self.inner.commit_boundary.subscribe();
+        loop {
+            if let Some(_commit_gate) = self.inner.commit_boundary.try_lock_durable_commit() {
+                {
+                    let mut state = self.lock_state();
+                    if state.has_explicit_transaction() {
+                        return Err(active_transaction_error());
+                    }
+                    let active_operations = state.active_operations();
+                    *state = if active_operations == 0 {
+                        SessionTransactionState::Closed
+                    } else {
+                        SessionTransactionState::Closing { active_operations }
+                    };
+                }
+                self.inner.state_changed.notify_waiters();
+                break;
+            }
+
+            let notified = self.inner.state_changed.notified();
+            tokio::select! {
+                _ = notified => {}
+                result = commit_rx.changed() => {
+                    if result.is_err() {
+                        self.inner.state_changed.notify_waiters();
+                    }
+                }
+            }
+        }
+
+        loop {
+            let notified = self.inner.state_changed.notified();
+            {
+                let mut state = self.lock_state();
+                let commit_count = *commit_rx.borrow_and_update();
+                if state.active_operations() == 0
+                    && commit_count == 0
+                    && !self.inner.commit_boundary.is_active()
+                {
+                    *state = SessionTransactionState::Closed;
+                    break;
+                }
+            }
+            tokio::select! {
+                _ = notified => {}
+                result = commit_rx.changed() => {
+                    if result.is_err() {
+                        self.inner.state_changed.notify_waiters();
+                    }
+                }
+            }
+        }
+        Ok(())
+    }
+
+    pub(super) fn is_closed(&self) -> bool {
+        self.lock_state().is_closed()
+    }
+
+    pub(super) fn ensure_open(&self) -> Result<(), LixError> {
+        if self.is_closed() {
+            return Err(closed_error());
+        }
+        Ok(())
+    }
+
+    pub(super) fn begin_session_operation(&self) -> Result<SessionOperationGuard, LixError> {
+        self.begin_operation(SessionOperationScope::Session)
+    }
+
+    pub(super) fn begin_transaction_operation(&self) -> Result<SessionOperationGuard, LixError> {
+        self.begin_operation(SessionOperationScope::Transaction)
+    }
+
+    pub(super) fn begin_transaction_commit_operation(
+        &self,
+    ) -> Result<SessionOperationGuard, LixError> {
+        self.begin_operation(SessionOperationScope::TransactionCommit)
+    }
+
+    fn begin_operation(
+        &self,
+        scope: SessionOperationScope,
+    ) -> Result<SessionOperationGuard, LixError> {
+        {
+            let mut state = self.lock_state();
+            state.begin_operation(scope)?;
+        }
+        self.inner.state_changed.notify_waiters();
+
+        if let Err(error) = self.ensure_open() {
+            self.finish_operation();
+            return Err(error);
+        }
+
+        Ok(SessionOperationGuard {
+            manager: self.clone(),
+        })
+    }
+
+    pub(super) fn begin_write_lease(&self) -> Result<SessionWriteLease, LixError> {
+        self.begin_write_lease_for(TransactionOwner::Automatic)
+    }
+
+    pub(super) fn begin_explicit_write_lease(&self) -> Result<SessionWriteLease, LixError> {
+        self.begin_write_lease_for(TransactionOwner::ExplicitOpening)
+    }
+
+    fn begin_write_lease_for(
+        &self,
+        owner: TransactionOwner,
+    ) -> Result<SessionWriteLease, LixError> {
+        {
+            let mut state = self.lock_state();
+            state.begin_write_lease(owner)?;
+        }
+        self.inner.state_changed.notify_waiters();
+
+        let operation_guard = SessionOperationGuard {
+            manager: self.clone(),
+        };
+        if let Err(error) = self.ensure_open() {
+            drop(operation_guard);
+            self.finish_transaction();
+            return Err(error);
+        }
+        let transaction_guard = SessionTransactionGuard {
+            manager: self.clone(),
+        };
+        Ok(SessionWriteLease {
+            _transaction_guard: transaction_guard,
+            _operation_guard: operation_guard,
+        })
+    }
+
+    pub(super) fn begin_commit(&self) -> CommitBoundaryGuard {
+        self.inner.commit_boundary.begin()
+    }
+
+    pub(super) fn mark_explicit_transaction_open(&self) -> Result<(), LixError> {
+        {
+            let mut state = self.lock_state();
+            state.mark_explicit_transaction_open()?;
+        }
+        self.inner.state_changed.notify_waiters();
+        Ok(())
+    }
+
+    pub(super) fn transaction_commit_boundary(&self) -> TransactionCommitBoundary {
+        let manager = self.clone();
+        TransactionCommitBoundary::new(
+            self.inner.commit_boundary.clone(),
+            Arc::new(move || manager.ensure_open()),
+        )
+    }
+
+    fn finish_operation(&self) {
+        {
+            let mut state = self.lock_state();
+            state.finish_operation();
+        }
+        self.inner.state_changed.notify_waiters();
+    }
+
+    fn finish_transaction(&self) {
+        {
+            let mut state = self.lock_state();
+            state.finish_transaction();
+        }
+        self.inner.state_changed.notify_waiters();
+    }
+
+    fn lock_state(&self) -> std::sync::MutexGuard<'_, SessionTransactionState> {
+        self.inner
+            .state
+            .lock()
+            .expect("session transaction manager lock should not poison")
+    }
+
+    #[cfg(test)]
+    pub(super) fn operation_count_for_test(&self) -> usize {
+        self.lock_state().active_operations()
+    }
+
+    #[cfg(test)]
+    pub(super) fn commit_in_progress_for_test(&self) -> bool {
+        self.inner.commit_boundary.is_active()
+    }
+
+    #[cfg(test)]
+    pub(super) fn active_transaction_for_test(&self) -> bool {
+        matches!(
+            *self.lock_state(),
+            SessionTransactionState::OpenTransaction { .. }
+        )
+    }
+}
+
+impl SessionTransactionState {
+    fn is_closed(&self) -> bool {
+        matches!(self, Self::Closing { .. } | Self::Closed)
+    }
+
+    fn active_operations(&self) -> usize {
+        match self {
+            Self::OpenIdle | Self::Closed => 0,
+            Self::OpenOperation { active_operations }
+            | Self::OpenTransaction {
+                active_operations, ..
+            }
+            | Self::Closing { active_operations } => *active_operations,
+        }
+    }
+
+    fn has_explicit_transaction(&self) -> bool {
+        matches!(
+            self,
+            Self::OpenTransaction {
+                owner: TransactionOwner::Explicit,
+                ..
+            }
+        )
+    }
+
+    fn begin_operation(&mut self, scope: SessionOperationScope) -> Result<(), LixError> {
+        match self {
+            Self::OpenIdle => {
+                if matches!(scope, SessionOperationScope::Session) {
+                    *self = Self::OpenOperation {
+                        active_operations: 1,
+                    };
+                    Ok(())
+                } else {
+                    Err(active_transaction_error())
+                }
+            }
+            Self::OpenOperation { active_operations } => {
+                if matches!(scope, SessionOperationScope::Session) {
+                    *active_operations += 1;
+                    Ok(())
+                } else {
+                    Err(active_transaction_error())
+                }
+            }
+            Self::OpenTransaction {
+                active_operations,
+                owner,
+            } => match scope {
+                SessionOperationScope::Session => Err(active_transaction_error()),
+                SessionOperationScope::Transaction => {
+                    *active_operations += 1;
+                    Ok(())
+                }
+                SessionOperationScope::TransactionCommit => {
+                    if *owner != TransactionOwner::Explicit {
+                        return Err(active_transaction_error());
+                    }
+                    *owner = TransactionOwner::ExplicitCommitting;
+                    *active_operations += 1;
+                    Ok(())
+                }
+            },
+            Self::Closing { .. } | Self::Closed => Err(closed_error()),
+        }
+    }
+
+    fn begin_write_lease(&mut self, owner: TransactionOwner) -> Result<(), LixError> {
+        match self {
+            Self::OpenIdle => {
+                *self = Self::OpenTransaction {
+                    active_operations: 1,
+                    owner,
+                };
+                Ok(())
+            }
+            Self::OpenOperation { .. } | Self::OpenTransaction { .. } => {
+                Err(active_transaction_error())
+            }
+            Self::Closing { .. } | Self::Closed => Err(closed_error()),
+        }
+    }
+
+    fn mark_explicit_transaction_open(&mut self) -> Result<(), LixError> {
+        match self {
+            Self::OpenTransaction {
+                active_operations: 1,
+                owner,
+            } if *owner == TransactionOwner::ExplicitOpening => {
+                *owner = TransactionOwner::Explicit;
+                Ok(())
+            }
+            Self::Closing { .. } | Self::Closed => Err(closed_error()),
+            _ => {
+                panic!("explicit transaction should be opening before it is marked open");
+            }
+        }
+    }
+
+    fn finish_operation(&mut self) {
+        match self {
+            Self::OpenOperation { active_operations } => {
+                *active_operations = active_operations
+                    .checked_sub(1)
+                    .expect("session operation count should not underflow");
+                if *active_operations == 0 {
+                    *self = Self::OpenIdle;
+                }
+            }
+            Self::OpenTransaction {
+                active_operations, ..
+            } => {
+                *active_operations = active_operations
+                    .checked_sub(1)
+                    .expect("session operation count should not underflow");
+            }
+            Self::Closing { active_operations } => {
+                *active_operations = active_operations
+                    .checked_sub(1)
+                    .expect("session operation count should not underflow");
+                if *active_operations == 0 {
+                    *self = Self::Closed;
+                }
+            }
+            Self::OpenIdle | Self::Closed => {
+                panic!("session operation count should not underflow");
+            }
+        }
+    }
+
+    fn finish_transaction(&mut self) {
+        match self {
+            Self::OpenTransaction {
+                active_operations: 0,
+                ..
+            } => {
+                *self = Self::OpenIdle;
+            }
+            Self::OpenTransaction { .. } => {}
+            Self::Closing { .. } | Self::Closed => {}
+            Self::OpenIdle | Self::OpenOperation { .. } => {
+                panic!("session transaction should be active before it is finished");
+            }
+        }
+    }
+}
+
+#[derive(Clone, Copy)]
+enum SessionOperationScope {
+    Session,
+    Transaction,
+    TransactionCommit,
+}
+
+fn active_transaction_error() -> LixError {
+    transaction_state_error(
+        "Lix handle has an active transaction; use the transaction handle for reads and writes until it is committed or rolled back",
+    )
+}
+
+pub(super) struct SessionWriteLease {
+    _operation_guard: SessionOperationGuard,
+    _transaction_guard: SessionTransactionGuard,
+}
+
+pub(super) struct SessionTransactionGuard {
+    manager: SessionTransactionManager,
+}
+
+impl Drop for SessionTransactionGuard {
+    fn drop(&mut self) {
+        self.manager.finish_transaction();
+    }
+}
+
+pub(super) struct SessionOperationGuard {
+    manager: SessionTransactionManager,
+}
+
+impl Drop for SessionOperationGuard {
+    fn drop(&mut self) {
+        self.manager.finish_operation();
+    }
+}

package/dist-engine-src/src/sql2/bind/classify.rs

@@ -0,0 +1,102 @@
+use datafusion::sql::parser::Statement as DataFusionStatement;
+use datafusion::sql::sqlparser::ast::{Query, SetExpr, Statement as SqlStatement};
+
+use crate::LixError;
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub(crate) enum SqlStatementKind {
+    Read,
+    Write,
+    Other,
+}
+
+pub(crate) fn validate_supported_datafusion_statement_ast(
+    statement: &DataFusionStatement,
+) -> Result<(), LixError> {
+    match statement {
+        DataFusionStatement::Statement(statement) => validate_supported_ast_statement(statement),
+        DataFusionStatement::Explain(explain) => {
+            if classify_datafusion_statement(explain.statement.as_ref()) == SqlStatementKind::Write
+            {
+                return Err(unsupported_sql_error(
+                    "EXPLAIN of write statements is not supported by Lix SQL",
+                ));
+            }
+            validate_supported_datafusion_statement_ast(explain.statement.as_ref())
+        }
+        _ => Err(unsupported_sql_error(format!(
+            "SQL statement is not supported by Lix SQL: {statement}"
+        ))),
+    }
+}
+
+pub(crate) fn classify_datafusion_statement(statement: &DataFusionStatement) -> SqlStatementKind {
+    match statement {
+        DataFusionStatement::Statement(statement) => classify_ast_statement(statement),
+        DataFusionStatement::Explain(explain) => {
+            classify_datafusion_statement(explain.statement.as_ref())
+        }
+        _ => SqlStatementKind::Other,
+    }
+}
+
+fn classify_ast_statement(statement: &SqlStatement) -> SqlStatementKind {
+    match statement {
+        SqlStatement::Insert(_) | SqlStatement::Update(_) | SqlStatement::Delete(_) => {
+            SqlStatementKind::Write
+        }
+        SqlStatement::Query(_) => SqlStatementKind::Read,
+        SqlStatement::Explain { .. } => SqlStatementKind::Read,
+        _ => SqlStatementKind::Other,
+    }
+}
+
+fn validate_supported_ast_statement(statement: &SqlStatement) -> Result<(), LixError> {
+    match statement {
+        SqlStatement::Query(query) => validate_supported_query(query),
+        SqlStatement::Insert(_) | SqlStatement::Update(_) | SqlStatement::Delete(_) => Ok(()),
+        SqlStatement::Explain { statement, .. } => {
+            if classify_ast_statement(statement.as_ref()) == SqlStatementKind::Write {
+                return Err(unsupported_sql_error(
+                    "EXPLAIN of write statements is not supported by Lix SQL",
+                ));
+            }
+            validate_supported_ast_statement(statement)
+        }
+        _ => Err(unsupported_sql_error(format!(
+            "SQL statement is not supported by Lix SQL: {statement}"
+        ))),
+    }
+}
+
+fn validate_supported_query(query: &Query) -> Result<(), LixError> {
+    if query.with.as_ref().is_some_and(|with| with.recursive) {
+        return Err(
+            unsupported_sql_error("recursive CTEs are not supported by Lix SQL").with_hint(
+                "Use explicit commit graph surfaces such as lix_commit, lix_commit_edge, and lix_state_history instead of WITH RECURSIVE.",
+            ),
+        );
+    }
+
+    if let Some(with) = &query.with {
+        for cte in &with.cte_tables {
+            validate_supported_query(&cte.query)?;
+        }
+    }
+    validate_supported_set_expr(&query.body)
+}
+
+fn validate_supported_set_expr(expr: &SetExpr) -> Result<(), LixError> {
+    match expr {
+        SetExpr::Query(query) => validate_supported_query(query),
+        SetExpr::SetOperation { left, right, .. } => {
+            validate_supported_set_expr(left)?;
+            validate_supported_set_expr(right)
+        }
+        _ => Ok(()),
+    }
+}
+
+fn unsupported_sql_error(message: impl Into<String>) -> LixError {
+    LixError::new(LixError::CODE_UNSUPPORTED_SQL, message)
+}

package/dist-engine-src/src/sql2/bind/error.rs

@@ -0,0 +1,5 @@
+use crate::LixError;
+
+pub(crate) fn unsupported(message: impl Into<String>) -> LixError {
+    LixError::new(LixError::CODE_UNSUPPORTED_SQL, message.into())
+}

package/dist-engine-src/src/sql2/bind/expr.rs

@@ -0,0 +1,29 @@
+#[derive(Clone, Debug, Eq, PartialEq)]
+pub(crate) enum BoundExpr {
+    Column(BoundColumnRef),
+    Param(BoundParamRef),
+    Literal(BoundLiteral),
+    Function { name: String, args: Vec<BoundExpr> },
+}
+
+#[derive(Clone, Debug, Eq, PartialEq)]
+pub(crate) enum BoundLiteral {
+    Null,
+    Bool(bool),
+    Integer(i64),
+    Text(String),
+    Json(serde_json::Value),
+    Blob(Vec<u8>),
+}
+
+#[derive(Clone, Debug, Eq, PartialEq, Ord, PartialOrd)]
+pub(crate) struct BoundColumnRef {
+    pub(crate) table: String,
+    pub(crate) column_id: usize,
+    pub(crate) name: String,
+}
+
+#[derive(Clone, Copy, Debug, Eq, PartialEq, Ord, PartialOrd)]
+pub(crate) struct BoundParamRef {
+    pub(crate) index: usize,
+}

package/dist-engine-src/src/sql2/bind/mod.rs

@@ -0,0 +1,12 @@
+pub(crate) mod classify;
+pub(crate) mod error;
+pub(crate) mod expr;
+mod public_udf;
+pub(crate) mod read;
+pub(crate) mod statement;
+pub(crate) mod table;
+pub(crate) mod write;
+
+pub(crate) use public_udf::statement_has_durable_runtime_function;
+pub(crate) use read::{bind_read_statement, bind_statement_route, BoundStatementRoute};
+pub(crate) use statement::bind_statement;