@lix-js/sdk 0.6.0-preview.4 → 0.6.0

package/dist-engine-src/src/session/switch_version.rs

@@ -1,110 +0,0 @@
-use std::sync::Arc;
-
-use serde_json::json;
-
-use crate::transaction::types::{TransactionJson, TransactionWriteRow};
-use crate::version::{VersionLifecycle, VersionOperation, VersionReferenceRole};
-use crate::LixError;
-use crate::GLOBAL_VERSION_ID;
-
-use super::context::{SessionContext, SessionMode, WORKSPACE_VERSION_KEY};
-
-const KEY_VALUE_SCHEMA_KEY: &str = "lix_key_value";
-
-/// Options for switching a session to another version.
-#[derive(Debug, Clone, PartialEq, Eq)]
-pub struct SwitchVersionOptions {
-    pub version_id: String,
-}
-
-/// Receipt returned after switching to another version.
-#[derive(Debug, Clone, PartialEq, Eq)]
-pub struct SwitchVersionReceipt {
-    pub version_id: String,
-}
-
-impl SessionContext {
-    /// Switches the session's active version selector.
-    ///
-    /// Pinned sessions switch in memory and return a new pinned session.
-    /// Workspace sessions update the shared workspace selector so other
-    /// workspace sessions observe the new active version on their next use.
-    pub async fn switch_version(
-        &self,
-        options: SwitchVersionOptions,
-    ) -> Result<(SessionContext, SwitchVersionReceipt), LixError> {
-        let version_id = options.version_id;
-        let receipt_version_id = version_id.clone();
-        let current_mode = self.mode.clone();
-        let next_mode = self
-            .with_write_transaction(|transaction| {
-                Box::pin(async move {
-                    {
-                        let reader = transaction.version_ref_reader();
-                        VersionLifecycle::new(&reader)
-                            .require_existing_commit_id(
-                                &version_id,
-                                VersionOperation::SwitchVersion,
-                                VersionReferenceRole::Target,
-                            )
-                            .await?
-                    };
-
-                    match current_mode {
-                        SessionMode::Pinned { .. } => Ok(SessionMode::Pinned {
-                            version_id: version_id.clone(),
-                        }),
-                        SessionMode::Workspace => {
-                            transaction
-                                .stage_rows(vec![workspace_version_stage_row(&version_id)?])
-                                .await?;
-                            Ok(SessionMode::Workspace)
-                        }
-                    }
-                })
-            })
-            .await?;
-
-        let session = SessionContext::new_with_closed(
-            next_mode,
-            self.storage.clone(),
-            Arc::clone(&self.live_state),
-            Arc::clone(&self.tracked_state),
-            Arc::clone(&self.binary_cas),
-            Arc::clone(&self.commit_store),
-            Arc::clone(&self.version_ctx),
-            Arc::clone(&self.catalog_context),
-            self.closed_flag(),
-            self.active_transaction_flag(),
-        );
-        Ok((
-            session,
-            SwitchVersionReceipt {
-                version_id: receipt_version_id,
-            },
-        ))
-    }
-}
-
-fn workspace_version_stage_row(version_id: &str) -> Result<TransactionWriteRow, LixError> {
-    Ok(TransactionWriteRow {
-        entity_id: Some(crate::entity_identity::EntityIdentity::single(
-            WORKSPACE_VERSION_KEY,
-        )),
-        schema_key: KEY_VALUE_SCHEMA_KEY.to_string(),
-        file_id: None,
-        snapshot: Some(TransactionJson::from_value_unchecked(json!({
-            "key": WORKSPACE_VERSION_KEY,
-            "value": version_id,
-        }))),
-        metadata: None,
-        origin: None,
-        created_at: None,
-        updated_at: None,
-        global: true,
-        change_id: None,
-        commit_id: None,
-        untracked: true,
-        version_id: GLOBAL_VERSION_ID.to_string(),
-    })
-}

package/dist-engine-src/src/session/transaction.rs

@@ -1,76 +0,0 @@
-use std::sync::Arc;
-
-use crate::functions::FunctionContext;
-use crate::transaction::{open_transaction, Transaction};
-use crate::LixError;
-
-use super::context::SessionTransactionGuard;
-use super::SessionContext;
-
-pub struct SessionTransaction {
-    pub(super) transaction: Option<Transaction>,
-    pub(super) runtime_functions: FunctionContext,
-    _transaction_guard: SessionTransactionGuard,
-}
-
-impl SessionContext {
-    pub async fn begin_transaction(&self) -> Result<SessionTransaction, LixError> {
-        self.ensure_open()?;
-        let transaction_guard = self.reserve_session_transaction()?;
-        let opened = match open_transaction(
-            &self.mode,
-            self.storage.clone(),
-            Arc::clone(&self.live_state),
-            Arc::clone(&self.tracked_state),
-            Arc::clone(&self.binary_cas),
-            Arc::clone(&self.commit_store),
-            Arc::clone(&self.version_ctx),
-            Arc::clone(&self.catalog_context),
-        )
-        .await
-        {
-            Ok(opened) => opened,
-            Err(error) => {
-                return Err(error);
-            }
-        };
-        Ok(SessionTransaction {
-            transaction: Some(opened.transaction),
-            runtime_functions: opened.runtime_functions,
-            _transaction_guard: transaction_guard,
-        })
-    }
-}
-
-impl SessionTransaction {
-    pub(super) fn transaction_mut(&mut self) -> Result<&mut Transaction, LixError> {
-        self.transaction
-            .as_mut()
-            .ok_or_else(|| transaction_state_error("Lix transaction is closed"))
-    }
-
-    pub async fn commit(mut self) -> Result<(), LixError> {
-        let transaction = self
-            .transaction
-            .take()
-            .ok_or_else(|| transaction_state_error("Lix transaction is closed"))?;
-        let result = transaction
-            .commit(&self.runtime_functions)
-            .await
-            .map(|_| ());
-        result
-    }
-
-    pub async fn rollback(mut self) -> Result<(), LixError> {
-        let transaction = self
-            .transaction
-            .take()
-            .ok_or_else(|| transaction_state_error("Lix transaction is closed"))?;
-        let result = transaction.rollback().await;
-        result
-    }
-}
-
-pub(crate) fn transaction_state_error(message: impl Into<String>) -> LixError {
-    LixError::new("LIX_INVALID_TRANSACTION_STATE", message)
-}

package/dist-engine-src/src/sql2/change_provider.rs

@@ -1,331 +0,0 @@
-use std::any::Any;
-use std::sync::Arc;
-
-use async_trait::async_trait;
-use datafusion::arrow::array::{ArrayRef, StringArray};
-use datafusion::arrow::datatypes::{DataType, Field, Schema, SchemaRef};
-use datafusion::arrow::record_batch::RecordBatch;
-use datafusion::catalog::{Session, TableProvider};
-use datafusion::common::{DataFusionError, Result};
-use datafusion::datasource::TableType;
-use datafusion::execution::TaskContext;
-use datafusion::logical_expr::{Expr, TableProviderFilterPushDown};
-use datafusion::physical_expr::EquivalenceProperties;
-use datafusion::physical_plan::execution_plan::{Boundedness, EmissionType, PlanProperties};
-use datafusion::physical_plan::stream::RecordBatchStreamAdapter;
-use datafusion::physical_plan::{
-    DisplayAs, DisplayFormatType, ExecutionPlan, Partitioning, SendableRecordBatchStream,
-};
-use futures_util::stream;
-
-use crate::commit_store::ChangeScanRequest;
-use crate::serialize_row_metadata;
-use crate::LixError;
-
-use super::record_batch::record_batch_with_row_count;
-use super::result_metadata::json_field;
-use super::SqlCommitStoreQuerySource;
-use crate::commit_store::{materialize_change, MaterializedChange};
-
-pub(crate) async fn register_lix_change_provider(
-    session: &datafusion::prelude::SessionContext,
-    query_source: SqlCommitStoreQuerySource,
-) -> Result<(), LixError> {
-    session
-        .register_table("lix_change", Arc::new(LixChangeProvider::new(query_source)))
-        .map_err(datafusion_error_to_lix_error)?;
-    Ok(())
-}
-
-struct LixChangeProvider {
-    schema: SchemaRef,
-    query_source: SqlCommitStoreQuerySource,
-}
-
-impl std::fmt::Debug for LixChangeProvider {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        f.debug_struct("LixChangeProvider").finish()
-    }
-}
-
-impl LixChangeProvider {
-    fn new(query_source: SqlCommitStoreQuerySource) -> Self {
-        Self {
-            schema: lix_change_schema(),
-            query_source,
-        }
-    }
-}
-
-#[async_trait]
-impl TableProvider for LixChangeProvider {
-    fn as_any(&self) -> &dyn Any {
-        self
-    }
-
-    fn schema(&self) -> SchemaRef {
-        Arc::clone(&self.schema)
-    }
-
-    fn table_type(&self) -> TableType {
-        TableType::Base
-    }
-
-    fn supports_filters_pushdown(
-        &self,
-        filters: &[&Expr],
-    ) -> Result<Vec<TableProviderFilterPushDown>> {
-        Ok(filters
-            .iter()
-            .map(|_| TableProviderFilterPushDown::Unsupported)
-            .collect())
-    }
-
-    async fn scan(
-        &self,
-        _state: &dyn Session,
-        projection: Option<&Vec<usize>>,
-        _filters: &[Expr],
-        limit: Option<usize>,
-    ) -> Result<Arc<dyn ExecutionPlan>> {
-        Ok(Arc::new(LixChangeScanExec::new(
-            self.query_source.clone(),
-            projected_schema(&self.schema, projection),
-            projection.cloned(),
-            limit,
-        )))
-    }
-}
-
-struct LixChangeScanExec {
-    query_source: SqlCommitStoreQuerySource,
-    schema: SchemaRef,
-    projection: Option<Vec<usize>>,
-    limit: Option<usize>,
-    properties: Arc<PlanProperties>,
-}
-
-impl std::fmt::Debug for LixChangeScanExec {
-    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        f.debug_struct("LixChangeScanExec").finish()
-    }
-}
-
-impl LixChangeScanExec {
-    fn new(
-        query_source: SqlCommitStoreQuerySource,
-        schema: SchemaRef,
-        projection: Option<Vec<usize>>,
-        limit: Option<usize>,
-    ) -> Self {
-        let properties = PlanProperties::new(
-            EquivalenceProperties::new(schema.clone()),
-            Partitioning::UnknownPartitioning(1),
-            EmissionType::Incremental,
-            Boundedness::Bounded,
-        );
-        Self {
-            query_source,
-            schema,
-            projection,
-            limit,
-            properties: Arc::new(properties),
-        }
-    }
-}
-
-impl DisplayAs for LixChangeScanExec {
-    fn fmt_as(&self, t: DisplayFormatType, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
-        match t {
-            DisplayFormatType::Default | DisplayFormatType::Verbose => {
-                write!(f, "LixChangeScanExec")
-            }
-            DisplayFormatType::TreeRender => write!(f, "LixChangeScanExec"),
-        }
-    }
-}
-
-impl ExecutionPlan for LixChangeScanExec {
-    fn name(&self) -> &str {
-        "LixChangeScanExec"
-    }
-
-    fn as_any(&self) -> &dyn Any {
-        self
-    }
-
-    fn properties(&self) -> &Arc<PlanProperties> {
-        &self.properties
-    }
-
-    fn children(&self) -> Vec<&Arc<dyn ExecutionPlan>> {
-        Vec::new()
-    }
-
-    fn with_new_children(
-        self: Arc<Self>,
-        children: Vec<Arc<dyn ExecutionPlan>>,
-    ) -> Result<Arc<dyn ExecutionPlan>> {
-        if !children.is_empty() {
-            return Err(DataFusionError::Execution(
-                "LixChangeScanExec does not accept children".to_string(),
-            ));
-        }
-        Ok(self)
-    }
-
-    fn execute(
-        &self,
-        partition: usize,
-        _context: Arc<TaskContext>,
-    ) -> Result<SendableRecordBatchStream> {
-        if partition != 0 {
-            return Err(DataFusionError::Execution(format!(
-                "LixChangeScanExec only exposes one partition, got {partition}"
-            )));
-        }
-
-        let query_source = self.query_source.clone();
-        let projection = change_projection_for_scan(self.projection.as_ref());
-        let limit = self.limit;
-        let schema = Arc::clone(&self.schema);
-        let stream = stream::once(async move {
-            let mut json_reader = query_source.json_reader;
-            let canonical_changes = query_source
-                .commit_store_reader
-                .scan_changes(&ChangeScanRequest { limit })
-                .await
-                .map_err(lix_error_to_datafusion_error)?;
-            let mut changes = Vec::with_capacity(canonical_changes.len());
-            for change in canonical_changes {
-                changes.push(
-                    materialize_change(&mut json_reader, change)
-                        .await
-                        .map_err(lix_error_to_datafusion_error)?,
-                );
-            }
-            change_record_batch(&projection, &changes)
-        });
-        Ok(Box::pin(RecordBatchStreamAdapter::new(schema, stream)))
-    }
-}
-
-#[derive(Debug, Clone, Copy)]
-enum ChangeColumn {
-    Id,
-    EntityId,
-    SchemaKey,
-    FileId,
-    Metadata,
-    CreatedAt,
-    SnapshotContent,
-}
-
-fn lix_change_schema() -> SchemaRef {
-    Arc::new(Schema::new(vec![
-        Field::new("id", DataType::Utf8, false),
-        json_field("entity_id", false),
-        Field::new("schema_key", DataType::Utf8, false),
-        Field::new("file_id", DataType::Utf8, true),
-        json_field("metadata", true),
-        Field::new("created_at", DataType::Utf8, false),
-        json_field("snapshot_content", true),
-    ]))
-}
-
-fn change_projection_for_scan(projection: Option<&Vec<usize>>) -> Vec<ChangeColumn> {
-    let all_columns = vec![
-        ChangeColumn::Id,
-        ChangeColumn::EntityId,
-        ChangeColumn::SchemaKey,
-        ChangeColumn::FileId,
-        ChangeColumn::Metadata,
-        ChangeColumn::CreatedAt,
-        ChangeColumn::SnapshotContent,
-    ];
-    projection.map_or(all_columns.clone(), |indices| {
-        indices
-            .iter()
-            .filter_map(|index| all_columns.get(*index).copied())
-            .collect()
-    })
-}
-
-fn projected_schema(schema: &SchemaRef, projection: Option<&Vec<usize>>) -> SchemaRef {
-    match projection {
-        Some(projection) => Arc::new(schema.project(projection).expect("projection is valid")),
-        None => Arc::clone(schema),
-    }
-}
-
-fn change_record_batch(
-    projection: &[ChangeColumn],
-    changes: &[MaterializedChange],
-) -> Result<RecordBatch> {
-    let arrays = projection
-        .iter()
-        .map(|column| match column {
-            ChangeColumn::Id => string_array(changes.iter().map(|row| Some(row.id.as_str()))),
-            ChangeColumn::EntityId => Arc::new(StringArray::from(
-                changes
-                    .iter()
-                    .map(|row| {
-                        Some(
-                            row.entity_id
-                                .as_json_array_text()
-                                .expect("canonical change entity identity should project"),
-                        )
-                    })
-                    .collect::<Vec<_>>(),
-            )) as ArrayRef,
-            ChangeColumn::SchemaKey => {
-                string_array(changes.iter().map(|row| Some(row.schema_key.as_str())))
-            }
-            ChangeColumn::FileId => string_array(changes.iter().map(|row| row.file_id.as_deref())),
-            ChangeColumn::Metadata => Arc::new(StringArray::from(
-                changes
-                    .iter()
-                    .map(|row| row.metadata.as_ref().map(serialize_row_metadata))
-                    .collect::<Vec<_>>(),
-            )),
-            ChangeColumn::CreatedAt => {
-                string_array(changes.iter().map(|row| Some(row.created_at.as_str())))
-            }
-            ChangeColumn::SnapshotContent => {
-                string_array(changes.iter().map(|row| row.snapshot_content.as_deref()))
-            }
-        })
-        .collect::<Vec<_>>();
-    record_batch_with_row_count(change_schema(projection), arrays, changes.len()).map_err(|error| {
-        DataFusionError::Execution(format!("failed to build lix_change batch: {error}"))
-    })
-}
-
-fn change_schema(projection: &[ChangeColumn]) -> SchemaRef {
-    Arc::new(Schema::new(
-        projection
-            .iter()
-            .map(|column| match column {
-                ChangeColumn::Id => Field::new("id", DataType::Utf8, false),
-                ChangeColumn::EntityId => json_field("entity_id", false),
-                ChangeColumn::SchemaKey => Field::new("schema_key", DataType::Utf8, false),
-                ChangeColumn::FileId => Field::new("file_id", DataType::Utf8, true),
-                ChangeColumn::Metadata => json_field("metadata", true),
-                ChangeColumn::CreatedAt => Field::new("created_at", DataType::Utf8, false),
-                ChangeColumn::SnapshotContent => json_field("snapshot_content", true),
-            })
-            .collect::<Vec<_>>(),
-    ))
-}
-
-fn string_array<'a>(values: impl Iterator<Item = Option<&'a str>>) -> ArrayRef {
-    Arc::new(StringArray::from(values.collect::<Vec<_>>())) as ArrayRef
-}
-
-fn datafusion_error_to_lix_error(error: DataFusionError) -> LixError {
-    super::error::datafusion_error_to_lix_error(error)
-}
-
-fn lix_error_to_datafusion_error(error: LixError) -> DataFusionError {
-    super::error::lix_error_to_datafusion_error(error)
-}

package/dist-engine-src/src/sql2/classify.rs

@@ -1,174 +0,0 @@
-use datafusion::sql::parser::Statement as DataFusionStatement;
-use datafusion::sql::sqlparser::ast::{
-    FromTable, ObjectName, Query, SetExpr, Statement as SqlStatement, TableFactor, TableObject,
-    TableWithJoins,
-};
-
-use crate::LixError;
-
-#[derive(Debug, Clone, Copy, PartialEq, Eq)]
-pub(crate) enum SqlStatementKind {
-    Read,
-    Write,
-    Other,
-}
-
-pub(crate) fn validate_supported_datafusion_statement_ast(
-    statement: &DataFusionStatement,
-) -> Result<(), LixError> {
-    match statement {
-        DataFusionStatement::Statement(statement) => validate_supported_ast_statement(statement),
-        DataFusionStatement::Explain(explain) => {
-            if classify_datafusion_statement(explain.statement.as_ref()) == SqlStatementKind::Write
-            {
-                return Err(unsupported_sql_error(
-                    "EXPLAIN of write statements is not supported by Lix SQL",
-                ));
-            }
-            validate_supported_datafusion_statement_ast(explain.statement.as_ref())
-        }
-        _ => Err(unsupported_sql_error(format!(
-            "SQL statement is not supported by Lix SQL: {statement}"
-        ))),
-    }
-}
-
-pub(crate) fn classify_datafusion_statement(statement: &DataFusionStatement) -> SqlStatementKind {
-    match statement {
-        DataFusionStatement::Statement(statement) => classify_ast_statement(statement),
-        DataFusionStatement::Explain(_) => SqlStatementKind::Read,
-        _ => SqlStatementKind::Other,
-    }
-}
-
-pub(crate) fn datafusion_statement_dml_target_table_names(
-    statement: &DataFusionStatement,
-) -> Vec<String> {
-    let mut targets = Vec::new();
-    collect_datafusion_statement_dml_target_table_names(statement, &mut targets);
-    targets
-}
-
-fn collect_datafusion_statement_dml_target_table_names(
-    statement: &DataFusionStatement,
-    targets: &mut Vec<String>,
-) {
-    match statement {
-        DataFusionStatement::Statement(statement) => {
-            collect_dml_target_table_names(statement, targets);
-        }
-        DataFusionStatement::Explain(explain) => {
-            collect_datafusion_statement_dml_target_table_names(
-                explain.statement.as_ref(),
-                targets,
-            );
-        }
-        _ => {}
-    }
-}
-
-fn collect_dml_target_table_names(statement: &SqlStatement, targets: &mut Vec<String>) {
-    match statement {
-        SqlStatement::Insert(insert) => {
-            if let TableObject::TableName(name) = &insert.table {
-                if let Some(table_name) = object_name_table_part(name) {
-                    targets.push(table_name);
-                }
-            }
-        }
-        SqlStatement::Update(update) => {
-            collect_table_with_joins_target(&update.table, targets);
-        }
-        SqlStatement::Delete(delete) => {
-            let tables = match &delete.from {
-                FromTable::WithFromKeyword(tables) | FromTable::WithoutKeyword(tables) => tables,
-            };
-            for table in tables {
-                collect_table_with_joins_target(table, targets);
-            }
-        }
-        SqlStatement::Explain { statement, .. } => {
-            collect_dml_target_table_names(statement.as_ref(), targets);
-        }
-        _ => {}
-    }
-}
-
-fn collect_table_with_joins_target(table: &TableWithJoins, targets: &mut Vec<String>) {
-    if let TableFactor::Table { name, .. } = &table.relation {
-        if let Some(table_name) = object_name_table_part(name) {
-            targets.push(table_name);
-        }
-    }
-}
-
-fn object_name_table_part(name: &ObjectName) -> Option<String> {
-    name.0.last().and_then(|part| part.as_ident()).map(|ident| {
-        if ident.quote_style.is_some() {
-            ident.value.clone()
-        } else {
-            ident.value.to_ascii_lowercase()
-        }
-    })
-}
-
-fn classify_ast_statement(statement: &SqlStatement) -> SqlStatementKind {
-    match statement {
-        SqlStatement::Insert(_) | SqlStatement::Update(_) | SqlStatement::Delete(_) => {
-            SqlStatementKind::Write
-        }
-        SqlStatement::Query(_) => SqlStatementKind::Read,
-        SqlStatement::Explain { .. } => SqlStatementKind::Read,
-        _ => SqlStatementKind::Other,
-    }
-}
-
-fn validate_supported_ast_statement(statement: &SqlStatement) -> Result<(), LixError> {
-    match statement {
-        SqlStatement::Query(query) => validate_supported_query(query),
-        SqlStatement::Insert(_) | SqlStatement::Update(_) | SqlStatement::Delete(_) => Ok(()),
-        SqlStatement::Explain { statement, .. } => {
-            if classify_ast_statement(statement.as_ref()) == SqlStatementKind::Write {
-                return Err(unsupported_sql_error(
-                    "EXPLAIN of write statements is not supported by Lix SQL",
-                ));
-            }
-            validate_supported_ast_statement(statement)
-        }
-        _ => Err(unsupported_sql_error(format!(
-            "SQL statement is not supported by Lix SQL: {statement}"
-        ))),
-    }
-}
-
-fn validate_supported_query(query: &Query) -> Result<(), LixError> {
-    if query.with.as_ref().is_some_and(|with| with.recursive) {
-        return Err(
-            unsupported_sql_error("recursive CTEs are not supported by Lix SQL").with_hint(
-                "Use explicit commit graph surfaces such as lix_commit, lix_commit_edge, and lix_state_history instead of WITH RECURSIVE.",
-            ),
-        );
-    }
-
-    if let Some(with) = &query.with {
-        for cte in &with.cte_tables {
-            validate_supported_query(&cte.query)?;
-        }
-    }
-    validate_supported_set_expr(&query.body)
-}
-
-fn validate_supported_set_expr(expr: &SetExpr) -> Result<(), LixError> {
-    match expr {
-        SetExpr::Query(query) => validate_supported_query(query),
-        SetExpr::SetOperation { left, right, .. } => {
-            validate_supported_set_expr(left)?;
-            validate_supported_set_expr(right)
-        }
-        _ => Ok(()),
-    }
-}
-
-fn unsupported_sql_error(message: impl Into<String>) -> LixError {
-    LixError::new(LixError::CODE_UNSUPPORTED_SQL, message)
-}