@lix-js/sdk 0.6.0-preview.1 → 0.6.0-preview.3

package/dist-engine-src/src/session/switch_version.rs

@@ -0,0 +1,109 @@
+use std::sync::Arc;
+
+use serde_json::json;
+
+use crate::transaction::types::{TransactionJson, TransactionWriteRow};
+use crate::version::{VersionLifecycle, VersionOperation, VersionReferenceRole};
+use crate::LixError;
+use crate::GLOBAL_VERSION_ID;
+
+use super::context::{SessionContext, SessionMode, WORKSPACE_VERSION_KEY};
+
+const KEY_VALUE_SCHEMA_KEY: &str = "lix_key_value";
+
+/// Options for switching a session to another version.
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct SwitchVersionOptions {
+    pub version_id: String,
+}
+
+/// Receipt returned after switching to another version.
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct SwitchVersionReceipt {
+    pub version_id: String,
+}
+
+impl SessionContext {
+    /// Switches the session's active version selector.
+    ///
+    /// Pinned sessions switch in memory and return a new pinned session.
+    /// Workspace sessions update the shared workspace selector so other
+    /// workspace sessions observe the new active version on their next use.
+    pub async fn switch_version(
+        &self,
+        options: SwitchVersionOptions,
+    ) -> Result<(SessionContext, SwitchVersionReceipt), LixError> {
+        let version_id = options.version_id;
+        let receipt_version_id = version_id.clone();
+        let current_mode = self.mode.clone();
+        let next_mode = self
+            .with_write_transaction(|transaction| {
+                Box::pin(async move {
+                    {
+                        let reader = transaction.version_ref_reader();
+                        VersionLifecycle::new(&reader)
+                            .require_existing_commit_id(
+                                &version_id,
+                                VersionOperation::SwitchVersion,
+                                VersionReferenceRole::Target,
+                            )
+                            .await?
+                    };
+
+                    match current_mode {
+                        SessionMode::Pinned { .. } => Ok(SessionMode::Pinned {
+                            version_id: version_id.clone(),
+                        }),
+                        SessionMode::Workspace => {
+                            transaction
+                                .stage_rows(vec![workspace_version_stage_row(&version_id)?])
+                                .await?;
+                            Ok(SessionMode::Workspace)
+                        }
+                    }
+                })
+            })
+            .await?;
+
+        let session = SessionContext::new_with_closed(
+            next_mode,
+            self.storage.clone(),
+            Arc::clone(&self.live_state),
+            Arc::clone(&self.tracked_state),
+            Arc::clone(&self.binary_cas),
+            Arc::clone(&self.commit_store),
+            Arc::clone(&self.version_ctx),
+            Arc::clone(&self.catalog_context),
+            self.closed_flag(),
+        );
+        Ok((
+            session,
+            SwitchVersionReceipt {
+                version_id: receipt_version_id,
+            },
+        ))
+    }
+}
+
+fn workspace_version_stage_row(version_id: &str) -> Result<TransactionWriteRow, LixError> {
+    Ok(TransactionWriteRow {
+        entity_id: Some(crate::entity_identity::EntityIdentity::single(
+            WORKSPACE_VERSION_KEY,
+        )),
+        schema_key: KEY_VALUE_SCHEMA_KEY.to_string(),
+        file_id: None,
+        snapshot: Some(TransactionJson::from_value_unchecked(json!({
+            "key": WORKSPACE_VERSION_KEY,
+            "value": version_id,
+        }))),
+        metadata: None,
+        origin: None,
+        created_at: None,
+        updated_at: None,
+        global: true,
+        change_id: None,
+        commit_id: None,
+        untracked: true,
+        version_id: GLOBAL_VERSION_ID.to_string(),
+    })
+}

package/dist-engine-src/src/sql2/change_provider.rs

@@ -0,0 +1,331 @@
+use std::any::Any;
+use std::sync::Arc;
+
+use async_trait::async_trait;
+use datafusion::arrow::array::{ArrayRef, StringArray};
+use datafusion::arrow::datatypes::{DataType, Field, Schema, SchemaRef};
+use datafusion::arrow::record_batch::RecordBatch;
+use datafusion::catalog::{Session, TableProvider};
+use datafusion::common::{DataFusionError, Result};
+use datafusion::datasource::TableType;
+use datafusion::execution::TaskContext;
+use datafusion::logical_expr::{Expr, TableProviderFilterPushDown};
+use datafusion::physical_expr::EquivalenceProperties;
+use datafusion::physical_plan::execution_plan::{Boundedness, EmissionType, PlanProperties};
+use datafusion::physical_plan::stream::RecordBatchStreamAdapter;
+use datafusion::physical_plan::{
+    DisplayAs, DisplayFormatType, ExecutionPlan, Partitioning, SendableRecordBatchStream,
+};
+use futures_util::stream;
+
+use crate::commit_store::ChangeScanRequest;
+use crate::serialize_row_metadata;
+use crate::LixError;
+
+use super::record_batch::record_batch_with_row_count;
+use super::result_metadata::json_field;
+use super::SqlCommitStoreQuerySource;
+use crate::commit_store::{materialize_change, MaterializedChange};
+
+pub(crate) async fn register_lix_change_provider(
+    session: &datafusion::prelude::SessionContext,
+    query_source: SqlCommitStoreQuerySource,
+) -> Result<(), LixError> {
+    session
+        .register_table("lix_change", Arc::new(LixChangeProvider::new(query_source)))
+        .map_err(datafusion_error_to_lix_error)?;
+    Ok(())
+}
+
+struct LixChangeProvider {
+    schema: SchemaRef,
+    query_source: SqlCommitStoreQuerySource,
+}
+
+impl std::fmt::Debug for LixChangeProvider {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("LixChangeProvider").finish()
+    }
+}
+
+impl LixChangeProvider {
+    fn new(query_source: SqlCommitStoreQuerySource) -> Self {
+        Self {
+            schema: lix_change_schema(),
+            query_source,
+        }
+    }
+}
+
+#[async_trait]
+impl TableProvider for LixChangeProvider {
+    fn as_any(&self) -> &dyn Any {
+        self
+    }
+
+    fn schema(&self) -> SchemaRef {
+        Arc::clone(&self.schema)
+    }
+
+    fn table_type(&self) -> TableType {
+        TableType::Base
+    }
+
+    fn supports_filters_pushdown(
+        &self,
+        filters: &[&Expr],
+    ) -> Result<Vec<TableProviderFilterPushDown>> {
+        Ok(filters
+            .iter()
+            .map(|_| TableProviderFilterPushDown::Unsupported)
+            .collect())
+    }
+
+    async fn scan(
+        &self,
+        _state: &dyn Session,
+        projection: Option<&Vec<usize>>,
+        _filters: &[Expr],
+        limit: Option<usize>,
+    ) -> Result<Arc<dyn ExecutionPlan>> {
+        Ok(Arc::new(LixChangeScanExec::new(
+            self.query_source.clone(),
+            projected_schema(&self.schema, projection),
+            projection.cloned(),
+            limit,
+        )))
+    }
+}
+
+struct LixChangeScanExec {
+    query_source: SqlCommitStoreQuerySource,
+    schema: SchemaRef,
+    projection: Option<Vec<usize>>,
+    limit: Option<usize>,
+    properties: Arc<PlanProperties>,
+}
+
+impl std::fmt::Debug for LixChangeScanExec {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        f.debug_struct("LixChangeScanExec").finish()
+    }
+}
+
+impl LixChangeScanExec {
+    fn new(
+        query_source: SqlCommitStoreQuerySource,
+        schema: SchemaRef,
+        projection: Option<Vec<usize>>,
+        limit: Option<usize>,
+    ) -> Self {
+        let properties = PlanProperties::new(
+            EquivalenceProperties::new(schema.clone()),
+            Partitioning::UnknownPartitioning(1),
+            EmissionType::Incremental,
+            Boundedness::Bounded,
+        );
+        Self {
+            query_source,
+            schema,
+            projection,
+            limit,
+            properties: Arc::new(properties),
+        }
+    }
+}
+
+impl DisplayAs for LixChangeScanExec {
+    fn fmt_as(&self, t: DisplayFormatType, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        match t {
+            DisplayFormatType::Default | DisplayFormatType::Verbose => {
+                write!(f, "LixChangeScanExec")
+            }
+            DisplayFormatType::TreeRender => write!(f, "LixChangeScanExec"),
+        }
+    }
+}
+
+impl ExecutionPlan for LixChangeScanExec {
+    fn name(&self) -> &str {
+        "LixChangeScanExec"
+    }
+
+    fn as_any(&self) -> &dyn Any {
+        self
+    }
+
+    fn properties(&self) -> &Arc<PlanProperties> {
+        &self.properties
+    }
+
+    fn children(&self) -> Vec<&Arc<dyn ExecutionPlan>> {
+        Vec::new()
+    }
+
+    fn with_new_children(
+        self: Arc<Self>,
+        children: Vec<Arc<dyn ExecutionPlan>>,
+    ) -> Result<Arc<dyn ExecutionPlan>> {
+        if !children.is_empty() {
+            return Err(DataFusionError::Execution(
+                "LixChangeScanExec does not accept children".to_string(),
+            ));
+        }
+        Ok(self)
+    }
+
+    fn execute(
+        &self,
+        partition: usize,
+        _context: Arc<TaskContext>,
+    ) -> Result<SendableRecordBatchStream> {
+        if partition != 0 {
+            return Err(DataFusionError::Execution(format!(
+                "LixChangeScanExec only exposes one partition, got {partition}"
+            )));
+        }
+
+        let query_source = self.query_source.clone();
+        let projection = change_projection_for_scan(self.projection.as_ref());
+        let limit = self.limit;
+        let schema = Arc::clone(&self.schema);
+        let stream = stream::once(async move {
+            let mut json_reader = query_source.json_reader;
+            let canonical_changes = query_source
+                .commit_store_reader
+                .scan_changes(&ChangeScanRequest { limit })
+                .await
+                .map_err(lix_error_to_datafusion_error)?;
+            let mut changes = Vec::with_capacity(canonical_changes.len());
+            for change in canonical_changes {
+                changes.push(
+                    materialize_change(&mut json_reader, change)
+                        .await
+                        .map_err(lix_error_to_datafusion_error)?,
+                );
+            }
+            change_record_batch(&projection, &changes)
+        });
+        Ok(Box::pin(RecordBatchStreamAdapter::new(schema, stream)))
+    }
+}
+
+#[derive(Debug, Clone, Copy)]
+enum ChangeColumn {
+    Id,
+    EntityId,
+    SchemaKey,
+    FileId,
+    Metadata,
+    CreatedAt,
+    SnapshotContent,
+}
+
+fn lix_change_schema() -> SchemaRef {
+    Arc::new(Schema::new(vec![
+        Field::new("id", DataType::Utf8, false),
+        json_field("entity_id", false),
+        Field::new("schema_key", DataType::Utf8, false),
+        Field::new("file_id", DataType::Utf8, true),
+        json_field("metadata", true),
+        Field::new("created_at", DataType::Utf8, false),
+        json_field("snapshot_content", true),
+    ]))
+}
+
+fn change_projection_for_scan(projection: Option<&Vec<usize>>) -> Vec<ChangeColumn> {
+    let all_columns = vec![
+        ChangeColumn::Id,
+        ChangeColumn::EntityId,
+        ChangeColumn::SchemaKey,
+        ChangeColumn::FileId,
+        ChangeColumn::Metadata,
+        ChangeColumn::CreatedAt,
+        ChangeColumn::SnapshotContent,
+    ];
+    projection.map_or(all_columns.clone(), |indices| {
+        indices
+            .iter()
+            .filter_map(|index| all_columns.get(*index).copied())
+            .collect()
+    })
+}
+
+fn projected_schema(schema: &SchemaRef, projection: Option<&Vec<usize>>) -> SchemaRef {
+    match projection {
+        Some(projection) => Arc::new(schema.project(projection).expect("projection is valid")),
+        None => Arc::clone(schema),
+    }
+}
+
+fn change_record_batch(
+    projection: &[ChangeColumn],
+    changes: &[MaterializedChange],
+) -> Result<RecordBatch> {
+    let arrays = projection
+        .iter()
+        .map(|column| match column {
+            ChangeColumn::Id => string_array(changes.iter().map(|row| Some(row.id.as_str()))),
+            ChangeColumn::EntityId => Arc::new(StringArray::from(
+                changes
+                    .iter()
+                    .map(|row| {
+                        Some(
+                            row.entity_id
+                                .as_json_array_text()
+                                .expect("canonical change entity identity should project"),
+                        )
+                    })
+                    .collect::<Vec<_>>(),
+            )) as ArrayRef,
+            ChangeColumn::SchemaKey => {
+                string_array(changes.iter().map(|row| Some(row.schema_key.as_str())))
+            }
+            ChangeColumn::FileId => string_array(changes.iter().map(|row| row.file_id.as_deref())),
+            ChangeColumn::Metadata => Arc::new(StringArray::from(
+                changes
+                    .iter()
+                    .map(|row| row.metadata.as_ref().map(serialize_row_metadata))
+                    .collect::<Vec<_>>(),
+            )),
+            ChangeColumn::CreatedAt => {
+                string_array(changes.iter().map(|row| Some(row.created_at.as_str())))
+            }
+            ChangeColumn::SnapshotContent => {
+                string_array(changes.iter().map(|row| row.snapshot_content.as_deref()))
+            }
+        })
+        .collect::<Vec<_>>();
+    record_batch_with_row_count(change_schema(projection), arrays, changes.len()).map_err(|error| {
+        DataFusionError::Execution(format!("failed to build lix_change batch: {error}"))
+    })
+}
+
+fn change_schema(projection: &[ChangeColumn]) -> SchemaRef {
+    Arc::new(Schema::new(
+        projection
+            .iter()
+            .map(|column| match column {
+                ChangeColumn::Id => Field::new("id", DataType::Utf8, false),
+                ChangeColumn::EntityId => json_field("entity_id", false),
+                ChangeColumn::SchemaKey => Field::new("schema_key", DataType::Utf8, false),
+                ChangeColumn::FileId => Field::new("file_id", DataType::Utf8, true),
+                ChangeColumn::Metadata => json_field("metadata", true),
+                ChangeColumn::CreatedAt => Field::new("created_at", DataType::Utf8, false),
+                ChangeColumn::SnapshotContent => json_field("snapshot_content", true),
+            })
+            .collect::<Vec<_>>(),
+    ))
+}
+
+fn string_array<'a>(values: impl Iterator<Item = Option<&'a str>>) -> ArrayRef {
+    Arc::new(StringArray::from(values.collect::<Vec<_>>())) as ArrayRef
+}
+
+fn datafusion_error_to_lix_error(error: DataFusionError) -> LixError {
+    super::error::datafusion_error_to_lix_error(error)
+}
+
+fn lix_error_to_datafusion_error(error: LixError) -> DataFusionError {
+    super::error::lix_error_to_datafusion_error(error)
+}

package/dist-engine-src/src/sql2/classify.rs

@@ -0,0 +1,182 @@
+use datafusion::sql::parser::Statement as DataFusionStatement;
+use datafusion::sql::sqlparser::ast::{
+    FromTable, ObjectName, Query, SetExpr, Statement as SqlStatement, TableFactor, TableObject,
+    TableWithJoins,
+};
+use datafusion::sql::sqlparser::dialect::GenericDialect;
+use datafusion::sql::sqlparser::parser::Parser;
+
+use crate::LixError;
+
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub(crate) enum SqlStatementKind {
+    Read,
+    Write,
+    Other,
+}
+
+pub(crate) fn classify_statement(sql: &str) -> Result<SqlStatementKind, LixError> {
+    let statements = parse_sql_statements(sql)?;
+    let [statement] = statements.as_slice() else {
+        return Ok(SqlStatementKind::Other);
+    };
+    Ok(classify_ast_statement(statement))
+}
+
+pub(crate) fn validate_supported_statement_ast(sql: &str) -> Result<(), LixError> {
+    let statements = parse_sql_statements(sql)?;
+    let [statement] = statements.as_slice() else {
+        return Err(unsupported_sql_error(
+            "Lix SQL only supports one statement per execute() call",
+        ));
+    };
+    validate_supported_ast_statement(statement)
+}
+
+pub(crate) fn validate_supported_datafusion_statement_ast(
+    statement: &DataFusionStatement,
+) -> Result<(), LixError> {
+    match statement {
+        DataFusionStatement::Statement(statement) => validate_supported_ast_statement(statement),
+        DataFusionStatement::Explain(explain) => {
+            validate_supported_datafusion_statement_ast(explain.statement.as_ref())
+        }
+        _ => Err(unsupported_sql_error(format!(
+            "SQL statement is not supported by Lix SQL: {statement}"
+        ))),
+    }
+}
+
+pub(crate) fn datafusion_statement_dml_target_table_names(
+    statement: &DataFusionStatement,
+) -> Vec<String> {
+    let mut targets = Vec::new();
+    collect_datafusion_statement_dml_target_table_names(statement, &mut targets);
+    targets
+}
+
+fn parse_sql_statements(sql: &str) -> Result<Vec<SqlStatement>, LixError> {
+    Parser::parse_sql(&GenericDialect {}, sql).map_err(|error| {
+        LixError::new(
+            LixError::CODE_PARSE_ERROR,
+            format!("sql2 SQL parse error: {error}"),
+        )
+    })
+}
+
+fn collect_datafusion_statement_dml_target_table_names(
+    statement: &DataFusionStatement,
+    targets: &mut Vec<String>,
+) {
+    match statement {
+        DataFusionStatement::Statement(statement) => {
+            collect_dml_target_table_names(statement, targets);
+        }
+        DataFusionStatement::Explain(explain) => {
+            collect_datafusion_statement_dml_target_table_names(
+                explain.statement.as_ref(),
+                targets,
+            );
+        }
+        _ => {}
+    }
+}
+
+fn collect_dml_target_table_names(statement: &SqlStatement, targets: &mut Vec<String>) {
+    match statement {
+        SqlStatement::Insert(insert) => {
+            if let TableObject::TableName(name) = &insert.table {
+                if let Some(table_name) = object_name_table_part(name) {
+                    targets.push(table_name);
+                }
+            }
+        }
+        SqlStatement::Update(update) => {
+            collect_table_with_joins_target(&update.table, targets);
+        }
+        SqlStatement::Delete(delete) => {
+            let tables = match &delete.from {
+                FromTable::WithFromKeyword(tables) | FromTable::WithoutKeyword(tables) => tables,
+            };
+            for table in tables {
+                collect_table_with_joins_target(table, targets);
+            }
+        }
+        SqlStatement::Explain { statement, .. } => {
+            collect_dml_target_table_names(statement.as_ref(), targets);
+        }
+        _ => {}
+    }
+}
+
+fn collect_table_with_joins_target(table: &TableWithJoins, targets: &mut Vec<String>) {
+    if let TableFactor::Table { name, .. } = &table.relation {
+        if let Some(table_name) = object_name_table_part(name) {
+            targets.push(table_name);
+        }
+    }
+}
+
+fn object_name_table_part(name: &ObjectName) -> Option<String> {
+    name.0.last().and_then(|part| part.as_ident()).map(|ident| {
+        if ident.quote_style.is_some() {
+            ident.value.clone()
+        } else {
+            ident.value.to_ascii_lowercase()
+        }
+    })
+}
+
+fn classify_ast_statement(statement: &SqlStatement) -> SqlStatementKind {
+    match statement {
+        SqlStatement::Insert(_) | SqlStatement::Update(_) | SqlStatement::Delete(_) => {
+            SqlStatementKind::Write
+        }
+        SqlStatement::Query(_) => SqlStatementKind::Read,
+        SqlStatement::Explain { statement, .. } => classify_ast_statement(statement.as_ref()),
+        _ => SqlStatementKind::Other,
+    }
+}
+
+fn validate_supported_ast_statement(statement: &SqlStatement) -> Result<(), LixError> {
+    match statement {
+        SqlStatement::Query(query) => validate_supported_query(query),
+        SqlStatement::Insert(_) | SqlStatement::Update(_) | SqlStatement::Delete(_) => Ok(()),
+        SqlStatement::Explain { statement, .. } => validate_supported_ast_statement(statement),
+        _ => Err(unsupported_sql_error(format!(
+            "SQL statement is not supported by Lix SQL: {statement}"
+        ))),
+    }
+}
+
+fn validate_supported_query(query: &Query) -> Result<(), LixError> {
+    if query.with.as_ref().is_some_and(|with| with.recursive) {
+        return Err(
+            unsupported_sql_error("recursive CTEs are not supported by Lix SQL").with_hint(
+                "Use explicit commit graph surfaces such as lix_commit, lix_commit_edge, and lix_state_history instead of WITH RECURSIVE.",
+            ),
+        );
+    }
+
+    if let Some(with) = &query.with {
+        for cte in &with.cte_tables {
+            validate_supported_query(&cte.query)?;
+        }
+    }
+    validate_supported_set_expr(&query.body)
+}
+
+fn validate_supported_set_expr(expr: &SetExpr) -> Result<(), LixError> {
+    match expr {
+        SetExpr::Query(query) => validate_supported_query(query),
+        SetExpr::SetOperation { left, right, .. } => {
+            validate_supported_set_expr(left)?;
+            validate_supported_set_expr(right)
+        }
+        _ => Ok(()),
+    }
+}
+
+fn unsupported_sql_error(message: impl Into<String>) -> LixError {
+    LixError::new(LixError::CODE_UNSUPPORTED_SQL, message)
+}