@lix-js/sdk 0.6.0-preview.2 → 0.6.0-preview.4

package/dist-engine-src/src/sql2/classify.rs

@@ -1,8 +1,8 @@
+use datafusion::sql::parser::Statement as DataFusionStatement;
 use datafusion::sql::sqlparser::ast::{
-    FromTable, ObjectName, Query, SetExpr, Statement, TableFactor, TableObject, TableWithJoins,
+    FromTable, ObjectName, Query, SetExpr, Statement as SqlStatement, TableFactor, TableObject,
+    TableWithJoins,
 };
-use datafusion::sql::sqlparser::dialect::GenericDialect;
-use datafusion::sql::sqlparser::parser::Parser;
 
 use crate::LixError;
 
@@ -13,56 +13,73 @@ pub(crate) enum SqlStatementKind {
     Other,
 }
 
-pub(crate) fn classify_statement(sql: &str) -> Result<SqlStatementKind, LixError> {
-    let statements = parse_sql_statements(sql)?;
-    let [statement] = statements.as_slice() else {
-        return Ok(SqlStatementKind::Other);
-    };
-    Ok(classify_ast_statement(statement))
+pub(crate) fn validate_supported_datafusion_statement_ast(
+    statement: &DataFusionStatement,
+) -> Result<(), LixError> {
+    match statement {
+        DataFusionStatement::Statement(statement) => validate_supported_ast_statement(statement),
+        DataFusionStatement::Explain(explain) => {
+            if classify_datafusion_statement(explain.statement.as_ref()) == SqlStatementKind::Write
+            {
+                return Err(unsupported_sql_error(
+                    "EXPLAIN of write statements is not supported by Lix SQL",
+                ));
+            }
+            validate_supported_datafusion_statement_ast(explain.statement.as_ref())
+        }
+        _ => Err(unsupported_sql_error(format!(
+            "SQL statement is not supported by Lix SQL: {statement}"
+        ))),
+    }
 }
 
-pub(crate) fn validate_supported_statement_ast(sql: &str) -> Result<(), LixError> {
-    let statements = parse_sql_statements(sql)?;
-    let [statement] = statements.as_slice() else {
-        return Err(unsupported_sql_error(
-            "Lix SQL only supports one statement per execute() call",
-        ));
-    };
-    validate_supported_ast_statement(statement)
+pub(crate) fn classify_datafusion_statement(statement: &DataFusionStatement) -> SqlStatementKind {
+    match statement {
+        DataFusionStatement::Statement(statement) => classify_ast_statement(statement),
+        DataFusionStatement::Explain(_) => SqlStatementKind::Read,
+        _ => SqlStatementKind::Other,
+    }
 }
 
-pub(crate) fn dml_target_table_names(sql: &str) -> Result<Vec<String>, LixError> {
-    let statements = parse_sql_statements(sql)?;
-    let [statement] = statements.as_slice() else {
-        return Ok(Vec::new());
-    };
+pub(crate) fn datafusion_statement_dml_target_table_names(
+    statement: &DataFusionStatement,
+) -> Vec<String> {
     let mut targets = Vec::new();
-    collect_dml_target_table_names(statement, &mut targets);
-    Ok(targets)
+    collect_datafusion_statement_dml_target_table_names(statement, &mut targets);
+    targets
 }
 
-fn parse_sql_statements(sql: &str) -> Result<Vec<Statement>, LixError> {
-    Parser::parse_sql(&GenericDialect {}, sql).map_err(|error| {
-        LixError::new(
-            LixError::CODE_PARSE_ERROR,
-            format!("sql2 SQL parse error: {error}"),
-        )
-    })
+fn collect_datafusion_statement_dml_target_table_names(
+    statement: &DataFusionStatement,
+    targets: &mut Vec<String>,
+) {
+    match statement {
+        DataFusionStatement::Statement(statement) => {
+            collect_dml_target_table_names(statement, targets);
+        }
+        DataFusionStatement::Explain(explain) => {
+            collect_datafusion_statement_dml_target_table_names(
+                explain.statement.as_ref(),
+                targets,
+            );
+        }
+        _ => {}
+    }
 }
 
-fn collect_dml_target_table_names(statement: &Statement, targets: &mut Vec<String>) {
+fn collect_dml_target_table_names(statement: &SqlStatement, targets: &mut Vec<String>) {
     match statement {
-        Statement::Insert(insert) => {
+        SqlStatement::Insert(insert) => {
             if let TableObject::TableName(name) = &insert.table {
                 if let Some(table_name) = object_name_table_part(name) {
                     targets.push(table_name);
                 }
             }
         }
-        Statement::Update(update) => {
+        SqlStatement::Update(update) => {
             collect_table_with_joins_target(&update.table, targets);
         }
-        Statement::Delete(delete) => {
+        SqlStatement::Delete(delete) => {
             let tables = match &delete.from {
                 FromTable::WithFromKeyword(tables) | FromTable::WithoutKeyword(tables) => tables,
             };
@@ -70,7 +87,7 @@ fn collect_dml_target_table_names(statement: &Statement, targets: &mut Vec<Strin
                 collect_table_with_joins_target(table, targets);
             }
         }
-        Statement::Explain { statement, .. } => {
+        SqlStatement::Explain { statement, .. } => {
             collect_dml_target_table_names(statement.as_ref(), targets);
         }
         _ => {}
@@ -86,28 +103,38 @@ fn collect_table_with_joins_target(table: &TableWithJoins, targets: &mut Vec<Str
 }
 
 fn object_name_table_part(name: &ObjectName) -> Option<String> {
-    name.0
-        .last()
-        .and_then(|part| part.as_ident())
-        .map(|ident| ident.value.clone())
+    name.0.last().and_then(|part| part.as_ident()).map(|ident| {
+        if ident.quote_style.is_some() {
+            ident.value.clone()
+        } else {
+            ident.value.to_ascii_lowercase()
+        }
+    })
 }
 
-fn classify_ast_statement(statement: &Statement) -> SqlStatementKind {
+fn classify_ast_statement(statement: &SqlStatement) -> SqlStatementKind {
     match statement {
-        Statement::Insert(_) | Statement::Update(_) | Statement::Delete(_) => {
+        SqlStatement::Insert(_) | SqlStatement::Update(_) | SqlStatement::Delete(_) => {
             SqlStatementKind::Write
         }
-        Statement::Query(_) => SqlStatementKind::Read,
-        Statement::Explain { statement, .. } => classify_ast_statement(statement.as_ref()),
+        SqlStatement::Query(_) => SqlStatementKind::Read,
+        SqlStatement::Explain { .. } => SqlStatementKind::Read,
         _ => SqlStatementKind::Other,
     }
 }
 
-fn validate_supported_ast_statement(statement: &Statement) -> Result<(), LixError> {
+fn validate_supported_ast_statement(statement: &SqlStatement) -> Result<(), LixError> {
     match statement {
-        Statement::Query(query) => validate_supported_query(query),
-        Statement::Insert(_) | Statement::Update(_) | Statement::Delete(_) => Ok(()),
-        Statement::Explain { statement, .. } => validate_supported_ast_statement(statement),
+        SqlStatement::Query(query) => validate_supported_query(query),
+        SqlStatement::Insert(_) | SqlStatement::Update(_) | SqlStatement::Delete(_) => Ok(()),
+        SqlStatement::Explain { statement, .. } => {
+            if classify_ast_statement(statement.as_ref()) == SqlStatementKind::Write {
+                return Err(unsupported_sql_error(
+                    "EXPLAIN of write statements is not supported by Lix SQL",
+                ));
+            }
+            validate_supported_ast_statement(statement)
+        }
         _ => Err(unsupported_sql_error(format!(
             "SQL statement is not supported by Lix SQL: {statement}"
         ))),

package/dist-engine-src/src/sql2/context.rs

@@ -6,26 +6,27 @@ use serde_json::Value as JsonValue;
 use tokio::sync::Mutex;
 
 use crate::binary_cas::{BlobBytesBatch, BlobDataReader, BlobHash};
-use crate::changelog::ChangelogReader;
 use crate::commit_graph::CommitGraphReader;
+use crate::commit_store::CommitStoreReader;
 use crate::functions::FunctionProviderHandle;
 use crate::json_store::JsonStoreReader;
 use crate::live_state::{
-    LiveStateFilter, LiveStateReader, LiveStateRow, LiveStateRowRequest, LiveStateScanRequest,
+    LiveStateFilter, LiveStateReader, LiveStateRowRequest, LiveStateScanRequest,
+    MaterializedLiveStateRow,
 };
 use crate::storage::{ScopedStorageReader, StorageReadTransaction};
-use crate::transaction::types::{StageWrite, StageWriteOutcome};
+use crate::transaction::types::{TransactionWrite, TransactionWriteOutcome};
 use crate::version::{VersionHead, VersionRefReader};
 use crate::LixError;
 
 pub(crate) type SqlReadStore =
     ScopedStorageReader<Box<dyn StorageReadTransaction + Send + Sync + 'static>>;
-pub(crate) type SqlChangelogQuerySource = ChangelogQuerySource<SqlReadStore>;
+pub(crate) type SqlCommitStoreQuerySource = CommitStoreQuerySource<SqlReadStore>;
 pub(crate) type SqlJsonReader = JsonStoreReader<ScopedStorageReader<SqlReadStore>>;
 
 #[derive(Clone)]
-pub(crate) struct ChangelogQuerySource<S> {
-    pub(crate) changelog_reader: Arc<dyn ChangelogReader>,
+pub(crate) struct CommitStoreQuerySource<S> {
+    pub(crate) commit_store_reader: Arc<CommitStoreReader<ScopedStorageReader<S>>>,
     pub(crate) json_reader: JsonStoreReader<ScopedStorageReader<S>>,
 }
 
@@ -43,7 +44,7 @@ pub(crate) trait SqlExecutionContext {
     fn active_version_id(&self) -> &str;
     fn live_state(&self) -> Arc<dyn LiveStateReader>;
     fn functions(&self) -> FunctionProviderHandle;
-    fn changelog_query_source(&self) -> SqlChangelogQuerySource;
+    fn commit_store_query_source(&self) -> SqlCommitStoreQuerySource;
     fn commit_graph(&self) -> Box<dyn CommitGraphReader>;
     fn version_ref(&self) -> Arc<dyn VersionRefReader>;
     fn blob_reader(&self) -> Arc<dyn BlobDataReader>;
@@ -52,9 +53,9 @@ pub(crate) trait SqlExecutionContext {
 
 /// Write-capable SQL runtime boundary.
 ///
-/// Providers that mutate engine2 state should target this shape instead of
+/// Providers that mutate engine state should target this shape instead of
 /// reaching through session/backend escape hatches. The request and write
-/// payloads stay in the existing engine2 forms so this boundary centralizes
+/// payloads stay in the existing engine forms so this boundary centralizes
 /// authority without adding another translation layer.
 #[async_trait]
 #[allow(dead_code)]
@@ -68,11 +69,14 @@ pub(crate) trait SqlWriteExecutionContext {
     async fn scan_live_state(
         &mut self,
         request: &LiveStateScanRequest,
-    ) -> Result<Vec<LiveStateRow>, LixError>;
+    ) -> Result<Vec<MaterializedLiveStateRow>, LixError>;
 
     async fn load_version_head(&mut self, version_id: &str) -> Result<Option<String>, LixError>;
 
-    async fn stage_write(&mut self, write: StageWrite) -> Result<StageWriteOutcome, LixError>;
+    async fn stage_write(
+        &mut self,
+        write: TransactionWrite,
+    ) -> Result<TransactionWriteOutcome, LixError>;
 }
 
 #[derive(Clone)]
@@ -123,7 +127,7 @@ impl SqlWriteContext {
     pub(crate) async fn scan_live_state(
         &self,
         request: &LiveStateScanRequest,
-    ) -> Result<Vec<LiveStateRow>, LixError> {
+    ) -> Result<Vec<MaterializedLiveStateRow>, LixError> {
         let _guard = self.gate.lock().await;
         unsafe {
             self.ptr
@@ -170,8 +174,8 @@ impl SqlWriteContext {
 
     pub(crate) async fn stage_write(
         &self,
-        write: StageWrite,
-    ) -> Result<StageWriteOutcome, LixError> {
+        write: TransactionWrite,
+    ) -> Result<TransactionWriteOutcome, LixError> {
         let _guard = self.gate.lock().await;
         unsafe {
             self.ptr
@@ -219,12 +223,12 @@ impl WriteAccess {
 
     pub(crate) fn require_write(
         &self,
-        operation: &str,
+        action: &str,
     ) -> Result<SqlWriteContext, datafusion::error::DataFusionError> {
         match self {
             Self::Write { ctx } => Ok(ctx.clone()),
             Self::ReadOnly => Err(datafusion::error::DataFusionError::Execution(format!(
-                "{operation} requires a write transaction"
+                "{action} requires a write transaction"
             ))),
         }
     }
@@ -249,14 +253,14 @@ impl LiveStateReader for WriteContextLiveStateReader {
     async fn scan_rows(
         &self,
         request: &LiveStateScanRequest,
-    ) -> Result<Vec<LiveStateRow>, LixError> {
+    ) -> Result<Vec<MaterializedLiveStateRow>, LixError> {
         self.ctx.scan_live_state(request).await
     }
 
     async fn load_row(
         &self,
         request: &LiveStateRowRequest,
-    ) -> Result<Option<LiveStateRow>, LixError> {
+    ) -> Result<Option<MaterializedLiveStateRow>, LixError> {
         let mut rows = self
             .ctx
             .scan_live_state(&LiveStateScanRequest {

package/dist-engine-src/src/sql2/directory_history_provider.rs

@@ -21,7 +21,6 @@ use futures_util::stream;
 use serde::Deserialize;
 use tokio::sync::Mutex;
 
-use crate::changelog::MaterializedCanonicalChange;
 use crate::commit_graph::CommitGraphReader;
 use crate::serialize_row_metadata;
 use crate::LixError;
@@ -32,17 +31,18 @@ use super::history_route::{
     HistoryColumnStyle, HistoryEntry, HistoryRoute, HistoryViewDescriptor, HISTORY_COL_CHANGE_ID,
     HISTORY_COL_COMMIT_CREATED_AT, HISTORY_COL_DEPTH, HISTORY_COL_ENTITY_ID, HISTORY_COL_FILE_ID,
     HISTORY_COL_METADATA, HISTORY_COL_OBSERVED_COMMIT_ID, HISTORY_COL_SCHEMA_KEY,
-    HISTORY_COL_SCHEMA_VERSION, HISTORY_COL_SNAPSHOT_CONTENT, HISTORY_COL_START_COMMIT_ID,
+    HISTORY_COL_SNAPSHOT_CONTENT, HISTORY_COL_START_COMMIT_ID,
 };
 use super::result_metadata::json_field;
-use super::SqlChangelogQuerySource;
+use super::SqlCommitStoreQuerySource;
+use crate::commit_store::MaterializedChange;
 
 const DIRECTORY_DESCRIPTOR_SCHEMA_KEY: &str = "lix_directory_descriptor";
 
 pub(crate) async fn register_lix_directory_history_provider(
     session: &datafusion::prelude::SessionContext,
     commit_graph: Box<dyn CommitGraphReader>,
-    query_source: SqlChangelogQuerySource,
+    query_source: SqlCommitStoreQuerySource,
 ) -> Result<(), LixError> {
     session
         .register_table(
@@ -59,7 +59,7 @@ pub(crate) async fn register_lix_directory_history_provider(
 struct LixDirectoryHistoryProvider {
     schema: SchemaRef,
     commit_graph: Arc<Mutex<Box<dyn CommitGraphReader>>>,
-    query_source: SqlChangelogQuerySource,
+    query_source: SqlCommitStoreQuerySource,
 }
 
 impl std::fmt::Debug for LixDirectoryHistoryProvider {
@@ -71,7 +71,7 @@ impl std::fmt::Debug for LixDirectoryHistoryProvider {
 impl LixDirectoryHistoryProvider {
     fn new(
         commit_graph: Arc<Mutex<Box<dyn CommitGraphReader>>>,
-        query_source: SqlChangelogQuerySource,
+        query_source: SqlCommitStoreQuerySource,
     ) -> Self {
         Self {
             schema: lix_directory_history_schema(),
@@ -130,7 +130,7 @@ impl TableProvider for LixDirectoryHistoryProvider {
 
 struct LixDirectoryHistoryScanExec {
     commit_graph: Arc<Mutex<Box<dyn CommitGraphReader>>>,
-    query_source: SqlChangelogQuerySource,
+    query_source: SqlCommitStoreQuerySource,
     schema: SchemaRef,
     route: HistoryRoute,
     limit: Option<usize>,
@@ -149,7 +149,7 @@ impl std::fmt::Debug for LixDirectoryHistoryScanExec {
 impl LixDirectoryHistoryScanExec {
     fn new(
         commit_graph: Arc<Mutex<Box<dyn CommitGraphReader>>>,
-        query_source: SqlChangelogQuerySource,
+        query_source: SqlCommitStoreQuerySource,
         schema: SchemaRef,
         route: HistoryRoute,
         limit: Option<usize>,
@@ -265,7 +265,7 @@ struct DirectoryHistoryOutputRow {
     parent_id: Option<String>,
     name: Option<String>,
     hidden: Option<bool>,
-    descriptor_change: MaterializedCanonicalChange,
+    descriptor_change: MaterializedChange,
     event: DirectoryHistoryEvent,
 }
 
@@ -274,7 +274,7 @@ struct DirectoryHistoryEvent {
     directory_id: String,
     start_commit_id: String,
     depth: u32,
-    change: MaterializedCanonicalChange,
+    change: MaterializedChange,
     observed_commit_id: String,
     commit_created_at: String,
 }
@@ -289,7 +289,7 @@ struct DirectoryDescriptorSnapshot {
 
 async fn load_directory_history_rows(
     commit_graph: Arc<Mutex<Box<dyn CommitGraphReader>>>,
-    query_source: SqlChangelogQuerySource,
+    query_source: SqlCommitStoreQuerySource,
     route: &HistoryRoute,
 ) -> Result<Vec<DirectoryHistoryOutputRow>, LixError> {
     let event_route = route.traversal_only();
@@ -356,9 +356,10 @@ async fn load_directory_history_rows(
         });
     }
     output.retain(|row| {
+        let entity_id = entity_id_json_array(&row.entity_id).ok();
         route.matches_surface_row(
             DIRECTORY_DESCRIPTOR_SCHEMA_KEY,
-            &row.entity_id,
+            entity_id.as_deref().unwrap_or(&row.entity_id),
             None,
             row.event.depth,
         )
@@ -388,7 +389,7 @@ fn parse_directory_history_records(
         .map(|entry| {
             let Some(snapshot_content) = entry.change.snapshot_content.as_deref() else {
                 return Ok(DirectoryHistoryRecord {
-                    id: entry.change.entity_id.as_string()?,
+                    id: entry.change.entity_id.as_single_string_owned()?,
                     parent_id: None,
                     name: None,
                     hidden: None,
@@ -528,15 +529,15 @@ fn directory_history_column_array(
         "hidden" => Arc::new(BooleanArray::from(
             rows.iter().map(|row| row.hidden).collect::<Vec<_>>(),
         )) as ArrayRef,
-        HISTORY_COL_ENTITY_ID => string_array(rows.iter().map(|row| Some(row.entity_id.as_str()))),
+        HISTORY_COL_ENTITY_ID => Arc::new(StringArray::from(
+            rows.iter()
+                .map(|row| entity_id_json_array(&row.entity_id).map(Some))
+                .collect::<std::result::Result<Vec<_>, _>>()?,
+        )) as ArrayRef,
         HISTORY_COL_SCHEMA_KEY => {
             string_array(rows.iter().map(|_| Some(DIRECTORY_DESCRIPTOR_SCHEMA_KEY)))
         }
         HISTORY_COL_FILE_ID => string_array(rows.iter().map(|_| None)),
-        HISTORY_COL_SCHEMA_VERSION => string_array(
-            rows.iter()
-                .map(|row| Some(row.descriptor_change.schema_version.as_str())),
-        ),
         HISTORY_COL_CHANGE_ID => {
             string_array(rows.iter().map(|row| Some(row.event.change.id.as_str())))
         }
@@ -589,11 +590,10 @@ fn lix_directory_history_schema() -> SchemaRef {
         Field::new("parent_id", DataType::Utf8, true),
         Field::new("name", DataType::Utf8, true),
         Field::new("hidden", DataType::Boolean, true),
-        Field::new(HISTORY_COL_ENTITY_ID, DataType::Utf8, false),
+        json_field(HISTORY_COL_ENTITY_ID, false),
         Field::new(HISTORY_COL_SCHEMA_KEY, DataType::Utf8, false),
         Field::new(HISTORY_COL_FILE_ID, DataType::Utf8, true),
         json_field(HISTORY_COL_SNAPSHOT_CONTENT, true),
-        Field::new(HISTORY_COL_SCHEMA_VERSION, DataType::Utf8, false),
         Field::new(HISTORY_COL_CHANGE_ID, DataType::Utf8, false),
         json_field(HISTORY_COL_METADATA, true),
         Field::new(HISTORY_COL_OBSERVED_COMMIT_ID, DataType::Utf8, false),
@@ -618,6 +618,14 @@ fn datafusion_error_to_lix_error(error: DataFusionError) -> LixError {
     super::error::datafusion_error_to_lix_error(error)
 }
 
+fn entity_id_json_array(entity_id: &str) -> Result<String, LixError> {
+    serde_json::to_string(&[entity_id]).map_err(|error| {
+        LixError::unknown(format!(
+            "failed to encode history entity id as JSON: {error}"
+        ))
+    })
+}
+
 fn lix_error_to_datafusion_error(error: LixError) -> DataFusionError {
     super::error::lix_error_to_datafusion_error(error)
 }