@lix-js/sdk 0.6.0-preview.3 → 0.6.0-preview.5

package/dist-engine-src/src/sql2/providers/mod.rs

@@ -0,0 +1,508 @@
+use std::sync::Arc;
+
+use datafusion::prelude::SessionContext;
+
+use crate::LixError;
+
+mod branch;
+mod change;
+mod directory;
+mod directory_history;
+mod entity;
+mod entity_history;
+mod file;
+mod file_history;
+mod history;
+mod lix_state;
+
+use crate::sql2::catalog::{PublicCatalog, PublicSurfaceKind};
+use crate::sql2::session::SqlWriteSessionOptions;
+use crate::sql2::{SqlExecutionContext, SqlWriteContext};
+
+pub(crate) async fn register_read<C>(session: &SessionContext, ctx: &C) -> Result<(), LixError>
+where
+    C: SqlExecutionContext + ?Sized,
+{
+    let branch_ref = ctx.branch_ref();
+    let history_query_source = ctx.history_query_source();
+    let changelog_query_source = ctx.changelog_query_source();
+    let catalog = PublicCatalog::from_visible_schemas(&ctx.list_visible_schemas()?)?;
+    for surface in catalog.surfaces() {
+        match &surface.kind {
+            PublicSurfaceKind::LixState => {
+                lix_state::register_lix_state_active_provider(
+                    session,
+                    &surface.name,
+                    ctx.active_branch_id(),
+                    ctx.live_state(),
+                    Arc::clone(&branch_ref),
+                )
+                .await?;
+            }
+            PublicSurfaceKind::LixStateByBranch => {
+                lix_state::register_lix_state_by_branch_provider(
+                    session,
+                    &surface.name,
+                    ctx.live_state(),
+                    Arc::clone(&branch_ref),
+                )
+                .await?;
+            }
+            PublicSurfaceKind::Branch => {
+                branch::register_lix_branch_read_provider(
+                    session,
+                    &surface.name,
+                    ctx.live_state(),
+                    Arc::clone(&branch_ref),
+                )
+                .await?;
+            }
+            PublicSurfaceKind::Change => {
+                change::register_lix_change_read_provider(
+                    session,
+                    &surface.name,
+                    changelog_query_source.clone(),
+                )
+                .await?;
+            }
+            PublicSurfaceKind::History => {
+                history::register_history_provider(
+                    session,
+                    &surface.name,
+                    ctx.commit_graph(),
+                    history_query_source.clone(),
+                )
+                .await?;
+            }
+            PublicSurfaceKind::File => {
+                file::register_lix_file_active_provider(
+                    session,
+                    &surface.name,
+                    ctx.active_branch_id(),
+                    ctx.live_state(),
+                    Arc::clone(&branch_ref),
+                    ctx.blob_reader(),
+                    ctx.functions(),
+                )
+                .await?;
+            }
+            PublicSurfaceKind::FileByBranch => {
+                file::register_lix_file_by_branch_provider(
+                    session,
+                    &surface.name,
+                    ctx.live_state(),
+                    Arc::clone(&branch_ref),
+                    ctx.blob_reader(),
+                    ctx.functions(),
+                )
+                .await?;
+            }
+            PublicSurfaceKind::FileHistory => {
+                file_history::register_lix_file_history_surface(
+                    session,
+                    &surface.name,
+                    ctx.commit_graph(),
+                    history_query_source.clone(),
+                    ctx.blob_reader(),
+                )
+                .await?;
+            }
+            PublicSurfaceKind::Directory => {
+                directory::register_lix_directory_active_provider(
+                    session,
+                    &surface.name,
+                    ctx.active_branch_id(),
+                    ctx.live_state(),
+                    Arc::clone(&branch_ref),
+                    ctx.functions(),
+                )
+                .await?;
+            }
+            PublicSurfaceKind::DirectoryByBranch => {
+                directory::register_lix_directory_by_branch_provider(
+                    session,
+                    &surface.name,
+                    ctx.live_state(),
+                    Arc::clone(&branch_ref),
+                    ctx.functions(),
+                )
+                .await?;
+            }
+            PublicSurfaceKind::DirectoryHistory => {
+                directory_history::register_lix_directory_history_surface(
+                    session,
+                    &surface.name,
+                    ctx.commit_graph(),
+                    history_query_source.clone(),
+                )
+                .await?;
+            }
+            PublicSurfaceKind::EntityBase { .. }
+            | PublicSurfaceKind::EntityByBranch { .. }
+            | PublicSurfaceKind::EntityHistory { .. } => {}
+        }
+    }
+    entity::register_entity_providers(
+        session,
+        ctx.active_branch_id(),
+        ctx.live_state(),
+        Arc::clone(&branch_ref),
+        Arc::new(tokio::sync::Mutex::new(ctx.commit_graph())),
+        history_query_source,
+        &catalog,
+    )
+    .await?;
+
+    Ok(())
+}
+
+pub(crate) async fn register_write(
+    session: &SessionContext,
+    write_ctx: SqlWriteContext,
+    options: SqlWriteSessionOptions,
+) -> Result<(), LixError> {
+    let catalog = PublicCatalog::from_visible_schemas(&write_ctx.list_visible_schemas()?)?;
+    for surface in catalog.surfaces() {
+        match &surface.kind {
+            PublicSurfaceKind::LixState => {
+                replace_registered_table(session, &surface.name)?;
+                lix_state::register_lix_state_active_write_provider(
+                    session,
+                    &surface.name,
+                    write_ctx.clone(),
+                )
+                .await?;
+            }
+            PublicSurfaceKind::LixStateByBranch => {
+                replace_registered_table(session, &surface.name)?;
+                lix_state::register_lix_state_by_branch_write_provider(
+                    session,
+                    &surface.name,
+                    write_ctx.clone(),
+                )
+                .await?;
+            }
+            PublicSurfaceKind::Branch => {
+                replace_registered_table(session, &surface.name)?;
+                branch::register_write_provider(session, &surface.name, write_ctx.clone()).await?;
+            }
+            PublicSurfaceKind::File => {
+                replace_registered_table(session, &surface.name)?;
+                file::register_active_write_provider(
+                    session,
+                    &surface.name,
+                    write_ctx.clone(),
+                    options.clone(),
+                )
+                .await?;
+            }
+            PublicSurfaceKind::FileByBranch => {
+                replace_registered_table(session, &surface.name)?;
+                file::register_by_branch_write_provider(
+                    session,
+                    &surface.name,
+                    write_ctx.clone(),
+                    options.clone(),
+                )
+                .await?;
+            }
+            PublicSurfaceKind::Directory => {
+                replace_registered_table(session, &surface.name)?;
+                directory::register_active_write_provider(
+                    session,
+                    &surface.name,
+                    write_ctx.clone(),
+                )
+                .await?;
+            }
+            PublicSurfaceKind::DirectoryByBranch => {
+                replace_registered_table(session, &surface.name)?;
+                directory::register_by_branch_write_provider(
+                    session,
+                    &surface.name,
+                    write_ctx.clone(),
+                )
+                .await?;
+            }
+            PublicSurfaceKind::Change
+            | PublicSurfaceKind::History
+            | PublicSurfaceKind::FileHistory
+            | PublicSurfaceKind::DirectoryHistory => {}
+            PublicSurfaceKind::EntityBase { .. }
+            | PublicSurfaceKind::EntityByBranch { .. }
+            | PublicSurfaceKind::EntityHistory { .. } => {}
+        }
+    }
+    for surface in catalog.surfaces() {
+        if matches!(
+            surface.kind,
+            PublicSurfaceKind::EntityBase { .. } | PublicSurfaceKind::EntityByBranch { .. }
+        ) {
+            replace_registered_table(session, &surface.name)?;
+        }
+    }
+    entity::register_entity_write_providers(session, write_ctx.clone(), &catalog).await?;
+    Ok(())
+}
+
+fn replace_registered_table(session: &SessionContext, name: &str) -> Result<(), LixError> {
+    match session.deregister_table(name) {
+        Ok(_) => Ok(()),
+        Err(error) if error.to_string().contains("not found") => Ok(()),
+        Err(error) => Err(LixError::new(
+            LixError::CODE_UNKNOWN,
+            format!("sql2 DataFusion error: {error}"),
+        )),
+    }
+}
+
+#[cfg(test)]
+mod tests {
+    use std::sync::Arc;
+
+    use async_trait::async_trait;
+    use serde_json::json;
+
+    use datafusion::arrow::datatypes::SchemaRef;
+    use datafusion::prelude::SessionContext;
+
+    use crate::branch::{BranchHead, BranchRefReader};
+    use crate::commit_graph::{
+        CommitGraphChangeHistoryEntry, CommitGraphChangeHistoryRequest, CommitGraphCommit,
+        CommitGraphReader, ReachableCommitGraphCommit,
+    };
+    use crate::json_store::JsonStoreContext;
+    use crate::live_state::{
+        LiveStateReader, LiveStateRowRequest, LiveStateScanRequest, MaterializedLiveStateRow,
+    };
+    use crate::sql2::catalog::{derive_entity_surface_spec_from_schema, PublicCatalog};
+    use crate::sql2::HistoryQuerySource;
+    use crate::storage::{
+        InMemoryStorageBackend, InMemoryStorageRead, StorageContext, StorageReadOptions,
+        StorageReadScope,
+    };
+    use crate::LixError;
+
+    use super::{
+        branch, change, directory, directory_history, entity, file, file_history, history,
+        lix_state,
+    };
+
+    #[test]
+    fn provider_history_schemas_match_catalog_contract_order() {
+        let catalog = PublicCatalog::from_visible_schemas(&[]).expect("catalog should build");
+
+        assert_surface_schema_matches_provider_schema(
+            &catalog,
+            "lix_state",
+            lix_state::lix_state_schema(),
+        );
+        assert_surface_schema_matches_provider_schema(
+            &catalog,
+            "lix_state_by_branch",
+            lix_state::lix_state_by_branch_schema(),
+        );
+        assert_surface_schema_matches_provider_schema(
+            &catalog,
+            "lix_file",
+            file::lix_file_schema(),
+        );
+        assert_surface_schema_matches_provider_schema(
+            &catalog,
+            "lix_file_by_branch",
+            file::lix_file_by_branch_schema(),
+        );
+        assert_surface_schema_matches_provider_schema(
+            &catalog,
+            "lix_directory",
+            directory::lix_directory_schema(),
+        );
+        assert_surface_schema_matches_provider_schema(
+            &catalog,
+            "lix_directory_by_branch",
+            directory::lix_directory_by_branch_schema(),
+        );
+        assert_surface_schema_matches_provider_schema(
+            &catalog,
+            "lix_branch",
+            branch::lix_branch_schema(),
+        );
+        assert_surface_schema_matches_provider_schema(
+            &catalog,
+            "lix_change",
+            change::lix_change_schema(),
+        );
+        assert_surface_schema_matches_provider_schema(
+            &catalog,
+            "lix_state_history",
+            history::lix_state_history_schema(),
+        );
+        assert_surface_schema_matches_provider_schema(
+            &catalog,
+            "lix_file_history",
+            file_history::lix_file_history_schema(),
+        );
+        assert_surface_schema_matches_provider_schema(
+            &catalog,
+            "lix_directory_history",
+            directory_history::lix_directory_history_schema(),
+        );
+    }
+
+    #[tokio::test]
+    async fn provider_entity_schemas_match_catalog_contract_order() {
+        let schema = json!({
+            "x-lix-key": "phase8_entity",
+            "x-lix-primary-key": ["/id"],
+            "type": "object",
+            "properties": {
+                "id": { "type": "string" },
+                "count": { "type": "integer" },
+                "body": { "type": "object" }
+            }
+        });
+        let catalog =
+            PublicCatalog::from_visible_schemas(&[schema.clone()]).expect("catalog should build");
+        let _spec = derive_entity_surface_spec_from_schema(&schema).expect("schema should derive");
+        let session = SessionContext::new();
+        entity::register_entity_providers(
+            &session,
+            "branch-a",
+            Arc::new(EmptyLiveStateReader),
+            Arc::new(EmptyBranchRefReader),
+            Arc::new(tokio::sync::Mutex::new(Box::new(EmptyCommitGraphReader))),
+            empty_history_query_source().await,
+            &catalog,
+        )
+        .await
+        .expect("entity providers should register");
+
+        assert_registered_table_schema_matches_catalog(&session, &catalog, "phase8_entity").await;
+        assert_registered_table_schema_matches_catalog(
+            &session,
+            &catalog,
+            "phase8_entity_by_branch",
+        )
+        .await;
+        assert_registered_table_schema_matches_catalog(&session, &catalog, "phase8_entity_history")
+            .await;
+    }
+
+    async fn assert_registered_table_schema_matches_catalog(
+        session: &SessionContext,
+        catalog: &PublicCatalog,
+        surface_name: &str,
+    ) {
+        let provider = session
+            .table_provider(surface_name)
+            .await
+            .unwrap_or_else(|error| panic!("{surface_name} provider should load: {error}"));
+        assert_surface_schema_matches_provider_schema(catalog, surface_name, provider.schema());
+    }
+
+    fn assert_surface_schema_matches_provider_schema(
+        catalog: &PublicCatalog,
+        surface_name: &str,
+        provider_schema: SchemaRef,
+    ) {
+        let surface = catalog
+            .surface(surface_name)
+            .unwrap_or_else(|| panic!("{surface_name} should be in catalog"));
+        let catalog_column_names = surface
+            .columns
+            .iter()
+            .map(|column| column.name.as_str())
+            .collect::<Vec<_>>();
+        let provider_field_names = provider_schema
+            .fields()
+            .iter()
+            .map(|field| field.name().as_str())
+            .collect::<Vec<_>>();
+        assert_eq!(
+            catalog_column_names, provider_field_names,
+            "{surface_name} column order"
+        );
+
+        let catalog_schema = catalog
+            .surface_schema(surface_name)
+            .unwrap_or_else(|| panic!("{surface_name} should be in catalog"));
+        assert_eq!(
+            catalog_schema.fields(),
+            provider_schema.fields(),
+            "{surface_name}"
+        );
+    }
+
+    async fn empty_history_query_source(
+    ) -> crate::sql2::SqlHistoryQuerySource<StorageReadScope<InMemoryStorageRead>> {
+        let storage = StorageContext::new(InMemoryStorageBackend::new());
+        let read_scope = storage
+            .begin_read(StorageReadOptions::default())
+            .expect("read should open");
+        HistoryQuerySource {
+            json_reader: JsonStoreContext::new().reader(read_scope.store()),
+        }
+    }
+
+    struct EmptyLiveStateReader;
+
+    #[async_trait]
+    impl LiveStateReader for EmptyLiveStateReader {
+        async fn scan_rows(
+            &self,
+            _request: &LiveStateScanRequest,
+        ) -> Result<Vec<MaterializedLiveStateRow>, LixError> {
+            Ok(Vec::new())
+        }
+
+        async fn load_row(
+            &self,
+            _request: &LiveStateRowRequest,
+        ) -> Result<Option<MaterializedLiveStateRow>, LixError> {
+            Ok(None)
+        }
+    }
+
+    struct EmptyBranchRefReader;
+
+    #[async_trait]
+    impl BranchRefReader for EmptyBranchRefReader {
+        async fn load_head(&self, branch_id: &str) -> Result<Option<BranchHead>, LixError> {
+            Ok(Some(BranchHead {
+                branch_id: branch_id.to_string(),
+                commit_id: format!("commit-{branch_id}"),
+            }))
+        }
+
+        async fn scan_heads(&self) -> Result<Vec<BranchHead>, LixError> {
+            Ok(Vec::new())
+        }
+    }
+
+    struct EmptyCommitGraphReader;
+
+    #[async_trait]
+    impl CommitGraphReader for EmptyCommitGraphReader {
+        async fn load_commit(
+            &mut self,
+            _commit_id: &str,
+        ) -> Result<Option<CommitGraphCommit>, LixError> {
+            Ok(None)
+        }
+
+        async fn reachable_commits(
+            &mut self,
+            _head_commit_id: &str,
+        ) -> Result<Vec<ReachableCommitGraphCommit>, LixError> {
+            Ok(Vec::new())
+        }
+
+        async fn change_history_from_commit(
+            &mut self,
+            _start_commit_id: &str,
+            _request: &CommitGraphChangeHistoryRequest,
+        ) -> Result<Vec<CommitGraphChangeHistoryEntry>, LixError> {
+            Ok(Vec::new())
+        }
+    }
+}

package/dist-engine-src/src/sql2/read_only.rs

@@ -44,8 +44,8 @@ fn read_only_error(action: &str, schema_key: &str, message: &'static str) -> Dat
 
 fn read_only_schema_message(schema_key: &str) -> Option<&'static str> {
     match schema_key {
-        "lix_version_descriptor" | "lix_version_ref" => {
-            Some("Use the writable lix_version surface to create, update, or delete versions.")
+        "lix_branch_descriptor" | "lix_branch_ref" => {
+            Some("Use the writable lix_branch surface to create, update, or delete branches.")
         }
         "lix_file_descriptor" => {
             Some("Use the writable lix_file surface to create, update, or delete files.")

package/dist-engine-src/src/sql2/session.rs

@@ -1,118 +1,69 @@
-use std::sync::Arc;
-
 use datafusion::prelude::{SessionConfig, SessionContext};
+use std::collections::BTreeSet;
 
 use crate::LixError;
 
-use super::change_provider::register_lix_change_provider;
-use super::directory_history_provider::register_lix_directory_history_provider;
-use super::directory_provider::{
-    register_lix_directory_providers, register_lix_directory_write_providers,
-};
-use super::entity_provider::{register_entity_providers, register_entity_write_providers};
-use super::file_history_provider::register_lix_file_history_provider;
-use super::file_provider::{register_lix_file_providers, register_lix_file_write_providers};
-use super::history_provider::register_history_providers;
-use super::lix_state_provider::{register_lix_state_providers, register_lix_state_write_providers};
+use super::providers;
 use super::udfs::register_sql2_functions;
-use super::version_provider::{register_lix_version_provider, register_lix_version_write_provider};
 use super::{SqlExecutionContext, SqlWriteContext, SqlWriteExecutionContext};
 
-pub(crate) async fn build_read_session(
-    ctx: &dyn SqlExecutionContext,
-) -> Result<SessionContext, LixError> {
+pub(crate) async fn build_read_session<C>(ctx: &C) -> Result<SessionContext, LixError>
+where
+    C: SqlExecutionContext + ?Sized,
+{
     let session = new_sql_session_context();
-    let version_ref = ctx.version_ref();
-    let active_version_commit_id = version_ref
-        .load_head(ctx.active_version_id())
+    let branch_ref = ctx.branch_ref();
+    let active_branch_commit_id = branch_ref
+        .load_head(ctx.active_branch_id())
         .await?
         .map(|head| head.commit_id);
-    register_sql2_functions(&session, ctx.functions(), active_version_commit_id);
-    register_lix_state_providers(
-        &session,
-        ctx.active_version_id(),
-        ctx.live_state(),
-        Arc::clone(&version_ref),
-    )
-    .await?;
-    register_lix_version_provider(&session, ctx.live_state(), Arc::clone(&version_ref)).await?;
-    let commit_store_query_source = ctx.commit_store_query_source();
-    register_lix_change_provider(&session, commit_store_query_source.clone()).await?;
-    let state_history_commit_graph = ctx.commit_graph();
-    register_history_providers(
-        &session,
-        state_history_commit_graph,
-        commit_store_query_source.clone(),
-    )
-    .await?;
-    let file_history_commit_graph = ctx.commit_graph();
-    register_lix_file_history_provider(
-        &session,
-        file_history_commit_graph,
-        commit_store_query_source.clone(),
-        ctx.blob_reader(),
-    )
-    .await?;
-    let directory_history_commit_graph = ctx.commit_graph();
-    register_lix_directory_history_provider(
-        &session,
-        directory_history_commit_graph,
-        commit_store_query_source.clone(),
-    )
-    .await?;
-    let entity_commit_graph = Arc::new(tokio::sync::Mutex::new(ctx.commit_graph()));
-    register_lix_directory_providers(
-        &session,
-        ctx.active_version_id(),
-        ctx.live_state(),
-        Arc::clone(&version_ref),
-        ctx.functions(),
-    )
-    .await?;
-    register_lix_file_providers(
-        &session,
-        ctx.active_version_id(),
-        ctx.live_state(),
-        Arc::clone(&version_ref),
-        ctx.blob_reader(),
-        ctx.functions(),
-    )
-    .await?;
-    register_entity_providers(
-        &session,
-        ctx.active_version_id(),
-        ctx.live_state(),
-        Arc::clone(&version_ref),
-        entity_commit_graph,
-        commit_store_query_source,
-        &ctx.list_visible_schemas()?,
-    )
-    .await?;
+    register_sql2_functions(&session, ctx.functions(), active_branch_commit_id);
+    providers::register_read(&session, ctx).await?;
+
+    Ok(session)
+}
 
+pub(crate) async fn build_transaction_read_session<C>(
+    read_ctx: &C,
+    write_ctx: &mut dyn SqlWriteExecutionContext,
+) -> Result<SessionContext, LixError>
+where
+    C: SqlExecutionContext + ?Sized,
+{
+    let session = build_read_session(read_ctx).await?;
+    let write_ctx = SqlWriteContext::new(write_ctx);
+    providers::register_write(&session, write_ctx, SqlWriteSessionOptions::default()).await?;
     Ok(session)
 }
 
-pub(crate) async fn build_write_session(
+#[derive(Clone, Debug, Default)]
+pub(crate) struct SqlWriteSessionOptions {
+    pub(crate) omitted_insert_columns: BTreeSet<String>,
+}
+
+pub(crate) async fn build_write_session_with_options(
     ctx: &mut dyn SqlWriteExecutionContext,
+    options: SqlWriteSessionOptions,
 ) -> Result<SessionContext, LixError> {
     let session = new_sql_session_context();
     let write_ctx = SqlWriteContext::new(ctx);
-    let active_version_commit_id = write_ctx
-        .load_version_head(&write_ctx.active_version_id())
-        .await?;
-    register_sql2_functions(&session, write_ctx.functions(), active_version_commit_id);
-
-    register_lix_state_write_providers(&session, write_ctx.clone()).await?;
-    register_lix_version_write_provider(&session, write_ctx.clone()).await?;
-
-    register_lix_directory_write_providers(&session, write_ctx.clone()).await?;
-    register_lix_file_write_providers(&session, write_ctx.clone()).await?;
-    register_entity_write_providers(
+    let active_branch_id = write_ctx.active_branch_id();
+    let active_branch_commit_id = write_ctx
+        .load_branch_head(&active_branch_id)
+        .await?
+        .ok_or_else(|| {
+            LixError::branch_not_found(
+                active_branch_id.clone(),
+                "build SQL write session",
+                "active branch",
+            )
+        })?;
+    register_sql2_functions(
         &session,
-        write_ctx.clone(),
-        &write_ctx.list_visible_schemas()?,
-    )
-    .await?;
+        write_ctx.functions(),
+        Some(active_branch_commit_id),
+    );
+    providers::register_write(&session, write_ctx, options).await?;
 
     Ok(session)
 }

package/dist-engine-src/src/sql2/storage/constraints.rs

@@ -0,0 +1 @@
+//! Storage-level write constraints for sql2 plans.

package/dist-engine-src/src/sql2/storage/mod.rs

@@ -0,0 +1 @@
+pub(crate) mod constraints;