@lix-js/sdk 0.6.0-preview.5 → 0.6.1

package/dist-engine-src/src/sql2/providers/mod.rs

@@ -1,508 +0,0 @@
-use std::sync::Arc;
-
-use datafusion::prelude::SessionContext;
-
-use crate::LixError;
-
-mod branch;
-mod change;
-mod directory;
-mod directory_history;
-mod entity;
-mod entity_history;
-mod file;
-mod file_history;
-mod history;
-mod lix_state;
-
-use crate::sql2::catalog::{PublicCatalog, PublicSurfaceKind};
-use crate::sql2::session::SqlWriteSessionOptions;
-use crate::sql2::{SqlExecutionContext, SqlWriteContext};
-
-pub(crate) async fn register_read<C>(session: &SessionContext, ctx: &C) -> Result<(), LixError>
-where
-    C: SqlExecutionContext + ?Sized,
-{
-    let branch_ref = ctx.branch_ref();
-    let history_query_source = ctx.history_query_source();
-    let changelog_query_source = ctx.changelog_query_source();
-    let catalog = PublicCatalog::from_visible_schemas(&ctx.list_visible_schemas()?)?;
-    for surface in catalog.surfaces() {
-        match &surface.kind {
-            PublicSurfaceKind::LixState => {
-                lix_state::register_lix_state_active_provider(
-                    session,
-                    &surface.name,
-                    ctx.active_branch_id(),
-                    ctx.live_state(),
-                    Arc::clone(&branch_ref),
-                )
-                .await?;
-            }
-            PublicSurfaceKind::LixStateByBranch => {
-                lix_state::register_lix_state_by_branch_provider(
-                    session,
-                    &surface.name,
-                    ctx.live_state(),
-                    Arc::clone(&branch_ref),
-                )
-                .await?;
-            }
-            PublicSurfaceKind::Branch => {
-                branch::register_lix_branch_read_provider(
-                    session,
-                    &surface.name,
-                    ctx.live_state(),
-                    Arc::clone(&branch_ref),
-                )
-                .await?;
-            }
-            PublicSurfaceKind::Change => {
-                change::register_lix_change_read_provider(
-                    session,
-                    &surface.name,
-                    changelog_query_source.clone(),
-                )
-                .await?;
-            }
-            PublicSurfaceKind::History => {
-                history::register_history_provider(
-                    session,
-                    &surface.name,
-                    ctx.commit_graph(),
-                    history_query_source.clone(),
-                )
-                .await?;
-            }
-            PublicSurfaceKind::File => {
-                file::register_lix_file_active_provider(
-                    session,
-                    &surface.name,
-                    ctx.active_branch_id(),
-                    ctx.live_state(),
-                    Arc::clone(&branch_ref),
-                    ctx.blob_reader(),
-                    ctx.functions(),
-                )
-                .await?;
-            }
-            PublicSurfaceKind::FileByBranch => {
-                file::register_lix_file_by_branch_provider(
-                    session,
-                    &surface.name,
-                    ctx.live_state(),
-                    Arc::clone(&branch_ref),
-                    ctx.blob_reader(),
-                    ctx.functions(),
-                )
-                .await?;
-            }
-            PublicSurfaceKind::FileHistory => {
-                file_history::register_lix_file_history_surface(
-                    session,
-                    &surface.name,
-                    ctx.commit_graph(),
-                    history_query_source.clone(),
-                    ctx.blob_reader(),
-                )
-                .await?;
-            }
-            PublicSurfaceKind::Directory => {
-                directory::register_lix_directory_active_provider(
-                    session,
-                    &surface.name,
-                    ctx.active_branch_id(),
-                    ctx.live_state(),
-                    Arc::clone(&branch_ref),
-                    ctx.functions(),
-                )
-                .await?;
-            }
-            PublicSurfaceKind::DirectoryByBranch => {
-                directory::register_lix_directory_by_branch_provider(
-                    session,
-                    &surface.name,
-                    ctx.live_state(),
-                    Arc::clone(&branch_ref),
-                    ctx.functions(),
-                )
-                .await?;
-            }
-            PublicSurfaceKind::DirectoryHistory => {
-                directory_history::register_lix_directory_history_surface(
-                    session,
-                    &surface.name,
-                    ctx.commit_graph(),
-                    history_query_source.clone(),
-                )
-                .await?;
-            }
-            PublicSurfaceKind::EntityBase { .. }
-            | PublicSurfaceKind::EntityByBranch { .. }
-            | PublicSurfaceKind::EntityHistory { .. } => {}
-        }
-    }
-    entity::register_entity_providers(
-        session,
-        ctx.active_branch_id(),
-        ctx.live_state(),
-        Arc::clone(&branch_ref),
-        Arc::new(tokio::sync::Mutex::new(ctx.commit_graph())),
-        history_query_source,
-        &catalog,
-    )
-    .await?;
-
-    Ok(())
-}
-
-pub(crate) async fn register_write(
-    session: &SessionContext,
-    write_ctx: SqlWriteContext,
-    options: SqlWriteSessionOptions,
-) -> Result<(), LixError> {
-    let catalog = PublicCatalog::from_visible_schemas(&write_ctx.list_visible_schemas()?)?;
-    for surface in catalog.surfaces() {
-        match &surface.kind {
-            PublicSurfaceKind::LixState => {
-                replace_registered_table(session, &surface.name)?;
-                lix_state::register_lix_state_active_write_provider(
-                    session,
-                    &surface.name,
-                    write_ctx.clone(),
-                )
-                .await?;
-            }
-            PublicSurfaceKind::LixStateByBranch => {
-                replace_registered_table(session, &surface.name)?;
-                lix_state::register_lix_state_by_branch_write_provider(
-                    session,
-                    &surface.name,
-                    write_ctx.clone(),
-                )
-                .await?;
-            }
-            PublicSurfaceKind::Branch => {
-                replace_registered_table(session, &surface.name)?;
-                branch::register_write_provider(session, &surface.name, write_ctx.clone()).await?;
-            }
-            PublicSurfaceKind::File => {
-                replace_registered_table(session, &surface.name)?;
-                file::register_active_write_provider(
-                    session,
-                    &surface.name,
-                    write_ctx.clone(),
-                    options.clone(),
-                )
-                .await?;
-            }
-            PublicSurfaceKind::FileByBranch => {
-                replace_registered_table(session, &surface.name)?;
-                file::register_by_branch_write_provider(
-                    session,
-                    &surface.name,
-                    write_ctx.clone(),
-                    options.clone(),
-                )
-                .await?;
-            }
-            PublicSurfaceKind::Directory => {
-                replace_registered_table(session, &surface.name)?;
-                directory::register_active_write_provider(
-                    session,
-                    &surface.name,
-                    write_ctx.clone(),
-                )
-                .await?;
-            }
-            PublicSurfaceKind::DirectoryByBranch => {
-                replace_registered_table(session, &surface.name)?;
-                directory::register_by_branch_write_provider(
-                    session,
-                    &surface.name,
-                    write_ctx.clone(),
-                )
-                .await?;
-            }
-            PublicSurfaceKind::Change
-            | PublicSurfaceKind::History
-            | PublicSurfaceKind::FileHistory
-            | PublicSurfaceKind::DirectoryHistory => {}
-            PublicSurfaceKind::EntityBase { .. }
-            | PublicSurfaceKind::EntityByBranch { .. }
-            | PublicSurfaceKind::EntityHistory { .. } => {}
-        }
-    }
-    for surface in catalog.surfaces() {
-        if matches!(
-            surface.kind,
-            PublicSurfaceKind::EntityBase { .. } | PublicSurfaceKind::EntityByBranch { .. }
-        ) {
-            replace_registered_table(session, &surface.name)?;
-        }
-    }
-    entity::register_entity_write_providers(session, write_ctx.clone(), &catalog).await?;
-    Ok(())
-}
-
-fn replace_registered_table(session: &SessionContext, name: &str) -> Result<(), LixError> {
-    match session.deregister_table(name) {
-        Ok(_) => Ok(()),
-        Err(error) if error.to_string().contains("not found") => Ok(()),
-        Err(error) => Err(LixError::new(
-            LixError::CODE_UNKNOWN,
-            format!("sql2 DataFusion error: {error}"),
-        )),
-    }
-}
-
-#[cfg(test)]
-mod tests {
-    use std::sync::Arc;
-
-    use async_trait::async_trait;
-    use serde_json::json;
-
-    use datafusion::arrow::datatypes::SchemaRef;
-    use datafusion::prelude::SessionContext;
-
-    use crate::branch::{BranchHead, BranchRefReader};
-    use crate::commit_graph::{
-        CommitGraphChangeHistoryEntry, CommitGraphChangeHistoryRequest, CommitGraphCommit,
-        CommitGraphReader, ReachableCommitGraphCommit,
-    };
-    use crate::json_store::JsonStoreContext;
-    use crate::live_state::{
-        LiveStateReader, LiveStateRowRequest, LiveStateScanRequest, MaterializedLiveStateRow,
-    };
-    use crate::sql2::catalog::{derive_entity_surface_spec_from_schema, PublicCatalog};
-    use crate::sql2::HistoryQuerySource;
-    use crate::storage::{
-        InMemoryStorageBackend, InMemoryStorageRead, StorageContext, StorageReadOptions,
-        StorageReadScope,
-    };
-    use crate::LixError;
-
-    use super::{
-        branch, change, directory, directory_history, entity, file, file_history, history,
-        lix_state,
-    };
-
-    #[test]
-    fn provider_history_schemas_match_catalog_contract_order() {
-        let catalog = PublicCatalog::from_visible_schemas(&[]).expect("catalog should build");
-
-        assert_surface_schema_matches_provider_schema(
-            &catalog,
-            "lix_state",
-            lix_state::lix_state_schema(),
-        );
-        assert_surface_schema_matches_provider_schema(
-            &catalog,
-            "lix_state_by_branch",
-            lix_state::lix_state_by_branch_schema(),
-        );
-        assert_surface_schema_matches_provider_schema(
-            &catalog,
-            "lix_file",
-            file::lix_file_schema(),
-        );
-        assert_surface_schema_matches_provider_schema(
-            &catalog,
-            "lix_file_by_branch",
-            file::lix_file_by_branch_schema(),
-        );
-        assert_surface_schema_matches_provider_schema(
-            &catalog,
-            "lix_directory",
-            directory::lix_directory_schema(),
-        );
-        assert_surface_schema_matches_provider_schema(
-            &catalog,
-            "lix_directory_by_branch",
-            directory::lix_directory_by_branch_schema(),
-        );
-        assert_surface_schema_matches_provider_schema(
-            &catalog,
-            "lix_branch",
-            branch::lix_branch_schema(),
-        );
-        assert_surface_schema_matches_provider_schema(
-            &catalog,
-            "lix_change",
-            change::lix_change_schema(),
-        );
-        assert_surface_schema_matches_provider_schema(
-            &catalog,
-            "lix_state_history",
-            history::lix_state_history_schema(),
-        );
-        assert_surface_schema_matches_provider_schema(
-            &catalog,
-            "lix_file_history",
-            file_history::lix_file_history_schema(),
-        );
-        assert_surface_schema_matches_provider_schema(
-            &catalog,
-            "lix_directory_history",
-            directory_history::lix_directory_history_schema(),
-        );
-    }
-
-    #[tokio::test]
-    async fn provider_entity_schemas_match_catalog_contract_order() {
-        let schema = json!({
-            "x-lix-key": "phase8_entity",
-            "x-lix-primary-key": ["/id"],
-            "type": "object",
-            "properties": {
-                "id": { "type": "string" },
-                "count": { "type": "integer" },
-                "body": { "type": "object" }
-            }
-        });
-        let catalog =
-            PublicCatalog::from_visible_schemas(&[schema.clone()]).expect("catalog should build");
-        let _spec = derive_entity_surface_spec_from_schema(&schema).expect("schema should derive");
-        let session = SessionContext::new();
-        entity::register_entity_providers(
-            &session,
-            "branch-a",
-            Arc::new(EmptyLiveStateReader),
-            Arc::new(EmptyBranchRefReader),
-            Arc::new(tokio::sync::Mutex::new(Box::new(EmptyCommitGraphReader))),
-            empty_history_query_source().await,
-            &catalog,
-        )
-        .await
-        .expect("entity providers should register");
-
-        assert_registered_table_schema_matches_catalog(&session, &catalog, "phase8_entity").await;
-        assert_registered_table_schema_matches_catalog(
-            &session,
-            &catalog,
-            "phase8_entity_by_branch",
-        )
-        .await;
-        assert_registered_table_schema_matches_catalog(&session, &catalog, "phase8_entity_history")
-            .await;
-    }
-
-    async fn assert_registered_table_schema_matches_catalog(
-        session: &SessionContext,
-        catalog: &PublicCatalog,
-        surface_name: &str,
-    ) {
-        let provider = session
-            .table_provider(surface_name)
-            .await
-            .unwrap_or_else(|error| panic!("{surface_name} provider should load: {error}"));
-        assert_surface_schema_matches_provider_schema(catalog, surface_name, provider.schema());
-    }
-
-    fn assert_surface_schema_matches_provider_schema(
-        catalog: &PublicCatalog,
-        surface_name: &str,
-        provider_schema: SchemaRef,
-    ) {
-        let surface = catalog
-            .surface(surface_name)
-            .unwrap_or_else(|| panic!("{surface_name} should be in catalog"));
-        let catalog_column_names = surface
-            .columns
-            .iter()
-            .map(|column| column.name.as_str())
-            .collect::<Vec<_>>();
-        let provider_field_names = provider_schema
-            .fields()
-            .iter()
-            .map(|field| field.name().as_str())
-            .collect::<Vec<_>>();
-        assert_eq!(
-            catalog_column_names, provider_field_names,
-            "{surface_name} column order"
-        );
-
-        let catalog_schema = catalog
-            .surface_schema(surface_name)
-            .unwrap_or_else(|| panic!("{surface_name} should be in catalog"));
-        assert_eq!(
-            catalog_schema.fields(),
-            provider_schema.fields(),
-            "{surface_name}"
-        );
-    }
-
-    async fn empty_history_query_source(
-    ) -> crate::sql2::SqlHistoryQuerySource<StorageReadScope<InMemoryStorageRead>> {
-        let storage = StorageContext::new(InMemoryStorageBackend::new());
-        let read_scope = storage
-            .begin_read(StorageReadOptions::default())
-            .expect("read should open");
-        HistoryQuerySource {
-            json_reader: JsonStoreContext::new().reader(read_scope.store()),
-        }
-    }
-
-    struct EmptyLiveStateReader;
-
-    #[async_trait]
-    impl LiveStateReader for EmptyLiveStateReader {
-        async fn scan_rows(
-            &self,
-            _request: &LiveStateScanRequest,
-        ) -> Result<Vec<MaterializedLiveStateRow>, LixError> {
-            Ok(Vec::new())
-        }
-
-        async fn load_row(
-            &self,
-            _request: &LiveStateRowRequest,
-        ) -> Result<Option<MaterializedLiveStateRow>, LixError> {
-            Ok(None)
-        }
-    }
-
-    struct EmptyBranchRefReader;
-
-    #[async_trait]
-    impl BranchRefReader for EmptyBranchRefReader {
-        async fn load_head(&self, branch_id: &str) -> Result<Option<BranchHead>, LixError> {
-            Ok(Some(BranchHead {
-                branch_id: branch_id.to_string(),
-                commit_id: format!("commit-{branch_id}"),
-            }))
-        }
-
-        async fn scan_heads(&self) -> Result<Vec<BranchHead>, LixError> {
-            Ok(Vec::new())
-        }
-    }
-
-    struct EmptyCommitGraphReader;
-
-    #[async_trait]
-    impl CommitGraphReader for EmptyCommitGraphReader {
-        async fn load_commit(
-            &mut self,
-            _commit_id: &str,
-        ) -> Result<Option<CommitGraphCommit>, LixError> {
-            Ok(None)
-        }
-
-        async fn reachable_commits(
-            &mut self,
-            _head_commit_id: &str,
-        ) -> Result<Vec<ReachableCommitGraphCommit>, LixError> {
-            Ok(Vec::new())
-        }
-
-        async fn change_history_from_commit(
-            &mut self,
-            _start_commit_id: &str,
-            _request: &CommitGraphChangeHistoryRequest,
-        ) -> Result<Vec<CommitGraphChangeHistoryEntry>, LixError> {
-            Ok(Vec::new())
-        }
-    }
-}

package/dist-engine-src/src/sql2/read_only.rs

@@ -1,63 +0,0 @@
-use datafusion::error::DataFusionError;
-
-use crate::transaction::types::TransactionWriteRow;
-use crate::LixError;
-
-pub(crate) fn reject_read_only_entity_surface(
-    schema_key: &str,
-    action: &str,
-) -> Result<(), DataFusionError> {
-    if schema_key == "lix_directory_descriptor" {
-        return Err(read_only_error(
-            action,
-            schema_key,
-            "Use the writable lix_directory surface to create, update, or delete directories.",
-        ));
-    }
-    if let Some(message) = read_only_schema_message(schema_key) {
-        return Err(read_only_error(action, schema_key, message));
-    }
-    Ok(())
-}
-
-pub(crate) fn reject_read_only_stage_rows(
-    rows: &[TransactionWriteRow],
-    action: &str,
-) -> Result<(), DataFusionError> {
-    for row in rows {
-        if let Some(message) = read_only_schema_message(&row.schema_key) {
-            return Err(read_only_error(action, &row.schema_key, message));
-        }
-    }
-    Ok(())
-}
-
-fn read_only_error(action: &str, schema_key: &str, message: &'static str) -> DataFusionError {
-    super::error::lix_error_to_datafusion_error(
-        LixError::new(
-            LixError::CODE_READ_ONLY,
-            format!("{action} cannot write read-only surface '{schema_key}'"),
-        )
-        .with_hint(message),
-    )
-}
-
-fn read_only_schema_message(schema_key: &str) -> Option<&'static str> {
-    match schema_key {
-        "lix_branch_descriptor" | "lix_branch_ref" => {
-            Some("Use the writable lix_branch surface to create, update, or delete branches.")
-        }
-        "lix_file_descriptor" => {
-            Some("Use the writable lix_file surface to create, update, or delete files.")
-        }
-        "lix_binary_blob_ref" => {
-            Some("Use the writable lix_file data column to create, update, or delete file contents.")
-        }
-        "lix_commit"
-        | "lix_commit_edge"
-        | "lix_change" => Some(
-            "Commit graph and changelog surfaces are read-only; Lix creates them when transactions commit.",
-        ),
-        _ => None,
-    }
-}

package/dist-engine-src/src/sql2/record_batch.rs

@@ -1,17 +0,0 @@
-use datafusion::arrow::array::ArrayRef;
-use datafusion::arrow::datatypes::SchemaRef;
-use datafusion::arrow::record_batch::{RecordBatch, RecordBatchOptions};
-use datafusion::common::{DataFusionError, Result};
-
-pub(crate) fn record_batch_with_row_count(
-    schema: SchemaRef,
-    columns: Vec<ArrayRef>,
-    row_count: usize,
-) -> Result<RecordBatch> {
-    if schema.fields().is_empty() {
-        let options = RecordBatchOptions::new().with_row_count(Some(row_count));
-        return RecordBatch::try_new_with_options(schema, columns, &options)
-            .map_err(DataFusionError::from);
-    }
-    RecordBatch::try_new(schema, columns).map_err(DataFusionError::from)
-}

package/dist-engine-src/src/sql2/result_metadata.rs

@@ -1,29 +0,0 @@
-use std::collections::HashMap;
-
-use datafusion::arrow::datatypes::Field;
-
-pub(crate) const LIX_VALUE_TYPE_METADATA_KEY: &str = "lix.value_type";
-pub(crate) const LIX_VALUE_TYPE_JSON: &str = "json";
-
-pub(crate) fn json_field(name: impl Into<String>, nullable: bool) -> Field {
-    Field::new(name, datafusion::arrow::datatypes::DataType::Utf8, nullable)
-        .with_metadata(json_field_metadata_map())
-}
-
-pub(crate) fn mark_json_field(field: Field) -> Field {
-    field.with_metadata(json_field_metadata_map())
-}
-
-pub(crate) fn field_is_json(field: &Field) -> bool {
-    field
-        .metadata()
-        .get(LIX_VALUE_TYPE_METADATA_KEY)
-        .is_some_and(|value| value == LIX_VALUE_TYPE_JSON)
-}
-
-fn json_field_metadata_map() -> HashMap<String, String> {
-    HashMap::from([(
-        LIX_VALUE_TYPE_METADATA_KEY.to_string(),
-        LIX_VALUE_TYPE_JSON.to_string(),
-    )])
-}

package/dist-engine-src/src/sql2/runtime.rs

@@ -1,60 +0,0 @@
-use std::sync::Arc;
-
-use datafusion::arrow::record_batch::RecordBatch;
-use datafusion::dataframe::DataFrame;
-use datafusion::error::Result;
-use datafusion::execution::TaskContext;
-use datafusion::physical_plan::{ExecutionPlan, ExecutionPlanProperties};
-use futures_util::TryStreamExt;
-
-pub(crate) async fn collect_dataframe(dataframe: DataFrame) -> Result<Vec<RecordBatch>> {
-    let task_ctx = Arc::new(dataframe.task_ctx());
-    let plan = dataframe.create_physical_plan().await?;
-    collect_input_plan(plan, task_ctx).await
-}
-
-pub(crate) async fn collect_input_plan(
-    plan: Arc<dyn ExecutionPlan>,
-    task_ctx: Arc<TaskContext>,
-) -> Result<Vec<RecordBatch>> {
-    validate_physical_plan(&plan)?;
-    let partition_count = plan.output_partitioning().partition_count();
-    let mut batches = Vec::new();
-    for partition in 0..partition_count {
-        let partition_batches = plan
-            .execute(partition, Arc::clone(&task_ctx))?
-            .try_collect::<Vec<_>>()
-            .await?;
-        batches.extend(partition_batches);
-    }
-    Ok(batches)
-}
-
-#[cfg(not(target_arch = "wasm32"))]
-fn validate_physical_plan(_plan: &Arc<dyn ExecutionPlan>) -> Result<()> {
-    Ok(())
-}
-
-#[cfg(target_arch = "wasm32")]
-fn validate_physical_plan(plan: &Arc<dyn ExecutionPlan>) -> Result<()> {
-    let operator_name = plan.name();
-    if is_wasm_unsafe_operator(operator_name) {
-        return Err(datafusion::error::DataFusionError::Plan(format!(
-            "SQL physical operator '{operator_name}' is not supported by the WebAssembly runtime yet"
-        )));
-    }
-
-    for child in plan.children() {
-        validate_physical_plan(child)?;
-    }
-
-    Ok(())
-}
-
-#[cfg(target_arch = "wasm32")]
-fn is_wasm_unsafe_operator(operator_name: &str) -> bool {
-    matches!(
-        operator_name,
-        "CoalescePartitionsExec" | "RepartitionExec" | "SortPreservingMergeExec"
-    )
-}