@lix-js/sdk 0.6.0-preview.1 → 0.6.0-preview.3

package/dist-engine-src/src/sql2/public_bind/table.rs

@@ -0,0 +1,168 @@
+use std::collections::{BTreeMap, BTreeSet};
+
+use serde_json::Value as JsonValue;
+
+use crate::schema::schema_key_from_definition;
+use crate::LixError;
+
+#[derive(Clone, Copy, Debug, Eq, PartialEq)]
+pub(crate) enum Capability {
+    Allowed,
+    ReadOnly(&'static str),
+    Unsupported(&'static str),
+}
+
+#[derive(Clone, Debug)]
+pub(crate) struct ColumnContract {
+    pub(crate) writable: bool,
+}
+
+#[derive(Clone, Debug)]
+pub(crate) struct TableContract {
+    pub(crate) insert: Capability,
+    pub(crate) update: Capability,
+    pub(crate) delete: Capability,
+    pub(crate) columns: BTreeMap<String, ColumnContract>,
+}
+
+impl TableContract {
+    pub(crate) fn operation(&self, operation: super::DmlOperation) -> Capability {
+        match operation {
+            super::DmlOperation::Insert => self.insert,
+            super::DmlOperation::Update => self.update,
+            super::DmlOperation::Delete => self.delete,
+        }
+    }
+
+    pub(crate) fn column(&self, column: &str) -> Option<&ColumnContract> {
+        self.columns.get(column)
+    }
+}
+
+#[derive(Clone, Debug, Eq, PartialEq, Ord, PartialOrd)]
+pub(crate) struct PublicSurface {
+    name: String,
+}
+
+impl PublicSurface {
+    pub(crate) fn named(name: impl Into<String>) -> Self {
+        Self {
+            name: name.into().to_ascii_lowercase(),
+        }
+    }
+
+    pub(crate) fn name(&self) -> &str {
+        &self.name
+    }
+}
+
+#[derive(Clone, Debug)]
+pub(crate) struct PublicTableContracts {
+    contracts: BTreeMap<String, TableContract>,
+}
+
+impl PublicTableContracts {
+    pub(crate) fn new(visible_schemas: &[JsonValue]) -> Result<Self, LixError> {
+        let mut contracts = builtin_contracts();
+        for schema in visible_schemas {
+            let schema_key = schema_key_from_definition(schema)?.schema_key;
+            contracts.insert(
+                format!("{}_history", schema_key.to_ascii_lowercase()),
+                history_contract(),
+            );
+        }
+        Ok(Self { contracts })
+    }
+
+    pub(crate) fn get(&self, surface: &PublicSurface) -> Option<&TableContract> {
+        self.contracts.get(surface.name())
+    }
+}
+
+fn builtin_contracts() -> BTreeMap<String, TableContract> {
+    let mut contracts = BTreeMap::new();
+
+    for table in [
+        "lix_change",
+        "lix_commit",
+        "lix_commit_by_version",
+        "lix_commit_edge",
+        "lix_commit_edge_by_version",
+        "lix_change_set",
+        "lix_change_set_by_version",
+        "lix_change_set_element",
+        "lix_change_set_element_by_version",
+    ] {
+        contracts.insert(table.to_string(), commit_graph_contract());
+    }
+
+    for table in [
+        "lix_state_history",
+        "lix_file_history",
+        "lix_directory_history",
+    ] {
+        contracts.insert(table.to_string(), history_contract());
+    }
+
+    contracts.insert(
+        "lix_registered_schema".to_string(),
+        TableContract {
+            insert: Capability::Allowed,
+            update: Capability::Allowed,
+            delete: Capability::Unsupported(
+                "lix_registered_schema deletion is not supported; register an amended schema instead",
+            ),
+            columns: columns(&["value", "lixcol_metadata", "lixcol_global", "lixcol_untracked"]),
+        },
+    );
+
+    contracts.insert(
+        "lix_key_value".to_string(),
+        TableContract {
+            insert: Capability::Allowed,
+            update: Capability::Allowed,
+            delete: Capability::Allowed,
+            columns: columns(&["key", "value", "lixcol_metadata"]),
+        },
+    );
+
+    contracts
+}
+
+fn commit_graph_contract() -> TableContract {
+    TableContract {
+        insert: Capability::ReadOnly(
+            "Commit graph and changelog surfaces are read-only; Lix creates them when transactions commit.",
+        ),
+        update: Capability::ReadOnly(
+            "Commit graph and changelog surfaces are read-only; Lix creates them when transactions commit.",
+        ),
+        delete: Capability::ReadOnly(
+            "Commit graph and changelog surfaces are read-only; Lix creates them when transactions commit.",
+        ),
+        columns: BTreeMap::new(),
+    }
+}
+
+fn history_contract() -> TableContract {
+    TableContract {
+        insert: Capability::ReadOnly(
+            "History views are query-only; write to the live surface such as lix_state, lix_file, lix_directory, or the typed entity table.",
+        ),
+        update: Capability::ReadOnly(
+            "History views are query-only; write to the live surface such as lix_state, lix_file, lix_directory, or the typed entity table.",
+        ),
+        delete: Capability::ReadOnly(
+            "History views are query-only; write to the live surface such as lix_state, lix_file, lix_directory, or the typed entity table.",
+        ),
+        columns: BTreeMap::new(),
+    }
+}
+
+fn columns(writable: &[&str]) -> BTreeMap<String, ColumnContract> {
+    let writable = writable.iter().copied().collect::<BTreeSet<_>>();
+    writable
+        .into_iter()
+        .map(|column| (column.to_string(), ColumnContract { writable: true }))
+        .collect()
+}

package/dist-engine-src/src/sql2/read_only.rs

@@ -0,0 +1,63 @@
+use datafusion::error::DataFusionError;
+
+use crate::transaction::types::TransactionWriteRow;
+use crate::LixError;
+
+pub(crate) fn reject_read_only_entity_surface(
+    schema_key: &str,
+    action: &str,
+) -> Result<(), DataFusionError> {
+    if schema_key == "lix_directory_descriptor" {
+        return Err(read_only_error(
+            action,
+            schema_key,
+            "Use the writable lix_directory surface to create, update, or delete directories.",
+        ));
+    }
+    if let Some(message) = read_only_schema_message(schema_key) {
+        return Err(read_only_error(action, schema_key, message));
+    }
+    Ok(())
+}
+
+pub(crate) fn reject_read_only_stage_rows(
+    rows: &[TransactionWriteRow],
+    action: &str,
+) -> Result<(), DataFusionError> {
+    for row in rows {
+        if let Some(message) = read_only_schema_message(&row.schema_key) {
+            return Err(read_only_error(action, &row.schema_key, message));
+        }
+    }
+    Ok(())
+}
+
+fn read_only_error(action: &str, schema_key: &str, message: &'static str) -> DataFusionError {
+    super::error::lix_error_to_datafusion_error(
+        LixError::new(
+            LixError::CODE_READ_ONLY,
+            format!("{action} cannot write read-only surface '{schema_key}'"),
+        )
+        .with_hint(message),
+    )
+}
+
+fn read_only_schema_message(schema_key: &str) -> Option<&'static str> {
+    match schema_key {
+        "lix_version_descriptor" | "lix_version_ref" => {
+            Some("Use the writable lix_version surface to create, update, or delete versions.")
+        }
+        "lix_file_descriptor" => {
+            Some("Use the writable lix_file surface to create, update, or delete files.")
+        }
+        "lix_binary_blob_ref" => {
+            Some("Use the writable lix_file data column to create, update, or delete file contents.")
+        }
+        "lix_commit"
+        | "lix_commit_edge"
+        | "lix_change" => Some(
+            "Commit graph and changelog surfaces are read-only; Lix creates them when transactions commit.",
+        ),
+        _ => None,
+    }
+}

package/dist-engine-src/src/sql2/record_batch.rs

@@ -0,0 +1,17 @@
+use datafusion::arrow::array::ArrayRef;
+use datafusion::arrow::datatypes::SchemaRef;
+use datafusion::arrow::record_batch::{RecordBatch, RecordBatchOptions};
+use datafusion::common::{DataFusionError, Result};
+
+pub(crate) fn record_batch_with_row_count(
+    schema: SchemaRef,
+    columns: Vec<ArrayRef>,
+    row_count: usize,
+) -> Result<RecordBatch> {
+    if schema.fields().is_empty() {
+        let options = RecordBatchOptions::new().with_row_count(Some(row_count));
+        return RecordBatch::try_new_with_options(schema, columns, &options)
+            .map_err(DataFusionError::from);
+    }
+    RecordBatch::try_new(schema, columns).map_err(DataFusionError::from)
+}

package/dist-engine-src/src/sql2/result_metadata.rs

@@ -0,0 +1,29 @@
+use std::collections::HashMap;
+
+use datafusion::arrow::datatypes::Field;
+
+pub(crate) const LIX_VALUE_TYPE_METADATA_KEY: &str = "lix.value_type";
+pub(crate) const LIX_VALUE_TYPE_JSON: &str = "json";
+
+pub(crate) fn json_field(name: impl Into<String>, nullable: bool) -> Field {
+    Field::new(name, datafusion::arrow::datatypes::DataType::Utf8, nullable)
+        .with_metadata(json_field_metadata_map())
+}
+
+pub(crate) fn mark_json_field(field: Field) -> Field {
+    field.with_metadata(json_field_metadata_map())
+}
+
+pub(crate) fn field_is_json(field: &Field) -> bool {
+    field
+        .metadata()
+        .get(LIX_VALUE_TYPE_METADATA_KEY)
+        .is_some_and(|value| value == LIX_VALUE_TYPE_JSON)
+}
+
+fn json_field_metadata_map() -> HashMap<String, String> {
+    HashMap::from([(
+        LIX_VALUE_TYPE_METADATA_KEY.to_string(),
+        LIX_VALUE_TYPE_JSON.to_string(),
+    )])
+}

package/dist-engine-src/src/sql2/runtime.rs

@@ -0,0 +1,60 @@
+use std::sync::Arc;
+
+use datafusion::arrow::record_batch::RecordBatch;
+use datafusion::dataframe::DataFrame;
+use datafusion::error::Result;
+use datafusion::execution::TaskContext;
+use datafusion::physical_plan::{ExecutionPlan, ExecutionPlanProperties};
+use futures_util::TryStreamExt;
+
+pub(crate) async fn collect_dataframe(dataframe: DataFrame) -> Result<Vec<RecordBatch>> {
+    let task_ctx = Arc::new(dataframe.task_ctx());
+    let plan = dataframe.create_physical_plan().await?;
+    collect_input_plan(plan, task_ctx).await
+}
+
+pub(crate) async fn collect_input_plan(
+    plan: Arc<dyn ExecutionPlan>,
+    task_ctx: Arc<TaskContext>,
+) -> Result<Vec<RecordBatch>> {
+    validate_physical_plan(&plan)?;
+    let partition_count = plan.output_partitioning().partition_count();
+    let mut batches = Vec::new();
+    for partition in 0..partition_count {
+        let partition_batches = plan
+            .execute(partition, Arc::clone(&task_ctx))?
+            .try_collect::<Vec<_>>()
+            .await?;
+        batches.extend(partition_batches);
+    }
+    Ok(batches)
+}
+
+#[cfg(not(target_arch = "wasm32"))]
+fn validate_physical_plan(_plan: &Arc<dyn ExecutionPlan>) -> Result<()> {
+    Ok(())
+}
+
+#[cfg(target_arch = "wasm32")]
+fn validate_physical_plan(plan: &Arc<dyn ExecutionPlan>) -> Result<()> {
+    let operator_name = plan.name();
+    if is_wasm_unsafe_operator(operator_name) {
+        return Err(datafusion::error::DataFusionError::Plan(format!(
+            "SQL physical operator '{operator_name}' is not supported by the WebAssembly runtime yet"
+        )));
+    }
+
+    for child in plan.children() {
+        validate_physical_plan(child)?;
+    }
+
+    Ok(())
+}
+
+#[cfg(target_arch = "wasm32")]
+fn is_wasm_unsafe_operator(operator_name: &str) -> bool {
+    matches!(
+        operator_name,
+        "CoalescePartitionsExec" | "RepartitionExec" | "SortPreservingMergeExec"
+    )
+}

package/dist-engine-src/src/sql2/session.rs

@@ -0,0 +1,132 @@
+use std::sync::Arc;
+
+use datafusion::prelude::{SessionConfig, SessionContext};
+
+use crate::LixError;
+
+use super::change_provider::register_lix_change_provider;
+use super::directory_history_provider::register_lix_directory_history_provider;
+use super::directory_provider::{
+    register_lix_directory_providers, register_lix_directory_write_providers,
+};
+use super::entity_provider::{register_entity_providers, register_entity_write_providers};
+use super::file_history_provider::register_lix_file_history_provider;
+use super::file_provider::{register_lix_file_providers, register_lix_file_write_providers};
+use super::history_provider::register_history_providers;
+use super::lix_state_provider::{register_lix_state_providers, register_lix_state_write_providers};
+use super::udfs::register_sql2_functions;
+use super::version_provider::{register_lix_version_provider, register_lix_version_write_provider};
+use super::{SqlExecutionContext, SqlWriteContext, SqlWriteExecutionContext};
+
+pub(crate) async fn build_read_session(
+    ctx: &dyn SqlExecutionContext,
+) -> Result<SessionContext, LixError> {
+    let session = new_sql_session_context();
+    let version_ref = ctx.version_ref();
+    let active_version_commit_id = version_ref
+        .load_head(ctx.active_version_id())
+        .await?
+        .map(|head| head.commit_id);
+    register_sql2_functions(&session, ctx.functions(), active_version_commit_id);
+    register_lix_state_providers(
+        &session,
+        ctx.active_version_id(),
+        ctx.live_state(),
+        Arc::clone(&version_ref),
+    )
+    .await?;
+    register_lix_version_provider(&session, ctx.live_state(), Arc::clone(&version_ref)).await?;
+    let commit_store_query_source = ctx.commit_store_query_source();
+    register_lix_change_provider(&session, commit_store_query_source.clone()).await?;
+    let state_history_commit_graph = ctx.commit_graph();
+    register_history_providers(
+        &session,
+        state_history_commit_graph,
+        commit_store_query_source.clone(),
+    )
+    .await?;
+    let file_history_commit_graph = ctx.commit_graph();
+    register_lix_file_history_provider(
+        &session,
+        file_history_commit_graph,
+        commit_store_query_source.clone(),
+        ctx.blob_reader(),
+    )
+    .await?;
+    let directory_history_commit_graph = ctx.commit_graph();
+    register_lix_directory_history_provider(
+        &session,
+        directory_history_commit_graph,
+        commit_store_query_source.clone(),
+    )
+    .await?;
+    let entity_commit_graph = Arc::new(tokio::sync::Mutex::new(ctx.commit_graph()));
+    register_lix_directory_providers(
+        &session,
+        ctx.active_version_id(),
+        ctx.live_state(),
+        Arc::clone(&version_ref),
+        ctx.functions(),
+    )
+    .await?;
+    register_lix_file_providers(
+        &session,
+        ctx.active_version_id(),
+        ctx.live_state(),
+        Arc::clone(&version_ref),
+        ctx.blob_reader(),
+        ctx.functions(),
+    )
+    .await?;
+    register_entity_providers(
+        &session,
+        ctx.active_version_id(),
+        ctx.live_state(),
+        Arc::clone(&version_ref),
+        entity_commit_graph,
+        commit_store_query_source,
+        &ctx.list_visible_schemas()?,
+    )
+    .await?;
+
+    Ok(session)
+}
+
+pub(crate) async fn build_write_session(
+    ctx: &mut dyn SqlWriteExecutionContext,
+) -> Result<SessionContext, LixError> {
+    let session = new_sql_session_context();
+    let write_ctx = SqlWriteContext::new(ctx);
+    let active_version_commit_id = write_ctx
+        .load_version_head(&write_ctx.active_version_id())
+        .await?;
+    register_sql2_functions(&session, write_ctx.functions(), active_version_commit_id);
+
+    register_lix_state_write_providers(&session, write_ctx.clone()).await?;
+    register_lix_version_write_provider(&session, write_ctx.clone()).await?;
+
+    register_lix_directory_write_providers(&session, write_ctx.clone()).await?;
+    register_lix_file_write_providers(&session, write_ctx.clone()).await?;
+    register_entity_write_providers(
+        &session,
+        write_ctx.clone(),
+        &write_ctx.list_visible_schemas()?,
+    )
+    .await?;
+
+    Ok(session)
+}
+
+pub(crate) fn new_sql_session_context() -> SessionContext {
+    SessionContext::new_with_config(
+        SessionConfig::new()
+            .with_information_schema(true)
+            .with_target_partitions(1)
+            .set_bool("datafusion.optimizer.repartition_aggregations", false)
+            .set_bool("datafusion.optimizer.repartition_joins", false)
+            .set_bool("datafusion.optimizer.repartition_sorts", false)
+            .set_bool("datafusion.optimizer.repartition_windows", false)
+            .set_bool("datafusion.optimizer.repartition_file_scans", false)
+            .set_bool("datafusion.optimizer.enable_round_robin_repartition", false),
+    )
+}