@lix-js/sdk 0.6.0-preview.1 → 0.6.0-preview.3

package/dist-engine-src/src/sql2/mod.rs

@@ -0,0 +1,46 @@
+mod change_provider;
+mod classify;
+mod context;
+mod directory_history_provider;
+mod directory_provider;
+mod dml;
+mod entity_history_provider;
+mod entity_provider;
+mod error;
+mod execute;
+mod file_history_provider;
+mod file_provider;
+mod filesystem_planner;
+mod filesystem_predicates;
+mod filesystem_visibility;
+mod history_projection;
+mod history_provider;
+mod history_route;
+mod lix_state_provider;
+mod predicate_typecheck;
+mod public_bind;
+mod read_only;
+mod record_batch;
+mod result_metadata;
+mod runtime;
+mod session;
+mod udfs;
+mod version_provider;
+mod version_scope;
+mod write_normalization;
+
+pub(crate) use classify::{
+    classify_statement, datafusion_statement_dml_target_table_names,
+    validate_supported_datafusion_statement_ast, validate_supported_statement_ast,
+    SqlStatementKind,
+};
+pub(crate) use context::{
+    CommitStoreQuerySource, SqlCommitStoreQuerySource, SqlExecutionContext, SqlJsonReader,
+    SqlWriteContext, SqlWriteExecutionContext, WriteAccess, WriteContextLiveStateReader,
+    WriteContextVersionRefReader,
+};
+#[allow(unused_imports)]
+pub(crate) use execute::{
+    create_logical_plan, create_write_logical_plan, execute_logical_plan, execute_sql,
+    SqlLogicalPlan,
+};

package/dist-engine-src/src/sql2/predicate_typecheck.rs

@@ -0,0 +1,246 @@
+use datafusion::arrow::datatypes::{Field, Schema};
+use datafusion::common::{DFSchema, DataFusionError, ScalarValue};
+use datafusion::logical_expr::expr::{Between, InList};
+use datafusion::logical_expr::{BinaryExpr, Expr, Like, Operator};
+
+use crate::LixError;
+
+use super::error::lix_error_to_datafusion_error;
+use super::result_metadata::{field_is_json, LIX_VALUE_TYPE_JSON, LIX_VALUE_TYPE_METADATA_KEY};
+
+pub(crate) fn validate_json_predicate_filters(
+    schema: &Schema,
+    filters: &[Expr],
+) -> Result<(), DataFusionError> {
+    for filter in filters {
+        validate_json_predicate_expr_with_arrow_schema(schema, filter)
+            .map_err(lix_error_to_datafusion_error)?;
+    }
+    Ok(())
+}
+
+pub(crate) fn validate_json_predicate_expr_with_dfschema(
+    schema: &DFSchema,
+    expr: &Expr,
+) -> Result<(), LixError> {
+    validate_expr(expr, &|column| {
+        schema
+            .field_with_name(column.relation.as_ref(), &column.name)
+            .ok()
+            .map(|field| field.as_ref())
+    })
+}
+
+fn validate_json_predicate_expr_with_arrow_schema(
+    schema: &Schema,
+    expr: &Expr,
+) -> Result<(), LixError> {
+    validate_expr(expr, &|column| {
+        schema
+            .fields()
+            .iter()
+            .find(|field| field.name() == &column.name)
+            .map(|field| field.as_ref())
+    })
+}
+
+fn validate_expr<'a>(
+    expr: &'a Expr,
+    lookup_field: &impl Fn(&datafusion::common::Column) -> Option<&'a Field>,
+) -> Result<(), LixError> {
+    match expr {
+        Expr::BinaryExpr(binary) => validate_binary_expr(binary, lookup_field),
+        Expr::InList(in_list) => validate_in_list(in_list, lookup_field),
+        Expr::Between(between) => validate_between(between, lookup_field),
+        Expr::Like(like) | Expr::SimilarTo(like) => validate_like(like, lookup_field),
+        Expr::Alias(alias) => validate_expr(&alias.expr, lookup_field),
+        Expr::Not(inner)
+        | Expr::IsNotNull(inner)
+        | Expr::IsNull(inner)
+        | Expr::IsTrue(inner)
+        | Expr::IsFalse(inner)
+        | Expr::IsUnknown(inner)
+        | Expr::IsNotTrue(inner)
+        | Expr::IsNotFalse(inner)
+        | Expr::IsNotUnknown(inner)
+        | Expr::Negative(inner) => validate_expr(inner, lookup_field),
+        Expr::Cast(cast) => validate_expr(&cast.expr, lookup_field),
+        Expr::TryCast(cast) => validate_expr(&cast.expr, lookup_field),
+        Expr::ScalarFunction(function) => {
+            for arg in &function.args {
+                validate_expr(arg, lookup_field)?;
+            }
+            Ok(())
+        }
+        Expr::Case(case) => {
+            if let Some(expr) = &case.expr {
+                validate_expr(expr, lookup_field)?;
+            }
+            for (when, then) in &case.when_then_expr {
+                validate_expr(when, lookup_field)?;
+                validate_expr(then, lookup_field)?;
+            }
+            if let Some(expr) = &case.else_expr {
+                validate_expr(expr, lookup_field)?;
+            }
+            Ok(())
+        }
+        Expr::AggregateFunction(function) => {
+            for arg in &function.params.args {
+                validate_expr(arg, lookup_field)?;
+            }
+            Ok(())
+        }
+        Expr::WindowFunction(function) => {
+            for arg in &function.params.args {
+                validate_expr(arg, lookup_field)?;
+            }
+            Ok(())
+        }
+        _ => Ok(()),
+    }
+}
+
+fn validate_binary_expr<'a>(
+    binary: &'a BinaryExpr,
+    lookup_field: &impl Fn(&datafusion::common::Column) -> Option<&'a Field>,
+) -> Result<(), LixError> {
+    validate_expr(&binary.left, lookup_field)?;
+    validate_expr(&binary.right, lookup_field)?;
+
+    if !is_comparison_operator(binary.op) {
+        return Ok(());
+    }
+
+    validate_comparison_operands(&binary.left, &binary.right, lookup_field)
+}
+
+fn validate_in_list<'a>(
+    in_list: &'a InList,
+    lookup_field: &impl Fn(&datafusion::common::Column) -> Option<&'a Field>,
+) -> Result<(), LixError> {
+    validate_expr(&in_list.expr, lookup_field)?;
+    for item in &in_list.list {
+        validate_expr(item, lookup_field)?;
+    }
+
+    if is_json_expr(&in_list.expr, lookup_field) {
+        for item in &in_list.list {
+            require_json_comparison_operand(item, lookup_field)?;
+        }
+    }
+
+    for item in &in_list.list {
+        if is_json_expr(item, lookup_field) {
+            require_json_comparison_operand(&in_list.expr, lookup_field)?;
+        }
+    }
+
+    Ok(())
+}
+
+fn validate_between<'a>(
+    between: &'a Between,
+    lookup_field: &impl Fn(&datafusion::common::Column) -> Option<&'a Field>,
+) -> Result<(), LixError> {
+    validate_expr(&between.expr, lookup_field)?;
+    validate_expr(&between.low, lookup_field)?;
+    validate_expr(&between.high, lookup_field)?;
+
+    if is_json_expr(&between.expr, lookup_field) {
+        require_json_comparison_operand(&between.low, lookup_field)?;
+        require_json_comparison_operand(&between.high, lookup_field)?;
+    }
+
+    Ok(())
+}
+
+fn validate_like<'a>(
+    like: &'a Like,
+    lookup_field: &impl Fn(&datafusion::common::Column) -> Option<&'a Field>,
+) -> Result<(), LixError> {
+    validate_expr(&like.expr, lookup_field)?;
+    validate_expr(&like.pattern, lookup_field)?;
+
+    if is_json_expr(&like.expr, lookup_field) {
+        return Err(json_predicate_type_error(&like.expr));
+    }
+
+    Ok(())
+}
+
+fn validate_comparison_operands<'a>(
+    left: &'a Expr,
+    right: &'a Expr,
+    lookup_field: &impl Fn(&datafusion::common::Column) -> Option<&'a Field>,
+) -> Result<(), LixError> {
+    let left_is_json = is_json_expr(left, lookup_field);
+    let right_is_json = is_json_expr(right, lookup_field);
+
+    if left_is_json {
+        require_json_comparison_operand(right, lookup_field)?;
+    }
+    if right_is_json {
+        require_json_comparison_operand(left, lookup_field)?;
+    }
+
+    Ok(())
+}
+
+fn require_json_comparison_operand<'a>(
+    expr: &'a Expr,
+    lookup_field: &impl Fn(&datafusion::common::Column) -> Option<&'a Field>,
+) -> Result<(), LixError> {
+    if is_json_expr(expr, lookup_field)
+        || is_null_literal(expr)
+        || matches!(expr, Expr::Placeholder(_))
+    {
+        return Ok(());
+    }
+
+    Err(json_predicate_type_error(expr))
+}
+
+fn is_json_expr<'a>(
+    expr: &'a Expr,
+    lookup_field: &impl Fn(&datafusion::common::Column) -> Option<&'a Field>,
+) -> bool {
+    match expr {
+        Expr::Column(column) => lookup_field(column).is_some_and(field_is_json),
+        Expr::Literal(_, Some(metadata)) => metadata
+            .inner()
+            .get(LIX_VALUE_TYPE_METADATA_KEY)
+            .is_some_and(|value| value == LIX_VALUE_TYPE_JSON),
+        Expr::ScalarFunction(function) => matches!(function.name(), "lix_json" | "lix_json_get"),
+        Expr::Alias(alias) => is_json_expr(&alias.expr, lookup_field),
+        Expr::Cast(cast) => is_json_expr(&cast.expr, lookup_field),
+        Expr::TryCast(cast) => is_json_expr(&cast.expr, lookup_field),
+        _ => false,
+    }
+}
+
+fn is_null_literal(expr: &Expr) -> bool {
+    matches!(expr, Expr::Literal(value, _) if matches!(value, ScalarValue::Null))
+}
+
+fn is_comparison_operator(op: Operator) -> bool {
+    matches!(
+        op,
+        Operator::Eq
+            | Operator::NotEq
+            | Operator::Lt
+            | Operator::LtEq
+            | Operator::Gt
+            | Operator::GtEq
+            | Operator::IsDistinctFrom
+            | Operator::IsNotDistinctFrom
+    )
+}
+
+fn json_predicate_type_error(expr: &Expr) -> LixError {
+    LixError::new(
+        LixError::CODE_TYPE_MISMATCH,
+        format!("JSON columns can only be compared with JSON expressions, got {expr}"),
+    )
+    .with_hint("Wrap JSON text with lix_json(...), use lix_json_get(...) for JSON values, or use IS NULL for null checks.")
+}

package/dist-engine-src/src/sql2/public_bind/assignment.rs

@@ -0,0 +1,46 @@
+use std::collections::BTreeSet;
+
+use crate::LixError;
+
+use super::table::{PublicSurface, PublicTableContracts};
+
+pub(crate) fn validate_update_assignments(
+    surface: &PublicSurface,
+    columns: Vec<String>,
+    contracts: &PublicTableContracts,
+) -> Result<(), LixError> {
+    let Some(contract) = contracts.get(surface) else {
+        return Ok(());
+    };
+    let mut seen = BTreeSet::new();
+    for column in columns {
+        if !seen.insert(column.clone()) {
+            return Err(LixError::new(
+                LixError::CODE_INVALID_PARAM,
+                format!(
+                    "update {} assigns column '{column}' more than once",
+                    surface.name()
+                ),
+            ));
+        }
+        let Some(column_contract) = contract.column(&column) else {
+            return Err(LixError::new(
+                LixError::CODE_INVALID_PARAM,
+                format!(
+                    "update {} references unknown column '{column}'",
+                    surface.name()
+                ),
+            ));
+        };
+        if !column_contract.writable {
+            return Err(LixError::new(
+                LixError::CODE_UNSUPPORTED_SQL,
+                format!(
+                    "update {} cannot assign read-only column '{column}'",
+                    surface.name()
+                ),
+            ));
+        }
+    }
+    Ok(())
+}

package/dist-engine-src/src/sql2/public_bind/capability.rs

@@ -0,0 +1,41 @@
+use crate::LixError;
+
+use super::table::{Capability, PublicSurface, PublicTableContracts};
+use super::DmlOperation;
+
+pub(crate) fn validate_table_operation(
+    surface: &PublicSurface,
+    operation: DmlOperation,
+    contracts: &PublicTableContracts,
+) -> Result<(), LixError> {
+    let Some(contract) = contracts.get(surface) else {
+        return Ok(());
+    };
+    match contract.operation(operation) {
+        Capability::Allowed => Ok(()),
+        Capability::ReadOnly(hint) => {
+            let message = if surface.name().ends_with("_history") {
+                format!(
+                    "DML cannot write read-only history view '{}'",
+                    surface.name()
+                )
+            } else {
+                format!(
+                    "{} {} is not allowed because the SQL surface is read-only",
+                    operation.as_str(),
+                    surface.name()
+                )
+            };
+            Err(LixError::new(LixError::CODE_READ_ONLY, message).with_hint(hint))
+        }
+        Capability::Unsupported(hint) => Err(LixError::new(
+            LixError::CODE_UNSUPPORTED_SQL,
+            format!(
+                "{} {} is not supported by Lix SQL",
+                operation.as_str(),
+                surface.name()
+            ),
+        )
+        .with_hint(hint)),
+    }
+}

package/dist-engine-src/src/sql2/public_bind/dml.rs

@@ -0,0 +1,166 @@
+use datafusion::logical_expr::{LogicalPlan, WriteOp};
+use datafusion::sql::sqlparser::ast::{
+    Assignment, AssignmentTarget, Delete, FromTable, ObjectName, Statement, TableFactor,
+    TableObject, TableWithJoins, Update,
+};
+use datafusion::sql::sqlparser::dialect::GenericDialect;
+use datafusion::sql::sqlparser::parser::Parser;
+use serde_json::Value as JsonValue;
+
+use crate::LixError;
+
+use super::assignment::validate_update_assignments;
+use super::capability::validate_table_operation;
+use super::table::{PublicSurface, PublicTableContracts};
+
+#[derive(Clone, Copy, Debug, Eq, PartialEq)]
+pub(crate) enum DmlOperation {
+    Insert,
+    Update,
+    Delete,
+}
+
+impl DmlOperation {
+    pub(crate) fn as_str(self) -> &'static str {
+        match self {
+            Self::Insert => "insert",
+            Self::Update => "update",
+            Self::Delete => "delete",
+        }
+    }
+}
+
+pub(crate) fn validate_sql(sql: &str, visible_schemas: &[JsonValue]) -> Result<(), LixError> {
+    let statements = Parser::parse_sql(&GenericDialect {}, sql).map_err(|error| {
+        LixError::new(
+            LixError::CODE_PARSE_ERROR,
+            format!("sql2 SQL parse error: {error}"),
+        )
+    })?;
+    let [statement] = statements.as_slice() else {
+        return Ok(());
+    };
+    let contracts = PublicTableContracts::new(visible_schemas)?;
+    validate_statement(statement, &contracts)
+}
+
+pub(crate) fn validate_plan(
+    plan: &LogicalPlan,
+    visible_schemas: &[JsonValue],
+) -> Result<(), LixError> {
+    let contracts = PublicTableContracts::new(visible_schemas)?;
+    validate_plan_with_contracts(plan, &contracts)
+}
+
+fn validate_plan_with_contracts(
+    plan: &LogicalPlan,
+    contracts: &PublicTableContracts,
+) -> Result<(), LixError> {
+    if let LogicalPlan::Dml(dml) = plan {
+        let surface = PublicSurface::named(dml.table_name.table());
+        validate_table_operation(&surface, operation_from_write_op(&dml.op), contracts)?;
+    }
+    for input in plan.inputs() {
+        validate_plan_with_contracts(input, contracts)?;
+    }
+    Ok(())
+}
+
+fn operation_from_write_op(op: &WriteOp) -> DmlOperation {
+    match op {
+        WriteOp::Insert(_) | WriteOp::Ctas => DmlOperation::Insert,
+        WriteOp::Update => DmlOperation::Update,
+        WriteOp::Delete | WriteOp::Truncate => DmlOperation::Delete,
+    }
+}
+
+fn validate_statement(
+    statement: &Statement,
+    contracts: &PublicTableContracts,
+) -> Result<(), LixError> {
+    match statement {
+        Statement::Insert(insert) => {
+            let Some(table_name) = insert_target_name(&insert.table) else {
+                return Ok(());
+            };
+            let surface = PublicSurface::named(table_name);
+            validate_table_operation(&surface, DmlOperation::Insert, contracts)
+        }
+        Statement::Update(update) => validate_update(update, contracts),
+        Statement::Delete(delete) => validate_delete(delete, contracts),
+        Statement::Explain { statement, .. } => validate_statement(statement, contracts),
+        _ => Ok(()),
+    }
+}
+
+fn validate_update(update: &Update, contracts: &PublicTableContracts) -> Result<(), LixError> {
+    let Some(table_name) = table_with_joins_target_name(&update.table) else {
+        return Ok(());
+    };
+    let surface = PublicSurface::named(table_name);
+    validate_table_operation(&surface, DmlOperation::Update, contracts)?;
+    validate_update_assignments(
+        &surface,
+        assignment_column_names(&update.assignments)?,
+        contracts,
+    )
+}
+
+fn validate_delete(delete: &Delete, contracts: &PublicTableContracts) -> Result<(), LixError> {
+    for table in delete_from_tables(delete) {
+        let Some(table_name) = table_with_joins_target_name(table) else {
+            continue;
+        };
+        let surface = PublicSurface::named(table_name);
+        validate_table_operation(&surface, DmlOperation::Delete, contracts)?;
+    }
+    Ok(())
+}
+
+fn delete_from_tables(delete: &Delete) -> &[TableWithJoins] {
+    match &delete.from {
+        FromTable::WithFromKeyword(tables) | FromTable::WithoutKeyword(tables) => tables,
+    }
+}
+
+fn assignment_column_names(assignments: &[Assignment]) -> Result<Vec<String>, LixError> {
+    let mut columns = Vec::new();
+    for assignment in assignments {
+        match &assignment.target {
+            AssignmentTarget::ColumnName(name) => {
+                if let Some(column) = object_name_leaf(name) {
+                    columns.push(column);
+                }
+            }
+            AssignmentTarget::Tuple(names) => {
+                for name in names {
+                    if let Some(column) = object_name_leaf(name) {
+                        columns.push(column);
+                    }
+                }
+            }
+        }
+    }
+    Ok(columns)
+}
+
+fn insert_target_name(table: &TableObject) -> Option<String> {
+    match table {
+        TableObject::TableName(name) => object_name_leaf(name),
+        _ => None,
+    }
+}
+
+fn table_with_joins_target_name(table: &TableWithJoins) -> Option<String> {
+    match &table.relation {
+        TableFactor::Table { name, .. } => object_name_leaf(name),
+        _ => None,
+    }
+}
+
+fn object_name_leaf(name: &ObjectName) -> Option<String> {
+    name.0
+        .last()
+        .and_then(|part| part.as_ident())
+        .map(|ident| ident.value.to_ascii_lowercase())
+}

package/dist-engine-src/src/sql2/public_bind/mod.rs

@@ -0,0 +1,25 @@
+mod assignment;
+mod capability;
+mod dml;
+mod table;
+
+use datafusion::logical_expr::LogicalPlan;
+use serde_json::Value as JsonValue;
+
+use crate::LixError;
+
+pub(crate) use dml::DmlOperation;
+
+pub(crate) fn validate_public_dml_sql(
+    sql: &str,
+    visible_schemas: &[JsonValue],
+) -> Result<(), LixError> {
+    dml::validate_sql(sql, visible_schemas)
+}
+
+pub(crate) fn validate_public_dml_plan(
+    plan: &LogicalPlan,
+    visible_schemas: &[JsonValue],
+) -> Result<(), LixError> {
+    dml::validate_plan(plan, visible_schemas)
+}