deltalake-rb 0.1.1 → 0.1.2

data/ext/deltalake/src/lib.rs

@@ -1,4 +1,6 @@
 mod error;
+mod features;
+mod merge;
 mod schema;
 mod utils;
 
@@ -9,12 +11,16 @@ use std::str::FromStr;
 use std::time;
 
 use chrono::{DateTime, Duration, FixedOffset, Utc};
+use delta_kernel::schema::StructField;
 use deltalake::arrow::ffi_stream::{ArrowArrayStreamReader, FFI_ArrowArrayStream};
 use deltalake::arrow::record_batch::RecordBatchIterator;
+use deltalake::checkpoints::{cleanup_metadata, create_checkpoint};
 use deltalake::datafusion::physical_plan::ExecutionPlan;
 use deltalake::datafusion::prelude::SessionContext;
 use deltalake::errors::DeltaTableError;
 use deltalake::kernel::{scalars::ScalarExt, StructType, Transaction};
+use deltalake::operations::add_column::AddColumnBuilder;
+use deltalake::operations::add_feature::AddTableFeatureBuilder;
 use deltalake::operations::collect_sendable_stream;
 use deltalake::operations::constraints::ConstraintBuilder;
 use deltalake::operations::delete::DeleteBuilder;
@@ -23,25 +29,49 @@ use deltalake::operations::filesystem_check::FileSystemCheckBuilder;
 use deltalake::operations::load_cdf::CdfLoadBuilder;
 use deltalake::operations::optimize::{OptimizeBuilder, OptimizeType};
 use deltalake::operations::restore::RestoreBuilder;
-use deltalake::operations::transaction::TableReference;
+use deltalake::operations::set_tbl_properties::SetTablePropertiesBuilder;
+use deltalake::operations::transaction::{CommitProperties, TableReference};
 use deltalake::operations::vacuum::VacuumBuilder;
+use deltalake::parquet::basic::Compression;
+use deltalake::parquet::errors::ParquetError;
+use deltalake::parquet::file::properties::WriterProperties;
 use deltalake::partitions::PartitionFilter;
 use deltalake::storage::IORuntime;
-use deltalake::DeltaOps;
+use deltalake::{DeltaOps, DeltaResult};
 use error::DeltaError;
 use futures::future::join_all;
 
 use magnus::{
-    exception, function, method, prelude::*, Error, Integer, Module, RArray, RHash, Ruby, Value,
+    function, method, prelude::*, typed_data::Obj, Error, Integer, Module, RArray, RHash, Ruby,
+    TryConvert, Value,
 };
+use serde_json::Map;
 
 use crate::error::DeltaProtocolError;
+use crate::error::RbValueError;
 use crate::error::RubyError;
+use crate::features::TableFeatures;
+use crate::merge::RbMergeBuilder;
 use crate::schema::{schema_to_rbobject, Field};
 use crate::utils::rt;
 
 type RbResult<T> = Result<T, Error>;
 
+enum PartitionFilterValue {
+    Single(String),
+    Multiple(Vec<String>),
+}
+
+impl TryConvert for PartitionFilterValue {
+    fn try_convert(val: Value) -> RbResult<Self> {
+        if let Ok(v) = Vec::<String>::try_convert(val) {
+            Ok(PartitionFilterValue::Multiple(v))
+        } else {
+            Ok(PartitionFilterValue::Single(String::try_convert(val)?))
+        }
+    }
+}
+
 #[magnus::wrap(class = "DeltaLake::RawDeltaTable")]
 struct RawDeltaTable {
     _table: RefCell<deltalake::DeltaTable>,
@@ -57,14 +87,29 @@ struct RawDeltaTableMetaData {
     configuration: HashMap<String, Option<String>>,
 }
 
-#[magnus::wrap(class = "DeltaLake::ArrowArrayStream")]
-pub struct ArrowArrayStream {
-    stream: FFI_ArrowArrayStream,
-}
+impl RawDeltaTableMetaData {
+    fn id(&self) -> String {
+        self.id.clone()
+    }
 
-impl ArrowArrayStream {
-    pub fn to_i(&self) -> usize {
-        (&self.stream as *const _) as usize
+    fn name(&self) -> Option<String> {
+        self.name.clone()
+    }
+
+    fn description(&self) -> Option<String> {
+        self.description.clone()
+    }
+
+    fn partition_columns(&self) -> Vec<String> {
+        self.partition_columns.clone()
+    }
+
+    fn created_time(&self) -> Option<i64> {
+        self.created_time
+    }
+
+    fn configuration(&self) -> HashMap<String, Option<String>> {
+        self.configuration.clone()
     }
 }
 
@@ -218,20 +263,19 @@ impl RawDeltaTable {
     }
 
     pub fn load_with_datetime(&self, ds: String) -> RbResult<()> {
-        let datetime = DateTime::<Utc>::from(
-            DateTime::<FixedOffset>::parse_from_rfc3339(&ds).map_err(|err| {
-                Error::new(
-                    exception::arg_error(),
-                    format!("Failed to parse datetime string: {err}"),
-                )
-            })?,
-        );
+        let datetime =
+            DateTime::<Utc>::from(DateTime::<FixedOffset>::parse_from_rfc3339(&ds).map_err(
+                |err| RbValueError::new_err(format!("Failed to parse datetime string: {err}")),
+            )?);
         Ok(rt()
             .block_on(self._table.borrow_mut().load_with_datetime(datetime))
             .map_err(RubyError::from)?)
     }
 
-    pub fn files(&self, partition_filters: Option<Value>) -> RbResult<Vec<String>> {
+    pub fn files(
+        &self,
+        partition_filters: Option<Vec<(String, String, PartitionFilterValue)>>,
+    ) -> RbResult<Vec<String>> {
         if !self.has_files()? {
             return Err(DeltaError::new_err("Table is instantiated without files."));
         }
@@ -257,7 +301,10 @@ impl RawDeltaTable {
         }
     }
 
-    pub fn file_uris(&self, partition_filters: Option<Value>) -> RbResult<Vec<String>> {
+    pub fn file_uris(
+        &self,
+        partition_filters: Option<Vec<(String, String, PartitionFilterValue)>>,
+    ) -> RbResult<Vec<String>> {
         if !self._table.borrow().config.require_files {
             return Err(DeltaError::new_err("Table is initiated without files."));
         }
@@ -290,6 +337,8 @@ impl RawDeltaTable {
         dry_run: bool,
         retention_hours: Option<u64>,
         enforce_retention_duration: bool,
+        commit_properties: Option<RbCommitProperties>,
+        post_commithook_properties: Option<RbPostCommitHookProperties>,
     ) -> RbResult<Vec<String>> {
         let mut cmd = VacuumBuilder::new(
             self._table.borrow().log_store(),
@@ -305,6 +354,11 @@ impl RawDeltaTable {
             cmd = cmd.with_retention_period(Duration::hours(retention_period as i64));
         }
 
+        if let Some(commit_properties) =
+            maybe_create_commit_properties(commit_properties, post_commithook_properties)
+        {
+            cmd = cmd.with_commit_properties(commit_properties);
+        }
         let (table, metrics) = rt().block_on(cmd.into_future()).map_err(RubyError::from)?;
         self._table.borrow_mut().state = table.state;
         Ok(metrics.files_deleted)
@@ -312,9 +366,13 @@ impl RawDeltaTable {
 
     pub fn compact_optimize(
         &self,
+        partition_filters: Option<Vec<(String, String, PartitionFilterValue)>>,
         target_size: Option<i64>,
         max_concurrent_tasks: Option<usize>,
         min_commit_interval: Option<u64>,
+        writer_properties: Option<RbWriterProperties>,
+        commit_properties: Option<RbCommitProperties>,
+        post_commithook_properties: Option<RbPostCommitHookProperties>,
     ) -> RbResult<String> {
         let mut cmd = OptimizeBuilder::new(
             self._table.borrow().log_store(),
@@ -332,6 +390,22 @@ impl RawDeltaTable {
             cmd = cmd.with_min_commit_interval(time::Duration::from_secs(commit_interval));
         }
 
+        if let Some(writer_props) = writer_properties {
+            cmd = cmd.with_writer_properties(
+                set_writer_properties(writer_props).map_err(RubyError::from)?,
+            );
+        }
+
+        if let Some(commit_properties) =
+            maybe_create_commit_properties(commit_properties, post_commithook_properties)
+        {
+            cmd = cmd.with_commit_properties(commit_properties);
+        }
+
+        let converted_filters = convert_partition_filters(partition_filters.unwrap_or_default())
+            .map_err(RubyError::from)?;
+        cmd = cmd.with_filters(&converted_filters);
+
         let (table, metrics) = rt().block_on(cmd.into_future()).map_err(RubyError::from)?;
         self._table.borrow_mut().state = table.state;
         Ok(serde_json::to_string(&metrics).unwrap())
@@ -340,10 +414,14 @@ impl RawDeltaTable {
     pub fn z_order_optimize(
         &self,
         z_order_columns: Vec<String>,
+        partition_filters: Option<Vec<(String, String, PartitionFilterValue)>>,
         target_size: Option<i64>,
         max_concurrent_tasks: Option<usize>,
         max_spill_size: usize,
         min_commit_interval: Option<u64>,
+        writer_properties: Option<RbWriterProperties>,
+        commit_properties: Option<RbCommitProperties>,
+        post_commithook_properties: Option<RbPostCommitHookProperties>,
     ) -> RbResult<String> {
         let mut cmd = OptimizeBuilder::new(
             self._table.borrow().log_store(),
@@ -363,11 +441,75 @@ impl RawDeltaTable {
             cmd = cmd.with_min_commit_interval(time::Duration::from_secs(commit_interval));
         }
 
+        if let Some(writer_props) = writer_properties {
+            cmd = cmd.with_writer_properties(
+                set_writer_properties(writer_props).map_err(RubyError::from)?,
+            );
+        }
+
+        if let Some(commit_properties) =
+            maybe_create_commit_properties(commit_properties, post_commithook_properties)
+        {
+            cmd = cmd.with_commit_properties(commit_properties);
+        }
+
+        let converted_filters = convert_partition_filters(partition_filters.unwrap_or_default())
+            .map_err(RubyError::from)?;
+        cmd = cmd.with_filters(&converted_filters);
+
         let (table, metrics) = rt().block_on(cmd.into_future()).map_err(RubyError::from)?;
         self._table.borrow_mut().state = table.state;
         Ok(serde_json::to_string(&metrics).unwrap())
     }
 
+    pub fn add_columns(&self, fields: RArray) -> RbResult<()> {
+        let fields = fields.typecheck::<Obj<Field>>()?;
+        let mut cmd = AddColumnBuilder::new(
+            self._table.borrow().log_store(),
+            self._table
+                .borrow()
+                .snapshot()
+                .map_err(RubyError::from)?
+                .clone(),
+        );
+
+        let new_fields = fields
+            .iter()
+            .map(|v| v.inner.clone())
+            .collect::<Vec<StructField>>();
+
+        cmd = cmd.with_fields(new_fields);
+
+        let table = rt().block_on(cmd.into_future()).map_err(RubyError::from)?;
+        self._table.borrow_mut().state = table.state;
+        Ok(())
+    }
+
+    pub fn add_feature(
+        &self,
+        feature: RArray,
+        allow_protocol_versions_increase: bool,
+    ) -> RbResult<()> {
+        let feature = feature
+            .into_iter()
+            .map(|v| TableFeatures::try_convert(v))
+            .collect::<RbResult<Vec<_>>>()?;
+        let cmd = AddTableFeatureBuilder::new(
+            self._table.borrow().log_store(),
+            self._table
+                .borrow()
+                .snapshot()
+                .map_err(RubyError::from)?
+                .clone(),
+        )
+        .with_features(feature)
+        .with_allow_protocol_versions_increase(allow_protocol_versions_increase);
+
+        let table = rt().block_on(cmd.into_future()).map_err(RubyError::from)?;
+        self._table.borrow_mut().state = table.state;
+        Ok(())
+    }
+
     pub fn add_constraints(&self, constraints: HashMap<String, String>) -> RbResult<()> {
         let mut cmd = ConstraintBuilder::new(
             self._table.borrow().log_store(),
@@ -428,13 +570,13 @@ impl RawDeltaTable {
         }
         if let Some(st) = starting_timestamp {
             let starting_ts: DateTime<Utc> = DateTime::<Utc>::from_str(&st)
-                .map_err(|pe| Error::new(exception::arg_error(), pe.to_string()))?
+                .map_err(|pe| RbValueError::new_err(pe.to_string()))?
                 .to_utc();
             cdf_read = cdf_read.with_starting_timestamp(starting_ts);
         }
         if let Some(et) = ending_timestamp {
             let ending_ts = DateTime::<Utc>::from_str(&et)
-                .map_err(|pe| Error::new(exception::arg_error(), pe.to_string()))?
+                .map_err(|pe| RbValueError::new_err(pe.to_string()))?
                 .to_utc();
             cdf_read = cdf_read.with_starting_timestamp(ending_ts);
         }
@@ -470,11 +612,48 @@ impl RawDeltaTable {
         Ok(ArrowArrayStream { stream: ffi_stream })
     }
 
+    pub fn create_merge_builder(
+        &self,
+        source: RbArrowType<ArrowArrayStreamReader>,
+        predicate: String,
+        source_alias: Option<String>,
+        target_alias: Option<String>,
+        safe_cast: bool,
+        writer_properties: Option<RbWriterProperties>,
+        post_commithook_properties: Option<RbPostCommitHookProperties>,
+        commit_properties: Option<RbCommitProperties>,
+    ) -> RbResult<RbMergeBuilder> {
+        Ok(RbMergeBuilder::new(
+            self._table.borrow().log_store(),
+            self._table
+                .borrow()
+                .snapshot()
+                .map_err(RubyError::from)?
+                .clone(),
+            source.0,
+            predicate,
+            source_alias,
+            target_alias,
+            safe_cast,
+            writer_properties,
+            post_commithook_properties,
+            commit_properties,
+        )
+        .map_err(RubyError::from)?)
+    }
+
+    pub fn merge_execute(&self, merge_builder: &RbMergeBuilder) -> RbResult<String> {
+        let (table, metrics) = merge_builder.execute().map_err(RubyError::from)?;
+        self._table.borrow_mut().state = table.state;
+        Ok(metrics)
+    }
+
     pub fn restore(
         &self,
         target: Option<Value>,
         ignore_missing_files: bool,
         protocol_downgrade_allowed: bool,
+        commit_properties: Option<RbCommitProperties>,
     ) -> RbResult<String> {
         let mut cmd = RestoreBuilder::new(
             self._table.borrow().log_store(),
@@ -491,10 +670,7 @@ impl RawDeltaTable {
             if let Ok(ds) = String::try_convert(val) {
                 let datetime = DateTime::<Utc>::from(
                     DateTime::<FixedOffset>::parse_from_rfc3339(ds.as_ref()).map_err(|err| {
-                        Error::new(
-                            exception::arg_error(),
-                            format!("Failed to parse datetime string: {err}"),
-                        )
+                        RbValueError::new_err(format!("Failed to parse datetime string: {err}"))
                     })?,
                 );
                 cmd = cmd.with_datetime_to_restore(datetime)
@@ -503,6 +679,10 @@ impl RawDeltaTable {
         cmd = cmd.with_ignore_missing_files(ignore_missing_files);
         cmd = cmd.with_protocol_downgrade_allowed(protocol_downgrade_allowed);
 
+        if let Some(commit_properties) = maybe_create_commit_properties(commit_properties, None) {
+            cmd = cmd.with_commit_properties(commit_properties);
+        }
+
         let (table, metrics) = rt().block_on(cmd.into_future()).map_err(RubyError::from)?;
         self._table.borrow_mut().state = table.state;
         Ok(serde_json::to_string(&metrics).unwrap())
@@ -575,7 +755,39 @@ impl RawDeltaTable {
         Ok(RArray::from_iter(active_partitions))
     }
 
-    pub fn delete(&self, predicate: Option<String>) -> RbResult<String> {
+    pub fn create_checkpoint(&self) -> RbResult<()> {
+        rt().block_on(create_checkpoint(&self._table.borrow()))
+            .map_err(RubyError::from)?;
+
+        Ok(())
+    }
+
+    pub fn cleanup_metadata(&self) -> RbResult<()> {
+        rt().block_on(cleanup_metadata(&self._table.borrow()))
+            .map_err(RubyError::from)?;
+
+        Ok(())
+    }
+
+    pub fn get_add_file_sizes(&self) -> RbResult<HashMap<String, i64>> {
+        Ok(self
+            ._table
+            .borrow()
+            .snapshot()
+            .map_err(RubyError::from)?
+            .eager_snapshot()
+            .files()
+            .map(|f| (f.path().to_string(), f.size()))
+            .collect::<HashMap<String, i64>>())
+    }
+
+    pub fn delete(
+        &self,
+        predicate: Option<String>,
+        writer_properties: Option<RbWriterProperties>,
+        commit_properties: Option<RbCommitProperties>,
+        post_commithook_properties: Option<RbPostCommitHookProperties>,
+    ) -> RbResult<String> {
         let mut cmd = DeleteBuilder::new(
             self._table.borrow().log_store(),
             self._table
@@ -587,14 +799,50 @@ impl RawDeltaTable {
         if let Some(predicate) = predicate {
             cmd = cmd.with_predicate(predicate);
         }
+        if let Some(writer_props) = writer_properties {
+            cmd = cmd.with_writer_properties(
+                set_writer_properties(writer_props).map_err(RubyError::from)?,
+            );
+        }
+        if let Some(commit_properties) =
+            maybe_create_commit_properties(commit_properties, post_commithook_properties)
+        {
+            cmd = cmd.with_commit_properties(commit_properties);
+        }
 
         let (table, metrics) = rt().block_on(cmd.into_future()).map_err(RubyError::from)?;
         self._table.borrow_mut().state = table.state;
         Ok(serde_json::to_string(&metrics).unwrap())
     }
 
-    pub fn repair(&self, dry_run: bool) -> RbResult<String> {
-        let cmd = FileSystemCheckBuilder::new(
+    pub fn set_table_properties(
+        &self,
+        properties: HashMap<String, String>,
+        raise_if_not_exists: bool,
+    ) -> RbResult<()> {
+        let cmd = SetTablePropertiesBuilder::new(
+            self._table.borrow().log_store(),
+            self._table
+                .borrow()
+                .snapshot()
+                .map_err(RubyError::from)?
+                .clone(),
+        )
+        .with_properties(properties)
+        .with_raise_if_not_exists(raise_if_not_exists);
+
+        let table = rt().block_on(cmd.into_future()).map_err(RubyError::from)?;
+        self._table.borrow_mut().state = table.state;
+        Ok(())
+    }
+
+    pub fn repair(
+        &self,
+        dry_run: bool,
+        commit_properties: Option<RbCommitProperties>,
+        post_commithook_properties: Option<RbPostCommitHookProperties>,
+    ) -> RbResult<String> {
+        let mut cmd = FileSystemCheckBuilder::new(
             self._table.borrow().log_store(),
             self._table
                 .borrow()
@@ -604,6 +852,12 @@ impl RawDeltaTable {
         )
         .with_dry_run(dry_run);
 
+        if let Some(commit_properties) =
+            maybe_create_commit_properties(commit_properties, post_commithook_properties)
+        {
+            cmd = cmd.with_commit_properties(commit_properties);
+        }
+
         let (table, metrics) = rt().block_on(cmd.into_future()).map_err(RubyError::from)?;
         self._table.borrow_mut().state = table.state;
         Ok(serde_json::to_string(&metrics).unwrap())
@@ -620,35 +874,240 @@ impl RawDeltaTable {
     }
 }
 
+fn set_post_commithook_properties(
+    mut commit_properties: CommitProperties,
+    post_commithook_properties: RbPostCommitHookProperties,
+) -> CommitProperties {
+    commit_properties =
+        commit_properties.with_create_checkpoint(post_commithook_properties.create_checkpoint);
+    commit_properties = commit_properties
+        .with_cleanup_expired_logs(post_commithook_properties.cleanup_expired_logs);
+    commit_properties
+}
+
+fn set_writer_properties(writer_properties: RbWriterProperties) -> DeltaResult<WriterProperties> {
+    let mut properties = WriterProperties::builder();
+    let data_page_size_limit = writer_properties.data_page_size_limit;
+    let dictionary_page_size_limit = writer_properties.dictionary_page_size_limit;
+    let data_page_row_count_limit = writer_properties.data_page_row_count_limit;
+    let write_batch_size = writer_properties.write_batch_size;
+    let max_row_group_size = writer_properties.max_row_group_size;
+    let compression = writer_properties.compression;
+    let statistics_truncate_length = writer_properties.statistics_truncate_length;
+    let default_column_properties = writer_properties.default_column_properties;
+    let column_properties = writer_properties.column_properties;
+
+    if let Some(data_page_size) = data_page_size_limit {
+        properties = properties.set_data_page_size_limit(data_page_size);
+    }
+    if let Some(dictionary_page_size) = dictionary_page_size_limit {
+        properties = properties.set_dictionary_page_size_limit(dictionary_page_size);
+    }
+    if let Some(data_page_row_count) = data_page_row_count_limit {
+        properties = properties.set_data_page_row_count_limit(data_page_row_count);
+    }
+    if let Some(batch_size) = write_batch_size {
+        properties = properties.set_write_batch_size(batch_size);
+    }
+    if let Some(row_group_size) = max_row_group_size {
+        properties = properties.set_max_row_group_size(row_group_size);
+    }
+    properties = properties.set_statistics_truncate_length(statistics_truncate_length);
+
+    if let Some(compression) = compression {
+        let compress: Compression = compression
+            .parse()
+            .map_err(|err: ParquetError| DeltaTableError::Generic(err.to_string()))?;
+
+        properties = properties.set_compression(compress);
+    }
+
+    if let Some(default_column_properties) = default_column_properties {
+        if let Some(dictionary_enabled) = default_column_properties.dictionary_enabled {
+            properties = properties.set_dictionary_enabled(dictionary_enabled);
+        }
+        if let Some(max_statistics_size) = default_column_properties.max_statistics_size {
+            properties = properties.set_max_statistics_size(max_statistics_size);
+        }
+        if let Some(bloom_filter_properties) = default_column_properties.bloom_filter_properties {
+            if let Some(set_bloom_filter_enabled) = bloom_filter_properties.set_bloom_filter_enabled
+            {
+                properties = properties.set_bloom_filter_enabled(set_bloom_filter_enabled);
+            }
+            if let Some(bloom_filter_fpp) = bloom_filter_properties.fpp {
+                properties = properties.set_bloom_filter_fpp(bloom_filter_fpp);
+            }
+            if let Some(bloom_filter_ndv) = bloom_filter_properties.ndv {
+                properties = properties.set_bloom_filter_ndv(bloom_filter_ndv);
+            }
+        }
+    }
+    if let Some(column_properties) = column_properties {
+        for (column_name, column_prop) in column_properties {
+            if let Some(column_prop) = column_prop {
+                if let Some(dictionary_enabled) = column_prop.dictionary_enabled {
+                    properties = properties.set_column_dictionary_enabled(
+                        column_name.clone().into(),
+                        dictionary_enabled,
+                    );
+                }
+                if let Some(bloom_filter_properties) = column_prop.bloom_filter_properties {
+                    if let Some(set_bloom_filter_enabled) =
+                        bloom_filter_properties.set_bloom_filter_enabled
+                    {
+                        properties = properties.set_column_bloom_filter_enabled(
+                            column_name.clone().into(),
+                            set_bloom_filter_enabled,
+                        );
+                    }
+                    if let Some(bloom_filter_fpp) = bloom_filter_properties.fpp {
+                        properties = properties.set_column_bloom_filter_fpp(
+                            column_name.clone().into(),
+                            bloom_filter_fpp,
+                        );
+                    }
+                    if let Some(bloom_filter_ndv) = bloom_filter_properties.ndv {
+                        properties = properties
+                            .set_column_bloom_filter_ndv(column_name.into(), bloom_filter_ndv);
+                    }
+                }
+            }
+        }
+    }
+    Ok(properties.build())
+}
+
 fn convert_partition_filters(
-    _partitions_filters: Value,
+    partitions_filters: Vec<(String, String, PartitionFilterValue)>,
 ) -> Result<Vec<PartitionFilter>, DeltaTableError> {
-    todo!()
+    partitions_filters
+        .into_iter()
+        .map(|filter| match filter {
+            (key, op, PartitionFilterValue::Single(v)) => {
+                let key: &'_ str = key.as_ref();
+                let op: &'_ str = op.as_ref();
+                let v: &'_ str = v.as_ref();
+                PartitionFilter::try_from((key, op, v))
+            }
+            (key, op, PartitionFilterValue::Multiple(v)) => {
+                let key: &'_ str = key.as_ref();
+                let op: &'_ str = op.as_ref();
+                let v: Vec<&'_ str> = v.iter().map(|v| v.as_ref()).collect();
+                PartitionFilter::try_from((key, op, v.as_slice()))
+            }
+        })
+        .collect()
 }
 
-impl RawDeltaTableMetaData {
-    fn id(&self) -> String {
-        self.id.clone()
+fn maybe_create_commit_properties(
+    maybe_commit_properties: Option<RbCommitProperties>,
+    post_commithook_properties: Option<RbPostCommitHookProperties>,
+) -> Option<CommitProperties> {
+    if maybe_commit_properties.is_none() && post_commithook_properties.is_none() {
+        return None;
+    }
+    let mut commit_properties = CommitProperties::default();
+
+    if let Some(commit_props) = maybe_commit_properties {
+        if let Some(metadata) = commit_props.custom_metadata {
+            let json_metadata: Map<String, serde_json::Value> =
+                metadata.into_iter().map(|(k, v)| (k, v.into())).collect();
+            commit_properties = commit_properties.with_metadata(json_metadata);
+        };
+
+        if let Some(max_retries) = commit_props.max_commit_retries {
+            commit_properties = commit_properties.with_max_retries(max_retries);
+        };
+
+        if let Some(app_transactions) = commit_props.app_transactions {
+            let app_transactions = app_transactions.iter().map(Transaction::from).collect();
+            commit_properties = commit_properties.with_application_transactions(app_transactions);
+        }
     }
 
-    fn name(&self) -> Option<String> {
-        self.name.clone()
+    if let Some(post_commit_hook_props) = post_commithook_properties {
+        commit_properties =
+            set_post_commithook_properties(commit_properties, post_commit_hook_props)
     }
+    Some(commit_properties)
+}
 
-    fn description(&self) -> Option<String> {
-        self.description.clone()
+fn rust_core_version() -> String {
+    deltalake::crate_version().to_string()
+}
+
+pub struct BloomFilterProperties {
+    pub set_bloom_filter_enabled: Option<bool>,
+    pub fpp: Option<f64>,
+    pub ndv: Option<u64>,
+}
+
+impl TryConvert for BloomFilterProperties {
+    fn try_convert(val: Value) -> RbResult<Self> {
+        Ok(BloomFilterProperties {
+            set_bloom_filter_enabled: val.funcall("set_bloom_filter_enabled", ())?,
+            fpp: val.funcall("fpp", ())?,
+            ndv: val.funcall("ndv", ())?,
+        })
     }
+}
 
-    fn partition_columns(&self) -> Vec<String> {
-        self.partition_columns.clone()
+pub struct ColumnProperties {
+    pub dictionary_enabled: Option<bool>,
+    pub max_statistics_size: Option<usize>,
+    pub bloom_filter_properties: Option<BloomFilterProperties>,
+}
+
+impl TryConvert for ColumnProperties {
+    fn try_convert(val: Value) -> RbResult<Self> {
+        Ok(ColumnProperties {
+            dictionary_enabled: val.funcall("dictionary_enabled", ())?,
+            max_statistics_size: val.funcall("max_statistics_size", ())?,
+            bloom_filter_properties: val.funcall("bloom_filter_properties", ())?,
+        })
     }
+}
 
-    fn created_time(&self) -> Option<i64> {
-        self.created_time
+pub struct RbWriterProperties {
+    data_page_size_limit: Option<usize>,
+    dictionary_page_size_limit: Option<usize>,
+    data_page_row_count_limit: Option<usize>,
+    write_batch_size: Option<usize>,
+    max_row_group_size: Option<usize>,
+    statistics_truncate_length: Option<usize>,
+    compression: Option<String>,
+    default_column_properties: Option<ColumnProperties>,
+    column_properties: Option<HashMap<String, Option<ColumnProperties>>>,
+}
+
+impl TryConvert for RbWriterProperties {
+    fn try_convert(val: Value) -> RbResult<Self> {
+        Ok(RbWriterProperties {
+            data_page_size_limit: val.funcall("data_page_size_limit", ())?,
+            dictionary_page_size_limit: val.funcall("dictionary_page_size_limit", ())?,
+            data_page_row_count_limit: val.funcall("data_page_row_count_limit", ())?,
+            write_batch_size: val.funcall("write_batch_size", ())?,
+            max_row_group_size: val.funcall("max_row_group_size", ())?,
+            statistics_truncate_length: val.funcall("statistics_truncate_length", ())?,
+            compression: val.funcall("compression", ())?,
+            default_column_properties: val.funcall("default_column_properties", ())?,
+            // TODO fix
+            column_properties: None,
+        })
     }
+}
 
-    fn configuration(&self) -> HashMap<String, Option<String>> {
-        self.configuration.clone()
+pub struct RbPostCommitHookProperties {
+    create_checkpoint: bool,
+    cleanup_expired_logs: Option<bool>,
+}
+
+impl TryConvert for RbPostCommitHookProperties {
+    fn try_convert(val: Value) -> RbResult<Self> {
+        Ok(RbPostCommitHookProperties {
+            create_checkpoint: val.funcall("create_checkpoint", ())?,
+            cleanup_expired_logs: val.funcall("cleanup_expired_logs", ())?,
+        })
     }
 }
 
@@ -669,10 +1128,37 @@ impl From<Transaction> for RbTransaction {
     }
 }
 
+impl From<&RbTransaction> for Transaction {
+    fn from(value: &RbTransaction) -> Self {
+        Transaction {
+            app_id: value.app_id.clone(),
+            version: value.version,
+            last_updated: value.last_updated,
+        }
+    }
+}
+
+pub struct RbCommitProperties {
+    custom_metadata: Option<HashMap<String, String>>,
+    max_commit_retries: Option<usize>,
+    app_transactions: Option<Vec<RbTransaction>>,
+}
+
+impl TryConvert for RbCommitProperties {
+    fn try_convert(val: Value) -> RbResult<Self> {
+        Ok(RbCommitProperties {
+            custom_metadata: val.funcall("custom_metadata", ())?,
+            max_commit_retries: val.funcall("max_commit_retries", ())?,
+            // TODO fix
+            app_transactions: None,
+        })
+    }
+}
+
 #[allow(clippy::too_many_arguments)]
 fn write_to_deltalake(
     table_uri: String,
-    data: Value,
+    data: RbArrowType<ArrowArrayStreamReader>,
     mode: String,
     table: Option<&RawDeltaTable>,
     schema_mode: Option<String>,
@@ -683,16 +1169,11 @@ fn write_to_deltalake(
     description: Option<String>,
     configuration: Option<HashMap<String, Option<String>>>,
     storage_options: Option<HashMap<String, String>>,
+    writer_properties: Option<RbWriterProperties>,
+    commit_properties: Option<RbCommitProperties>,
+    post_commithook_properties: Option<RbPostCommitHookProperties>,
 ) -> RbResult<()> {
-    let capsule_pointer: usize = data.funcall("to_i", ())?;
-
-    // use similar approach as Polars to avoid copy
-    let stream_ptr =
-        Box::new(unsafe { std::ptr::replace(capsule_pointer as _, FFI_ArrowArrayStream::empty()) });
-    let stream = ArrowArrayStreamReader::try_new(*stream_ptr)
-        .map_err(|err| DeltaError::new_err(err.to_string()))?;
-
-    let batches = stream.map(|batch| batch.unwrap()).collect::<Vec<_>>();
+    let batches = data.0.map(|batch| batch.unwrap()).collect::<Vec<_>>();
     let save_mode = mode.parse().map_err(RubyError::from)?;
 
     let options = storage_options.clone().unwrap_or_default();
@@ -713,6 +1194,12 @@ fn write_to_deltalake(
         builder = builder.with_partition_columns(partition_columns);
     }
 
+    if let Some(writer_props) = writer_properties {
+        builder = builder.with_writer_properties(
+            set_writer_properties(writer_props).map_err(RubyError::from)?,
+        );
+    }
+
     if let Some(name) = &name {
         builder = builder.with_table_name(name);
     };
@@ -733,18 +1220,55 @@ fn write_to_deltalake(
         builder = builder.with_configuration(config);
     };
 
+    if let Some(commit_properties) =
+        maybe_create_commit_properties(commit_properties, post_commithook_properties)
+    {
+        builder = builder.with_commit_properties(commit_properties);
+    };
+
     rt().block_on(builder.into_future())
         .map_err(RubyError::from)?;
 
     Ok(())
 }
 
+pub struct RbArrowType<T>(pub T);
+
+impl TryConvert for RbArrowType<ArrowArrayStreamReader> {
+    fn try_convert(val: Value) -> RbResult<Self> {
+        let addr: usize = val.funcall("to_i", ())?;
+
+        // use similar approach as Polars to consume pointer and avoid copy
+        let stream_ptr =
+            Box::new(unsafe { std::ptr::replace(addr as _, FFI_ArrowArrayStream::empty()) });
+
+        Ok(RbArrowType(
+            ArrowArrayStreamReader::try_new(*stream_ptr)
+                .map_err(|err| DeltaError::new_err(err.to_string()))?,
+        ))
+    }
+}
+
+#[magnus::wrap(class = "DeltaLake::ArrowArrayStream")]
+pub struct ArrowArrayStream {
+    stream: FFI_ArrowArrayStream,
+}
+
+impl ArrowArrayStream {
+    pub fn to_i(&self) -> usize {
+        (&self.stream as *const _) as usize
+    }
+}
+
 #[magnus::init]
 fn init(ruby: &Ruby) -> RbResult<()> {
     deltalake::aws::register_handlers(None);
+    deltalake::azure::register_handlers(None);
+    deltalake::gcp::register_handlers(None);
 
     let module = ruby.define_module("DeltaLake")?;
-    module.define_singleton_method("write_deltalake_rust", function!(write_to_deltalake, 12))?;
+    module.define_singleton_method("write_deltalake_rust", function!(write_to_deltalake, 15))?;
+    module.define_singleton_method("rust_core_version", function!(rust_core_version, 0))?;
 
     let class = module.define_class("RawDeltaTable", ruby.class_object())?;
     class.define_singleton_method("new", function!(RawDeltaTable::new, 5))?;
@@ -781,15 +1305,17 @@ fn init(ruby: &Ruby) -> RbResult<()> {
     class.define_method("files", method!(RawDeltaTable::files, 1))?;
     class.define_method("file_uris", method!(RawDeltaTable::file_uris, 1))?;
     class.define_method("schema", method!(RawDeltaTable::schema, 0))?;
-    class.define_method("vacuum", method!(RawDeltaTable::vacuum, 3))?;
+    class.define_method("vacuum", method!(RawDeltaTable::vacuum, 5))?;
     class.define_method(
         "compact_optimize",
-        method!(RawDeltaTable::compact_optimize, 3),
+        method!(RawDeltaTable::compact_optimize, 7),
     )?;
     class.define_method(
         "z_order_optimize",
-        method!(RawDeltaTable::z_order_optimize, 5),
+        method!(RawDeltaTable::z_order_optimize, 9),
     )?;
+    class.define_method("add_columns", method!(RawDeltaTable::add_columns, 1))?;
+    class.define_method("add_feature", method!(RawDeltaTable::add_feature, 2))?;
     class.define_method(
         "add_constraints",
         method!(RawDeltaTable::add_constraints, 1),
@@ -799,7 +1325,12 @@ fn init(ruby: &Ruby) -> RbResult<()> {
         method!(RawDeltaTable::drop_constraints, 2),
     )?;
     class.define_method("load_cdf", method!(RawDeltaTable::load_cdf, 5))?;
-    class.define_method("restore", method!(RawDeltaTable::restore, 3))?;
+    class.define_method(
+        "create_merge_builder",
+        method!(RawDeltaTable::create_merge_builder, 8),
+    )?;
+    class.define_method("merge_execute", method!(RawDeltaTable::merge_execute, 1))?;
+    class.define_method("restore", method!(RawDeltaTable::restore, 4))?;
     class.define_method("history", method!(RawDeltaTable::history, 1))?;
     class.define_method(
         "update_incremental",
@@ -809,8 +1340,24 @@ fn init(ruby: &Ruby) -> RbResult<()> {
         "get_active_partitions",
         method!(RawDeltaTable::get_active_partitions, 0),
     )?;
-    class.define_method("delete", method!(RawDeltaTable::delete, 1))?;
-    class.define_method("repair", method!(RawDeltaTable::repair, 1))?;
+    class.define_method(
+        "create_checkpoint",
+        method!(RawDeltaTable::create_checkpoint, 0),
+    )?;
+    class.define_method(
+        "cleanup_metadata",
+        method!(RawDeltaTable::cleanup_metadata, 0),
+    )?;
+    class.define_method(
+        "get_add_file_sizes",
+        method!(RawDeltaTable::get_add_file_sizes, 0),
+    )?;
+    class.define_method("delete", method!(RawDeltaTable::delete, 4))?;
+    class.define_method(
+        "set_table_properties",
+        method!(RawDeltaTable::set_table_properties, 2),
+    )?;
+    class.define_method("repair", method!(RawDeltaTable::repair, 3))?;
     class.define_method(
         "transaction_versions",
         method!(RawDeltaTable::transaction_versions, 0),
@@ -844,5 +1391,29 @@ fn init(ruby: &Ruby) -> RbResult<()> {
     class.define_method("type", method!(Field::get_type, 0))?;
     class.define_method("nullable", method!(Field::nullable, 0))?;
 
+    let class = module.define_class("RbMergeBuilder", ruby.class_object())?;
+    class.define_method("source_alias", method!(RbMergeBuilder::source_alias, 0))?;
+    class.define_method("target_alias", method!(RbMergeBuilder::target_alias, 0))?;
+    class.define_method(
+        "when_matched_update",
+        method!(RbMergeBuilder::when_matched_update, 2),
+    )?;
+    class.define_method(
+        "when_matched_delete",
+        method!(RbMergeBuilder::when_matched_delete, 1),
+    )?;
+    class.define_method(
+        "when_not_matched_insert",
+        method!(RbMergeBuilder::when_not_matched_insert, 2),
+    )?;
+    class.define_method(
+        "when_not_matched_by_source_update",
+        method!(RbMergeBuilder::when_not_matched_by_source_update, 2),
+    )?;
+    class.define_method(
+        "when_not_matched_by_source_delete",
+        method!(RbMergeBuilder::when_not_matched_by_source_delete, 1),
+    )?;
+
     Ok(())
 }