deltalake-rb 0.2.9 → 0.3.1

data/ext/deltalake/src/lib.rs

@@ -1,30 +1,33 @@
 mod error;
 mod features;
 mod merge;
+mod ruby;
 mod schema;
 mod utils;
 
 use chrono::{DateTime, Duration, FixedOffset, Utc};
-use delta_kernel::schema::StructField;
+use delta_kernel::schema::{MetadataValue, StructField};
 use delta_kernel::table_properties::DataSkippingNumIndexedCols;
 use deltalake::arrow::ffi_stream::{ArrowArrayStreamReader, FFI_ArrowArrayStream};
 use deltalake::arrow::record_batch::RecordBatchIterator;
 use deltalake::checkpoints::{cleanup_metadata, create_checkpoint};
 use deltalake::datafusion::catalog::TableProvider;
 use deltalake::datafusion::prelude::SessionContext;
-use deltalake::delta_datafusion::DeltaCdfTableProvider;
+use deltalake::delta_datafusion::{create_session_state_with_spill_config, DeltaCdfTableProvider};
 use deltalake::errors::DeltaTableError;
 use deltalake::kernel::transaction::{CommitProperties, TableReference};
 use deltalake::kernel::{scalars::ScalarExt, Transaction};
-use deltalake::kernel::{EagerSnapshot, StructDataExt};
+use deltalake::kernel::{EagerSnapshot, StructDataExt, Version};
 use deltalake::logstore::IORuntime;
 use deltalake::logstore::LogStoreRef;
 use deltalake::operations::collect_sendable_stream;
-use deltalake::operations::optimize::{create_session_state_for_optimize, OptimizeType};
+use deltalake::operations::optimize::OptimizeType;
+use deltalake::operations::update_table_metadata::TableMetadataUpdate;
 use deltalake::parquet::basic::Compression;
 use deltalake::parquet::errors::ParquetError;
 use deltalake::parquet::file::properties::WriterProperties;
 use deltalake::partitions::PartitionFilter;
+use deltalake::protocol::log_compaction::compact_logs;
 use deltalake::table::config::TablePropertiesExt;
 use deltalake::table::state::DeltaTableState;
 use deltalake::{DeltaResult, DeltaTable};
@@ -32,20 +35,22 @@ use error::DeltaError;
 use futures::future::join_all;
 use futures::TryStreamExt;
 use magnus::{
-    function, method, prelude::*, try_convert::TryConvertOwned, typed_data::Obj, Error as RbErr,
-    Integer, Module, RArray, Ruby, TryConvert, Value,
+    function, method, prelude::*, try_convert::TryConvertOwned, Error as RbErr, Integer, Module,
+    RArray, Ruby, TryConvert, Value,
 };
 use serde_json::Map;
 use std::collections::{HashMap, HashSet};
 use std::future::IntoFuture;
+use std::num::NonZeroU64;
 use std::str::FromStr;
 use std::sync::{Arc, Mutex};
 use std::time;
 use uuid::Uuid;
 
-use crate::error::{to_rt_err, RbRuntimeError, RbValueError, RubyError};
+use crate::error::{to_rt_err, to_rt_err2, RubyError};
 use crate::features::TableFeatures;
 use crate::merge::RbMergeBuilder;
+use crate::ruby::{GvlExt, RbRuntimeError, RbValueError};
 use crate::schema::{schema_to_rbobject, Field};
 use crate::utils::rt;
 
@@ -111,6 +116,24 @@ impl RawDeltaTableMetaData {
 
 type StringVec = Vec<String>;
 
+const MAX_OPTIMIZE_TARGET_SIZE: u64 = i64::MAX as u64;
+
+fn parse_optimize_target_size(target_size: u64) -> Result<NonZeroU64, RubyError> {
+    let target_size = NonZeroU64::new(target_size).ok_or_else(|| {
+        RubyError::ValueError(format!(
+            "target_file_size must be between 1 and {MAX_OPTIMIZE_TARGET_SIZE}"
+        ))
+    })?;
+
+    if target_size.get() > MAX_OPTIMIZE_TARGET_SIZE {
+        return Err(RubyError::ValueError(format!(
+            "target_file_size must be between 1 and {MAX_OPTIMIZE_TARGET_SIZE}"
+        )));
+    }
+
+    Ok(target_size)
+}
+
 impl RawDeltaTable {
     fn with_table<T>(&self, func: impl Fn(&deltalake::DeltaTable) -> RbResult<T>) -> RbResult<T> {
         match self._table.lock() {
@@ -119,6 +142,16 @@ impl RawDeltaTable {
         }
     }
 
+    fn with_table2<T>(
+        &self,
+        func: impl Fn(&deltalake::DeltaTable) -> Result<T, RubyError>,
+    ) -> Result<T, RubyError> {
+        match self._table.lock() {
+            Ok(table) => func(&table),
+            Err(e) => Err(RubyError::RuntimeError(e.to_string())),
+        }
+    }
+
     fn cloned_state(&self) -> RbResult<EagerSnapshot> {
         self.with_table(|t| {
             t.snapshot()
@@ -145,37 +178,41 @@ impl RawDeltaTable {
 
 impl RawDeltaTable {
     pub fn new(
+        rb: &Ruby,
         table_uri: String,
-        version: Option<i64>,
+        version: Option<Version>,
         storage_options: Option<HashMap<String, String>>,
         without_files: bool,
         log_buffer_size: Option<usize>,
     ) -> RbResult<Self> {
-        let table_url = deltalake::table::builder::parse_table_uri(table_uri)
-            .map_err(error::RubyError::from)?;
-        let mut builder = deltalake::DeltaTableBuilder::from_url(table_url)
-            .map_err(error::RubyError::from)?
-            .with_io_runtime(IORuntime::default());
-
-        if let Some(storage_options) = storage_options {
-            builder = builder.with_storage_options(storage_options)
-        }
-        if let Some(version) = version {
-            builder = builder.with_version(version)
-        }
-        if without_files {
-            builder = builder.without_files()
-        }
-        if let Some(buf_size) = log_buffer_size {
-            builder = builder
-                .with_log_buffer_size(buf_size)
-                .map_err(RubyError::from)?;
-        }
+        rb.detach(|| {
+            let table_url = deltalake::table::builder::parse_table_uri(table_uri)
+                .map_err(error::RubyError::from)?;
+            let mut builder = deltalake::DeltaTableBuilder::from_url(table_url)
+                .map_err(error::RubyError::from)?
+                .with_io_runtime(IORuntime::default());
+
+            if let Some(storage_options) = storage_options {
+                builder = builder.with_storage_options(storage_options)
+            }
+            if let Some(version) = version {
+                builder = builder.with_version(version)
+            }
+            if without_files {
+                builder = builder.without_files()
+            }
+            if let Some(buf_size) = log_buffer_size {
+                builder = builder
+                    .with_log_buffer_size(buf_size)
+                    .map_err(RubyError::from)?;
+            }
 
-        let table = rt().block_on(builder.load()).map_err(RubyError::from)?;
-        Ok(RawDeltaTable {
-            _table: Arc::new(Mutex::new(table)),
+            let table = rt().block_on(builder.load()).map_err(RubyError::from)?;
+            Ok::<_, RubyError>(RawDeltaTable {
+                _table: Arc::new(Mutex::new(table)),
+            })
         })
+        .map_err(RbErr::from)
     }
 
     pub fn is_deltatable(
@@ -203,7 +240,7 @@ impl RawDeltaTable {
         self.with_table(|t| Ok(t.table_url().to_string()))
     }
 
-    pub fn version(&self) -> RbResult<Option<i64>> {
+    pub fn version(&self) -> RbResult<Option<Version>> {
         self.with_table(|t| Ok(t.version()))
     }
 
@@ -220,7 +257,7 @@ impl RawDeltaTable {
             id: metadata.id().to_string(),
             name: metadata.name().map(String::from),
             description: metadata.description().map(String::from),
-            partition_columns: metadata.partition_columns().clone(),
+            partition_columns: metadata.partition_columns().to_vec(),
             created_time: metadata.created_time(),
             configuration: metadata.configuration().clone(),
         })
@@ -255,33 +292,34 @@ impl RawDeltaTable {
         ))
     }
 
-    pub fn load_version(&self, version: i64) -> RbResult<()> {
-        #[allow(clippy::await_holding_lock)]
-        rt().block_on(async {
-            let mut table = self
-                ._table
-                .lock()
-                .map_err(|e| RbRuntimeError::new_err(e.to_string()))?;
-            (*table)
-                .load_version(version)
-                .await
-                .map_err(RubyError::from)
-                .map_err(RbErr::from)
+    pub fn load_version(rb: &Ruby, self_: &Self, version: Version) -> RbResult<()> {
+        rb.detach(|| {
+            #[allow(clippy::await_holding_lock)]
+            rt().block_on(async {
+                let mut table = self_
+                    ._table
+                    .lock()
+                    .map_err(|e| RubyError::RuntimeError(e.to_string()))?;
+                (*table)
+                    .load_version(version)
+                    .await
+                    .map_err(RubyError::from)
+            })
         })
+        .map_err(RbErr::from)
     }
 
-    pub fn get_latest_version(&self) -> RbResult<i64> {
-        #[allow(clippy::await_holding_lock)]
-        rt().block_on(async {
-            match self._table.lock() {
-                Ok(table) => table
-                    .get_latest_version()
-                    .await
-                    .map_err(RubyError::from)
-                    .map_err(RbErr::from),
-                Err(e) => Err(RbRuntimeError::new_err(e.to_string())),
-            }
+    pub fn get_latest_version(rb: &Ruby, self_: &Self) -> RbResult<Version> {
+        rb.detach(|| {
+            #[allow(clippy::await_holding_lock)]
+            rt().block_on(async {
+                match self_._table.lock() {
+                    Ok(table) => table.get_latest_version().await.map_err(RubyError::from),
+                    Err(e) => Err(RubyError::RuntimeError(e.to_string())),
+                }
+            })
         })
+        .map_err(RbErr::from)
     }
 
     pub fn get_num_index_cols(&self) -> RbResult<i32> {
@@ -309,57 +347,61 @@ impl RawDeltaTable {
         })
     }
 
-    pub fn load_with_datetime(&self, ds: String) -> RbResult<()> {
-        let datetime =
-            DateTime::<Utc>::from(DateTime::<FixedOffset>::parse_from_rfc3339(&ds).map_err(
-                |err| RbValueError::new_err(format!("Failed to parse datetime string: {err}")),
-            )?);
-        #[allow(clippy::await_holding_lock)]
-        rt().block_on(async {
-            let mut table = self
-                ._table
-                .lock()
-                .map_err(|e| RbRuntimeError::new_err(e.to_string()))?;
-            (*table)
-                .load_with_datetime(datetime)
-                .await
-                .map_err(RubyError::from)
-                .map_err(RbErr::from)
+    pub fn load_with_datetime(rb: &Ruby, self_: &Self, ds: String) -> RbResult<()> {
+        rb.detach(|| {
+            let datetime =
+                DateTime::<Utc>::from(DateTime::<FixedOffset>::parse_from_rfc3339(&ds).map_err(
+                    |err| RubyError::ValueError(format!("Failed to parse datetime string: {err}")),
+                )?);
+            #[allow(clippy::await_holding_lock)]
+            rt().block_on(async {
+                let mut table = self_
+                    ._table
+                    .lock()
+                    .map_err(|e| RubyError::RuntimeError(e.to_string()))?;
+                (*table)
+                    .load_with_datetime(datetime)
+                    .await
+                    .map_err(RubyError::from)
+            })
         })
+        .map_err(RbErr::from)
     }
 
     pub fn files(
-        &self,
+        rb: &Ruby,
+        self_: &Self,
         partition_filters: Option<Vec<(String, String, PartitionFilterValue)>>,
     ) -> RbResult<Vec<String>> {
-        if !self.has_files()? {
+        if !self_.has_files()? {
             return Err(DeltaError::new_err("Table is instantiated without files."));
         }
-
-        if let Some(filters) = partition_filters {
-            let filters = convert_partition_filters(filters).map_err(RubyError::from)?;
-            Ok(self
-                .with_table(|t| {
-                    rt().block_on(async {
-                        t.get_files_by_partitions(&filters)
-                            .await
-                            .map_err(RubyError::from)
-                            .map_err(RbErr::from)
-                    })
-                })?
-                .into_iter()
-                .map(|p| p.to_string())
-                .collect())
-        } else {
-            match self._table.lock() {
-                Ok(table) => Ok(table
-                    .get_file_uris()
-                    .map_err(RubyError::from)?
-                    .map(|f| f.to_string())
-                    .collect()),
-                Err(e) => Err(RbRuntimeError::new_err(e.to_string())),
+        rb.detach(|| {
+            if let Some(filters) = partition_filters {
+                let filters = convert_partition_filters(filters).map_err(RubyError::from)?;
+                Ok(self_
+                    .with_table2(|t| {
+                        rt().block_on(async {
+                            t.get_files_by_partitions(&filters)
+                                .await
+                                .map_err(RubyError::from)
+                        })
+                    })?
+                    .into_iter()
+                    .map(|p| p.to_string())
+                    .collect())
+            } else {
+                match self_._table.lock() {
+                    Ok(table) => Ok(table
+                        .get_file_uris()
+                        .map_err(RubyError::from)?
+                        .map(|f| f.to_string())
+                        .collect()),
+                    Err(e) => Err(RubyError::RuntimeError(e.to_string())),
+                }
             }
-        }
+        })
+        .map_err(RbErr::from)
     }
 
     pub fn file_uris(
@@ -399,36 +441,84 @@ impl RawDeltaTable {
     }
 
     pub fn vacuum(
-        &self,
+        rb: &Ruby,
+        self_: &Self,
         dry_run: bool,
         retention_hours: Option<u64>,
         enforce_retention_duration: bool,
         commit_properties: Option<RbCommitProperties>,
         post_commithook_properties: Option<RbPostCommitHookProperties>,
     ) -> RbResult<Vec<String>> {
-        let table = self._table.lock().map_err(to_rt_err)?.clone();
-        let mut cmd = table
-            .vacuum()
-            .with_enforce_retention_duration(enforce_retention_duration)
-            .with_dry_run(dry_run);
+        let (table, metrics) = rb.detach(|| {
+            let table = self_._table.lock().map_err(to_rt_err2)?.clone();
+            let mut cmd = table
+                .vacuum()
+                .with_enforce_retention_duration(enforce_retention_duration)
+                .with_dry_run(dry_run);
+
+            if let Some(retention_period) = retention_hours {
+                cmd = cmd.with_retention_period(Duration::hours(retention_period as i64));
+            }
 
-        if let Some(retention_period) = retention_hours {
-            cmd = cmd.with_retention_period(Duration::hours(retention_period as i64));
-        }
+            if let Some(commit_properties) =
+                maybe_create_commit_properties(commit_properties, post_commithook_properties)
+            {
+                cmd = cmd.with_commit_properties(commit_properties);
+            }
 
-        if let Some(commit_properties) =
-            maybe_create_commit_properties(commit_properties, post_commithook_properties)
-        {
-            cmd = cmd.with_commit_properties(commit_properties);
-        }
-        let (table, metrics) = rt().block_on(cmd.into_future()).map_err(RubyError::from)?;
-        self.set_state(table.state)?;
+            rt().block_on(cmd.into_future()).map_err(RubyError::from)
+        })?;
+        self_.set_state(table.state)?;
         Ok(metrics.files_deleted)
     }
 
+    #[allow(clippy::too_many_arguments)]
+    pub fn update(
+        rb: &Ruby,
+        self_: &Self,
+        updates: HashMap<String, String>,
+        predicate: Option<String>,
+        writer_properties: Option<RbWriterProperties>,
+        safe_cast: bool,
+        commit_properties: Option<RbCommitProperties>,
+        post_commithook_properties: Option<RbPostCommitHookProperties>,
+    ) -> RbResult<String> {
+        let (table, metrics) = rb
+            .detach(|| {
+                let table = self_._table.lock().map_err(to_rt_err2)?.clone();
+                let mut cmd = table.update().with_safe_cast(safe_cast);
+
+                if let Some(writer_props) = writer_properties {
+                    cmd = cmd.with_writer_properties(
+                        set_writer_properties(writer_props).map_err(RubyError::from)?,
+                    );
+                }
+
+                for (col_name, expression) in updates {
+                    cmd = cmd.with_update(col_name.clone(), expression.clone());
+                }
+
+                if let Some(update_predicate) = predicate {
+                    cmd = cmd.with_predicate(update_predicate);
+                }
+
+                if let Some(commit_properties) =
+                    maybe_create_commit_properties(commit_properties, post_commithook_properties)
+                {
+                    cmd = cmd.with_commit_properties(commit_properties);
+                }
+
+                rt().block_on(cmd.into_future()).map_err(RubyError::from)
+            })
+            .map_err(RbErr::from)?;
+        self_.set_state(table.state)?;
+        Ok(serde_json::to_string(&metrics).unwrap())
+    }
+
     #[allow(clippy::too_many_arguments)]
     pub fn compact_optimize(
-        &self,
+        rb: &Ruby,
+        self_: &Self,
         partition_filters: Option<Vec<(String, String, PartitionFilterValue)>>,
         target_size: Option<u64>,
         max_concurrent_tasks: Option<usize>,
@@ -437,42 +527,47 @@ impl RawDeltaTable {
         commit_properties: Option<RbCommitProperties>,
         post_commithook_properties: Option<RbPostCommitHookProperties>,
     ) -> RbResult<String> {
-        let table = self._table.lock().map_err(to_rt_err)?.clone();
-        let mut cmd = table
-            .optimize()
-            .with_max_concurrent_tasks(max_concurrent_tasks.unwrap_or_else(num_cpus::get));
-
-        if let Some(size) = target_size {
-            cmd = cmd.with_target_size(size);
-        }
-        if let Some(commit_interval) = min_commit_interval {
-            cmd = cmd.with_min_commit_interval(time::Duration::from_secs(commit_interval));
-        }
+        let (table, metrics) = rb.detach(|| {
+            let table = self_._table.lock().map_err(to_rt_err2)?.clone();
+            let mut cmd = table
+                .optimize()
+                .with_max_concurrent_tasks(max_concurrent_tasks.unwrap_or_else(num_cpus::get));
+
+            if let Some(target_size) = target_size {
+                let target_size = parse_optimize_target_size(target_size)?;
+                cmd = cmd.with_target_size(target_size);
+            }
+            if let Some(commit_interval) = min_commit_interval {
+                cmd = cmd.with_min_commit_interval(time::Duration::from_secs(commit_interval));
+            }
 
-        if let Some(writer_props) = writer_properties {
-            cmd = cmd.with_writer_properties(
-                set_writer_properties(writer_props).map_err(RubyError::from)?,
-            );
-        }
+            if let Some(writer_props) = writer_properties {
+                cmd = cmd.with_writer_properties(
+                    set_writer_properties(writer_props).map_err(RubyError::from)?,
+                );
+            }
 
-        if let Some(commit_properties) =
-            maybe_create_commit_properties(commit_properties, post_commithook_properties)
-        {
-            cmd = cmd.with_commit_properties(commit_properties);
-        }
+            if let Some(commit_properties) =
+                maybe_create_commit_properties(commit_properties, post_commithook_properties)
+            {
+                cmd = cmd.with_commit_properties(commit_properties);
+            }
 
-        let converted_filters = convert_partition_filters(partition_filters.unwrap_or_default())
-            .map_err(RubyError::from)?;
-        cmd = cmd.with_filters(&converted_filters);
+            let converted_filters =
+                convert_partition_filters(partition_filters.unwrap_or_default())
+                    .map_err(RubyError::from)?;
+            cmd = cmd.with_filters(&converted_filters);
 
-        let (table, metrics) = rt().block_on(cmd.into_future()).map_err(RubyError::from)?;
-        self.set_state(table.state)?;
+            rt().block_on(cmd.into_future()).map_err(RubyError::from)
+        })?;
+        self_.set_state(table.state)?;
         Ok(serde_json::to_string(&metrics).unwrap())
     }
 
     #[allow(clippy::too_many_arguments)]
     pub fn z_order_optimize(
-        &self,
+        rb: &Ruby,
+        self_: &Self,
         z_order_columns: Vec<String>,
         partition_filters: Option<Vec<(String, String, PartitionFilterValue)>>,
         target_size: Option<u64>,
@@ -484,67 +579,77 @@ impl RawDeltaTable {
         commit_properties: Option<RbCommitProperties>,
         post_commithook_properties: Option<RbPostCommitHookProperties>,
     ) -> RbResult<String> {
-        let table = self._table.lock().map_err(to_rt_err)?.clone();
-        let mut cmd = table
-            .clone()
-            .optimize()
-            .with_max_concurrent_tasks(max_concurrent_tasks.unwrap_or_else(num_cpus::get))
-            .with_type(OptimizeType::ZOrder(z_order_columns));
-
-        if max_spill_size.is_some() || max_temp_directory_size.is_some() {
-            let session =
-                create_session_state_for_optimize(max_spill_size, max_temp_directory_size);
-            cmd = cmd.with_session_state(Arc::new(session));
-        }
+        let (table, metrics) = rb.detach(|| {
+            let table = self_._table.lock().map_err(to_rt_err2)?.clone();
+            let mut cmd = table
+                .clone()
+                .optimize()
+                .with_max_concurrent_tasks(max_concurrent_tasks.unwrap_or_else(num_cpus::get))
+                .with_type(OptimizeType::ZOrder(z_order_columns));
+
+            if max_spill_size.is_some() || max_temp_directory_size.is_some() {
+                let session =
+                    create_session_state_with_spill_config(max_spill_size, max_temp_directory_size);
+                cmd = cmd.with_session_state(Arc::new(session));
+            }
 
-        if let Some(size) = target_size {
-            cmd = cmd.with_target_size(size);
-        }
-        if let Some(commit_interval) = min_commit_interval {
-            cmd = cmd.with_min_commit_interval(time::Duration::from_secs(commit_interval));
-        }
+            if let Some(target_size) = target_size {
+                let target_size = parse_optimize_target_size(target_size)?;
+                cmd = cmd.with_target_size(target_size);
+            }
+            if let Some(commit_interval) = min_commit_interval {
+                cmd = cmd.with_min_commit_interval(time::Duration::from_secs(commit_interval));
+            }
 
-        if let Some(writer_props) = writer_properties {
-            cmd = cmd.with_writer_properties(
-                set_writer_properties(writer_props).map_err(RubyError::from)?,
-            );
-        }
+            if let Some(writer_props) = writer_properties {
+                cmd = cmd.with_writer_properties(
+                    set_writer_properties(writer_props).map_err(RubyError::from)?,
+                );
+            }
 
-        if let Some(commit_properties) =
-            maybe_create_commit_properties(commit_properties, post_commithook_properties)
-        {
-            cmd = cmd.with_commit_properties(commit_properties);
-        }
+            if let Some(commit_properties) =
+                maybe_create_commit_properties(commit_properties, post_commithook_properties)
+            {
+                cmd = cmd.with_commit_properties(commit_properties);
+            }
 
-        let converted_filters = convert_partition_filters(partition_filters.unwrap_or_default())
-            .map_err(RubyError::from)?;
-        cmd = cmd.with_filters(&converted_filters);
+            let converted_filters =
+                convert_partition_filters(partition_filters.unwrap_or_default())
+                    .map_err(RubyError::from)?;
+            cmd = cmd.with_filters(&converted_filters);
 
-        let (table, metrics) = rt().block_on(cmd.into_future()).map_err(RubyError::from)?;
-        self.set_state(table.state)?;
+            rt().block_on(cmd.into_future()).map_err(RubyError::from)
+        })?;
+        self_.set_state(table.state)?;
         Ok(serde_json::to_string(&metrics).unwrap())
     }
 
-    pub fn add_columns(&self, fields: RArray) -> RbResult<()> {
-        let fields = fields.typecheck::<Obj<Field>>()?;
+    pub fn add_columns(rb: &Ruby, self_: &Self, fields: RArray) -> RbResult<()> {
+        let fields = fields
+            .into_iter()
+            .map(|v| <&Field>::try_convert(v).cloned())
+            .collect::<RbResult<Vec<_>>>()?;
 
-        let table = self._table.lock().map_err(to_rt_err)?.clone();
-        let mut cmd = table.add_columns();
+        let table = rb.detach(|| {
+            let table = self_._table.lock().map_err(to_rt_err2)?.clone();
+            let mut cmd = table.add_columns();
 
-        let new_fields = fields
-            .iter()
-            .map(|v| v.inner.clone())
-            .collect::<Vec<StructField>>();
+            let new_fields = fields
+                .iter()
+                .map(|v| v.inner.clone())
+                .collect::<Vec<StructField>>();
 
-        cmd = cmd.with_fields(new_fields);
+            cmd = cmd.with_fields(new_fields);
 
-        let table = rt().block_on(cmd.into_future()).map_err(RubyError::from)?;
-        self.set_state(table.state)?;
+            rt().block_on(cmd.into_future()).map_err(RubyError::from)
+        })?;
+        self_.set_state(table.state)?;
         Ok(())
     }
 
     pub fn add_feature(
-        &self,
+        rb: &Ruby,
+        self_: &Self,
         feature: RArray,
         allow_protocol_versions_increase: bool,
     ) -> RbResult<()> {
@@ -553,52 +658,75 @@ impl RawDeltaTable {
             .map(TableFeatures::try_convert)
             .collect::<RbResult<Vec<_>>>()?;
 
-        let table = self._table.lock().map_err(to_rt_err)?.clone();
-        let cmd = table
-            .add_feature()
-            .with_features(feature)
-            .with_allow_protocol_versions_increase(allow_protocol_versions_increase);
+        let table = rb.detach(|| {
+            let table = self_._table.lock().map_err(to_rt_err2)?.clone();
+            let cmd = table
+                .add_feature()
+                .with_features(feature)
+                .with_allow_protocol_versions_increase(allow_protocol_versions_increase);
 
-        let table = rt().block_on(cmd.into_future()).map_err(RubyError::from)?;
-        self.set_state(table.state)?;
+            rt().block_on(cmd.into_future()).map_err(RubyError::from)
+        })?;
+        self_.set_state(table.state)?;
         Ok(())
     }
 
-    pub fn add_constraints(&self, constraints: HashMap<String, String>) -> RbResult<()> {
-        let table = self._table.lock().map_err(to_rt_err)?.clone();
-        let mut cmd = table.add_constraint();
+    pub fn add_constraints(
+        rb: &Ruby,
+        self_: &Self,
+        constraints: HashMap<String, String>,
+    ) -> RbResult<()> {
+        let table = rb.detach(|| {
+            let table = self_._table.lock().map_err(to_rt_err2)?.clone();
+            let mut cmd = table.add_constraint();
 
-        for (col_name, expression) in constraints {
-            cmd = cmd.with_constraint(col_name.clone(), expression.clone());
-        }
+            for (col_name, expression) in constraints {
+                cmd = cmd.with_constraint(col_name.clone(), expression.clone());
+            }
 
-        let table = rt().block_on(cmd.into_future()).map_err(RubyError::from)?;
-        self.set_state(table.state)?;
+            rt().block_on(cmd.into_future()).map_err(RubyError::from)
+        })?;
+        self_.set_state(table.state)?;
         Ok(())
     }
 
-    pub fn drop_constraints(&self, name: String, raise_if_not_exists: bool) -> RbResult<()> {
-        let table = self._table.lock().map_err(to_rt_err)?.clone();
-        let cmd = table
-            .drop_constraints()
-            .with_constraint(name)
-            .with_raise_if_not_exists(raise_if_not_exists);
+    pub fn drop_constraints(
+        rb: &Ruby,
+        self_: &Self,
+        name: String,
+        raise_if_not_exists: bool,
+    ) -> RbResult<()> {
+        let table = rb.detach(|| {
+            let table = self_._table.lock().map_err(to_rt_err2)?.clone();
+            let cmd = table
+                .drop_constraints()
+                .with_constraint(name)
+                .with_raise_if_not_exists(raise_if_not_exists);
+
+            rt().block_on(cmd.into_future()).map_err(RubyError::from)
+        })?;
+        self_.set_state(table.state)?;
+        Ok(())
+    }
 
-        let table = rt().block_on(cmd.into_future()).map_err(RubyError::from)?;
-        self.set_state(table.state)?;
+    pub fn generate(&self) -> RbResult<()> {
+        let table = self._table.lock().map_err(to_rt_err)?.clone();
+        rt().block_on(async { table.generate().await })
+            .map_err(RubyError::from)?;
         Ok(())
     }
 
     pub fn load_cdf(
-        &self,
-        starting_version: Option<i64>,
-        ending_version: Option<i64>,
+        rb: &Ruby,
+        self_: &Self,
+        starting_version: Option<Version>,
+        ending_version: Option<Version>,
         starting_timestamp: Option<String>,
         ending_timestamp: Option<String>,
         columns: Option<Vec<String>>,
     ) -> RbResult<ArrowArrayStream> {
         let ctx = SessionContext::new();
-        let table = self._table.lock().map_err(to_rt_err)?.clone();
+        let table = self_._table.lock().map_err(to_rt_err)?.clone();
         let mut cmd = table.scan_cdf();
 
         if let Some(sv) = starting_version {
@@ -623,38 +751,41 @@ impl RawDeltaTable {
         let table_provider: Arc<dyn TableProvider> =
             Arc::new(DeltaCdfTableProvider::try_new(cmd).map_err(RubyError::from)?);
 
-        let plan = rt()
-            .block_on(async {
-                let mut df = ctx.read_table(table_provider)?;
-                if let Some(columns) = columns {
-                    let cols: Vec<_> = columns.iter().map(|c| c.as_ref()).collect();
-                    df = df.select_columns(&cols)?;
-                }
-                df.create_physical_plan().await
-            })
-            .map_err(RubyError::from)?;
+        rb.detach(|| {
+            let plan = rt()
+                .block_on(async {
+                    let mut df = ctx.read_table(table_provider)?;
+                    if let Some(columns) = columns {
+                        let cols: Vec<_> = columns.iter().map(|c| c.as_ref()).collect();
+                        df = df.select_columns(&cols)?;
+                    }
+                    df.create_physical_plan().await
+                })
+                .map_err(RubyError::from)?;
 
-        let mut tasks = vec![];
-        for p in 0..plan.properties().output_partitioning().partition_count() {
-            let inner_plan = plan.clone();
-            let partition_batch = inner_plan.execute(p, ctx.task_ctx()).unwrap();
-            let handle = rt().spawn(collect_sendable_stream(partition_batch));
-            tasks.push(handle);
-        }
+            let mut tasks = vec![];
+            for p in 0..plan.properties().output_partitioning().partition_count() {
+                let inner_plan = plan.clone();
+                let partition_batch = inner_plan.execute(p, ctx.task_ctx()).unwrap();
+                let handle = rt().spawn(collect_sendable_stream(partition_batch));
+                tasks.push(handle);
+            }
 
-        // This is unfortunate.
-        let batches = rt()
-            .block_on(join_all(tasks))
-            .into_iter()
-            .flatten()
-            .collect::<Result<Vec<Vec<_>>, _>>()
-            .unwrap()
-            .into_iter()
-            .flatten()
-            .map(Ok);
-        let batch_iter = RecordBatchIterator::new(batches, plan.schema());
-        let ffi_stream = FFI_ArrowArrayStream::new(Box::new(batch_iter));
-        Ok(ArrowArrayStream { stream: ffi_stream })
+            // This is unfortunate.
+            let batches = rt()
+                .block_on(join_all(tasks))
+                .into_iter()
+                .flatten()
+                .collect::<Result<Vec<Vec<_>>, _>>()
+                .unwrap()
+                .into_iter()
+                .flatten()
+                .map(Ok);
+            let batch_iter = RecordBatchIterator::new(batches, plan.schema());
+            let ffi_stream = FFI_ArrowArrayStream::new(Box::new(batch_iter));
+            Ok::<_, RubyError>(ArrowArrayStream { stream: ffi_stream })
+        })
+        .map_err(RbErr::from)
     }
 
     #[allow(clippy::too_many_arguments)]
@@ -701,7 +832,7 @@ impl RawDeltaTable {
         let mut cmd = table.restore();
         if let Some(val) = target {
             if let Some(version) = Integer::from_value(val) {
-                cmd = cmd.with_version_to_restore(version.to_i64()?)
+                cmd = cmd.with_version_to_restore(version.to_u64()?)
             }
             if let Ok(ds) = String::try_convert(val) {
                 let datetime = DateTime::<Utc>::from(
@@ -809,35 +940,64 @@ impl RawDeltaTable {
         Ok(ruby.ary_from_iter(active_partitions))
     }
 
-    pub fn create_checkpoint(&self) -> RbResult<()> {
-        let operation_id = Uuid::new_v4();
+    pub fn create_checkpoint(rb: &Ruby, self_: &Self) -> RbResult<()> {
+        rb.detach(|| {
+            let operation_id = Uuid::new_v4();
 
-        #[allow(clippy::await_holding_lock)]
-        let _result = rt().block_on(async {
-            match self._table.lock() {
-                Ok(table) => create_checkpoint(&table, Some(operation_id))
-                    .await
-                    .map_err(RubyError::from)
-                    .map_err(RbErr::from),
-                Err(e) => Err(RbRuntimeError::new_err(e.to_string())),
-            }
-        });
+            #[allow(clippy::await_holding_lock)]
+            let _result = rt().block_on(async {
+                match self_._table.lock() {
+                    Ok(table) => create_checkpoint(&table, Some(operation_id))
+                        .await
+                        .map_err(RubyError::from),
+                    Err(e) => Err(RubyError::RuntimeError(e.to_string())),
+                }
+            });
+
+            Ok::<_, RubyError>(())
+        })
+        .map_err(RbErr::from)
+    }
+
+    pub fn compact_logs(
+        rb: &Ruby,
+        self_: &Self,
+        starting_version: u64,
+        ending_version: u64,
+    ) -> RbResult<()> {
+        rb.detach(|| {
+            let operation_id = Uuid::new_v4();
+
+            #[allow(clippy::await_holding_lock)]
+            let result = rt().block_on(async {
+                match self_._table.lock() {
+                    Ok(table) => {
+                        compact_logs(&table, starting_version, ending_version, Some(operation_id))
+                            .await
+                            .map_err(RubyError::from)
+                    }
+                    Err(e) => Err(RubyError::RuntimeError(e.to_string())),
+                }
+            });
+
+            result
+        })
+        .map_err(RbErr::from)?;
 
         Ok(())
     }
 
-    pub fn cleanup_metadata(&self) -> RbResult<()> {
-        let (_result, new_state) = {
+    pub fn cleanup_metadata(rb: &Ruby, self_: &Self) -> RbResult<()> {
+        let (_result, new_state) = rb.detach(|| {
             let operation_id = Uuid::new_v4();
 
             #[allow(clippy::await_holding_lock)]
             let result = rt().block_on(async {
-                match self._table.lock() {
+                match self_._table.lock() {
                     Ok(table) => {
                         let result = cleanup_metadata(&table, Some(operation_id))
                             .await
-                            .map_err(RubyError::from)
-                            .map_err(RbErr::from)?;
+                            .map_err(RubyError::from)?;
 
                         let new_state = if result > 0 {
                             Some(
@@ -855,15 +1015,15 @@ impl RawDeltaTable {
 
                         Ok((result, new_state))
                     }
-                    Err(e) => Err(RbRuntimeError::new_err(e.to_string())),
+                    Err(e) => Err(RubyError::RuntimeError(e.to_string())),
                 }
             });
 
             result
-        }?;
+        })?;
 
         if new_state.is_some() {
-            self.set_state(new_state)?;
+            self_.set_state(new_state)?;
         }
 
         Ok(())
@@ -887,30 +1047,33 @@ impl RawDeltaTable {
     }
 
     pub fn delete(
-        &self,
+        rb: &Ruby,
+        self_: &Self,
         predicate: Option<String>,
         writer_properties: Option<RbWriterProperties>,
         commit_properties: Option<RbCommitProperties>,
         post_commithook_properties: Option<RbPostCommitHookProperties>,
     ) -> RbResult<String> {
-        let table = self._table.lock().map_err(to_rt_err)?.clone();
-        let mut cmd = table.delete();
-        if let Some(predicate) = predicate {
-            cmd = cmd.with_predicate(predicate);
-        }
-        if let Some(writer_props) = writer_properties {
-            cmd = cmd.with_writer_properties(
-                set_writer_properties(writer_props).map_err(RubyError::from)?,
-            );
-        }
-        if let Some(commit_properties) =
-            maybe_create_commit_properties(commit_properties, post_commithook_properties)
-        {
-            cmd = cmd.with_commit_properties(commit_properties);
-        }
+        let (table, metrics) = rb.detach(|| {
+            let table = self_._table.lock().map_err(to_rt_err2)?.clone();
+            let mut cmd = table.delete();
+            if let Some(predicate) = predicate {
+                cmd = cmd.with_predicate(predicate);
+            }
+            if let Some(writer_props) = writer_properties {
+                cmd = cmd.with_writer_properties(
+                    set_writer_properties(writer_props).map_err(RubyError::from)?,
+                );
+            }
+            if let Some(commit_properties) =
+                maybe_create_commit_properties(commit_properties, post_commithook_properties)
+            {
+                cmd = cmd.with_commit_properties(commit_properties);
+            }
 
-        let (table, metrics) = rt().block_on(cmd.into_future()).map_err(RubyError::from)?;
-        self.set_state(table.state)?;
+            rt().block_on(cmd.into_future()).map_err(RubyError::from)
+        })?;
+        self_.set_state(table.state)?;
         Ok(serde_json::to_string(&metrics).unwrap())
     }
 
@@ -930,6 +1093,54 @@ impl RawDeltaTable {
         Ok(())
     }
 
+    pub fn set_table_name(
+        &self,
+        name: String,
+        commit_properties: Option<RbCommitProperties>,
+        post_commithook_properties: Option<RbPostCommitHookProperties>,
+    ) -> RbResult<()> {
+        let update = TableMetadataUpdate {
+            name: Some(name),
+            description: None,
+        };
+        let table = self._table.lock().map_err(to_rt_err)?.clone();
+        let mut cmd = table.update_table_metadata().with_update(update);
+
+        if let Some(commit_properties) =
+            maybe_create_commit_properties(commit_properties, post_commithook_properties)
+        {
+            cmd = cmd.with_commit_properties(commit_properties);
+        }
+
+        let table = rt().block_on(cmd.into_future()).map_err(RubyError::from)?;
+        self.set_state(table.state)?;
+        Ok(())
+    }
+
+    pub fn set_table_description(
+        &self,
+        description: String,
+        commit_properties: Option<RbCommitProperties>,
+        post_commithook_properties: Option<RbPostCommitHookProperties>,
+    ) -> RbResult<()> {
+        let update = TableMetadataUpdate {
+            name: None,
+            description: Some(description),
+        };
+        let table = self._table.lock().map_err(to_rt_err)?.clone();
+        let mut cmd = table.update_table_metadata().with_update(update);
+
+        if let Some(commit_properties) =
+            maybe_create_commit_properties(commit_properties, post_commithook_properties)
+        {
+            cmd = cmd.with_commit_properties(commit_properties);
+        }
+
+        let table = rt().block_on(cmd.into_future()).map_err(RubyError::from)?;
+        self.set_state(table.state)?;
+        Ok(())
+    }
+
     pub fn repair(
         &self,
         dry_run: bool,
@@ -959,15 +1170,50 @@ impl RawDeltaTable {
             .map_err(RubyError::from)?)
     }
 
+    pub fn set_column_metadata(
+        rb: &Ruby,
+        self_: &Self,
+        field_name: String,
+        metadata: HashMap<String, String>,
+        commit_properties: Option<RbCommitProperties>,
+        post_commithook_properties: Option<RbPostCommitHookProperties>,
+    ) -> RbResult<()> {
+        let table = rb
+            .detach(|| {
+                let table = self_._table.lock().map_err(to_rt_err2)?.clone();
+                let mut cmd = table
+                    .update_field_metadata()
+                    .with_field_name(&field_name)
+                    .with_metadata(
+                        metadata
+                            .iter()
+                            .map(|(k, v)| (k.clone(), MetadataValue::String(v.clone())))
+                            .collect(),
+                    );
+
+                if let Some(commit_properties) =
+                    maybe_create_commit_properties(commit_properties, post_commithook_properties)
+                {
+                    cmd = cmd.with_commit_properties(commit_properties)
+                }
+
+                rt().block_on(cmd.into_future()).map_err(RubyError::from)
+            })
+            .map_err(RbErr::from)?;
+        self_.set_state(table.state)?;
+        Ok(())
+    }
+
     #[allow(clippy::too_many_arguments)]
     pub fn write(
-        &self,
+        rb: &Ruby,
+        self_: &Self,
         data: RbArrowType<ArrowArrayStreamReader>,
         mode: String,
         schema_mode: Option<String>,
         partition_by: Option<Vec<String>>,
         predicate: Option<String>,
-        target_file_size: Option<usize>,
+        target_file_size: Option<u64>,
         name: Option<String>,
         description: Option<String>,
         configuration: Option<HashMap<String, Option<String>>>,
@@ -975,8 +1221,8 @@ impl RawDeltaTable {
         commit_properties: Option<RbCommitProperties>,
         post_commithook_properties: Option<RbPostCommitHookProperties>,
     ) -> RbResult<()> {
-        let table = {
-            let table = self._table.lock().map_err(to_rt_err)?.clone();
+        let table = rb.detach(|| {
+            let table = self_._table.lock().map_err(to_rt_err2)?.clone();
             let batches = data.0.map(|batch| batch.unwrap()).collect::<Vec<_>>();
 
             let save_mode = mode.parse().map_err(RubyError::from)?;
@@ -1008,7 +1254,10 @@ impl RawDeltaTable {
             };
 
             if let Some(target_file_size) = target_file_size {
-                builder = builder.with_target_file_size(target_file_size)
+                let target_file_size = NonZeroU64::new(target_file_size).ok_or_else(|| {
+                    RubyError::ValueError("target_file_size must be greater than 0".to_string())
+                })?;
+                builder = builder.with_target_file_size(Some(target_file_size))
             };
 
             if let Some(config) = configuration {
@@ -1023,10 +1272,9 @@ impl RawDeltaTable {
 
             rt().block_on(builder.into_future())
                 .map_err(RubyError::from)
-                .map_err(RbErr::from)
-        }?;
+        })?;
 
-        self.set_state(table.state)?;
+        self_.set_state(table.state)?;
         Ok(())
     }
 }
@@ -1067,7 +1315,7 @@ fn set_writer_properties(writer_properties: RbWriterProperties) -> DeltaResult<W
         properties = properties.set_write_batch_size(batch_size);
     }
     if let Some(row_group_size) = max_row_group_size {
-        properties = properties.set_max_row_group_size(row_group_size);
+        properties = properties.set_max_row_group_row_count(Some(row_group_size));
     }
     properties = properties.set_statistics_truncate_length(statistics_truncate_length);
 
@@ -1311,13 +1559,14 @@ impl TryConvert for RbCommitProperties {
 
 #[allow(clippy::too_many_arguments)]
 fn write_to_deltalake(
+    rb: &Ruby,
     table_uri: String,
     data: RbArrowType<ArrowArrayStreamReader>,
     mode: String,
     schema_mode: Option<String>,
     partition_by: Option<Vec<String>>,
     predicate: Option<String>,
-    target_file_size: Option<usize>,
+    target_file_size: Option<u64>,
     name: Option<String>,
     description: Option<String>,
     configuration: Option<HashMap<String, Option<String>>>,
@@ -1326,62 +1575,36 @@ fn write_to_deltalake(
     commit_properties: Option<RbCommitProperties>,
     post_commithook_properties: Option<RbPostCommitHookProperties>,
 ) -> RbResult<()> {
-    let batches = data.0.map(|batch| batch.unwrap()).collect::<Vec<_>>();
-    let save_mode = mode.parse().map_err(RubyError::from)?;
-
-    let options = storage_options.clone().unwrap_or_default();
-    let table_url =
-        deltalake::table::builder::ensure_table_uri(&table_uri).map_err(RubyError::from)?;
-    let table = rt()
-        .block_on(DeltaTable::try_from_url_with_storage_options(
+    let raw_table: DeltaResult<RawDeltaTable> = rb.detach(|| {
+        let options = storage_options.clone().unwrap_or_default();
+        let table_url = deltalake::table::builder::ensure_table_uri(&table_uri)?;
+        let table = rt().block_on(DeltaTable::try_from_url_with_storage_options(
             table_url.clone(),
             options.clone(),
-        ))
-        .map_err(RubyError::from)?;
-
-    let mut builder = table.write(batches).with_save_mode(save_mode);
-    if let Some(schema_mode) = schema_mode {
-        builder = builder.with_schema_mode(schema_mode.parse().map_err(RubyError::from)?);
-    }
-    if let Some(partition_columns) = partition_by {
-        builder = builder.with_partition_columns(partition_columns);
-    }
-
-    if let Some(writer_props) = writer_properties {
-        builder = builder
-            .with_writer_properties(set_writer_properties(writer_props).map_err(RubyError::from)?);
-    }
-
-    if let Some(name) = &name {
-        builder = builder.with_table_name(name);
-    };
-
-    if let Some(description) = &description {
-        builder = builder.with_description(description);
-    };
-
-    if let Some(predicate) = predicate {
-        builder = builder.with_replace_where(predicate);
-    };
-
-    if let Some(target_file_size) = target_file_size {
-        builder = builder.with_target_file_size(target_file_size)
-    };
+        ))?;
 
-    if let Some(config) = configuration {
-        builder = builder.with_configuration(config);
-    };
-
-    if let Some(commit_properties) =
-        maybe_create_commit_properties(commit_properties, post_commithook_properties)
-    {
-        builder = builder.with_commit_properties(commit_properties);
-    };
-
-    rt().block_on(builder.into_future())
-        .map_err(RubyError::from)?;
-
-    Ok(())
+        let raw_table = RawDeltaTable {
+            _table: Arc::new(Mutex::new(table)),
+        };
+        Ok(raw_table)
+    });
+
+    RawDeltaTable::write(
+        rb,
+        &raw_table.map_err(RubyError::from)?,
+        data,
+        mode,
+        schema_mode,
+        partition_by,
+        predicate,
+        target_file_size,
+        name,
+        description,
+        configuration,
+        writer_properties,
+        commit_properties,
+        post_commithook_properties,
+    )
 }
 
 pub struct RbArrowType<T>(pub T);
@@ -1454,6 +1677,7 @@ fn init(ruby: &Ruby) -> RbResult<()> {
     class.define_method("file_uris", method!(RawDeltaTable::file_uris, 1))?;
     class.define_method("schema", method!(RawDeltaTable::schema, 0))?;
     class.define_method("vacuum", method!(RawDeltaTable::vacuum, 5))?;
+    class.define_method("update", method!(RawDeltaTable::update, 6))?;
     class.define_method(
         "compact_optimize",
         method!(RawDeltaTable::compact_optimize, 7),
@@ -1472,6 +1696,7 @@ fn init(ruby: &Ruby) -> RbResult<()> {
         "drop_constraints",
         method!(RawDeltaTable::drop_constraints, 2),
     )?;
+    class.define_method("generate", method!(RawDeltaTable::generate, 0))?;
     class.define_method("load_cdf", method!(RawDeltaTable::load_cdf, 5))?;
     class.define_method(
         "create_merge_builder",
@@ -1492,6 +1717,7 @@ fn init(ruby: &Ruby) -> RbResult<()> {
         "create_checkpoint",
         method!(RawDeltaTable::create_checkpoint, 0),
     )?;
+    class.define_method("compact_logs", method!(RawDeltaTable::compact_logs, 2))?;
     class.define_method(
         "cleanup_metadata",
         method!(RawDeltaTable::cleanup_metadata, 0),
@@ -1505,11 +1731,20 @@ fn init(ruby: &Ruby) -> RbResult<()> {
         "set_table_properties",
         method!(RawDeltaTable::set_table_properties, 2),
     )?;
+    class.define_method("set_table_name", method!(RawDeltaTable::set_table_name, 3))?;
+    class.define_method(
+        "set_table_description",
+        method!(RawDeltaTable::set_table_description, 3),
+    )?;
     class.define_method("repair", method!(RawDeltaTable::repair, 3))?;
     class.define_method(
         "transaction_version",
         method!(RawDeltaTable::transaction_version, 1),
     )?;
+    class.define_method(
+        "set_column_metadata",
+        method!(RawDeltaTable::set_column_metadata, 4),
+    )?;
     class.define_method("write", method!(RawDeltaTable::write, 12))?;
 
     let class = module.define_class("RawDeltaTableMetaData", ruby.class_object())?;