deltalake-rb 0.1.5 → 0.1.6

data/ext/deltalake/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltalake"
-version = "0.1.5"
+version = "0.1.6"
 license = "Apache-2.0"
 authors = ["Andrew Kane <andrew@ankane.org>"]
 edition = "2021"
@@ -11,11 +11,11 @@ publish = false
 crate-type = ["cdylib"]
 
 [dependencies]
-arrow = { version = "53", features = ["ffi"] }
-arrow-schema = { version = "53", features = ["serde"] }
+arrow = { version = "54", features = ["ffi"] }
+arrow-schema = { version = "54", features = ["serde"] }
 chrono = "0.4"
-delta_kernel = "=0.6.0"
-deltalake = { version = "=0.24.0", features = ["azure", "datafusion", "gcs", "s3"] }
+delta_kernel = "=0.8.0"
+deltalake = { version = "=0.25.0", features = ["azure", "datafusion", "gcs", "s3"] }
 futures = "0.3"
 magnus = "0.7"
 num_cpus = "1"

data/ext/deltalake/src/error.rs

@@ -1,4 +1,5 @@
 use arrow_schema::ArrowError;
+use deltalake::datafusion::error::DataFusionError;
 use deltalake::protocol::ProtocolError;
 use deltalake::{errors::DeltaTableError, ObjectStoreError};
 use magnus::{exception, Error as RbErr, Module, RModule, Ruby};
@@ -98,9 +99,14 @@ fn checkpoint_to_rb(err: ProtocolError) -> RbErr {
     }
 }
 
+fn datafusion_to_rb(err: DataFusionError) -> RbErr {
+    DeltaError::new_err(err.to_string())
+}
+
 pub enum RubyError {
     DeltaTable(DeltaTableError),
     Protocol(ProtocolError),
+    DataFusion(DataFusionError),
 }
 
 impl From<DeltaTableError> for RubyError {
@@ -115,11 +121,18 @@ impl From<ProtocolError> for RubyError {
     }
 }
 
+impl From<DataFusionError> for RubyError {
+    fn from(err: DataFusionError) -> Self {
+        RubyError::DataFusion(err)
+    }
+}
+
 impl From<RubyError> for RbErr {
     fn from(value: RubyError) -> Self {
         match value {
             RubyError::DeltaTable(err) => inner_to_rb_err(err),
             RubyError::Protocol(err) => checkpoint_to_rb(err),
+            RubyError::DataFusion(err) => datafusion_to_rb(err),
         }
     }
 }

data/ext/deltalake/src/lib.rs

@@ -8,6 +8,7 @@ use std::cell::RefCell;
 use std::collections::{HashMap, HashSet};
 use std::future::IntoFuture;
 use std::str::FromStr;
+use std::sync::Arc;
 use std::time;
 
 use chrono::{DateTime, Duration, FixedOffset, Utc};
@@ -15,8 +16,9 @@ use delta_kernel::schema::StructField;
 use deltalake::arrow::ffi_stream::{ArrowArrayStreamReader, FFI_ArrowArrayStream};
 use deltalake::arrow::record_batch::RecordBatchIterator;
 use deltalake::checkpoints::{cleanup_metadata, create_checkpoint};
-use deltalake::datafusion::physical_plan::ExecutionPlan;
+use deltalake::datafusion::catalog::TableProvider;
 use deltalake::datafusion::prelude::SessionContext;
+use deltalake::delta_datafusion::DeltaCdfTableProvider;
 use deltalake::errors::DeltaTableError;
 use deltalake::kernel::{scalars::ScalarExt, StructType, Transaction};
 use deltalake::operations::add_column::AddColumnBuilder;
@@ -364,6 +366,7 @@ impl RawDeltaTable {
         Ok(metrics.files_deleted)
     }
 
+    #[allow(clippy::too_many_arguments)]
     pub fn compact_optimize(
         &self,
         partition_filters: Option<Vec<(String, String, PartitionFilterValue)>>,
@@ -411,6 +414,7 @@ impl RawDeltaTable {
         Ok(serde_json::to_string(&metrics).unwrap())
     }
 
+    #[allow(clippy::too_many_arguments)]
     pub fn z_order_optimize(
         &self,
         z_order_columns: Vec<String>,
@@ -492,7 +496,7 @@ impl RawDeltaTable {
     ) -> RbResult<()> {
         let feature = feature
             .into_iter()
-            .map(|v| TableFeatures::try_convert(v))
+            .map(TableFeatures::try_convert)
             .collect::<RbResult<Vec<_>>>()?;
         let cmd = AddTableFeatureBuilder::new(
             self._table.borrow().log_store(),
@@ -581,13 +585,19 @@ impl RawDeltaTable {
             cdf_read = cdf_read.with_starting_timestamp(ending_ts);
         }
 
-        if let Some(columns) = columns {
-            cdf_read = cdf_read.with_columns(columns);
-        }
-
-        cdf_read = cdf_read.with_session_ctx(ctx.clone());
+        let table_provider: Arc<dyn TableProvider> =
+            Arc::new(DeltaCdfTableProvider::try_new(cdf_read).map_err(RubyError::from)?);
 
-        let plan = rt().block_on(cdf_read.build()).map_err(RubyError::from)?;
+        let plan = rt()
+            .block_on(async {
+                let mut df = ctx.read_table(table_provider)?;
+                if let Some(columns) = columns {
+                    let cols: Vec<_> = columns.iter().map(|c| c.as_ref()).collect();
+                    df = df.select_columns(&cols)?;
+                }
+                df.create_physical_plan().await
+            })
+            .map_err(RubyError::from)?;
 
         let mut tasks = vec![];
         for p in 0..plan.properties().output_partitioning().partition_count() {
@@ -612,6 +622,7 @@ impl RawDeltaTable {
         Ok(ArrowArrayStream { stream: ffi_stream })
     }
 
+    #[allow(clippy::too_many_arguments)]
     pub fn create_merge_builder(
         &self,
         source: RbArrowType<ArrowArrayStreamReader>,
@@ -926,9 +937,6 @@ fn set_writer_properties(writer_properties: RbWriterProperties) -> DeltaResult<W
         if let Some(dictionary_enabled) = default_column_properties.dictionary_enabled {
             properties = properties.set_dictionary_enabled(dictionary_enabled);
         }
-        if let Some(max_statistics_size) = default_column_properties.max_statistics_size {
-            properties = properties.set_max_statistics_size(max_statistics_size);
-        }
         if let Some(bloom_filter_properties) = default_column_properties.bloom_filter_properties {
             if let Some(set_bloom_filter_enabled) = bloom_filter_properties.set_bloom_filter_enabled
             {
@@ -1195,9 +1203,8 @@ fn write_to_deltalake(
     }
 
     if let Some(writer_props) = writer_properties {
-        builder = builder.with_writer_properties(
-            set_writer_properties(writer_props).map_err(RubyError::from)?,
-        );
+        builder = builder
+            .with_writer_properties(set_writer_properties(writer_props).map_err(RubyError::from)?);
     }
 
     if let Some(name) = &name {

data/ext/deltalake/src/merge.rs

@@ -42,6 +42,7 @@ impl RbMergeBuilder {
 }
 
 impl RbMergeBuilder {
+    #[allow(clippy::too_many_arguments)]
     pub fn new(
         log_store: LogStoreRef,
         snapshot: DeltaTableState,

data/lib/deltalake/table.rb

@@ -1,5 +1,7 @@
 module DeltaLake
   class Table
+    FSCK_METRICS_FILES_REMOVED_LABEL = "files_removed"
+
     def initialize(
       table_uri,
       version: nil,
@@ -244,7 +246,11 @@ module DeltaLake
           commit_properties,
           post_commithook_properties
         )
-      JSON.parse(metrics).transform_keys(&:to_sym)
+      deserialized_metrics = JSON.parse(metrics)
+      deserialized_metrics[FSCK_METRICS_FILES_REMOVED_LABEL] = JSON.parse(
+        deserialized_metrics[FSCK_METRICS_FILES_REMOVED_LABEL]
+      )
+      deserialized_metrics.transform_keys(&:to_sym)
     end
 
     def transaction_versions

data/lib/deltalake/version.rb

@@ -1,3 +1,3 @@
 module DeltaLake
-  VERSION = "0.1.5"
+  VERSION = "0.1.6"
 end

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification
 name: deltalake-rb
 version: !ruby/object:Gem::Version
-  version: 0.1.5
+  version: 0.1.6
 platform: ruby
 authors:
 - Andrew Kane
 bindir: bin
 cert_chain: []
-date: 2025-01-28 00:00:00.000000000 Z
+date: 2025-03-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rb_sys