deltalake-rb 0.1.7 → 0.2.0

data/ext/deltalake/Cargo.toml

@@ -1,21 +1,21 @@
 [package]
 name = "deltalake"
-version = "0.1.7"
+version = "0.2.0"
 license = "Apache-2.0"
 authors = ["Andrew Kane <andrew@ankane.org>"]
 edition = "2021"
-rust-version = "1.82.0"
+rust-version = "1.82"
 publish = false
 
 [lib]
 crate-type = ["cdylib"]
 
 [dependencies]
-arrow = { version = "55", features = ["ffi"] }
-arrow-schema = { version = "55", features = ["serde"] }
+arrow = { version = "55.2", features = ["ffi"] }
+arrow-schema = { version = "55.2", features = ["serde"] }
 chrono = "0.4"
-delta_kernel = "=0.10.0"
-deltalake = { version = "=0.26.0", features = ["azure", "datafusion", "gcs", "s3"] }
+delta_kernel = "0.13"
+deltalake = { version = "=0.27.0", features = ["azure", "datafusion", "gcs", "s3"] }
 futures = "0.3"
 magnus = "0.7"
 num_cpus = "1"

data/ext/deltalake/src/error.rs

@@ -1,6 +1,5 @@
 use arrow_schema::ArrowError;
 use deltalake::datafusion::error::DataFusionError;
-use deltalake::protocol::ProtocolError;
 use deltalake::{errors::DeltaTableError, ObjectStoreError};
 use magnus::{exception, Error as RbErr, Module, RModule, Ruby};
 use std::borrow::Cow;
@@ -81,31 +80,12 @@ fn arrow_to_rb(err: ArrowError) -> RbErr {
     }
 }
 
-fn checkpoint_to_rb(err: ProtocolError) -> RbErr {
-    match err {
-        ProtocolError::Arrow { source } => arrow_to_rb(source),
-        ProtocolError::ObjectStore { source } => object_store_to_rb(source),
-        ProtocolError::EndOfLog => DeltaProtocolError::new_err("End of log"),
-        ProtocolError::NoMetaData => DeltaProtocolError::new_err("Table metadata missing"),
-        ProtocolError::CheckpointNotFound => DeltaProtocolError::new_err(err.to_string()),
-        ProtocolError::InvalidField(err) => RbValueError::new_err(err),
-        ProtocolError::InvalidRow(err) => RbValueError::new_err(err),
-        ProtocolError::InvalidDeletionVectorStorageType(err) => RbValueError::new_err(err),
-        ProtocolError::SerializeOperation { source } => RbValueError::new_err(source.to_string()),
-        ProtocolError::ParquetParseError { source } => RbIOError::new_err(source.to_string()),
-        ProtocolError::IO { source } => RbIOError::new_err(source.to_string()),
-        ProtocolError::Generic(msg) => DeltaError::new_err(msg),
-        ProtocolError::Kernel { source } => DeltaError::new_err(source.to_string()),
-    }
-}
-
 fn datafusion_to_rb(err: DataFusionError) -> RbErr {
     DeltaError::new_err(err.to_string())
 }
 
 pub enum RubyError {
     DeltaTable(DeltaTableError),
-    Protocol(ProtocolError),
     DataFusion(DataFusionError),
 }
 
@@ -115,12 +95,6 @@ impl From<DeltaTableError> for RubyError {
     }
 }
 
-impl From<ProtocolError> for RubyError {
-    fn from(err: ProtocolError) -> Self {
-        RubyError::Protocol(err)
-    }
-}
-
 impl From<DataFusionError> for RubyError {
     fn from(err: DataFusionError) -> Self {
         RubyError::DataFusion(err)
@@ -131,7 +105,6 @@ impl From<RubyError> for RbErr {
     fn from(value: RubyError) -> Self {
         match value {
             RubyError::DeltaTable(err) => inner_to_rb_err(err),
-            RubyError::Protocol(err) => checkpoint_to_rb(err),
             RubyError::DataFusion(err) => datafusion_to_rb(err),
         }
     }

data/ext/deltalake/src/lib.rs

@@ -23,6 +23,7 @@ use deltalake::errors::DeltaTableError;
 use deltalake::kernel::transaction::{CommitProperties, TableReference};
 use deltalake::kernel::{scalars::ScalarExt, StructType, Transaction};
 use deltalake::logstore::IORuntime;
+use deltalake::logstore::LogStoreRef;
 use deltalake::operations::add_column::AddColumnBuilder;
 use deltalake::operations::add_feature::AddTableFeatureBuilder;
 use deltalake::operations::collect_sendable_stream;
@@ -44,20 +45,18 @@ use error::DeltaError;
 use futures::future::join_all;
 
 use magnus::{
-    function, method, prelude::*, typed_data::Obj, Error, Integer, Module, RArray, RHash, Ruby,
+    function, method, prelude::*, typed_data::Obj, Error as RbErr, Integer, Module, RArray, Ruby,
     TryConvert, Value,
 };
 use serde_json::Map;
 
-use crate::error::DeltaProtocolError;
-use crate::error::RbValueError;
-use crate::error::RubyError;
+use crate::error::{DeltaProtocolError, RbValueError, RubyError};
 use crate::features::TableFeatures;
 use crate::merge::RbMergeBuilder;
 use crate::schema::{schema_to_rbobject, Field};
 use crate::utils::rt;
 
-type RbResult<T> = Result<T, Error>;
+type RbResult<T> = Result<T, RbErr>;
 
 enum PartitionFilterValue {
     Single(String),
@@ -86,7 +85,7 @@ struct RawDeltaTableMetaData {
     description: Option<String>,
     partition_columns: Vec<String>,
     created_time: Option<i64>,
-    configuration: HashMap<String, Option<String>>,
+    configuration: HashMap<String, String>,
 }
 
 impl RawDeltaTableMetaData {
@@ -110,13 +109,23 @@ impl RawDeltaTableMetaData {
         self.created_time
     }
 
-    fn configuration(&self) -> HashMap<String, Option<String>> {
+    fn configuration(&self) -> HashMap<String, String> {
         self.configuration.clone()
     }
 }
 
 type StringVec = Vec<String>;
 
+impl RawDeltaTable {
+    fn with_table<T>(&self, func: impl Fn(&deltalake::DeltaTable) -> RbResult<T>) -> RbResult<T> {
+        func(&self._table.borrow())
+    }
+
+    fn log_store(&self) -> RbResult<LogStoreRef> {
+        self.with_table(|t| Ok(t.log_store().clone()))
+    }
+}
+
 impl RawDeltaTable {
     pub fn new(
         table_uri: String,
@@ -168,60 +177,62 @@ impl RawDeltaTable {
     }
 
     pub fn table_uri(&self) -> RbResult<String> {
-        Ok(self._table.borrow().table_uri())
+        self.with_table(|t| Ok(t.table_uri()))
     }
 
-    pub fn version(&self) -> RbResult<i64> {
-        Ok(self._table.borrow().version())
+    pub fn version(&self) -> RbResult<Option<i64>> {
+        self.with_table(|t| Ok(t.version()))
     }
 
     pub fn has_files(&self) -> RbResult<bool> {
-        Ok(self._table.borrow().config.require_files)
+        self.with_table(|t| Ok(t.config.require_files))
     }
 
     pub fn metadata(&self) -> RbResult<RawDeltaTableMetaData> {
-        let binding = self._table.borrow();
-        let metadata = binding.metadata().map_err(RubyError::from)?;
+        let metadata = self.with_table(|t| {
+            t.metadata()
+                .cloned()
+                .map_err(RubyError::from)
+                .map_err(RbErr::from)
+        })?;
         Ok(RawDeltaTableMetaData {
-            id: metadata.id.clone(),
-            name: metadata.name.clone(),
-            description: metadata.description.clone(),
-            partition_columns: metadata.partition_columns.clone(),
-            created_time: metadata.created_time,
-            configuration: metadata.configuration.clone(),
+            id: metadata.id().to_string(),
+            name: metadata.name().map(String::from),
+            description: metadata.description().map(String::from),
+            partition_columns: metadata.partition_columns().clone(),
+            created_time: metadata.created_time(),
+            configuration: metadata.configuration().clone(),
         })
     }
 
     pub fn protocol_versions(&self) -> RbResult<(i32, i32, Option<StringVec>, Option<StringVec>)> {
-        let binding = self._table.borrow();
-        let table_protocol = binding.protocol().map_err(RubyError::from)?;
+        let table_protocol = self.with_table(|t| {
+            t.protocol()
+                .cloned()
+                .map_err(RubyError::from)
+                .map_err(RbErr::from)
+        })?;
         Ok((
-            table_protocol.min_reader_version,
-            table_protocol.min_writer_version,
-            table_protocol
-                .writer_features
-                .as_ref()
-                .and_then(|features| {
-                    let empty_set = !features.is_empty();
-                    empty_set.then(|| {
-                        features
-                            .iter()
-                            .map(|v| v.to_string())
-                            .collect::<Vec<String>>()
-                    })
-                }),
-            table_protocol
-                .reader_features
-                .as_ref()
-                .and_then(|features| {
-                    let empty_set = !features.is_empty();
-                    empty_set.then(|| {
-                        features
-                            .iter()
-                            .map(|v| v.to_string())
-                            .collect::<Vec<String>>()
-                    })
-                }),
+            table_protocol.min_reader_version(),
+            table_protocol.min_writer_version(),
+            table_protocol.writer_features().and_then(|features| {
+                let empty_set = !features.is_empty();
+                empty_set.then(|| {
+                    features
+                        .iter()
+                        .map(|v| v.to_string())
+                        .collect::<Vec<String>>()
+                })
+            }),
+            table_protocol.reader_features().and_then(|features| {
+                let empty_set = !features.is_empty();
+                empty_set.then(|| {
+                    features
+                        .iter()
+                        .map(|v| v.to_string())
+                        .collect::<Vec<String>>()
+                })
+            }),
         ))
     }
 
@@ -237,31 +248,23 @@ impl RawDeltaTable {
             .map_err(RubyError::from)?)
     }
 
-    pub fn get_earliest_version(&self) -> RbResult<i64> {
-        Ok(rt()
-            .block_on(self._table.borrow().get_earliest_version())
-            .map_err(RubyError::from)?)
-    }
-
     pub fn get_num_index_cols(&self) -> RbResult<i32> {
-        Ok(self
-            ._table
-            .borrow()
-            .snapshot()
-            .map_err(RubyError::from)?
-            .config()
-            .num_indexed_cols())
+        self.with_table(|t| {
+            Ok(t.snapshot()
+                .map_err(RubyError::from)?
+                .config()
+                .num_indexed_cols())
+        })
     }
 
     pub fn get_stats_columns(&self) -> RbResult<Option<Vec<String>>> {
-        Ok(self
-            ._table
-            .borrow()
-            .snapshot()
-            .map_err(RubyError::from)?
-            .config()
-            .stats_columns()
-            .map(|v| v.iter().map(|v| v.to_string()).collect::<Vec<String>>()))
+        self.with_table(|t| {
+            Ok(t.snapshot()
+                .map_err(RubyError::from)?
+                .config()
+                .stats_columns()
+                .map(|v| v.iter().map(|s| s.to_string()).collect::<Vec<String>>()))
+        })
     }
 
     pub fn load_with_datetime(&self, ds: String) -> RbResult<()> {
@@ -285,10 +288,11 @@ impl RawDeltaTable {
         if let Some(filters) = partition_filters {
             let filters = convert_partition_filters(filters).map_err(RubyError::from)?;
             Ok(self
-                ._table
-                .borrow()
-                .get_files_by_partitions(&filters)
-                .map_err(RubyError::from)?
+                .with_table(|t| {
+                    t.get_files_by_partitions(&filters)
+                        .map_err(RubyError::from)
+                        .map_err(RbErr::from)
+                })?
                 .into_iter()
                 .map(|p| p.to_string())
                 .collect())
@@ -307,30 +311,34 @@ impl RawDeltaTable {
         &self,
         partition_filters: Option<Vec<(String, String, PartitionFilterValue)>>,
     ) -> RbResult<Vec<String>> {
-        if !self._table.borrow().config.require_files {
+        if !self.with_table(|t| Ok(t.config.require_files))? {
             return Err(DeltaError::new_err("Table is initiated without files."));
         }
 
         if let Some(filters) = partition_filters {
             let filters = convert_partition_filters(filters).map_err(RubyError::from)?;
-            Ok(self
-                ._table
-                .borrow()
-                .get_file_uris_by_partitions(&filters)
-                .map_err(RubyError::from)?)
+            self.with_table(|t| {
+                t.get_file_uris_by_partitions(&filters)
+                    .map_err(RubyError::from)
+                    .map_err(RbErr::from)
+            })
         } else {
-            Ok(self
-                ._table
-                .borrow()
-                .get_file_uris()
-                .map_err(RubyError::from)?
-                .collect())
+            self.with_table(|t| {
+                Ok(t.get_file_uris()
+                    .map_err(RubyError::from)
+                    .map_err(RbErr::from)?
+                    .collect::<Vec<String>>())
+            })
         }
     }
 
     pub fn schema(&self) -> RbResult<Value> {
-        let binding = self._table.borrow();
-        let schema: &StructType = binding.get_schema().map_err(RubyError::from)?;
+        let schema: StructType = self.with_table(|t| {
+            t.get_schema()
+                .map_err(RubyError::from)
+                .map_err(RbErr::from)
+                .map(|s| s.to_owned())
+        })?;
         schema_to_rbobject(schema.to_owned())
     }
 
@@ -727,7 +735,7 @@ impl RawDeltaTable {
         let partition_columns: HashSet<&str> = binding
             .metadata()
             .map_err(RubyError::from)?
-            .partition_columns
+            .partition_columns()
             .iter()
             .map(|col| col.as_str())
             .collect();
@@ -874,14 +882,13 @@ impl RawDeltaTable {
         Ok(serde_json::to_string(&metrics).unwrap())
     }
 
-    pub fn transaction_versions(&self) -> RHash {
-        RHash::from_iter(
-            self._table
-                .borrow()
-                .get_app_transaction_version()
-                .into_iter()
-                .map(|(app_id, transaction)| (app_id, RbTransaction::from(transaction))),
-        )
+    pub fn transaction_version(&self, app_id: String) -> RbResult<Option<i64>> {
+        // NOTE: this will simplify once we have moved logstore onto state.
+        let log_store = self.log_store()?;
+        let snapshot = self.with_table(|t| Ok(t.snapshot().map_err(RubyError::from)?.clone()))?;
+        Ok(rt()
+            .block_on(snapshot.transaction_version(log_store.as_ref(), app_id))
+            .map_err(RubyError::from)?)
     }
 }
 
@@ -1293,10 +1300,6 @@ fn init(ruby: &Ruby) -> RbResult<()> {
         "get_latest_version",
         method!(RawDeltaTable::get_latest_version, 0),
     )?;
-    class.define_method(
-        "get_earliest_version",
-        method!(RawDeltaTable::get_earliest_version, 0),
-    )?;
     class.define_method(
         "get_num_index_cols",
         method!(RawDeltaTable::get_num_index_cols, 0),
@@ -1366,8 +1369,8 @@ fn init(ruby: &Ruby) -> RbResult<()> {
     )?;
     class.define_method("repair", method!(RawDeltaTable::repair, 3))?;
     class.define_method(
-        "transaction_versions",
-        method!(RawDeltaTable::transaction_versions, 0),
+        "transaction_version",
+        method!(RawDeltaTable::transaction_version, 1),
     )?;
 
     let class = module.define_class("RawDeltaTableMetaData", ruby.class_object())?;

data/lib/deltalake/table.rb

@@ -204,7 +204,13 @@ module DeltaLake
           "DELTA_DYNAMO_TABLE_NAME"
         ]
         storage_options = @storage_options&.reject { |k, _| delta_keys.include?(k.to_s.upcase) }
-        lf = Polars.scan_parquet(sources, storage_options: storage_options, rechunk: rechunk)
+        lf =
+          Polars.scan_parquet(
+            sources,
+            hive_partitioning: true,
+            storage_options: storage_options,
+            rechunk: rechunk
+          )
 
         if columns
           # by_name requires polars-df > 0.15.0
@@ -253,8 +259,8 @@ module DeltaLake
       deserialized_metrics.transform_keys(&:to_sym)
     end
 
-    def transaction_versions
-      @table.transaction_versions
+    def transaction_version(app_id)
+      @table.transaction_version(app_id)
     end
 
     # private

data/lib/deltalake/version.rb

@@ -1,3 +1,3 @@
 module DeltaLake
-  VERSION = "0.1.7"
+  VERSION = "0.2.0"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: deltalake-rb
 version: !ruby/object:Gem::Version
-  version: 0.1.7
+  version: 0.2.0
 platform: ruby
 authors:
 - Andrew Kane
@@ -64,7 +64,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '3.1'
+      version: '3.2'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="