deltalake-rb 0.2.0 → 0.2.2

data/ext/deltalake/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "deltalake"
-version = "0.2.0"
+version = "0.2.2"
 license = "Apache-2.0"
 authors = ["Andrew Kane <andrew@ankane.org>"]
 edition = "2021"
@@ -14,11 +14,19 @@ crate-type = ["cdylib"]
 arrow = { version = "55.2", features = ["ffi"] }
 arrow-schema = { version = "55.2", features = ["serde"] }
 chrono = "0.4"
-delta_kernel = "0.13"
-deltalake = { version = "=0.27.0", features = ["azure", "datafusion", "gcs", "s3"] }
+delta_kernel = { version = "0.15", features = ["arrow-55", "default-engine-rustls"] }
 futures = "0.3"
-magnus = "0.7"
+magnus = "0.8"
 num_cpus = "1"
 serde = "1"
 serde_json = "1"
 tokio = { version = "1", features = ["rt-multi-thread"] }
+
+[dependencies.deltalake]
+version = "=0.28.1"
+features = [
+    "azure",
+    "datafusion",
+    "gcs",
+    "s3"
+]

data/ext/deltalake/src/error.rs

@@ -1,7 +1,7 @@
 use arrow_schema::ArrowError;
 use deltalake::datafusion::error::DataFusionError;
 use deltalake::{errors::DeltaTableError, ObjectStoreError};
-use magnus::{exception, Error as RbErr, Module, RModule, Ruby};
+use magnus::{Error as RbErr, Module, RModule, Ruby};
 use std::borrow::Cow;
 
 macro_rules! create_exception {
@@ -41,7 +41,7 @@ fn inner_to_rb_err(err: DeltaTableError) -> RbErr {
         DeltaTableError::InvalidJsonLog { .. } => DeltaProtocolError::new_err(err.to_string()),
         DeltaTableError::InvalidStatsJson { .. } => DeltaProtocolError::new_err(err.to_string()),
         DeltaTableError::InvalidData { violations } => {
-            DeltaProtocolError::new_err(format!("Invariant violations: {:?}", violations))
+            DeltaProtocolError::new_err(format!("Invariant violations: {violations:?}"))
         }
 
         // commit errors
@@ -111,7 +111,7 @@ impl From<RubyError> for RbErr {
 }
 
 macro_rules! create_builtin_exception {
-    ($type:ident, $class:expr) => {
+    ($type:ident, $method:ident) => {
         pub struct $type {}
 
         impl $type {
@@ -119,13 +119,14 @@ macro_rules! create_builtin_exception {
             where
                 T: Into<Cow<'static, str>>,
             {
-                RbErr::new($class, message)
+                let ruby = Ruby::get().unwrap();
+                RbErr::new(ruby.$method(), message)
             }
         }
     };
 }
 
-create_builtin_exception!(RbException, exception::runtime_error());
-create_builtin_exception!(RbIOError, exception::io_error());
-create_builtin_exception!(RbNotImplementedError, exception::not_imp_error());
-create_builtin_exception!(RbValueError, exception::arg_error());
+create_builtin_exception!(RbException, exception_runtime_error);
+create_builtin_exception!(RbIOError, exception_io_error);
+create_builtin_exception!(RbNotImplementedError, exception_not_imp_error);
+create_builtin_exception!(RbValueError, exception_arg_error);

data/ext/deltalake/src/lib.rs

@@ -13,6 +13,7 @@ use std::time;
 
 use chrono::{DateTime, Duration, FixedOffset, Utc};
 use delta_kernel::schema::StructField;
+use delta_kernel::table_properties::DataSkippingNumIndexedCols;
 use deltalake::arrow::ffi_stream::{ArrowArrayStreamReader, FFI_ArrowArrayStream};
 use deltalake::arrow::record_batch::RecordBatchIterator;
 use deltalake::checkpoints::{cleanup_metadata, create_checkpoint};
@@ -21,6 +22,7 @@ use deltalake::datafusion::prelude::SessionContext;
 use deltalake::delta_datafusion::DeltaCdfTableProvider;
 use deltalake::errors::DeltaTableError;
 use deltalake::kernel::transaction::{CommitProperties, TableReference};
+use deltalake::kernel::StructDataExt;
 use deltalake::kernel::{scalars::ScalarExt, StructType, Transaction};
 use deltalake::logstore::IORuntime;
 use deltalake::logstore::LogStoreRef;
@@ -40,17 +42,20 @@ use deltalake::parquet::basic::Compression;
 use deltalake::parquet::errors::ParquetError;
 use deltalake::parquet::file::properties::WriterProperties;
 use deltalake::partitions::PartitionFilter;
+use deltalake::table::config::TablePropertiesExt;
+use deltalake::table::state::DeltaTableState;
 use deltalake::{DeltaOps, DeltaResult};
 use error::DeltaError;
 use futures::future::join_all;
+use futures::TryStreamExt;
 
 use magnus::{
-    function, method, prelude::*, typed_data::Obj, Error as RbErr, Integer, Module, RArray, Ruby,
-    TryConvert, Value,
+    function, method, prelude::*, try_convert::TryConvertOwned, typed_data::Obj, Error as RbErr,
+    Integer, Module, RArray, Ruby, TryConvert, Value,
 };
 use serde_json::Map;
 
-use crate::error::{DeltaProtocolError, RbValueError, RubyError};
+use crate::error::{RbValueError, RubyError};
 use crate::features::TableFeatures;
 use crate::merge::RbMergeBuilder;
 use crate::schema::{schema_to_rbobject, Field};
@@ -73,6 +78,8 @@ impl TryConvert for PartitionFilterValue {
     }
 }
 
+unsafe impl TryConvertOwned for PartitionFilterValue {}
+
 #[magnus::wrap(class = "DeltaLake::RawDeltaTable")]
 struct RawDeltaTable {
     _table: RefCell<deltalake::DeltaTable>,
@@ -121,6 +128,15 @@ impl RawDeltaTable {
         func(&self._table.borrow())
     }
 
+    fn cloned_state(&self) -> RbResult<DeltaTableState> {
+        self.with_table(|t| {
+            t.snapshot()
+                .cloned()
+                .map_err(RubyError::from)
+                .map_err(RbErr::from)
+        })
+    }
+
     fn log_store(&self) -> RbResult<LogStoreRef> {
         self.with_table(|t| Ok(t.log_store().clone()))
     }
@@ -190,10 +206,8 @@ impl RawDeltaTable {
 
     pub fn metadata(&self) -> RbResult<RawDeltaTableMetaData> {
         let metadata = self.with_table(|t| {
-            t.metadata()
-                .cloned()
-                .map_err(RubyError::from)
-                .map_err(RbErr::from)
+            let snapshot = t.snapshot().map_err(RubyError::from).map_err(RbErr::from)?;
+            Ok(snapshot.metadata().clone())
         })?;
         Ok(RawDeltaTableMetaData {
             id: metadata.id().to_string(),
@@ -207,10 +221,8 @@ impl RawDeltaTable {
 
     pub fn protocol_versions(&self) -> RbResult<(i32, i32, Option<StringVec>, Option<StringVec>)> {
         let table_protocol = self.with_table(|t| {
-            t.protocol()
-                .cloned()
-                .map_err(RubyError::from)
-                .map_err(RbErr::from)
+            let snapshot = t.snapshot().map_err(RubyError::from).map_err(RbErr::from)?;
+            Ok(snapshot.protocol().clone())
         })?;
         Ok((
             table_protocol.min_reader_version(),
@@ -250,10 +262,15 @@ impl RawDeltaTable {
 
     pub fn get_num_index_cols(&self) -> RbResult<i32> {
         self.with_table(|t| {
-            Ok(t.snapshot()
+            let n_cols = t
+                .snapshot()
                 .map_err(RubyError::from)?
                 .config()
-                .num_indexed_cols())
+                .num_indexed_cols();
+            Ok(match n_cols {
+                DataSkippingNumIndexedCols::NumColumns(n_cols) => n_cols as i32,
+                DataSkippingNumIndexedCols::AllColumns => -1,
+            })
         })
     }
 
@@ -262,7 +279,8 @@ impl RawDeltaTable {
             Ok(t.snapshot()
                 .map_err(RubyError::from)?
                 .config()
-                .stats_columns()
+                .data_skipping_stats_columns
+                .as_ref()
                 .map(|v| v.iter().map(|s| s.to_string()).collect::<Vec<String>>()))
         })
     }
@@ -289,9 +307,12 @@ impl RawDeltaTable {
             let filters = convert_partition_filters(filters).map_err(RubyError::from)?;
             Ok(self
                 .with_table(|t| {
-                    t.get_files_by_partitions(&filters)
-                        .map_err(RubyError::from)
-                        .map_err(RbErr::from)
+                    rt().block_on(async {
+                        t.get_files_by_partitions(&filters)
+                            .await
+                            .map_err(RubyError::from)
+                            .map_err(RbErr::from)
+                    })
                 })?
                 .into_iter()
                 .map(|p| p.to_string())
@@ -300,8 +321,9 @@ impl RawDeltaTable {
             Ok(self
                 ._table
                 .borrow()
-                .get_files_iter()
+                .snapshot()
                 .map_err(RubyError::from)?
+                .file_paths_iter()
                 .map(|f| f.to_string())
                 .collect())
         }
@@ -318,9 +340,12 @@ impl RawDeltaTable {
         if let Some(filters) = partition_filters {
             let filters = convert_partition_filters(filters).map_err(RubyError::from)?;
             self.with_table(|t| {
-                t.get_file_uris_by_partitions(&filters)
-                    .map_err(RubyError::from)
-                    .map_err(RbErr::from)
+                rt().block_on(async {
+                    t.get_file_uris_by_partitions(&filters)
+                        .await
+                        .map_err(RubyError::from)
+                        .map_err(RbErr::from)
+                })
             })
         } else {
             self.with_table(|t| {
@@ -332,14 +357,12 @@ impl RawDeltaTable {
         }
     }
 
-    pub fn schema(&self) -> RbResult<Value> {
-        let schema: StructType = self.with_table(|t| {
-            t.get_schema()
-                .map_err(RubyError::from)
-                .map_err(RbErr::from)
-                .map(|s| s.to_owned())
+    pub fn schema(ruby: &Ruby, rb_self: &Self) -> RbResult<Value> {
+        let schema: StructType = rb_self.with_table(|t| {
+            let snapshot = t.snapshot().map_err(RubyError::from).map_err(RbErr::from)?;
+            Ok(snapshot.schema().clone())
         })?;
-        schema_to_rbobject(schema.to_owned())
+        schema_to_rbobject(schema.to_owned(), ruby)
     }
 
     pub fn vacuum(
@@ -378,7 +401,7 @@ impl RawDeltaTable {
     pub fn compact_optimize(
         &self,
         partition_filters: Option<Vec<(String, String, PartitionFilterValue)>>,
-        target_size: Option<i64>,
+        target_size: Option<u64>,
         max_concurrent_tasks: Option<usize>,
         min_commit_interval: Option<u64>,
         writer_properties: Option<RbWriterProperties>,
@@ -427,7 +450,7 @@ impl RawDeltaTable {
         &self,
         z_order_columns: Vec<String>,
         partition_filters: Option<Vec<(String, String, PartitionFilterValue)>>,
-        target_size: Option<i64>,
+        target_size: Option<u64>,
         max_concurrent_tasks: Option<usize>,
         max_spill_size: usize,
         min_commit_interval: Option<u64>,
@@ -724,17 +747,18 @@ impl RawDeltaTable {
             .map_err(RubyError::from)?)
     }
 
-    fn get_active_partitions(&self) -> RbResult<RArray> {
-        let binding = self._table.borrow();
-        let _column_names: HashSet<&str> = binding
-            .get_schema()
-            .map_err(|_| DeltaProtocolError::new_err("table does not yet have a schema"))?
-            .fields()
-            .map(|field| field.name().as_str())
-            .collect();
-        let partition_columns: HashSet<&str> = binding
-            .metadata()
-            .map_err(RubyError::from)?
+    fn get_active_partitions(ruby: &Ruby, rb_self: &Self) -> RbResult<RArray> {
+        let schema = rb_self.with_table(|t| {
+            let snapshot = t.snapshot().map_err(RubyError::from).map_err(RbErr::from)?;
+            Ok(snapshot.schema().clone())
+        })?;
+        let metadata = rb_self.with_table(|t| {
+            let snapshot = t.snapshot().map_err(RubyError::from).map_err(RbErr::from)?;
+            Ok(snapshot.metadata().clone())
+        })?;
+        let _column_names: HashSet<&str> =
+            schema.fields().map(|field| field.name().as_str()).collect();
+        let partition_columns: HashSet<&str> = metadata
             .partition_columns()
             .iter()
             .map(|col| col.as_str())
@@ -744,12 +768,15 @@ impl RawDeltaTable {
 
         let partition_columns: Vec<&str> = partition_columns.into_iter().collect();
 
-        let adds = binding
-            .snapshot()
-            .map_err(RubyError::from)?
-            .get_active_add_actions_by_partitions(&converted_filters)
-            .map_err(RubyError::from)?
-            .collect::<Result<Vec<_>, _>>()
+        let state = rb_self.cloned_state()?;
+        let log_store = rb_self.log_store()?;
+        let adds: Vec<_> = rt()
+            .block_on(async {
+                state
+                    .get_active_add_actions_by_partitions(&log_store, &converted_filters)
+                    .try_collect()
+                    .await
+            })
             .map_err(RubyError::from)?;
         let active_partitions: HashSet<Vec<(&str, Option<String>)>> = adds
             .iter()
@@ -757,21 +784,22 @@ impl RawDeltaTable {
                 Ok::<_, RubyError>(
                     partition_columns
                         .iter()
-                        .flat_map(|col| {
-                            Ok::<_, RubyError>((
+                        .map(|col| {
+                            (
                                 *col,
                                 add.partition_values()
-                                    .map_err(RubyError::from)?
-                                    .get(*col)
+                                    .and_then(|v| {
+                                        v.index_of(col).and_then(|idx| v.value(idx).cloned())
+                                    })
                                     .map(|v| v.serialize()),
-                            ))
+                            )
                         })
                         .collect(),
                 )
             })
             .collect();
 
-        Ok(RArray::from_iter(active_partitions))
+        Ok(ruby.ary_from_iter(active_partitions))
     }
 
     pub fn create_checkpoint(&self) -> RbResult<()> {
@@ -789,15 +817,20 @@ impl RawDeltaTable {
     }
 
     pub fn get_add_file_sizes(&self) -> RbResult<HashMap<String, i64>> {
-        Ok(self
-            ._table
-            .borrow()
-            .snapshot()
-            .map_err(RubyError::from)?
-            .eager_snapshot()
-            .files()
-            .map(|f| (f.path().to_string(), f.size()))
-            .collect::<HashMap<String, i64>>())
+        self.with_table(|t| {
+            let log_store = t.log_store();
+            let sizes: HashMap<String, i64> = rt()
+                .block_on(async {
+                    t.snapshot()?
+                        .snapshot()
+                        .files(&log_store, None)
+                        .map_ok(|f| (f.path().to_string(), f.size()))
+                        .try_collect()
+                        .await
+                })
+                .map_err(RubyError::from)?;
+            Ok(sizes)
+        })
     }
 
     pub fn delete(
@@ -1397,6 +1430,7 @@ fn init(ruby: &Ruby) -> RbResult<()> {
     class.define_method("to_i", method!(ArrowArrayStream::to_i, 0))?;
 
     let class = module.define_class("Field", ruby.class_object())?;
+    class.define_singleton_method("new", function!(Field::new, 2))?;
     class.define_method("name", method!(Field::name, 0))?;
     class.define_method("type", method!(Field::get_type, 0))?;
     class.define_method("nullable", method!(Field::nullable, 0))?;

data/ext/deltalake/src/schema.rs

@@ -1,20 +1,53 @@
-use deltalake::kernel::{StructField, StructType as DeltaStructType};
-use magnus::{value::ReprValue, Module, RArray, RModule, Ruby, Value};
+use deltalake::kernel::{
+    DataType, PrimitiveType as DeltaPrimitive, StructField, StructType as DeltaStructType,
+};
+use magnus::{value::ReprValue, Module, RModule, Ruby, TryConvert, Value};
 
-use crate::RbResult;
+use crate::{RbResult, RbValueError};
 
-pub fn schema_to_rbobject(schema: DeltaStructType) -> RbResult<Value> {
+pub fn schema_to_rbobject(schema: DeltaStructType, ruby: &Ruby) -> RbResult<Value> {
     let fields = schema.fields().map(|field| Field {
         inner: field.clone(),
     });
 
-    let rb_schema: Value = Ruby::get()
-        .unwrap()
+    let rb_schema: Value = ruby
         .class_object()
         .const_get::<_, RModule>("DeltaLake")?
         .const_get("Schema")?;
 
-    rb_schema.funcall("new", (RArray::from_iter(fields),))
+    rb_schema.funcall("new", (ruby.ary_from_iter(fields),))
+}
+
+fn ruby_type_to_schema(ob: Value) -> RbResult<DataType> {
+    if let Ok(raw_primitive) = String::try_convert(ob) {
+        // Pass through PrimitiveType::new() to do validation
+        return PrimitiveType::new(raw_primitive)
+            .map(|data_type| DataType::Primitive(data_type.inner_type));
+    }
+    Err(RbValueError::new_err("Invalid data type"))
+}
+
+pub struct PrimitiveType {
+    inner_type: DeltaPrimitive,
+}
+
+impl PrimitiveType {
+    fn new(data_type: String) -> RbResult<Self> {
+        let data_type: DeltaPrimitive =
+            serde_json::from_str(&format!("\"{data_type}\"")).map_err(|_| {
+                if data_type.starts_with("decimal") {
+                    RbValueError::new_err(format!(
+                        "invalid type string: {data_type}, precision/scale can't be larger than 38"
+                    ))
+                } else {
+                    RbValueError::new_err(format!("invalid type string: {data_type}"))
+                }
+            })?;
+
+        Ok(Self {
+            inner_type: data_type,
+        })
+    }
 }
 
 #[magnus::wrap(class = "DeltaLake::Field")]
@@ -23,6 +56,13 @@ pub struct Field {
 }
 
 impl Field {
+    pub fn new(name: String, r#type: Value) -> RbResult<Self> {
+        let ty = ruby_type_to_schema(r#type)?;
+        Ok(Self {
+            inner: StructField::new(name, ty, true),
+        })
+    }
+
     pub fn name(&self) -> String {
         self.inner.name().to_string()
     }

data/ext/deltalake/src/utils.rs

@@ -10,11 +10,10 @@ pub fn rt() -> &'static Runtime {
     let runtime_pid = *PID.get_or_init(|| pid);
     if pid != runtime_pid {
         panic!(
-            "Forked process detected - current PID is {} but the tokio runtime was created by {}. The tokio \
+            "Forked process detected - current PID is {pid} but the tokio runtime was created by {runtime_pid}. The tokio \
             runtime does not support forked processes https://github.com/tokio-rs/tokio/issues/4301. If you are \
             seeing this message while using Ruby multithreading make sure to use the `spawn` or `forkserver` \
-            mode.",
-            pid, runtime_pid
+            mode."
         );
     }
     TOKIO_RT.get_or_init(|| Runtime::new().expect("Failed to create a tokio runtime."))

data/lib/deltalake/table.rb

@@ -195,6 +195,7 @@ module DeltaLake
 
       sources = file_uris
       if sources.empty?
+        # TODO pass schema
         lf = Polars::LazyFrame.new
       else
         delta_keys = [
@@ -209,7 +210,8 @@ module DeltaLake
             sources,
             hive_partitioning: true,
             storage_options: storage_options,
-            rechunk: rechunk
+            rechunk: rechunk,
+            allow_missing_columns: true
           )
 
         if columns

data/lib/deltalake/version.rb

@@ -1,3 +1,3 @@
 module DeltaLake
-  VERSION = "0.2.0"
+  VERSION = "0.2.2"
 end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: deltalake-rb
 version: !ruby/object:Gem::Version
-  version: 0.2.0
+  version: 0.2.2
 platform: ruby
 authors:
 - Andrew Kane
@@ -71,7 +71,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.6.7
+rubygems_version: 3.6.9
 specification_version: 4
 summary: Delta Lake for Ruby
 test_files: []