polars-df 0.21.1 → 0.23.0

data/ext/polars/src/conversion/chunked_array.rs

@@ -1,10 +1,11 @@
-use magnus::{IntoValue, RArray, RString, Ruby, TryConvert, Value, prelude::*};
+use magnus::{IntoValue, RString, Ruby, TryConvert, Value, prelude::*};
 use polars::prelude::*;
+use polars_compute::decimal::DecimalFmtBuffer;
 
 use super::{Wrap, get_rbseq, struct_dict};
 
 use crate::RbResult;
-use crate::rb_modules::utils;
+use crate::rb_modules::pl_utils;
 
 impl TryConvert for Wrap<StringChunked> {
     fn try_convert(obj: Value) -> RbResult<Self> {
@@ -39,19 +40,19 @@ impl TryConvert for Wrap<BinaryChunked> {
 }
 
 impl IntoValue for Wrap<&StringChunked> {
-    fn into_value_with(self, _: &Ruby) -> Value {
+    fn into_value_with(self, ruby: &Ruby) -> Value {
         let iter = self.0.into_iter();
-        RArray::from_iter(iter).into_value()
+        ruby.ary_from_iter(iter).as_value()
     }
 }
 
 impl IntoValue for Wrap<&BinaryChunked> {
-    fn into_value_with(self, _: &Ruby) -> Value {
+    fn into_value_with(self, ruby: &Ruby) -> Value {
         let iter = self
             .0
             .into_iter()
-            .map(|opt_bytes| opt_bytes.map(RString::from_slice));
-        RArray::from_iter(iter).into_value()
+            .map(|opt_bytes| opt_bytes.map(|v| ruby.str_from_slice(v)));
+        ruby.ary_from_iter(iter).as_value()
     }
 }
 
@@ -62,19 +63,19 @@ impl IntoValue for Wrap<&StructChunked> {
         // make series::iter() accept a chunk index.
         let s = s.rechunk();
         let iter = s.iter().map(|av| match av {
-            AnyValue::Struct(_, _, flds) => struct_dict(av._iter_struct_av(), flds),
+            AnyValue::Struct(_, _, flds) => struct_dict(ruby, av._iter_struct_av(), flds),
             AnyValue::Null => ruby.qnil().as_value(),
             _ => unreachable!(),
         });
 
-        RArray::from_iter(iter).into_value()
+        ruby.ary_from_iter(iter).as_value()
     }
 }
 
 impl IntoValue for Wrap<&DurationChunked> {
-    fn into_value_with(self, _: &Ruby) -> Value {
-        let utils = utils();
-        let time_unit = Wrap(self.0.time_unit()).into_value();
+    fn into_value_with(self, ruby: &Ruby) -> Value {
+        let utils = pl_utils();
+        let time_unit = Wrap(self.0.time_unit()).into_value_with(ruby);
         let iter = self.0.physical().into_iter().map(|opt_v| {
             opt_v.map(|v| {
                 utils
@@ -82,15 +83,19 @@ impl IntoValue for Wrap<&DurationChunked> {
                     .unwrap()
             })
         });
-        RArray::from_iter(iter).into_value()
+        ruby.ary_from_iter(iter).as_value()
     }
 }
 
 impl IntoValue for Wrap<&DatetimeChunked> {
-    fn into_value_with(self, _: &Ruby) -> Value {
-        let utils = utils();
-        let time_unit = Wrap(self.0.time_unit()).into_value();
-        let time_zone = self.0.time_zone().as_deref().map(|v| v.into_value());
+    fn into_value_with(self, ruby: &Ruby) -> Value {
+        let utils = pl_utils();
+        let time_unit = Wrap(self.0.time_unit()).into_value_with(ruby);
+        let time_zone = self
+            .0
+            .time_zone()
+            .as_deref()
+            .map(|v| v.into_value_with(ruby));
         let iter = self.0.physical().into_iter().map(|opt_v| {
             opt_v.map(|v| {
                 utils
@@ -98,41 +103,43 @@ impl IntoValue for Wrap<&DatetimeChunked> {
                     .unwrap()
             })
         });
-        RArray::from_iter(iter).into_value()
+        ruby.ary_from_iter(iter).as_value()
     }
 }
 
 impl IntoValue for Wrap<&TimeChunked> {
-    fn into_value_with(self, _: &Ruby) -> Value {
-        let utils = utils();
+    fn into_value_with(self, ruby: &Ruby) -> Value {
+        let utils = pl_utils();
         let iter = self.0.physical().into_iter().map(|opt_v| {
             opt_v.map(|v| utils.funcall::<_, _, Value>("_to_ruby_time", (v,)).unwrap())
         });
-        RArray::from_iter(iter).into_value()
+        ruby.ary_from_iter(iter).as_value()
     }
 }
 
 impl IntoValue for Wrap<&DateChunked> {
-    fn into_value_with(self, _: &Ruby) -> Value {
-        let utils = utils();
+    fn into_value_with(self, ruby: &Ruby) -> Value {
+        let utils = pl_utils();
         let iter = self.0.physical().into_iter().map(|opt_v| {
             opt_v.map(|v| utils.funcall::<_, _, Value>("_to_ruby_date", (v,)).unwrap())
         });
-        RArray::from_iter(iter).into_value()
+        ruby.ary_from_iter(iter).as_value()
     }
 }
 
 impl IntoValue for Wrap<&DecimalChunked> {
-    fn into_value_with(self, _: &Ruby) -> Value {
-        let utils = utils();
-        let rb_scale = (-(self.0.scale() as i32)).into_value();
+    fn into_value_with(self, ruby: &Ruby) -> Value {
+        let utils = pl_utils();
+        let rb_precision = self.0.precision().into_value_with(ruby);
+        let mut buf = DecimalFmtBuffer::new();
         let iter = self.0.physical().into_iter().map(|opt_v| {
             opt_v.map(|v| {
+                let s = buf.format_dec128(v, self.0.scale(), false, false);
                 utils
-                    .funcall::<_, _, Value>("_to_ruby_decimal", (v.to_string(), rb_scale))
+                    .funcall::<_, _, Value>("_to_ruby_decimal", (rb_precision, s))
                     .unwrap()
             })
         });
-        RArray::from_iter(iter).into_value()
+        ruby.ary_from_iter(iter).as_value()
     }
 }

data/ext/polars/src/conversion/datetime.rs

@@ -0,0 +1,11 @@
+use magnus::{Value, prelude::*};
+use polars::prelude::*;
+
+use crate::rb_modules::pl_utils;
+
+pub fn datetime_to_rb_object(v: i64, tu: TimeUnit, tz: Option<&TimeZone>) -> Value {
+    let tu = tu.to_ascii();
+    pl_utils()
+        .funcall("_to_ruby_datetime", (v, tu, tz.map(|v| v.to_string())))
+        .unwrap()
+}

data/ext/polars/src/conversion/mod.rs

@@ -1,16 +1,17 @@
 pub(crate) mod any_value;
 mod categorical;
 mod chunked_array;
+mod datetime;
 
+use std::collections::BTreeMap;
 use std::fmt::{Debug, Display, Formatter};
 use std::fs::File;
 use std::hash::{Hash, Hasher};
-use std::num::NonZeroUsize;
 
 pub use categorical::RbCategories;
 use magnus::{
-    IntoValue, Module, RArray, RHash, Ruby, Symbol, TryConvert, Value, class, exception,
-    prelude::*, r_hash::ForEach, try_convert::TryConvertOwned, value::Opaque,
+    IntoValue, Module, RArray, RHash, Ruby, Symbol, TryConvert, Value, prelude::*, r_hash::ForEach,
+    try_convert::TryConvertOwned, value::Opaque,
 };
 use polars::chunked_array::object::PolarsObjectSafe;
 use polars::chunked_array::ops::{FillNullLimit, FillNullStrategy};
@@ -18,9 +19,13 @@ use polars::datatypes::AnyValue;
 use polars::frame::row::Row;
 use polars::io::avro::AvroCompression;
 use polars::io::cloud::CloudOptions;
+use polars::prelude::default_values::{
+    DefaultFieldValues, IcebergIdentityTransformedPartitionFields,
+};
 use polars::prelude::deletion::DeletionFilesList;
 use polars::prelude::*;
 use polars::series::ops::NullBehavior;
+use polars_compute::decimal::dec128_verify_prec_scale;
 use polars_core::schema::iceberg::IcebergSchema;
 use polars_core::utils::arrow::array::Array;
 use polars_core::utils::materialize_dyn_int;
@@ -30,7 +35,7 @@ use polars_utils::total_ord::{TotalEq, TotalHash};
 
 use crate::file::{RubyScanSourceInput, get_ruby_scan_source_input};
 use crate::object::OBJECT_NAME;
-use crate::rb_modules::series;
+use crate::rb_modules::pl_series;
 use crate::utils::to_rb_err;
 use crate::{RbDataFrame, RbLazyFrame, RbPolarsErr, RbResult, RbSeries, RbTypeError, RbValueError};
 
@@ -86,7 +91,7 @@ pub(crate) fn get_series(obj: Value) -> RbResult<Series> {
 }
 
 pub(crate) fn to_series(s: RbSeries) -> Value {
-    let series = series();
+    let series = pl_series();
     series
         .funcall::<_, _, Value>("_from_rbseries", (s,))
         .unwrap()
@@ -121,16 +126,16 @@ impl TryConvert for Wrap<NullValues> {
     }
 }
 
-fn struct_dict<'a>(vals: impl Iterator<Item = AnyValue<'a>>, flds: &[Field]) -> Value {
-    let dict = RHash::new();
+fn struct_dict<'a>(ruby: &Ruby, vals: impl Iterator<Item = AnyValue<'a>>, flds: &[Field]) -> Value {
+    let dict = ruby.hash_new();
     for (fld, val) in flds.iter().zip(vals) {
         dict.aset(fld.name().as_str(), Wrap(val)).unwrap()
     }
-    dict.into_value()
+    dict.as_value()
 }
 
 impl IntoValue for Wrap<DataType> {
-    fn into_value_with(self, _: &Ruby) -> Value {
+    fn into_value_with(self, ruby: &Ruby) -> Value {
         let pl = crate::rb_modules::polars();
 
         match self.0 {
@@ -170,6 +175,10 @@ impl IntoValue for Wrap<DataType> {
                 let class = pl.const_get::<_, Value>("UInt64").unwrap();
                 class.funcall("new", ()).unwrap()
             }
+            DataType::UInt128 => {
+                let class = pl.const_get::<_, Value>("UInt128").unwrap();
+                class.funcall("new", ()).unwrap()
+            }
             DataType::Float32 => {
                 let class = pl.const_get::<_, Value>("Float32").unwrap();
                 class.funcall("new", ()).unwrap()
@@ -236,8 +245,10 @@ impl IntoValue for Wrap<DataType> {
                 let categories: Value = categories_class
                     .funcall("_from_rb_categories", (RbCategories::from(cats.clone()),))
                     .unwrap();
-                let kwargs = RHash::new();
-                kwargs.aset(Symbol::new("categories"), categories).unwrap();
+                let kwargs = ruby.hash_new();
+                kwargs
+                    .aset(ruby.to_symbol("categories"), categories)
+                    .unwrap();
                 categorical_class.funcall("new", (kwargs,)).unwrap()
             }
             DataType::Enum(_, mapping) => {
@@ -264,7 +275,7 @@ impl IntoValue for Wrap<DataType> {
                         .funcall::<_, _, Value>("new", (name, dtype))
                         .unwrap()
                 });
-                let fields = RArray::from_iter(iter);
+                let fields = ruby.ary_from_iter(iter);
                 let struct_class = pl.const_get::<_, Value>("Struct").unwrap();
                 struct_class
                     .funcall::<_, _, Value>("new", (fields,))
@@ -275,7 +286,7 @@ impl IntoValue for Wrap<DataType> {
                 class.funcall("new", ()).unwrap()
             }
             DataType::Unknown(UnknownKind::Int(v)) => {
-                Wrap(materialize_dyn_int(v).dtype()).into_value()
+                Wrap(materialize_dyn_int(v).dtype()).into_value_with(ruby)
             }
             DataType::Unknown(_) => {
                 let class = pl.const_get::<_, Value>("Unknown").unwrap();
@@ -293,19 +304,19 @@ enum CategoricalOrdering {
 }
 
 impl IntoValue for Wrap<CategoricalOrdering> {
-    fn into_value_with(self, _: &Ruby) -> Value {
-        "lexical".into_value()
+    fn into_value_with(self, ruby: &Ruby) -> Value {
+        "lexical".into_value_with(ruby)
     }
 }
 
 impl IntoValue for Wrap<TimeUnit> {
-    fn into_value_with(self, _: &Ruby) -> Value {
+    fn into_value_with(self, ruby: &Ruby) -> Value {
         let tu = match self.0 {
             TimeUnit::Nanoseconds => "ns",
             TimeUnit::Microseconds => "us",
             TimeUnit::Milliseconds => "ms",
         };
-        tu.into_value()
+        tu.into_value_with(ruby)
     }
 }
 
@@ -319,7 +330,8 @@ impl TryConvert for Wrap<Field> {
 
 impl TryConvert for Wrap<DataType> {
     fn try_convert(ob: Value) -> RbResult<Self> {
-        let dtype = if ob.is_kind_of(class::class()) {
+        let ruby = Ruby::get_with(ob);
+        let dtype = if ob.is_kind_of(ruby.class_class()) {
             let name = ob.funcall::<_, _, String>("name", ())?;
             match name.as_str() {
                 "Polars::Int8" => DataType::Int8,
@@ -343,7 +355,11 @@ impl TryConvert for Wrap<DataType> {
                 "Polars::Time" => DataType::Time,
                 "Polars::Datetime" => DataType::Datetime(TimeUnit::Microseconds, None),
                 "Polars::Duration" => DataType::Duration(TimeUnit::Microseconds),
-                "Polars::Decimal" => DataType::Decimal(None, None),
+                "Polars::Decimal" => {
+                    return Err(RbTypeError::new_err(
+                        "Decimal without precision/scale set is not a valid Polars datatype",
+                    ));
+                }
                 "Polars::List" => DataType::List(Box::new(DataType::Null)),
                 "Polars::Array" => DataType::Array(Box::new(DataType::Null), 0),
                 "Polars::Struct" => DataType::Struct(vec![]),
@@ -408,7 +424,8 @@ impl TryConvert for Wrap<DataType> {
                 "Polars::Decimal" => {
                     let precision = ob.funcall("precision", ())?;
                     let scale = ob.funcall("scale", ())?;
-                    DataType::Decimal(precision, Some(scale))
+                    dec128_verify_prec_scale(precision, scale).map_err(to_rb_err)?;
+                    DataType::Decimal(precision, scale)
                 }
                 "Polars::List" => {
                     let inner: Value = ob.funcall("inner", ()).unwrap();
@@ -527,8 +544,56 @@ impl TryConvert for Wrap<Schema> {
 }
 
 impl TryConvert for Wrap<ArrowSchema> {
-    fn try_convert(_ob: Value) -> RbResult<Self> {
-        todo!();
+    fn try_convert(ob: Value) -> RbResult<Self> {
+        let ruby = Ruby::get_with(ob);
+        // TODO improve
+        let ob = RHash::try_convert(ob)?;
+        let fields: RArray = ob.aref(ruby.to_symbol("fields"))?;
+        let mut arrow_schema = ArrowSchema::with_capacity(fields.len());
+        for f in fields {
+            let f = RHash::try_convert(f)?;
+            let name: String = f.aref(ruby.to_symbol("name"))?;
+            let rb_dtype: String = f.aref(ruby.to_symbol("type"))?;
+            let dtype = match rb_dtype.as_str() {
+                "null" => ArrowDataType::Null,
+                "boolean" => ArrowDataType::Boolean,
+                "int8" => ArrowDataType::Int8,
+                "int16" => ArrowDataType::Int16,
+                "int32" => ArrowDataType::Int32,
+                "int64" => ArrowDataType::Int64,
+                "uint8" => ArrowDataType::UInt8,
+                "uint16" => ArrowDataType::UInt16,
+                "uint32" => ArrowDataType::UInt32,
+                "uint64" => ArrowDataType::UInt64,
+                "float16" => ArrowDataType::Float16,
+                "float32" => ArrowDataType::Float32,
+                "float64" => ArrowDataType::Float64,
+                "date32" => ArrowDataType::Date32,
+                "date64" => ArrowDataType::Date64,
+                "binary" => ArrowDataType::Binary,
+                "large_binary" => ArrowDataType::LargeBinary,
+                "string" => ArrowDataType::Utf8,
+                "large_string" => ArrowDataType::LargeUtf8,
+                "binary_view" => ArrowDataType::BinaryView,
+                "string_view" => ArrowDataType::Utf8View,
+                "unknown" => ArrowDataType::Unknown,
+                _ => todo!(),
+            };
+            let is_nullable = f.aref(ruby.to_symbol("nullable"))?;
+            let rb_metadata: RHash = f.aref(ruby.to_symbol("metadata"))?;
+            let mut metadata = BTreeMap::new();
+            rb_metadata.foreach(|k: String, v: String| {
+                metadata.insert(k.into(), v.into());
+                Ok(ForEach::Continue)
+            })?;
+            arrow_schema
+                .try_insert(
+                    name.clone().into(),
+                    ArrowField::new(name.into(), dtype, is_nullable).with_metadata(metadata),
+                )
+                .map_err(to_rb_err)?;
+        }
+        Ok(Wrap(arrow_schema))
     }
 }
 
@@ -673,7 +738,7 @@ impl From<&dyn PolarsObjectSafe> for &ObjectValue {
 
 impl ObjectValue {
     pub fn to_value(&self) -> Value {
-        self.clone().into_value()
+        self.clone().into_value_with(&Ruby::get().unwrap())
     }
 }
 
@@ -827,7 +892,7 @@ impl TryConvert for Wrap<Option<IpcCompression>> {
         let parsed = match String::try_convert(ob)?.as_str() {
             "uncompressed" => None,
             "lz4" => Some(IpcCompression::LZ4),
-            "zstd" => Some(IpcCompression::ZSTD),
+            "zstd" => Some(IpcCompression::ZSTD(Default::default())),
             v => {
                 return Err(RbValueError::new_err(format!(
                     "compression must be one of {{'uncompressed', 'lz4', 'zstd'}}, got {v}"
@@ -1036,21 +1101,6 @@ impl TryConvert for Wrap<UniqueKeepStrategy> {
     }
 }
 
-impl TryConvert for Wrap<IpcCompression> {
-    fn try_convert(ob: Value) -> RbResult<Self> {
-        let parsed = match String::try_convert(ob)?.as_str() {
-            "lz4" => IpcCompression::LZ4,
-            "zstd" => IpcCompression::ZSTD,
-            v => {
-                return Err(RbValueError::new_err(format!(
-                    "compression must be one of {{'lz4', 'zstd'}}, got {v}"
-                )));
-            }
-        };
-        Ok(Wrap(parsed))
-    }
-}
-
 impl TryConvert for Wrap<SearchSortedSide> {
     fn try_convert(ob: Value) -> RbResult<Self> {
         let parsed = match String::try_convert(ob)?.as_str() {
@@ -1153,7 +1203,8 @@ impl TryConvert for Wrap<QuoteStyle> {
 }
 
 pub(crate) fn parse_cloud_options(uri: &str, kv: Vec<(String, String)>) -> RbResult<CloudOptions> {
-    let out = CloudOptions::from_untyped_config(uri, kv).map_err(RbPolarsErr::from)?;
+    let out = CloudOptions::from_untyped_config(CloudScheme::from_uri(uri).as_ref(), kv)
+        .map_err(RbPolarsErr::from)?;
     Ok(out)
 }
 
@@ -1180,7 +1231,121 @@ impl TryConvert for Wrap<CastColumnsPolicy> {
             let out = Wrap(CastColumnsPolicy::ERROR_ON_MISMATCH);
             return Ok(out);
         }
-        todo!();
+
+        let integer_upcast = match &*ob.funcall::<_, _, String>("integer_cast", ())? {
+            "upcast" => true,
+            "forbid" => false,
+            v => {
+                return Err(RbValueError::new_err(format!(
+                    "unknown option for integer_cast: {v}"
+                )));
+            }
+        };
+
+        let mut float_upcast = false;
+        let mut float_downcast = false;
+
+        let float_cast_object: Value = ob.funcall("float_cast", ())?;
+
+        parse_multiple_options("float_cast", float_cast_object, |v| {
+            match v {
+                "forbid" => {}
+                "upcast" => float_upcast = true,
+                "downcast" => float_downcast = true,
+                v => {
+                    return Err(RbValueError::new_err(format!(
+                        "unknown option for float_cast: {v}"
+                    )));
+                }
+            }
+
+            Ok(())
+        })?;
+
+        let mut datetime_nanoseconds_downcast = false;
+        let mut datetime_convert_timezone = false;
+
+        let datetime_cast_object: Value = ob.funcall("datetime_cast", ())?;
+
+        parse_multiple_options("datetime_cast", datetime_cast_object, |v| {
+            match v {
+                "forbid" => {}
+                "nanosecond-downcast" => datetime_nanoseconds_downcast = true,
+                "convert-timezone" => datetime_convert_timezone = true,
+                v => {
+                    return Err(RbValueError::new_err(format!(
+                        "unknown option for datetime_cast: {v}"
+                    )));
+                }
+            };
+
+            Ok(())
+        })?;
+
+        let missing_struct_fields =
+            match &*ob.funcall::<_, _, String>("missing_struct_fields", ())? {
+                "insert" => MissingColumnsPolicy::Insert,
+                "raise" => MissingColumnsPolicy::Raise,
+                v => {
+                    return Err(RbValueError::new_err(format!(
+                        "unknown option for missing_struct_fields: {v}"
+                    )));
+                }
+            };
+
+        let extra_struct_fields = match &*ob.funcall::<_, _, String>("extra_struct_fields", ())? {
+            "ignore" => ExtraColumnsPolicy::Ignore,
+            "raise" => ExtraColumnsPolicy::Raise,
+            v => {
+                return Err(RbValueError::new_err(format!(
+                    "unknown option for extra_struct_fields: {v}"
+                )));
+            }
+        };
+
+        let categorical_to_string =
+            match &*ob.funcall::<_, _, String>("categorical_to_string", ())? {
+                "allow" => true,
+                "forbid" => false,
+                v => {
+                    return Err(RbValueError::new_err(format!(
+                        "unknown option for categorical_to_string: {v}"
+                    )));
+                }
+            };
+
+        return Ok(Wrap(CastColumnsPolicy {
+            integer_upcast,
+            float_upcast,
+            float_downcast,
+            datetime_nanoseconds_downcast,
+            datetime_microseconds_downcast: false,
+            datetime_convert_timezone,
+            null_upcast: true,
+            categorical_to_string,
+            missing_struct_fields,
+            extra_struct_fields,
+        }));
+
+        fn parse_multiple_options(
+            parameter_name: &'static str,
+            rb_object: Value,
+            mut parser_func: impl FnMut(&str) -> RbResult<()>,
+        ) -> RbResult<()> {
+            if let Ok(v) = String::try_convert(rb_object) {
+                parser_func(&v)?;
+            } else if let Ok(v) = RArray::try_convert(rb_object) {
+                for v in v {
+                    parser_func(&String::try_convert(v)?)?;
+                }
+            } else {
+                return Err(RbValueError::new_err(format!(
+                    "unknown type for {parameter_name}: {rb_object}"
+                )));
+            }
+
+            Ok(())
+        }
     }
 }
 
@@ -1198,7 +1363,7 @@ pub fn parse_fill_null_strategy(
         "one" => FillNullStrategy::One,
         e => {
             return Err(magnus::Error::new(
-                exception::runtime_error(),
+                Ruby::get().unwrap().exception_runtime_error(),
                 format!(
                     "strategy must be one of {{'forward', 'backward', 'min', 'max', 'mean', 'zero', 'one'}}, got {e}",
                 ),
@@ -1249,15 +1414,6 @@ pub fn parse_parquet_compression(
     Ok(parsed)
 }
 
-impl TryConvert for Wrap<NonZeroUsize> {
-    fn try_convert(ob: Value) -> RbResult<Self> {
-        let v = usize::try_convert(ob)?;
-        NonZeroUsize::new(v)
-            .map(Wrap)
-            .ok_or(RbValueError::new_err("must be non-zero"))
-    }
-}
-
 pub(crate) fn strings_to_pl_smallstr<I, S>(container: I) -> Vec<PlSmallStr>
 where
     I: IntoIterator<Item = S>,
@@ -1411,3 +1567,40 @@ impl TryConvert for Wrap<DeletionFilesList> {
         }))
     }
 }
+
+impl TryConvert for Wrap<DefaultFieldValues> {
+    fn try_convert(ob: Value) -> RbResult<Self> {
+        let (default_values_type, ob) = <(String, Value)>::try_convert(ob)?;
+
+        Ok(Wrap(match &*default_values_type {
+            "iceberg" => {
+                let dict = RHash::try_convert(ob)?;
+
+                let mut out = PlIndexMap::new();
+
+                dict.foreach(|k: u32, v: Value| {
+                    let v: Result<Column, String> = if let Ok(s) = get_series(v) {
+                        Ok(s.into_column())
+                    } else {
+                        let err_msg = String::try_convert(v)?;
+                        Err(err_msg)
+                    };
+
+                    out.insert(k, v);
+
+                    Ok(ForEach::Continue)
+                })?;
+
+                DefaultFieldValues::Iceberg(Arc::new(IcebergIdentityTransformedPartitionFields(
+                    out,
+                )))
+            }
+
+            v => {
+                return Err(RbValueError::new_err(format!(
+                    "unknown deletion file type: {v}"
+                )));
+            }
+        }))
+    }
+}

data/ext/polars/src/dataframe/construction.rs

@@ -1,4 +1,4 @@
-use magnus::{RArray, RHash, Symbol, Value, prelude::*, r_hash::ForEach};
+use magnus::{RArray, RHash, Ruby, Symbol, Value, prelude::*, r_hash::ForEach};
 use polars::frame::row::{Row, rows_to_schema_supertypes, rows_to_supertypes};
 use polars::prelude::*;
 
@@ -54,7 +54,7 @@ fn finish_from_rows(
     schema_overrides: Option<Schema>,
     infer_schema_length: Option<usize>,
 ) -> RbResult<RbDataFrame> {
-    let mut schema = if let Some(mut schema) = schema {
+    let schema = if let Some(mut schema) = schema {
         resolve_schema_overrides(&mut schema, schema_overrides);
         update_schema_from_rows(&mut schema, &rows, infer_schema_length)?;
         schema
@@ -62,11 +62,6 @@ fn finish_from_rows(
         rows_to_schema_supertypes(&rows, infer_schema_length).map_err(RbPolarsErr::from)?
     };
 
-    // TODO: Remove this step when Decimals are supported properly.
-    // Erasing the decimal precision/scale here will just require us to infer it again later.
-    // https://github.com/pola-rs/polars/issues/14427
-    erase_decimal_precision_scale(&mut schema);
-
     let df = DataFrame::from_rows_and_schema(&rows, &schema).map_err(RbPolarsErr::from)?;
     Ok(df.into())
 }
@@ -106,14 +101,6 @@ fn resolve_schema_overrides(schema: &mut Schema, schema_overrides: Option<Schema
     }
 }
 
-fn erase_decimal_precision_scale(schema: &mut Schema) {
-    for dtype in schema.iter_values_mut() {
-        if let DataType::Decimal(_, _) = dtype {
-            *dtype = DataType::Decimal(None, None)
-        }
-    }
-}
-
 fn columns_names_to_empty_schema<'a, I>(column_names: I) -> Schema
 where
     I: IntoIterator<Item = &'a str>,
@@ -125,6 +112,7 @@ where
 }
 
 fn dicts_to_rows<'a>(data: &Value, names: &'a [String], _strict: bool) -> RbResult<Vec<Row<'a>>> {
+    let ruby = Ruby::get_with(*data);
     let (data, len) = get_rbseq(*data)?;
     let mut rows = Vec::with_capacity(len);
     for d in data.into_iter() {
@@ -132,8 +120,8 @@ fn dicts_to_rows<'a>(data: &Value, names: &'a [String], _strict: bool) -> RbResu
 
         let mut row = Vec::with_capacity(names.len());
         for k in names.iter() {
-            // TODO improve performance
-            let val = match d.get(k.clone()).or_else(|| d.get(Symbol::new(k))) {
+            // TODO improve performance (must work with GC)
+            let val = match d.get(k.clone()).or_else(|| d.get(ruby.to_symbol(k))) {
                 None => AnyValue::Null,
                 Some(val) => Wrap::<AnyValue>::try_convert(val)?.0,
             };