polars-df 0.15.0 → 0.26.0

data/ext/polars/src/conversion/mod.rs

@@ -1,46 +1,48 @@
 pub(crate) mod any_value;
+mod categorical;
 mod chunked_array;
+mod datetime;
 
+use std::collections::BTreeMap;
 use std::fmt::{Debug, Display, Formatter};
 use std::fs::File;
 use std::hash::{Hash, Hasher};
-use std::num::NonZeroUsize;
-use std::path::PathBuf;
 
+pub use categorical::RbCategories;
 use magnus::{
-    class, exception, prelude::*, r_hash::ForEach, try_convert::TryConvertOwned, value::Opaque,
-    IntoValue, Module, RArray, RHash, Ruby, Symbol, TryConvert, Value,
+    IntoValue, Module, RArray, RHash, Ruby, Symbol, TryConvert, Value, prelude::*, r_hash::ForEach,
+    try_convert::TryConvertOwned, value::Opaque,
 };
 use polars::chunked_array::object::PolarsObjectSafe;
 use polars::chunked_array::ops::{FillNullLimit, FillNullStrategy};
 use polars::datatypes::AnyValue;
+use polars::frame::PivotColumnNaming;
 use polars::frame::row::Row;
-use polars::frame::NullStrategy;
 use polars::io::avro::AvroCompression;
-use polars::io::cloud::CloudOptions;
+use polars::prelude::default_values::{
+    DefaultFieldValues, IcebergIdentityTransformedPartitionFields,
+};
+use polars::prelude::deletion::DeletionFilesList;
 use polars::prelude::*;
 use polars::series::ops::NullBehavior;
+use polars_buffer::Buffer;
+use polars_compute::decimal::dec128_verify_prec_scale;
+use polars_core::schema::iceberg::IcebergSchema;
 use polars_core::utils::arrow::array::Array;
 use polars_core::utils::materialize_dyn_int;
-use polars_plan::plans::ScanSources;
+use polars_plan::dsl::ScanSources;
+use polars_utils::compression::{BrotliLevel, GzipLevel, ZstdLevel};
 use polars_utils::total_ord::{TotalEq, TotalHash};
 
-use crate::file::{get_ruby_scan_source_input, RubyScanSourceInput};
+use crate::file::{RubyScanSourceInput, get_ruby_scan_source_input};
 use crate::object::OBJECT_NAME;
-use crate::rb_modules::series;
-use crate::{RbDataFrame, RbLazyFrame, RbPolarsErr, RbResult, RbSeries, RbTypeError, RbValueError};
-
-pub(crate) fn slice_extract_wrapped<T>(slice: &[Wrap<T>]) -> &[T] {
-    // Safety:
-    // Wrap is transparent.
-    unsafe { std::mem::transmute(slice) }
-}
-
-pub(crate) fn vec_extract_wrapped<T>(buf: Vec<Wrap<T>>) -> Vec<T> {
-    // Safety:
-    // Wrap is transparent.
-    unsafe { std::mem::transmute(buf) }
-}
+use crate::rb_modules::pl_series;
+use crate::ruby::gvl::GvlExt;
+use crate::ruby::utils::TryIntoValue;
+use crate::utils::to_rb_err;
+use crate::{
+    RbDataFrame, RbExpr, RbLazyFrame, RbPolarsErr, RbResult, RbSeries, RbTypeError, RbValueError,
+};
 
 #[repr(transparent)]
 pub struct Wrap<T>(pub T);
@@ -68,24 +70,22 @@ pub(crate) fn get_rbseq(obj: Value) -> RbResult<(RArray, usize)> {
 
 pub(crate) fn get_df(obj: Value) -> RbResult<DataFrame> {
     let rbdf = obj.funcall::<_, _, &RbDataFrame>("_df", ())?;
-    Ok(rbdf.df.borrow().clone())
+    Ok(rbdf.df.read().clone())
 }
 
 pub(crate) fn get_lf(obj: Value) -> RbResult<LazyFrame> {
     let rbdf = obj.funcall::<_, _, &RbLazyFrame>("_ldf", ())?;
-    Ok(rbdf.ldf.borrow().clone())
+    Ok(rbdf.ldf.read().clone())
 }
 
 pub(crate) fn get_series(obj: Value) -> RbResult<Series> {
     let rbs = obj.funcall::<_, _, &RbSeries>("_s", ())?;
-    Ok(rbs.series.borrow().clone())
+    Ok(rbs.series.read().clone())
 }
 
-pub(crate) fn to_series(s: RbSeries) -> Value {
-    let series = series();
-    series
-        .funcall::<_, _, Value>("_from_rbseries", (s,))
-        .unwrap()
+pub(crate) fn to_series(rb: &Ruby, s: RbSeries) -> RbResult<Value> {
+    let series = pl_series(rb);
+    series.funcall::<_, _, Value>("_from_rbseries", (s,))
 }
 
 impl TryConvert for Wrap<PlSmallStr> {
@@ -117,151 +117,175 @@ impl TryConvert for Wrap<NullValues> {
     }
 }
 
-fn struct_dict<'a>(vals: impl Iterator<Item = AnyValue<'a>>, flds: &[Field]) -> Value {
-    let dict = RHash::new();
+fn struct_dict<'a>(
+    ruby: &Ruby,
+    vals: impl Iterator<Item = AnyValue<'a>>,
+    flds: &[Field],
+) -> RbResult<Value> {
+    let dict = ruby.hash_new();
     for (fld, val) in flds.iter().zip(vals) {
-        dict.aset(fld.name().as_str(), Wrap(val)).unwrap()
+        dict.aset(fld.name().as_str(), Wrap(val).try_into_value_with(ruby)?)?;
     }
-    dict.into_value()
+    Ok(dict.as_value())
 }
 
-impl IntoValue for Wrap<DataType> {
-    fn into_value_with(self, _: &Ruby) -> Value {
-        let pl = crate::rb_modules::polars();
+impl TryIntoValue for Wrap<Series> {
+    fn try_into_value_with(self, ruby: &Ruby) -> RbResult<Value> {
+        to_series(ruby, RbSeries::new(self.0))
+    }
+}
+
+impl TryIntoValue for Wrap<DataType> {
+    fn try_into_value_with(self, ruby: &Ruby) -> RbResult<Value> {
+        let pl = crate::rb_modules::polars(ruby);
 
         match self.0 {
             DataType::Int8 => {
-                let class = pl.const_get::<_, Value>("Int8").unwrap();
-                class.funcall("new", ()).unwrap()
+                let class = pl.const_get::<_, Value>("Int8")?;
+                class.funcall("new", ())
             }
             DataType::Int16 => {
-                let class = pl.const_get::<_, Value>("Int16").unwrap();
-                class.funcall("new", ()).unwrap()
+                let class = pl.const_get::<_, Value>("Int16")?;
+                class.funcall("new", ())
             }
             DataType::Int32 => {
-                let class = pl.const_get::<_, Value>("Int32").unwrap();
-                class.funcall("new", ()).unwrap()
+                let class = pl.const_get::<_, Value>("Int32")?;
+                class.funcall("new", ())
             }
             DataType::Int64 => {
-                let class = pl.const_get::<_, Value>("Int64").unwrap();
-                class.funcall("new", ()).unwrap()
+                let class = pl.const_get::<_, Value>("Int64")?;
+                class.funcall("new", ())
+            }
+            DataType::Int128 => {
+                let class = pl.const_get::<_, Value>("Int128")?;
+                class.funcall("new", ())
             }
             DataType::UInt8 => {
-                let class = pl.const_get::<_, Value>("UInt8").unwrap();
-                class.funcall("new", ()).unwrap()
+                let class = pl.const_get::<_, Value>("UInt8")?;
+                class.funcall("new", ())
             }
             DataType::UInt16 => {
-                let class = pl.const_get::<_, Value>("UInt16").unwrap();
-                class.funcall("new", ()).unwrap()
+                let class = pl.const_get::<_, Value>("UInt16")?;
+                class.funcall("new", ())
             }
             DataType::UInt32 => {
-                let class = pl.const_get::<_, Value>("UInt32").unwrap();
-                class.funcall("new", ()).unwrap()
+                let class = pl.const_get::<_, Value>("UInt32")?;
+                class.funcall("new", ())
             }
             DataType::UInt64 => {
-                let class = pl.const_get::<_, Value>("UInt64").unwrap();
-                class.funcall("new", ()).unwrap()
+                let class = pl.const_get::<_, Value>("UInt64")?;
+                class.funcall("new", ())
+            }
+            DataType::UInt128 => {
+                let class = pl.const_get::<_, Value>("UInt128")?;
+                class.funcall("new", ())
+            }
+            DataType::Float16 => {
+                let class = pl.const_get::<_, Value>("Float16")?;
+                class.funcall("new", ())
             }
             DataType::Float32 => {
-                let class = pl.const_get::<_, Value>("Float32").unwrap();
-                class.funcall("new", ()).unwrap()
+                let class = pl.const_get::<_, Value>("Float32")?;
+                class.funcall("new", ())
             }
             DataType::Float64 | DataType::Unknown(UnknownKind::Float) => {
-                let class = pl.const_get::<_, Value>("Float64").unwrap();
-                class.funcall("new", ()).unwrap()
+                let class = pl.const_get::<_, Value>("Float64")?;
+                class.funcall("new", ())
             }
             DataType::Decimal(precision, scale) => {
-                let class = pl.const_get::<_, Value>("Decimal").unwrap();
-                class
-                    .funcall::<_, _, Value>("new", (precision, scale))
-                    .unwrap()
+                let class = pl.const_get::<_, Value>("Decimal")?;
+                class.funcall::<_, _, Value>("new", (precision, scale))
             }
             DataType::Boolean => {
-                let class = pl.const_get::<_, Value>("Boolean").unwrap();
-                class.funcall("new", ()).unwrap()
+                let class = pl.const_get::<_, Value>("Boolean")?;
+                class.funcall("new", ())
             }
             DataType::String | DataType::Unknown(UnknownKind::Str) => {
-                let class = pl.const_get::<_, Value>("String").unwrap();
-                class.funcall("new", ()).unwrap()
+                let class = pl.const_get::<_, Value>("String")?;
+                class.funcall("new", ())
             }
             DataType::Binary => {
-                let class = pl.const_get::<_, Value>("Binary").unwrap();
-                class.funcall("new", ()).unwrap()
+                let class = pl.const_get::<_, Value>("Binary")?;
+                class.funcall("new", ())
             }
             DataType::Array(inner, size) => {
-                let class = pl.const_get::<_, Value>("Array").unwrap();
-                let inner = Wrap(*inner);
+                let class = pl.const_get::<_, Value>("Array")?;
+                let inner = Wrap(*inner).try_into_value_with(ruby)?;
                 let args = (inner, size);
-                class.funcall::<_, _, Value>("new", args).unwrap()
+                class.funcall::<_, _, Value>("new", args)
             }
             DataType::List(inner) => {
-                let class = pl.const_get::<_, Value>("List").unwrap();
-                let inner = Wrap(*inner);
-                class.funcall::<_, _, Value>("new", (inner,)).unwrap()
+                let class = pl.const_get::<_, Value>("List")?;
+                let inner = Wrap(*inner).try_into_value_with(ruby)?;
+                class.funcall::<_, _, Value>("new", (inner,))
             }
             DataType::Date => {
-                let class = pl.const_get::<_, Value>("Date").unwrap();
-                class.funcall("new", ()).unwrap()
+                let class = pl.const_get::<_, Value>("Date")?;
+                class.funcall("new", ())
             }
             DataType::Datetime(tu, tz) => {
-                let datetime_class = pl.const_get::<_, Value>("Datetime").unwrap();
-                datetime_class
-                    .funcall::<_, _, Value>("new", (tu.to_ascii(), tz.as_deref()))
-                    .unwrap()
+                let datetime_class = pl.const_get::<_, Value>("Datetime")?;
+                datetime_class.funcall::<_, _, Value>(
+                    "new",
+                    (tu.to_ascii(), tz.as_deref().map(|x| x.as_str())),
+                )
             }
             DataType::Duration(tu) => {
-                let duration_class = pl.const_get::<_, Value>("Duration").unwrap();
-                duration_class
-                    .funcall::<_, _, Value>("new", (tu.to_ascii(),))
-                    .unwrap()
-            }
-            DataType::Object(_, _) => {
-                let class = pl.const_get::<_, Value>("Object").unwrap();
-                class.funcall("new", ()).unwrap()
-            }
-            DataType::Categorical(_, ordering) => {
-                let class = pl.const_get::<_, Value>("Categorical").unwrap();
-                class.funcall("new", (Wrap(ordering),)).unwrap()
-            }
-            DataType::Enum(rev_map, _) => {
-                // we should always have an initialized rev_map coming from rust
-                let categories = rev_map.as_ref().unwrap().get_categories();
-                let class = pl.const_get::<_, Value>("Enum").unwrap();
-                let s =
-                    Series::from_arrow(PlSmallStr::from_static("category"), categories.to_boxed())
-                        .unwrap();
-                let series = to_series(s.into());
-                class.funcall::<_, _, Value>("new", (series,)).unwrap()
+                let duration_class = pl.const_get::<_, Value>("Duration")?;
+                duration_class.funcall::<_, _, Value>("new", (tu.to_ascii(),))
+            }
+            DataType::Object(_) => {
+                let class = pl.const_get::<_, Value>("Object")?;
+                class.funcall("new", ())
+            }
+            DataType::Categorical(cats, _) => {
+                let categories_class = pl.const_get::<_, Value>("Categories")?;
+                let categorical_class = pl.const_get::<_, Value>("Categorical")?;
+                let categories: Value = categories_class
+                    .funcall("_from_rb_categories", (RbCategories::from(cats.clone()),))?;
+                let kwargs = ruby.hash_new();
+                kwargs.aset(ruby.to_symbol("categories"), categories)?;
+                categorical_class.funcall("new", (kwargs,))
+            }
+            DataType::Enum(_, mapping) => {
+                let categories = unsafe {
+                    StringChunked::from_chunks(
+                        PlSmallStr::from_static("category"),
+                        vec![mapping.to_arrow(true)],
+                    )
+                };
+                let class = pl.const_get::<_, Value>("Enum")?;
+                let series = to_series(ruby, categories.into_series().into())?;
+                class.funcall::<_, _, Value>("new", (series,))
             }
             DataType::Time => {
-                let class = pl.const_get::<_, Value>("Time").unwrap();
-                class.funcall("new", ()).unwrap()
+                let class = pl.const_get::<_, Value>("Time")?;
+                class.funcall("new", ())
             }
             DataType::Struct(fields) => {
-                let field_class = pl.const_get::<_, Value>("Field").unwrap();
+                let field_class = pl.const_get::<_, Value>("Field")?;
                 let iter = fields.iter().map(|fld| {
                     let name = fld.name().as_str();
-                    let dtype = Wrap(fld.dtype().clone());
-                    field_class
-                        .funcall::<_, _, Value>("new", (name, dtype))
-                        .unwrap()
+                    let dtype = Wrap(fld.dtype().clone()).try_into_value_with(ruby);
+                    dtype.and_then(|dt| field_class.funcall::<_, _, Value>("new", (name, dt)))
                 });
-                let fields = RArray::from_iter(iter);
-                let struct_class = pl.const_get::<_, Value>("Struct").unwrap();
-                struct_class
-                    .funcall::<_, _, Value>("new", (fields,))
-                    .unwrap()
+                let fields = ruby.ary_try_from_iter(iter)?;
+                let struct_class = pl.const_get::<_, Value>("Struct")?;
+                struct_class.funcall::<_, _, Value>("new", (fields,))
             }
             DataType::Null => {
-                let class = pl.const_get::<_, Value>("Null").unwrap();
-                class.funcall("new", ()).unwrap()
+                let class = pl.const_get::<_, Value>("Null")?;
+                class.funcall("new", ())
+            }
+            DataType::Extension(_typ, _storage) => {
+                todo!();
             }
             DataType::Unknown(UnknownKind::Int(v)) => {
-                Wrap(materialize_dyn_int(v).dtype()).into_value()
+                Wrap(materialize_dyn_int(v).dtype()).try_into_value_with(ruby)
             }
             DataType::Unknown(_) => {
-                let class = pl.const_get::<_, Value>("Unknown").unwrap();
-                class.funcall("new", ()).unwrap()
+                let class = pl.const_get::<_, Value>("Unknown")?;
+                class.funcall("new", ())
             }
             DataType::BinaryOffset => {
                 unimplemented!()
@@ -270,24 +294,24 @@ impl IntoValue for Wrap<DataType> {
     }
 }
 
+enum CategoricalOrdering {
+    Lexical,
+}
+
 impl IntoValue for Wrap<CategoricalOrdering> {
-    fn into_value_with(self, _: &Ruby) -> Value {
-        let ordering = match self.0 {
-            CategoricalOrdering::Physical => "physical",
-            CategoricalOrdering::Lexical => "lexical",
-        };
-        ordering.into_value()
+    fn into_value_with(self, ruby: &Ruby) -> Value {
+        "lexical".into_value_with(ruby)
     }
 }
 
 impl IntoValue for Wrap<TimeUnit> {
-    fn into_value_with(self, _: &Ruby) -> Value {
+    fn into_value_with(self, ruby: &Ruby) -> Value {
         let tu = match self.0 {
             TimeUnit::Nanoseconds => "ns",
             TimeUnit::Microseconds => "us",
             TimeUnit::Milliseconds => "ms",
         };
-        tu.into_value()
+        tu.into_value_with(ruby)
     }
 }
 
@@ -301,93 +325,120 @@ impl TryConvert for Wrap<Field> {
 
 impl TryConvert for Wrap<DataType> {
     fn try_convert(ob: Value) -> RbResult<Self> {
-        let dtype = if ob.is_kind_of(class::class()) {
+        let ruby = Ruby::get_with(ob);
+        let dtype = if ob.is_kind_of(ruby.class_class()) {
             let name = ob.funcall::<_, _, String>("name", ())?;
             match name.as_str() {
-                "Polars::UInt8" => DataType::UInt8,
-                "Polars::UInt16" => DataType::UInt16,
-                "Polars::UInt32" => DataType::UInt32,
-                "Polars::UInt64" => DataType::UInt64,
                 "Polars::Int8" => DataType::Int8,
                 "Polars::Int16" => DataType::Int16,
                 "Polars::Int32" => DataType::Int32,
                 "Polars::Int64" => DataType::Int64,
+                "Polars::Int128" => DataType::Int64,
+                "Polars::UInt8" => DataType::UInt8,
+                "Polars::UInt16" => DataType::UInt16,
+                "Polars::UInt32" => DataType::UInt32,
+                "Polars::UInt64" => DataType::UInt64,
+                "Polars::UInt128" => DataType::UInt128,
+                "Polars::Float16" => DataType::Float16,
+                "Polars::Float32" => DataType::Float32,
+                "Polars::Float64" => DataType::Float64,
+                "Polars::Boolean" => DataType::Boolean,
                 "Polars::String" => DataType::String,
                 "Polars::Binary" => DataType::Binary,
-                "Polars::Boolean" => DataType::Boolean,
-                "Polars::Categorical" => DataType::Categorical(None, Default::default()),
-                "Polars::Enum" => DataType::Enum(None, Default::default()),
+                "Polars::Categorical" => DataType::from_categories(Categories::global()),
+                "Polars::Enum" => {
+                    DataType::from_frozen_categories(FrozenCategories::new([]).unwrap())
+                }
                 "Polars::Date" => DataType::Date,
-                "Polars::Datetime" => DataType::Datetime(TimeUnit::Microseconds, None),
                 "Polars::Time" => DataType::Time,
+                "Polars::Datetime" => DataType::Datetime(TimeUnit::Microseconds, None),
                 "Polars::Duration" => DataType::Duration(TimeUnit::Microseconds),
-                "Polars::Decimal" => DataType::Decimal(None, None),
-                "Polars::Float32" => DataType::Float32,
-                "Polars::Float64" => DataType::Float64,
-                "Polars::Object" => DataType::Object(OBJECT_NAME, None),
+                "Polars::Decimal" => {
+                    return Err(RbTypeError::new_err(
+                        "Decimal without precision/scale set is not a valid Polars datatype",
+                    ));
+                }
                 "Polars::List" => DataType::List(Box::new(DataType::Null)),
+                "Polars::Array" => DataType::Array(Box::new(DataType::Null), 0),
+                "Polars::Struct" => DataType::Struct(vec![]),
                 "Polars::Null" => DataType::Null,
+                "Polars::Object" => DataType::Object(OBJECT_NAME),
                 "Polars::Unknown" => DataType::Unknown(Default::default()),
                 dt => {
-                    return Err(RbValueError::new_err(format!(
-                        "{dt} is not a correct polars DataType.",
-                    )))
+                    return Err(RbTypeError::new_err(format!(
+                        "'{dt}' is not a Polars data type",
+                    )));
                 }
             }
-        } else if String::try_convert(ob).is_err() {
-            let name = unsafe { ob.class().name() }.into_owned();
+        } else {
+            let cls = ob.class();
+            let name = unsafe { cls.name() }.into_owned();
             match name.as_str() {
                 "Polars::Int8" => DataType::Int8,
                 "Polars::Int16" => DataType::Int16,
                 "Polars::Int32" => DataType::Int32,
                 "Polars::Int64" => DataType::Int64,
+                "Polars::Int128" => DataType::Int128,
                 "Polars::UInt8" => DataType::UInt8,
                 "Polars::UInt16" => DataType::UInt16,
                 "Polars::UInt32" => DataType::UInt32,
                 "Polars::UInt64" => DataType::UInt64,
+                "Polars::UInt128" => DataType::UInt128,
+                "Polars::Float16" => DataType::Float16,
+                "Polars::Float32" => DataType::Float32,
+                "Polars::Float64" => DataType::Float64,
+                "Polars::Boolean" => DataType::Boolean,
                 "Polars::String" => DataType::String,
                 "Polars::Binary" => DataType::Binary,
-                "Polars::Boolean" => DataType::Boolean,
                 "Polars::Categorical" => {
-                    let ordering = ob
-                        .funcall::<_, _, Wrap<CategoricalOrdering>>("ordering", ())?
-                        .0;
-                    DataType::Categorical(None, ordering)
+                    let categories: Value = ob.funcall("categories", ())?;
+                    let rb_categories: &RbCategories = categories.funcall("_categories", ())?;
+                    DataType::from_categories(rb_categories.categories().clone())
                 }
                 "Polars::Enum" => {
-                    let categories = ob.funcall("categories", ()).unwrap();
+                    let categories: Value = ob.funcall("categories", ())?;
                     let s = get_series(categories)?;
                     let ca = s.str().map_err(RbPolarsErr::from)?;
                     let categories = ca.downcast_iter().next().unwrap().clone();
-                    create_enum_dtype(categories)
+                    assert!(!categories.has_nulls());
+                    DataType::from_frozen_categories(
+                        FrozenCategories::new(categories.values_iter()).unwrap(),
+                    )
                 }
                 "Polars::Date" => DataType::Date,
                 "Polars::Time" => DataType::Time,
-                "Polars::Float32" => DataType::Float32,
-                "Polars::Float64" => DataType::Float64,
-                "Polars::Null" => DataType::Null,
-                "Polars::Unknown" => DataType::Unknown(Default::default()),
-                "Polars::Duration" => {
-                    let time_unit: Value = ob.funcall("time_unit", ()).unwrap();
-                    let time_unit = Wrap::<TimeUnit>::try_convert(time_unit)?.0;
-                    DataType::Duration(time_unit)
-                }
                 "Polars::Datetime" => {
-                    let time_unit: Value = ob.funcall("time_unit", ()).unwrap();
+                    let time_unit: Value = ob.funcall("time_unit", ())?;
                     let time_unit = Wrap::<TimeUnit>::try_convert(time_unit)?.0;
                     let time_zone: Option<String> = ob.funcall("time_zone", ())?;
-                    DataType::Datetime(time_unit, time_zone.as_deref().map(|x| x.into()))
+                    DataType::Datetime(
+                        time_unit,
+                        TimeZone::opt_try_new(time_zone.as_deref()).map_err(RbPolarsErr::from)?,
+                    )
+                }
+                "Polars::Duration" => {
+                    let time_unit: Value = ob.funcall("time_unit", ())?;
+                    let time_unit = Wrap::<TimeUnit>::try_convert(time_unit)?.0;
+                    DataType::Duration(time_unit)
                 }
                 "Polars::Decimal" => {
                     let precision = ob.funcall("precision", ())?;
                     let scale = ob.funcall("scale", ())?;
-                    DataType::Decimal(precision, Some(scale))
+                    dec128_verify_prec_scale(precision, scale).map_err(to_rb_err)?;
+                    DataType::Decimal(precision, scale)
                 }
                 "Polars::List" => {
-                    let inner: Value = ob.funcall("inner", ()).unwrap();
+                    let inner: Value = ob.funcall("inner", ())?;
                     let inner = Wrap::<DataType>::try_convert(inner)?;
                     DataType::List(Box::new(inner.0))
                 }
+                "Polars::Array" => {
+                    let inner: Value = ob.funcall("inner", ())?;
+                    let size: Value = ob.funcall("size", ())?;
+                    let inner = Wrap::<DataType>::try_convert(inner)?;
+                    let size = usize::try_convert(size)?;
+                    DataType::Array(Box::new(inner.0), size)
+                }
                 "Polars::Struct" => {
                     let arr: RArray = ob.funcall("fields", ())?;
                     let mut fields = Vec::with_capacity(arr.len());
@@ -396,42 +447,13 @@ impl TryConvert for Wrap<DataType> {
                     }
                     DataType::Struct(fields)
                 }
+                "Polars::Null" => DataType::Null,
+                "Polars::Object" => DataType::Object(OBJECT_NAME),
+                "Polars::Unknown" => DataType::Unknown(Default::default()),
                 dt => {
                     return Err(RbTypeError::new_err(format!(
-                        "A {dt} object is not a correct polars DataType. \
-                        Hint: use the class without instantiating it.",
-                    )))
-                }
-            }
-        } else {
-            match String::try_convert(ob)?.as_str() {
-                "u8" => DataType::UInt8,
-                "u16" => DataType::UInt16,
-                "u32" => DataType::UInt32,
-                "u64" => DataType::UInt64,
-                "i8" => DataType::Int8,
-                "i16" => DataType::Int16,
-                "i32" => DataType::Int32,
-                "i64" => DataType::Int64,
-                "str" => DataType::String,
-                "bin" => DataType::Binary,
-                "bool" => DataType::Boolean,
-                "cat" => DataType::Categorical(None, Default::default()),
-                "date" => DataType::Date,
-                "datetime" => DataType::Datetime(TimeUnit::Microseconds, None),
-                "f32" => DataType::Float32,
-                "time" => DataType::Time,
-                "dur" => DataType::Duration(TimeUnit::Microseconds),
-                "f64" => DataType::Float64,
-                "obj" => DataType::Object(OBJECT_NAME, None),
-                "list" => DataType::List(Box::new(DataType::Boolean)),
-                "null" => DataType::Null,
-                "unk" => DataType::Unknown(Default::default()),
-                _ => {
-                    return Err(RbValueError::new_err(format!(
-                        "{} is not a supported DataType.",
-                        ob
-                    )))
+                        "'{dt}' is not a Polars data type",
+                    )));
                 }
             }
         };
@@ -455,7 +477,7 @@ impl TryConvert for Wrap<StatisticsOptions> {
                 _ => {
                     return Err(RbTypeError::new_err(format!(
                         "'{key}' is not a valid statistic option",
-                    )))
+                    )));
                 }
             }
             Ok(ForEach::Continue)
@@ -478,7 +500,7 @@ impl<'s> TryConvert for Wrap<Row<'s>> {
 
 impl TryConvert for Wrap<Schema> {
     fn try_convert(ob: Value) -> RbResult<Self> {
-        let dict = RHash::try_convert(ob)?;
+        let dict: RHash = ob.funcall("to_h", ())?;
 
         let mut schema = Vec::new();
         dict.foreach(|key: String, val: Wrap<DataType>| {
@@ -490,6 +512,60 @@ impl TryConvert for Wrap<Schema> {
     }
 }
 
+impl TryConvert for Wrap<ArrowSchema> {
+    fn try_convert(ob: Value) -> RbResult<Self> {
+        let ruby = Ruby::get_with(ob);
+        // TODO improve
+        let ob = RHash::try_convert(ob)?;
+        let fields: RArray = ob.aref(ruby.to_symbol("fields"))?;
+        let mut arrow_schema = ArrowSchema::with_capacity(fields.len());
+        for f in fields {
+            let f = RHash::try_convert(f)?;
+            let name: String = f.aref(ruby.to_symbol("name"))?;
+            let rb_dtype: String = f.aref(ruby.to_symbol("type"))?;
+            let dtype = match rb_dtype.as_str() {
+                "null" => ArrowDataType::Null,
+                "boolean" => ArrowDataType::Boolean,
+                "int8" => ArrowDataType::Int8,
+                "int16" => ArrowDataType::Int16,
+                "int32" => ArrowDataType::Int32,
+                "int64" => ArrowDataType::Int64,
+                "uint8" => ArrowDataType::UInt8,
+                "uint16" => ArrowDataType::UInt16,
+                "uint32" => ArrowDataType::UInt32,
+                "uint64" => ArrowDataType::UInt64,
+                "float16" => ArrowDataType::Float16,
+                "float32" => ArrowDataType::Float32,
+                "float64" => ArrowDataType::Float64,
+                "date32" => ArrowDataType::Date32,
+                "date64" => ArrowDataType::Date64,
+                "binary" => ArrowDataType::Binary,
+                "large_binary" => ArrowDataType::LargeBinary,
+                "string" => ArrowDataType::Utf8,
+                "large_string" => ArrowDataType::LargeUtf8,
+                "binary_view" => ArrowDataType::BinaryView,
+                "string_view" => ArrowDataType::Utf8View,
+                "unknown" => ArrowDataType::Unknown,
+                _ => todo!(),
+            };
+            let is_nullable = f.aref(ruby.to_symbol("nullable"))?;
+            let rb_metadata: RHash = f.aref(ruby.to_symbol("metadata"))?;
+            let mut metadata = BTreeMap::new();
+            rb_metadata.foreach(|k: String, v: String| {
+                metadata.insert(k.into(), v.into());
+                Ok(ForEach::Continue)
+            })?;
+            arrow_schema
+                .try_insert(
+                    name.clone().into(),
+                    ArrowField::new(name.into(), dtype, is_nullable).with_metadata(metadata),
+                )
+                .map_err(to_rb_err)?;
+        }
+        Ok(Wrap(arrow_schema))
+    }
+}
+
 impl TryConvert for Wrap<ScanSources> {
     fn try_convert(ob: Value) -> RbResult<Self> {
         let list = RArray::try_convert(ob)?;
@@ -499,9 +575,9 @@ impl TryConvert for Wrap<ScanSources> {
         }
 
         enum MutableSources {
-            Paths(Vec<PathBuf>),
+            Paths(Vec<PlRefPath>),
             Files(Vec<File>),
-            Buffers(Vec<bytes::Bytes>),
+            Buffers(Vec<Buffer<u8>>),
         }
 
         let num_items = list.len();
@@ -540,7 +616,7 @@ impl TryConvert for Wrap<ScanSources> {
                     return Err(RbTypeError::new_err(
                         "Cannot combine in-memory bytes, paths and files for scan sources"
                             .to_string(),
-                    ))
+                    ));
                 }
             }
         }
@@ -553,6 +629,16 @@ impl TryConvert for Wrap<ScanSources> {
     }
 }
 
+impl TryIntoValue for Wrap<Schema> {
+    fn try_into_value_with(self, ruby: &Ruby) -> RbResult<Value> {
+        let dict = ruby.hash_new();
+        for (k, v) in self.0.iter() {
+            dict.aset(k.as_str(), Wrap(v.clone()).try_into_value_with(ruby)?)?;
+        }
+        Ok(dict.as_value())
+    }
+}
+
 #[derive(Clone)]
 pub struct ObjectValue {
     pub inner: Opaque<Value>,
@@ -560,18 +646,17 @@ pub struct ObjectValue {
 
 impl Debug for ObjectValue {
     fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
-        f.debug_struct("ObjectValue")
-            .field("inner", &self.to_value())
-            .finish()
+        write!(f, "{}", self)
     }
 }
 
 impl Hash for ObjectValue {
     fn hash<H: Hasher>(&self, state: &mut H) {
-        let h = self
-            .to_value()
-            .funcall::<_, _, isize>("hash", ())
-            .expect("should be hashable");
+        let h = Ruby::attach(|rb| {
+            rb.get_inner(self.inner)
+                .funcall::<_, _, isize>("hash", ())
+                .expect("should be hashable")
+        });
         state.write_isize(h)
     }
 }
@@ -580,7 +665,11 @@ impl Eq for ObjectValue {}
 
 impl PartialEq for ObjectValue {
     fn eq(&self, other: &Self) -> bool {
-        self.to_value().eql(other.to_value()).unwrap_or(false)
+        Ruby::attach(|ruby| {
+            ruby.get_inner(self.inner)
+                .eql(ruby.get_inner(other.inner))
+                .unwrap_or(false)
+        })
     }
 }
 
@@ -601,7 +690,10 @@ impl TotalHash for ObjectValue {
 
 impl Display for ObjectValue {
     fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
-        write!(f, "{}", self.to_value())
+        Ruby::attach(|rb| {
+            let v = rb.get_inner(self.inner);
+            write!(f, "{}", v)
+        })
     }
 }
 
@@ -629,12 +721,6 @@ impl From<&dyn PolarsObjectSafe> for &ObjectValue {
     }
 }
 
-impl ObjectValue {
-    pub fn to_value(&self) -> Value {
-        self.clone().into_value()
-    }
-}
-
 impl IntoValue for ObjectValue {
     fn into_value_with(self, ruby: &Ruby) -> Value {
         ruby.get_inner(self.inner)
@@ -643,9 +729,9 @@ impl IntoValue for ObjectValue {
 
 impl Default for ObjectValue {
     fn default() -> Self {
-        ObjectValue {
-            inner: Ruby::get().unwrap().qnil().as_value().into(),
-        }
+        Ruby::attach(|rb| ObjectValue {
+            inner: rb.qnil().as_value().into(),
+        })
     }
 }
 
@@ -658,7 +744,7 @@ impl TryConvert for Wrap<AsofStrategy> {
             v => {
                 return Err(RbValueError::new_err(format!(
                     "asof `strategy` must be one of {{'backward', 'forward', 'nearest'}}, got {v}",
-                )))
+                )));
             }
         };
         Ok(Wrap(parsed))
@@ -673,7 +759,7 @@ impl TryConvert for Wrap<InterpolationMethod> {
             v => {
                 return Err(RbValueError::new_err(format!(
                     "method must be one of {{'linear', 'nearest'}}, got {v}",
-                )))
+                )));
             }
         };
         Ok(Wrap(parsed))
@@ -688,9 +774,8 @@ impl TryConvert for Wrap<Option<AvroCompression>> {
             "deflate" => Some(AvroCompression::Deflate),
             v => {
                 return Err(RbValueError::new_err(format!(
-                    "compression must be one of {{'uncompressed', 'snappy', 'deflate'}}, got {}",
-                    v
-                )))
+                    "compression must be one of {{'uncompressed', 'snappy', 'deflate'}}, got {v}"
+                )));
             }
         };
         Ok(Wrap(parsed))
@@ -700,13 +785,18 @@ impl TryConvert for Wrap<Option<AvroCompression>> {
 impl TryConvert for Wrap<CategoricalOrdering> {
     fn try_convert(ob: Value) -> RbResult<Self> {
         let parsed = match String::try_convert(ob)?.as_str() {
-            "physical" => CategoricalOrdering::Physical,
             "lexical" => CategoricalOrdering::Lexical,
+            "physical" => {
+                polars_warn!(
+                    Deprecation,
+                    "physical ordering is deprecated, will use lexical ordering instead"
+                );
+                CategoricalOrdering::Lexical
+            }
             v => {
                 return Err(RbValueError::new_err(format!(
-                    "ordering must be one of {{'physical', 'lexical'}}, got {}",
-                    v
-                )))
+                    "ordering must be one of {{'physical', 'lexical'}}, got {v}"
+                )));
             }
         };
         Ok(Wrap(parsed))
@@ -722,7 +812,7 @@ impl TryConvert for Wrap<StartBy> {
             v => {
                 return Err(RbValueError::new_err(format!(
                     "closed must be one of {{'window', 'datapoint', 'monday'}}, got {v}",
-                )))
+                )));
             }
         };
         Ok(Wrap(parsed))
@@ -738,9 +828,23 @@ impl TryConvert for Wrap<ClosedWindow> {
             "none" => ClosedWindow::None,
             v => {
                 return Err(RbValueError::new_err(format!(
-                    "closed must be one of {{'left', 'right', 'both', 'none'}}, got {}",
-                    v
-                )))
+                    "closed must be one of {{'left', 'right', 'both', 'none'}}, got {v}"
+                )));
+            }
+        };
+        Ok(Wrap(parsed))
+    }
+}
+
+impl TryConvert for Wrap<RoundMode> {
+    fn try_convert(ob: Value) -> RbResult<Self> {
+        let parsed = match String::try_convert(ob)?.as_str() {
+            "half_to_even" => RoundMode::HalfToEven,
+            "half_away_from_zero" => RoundMode::HalfAwayFromZero,
+            v => {
+                return Err(RbValueError::new_err(format!(
+                    "`mode` must be one of {{'half_to_even', 'half_away_from_zero'}}, got {v}",
+                )));
             }
         };
         Ok(Wrap(parsed))
@@ -754,9 +858,8 @@ impl TryConvert for Wrap<CsvEncoding> {
             "utf8-lossy" => CsvEncoding::LossyUtf8,
             v => {
                 return Err(RbValueError::new_err(format!(
-                    "encoding must be one of {{'utf8', 'utf8-lossy'}}, got {}",
-                    v
-                )))
+                    "encoding must be one of {{'utf8', 'utf8-lossy'}}, got {v}"
+                )));
             }
         };
         Ok(Wrap(parsed))
@@ -768,12 +871,11 @@ impl TryConvert for Wrap<Option<IpcCompression>> {
         let parsed = match String::try_convert(ob)?.as_str() {
             "uncompressed" => None,
             "lz4" => Some(IpcCompression::LZ4),
-            "zstd" => Some(IpcCompression::ZSTD),
+            "zstd" => Some(IpcCompression::ZSTD(Default::default())),
             v => {
                 return Err(RbValueError::new_err(format!(
-                    "compression must be one of {{'uncompressed', 'lz4', 'zstd'}}, got {}",
-                    v
-                )))
+                    "compression must be one of {{'uncompressed', 'lz4', 'zstd'}}, got {v}"
+                )));
             }
         };
         Ok(Wrap(parsed))
@@ -791,9 +893,8 @@ impl TryConvert for Wrap<JoinType> {
             "cross" => JoinType::Cross,
             v => {
                 return Err(RbValueError::new_err(format!(
-                "how must be one of {{'inner', 'left', 'full', 'semi', 'anti', 'cross'}}, got {}",
-                v
-            )))
+                    "how must be one of {{'inner', 'left', 'full', 'semi', 'anti', 'cross'}}, got {v}"
+                )));
             }
         };
         Ok(Wrap(parsed))
@@ -809,7 +910,7 @@ impl TryConvert for Wrap<Label> {
             v => {
                 return Err(RbValueError::new_err(format!(
                     "`label` must be one of {{'left', 'right', 'datapoint'}}, got {v}",
-                )))
+                )));
             }
         };
         Ok(Wrap(parsed))
@@ -823,9 +924,8 @@ impl TryConvert for Wrap<ListToStructWidthStrategy> {
             "max_width" => ListToStructWidthStrategy::MaxWidth,
             v => {
                 return Err(RbValueError::new_err(format!(
-                    "n_field_strategy must be one of {{'first_non_null', 'max_width'}}, got {}",
-                    v
-                )))
+                    "n_field_strategy must be one of {{'first_non_null', 'max_width'}}, got {v}"
+                )));
             }
         };
         Ok(Wrap(parsed))
@@ -840,7 +940,7 @@ impl TryConvert for Wrap<NonExistent> {
             v => {
                 return Err(RbValueError::new_err(format!(
                     "`non_existent` must be one of {{'null', 'raise'}}, got {v}",
-                )))
+                )));
             }
         };
         Ok(Wrap(parsed))
@@ -854,9 +954,8 @@ impl TryConvert for Wrap<NullBehavior> {
             "ignore" => NullBehavior::Ignore,
             v => {
                 return Err(RbValueError::new_err(format!(
-                    "null behavior must be one of {{'drop', 'ignore'}}, got {}",
-                    v
-                )))
+                    "null behavior must be one of {{'drop', 'ignore'}}, got {v}"
+                )));
             }
         };
         Ok(Wrap(parsed))
@@ -870,9 +969,8 @@ impl TryConvert for Wrap<NullStrategy> {
             "propagate" => NullStrategy::Propagate,
             v => {
                 return Err(RbValueError::new_err(format!(
-                    "null strategy must be one of {{'ignore', 'propagate'}}, got {}",
-                    v
-                )))
+                    "null strategy must be one of {{'ignore', 'propagate'}}, got {v}"
+                )));
             }
         };
         Ok(Wrap(parsed))
@@ -888,9 +986,8 @@ impl TryConvert for Wrap<ParallelStrategy> {
             "none" => ParallelStrategy::None,
             v => {
                 return Err(RbValueError::new_err(format!(
-                    "parallel must be one of {{'auto', 'columns', 'row_groups', 'none'}}, got {}",
-                    v
-                )))
+                    "parallel must be one of {{'auto', 'columns', 'row_groups', 'none'}}, got {v}"
+                )));
             }
         };
         Ok(Wrap(parsed))
@@ -907,9 +1004,8 @@ impl TryConvert for Wrap<QuantileMethod> {
             "midpoint" => QuantileMethod::Midpoint,
             v => {
                 return Err(RbValueError::new_err(format!(
-                    "interpolation must be one of {{'lower', 'higher', 'nearest', 'linear', 'midpoint'}}, got {}",
-                    v
-                )))
+                    "interpolation must be one of {{'lower', 'higher', 'nearest', 'linear', 'midpoint'}}, got {v}"
+                )));
             }
         };
         Ok(Wrap(parsed))
@@ -927,9 +1023,42 @@ impl TryConvert for Wrap<RankMethod> {
             "random" => RankMethod::Random,
             v => {
                 return Err(RbValueError::new_err(format!(
-                    "method must be one of {{'min', 'max', 'average', 'dense', 'ordinal', 'random'}}, got {}",
-                    v
-                )))
+                    "method must be one of {{'min', 'max', 'average', 'dense', 'ordinal', 'random'}}, got {v}"
+                )));
+            }
+        };
+        Ok(Wrap(parsed))
+    }
+}
+
+impl TryConvert for Wrap<RollingRankMethod> {
+    fn try_convert(ob: Value) -> RbResult<Self> {
+        let parsed = match String::try_convert(ob)?.as_str() {
+            "min" => RollingRankMethod::Min,
+            "max" => RollingRankMethod::Max,
+            "average" => RollingRankMethod::Average,
+            "dense" => RollingRankMethod::Dense,
+            "random" => RollingRankMethod::Random,
+            v => {
+                return Err(RbValueError::new_err(format!(
+                    "rank `method` must be one of {{'min', 'max', 'average', 'dense', 'random'}}, got {v}",
+                )));
+            }
+        };
+        Ok(Wrap(parsed))
+    }
+}
+
+impl TryConvert for Wrap<Roll> {
+    fn try_convert(ob: Value) -> RbResult<Self> {
+        let parsed = match String::try_convert(ob)?.as_str() {
+            "raise" => Roll::Raise,
+            "forward" => Roll::Forward,
+            "backward" => Roll::Backward,
+            v => {
+                return Err(RbValueError::new_err(format!(
+                    "`roll` must be one of {{'raise', 'forward', 'backward'}}, got {v}",
+                )));
             }
         };
         Ok(Wrap(parsed))
@@ -944,57 +1073,58 @@ impl TryConvert for Wrap<TimeUnit> {
             "ms" => TimeUnit::Milliseconds,
             v => {
                 return Err(RbValueError::new_err(format!(
-                    "time unit must be one of {{'ns', 'us', 'ms'}}, got {}",
-                    v
-                )))
+                    "time unit must be one of {{'ns', 'us', 'ms'}}, got {v}"
+                )));
             }
         };
         Ok(Wrap(parsed))
     }
 }
 
+unsafe impl TryConvertOwned for Wrap<TimeUnit> {}
+
 impl TryConvert for Wrap<UniqueKeepStrategy> {
     fn try_convert(ob: Value) -> RbResult<Self> {
         let parsed = match String::try_convert(ob)?.as_str() {
             "first" => UniqueKeepStrategy::First,
             "last" => UniqueKeepStrategy::Last,
+            "none" => UniqueKeepStrategy::None,
+            "any" => UniqueKeepStrategy::Any,
             v => {
                 return Err(RbValueError::new_err(format!(
-                    "keep must be one of {{'first', 'last'}}, got {}",
-                    v
-                )))
+                    "`keep` must be one of {{'first', 'last', 'any', 'none'}}, got {v}",
+                )));
             }
         };
         Ok(Wrap(parsed))
     }
 }
 
-impl TryConvert for Wrap<IpcCompression> {
+impl TryConvert for Wrap<SearchSortedSide> {
     fn try_convert(ob: Value) -> RbResult<Self> {
         let parsed = match String::try_convert(ob)?.as_str() {
-            "lz4" => IpcCompression::LZ4,
-            "zstd" => IpcCompression::ZSTD,
+            "any" => SearchSortedSide::Any,
+            "left" => SearchSortedSide::Left,
+            "right" => SearchSortedSide::Right,
             v => {
                 return Err(RbValueError::new_err(format!(
-                    "compression must be one of {{'lz4', 'zstd'}}, got {}",
-                    v
-                )))
+                    "side must be one of {{'any', 'left', 'right'}}, got {v}",
+                )));
             }
         };
         Ok(Wrap(parsed))
     }
 }
 
-impl TryConvert for Wrap<SearchSortedSide> {
+impl TryConvert for Wrap<PivotColumnNaming> {
     fn try_convert(ob: Value) -> RbResult<Self> {
         let parsed = match String::try_convert(ob)?.as_str() {
-            "any" => SearchSortedSide::Any,
-            "left" => SearchSortedSide::Left,
-            "right" => SearchSortedSide::Right,
+            "auto" => PivotColumnNaming::Auto,
+            "combine" => PivotColumnNaming::Combine,
             v => {
                 return Err(RbValueError::new_err(format!(
-                    "side must be one of {{'any', 'left', 'right'}}, got {v}",
-                )))
+                    "`column_naming` must be one of {{'auto', 'combine'}}, got {v}",
+                )));
             }
         };
         Ok(Wrap(parsed))
@@ -1011,7 +1141,7 @@ impl TryConvert for Wrap<ClosedInterval> {
             v => {
                 return Err(RbValueError::new_err(format!(
                     "`closed` must be one of {{'both', 'left', 'right', 'none'}}, got {v}",
-                )))
+                )));
             }
         };
         Ok(Wrap(parsed))
@@ -1026,8 +1156,8 @@ impl TryConvert for Wrap<WindowMapping> {
             "explode" => WindowMapping::Explode,
             v => {
                 return Err(RbValueError::new_err(format!(
-                "`mapping_strategy` must be one of {{'group_to_rows', 'join', 'explode'}}, got {v}",
-            )))
+                    "`mapping_strategy` must be one of {{'group_to_rows', 'join', 'explode'}}, got {v}",
+                )));
             }
         };
         Ok(Wrap(parsed))
@@ -1044,7 +1174,25 @@ impl TryConvert for Wrap<JoinValidation> {
             v => {
                 return Err(RbValueError::new_err(format!(
                     "`validate` must be one of {{'m:m', 'm:1', '1:m', '1:1'}}, got {v}",
-                )))
+                )));
+            }
+        };
+        Ok(Wrap(parsed))
+    }
+}
+
+impl TryConvert for Wrap<MaintainOrderJoin> {
+    fn try_convert(ob: Value) -> RbResult<Self> {
+        let parsed = match String::try_convert(ob)?.as_str() {
+            "none" => MaintainOrderJoin::None,
+            "left" => MaintainOrderJoin::Left,
+            "right" => MaintainOrderJoin::Right,
+            "left_right" => MaintainOrderJoin::LeftRight,
+            "right_left" => MaintainOrderJoin::RightLeft,
+            v => {
+                return Err(RbValueError::new_err(format!(
+                    "`maintain_order` must be one of {{'none', 'left', 'right', 'left_right', 'right_left'}}, got {v}",
+                )));
             }
         };
         Ok(Wrap(parsed))
@@ -1061,16 +1209,163 @@ impl TryConvert for Wrap<QuoteStyle> {
             v => {
                 return Err(RbValueError::new_err(format!(
                     "`quote_style` must be one of {{'always', 'necessary', 'non_numeric', 'never'}}, got {v}",
-                )))
-            },
+                )));
+            }
         };
         Ok(Wrap(parsed))
     }
 }
 
-pub(crate) fn parse_cloud_options(uri: &str, kv: Vec<(String, String)>) -> RbResult<CloudOptions> {
-    let out = CloudOptions::from_untyped_config(uri, kv).map_err(RbPolarsErr::from)?;
-    Ok(out)
+impl TryConvert for Wrap<SetOperation> {
+    fn try_convert(ob: Value) -> RbResult<Self> {
+        let parsed = match String::try_convert(ob)?.as_str() {
+            "union" => SetOperation::Union,
+            "difference" => SetOperation::Difference,
+            "intersection" => SetOperation::Intersection,
+            "symmetric_difference" => SetOperation::SymmetricDifference,
+            v => {
+                return Err(RbValueError::new_err(format!(
+                    "set operation must be one of {{'union', 'difference', 'intersection', 'symmetric_difference'}}, got {v}",
+                )));
+            }
+        };
+        Ok(Wrap(parsed))
+    }
+}
+
+impl TryConvert for Wrap<CastColumnsPolicy> {
+    fn try_convert(ob: Value) -> RbResult<Self> {
+        if ob.is_nil() {
+            let out = Wrap(CastColumnsPolicy::ERROR_ON_MISMATCH);
+            return Ok(out);
+        }
+
+        let mut integer_upcast = false;
+        let mut integer_to_float_cast = false;
+
+        let integer_cast_object: Value = ob.funcall("integer_cast", ())?;
+
+        parse_multiple_options("integer_cast", integer_cast_object, |v| {
+            match v {
+                "upcast" => integer_upcast = true,
+                "allow-float" => integer_to_float_cast = true,
+                "forbid" => {}
+                v => {
+                    return Err(RbValueError::new_err(format!(
+                        "unknown option for integer_cast: {v}"
+                    )));
+                }
+            }
+
+            Ok(())
+        })?;
+
+        let mut float_upcast = false;
+        let mut float_downcast = false;
+
+        let float_cast_object: Value = ob.funcall("float_cast", ())?;
+
+        parse_multiple_options("float_cast", float_cast_object, |v| {
+            match v {
+                "forbid" => {}
+                "upcast" => float_upcast = true,
+                "downcast" => float_downcast = true,
+                v => {
+                    return Err(RbValueError::new_err(format!(
+                        "unknown option for float_cast: {v}"
+                    )));
+                }
+            }
+
+            Ok(())
+        })?;
+
+        let mut datetime_nanoseconds_downcast = false;
+        let mut datetime_convert_timezone = false;
+
+        let datetime_cast_object: Value = ob.funcall("datetime_cast", ())?;
+
+        parse_multiple_options("datetime_cast", datetime_cast_object, |v| {
+            match v {
+                "forbid" => {}
+                "nanosecond-downcast" => datetime_nanoseconds_downcast = true,
+                "convert-timezone" => datetime_convert_timezone = true,
+                v => {
+                    return Err(RbValueError::new_err(format!(
+                        "unknown option for datetime_cast: {v}"
+                    )));
+                }
+            };
+
+            Ok(())
+        })?;
+
+        let missing_struct_fields =
+            match &*ob.funcall::<_, _, String>("missing_struct_fields", ())? {
+                "insert" => MissingColumnsPolicy::Insert,
+                "raise" => MissingColumnsPolicy::Raise,
+                v => {
+                    return Err(RbValueError::new_err(format!(
+                        "unknown option for missing_struct_fields: {v}"
+                    )));
+                }
+            };
+
+        let extra_struct_fields = match &*ob.funcall::<_, _, String>("extra_struct_fields", ())? {
+            "ignore" => ExtraColumnsPolicy::Ignore,
+            "raise" => ExtraColumnsPolicy::Raise,
+            v => {
+                return Err(RbValueError::new_err(format!(
+                    "unknown option for extra_struct_fields: {v}"
+                )));
+            }
+        };
+
+        let categorical_to_string =
+            match &*ob.funcall::<_, _, String>("categorical_to_string", ())? {
+                "allow" => true,
+                "forbid" => false,
+                v => {
+                    return Err(RbValueError::new_err(format!(
+                        "unknown option for categorical_to_string: {v}"
+                    )));
+                }
+            };
+
+        return Ok(Wrap(CastColumnsPolicy {
+            integer_upcast,
+            integer_to_float_cast,
+            float_upcast,
+            float_downcast,
+            datetime_nanoseconds_downcast,
+            datetime_microseconds_downcast: false,
+            datetime_convert_timezone,
+            null_upcast: true,
+            categorical_to_string,
+            missing_struct_fields,
+            extra_struct_fields,
+        }));
+
+        fn parse_multiple_options(
+            parameter_name: &'static str,
+            rb_object: Value,
+            mut parser_func: impl FnMut(&str) -> RbResult<()>,
+        ) -> RbResult<()> {
+            if let Ok(v) = String::try_convert(rb_object) {
+                parser_func(&v)?;
+            } else if let Ok(v) = RArray::try_convert(rb_object) {
+                for v in v {
+                    parser_func(&String::try_convert(v)?)?;
+                }
+            } else {
+                return Err(RbValueError::new_err(format!(
+                    "unknown type for {parameter_name}: {rb_object}"
+                )));
+            }
+
+            Ok(())
+        }
+    }
 }
 
 pub fn parse_fill_null_strategy(
@@ -1086,10 +1381,9 @@ pub fn parse_fill_null_strategy(
         "zero" => FillNullStrategy::Zero,
         "one" => FillNullStrategy::One,
         e => {
-            return Err(magnus::Error::new(exception::runtime_error(), format!(
-                "strategy must be one of {{'forward', 'backward', 'min', 'max', 'mean', 'zero', 'one'}}, got {}",
-                e,
-            )))
+            return Err(RbValueError::new_err(format!(
+                "`strategy` must be one of {{'forward', 'backward', 'min', 'max', 'mean', 'zero', 'one'}}, got {e}",
+            )));
         }
     };
     Ok(parsed)
@@ -1106,16 +1400,15 @@ pub fn parse_parquet_compression(
             compression_level
                 .map(|lvl| {
                     GzipLevel::try_new(lvl as u8)
-                        .map_err(|e| RbValueError::new_err(format!("{:?}", e)))
+                        .map_err(|e| RbValueError::new_err(format!("{e:?}")))
                 })
                 .transpose()?,
         ),
-        "lzo" => ParquetCompression::Lzo,
         "brotli" => ParquetCompression::Brotli(
             compression_level
                 .map(|lvl| {
                     BrotliLevel::try_new(lvl as u32)
-                        .map_err(|e| RbValueError::new_err(format!("{:?}", e)))
+                        .map_err(|e| RbValueError::new_err(format!("{e:?}")))
                 })
                 .transpose()?,
         ),
@@ -1123,30 +1416,19 @@ pub fn parse_parquet_compression(
         "zstd" => ParquetCompression::Zstd(
             compression_level
                 .map(|lvl| {
-                    ZstdLevel::try_new(lvl)
-                        .map_err(|e| RbValueError::new_err(format!("{:?}", e)))
+                    ZstdLevel::try_new(lvl).map_err(|e| RbValueError::new_err(format!("{e:?}")))
                 })
                 .transpose()?,
         ),
         e => {
             return Err(RbValueError::new_err(format!(
-                "compression must be one of {{'uncompressed', 'snappy', 'gzip', 'lzo', 'brotli', 'lz4', 'zstd'}}, got {}",
-                e
-            )))
+                "compression must be one of {{'uncompressed', 'snappy', 'gzip', 'lzo', 'brotli', 'lz4', 'zstd'}}, got {e}"
+            )));
         }
     };
     Ok(parsed)
 }
 
-impl TryConvert for Wrap<NonZeroUsize> {
-    fn try_convert(ob: Value) -> RbResult<Self> {
-        let v = usize::try_convert(ob)?;
-        NonZeroUsize::new(v)
-            .map(Wrap)
-            .ok_or(RbValueError::new_err("must be non-zero"))
-    }
-}
-
 pub(crate) fn strings_to_pl_smallstr<I, S>(container: I) -> Vec<PlSmallStr>
 where
     I: IntoIterator<Item = S>,
@@ -1182,3 +1464,205 @@ impl TryConvert for RbCompatLevel {
         }))
     }
 }
+
+impl TryConvert for Wrap<UnicodeForm> {
+    fn try_convert(ob: Value) -> RbResult<Self> {
+        let parsed = match String::try_convert(ob)?.as_str() {
+            "NFC" => UnicodeForm::NFC,
+            "NFKC" => UnicodeForm::NFKC,
+            "NFD" => UnicodeForm::NFD,
+            "NFKD" => UnicodeForm::NFKD,
+            v => {
+                return Err(RbValueError::new_err(format!(
+                    "`form` must be one of {{'NFC', 'NFKC', 'NFD', 'NFKD'}}, got {v}",
+                )));
+            }
+        };
+        Ok(Wrap(parsed))
+    }
+}
+
+impl TryConvert for Wrap<Option<KeyValueMetadata>> {
+    fn try_convert(ob: Value) -> RbResult<Self> {
+        if ob.is_nil() {
+            return Ok(Wrap(None));
+        }
+        todo!();
+    }
+}
+
+unsafe impl TryConvertOwned for Wrap<Option<KeyValueMetadata>> {}
+
+impl TryConvert for Wrap<Option<TimeZone>> {
+    fn try_convert(ob: Value) -> RbResult<Self> {
+        let tz = Option::<Wrap<PlSmallStr>>::try_convert(ob)?;
+
+        let tz = tz.map(|x| x.0);
+
+        Ok(Wrap(TimeZone::opt_try_new(tz).map_err(RbPolarsErr::from)?))
+    }
+}
+
+unsafe impl TryConvertOwned for Wrap<Option<TimeZone>> {}
+
+impl TryConvert for Wrap<UpcastOrForbid> {
+    fn try_convert(ob: Value) -> RbResult<Self> {
+        let parsed = match String::try_convert(ob)?.as_str() {
+            "upcast" => UpcastOrForbid::Upcast,
+            "forbid" => UpcastOrForbid::Forbid,
+            v => {
+                return Err(RbValueError::new_err(format!(
+                    "cast parameter must be one of {{'upcast', 'forbid'}}, got {v}",
+                )));
+            }
+        };
+        Ok(Wrap(parsed))
+    }
+}
+
+impl TryConvert for Wrap<ExtraColumnsPolicy> {
+    fn try_convert(ob: Value) -> RbResult<Self> {
+        let parsed = match String::try_convert(ob)?.as_str() {
+            "ignore" => ExtraColumnsPolicy::Ignore,
+            "raise" => ExtraColumnsPolicy::Raise,
+            v => {
+                return Err(RbValueError::new_err(format!(
+                    "extra column/field parameter must be one of {{'ignore', 'raise'}}, got {v}",
+                )));
+            }
+        };
+        Ok(Wrap(parsed))
+    }
+}
+
+impl TryConvert for Wrap<MissingColumnsPolicy> {
+    fn try_convert(ob: Value) -> RbResult<Self> {
+        let parsed = match String::try_convert(ob)?.as_str() {
+            "insert" => MissingColumnsPolicy::Insert,
+            "raise" => MissingColumnsPolicy::Raise,
+            v => {
+                return Err(RbValueError::new_err(format!(
+                    "missing column/field parameter must be one of {{'insert', 'raise'}}, got {v}",
+                )));
+            }
+        };
+        Ok(Wrap(parsed))
+    }
+}
+
+impl TryConvert for Wrap<MissingColumnsPolicyOrExpr> {
+    fn try_convert(ob: Value) -> RbResult<Self> {
+        if let Ok(rbexpr) = <&RbExpr>::try_convert(ob) {
+            return Ok(Wrap(MissingColumnsPolicyOrExpr::InsertWith(
+                rbexpr.inner.clone(),
+            )));
+        }
+
+        let parsed = match String::try_convert(ob)?.as_str() {
+            "insert" => MissingColumnsPolicyOrExpr::Insert,
+            "raise" => MissingColumnsPolicyOrExpr::Raise,
+            v => {
+                return Err(RbValueError::new_err(format!(
+                    "missing column/field parameter must be one of {{'insert', 'raise', expression}}, got {v}",
+                )));
+            }
+        };
+        Ok(Wrap(parsed))
+    }
+}
+
+impl TryConvert for Wrap<ColumnMapping> {
+    fn try_convert(ob: Value) -> RbResult<Self> {
+        let (column_mapping_type, ob) = <(String, Value)>::try_convert(ob)?;
+
+        Ok(Wrap(match column_mapping_type.as_str() {
+            "iceberg-column-mapping" => {
+                let arrow_schema = Wrap::<ArrowSchema>::try_convert(ob)?;
+                ColumnMapping::Iceberg(Arc::new(
+                    IcebergSchema::from_arrow_schema(&arrow_schema.0).map_err(to_rb_err)?,
+                ))
+            }
+
+            v => {
+                return Err(RbValueError::new_err(format!(
+                    "unknown column mapping type: {v}"
+                )));
+            }
+        }))
+    }
+}
+
+impl TryConvert for Wrap<DeletionFilesList> {
+    fn try_convert(ob: Value) -> RbResult<Self> {
+        let (deletion_file_type, ob) = <(String, Value)>::try_convert(ob)?;
+
+        Ok(Wrap(match deletion_file_type.as_str() {
+            "iceberg-position-delete" => {
+                let dict = RHash::try_convert(ob)?;
+
+                let mut out = PlIndexMap::new();
+
+                dict.foreach(|k: usize, v: RArray| {
+                    let files = v
+                        .into_iter()
+                        .map(|x| {
+                            let x = String::try_convert(x)?;
+                            Ok(x)
+                        })
+                        .collect::<RbResult<Arc<[String]>>>()?;
+
+                    if !files.is_empty() {
+                        out.insert(k, files);
+                    }
+
+                    Ok(ForEach::Continue)
+                })?;
+
+                DeletionFilesList::IcebergPositionDelete(Arc::new(out))
+            }
+
+            v => {
+                return Err(RbValueError::new_err(format!(
+                    "unknown deletion file type: {v}"
+                )));
+            }
+        }))
+    }
+}
+
+impl TryConvert for Wrap<DefaultFieldValues> {
+    fn try_convert(ob: Value) -> RbResult<Self> {
+        let (default_values_type, ob) = <(String, Value)>::try_convert(ob)?;
+
+        Ok(Wrap(match &*default_values_type {
+            "iceberg" => {
+                let dict = RHash::try_convert(ob)?;
+
+                let mut out = PlIndexMap::new();
+
+                dict.foreach(|k: u32, v: Value| {
+                    let v: Result<Column, String> = if let Ok(s) = get_series(v) {
+                        Ok(s.into_column())
+                    } else {
+                        let err_msg = String::try_convert(v)?;
+                        Err(err_msg)
+                    };
+
+                    out.insert(k, v);
+
+                    Ok(ForEach::Continue)
+                })?;
+
+                DefaultFieldValues::Iceberg(Arc::new(IcebergIdentityTransformedPartitionFields(
+                    out,
+                )))
+            }
+
+            v => {
+                return Err(RbValueError::new_err(format!(
+                    "unknown deletion file type: {v}"
+                )));
+            }
+        }))
+    }
+}