polars-df 0.7.0 → 0.9.0

data/ext/polars/src/{conversion.rs → conversion/mod.rs}

@@ -1,24 +1,28 @@
+pub(crate) mod anyvalue;
+mod chunked_array;
+
 use std::fmt::{Debug, Display, Formatter};
 use std::hash::{Hash, Hasher};
+use std::num::NonZeroUsize;
 
-use magnus::encoding::{EncodingCapable, Index};
 use magnus::{
-    class, exception, prelude::*, r_hash::ForEach, value::Opaque, Float, Integer, IntoValue,
-    Module, RArray, RHash, RString, Ruby, Symbol, TryConvert, Value,
+    class, exception, prelude::*, r_hash::ForEach, value::Opaque, IntoValue, Module, RArray, RHash,
+    Ruby, Symbol, TryConvert, Value,
 };
 use polars::chunked_array::object::PolarsObjectSafe;
 use polars::chunked_array::ops::{FillNullLimit, FillNullStrategy};
 use polars::datatypes::AnyValue;
-use polars::frame::row::{any_values_to_dtype, Row};
+use polars::frame::row::Row;
 use polars::frame::NullStrategy;
 use polars::io::avro::AvroCompression;
 use polars::prelude::*;
 use polars::series::ops::NullBehavior;
-use polars_core::utils::arrow::util::total_ord::TotalEq;
+use polars_core::utils::arrow::array::Array;
+use polars_utils::total_ord::{TotalEq, TotalHash};
 use smartstring::alias::String as SmartString;
 
 use crate::object::OBJECT_NAME;
-use crate::rb_modules::utils;
+use crate::rb_modules::series;
 use crate::{RbDataFrame, RbLazyFrame, RbPolarsErr, RbResult, RbSeries, RbTypeError, RbValueError};
 
 pub(crate) fn slice_to_wrapped<T>(slice: &[T]) -> &[Wrap<T>] {
@@ -78,36 +82,11 @@ pub(crate) fn get_series(obj: Value) -> RbResult<Series> {
     Ok(rbs.series.borrow().clone())
 }
 
-impl TryConvert for Wrap<Utf8Chunked> {
-    fn try_convert(obj: Value) -> RbResult<Self> {
-        let (seq, len) = get_rbseq(obj)?;
-        let mut builder = Utf8ChunkedBuilder::new("", len, len * 25);
-
-        for res in seq.each() {
-            let item = res?;
-            match String::try_convert(item) {
-                Ok(val) => builder.append_value(&val),
-                Err(_) => builder.append_null(),
-            }
-        }
-        Ok(Wrap(builder.finish()))
-    }
-}
-
-impl TryConvert for Wrap<BinaryChunked> {
-    fn try_convert(obj: Value) -> RbResult<Self> {
-        let (seq, len) = get_rbseq(obj)?;
-        let mut builder = BinaryChunkedBuilder::new("", len, len * 25);
-
-        for res in seq.each() {
-            let item = res?;
-            match RString::try_convert(item) {
-                Ok(val) => builder.append_value(unsafe { val.as_slice() }),
-                Err(_) => builder.append_null(),
-            }
-        }
-        Ok(Wrap(builder.finish()))
-    }
+pub(crate) fn to_series(s: RbSeries) -> Value {
+    let series = series();
+    series
+        .funcall::<_, _, Value>("_from_rbseries", (s,))
+        .unwrap()
 }
 
 impl TryConvert for Wrap<NullValues> {
@@ -134,102 +113,84 @@ fn struct_dict<'a>(vals: impl Iterator<Item = AnyValue<'a>>, flds: &[Field]) ->
     dict.into_value()
 }
 
-impl IntoValue for Wrap<AnyValue<'_>> {
-    fn into_value_with(self, ruby: &Ruby) -> Value {
+impl IntoValue for Wrap<DataType> {
+    fn into_value_with(self, _: &Ruby) -> Value {
+        let pl = crate::rb_modules::polars();
+
         match self.0 {
-            AnyValue::UInt8(v) => ruby.into_value(v),
-            AnyValue::UInt16(v) => ruby.into_value(v),
-            AnyValue::UInt32(v) => ruby.into_value(v),
-            AnyValue::UInt64(v) => ruby.into_value(v),
-            AnyValue::Int8(v) => ruby.into_value(v),
-            AnyValue::Int16(v) => ruby.into_value(v),
-            AnyValue::Int32(v) => ruby.into_value(v),
-            AnyValue::Int64(v) => ruby.into_value(v),
-            AnyValue::Float32(v) => ruby.into_value(v),
-            AnyValue::Float64(v) => ruby.into_value(v),
-            AnyValue::Null => ruby.qnil().as_value(),
-            AnyValue::Boolean(v) => ruby.into_value(v),
-            AnyValue::Utf8(v) => ruby.into_value(v),
-            AnyValue::Utf8Owned(v) => ruby.into_value(v.as_str()),
-            AnyValue::Categorical(idx, rev, arr) => {
-                let s = if arr.is_null() {
-                    rev.get(idx)
-                } else {
-                    unsafe { arr.deref_unchecked().value(idx as usize) }
-                };
-                s.into_value()
+            DataType::Int8 => {
+                let class = pl.const_get::<_, Value>("Int8").unwrap();
+                class.funcall("new", ()).unwrap()
             }
-            AnyValue::Date(v) => utils().funcall("_to_ruby_date", (v,)).unwrap(),
-            AnyValue::Datetime(v, time_unit, time_zone) => {
-                let time_unit = time_unit.to_ascii();
-                utils()
-                    .funcall("_to_ruby_datetime", (v, time_unit, time_zone.clone()))
-                    .unwrap()
+            DataType::Int16 => {
+                let class = pl.const_get::<_, Value>("Int16").unwrap();
+                class.funcall("new", ()).unwrap()
             }
-            AnyValue::Duration(v, time_unit) => {
-                let time_unit = time_unit.to_ascii();
-                utils()
-                    .funcall("_to_ruby_duration", (v, time_unit))
-                    .unwrap()
+            DataType::Int32 => {
+                let class = pl.const_get::<_, Value>("Int32").unwrap();
+                class.funcall("new", ()).unwrap()
             }
-            AnyValue::Time(v) => utils().funcall("_to_ruby_time", (v,)).unwrap(),
-            AnyValue::Array(v, _) | AnyValue::List(v) => RbSeries::new(v).to_a().into_value(),
-            ref av @ AnyValue::Struct(_, _, flds) => struct_dict(av._iter_struct_av(), flds),
-            AnyValue::StructOwned(payload) => struct_dict(payload.0.into_iter(), &payload.1),
-            AnyValue::Object(v) => {
-                let object = v.as_any().downcast_ref::<ObjectValue>().unwrap();
-                object.to_object()
+            DataType::Int64 => {
+                let class = pl.const_get::<_, Value>("Int64").unwrap();
+                class.funcall("new", ()).unwrap()
             }
-            AnyValue::ObjectOwned(v) => {
-                let object = v.0.as_any().downcast_ref::<ObjectValue>().unwrap();
-                object.to_object()
+            DataType::UInt8 => {
+                let class = pl.const_get::<_, Value>("UInt8").unwrap();
+                class.funcall("new", ()).unwrap()
+            }
+            DataType::UInt16 => {
+                let class = pl.const_get::<_, Value>("UInt16").unwrap();
+                class.funcall("new", ()).unwrap()
+            }
+            DataType::UInt32 => {
+                let class = pl.const_get::<_, Value>("UInt32").unwrap();
+                class.funcall("new", ()).unwrap()
+            }
+            DataType::UInt64 => {
+                let class = pl.const_get::<_, Value>("UInt64").unwrap();
+                class.funcall("new", ()).unwrap()
+            }
+            DataType::Float32 => {
+                let class = pl.const_get::<_, Value>("Float32").unwrap();
+                class.funcall("new", ()).unwrap()
+            }
+            DataType::Float64 => {
+                let class = pl.const_get::<_, Value>("Float64").unwrap();
+                class.funcall("new", ()).unwrap()
             }
-            AnyValue::Binary(v) => RString::from_slice(v).into_value(),
-            AnyValue::BinaryOwned(v) => RString::from_slice(&v).into_value(),
-            AnyValue::Decimal(v, scale) => utils()
-                .funcall("_to_ruby_decimal", (v.to_string(), -(scale as i32)))
-                .unwrap(),
-        }
-    }
-}
-
-impl IntoValue for Wrap<DataType> {
-    fn into_value_with(self, _: &Ruby) -> Value {
-        let pl = crate::rb_modules::polars();
-
-        match self.0 {
-            DataType::Int8 => pl.const_get::<_, Value>("Int8").unwrap(),
-            DataType::Int16 => pl.const_get::<_, Value>("Int16").unwrap(),
-            DataType::Int32 => pl.const_get::<_, Value>("Int32").unwrap(),
-            DataType::Int64 => pl.const_get::<_, Value>("Int64").unwrap(),
-            DataType::UInt8 => pl.const_get::<_, Value>("UInt8").unwrap(),
-            DataType::UInt16 => pl.const_get::<_, Value>("UInt16").unwrap(),
-            DataType::UInt32 => pl.const_get::<_, Value>("UInt32").unwrap(),
-            DataType::UInt64 => pl.const_get::<_, Value>("UInt64").unwrap(),
-            DataType::Float32 => pl.const_get::<_, Value>("Float32").unwrap(),
-            DataType::Float64 => pl.const_get::<_, Value>("Float64").unwrap(),
             DataType::Decimal(precision, scale) => {
-                let decimal_class = pl.const_get::<_, Value>("Decimal").unwrap();
-                decimal_class
+                let class = pl.const_get::<_, Value>("Decimal").unwrap();
+                class
                     .funcall::<_, _, Value>("new", (precision, scale))
                     .unwrap()
             }
-            DataType::Boolean => pl.const_get::<_, Value>("Boolean").unwrap(),
-            DataType::Utf8 => pl.const_get::<_, Value>("Utf8").unwrap(),
-            DataType::Binary => pl.const_get::<_, Value>("Binary").unwrap(),
+            DataType::Boolean => {
+                let class = pl.const_get::<_, Value>("Boolean").unwrap();
+                class.funcall("new", ()).unwrap()
+            }
+            DataType::String => {
+                let class = pl.const_get::<_, Value>("String").unwrap();
+                class.funcall("new", ()).unwrap()
+            }
+            DataType::Binary => {
+                let class = pl.const_get::<_, Value>("Binary").unwrap();
+                class.funcall("new", ()).unwrap()
+            }
             DataType::Array(inner, size) => {
+                let class = pl.const_get::<_, Value>("Array").unwrap();
                 let inner = Wrap(*inner);
-                let list_class = pl.const_get::<_, Value>("Array").unwrap();
-                list_class
-                    .funcall::<_, _, Value>("new", (size, inner))
-                    .unwrap()
+                let args = (inner, size);
+                class.funcall::<_, _, Value>("new", args).unwrap()
             }
             DataType::List(inner) => {
+                let class = pl.const_get::<_, Value>("List").unwrap();
                 let inner = Wrap(*inner);
-                let list_class = pl.const_get::<_, Value>("List").unwrap();
-                list_class.funcall::<_, _, Value>("new", (inner,)).unwrap()
+                class.funcall::<_, _, Value>("new", (inner,)).unwrap()
+            }
+            DataType::Date => {
+                let class = pl.const_get::<_, Value>("Date").unwrap();
+                class.funcall("new", ()).unwrap()
             }
-            DataType::Date => pl.const_get::<_, Value>("Date").unwrap(),
             DataType::Datetime(tu, tz) => {
                 let datetime_class = pl.const_get::<_, Value>("Datetime").unwrap();
                 datetime_class
@@ -242,9 +203,29 @@ impl IntoValue for Wrap<DataType> {
                     .funcall::<_, _, Value>("new", (tu.to_ascii(),))
                     .unwrap()
             }
-            DataType::Object(_) => pl.const_get::<_, Value>("Object").unwrap(),
-            DataType::Categorical(_) => pl.const_get::<_, Value>("Categorical").unwrap(),
-            DataType::Time => pl.const_get::<_, Value>("Time").unwrap(),
+            DataType::Object(_, _) => {
+                let class = pl.const_get::<_, Value>("Object").unwrap();
+                class.funcall("new", ()).unwrap()
+            }
+            DataType::Categorical(_, ordering) => {
+                let class = pl.const_get::<_, Value>("Categorical").unwrap();
+                class.funcall("new", (Wrap(ordering),)).unwrap()
+            }
+            DataType::Enum(rev_map, _) => {
+                // we should always have an initialized rev_map coming from rust
+                let categories = rev_map.as_ref().unwrap().get_categories();
+                let class = pl.const_get::<_, Value>("Enum").unwrap();
+                let s = Series::from_arrow("category", categories.to_boxed()).unwrap();
+                let series = to_series(s.into());
+                class
+                    .funcall::<_, _, Value>("new", (series,))
+                    .unwrap()
+                    .into()
+            }
+            DataType::Time => {
+                let class = pl.const_get::<_, Value>("Time").unwrap();
+                class.funcall("new", ()).unwrap()
+            }
             DataType::Struct(fields) => {
                 let field_class = pl.const_get::<_, Value>("Field").unwrap();
                 let iter = fields.iter().map(|fld| {
@@ -260,12 +241,31 @@ impl IntoValue for Wrap<DataType> {
                     .funcall::<_, _, Value>("new", (fields,))
                     .unwrap()
             }
-            DataType::Null => pl.const_get::<_, Value>("Null").unwrap(),
-            DataType::Unknown => pl.const_get::<_, Value>("Unknown").unwrap(),
+            DataType::Null => {
+                let class = pl.const_get::<_, Value>("Null").unwrap();
+                class.funcall("new", ()).unwrap()
+            }
+            DataType::Unknown => {
+                let class = pl.const_get::<_, Value>("Unknown").unwrap();
+                class.funcall("new", ()).unwrap()
+            }
+            DataType::BinaryOffset => {
+                unimplemented!()
+            }
         }
     }
 }
 
+impl IntoValue for Wrap<CategoricalOrdering> {
+    fn into_value_with(self, _: &Ruby) -> Value {
+        let ordering = match self.0 {
+            CategoricalOrdering::Physical => "physical",
+            CategoricalOrdering::Lexical => "lexical",
+        };
+        ordering.into_value()
+    }
+}
+
 impl IntoValue for Wrap<TimeUnit> {
     fn into_value_with(self, _: &Ruby) -> Value {
         let tu = match self.0 {
@@ -277,114 +277,6 @@ impl IntoValue for Wrap<TimeUnit> {
     }
 }
 
-impl IntoValue for Wrap<&Utf8Chunked> {
-    fn into_value_with(self, _: &Ruby) -> Value {
-        let iter = self.0.into_iter();
-        RArray::from_iter(iter).into_value()
-    }
-}
-
-impl IntoValue for Wrap<&BinaryChunked> {
-    fn into_value_with(self, _: &Ruby) -> Value {
-        let iter = self
-            .0
-            .into_iter()
-            .map(|opt_bytes| opt_bytes.map(RString::from_slice));
-        RArray::from_iter(iter).into_value()
-    }
-}
-
-impl IntoValue for Wrap<&StructChunked> {
-    fn into_value_with(self, _: &Ruby) -> Value {
-        let s = self.0.clone().into_series();
-        // todo! iterate its chunks and flatten.
-        // make series::iter() accept a chunk index.
-        let s = s.rechunk();
-        let iter = s.iter().map(|av| {
-            if let AnyValue::Struct(_, _, flds) = av {
-                struct_dict(av._iter_struct_av(), flds)
-            } else {
-                unreachable!()
-            }
-        });
-
-        RArray::from_iter(iter).into_value()
-    }
-}
-
-impl IntoValue for Wrap<&DurationChunked> {
-    fn into_value_with(self, _: &Ruby) -> Value {
-        let utils = utils();
-        let time_unit = Wrap(self.0.time_unit()).into_value();
-        let iter = self.0.into_iter().map(|opt_v| {
-            opt_v.map(|v| {
-                utils
-                    .funcall::<_, _, Value>("_to_ruby_duration", (v, time_unit))
-                    .unwrap()
-            })
-        });
-        RArray::from_iter(iter).into_value()
-    }
-}
-
-impl IntoValue for Wrap<&DatetimeChunked> {
-    fn into_value_with(self, _: &Ruby) -> Value {
-        let utils = utils();
-        let time_unit = Wrap(self.0.time_unit()).into_value();
-        let time_zone = self.0.time_zone().clone().into_value();
-        let iter = self.0.into_iter().map(|opt_v| {
-            opt_v.map(|v| {
-                utils
-                    .funcall::<_, _, Value>("_to_ruby_datetime", (v, time_unit, time_zone))
-                    .unwrap()
-            })
-        });
-        RArray::from_iter(iter).into_value()
-    }
-}
-
-impl IntoValue for Wrap<&TimeChunked> {
-    fn into_value_with(self, _: &Ruby) -> Value {
-        let utils = utils();
-        let iter = self.0.into_iter().map(|opt_v| {
-            opt_v.map(|v| utils.funcall::<_, _, Value>("_to_ruby_time", (v,)).unwrap())
-        });
-        RArray::from_iter(iter).into_value()
-    }
-}
-
-impl IntoValue for Wrap<&DateChunked> {
-    fn into_value_with(self, _: &Ruby) -> Value {
-        let utils = utils();
-        let iter = self.0.into_iter().map(|opt_v| {
-            opt_v.map(|v| utils.funcall::<_, _, Value>("_to_ruby_date", (v,)).unwrap())
-        });
-        RArray::from_iter(iter).into_value()
-    }
-}
-
-impl IntoValue for Wrap<&DecimalChunked> {
-    fn into_value_with(self, _: &Ruby) -> Value {
-        let utils = utils();
-        let rb_scale = (-(self.0.scale() as i32)).into_value();
-        let iter = self.0.into_iter().map(|opt_v| {
-            opt_v.map(|v| {
-                utils
-                    .funcall::<_, _, Value>("_to_ruby_decimal", (v.to_string(), rb_scale))
-                    .unwrap()
-            })
-        });
-        RArray::from_iter(iter).into_value()
-    }
-}
-
-fn abs_decimal_from_digits(digits: String, exp: i32) -> Option<(i128, usize)> {
-    match digits.parse::<i128>() {
-        Ok(v) => Some((v, ((digits.len() as i32) - exp) as usize)),
-        Err(_) => None,
-    }
-}
-
 impl TryConvert for Wrap<Field> {
     fn try_convert(ob: Value) -> RbResult<Self> {
         let name: String = ob.funcall("name", ())?;
@@ -406,10 +298,11 @@ impl TryConvert for Wrap<DataType> {
                 "Polars::Int16" => DataType::Int16,
                 "Polars::Int32" => DataType::Int32,
                 "Polars::Int64" => DataType::Int64,
-                "Polars::Utf8" => DataType::Utf8,
+                "Polars::String" => DataType::String,
                 "Polars::Binary" => DataType::Binary,
                 "Polars::Boolean" => DataType::Boolean,
-                "Polars::Categorical" => DataType::Categorical(None),
+                "Polars::Categorical" => DataType::Categorical(None, Default::default()),
+                "Polars::Enum" => DataType::Enum(None, Default::default()),
                 "Polars::Date" => DataType::Date,
                 "Polars::Datetime" => DataType::Datetime(TimeUnit::Microseconds, None),
                 "Polars::Time" => DataType::Time,
@@ -417,7 +310,7 @@ impl TryConvert for Wrap<DataType> {
                 "Polars::Decimal" => DataType::Decimal(None, None),
                 "Polars::Float32" => DataType::Float32,
                 "Polars::Float64" => DataType::Float64,
-                "Polars::Object" => DataType::Object(OBJECT_NAME),
+                "Polars::Object" => DataType::Object(OBJECT_NAME, None),
                 "Polars::List" => DataType::List(Box::new(DataType::Null)),
                 "Polars::Null" => DataType::Null,
                 "Polars::Unknown" => DataType::Unknown,
@@ -431,6 +324,36 @@ impl TryConvert for Wrap<DataType> {
         } else if String::try_convert(ob).is_err() {
             let name = unsafe { ob.class().name() }.into_owned();
             match name.as_str() {
+                "Polars::Int8" => DataType::Int8,
+                "Polars::Int16" => DataType::Int16,
+                "Polars::Int32" => DataType::Int32,
+                "Polars::Int64" => DataType::Int64,
+                "Polars::UInt8" => DataType::UInt8,
+                "Polars::UInt16" => DataType::UInt16,
+                "Polars::UInt32" => DataType::UInt32,
+                "Polars::UInt64" => DataType::UInt64,
+                "Polars::String" => DataType::String,
+                "Polars::Binary" => DataType::Binary,
+                "Polars::Boolean" => DataType::Boolean,
+                "Polars::Categorical" => {
+                    let ordering = ob
+                        .funcall::<_, _, Wrap<CategoricalOrdering>>("ordering", ())?
+                        .0;
+                    DataType::Categorical(None, ordering)
+                }
+                "Polars::Enum" => {
+                    let categories = ob.funcall("categories", ()).unwrap();
+                    let s = get_series(categories)?;
+                    let ca = s.str().map_err(RbPolarsErr::from)?;
+                    let categories = ca.downcast_iter().next().unwrap().clone();
+                    create_enum_data_type(categories)
+                }
+                "Polars::Date" => DataType::Date,
+                "Polars::Time" => DataType::Time,
+                "Polars::Float32" => DataType::Float32,
+                "Polars::Float64" => DataType::Float64,
+                "Polars::Null" => DataType::Null,
+                "Polars::Unknown" => DataType::Unknown,
                 "Polars::Duration" => {
                     let time_unit: Value = ob.funcall("time_unit", ()).unwrap();
                     let time_unit = Wrap::<TimeUnit>::try_convert(time_unit)?.0;
@@ -477,17 +400,17 @@ impl TryConvert for Wrap<DataType> {
                 "i16" => DataType::Int16,
                 "i32" => DataType::Int32,
                 "i64" => DataType::Int64,
-                "str" => DataType::Utf8,
+                "str" => DataType::String,
                 "bin" => DataType::Binary,
                 "bool" => DataType::Boolean,
-                "cat" => DataType::Categorical(None),
+                "cat" => DataType::Categorical(None, Default::default()),
                 "date" => DataType::Date,
                 "datetime" => DataType::Datetime(TimeUnit::Microseconds, None),
                 "f32" => DataType::Float32,
                 "time" => DataType::Time,
                 "dur" => DataType::Duration(TimeUnit::Microseconds),
                 "f64" => DataType::Float64,
-                "obj" => DataType::Object(OBJECT_NAME),
+                "obj" => DataType::Object(OBJECT_NAME, None),
                 "list" => DataType::List(Box::new(DataType::Boolean)),
                 "null" => DataType::Null,
                 "unk" => DataType::Unknown,
@@ -503,102 +426,6 @@ impl TryConvert for Wrap<DataType> {
     }
 }
 
-impl<'s> TryConvert for Wrap<AnyValue<'s>> {
-    fn try_convert(ob: Value) -> RbResult<Self> {
-        if ob.is_kind_of(class::true_class()) || ob.is_kind_of(class::false_class()) {
-            Ok(AnyValue::Boolean(bool::try_convert(ob)?).into())
-        } else if let Some(v) = Integer::from_value(ob) {
-            Ok(AnyValue::Int64(v.to_i64()?).into())
-        } else if let Some(v) = Float::from_value(ob) {
-            Ok(AnyValue::Float64(v.to_f64()).into())
-        } else if let Some(v) = RString::from_value(ob) {
-            if v.enc_get() == Index::utf8() {
-                Ok(AnyValue::Utf8Owned(v.to_string()?.into()).into())
-            } else {
-                Ok(AnyValue::BinaryOwned(unsafe { v.as_slice() }.to_vec()).into())
-            }
-        // call is_a? for ActiveSupport::TimeWithZone
-        } else if ob.funcall::<_, _, bool>("is_a?", (class::time(),))? {
-            let sec = ob.funcall::<_, _, i64>("to_i", ())?;
-            let nsec = ob.funcall::<_, _, i64>("nsec", ())?;
-            let v = sec * 1_000_000_000 + nsec;
-            // TODO support time zone when possible
-            // https://github.com/pola-rs/polars/issues/9103
-            Ok(AnyValue::Datetime(v, TimeUnit::Nanoseconds, &None).into())
-        } else if ob.is_nil() {
-            Ok(AnyValue::Null.into())
-        } else if let Some(dict) = RHash::from_value(ob) {
-            let len = dict.len();
-            let mut keys = Vec::with_capacity(len);
-            let mut vals = Vec::with_capacity(len);
-            dict.foreach(|k: Value, v: Value| {
-                let key = String::try_convert(k)?;
-                let val = Wrap::<AnyValue>::try_convert(v)?.0;
-                let dtype = DataType::from(&val);
-                keys.push(Field::new(&key, dtype));
-                vals.push(val);
-                Ok(ForEach::Continue)
-            })?;
-            Ok(Wrap(AnyValue::StructOwned(Box::new((vals, keys)))))
-        } else if let Some(v) = RArray::from_value(ob) {
-            if v.is_empty() {
-                Ok(Wrap(AnyValue::List(Series::new_empty("", &DataType::Null))))
-            } else {
-                let list = v;
-
-                let mut avs = Vec::with_capacity(25);
-                let mut iter = list.each();
-
-                for item in (&mut iter).take(25) {
-                    avs.push(Wrap::<AnyValue>::try_convert(item?)?.0)
-                }
-
-                let (dtype, _n_types) = any_values_to_dtype(&avs).map_err(RbPolarsErr::from)?;
-
-                // push the rest
-                avs.reserve(list.len());
-                for item in iter {
-                    avs.push(Wrap::<AnyValue>::try_convert(item?)?.0)
-                }
-
-                let s = Series::from_any_values_and_dtype("", &avs, &dtype, true)
-                    .map_err(RbPolarsErr::from)?;
-                Ok(Wrap(AnyValue::List(s)))
-            }
-        } else if ob.is_kind_of(crate::rb_modules::datetime()) {
-            let sec: i64 = ob.funcall("to_i", ())?;
-            let nsec: i64 = ob.funcall("nsec", ())?;
-            Ok(Wrap(AnyValue::Datetime(
-                sec * 1_000_000_000 + nsec,
-                TimeUnit::Nanoseconds,
-                &None,
-            )))
-        } else if ob.is_kind_of(crate::rb_modules::date()) {
-            // convert to DateTime for UTC
-            let v = ob
-                .funcall::<_, _, Value>("to_datetime", ())?
-                .funcall::<_, _, Value>("to_time", ())?
-                .funcall::<_, _, i64>("to_i", ())?;
-            Ok(Wrap(AnyValue::Date((v / 86400) as i32)))
-        } else if ob.is_kind_of(crate::rb_modules::bigdecimal()) {
-            let (sign, digits, _, exp): (i8, String, i32, i32) = ob.funcall("split", ()).unwrap();
-            let (mut v, scale) = abs_decimal_from_digits(digits, exp).ok_or_else(|| {
-                RbPolarsErr::other("BigDecimal is too large to fit in Decimal128".into())
-            })?;
-            if sign < 0 {
-                // TODO better error
-                v = v.checked_neg().unwrap();
-            }
-            Ok(Wrap(AnyValue::Decimal(v, scale)))
-        } else {
-            Err(RbPolarsErr::other(format!(
-                "object type not supported {:?}",
-                ob
-            )))
-        }
-    }
-}
-
 impl<'s> TryConvert for Wrap<Row<'s>> {
     fn try_convert(ob: Value) -> RbResult<Self> {
         let mut vals: Vec<Wrap<AnyValue<'s>>> = Vec::new();
@@ -662,6 +489,15 @@ impl TotalEq for ObjectValue {
     }
 }
 
+impl TotalHash for ObjectValue {
+    fn tot_hash<H>(&self, state: &mut H)
+    where
+        H: Hasher,
+    {
+        self.hash(state);
+    }
+}
+
 impl Display for ObjectValue {
     fn fmt(&self, f: &mut Formatter<'_>) -> std::fmt::Result {
         write!(f, "{}", self.to_object())
@@ -715,24 +551,33 @@ impl Default for ObjectValue {
 
 pub(crate) fn dicts_to_rows(
     records: &Value,
-    infer_schema_len: usize,
+    infer_schema_len: Option<usize>,
+    schema_columns: PlIndexSet<String>,
 ) -> RbResult<(Vec<Row>, Vec<String>)> {
+    let infer_schema_len = infer_schema_len.map(|n| std::cmp::max(1, n));
     let (dicts, len) = get_rbseq(*records)?;
 
-    let mut key_names = PlIndexSet::new();
-    for d in dicts.each().take(infer_schema_len) {
-        let d = d?;
-        let d = RHash::try_convert(d)?;
-
-        d.foreach(|name: Value, _value: Value| {
-            if let Some(v) = Symbol::from_value(name) {
-                key_names.insert(v.name()?.into());
-            } else {
-                key_names.insert(String::try_convert(name)?);
-            };
-            Ok(ForEach::Continue)
-        })?;
-    }
+    let key_names = {
+        if !schema_columns.is_empty() {
+            schema_columns
+        } else {
+            let mut inferred_keys = PlIndexSet::new();
+            for d in dicts.each().take(infer_schema_len.unwrap_or(usize::MAX)) {
+                let d = d?;
+                let d = RHash::try_convert(d)?;
+
+                d.foreach(|name: Value, _value: Value| {
+                    if let Some(v) = Symbol::from_value(name) {
+                        inferred_keys.insert(v.name()?.into());
+                    } else {
+                        inferred_keys.insert(String::try_convert(name)?);
+                    };
+                    Ok(ForEach::Continue)
+                })?;
+            }
+            inferred_keys
+        }
+    };
 
     let mut rows = Vec::with_capacity(len);
 
@@ -891,7 +736,8 @@ impl TryConvert for Wrap<JoinType> {
         let parsed = match String::try_convert(ob)?.as_str() {
             "inner" => JoinType::Inner,
             "left" => JoinType::Left,
-            "outer" => JoinType::Outer,
+            "outer" => JoinType::Outer { coalesce: false },
+            "outer_coalesce" => JoinType::Outer { coalesce: true },
             "semi" => JoinType::Semi,
             "anti" => JoinType::Anti,
             // #[cfg(feature = "cross_join")]
@@ -1061,6 +907,22 @@ impl TryConvert for Wrap<UniqueKeepStrategy> {
     }
 }
 
+impl TryConvert for Wrap<IpcCompression> {
+    fn try_convert(ob: Value) -> RbResult<Self> {
+        let parsed = match String::try_convert(ob)?.as_str() {
+            "lz4" => IpcCompression::LZ4,
+            "zstd" => IpcCompression::ZSTD,
+            v => {
+                return Err(RbValueError::new_err(format!(
+                    "compression must be one of {{'lz4', 'zstd'}}, got {}",
+                    v
+                )))
+            }
+        };
+        Ok(Wrap(parsed))
+    }
+}
+
 impl TryConvert for Wrap<SearchSortedSide> {
     fn try_convert(ob: Value) -> RbResult<Self> {
         let parsed = match String::try_convert(ob)?.as_str() {
@@ -1077,6 +939,56 @@ impl TryConvert for Wrap<SearchSortedSide> {
     }
 }
 
+impl TryConvert for Wrap<WindowMapping> {
+    fn try_convert(ob: Value) -> RbResult<Self> {
+        let parsed = match String::try_convert(ob)?.as_str() {
+            "group_to_rows" => WindowMapping::GroupsToRows,
+            "join" => WindowMapping::Join,
+            "explode" => WindowMapping::Explode,
+            v => {
+                return Err(RbValueError::new_err(format!(
+                "`mapping_strategy` must be one of {{'group_to_rows', 'join', 'explode'}}, got {v}",
+            )))
+            }
+        };
+        Ok(Wrap(parsed))
+    }
+}
+
+impl TryConvert for Wrap<JoinValidation> {
+    fn try_convert(ob: Value) -> RbResult<Self> {
+        let parsed = match String::try_convert(ob)?.as_str() {
+            "1:1" => JoinValidation::OneToOne,
+            "1:m" => JoinValidation::OneToMany,
+            "m:m" => JoinValidation::ManyToMany,
+            "m:1" => JoinValidation::ManyToOne,
+            v => {
+                return Err(RbValueError::new_err(format!(
+                    "`validate` must be one of {{'m:m', 'm:1', '1:m', '1:1'}}, got {v}",
+                )))
+            }
+        };
+        Ok(Wrap(parsed))
+    }
+}
+
+impl TryConvert for Wrap<QuoteStyle> {
+    fn try_convert(ob: Value) -> RbResult<Self> {
+        let parsed = match String::try_convert(ob)?.as_str() {
+            "always" => QuoteStyle::Always,
+            "necessary" => QuoteStyle::Necessary,
+            "non_numeric" => QuoteStyle::NonNumeric,
+            "never" => QuoteStyle::Never,
+            v => {
+                return Err(RbValueError::new_err(format!(
+                    "`quote_style` must be one of {{'always', 'necessary', 'non_numeric', 'never'}}, got {v}",
+                )))
+            },
+        };
+        Ok(Wrap(parsed))
+    }
+}
+
 pub fn parse_fill_null_strategy(
     strategy: &str,
     limit: FillNullLimit,
@@ -1149,3 +1061,12 @@ where
 {
     container.into_iter().map(|s| s.as_ref().into()).collect()
 }
+
+impl TryConvert for Wrap<NonZeroUsize> {
+    fn try_convert(ob: Value) -> RbResult<Self> {
+        let v = usize::try_convert(ob)?;
+        NonZeroUsize::new(v)
+            .map(|v| Wrap(v))
+            .ok_or(RbValueError::new_err("must be non-zero".into()))
+    }
+}