polars-df 0.19.0 → 0.21.0

data/ext/polars/src/map/dataframe.rs

@@ -1,6 +1,6 @@
-use magnus::{class, prelude::*, typed_data::Obj, IntoValue, RArray, TryConvert, Value};
+use magnus::{IntoValue, RArray, TryConvert, Value, class, prelude::*, typed_data::Obj};
 use polars::prelude::*;
-use polars_core::frame::row::{rows_to_schema_first_non_null, Row};
+use polars_core::frame::row::{Row, rows_to_schema_first_non_null};
 use polars_core::series::SeriesIter;
 
 use super::*;
@@ -144,7 +144,7 @@ where
         let tpl = (RArray::from_iter(iter),);
         match lambda.funcall::<_, _, Value>("call", tpl) {
             Ok(val) => T::try_convert(val).ok(),
-            Err(e) => panic!("ruby function failed {}", e),
+            Err(e) => panic!("ruby function failed {e}"),
         }
     })
 }
@@ -157,7 +157,7 @@ pub fn apply_lambda_with_primitive_out_type<D>(
     first_value: Option<D::Native>,
 ) -> ChunkedArray<D>
 where
-    D: RbArrowPrimitiveType,
+    D: RbPolarsNumericType,
     D::Native: IntoValue + TryConvert,
 {
     let skip = usize::from(first_value.is_some());
@@ -247,11 +247,11 @@ pub fn apply_lambda_with_list_out_type(
                         if val.is_nil() {
                             None
                         } else {
-                            panic!("should return a Series, got a {:?}", val)
+                            panic!("should return a Series, got a {val:?}")
                         }
                     }
                 },
-                Err(e) => panic!("ruby function failed {}", e),
+                Err(e) => panic!("ruby function failed {e}"),
             }
         });
         iterator_to_list(
@@ -302,7 +302,7 @@ pub fn apply_lambda_with_rows_output<'a>(
                     None => Ok(&null_row),
                 }
             }
-            Err(e) => panic!("ruby function failed {}", e),
+            Err(e) => panic!("ruby function failed {e}"),
         }
     });
 

data/ext/polars/src/map/lazy.rs

@@ -1,4 +1,4 @@
-use magnus::{prelude::*, RArray, Value};
+use magnus::{RArray, Value, prelude::*};
 use polars::prelude::*;
 
 use crate::rb_modules::*;

data/ext/polars/src/map/mod.rs

@@ -2,27 +2,37 @@ pub mod dataframe;
 pub mod lazy;
 pub mod series;
 
-use magnus::{prelude::*, RHash, Value};
+use magnus::{RHash, Value, prelude::*};
 use polars::chunked_array::builder::get_list_builder;
 use polars::prelude::*;
-use polars_core::utils::CustomIterTools;
 use polars_core::POOL;
+use polars_core::utils::CustomIterTools;
 use rayon::prelude::*;
 
-use crate::{ObjectValue, RbPolarsErr, RbResult, RbSeries, Wrap};
+use crate::{ObjectValue, RbPolarsErr, RbResult, RbSeries, RbValueError, Wrap};
+
+pub trait RbPolarsNumericType: PolarsNumericType {}
 
-pub trait RbArrowPrimitiveType: PolarsNumericType {}
+impl RbPolarsNumericType for UInt8Type {}
+impl RbPolarsNumericType for UInt16Type {}
+impl RbPolarsNumericType for UInt32Type {}
+impl RbPolarsNumericType for UInt64Type {}
+impl RbPolarsNumericType for Int8Type {}
+impl RbPolarsNumericType for Int16Type {}
+impl RbPolarsNumericType for Int32Type {}
+impl RbPolarsNumericType for Int64Type {}
+impl RbPolarsNumericType for Float32Type {}
+impl RbPolarsNumericType for Float64Type {}
 
-impl RbArrowPrimitiveType for UInt8Type {}
-impl RbArrowPrimitiveType for UInt16Type {}
-impl RbArrowPrimitiveType for UInt32Type {}
-impl RbArrowPrimitiveType for UInt64Type {}
-impl RbArrowPrimitiveType for Int8Type {}
-impl RbArrowPrimitiveType for Int16Type {}
-impl RbArrowPrimitiveType for Int32Type {}
-impl RbArrowPrimitiveType for Int64Type {}
-impl RbArrowPrimitiveType for Float32Type {}
-impl RbArrowPrimitiveType for Float64Type {}
+pub(super) fn check_nested_object(dt: &DataType) -> RbResult<()> {
+    if dt.contains_objects() {
+        Err(RbValueError::new_err(
+            "nested objects are not allowed\n\nSet `return_dtype: Polars::Object` to use Ruby's native nesting.",
+        ))
+    } else {
+        Ok(())
+    }
+}
 
 fn iterator_to_struct(
     it: impl Iterator<Item = Option<Value>>,
@@ -37,7 +47,7 @@ fn iterator_to_struct(
         _ => {
             return Err(crate::exceptions::ComputeError::new_err(format!(
                 "expected struct got {first_value:?}",
-            )))
+            )));
         }
     };
 
@@ -70,9 +80,11 @@ fn iterator_to_struct(
             Some(dict) => {
                 let dict = RHash::try_convert(dict)?;
                 if dict.len() != struct_width {
-                    return Err(crate::exceptions::ComputeError::new_err(
-                        format!("Cannot create struct type.\n> The struct dtype expects {} fields, but it got a dict with {} fields.", struct_width, dict.len())
-                    ));
+                    return Err(crate::exceptions::ComputeError::new_err(format!(
+                        "Cannot create struct type.\n> The struct dtype expects {} fields, but it got a dict with {} fields.",
+                        struct_width,
+                        dict.len()
+                    )));
                 }
                 // we ignore the keys of the rest of the dicts
                 // the first item determines the output name
@@ -109,7 +121,7 @@ fn iterator_to_primitive<T>(
     capacity: usize,
 ) -> ChunkedArray<T>
 where
-    T: RbArrowPrimitiveType,
+    T: RbPolarsNumericType,
 {
     // safety: we know the iterators len
     let mut ca: ChunkedArray<T> = unsafe {

data/ext/polars/src/map/series.rs

@@ -1,4 +1,4 @@
-use magnus::{class, prelude::*, typed_data::Obj, IntoValue, TryConvert, Value};
+use magnus::{IntoValue, TryConvert, Value, class, prelude::*, typed_data::Obj};
 use polars::prelude::*;
 
 use super::*;
@@ -85,7 +85,7 @@ pub trait ApplyLambda<'a> {
         first_value: Option<D::Native>,
     ) -> RbResult<ChunkedArray<D>>
     where
-        D: RbArrowPrimitiveType,
+        D: RbPolarsNumericType,
         D::Native: IntoValue + TryConvert;
 
     /// Apply a lambda with a boolean output type
@@ -143,7 +143,7 @@ where
 {
     match call_lambda(lambda, in_val) {
         Ok(out) => S::try_convert(out),
-        Err(e) => panic!("ruby function failed {}", e),
+        Err(e) => panic!("ruby function failed {e}"),
     }
 }
 
@@ -219,7 +219,7 @@ impl<'a> ApplyLambda<'a> for BooleanChunked {
         first_value: Option<D::Native>,
     ) -> RbResult<ChunkedArray<D>>
     where
-        D: RbArrowPrimitiveType,
+        D: RbPolarsNumericType,
         D::Native: IntoValue + TryConvert,
     {
         let skip = usize::from(first_value.is_some());
@@ -438,7 +438,7 @@ impl<'a> ApplyLambda<'a> for BooleanChunked {
 
 impl<'a, T> ApplyLambda<'a> for ChunkedArray<T>
 where
-    T: RbArrowPrimitiveType + PolarsNumericType,
+    T: RbPolarsNumericType + PolarsNumericType,
     T::Native: IntoValue + TryConvert,
     ChunkedArray<T>: IntoSeries,
 {
@@ -503,7 +503,7 @@ where
         first_value: Option<D::Native>,
     ) -> RbResult<ChunkedArray<D>>
     where
-        D: RbArrowPrimitiveType,
+        D: RbPolarsNumericType,
         D::Native: IntoValue + TryConvert,
     {
         let skip = usize::from(first_value.is_some());
@@ -782,7 +782,7 @@ impl<'a> ApplyLambda<'a> for StringChunked {
         first_value: Option<D::Native>,
     ) -> RbResult<ChunkedArray<D>>
     where
-        D: RbArrowPrimitiveType,
+        D: RbPolarsNumericType,
         D::Native: IntoValue + TryConvert,
     {
         let skip = usize::from(first_value.is_some());
@@ -1047,7 +1047,7 @@ impl<'a> ApplyLambda<'a> for StructChunked {
         first_value: Option<D::Native>,
     ) -> RbResult<ChunkedArray<D>>
     where
-        D: RbArrowPrimitiveType,
+        D: RbPolarsNumericType,
         D::Native: IntoValue + TryConvert,
     {
         let skip = usize::from(first_value.is_some());

data/ext/polars/src/on_startup.rs

@@ -9,8 +9,8 @@ use polars_core::chunked_array::object::registry;
 use polars_core::chunked_array::object::registry::AnonymousObjectBuilder;
 use polars_core::prelude::AnyValue;
 
-use crate::prelude::ObjectValue;
 use crate::Wrap;
+use crate::prelude::ObjectValue;
 
 static POLARS_REGISTRY_INIT_LOCK: OnceLock<()> = OnceLock::new();
 
@@ -39,6 +39,9 @@ pub(crate) fn register_startup_deps() {
             object_converter,
             rbobject_converter,
             physical_dtype,
-        )
+        );
+        // TODO
+        // Register warning function for `polars_warn!`.
+        // polars_error::set_warning_function(warning_function);
     });
 }

data/ext/polars/src/rb_modules.rs

@@ -1,4 +1,4 @@
-use magnus::{value::Lazy, ExceptionClass, Module, RClass, RModule, Ruby};
+use magnus::{ExceptionClass, Module, RClass, RModule, Ruby, value::Lazy};
 
 static POLARS: Lazy<RModule> = Lazy::new(|ruby| ruby.class_object().const_get("Polars").unwrap());
 

data/ext/polars/src/series/construction.rs

@@ -1,8 +1,8 @@
-use magnus::{prelude::*, RArray, RString};
+use magnus::{RArray, RString, prelude::*};
 use polars_core::prelude::*;
 
 use crate::any_value::rb_object_to_any_value;
-use crate::conversion::{slice_extract_wrapped, vec_extract_wrapped, Wrap};
+use crate::conversion::{Wrap, slice_extract_wrapped, vec_extract_wrapped};
 use crate::prelude::ObjectValue;
 use crate::series::to_series;
 use crate::{RbPolarsErr, RbResult, RbSeries, RbTypeError, RbValueError};
@@ -122,13 +122,17 @@ impl RbSeries {
             .into_iter()
             .map(|v| rb_object_to_any_value(v, strict))
             .collect::<RbResult<Vec<AnyValue>>>()?;
-        let s =
-            Series::from_any_values_and_dtype(name.into(), any_values.as_slice(), &dtype.0, strict)
-                .map_err(|e| {
-                    RbTypeError::new_err(format!(
+        let s = Series::from_any_values_and_dtype(
+            name.into(),
+            any_values.as_slice(),
+            &dtype.0,
+            strict,
+        )
+        .map_err(|e| {
+            RbTypeError::new_err(format!(
                 "{e}\n\nHint: Try setting `strict: false` to allow passing data with mixed types."
             ))
-                })?;
+        })?;
         Ok(s.into())
     }
 

data/ext/polars/src/series/export.rs

@@ -1,8 +1,8 @@
-use magnus::{value::qnil, IntoValue, RArray, Value};
+use magnus::{IntoValue, RArray, Value, value::qnil};
 use polars_core::prelude::*;
 
-use crate::prelude::*;
 use crate::RbSeries;
+use crate::prelude::*;
 
 impl RbSeries {
     /// Convert this Series to a Ruby array.
@@ -25,7 +25,9 @@ impl RbSeries {
                 DataType::Float32 => RArray::from_iter(series.f32().unwrap()).into_value(),
                 DataType::Float64 => RArray::from_iter(series.f64().unwrap()).into_value(),
                 DataType::Categorical(_, _) | DataType::Enum(_, _) => {
-                    RArray::from_iter(series.categorical().unwrap().iter_str()).into_value()
+                    with_match_categorical_physical_type!(series.dtype().cat_physical().unwrap(), |$C| {
+                        RArray::from_iter(series.cat::<$C>().unwrap().iter_str()).into_value()
+                    })
                 }
                 DataType::Object(_) => {
                     let v = RArray::with_capacity(series.len());
@@ -132,7 +134,7 @@ impl RbSeries {
                     unreachable!()
                 }
             };
-            rblist
+            rblist.into_value()
         }
 
         to_a_recursive(series)

data/ext/polars/src/series/general.rs

@@ -1,10 +1,8 @@
-use magnus::{exception, Error, IntoValue, Value};
+use magnus::{Error, IntoValue, Value, exception};
 use polars::prelude::*;
 use polars::series::IsSorted;
 
-use crate::apply_method_all_arrow_series2;
 use crate::conversion::*;
-use crate::map::series::{call_lambda_and_extract, ApplyLambda};
 use crate::{RbDataFrame, RbPolarsErr, RbResult, RbSeries};
 
 impl RbSeries {
@@ -42,21 +40,15 @@ impl RbSeries {
     }
 
     pub fn cat_uses_lexical_ordering(&self) -> RbResult<bool> {
-        let binding = self.series.borrow();
-        let ca = binding.categorical().map_err(RbPolarsErr::from)?;
-        Ok(ca.uses_lexical_ordering())
+        Ok(true)
     }
 
     pub fn cat_is_local(&self) -> RbResult<bool> {
-        let binding = self.series.borrow();
-        let ca = binding.categorical().map_err(RbPolarsErr::from)?;
-        Ok(ca.get_rev_map().is_local())
+        Ok(false)
     }
 
     pub fn cat_to_local(&self) -> RbResult<Self> {
-        let binding = self.series.borrow();
-        let ca = binding.categorical().map_err(RbPolarsErr::from)?;
-        Ok(ca.to_local().into_series().into())
+        Ok(self.clone())
     }
 
     pub fn estimated_size(&self) -> usize {
@@ -75,7 +67,7 @@ impl RbSeries {
             if val == v_trunc {
                 val
             } else {
-                format!("{}…", v_trunc)
+                format!("{v_trunc}…")
             }
         } else {
             val
@@ -305,198 +297,6 @@ impl RbSeries {
         RbSeries::new(self.series.borrow().clone())
     }
 
-    pub fn apply_lambda(
-        &self,
-        lambda: Value,
-        output_type: Option<Wrap<DataType>>,
-        skip_nulls: bool,
-    ) -> RbResult<Self> {
-        let series = &self.series.borrow();
-
-        let output_type = output_type.map(|dt| dt.0);
-
-        macro_rules! dispatch_apply {
-            ($self:expr, $method:ident, $($args:expr),*) => {
-                if matches!($self.dtype(), DataType::Object(_)) {
-                    // let ca = $self.0.unpack::<ObjectType<ObjectValue>>().unwrap();
-                    // ca.$method($($args),*)
-                    todo!()
-                } else {
-                    apply_method_all_arrow_series2!(
-                        $self,
-                        $method,
-                        $($args),*
-                    )
-                }
-
-            }
-
-        }
-
-        if matches!(
-            series.dtype(),
-            DataType::Datetime(_, _)
-                | DataType::Date
-                | DataType::Duration(_)
-                | DataType::Categorical(_, _)
-                | DataType::Time
-        ) || !skip_nulls
-        {
-            let mut avs = Vec::with_capacity(series.len());
-            let iter = series.iter().map(|av| {
-                let input = Wrap(av);
-                call_lambda_and_extract::<_, Wrap<AnyValue>>(lambda, input)
-                    .unwrap()
-                    .0
-            });
-            avs.extend(iter);
-            return Ok(Series::new(self.name().into(), &avs).into());
-        }
-
-        let out = match output_type {
-            Some(DataType::Int8) => {
-                let ca: Int8Chunked = dispatch_apply!(
-                    series,
-                    apply_lambda_with_primitive_out_type,
-                    lambda,
-                    0,
-                    None
-                )?;
-                ca.into_series()
-            }
-            Some(DataType::Int16) => {
-                let ca: Int16Chunked = dispatch_apply!(
-                    series,
-                    apply_lambda_with_primitive_out_type,
-                    lambda,
-                    0,
-                    None
-                )?;
-                ca.into_series()
-            }
-            Some(DataType::Int32) => {
-                let ca: Int32Chunked = dispatch_apply!(
-                    series,
-                    apply_lambda_with_primitive_out_type,
-                    lambda,
-                    0,
-                    None
-                )?;
-                ca.into_series()
-            }
-            Some(DataType::Int64) => {
-                let ca: Int64Chunked = dispatch_apply!(
-                    series,
-                    apply_lambda_with_primitive_out_type,
-                    lambda,
-                    0,
-                    None
-                )?;
-                ca.into_series()
-            }
-            Some(DataType::UInt8) => {
-                let ca: UInt8Chunked = dispatch_apply!(
-                    series,
-                    apply_lambda_with_primitive_out_type,
-                    lambda,
-                    0,
-                    None
-                )?;
-                ca.into_series()
-            }
-            Some(DataType::UInt16) => {
-                let ca: UInt16Chunked = dispatch_apply!(
-                    series,
-                    apply_lambda_with_primitive_out_type,
-                    lambda,
-                    0,
-                    None
-                )?;
-                ca.into_series()
-            }
-            Some(DataType::UInt32) => {
-                let ca: UInt32Chunked = dispatch_apply!(
-                    series,
-                    apply_lambda_with_primitive_out_type,
-                    lambda,
-                    0,
-                    None
-                )?;
-                ca.into_series()
-            }
-            Some(DataType::UInt64) => {
-                let ca: UInt64Chunked = dispatch_apply!(
-                    series,
-                    apply_lambda_with_primitive_out_type,
-                    lambda,
-                    0,
-                    None
-                )?;
-                ca.into_series()
-            }
-            Some(DataType::Float32) => {
-                let ca: Float32Chunked = dispatch_apply!(
-                    series,
-                    apply_lambda_with_primitive_out_type,
-                    lambda,
-                    0,
-                    None
-                )?;
-                ca.into_series()
-            }
-            Some(DataType::Float64) => {
-                let ca: Float64Chunked = dispatch_apply!(
-                    series,
-                    apply_lambda_with_primitive_out_type,
-                    lambda,
-                    0,
-                    None
-                )?;
-                ca.into_series()
-            }
-            Some(DataType::Boolean) => {
-                let ca: BooleanChunked =
-                    dispatch_apply!(series, apply_lambda_with_bool_out_type, lambda, 0, None)?;
-                ca.into_series()
-            }
-            Some(DataType::Date) => {
-                let ca: Int32Chunked = dispatch_apply!(
-                    series,
-                    apply_lambda_with_primitive_out_type,
-                    lambda,
-                    0,
-                    None
-                )?;
-                ca.into_date().into_series()
-            }
-            Some(DataType::Datetime(tu, tz)) => {
-                let ca: Int64Chunked = dispatch_apply!(
-                    series,
-                    apply_lambda_with_primitive_out_type,
-                    lambda,
-                    0,
-                    None
-                )?;
-                ca.into_datetime(tu, tz).into_series()
-            }
-            Some(DataType::String) => {
-                let ca = dispatch_apply!(series, apply_lambda_with_utf8_out_type, lambda, 0, None)?;
-
-                ca.into_series()
-            }
-            Some(DataType::Object(_)) => {
-                let ca =
-                    dispatch_apply!(series, apply_lambda_with_object_out_type, lambda, 0, None)?;
-                ca.into_series()
-            }
-            None => return dispatch_apply!(series, apply_lambda_unknown, lambda),
-
-            _ => return dispatch_apply!(series, apply_lambda_unknown, lambda),
-        };
-
-        Ok(RbSeries::new(out))
-    }
-
     pub fn zip_with(&self, mask: &RbSeries, other: &RbSeries) -> RbResult<Self> {
         let binding = mask.series.borrow();
         let mask = binding.bool().map_err(RbPolarsErr::from)?;
@@ -508,11 +308,16 @@ impl RbSeries {
         Ok(RbSeries::new(s))
     }
 
-    pub fn to_dummies(&self, sep: Option<String>, drop_first: bool) -> RbResult<RbDataFrame> {
+    pub fn to_dummies(
+        &self,
+        sep: Option<String>,
+        drop_first: bool,
+        drop_nulls: bool,
+    ) -> RbResult<RbDataFrame> {
         let df = self
             .series
             .borrow()
-            .to_dummies(sep.as_deref(), drop_first)
+            .to_dummies(sep.as_deref(), drop_first, drop_nulls)
             .map_err(RbPolarsErr::from)?;
         Ok(df.into())
     }

data/ext/polars/src/series/import.rs

@@ -1,13 +1,13 @@
 use arrow::array::Array;
 use arrow::ffi::{ArrowArrayStream, ArrowArrayStreamReader};
-use magnus::prelude::*;
 use magnus::Value;
+use magnus::prelude::*;
 use polars::prelude::*;
 
 use super::RbSeries;
 
-use crate::exceptions::RbValueError;
 use crate::RbResult;
+use crate::exceptions::RbValueError;
 
 /// Import `arrow_c_stream` across Ruby boundary.
 fn call_arrow_c_stream(ob: Value) -> RbResult<Value> {