polars-df 0.10.0 → 0.12.0

data/ext/polars/src/series/construction.rs

@@ -1,10 +1,11 @@
 use magnus::{prelude::*, RArray};
 use polars_core::prelude::*;
 
+use crate::any_value::rb_object_to_any_value;
 use crate::conversion::{slice_extract_wrapped, vec_extract_wrapped, Wrap};
 use crate::prelude::ObjectValue;
 use crate::series::to_series_collection;
-use crate::{RbPolarsErr, RbResult, RbSeries, RbValueError};
+use crate::{RbPolarsErr, RbResult, RbSeries, RbTypeError, RbValueError};
 
 impl RbSeries {
     pub fn new_opt_bool(name: String, obj: RArray, strict: bool) -> RbResult<RbSeries> {
@@ -35,36 +36,28 @@ impl RbSeries {
     }
 }
 
-fn new_primitive<T>(name: &str, obj: RArray, strict: bool) -> RbResult<RbSeries>
+fn new_primitive<T>(name: &str, values: RArray, _strict: bool) -> RbResult<RbSeries>
 where
     T: PolarsNumericType,
     ChunkedArray<T>: IntoSeries,
     T::Native: magnus::TryConvert,
 {
-    let len = obj.len();
+    let len = values.len();
     let mut builder = PrimitiveChunkedBuilder::<T>::new(name, len);
 
-    unsafe {
-        for item in obj.as_slice().iter() {
-            if item.is_nil() {
-                builder.append_null()
-            } else {
-                match T::Native::try_convert(*item) {
-                    Ok(val) => builder.append_value(val),
-                    Err(e) => {
-                        if strict {
-                            return Err(e);
-                        }
-                        builder.append_null()
-                    }
-                }
-            }
+    for res in values.into_iter() {
+        let value = res;
+        if value.is_nil() {
+            builder.append_null()
+        } else {
+            let v = <T::Native>::try_convert(value)?;
+            builder.append_value(v)
         }
     }
-    let ca = builder.finish();
 
+    let ca = builder.finish();
     let s = ca.into_series();
-    Ok(RbSeries::new(s))
+    Ok(s.into())
 }
 
 // Init with lists that can contain Nones
@@ -91,18 +84,50 @@ init_method_opt!(new_opt_f64, Float64Type, f64);
 
 fn vec_wrap_any_value<'s>(arr: RArray) -> RbResult<Vec<Wrap<AnyValue<'s>>>> {
     let mut val = Vec::with_capacity(arr.len());
-    for v in arr.each() {
-        val.push(Wrap::<AnyValue<'s>>::try_convert(v?)?);
+    for v in arr.into_iter() {
+        val.push(Wrap::<AnyValue<'s>>::try_convert(v)?);
     }
     Ok(val)
 }
 
 impl RbSeries {
-    pub fn new_from_anyvalues(name: String, val: RArray, strict: bool) -> RbResult<Self> {
-        let val = vec_wrap_any_value(val)?;
-        let avs = slice_extract_wrapped(&val);
+    pub fn new_from_any_values(name: String, values: RArray, strict: bool) -> RbResult<Self> {
+        let any_values_result = vec_wrap_any_value(values);
         // from anyvalues is fallible
-        let s = Series::from_any_values(&name, avs, strict).map_err(RbPolarsErr::from)?;
+        let result = any_values_result.and_then(|avs| {
+            let avs = slice_extract_wrapped(&avs);
+            let s = Series::from_any_values(&name, avs, strict).map_err(|e| {
+                RbTypeError::new_err(format!(
+                    "{e}\n\nHint: Try setting `strict: false` to allow passing data with mixed types."
+                ))
+            })?;
+            Ok(s.into())
+        });
+
+        // Fall back to Object type for non-strict construction.
+        if !strict && result.is_err() {
+            return Self::new_object(name, values, strict);
+        }
+
+        result
+    }
+
+    pub fn new_from_any_values_and_dtype(
+        name: String,
+        values: RArray,
+        dtype: Wrap<DataType>,
+        strict: bool,
+    ) -> RbResult<Self> {
+        let any_values = values
+            .into_iter()
+            .map(|v| rb_object_to_any_value(v, strict))
+            .collect::<RbResult<Vec<AnyValue>>>()?;
+        let s = Series::from_any_values_and_dtype(&name, any_values.as_slice(), &dtype.0, strict)
+            .map_err(|e| {
+            RbTypeError::new_err(format!(
+                "{e}\n\nHint: Try setting `strict: false` to allow passing data with mixed types."
+            ))
+        })?;
         Ok(s.into())
     }
 
@@ -125,9 +150,9 @@ impl RbSeries {
 
     pub fn new_object(name: String, val: RArray, _strict: bool) -> RbResult<Self> {
         let val = val
-            .each()
-            .map(|v| v.map(ObjectValue::from))
-            .collect::<RbResult<Vec<ObjectValue>>>()?;
+            .into_iter()
+            .map(ObjectValue::from)
+            .collect::<Vec<ObjectValue>>();
         let s = ObjectChunked::<ObjectValue>::new_from_vec(&name, val).into_series();
         Ok(s.into())
     }
@@ -163,15 +188,8 @@ impl RbSeries {
         }
     }
 
-    pub fn new_decimal(name: String, val: RArray, strict: bool) -> RbResult<Self> {
-        let val = vec_wrap_any_value(val)?;
-        // TODO: do we have to respect 'strict' here? it's possible if we want to
-        let avs = slice_extract_wrapped(&val);
-        // create a fake dtype with a placeholder "none" scale, to be inferred later
-        let dtype = DataType::Decimal(None, None);
-        let s = Series::from_any_values_and_dtype(&name, avs, &dtype, strict)
-            .map_err(RbPolarsErr::from)?;
-        Ok(s.into())
+    pub fn new_decimal(name: String, values: RArray, strict: bool) -> RbResult<Self> {
+        Self::new_from_any_values(name, values, strict)
     }
 
     pub fn repeat(

data/ext/polars/src/series/export.rs

@@ -1,57 +1,139 @@
-use magnus::{class, prelude::*, Module, RArray, RClass, RModule, Value};
+use magnus::{value::qnil, IntoValue, RArray, Value};
 use polars_core::prelude::*;
 
-use crate::{raise_err, RbPolarsErr, RbResult, RbSeries};
+use crate::prelude::*;
+use crate::RbSeries;
 
 impl RbSeries {
-    /// For numeric types, this should only be called for Series with null types.
-    /// This will cast to floats so that `nil = NAN`
-    pub fn to_numo(&self) -> RbResult<Value> {
-        let s = &self.series.borrow();
-        match s.dtype() {
-            DataType::String => {
-                let ca = s.str().unwrap();
+    /// Convert this Series to a Ruby array.
+    /// This operation copies data.
+    pub fn to_a(&self) -> Value {
+        let series = &self.series.borrow();
 
-                // TODO make more efficient
-                let np_arr = RArray::from_iter(ca);
-                class::object()
-                    .const_get::<_, RModule>("Numo")?
-                    .const_get::<_, RClass>("RObject")?
-                    .funcall("cast", (np_arr,))
-            }
-            dt if dt.is_numeric() => {
-                if s.bit_repr_is_large() {
-                    let s = s.cast(&DataType::Float64).unwrap();
-                    let ca = s.f64().unwrap();
-                    // TODO make more efficient
-                    let np_arr = RArray::from_iter(ca.into_iter().map(|opt_v| match opt_v {
-                        Some(v) => v,
-                        None => f64::NAN,
-                    }));
-                    class::object()
-                        .const_get::<_, RModule>("Numo")?
-                        .const_get::<_, RClass>("DFloat")?
-                        .funcall("cast", (np_arr,))
-                } else {
-                    let s = s.cast(&DataType::Float32).unwrap();
-                    let ca = s.f32().unwrap();
-                    // TODO make more efficient
-                    let np_arr = RArray::from_iter(ca.into_iter().map(|opt_v| match opt_v {
-                        Some(v) => v,
-                        None => f32::NAN,
-                    }));
-                    class::object()
-                        .const_get::<_, RModule>("Numo")?
-                        .const_get::<_, RClass>("SFloat")?
-                        .funcall("cast", (np_arr,))
-                }
-            }
-            dt => {
-                raise_err!(
-                    format!("'to_numo' not supported for dtype: {dt:?}"),
-                    ComputeError
-                );
-            }
+        fn to_a_recursive(series: &Series) -> Value {
+            let rblist = match series.dtype() {
+                DataType::Boolean => RArray::from_iter(series.bool().unwrap()).into_value(),
+                DataType::UInt8 => RArray::from_iter(series.u8().unwrap()).into_value(),
+                DataType::UInt16 => RArray::from_iter(series.u16().unwrap()).into_value(),
+                DataType::UInt32 => RArray::from_iter(series.u32().unwrap()).into_value(),
+                DataType::UInt64 => RArray::from_iter(series.u64().unwrap()).into_value(),
+                DataType::Int8 => RArray::from_iter(series.i8().unwrap()).into_value(),
+                DataType::Int16 => RArray::from_iter(series.i16().unwrap()).into_value(),
+                DataType::Int32 => RArray::from_iter(series.i32().unwrap()).into_value(),
+                DataType::Int64 => RArray::from_iter(series.i64().unwrap()).into_value(),
+                DataType::Float32 => RArray::from_iter(series.f32().unwrap()).into_value(),
+                DataType::Float64 => RArray::from_iter(series.f64().unwrap()).into_value(),
+                DataType::Categorical(_, _) | DataType::Enum(_, _) => {
+                    RArray::from_iter(series.categorical().unwrap().iter_str()).into_value()
+                }
+                DataType::Object(_, _) => {
+                    let v = RArray::with_capacity(series.len());
+                    for i in 0..series.len() {
+                        let obj: Option<&ObjectValue> = series.get_object(i).map(|any| any.into());
+                        match obj {
+                            Some(val) => v.push(val.to_object()).unwrap(),
+                            None => v.push(qnil()).unwrap(),
+                        };
+                    }
+                    v.into_value()
+                }
+                DataType::List(_) => {
+                    let v = RArray::new();
+                    let ca = series.list().unwrap();
+                    for opt_s in ca.amortized_iter() {
+                        match opt_s {
+                            None => {
+                                v.push(qnil()).unwrap();
+                            }
+                            Some(s) => {
+                                let rblst = to_a_recursive(s.as_ref());
+                                v.push(rblst).unwrap();
+                            }
+                        }
+                    }
+                    v.into_value()
+                }
+                DataType::Array(_, _) => {
+                    let v = RArray::new();
+                    let ca = series.array().unwrap();
+                    for opt_s in ca.amortized_iter() {
+                        match opt_s {
+                            None => {
+                                v.push(qnil()).unwrap();
+                            }
+                            Some(s) => {
+                                let rblst = to_a_recursive(s.as_ref());
+                                v.push(rblst).unwrap();
+                            }
+                        }
+                    }
+                    v.into_value()
+                }
+                DataType::Date => {
+                    let ca = series.date().unwrap();
+                    return Wrap(ca).into_value();
+                }
+                DataType::Time => {
+                    let ca = series.time().unwrap();
+                    return Wrap(ca).into_value();
+                }
+                DataType::Datetime(_, _) => {
+                    let ca = series.datetime().unwrap();
+                    return Wrap(ca).into_value();
+                }
+                DataType::Decimal(_, _) => {
+                    let ca = series.decimal().unwrap();
+                    return Wrap(ca).into_value();
+                }
+                DataType::String => {
+                    let ca = series.str().unwrap();
+                    return Wrap(ca).into_value();
+                }
+                DataType::Struct(_) => {
+                    let ca = series.struct_().unwrap();
+                    return Wrap(ca).into_value();
+                }
+                DataType::Duration(_) => {
+                    let ca = series.duration().unwrap();
+                    return Wrap(ca).into_value();
+                }
+                DataType::Binary => {
+                    let ca = series.binary().unwrap();
+                    return Wrap(ca).into_value();
+                }
+                DataType::Null => {
+                    let null: Option<u8> = None;
+                    let n = series.len();
+                    let iter = std::iter::repeat(null).take(n);
+                    use std::iter::{Repeat, Take};
+                    struct NullIter {
+                        iter: Take<Repeat<Option<u8>>>,
+                        n: usize,
+                    }
+                    impl Iterator for NullIter {
+                        type Item = Option<u8>;
+
+                        fn next(&mut self) -> Option<Self::Item> {
+                            self.iter.next()
+                        }
+                        fn size_hint(&self) -> (usize, Option<usize>) {
+                            (self.n, Some(self.n))
+                        }
+                    }
+                    impl ExactSizeIterator for NullIter {}
+
+                    RArray::from_iter(NullIter { iter, n }).into_value()
+                }
+                DataType::Unknown(_) => {
+                    panic!("to_a not implemented for unknown")
+                }
+                DataType::BinaryOffset => {
+                    unreachable!()
+                }
+            };
+            rblist
         }
+
+        to_a_recursive(series)
     }
 }

data/ext/polars/src/series/mod.rs

@@ -5,7 +5,7 @@ mod construction;
 mod export;
 mod scatter;
 
-use magnus::{exception, prelude::*, value::qnil, Error, IntoValue, RArray, Value};
+use magnus::{exception, prelude::*, Error, IntoValue, RArray, Value};
 use polars::prelude::*;
 use polars::series::IsSorted;
 use std::cell::RefCell;
@@ -36,8 +36,8 @@ impl RbSeries {
 
 pub fn to_series_collection(rs: RArray) -> RbResult<Vec<Series>> {
     let mut series = Vec::new();
-    for item in rs.each() {
-        series.push(<&RbSeries>::try_convert(item?)?.series.borrow().clone());
+    for item in rs.into_iter() {
+        series.push(<&RbSeries>::try_convert(item)?.series.borrow().clone());
     }
     Ok(series)
 }
@@ -247,13 +247,24 @@ impl RbSeries {
             .into())
     }
 
-    pub fn value_counts(&self, sorted: bool) -> RbResult<RbDataFrame> {
-        let df = self
+    pub fn value_counts(
+        &self,
+        sort: bool,
+        parallel: bool,
+        name: String,
+        normalize: bool,
+    ) -> RbResult<RbDataFrame> {
+        let out = self
             .series
             .borrow()
-            .value_counts(true, sorted)
+            .value_counts(sort, parallel, name, normalize)
             .map_err(RbPolarsErr::from)?;
-        Ok(df.into())
+        Ok(out.into())
+    }
+
+    pub fn slice(&self, offset: i64, length: Option<usize>) -> Self {
+        let length = length.unwrap_or_else(|| self.series.borrow().len());
+        self.series.borrow().slice(offset, length).into()
     }
 
     pub fn take_with_series(&self, indices: &RbSeries) -> RbResult<Self> {
@@ -301,10 +312,20 @@ impl RbSeries {
         Ok(s.into())
     }
 
-    pub fn equals(&self, other: &RbSeries, null_equal: bool, strict: bool) -> bool {
-        if strict {
-            self.series.borrow().eq(&other.series.borrow())
-        } else if null_equal {
+    pub fn equals(
+        &self,
+        other: &RbSeries,
+        check_dtypes: bool,
+        check_names: bool,
+        null_equal: bool,
+    ) -> bool {
+        if check_dtypes && (self.series.borrow().dtype() != other.series.borrow().dtype()) {
+            return false;
+        }
+        if check_names && (self.series.borrow().name() != other.series.borrow().name()) {
+            return false;
+        }
+        if null_equal {
             self.series.borrow().equals_missing(&other.series.borrow())
         } else {
             self.series.borrow().equals(&other.series.borrow())
@@ -325,136 +346,6 @@ impl RbSeries {
         self.series.borrow().len()
     }
 
-    pub fn to_a(&self) -> Value {
-        let series = &self.series.borrow();
-
-        fn to_a_recursive(series: &Series) -> Value {
-            let rblist = match series.dtype() {
-                DataType::Boolean => RArray::from_iter(series.bool().unwrap()).into_value(),
-                DataType::UInt8 => RArray::from_iter(series.u8().unwrap()).into_value(),
-                DataType::UInt16 => RArray::from_iter(series.u16().unwrap()).into_value(),
-                DataType::UInt32 => RArray::from_iter(series.u32().unwrap()).into_value(),
-                DataType::UInt64 => RArray::from_iter(series.u64().unwrap()).into_value(),
-                DataType::Int8 => RArray::from_iter(series.i8().unwrap()).into_value(),
-                DataType::Int16 => RArray::from_iter(series.i16().unwrap()).into_value(),
-                DataType::Int32 => RArray::from_iter(series.i32().unwrap()).into_value(),
-                DataType::Int64 => RArray::from_iter(series.i64().unwrap()).into_value(),
-                DataType::Float32 => RArray::from_iter(series.f32().unwrap()).into_value(),
-                DataType::Float64 => RArray::from_iter(series.f64().unwrap()).into_value(),
-                DataType::Categorical(_, _) | DataType::Enum(_, _) => {
-                    RArray::from_iter(series.categorical().unwrap().iter_str()).into_value()
-                }
-                DataType::Object(_, _) => {
-                    let v = RArray::with_capacity(series.len());
-                    for i in 0..series.len() {
-                        let obj: Option<&ObjectValue> = series.get_object(i).map(|any| any.into());
-                        match obj {
-                            Some(val) => v.push(val.to_object()).unwrap(),
-                            None => v.push(qnil()).unwrap(),
-                        };
-                    }
-                    v.into_value()
-                }
-                DataType::List(_) => {
-                    let v = RArray::new();
-                    let ca = series.list().unwrap();
-                    for opt_s in unsafe { ca.amortized_iter() } {
-                        match opt_s {
-                            None => {
-                                v.push(qnil()).unwrap();
-                            }
-                            Some(s) => {
-                                let rblst = to_a_recursive(s.as_ref());
-                                v.push(rblst).unwrap();
-                            }
-                        }
-                    }
-                    v.into_value()
-                }
-                DataType::Array(_, _) => {
-                    let v = RArray::new();
-                    let ca = series.array().unwrap();
-                    for opt_s in ca.amortized_iter() {
-                        match opt_s {
-                            None => {
-                                v.push(qnil()).unwrap();
-                            }
-                            Some(s) => {
-                                let rblst = to_a_recursive(s.as_ref());
-                                v.push(rblst).unwrap();
-                            }
-                        }
-                    }
-                    v.into_value()
-                }
-                DataType::Date => {
-                    let ca = series.date().unwrap();
-                    return Wrap(ca).into_value();
-                }
-                DataType::Time => {
-                    let ca = series.time().unwrap();
-                    return Wrap(ca).into_value();
-                }
-                DataType::Datetime(_, _) => {
-                    let ca = series.datetime().unwrap();
-                    return Wrap(ca).into_value();
-                }
-                DataType::Decimal(_, _) => {
-                    let ca = series.decimal().unwrap();
-                    return Wrap(ca).into_value();
-                }
-                DataType::String => {
-                    let ca = series.str().unwrap();
-                    return Wrap(ca).into_value();
-                }
-                DataType::Struct(_) => {
-                    let ca = series.struct_().unwrap();
-                    return Wrap(ca).into_value();
-                }
-                DataType::Duration(_) => {
-                    let ca = series.duration().unwrap();
-                    return Wrap(ca).into_value();
-                }
-                DataType::Binary => {
-                    let ca = series.binary().unwrap();
-                    return Wrap(ca).into_value();
-                }
-                DataType::Null => {
-                    let null: Option<u8> = None;
-                    let n = series.len();
-                    let iter = std::iter::repeat(null).take(n);
-                    use std::iter::{Repeat, Take};
-                    struct NullIter {
-                        iter: Take<Repeat<Option<u8>>>,
-                        n: usize,
-                    }
-                    impl Iterator for NullIter {
-                        type Item = Option<u8>;
-
-                        fn next(&mut self) -> Option<Self::Item> {
-                            self.iter.next()
-                        }
-                        fn size_hint(&self) -> (usize, Option<usize>) {
-                            (self.n, Some(self.n))
-                        }
-                    }
-                    impl ExactSizeIterator for NullIter {}
-
-                    RArray::from_iter(NullIter { iter, n }).into_value()
-                }
-                DataType::Unknown => {
-                    panic!("to_a not implemented for unknown")
-                }
-                DataType::BinaryOffset => {
-                    unreachable!()
-                }
-            };
-            rblist
-        }
-
-        to_a_recursive(series)
-    }
-
     pub fn clone(&self) -> Self {
         RbSeries::new(self.series.borrow().clone())
     }

data/ext/polars/src/sql.rs

@@ -37,7 +37,9 @@ impl RbSQLContext {
     }
 
     pub fn register(&self, name: String, lf: &RbLazyFrame) {
-        self.context.borrow_mut().register(&name, lf.ldf.clone())
+        self.context
+            .borrow_mut()
+            .register(&name, lf.ldf.borrow().clone())
     }
 
     pub fn unregister(&self, name: String) {

data/lib/polars/array_expr.rb

@@ -358,7 +358,7 @@ module Polars
     #   # │ [7, 8, 9]     ┆ 4   ┆ null │
     #   # └───────────────┴─────┴──────┘
     def get(index, null_on_oob: true)
-      index = Utils.parse_as_expression(index)
+      index = Utils.parse_into_expression(index)
       Utils.wrap_expr(_rbexpr.arr_get(index, null_on_oob))
     end
 
@@ -446,7 +446,7 @@ module Polars
     #   # │ ["x", "y"]    ┆ _         ┆ x_y  │
     #   # └───────────────┴───────────┴──────┘
     def join(separator, ignore_nulls: true)
-      separator = Utils.parse_as_expression(separator, str_as_lit: true)
+      separator = Utils.parse_into_expression(separator, str_as_lit: true)
       Utils.wrap_expr(_rbexpr.arr_join(separator, ignore_nulls))
     end
 
@@ -502,7 +502,7 @@ module Polars
     #   # │ ["a", "c"]    ┆ true     │
     #   # └───────────────┴──────────┘
     def contains(item)
-      item = Utils.parse_as_expression(item, str_as_lit: true)
+      item = Utils.parse_into_expression(item, str_as_lit: true)
       Utils.wrap_expr(_rbexpr.arr_contains(item))
     end
 
@@ -530,7 +530,7 @@ module Polars
     #   # │ [2, 2]        ┆ 2              │
     #   # └───────────────┴────────────────┘
     def count_matches(element)
-      element = Utils.parse_as_expression(element, str_as_lit: true)
+      element = Utils.parse_into_expression(element, str_as_lit: true)
       Utils.wrap_expr(_rbexpr.arr_count_matches(element))
     end
   end

data/lib/polars/batched_csv_reader.rb

@@ -13,6 +13,7 @@ module Polars
       skip_rows: 0,
       dtypes: nil,
       null_values: nil,
+      missing_utf8_is_empty_string: false,
       ignore_errors: false,
       parse_dates: false,
       n_threads: nil,
@@ -28,10 +29,12 @@ module Polars
       sample_size: 1024,
       eol_char: "\n",
       new_columns: nil,
-      truncate_ragged_lines: false
+      raise_if_empty: true,
+      truncate_ragged_lines: false,
+      decimal_comma: false
     )
       if Utils.pathlike?(file)
-        path = Utils.normalise_filepath(file)
+        path = Utils.normalize_filepath(file)
       end
 
       dtype_list = nil
@@ -39,7 +42,7 @@ module Polars
       if !dtypes.nil?
         if dtypes.is_a?(Hash)
           dtype_list = []
-          dtypes.each do|k, v|
+          dtypes.each do |k, v|
             dtype_list << [k, Utils.rb_type_to_dtype(v)]
           end
         elsif dtypes.is_a?(::Array)
@@ -72,12 +75,15 @@ module Polars
         comment_char,
         quote_char,
         processed_null_values,
+        missing_utf8_is_empty_string,
         parse_dates,
         skip_rows_after_header,
-        Utils._prepare_row_count_args(row_count_name, row_count_offset),
+        Utils.parse_row_index_args(row_count_name, row_count_offset),
         sample_size,
         eol_char,
-        truncate_ragged_lines
+        raise_if_empty,
+        truncate_ragged_lines,
+        decimal_comma
       )
       self.new_columns = new_columns
     end