polars-df 0.9.0 → 0.11.0

data/ext/polars/src/series/export.rs

@@ -1,57 +1,139 @@
-use magnus::{class, prelude::*, Module, RArray, RClass, RModule, Value};
+use magnus::{value::qnil, IntoValue, RArray, Value};
 use polars_core::prelude::*;
 
-use crate::{raise_err, RbPolarsErr, RbResult, RbSeries};
+use crate::prelude::*;
+use crate::RbSeries;
 
 impl RbSeries {
-    /// For numeric types, this should only be called for Series with null types.
-    /// This will cast to floats so that `nil = NAN`
-    pub fn to_numo(&self) -> RbResult<Value> {
-        let s = &self.series.borrow();
-        match s.dtype() {
-            DataType::String => {
-                let ca = s.str().unwrap();
+    /// Convert this Series to a Ruby array.
+    /// This operation copies data.
+    pub fn to_a(&self) -> Value {
+        let series = &self.series.borrow();
 
-                // TODO make more efficient
-                let np_arr = RArray::from_iter(ca);
-                class::object()
-                    .const_get::<_, RModule>("Numo")?
-                    .const_get::<_, RClass>("RObject")?
-                    .funcall("cast", (np_arr,))
-            }
-            dt if dt.is_numeric() => {
-                if s.bit_repr_is_large() {
-                    let s = s.cast(&DataType::Float64).unwrap();
-                    let ca = s.f64().unwrap();
-                    // TODO make more efficient
-                    let np_arr = RArray::from_iter(ca.into_iter().map(|opt_v| match opt_v {
-                        Some(v) => v,
-                        None => f64::NAN,
-                    }));
-                    class::object()
-                        .const_get::<_, RModule>("Numo")?
-                        .const_get::<_, RClass>("DFloat")?
-                        .funcall("cast", (np_arr,))
-                } else {
-                    let s = s.cast(&DataType::Float32).unwrap();
-                    let ca = s.f32().unwrap();
-                    // TODO make more efficient
-                    let np_arr = RArray::from_iter(ca.into_iter().map(|opt_v| match opt_v {
-                        Some(v) => v,
-                        None => f32::NAN,
-                    }));
-                    class::object()
-                        .const_get::<_, RModule>("Numo")?
-                        .const_get::<_, RClass>("SFloat")?
-                        .funcall("cast", (np_arr,))
-                }
-            }
-            dt => {
-                raise_err!(
-                    format!("'to_numo' not supported for dtype: {dt:?}"),
-                    ComputeError
-                );
-            }
+        fn to_a_recursive(series: &Series) -> Value {
+            let rblist = match series.dtype() {
+                DataType::Boolean => RArray::from_iter(series.bool().unwrap()).into_value(),
+                DataType::UInt8 => RArray::from_iter(series.u8().unwrap()).into_value(),
+                DataType::UInt16 => RArray::from_iter(series.u16().unwrap()).into_value(),
+                DataType::UInt32 => RArray::from_iter(series.u32().unwrap()).into_value(),
+                DataType::UInt64 => RArray::from_iter(series.u64().unwrap()).into_value(),
+                DataType::Int8 => RArray::from_iter(series.i8().unwrap()).into_value(),
+                DataType::Int16 => RArray::from_iter(series.i16().unwrap()).into_value(),
+                DataType::Int32 => RArray::from_iter(series.i32().unwrap()).into_value(),
+                DataType::Int64 => RArray::from_iter(series.i64().unwrap()).into_value(),
+                DataType::Float32 => RArray::from_iter(series.f32().unwrap()).into_value(),
+                DataType::Float64 => RArray::from_iter(series.f64().unwrap()).into_value(),
+                DataType::Categorical(_, _) | DataType::Enum(_, _) => {
+                    RArray::from_iter(series.categorical().unwrap().iter_str()).into_value()
+                }
+                DataType::Object(_, _) => {
+                    let v = RArray::with_capacity(series.len());
+                    for i in 0..series.len() {
+                        let obj: Option<&ObjectValue> = series.get_object(i).map(|any| any.into());
+                        match obj {
+                            Some(val) => v.push(val.to_object()).unwrap(),
+                            None => v.push(qnil()).unwrap(),
+                        };
+                    }
+                    v.into_value()
+                }
+                DataType::List(_) => {
+                    let v = RArray::new();
+                    let ca = series.list().unwrap();
+                    for opt_s in unsafe { ca.amortized_iter() } {
+                        match opt_s {
+                            None => {
+                                v.push(qnil()).unwrap();
+                            }
+                            Some(s) => {
+                                let rblst = to_a_recursive(s.as_ref());
+                                v.push(rblst).unwrap();
+                            }
+                        }
+                    }
+                    v.into_value()
+                }
+                DataType::Array(_, _) => {
+                    let v = RArray::new();
+                    let ca = series.array().unwrap();
+                    for opt_s in ca.amortized_iter() {
+                        match opt_s {
+                            None => {
+                                v.push(qnil()).unwrap();
+                            }
+                            Some(s) => {
+                                let rblst = to_a_recursive(s.as_ref());
+                                v.push(rblst).unwrap();
+                            }
+                        }
+                    }
+                    v.into_value()
+                }
+                DataType::Date => {
+                    let ca = series.date().unwrap();
+                    return Wrap(ca).into_value();
+                }
+                DataType::Time => {
+                    let ca = series.time().unwrap();
+                    return Wrap(ca).into_value();
+                }
+                DataType::Datetime(_, _) => {
+                    let ca = series.datetime().unwrap();
+                    return Wrap(ca).into_value();
+                }
+                DataType::Decimal(_, _) => {
+                    let ca = series.decimal().unwrap();
+                    return Wrap(ca).into_value();
+                }
+                DataType::String => {
+                    let ca = series.str().unwrap();
+                    return Wrap(ca).into_value();
+                }
+                DataType::Struct(_) => {
+                    let ca = series.struct_().unwrap();
+                    return Wrap(ca).into_value();
+                }
+                DataType::Duration(_) => {
+                    let ca = series.duration().unwrap();
+                    return Wrap(ca).into_value();
+                }
+                DataType::Binary => {
+                    let ca = series.binary().unwrap();
+                    return Wrap(ca).into_value();
+                }
+                DataType::Null => {
+                    let null: Option<u8> = None;
+                    let n = series.len();
+                    let iter = std::iter::repeat(null).take(n);
+                    use std::iter::{Repeat, Take};
+                    struct NullIter {
+                        iter: Take<Repeat<Option<u8>>>,
+                        n: usize,
+                    }
+                    impl Iterator for NullIter {
+                        type Item = Option<u8>;
+
+                        fn next(&mut self) -> Option<Self::Item> {
+                            self.iter.next()
+                        }
+                        fn size_hint(&self) -> (usize, Option<usize>) {
+                            (self.n, Some(self.n))
+                        }
+                    }
+                    impl ExactSizeIterator for NullIter {}
+
+                    RArray::from_iter(NullIter { iter, n }).into_value()
+                }
+                DataType::Unknown(_) => {
+                    panic!("to_a not implemented for unknown")
+                }
+                DataType::BinaryOffset => {
+                    unreachable!()
+                }
+            };
+            rblist
         }
+
+        to_a_recursive(series)
     }
 }

data/ext/polars/src/series/mod.rs

@@ -5,7 +5,7 @@ mod construction;
 mod export;
 mod scatter;
 
-use magnus::{exception, prelude::*, value::qnil, Error, IntoValue, RArray, Value};
+use magnus::{exception, prelude::*, Error, IntoValue, RArray, Value};
 use polars::prelude::*;
 use polars::series::IsSorted;
 use std::cell::RefCell;
@@ -233,8 +233,18 @@ impl RbSeries {
         }
     }
 
-    pub fn sort(&self, descending: bool, nulls_last: bool) -> Self {
-        (self.series.borrow_mut().sort(descending, nulls_last)).into()
+    pub fn sort(&self, descending: bool, nulls_last: bool, multithreaded: bool) -> RbResult<Self> {
+        Ok(self
+            .series
+            .borrow_mut()
+            .sort(
+                SortOptions::default()
+                    .with_order_descending(descending)
+                    .with_nulls_last(nulls_last)
+                    .with_multithreaded(multithreaded),
+            )
+            .map_err(RbPolarsErr::from)?
+            .into())
     }
 
     pub fn value_counts(&self, sorted: bool) -> RbResult<RbDataFrame> {
@@ -315,136 +325,6 @@ impl RbSeries {
         self.series.borrow().len()
     }
 
-    pub fn to_a(&self) -> Value {
-        let series = &self.series.borrow();
-
-        fn to_a_recursive(series: &Series) -> Value {
-            let rblist = match series.dtype() {
-                DataType::Boolean => RArray::from_iter(series.bool().unwrap()).into_value(),
-                DataType::UInt8 => RArray::from_iter(series.u8().unwrap()).into_value(),
-                DataType::UInt16 => RArray::from_iter(series.u16().unwrap()).into_value(),
-                DataType::UInt32 => RArray::from_iter(series.u32().unwrap()).into_value(),
-                DataType::UInt64 => RArray::from_iter(series.u64().unwrap()).into_value(),
-                DataType::Int8 => RArray::from_iter(series.i8().unwrap()).into_value(),
-                DataType::Int16 => RArray::from_iter(series.i16().unwrap()).into_value(),
-                DataType::Int32 => RArray::from_iter(series.i32().unwrap()).into_value(),
-                DataType::Int64 => RArray::from_iter(series.i64().unwrap()).into_value(),
-                DataType::Float32 => RArray::from_iter(series.f32().unwrap()).into_value(),
-                DataType::Float64 => RArray::from_iter(series.f64().unwrap()).into_value(),
-                DataType::Categorical(_, _) | DataType::Enum(_, _) => {
-                    RArray::from_iter(series.categorical().unwrap().iter_str()).into_value()
-                }
-                DataType::Object(_, _) => {
-                    let v = RArray::with_capacity(series.len());
-                    for i in 0..series.len() {
-                        let obj: Option<&ObjectValue> = series.get_object(i).map(|any| any.into());
-                        match obj {
-                            Some(val) => v.push(val.to_object()).unwrap(),
-                            None => v.push(qnil()).unwrap(),
-                        };
-                    }
-                    v.into_value()
-                }
-                DataType::List(_) => {
-                    let v = RArray::new();
-                    let ca = series.list().unwrap();
-                    for opt_s in unsafe { ca.amortized_iter() } {
-                        match opt_s {
-                            None => {
-                                v.push(qnil()).unwrap();
-                            }
-                            Some(s) => {
-                                let rblst = to_a_recursive(s.as_ref());
-                                v.push(rblst).unwrap();
-                            }
-                        }
-                    }
-                    v.into_value()
-                }
-                DataType::Array(_, _) => {
-                    let v = RArray::new();
-                    let ca = series.array().unwrap();
-                    for opt_s in ca.amortized_iter() {
-                        match opt_s {
-                            None => {
-                                v.push(qnil()).unwrap();
-                            }
-                            Some(s) => {
-                                let rblst = to_a_recursive(s.as_ref());
-                                v.push(rblst).unwrap();
-                            }
-                        }
-                    }
-                    v.into_value()
-                }
-                DataType::Date => {
-                    let ca = series.date().unwrap();
-                    return Wrap(ca).into_value();
-                }
-                DataType::Time => {
-                    let ca = series.time().unwrap();
-                    return Wrap(ca).into_value();
-                }
-                DataType::Datetime(_, _) => {
-                    let ca = series.datetime().unwrap();
-                    return Wrap(ca).into_value();
-                }
-                DataType::Decimal(_, _) => {
-                    let ca = series.decimal().unwrap();
-                    return Wrap(ca).into_value();
-                }
-                DataType::String => {
-                    let ca = series.str().unwrap();
-                    return Wrap(ca).into_value();
-                }
-                DataType::Struct(_) => {
-                    let ca = series.struct_().unwrap();
-                    return Wrap(ca).into_value();
-                }
-                DataType::Duration(_) => {
-                    let ca = series.duration().unwrap();
-                    return Wrap(ca).into_value();
-                }
-                DataType::Binary => {
-                    let ca = series.binary().unwrap();
-                    return Wrap(ca).into_value();
-                }
-                DataType::Null => {
-                    let null: Option<u8> = None;
-                    let n = series.len();
-                    let iter = std::iter::repeat(null).take(n);
-                    use std::iter::{Repeat, Take};
-                    struct NullIter {
-                        iter: Take<Repeat<Option<u8>>>,
-                        n: usize,
-                    }
-                    impl Iterator for NullIter {
-                        type Item = Option<u8>;
-
-                        fn next(&mut self) -> Option<Self::Item> {
-                            self.iter.next()
-                        }
-                        fn size_hint(&self) -> (usize, Option<usize>) {
-                            (self.n, Some(self.n))
-                        }
-                    }
-                    impl ExactSizeIterator for NullIter {}
-
-                    RArray::from_iter(NullIter { iter, n }).into_value()
-                }
-                DataType::Unknown => {
-                    panic!("to_a not implemented for unknown")
-                }
-                DataType::BinaryOffset => {
-                    unreachable!()
-                }
-            };
-            rblist
-        }
-
-        to_a_recursive(series)
-    }
-
     pub fn clone(&self) -> Self {
         RbSeries::new(self.series.borrow().clone())
     }

data/lib/polars/array_expr.rb

@@ -333,6 +333,10 @@ module Polars
     #
     # @param index [Integer]
     #   Index to return per sub-array
+    # @param null_on_oob [Boolean]
+    #   Behavior if an index is out of bounds:
+    #   true -> set as null
+    #   false -> raise an error
     #
     # @return [Expr]
     #
@@ -353,9 +357,9 @@ module Polars
     #   # │ [4, 5, 6]     ┆ -2  ┆ 5    │
     #   # │ [7, 8, 9]     ┆ 4   ┆ null │
     #   # └───────────────┴─────┴──────┘
-    def get(index)
+    def get(index, null_on_oob: true)
       index = Utils.parse_as_expression(index)
-      Utils.wrap_expr(_rbexpr.arr_get(index))
+      Utils.wrap_expr(_rbexpr.arr_get(index, null_on_oob))
     end
 
     # Get the first value of the sub-arrays.

data/lib/polars/batched_csv_reader.rb

@@ -13,6 +13,7 @@ module Polars
       skip_rows: 0,
       dtypes: nil,
       null_values: nil,
+      missing_utf8_is_empty_string: false,
       ignore_errors: false,
       parse_dates: false,
       n_threads: nil,
@@ -27,10 +28,13 @@ module Polars
       row_count_offset: 0,
       sample_size: 1024,
       eol_char: "\n",
-      new_columns: nil
+      new_columns: nil,
+      raise_if_empty: true,
+      truncate_ragged_lines: false,
+      decimal_comma: false
     )
       if Utils.pathlike?(file)
-        path = Utils.normalise_filepath(file)
+        path = Utils.normalize_filepath(file)
       end
 
       dtype_list = nil
@@ -71,11 +75,15 @@ module Polars
         comment_char,
         quote_char,
         processed_null_values,
+        missing_utf8_is_empty_string,
         parse_dates,
         skip_rows_after_header,
         Utils._prepare_row_count_args(row_count_name, row_count_offset),
         sample_size,
-        eol_char
+        eol_char,
+        raise_if_empty,
+        truncate_ragged_lines,
+        decimal_comma
       )
       self.new_columns = new_columns
     end

data/lib/polars/convert.rb

@@ -27,7 +27,12 @@ module Polars
     #   # │ 2   ┆ 4   │
     #   # └─────┴─────┘
     def from_hash(data, schema: nil, columns: nil)
-      DataFrame._from_hash(data, schema: schema || columns)
+      Utils.wrap_df(
+        DataFrame.hash_to_rbdf(
+          data,
+          schema: schema || columns
+        )
+      )
     end
 
     # Construct a DataFrame from a sequence of dictionaries. This operation clones data.