polars-df 0.17.0 → 0.18.0

data/README.md

@@ -1,6 +1,6 @@
 # Ruby Polars
 
-:fire: Blazingly fast DataFrames for Ruby, powered by [Polars](https://github.com/pola-rs/polars)
+🔥 Blazingly fast DataFrames for Ruby, powered by [Polars](https://github.com/pola-rs/polars)
 
 [![Build Status](https://github.com/ankane/ruby-polars/actions/workflows/build.yml/badge.svg)](https://github.com/ankane/ruby-polars/actions)
 
@@ -448,7 +448,7 @@ df.group_by("c").plot("a", "b", stacked: true)
 
 ## History
 
-View the [changelog](CHANGELOG.md)
+View the [changelog](https://github.com/ankane/ruby-polars/blob/master/CHANGELOG.md)
 
 ## Contributing
 

data/ext/polars/Cargo.toml

@@ -1,10 +1,10 @@
 [package]
 name = "polars"
-version = "0.17.0"
+version = "0.18.0"
 license = "MIT"
 authors = ["Andrew Kane <andrew@ankane.org>"]
 edition = "2021"
-rust-version = "1.80.0"
+rust-version = "1.85.0"
 publish = false
 
 [lib]
@@ -12,21 +12,22 @@ crate-type = ["cdylib"]
 
 [dependencies]
 ahash = "0.8"
-arrow = { package = "polars-arrow", version = "=0.46.0" }
+arrow = { package = "polars-arrow", version = "=0.47.1" }
 bytes = "1"
 chrono = "0.4"
 either = "1.8"
 magnus = "0.7"
-polars-core = "=0.46.0"
-polars-plan = "=0.46.0"
-polars-parquet = "=0.46.0"
-polars-utils = "=0.46.0"
+num-traits = "0.2"
+polars-core = "=0.47.1"
+polars-plan = "=0.47.1"
+polars-parquet = "=0.47.1"
+polars-utils = "=0.47.1"
 rayon = "1.9"
 regex = "1"
 serde_json = "1"
 
 [dependencies.polars]
-version = "=0.46.0"
+version = "=0.47.1"
 features = [
     "abs",
     "approx_unique",
@@ -63,6 +64,7 @@ features = [
     "gcp",
     "http",
     "interpolate",
+    "interpolate_by",
     "ipc",
     "ipc_streaming",
     "is_between",

data/ext/polars/src/conversion/any_value.rs

@@ -17,7 +17,7 @@ impl IntoValue for Wrap<AnyValue<'_>> {
     }
 }
 
-impl<'s> TryConvert for Wrap<AnyValue<'s>> {
+impl TryConvert for Wrap<AnyValue<'_>> {
     fn try_convert(ob: Value) -> RbResult<Self> {
         rb_object_to_any_value(ob, true).map(Wrap)
     }

data/ext/polars/src/conversion/mod.rs

@@ -21,7 +21,7 @@ use polars::prelude::*;
 use polars::series::ops::NullBehavior;
 use polars_core::utils::arrow::array::Array;
 use polars_core::utils::materialize_dyn_int;
-use polars_plan::plans::ScanSources;
+use polars_plan::dsl::ScanSources;
 use polars_utils::mmap::MemSlice;
 use polars_utils::total_ord::{TotalEq, TotalHash};
 
@@ -219,7 +219,7 @@ impl IntoValue for Wrap<DataType> {
                     .funcall::<_, _, Value>("new", (tu.to_ascii(),))
                     .unwrap()
             }
-            DataType::Object(_, _) => {
+            DataType::Object(_) => {
                 let class = pl.const_get::<_, Value>("Object").unwrap();
                 class.funcall("new", ()).unwrap()
             }
@@ -332,7 +332,7 @@ impl TryConvert for Wrap<DataType> {
                 "Polars::Array" => DataType::Array(Box::new(DataType::Null), 0),
                 "Polars::Struct" => DataType::Struct(vec![]),
                 "Polars::Null" => DataType::Null,
-                "Polars::Object" => DataType::Object(OBJECT_NAME, None),
+                "Polars::Object" => DataType::Object(OBJECT_NAME),
                 "Polars::Unknown" => DataType::Unknown(Default::default()),
                 dt => {
                     return Err(RbValueError::new_err(format!(
@@ -408,7 +408,7 @@ impl TryConvert for Wrap<DataType> {
                     DataType::Struct(fields)
                 }
                 "Polars::Null" => DataType::Null,
-                "Object" => DataType::Object(OBJECT_NAME, None),
+                "Object" => DataType::Object(OBJECT_NAME),
                 "Polars::Unknown" => DataType::Unknown(Default::default()),
                 dt => {
                     return Err(RbTypeError::new_err(format!(
@@ -437,7 +437,7 @@ impl TryConvert for Wrap<DataType> {
                 "time" => DataType::Time,
                 "dur" => DataType::Duration(TimeUnit::Microseconds),
                 "f64" => DataType::Float64,
-                "obj" => DataType::Object(OBJECT_NAME, None),
+                "obj" => DataType::Object(OBJECT_NAME),
                 "list" => DataType::List(Box::new(DataType::Boolean)),
                 "null" => DataType::Null,
                 "unk" => DataType::Unknown(Default::default()),
@@ -761,6 +761,21 @@ impl TryConvert for Wrap<ClosedWindow> {
     }
 }
 
+impl TryConvert for Wrap<RoundMode> {
+    fn try_convert(ob: Value) -> RbResult<Self> {
+        let parsed = match String::try_convert(ob)?.as_str() {
+            "half_to_even" => RoundMode::HalfToEven,
+            "half_away_from_zero" => RoundMode::HalfAwayFromZero,
+            v => {
+                return Err(RbValueError::new_err(format!(
+                    "`mode` must be one of {{'half_to_even', 'half_away_from_zero'}}, got {v}",
+                )));
+            }
+        };
+        Ok(Wrap(parsed))
+    }
+}
+
 impl TryConvert for Wrap<CsvEncoding> {
     fn try_convert(ob: Value) -> RbResult<Self> {
         let parsed = match String::try_convert(ob)?.as_str() {
@@ -1065,6 +1080,24 @@ impl TryConvert for Wrap<JoinValidation> {
     }
 }
 
+impl TryConvert for Wrap<MaintainOrderJoin> {
+    fn try_convert(ob: Value) -> RbResult<Self> {
+        let parsed = match String::try_convert(ob)?.as_str() {
+            "none" => MaintainOrderJoin::None,
+            "left" => MaintainOrderJoin::Left,
+            "right" => MaintainOrderJoin::Right,
+            "left_right" => MaintainOrderJoin::LeftRight,
+            "right_left" => MaintainOrderJoin::RightLeft,
+            v => {
+                return Err(RbValueError::new_err(format!(
+                    "`maintain_order` must be one of {{'none', 'left', 'right', 'left_right', 'right_left'}}, got {v}",
+                )))
+            },
+        };
+        Ok(Wrap(parsed))
+    }
+}
+
 impl TryConvert for Wrap<QuoteStyle> {
     fn try_convert(ob: Value) -> RbResult<Self> {
         let parsed = match String::try_convert(ob)?.as_str() {

data/ext/polars/src/dataframe/export.rs

@@ -18,7 +18,7 @@ impl RbDataFrame {
                 .get_columns()
                 .iter()
                 .map(|s| match s.dtype() {
-                    DataType::Object(_, _) => {
+                    DataType::Object(_) => {
                         let obj: Option<&ObjectValue> = s.get_object(idx).map(|any| any.into());
                         obj.unwrap().to_value()
                     }
@@ -37,7 +37,7 @@ impl RbDataFrame {
                     .get_columns()
                     .iter()
                     .map(|s| match s.dtype() {
-                        DataType::Object(_, _) => {
+                        DataType::Object(_) => {
                             let obj: Option<&ObjectValue> = s.get_object(idx).map(|any| any.into());
                             obj.unwrap().to_value()
                         }

data/ext/polars/src/dataframe/general.rs

@@ -1,3 +1,5 @@
+use std::hash::BuildHasher;
+
 use either::Either;
 use magnus::{prelude::*, typed_data::Obj, IntoValue, RArray, Value};
 use polars::prelude::pivot::{pivot, pivot_stable};
@@ -295,11 +297,11 @@ impl RbDataFrame {
         Ok(())
     }
 
-    pub fn slice(&self, offset: usize, length: Option<usize>) -> Self {
-        let df = self.df.borrow().slice(
-            offset as i64,
-            length.unwrap_or_else(|| self.df.borrow().height()),
-        );
+    pub fn slice(&self, offset: i64, length: Option<usize>) -> Self {
+        let df = self
+            .df
+            .borrow()
+            .slice(offset, length.unwrap_or_else(|| self.df.borrow().height()));
         df.into()
     }
 
@@ -494,7 +496,8 @@ impl RbDataFrame {
     }
 
     pub fn hash_rows(&self, k0: u64, k1: u64, k2: u64, k3: u64) -> RbResult<RbSeries> {
-        let hb = ahash::RandomState::with_seeds(k0, k1, k2, k3);
+        let seed = PlFixedStateQuality::default().hash_one((k0, k1, k2, k3));
+        let hb = PlSeedableRandomStateQuality::seed_from_u64(seed);
         let hash = self
             .df
             .borrow_mut()

data/ext/polars/src/expr/general.rs

@@ -371,14 +371,6 @@ impl RbExpr {
             .into())
     }
 
-    pub fn backward_fill(&self, limit: FillNullLimit) -> Self {
-        self.inner.clone().backward_fill(limit).into()
-    }
-
-    pub fn forward_fill(&self, limit: FillNullLimit) -> Self {
-        self.inner.clone().forward_fill(limit).into()
-    }
-
     pub fn shift(&self, n: &Self, fill_value: Option<&Self>) -> Self {
         let expr = self.inner.clone();
         let out = match fill_value {
@@ -497,8 +489,8 @@ impl RbExpr {
             .into()
     }
 
-    pub fn round(&self, decimals: u32) -> Self {
-        self.inner.clone().round(decimals).into()
+    pub fn round(&self, decimals: u32, mode: Wrap<RoundMode>) -> Self {
+        self.inner.clone().round(decimals, mode.0).into()
     }
 
     pub fn floor(&self) -> Self {
@@ -597,8 +589,11 @@ impl RbExpr {
         self.inner.clone().or(expr.inner.clone()).into()
     }
 
-    pub fn is_in(&self, expr: &Self) -> Self {
-        self.inner.clone().is_in(expr.inner.clone()).into()
+    pub fn is_in(&self, expr: &Self, nulls_equal: bool) -> Self {
+        self.inner
+            .clone()
+            .is_in(expr.inner.clone(), nulls_equal)
+            .into()
     }
 
     pub fn repeat_by(&self, by: &Self) -> Self {
@@ -671,6 +666,10 @@ impl RbExpr {
         self.inner.clone().interpolate(method.0).into()
     }
 
+    pub fn interpolate_by(&self, by: &Self) -> Self {
+        self.inner.clone().interpolate_by(by.inner.clone()).into()
+    }
+
     pub fn lower_bound(&self) -> Self {
         self.inner.clone().lower_bound().into()
     }
@@ -694,8 +693,11 @@ impl RbExpr {
         self.inner.clone().rank(options, seed).into()
     }
 
-    pub fn diff(&self, n: i64, null_behavior: Wrap<NullBehavior>) -> Self {
-        self.inner.clone().diff(n, null_behavior.0).into()
+    pub fn diff(&self, n: &Self, null_behavior: Wrap<NullBehavior>) -> Self {
+        self.inner
+            .clone()
+            .diff(n.inner.clone(), null_behavior.0)
+            .into()
     }
 
     pub fn pct_change(&self, n: &Self) -> Self {

data/ext/polars/src/expr/rolling.rs

@@ -319,7 +319,22 @@ impl RbExpr {
             .into()
     }
 
-    pub fn rolling_skew(&self, window_size: usize, bias: bool) -> Self {
-        self.inner.clone().rolling_skew(window_size, bias).into()
+    pub fn rolling_skew(
+        &self,
+        window_size: usize,
+        bias: bool,
+        min_periods: Option<usize>,
+        center: bool,
+    ) -> Self {
+        let min_periods = min_periods.unwrap_or(window_size);
+        let options = RollingOptionsFixedWindow {
+            window_size,
+            weights: None,
+            min_periods,
+            center,
+            fn_params: Some(RollingFnParams::Skew { bias }),
+        };
+
+        self.inner.clone().rolling_skew(options).into()
     }
 }

data/ext/polars/src/file.rs

@@ -3,9 +3,12 @@ use std::io;
 use std::io::{Cursor, Read, Seek, SeekFrom, Write};
 use std::path::PathBuf;
 
-use magnus::{exception, prelude::*, Error, RString, Value};
+use magnus::{exception, prelude::*, value::Opaque, Error, RString, Ruby, Value};
 use polars::io::cloud::CloudOptions;
 use polars::io::mmap::MmapBytesReader;
+use polars::prelude::file::DynWriteable;
+use polars::prelude::sync_on_close::SyncOnCloseType;
+use polars_utils::file::ClosableFile;
 use polars_utils::mmap::MemSlice;
 
 use crate::error::RbPolarsErr;
@@ -14,7 +17,22 @@ use crate::RbResult;
 
 #[derive(Clone)]
 pub struct RbFileLikeObject {
-    inner: Value,
+    inner: Opaque<Value>,
+}
+
+impl DynWriteable for RbFileLikeObject {
+    fn as_dyn_write(&self) -> &(dyn io::Write + Send + 'static) {
+        self as _
+    }
+    fn as_mut_dyn_write(&mut self) -> &mut (dyn io::Write + Send + 'static) {
+        self as _
+    }
+    fn close(self: Box<Self>) -> io::Result<()> {
+        Ok(())
+    }
+    fn sync_on_close(&mut self, _sync_on_close: SyncOnCloseType) -> io::Result<()> {
+        Ok(())
+    }
 }
 
 /// Wraps a `Value`, and implements read, seek, and write for it.
@@ -23,7 +41,9 @@ impl RbFileLikeObject {
     /// To assert the object has the required methods methods,
     /// instantiate it with `RbFileLikeObject::require`
     pub fn new(object: Value) -> Self {
-        RbFileLikeObject { inner: object }
+        RbFileLikeObject {
+            inner: object.into(),
+        }
     }
 
     pub fn as_bytes(&self) -> bytes::Bytes {
@@ -31,8 +51,9 @@ impl RbFileLikeObject {
     }
 
     pub fn as_file_buffer(&self) -> Cursor<Vec<u8>> {
-        let bytes = self
-            .inner
+        let bytes = Ruby::get()
+            .unwrap()
+            .get_inner(self.inner)
             .funcall::<_, _, RString>("read", ())
             .expect("no read method found");
 
@@ -77,8 +98,9 @@ fn rberr_to_io_err(e: Error) -> io::Error {
 
 impl Read for RbFileLikeObject {
     fn read(&mut self, mut buf: &mut [u8]) -> Result<usize, io::Error> {
-        let bytes = self
-            .inner
+        let bytes = Ruby::get()
+            .unwrap()
+            .get_inner(self.inner)
             .funcall::<_, _, RString>("read", (buf.len(),))
             .map_err(rberr_to_io_err)?;
 
@@ -92,8 +114,9 @@ impl Write for RbFileLikeObject {
     fn write(&mut self, buf: &[u8]) -> Result<usize, io::Error> {
         let rbbytes = RString::from_slice(buf);
 
-        let number_bytes_written = self
-            .inner
+        let number_bytes_written = Ruby::get()
+            .unwrap()
+            .get_inner(self.inner)
             .funcall::<_, _, usize>("write", (rbbytes,))
             .map_err(rberr_to_io_err)?;
 
@@ -101,7 +124,9 @@ impl Write for RbFileLikeObject {
     }
 
     fn flush(&mut self) -> Result<(), io::Error> {
-        self.inner
+        Ruby::get()
+            .unwrap()
+            .get_inner(self.inner)
             .funcall::<_, _, Value>("flush", ())
             .map_err(rberr_to_io_err)?;
 
@@ -117,8 +142,9 @@ impl Seek for RbFileLikeObject {
             SeekFrom::End(i) => (2, i),
         };
 
-        let new_position = self
-            .inner
+        let new_position = Ruby::get()
+            .unwrap()
+            .get_inner(self.inner)
             .funcall("seek", (offset, whence))
             .map_err(rberr_to_io_err)?;
 
@@ -129,11 +155,12 @@ impl Seek for RbFileLikeObject {
 pub trait FileLike: Read + Write + Seek {}
 
 impl FileLike for File {}
+impl FileLike for ClosableFile {}
 impl FileLike for RbFileLikeObject {}
 
 pub enum EitherRustRubyFile {
     Rb(RbFileLikeObject),
-    Rust(File),
+    Rust(ClosableFile),
 }
 
 impl EitherRustRubyFile {
@@ -144,6 +171,13 @@ impl EitherRustRubyFile {
         }
     }
 
+    pub(crate) fn into_writeable(self) -> Box<dyn DynWriteable> {
+        match self {
+            Self::Rb(f) => Box::new(f),
+            Self::Rust(f) => Box::new(f),
+        }
+    }
+
     pub fn into_dyn_writeable(self) -> Box<dyn Write> {
         match self {
             EitherRustRubyFile::Rb(f) => Box::new(f),
@@ -159,6 +193,14 @@ pub enum RubyScanSourceInput {
     File(File),
 }
 
+pub(crate) fn try_get_rbfile(
+    rb_f: Value,
+    write: bool,
+) -> RbResult<(EitherRustRubyFile, Option<PathBuf>)> {
+    let f = RbFileLikeObject::with_requirements(rb_f, !write, write, !write)?;
+    Ok((EitherRustRubyFile::Rb(f), None))
+}
+
 pub fn get_ruby_scan_source_input(rb_f: Value, write: bool) -> RbResult<RubyScanSourceInput> {
     if let Ok(file_path) = PathBuf::try_convert(rb_f) {
         // TODO resolve_homedir
@@ -184,7 +226,7 @@ pub fn get_either_file(rb_f: Value, truncate: bool) -> RbResult<EitherRustRubyFi
         } else {
             polars_utils::open_file(&file_path).map_err(RbPolarsErr::from)?
         };
-        Ok(EitherRustRubyFile::Rust(f))
+        Ok(EitherRustRubyFile::Rust(f.into()))
     } else {
         let f = RbFileLikeObject::with_requirements(rb_f, !truncate, truncate, !truncate)?;
         Ok(EitherRustRubyFile::Rb(f))

data/ext/polars/src/functions/lazy.rs

@@ -205,6 +205,21 @@ pub fn concat_lf_diagonal(
     Ok(lf.into())
 }
 
+pub fn concat_lf_horizontal(lfs: RArray, parallel: bool) -> RbResult<RbLazyFrame> {
+    let iter = lfs.into_iter();
+
+    let lfs = iter.map(get_lf).collect::<RbResult<Vec<_>>>()?;
+
+    let args = UnionArgs {
+        rechunk: false, // No need to rechunk with horizontal concatenation
+        parallel,
+        to_supertypes: false,
+        ..Default::default()
+    };
+    let lf = dsl::functions::concat_lf_horizontal(lfs, args).map_err(RbPolarsErr::from)?;
+    Ok(lf.into())
+}
+
 pub fn dtype_cols(dtypes: RArray) -> RbResult<RbExpr> {
     let dtypes = dtypes
         .into_iter()
@@ -263,7 +278,13 @@ pub fn first() -> RbExpr {
     dsl::first().into()
 }
 
-pub fn fold(acc: &RbExpr, lambda: Value, exprs: RArray) -> RbResult<RbExpr> {
+pub fn fold(
+    acc: &RbExpr,
+    lambda: Value,
+    exprs: RArray,
+    returns_scalar: bool,
+    return_dtype: Option<Wrap<DataType>>,
+) -> RbResult<RbExpr> {
     let exprs = rb_exprs_to_exprs(exprs)?;
     let lambda = Opaque::from(lambda);
 
@@ -275,7 +296,14 @@ pub fn fold(acc: &RbExpr, lambda: Value, exprs: RArray) -> RbResult<RbExpr> {
         )
         .map(|v| v.map(Column::from))
     };
-    Ok(dsl::fold_exprs(acc.inner.clone(), func, exprs).into())
+    Ok(dsl::fold_exprs(
+        acc.inner.clone(),
+        func,
+        exprs,
+        returns_scalar,
+        return_dtype.map(|w| w.0),
+    )
+    .into())
 }
 
 pub fn last() -> RbExpr {

data/ext/polars/src/interop/numo/mod.rs

@@ -1,2 +1,3 @@
+pub mod numo_rs;
 pub mod to_numo_df;
 pub mod to_numo_series;

data/ext/polars/src/interop/numo/numo_rs.rs

@@ -0,0 +1,52 @@
+use magnus::{class, prelude::*, IntoValue, Module, RArray, RClass, RModule, Value};
+
+use crate::RbResult;
+
+pub trait Element: IntoValue {
+    fn class_name() -> &'static str;
+}
+
+macro_rules! create_element {
+    ($type:ty, $name:expr) => {
+        impl Element for $type {
+            fn class_name() -> &'static str {
+                $name
+            }
+        }
+    };
+}
+
+create_element!(i8, "Int8");
+create_element!(i16, "Int16");
+create_element!(i32, "Int32");
+create_element!(i64, "Int64");
+create_element!(u8, "UInt8");
+create_element!(u16, "UInt16");
+create_element!(u32, "UInt32");
+create_element!(u64, "UInt64");
+create_element!(f32, "SFloat");
+create_element!(f64, "DFloat");
+create_element!(bool, "Bit");
+
+impl<T> Element for Option<T>
+where
+    Option<T>: IntoValue,
+{
+    fn class_name() -> &'static str {
+        "RObject"
+    }
+}
+
+pub struct RbArray1<T>(T);
+
+impl<T: Element> RbArray1<T> {
+    pub fn from_iter<I>(values: I) -> RbResult<Value>
+    where
+        I: IntoIterator<Item = T>,
+    {
+        class::object()
+            .const_get::<_, RModule>("Numo")?
+            .const_get::<_, RClass>(T::class_name())?
+            .funcall("cast", (RArray::from_iter(values),))
+    }
+}