polars-df 0.17.0 → 0.17.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: de79849ac873f2360b7768ae422cea5f16b0103b910e4e80b88310365cf4a060
-  data.tar.gz: a479ccd22196787a6e19d9ab715a2ce7471dcba6c3fe9292c44c33958b2cc1fd
+  metadata.gz: b2df0748eeb77638cc93667d89231cfd03510a856a84a9e01a9cdc97b0b6ae57
+  data.tar.gz: fabc2dedf44a22a4e657be770a1cc34ce2f8001e9f4cafbbd4346eabc4d1a4ec
 SHA512:
-  metadata.gz: d19d168b4a3fe616c4076e49e60d8cc6da05fa3df2042c9d6c8fc475daec3335488fa623908229509271430957ee414a852f571b9e750962319c3623493b66ca
-  data.tar.gz: 32f97daa9f7743364e3ae0bc8087e60f4f64979a0163218a912b5c5d6a5917ad3a9e7ab98c0b510cca46d8582e8b2bd16ee5692eba4f6982116f9aebde40fa4f
+  metadata.gz: df7d294861508ccbc11a0d5ebb90880e897a6bc2f91396b1e08839c1fe019386edf0f0c80a9c6db256ca1d917dd8253bee78d353bf692a28e76b20bd048cea2b
+  data.tar.gz: be4eb782c03b0b41f42b0ce3aad8a64f388a4c79444be282a5802499e88f23e8e8efeea85f6ff6f684f9c004502b9be44aba33688e54b70ca4dd0a8596c0960a

data/CHANGELOG.md

@@ -1,3 +1,15 @@
+## 0.17.1 (2025-04-12)
+
+- Added support for horizontal concatenation of `LazyFrame`s
+- Added `diagonal_relaxed` and `align` strategies to `concat` method
+- Added `interpolate_by` method to `Series` and `Expr`
+- Added `iter_columns` and `iter_slices` methods to `DataFrame`
+- Added `maintain_order` option to `join` method
+- Added `collect_schema` method to `DataFrame`
+- Added `write_database` method to `DataFrame` (experimental)
+- Fixed error with `to_numo` method for `Boolean` series with null values
+- Fixed error with `slice` method and negative offset
+
 ## 0.17.0 (2025-01-28)
 
 - Updated Polars to 0.46.0

data/Cargo.lock

@@ -1645,7 +1645,7 @@ dependencies = [
 
 [[package]]
 name = "polars"
-version = "0.17.0"
+version = "0.17.1"
 dependencies = [
  "ahash",
  "bytes",
@@ -1654,6 +1654,7 @@ dependencies = [
  "jemallocator",
  "magnus",
  "mimalloc",
+ "num-traits",
  "polars 0.46.0",
  "polars-arrow",
  "polars-core",
@@ -2545,15 +2546,14 @@ dependencies = [
 
 [[package]]
 name = "ring"
-version = "0.17.8"
+version = "0.17.13"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c17fa4cb658e3583423e915b9f3acc01cceaee1860e33d59ebae66adc3a2dc0d"
+checksum = "70ac5d832aa16abd7d1def883a8545280c20a60f523a370aa3a9617c2b8550ee"
 dependencies = [
  "cc",
  "cfg-if",
  "getrandom",
  "libc",
- "spin",
  "untrusted",
  "windows-sys 0.52.0",
 ]
@@ -2862,12 +2862,6 @@ dependencies = [
  "windows-sys 0.52.0",
 ]
 
-[[package]]
-name = "spin"
-version = "0.9.8"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67"
-
 [[package]]
 name = "sqlparser"
 version = "0.53.0"

data/README.md

@@ -1,6 +1,6 @@
 # Ruby Polars
 
-:fire: Blazingly fast DataFrames for Ruby, powered by [Polars](https://github.com/pola-rs/polars)
+🔥 Blazingly fast DataFrames for Ruby, powered by [Polars](https://github.com/pola-rs/polars)
 
 [![Build Status](https://github.com/ankane/ruby-polars/actions/workflows/build.yml/badge.svg)](https://github.com/ankane/ruby-polars/actions)
 
@@ -448,7 +448,7 @@ df.group_by("c").plot("a", "b", stacked: true)
 
 ## History
 
-View the [changelog](CHANGELOG.md)
+View the [changelog](https://github.com/ankane/ruby-polars/blob/master/CHANGELOG.md)
 
 ## Contributing
 

data/ext/polars/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "polars"
-version = "0.17.0"
+version = "0.17.1"
 license = "MIT"
 authors = ["Andrew Kane <andrew@ankane.org>"]
 edition = "2021"
@@ -17,6 +17,7 @@ bytes = "1"
 chrono = "0.4"
 either = "1.8"
 magnus = "0.7"
+num-traits = "0.2"
 polars-core = "=0.46.0"
 polars-plan = "=0.46.0"
 polars-parquet = "=0.46.0"
@@ -63,6 +64,7 @@ features = [
     "gcp",
     "http",
     "interpolate",
+    "interpolate_by",
     "ipc",
     "ipc_streaming",
     "is_between",

data/ext/polars/src/conversion/any_value.rs

@@ -17,7 +17,7 @@ impl IntoValue for Wrap<AnyValue<'_>> {
     }
 }
 
-impl<'s> TryConvert for Wrap<AnyValue<'s>> {
+impl TryConvert for Wrap<AnyValue<'_>> {
     fn try_convert(ob: Value) -> RbResult<Self> {
         rb_object_to_any_value(ob, true).map(Wrap)
     }

data/ext/polars/src/conversion/mod.rs

@@ -1065,6 +1065,24 @@ impl TryConvert for Wrap<JoinValidation> {
     }
 }
 
+impl TryConvert for Wrap<MaintainOrderJoin> {
+    fn try_convert(ob: Value) -> RbResult<Self> {
+        let parsed = match String::try_convert(ob)?.as_str() {
+            "none" => MaintainOrderJoin::None,
+            "left" => MaintainOrderJoin::Left,
+            "right" => MaintainOrderJoin::Right,
+            "left_right" => MaintainOrderJoin::LeftRight,
+            "right_left" => MaintainOrderJoin::RightLeft,
+            v => {
+                return Err(RbValueError::new_err(format!(
+                    "`maintain_order` must be one of {{'none', 'left', 'right', 'left_right', 'right_left'}}, got {v}",
+                )))
+            },
+        };
+        Ok(Wrap(parsed))
+    }
+}
+
 impl TryConvert for Wrap<QuoteStyle> {
     fn try_convert(ob: Value) -> RbResult<Self> {
         let parsed = match String::try_convert(ob)?.as_str() {

data/ext/polars/src/dataframe/general.rs

@@ -295,11 +295,11 @@ impl RbDataFrame {
         Ok(())
     }
 
-    pub fn slice(&self, offset: usize, length: Option<usize>) -> Self {
-        let df = self.df.borrow().slice(
-            offset as i64,
-            length.unwrap_or_else(|| self.df.borrow().height()),
-        );
+    pub fn slice(&self, offset: i64, length: Option<usize>) -> Self {
+        let df = self
+            .df
+            .borrow()
+            .slice(offset, length.unwrap_or_else(|| self.df.borrow().height()));
         df.into()
     }
 

data/ext/polars/src/expr/general.rs

@@ -671,6 +671,10 @@ impl RbExpr {
         self.inner.clone().interpolate(method.0).into()
     }
 
+    pub fn interpolate_by(&self, by: &Self) -> Self {
+        self.inner.clone().interpolate_by(by.inner.clone()).into()
+    }
+
     pub fn lower_bound(&self) -> Self {
         self.inner.clone().lower_bound().into()
     }

data/ext/polars/src/functions/lazy.rs

@@ -205,6 +205,21 @@ pub fn concat_lf_diagonal(
     Ok(lf.into())
 }
 
+pub fn concat_lf_horizontal(lfs: RArray, parallel: bool) -> RbResult<RbLazyFrame> {
+    let iter = lfs.into_iter();
+
+    let lfs = iter.map(get_lf).collect::<RbResult<Vec<_>>>()?;
+
+    let args = UnionArgs {
+        rechunk: false, // No need to rechunk with horizontal concatenation
+        parallel,
+        to_supertypes: false,
+        ..Default::default()
+    };
+    let lf = dsl::functions::concat_lf_horizontal(lfs, args).map_err(RbPolarsErr::from)?;
+    Ok(lf.into())
+}
+
 pub fn dtype_cols(dtypes: RArray) -> RbResult<RbExpr> {
     let dtypes = dtypes
         .into_iter()

data/ext/polars/src/interop/numo/mod.rs

@@ -1,2 +1,3 @@
+pub mod numo_rs;
 pub mod to_numo_df;
 pub mod to_numo_series;

data/ext/polars/src/interop/numo/numo_rs.rs

@@ -0,0 +1,52 @@
+use magnus::{class, prelude::*, IntoValue, Module, RArray, RClass, RModule, Value};
+
+use crate::RbResult;
+
+pub trait Element: IntoValue {
+    fn class_name() -> &'static str;
+}
+
+macro_rules! create_element {
+    ($type:ty, $name:expr) => {
+        impl Element for $type {
+            fn class_name() -> &'static str {
+                $name
+            }
+        }
+    };
+}
+
+create_element!(i8, "Int8");
+create_element!(i16, "Int16");
+create_element!(i32, "Int32");
+create_element!(i64, "Int64");
+create_element!(u8, "UInt8");
+create_element!(u16, "UInt16");
+create_element!(u32, "UInt32");
+create_element!(u64, "UInt64");
+create_element!(f32, "SFloat");
+create_element!(f64, "DFloat");
+create_element!(bool, "Bit");
+
+impl<T> Element for Option<T>
+where
+    Option<T>: IntoValue,
+{
+    fn class_name() -> &'static str {
+        "RObject"
+    }
+}
+
+pub struct RbArray1<T>(T);
+
+impl<T: Element> RbArray1<T> {
+    pub fn from_iter<I>(values: I) -> RbResult<Value>
+    where
+        I: IntoIterator<Item = T>,
+    {
+        class::object()
+            .const_get::<_, RModule>("Numo")?
+            .const_get::<_, RClass>(T::class_name())?
+            .funcall("cast", (RArray::from_iter(values),))
+    }
+}

data/ext/polars/src/interop/numo/to_numo_series.rs

@@ -1,61 +1,82 @@
-use magnus::{class, prelude::*, Module, RArray, RClass, RModule, Value};
-use polars::series::BitRepr;
+use magnus::Value;
+use num_traits::{Float, NumCast};
 use polars_core::prelude::*;
 
+use super::numo_rs::{Element, RbArray1};
 use crate::error::RbPolarsErr;
 use crate::raise_err;
 use crate::series::RbSeries;
 use crate::RbResult;
 
 impl RbSeries {
-    /// For numeric types, this should only be called for Series with null types.
-    /// This will cast to floats so that `nil = NAN`
+    /// Convert this Series to a Numo array.
     pub fn to_numo(&self) -> RbResult<Value> {
-        let s = &self.series.borrow();
-        match s.dtype() {
-            DataType::String => {
-                let ca = s.str().unwrap();
+        series_to_numo(&self.series.borrow())
+    }
+}
+
+/// Convert a Series to a Numo array.
+fn series_to_numo(s: &Series) -> RbResult<Value> {
+    series_to_numo_with_copy(s)
+}
 
-                // TODO make more efficient
-                let np_arr = RArray::from_iter(ca);
-                class::object()
-                    .const_get::<_, RModule>("Numo")?
-                    .const_get::<_, RClass>("RObject")?
-                    .funcall("cast", (np_arr,))
-            }
-            dt if dt.is_primitive_numeric() => {
-                if let Some(BitRepr::Large(_)) = s.bit_repr() {
-                    let s = s.cast(&DataType::Float64).unwrap();
-                    let ca = s.f64().unwrap();
-                    // TODO make more efficient
-                    let np_arr = RArray::from_iter(ca.into_iter().map(|opt_v| match opt_v {
-                        Some(v) => v,
-                        None => f64::NAN,
-                    }));
-                    class::object()
-                        .const_get::<_, RModule>("Numo")?
-                        .const_get::<_, RClass>("DFloat")?
-                        .funcall("cast", (np_arr,))
-                } else {
-                    let s = s.cast(&DataType::Float32).unwrap();
-                    let ca = s.f32().unwrap();
-                    // TODO make more efficient
-                    let np_arr = RArray::from_iter(ca.into_iter().map(|opt_v| match opt_v {
-                        Some(v) => v,
-                        None => f32::NAN,
-                    }));
-                    class::object()
-                        .const_get::<_, RModule>("Numo")?
-                        .const_get::<_, RClass>("SFloat")?
-                        .funcall("cast", (np_arr,))
-                }
-            }
-            dt => {
-                raise_err!(
-                    format!("'to_numo' not supported for dtype: {dt:?}"),
-                    ComputeError
-                );
-            }
+/// Convert a Series to a Numo array, copying data in the process.
+fn series_to_numo_with_copy(s: &Series) -> RbResult<Value> {
+    use DataType::*;
+    match s.dtype() {
+        Int8 => numeric_series_to_numpy::<Int8Type, f32>(s),
+        Int16 => numeric_series_to_numpy::<Int16Type, f32>(s),
+        Int32 => numeric_series_to_numpy::<Int32Type, f64>(s),
+        Int64 => numeric_series_to_numpy::<Int64Type, f64>(s),
+        UInt8 => numeric_series_to_numpy::<UInt8Type, f32>(s),
+        UInt16 => numeric_series_to_numpy::<UInt16Type, f32>(s),
+        UInt32 => numeric_series_to_numpy::<UInt32Type, f64>(s),
+        UInt64 => numeric_series_to_numpy::<UInt64Type, f64>(s),
+        Float32 => numeric_series_to_numpy::<Float32Type, f32>(s),
+        Float64 => numeric_series_to_numpy::<Float64Type, f64>(s),
+        Boolean => boolean_series_to_numo(s),
+        String => {
+            let ca = s.str().unwrap();
+            RbArray1::from_iter(ca)
+        }
+        dt => {
+            raise_err!(
+                format!("'to_numo' not supported for dtype: {dt:?}"),
+                ComputeError
+            );
         }
     }
 }
+
+/// Convert numeric types to f32 or f64 with NaN representing a null value.
+fn numeric_series_to_numpy<T, U>(s: &Series) -> RbResult<Value>
+where
+    T: PolarsNumericType,
+    T::Native: Element,
+    U: Float + Element,
+{
+    let ca: &ChunkedArray<T> = s.as_ref().as_ref();
+    if s.null_count() == 0 {
+        let values = ca.into_no_null_iter();
+        RbArray1::<T::Native>::from_iter(values)
+    } else {
+        let mapper = |opt_v: Option<T::Native>| match opt_v {
+            Some(v) => NumCast::from(v).unwrap(),
+            None => U::nan(),
+        };
+        let values = ca.iter().map(mapper);
+        RbArray1::from_iter(values)
+    }
+}
+
+/// Convert booleans to bit if no nulls are present, otherwise convert to objects.
+fn boolean_series_to_numo(s: &Series) -> RbResult<Value> {
+    let ca = s.bool().unwrap();
+    if s.null_count() == 0 {
+        let values = ca.into_no_null_iter();
+        RbArray1::<bool>::from_iter(values)
+    } else {
+        let values = ca.iter();
+        RbArray1::from_iter(values)
+    }
+}

data/ext/polars/src/lazyframe/general.rs

@@ -431,27 +431,24 @@ impl RbLazyFrame {
         Ok(())
     }
 
-    #[allow(clippy::too_many_arguments)]
-    pub fn sink_csv(
-        &self,
-        path: PathBuf,
-        include_bom: bool,
-        include_header: bool,
-        separator: u8,
-        line_terminator: String,
-        quote_char: u8,
-        batch_size: Wrap<NonZeroUsize>,
-        datetime_format: Option<String>,
-        date_format: Option<String>,
-        time_format: Option<String>,
-        float_scientific: Option<bool>,
-        float_precision: Option<usize>,
-        null_value: Option<String>,
-        quote_style: Option<Wrap<QuoteStyle>>,
-        maintain_order: bool,
-    ) -> RbResult<()> {
-        // TODO
-        let cloud_options = None;
+    pub fn sink_csv(&self, arguments: &[Value]) -> RbResult<()> {
+        let path = PathBuf::try_convert(arguments[0])?;
+        let include_bom = bool::try_convert(arguments[1])?;
+        let include_header = bool::try_convert(arguments[2])?;
+        let separator = u8::try_convert(arguments[3])?;
+        let line_terminator = String::try_convert(arguments[4])?;
+        let quote_char = u8::try_convert(arguments[5])?;
+        let batch_size = Wrap::<NonZeroUsize>::try_convert(arguments[6])?;
+        let datetime_format = Option::<String>::try_convert(arguments[7])?;
+        let date_format = Option::<String>::try_convert(arguments[8])?;
+        let time_format = Option::<String>::try_convert(arguments[9])?;
+        let float_scientific = Option::<bool>::try_convert(arguments[10])?;
+        let float_precision = Option::<usize>::try_convert(arguments[11])?;
+        let null_value = Option::<String>::try_convert(arguments[12])?;
+        let quote_style = Option::<Wrap<QuoteStyle>>::try_convert(arguments[13])?;
+        let maintain_order = bool::try_convert(arguments[14])?;
+        let cloud_options = Option::<Vec<(String, String)>>::try_convert(arguments[15])?;
+        let retries = usize::try_convert(arguments[16])?;
 
         let quote_style = quote_style.map_or(QuoteStyle::default(), |wrap| wrap.0);
         let null_value = null_value.unwrap_or(SerializeOptions::default().null);
@@ -480,7 +477,7 @@ impl RbLazyFrame {
         let cloud_options = {
             let cloud_options =
                 parse_cloud_options(path.to_str().unwrap(), cloud_options.unwrap_or_default())?;
-            Some(cloud_options)
+            Some(cloud_options.with_max_retries(retries))
         };
 
         let ldf = self.ldf.borrow().clone();
@@ -679,6 +676,7 @@ impl RbLazyFrame {
         how: Wrap<JoinType>,
         suffix: String,
         validate: Wrap<JoinValidation>,
+        maintain_order: Wrap<MaintainOrderJoin>,
         coalesce: Option<bool>,
     ) -> RbResult<Self> {
         let coalesce = match coalesce {
@@ -702,6 +700,7 @@ impl RbLazyFrame {
             .how(how.0)
             .validate(validate.0)
             .coalesce(coalesce)
+            .maintain_order(maintain_order.0)
             .suffix(suffix)
             .finish()
             .into())

data/ext/polars/src/lib.rs

@@ -430,6 +430,7 @@ fn init(ruby: &Ruby) -> RbResult<()> {
     class.define_method("mode", method!(RbExpr::mode, 0))?;
     class.define_method("exclude", method!(RbExpr::exclude, 1))?;
     class.define_method("interpolate", method!(RbExpr::interpolate, 1))?;
+    class.define_method("interpolate_by", method!(RbExpr::interpolate_by, 1))?;
     class.define_method("rolling_sum", method!(RbExpr::rolling_sum, 4))?;
     class.define_method("rolling_sum_by", method!(RbExpr::rolling_sum_by, 4))?;
     class.define_method("rolling_min", method!(RbExpr::rolling_min, 4))?;
@@ -614,6 +615,10 @@ fn init(ruby: &Ruby) -> RbResult<()> {
         "concat_lf_diagonal",
         function!(functions::lazy::concat_lf_diagonal, 4),
     )?;
+    class.define_singleton_method(
+        "concat_lf_horizontal",
+        function!(functions::lazy::concat_lf_horizontal, 2),
+    )?;
     class.define_singleton_method("concat_df", function!(functions::eager::concat_df, 1))?;
     class.define_singleton_method("concat_lf", function!(functions::lazy::concat_lf, 4))?;
     class.define_singleton_method(
@@ -745,7 +750,7 @@ fn init(ruby: &Ruby) -> RbResult<()> {
     class.define_method("collect", method!(RbLazyFrame::collect, 0))?;
     class.define_method("sink_parquet", method!(RbLazyFrame::sink_parquet, 9))?;
     class.define_method("sink_ipc", method!(RbLazyFrame::sink_ipc, 5))?;
-    class.define_method("sink_csv", method!(RbLazyFrame::sink_csv, 15))?;
+    class.define_method("sink_csv", method!(RbLazyFrame::sink_csv, -1))?;
     class.define_method("sink_json", method!(RbLazyFrame::sink_json, 4))?;
     class.define_method("fetch", method!(RbLazyFrame::fetch, 1))?;
     class.define_method("filter", method!(RbLazyFrame::filter, 1))?;
@@ -759,7 +764,7 @@ fn init(ruby: &Ruby) -> RbResult<()> {
     )?;
     class.define_method("with_context", method!(RbLazyFrame::with_context, 1))?;
     class.define_method("join_asof", method!(RbLazyFrame::join_asof, 14))?;
-    class.define_method("join", method!(RbLazyFrame::join, 10))?;
+    class.define_method("join", method!(RbLazyFrame::join, 11))?;
     class.define_method("with_column", method!(RbLazyFrame::with_column, 1))?;
     class.define_method("with_columns", method!(RbLazyFrame::with_columns, 1))?;
     class.define_method(