polars-df 0.12.0 → 0.13.0

data/ext/polars/src/map/series.rs

@@ -1,8 +1,7 @@
-use magnus::{class, prelude::*, typed_data::Obj, IntoValue, RHash, TryConvert, Value};
+use magnus::{class, prelude::*, typed_data::Obj, IntoValue, TryConvert, Value};
 use polars::prelude::*;
 
 use super::*;
-use crate::conversion::slice_to_wrapped;
 use crate::series::RbSeries;
 use crate::{ObjectValue, RbResult};
 
@@ -43,7 +42,7 @@ fn infer_and_finish<'a, A: ApplyLambda<'a>>(
         todo!()
     } else if out.is_kind_of(class::hash()) {
         let first = Wrap::<AnyValue<'_>>::try_convert(out)?;
-        applyer.apply_to_struct(lambda, null_count, first.0)
+        applyer.apply_into_struct(lambda, null_count, first.0)
     }
     // this succeeds for numpy ints as well, where checking if it is pyint fails
     // we do this later in the chain so that we don't extract integers from string chars.
@@ -70,12 +69,8 @@ fn infer_and_finish<'a, A: ApplyLambda<'a>>(
 pub trait ApplyLambda<'a> {
     fn apply_lambda_unknown(&'a self, _lambda: Value) -> RbResult<RbSeries>;
 
-    /// Apply a lambda that doesn't change output types
-    #[allow(dead_code)]
-    fn apply_lambda(&'a self, _lambda: Value) -> RbResult<RbSeries>;
-
     // Used to store a struct type
-    fn apply_to_struct(
+    fn apply_into_struct(
         &'a self,
         lambda: Value,
         init_null_count: usize,
@@ -183,19 +178,14 @@ impl<'a> ApplyLambda<'a> for BooleanChunked {
             .into())
     }
 
-    fn apply_lambda(&'a self, lambda: Value) -> RbResult<RbSeries> {
-        self.apply_lambda_with_bool_out_type(lambda, 0, None)
-            .map(|ca| RbSeries::new(ca.into_series()))
-    }
-
-    fn apply_to_struct(
+    fn apply_into_struct(
         &'a self,
         lambda: Value,
         init_null_count: usize,
         first_value: AnyValue<'a>,
     ) -> RbResult<RbSeries> {
         let skip = 1;
-        if !self.has_validity() {
+        if !self.has_nulls() {
             let it = self
                 .into_no_null_iter()
                 .skip(init_null_count + skip)
@@ -223,7 +213,7 @@ impl<'a> ApplyLambda<'a> for BooleanChunked {
         let skip = usize::from(first_value.is_some());
         if init_null_count == self.len() {
             Ok(ChunkedArray::full_null(self.name(), self.len()))
-        } else if !self.has_validity() {
+        } else if !self.has_nulls() {
             let it = self
                 .into_no_null_iter()
                 .skip(init_null_count + skip)
@@ -259,7 +249,7 @@ impl<'a> ApplyLambda<'a> for BooleanChunked {
         let skip = usize::from(first_value.is_some());
         if init_null_count == self.len() {
             Ok(ChunkedArray::full_null(self.name(), self.len()))
-        } else if !self.has_validity() {
+        } else if !self.has_nulls() {
             let it = self
                 .into_no_null_iter()
                 .skip(init_null_count + skip)
@@ -295,7 +285,7 @@ impl<'a> ApplyLambda<'a> for BooleanChunked {
         let skip = usize::from(first_value.is_some());
         if init_null_count == self.len() {
             Ok(ChunkedArray::full_null(self.name(), self.len()))
-        } else if !self.has_validity() {
+        } else if !self.has_nulls() {
             let it = self
                 .into_no_null_iter()
                 .skip(init_null_count + skip)
@@ -333,7 +323,7 @@ impl<'a> ApplyLambda<'a> for BooleanChunked {
         let skip = 1;
         if init_null_count == self.len() {
             Ok(ChunkedArray::full_null(self.name(), self.len()))
-        } else if !self.has_validity() {
+        } else if !self.has_nulls() {
             let it = self
                 .into_no_null_iter()
                 .skip(init_null_count + skip)
@@ -405,7 +395,7 @@ impl<'a> ApplyLambda<'a> for BooleanChunked {
         let skip = usize::from(first_value.is_some());
         if init_null_count == self.len() {
             Ok(ChunkedArray::full_null(self.name(), self.len()))
-        } else if !self.has_validity() {
+        } else if !self.has_nulls() {
             let it = self
                 .into_no_null_iter()
                 .skip(init_null_count + skip)
@@ -460,19 +450,14 @@ where
             .into())
     }
 
-    fn apply_lambda(&'a self, lambda: Value) -> RbResult<RbSeries> {
-        self.apply_lambda_with_primitive_out_type::<T>(lambda, 0, None)
-            .map(|ca| RbSeries::new(ca.into_series()))
-    }
-
-    fn apply_to_struct(
+    fn apply_into_struct(
         &'a self,
         lambda: Value,
         init_null_count: usize,
         first_value: AnyValue<'a>,
     ) -> RbResult<RbSeries> {
         let skip = 1;
-        if !self.has_validity() {
+        if !self.has_nulls() {
             let it = self
                 .into_no_null_iter()
                 .skip(init_null_count + skip)
@@ -500,7 +485,7 @@ where
         let skip = usize::from(first_value.is_some());
         if init_null_count == self.len() {
             Ok(ChunkedArray::full_null(self.name(), self.len()))
-        } else if !self.has_validity() {
+        } else if !self.has_nulls() {
             let it = self
                 .into_no_null_iter()
                 .skip(init_null_count + skip)
@@ -536,7 +521,7 @@ where
         let skip = usize::from(first_value.is_some());
         if init_null_count == self.len() {
             Ok(ChunkedArray::full_null(self.name(), self.len()))
-        } else if !self.has_validity() {
+        } else if !self.has_nulls() {
             let it = self
                 .into_no_null_iter()
                 .skip(init_null_count + skip)
@@ -572,7 +557,7 @@ where
         let skip = usize::from(first_value.is_some());
         if init_null_count == self.len() {
             Ok(ChunkedArray::full_null(self.name(), self.len()))
-        } else if !self.has_validity() {
+        } else if !self.has_nulls() {
             let it = self
                 .into_no_null_iter()
                 .skip(init_null_count + skip)
@@ -610,7 +595,7 @@ where
         let skip = 1;
         if init_null_count == self.len() {
             Ok(ChunkedArray::full_null(self.name(), self.len()))
-        } else if !self.has_validity() {
+        } else if !self.has_nulls() {
             let it = self
                 .into_no_null_iter()
                 .skip(init_null_count + skip)
@@ -682,7 +667,7 @@ where
         let skip = usize::from(first_value.is_some());
         if init_null_count == self.len() {
             Ok(ChunkedArray::full_null(self.name(), self.len()))
-        } else if !self.has_validity() {
+        } else if !self.has_nulls() {
             let it = self
                 .into_no_null_iter()
                 .skip(init_null_count + skip)
@@ -732,19 +717,14 @@ impl<'a> ApplyLambda<'a> for StringChunked {
             .into())
     }
 
-    fn apply_lambda(&'a self, lambda: Value) -> RbResult<RbSeries> {
-        let ca = self.apply_lambda_with_utf8_out_type(lambda, 0, None)?;
-        Ok(ca.into_series().into())
-    }
-
-    fn apply_to_struct(
+    fn apply_into_struct(
         &'a self,
         lambda: Value,
         init_null_count: usize,
         first_value: AnyValue<'a>,
     ) -> RbResult<RbSeries> {
         let skip = 1;
-        if !self.has_validity() {
+        if !self.has_nulls() {
             let it = self
                 .into_no_null_iter()
                 .skip(init_null_count + skip)
@@ -772,7 +752,7 @@ impl<'a> ApplyLambda<'a> for StringChunked {
         let skip = usize::from(first_value.is_some());
         if init_null_count == self.len() {
             Ok(ChunkedArray::full_null(self.name(), self.len()))
-        } else if !self.has_validity() {
+        } else if !self.has_nulls() {
             let it = self
                 .into_no_null_iter()
                 .skip(init_null_count + skip)
@@ -808,7 +788,7 @@ impl<'a> ApplyLambda<'a> for StringChunked {
         let skip = usize::from(first_value.is_some());
         if init_null_count == self.len() {
             Ok(ChunkedArray::full_null(self.name(), self.len()))
-        } else if !self.has_validity() {
+        } else if !self.has_nulls() {
             let it = self
                 .into_no_null_iter()
                 .skip(init_null_count + skip)
@@ -844,7 +824,7 @@ impl<'a> ApplyLambda<'a> for StringChunked {
         let skip = usize::from(first_value.is_some());
         if init_null_count == self.len() {
             Ok(ChunkedArray::full_null(self.name(), self.len()))
-        } else if !self.has_validity() {
+        } else if !self.has_nulls() {
             let it = self
                 .into_no_null_iter()
                 .skip(init_null_count + skip)
@@ -882,7 +862,7 @@ impl<'a> ApplyLambda<'a> for StringChunked {
         let skip = 1;
         if init_null_count == self.len() {
             Ok(ChunkedArray::full_null(self.name(), self.len()))
-        } else if !self.has_validity() {
+        } else if !self.has_nulls() {
             let it = self
                 .into_no_null_iter()
                 .skip(init_null_count + skip)
@@ -954,7 +934,7 @@ impl<'a> ApplyLambda<'a> for StringChunked {
         let skip = usize::from(first_value.is_some());
         if init_null_count == self.len() {
             Ok(ChunkedArray::full_null(self.name(), self.len()))
-        } else if !self.has_validity() {
+        } else if !self.has_nulls() {
             let it = self
                 .into_no_null_iter()
                 .skip(init_null_count + skip)
@@ -983,21 +963,16 @@ impl<'a> ApplyLambda<'a> for StringChunked {
     }
 }
 
-fn make_dict_arg(names: &[&str], vals: &[AnyValue]) -> RHash {
-    let dict = RHash::new();
-    for (name, val) in names.iter().zip(slice_to_wrapped(vals)) {
-        dict.aset(name.to_string(), (*val).clone()).unwrap()
-    }
-    dict
+fn iter_struct(ca: &StructChunked) -> impl Iterator<Item = AnyValue> {
+    (0..ca.len()).map(|i| unsafe { ca.get_any_value_unchecked(i) })
 }
 
 impl<'a> ApplyLambda<'a> for StructChunked {
     fn apply_lambda_unknown(&'a self, lambda: Value) -> RbResult<RbSeries> {
-        let names = self.fields().iter().map(|s| s.name()).collect::<Vec<_>>();
         let mut null_count = 0;
-        for val in self.into_iter() {
-            let arg = make_dict_arg(&names, val);
-            let out: Value = lambda.funcall("call", (arg,))?;
+
+        for val in iter_struct(self) {
+            let out: Value = lambda.funcall("call", (Wrap(val),))?;
             if out.is_nil() {
                 null_count += 1;
                 continue;
@@ -1009,22 +984,15 @@ impl<'a> ApplyLambda<'a> for StructChunked {
         Ok(self.clone().into_series().into())
     }
 
-    fn apply_lambda(&'a self, lambda: Value) -> RbResult<RbSeries> {
-        self.apply_lambda_unknown(lambda)
-    }
-
-    fn apply_to_struct(
+    fn apply_into_struct(
         &'a self,
         lambda: Value,
         init_null_count: usize,
         first_value: AnyValue<'a>,
     ) -> RbResult<RbSeries> {
-        let names = self.fields().iter().map(|s| s.name()).collect::<Vec<_>>();
-
         let skip = 1;
-        let it = self.into_iter().skip(init_null_count + skip).map(|val| {
-            let arg = make_dict_arg(&names, val);
-            let out = lambda.funcall("call", (arg,)).unwrap();
+        let it = iter_struct(self).skip(init_null_count + skip).map(|val| {
+            let out = lambda.funcall("call", (Wrap(val),)).unwrap();
             Some(out)
         });
         iterator_to_struct(it, init_null_count, first_value, self.name(), self.len())
@@ -1040,13 +1008,10 @@ impl<'a> ApplyLambda<'a> for StructChunked {
         D: RbArrowPrimitiveType,
         D::Native: IntoValue + TryConvert,
     {
-        let names = self.fields().iter().map(|s| s.name()).collect::<Vec<_>>();
-
         let skip = usize::from(first_value.is_some());
-        let it = self.into_iter().skip(init_null_count + skip).map(|val| {
-            let arg = make_dict_arg(&names, val);
-            call_lambda_and_extract(lambda, arg).ok()
-        });
+        let it = iter_struct(self)
+            .skip(init_null_count + skip)
+            .map(|val| call_lambda_and_extract(lambda, Wrap(val)).ok());
 
         Ok(iterator_to_primitive(
             it,
@@ -1063,13 +1028,10 @@ impl<'a> ApplyLambda<'a> for StructChunked {
         init_null_count: usize,
         first_value: Option<bool>,
     ) -> RbResult<BooleanChunked> {
-        let names = self.fields().iter().map(|s| s.name()).collect::<Vec<_>>();
-
         let skip = usize::from(first_value.is_some());
-        let it = self.into_iter().skip(init_null_count + skip).map(|val| {
-            let arg = make_dict_arg(&names, val);
-            call_lambda_and_extract(lambda, arg).ok()
-        });
+        let it = iter_struct(self)
+            .skip(init_null_count + skip)
+            .map(|val| call_lambda_and_extract(lambda, Wrap(val)).ok());
 
         Ok(iterator_to_bool(
             it,
@@ -1086,13 +1048,10 @@ impl<'a> ApplyLambda<'a> for StructChunked {
         init_null_count: usize,
         first_value: Option<&str>,
     ) -> RbResult<StringChunked> {
-        let names = self.fields().iter().map(|s| s.name()).collect::<Vec<_>>();
-
         let skip = usize::from(first_value.is_some());
-        let it = self.into_iter().skip(init_null_count + skip).map(|val| {
-            let arg = make_dict_arg(&names, val);
-            call_lambda_and_extract(lambda, arg).ok()
-        });
+        let it = iter_struct(self)
+            .skip(init_null_count + skip)
+            .map(|val| call_lambda_and_extract(lambda, Wrap(val)).ok());
 
         Ok(iterator_to_utf8(
             it,
@@ -1111,13 +1070,9 @@ impl<'a> ApplyLambda<'a> for StructChunked {
         dt: &DataType,
     ) -> RbResult<ListChunked> {
         let skip = 1;
-
-        let names = self.fields().iter().map(|s| s.name()).collect::<Vec<_>>();
-
-        let it = self.into_iter().skip(init_null_count + skip).map(|val| {
-            let arg = make_dict_arg(&names, val);
-            call_lambda_series_out(lambda, arg).ok()
-        });
+        let it = iter_struct(self)
+            .skip(init_null_count + skip)
+            .map(|val| call_lambda_series_out(lambda, Wrap(val)).ok());
         iterator_to_list(
             dt,
             it,
@@ -1134,14 +1089,12 @@ impl<'a> ApplyLambda<'a> for StructChunked {
         init_null_count: usize,
         first_value: AnyValue<'a>,
     ) -> RbResult<Series> {
-        let names = self.fields().iter().map(|s| s.name()).collect::<Vec<_>>();
         let mut avs = Vec::with_capacity(self.len());
         avs.extend(std::iter::repeat(AnyValue::Null).take(init_null_count));
         avs.push(first_value);
 
-        let iter = self.into_iter().skip(init_null_count + 1).map(|val| {
-            let arg = make_dict_arg(&names, val);
-            call_lambda_and_extract::<_, Wrap<AnyValue>>(lambda, arg)
+        let iter = iter_struct(self).skip(init_null_count + 1).map(|val| {
+            call_lambda_and_extract::<_, Wrap<AnyValue>>(lambda, Wrap(val))
                 .unwrap()
                 .0
         });
@@ -1156,13 +1109,10 @@ impl<'a> ApplyLambda<'a> for StructChunked {
         init_null_count: usize,
         first_value: Option<ObjectValue>,
     ) -> RbResult<ObjectChunked<ObjectValue>> {
-        let names = self.fields().iter().map(|s| s.name()).collect::<Vec<_>>();
-
         let skip = usize::from(first_value.is_some());
-        let it = self.into_iter().skip(init_null_count + skip).map(|val| {
-            let arg = make_dict_arg(&names, val);
-            call_lambda_and_extract(lambda, arg).ok()
-        });
+        let it = iter_struct(self)
+            .skip(init_null_count + skip)
+            .map(|val| call_lambda_and_extract(lambda, Wrap(val)).ok());
 
         Ok(iterator_to_object(
             it,

data/ext/polars/src/series/mod.rs

@@ -50,7 +50,7 @@ impl RbSeries {
     pub fn struct_unnest(&self) -> RbResult<RbDataFrame> {
         let binding = self.series.borrow();
         let ca = binding.struct_().map_err(RbPolarsErr::from)?;
-        let df: DataFrame = ca.clone().into();
+        let df: DataFrame = ca.clone().unnest();
         Ok(df.into())
     }
 
@@ -278,8 +278,8 @@ impl RbSeries {
         Ok(self.series.borrow().null_count())
     }
 
-    pub fn has_validity(&self) -> bool {
-        self.series.borrow().has_validity()
+    pub fn has_nulls(&self) -> bool {
+        self.series.borrow().has_nulls()
     }
 
     pub fn sample_n(

data/lib/polars/io/ipc.rb

@@ -193,6 +193,18 @@ module Polars
     #   Try to memory map the file. This can greatly improve performance on repeated
     #   queries as the OS may cache pages.
     #   Only uncompressed IPC files can be memory mapped.
+    # @param hive_partitioning [Boolean]
+    #   Infer statistics and schema from Hive partitioned URL and use them
+    #   to prune reads. This is unset by default (i.e. `nil`), meaning it is
+    #   automatically enabled when a single directory is passed, and otherwise
+    #   disabled.
+    # @param hive_schema [Hash]
+    #   The column names and data types of the columns by which the data is partitioned.
+    #   If set to `nil` (default), the schema of the Hive partitions is inferred.
+    # @param try_parse_hive_dates [Boolean]
+    #   Whether to try parsing hive values as date/datetime types.
+    # @param include_file_paths [String]
+    #   Include the path of the source file(s) as a column with this name.
     #
     # @return [LazyFrame]
     def scan_ipc(
@@ -203,7 +215,11 @@ module Polars
       row_count_name: nil,
       row_count_offset: 0,
       storage_options: nil,
-      memory_map: true
+      memory_map: true,
+      hive_partitioning: nil,
+      hive_schema: nil,
+      try_parse_hive_dates: true,
+      include_file_paths: nil
     )
       _scan_ipc_impl(
         source,
@@ -213,7 +229,11 @@ module Polars
         row_count_name: row_count_name,
         row_count_offset: row_count_offset,
         storage_options: storage_options,
-        memory_map: memory_map
+        memory_map: memory_map,
+        hive_partitioning: hive_partitioning,
+        hive_schema: hive_schema,
+        try_parse_hive_dates: try_parse_hive_dates,
+        include_file_paths: include_file_paths
       )
     end
 
@@ -226,7 +246,11 @@ module Polars
       row_count_name: nil,
       row_count_offset: 0,
       storage_options: nil,
-      memory_map: true
+      memory_map: true,
+      hive_partitioning: nil,
+      hive_schema: nil,
+      try_parse_hive_dates: true,
+      include_file_paths: nil
     )
       if Utils.pathlike?(file)
         file = Utils.normalize_filepath(file)
@@ -239,7 +263,11 @@ module Polars
           cache,
           rechunk,
           Utils.parse_row_index_args(row_count_name, row_count_offset),
-          memory_map
+          memory_map,
+          hive_partitioning,
+          hive_schema,
+          try_parse_hive_dates,
+          include_file_paths
         )
       Utils.wrap_ldf(rblf)
     end

data/lib/polars/io/parquet.rb

@@ -158,6 +158,8 @@ module Polars
     #   Extra options that make sense for a particular storage connection.
     # @param low_memory [Boolean]
     #   Reduce memory pressure at the expense of performance.
+    # @param include_file_paths [String]
+    #  Include the path of the source file(s) as a column with this name.
     #
     # @return [LazyFrame]
     def scan_parquet(
@@ -170,7 +172,8 @@ module Polars
       row_count_name: nil,
       row_count_offset: 0,
       storage_options: nil,
-      low_memory: false
+      low_memory: false,
+      include_file_paths: nil
     )
       if Utils.pathlike?(source)
         source = Utils.normalize_filepath(source)
@@ -186,7 +189,8 @@ module Polars
         row_count_offset: row_count_offset,
         storage_options: storage_options,
         low_memory: low_memory,
-        glob: glob
+        glob: glob,
+        include_file_paths: include_file_paths
       )
     end
 
@@ -203,7 +207,8 @@ module Polars
       low_memory: false,
       use_statistics: true,
       hive_partitioning: nil,
-      glob: true
+      glob: true,
+      include_file_paths: nil
     )
       rblf =
         RbLazyFrame.new_from_parquet(
@@ -219,7 +224,8 @@ module Polars
           hive_partitioning,
           nil,
           true,
-          glob
+          glob,
+          include_file_paths
         )
       Utils.wrap_ldf(rblf)
     end

data/lib/polars/lazy_frame.rb

@@ -2526,11 +2526,15 @@ module Polars
       value_name: nil,
       streamable: true
     )
+      if !streamable
+        warn "The `streamable` parameter for `LazyFrame.unpivot` is deprecated"
+      end
+
       on = on.nil? ? [] : Utils._expand_selectors(self, on)
       index = index.nil? ? [] : Utils._expand_selectors(self, index)
 
       _from_rbldf(
-        _ldf.unpivot(on, index, value_name, variable_name, streamable)
+        _ldf.unpivot(on, index, value_name, variable_name)
       )
     end
     alias_method :melt, :unpivot

data/lib/polars/series.rb

@@ -1790,9 +1790,10 @@ module Polars
     # Use this to swiftly assert a Series does not have null values.
     #
     # @return [Boolean]
-    def has_validity
-      _s.has_validity
+    def has_nulls
+      _s.has_nulls
     end
+    alias_method :has_validity, :has_nulls
 
     # Check if the Series is empty.
     #

data/lib/polars/string_expr.rb

@@ -792,15 +792,15 @@ module Polars
     #   df.select(Polars.col("json").str.json_decode(dtype))
     #   # =>
     #   # shape: (3, 1)
-    #   # ┌─────────────┐
-    #   # │ json        │
-    #   # │ ---         │
-    #   # │ struct[2]   │
-    #   # ╞═════════════╡
-    #   # │ {1,true}    │
-    #   # │ {null,null} │
-    #   # │ {2,false}   │
-    #   # └─────────────┘
+    #   # ┌───────────┐
+    #   # │ json      │
+    #   # │ ---       │
+    #   # │ struct[2] │
+    #   # ╞═══════════╡
+    #   # │ {1,true}  │
+    #   # │ null      │
+    #   # │ {2,false} │
+    #   # └───────────┘
     def json_decode(dtype = nil, infer_schema_length: 100)
       if !dtype.nil?
         dtype = Utils.rb_type_to_dtype(dtype)

data/lib/polars/version.rb

@@ -1,4 +1,4 @@
 module Polars
   # @private
-  VERSION = "0.12.0"
+  VERSION = "0.13.0"
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: polars-df
 version: !ruby/object:Gem::Version
-  version: 0.12.0
+  version: 0.13.0
 platform: ruby
 authors:
 - Andrew Kane
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2024-07-11 00:00:00.000000000 Z
+date: 2024-09-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bigdecimal
@@ -53,6 +53,7 @@ files:
 - README.md
 - ext/polars/Cargo.toml
 - ext/polars/extconf.rb
+- ext/polars/src/allocator.rs
 - ext/polars/src/batched_csv.rs
 - ext/polars/src/conversion/any_value.rs
 - ext/polars/src/conversion/chunked_array.rs