polars-df 0.12.0 → 0.13.0

data/ext/polars/Cargo.toml

@@ -1,10 +1,10 @@
 [package]
 name = "polars"
-version = "0.12.0"
+version = "0.13.0"
 license = "MIT"
 authors = ["Andrew Kane <andrew@ankane.org>"]
 edition = "2021"
-rust-version = "1.76.0"
+rust-version = "1.80.0"
 publish = false
 
 [lib]
@@ -15,14 +15,14 @@ ahash = "0.8"
 chrono = "0.4"
 either = "1.8"
 magnus = "0.7"
-polars-core = "=0.41.3"
-polars-parquet = "=0.41.3"
-polars-utils = "=0.41.3"
+polars-core = "=0.42.0"
+polars-parquet = "=0.42.0"
+polars-utils = "=0.42.0"
 serde_json = "1"
 smartstring = "1"
 
 [dependencies.polars]
-version = "=0.41.3"
+version = "=0.42.0"
 features = [
     "abs",
     "approx_unique",
@@ -117,5 +117,5 @@ features = [
 [target.'cfg(target_os = "linux")'.dependencies]
 jemallocator = { version = "0.5", features = ["disable_initial_exec_tls"] }
 
-[target.'cfg(not(target_os = "linux"))'.dependencies]
+[target.'cfg(not(any(target_os = "linux", target_os = "windows")))'.dependencies]
 mimalloc = { version = "0.1", default-features = false }

data/ext/polars/src/allocator.rs

@@ -0,0 +1,13 @@
+#[cfg(target_os = "linux")]
+use jemallocator::Jemalloc;
+
+#[cfg(not(any(target_os = "linux", target_os = "windows")))]
+use mimalloc::MiMalloc;
+
+#[global_allocator]
+#[cfg(target_os = "linux")]
+static ALLOC: Jemalloc = Jemalloc;
+
+#[global_allocator]
+#[cfg(not(any(target_os = "linux", target_os = "windows")))]
+static ALLOC: MiMalloc = MiMalloc;

data/ext/polars/src/conversion/chunked_array.rs

@@ -56,17 +56,15 @@ impl IntoValue for Wrap<&BinaryChunked> {
 }
 
 impl IntoValue for Wrap<&StructChunked> {
-    fn into_value_with(self, _: &Ruby) -> Value {
+    fn into_value_with(self, ruby: &Ruby) -> Value {
         let s = self.0.clone().into_series();
         // todo! iterate its chunks and flatten.
         // make series::iter() accept a chunk index.
         let s = s.rechunk();
-        let iter = s.iter().map(|av| {
-            if let AnyValue::Struct(_, _, flds) = av {
-                struct_dict(av._iter_struct_av(), flds)
-            } else {
-                unreachable!()
-            }
+        let iter = s.iter().map(|av| match av {
+            AnyValue::Struct(_, _, flds) => struct_dict(av._iter_struct_av(), flds),
+            AnyValue::Null => ruby.qnil().as_value(),
+            _ => unreachable!(),
         });
 
         RArray::from_iter(iter).into_value()

data/ext/polars/src/conversion/mod.rs

@@ -26,12 +26,6 @@ use crate::object::OBJECT_NAME;
 use crate::rb_modules::series;
 use crate::{RbDataFrame, RbLazyFrame, RbPolarsErr, RbResult, RbSeries, RbTypeError, RbValueError};
 
-pub(crate) fn slice_to_wrapped<T>(slice: &[T]) -> &[Wrap<T>] {
-    // Safety:
-    // Wrap is transparent.
-    unsafe { std::mem::transmute(slice) }
-}
-
 pub(crate) fn slice_extract_wrapped<T>(slice: &[Wrap<T>]) -> &[T] {
     // Safety:
     // Wrap is transparent.

data/ext/polars/src/dataframe/general.rs

@@ -348,12 +348,11 @@ impl RbDataFrame {
         value_name: Option<String>,
         variable_name: Option<String>,
     ) -> RbResult<Self> {
-        let args = UnpivotArgs {
+        let args = UnpivotArgsIR {
             on: strings_to_smartstrings(on),
             index: strings_to_smartstrings(index),
             value_name: value_name.map(|s| s.into()),
             variable_name: variable_name.map(|s| s.into()),
-            streamable: false,
         };
 
         let df = self.df.borrow().unpivot2(args).map_err(RbPolarsErr::from)?;

data/ext/polars/src/dataframe/io.rs

@@ -145,7 +145,7 @@ impl RbDataFrame {
                     .with_projection(projection)
                     .with_columns(columns)
                     .read_parallel(parallel.0)
-                    .with_n_rows(n_rows)
+                    .with_slice(n_rows.map(|x| (0, x)))
                     .with_row_index(row_index)
                     .set_low_memory(low_memory)
                     .use_statistics(use_statistics)
@@ -156,7 +156,7 @@ impl RbDataFrame {
                 .with_projection(projection)
                 .with_columns(columns)
                 .read_parallel(parallel.0)
-                .with_n_rows(n_rows)
+                .with_slice(n_rows.map(|x| (0, x)))
                 .with_row_index(row_index)
                 .use_statistics(use_statistics)
                 .set_rechunk(rechunk)

data/ext/polars/src/expr/meta.rs

@@ -7,15 +7,11 @@ impl RbExpr {
         self.inner == other.inner
     }
 
-    pub fn meta_pop(&self) -> RArray {
-        RArray::from_iter(
-            self.inner
-                .clone()
-                .meta()
-                .pop()
-                .into_iter()
-                .map(RbExpr::from),
-        )
+    pub fn meta_pop(&self) -> RbResult<RArray> {
+        let exprs = self.inner.clone().meta().pop().map_err(RbPolarsErr::from)?;
+        Ok(RArray::from_iter(
+            exprs.iter().map(|e| RbExpr::from(e.clone())),
+        ))
     }
 
     pub fn meta_root_names(&self) -> Vec<String> {

data/ext/polars/src/functions/eager.rs

@@ -64,6 +64,6 @@ pub fn concat_df_horizontal(seq: RArray) -> RbResult<RbDataFrame> {
     for item in seq.into_iter() {
         dfs.push(get_df(item)?);
     }
-    let df = functions::concat_df_horizontal(&dfs).map_err(RbPolarsErr::from)?;
+    let df = functions::concat_df_horizontal(&dfs, true).map_err(RbPolarsErr::from)?;
     Ok(df.into())
 }

data/ext/polars/src/lazyframe/mod.rs

@@ -162,6 +162,7 @@ impl RbLazyFrame {
         hive_schema: Option<Wrap<Schema>>,
         try_parse_hive_dates: bool,
         glob: bool,
+        include_file_paths: Option<String>,
     ) -> RbResult<Self> {
         let parallel = parallel.0;
         let hive_schema = hive_schema.map(|s| Arc::new(s.0));
@@ -196,6 +197,7 @@ impl RbLazyFrame {
             use_statistics,
             hive_options,
             glob,
+            include_file_paths: include_file_paths.map(Arc::from),
         };
 
         let lf = if path.is_some() {
@@ -207,6 +209,7 @@ impl RbLazyFrame {
         Ok(lf.into())
     }
 
+    #[allow(clippy::too_many_arguments)]
     pub fn new_from_ipc(
         path: String,
         n_rows: Option<usize>,
@@ -214,12 +217,23 @@ impl RbLazyFrame {
         rechunk: bool,
         row_index: Option<(String, IdxSize)>,
         memory_map: bool,
+        hive_partitioning: Option<bool>,
+        hive_schema: Option<Wrap<Schema>>,
+        try_parse_hive_dates: bool,
+        include_file_paths: Option<String>,
     ) -> RbResult<Self> {
         let row_index = row_index.map(|(name, offset)| RowIndex {
             name: Arc::from(name.as_str()),
             offset,
         });
 
+        let hive_options = HiveOptions {
+            enabled: hive_partitioning,
+            hive_start_idx: 0,
+            schema: hive_schema.map(|x| Arc::new(x.0)),
+            try_parse_dates: try_parse_hive_dates,
+        };
+
         let args = ScanArgsIpc {
             n_rows,
             cache,
@@ -227,6 +241,8 @@ impl RbLazyFrame {
             row_index,
             memory_map,
             cloud_options: None,
+            hive_options,
+            include_file_paths: include_file_paths.map(Arc::from),
         };
         let lf = LazyFrame::scan_ipc(path, args).map_err(RbPolarsErr::from)?;
         Ok(lf.into())
@@ -348,7 +364,7 @@ impl RbLazyFrame {
         compression_level: Option<i32>,
         statistics: Wrap<StatisticsOptions>,
         row_group_size: Option<usize>,
-        data_pagesize_limit: Option<usize>,
+        data_page_size: Option<usize>,
         maintain_order: bool,
     ) -> RbResult<()> {
         let compression = parse_parquet_compression(&compression, compression_level)?;
@@ -357,7 +373,7 @@ impl RbLazyFrame {
             compression,
             statistics: statistics.0,
             row_group_size,
-            data_pagesize_limit,
+            data_page_size,
             maintain_order,
         };
 
@@ -752,22 +768,22 @@ impl RbLazyFrame {
 
     pub fn unpivot(
         &self,
-        on: Vec<String>,
-        index: Vec<String>,
+        on: RArray,
+        index: RArray,
         value_name: Option<String>,
         variable_name: Option<String>,
-        streamable: bool,
-    ) -> Self {
-        let args = UnpivotArgs {
-            on: strings_to_smartstrings(on),
-            index: strings_to_smartstrings(index),
+    ) -> RbResult<Self> {
+        let on = rb_exprs_to_exprs(on)?;
+        let index = rb_exprs_to_exprs(index)?;
+        let args = UnpivotArgsDSL {
+            on: on.into_iter().map(|e| e.into()).collect(),
+            index: index.into_iter().map(|e| e.into()).collect(),
             value_name: value_name.map(|s| s.into()),
             variable_name: variable_name.map(|s| s.into()),
-            streamable,
         };
 
         let ldf = self.ldf.borrow().clone();
-        ldf.unpivot(args).into()
+        Ok(ldf.unpivot(args).into())
     }
 
     pub fn with_row_index(&self, name: String, offset: Option<IdxSize>) -> Self {

data/ext/polars/src/lib.rs

@@ -1,3 +1,4 @@
+mod allocator;
 mod batched_csv;
 mod conversion;
 mod dataframe;
@@ -31,20 +32,6 @@ use magnus::{define_module, function, method, prelude::*, Error, Ruby};
 use series::RbSeries;
 use sql::RbSQLContext;
 
-#[cfg(target_os = "linux")]
-use jemallocator::Jemalloc;
-
-#[cfg(not(target_os = "linux"))]
-use mimalloc::MiMalloc;
-
-#[global_allocator]
-#[cfg(target_os = "linux")]
-static GLOBAL: Jemalloc = Jemalloc;
-
-#[global_allocator]
-#[cfg(not(target_os = "linux"))]
-static GLOBAL: MiMalloc = MiMalloc;
-
 type RbResult<T> = Result<T, Error>;
 
 #[magnus::init]
@@ -722,9 +709,9 @@ fn init(ruby: &Ruby) -> RbResult<()> {
     class.define_singleton_method("new_from_csv", function!(RbLazyFrame::new_from_csv, -1))?;
     class.define_singleton_method(
         "new_from_parquet",
-        function!(RbLazyFrame::new_from_parquet, 13),
+        function!(RbLazyFrame::new_from_parquet, 14),
     )?;
-    class.define_singleton_method("new_from_ipc", function!(RbLazyFrame::new_from_ipc, 6))?;
+    class.define_singleton_method("new_from_ipc", function!(RbLazyFrame::new_from_ipc, 10))?;
     class.define_method("write_json", method!(RbLazyFrame::write_json, 1))?;
     class.define_method("describe_plan", method!(RbLazyFrame::describe_plan, 0))?;
     class.define_method(
@@ -780,7 +767,7 @@ fn init(ruby: &Ruby) -> RbResult<()> {
     class.define_method("drop_nulls", method!(RbLazyFrame::drop_nulls, 1))?;
     class.define_method("slice", method!(RbLazyFrame::slice, 2))?;
     class.define_method("tail", method!(RbLazyFrame::tail, 1))?;
-    class.define_method("unpivot", method!(RbLazyFrame::unpivot, 5))?;
+    class.define_method("unpivot", method!(RbLazyFrame::unpivot, 4))?;
     class.define_method("with_row_index", method!(RbLazyFrame::with_row_index, 2))?;
     class.define_method("drop", method!(RbLazyFrame::drop, 1))?;
     class.define_method("cast_all", method!(RbLazyFrame::cast_all, 2))?;
@@ -878,7 +865,7 @@ fn init(ruby: &Ruby) -> RbResult<()> {
     class.define_method("arg_max", method!(RbSeries::arg_max, 0))?;
     class.define_method("take_with_series", method!(RbSeries::take_with_series, 1))?;
     class.define_method("null_count", method!(RbSeries::null_count, 0))?;
-    class.define_method("has_validity", method!(RbSeries::has_validity, 0))?;
+    class.define_method("has_nulls", method!(RbSeries::has_nulls, 0))?;
     class.define_method("sample_n", method!(RbSeries::sample_n, 4))?;
     class.define_method("sample_frac", method!(RbSeries::sample_frac, 4))?;
     class.define_method("equals", method!(RbSeries::equals, 4))?;

data/ext/polars/src/map/mod.rs

@@ -93,7 +93,7 @@ fn iterator_to_struct(
             .collect::<Vec<_>>()
     });
 
-    Ok(StructChunked::new(name, &fields)
+    Ok(StructChunked::from_series(name, &fields)
         .unwrap()
         .into_series()
         .into())