polars-df 0.26.1 → 0.27.0

data/ext/polars/src/file.rs

@@ -6,9 +6,8 @@ use std::path::PathBuf;
 use magnus::{Error, RString, Ruby, Value, prelude::*, value::Opaque};
 use polars::io::mmap::MmapBytesReader;
 use polars::prelude::PlRefPath;
-use polars::prelude::file::{Writeable, WriteableTrait};
+use polars::prelude::file::{Writable, WritableTrait};
 use polars_buffer::Buffer;
-use polars_utils::create_file;
 
 use crate::error::RbPolarsErr;
 use crate::prelude::resolve_homedir;
@@ -23,7 +22,7 @@ pub struct RbFileLikeObject {
     has_flush: bool,
 }
 
-impl WriteableTrait for RbFileLikeObject {
+impl WritableTrait for RbFileLikeObject {
     fn close(&mut self) -> io::Result<()> {
         Ok(())
     }
@@ -192,7 +191,7 @@ impl Seek for RbFileLikeObject {
     }
 }
 
-pub trait FileLike: Read + Write + Seek {}
+pub trait FileLike: Read + Write + Seek + Sync + Send {}
 
 impl FileLike for File {}
 impl FileLike for RbFileLikeObject {}
@@ -217,10 +216,10 @@ impl EitherRustRubyFile {
         }
     }
 
-    pub(crate) fn into_writeable(self) -> Writeable {
+    pub(crate) fn into_writable(self) -> Writable {
         match self {
-            Self::Rb(f) => Writeable::Dyn(Box::new(f)),
-            Self::Rust(f) => Writeable::Local(f),
+            Self::Rb(f) => Writable::Dyn(Box::new(f)),
+            Self::Rust(f) => Writable::Local(f),
         }
     }
 }
@@ -272,9 +271,9 @@ pub fn get_either_buffer_or_path(
             let file_path = std::path::Path::new(&s);
             let file_path = resolve_homedir(&file_path);
             let f = if write {
-                create_file(&file_path).map_err(RbPolarsErr::from)?
+                polars_utils::io::create_file(&file_path).map_err(RbPolarsErr::from)?
             } else {
-                polars_utils::open_file(&file_path).map_err(RbPolarsErr::from)?
+                polars_utils::io::open_file(&file_path).map_err(RbPolarsErr::from)?
             };
             Ok((EitherRustRubyFile::Rust(f), Some(file_path.into_owned())))
         } else {

data/ext/polars/src/functions/io.rs

@@ -62,7 +62,8 @@ pub fn read_parquet_metadata(
                 })
                 .map_err(RbPolarsErr::from)?
             } else {
-                let file = polars_utils::open_file(p.as_std_path()).map_err(RbPolarsErr::from)?;
+                let file =
+                    polars_utils::io::open_file(p.as_std_path()).map_err(RbPolarsErr::from)?;
                 read_metadata(&mut BufReader::new(file)).map_err(RbPolarsErr::from)?
             }
         }

data/ext/polars/src/interop/arrow/to_rb.rs

@@ -1,28 +1,15 @@
 use arrow::datatypes::ArrowDataType;
 use arrow::ffi;
-use magnus::{IntoValue, Ruby, Value};
+use magnus::{Ruby, Value};
 use polars::datatypes::CompatLevel;
 use polars::frame::DataFrame;
 use polars::prelude::{ArrayRef, ArrowField, PlSmallStr, PolarsResult, SchemaExt};
 use polars::series::Series;
 use polars_core::utils::arrow;
-use std::ffi::CString;
 
 use crate::RbResult;
 use crate::ruby::capsule::RbCapsule;
 
-// TODO switch to RbCapsule in 0.27.0
-#[magnus::wrap(class = "Polars::ArrowSchema")]
-pub struct RbArrowSchema {
-    pub(crate) schema: ffi::ArrowSchema,
-}
-
-impl RbArrowSchema {
-    pub fn to_i(&self) -> usize {
-        (&self.schema as *const _) as usize
-    }
-}
-
 pub(crate) fn series_to_stream(series: &Series, ruby: &Ruby) -> RbResult<Value> {
     let field = series.field().to_arrow(CompatLevel::newest());
     let series = series.clone();
@@ -31,16 +18,14 @@ pub(crate) fn series_to_stream(series: &Series, ruby: &Ruby) -> RbResult<Value>
     ) as _;
 
     let stream = ffi::export_iterator(iter, field);
-    let stream_capsule_name = CString::new("arrow_array_stream").unwrap();
-    Ok(RbCapsule::new(stream, Some(stream_capsule_name)).into_value_with(ruby))
+    RbCapsule::new_with_value(ruby, stream, c"arrow_array_stream")
 }
 
 pub(crate) fn dataframe_to_stream(df: &DataFrame, ruby: &Ruby) -> RbResult<Value> {
     let iter = Box::new(DataFrameStreamIterator::new(df));
     let field = iter.field();
     let stream = ffi::export_iterator(iter, field);
-    let stream_capsule_name = CString::new("arrow_array_stream").unwrap();
-    Ok(RbCapsule::new(stream, Some(stream_capsule_name)).into_value_with(ruby))
+    RbCapsule::new_with_value(ruby, stream, c"arrow_array_stream")
 }
 
 pub(crate) fn polars_schema_to_rbcapsule(
@@ -59,7 +44,7 @@ pub(crate) fn polars_schema_to_rbcapsule(
         false,
     ));
 
-    Ok(RbArrowSchema { schema }.into_value_with(ruby))
+    RbCapsule::new_with_value(ruby, schema, c"arrow_schema")
 }
 
 pub struct DataFrameStreamIterator {

data/ext/polars/src/lazyframe/general.rs

@@ -1224,7 +1224,12 @@ impl RbLazyFrame {
         ldf.count().into()
     }
 
-    pub fn merge_sorted(&self, other: &Self, key: String, maintain_order: bool) -> RbResult<Self> {
+    pub fn merge_sorted(
+        &self,
+        other: &Self,
+        key: Vec<String>,
+        maintain_order: bool,
+    ) -> RbResult<Self> {
         let out = self
             .ldf
             .read()

data/ext/polars/src/lazyframe/sink.rs

@@ -18,7 +18,7 @@ impl TryConvert for Wrap<polars_plan::dsl::SinkTarget> {
                 RbResult::Ok(
                     crate::file::try_get_rbfile(rb, rb_f, true)?
                         .0
-                        .into_writeable(),
+                        .into_writable(),
                 )
             })?;
 

data/ext/polars/src/lib.rs

@@ -37,7 +37,6 @@ use expr::RbExpr;
 use expr::datatype::RbDataTypeExpr;
 use expr::selector::RbSelector;
 use functions::whenthen::{RbChainedThen, RbChainedWhen, RbThen, RbWhen};
-use interop::arrow::to_rb::RbArrowSchema;
 use lazyframe::{RbCollectBatches, RbInProcessQuery, RbLazyFrame, RbOptFlags};
 use lazygroupby::RbLazyGroupBy;
 use magnus::{Ruby, function, method, prelude::*};
@@ -138,6 +137,7 @@ fn init(ruby: &Ruby) -> RbResult<()> {
     class.define_method("tail", method!(RbDataFrame::tail, 1))?;
     class.define_method("is_unique", method!(RbDataFrame::is_unique, 0))?;
     class.define_method("is_duplicated", method!(RbDataFrame::is_duplicated, 0))?;
+    class.define_method("is_sorted", method!(RbDataFrame::is_sorted, 3))?;
     class.define_method("equals", method!(RbDataFrame::equals, 2))?;
     class.define_method("with_row_index", method!(RbDataFrame::with_row_index, 2))?;
     class.define_method(
@@ -252,6 +252,7 @@ fn init(ruby: &Ruby) -> RbResult<()> {
     class.define_method("is_unique", method!(RbExpr::is_unique, 0))?;
     class.define_method("is_between", method!(RbExpr::is_between, 3))?;
     class.define_method("is_close", method!(RbExpr::is_close, 4))?;
+    class.define_method("is_sorted", method!(RbExpr::is_sorted, 2))?;
     class.define_method("approx_n_unique", method!(RbExpr::approx_n_unique, 0))?;
     class.define_method("is_first_distinct", method!(RbExpr::is_first_distinct, 0))?;
     class.define_method("is_last_distinct", method!(RbExpr::is_last_distinct, 0))?;
@@ -1347,10 +1348,6 @@ fn init(ruby: &Ruby) -> RbResult<()> {
     class.define_method("to_i", method!(RbCapsule::to_i, 0))?;
     class.define_method("name", method!(RbCapsule::name, 0))?;
 
-    // arrow schema
-    let class = module.define_class("ArrowSchema", ruby.class_object())?;
-    class.define_method("to_i", method!(RbArrowSchema::to_i, 0))?;
-
     // catalog
     let class = module.define_class("RbCatalogClient", ruby.class_object())?;
     class.define_singleton_method("new", function!(RbCatalogClient::new, 2))?;

data/ext/polars/src/on_startup.rs

@@ -12,7 +12,7 @@ use polars_core::chunked_array::object::registry;
 use polars_core::chunked_array::object::registry::AnonymousObjectBuilder;
 use polars_core::prelude::AnyValue;
 use polars_error::PolarsWarning;
-use polars_error::signals::register_polars_keyboard_interrupt_hook;
+use polars_error::abort::register_polars_abort_mechanism;
 
 use crate::Wrap;
 use crate::dataframe::RbDataFrame;
@@ -209,7 +209,7 @@ pub unsafe fn register_startup_deps(catch_keyboard_interrupt: bool) {
         polars_error::set_warning_function(warning_function);
 
         if catch_keyboard_interrupt {
-            register_polars_keyboard_interrupt_hook();
+            register_polars_abort_mechanism();
         }
     });
 }

data/ext/polars/src/ruby/capsule.rs

@@ -1,5 +1,9 @@
 use std::any::Any;
-use std::ffi::CString;
+use std::ffi::{CStr, CString};
+
+use magnus::{IntoValue, Ruby, Value};
+
+use crate::RbResult;
 
 #[magnus::wrap(class = "Polars::Capsule")]
 pub struct RbCapsule {
@@ -8,6 +12,14 @@ pub struct RbCapsule {
 }
 
 impl RbCapsule {
+    pub fn new_with_value<T: 'static + Send>(
+        ruby: &Ruby,
+        value: T,
+        name: &CStr,
+    ) -> RbResult<Value> {
+        Ok(Self::new(value, Some(name.into())).into_value_with(ruby))
+    }
+
     pub fn new<T: 'static + Send>(value: T, name: Option<CString>) -> Self {
         Self {
             value: Box::new(value),

data/ext/polars/src/utils.rs

@@ -4,7 +4,7 @@ use magnus::Ruby;
 use polars::frame::DataFrame;
 use polars::series::IntoSeries;
 use polars_error::PolarsResult;
-use polars_error::signals::{KeyboardInterrupt, catch_keyboard_interrupt};
+use polars_error::abort::{QueryAborted, catch_polars_abort};
 
 use crate::ruby::exceptions::RbKeyboardInterrupt;
 use crate::ruby::gvl::GvlExt;
@@ -114,12 +114,16 @@ impl EnterPolarsExt for &Ruby {
         E: Send + Into<RbPolarsErr>,
     {
         let timeout = schedule_polars_timeout();
-        let ret = self.detach(|| catch_keyboard_interrupt(AssertUnwindSafe(f)));
+        let ret = self.detach(|| catch_polars_abort(AssertUnwindSafe(f)));
         cancel_polars_timeout(timeout);
         match ret {
             Ok(Ok(ret)) => Ok(ret),
             Ok(Err(err)) => Err(RbErr::from(err.into())),
-            Err(KeyboardInterrupt) => Err(RbKeyboardInterrupt::new_err("")),
+            Err(QueryAborted::KeyboardInterrupt) => Err(RbKeyboardInterrupt::new_err("")),
+            Err(QueryAborted::OocOutOfDisk) => Err(RbPolarsErr::Other(
+                "query aborted, raise POLARS_OOC_DISK_BUDGET_MB".to_string(),
+            )
+            .into()),
         }
     }
 }

data/lib/polars/array_expr.rb

@@ -776,7 +776,7 @@ module Polars
     #   df = Polars::DataFrame.new(
     #     {"a" => [[1, 2, 3], [4, 5, 6]]}, schema: {"a" => Polars::Array.new(Polars::Int64, 3)}
     #   )
-    #   df.select(Polars.col("a").arr.explode)
+    #   df.select(Polars.col("a").arr.explode(empty_as_null: false))
     #   # =>
     #   # shape: (6, 1)
     #   # ┌─────┐
@@ -791,8 +791,16 @@ module Polars
     #   # │ 5   │
     #   # │ 6   │
     #   # └─────┘
-    def explode(empty_as_null: true, keep_nulls: true)
-      Utils.wrap_expr(_rbexpr.explode(empty_as_null, keep_nulls))
+    def explode(empty_as_null: OMITTED, keep_nulls: true)
+      if empty_as_null == OMITTED
+        Utils.issue_deprecation_warning(
+          "The default behavior for `empty_as_null` will change to `false`. " +
+          "To keep the current behavior, explicitly set `empty_as_null: true`."
+        )
+        empty_as_null = true
+      end
+
+      Utils.wrap_expr(_rbexpr.arr_explode(empty_as_null, keep_nulls))
     end
 
     # Check if sub-arrays contain the given item.

data/lib/polars/array_name_space.rb

@@ -614,7 +614,7 @@ module Polars
     #
     # @example
     #   s = Polars::Series.new("a", [[1, 2, 3], [4, 5, 6]], dtype: Polars::Array.new(Polars::Int64, 3))
-    #   s.arr.explode
+    #   s.arr.explode(empty_as_null: false)
     #   # =>
     #   # shape: (6,)
     #   # Series: 'a' [i64]
@@ -626,7 +626,7 @@ module Polars
     #   #         5
     #   #         6
     #   # ]
-    def explode(empty_as_null: true, keep_nulls: true)
+    def explode(empty_as_null: nil, keep_nulls: true)
       super
     end
 

data/lib/polars/data_frame.rb

@@ -4535,7 +4535,7 @@ module Polars
     #       "numbers" => [[1], [2, 3], [4, 5], [6, 7, 8]]
     #     }
     #   )
-    #   df.explode("numbers")
+    #   df.explode("numbers", empty_as_null: false)
     #   # =>
     #   # shape: (8, 2)
     #   # ┌─────────┬─────────┐
@@ -4555,7 +4555,7 @@ module Polars
     def explode(
       columns,
       *more_columns,
-      empty_as_null: true,
+      empty_as_null: OMITTED,
       keep_nulls: true
     )
       lazy
@@ -4970,6 +4970,51 @@ module Polars
       lazy.shift(n, fill_value: fill_value).collect(optimizations: QueryOptFlags._eager)
     end
 
+    # Check whether the DataFrame is sorted by the given columns.
+    #
+    # @param by [Object]
+    #   Column name(s) to check.
+    # @param more_by [Array]
+    #   Additional column names.
+    # @param descending [Boolean]
+    #   Sort in descending order. When sorting by multiple columns, can be
+    #   specified per column by passing a sequence of booleans.
+    # @param nulls_last [Boolean]
+    #   Place null values last. When sorting by multiple columns, can be
+    #   specified per column by passing a sequence of booleans.
+    #
+    # @return [Boolean]
+    #
+    # @example
+    #   df = Polars::DataFrame.new({"a" => [1, 2, 3], "b" => [5, 4, 3]})
+    #   df.is_sorted("a")
+    #   # => true
+    #
+    # @example
+    #   df.is_sorted("b", descending: true)
+    #   # => true
+    #
+    # @example
+    #   df.is_sorted("a", "b")
+    #   # => true
+    def is_sorted(
+      by,
+      *more_by,
+      descending: false,
+      nulls_last: false
+    )
+      by = Array(by)
+      by.concat(more_by)
+      n = by.length
+      if Utils.bool?(descending)
+        descending = [descending] * n
+      end
+      if Utils.bool?(nulls_last)
+        nulls_last = [nulls_last] * n
+      end
+      _df.is_sorted(by, descending, nulls_last)
+    end
+
     # Get a mask of all duplicated rows in this DataFrame.
     #
     # @return [Series]

data/lib/polars/expr.rb

@@ -1576,7 +1576,7 @@ module Polars
     # @example Get the top 2 rows by column `a` in each group.
     #   df.group_by("c", maintain_order: true)
     #     .agg(Polars.all.top_k_by("a", k: 2))
-    #     .explode(Polars.all.exclude("c"))
+    #     .explode(Polars.all.exclude("c"), empty_as_null: false)
     #   # =>
     #   # shape: (5, 3)
     #   # ┌────────┬─────┬─────┐
@@ -1721,7 +1721,7 @@ module Polars
     # @example Get the bottom 2 rows by column `a` in each group.
     #   df.group_by("c", maintain_order: true)
     #     .agg(Polars.all.bottom_k_by("a", k: 2))
-    #     .explode(Polars.all.exclude("c"))
+    #     .explode(Polars.all.exclude("c"), empty_as_null: false)
     #   # =>
     #   # shape: (5, 3)
     #   # ┌────────┬─────┬─────┐
@@ -3677,7 +3677,7 @@ module Polars
     #
     # @example
     #   df = Polars::DataFrame.new({"b" => [[1, 2, 3], [4, 5, 6]]})
-    #   df.select(Polars.col("b").explode)
+    #   df.select(Polars.col("b").explode(empty_as_null: false))
     #   # =>
     #   # shape: (6, 1)
     #   # ┌─────┐
@@ -3692,7 +3692,15 @@ module Polars
     #   # │ 5   │
     #   # │ 6   │
     #   # └─────┘
-    def explode(empty_as_null: true, keep_nulls: true)
+    def explode(empty_as_null: OMITTED, keep_nulls: true)
+      if empty_as_null == OMITTED
+        Utils.issue_deprecation_warning(
+          "The default behavior for `empty_as_null` will change to `false`. " +
+          "To keep the current behavior, explicitly set `empty_as_null: true`."
+        )
+        empty_as_null = true
+      end
+
       wrap_expr(_rbexpr.explode(empty_as_null, keep_nulls))
     end
 
@@ -4604,6 +4612,66 @@ module Polars
       wrap_expr(_rbexpr.is_close(other, abs_tol, rel_tol, nans_equal))
     end
 
+    # Checks if an expression is sorted.
+    #
+    # If `descending` and/or `nulls_last` are nil, it will check `true` and `false`
+    # for the unspecified option(s), and return `true` if the expression is sorted
+    # under any combination of those settings.
+    #
+    # @param descending [Boolean]
+    #   Checks if the expression is sorted in descending order.
+    #   Defaults to false.
+    # @param nulls_last [Boolean]
+    #   Consider null values as being ordered last when checking sortedness.
+    #   Defaults to false.
+    #
+    # @return [Expr]
+    #
+    # @example Check if a column is sorted in ascending order.
+    #   df = Polars::DataFrame.new({"a" => [1, 2, 3, 4]})
+    #   df.select(Polars.col("a").is_sorted)
+    #   # =>
+    #   # shape: (1, 1)
+    #   # ┌──────┐
+    #   # │ a    │
+    #   # │ ---  │
+    #   # │ bool │
+    #   # ╞══════╡
+    #   # │ true │
+    #   # └──────┘
+    #
+    # @example Check if a column is sorted in descending order.
+    #   df = Polars::DataFrame.new({"a" => [4, 3, 2, 1]})
+    #   df.select(Polars.col("a").is_sorted(descending: true))
+    #   # =>
+    #   # shape: (1, 1)
+    #   # ┌──────┐
+    #   # │ a    │
+    #   # │ ---  │
+    #   # │ bool │
+    #   # ╞══════╡
+    #   # │ true │
+    #   # └──────┘
+    #
+    # @example Check if a column is sorted in either direction.
+    #   df = Polars::DataFrame.new({"a" => [4, 3, 2, 1]})
+    #   df.select(Polars.col("a").is_sorted(descending: nil))
+    #   # =>
+    #   # shape: (1, 1)
+    #   # ┌──────┐
+    #   # │ a    │
+    #   # │ ---  │
+    #   # │ bool │
+    #   # ╞══════╡
+    #   # │ true │
+    #   # └──────┘
+    def is_sorted(
+      descending: false,
+      nulls_last: false
+    )
+      wrap_expr(_rbexpr.is_sorted(descending, nulls_last))
+    end
+
     # Hash the elements in the selection.
     #
     # The hash value is of type `UInt64`.
@@ -7583,7 +7651,7 @@ module Polars
     #   # │ ---      │
     #   # │ f64      │
     #   # ╞══════════╡
-    #   # │ 0.0      │
+    #   # │ null     │
     #   # │ 0.707107 │
     #   # │ 0.963624 │
     #   # └──────────┘
@@ -7615,7 +7683,7 @@ module Polars
     #   # │ ---      │
     #   # │ f64      │
     #   # ╞══════════╡
-    #   # │ 0.0      │
+    #   # │ null     │
     #   # │ 0.5      │
     #   # │ 0.928571 │
     #   # └──────────┘

data/lib/polars/functions/eager.rb

@@ -95,7 +95,7 @@ module Polars
     #   # │ 2   ┆ 4    ┆ 5    ┆ null │
     #   # │ 3   ┆ null ┆ 6    ┆ 8    │
     #   # └─────┴──────┴──────┴──────┘
-    def concat(items, rechunk: false, how: "vertical", parallel: true, strict: false)
+    def concat(items, rechunk: false, how: "vertical", parallel: true, strict: nil)
       elems = items.to_a
 
       if elems.empty?
@@ -303,7 +303,7 @@ module Polars
     def union(
       items,
       how: "vertical",
-      strict: false
+      strict: nil
     )
       elems = items.to_a