polars-df 0.25.1 → 0.26.1

data/ext/polars/src/interop/numo/to_numo_series.rs

@@ -1,4 +1,4 @@
-use magnus::Value;
+use magnus::{Ruby, Value};
 use num_traits::{Float, NumCast};
 use polars_core::prelude::*;
 
@@ -10,34 +10,35 @@ use crate::series::RbSeries;
 
 impl RbSeries {
     /// Convert this Series to a Numo array.
-    pub fn to_numo(&self) -> RbResult<Value> {
-        series_to_numo(&self.series.read())
+    pub fn to_numo(rb: &Ruby, self_: &Self) -> RbResult<Value> {
+        series_to_numo(rb, &self_.series.read())
     }
 }
 
 /// Convert a Series to a Numo array.
-fn series_to_numo(s: &Series) -> RbResult<Value> {
-    series_to_numo_with_copy(s)
+fn series_to_numo(rb: &Ruby, s: &Series) -> RbResult<Value> {
+    series_to_numo_with_copy(rb, s)
 }
 
 /// Convert a Series to a Numo array, copying data in the process.
-fn series_to_numo_with_copy(s: &Series) -> RbResult<Value> {
+fn series_to_numo_with_copy(rb: &Ruby, s: &Series) -> RbResult<Value> {
     use DataType::*;
     match s.dtype() {
-        Int8 => numeric_series_to_numpy::<Int8Type, f32>(s),
-        Int16 => numeric_series_to_numpy::<Int16Type, f32>(s),
-        Int32 => numeric_series_to_numpy::<Int32Type, f64>(s),
-        Int64 => numeric_series_to_numpy::<Int64Type, f64>(s),
-        UInt8 => numeric_series_to_numpy::<UInt8Type, f32>(s),
-        UInt16 => numeric_series_to_numpy::<UInt16Type, f32>(s),
-        UInt32 => numeric_series_to_numpy::<UInt32Type, f64>(s),
-        UInt64 => numeric_series_to_numpy::<UInt64Type, f64>(s),
-        Float32 => numeric_series_to_numpy::<Float32Type, f32>(s),
-        Float64 => numeric_series_to_numpy::<Float64Type, f64>(s),
-        Boolean => boolean_series_to_numo(s),
+        Int8 => numeric_series_to_numo::<Int8Type, f32>(rb, s),
+        Int16 => numeric_series_to_numo::<Int16Type, f32>(rb, s),
+        Int32 => numeric_series_to_numo::<Int32Type, f64>(rb, s),
+        Int64 => numeric_series_to_numo::<Int64Type, f64>(rb, s),
+        UInt8 => numeric_series_to_numo::<UInt8Type, f32>(rb, s),
+        UInt16 => numeric_series_to_numo::<UInt16Type, f32>(rb, s),
+        UInt32 => numeric_series_to_numo::<UInt32Type, f64>(rb, s),
+        UInt64 => numeric_series_to_numo::<UInt64Type, f64>(rb, s),
+        Float32 => numeric_series_to_numo::<Float32Type, f32>(rb, s),
+        Float64 => numeric_series_to_numo::<Float64Type, f64>(rb, s),
+        Boolean => boolean_series_to_numo(rb, s),
         String => {
             let ca = s.str().unwrap();
-            RbArray1::from_iter(ca)
+            let values = ca.iter();
+            RbArray1::from_iter(rb, values)
         }
         dt => {
             raise_err!(
@@ -49,7 +50,7 @@ fn series_to_numo_with_copy(s: &Series) -> RbResult<Value> {
 }
 
 /// Convert numeric types to f32 or f64 with NaN representing a null value.
-fn numeric_series_to_numpy<T, U>(s: &Series) -> RbResult<Value>
+fn numeric_series_to_numo<T, U>(rb: &Ruby, s: &Series) -> RbResult<Value>
 where
     T: PolarsNumericType,
     T::Native: Element,
@@ -58,25 +59,25 @@ where
     let ca: &ChunkedArray<T> = s.as_ref().as_ref();
     if s.null_count() == 0 {
         let values = ca.into_no_null_iter();
-        RbArray1::<T::Native>::from_iter(values)
+        RbArray1::<T::Native>::from_iter(rb, values)
     } else {
         let mapper = |opt_v: Option<T::Native>| match opt_v {
             Some(v) => NumCast::from(v).unwrap(),
             None => U::nan(),
         };
         let values = ca.iter().map(mapper);
-        RbArray1::from_iter(values)
+        RbArray1::from_iter(rb, values)
     }
 }
 
 /// Convert booleans to bit if no nulls are present, otherwise convert to objects.
-fn boolean_series_to_numo(s: &Series) -> RbResult<Value> {
+fn boolean_series_to_numo(rb: &Ruby, s: &Series) -> RbResult<Value> {
     let ca = s.bool().unwrap();
     if s.null_count() == 0 {
-        let values = ca.into_no_null_iter();
-        RbArray1::<bool>::from_iter(values)
+        let values = ca.no_null_iter();
+        RbArray1::<bool>::from_iter(rb, values)
     } else {
         let values = ca.iter();
-        RbArray1::from_iter(values)
+        RbArray1::from_iter(rb, values)
     }
 }

data/ext/polars/src/io/scan_options.rs

@@ -11,9 +11,9 @@ use polars_io::{HiveOptions, RowIndex};
 use polars_utils::IdxSize;
 use polars_utils::slice_enum::Slice;
 
-use crate::RbResult;
 use crate::io::cloud_options::OptRbCloudOptions;
 use crate::prelude::Wrap;
+use crate::{RbDataFrame, RbResult};
 
 /// Interface to `class ScanOptions` on the Ruby side
 pub struct RbScanOptions(Value);
@@ -25,8 +25,11 @@ impl TryConvert for RbScanOptions {
 }
 
 impl TryConvert for Wrap<TableStatistics> {
-    fn try_convert(_ob: Value) -> RbResult<Self> {
-        todo!();
+    fn try_convert(ob: Value) -> RbResult<Self> {
+        let attr: Value = ob.funcall("_df", ())?;
+        Ok(Wrap(TableStatistics(Arc::new(
+            <&RbDataFrame>::try_convert(attr)?.clone().df.into_inner(),
+        ))))
     }
 }
 

data/ext/polars/src/io/sink_options.rs

@@ -26,6 +26,7 @@ impl RbSinkOptions {
         let sync_on_close: Option<Wrap<SyncOnCloseType>> = self.0.funcall("sync_on_close", ())?;
         let storage_options: OptRbCloudOptions = self.0.funcall("storage_options", ())?;
         let credential_provider: Option<Value> = self.0.funcall("credential_provider", ())?;
+        let sinked_paths_callback: Option<Value> = self.0.funcall("sinked_paths_callback", ())?;
 
         let cloud_options =
             storage_options.extract_opt_cloud_options(cloud_scheme, credential_provider)?;
@@ -37,6 +38,7 @@ impl RbSinkOptions {
             maintain_order,
             sync_on_close,
             cloud_options: cloud_options.map(Arc::new),
+            sinked_paths_callback: sinked_paths_callback.map(|_| todo!()),
         };
 
         Ok(unified_sink_args)

data/ext/polars/src/lazyframe/general.rs

@@ -3,11 +3,14 @@ use magnus::{
     IntoValue, RArray, RHash, Ruby, TryConvert, Value, r_hash::ForEach, value::ReprValue,
 };
 use parking_lot::Mutex;
+use polars::frame::PivotColumnNaming;
 use polars::io::RowIndex;
 use polars::lazy::frame::LazyFrame;
 use polars::prelude::*;
+use polars_core::query_result::QueryResult;
 use polars_plan::dsl::ScanSources;
 use polars_plan::plans::{HintIR, Sorted};
+use std::ffi::CString;
 use std::num::NonZeroUsize;
 
 use super::{RbLazyFrame, RbOptFlags};
@@ -22,10 +25,10 @@ use crate::ruby::gvl::GvlExt;
 use crate::ruby::lazy::RubyUdfLazyFrameExt;
 use crate::ruby::plan_callback::PlanCallbackExt;
 use crate::ruby::ruby_function::RubyObject;
+use crate::ruby::utils::TryIntoValue;
 use crate::utils::{EnterPolarsExt, to_rb_err};
 use crate::{
-    RbArrowArrayStream, RbDataFrame, RbExpr, RbLazyGroupBy, RbPolarsErr, RbResult, RbTypeError,
-    RbValueError,
+    RbCapsule, RbDataFrame, RbExpr, RbLazyGroupBy, RbPolarsErr, RbResult, RbTypeError, RbValueError,
 };
 
 fn rbobject_to_first_path_and_scan_sources(
@@ -134,6 +137,7 @@ impl RbLazyFrame {
         let cloud_options = OptRbCloudOptions::try_convert(arguments[28])?;
         let credential_provider = Option::<Value>::try_convert(arguments[29])?;
         let include_file_paths = Option::<String>::try_convert(arguments[30])?;
+        let missing_columns = Option::<Wrap<MissingColumnsPolicy>>::try_convert(arguments[31])?;
         // end arguments
 
         let null_values = null_values.map(|w| w.0);
@@ -195,7 +199,8 @@ impl RbLazyFrame {
             .with_decimal_comma(decimal_comma)
             .with_glob(glob)
             .with_raise_if_empty(raise_if_empty)
-            .with_include_file_paths(include_file_paths.map(|x| x.into()));
+            .with_include_file_paths(include_file_paths.map(|x| x.into()))
+            .with_missing_columns_policy(missing_columns.map(|x| x.0));
 
         if let Some(lambda) = with_schema_modify {
             let f = |schema: Schema| {
@@ -413,7 +418,11 @@ impl RbLazyFrame {
     pub fn collect(rb: &Ruby, self_: &Self, engine: Wrap<Engine>) -> RbResult<RbDataFrame> {
         rb.enter_polars_df(|| {
             let ldf = self_.ldf.read().clone();
-            ldf.collect_with_engine(engine.0)
+            ldf.collect_with_engine(engine.0).map(|r| match r {
+                QueryResult::Single(df) => df,
+                // TODO: Should return query results
+                QueryResult::Multiple(_) => DataFrame::empty(),
+            })
         })
     }
 
@@ -810,6 +819,12 @@ impl RbLazyFrame {
             .into())
     }
 
+    pub fn gather(&self, idxs: &Self, null_on_oob: bool) -> Self {
+        let ldf = self.ldf.read().clone();
+        let idxs = idxs.clone().ldf.into_inner();
+        ldf.gather(idxs, null_on_oob).into()
+    }
+
     pub fn with_column(&self, expr: &RbExpr) -> Self {
         let ldf = self.ldf.read().clone();
         ldf.with_column(expr.inner.clone()).into()
@@ -1090,6 +1105,7 @@ impl RbLazyFrame {
         agg: &RbExpr,
         maintain_order: bool,
         separator: String,
+        column_naming: Wrap<PivotColumnNaming>,
     ) -> Self {
         let ldf = self.ldf.read().clone();
         ldf.pivot(
@@ -1100,6 +1116,7 @@ impl RbLazyFrame {
             agg.inner.clone(),
             maintain_order,
             separator.into(),
+            column_naming.0,
         )
         .into()
     }
@@ -1141,7 +1158,7 @@ impl RbLazyFrame {
         opt.set(OptFlags::PREDICATE_PUSHDOWN, predicate_pushdown);
         opt.set(OptFlags::PROJECTION_PUSHDOWN, projection_pushdown);
         opt.set(OptFlags::SLICE_PUSHDOWN, slice_pushdown);
-        opt.set(OptFlags::NEW_STREAMING, streamable);
+        opt.set(OptFlags::STREAMING, streamable);
 
         self.ldf
             .read()
@@ -1182,14 +1199,12 @@ impl RbLazyFrame {
         let schema = rb.enter_polars(|| self_.ldf.write().collect_schema())?;
 
         let schema_dict = rb.hash_new();
-        schema.iter_fields().for_each(|fld| {
-            schema_dict
-                .aset::<String, Value>(
-                    fld.name().to_string(),
-                    Wrap(fld.dtype().clone()).into_value_with(rb),
-                )
-                .unwrap();
-        });
+        for fld in schema.iter_fields() {
+            schema_dict.aset(
+                fld.name().to_string(),
+                Wrap(fld.dtype().clone()).try_into_value_with(rb)?,
+            )?;
+        }
         Ok(schema_dict)
     }
 
@@ -1209,12 +1224,12 @@ impl RbLazyFrame {
         ldf.count().into()
     }
 
-    pub fn merge_sorted(&self, other: &Self, key: String) -> RbResult<Self> {
+    pub fn merge_sorted(&self, other: &Self, key: String, maintain_order: bool) -> RbResult<Self> {
         let out = self
             .ldf
             .read()
             .clone()
-            .merge_sorted(other.ldf.read().clone(), &key)
+            .merge_sorted(other.ldf.read().clone(), &key, maintain_order)
             .map_err(RbPolarsErr::from)?;
         Ok(out.into())
     }
@@ -1299,7 +1314,8 @@ impl RbCollectBatches {
         let iter = Box::new(ArrowStreamIterator::new(self_.inner.clone(), dtype.clone()));
         let field = ArrowField::new(PlSmallStr::EMPTY, dtype, false);
         let stream = export_iterator(iter, field);
-        Ok(RbArrowArrayStream { stream }.into_value_with(rb))
+        let stream_capsule_name = CString::new("arrow_array_stream").unwrap();
+        Ok(RbCapsule::new(stream, Some(stream_capsule_name)).into_value_with(rb))
     }
 }
 

data/ext/polars/src/lazyframe/optflags.rs

@@ -52,7 +52,8 @@ flag_getter_setters! {
     (COMM_SUBEXPR_ELIM, get_comm_subexpr_elim, set_comm_subexpr_elim, clear=true)
     (CHECK_ORDER_OBSERVE, get_check_order_observe, set_check_order_observe, clear=true)
     (FAST_PROJECTION, get_fast_projection, set_fast_projection, clear=true)
+    (SORT_COLLAPSE, get_sort_collapse, set_sort_collapse, clear=true)
 
     (EAGER, get_eager, set_eager, clear=true)
-    (NEW_STREAMING, get_streaming, set_streaming, clear=true)
+    (STREAMING, get_streaming, set_streaming, clear=true)
 }

data/ext/polars/src/lib.rs

@@ -25,6 +25,10 @@ mod testing;
 mod timeout;
 mod utils;
 
+use crate::conversion::Wrap;
+
+pub type RbDataType = Wrap<polars_core::datatypes::DataType>;
+
 use catalog::unity::RbCatalogClient;
 use conversion::*;
 use dataframe::RbDataFrame;
@@ -32,12 +36,12 @@ use error::RbPolarsErr;
 use expr::RbExpr;
 use expr::datatype::RbDataTypeExpr;
 use expr::selector::RbSelector;
-use functions::string_cache::RbStringCacheHolder;
 use functions::whenthen::{RbChainedThen, RbChainedWhen, RbThen, RbWhen};
-use interop::arrow::to_rb::{RbArrowArrayStream, RbArrowSchema};
+use interop::arrow::to_rb::RbArrowSchema;
 use lazyframe::{RbCollectBatches, RbInProcessQuery, RbLazyFrame, RbOptFlags};
 use lazygroupby::RbLazyGroupBy;
 use magnus::{Ruby, function, method, prelude::*};
+use ruby::capsule::RbCapsule;
 use ruby::exceptions::{RbTypeError, RbValueError};
 use series::RbSeries;
 use sql::RbSQLContext;
@@ -203,7 +207,7 @@ fn init(ruby: &Ruby) -> RbResult<()> {
     class.define_method("first", method!(RbExpr::first, 1))?;
     class.define_method("last", method!(RbExpr::last, 1))?;
     class.define_method("item", method!(RbExpr::item, 1))?;
-    class.define_method("implode", method!(RbExpr::implode, 0))?;
+    class.define_method("implode", method!(RbExpr::implode, 1))?;
     class.define_method("quantile", method!(RbExpr::quantile, 2))?;
     class.define_method("cut", method!(RbExpr::cut, 4))?;
     class.define_method("qcut", method!(RbExpr::qcut, 5))?;
@@ -229,7 +233,7 @@ fn init(ruby: &Ruby) -> RbResult<()> {
     class.define_method("arg_min", method!(RbExpr::arg_min, 0))?;
     class.define_method("index_of", method!(RbExpr::index_of, 1))?;
     class.define_method("search_sorted", method!(RbExpr::search_sorted, 3))?;
-    class.define_method("gather", method!(RbExpr::gather, 1))?;
+    class.define_method("gather", method!(RbExpr::gather, 2))?;
     class.define_method("get", method!(RbExpr::get, 2))?;
     class.define_method("sort_by", method!(RbExpr::sort_by, 5))?;
     class.define_method("shift", method!(RbExpr::shift, 2))?;
@@ -260,6 +264,7 @@ fn init(ruby: &Ruby) -> RbResult<()> {
     class.define_method("rechunk", method!(RbExpr::rechunk, 0))?;
     class.define_method("round", method!(RbExpr::round, 2))?;
     class.define_method("round_sig_figs", method!(RbExpr::round_sig_figs, 1))?;
+    class.define_method("truncate", method!(RbExpr::truncate, 1))?;
     class.define_method("floor", method!(RbExpr::floor, 0))?;
     class.define_method("ceil", method!(RbExpr::ceil, 0))?;
     class.define_method("clip", method!(RbExpr::clip, 2))?;
@@ -337,13 +342,8 @@ fn init(ruby: &Ruby) -> RbResult<()> {
     class.define_method("arr_var", method!(RbExpr::arr_var, 1))?;
     class.define_method("arr_mean", method!(RbExpr::arr_mean, 0))?;
     class.define_method("arr_median", method!(RbExpr::arr_median, 0))?;
-    class.define_method("arr_unique", method!(RbExpr::arr_unique, 1))?;
-    class.define_method("arr_n_unique", method!(RbExpr::arr_n_unique, 0))?;
     class.define_method("arr_to_list", method!(RbExpr::arr_to_list, 0))?;
-    class.define_method("arr_all", method!(RbExpr::arr_all, 0))?;
-    class.define_method("arr_any", method!(RbExpr::arr_any, 0))?;
     class.define_method("arr_sort", method!(RbExpr::arr_sort, 2))?;
-    class.define_method("arr_reverse", method!(RbExpr::arr_reverse, 0))?;
     class.define_method("arr_arg_min", method!(RbExpr::arr_arg_min, 0))?;
     class.define_method("arr_arg_max", method!(RbExpr::arr_arg_max, 0))?;
     class.define_method("arr_get", method!(RbExpr::arr_get, 2))?;
@@ -479,7 +479,7 @@ fn init(ruby: &Ruby) -> RbResult<()> {
     class.define_method("dt_replace", method!(RbExpr::dt_replace, 8))?;
     class.define_method("dt_combine", method!(RbExpr::dt_combine, 2))?;
     class.define_method("dot", method!(RbExpr::dot, 1))?;
-    class.define_method("reinterpret", method!(RbExpr::reinterpret, 1))?;
+    class.define_method("reinterpret", method!(RbExpr::reinterpret, 2))?;
     class.define_method("mode", method!(RbExpr::mode, 1))?;
     class.define_method("interpolate", method!(RbExpr::interpolate, 1))?;
     class.define_method("interpolate_by", method!(RbExpr::interpolate_by, 1))?;
@@ -527,16 +527,11 @@ fn init(ruby: &Ruby) -> RbResult<()> {
     class.define_method("list_var", method!(RbExpr::list_var, 1))?;
     class.define_method("list_tail", method!(RbExpr::list_tail, 1))?;
     class.define_method("list_sort", method!(RbExpr::list_sort, 2))?;
-    class.define_method("list_reverse", method!(RbExpr::list_reverse, 0))?;
-    class.define_method("list_n_unique", method!(RbExpr::list_n_unique, 0))?;
-    class.define_method("list_unique", method!(RbExpr::list_unique, 1))?;
     class.define_method("list_set_operation", method!(RbExpr::list_set_operation, 2))?;
     class.define_method("list_get", method!(RbExpr::list_get, 2))?;
     class.define_method("list_join", method!(RbExpr::list_join, 2))?;
     class.define_method("list_arg_min", method!(RbExpr::list_arg_min, 0))?;
     class.define_method("list_arg_max", method!(RbExpr::list_arg_max, 0))?;
-    class.define_method("list_all", method!(RbExpr::list_all, 0))?;
-    class.define_method("list_any", method!(RbExpr::list_any, 0))?;
     class.define_method("list_diff", method!(RbExpr::list_diff, 2))?;
     class.define_method("list_shift", method!(RbExpr::list_shift, 1))?;
     class.define_method("list_slice", method!(RbExpr::list_slice, 2))?;
@@ -570,6 +565,7 @@ fn init(ruby: &Ruby) -> RbResult<()> {
     class.define_method("extend_constant", method!(RbExpr::extend_constant, 2))?;
     class.define_method("any", method!(RbExpr::any, 1))?;
     class.define_method("all", method!(RbExpr::all, 1))?;
+    class.define_method("is_empty", method!(RbExpr::is_empty, 1))?;
     class.define_method(
         "struct_field_by_name",
         method!(RbExpr::struct_field_by_name, 1),
@@ -593,7 +589,7 @@ fn init(ruby: &Ruby) -> RbResult<()> {
     class.define_method("exp", method!(RbExpr::exp, 0))?;
     class.define_method("entropy", method!(RbExpr::entropy, 2))?;
     class.define_method("_hash", method!(RbExpr::hash, 4))?;
-    class.define_method("set_sorted_flag", method!(RbExpr::set_sorted_flag, 1))?;
+    class.define_method("set_sorted_flag", method!(RbExpr::set_sorted_flag, 2))?;
     class.define_method("replace", method!(RbExpr::replace, 2))?;
     class.define_method("replace_strict", method!(RbExpr::replace_strict, 4))?;
     class.define_method("hist", method!(RbExpr::hist, 4))?;
@@ -801,18 +797,6 @@ fn init(ruby: &Ruby) -> RbResult<()> {
         "thread_pool_size",
         function!(functions::meta::thread_pool_size, 0),
     )?;
-    class.define_singleton_method(
-        "enable_string_cache",
-        function!(functions::string_cache::enable_string_cache, 0),
-    )?;
-    class.define_singleton_method(
-        "disable_string_cache",
-        function!(functions::string_cache::disable_string_cache, 0),
-    )?;
-    class.define_singleton_method(
-        "using_string_cache",
-        function!(functions::string_cache::using_string_cache, 0),
-    )?;
     class.define_singleton_method(
         "set_float_fmt",
         function!(functions::meta::set_float_fmt, 1),
@@ -961,6 +945,7 @@ fn init(ruby: &Ruby) -> RbResult<()> {
     class.define_method("join_asof", method!(RbLazyFrame::join_asof, 14))?;
     class.define_method("join", method!(RbLazyFrame::join, 11))?;
     class.define_method("join_where", method!(RbLazyFrame::join_where, 3))?;
+    class.define_method("gather", method!(RbLazyFrame::gather, 2))?;
     class.define_method("with_column", method!(RbLazyFrame::with_column, 1))?;
     class.define_method("with_columns", method!(RbLazyFrame::with_columns, 1))?;
     class.define_method(
@@ -991,7 +976,7 @@ fn init(ruby: &Ruby) -> RbResult<()> {
     class.define_method("drop_nulls", method!(RbLazyFrame::drop_nulls, 1))?;
     class.define_method("slice", method!(RbLazyFrame::slice, 2))?;
     class.define_method("tail", method!(RbLazyFrame::tail, 1))?;
-    class.define_method("pivot", method!(RbLazyFrame::pivot, 7))?;
+    class.define_method("pivot", method!(RbLazyFrame::pivot, 8))?;
     class.define_method("unpivot", method!(RbLazyFrame::unpivot, 4))?;
     class.define_method("with_row_index", method!(RbLazyFrame::with_row_index, 2))?;
     class.define_method("map_batches", method!(RbLazyFrame::map_batches, 7))?;
@@ -1002,7 +987,7 @@ fn init(ruby: &Ruby) -> RbResult<()> {
     class.define_method("collect_schema", method!(RbLazyFrame::collect_schema, 0))?;
     class.define_method("unnest", method!(RbLazyFrame::unnest, 2))?;
     class.define_method("count", method!(RbLazyFrame::count, 0))?;
-    class.define_method("merge_sorted", method!(RbLazyFrame::merge_sorted, 2))?;
+    class.define_method("merge_sorted", method!(RbLazyFrame::merge_sorted, 3))?;
     class.define_method("hint_sorted", method!(RbLazyFrame::hint_sorted, 3))?;
     class.define_method(
         "collect_concurrently",
@@ -1124,6 +1109,7 @@ fn init(ruby: &Ruby) -> RbResult<()> {
     class.define_method("slice", method!(RbSeries::slice, 2))?;
     class.define_method("any", method!(RbSeries::any, 1))?;
     class.define_method("all", method!(RbSeries::all, 1))?;
+    class.define_method("is_empty", method!(RbSeries::is_empty, 1))?;
     class.define_method("arg_min", method!(RbSeries::arg_min, 0))?;
     class.define_method("arg_max", method!(RbSeries::arg_max, 0))?;
     class.define_method(
@@ -1150,6 +1136,7 @@ fn init(ruby: &Ruby) -> RbResult<()> {
     class.define_method("to_s", method!(RbSeries::as_str, 0))?;
     class.define_method("len", method!(RbSeries::len, 0))?;
     class.define_method("to_a", method!(RbSeries::to_a, 0))?;
+    class.define_method("arrow_c_stream", method!(RbSeries::__arrow_c_stream__, 0))?;
     class.define_method("median", method!(RbSeries::median, 0))?;
     class.define_method("quantile", method!(RbSeries::quantile, 2))?;
     class.define_method("_clone", method!(RbSeries::clone, 0))?;
@@ -1355,13 +1342,10 @@ fn init(ruby: &Ruby) -> RbResult<()> {
     class.define_method("register", method!(RbSQLContext::register, 2))?;
     class.define_method("unregister", method!(RbSQLContext::unregister, 1))?;
 
-    // string cache
-    let class = module.define_class("RbStringCacheHolder", ruby.class_object())?;
-    class.define_singleton_method("hold", function!(RbStringCacheHolder::hold, 0))?;
-
-    // arrow array stream
-    let class = module.define_class("ArrowArrayStream", ruby.class_object())?;
-    class.define_method("to_i", method!(RbArrowArrayStream::to_i, 0))?;
+    // capsule
+    let class = module.define_class("Capsule", ruby.class_object())?;
+    class.define_method("to_i", method!(RbCapsule::to_i, 0))?;
+    class.define_method("name", method!(RbCapsule::name, 0))?;
 
     // arrow schema
     let class = module.define_class("ArrowSchema", ruby.class_object())?;

data/ext/polars/src/map/lazy.rs

@@ -4,7 +4,7 @@ use polars::prelude::*;
 use crate::expr::ToExprs;
 use crate::expr::datatype::RbDataTypeExpr;
 use crate::ruby::ruby_udf::{RubyUdfExpression, RubyUdfExt};
-use crate::ruby::utils::to_pl_err;
+use crate::ruby::utils::{TryIntoValue, to_pl_err};
 use crate::{RbExpr, RbResult, RbSeries, Wrap};
 
 pub(crate) fn call_lambda_with_series(
@@ -17,7 +17,10 @@ pub(crate) fn call_lambda_with_series(
 
     // Set return_dtype in kwargs
     let dict = rb.hash_new();
-    let output_dtype = output_dtype.map(Wrap);
+    let output_dtype = output_dtype
+        .map(|v| Wrap(v).try_into_value_with(rb))
+        .transpose()
+        .map_err(to_pl_err)?;
     dict.aset(rb.sym_new("return_dtype"), output_dtype)
         .map_err(to_pl_err)?;
 

data/ext/polars/src/map/series.rs

@@ -4,6 +4,7 @@ use super::*;
 use crate::Wrap;
 use crate::error::RbPolarsErr;
 use crate::prelude::ObjectValue;
+use crate::ruby::utils::TryIntoValue;
 
 pub trait ApplyLambdaGeneric {
     fn apply_generic(&self, rb: &Ruby, lambda: Value, skip_nulls: bool) -> RbResult<Series>;
@@ -25,7 +26,7 @@ fn call_and_collect_anyvalues<T, I>(
     skip_nulls: bool,
 ) -> RbResult<Vec<AnyValue<'static>>>
 where
-    T: IntoValue,
+    T: TryIntoValue,
     I: Iterator<Item = Option<T>>,
 {
     let mut avs = Vec::with_capacity(len);
@@ -36,7 +37,7 @@ where
                 continue;
             }
             None => rb.qnil().into_value_with(rb),
-            Some(val) => val.into_value_with(rb),
+            Some(val) => val.try_into_value_with(rb)?,
         };
         let av: Option<Wrap<AnyValue>> = lambda.funcall("call", (arg,))?;
         avs.push(av.map(|w| w.0).unwrap_or(AnyValue::Null));
@@ -46,7 +47,7 @@ where
 
 impl ApplyLambdaGeneric for BooleanChunked {
     fn apply_generic(&self, rb: &Ruby, lambda: Value, skip_nulls: bool) -> RbResult<Series> {
-        let avs = call_and_collect_anyvalues(rb, lambda, self.len(), self.into_iter(), skip_nulls)?;
+        let avs = call_and_collect_anyvalues(rb, lambda, self.len(), self.iter(), skip_nulls)?;
         Ok(Series::from_any_values(self.name().clone(), &avs, true).map_err(RbPolarsErr::from)?)
     }
 
@@ -57,7 +58,7 @@ impl ApplyLambdaGeneric for BooleanChunked {
         datatype: &DataType,
         skip_nulls: bool,
     ) -> RbResult<Series> {
-        let avs = call_and_collect_anyvalues(rb, lambda, self.len(), self.into_iter(), skip_nulls)?;
+        let avs = call_and_collect_anyvalues(rb, lambda, self.len(), self.iter(), skip_nulls)?;
         Ok(
             Series::from_any_values_and_dtype(self.name().clone(), &avs, datatype, true)
                 .map_err(RbPolarsErr::from)?,
@@ -71,7 +72,7 @@ where
     T::Native: IntoValue + TryConvert,
 {
     fn apply_generic(&self, rb: &Ruby, lambda: Value, skip_nulls: bool) -> RbResult<Series> {
-        let avs = call_and_collect_anyvalues(rb, lambda, self.len(), self.into_iter(), skip_nulls)?;
+        let avs = call_and_collect_anyvalues(rb, lambda, self.len(), self.iter(), skip_nulls)?;
         Ok(Series::from_any_values(self.name().clone(), &avs, true).map_err(RbPolarsErr::from)?)
     }
 
@@ -82,7 +83,7 @@ where
         datatype: &DataType,
         skip_nulls: bool,
     ) -> RbResult<Series> {
-        let avs = call_and_collect_anyvalues(rb, lambda, self.len(), self.into_iter(), skip_nulls)?;
+        let avs = call_and_collect_anyvalues(rb, lambda, self.len(), self.iter(), skip_nulls)?;
         Ok(
             Series::from_any_values_and_dtype(self.name().clone(), &avs, datatype, true)
                 .map_err(RbPolarsErr::from)?,
@@ -92,7 +93,7 @@ where
 
 impl ApplyLambdaGeneric for StringChunked {
     fn apply_generic(&self, rb: &Ruby, lambda: Value, skip_nulls: bool) -> RbResult<Series> {
-        let avs = call_and_collect_anyvalues(rb, lambda, self.len(), self.into_iter(), skip_nulls)?;
+        let avs = call_and_collect_anyvalues(rb, lambda, self.len(), self.iter(), skip_nulls)?;
         Ok(Series::from_any_values(self.name().clone(), &avs, true).map_err(RbPolarsErr::from)?)
     }
 
@@ -103,7 +104,7 @@ impl ApplyLambdaGeneric for StringChunked {
         datatype: &DataType,
         skip_nulls: bool,
     ) -> RbResult<Series> {
-        let avs = call_and_collect_anyvalues(rb, lambda, self.len(), self.into_iter(), skip_nulls)?;
+        let avs = call_and_collect_anyvalues(rb, lambda, self.len(), self.iter(), skip_nulls)?;
         Ok(
             Series::from_any_values_and_dtype(self.name().clone(), &avs, datatype, true)
                 .map_err(RbPolarsErr::from)?,
@@ -113,7 +114,7 @@ impl ApplyLambdaGeneric for StringChunked {
 
 impl ApplyLambdaGeneric for ListChunked {
     fn apply_generic(&self, rb: &Ruby, lambda: Value, skip_nulls: bool) -> RbResult<Series> {
-        let it = self.into_iter().map(|opt_s| opt_s.map(Wrap));
+        let it = self.series_iter().map(|opt_s| opt_s.map(Wrap));
         let avs = call_and_collect_anyvalues(rb, lambda, self.len(), it, skip_nulls)?;
         Ok(Series::from_any_values(self.name().clone(), &avs, true).map_err(RbPolarsErr::from)?)
     }
@@ -125,7 +126,7 @@ impl ApplyLambdaGeneric for ListChunked {
         datatype: &DataType,
         skip_nulls: bool,
     ) -> RbResult<Series> {
-        let it = self.into_iter().map(|opt_s| opt_s.map(Wrap));
+        let it = self.series_iter().map(|opt_s| opt_s.map(Wrap));
         let avs = call_and_collect_anyvalues(rb, lambda, self.len(), it, skip_nulls)?;
         Ok(
             Series::from_any_values_and_dtype(self.name().clone(), &avs, datatype, true)
@@ -136,7 +137,7 @@ impl ApplyLambdaGeneric for ListChunked {
 
 impl ApplyLambdaGeneric for ArrayChunked {
     fn apply_generic(&self, rb: &Ruby, lambda: Value, skip_nulls: bool) -> RbResult<Series> {
-        let it = self.into_iter().map(|opt_s| Some(RbSeries::new(opt_s?)));
+        let it = self.series_iter().map(|opt_s| Some(RbSeries::new(opt_s?)));
         let avs = call_and_collect_anyvalues(rb, lambda, self.len(), it, skip_nulls)?;
         Ok(Series::from_any_values(self.name().clone(), &avs, true).map_err(RbPolarsErr::from)?)
     }
@@ -148,7 +149,7 @@ impl ApplyLambdaGeneric for ArrayChunked {
         datatype: &DataType,
         skip_nulls: bool,
     ) -> RbResult<Series> {
-        let it = self.into_iter().map(|opt_s| Some(RbSeries::new(opt_s?)));
+        let it = self.series_iter().map(|opt_s| Some(RbSeries::new(opt_s?)));
         let avs = call_and_collect_anyvalues(rb, lambda, self.len(), it, skip_nulls)?;
         Ok(
             Series::from_any_values_and_dtype(self.name().clone(), &avs, datatype, true)
@@ -159,12 +160,12 @@ impl ApplyLambdaGeneric for ArrayChunked {
 
 impl ApplyLambdaGeneric for ObjectChunked<ObjectValue> {
     fn apply_generic(&self, rb: &Ruby, lambda: Value, skip_nulls: bool) -> RbResult<Series> {
-        // TODO improve into_iter
+        // TODO improve iter
         let avs = call_and_collect_anyvalues(
             rb,
             lambda,
             self.len(),
-            self.into_iter().map(|v| v.cloned()),
+            self.iter().map(|v| v.cloned()),
             skip_nulls,
         )?;
         Ok(Series::from_any_values(self.name().clone(), &avs, true).map_err(RbPolarsErr::from)?)
@@ -177,12 +178,12 @@ impl ApplyLambdaGeneric for ObjectChunked<ObjectValue> {
         datatype: &DataType,
         skip_nulls: bool,
     ) -> RbResult<Series> {
-        // TODO improve into_iter
+        // TODO improve iter
         let avs = call_and_collect_anyvalues(
             rb,
             lambda,
             self.len(),
-            self.into_iter().map(|v| v.cloned()),
+            self.iter().map(|v| v.cloned()),
             skip_nulls,
         )?;
         Ok(
@@ -223,7 +224,7 @@ impl ApplyLambdaGeneric for BinaryChunked {
             rb,
             lambda,
             self.len(),
-            self.into_iter().map(|v| v.map(|v2| rb.str_from_slice(v2))),
+            self.iter().map(|v| v.map(|v2| rb.str_from_slice(v2))),
             skip_nulls,
         )?;
         Ok(Series::from_any_values(self.name().clone(), &avs, true).map_err(RbPolarsErr::from)?)
@@ -240,7 +241,7 @@ impl ApplyLambdaGeneric for BinaryChunked {
             rb,
             lambda,
             self.len(),
-            self.into_iter().map(|v| v.map(|v2| rb.str_from_slice(v2))),
+            self.iter().map(|v| v.map(|v2| rb.str_from_slice(v2))),
             skip_nulls,
         )?;
         Ok(