polars-df 0.25.0 → 0.25.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: f29eb603b53abf84e780389ff4c04ff58ae16ccb4336747150caf662166c8ff7
-  data.tar.gz: fea822884078c5f8ea7cc060d24f92304465b11efdb42eef36a63ee0f41e02a7
+  metadata.gz: 3f8d65cfb4e8577c4f9e92f2f2271f97f44b0bc66bd16e9b517a5ab5dae9a877
+  data.tar.gz: 7b9254cc522bfa6eae53986384d33c96933ffa788b47f4f6e82bc46d2cbf6395
 SHA512:
-  metadata.gz: ed7f845b117e6dcd7adb90b77cd7dbe0e53607d2c2551627c5ca2bbfbbc42fe94970bd98c31dd752b7a61ff97570f96441a403d347e2e45aae0062f78c9e3676
-  data.tar.gz: 40c417f1de724293f8800e02da735a6fd5631607e1da8c505f3a42e6056b98a1611e720f9d9522e56838161015e6ad8be4722391eeda57cce3d4614e2bad5dab
+  metadata.gz: 97754dc2c36dbc5e2d7d99bf71973d05107d9124d1dc7bea1bd760ee77d9ac303898a72ac301c798fe4ae77cd24a086a28b20db2e8254c8ef300b3df205e33a5
+  data.tar.gz: 61cbe25d941e7451bcce590a263912967430aa31638db0cfe699d05754ebae71da3495bcc6ac7305afcfd9cfb3bb39670785501e196eae5f6fd6ba631bc92d27

data/CHANGELOG.md

@@ -1,3 +1,10 @@
+## 0.25.1 (2026-03-23)
+
+- Added `explain_all` method to `Polars`
+- Fixed `storage_options` option for `scan_parquet` and `scan_ipc` methods
+- Fixed `Struct` construction with `nil` values
+- Fixed `read_database` for JSON columns
+
 ## 0.25.0 (2026-02-18)
 
 - Updated Polars to 0.53.0

data/Cargo.lock

@@ -2160,7 +2160,7 @@ dependencies = [
 
 [[package]]
 name = "polars-ruby"
-version = "0.25.0"
+version = "0.25.1"
 dependencies = [
  "ahash",
  "bytes",
@@ -2408,9 +2408,9 @@ dependencies = [
 
 [[package]]
 name = "quinn-proto"
-version = "0.11.12"
+version = "0.11.14"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "49df843a9161c85bb8aae55f101bc0bac8bcafd637a620d9122fd7e0b2f7422e"
+checksum = "434b42fec591c96ef50e21e886936e66d3cc3f737104fdb9b737c40ffb94c098"
 dependencies = [
  "bytes",
  "getrandom 0.3.3",
@@ -2811,9 +2811,9 @@ dependencies = [
 
 [[package]]
 name = "rustls-webpki"
-version = "0.103.3"
+version = "0.103.10"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e4a72fe2bcf7a6ac6fd7d0b9e5cb68aeb7d4c0a0271730218b3e92d43b4eb435"
+checksum = "df33b2b81ac578cabaf06b89b0631153a3f416b0a886e8a7a1707fb51abbd1ef"
 dependencies = [
  "ring",
  "rustls-pki-types",

data/ext/polars/Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "polars-ruby"
-version = "0.25.0"
+version = "0.25.1"
 license = "MIT"
 authors = ["Andrew Kane <andrew@ankane.org>"]
 edition = "2024"
@@ -130,6 +130,7 @@ features = [
     "rolling_window_by",
     "round_series",
     "row_hash",
+    "scan_lines",
     "search_sorted",
     "semi_anti_join",
     "serde-lazy",

data/ext/polars/src/conversion/any_value.rs

@@ -1,8 +1,10 @@
 use magnus::encoding::EncodingCapable;
 use magnus::{
-    IntoValue, RArray, RHash, RString, Ruby, TryConvert, Value, prelude::*, r_hash::ForEach,
+    IntoValue, RArray, RHash, RString, Ruby, Symbol, TryConvert, Value, prelude::*,
+    r_hash::ForEach, value::Opaque,
 };
 use num_traits::ToPrimitive;
+use polars::chunked_array::object::PolarsObjectSafe;
 use polars::prelude::*;
 use polars_compute::decimal::{DEC128_MAX_PREC, DecimalFmtBuffer, dec128_fits};
 use polars_core::utils::any_values_to_supertype_and_n_dtypes;
@@ -22,7 +24,7 @@ impl IntoValue for Wrap<AnyValue<'_>> {
 
 impl TryConvert for Wrap<AnyValue<'_>> {
     fn try_convert(ob: Value) -> RbResult<Self> {
-        rb_object_to_any_value(ob, true).map(Wrap)
+        rb_object_to_any_value(ob, true, true).map(Wrap)
     }
 }
 
@@ -88,7 +90,11 @@ pub(crate) fn any_value_into_rb_object(av: AnyValue, ruby: &Ruby) -> Value {
     }
 }
 
-pub(crate) fn rb_object_to_any_value<'s>(ob: Value, strict: bool) -> RbResult<AnyValue<'s>> {
+pub(crate) fn rb_object_to_any_value<'s>(
+    ob: Value,
+    strict: bool,
+    allow_object: bool,
+) -> RbResult<AnyValue<'s>> {
     // Conversion functions.
     fn get_null(_ob: Value, _strict: bool) -> RbResult<AnyValue<'static>> {
         Ok(AnyValue::Null)
@@ -174,7 +180,12 @@ pub(crate) fn rb_object_to_any_value<'s>(ob: Value, strict: bool) -> RbResult<An
         let len = dict.len();
         let mut keys = Vec::with_capacity(len);
         let mut vals = Vec::with_capacity(len);
-        dict.foreach(|key: String, val: Wrap<AnyValue>| {
+        dict.foreach(|key: Value, val: Wrap<AnyValue>| {
+            let key = if let Some(s) = Symbol::from_value(key) {
+                s.name()?.to_string()
+            } else {
+                String::try_convert(key)?
+            };
             let val = val.0;
             let dtype = DataType::from(&val);
             keys.push(Field::new(key.into(), dtype));
@@ -184,6 +195,14 @@ pub(crate) fn rb_object_to_any_value<'s>(ob: Value, strict: bool) -> RbResult<An
         Ok(AnyValue::StructOwned(Box::new((vals, keys))))
     }
 
+    fn get_object(ob: Value, _strict: bool) -> RbResult<AnyValue<'static>> {
+        // This is slow, but hey don't use objects.
+        let v = &ObjectValue {
+            inner: Opaque::from(ob),
+        };
+        Ok(AnyValue::ObjectOwned(OwnedObject(v.to_boxed())))
+    }
+
     fn get_date(ob: Value, _strict: bool) -> RbResult<AnyValue<'static>> {
         // convert to DateTime for UTC
         let v = ob
@@ -263,6 +282,10 @@ pub(crate) fn rb_object_to_any_value<'s>(ob: Value, strict: bool) -> RbResult<An
     } else if ob.is_kind_of(crate::ruby::rb_modules::bigdecimal(&ruby)) {
         get_decimal(ob, strict)
     } else {
-        Err(RbValueError::new_err(format!("Cannot convert {ob}")))
+        if allow_object {
+            get_object(ob, strict)
+        } else {
+            Err(RbValueError::new_err(format!("Cannot convert {ob}")))
+        }
     }
 }

data/ext/polars/src/conversion/mod.rs

@@ -37,19 +37,9 @@ use crate::file::{RubyScanSourceInput, get_ruby_scan_source_input};
 use crate::object::OBJECT_NAME;
 use crate::rb_modules::pl_series;
 use crate::utils::to_rb_err;
-use crate::{RbDataFrame, RbLazyFrame, RbPolarsErr, RbResult, RbSeries, RbTypeError, RbValueError};
-
-pub(crate) fn slice_extract_wrapped<T>(slice: &[Wrap<T>]) -> &[T] {
-    // Safety:
-    // Wrap is transparent.
-    unsafe { std::mem::transmute(slice) }
-}
-
-pub(crate) fn vec_extract_wrapped<T>(buf: Vec<Wrap<T>>) -> Vec<T> {
-    // Safety:
-    // Wrap is transparent.
-    unsafe { std::mem::transmute(buf) }
-}
+use crate::{
+    RbDataFrame, RbExpr, RbLazyFrame, RbPolarsErr, RbResult, RbSeries, RbTypeError, RbValueError,
+};
 
 #[repr(transparent)]
 pub struct Wrap<T>(pub T);
@@ -648,6 +638,16 @@ impl TryConvert for Wrap<ScanSources> {
     }
 }
 
+impl IntoValue for Wrap<Schema> {
+    fn into_value_with(self, ruby: &Ruby) -> Value {
+        let dict = ruby.hash_new();
+        for (k, v) in self.0.iter() {
+            dict.aset(k.as_str(), Wrap(v.clone())).unwrap();
+        }
+        dict.as_value()
+    }
+}
+
 #[derive(Clone)]
 pub struct ObjectValue {
     pub inner: Opaque<Value>,
@@ -1491,6 +1491,21 @@ impl TryConvert for Wrap<Option<TimeZone>> {
 
 unsafe impl TryConvertOwned for Wrap<Option<TimeZone>> {}
 
+impl TryConvert for Wrap<UpcastOrForbid> {
+    fn try_convert(ob: Value) -> RbResult<Self> {
+        let parsed = match String::try_convert(ob)?.as_str() {
+            "upcast" => UpcastOrForbid::Upcast,
+            "forbid" => UpcastOrForbid::Forbid,
+            v => {
+                return Err(RbValueError::new_err(format!(
+                    "cast parameter must be one of {{'upcast', 'forbid'}}, got {v}",
+                )));
+            }
+        };
+        Ok(Wrap(parsed))
+    }
+}
+
 impl TryConvert for Wrap<ExtraColumnsPolicy> {
     fn try_convert(ob: Value) -> RbResult<Self> {
         let parsed = match String::try_convert(ob)?.as_str() {
@@ -1521,6 +1536,27 @@ impl TryConvert for Wrap<MissingColumnsPolicy> {
     }
 }
 
+impl TryConvert for Wrap<MissingColumnsPolicyOrExpr> {
+    fn try_convert(ob: Value) -> RbResult<Self> {
+        if let Ok(rbexpr) = <&RbExpr>::try_convert(ob) {
+            return Ok(Wrap(MissingColumnsPolicyOrExpr::InsertWith(
+                rbexpr.inner.clone(),
+            )));
+        }
+
+        let parsed = match String::try_convert(ob)?.as_str() {
+            "insert" => MissingColumnsPolicyOrExpr::Insert,
+            "raise" => MissingColumnsPolicyOrExpr::Raise,
+            v => {
+                return Err(RbValueError::new_err(format!(
+                    "missing column/field parameter must be one of {{'insert', 'raise', expression}}, got {v}",
+                )));
+            }
+        };
+        Ok(Wrap(parsed))
+    }
+}
+
 impl TryConvert for Wrap<ColumnMapping> {
     fn try_convert(ob: Value) -> RbResult<Self> {
         let (column_mapping_type, ob) = <(String, Value)>::try_convert(ob)?;

data/ext/polars/src/dataframe/general.rs

@@ -1,5 +1,6 @@
 use std::hash::BuildHasher;
 
+use arrow::bitmap::MutableBitmap;
 use either::Either;
 use magnus::{IntoValue, RArray, Ruby, Value, prelude::*, value::Opaque};
 use polars::prelude::*;
@@ -503,10 +504,26 @@ impl RbDataFrame {
         })
     }
 
-    pub fn to_struct(rb: &Ruby, self_: &Self, name: String) -> RbResult<RbSeries> {
+    pub fn to_struct(
+        rb: &Ruby,
+        self_: &Self,
+        name: String,
+        invalid_indices: Vec<usize>,
+    ) -> RbResult<RbSeries> {
         rb.enter_polars_series(|| {
-            let ca = self_.df.read().clone().into_struct(name.into());
-            Ok(ca)
+            let mut ca = self_.df.read().clone().into_struct(name.into());
+
+            if !invalid_indices.is_empty() {
+                let mut validity = MutableBitmap::with_capacity(ca.len());
+                validity.extend_constant(ca.len(), true);
+                for i in invalid_indices {
+                    validity.set(i, false);
+                }
+                ca.rechunk_mut();
+                Ok(ca.with_outer_validity(Some(validity.freeze())))
+            } else {
+                Ok(ca)
+            }
         })
     }
 

data/ext/polars/src/expr/datatype.rs

@@ -1,5 +1,5 @@
-use magnus::{IntoValue, Ruby, Value};
-use polars::prelude::{DataType, DataTypeExpr, Schema};
+use magnus::{IntoValue, RArray, Ruby, TryConvert, Value};
+use polars::prelude::{DataType, DataTypeExpr, PlSmallStr, Schema};
 
 use crate::prelude::Wrap;
 use crate::{RbExpr, RbPolarsErr, RbResult};
@@ -38,4 +38,13 @@ impl RbDataTypeExpr {
             .map_err(RbPolarsErr::from)?;
         Ok(Wrap(dtype).into_value_with(ruby))
     }
+
+    pub fn struct_with_fields(rb_fields: RArray) -> RbResult<Self> {
+        let mut fields = Vec::new();
+        for v in rb_fields.into_iter() {
+            let (name, dt_expr) = <(String, &RbDataTypeExpr)>::try_convert(v)?;
+            fields.push((PlSmallStr::from_string(name), dt_expr.inner.clone()));
+        }
+        Ok(DataTypeExpr::StructWithFields(fields).into())
+    }
 }

data/ext/polars/src/expr/general.rs

@@ -118,6 +118,14 @@ impl RbExpr {
         self.inner.clone().max().into()
     }
 
+    pub fn min_by(&self, by: &Self) -> Self {
+        self.inner.clone().min_by(by.inner.clone()).into()
+    }
+
+    pub fn max_by(&self, by: &Self) -> Self {
+        self.inner.clone().max_by(by.inner.clone()).into()
+    }
+
     pub fn nan_max(&self) -> Self {
         self.inner.clone().nan_max().into()
     }

data/ext/polars/src/expr/rolling.rs

@@ -1,6 +1,9 @@
+use magnus::Value;
 use polars::prelude::*;
 
 use crate::conversion::Wrap;
+use crate::ruby::plan_callback::PlanCallbackExt;
+use crate::ruby::ruby_function::RubyObject;
 use crate::{RbExpr, RbPolarsErr, RbResult};
 
 impl RbExpr {
@@ -406,4 +409,25 @@ impl RbExpr {
 
         self.inner.clone().rolling_kurtosis(options).into()
     }
+
+    pub fn rolling_map(
+        &self,
+        lambda: Value,
+        window_size: usize,
+        weights: Option<Vec<f64>>,
+        min_periods: Option<usize>,
+        center: bool,
+    ) -> Self {
+        let min_periods = min_periods.unwrap_or(window_size);
+        let options = RollingOptionsFixedWindow {
+            window_size,
+            weights,
+            min_periods,
+            center,
+            ..Default::default()
+        };
+        let function = PlanCallback::new_ruby(RubyObject::from(lambda));
+
+        self.inner.clone().rolling_map(function, options).into()
+    }
 }

data/ext/polars/src/functions/lazy.rs

@@ -130,6 +130,13 @@ pub fn collect_all(
     Ok(ruby.ary_from_iter(dfs.into_iter().map(Into::<RbDataFrame>::into)))
 }
 
+pub fn explain_all(rb: &Ruby, lfs: RArray, optflags: &RbOptFlags) -> RbResult<String> {
+    let plans = lfs_to_plans(lfs)?;
+    let explained =
+        rb.enter_polars(|| LazyFrame::explain_all(plans, optflags.clone().inner.into_inner()))?;
+    Ok(explained)
+}
+
 pub fn collect_all_lazy(lfs: RArray, optflags: &RbOptFlags) -> RbResult<RbLazyFrame> {
     let plans = lfs_to_plans(lfs)?;
 

data/ext/polars/src/functions/range.rs

@@ -138,6 +138,19 @@ pub fn time_ranges(
     Ok(dsl::time_ranges(start, end, every, closed).into())
 }
 
+pub fn linear_space(
+    start: &RbExpr,
+    end: &RbExpr,
+    num_samples: &RbExpr,
+    closed: Wrap<ClosedInterval>,
+) -> RbResult<RbExpr> {
+    let start = start.inner.clone();
+    let end = end.inner.clone();
+    let num_samples = num_samples.inner.clone();
+    let closed = closed.0;
+    Ok(dsl::linear_space(start, end, num_samples, closed).into())
+}
+
 pub fn linear_spaces(
     start: &RbExpr,
     end: &RbExpr,

data/ext/polars/src/interop/arrow/to_rb.rs

@@ -11,7 +11,7 @@ use crate::RbResult;
 
 #[magnus::wrap(class = "Polars::ArrowArrayStream")]
 pub struct RbArrowArrayStream {
-    stream: ffi::ArrowArrayStream,
+    pub(crate) stream: ffi::ArrowArrayStream,
 }
 
 impl RbArrowArrayStream {

data/ext/polars/src/lazyframe/general.rs

@@ -1,3 +1,4 @@
+use arrow::ffi::export_iterator;
 use magnus::{
     IntoValue, RArray, RHash, Ruby, TryConvert, Value, r_hash::ForEach, value::ReprValue,
 };
@@ -19,8 +20,13 @@ use crate::io::sink_options::RbSinkOptions;
 use crate::io::sink_output::RbFileSinkDestination;
 use crate::ruby::gvl::GvlExt;
 use crate::ruby::lazy::RubyUdfLazyFrameExt;
-use crate::utils::EnterPolarsExt;
-use crate::{RbDataFrame, RbExpr, RbLazyGroupBy, RbPolarsErr, RbResult, RbValueError};
+use crate::ruby::plan_callback::PlanCallbackExt;
+use crate::ruby::ruby_function::RubyObject;
+use crate::utils::{EnterPolarsExt, to_rb_err};
+use crate::{
+    RbArrowArrayStream, RbDataFrame, RbExpr, RbLazyGroupBy, RbPolarsErr, RbResult, RbTypeError,
+    RbValueError,
+};
 
 fn rbobject_to_first_path_and_scan_sources(
     obj: Value,
@@ -272,6 +278,25 @@ impl RbLazyFrame {
         Ok(lf.into())
     }
 
+    pub fn new_from_scan_lines(
+        sources: Wrap<ScanSources>,
+        scan_options: RbScanOptions,
+        name: String,
+    ) -> RbResult<Self> {
+        let sources = sources.0;
+        let first_path = sources.first_path();
+
+        let unified_scan_args =
+            scan_options.extract_unified_scan_args(first_path.and_then(|x| x.scheme()))?;
+
+        let dsl: DslPlan = DslBuilder::scan_lines(sources, unified_scan_args, (&*name).into())
+            .map_err(to_rb_err)?
+            .build();
+        let lf: LazyFrame = dsl.into();
+
+        Ok(lf.into())
+    }
+
     pub fn describe_plan(rb: &Ruby, self_: &Self) -> RbResult<String> {
         rb.enter_polars(|| self_.ldf.read().describe_plan())
     }
@@ -409,7 +434,7 @@ impl RbLazyFrame {
 
             PolarsResult::Ok(RbCollectBatches {
                 inner: Arc::new(Mutex::new(collect_batches)),
-                _ldf: ldf,
+                ldf,
             })
         })
     }
@@ -573,6 +598,20 @@ impl RbLazyFrame {
         .map(Into::into)
     }
 
+    pub fn sink_batches(
+        rb: &Ruby,
+        self_: &Self,
+        function: Value,
+        maintain_order: bool,
+        chunk_size: Option<NonZeroUsize>,
+    ) -> RbResult<RbLazyFrame> {
+        let ldf = self_.ldf.read().clone();
+        // ensure new_ruby is called with GVL
+        let callback = PlanCallback::new_ruby(RubyObject::from(function));
+        rb.enter_polars(|| ldf.sink_batches(callback, maintain_order, chunk_size))
+            .map(Into::into)
+    }
+
     pub fn filter(&self, predicate: &RbExpr) -> Self {
         let ldf = self.ldf.read().clone();
         ldf.filter(predicate.inner.clone()).into()
@@ -786,6 +825,127 @@ impl RbLazyFrame {
         Ok(ldf.with_columns_seq(exprs.to_exprs()?).into())
     }
 
+    pub fn match_to_schema(
+        &self,
+        schema: Wrap<Schema>,
+        missing_columns: Value,
+        missing_struct_fields: Value,
+        extra_columns: Wrap<ExtraColumnsPolicy>,
+        extra_struct_fields: Value,
+        integer_cast: Value,
+        float_cast: Value,
+    ) -> RbResult<Self> {
+        fn parse_missing_columns(
+            schema: &Schema,
+            missing_columns: Value,
+        ) -> RbResult<Vec<MissingColumnsPolicyOrExpr>> {
+            let mut out = Vec::with_capacity(schema.len());
+            if let Ok(policy) = Wrap::<MissingColumnsPolicyOrExpr>::try_convert(missing_columns) {
+                out.extend(std::iter::repeat_n(policy.0, schema.len()));
+            } else if let Ok(dict) = RHash::try_convert(missing_columns) {
+                out.extend(std::iter::repeat_n(
+                    MissingColumnsPolicyOrExpr::Raise,
+                    schema.len(),
+                ));
+                dict.foreach(|key: String, value: Wrap<MissingColumnsPolicyOrExpr>| {
+                    out[schema.try_index_of(&key).map_err(to_rb_err)?] = value.0;
+                    Ok(ForEach::Continue)
+                })?;
+            } else {
+                return Err(RbTypeError::new_err("Invalid value for `missing_columns`"));
+            }
+            Ok(out)
+        }
+        fn parse_missing_struct_fields(
+            schema: &Schema,
+            missing_struct_fields: Value,
+        ) -> RbResult<Vec<MissingColumnsPolicy>> {
+            let mut out = Vec::with_capacity(schema.len());
+            if let Ok(policy) = Wrap::<MissingColumnsPolicy>::try_convert(missing_struct_fields) {
+                out.extend(std::iter::repeat_n(policy.0, schema.len()));
+            } else if let Ok(dict) = RHash::try_convert(missing_struct_fields) {
+                out.extend(std::iter::repeat_n(
+                    MissingColumnsPolicy::Raise,
+                    schema.len(),
+                ));
+                dict.foreach(|key: String, value: Wrap<MissingColumnsPolicy>| {
+                    out[schema.try_index_of(&key).map_err(to_rb_err)?] = value.0;
+                    Ok(ForEach::Continue)
+                })?;
+            } else {
+                return Err(RbTypeError::new_err(
+                    "Invalid value for `missing_struct_fields`",
+                ));
+            }
+            Ok(out)
+        }
+        fn parse_extra_struct_fields(
+            schema: &Schema,
+            extra_struct_fields: Value,
+        ) -> RbResult<Vec<ExtraColumnsPolicy>> {
+            let mut out = Vec::with_capacity(schema.len());
+            if let Ok(policy) = Wrap::<ExtraColumnsPolicy>::try_convert(extra_struct_fields) {
+                out.extend(std::iter::repeat_n(policy.0, schema.len()));
+            } else if let Ok(dict) = RHash::try_convert(extra_struct_fields) {
+                out.extend(std::iter::repeat_n(ExtraColumnsPolicy::Raise, schema.len()));
+                dict.foreach(|key: String, value: Wrap<ExtraColumnsPolicy>| {
+                    out[schema.try_index_of(&key).map_err(to_rb_err)?] = value.0;
+                    Ok(ForEach::Continue)
+                })?;
+            } else {
+                return Err(RbTypeError::new_err(
+                    "Invalid value for `extra_struct_fields`",
+                ));
+            }
+            Ok(out)
+        }
+        fn parse_cast(schema: &Schema, cast: Value) -> RbResult<Vec<UpcastOrForbid>> {
+            let mut out = Vec::with_capacity(schema.len());
+            if let Ok(policy) = Wrap::<UpcastOrForbid>::try_convert(cast) {
+                out.extend(std::iter::repeat_n(policy.0, schema.len()));
+            } else if let Ok(dict) = RHash::try_convert(cast) {
+                out.extend(std::iter::repeat_n(UpcastOrForbid::Forbid, schema.len()));
+                dict.foreach(|key: String, value: Wrap<UpcastOrForbid>| {
+                    out[schema.try_index_of(&key).map_err(to_rb_err)?] = value.0;
+                    Ok(ForEach::Continue)
+                })?;
+            } else {
+                return Err(RbTypeError::new_err(
+                    "Invalid value for `integer_cast` / `float_cast`",
+                ));
+            }
+            Ok(out)
+        }
+
+        let missing_columns = parse_missing_columns(&schema.0, missing_columns)?;
+        let missing_struct_fields = parse_missing_struct_fields(&schema.0, missing_struct_fields)?;
+        let extra_struct_fields = parse_extra_struct_fields(&schema.0, extra_struct_fields)?;
+        let integer_cast = parse_cast(&schema.0, integer_cast)?;
+        let float_cast = parse_cast(&schema.0, float_cast)?;
+
+        let per_column = (0..schema.0.len())
+            .map(|i| MatchToSchemaPerColumn {
+                missing_columns: missing_columns[i].clone(),
+                missing_struct_fields: missing_struct_fields[i],
+                extra_struct_fields: extra_struct_fields[i],
+                integer_cast: integer_cast[i],
+                float_cast: float_cast[i],
+            })
+            .collect();
+
+        let ldf = self.ldf.read().clone();
+        Ok(ldf
+            .match_to_schema(Arc::new(schema.0), per_column, extra_columns.0)
+            .into())
+    }
+
+    pub fn pipe_with_schema(&self, callback: Value) -> Self {
+        let ldf = self.ldf.read().clone();
+        let function = RubyObject::from(callback);
+        ldf.pipe_with_schema(PlanCallback::new_ruby(function))
+            .into()
+    }
+
     pub fn rename(&self, existing: Vec<String>, new: Vec<String>, strict: bool) -> Self {
         let ldf = self.ldf.read().clone();
         ldf.rename(existing, new, strict).into()
@@ -1118,7 +1278,7 @@ impl RbLazyFrame {
 #[magnus::wrap(class = "Polars::RbCollectBatches")]
 pub struct RbCollectBatches {
     inner: Arc<Mutex<CollectBatches>>,
-    _ldf: LazyFrame,
+    ldf: LazyFrame,
 }
 
 impl RbCollectBatches {
@@ -1126,4 +1286,55 @@ impl RbCollectBatches {
         let inner = Arc::clone(&slf.inner);
         rb.enter_polars(|| PolarsResult::Ok(inner.lock().next().transpose()?.map(RbDataFrame::new)))
     }
+
+    pub fn __arrow_c_stream__(rb: &Ruby, self_: &Self) -> RbResult<Value> {
+        let mut ldf = self_.ldf.clone();
+        let schema = ldf
+            .collect_schema()
+            .map_err(RbPolarsErr::from)?
+            .to_arrow(CompatLevel::newest());
+
+        let dtype = ArrowDataType::Struct(schema.into_iter_values().collect());
+
+        let iter = Box::new(ArrowStreamIterator::new(self_.inner.clone(), dtype.clone()));
+        let field = ArrowField::new(PlSmallStr::EMPTY, dtype, false);
+        let stream = export_iterator(iter, field);
+        Ok(RbArrowArrayStream { stream }.into_value_with(rb))
+    }
+}
+
+pub struct ArrowStreamIterator {
+    inner: Arc<Mutex<CollectBatches>>,
+    dtype: ArrowDataType,
+}
+
+impl ArrowStreamIterator {
+    fn new(inner: Arc<Mutex<CollectBatches>>, schema: ArrowDataType) -> Self {
+        Self {
+            inner,
+            dtype: schema,
+        }
+    }
+}
+
+impl Iterator for ArrowStreamIterator {
+    type Item = PolarsResult<ArrayRef>;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        let next = self.inner.lock().next();
+        match next {
+            None => None,
+            Some(Err(err)) => Some(Err(err)),
+            Some(Ok(df)) => {
+                let height = df.height();
+                let arrays = df.rechunk_into_arrow(CompatLevel::newest());
+                Some(Ok(Box::new(arrow::array::StructArray::new(
+                    self.dtype.clone(),
+                    height,
+                    arrays,
+                    None,
+                ))))
+            }
+        }
+    }
 }