polars-df 0.21.1 → 0.23.0

data/ext/polars/src/file.rs

@@ -3,7 +3,7 @@ use std::io;
 use std::io::{Cursor, Read, Seek, SeekFrom, Write};
 use std::path::PathBuf;
 
-use magnus::{Error, RString, Ruby, Value, exception, prelude::*, value::Opaque};
+use magnus::{Error, RString, Ruby, Value, prelude::*, value::Opaque};
 use polars::io::cloud::CloudOptions;
 use polars::io::mmap::MmapBytesReader;
 use polars::prelude::PlPath;
@@ -67,23 +67,25 @@ impl RbFileLikeObject {
     /// ruby object has a `read`, `write`, and `seek` methods in respect to parameters.
     /// Will return a `TypeError` if object does not have `read`, `seek`, and `write` methods.
     pub fn with_requirements(object: Value, read: bool, write: bool, seek: bool) -> RbResult<Self> {
+        let ruby = Ruby::get_with(object);
+
         if read && !object.respond_to("read", false)? {
             return Err(Error::new(
-                exception::type_error(),
+                ruby.exception_type_error(),
                 "Object does not have a .read() method.",
             ));
         }
 
         if seek && !object.respond_to("seek", false)? {
             return Err(Error::new(
-                exception::type_error(),
+                ruby.exception_type_error(),
                 "Object does not have a .seek() method.",
             ));
         }
 
         if write && !object.respond_to("write", false)? {
             return Err(Error::new(
-                exception::type_error(),
+                ruby.exception_type_error(),
                 "Object does not have a .write() method.",
             ));
         }
@@ -113,10 +115,10 @@ impl Read for RbFileLikeObject {
 
 impl Write for RbFileLikeObject {
     fn write(&mut self, buf: &[u8]) -> Result<usize, io::Error> {
-        let rbbytes = RString::from_slice(buf);
+        let ruby = Ruby::get().unwrap();
+        let rbbytes = ruby.str_from_slice(buf);
 
-        let number_bytes_written = Ruby::get()
-            .unwrap()
+        let number_bytes_written = ruby
             .get_inner(self.inner)
             .funcall::<_, _, usize>("write", (rbbytes,))
             .map_err(rberr_to_io_err)?;
@@ -143,13 +145,13 @@ impl Seek for RbFileLikeObject {
             SeekFrom::End(i) => (2, i),
         };
 
-        let new_position = Ruby::get()
-            .unwrap()
-            .get_inner(self.inner)
-            .funcall("seek", (offset, whence))
+        let inner = Ruby::get().unwrap().get_inner(self.inner);
+
+        inner
+            .funcall::<_, _, Value>("seek", (offset, whence))
             .map_err(rberr_to_io_err)?;
 
-        Ok(new_position)
+        inner.funcall("tell", ()).map_err(rberr_to_io_err)
     }
 }
 
@@ -265,8 +267,12 @@ pub fn get_mmap_bytes_reader_and_path<'a>(
         RbReadBytes::Bytes(v) => Ok((Box::new(Cursor::new(unsafe { v.as_slice() })), None)),
         RbReadBytes::Other(v) => {
             let path = PathBuf::try_convert(*v)?;
-            let f = File::open(&path)
-                .map_err(|e| Error::new(exception::runtime_error(), e.to_string()))?;
+            let f = File::open(&path).map_err(|e| {
+                Error::new(
+                    Ruby::get().unwrap().exception_runtime_error(),
+                    e.to_string(),
+                )
+            })?;
             Ok((Box::new(f), Some(path)))
         }
     }

data/ext/polars/src/functions/business.rs

@@ -2,7 +2,6 @@ use polars::lazy::dsl;
 
 use crate::RbExpr;
 
-// TODO add to Ruby
 pub fn business_day_count(
     start: &RbExpr,
     end: &RbExpr,

data/ext/polars/src/functions/io.rs

@@ -1,6 +1,6 @@
 use std::io::BufReader;
 
-use magnus::{RHash, Value};
+use magnus::{RHash, Ruby, Value};
 use polars::prelude::ArrowSchema;
 
 use crate::conversion::Wrap;
@@ -16,7 +16,8 @@ pub fn read_ipc_schema(rb_f: Value) -> RbResult<RHash> {
         EitherRustRubyFile::Rb(mut r) => read_file_metadata(&mut r).map_err(RbPolarsErr::from)?,
     };
 
-    let dict = RHash::new();
+    let ruby = Ruby::get_with(rb_f);
+    let dict = ruby.hash_new();
     fields_to_rbdict(&metadata.schema, &dict)?;
     Ok(dict)
 }
@@ -33,7 +34,8 @@ pub fn read_parquet_metadata(rb_f: Value) -> RbResult<RHash> {
     };
 
     let key_value_metadata = read_custom_key_value_metadata(metadata.key_value_metadata());
-    let dict = RHash::new();
+    let ruby = Ruby::get_with(rb_f);
+    let dict = ruby.hash_new();
     for (key, value) in key_value_metadata.into_iter() {
         dict.aset(key.as_str(), value.as_str())?;
     }
@@ -51,7 +53,8 @@ pub fn read_parquet_schema(rb_f: Value) -> RbResult<RHash> {
     };
     let arrow_schema = infer_schema(&metadata).map_err(RbPolarsErr::from)?;
 
-    let dict = RHash::new();
+    let ruby = Ruby::get_with(rb_f);
+    let dict = ruby.hash_new();
     fields_to_rbdict(&arrow_schema, &dict)?;
     Ok(dict)
 }

data/ext/polars/src/functions/lazy.rs

@@ -1,6 +1,6 @@
-use magnus::encoding::{self, EncodingCapable};
+use magnus::encoding::EncodingCapable;
 use magnus::{
-    Float, Integer, RArray, RString, Ruby, Value, class, prelude::*, typed_data::Obj, value::Opaque,
+    Float, Integer, RArray, RString, Ruby, Value, prelude::*, typed_data::Obj, value::Opaque,
 };
 use polars::lazy::dsl;
 use polars::prelude::*;
@@ -98,10 +98,10 @@ pub fn col(name: String) -> RbExpr {
     dsl::col(&name).into()
 }
 
-pub fn collect_all(lfs: RArray) -> RbResult<RArray> {
+pub fn collect_all(ruby: &Ruby, lfs: RArray) -> RbResult<RArray> {
     let lfs = lfs.typecheck::<Obj<RbLazyFrame>>()?;
 
-    Ok(RArray::from_iter(lfs.iter().map(|lf| {
+    Ok(ruby.ary_from_iter(lfs.iter().map(|lf| {
         let df = lf.ldf.borrow().clone().collect().unwrap();
         RbDataFrame::new(df)
     })))
@@ -283,7 +283,8 @@ pub fn fold(
 }
 
 pub fn lit(value: Value, allow_object: bool, is_scalar: bool) -> RbResult<RbExpr> {
-    if value.is_kind_of(class::true_class()) || value.is_kind_of(class::false_class()) {
+    let ruby = Ruby::get_with(value);
+    if value.is_kind_of(ruby.class_true_class()) || value.is_kind_of(ruby.class_false_class()) {
         Ok(dsl::lit(bool::try_convert(value)?).into())
     } else if let Some(v) = Integer::from_value(value) {
         match v.to_i64() {
@@ -302,7 +303,7 @@ pub fn lit(value: Value, allow_object: bool, is_scalar: bool) -> RbResult<RbExpr
     } else if let Some(v) = Float::from_value(value) {
         Ok(dsl::lit(v.to_f64()).into())
     } else if let Some(v) = RString::from_value(value) {
-        if v.enc_get() == encoding::Index::utf8() {
+        if v.enc_get() == ruby.utf8_encindex() {
             Ok(dsl::lit(v.to_string()?).into())
         } else {
             Ok(dsl::lit(unsafe { v.as_slice() }).into())

data/ext/polars/src/functions/meta.rs

@@ -1,4 +1,4 @@
-use magnus::{IntoValue, Value};
+use magnus::{IntoValue, Ruby, Value};
 use polars_core;
 use polars_core::POOL;
 use polars_core::fmt::FloatFmt;
@@ -7,8 +7,8 @@ use polars_core::prelude::IDX_DTYPE;
 use crate::conversion::Wrap;
 use crate::{RbResult, RbValueError};
 
-pub fn get_index_type() -> Value {
-    Wrap(IDX_DTYPE).into_value()
+pub fn get_index_type(ruby: &Ruby) -> Value {
+    Wrap(IDX_DTYPE).into_value_with(ruby)
 }
 
 pub fn thread_pool_size() -> usize {

data/ext/polars/src/functions/string_cache.rs

@@ -1,5 +1,5 @@
 use crate::RbResult;
-use magnus::{RArray, Ruby, Value};
+use magnus::{Ruby, Value};
 
 pub fn enable_string_cache() {
     // The string cache no longer exists.
@@ -18,7 +18,7 @@ pub fn using_string_cache() -> bool {
 pub struct RbStringCacheHolder {}
 
 impl RbStringCacheHolder {
-    pub fn hold() -> RbResult<Value> {
-        Ruby::get().unwrap().yield_splat(RArray::new())
+    pub fn hold(ruby: &Ruby) -> RbResult<Value> {
+        ruby.yield_splat(ruby.ary_new())
     }
 }

data/ext/polars/src/interop/arrow/to_ruby.rs

@@ -1,6 +1,6 @@
 use arrow::datatypes::ArrowDataType;
 use arrow::ffi;
-use magnus::{IntoValue, Value};
+use magnus::{IntoValue, Ruby, Value};
 use polars::datatypes::CompatLevel;
 use polars::frame::DataFrame;
 use polars::prelude::{ArrayRef, ArrowField, PlSmallStr, PolarsResult, SchemaExt};
@@ -20,11 +20,11 @@ impl RbArrowArrayStream {
     }
 }
 
-pub(crate) fn dataframe_to_stream(df: &DataFrame) -> RbResult<Value> {
+pub(crate) fn dataframe_to_stream(df: &DataFrame, ruby: &Ruby) -> RbResult<Value> {
     let iter = Box::new(DataFrameStreamIterator::new(df));
     let field = iter.field();
     let stream = ffi::export_iterator(iter, field);
-    Ok(RbArrowArrayStream { stream }.into_value())
+    Ok(RbArrowArrayStream { stream }.into_value_with(ruby))
 }
 
 pub struct DataFrameStreamIterator {

data/ext/polars/src/interop/numo/numo_rs.rs

@@ -1,4 +1,4 @@
-use magnus::{IntoValue, Module, RArray, RClass, RModule, Value, class, prelude::*};
+use magnus::{IntoValue, Module, RClass, RModule, Ruby, Value, prelude::*};
 
 use crate::RbResult;
 
@@ -44,9 +44,10 @@ impl<T: Element> RbArray1<T> {
     where
         I: IntoIterator<Item = T>,
     {
-        class::object()
+        let ruby = Ruby::get().unwrap();
+        ruby.class_object()
             .const_get::<_, RModule>("Numo")?
             .const_get::<_, RClass>(T::class_name())?
-            .funcall("cast", (RArray::from_iter(values),))
+            .funcall("cast", (ruby.ary_from_iter(values),))
     }
 }

data/ext/polars/src/io/mod.rs

@@ -1,10 +1,11 @@
 use std::sync::Arc;
 
 use magnus::{TryConvert, Value, value::ReprValue};
+use polars::prelude::default_values::DefaultFieldValues;
 use polars::prelude::deletion::DeletionFilesList;
 use polars::prelude::{
     CastColumnsPolicy, ColumnMapping, ExtraColumnsPolicy, MissingColumnsPolicy, PlSmallStr, Schema,
-    UnifiedScanArgs,
+    TableStatistics, UnifiedScanArgs,
 };
 use polars_io::{HiveOptions, RowIndex};
 use polars_utils::IdxSize;
@@ -23,6 +24,12 @@ impl TryConvert for RbScanOptions {
     }
 }
 
+impl TryConvert for Wrap<TableStatistics> {
+    fn try_convert(_ob: Value) -> RbResult<Self> {
+        todo!();
+    }
+}
+
 impl RbScanOptions {
     pub fn extract_unified_scan_args(
         &self,
@@ -37,6 +44,10 @@ impl RbScanOptions {
         let include_file_paths: Option<Wrap<PlSmallStr>> =
             self.0.funcall("include_file_paths", ())?;
         let glob: bool = self.0.funcall("glob", ())?;
+        let hidden_file_prefix: Option<Vec<String>> = self.0.funcall("hidden_file_prefix", ())?;
+        let column_mapping: Option<Wrap<ColumnMapping>> = self.0.funcall("column_mapping", ())?;
+        let default_values: Option<Wrap<DefaultFieldValues>> =
+            self.0.funcall("default_values", ())?;
         let hive_partitioning: Option<bool> = self.0.funcall("hive_partitioning", ())?;
         let hive_schema: Option<Wrap<Schema>> = self.0.funcall("hive_schema", ())?;
         let try_parse_hive_dates: bool = self.0.funcall("try_parse_hive_dates", ())?;
@@ -47,7 +58,9 @@ impl RbScanOptions {
         let retries: usize = self.0.funcall("retries", ())?;
         let deletion_files: Option<Wrap<DeletionFilesList>> =
             self.0.funcall("deletion_files", ())?;
-        let column_mapping: Option<Wrap<ColumnMapping>> = self.0.funcall("column_mapping", ())?;
+        let table_statistics: Option<Wrap<TableStatistics>> =
+            self.0.funcall("table_statistics", ())?;
+        let row_count: Option<(u64, u64)> = self.0.funcall("row_count", ())?;
 
         let cloud_options = storage_options;
 
@@ -86,7 +99,13 @@ impl RbScanOptions {
             rechunk,
             cache,
             glob,
+            hidden_file_prefix: hidden_file_prefix
+                .map(|x| x.into_iter().map(|x| (*x).into()).collect()),
             projection: None,
+            column_mapping: column_mapping.map(|x| x.0),
+            default_values: default_values
+                .map(|x| x.0)
+                .filter(|DefaultFieldValues::Iceberg(v)| !v.is_empty()),
             row_index,
             pre_slice: pre_slice.map(Slice::from),
             cast_columns_policy: cast_options.0,
@@ -94,7 +113,8 @@ impl RbScanOptions {
             extra_columns_policy: extra_columns.0,
             include_file_paths: include_file_paths.map(|x| x.0),
             deletion_files: DeletionFilesList::filter_empty(deletion_files.map(|x| x.0)),
-            column_mapping: column_mapping.map(|x| x.0),
+            table_statistics: table_statistics.map(|x| x.0),
+            row_count,
         };
 
         Ok(unified_scan_args)

data/ext/polars/src/lazyframe/general.rs

@@ -1,5 +1,5 @@
-use magnus::{IntoValue, RArray, RHash, TryConvert, Value, r_hash::ForEach, typed_data::Obj};
-use polars::io::{HiveOptions, RowIndex};
+use magnus::{IntoValue, RArray, RHash, Ruby, TryConvert, Value, r_hash::ForEach, typed_data::Obj};
+use polars::io::RowIndex;
 use polars::lazy::frame::LazyFrame;
 use polars::prelude::*;
 use polars_plan::dsl::ScanSources;
@@ -18,7 +18,10 @@ use crate::{RbDataFrame, RbExpr, RbLazyFrame, RbLazyGroupBy, RbPolarsErr, RbResu
 fn rbobject_to_first_path_and_scan_sources(obj: Value) -> RbResult<(Option<PlPath>, ScanSources)> {
     use crate::file::{RubyScanSourceInput, get_ruby_scan_source_input};
     Ok(match get_ruby_scan_source_input(obj, false)? {
-        RubyScanSourceInput::Path(path) => (Some(path.clone()), ScanSources::Paths([path].into())),
+        RubyScanSourceInput::Path(path) => (
+            Some(path.clone()),
+            ScanSources::Paths(FromIterator::from_iter([path])),
+        ),
         RubyScanSourceInput::File(file) => (None, ScanSources::Files([file].into())),
         RubyScanSourceInput::Buffer(buff) => (None, ScanSources::Buffers([buff].into())),
     })
@@ -30,13 +33,12 @@ impl RbLazyFrame {
         source: Option<Value>,
         sources: Wrap<ScanSources>,
         infer_schema_length: Option<usize>,
-        batch_size: Option<Wrap<NonZeroUsize>>,
+        batch_size: Option<NonZeroUsize>,
         n_rows: Option<usize>,
         low_memory: bool,
         rechunk: bool,
         row_index: Option<(String, IdxSize)>,
     ) -> RbResult<Self> {
-        let batch_size = batch_size.map(|v| v.0);
         let row_index = row_index.map(|(name, offset)| RowIndex {
             name: name.into(),
             offset,
@@ -181,48 +183,28 @@ impl RbLazyFrame {
         Ok(lf.into())
     }
 
-    #[allow(clippy::too_many_arguments)]
     pub fn new_from_ipc(
-        source: Option<Value>,
         sources: Wrap<ScanSources>,
-        n_rows: Option<usize>,
-        cache: bool,
-        rechunk: bool,
-        row_index: Option<(String, IdxSize)>,
-        hive_partitioning: Option<bool>,
-        hive_schema: Option<Wrap<Schema>>,
-        try_parse_hive_dates: bool,
-        include_file_paths: Option<String>,
+        scan_options: RbScanOptions,
+        file_cache_ttl: Option<u64>,
     ) -> RbResult<Self> {
-        let row_index = row_index.map(|(name, offset)| RowIndex {
-            name: name.into(),
-            offset,
-        });
+        let options = IpcScanOptions;
 
-        let hive_options = HiveOptions {
-            enabled: hive_partitioning,
-            hive_start_idx: 0,
-            schema: hive_schema.map(|x| Arc::new(x.0)),
-            try_parse_dates: try_parse_hive_dates,
-        };
+        let sources = sources.0;
+        let first_path = sources.first_path().map(|p| p.into_owned());
 
-        let args = ScanArgsIpc {
-            n_rows,
-            cache,
-            rechunk,
-            row_index,
-            cloud_options: None,
-            hive_options,
-            include_file_paths: include_file_paths.map(|x| x.into()),
-        };
+        let mut unified_scan_args =
+            scan_options.extract_unified_scan_args(first_path.as_ref().map(|p| p.as_ref()))?;
 
-        let sources = sources.0;
-        let (_first_path, sources) = match source {
-            None => (sources.first_path().map(|p| p.into_owned()), sources),
-            Some(source) => rbobject_to_first_path_and_scan_sources(source)?,
-        };
+        if let Some(file_cache_ttl) = file_cache_ttl {
+            unified_scan_args
+                .cloud_options
+                .get_or_insert_default()
+                .file_cache_ttl = file_cache_ttl;
+        }
 
-        let lf = LazyFrame::scan_ipc_sources(sources, args).map_err(RbPolarsErr::from)?;
+        let lf = LazyFrame::scan_ipc_sources(sources, options, unified_scan_args)
+            .map_err(RbPolarsErr::from)?;
         Ok(lf.into())
     }
 
@@ -407,13 +389,13 @@ impl RbLazyFrame {
     pub fn sink_ipc(
         &self,
         target: SinkTarget,
-        compression: Option<Wrap<IpcCompression>>,
+        compression: Wrap<Option<IpcCompression>>,
         cloud_options: Option<Vec<(String, String)>>,
         retries: usize,
         sink_options: Wrap<SinkOptions>,
     ) -> RbResult<RbLazyFrame> {
         let options = IpcWriterOptions {
-            compression: compression.map(|c| c.0),
+            compression: compression.0,
             ..Default::default()
         };
 
@@ -444,7 +426,7 @@ impl RbLazyFrame {
         let separator = u8::try_convert(arguments[3])?;
         let line_terminator = String::try_convert(arguments[4])?;
         let quote_char = u8::try_convert(arguments[5])?;
-        let batch_size = Wrap::<NonZeroUsize>::try_convert(arguments[6])?;
+        let batch_size = NonZeroUsize::try_convert(arguments[6])?;
         let datetime_format = Option::<String>::try_convert(arguments[7])?;
         let date_format = Option::<String>::try_convert(arguments[8])?;
         let time_format = Option::<String>::try_convert(arguments[9])?;
@@ -477,7 +459,7 @@ impl RbLazyFrame {
         let options = CsvWriterOptions {
             include_bom,
             include_header,
-            batch_size: batch_size.0,
+            batch_size,
             serialize_options,
         };
 
@@ -931,30 +913,33 @@ impl RbLazyFrame {
         self.ldf.borrow().clone().into()
     }
 
-    pub fn collect_schema(&self) -> RbResult<RHash> {
-        let schema = self
+    pub fn collect_schema(ruby: &Ruby, rb_self: &Self) -> RbResult<RHash> {
+        let schema = rb_self
             .ldf
             .borrow_mut()
             .collect_schema()
             .map_err(RbPolarsErr::from)?;
 
-        let schema_dict = RHash::new();
+        let schema_dict = ruby.hash_new();
         schema.iter_fields().for_each(|fld| {
             schema_dict
                 .aset::<String, Value>(
                     fld.name().to_string(),
-                    Wrap(fld.dtype().clone()).into_value(),
+                    Wrap(fld.dtype().clone()).into_value_with(ruby),
                 )
                 .unwrap();
         });
         Ok(schema_dict)
     }
 
-    pub fn unnest(&self, columns: &RbSelector) -> Self {
+    pub fn unnest(&self, columns: &RbSelector, separator: Option<String>) -> Self {
         self.ldf
             .borrow()
             .clone()
-            .unnest(columns.inner.clone())
+            .unnest(
+                columns.inner.clone(),
+                separator.as_deref().map(PlSmallStr::from_str),
+            )
             .into()
     }
 

data/ext/polars/src/lazyframe/mod.rs

@@ -1,8 +1,9 @@
 mod general;
+mod optflags;
 mod serde;
 mod sink;
 
-use polars::lazy::frame::LazyFrame;
+use polars::prelude::{LazyFrame, OptFlags};
 pub use sink::SinkTarget;
 use std::cell::RefCell;
 
@@ -12,6 +13,12 @@ pub struct RbLazyFrame {
     pub ldf: RefCell<LazyFrame>,
 }
 
+#[magnus::wrap(class = "Polars::RbOptFlags")]
+#[derive(Clone)]
+pub struct RbOptFlags {
+    pub inner: RefCell<OptFlags>,
+}
+
 impl From<LazyFrame> for RbLazyFrame {
     fn from(ldf: LazyFrame) -> Self {
         RbLazyFrame {
@@ -19,3 +26,11 @@ impl From<LazyFrame> for RbLazyFrame {
         }
     }
 }
+
+impl From<OptFlags> for RbOptFlags {
+    fn from(inner: OptFlags) -> Self {
+        RbOptFlags {
+            inner: RefCell::new(inner),
+        }
+    }
+}

data/ext/polars/src/lazyframe/optflags.rs

@@ -0,0 +1,57 @@
+use polars::prelude::OptFlags;
+
+use super::RbOptFlags;
+
+macro_rules! flag_getter_setters {
+    ($(($flag:ident, $getter:ident, $setter:ident, clear=$clear:literal))+) => {
+        impl RbOptFlags {
+            pub fn empty() -> Self {
+                Self {
+                    inner: OptFlags::empty().into()
+                }
+            }
+
+            #[allow(clippy::should_implement_trait)]
+            pub fn default() -> Self {
+                Self { inner: OptFlags::default().into() }
+            }
+
+            pub fn no_optimizations(&self) {
+                $(if $clear {
+                    self.inner.borrow_mut().remove(OptFlags::$flag);
+                })+
+            }
+
+            pub fn copy(&self) -> Self {
+                Self { inner: self.inner.clone() }
+            }
+
+            $(
+            pub fn $getter(&self) -> bool {
+                self.inner.borrow().contains(OptFlags::$flag)
+            }
+            pub fn $setter(&self, value: bool) {
+                self.inner.borrow_mut().set(OptFlags::$flag, value)
+            }
+            )+
+        }
+    };
+}
+
+flag_getter_setters! {
+    (TYPE_COERCION, get_type_coercion, set_type_coercion, clear=false)
+    (TYPE_CHECK, get_type_check, set_type_check, clear=false)
+
+    (PROJECTION_PUSHDOWN, get_projection_pushdown, set_projection_pushdown, clear=true)
+    (PREDICATE_PUSHDOWN, get_predicate_pushdown, set_predicate_pushdown, clear=true)
+    (CLUSTER_WITH_COLUMNS, get_cluster_with_columns, set_cluster_with_columns, clear=true)
+    (SIMPLIFY_EXPR, get_simplify_expression, set_simplify_expression, clear=true)
+    (SLICE_PUSHDOWN, get_slice_pushdown, set_slice_pushdown, clear=true)
+    (COMM_SUBPLAN_ELIM, get_comm_subplan_elim, set_comm_subplan_elim, clear=true)
+    (COMM_SUBEXPR_ELIM, get_comm_subexpr_elim, set_comm_subexpr_elim, clear=true)
+    (CHECK_ORDER_OBSERVE, get_check_order_observe, set_check_order_observe, clear=true)
+    (FAST_PROJECTION, get_fast_projection, set_fast_projection, clear=true)
+
+    (EAGER, get_eager, set_eager, clear=true)
+    (NEW_STREAMING, get_streaming, set_streaming, clear=true)
+}

data/ext/polars/src/lazyframe/serde.rs

@@ -1,14 +1,38 @@
+use std::io::Read;
+#[cfg(feature = "serialize_binary")]
+use std::io::{BufReader, BufWriter};
+
 use magnus::Value;
 use polars::lazy::frame::LazyFrame;
 use polars::prelude::*;
-use std::io::Read;
 
 use crate::file::get_file_like;
+#[cfg(feature = "serialize_binary")]
+use crate::utils::to_rb_err;
 use crate::{RbLazyFrame, RbResult, RbValueError};
 
 impl RbLazyFrame {
-    // TODO change to serialize_json
-    pub fn read_json(rb_f: Value) -> RbResult<Self> {
+    #[cfg(feature = "serialize_binary")]
+    pub fn serialize_binary(&self, rb_f: Value) -> RbResult<()> {
+        let file = get_file_like(rb_f, true)?;
+        let writer = BufWriter::new(file);
+        self.ldf
+            .borrow()
+            .logical_plan
+            .serialize_versioned(writer, Default::default())
+            .map_err(to_rb_err)
+    }
+
+    #[cfg(feature = "serialize_binary")]
+    pub fn deserialize_binary(rb_f: Value) -> RbResult<Self> {
+        let file = get_file_like(rb_f, false)?;
+        let reader = BufReader::new(file);
+
+        let lp: DslPlan = DslPlan::deserialize_versioned(reader).map_err(to_rb_err)?;
+        Ok(LazyFrame::from(lp).into())
+    }
+
+    pub fn deserialize_json(rb_f: Value) -> RbResult<Self> {
         // it is faster to first read to memory and then parse: https://github.com/serde-rs/json/issues/160
         // so don't bother with files.
         let mut json = String::new();