polars-df 0.19.0 → 0.21.0

data/ext/polars/src/lazyframe/general.rs

@@ -1,4 +1,4 @@
-use magnus::{r_hash::ForEach, typed_data::Obj, IntoValue, RArray, RHash, TryConvert, Value};
+use magnus::{IntoValue, RArray, RHash, TryConvert, Value, r_hash::ForEach, typed_data::Obj};
 use polars::io::{HiveOptions, RowIndex};
 use polars::lazy::frame::LazyFrame;
 use polars::prelude::*;
@@ -6,16 +6,17 @@ use polars_plan::dsl::ScanSources;
 use std::cell::RefCell;
 use std::io::BufWriter;
 use std::num::NonZeroUsize;
-use std::path::PathBuf;
 
 use super::SinkTarget;
 use crate::conversion::*;
 use crate::expr::rb_exprs_to_exprs;
+use crate::expr::selector::RbSelector;
 use crate::file::get_file_like;
+use crate::io::RbScanOptions;
 use crate::{RbDataFrame, RbExpr, RbLazyFrame, RbLazyGroupBy, RbPolarsErr, RbResult, RbValueError};
 
-fn rbobject_to_first_path_and_scan_sources(obj: Value) -> RbResult<(Option<PathBuf>, ScanSources)> {
-    use crate::file::{get_ruby_scan_source_input, RubyScanSourceInput};
+fn rbobject_to_first_path_and_scan_sources(obj: Value) -> RbResult<(Option<PlPath>, ScanSources)> {
+    use crate::file::{RubyScanSourceInput, get_ruby_scan_source_input};
     Ok(match get_ruby_scan_source_input(obj, false)? {
         RubyScanSourceInput::Path(path) => (Some(path.clone()), ScanSources::Paths([path].into())),
         RubyScanSourceInput::File(file) => (None, ScanSources::Files([file].into())),
@@ -43,7 +44,7 @@ impl RbLazyFrame {
 
         let sources = sources.0;
         let (_first_path, sources) = match source {
-            None => (sources.first_path().map(|p| p.to_path_buf()), sources),
+            None => (sources.first_path().map(|p| p.into_owned()), sources),
             Some(source) => rbobject_to_first_path_and_scan_sources(source)?,
         };
 
@@ -111,7 +112,7 @@ impl RbLazyFrame {
 
         let sources = sources.0;
         let (_first_path, sources) = match source {
-            None => (sources.first_path().map(|p| p.to_path_buf()), sources),
+            None => (sources.first_path().map(|p| p.into_owned()), sources),
             Some(source) => rbobject_to_first_path_and_scan_sources(source)?,
         };
 
@@ -147,72 +148,35 @@ impl RbLazyFrame {
         Ok(r.finish().map_err(RbPolarsErr::from)?.into())
     }
 
-    pub fn new_from_parquet(arguments: &[Value]) -> RbResult<Self> {
-        let source = Option::<Value>::try_convert(arguments[0])?;
-        let sources = Wrap::<ScanSources>::try_convert(arguments[1])?;
-        let n_rows = Option::<usize>::try_convert(arguments[2])?;
-        let cache = bool::try_convert(arguments[3])?;
-        let parallel = Wrap::<ParallelStrategy>::try_convert(arguments[4])?;
-        let rechunk = bool::try_convert(arguments[5])?;
-        let row_index = Option::<(String, IdxSize)>::try_convert(arguments[6])?;
-        let low_memory = bool::try_convert(arguments[7])?;
-        let cloud_options = Option::<Vec<(String, String)>>::try_convert(arguments[8])?;
-        let _credential_provider = Option::<Value>::try_convert(arguments[9])?;
-        let use_statistics = bool::try_convert(arguments[10])?;
-        let hive_partitioning = Option::<bool>::try_convert(arguments[11])?;
-        let schema = Option::<Wrap<Schema>>::try_convert(arguments[12])?;
-        let hive_schema = Option::<Wrap<Schema>>::try_convert(arguments[13])?;
-        let try_parse_hive_dates = bool::try_convert(arguments[14])?;
-        let retries = usize::try_convert(arguments[15])?;
-        let glob = bool::try_convert(arguments[16])?;
-        let include_file_paths = Option::<String>::try_convert(arguments[17])?;
-        let allow_missing_columns = bool::try_convert(arguments[18])?;
+    pub fn new_from_parquet(
+        sources: Wrap<ScanSources>,
+        schema: Option<Wrap<Schema>>,
+        scan_options: RbScanOptions,
+        parallel: Wrap<ParallelStrategy>,
+        low_memory: bool,
+        use_statistics: bool,
+    ) -> RbResult<Self> {
+        use crate::utils::to_rb_err;
 
         let parallel = parallel.0;
-        let hive_schema = hive_schema.map(|s| Arc::new(s.0));
-
-        let row_index = row_index.map(|(name, offset)| RowIndex {
-            name: name.into(),
-            offset,
-        });
 
-        let hive_options = HiveOptions {
-            enabled: hive_partitioning,
-            hive_start_idx: 0,
-            schema: hive_schema,
-            try_parse_dates: try_parse_hive_dates,
-        };
-
-        let mut args = ScanArgsParquet {
-            n_rows,
-            cache,
+        let options = ParquetOptions {
+            schema: schema.map(|x| Arc::new(x.0)),
             parallel,
-            rechunk,
-            row_index,
             low_memory,
-            cloud_options: None,
             use_statistics,
-            schema: schema.map(|x| Arc::new(x.0)),
-            hive_options,
-            glob,
-            include_file_paths: include_file_paths.map(|x| x.into()),
-            allow_missing_columns,
         };
 
         let sources = sources.0;
-        let (first_path, sources) = match source {
-            None => (sources.first_path().map(|p| p.to_path_buf()), sources),
-            Some(source) => rbobject_to_first_path_and_scan_sources(source)?,
-        };
+        let first_path = sources.first_path().map(|p| p.into_owned());
 
-        if let Some(first_path) = first_path {
-            let first_path_url = first_path.to_string_lossy();
-            let cloud_options =
-                parse_cloud_options(&first_path_url, cloud_options.unwrap_or_default())?;
-            args.cloud_options = Some(cloud_options.with_max_retries(retries));
-        }
+        let unified_scan_args =
+            scan_options.extract_unified_scan_args(first_path.as_ref().map(|p| p.as_ref()))?;
 
-        let lf = LazyFrame::scan_parquet_sources(sources, args).map_err(RbPolarsErr::from)?;
+        let lf: LazyFrame = DslBuilder::scan_parquet(sources, options, unified_scan_args)
+            .map_err(to_rb_err)?
+            .build()
+            .into();
 
         Ok(lf.into())
     }
@@ -254,7 +218,7 @@ impl RbLazyFrame {
 
         let sources = sources.0;
         let (_first_path, sources) = match source {
-            None => (sources.first_path().map(|p| p.to_path_buf()), sources),
+            None => (sources.first_path().map(|p| p.into_owned()), sources),
             Some(source) => rbobject_to_first_path_and_scan_sources(source)?,
         };
 
@@ -265,7 +229,7 @@ impl RbLazyFrame {
     pub fn write_json(&self, rb_f: Value) -> RbResult<()> {
         let file = BufWriter::new(get_file_like(rb_f, true)?);
         serde_json::to_writer(file, &self.ldf.borrow().logical_plan)
-            .map_err(|err| RbValueError::new_err(format!("{:?}", err)))?;
+            .map_err(|err| RbValueError::new_err(format!("{err:?}")))?;
         Ok(())
     }
 
@@ -305,7 +269,7 @@ impl RbLazyFrame {
             .with_predicate_pushdown(predicate_pushdown)
             .with_simplify_expr(simplify_expr)
             .with_slice_pushdown(slice_pushdown)
-            .with_streaming(allow_streaming)
+            .with_new_streaming(allow_streaming)
             ._with_eager(_eager)
             .with_projection_pushdown(projection_pushdown);
 
@@ -399,10 +363,8 @@ impl RbLazyFrame {
         let cloud_options = match target.base_path() {
             None => None,
             Some(base_path) => {
-                let cloud_options = parse_cloud_options(
-                    base_path.to_str().unwrap(),
-                    cloud_options.unwrap_or_default(),
-                )?;
+                let cloud_options =
+                    parse_cloud_options(base_path.to_str(), cloud_options.unwrap_or_default())?;
                 Some(cloud_options.with_max_retries(retries))
             }
         };
@@ -434,10 +396,8 @@ impl RbLazyFrame {
         let cloud_options = match target.base_path() {
             None => None,
             Some(base_path) => {
-                let cloud_options = parse_cloud_options(
-                    base_path.to_str().unwrap(),
-                    cloud_options.unwrap_or_default(),
-                )?;
+                let cloud_options =
+                    parse_cloud_options(base_path.to_str(), cloud_options.unwrap_or_default())?;
                 Some(cloud_options.with_max_retries(retries))
             }
         };
@@ -466,11 +426,12 @@ impl RbLazyFrame {
         let time_format = Option::<String>::try_convert(arguments[9])?;
         let float_scientific = Option::<bool>::try_convert(arguments[10])?;
         let float_precision = Option::<usize>::try_convert(arguments[11])?;
-        let null_value = Option::<String>::try_convert(arguments[12])?;
-        let quote_style = Option::<Wrap<QuoteStyle>>::try_convert(arguments[13])?;
-        let cloud_options = Option::<Vec<(String, String)>>::try_convert(arguments[14])?;
-        let retries = usize::try_convert(arguments[15])?;
-        let sink_options = Wrap::<SinkOptions>::try_convert(arguments[16])?;
+        let decimal_comma = bool::try_convert(arguments[12])?;
+        let null_value = Option::<String>::try_convert(arguments[13])?;
+        let quote_style = Option::<Wrap<QuoteStyle>>::try_convert(arguments[14])?;
+        let cloud_options = Option::<Vec<(String, String)>>::try_convert(arguments[15])?;
+        let retries = usize::try_convert(arguments[16])?;
+        let sink_options = Wrap::<SinkOptions>::try_convert(arguments[17])?;
 
         let quote_style = quote_style.map_or(QuoteStyle::default(), |wrap| wrap.0);
         let null_value = null_value.unwrap_or(SerializeOptions::default().null);
@@ -481,6 +442,7 @@ impl RbLazyFrame {
             datetime_format,
             float_scientific,
             float_precision,
+            decimal_comma,
             separator,
             quote_char,
             null: null_value,
@@ -498,10 +460,8 @@ impl RbLazyFrame {
         let cloud_options = match target.base_path() {
             None => None,
             Some(base_path) => {
-                let cloud_options = parse_cloud_options(
-                    base_path.to_str().unwrap(),
-                    cloud_options.unwrap_or_default(),
-                )?;
+                let cloud_options =
+                    parse_cloud_options(base_path.to_str(), cloud_options.unwrap_or_default())?;
                 Some(cloud_options.with_max_retries(retries))
             }
         };
@@ -529,10 +489,8 @@ impl RbLazyFrame {
         let cloud_options = match target.base_path() {
             None => None,
             Some(base_path) => {
-                let cloud_options = parse_cloud_options(
-                    base_path.to_str().unwrap(),
-                    cloud_options.unwrap_or_default(),
-                )?;
+                let cloud_options =
+                    parse_cloud_options(base_path.to_str(), cloud_options.unwrap_or_default())?;
                 Some(cloud_options.with_max_retries(retries))
             }
         };
@@ -546,12 +504,6 @@ impl RbLazyFrame {
         .map_err(Into::into)
     }
 
-    pub fn fetch(&self, n_rows: usize) -> RbResult<RbDataFrame> {
-        let ldf = self.ldf.borrow().clone();
-        let df = ldf.fetch(n_rows).map_err(RbPolarsErr::from)?;
-        Ok(df.into())
-    }
-
     pub fn filter(&self, predicate: &RbExpr) -> Self {
         let ldf = self.ldf.borrow().clone();
         ldf.filter(predicate.inner.clone()).into()
@@ -689,15 +641,19 @@ impl RbLazyFrame {
             .allow_parallel(allow_parallel)
             .force_parallel(force_parallel)
             .coalesce(coalesce)
-            .how(JoinType::AsOf(AsOfOptions {
+            .how(JoinType::AsOf(Box::new(AsOfOptions {
                 strategy: strategy.0,
                 left_by: left_by.map(strings_to_pl_smallstr),
                 right_by: right_by.map(strings_to_pl_smallstr),
-                tolerance: tolerance.map(|t| t.0.into_static()),
+                tolerance: tolerance.map(|t| {
+                    let av = t.0.into_static();
+                    let dtype = av.dtype();
+                    Scalar::new(dtype, av)
+                }),
                 tolerance_str: tolerance_str.map(|s| s.into()),
                 allow_eq,
                 check_sortedness,
-            }))
+            })))
             .suffix(suffix)
             .finish()
             .into())
@@ -832,10 +788,12 @@ impl RbLazyFrame {
         out.into()
     }
 
-    pub fn explode(&self, column: RArray) -> RbResult<Self> {
-        let ldf = self.ldf.borrow().clone();
-        let column = rb_exprs_to_exprs(column)?;
-        Ok(ldf.explode(column).into())
+    pub fn explode(&self, subset: &RbSelector) -> Self {
+        self.ldf
+            .borrow()
+            .clone()
+            .explode(subset.inner.clone())
+            .into()
     }
 
     pub fn null_count(&self) -> Self {
@@ -846,10 +804,11 @@ impl RbLazyFrame {
     pub fn unique(
         &self,
         maintain_order: bool,
-        subset: Option<Vec<String>>,
+        subset: Option<&RbSelector>,
         keep: Wrap<UniqueKeepStrategy>,
     ) -> RbResult<Self> {
         let ldf = self.ldf.borrow().clone();
+        let subset = subset.map(|e| e.inner.clone());
         Ok(match maintain_order {
             true => ldf.unique_stable_generic(subset, keep.0),
             false => ldf.unique_generic(subset, keep.0),
@@ -857,9 +816,11 @@ impl RbLazyFrame {
         .into())
     }
 
-    pub fn drop_nulls(&self, subset: Option<Vec<String>>) -> Self {
-        let ldf = self.ldf.borrow().clone();
-        ldf.drop_nulls(subset.map(|v| v.into_iter().map(|s| col(&s)).collect()))
+    pub fn drop_nulls(&self, subset: Option<&RbSelector>) -> Self {
+        self.ldf
+            .borrow()
+            .clone()
+            .drop_nulls(subset.map(|e| e.inner.clone()))
             .into()
     }
 
@@ -875,16 +836,14 @@ impl RbLazyFrame {
 
     pub fn unpivot(
         &self,
-        on: RArray,
-        index: RArray,
+        on: &RbSelector,
+        index: &RbSelector,
         value_name: Option<String>,
         variable_name: Option<String>,
     ) -> RbResult<Self> {
-        let on = rb_exprs_to_exprs(on)?;
-        let index = rb_exprs_to_exprs(index)?;
         let args = UnpivotArgsDSL {
-            on: on.into_iter().map(|e| e.into()).collect(),
-            index: index.into_iter().map(|e| e.into()).collect(),
+            on: on.inner.clone(),
+            index: index.inner.clone(),
             value_name: value_name.map(|s| s.into()),
             variable_name: variable_name.map(|s| s.into()),
         };
@@ -898,9 +857,8 @@ impl RbLazyFrame {
         ldf.with_row_index(&name, offset).into()
     }
 
-    pub fn drop(&self, cols: Vec<String>) -> Self {
-        let ldf = self.ldf.borrow().clone();
-        ldf.drop(cols).into()
+    pub fn drop(&self, columns: &RbSelector) -> Self {
+        self.ldf.borrow().clone().drop(columns.inner.clone()).into()
     }
 
     pub fn cast(&self, rb_dtypes: RHash, strict: bool) -> RbResult<Self> {
@@ -941,8 +899,12 @@ impl RbLazyFrame {
         Ok(schema_dict)
     }
 
-    pub fn unnest(&self, cols: Vec<String>) -> Self {
-        self.ldf.borrow().clone().unnest(cols).into()
+    pub fn unnest(&self, columns: &RbSelector) -> Self {
+        self.ldf
+            .borrow()
+            .clone()
+            .unnest(columns.inner.clone())
+            .into()
     }
 
     pub fn count(&self) -> Self {

data/ext/polars/src/lazyframe/serde.rs

@@ -25,7 +25,7 @@ impl RbLazyFrame {
         let json = unsafe { std::mem::transmute::<&'_ str, &'static str>(json.as_str()) };
 
         let lp = serde_json::from_str::<DslPlan>(json)
-            .map_err(|err| RbValueError::new_err(format!("{:?}", err)))?;
+            .map_err(|err| RbValueError::new_err(format!("{err:?}")))?;
         Ok(LazyFrame::from(lp).into())
     }
 }

data/ext/polars/src/lazyframe/sink.rs

@@ -1,9 +1,9 @@
-use std::path::{Path, PathBuf};
 use std::sync::{Arc, Mutex};
 
 use magnus::{RHash, TryConvert, Value};
 use polars::prelude::sync_on_close::SyncOnCloseType;
-use polars::prelude::{SinkOptions, SpecialEq};
+use polars::prelude::{PlPath, SinkOptions, SpecialEq};
+use polars_utils::plpath::PlPathRef;
 
 use crate::prelude::Wrap;
 use crate::{RbResult, RbValueError};
@@ -15,8 +15,8 @@ pub enum SinkTarget {
 
 impl TryConvert for Wrap<polars_plan::dsl::SinkTarget> {
     fn try_convert(ob: Value) -> RbResult<Self> {
-        if let Ok(v) = PathBuf::try_convert(ob) {
-            Ok(Wrap(polars::prelude::SinkTarget::Path(Arc::new(v))))
+        if let Ok(v) = String::try_convert(ob) {
+            Ok(Wrap(polars::prelude::SinkTarget::Path(PlPath::new(&v))))
         } else {
             let writer = {
                 let rb_f = ob;
@@ -39,10 +39,10 @@ impl TryConvert for SinkTarget {
 }
 
 impl SinkTarget {
-    pub fn base_path(&self) -> Option<&Path> {
+    pub fn base_path(&self) -> Option<PlPathRef<'_>> {
         match self {
             Self::File(t) => match t {
-                polars::prelude::SinkTarget::Path(p) => Some(p.as_path()),
+                polars::prelude::SinkTarget::Path(p) => Some(p.as_ref()),
                 polars::prelude::SinkTarget::Dyn(_) => None,
             },
         }

data/ext/polars/src/lib.rs

@@ -1,5 +1,6 @@
 mod allocator;
 mod batched_csv;
+mod catalog;
 mod conversion;
 mod dataframe;
 mod error;
@@ -8,6 +9,7 @@ mod expr;
 mod file;
 mod functions;
 mod interop;
+mod io;
 mod lazyframe;
 mod lazygroupby;
 mod map;
@@ -20,23 +22,25 @@ mod sql;
 mod utils;
 
 use batched_csv::RbBatchedCsv;
+use catalog::unity::RbCatalogClient;
 use conversion::*;
 use dataframe::RbDataFrame;
 use error::RbPolarsErr;
 use exceptions::{RbTypeError, RbValueError};
-use expr::rb_exprs_to_exprs;
 use expr::RbExpr;
+use expr::rb_exprs_to_exprs;
+use expr::selector::RbSelector;
 use functions::string_cache::RbStringCacheHolder;
 use functions::whenthen::{RbChainedThen, RbChainedWhen, RbThen, RbWhen};
 use interop::arrow::to_ruby::RbArrowArrayStream;
 use lazyframe::RbLazyFrame;
 use lazygroupby::RbLazyGroupBy;
-use magnus::{define_module, function, method, prelude::*, Ruby};
+use magnus::{Ruby, define_module, function, method, prelude::*};
 use series::RbSeries;
 use sql::RbSQLContext;
 
-use magnus::error::Result as RbResult;
 use magnus::Error as RbErr;
+use magnus::error::Result as RbResult;
 
 // TODO move
 fn re_escape(pattern: String) -> String {
@@ -143,7 +147,7 @@ fn init(ruby: &Ruby) -> RbResult<()> {
     class.define_method("pivot_expr", method!(RbDataFrame::pivot_expr, 7))?;
     class.define_method("partition_by", method!(RbDataFrame::partition_by, 3))?;
     class.define_method("lazy", method!(RbDataFrame::lazy, 0))?;
-    class.define_method("to_dummies", method!(RbDataFrame::to_dummies, 3))?;
+    class.define_method("to_dummies", method!(RbDataFrame::to_dummies, 4))?;
     class.define_method("null_count", method!(RbDataFrame::null_count, 0))?;
     class.define_method("map_rows", method!(RbDataFrame::map_rows, 3))?;
     class.define_method("shrink_to_fit", method!(RbDataFrame::shrink_to_fit, 0))?;
@@ -217,7 +221,7 @@ fn init(ruby: &Ruby) -> RbResult<()> {
     class.define_method("peak_max", method!(RbExpr::peak_max, 0))?;
     class.define_method("arg_max", method!(RbExpr::arg_max, 0))?;
     class.define_method("arg_min", method!(RbExpr::arg_min, 0))?;
-    class.define_method("search_sorted", method!(RbExpr::search_sorted, 2))?;
+    class.define_method("search_sorted", method!(RbExpr::search_sorted, 3))?;
     class.define_method("gather", method!(RbExpr::gather, 1))?;
     class.define_method("get", method!(RbExpr::get, 1))?;
     class.define_method("sort_by", method!(RbExpr::sort_by, 5))?;
@@ -329,7 +333,7 @@ fn init(ruby: &Ruby) -> RbResult<()> {
     class.define_method("str_hex_decode", method!(RbExpr::str_hex_decode, 1))?;
     class.define_method("str_base64_encode", method!(RbExpr::str_base64_encode, 0))?;
     class.define_method("str_base64_decode", method!(RbExpr::str_base64_decode, 1))?;
-    class.define_method("str_to_integer", method!(RbExpr::str_to_integer, 2))?;
+    class.define_method("str_to_integer", method!(RbExpr::str_to_integer, 3))?;
     class.define_method("str_json_decode", method!(RbExpr::str_json_decode, 2))?;
     class.define_method("binary_hex_encode", method!(RbExpr::bin_hex_encode, 0))?;
     class.define_method("binary_hex_decode", method!(RbExpr::bin_hex_decode, 1))?;
@@ -426,7 +430,6 @@ fn init(ruby: &Ruby) -> RbResult<()> {
     class.define_method("dot", method!(RbExpr::dot, 1))?;
     class.define_method("reinterpret", method!(RbExpr::reinterpret, 1))?;
     class.define_method("mode", method!(RbExpr::mode, 0))?;
-    class.define_method("exclude", method!(RbExpr::exclude, 1))?;
     class.define_method("interpolate", method!(RbExpr::interpolate, 1))?;
     class.define_method("interpolate_by", method!(RbExpr::interpolate_by, 1))?;
     class.define_method("rolling_sum", method!(RbExpr::rolling_sum, 4))?;
@@ -464,9 +467,10 @@ fn init(ruby: &Ruby) -> RbResult<()> {
     class.define_method("list_to_array", method!(RbExpr::list_to_array, 1))?;
     class.define_method("list_mean", method!(RbExpr::list_mean, 0))?;
     class.define_method("list_tail", method!(RbExpr::list_tail, 1))?;
-    class.define_method("list_sort", method!(RbExpr::list_sort, 1))?;
+    class.define_method("list_sort", method!(RbExpr::list_sort, 2))?;
     class.define_method("list_reverse", method!(RbExpr::list_reverse, 0))?;
     class.define_method("list_unique", method!(RbExpr::list_unique, 1))?;
+    class.define_method("list_set_operation", method!(RbExpr::list_set_operation, 2))?;
     class.define_method("list_get", method!(RbExpr::list_get, 2))?;
     class.define_method("list_join", method!(RbExpr::list_join, 2))?;
     class.define_method("list_arg_min", method!(RbExpr::list_arg_min, 0))?;
@@ -476,8 +480,9 @@ fn init(ruby: &Ruby) -> RbResult<()> {
     class.define_method("list_diff", method!(RbExpr::list_diff, 2))?;
     class.define_method("list_shift", method!(RbExpr::list_shift, 1))?;
     class.define_method("list_slice", method!(RbExpr::list_slice, 2))?;
-    class.define_method("list_eval", method!(RbExpr::list_eval, 2))?;
-    class.define_method("cumulative_eval", method!(RbExpr::cumulative_eval, 3))?;
+    class.define_method("list_eval", method!(RbExpr::list_eval, 1))?;
+    class.define_method("list_filter", method!(RbExpr::list_filter, 1))?;
+    class.define_method("cumulative_eval", method!(RbExpr::cumulative_eval, 2))?;
     class.define_method("list_to_struct", method!(RbExpr::list_to_struct, 3))?;
     class.define_method("rank", method!(RbExpr::rank, 3))?;
     class.define_method("diff", method!(RbExpr::diff, 2))?;
@@ -518,9 +523,11 @@ fn init(ruby: &Ruby) -> RbResult<()> {
     class.define_method("set_sorted_flag", method!(RbExpr::set_sorted_flag, 1))?;
     class.define_method("replace", method!(RbExpr::replace, 2))?;
     class.define_method("replace_strict", method!(RbExpr::replace_strict, 4))?;
+    class.define_method("into_selector", method!(RbExpr::into_selector, 0))?;
+    class.define_singleton_method("new_selector", function!(RbExpr::new_selector, 1))?;
 
     // meta
-    class.define_method("meta_pop", method!(RbExpr::meta_pop, 0))?;
+    class.define_method("meta_pop", method!(RbExpr::meta_pop, 1))?;
     class.define_method("meta_eq", method!(RbExpr::meta_eq, 1))?;
     class.define_method("meta_roots", method!(RbExpr::meta_root_names, 0))?;
     class.define_method("meta_output_name", method!(RbExpr::meta_output_name, 0))?;
@@ -534,11 +541,7 @@ fn init(ruby: &Ruby) -> RbResult<()> {
         "meta_is_regex_projection",
         method!(RbExpr::meta_is_regex_projection, 0),
     )?;
-    class.define_method("_meta_selector_add", method!(RbExpr::_meta_selector_add, 1))?;
-    class.define_method("_meta_selector_sub", method!(RbExpr::_meta_selector_sub, 1))?;
-    class.define_method("_meta_selector_and", method!(RbExpr::_meta_selector_and, 1))?;
-    class.define_method("_meta_as_selector", method!(RbExpr::_meta_as_selector, 0))?;
-    class.define_method("meta_tree_format", method!(RbExpr::meta_tree_format, 0))?;
+    class.define_method("meta_tree_format", method!(RbExpr::meta_tree_format, 1))?;
 
     // name
     class.define_method("name_keep", method!(RbExpr::name_keep, 0))?;
@@ -550,15 +553,10 @@ fn init(ruby: &Ruby) -> RbResult<()> {
 
     // maybe add to different class
     let class = module.define_module("Plr")?;
-    class.define_singleton_method("dtype_cols", function!(functions::lazy::dtype_cols, 1))?;
-    class.define_singleton_method("index_cols", function!(functions::lazy::index_cols, 1))?;
     class.define_singleton_method("col", function!(functions::lazy::col, 1))?;
     class.define_singleton_method("len", function!(functions::lazy::len, 0))?;
-    class.define_singleton_method("first", function!(functions::lazy::first, 0))?;
-    class.define_singleton_method("last", function!(functions::lazy::last, 0))?;
-    class.define_singleton_method("cols", function!(functions::lazy::cols, 1))?;
     class.define_singleton_method("fold", function!(functions::lazy::fold, 5))?;
-    class.define_singleton_method("cum_fold", function!(functions::lazy::cum_fold, 4))?;
+    class.define_singleton_method("cum_fold", function!(functions::lazy::cum_fold, 6))?;
     class.define_singleton_method("lit", function!(functions::lazy::lit, 3))?;
     class.define_singleton_method("int_range", function!(functions::range::int_range, 4))?;
     class.define_singleton_method("int_ranges", function!(functions::range::int_ranges, 4))?;
@@ -729,7 +727,7 @@ fn init(ruby: &Ruby) -> RbResult<()> {
     class.define_singleton_method("new_from_csv", function!(RbLazyFrame::new_from_csv, -1))?;
     class.define_singleton_method(
         "new_from_parquet",
-        function!(RbLazyFrame::new_from_parquet, -1),
+        function!(RbLazyFrame::new_from_parquet, 6),
     )?;
     class.define_singleton_method("new_from_ipc", function!(RbLazyFrame::new_from_ipc, 10))?;
     class.define_method("write_json", method!(RbLazyFrame::write_json, 1))?;
@@ -750,7 +748,6 @@ fn init(ruby: &Ruby) -> RbResult<()> {
     class.define_method("sink_ipc", method!(RbLazyFrame::sink_ipc, 5))?;
     class.define_method("sink_csv", method!(RbLazyFrame::sink_csv, -1))?;
     class.define_method("sink_json", method!(RbLazyFrame::sink_json, 4))?;
-    class.define_method("fetch", method!(RbLazyFrame::fetch, 1))?;
     class.define_method("filter", method!(RbLazyFrame::filter, 1))?;
     class.define_method("select", method!(RbLazyFrame::select, 1))?;
     class.define_method("select_seq", method!(RbLazyFrame::select_seq, 1))?;
@@ -908,9 +905,9 @@ fn init(ruby: &Ruby) -> RbResult<()> {
     class.define_method("median", method!(RbSeries::median, 0))?;
     class.define_method("quantile", method!(RbSeries::quantile, 2))?;
     class.define_method("_clone", method!(RbSeries::clone, 0))?;
-    class.define_method("apply_lambda", method!(RbSeries::apply_lambda, 3))?;
+    class.define_method("map_elements", method!(RbSeries::map_elements, 3))?;
     class.define_method("zip_with", method!(RbSeries::zip_with, 2))?;
-    class.define_method("to_dummies", method!(RbSeries::to_dummies, 2))?;
+    class.define_method("to_dummies", method!(RbSeries::to_dummies, 3))?;
     class.define_method("n_unique", method!(RbSeries::n_unique, 0))?;
     class.define_method("floor", method!(RbSeries::floor, 0))?;
     class.define_method("shrink_to_fit", method!(RbSeries::shrink_to_fit, 0))?;
@@ -1111,5 +1108,86 @@ fn init(ruby: &Ruby) -> RbResult<()> {
     let class = module.define_class("ArrowArrayStream", ruby.class_object())?;
     class.define_method("to_i", method!(RbArrowArrayStream::to_i, 0))?;
 
+    // catalog
+    let class = module.define_class("RbCatalogClient", ruby.class_object())?;
+    class.define_singleton_method("new", function!(RbCatalogClient::new, 2))?;
+    class.define_singleton_method(
+        "type_json_to_polars_type",
+        function!(RbCatalogClient::type_json_to_polars_type, 1),
+    )?;
+    class.define_method("list_catalogs", method!(RbCatalogClient::list_catalogs, 0))?;
+    class.define_method(
+        "list_namespaces",
+        method!(RbCatalogClient::list_namespaces, 1),
+    )?;
+    class.define_method("list_tables", method!(RbCatalogClient::list_tables, 2))?;
+    class.define_method(
+        "get_table_info",
+        method!(RbCatalogClient::get_table_info, 3),
+    )?;
+    class.define_method(
+        "create_catalog",
+        method!(RbCatalogClient::create_catalog, 3),
+    )?;
+    class.define_method(
+        "delete_catalog",
+        method!(RbCatalogClient::delete_catalog, 2),
+    )?;
+    class.define_method(
+        "create_namespace",
+        method!(RbCatalogClient::create_namespace, 4),
+    )?;
+    class.define_method(
+        "delete_namespace",
+        method!(RbCatalogClient::delete_namespace, 3),
+    )?;
+    class.define_method("create_table", method!(RbCatalogClient::create_table, 9))?;
+    class.define_method("delete_table", method!(RbCatalogClient::delete_table, 3))?;
+
+    // categories
+    let class = module.define_class("RbCategories", ruby.class_object())?;
+    class.define_singleton_method(
+        "global_categories",
+        function!(RbCategories::global_categories, 0),
+    )?;
+
+    // data type expr
+    let _class = module.define_class("RbDataTypeExpr", ruby.class_object())?;
+
+    // selector
+    let class = module.define_class("RbSelector", ruby.class_object())?;
+    class.define_method("union", method!(RbSelector::union, 1))?;
+    class.define_method("difference", method!(RbSelector::difference, 1))?;
+    class.define_method("exclusive_or", method!(RbSelector::exclusive_or, 1))?;
+    class.define_method("intersect", method!(RbSelector::intersect, 1))?;
+    class.define_singleton_method("by_dtype", function!(RbSelector::by_dtype, 1))?;
+    class.define_singleton_method("by_name", function!(RbSelector::by_name, 2))?;
+    class.define_singleton_method("by_index", function!(RbSelector::by_index, 2))?;
+    class.define_singleton_method("first", function!(RbSelector::first, 1))?;
+    class.define_singleton_method("last", function!(RbSelector::last, 1))?;
+    class.define_singleton_method("matches", function!(RbSelector::matches, 1))?;
+    class.define_singleton_method("enum_", function!(RbSelector::enum_, 0))?;
+    class.define_singleton_method("categorical", function!(RbSelector::categorical, 0))?;
+    class.define_singleton_method("nested", function!(RbSelector::nested, 0))?;
+    class.define_singleton_method("list", function!(RbSelector::list, 1))?;
+    class.define_singleton_method("array", function!(RbSelector::array, 2))?;
+    class.define_singleton_method("struct_", function!(RbSelector::struct_, 0))?;
+    class.define_singleton_method("integer", function!(RbSelector::integer, 0))?;
+    class.define_singleton_method("signed_integer", function!(RbSelector::signed_integer, 0))?;
+    class.define_singleton_method(
+        "unsigned_integer",
+        function!(RbSelector::unsigned_integer, 0),
+    )?;
+    class.define_singleton_method("float", function!(RbSelector::float, 0))?;
+    class.define_singleton_method("decimal", function!(RbSelector::decimal, 0))?;
+    class.define_singleton_method("numeric", function!(RbSelector::numeric, 0))?;
+    class.define_singleton_method("temporal", function!(RbSelector::temporal, 0))?;
+    class.define_singleton_method("datetime", function!(RbSelector::datetime, 2))?;
+    class.define_singleton_method("duration", function!(RbSelector::duration, 1))?;
+    class.define_singleton_method("object", function!(RbSelector::object, 0))?;
+    class.define_singleton_method("empty", function!(RbSelector::empty, 0))?;
+    class.define_singleton_method("all", function!(RbSelector::all, 0))?;
+    class.define_method("_hash", method!(RbSelector::hash, 0))?;
+
     Ok(())
 }