polars-df 0.1.3 → 0.1.5

data/ext/polars/src/lazy/dsl.rs

@@ -1,3 +1,4 @@
+use magnus::block::Proc;
 use magnus::{class, RArray, RString, Value};
 use polars::chunked_array::ops::SortOptions;
 use polars::lazy::dsl;
@@ -659,6 +660,65 @@ impl RbExpr {
         self.inner.clone().str().starts_with(sub).into()
     }
 
+    pub fn str_hex_encode(&self) -> Self {
+        self.clone()
+            .inner
+            .map(
+                move |s| s.utf8().map(|s| s.hex_encode().into_series()),
+                GetOutput::same_type(),
+            )
+            .with_fmt("str.hex_encode")
+            .into()
+    }
+
+    pub fn str_hex_decode(&self, strict: Option<bool>) -> Self {
+        self.clone()
+            .inner
+            .map(
+                move |s| s.utf8()?.hex_decode(strict).map(|s| s.into_series()),
+                GetOutput::same_type(),
+            )
+            .with_fmt("str.hex_decode")
+            .into()
+    }
+
+    pub fn str_base64_encode(&self) -> Self {
+        self.clone()
+            .inner
+            .map(
+                move |s| s.utf8().map(|s| s.base64_encode().into_series()),
+                GetOutput::same_type(),
+            )
+            .with_fmt("str.base64_encode")
+            .into()
+    }
+
+    pub fn str_base64_decode(&self, strict: Option<bool>) -> Self {
+        self.clone()
+            .inner
+            .map(
+                move |s| s.utf8()?.base64_decode(strict).map(|s| s.into_series()),
+                GetOutput::same_type(),
+            )
+            .with_fmt("str.base64_decode")
+            .into()
+    }
+
+    pub fn str_json_path_match(&self, pat: String) -> Self {
+        let function = move |s: Series| {
+            let ca = s.utf8()?;
+            match ca.json_path_match(&pat) {
+                Ok(ca) => Ok(ca.into_series()),
+                Err(e) => Err(PolarsError::ComputeError(format!("{:?}", e).into())),
+            }
+        };
+        self.clone()
+            .inner
+            .map(function, GetOutput::from_type(DataType::Utf8))
+            .with_fmt("str.json_path_match")
+            .into()
+    }
+
     pub fn str_extract(&self, pat: String, group_index: usize) -> Self {
         self.inner.clone().str().extract(&pat, group_index).into()
     }
@@ -887,6 +947,14 @@ impl RbExpr {
         self.inner.clone().dt().round(&every, &offset).into()
     }
 
+    pub fn map(&self, lambda: Value, output_type: Option<Wrap<DataType>>, agg_list: bool) -> Self {
+        map_single(self, lambda, output_type, agg_list)
+    }
+
+    pub fn dot(&self, other: &RbExpr) -> Self {
+        self.inner.clone().dot(other.inner.clone()).into()
+    }
+
     pub fn reinterpret(&self, signed: bool) -> Self {
         let function = move |s: Series| reinterpret(&s, signed);
         let dt = if signed {
@@ -916,6 +984,23 @@ impl RbExpr {
         self.inner.clone().suffix(&suffix).into()
     }
 
+    pub fn map_alias(&self, lambda: Proc) -> Self {
+        self.inner
+            .clone()
+            .map_alias(move |name| {
+                let out = lambda.call::<_, String>((name,));
+                // TODO switch to match
+                out.unwrap()
+                // match out {
+                //     Ok(out) => Ok(out.to_string()),
+                //     Err(e) => Err(PolarsError::ComputeError(
+                //         format!("Ruby function in 'map_alias' produced an error: {}.", e).into(),
+                //     )),
+                // }
+            })
+            .into()
+    }
+
     pub fn exclude(&self, columns: Vec<String>) -> Self {
         self.inner.clone().exclude(columns).into()
     }
@@ -1208,6 +1293,28 @@ impl RbExpr {
             .into()
     }
 
+    pub fn lst_to_struct(
+        &self,
+        width_strat: Wrap<ListToStructWidthStrategy>,
+        _name_gen: Option<Value>,
+    ) -> RbResult<Self> {
+        // TODO fix
+        let name_gen = None;
+        // let name_gen = name_gen.map(|lambda| {
+        //     Arc::new(move |idx: usize| {
+        //         let out: Value = lambda.funcall("call", (idx,)).unwrap();
+        //         out.try_convert::<String>().unwrap()
+        //     }) as NameGenerator
+        // });
+
+        Ok(self
+            .inner
+            .clone()
+            .arr()
+            .to_struct(width_strat.0, name_gen)
+            .into())
+    }
+
     pub fn rank(&self, method: Wrap<RankMethod>, reverse: bool) -> Self {
         let options = RankOptions {
             method: method.0,
@@ -1365,6 +1472,10 @@ impl RbExpr {
     pub fn entropy(&self, base: f64, normalize: bool) -> Self {
         self.inner.clone().entropy(base, normalize).into()
     }
+
+    pub fn hash(&self, seed: u64, seed_1: u64, seed_2: u64, seed_3: u64) -> Self {
+        self.inner.clone().hash(seed, seed_1, seed_2, seed_3).into()
+    }
 }
 
 pub fn col(name: String) -> RbExpr {
@@ -1394,6 +1505,13 @@ pub fn fold(acc: &RbExpr, lambda: Value, exprs: RArray) -> RbResult<RbExpr> {
     Ok(polars::lazy::dsl::fold_exprs(acc.inner.clone(), func, exprs).into())
 }
 
+pub fn cumfold(acc: &RbExpr, lambda: Value, exprs: RArray, include_init: bool) -> RbResult<RbExpr> {
+    let exprs = rb_exprs_to_exprs(exprs)?;
+
+    let func = move |a: Series, b: Series| binary_lambda(lambda, a, b);
+    Ok(polars::lazy::dsl::cumfold_exprs(acc.inner.clone(), func, exprs, include_init).into())
+}
+
 // TODO improve
 pub fn lit(value: Value) -> RbResult<RbExpr> {
     if value.is_nil() {
@@ -1433,6 +1551,24 @@ pub fn repeat(value: Value, n_times: &RbExpr) -> RbResult<RbExpr> {
     }
 }
 
+pub fn pearson_corr(a: &RbExpr, b: &RbExpr, ddof: u8) -> RbExpr {
+    polars::lazy::dsl::pearson_corr(a.inner.clone(), b.inner.clone(), ddof).into()
+}
+
+pub fn spearman_rank_corr(a: &RbExpr, b: &RbExpr, ddof: u8, propagate_nans: bool) -> RbExpr {
+    polars::lazy::dsl::spearman_rank_corr(a.inner.clone(), b.inner.clone(), ddof, propagate_nans)
+        .into()
+}
+
+pub fn cov(a: &RbExpr, b: &RbExpr) -> RbExpr {
+    polars::lazy::dsl::cov(a.inner.clone(), b.inner.clone()).into()
+}
+
+pub fn argsort_by(by: RArray, reverse: Vec<bool>) -> RbResult<RbExpr> {
+    let by = rb_exprs_to_exprs(by)?;
+    Ok(polars::lazy::dsl::argsort_by(by, &reverse).into())
+}
+
 #[magnus::wrap(class = "Polars::RbWhen")]
 #[derive(Clone)]
 pub struct RbWhen {

data/ext/polars/src/lib.rs

@@ -1,9 +1,11 @@
+mod apply;
 mod batched_csv;
 mod conversion;
 mod dataframe;
 mod error;
 mod file;
 mod lazy;
+mod list_construction;
 mod series;
 mod set;
 mod utils;
@@ -24,9 +26,23 @@ use polars::datatypes::{DataType, TimeUnit};
 use polars::error::PolarsResult;
 use polars::frame::DataFrame;
 use polars::functions::{diag_concat_df, hor_concat_df};
-use polars::prelude::{ClosedWindow, Duration, IntoSeries, TimeZone};
+use polars::prelude::{ClosedWindow, Duration, DurationArgs, IntoSeries, TimeZone};
 use series::RbSeries;
 
+#[cfg(target_os = "linux")]
+use jemallocator::Jemalloc;
+
+#[cfg(not(target_os = "linux"))]
+use mimalloc::MiMalloc;
+
+#[global_allocator]
+#[cfg(target_os = "linux")]
+static GLOBAL: Jemalloc = Jemalloc;
+
+#[global_allocator]
+#[cfg(not(target_os = "linux"))]
+static GLOBAL: MiMalloc = MiMalloc;
+
 type RbResult<T> = Result<T, Error>;
 
 fn module() -> RModule {
@@ -40,13 +56,18 @@ fn series() -> RClass {
 #[magnus::init]
 fn init() -> RbResult<()> {
     let module = module();
+    module.define_singleton_method("_rb_duration", function!(rb_duration, 8))?;
     module.define_singleton_method("_concat_df", function!(concat_df, 1))?;
+    module.define_singleton_method("_concat_lf", function!(concat_lf, 3))?;
     module.define_singleton_method("_diag_concat_df", function!(rb_diag_concat_df, 1))?;
     module.define_singleton_method("_hor_concat_df", function!(rb_hor_concat_df, 1))?;
     module.define_singleton_method("_concat_series", function!(concat_series, 1))?;
     module.define_singleton_method("_ipc_schema", function!(ipc_schema, 1))?;
     module.define_singleton_method("_parquet_schema", function!(parquet_schema, 1))?;
+    module.define_singleton_method("_collect_all", function!(collect_all, 1))?;
     module.define_singleton_method("_rb_date_range", function!(rb_date_range, 7))?;
+    module.define_singleton_method("_coalesce_exprs", function!(coalesce_exprs, 1))?;
+    module.define_singleton_method("_sum_exprs", function!(sum_exprs, 1))?;
     module.define_singleton_method("_as_struct", function!(as_struct, 1))?;
     module.define_singleton_method("_arg_where", function!(arg_where, 1))?;
 
@@ -59,10 +80,13 @@ fn init() -> RbResult<()> {
     class.define_singleton_method("read_csv", function!(RbDataFrame::read_csv, -1))?;
     class.define_singleton_method("read_parquet", function!(RbDataFrame::read_parquet, 7))?;
     class.define_singleton_method("read_ipc", function!(RbDataFrame::read_ipc, 6))?;
+    class.define_singleton_method("read_avro", function!(RbDataFrame::read_avro, 4))?;
+    class.define_singleton_method("read_hashes", function!(RbDataFrame::read_hashes, 3))?;
     class.define_singleton_method("read_hash", function!(RbDataFrame::read_hash, 1))?;
     class.define_singleton_method("read_json", function!(RbDataFrame::read_json, 1))?;
     class.define_singleton_method("read_ndjson", function!(RbDataFrame::read_ndjson, 1))?;
     class.define_method("estimated_size", method!(RbDataFrame::estimated_size, 0))?;
+    class.define_method("write_avro", method!(RbDataFrame::write_avro, 2))?;
     class.define_method("write_json", method!(RbDataFrame::write_json, 3))?;
     class.define_method("write_ndjson", method!(RbDataFrame::write_ndjson, 1))?;
     class.define_method("write_csv", method!(RbDataFrame::write_csv, 10))?;
@@ -128,6 +152,7 @@ fn init() -> RbResult<()> {
     class.define_method("with_row_count", method!(RbDataFrame::with_row_count, 2))?;
     class.define_method("_clone", method!(RbDataFrame::clone, 0))?;
     class.define_method("melt", method!(RbDataFrame::melt, 4))?;
+    class.define_method("pivot_expr", method!(RbDataFrame::pivot_expr, 6))?;
     class.define_method("partition_by", method!(RbDataFrame::partition_by, 2))?;
     class.define_method("shift", method!(RbDataFrame::shift, 1))?;
     class.define_method("unique", method!(RbDataFrame::unique, 3))?;
@@ -146,7 +171,9 @@ fn init() -> RbResult<()> {
     class.define_method("quantile", method!(RbDataFrame::quantile, 2))?;
     class.define_method("to_dummies", method!(RbDataFrame::to_dummies, 1))?;
     class.define_method("null_count", method!(RbDataFrame::null_count, 0))?;
+    class.define_method("apply", method!(RbDataFrame::apply, 3))?;
     class.define_method("shrink_to_fit", method!(RbDataFrame::shrink_to_fit, 0))?;
+    class.define_method("hash_rows", method!(RbDataFrame::hash_rows, 4))?;
     class.define_method("transpose", method!(RbDataFrame::transpose, 2))?;
     class.define_method("upsample", method!(RbDataFrame::upsample, 5))?;
     class.define_method("to_struct", method!(RbDataFrame::to_struct, 1))?;
@@ -281,6 +308,14 @@ fn init() -> RbResult<()> {
     class.define_method("str_contains", method!(RbExpr::str_contains, 2))?;
     class.define_method("str_ends_with", method!(RbExpr::str_ends_with, 1))?;
     class.define_method("str_starts_with", method!(RbExpr::str_starts_with, 1))?;
+    class.define_method("str_hex_encode", method!(RbExpr::str_hex_encode, 0))?;
+    class.define_method("str_hex_decode", method!(RbExpr::str_hex_decode, 1))?;
+    class.define_method("str_base64_encode", method!(RbExpr::str_base64_encode, 0))?;
+    class.define_method("str_base64_decode", method!(RbExpr::str_base64_decode, 1))?;
+    class.define_method(
+        "str_json_path_match",
+        method!(RbExpr::str_json_path_match, 1),
+    )?;
     class.define_method("str_extract", method!(RbExpr::str_extract, 2))?;
     class.define_method("str_extract_all", method!(RbExpr::str_extract_all, 1))?;
     class.define_method("count_match", method!(RbExpr::count_match, 1))?;
@@ -338,11 +373,14 @@ fn init() -> RbResult<()> {
     class.define_method("dt_tz_localize", method!(RbExpr::dt_tz_localize, 1))?;
     class.define_method("dt_truncate", method!(RbExpr::dt_truncate, 2))?;
     class.define_method("dt_round", method!(RbExpr::dt_round, 2))?;
+    class.define_method("map", method!(RbExpr::map, 3))?;
+    class.define_method("dot", method!(RbExpr::dot, 1))?;
     class.define_method("reinterpret", method!(RbExpr::reinterpret, 1))?;
     class.define_method("mode", method!(RbExpr::mode, 0))?;
     class.define_method("keep_name", method!(RbExpr::keep_name, 0))?;
     class.define_method("prefix", method!(RbExpr::prefix, 1))?;
     class.define_method("suffix", method!(RbExpr::suffix, 1))?;
+    class.define_method("map_alias", method!(RbExpr::map_alias, 1))?;
     class.define_method("exclude", method!(RbExpr::exclude, 1))?;
     class.define_method("interpolate", method!(RbExpr::interpolate, 0))?;
     class.define_method("rolling_sum", method!(RbExpr::rolling_sum, 6))?;
@@ -372,6 +410,7 @@ fn init() -> RbResult<()> {
     class.define_method("lst_slice", method!(RbExpr::lst_slice, 2))?;
     class.define_method("lst_eval", method!(RbExpr::lst_eval, 2))?;
     class.define_method("cumulative_eval", method!(RbExpr::cumulative_eval, 3))?;
+    class.define_method("lst_to_struct", method!(RbExpr::lst_to_struct, 2))?;
     class.define_method("rank", method!(RbExpr::rank, 2))?;
     class.define_method("diff", method!(RbExpr::diff, 2))?;
     class.define_method("pct_change", method!(RbExpr::pct_change, 1))?;
@@ -406,6 +445,7 @@ fn init() -> RbResult<()> {
     class.define_method("log", method!(RbExpr::log, 1))?;
     class.define_method("exp", method!(RbExpr::exp, 0))?;
     class.define_method("entropy", method!(RbExpr::entropy, 2))?;
+    class.define_method("_hash", method!(RbExpr::hash, 4))?;
 
     // meta
     class.define_method("meta_pop", method!(RbExpr::meta_pop, 0))?;
@@ -421,14 +461,23 @@ fn init() -> RbResult<()> {
     class.define_singleton_method("last", function!(crate::lazy::dsl::last, 0))?;
     class.define_singleton_method("cols", function!(crate::lazy::dsl::cols, 1))?;
     class.define_singleton_method("fold", function!(crate::lazy::dsl::fold, 3))?;
+    class.define_singleton_method("cumfold", function!(crate::lazy::dsl::cumfold, 4))?;
     class.define_singleton_method("lit", function!(crate::lazy::dsl::lit, 1))?;
     class.define_singleton_method("arange", function!(crate::lazy::dsl::arange, 3))?;
     class.define_singleton_method("repeat", function!(crate::lazy::dsl::repeat, 2))?;
+    class.define_singleton_method("pearson_corr", function!(crate::lazy::dsl::pearson_corr, 3))?;
+    class.define_singleton_method(
+        "spearman_rank_corr",
+        function!(crate::lazy::dsl::spearman_rank_corr, 4),
+    )?;
+    class.define_singleton_method("cov", function!(crate::lazy::dsl::cov, 2))?;
+    class.define_singleton_method("argsort_by", function!(crate::lazy::dsl::argsort_by, 2))?;
     class.define_singleton_method("when", function!(crate::lazy::dsl::when, 1))?;
     class.define_singleton_method("concat_str", function!(crate::lazy::dsl::concat_str, 2))?;
     class.define_singleton_method("concat_lst", function!(crate::lazy::dsl::concat_lst, 1))?;
 
     let class = module.define_class("RbLazyFrame", Default::default())?;
+    class.define_singleton_method("read_json", function!(RbLazyFrame::read_json, 1))?;
     class.define_singleton_method(
         "new_from_ndjson",
         function!(RbLazyFrame::new_from_ndjson, 7),
@@ -459,6 +508,8 @@ fn init() -> RbResult<()> {
     class.define_method("groupby", method!(RbLazyFrame::groupby, 2))?;
     class.define_method("groupby_rolling", method!(RbLazyFrame::groupby_rolling, 5))?;
     class.define_method("groupby_dynamic", method!(RbLazyFrame::groupby_dynamic, 8))?;
+    class.define_method("with_context", method!(RbLazyFrame::with_context, 1))?;
+    class.define_method("join_asof", method!(RbLazyFrame::join_asof, 11))?;
     class.define_method("join", method!(RbLazyFrame::join, 7))?;
     class.define_method("with_columns", method!(RbLazyFrame::with_columns, 1))?;
     class.define_method("rename", method!(RbLazyFrame::rename, 2))?;
@@ -507,7 +558,10 @@ fn init() -> RbResult<()> {
     class.define_singleton_method("new_opt_f32", function!(RbSeries::new_opt_f32, 3))?;
     class.define_singleton_method("new_opt_f64", function!(RbSeries::new_opt_f64, 3))?;
     class.define_singleton_method("new_str", function!(RbSeries::new_str, 3))?;
+    class.define_singleton_method("new_object", function!(RbSeries::new_object, 3))?;
+    class.define_singleton_method("new_list", function!(RbSeries::new_list, 3))?;
     class.define_singleton_method("new_opt_date", function!(RbSeries::new_opt_date, 3))?;
+    class.define_singleton_method("new_opt_datetime", function!(RbSeries::new_opt_datetime, 3))?;
     class.define_method("is_sorted_flag", method!(RbSeries::is_sorted_flag, 0))?;
     class.define_method(
         "is_sorted_reverse_flag",
@@ -563,6 +617,7 @@ fn init() -> RbResult<()> {
     class.define_method("median", method!(RbSeries::median, 0))?;
     class.define_method("quantile", method!(RbSeries::quantile, 2))?;
     class.define_method("_clone", method!(RbSeries::clone, 0))?;
+    class.define_method("apply_lambda", method!(RbSeries::apply_lambda, 3))?;
     class.define_method("zip_with", method!(RbSeries::zip_with, 2))?;
     class.define_method("to_dummies", method!(RbSeries::to_dummies, 0))?;
     class.define_method("peak_max", method!(RbSeries::peak_max, 0))?;
@@ -577,6 +632,79 @@ fn init() -> RbResult<()> {
     class.define_method("time_unit", method!(RbSeries::time_unit, 0))?;
     class.define_method("set_at_idx", method!(RbSeries::set_at_idx, 2))?;
 
+    // set
+    // class.define_method("set_with_mask_str", method!(RbSeries::set_with_mask_str, 2))?;
+    class.define_method("set_with_mask_f64", method!(RbSeries::set_with_mask_f64, 2))?;
+    class.define_method("set_with_mask_f32", method!(RbSeries::set_with_mask_f32, 2))?;
+    class.define_method("set_with_mask_u8", method!(RbSeries::set_with_mask_u8, 2))?;
+    class.define_method("set_with_mask_u16", method!(RbSeries::set_with_mask_u16, 2))?;
+    class.define_method("set_with_mask_u32", method!(RbSeries::set_with_mask_u32, 2))?;
+    class.define_method("set_with_mask_u64", method!(RbSeries::set_with_mask_u64, 2))?;
+    class.define_method("set_with_mask_i8", method!(RbSeries::set_with_mask_i8, 2))?;
+    class.define_method("set_with_mask_i16", method!(RbSeries::set_with_mask_i16, 2))?;
+    class.define_method("set_with_mask_i32", method!(RbSeries::set_with_mask_i32, 2))?;
+    class.define_method("set_with_mask_i64", method!(RbSeries::set_with_mask_i64, 2))?;
+    class.define_method(
+        "set_with_mask_bool",
+        method!(RbSeries::set_with_mask_bool, 2),
+    )?;
+
+    // arithmetic
+    class.define_method("add_u8", method!(RbSeries::add_u8, 1))?;
+    class.define_method("add_u16", method!(RbSeries::add_u16, 1))?;
+    class.define_method("add_u32", method!(RbSeries::add_u32, 1))?;
+    class.define_method("add_u64", method!(RbSeries::add_u64, 1))?;
+    class.define_method("add_i8", method!(RbSeries::add_i8, 1))?;
+    class.define_method("add_i16", method!(RbSeries::add_i16, 1))?;
+    class.define_method("add_i32", method!(RbSeries::add_i32, 1))?;
+    class.define_method("add_i64", method!(RbSeries::add_i64, 1))?;
+    class.define_method("add_datetime", method!(RbSeries::add_datetime, 1))?;
+    class.define_method("add_duration", method!(RbSeries::add_duration, 1))?;
+    class.define_method("add_f32", method!(RbSeries::add_f32, 1))?;
+    class.define_method("add_f64", method!(RbSeries::add_f64, 1))?;
+    class.define_method("sub_u8", method!(RbSeries::sub_u8, 1))?;
+    class.define_method("sub_u16", method!(RbSeries::sub_u16, 1))?;
+    class.define_method("sub_u32", method!(RbSeries::sub_u32, 1))?;
+    class.define_method("sub_u64", method!(RbSeries::sub_u64, 1))?;
+    class.define_method("sub_i8", method!(RbSeries::sub_i8, 1))?;
+    class.define_method("sub_i16", method!(RbSeries::sub_i16, 1))?;
+    class.define_method("sub_i32", method!(RbSeries::sub_i32, 1))?;
+    class.define_method("sub_i64", method!(RbSeries::sub_i64, 1))?;
+    class.define_method("sub_datetime", method!(RbSeries::sub_datetime, 1))?;
+    class.define_method("sub_duration", method!(RbSeries::sub_duration, 1))?;
+    class.define_method("sub_f32", method!(RbSeries::sub_f32, 1))?;
+    class.define_method("sub_f64", method!(RbSeries::sub_f64, 1))?;
+    class.define_method("div_u8", method!(RbSeries::div_u8, 1))?;
+    class.define_method("div_u16", method!(RbSeries::div_u16, 1))?;
+    class.define_method("div_u32", method!(RbSeries::div_u32, 1))?;
+    class.define_method("div_u64", method!(RbSeries::div_u64, 1))?;
+    class.define_method("div_i8", method!(RbSeries::div_i8, 1))?;
+    class.define_method("div_i16", method!(RbSeries::div_i16, 1))?;
+    class.define_method("div_i32", method!(RbSeries::div_i32, 1))?;
+    class.define_method("div_i64", method!(RbSeries::div_i64, 1))?;
+    class.define_method("div_f32", method!(RbSeries::div_f32, 1))?;
+    class.define_method("div_f64", method!(RbSeries::div_f64, 1))?;
+    class.define_method("mul_u8", method!(RbSeries::mul_u8, 1))?;
+    class.define_method("mul_u16", method!(RbSeries::mul_u16, 1))?;
+    class.define_method("mul_u32", method!(RbSeries::mul_u32, 1))?;
+    class.define_method("mul_u64", method!(RbSeries::mul_u64, 1))?;
+    class.define_method("mul_i8", method!(RbSeries::mul_i8, 1))?;
+    class.define_method("mul_i16", method!(RbSeries::mul_i16, 1))?;
+    class.define_method("mul_i32", method!(RbSeries::mul_i32, 1))?;
+    class.define_method("mul_i64", method!(RbSeries::mul_i64, 1))?;
+    class.define_method("mul_f32", method!(RbSeries::mul_f32, 1))?;
+    class.define_method("mul_f64", method!(RbSeries::mul_f64, 1))?;
+    class.define_method("rem_u8", method!(RbSeries::rem_u8, 1))?;
+    class.define_method("rem_u16", method!(RbSeries::rem_u16, 1))?;
+    class.define_method("rem_u32", method!(RbSeries::rem_u32, 1))?;
+    class.define_method("rem_u64", method!(RbSeries::rem_u64, 1))?;
+    class.define_method("rem_i8", method!(RbSeries::rem_i8, 1))?;
+    class.define_method("rem_i16", method!(RbSeries::rem_i16, 1))?;
+    class.define_method("rem_i32", method!(RbSeries::rem_i32, 1))?;
+    class.define_method("rem_i64", method!(RbSeries::rem_i64, 1))?;
+    class.define_method("rem_f32", method!(RbSeries::rem_f32, 1))?;
+    class.define_method("rem_f64", method!(RbSeries::rem_f64, 1))?;
+
     // eq
     class.define_method("eq_u8", method!(RbSeries::eq_u8, 1))?;
     class.define_method("eq_u16", method!(RbSeries::eq_u16, 1))?;
@@ -664,6 +792,31 @@ fn init() -> RbResult<()> {
     Ok(())
 }
 
+#[allow(clippy::too_many_arguments)]
+fn rb_duration(
+    days: Option<&RbExpr>,
+    seconds: Option<&RbExpr>,
+    nanoseconds: Option<&RbExpr>,
+    microseconds: Option<&RbExpr>,
+    milliseconds: Option<&RbExpr>,
+    minutes: Option<&RbExpr>,
+    hours: Option<&RbExpr>,
+    weeks: Option<&RbExpr>,
+) -> RbExpr {
+    let args = DurationArgs {
+        days: days.map(|e| e.inner.clone()),
+        seconds: seconds.map(|e| e.inner.clone()),
+        nanoseconds: nanoseconds.map(|e| e.inner.clone()),
+        microseconds: microseconds.map(|e| e.inner.clone()),
+        milliseconds: milliseconds.map(|e| e.inner.clone()),
+        minutes: minutes.map(|e| e.inner.clone()),
+        hours: hours.map(|e| e.inner.clone()),
+        weeks: weeks.map(|e| e.inner.clone()),
+    };
+
+    polars::lazy::dsl::duration(args).into()
+}
+
 fn concat_df(seq: RArray) -> RbResult<RbDataFrame> {
     let mut iter = seq.each();
     let first = iter.next().unwrap()?;
@@ -692,6 +845,20 @@ fn concat_df(seq: RArray) -> RbResult<RbDataFrame> {
     Ok(df.into())
 }
 
+fn concat_lf(lfs: Value, rechunk: bool, parallel: bool) -> RbResult<RbLazyFrame> {
+    let (seq, len) = get_rbseq(lfs)?;
+    let mut lfs = Vec::with_capacity(len);
+
+    for res in seq.each() {
+        let item = res?;
+        let lf = get_lf(item)?;
+        lfs.push(lf);
+    }
+
+    let lf = polars::lazy::dsl::concat(lfs, rechunk, parallel).map_err(RbPolarsErr::from)?;
+    Ok(lf.into())
+}
+
 fn rb_diag_concat_df(seq: RArray) -> RbResult<RbDataFrame> {
     let mut dfs = Vec::new();
     for item in seq.each() {
@@ -752,6 +919,27 @@ fn parquet_schema(rb_f: Value) -> RbResult<Value> {
     Ok(dict.into())
 }
 
+fn collect_all(lfs: RArray) -> RbResult<Vec<RbDataFrame>> {
+    use polars_core::utils::rayon::prelude::*;
+
+    let lfs = lfs
+        .each()
+        .map(|v| v?.try_convert::<&RbLazyFrame>())
+        .collect::<RbResult<Vec<&RbLazyFrame>>>()?;
+
+    let out = polars_core::POOL.install(|| {
+        lfs.par_iter()
+            .map(|lf| {
+                let df = lf.ldf.clone().collect()?;
+                Ok(RbDataFrame::new(df))
+            })
+            .collect::<polars_core::error::PolarsResult<Vec<_>>>()
+            .map_err(RbPolarsErr::from)
+    });
+
+    Ok(out?)
+}
+
 fn rb_date_range(
     start: i64,
     stop: i64,
@@ -774,6 +962,16 @@ fn rb_date_range(
     .into()
 }
 
+fn coalesce_exprs(exprs: RArray) -> RbResult<RbExpr> {
+    let exprs = rb_exprs_to_exprs(exprs)?;
+    Ok(polars::lazy::dsl::coalesce(&exprs).into())
+}
+
+fn sum_exprs(exprs: RArray) -> RbResult<RbExpr> {
+    let exprs = rb_exprs_to_exprs(exprs)?;
+    Ok(polars::lazy::dsl::sum_exprs(exprs).into())
+}
+
 fn as_struct(exprs: RArray) -> RbResult<RbExpr> {
     let exprs = rb_exprs_to_exprs(exprs)?;
     Ok(polars::lazy::dsl::as_struct(&exprs).into())

data/ext/polars/src/list_construction.rs

@@ -0,0 +1,100 @@
+use magnus::Value;
+use polars::prelude::*;
+use polars_core::utils::CustomIterTools;
+
+use crate::conversion::get_rbseq;
+use crate::{RbPolarsErr, RbResult};
+
+pub fn rb_seq_to_list(name: &str, seq: Value, dtype: &DataType) -> RbResult<Series> {
+    let (seq, len) = get_rbseq(seq)?;
+
+    let s = match dtype {
+        DataType::Int64 => {
+            let mut builder =
+                ListPrimitiveChunkedBuilder::<Int64Type>::new(name, len, len * 5, DataType::Int64);
+            for sub_seq in seq.each() {
+                let sub_seq = sub_seq?;
+                let (sub_seq, len) = get_rbseq(sub_seq)?;
+
+                // safety: we know the iterators len
+                let iter = unsafe {
+                    sub_seq
+                        .each()
+                        .map(|v| {
+                            let v = v.unwrap();
+                            if v.is_nil() {
+                                None
+                            } else {
+                                Some(v.try_convert::<i64>().unwrap())
+                            }
+                        })
+                        .trust_my_length(len)
+                };
+                builder.append_iter(iter)
+            }
+            builder.finish().into_series()
+        }
+        DataType::Float64 => {
+            let mut builder = ListPrimitiveChunkedBuilder::<Float64Type>::new(
+                name,
+                len,
+                len * 5,
+                DataType::Float64,
+            );
+            for sub_seq in seq.each() {
+                let sub_seq = sub_seq?;
+                let (sub_seq, len) = get_rbseq(sub_seq)?;
+                // safety: we know the iterators len
+                let iter = unsafe {
+                    sub_seq
+                        .each()
+                        .map(|v| {
+                            let v = v.unwrap();
+                            if v.is_nil() {
+                                None
+                            } else {
+                                Some(v.try_convert::<f64>().unwrap())
+                            }
+                        })
+                        .trust_my_length(len)
+                };
+                builder.append_iter(iter)
+            }
+            builder.finish().into_series()
+        }
+        DataType::Boolean => {
+            let mut builder = ListBooleanChunkedBuilder::new(name, len, len * 5);
+            for sub_seq in seq.each() {
+                let sub_seq = sub_seq?;
+                let (sub_seq, len) = get_rbseq(sub_seq)?;
+                // safety: we know the iterators len
+                let iter = unsafe {
+                    sub_seq
+                        .each()
+                        .map(|v| {
+                            let v = v.unwrap();
+                            if v.is_nil() {
+                                None
+                            } else {
+                                Some(v.try_convert::<bool>().unwrap())
+                            }
+                        })
+                        .trust_my_length(len)
+                };
+                builder.append_iter(iter)
+            }
+            builder.finish().into_series()
+        }
+        DataType::Utf8 => {
+            return Err(RbPolarsErr::todo());
+        }
+        dt => {
+            return Err(RbPolarsErr::other(format!(
+                "cannot create list array from {:?}",
+                dt
+            )));
+        }
+    };
+
+    Ok(s)
+}