polars-df 0.15.0 → 0.16.0

data/ext/polars/src/lazyframe/general.rs

@@ -330,6 +330,7 @@ impl RbLazyFrame {
                 nulls_last: vec![nulls_last],
                 multithreaded,
                 maintain_order,
+                limit: None,
             },
         )
         .into()
@@ -353,6 +354,7 @@ impl RbLazyFrame {
                     nulls_last,
                     maintain_order,
                     multithreaded,
+                    limit: None,
                 },
             )
             .into())
@@ -379,6 +381,8 @@ impl RbLazyFrame {
         row_group_size: Option<usize>,
         data_page_size: Option<usize>,
         maintain_order: bool,
+        cloud_options: Option<Vec<(String, String)>>,
+        retries: usize,
     ) -> RbResult<()> {
         let compression = parse_parquet_compression(&compression, compression_level)?;
 
@@ -390,8 +394,15 @@ impl RbLazyFrame {
             maintain_order,
         };
 
+        let cloud_options = {
+            let cloud_options =
+                parse_cloud_options(path.to_str().unwrap(), cloud_options.unwrap_or_default())?;
+            Some(cloud_options.with_max_retries(retries))
+        };
+
         let ldf = self.ldf.borrow().clone();
-        ldf.sink_parquet(path, options).map_err(RbPolarsErr::from)?;
+        ldf.sink_parquet(&path, options, cloud_options)
+            .map_err(RbPolarsErr::from)?;
         Ok(())
     }
 
@@ -400,14 +411,23 @@ impl RbLazyFrame {
         path: PathBuf,
         compression: Option<Wrap<IpcCompression>>,
         maintain_order: bool,
+        cloud_options: Option<Vec<(String, String)>>,
+        retries: usize,
     ) -> RbResult<()> {
         let options = IpcWriterOptions {
             compression: compression.map(|c| c.0),
             maintain_order,
         };
 
+        let cloud_options = {
+            let cloud_options =
+                parse_cloud_options(path.to_str().unwrap(), cloud_options.unwrap_or_default())?;
+            Some(cloud_options.with_max_retries(retries))
+        };
+
         let ldf = self.ldf.borrow().clone();
-        ldf.sink_ipc(path, options).map_err(RbPolarsErr::from)?;
+        ldf.sink_ipc(&path, options, cloud_options)
+            .map_err(RbPolarsErr::from)?;
         Ok(())
     }
 
@@ -430,6 +450,9 @@ impl RbLazyFrame {
         quote_style: Option<Wrap<QuoteStyle>>,
         maintain_order: bool,
     ) -> RbResult<()> {
+        // TODO
+        let cloud_options = None;
+
         let quote_style = quote_style.map_or(QuoteStyle::default(), |wrap| wrap.0);
         let null_value = null_value.unwrap_or(SerializeOptions::default().null);
 
@@ -454,16 +477,36 @@ impl RbLazyFrame {
             serialize_options,
         };
 
+        let cloud_options = {
+            let cloud_options =
+                parse_cloud_options(path.to_str().unwrap(), cloud_options.unwrap_or_default())?;
+            Some(cloud_options)
+        };
+
         let ldf = self.ldf.borrow().clone();
-        ldf.sink_csv(path, options).map_err(RbPolarsErr::from)?;
+        ldf.sink_csv(&path, options, cloud_options)
+            .map_err(RbPolarsErr::from)?;
         Ok(())
     }
 
-    pub fn sink_json(&self, path: PathBuf, maintain_order: bool) -> RbResult<()> {
+    pub fn sink_json(
+        &self,
+        path: PathBuf,
+        maintain_order: bool,
+        cloud_options: Option<Vec<(String, String)>>,
+        retries: usize,
+    ) -> RbResult<()> {
         let options = JsonWriterOptions { maintain_order };
 
+        let cloud_options = {
+            let cloud_options =
+                parse_cloud_options(path.to_str().unwrap(), cloud_options.unwrap_or_default())?;
+            Some(cloud_options.with_max_retries(retries))
+        };
+
         let ldf = self.ldf.borrow().clone();
-        ldf.sink_json(path, options).map_err(RbPolarsErr::from)?;
+        ldf.sink_json(&path, options, cloud_options)
+            .map_err(RbPolarsErr::from)?;
         Ok(())
     }
 

data/ext/polars/src/lib.rs

@@ -72,7 +72,7 @@ fn init(ruby: &Ruby) -> RbResult<()> {
     class.define_method("write_json", method!(RbDataFrame::write_json, 3))?;
     class.define_method("write_ndjson", method!(RbDataFrame::write_ndjson, 1))?;
     class.define_method("write_csv", method!(RbDataFrame::write_csv, 10))?;
-    class.define_method("write_ipc", method!(RbDataFrame::write_ipc, 3))?;
+    class.define_method("write_ipc", method!(RbDataFrame::write_ipc, 5))?;
     class.define_method(
         "write_ipc_stream",
         method!(RbDataFrame::write_ipc_stream, 3),
@@ -143,10 +143,6 @@ fn init(ruby: &Ruby) -> RbResult<()> {
     class.define_method("pivot_expr", method!(RbDataFrame::pivot_expr, 7))?;
     class.define_method("partition_by", method!(RbDataFrame::partition_by, 3))?;
     class.define_method("lazy", method!(RbDataFrame::lazy, 0))?;
-    class.define_method("mean_horizontal", method!(RbDataFrame::mean_horizontal, 1))?;
-    class.define_method("max_horizontal", method!(RbDataFrame::max_horizontal, 0))?;
-    class.define_method("min_horizontal", method!(RbDataFrame::min_horizontal, 0))?;
-    class.define_method("sum_horizontal", method!(RbDataFrame::sum_horizontal, 1))?;
     class.define_method("to_dummies", method!(RbDataFrame::to_dummies, 3))?;
     class.define_method("null_count", method!(RbDataFrame::null_count, 0))?;
     class.define_method("map_rows", method!(RbDataFrame::map_rows, 3))?;
@@ -568,10 +564,10 @@ fn init(ruby: &Ruby) -> RbResult<()> {
     class.define_singleton_method("int_range", function!(functions::range::int_range, 4))?;
     class.define_singleton_method("int_ranges", function!(functions::range::int_ranges, 4))?;
     class.define_singleton_method("repeat", function!(functions::lazy::repeat, 3))?;
-    class.define_singleton_method("pearson_corr", function!(functions::lazy::pearson_corr, 3))?;
+    class.define_singleton_method("pearson_corr", function!(functions::lazy::pearson_corr, 2))?;
     class.define_singleton_method(
         "spearman_rank_corr",
-        function!(functions::lazy::spearman_rank_corr, 4),
+        function!(functions::lazy::spearman_rank_corr, 3),
     )?;
     class.define_singleton_method("sql_expr", function!(functions::lazy::sql_expr, 1))?;
     class.define_singleton_method("cov", function!(functions::lazy::cov, 3))?;
@@ -605,11 +601,11 @@ fn init(ruby: &Ruby) -> RbResult<()> {
     )?;
     class.define_singleton_method(
         "sum_horizontal",
-        function!(functions::aggregation::sum_horizontal, 1),
+        function!(functions::aggregation::sum_horizontal, 2),
     )?;
     class.define_singleton_method(
         "mean_horizontal",
-        function!(functions::aggregation::mean_horizontal, 1),
+        function!(functions::aggregation::mean_horizontal, 2),
     )?;
     class.define_singleton_method("as_struct", function!(functions::lazy::as_struct, 1))?;
     class.define_singleton_method("coalesce", function!(functions::lazy::coalesce, 1))?;
@@ -660,8 +656,8 @@ fn init(ruby: &Ruby) -> RbResult<()> {
         function!(functions::meta::get_index_type, 0),
     )?;
     class.define_singleton_method(
-        "threadpool_size",
-        function!(functions::meta::threadpool_size, 0),
+        "thread_pool_size",
+        function!(functions::meta::thread_pool_size, 0),
     )?;
     class.define_singleton_method(
         "enable_string_cache",
@@ -747,10 +743,10 @@ fn init(ruby: &Ruby) -> RbResult<()> {
     class.define_method("sort_by_exprs", method!(RbLazyFrame::sort_by_exprs, 5))?;
     class.define_method("cache", method!(RbLazyFrame::cache, 0))?;
     class.define_method("collect", method!(RbLazyFrame::collect, 0))?;
-    class.define_method("sink_parquet", method!(RbLazyFrame::sink_parquet, 7))?;
-    class.define_method("sink_ipc", method!(RbLazyFrame::sink_ipc, 3))?;
+    class.define_method("sink_parquet", method!(RbLazyFrame::sink_parquet, 9))?;
+    class.define_method("sink_ipc", method!(RbLazyFrame::sink_ipc, 5))?;
     class.define_method("sink_csv", method!(RbLazyFrame::sink_csv, 15))?;
-    class.define_method("sink_json", method!(RbLazyFrame::sink_json, 2))?;
+    class.define_method("sink_json", method!(RbLazyFrame::sink_json, 4))?;
     class.define_method("fetch", method!(RbLazyFrame::fetch, 1))?;
     class.define_method("filter", method!(RbLazyFrame::filter, 1))?;
     class.define_method("select", method!(RbLazyFrame::select, 1))?;
@@ -1109,7 +1105,7 @@ fn init(ruby: &Ruby) -> RbResult<()> {
     class.define_singleton_method("hold", function!(RbStringCacheHolder::hold, 0))?;
 
     // arrow array stream
-    let class = module.define_class("RbArrowArrayStream", ruby.class_object())?;
+    let class = module.define_class("ArrowArrayStream", ruby.class_object())?;
     class.define_method("to_i", method!(RbArrowArrayStream::to_i, 0))?;
 
     Ok(())

data/ext/polars/src/series/general.rs

@@ -97,29 +97,17 @@ impl RbSeries {
     }
 
     pub fn bitand(&self, other: &RbSeries) -> RbResult<Self> {
-        let out = self
-            .series
-            .borrow()
-            .bitand(&other.series.borrow())
-            .map_err(RbPolarsErr::from)?;
+        let out = (&*self.series.borrow() & &*other.series.borrow()).map_err(RbPolarsErr::from)?;
         Ok(out.into())
     }
 
     pub fn bitor(&self, other: &RbSeries) -> RbResult<Self> {
-        let out = self
-            .series
-            .borrow()
-            .bitor(&other.series.borrow())
-            .map_err(RbPolarsErr::from)?;
+        let out = (&*self.series.borrow() | &*other.series.borrow()).map_err(RbPolarsErr::from)?;
         Ok(out.into())
     }
 
     pub fn bitxor(&self, other: &RbSeries) -> RbResult<Self> {
-        let out = self
-            .series
-            .borrow()
-            .bitxor(&other.series.borrow())
-            .map_err(RbPolarsErr::from)?;
+        let out = (&*self.series.borrow() ^ &*other.series.borrow()).map_err(RbPolarsErr::from)?;
         Ok(out.into())
     }
 

data/ext/polars/src/series/import.rs

@@ -39,7 +39,7 @@ pub(crate) fn import_stream_rbcapsule(capsule: Value) -> RbResult<RbSeries> {
 
     // Series::try_from fails for an empty vec of chunks
     let s = if produced_arrays.is_empty() {
-        let polars_dt = DataType::from_arrow(stream.field().dtype(), false);
+        let polars_dt = DataType::from_arrow_field(stream.field());
         Series::new_empty(stream.field().name.clone(), &polars_dt)
     } else {
         Series::try_from((stream.field(), produced_arrays)).unwrap()

data/lib/polars/data_frame.rb

@@ -831,7 +831,13 @@ module Polars
     #   Compression method. Defaults to "uncompressed".
     #
     # @return [nil]
-    def write_ipc(file, compression: "uncompressed", compat_level: nil)
+    def write_ipc(
+      file,
+      compression: "uncompressed",
+      compat_level: nil,
+      storage_options: nil,
+      retries: 2
+    )
       return_bytes = file.nil?
       if return_bytes
         file = StringIO.new
@@ -849,7 +855,13 @@ module Polars
         compression = "uncompressed"
       end
 
-      _df.write_ipc(file, compression, compat_level)
+      if storage_options&.any?
+        storage_options = storage_options.to_a
+      else
+        storage_options = nil
+      end
+
+      _df.write_ipc(file, compression, compat_level, storage_options, retries)
       return_bytes ? file.string : nil
     end
 
@@ -961,6 +973,61 @@ module Polars
       )
     end
 
+    # Write DataFrame as delta table.
+    #
+    # @param target [Object]
+    #   URI of a table or a DeltaTable object.
+    # @param mode ["error", "append", "overwrite", "ignore", "merge"]
+    #   How to handle existing data.
+    # @param storage_options [Hash]
+    #   Extra options for the storage backends supported by `deltalake-rb`.
+    # @param delta_write_options [Hash]
+    #   Additional keyword arguments while writing a Delta lake Table.
+    # @param delta_merge_options [Hash]
+    #   Keyword arguments which are required to `MERGE` a Delta lake Table.
+    #
+    # @return [nil]
+    def write_delta(
+      target,
+      mode: "error",
+      storage_options: nil,
+      delta_write_options: nil,
+      delta_merge_options: nil
+    )
+      Polars.send(:_check_if_delta_available)
+
+      if Utils.pathlike?(target)
+        target = Polars.send(:_resolve_delta_lake_uri, target.to_s, strict: false)
+      end
+
+      data = self
+
+      if mode == "merge"
+        if delta_merge_options.nil?
+          msg = "You need to pass delta_merge_options with at least a given predicate for `MERGE` to work."
+          raise ArgumentError, msg
+        end
+        if target.is_a?(::String)
+          dt = DeltaLake::Table.new(target, storage_options: storage_options)
+        else
+          dt = target
+        end
+
+        predicate = delta_merge_options.delete(:predicate)
+        dt.merge(data, predicate, **delta_merge_options)
+      else
+        delta_write_options ||= {}
+
+        DeltaLake.write(
+          target,
+          data,
+          mode: mode,
+          storage_options: storage_options,
+          **delta_write_options
+        )
+      end
+    end
+
     # Return an estimation of the total (heap) allocated size of the DataFrame.
     #
     # Estimated size is given in the specified unit (bytes by default).
@@ -3939,14 +4006,32 @@ module Polars
     #   # ╞═════╪═════╪═════╡
     #   # │ 3   ┆ 8   ┆ c   │
     #   # └─────┴─────┴─────┘
-    def max(axis: 0)
-      if axis == 0
-        lazy.max.collect(_eager: true)
-      elsif axis == 1
-        Utils.wrap_s(_df.max_horizontal)
-      else
-        raise ArgumentError, "Axis should be 0 or 1."
-      end
+    def max
+      lazy.max.collect(_eager: true)
+    end
+
+    # Get the maximum value horizontally across columns.
+    #
+    # @return [Series]
+    #
+    # @example
+    #   df = Polars::DataFrame.new(
+    #     {
+    #       "foo" => [1, 2, 3],
+    #       "bar" => [4.0, 5.0, 6.0]
+    #     }
+    #   )
+    #   df.max_horizontal
+    #   # =>
+    #   # shape: (3,)
+    #   # Series: 'max' [f64]
+    #   # [
+    #   #         4.0
+    #   #         5.0
+    #   #         6.0
+    #   # ]
+    def max_horizontal
+      select(max: F.max_horizontal(F.all)).to_series
     end
 
     # Aggregate the columns of this DataFrame to their minimum value.
@@ -3971,22 +4056,35 @@ module Polars
     #   # ╞═════╪═════╪═════╡
     #   # │ 1   ┆ 6   ┆ a   │
     #   # └─────┴─────┴─────┘
-    def min(axis: 0)
-      if axis == 0
-        lazy.min.collect(_eager: true)
-      elsif axis == 1
-        Utils.wrap_s(_df.min_horizontal)
-      else
-        raise ArgumentError, "Axis should be 0 or 1."
-      end
+    def min
+      lazy.min.collect(_eager: true)
     end
 
-    # Aggregate the columns of this DataFrame to their sum value.
+    # Get the minimum value horizontally across columns.
     #
-    # @param axis [Integer]
-    #   Either 0 or 1.
-    # @param null_strategy ["ignore", "propagate"]
-    #   This argument is only used if axis == 1.
+    # @return [Series]
+    #
+    # @example
+    #   df = Polars::DataFrame.new(
+    #     {
+    #       "foo" => [1, 2, 3],
+    #       "bar" => [4.0, 5.0, 6.0]
+    #     }
+    #   )
+    #   df.min_horizontal
+    #   # =>
+    #   # shape: (3,)
+    #   # Series: 'min' [f64]
+    #   # [
+    #   #         1.0
+    #   #         2.0
+    #   #         3.0
+    #   # ]
+    def min_horizontal
+      select(min: F.min_horizontal(F.all)).to_series
+    end
+
+    # Aggregate the columns of this DataFrame to their sum value.
     #
     # @return [DataFrame]
     #
@@ -4008,35 +4106,42 @@ module Polars
     #   # ╞═════╪═════╪══════╡
     #   # │ 6   ┆ 21  ┆ null │
     #   # └─────┴─────┴──────┘
+    def sum
+      lazy.sum.collect(_eager: true)
+    end
+
+    # Sum all values horizontally across columns.
+    #
+    # @param ignore_nulls [Boolean]
+    #   Ignore null values (default).
+    #   If set to `false`, any null value in the input will lead to a null output.
+    #
+    # @return [Series]
     #
     # @example
-    #   df.sum(axis: 1)
+    #   df = Polars::DataFrame.new(
+    #     {
+    #       "foo" => [1, 2, 3],
+    #       "bar" => [4.0, 5.0, 6.0]
+    #     }
+    #   )
+    #   df.sum_horizontal
     #   # =>
     #   # shape: (3,)
-    #   # Series: 'foo' [str]
+    #   # Series: 'sum' [f64]
     #   # [
-    #   #         "16a"
-    #   #         "27b"
-    #   #         "38c"
+    #   #         5.0
+    #   #         7.0
+    #   #         9.0
     #   # ]
-    def sum(axis: 0, null_strategy: "ignore")
-      case axis
-      when 0
-        lazy.sum.collect(_eager: true)
-      when 1
-        Utils.wrap_s(_df.sum_horizontal(null_strategy))
-      else
-        raise ArgumentError, "Axis should be 0 or 1."
-      end
+    def sum_horizontal(ignore_nulls: true)
+      select(
+        sum: F.sum_horizontal(F.all, ignore_nulls: ignore_nulls)
+      ).to_series
     end
 
     # Aggregate the columns of this DataFrame to their mean value.
     #
-    # @param axis [Integer]
-    #   Either 0 or 1.
-    # @param null_strategy ["ignore", "propagate"]
-    #   This argument is only used if axis == 1.
-    #
     # @return [DataFrame]
     #
     # @example
@@ -4057,15 +4162,38 @@ module Polars
     #   # ╞═════╪═════╪══════╡
     #   # │ 2.0 ┆ 7.0 ┆ null │
     #   # └─────┴─────┴──────┘
-    def mean(axis: 0, null_strategy: "ignore")
-      case axis
-      when 0
-        lazy.mean.collect(_eager: true)
-      when 1
-        Utils.wrap_s(_df.mean_horizontal(null_strategy))
-      else
-        raise ArgumentError, "Axis should be 0 or 1."
-      end
+    def mean
+      lazy.mean.collect(_eager: true)
+    end
+
+    # Take the mean of all values horizontally across columns.
+    #
+    # @param ignore_nulls [Boolean]
+    #   Ignore null values (default).
+    #   If set to `false`, any null value in the input will lead to a null output.
+    #
+    # @return [Series]
+    #
+    # @example
+    #   df = Polars::DataFrame.new(
+    #     {
+    #       "foo" => [1, 2, 3],
+    #       "bar" => [4.0, 5.0, 6.0]
+    #     }
+    #   )
+    #   df.mean_horizontal
+    #   # =>
+    #   # shape: (3,)
+    #   # Series: 'mean' [f64]
+    #   # [
+    #   #         2.5
+    #   #         3.5
+    #   #         4.5
+    #   # ]
+    def mean_horizontal(ignore_nulls: true)
+      select(
+        mean: F.mean_horizontal(F.all, ignore_nulls: ignore_nulls)
+      ).to_series
     end
 
     # Aggregate the columns of this DataFrame to their standard deviation value.

data/lib/polars/data_types.rb

@@ -311,7 +311,7 @@ module Polars
       end
 
       if categories.empty?
-        self.categories = Series.new("category", [], dtype: String)
+        @categories = Series.new("category", [], dtype: String)
         return
       end
 

data/lib/polars/functions/aggregation/horizontal.rb

@@ -143,6 +143,9 @@ module Polars
     # @param exprs [Array]
     #   Column(s) to use in the aggregation. Accepts expression input. Strings are
     #   parsed as column names, other non-expression inputs are parsed as literals.
+    # @param ignore_nulls [Boolean]
+    #   Ignore null values (default).
+    #   If set to `false`, any null value in the input will lead to a null output.
     #
     # @return [Expr]
     #
@@ -166,9 +169,9 @@ module Polars
     #   # │ 8   ┆ 5    ┆ y   ┆ 13  │
     #   # │ 3   ┆ null ┆ z   ┆ 3   │
     #   # └─────┴──────┴─────┴─────┘
-    def sum_horizontal(*exprs)
+    def sum_horizontal(*exprs, ignore_nulls: true)
       rbexprs = Utils.parse_into_list_of_expressions(*exprs)
-      Utils.wrap_expr(Plr.sum_horizontal(rbexprs))
+      Utils.wrap_expr(Plr.sum_horizontal(rbexprs, ignore_nulls))
     end
 
     # Compute the mean of all values horizontally across columns.
@@ -176,6 +179,9 @@ module Polars
     # @param exprs [Array]
     #   Column(s) to use in the aggregation. Accepts expression input. Strings are
     #   parsed as column names, other non-expression inputs are parsed as literals.
+    # @param ignore_nulls [Boolean]
+    #   Ignore null values (default).
+    #   If set to `false`, any null value in the input will lead to a null output.
     #
     # @return [Expr]
     #
@@ -199,9 +205,9 @@ module Polars
     #   # │ 8   ┆ 5    ┆ y   ┆ 6.5  │
     #   # │ 3   ┆ null ┆ z   ┆ 3.0  │
     #   # └─────┴──────┴─────┴──────┘
-    def mean_horizontal(*exprs)
+    def mean_horizontal(*exprs, ignore_nulls: true)
       rbexprs = Utils.parse_into_list_of_expressions(*exprs)
-      Utils.wrap_expr(Plr.mean_horizontal(rbexprs))
+      Utils.wrap_expr(Plr.mean_horizontal(rbexprs, ignore_nulls))
     end
 
     # Cumulatively sum all values horizontally across columns.

data/lib/polars/functions/lazy.rb

@@ -729,16 +729,20 @@ module Polars
       a,
       b,
       method: "pearson",
-      ddof: 1,
+      ddof: nil,
       propagate_nans: false
     )
+      if !ddof.nil?
+        warn "The `ddof` parameter has no effect. Do not use it."
+      end
+
       a = Utils.parse_into_expression(a)
       b = Utils.parse_into_expression(b)
 
       if method == "pearson"
-        Utils.wrap_expr(Plr.pearson_corr(a, b, ddof))
+        Utils.wrap_expr(Plr.pearson_corr(a, b))
       elsif method == "spearman"
-        Utils.wrap_expr(Plr.spearman_rank_corr(a, b, ddof, propagate_nans))
+        Utils.wrap_expr(Plr.spearman_rank_corr(a, b, propagate_nans))
       else
         msg = "method must be one of {{'pearson', 'spearman'}}, got #{method}"
         raise ArgumentError, msg