polars-df 0.17.0-aarch64-linux → 0.18.0-aarch64-linux

data/lib/polars/functions/eager.rb

@@ -6,8 +6,7 @@ module Polars
     #   DataFrames/Series/LazyFrames to concatenate.
     # @param rechunk [Boolean]
     #   Make sure that all data is in contiguous memory.
-    # @param how ["vertical", "vertical_relaxed", "diagonal", "horizontal"]
-    #   LazyFrames do not support the `horizontal` strategy.
+    # @param how ["vertical", "vertical_relaxed", "diagonal", "diagonal_relaxed", "horizontal"]
     #
     #   - Vertical: applies multiple `vstack` operations.
     #   - Diagonal: finds a union between the column schemas and fills missing column values with null.
@@ -21,7 +20,7 @@ module Polars
     # @example
     #   df1 = Polars::DataFrame.new({"a" => [1], "b" => [3]})
     #   df2 = Polars::DataFrame.new({"a" => [2], "b" => [4]})
-    #   Polars.concat([df1, df2])
+    #   Polars.concat([df1, df2])  # default is 'vertical' strategy
     #   # =>
     #   # shape: (2, 2)
     #   # ┌─────┬─────┐
@@ -32,38 +31,168 @@ module Polars
     #   # │ 1   ┆ 3   │
     #   # │ 2   ┆ 4   │
     #   # └─────┴─────┘
+    #
+    # @example
+    #   df1 = Polars::DataFrame.new({"a" => [1], "b" => [3]})
+    #   df2 = Polars::DataFrame.new({"a" => [2.5], "b" => [4]})
+    #   Polars.concat([df1, df2], how: "vertical_relaxed")  # 'a' coerced into f64
+    #   # =>
+    #   # shape: (2, 2)
+    #   # ┌─────┬─────┐
+    #   # │ a   ┆ b   │
+    #   # │ --- ┆ --- │
+    #   # │ f64 ┆ i64 │
+    #   # ╞═════╪═════╡
+    #   # │ 1.0 ┆ 3   │
+    #   # │ 2.5 ┆ 4   │
+    #   # └─────┴─────┘
+    #
+    # @example
+    #   df_h1 = Polars::DataFrame.new({"l1" => [1, 2], "l2" => [3, 4]})
+    #   df_h2 = Polars::DataFrame.new({"r1" => [5, 6], "r2" => [7, 8], "r3" => [9, 10]})
+    #   Polars.concat([df_h1, df_h2], how: "horizontal")
+    #   # =>
+    #   # shape: (2, 5)
+    #   # ┌─────┬─────┬─────┬─────┬─────┐
+    #   # │ l1  ┆ l2  ┆ r1  ┆ r2  ┆ r3  │
+    #   # │ --- ┆ --- ┆ --- ┆ --- ┆ --- │
+    #   # │ i64 ┆ i64 ┆ i64 ┆ i64 ┆ i64 │
+    #   # ╞═════╪═════╪═════╪═════╪═════╡
+    #   # │ 1   ┆ 3   ┆ 5   ┆ 7   ┆ 9   │
+    #   # │ 2   ┆ 4   ┆ 6   ┆ 8   ┆ 10  │
+    #   # └─────┴─────┴─────┴─────┴─────┘
+    #
+    # @example
+    #   df_d1 = Polars::DataFrame.new({"a" => [1], "b" => [3]})
+    #   df_d2 = Polars::DataFrame.new({"a" => [2], "c" => [4]})
+    #   Polars.concat([df_d1, df_d2], how: "diagonal")
+    #   # =>
+    #   # shape: (2, 3)
+    #   # ┌─────┬──────┬──────┐
+    #   # │ a   ┆ b    ┆ c    │
+    #   # │ --- ┆ ---  ┆ ---  │
+    #   # │ i64 ┆ i64  ┆ i64  │
+    #   # ╞═════╪══════╪══════╡
+    #   # │ 1   ┆ 3    ┆ null │
+    #   # │ 2   ┆ null ┆ 4    │
+    #   # └─────┴──────┴──────┘
+    #
+    # @example
+    #   df_a1 = Polars::DataFrame.new({"id" => [1, 2], "x" => [3, 4]})
+    #   df_a2 = Polars::DataFrame.new({"id" => [2, 3], "y" => [5, 6]})
+    #   df_a3 = Polars::DataFrame.new({"id" => [1, 3], "z" => [7, 8]})
+    #   Polars.concat([df_a1, df_a2, df_a3], how: "align")
+    #   # =>
+    #   # shape: (3, 4)
+    #   # ┌─────┬──────┬──────┬──────┐
+    #   # │ id  ┆ x    ┆ y    ┆ z    │
+    #   # │ --- ┆ ---  ┆ ---  ┆ ---  │
+    #   # │ i64 ┆ i64  ┆ i64  ┆ i64  │
+    #   # ╞═════╪══════╪══════╪══════╡
+    #   # │ 1   ┆ 3    ┆ null ┆ 7    │
+    #   # │ 2   ┆ 4    ┆ 5    ┆ null │
+    #   # │ 3   ┆ null ┆ 6    ┆ 8    │
+    #   # └─────┴──────┴──────┴──────┘
     def concat(items, rechunk: true, how: "vertical", parallel: true)
-      if items.empty?
+      elems = items.to_a
+
+      if elems.empty?
         raise ArgumentError, "cannot concat empty list"
       end
 
-      first = items[0]
+      if how == "align"
+        if !elems[0].is_a?(DataFrame) && !elems[0].is_a?(LazyFrame)
+          msg = "'align' strategy is not supported for #{elems[0].class.name}"
+          raise TypeError, msg
+        end
+
+        # establish common columns, maintaining the order in which they appear
+        all_columns = elems.flat_map { |e| e.collect_schema.names }
+        key = all_columns.uniq.map.with_index.to_h
+        common_cols = elems.map { |e| e.collect_schema.names }
+          .reduce { |x, y| Set.new(x) & Set.new(y) }
+          .sort_by { |k| key[k] }
+        # we require at least one key column for 'align'
+        if common_cols.empty?
+          msg = "'align' strategy requires at least one common column"
+          raise InvalidOperationError, msg
+        end
+
+        # align the frame data using a full outer join with no suffix-resolution
+        # (so we raise an error in case of column collision, like "horizontal")
+        lf = elems.map { |df| df.lazy }.reduce do |x, y|
+          x.join(
+            y,
+            how: "full",
+            on: common_cols,
+            suffix: "_PL_CONCAT_RIGHT",
+            maintain_order: "right_left"
+          )
+          # Coalesce full outer join columns
+          .with_columns(
+            common_cols.map { |name| F.coalesce([name, "#{name}_PL_CONCAT_RIGHT"]) }
+          )
+          .drop(common_cols.map { |name| "#{name}_PL_CONCAT_RIGHT" })
+        end.sort(common_cols)
+
+        eager = elems[0].is_a?(DataFrame)
+        return eager ? lf.collect : lf
+      end
+
+      first = elems[0]
+
       if first.is_a?(DataFrame)
         if how == "vertical"
-          out = Utils.wrap_df(Plr.concat_df(items))
+          out = Utils.wrap_df(Plr.concat_df(elems))
+        elsif how == "vertical_relaxed"
+          out = Utils.wrap_ldf(
+            Plr.concat_lf(
+              elems.map { |df| df.lazy },
+              rechunk,
+              parallel,
+              true
+            )
+          ).collect(no_optimization: true)
         elsif how == "diagonal"
-          out = Utils.wrap_df(Plr.concat_df_diagonal(items))
+          out = Utils.wrap_df(Plr.concat_df_diagonal(elems))
+        elsif how == "diagonal_relaxed"
+          out = Utils.wrap_ldf(
+            Plr.concat_lf_diagonal(
+              elems.map { |df| df.lazy },
+              rechunk,
+              parallel,
+              true
+            )
+          ).collect(no_optimization: true)
         elsif how == "horizontal"
-          out = Utils.wrap_df(Plr.concat_df_horizontal(items))
+          out = Utils.wrap_df(Plr.concat_df_horizontal(elems))
         else
-          raise ArgumentError, "how must be one of {{'vertical', 'diagonal', 'horizontal'}}, got #{how}"
+          raise ArgumentError, "how must be one of {{'vertical', 'vertical_relaxed', 'diagonal', 'diagonal_relaxed', 'horizontal'}}, got #{how}"
         end
       elsif first.is_a?(LazyFrame)
         if how == "vertical"
-          return Utils.wrap_ldf(Plr.concat_lf(items, rechunk, parallel, false))
+          return Utils.wrap_ldf(Plr.concat_lf(elems, rechunk, parallel, false))
         elsif how == "vertical_relaxed"
-          return Utils.wrap_ldf(Plr.concat_lf(items, rechunk, parallel, true))
+          return Utils.wrap_ldf(Plr.concat_lf(elems, rechunk, parallel, true))
         elsif how == "diagonal"
-          return Utils.wrap_ldf(Plr.concat_lf_diagonal(items, rechunk, parallel, false))
+          return Utils.wrap_ldf(Plr.concat_lf_diagonal(elems, rechunk, parallel, false))
+        elsif how == "diagonal_relaxed"
+          return Utils.wrap_ldf(Plr.concat_lf_diagonal(elems, rechunk, parallel, true))
+        elsif how == "horizontal"
+          return Utils.wrap_ldf(Plr.concat_lf_horizontal(elems, parallel))
         else
-          raise ArgumentError, "Lazy only allows 'vertical', 'vertical_relaxed', and 'diagonal' concat strategy."
+          raise ArgumentError, "Lazy only allows 'vertical', 'vertical_relaxed', 'diagonal', and 'diagonal_relaxed' concat strategy."
         end
       elsif first.is_a?(Series)
-        # TODO
-        out = Utils.wrap_s(Plr.concat_series(items))
+        if how == "vertical"
+          out = Utils.wrap_s(Plr.concat_series(elems))
+        else
+          msg = "Series only supports 'vertical' concat strategy"
+          raise ArgumentError, msg
+        end
       elsif first.is_a?(Expr)
         out = first
-        items[1..-1].each do |e|
+        elems[1..-1].each do |e|
           out = out.append(e)
         end
       else

data/lib/polars/io/database.rb

@@ -51,8 +51,25 @@ module Polars
           when :decimal
             Decimal
           when :float
+            # TODO uncomment in future release
+            # if column_type.limit && column_type.limit <= 24
+            #   Float32
+            # else
+            #   Float64
+            # end
             Float64
           when :integer
+            # TODO uncomment in future release
+            # case column_type.limit
+            # when 1
+            #   Int8
+            # when 2
+            #   Int16
+            # when 4
+            #   Int32
+            # else
+            #   Int64
+            # end
             Int64
           when :string, :text
             String

data/lib/polars/lazy_frame.rb

@@ -433,7 +433,10 @@ module Polars
       no_optimization: false,
       slice_pushdown: true,
       storage_options: nil,
-      retries: 2
+      retries: 2,
+      sync_on_close: nil,
+      mkdir: false,
+      lazy: false
     )
       lf = _set_sink_optimizations(
         type_coercion: type_coercion,
@@ -468,17 +471,30 @@ module Polars
         storage_options = nil
       end
 
-      lf.sink_parquet(
+      sink_options = {
+        "sync_on_close" => sync_on_close || "none",
+        "maintain_order" => maintain_order,
+        "mkdir" => mkdir
+      }
+
+      lf = lf.sink_parquet(
         path,
         compression,
         compression_level,
         statistics,
         row_group_size,
         data_pagesize_limit,
-        maintain_order,
         storage_options,
-        retries
+        retries,
+        sink_options
       )
+      lf = LazyFrame._from_rbldf(lf)
+
+      if !lazy
+        lf.collect
+        return nil
+      end
+      lf
     end
 
     # Evaluate the query in streaming mode and write to an IPC file.
@@ -520,7 +536,10 @@ module Polars
       projection_pushdown: true,
       simplify_expression: true,
       slice_pushdown: true,
-      no_optimization: false
+      no_optimization: false,
+      sync_on_close: nil,
+      mkdir: false,
+      lazy: false
     )
       # TODO support storage options in Rust
       storage_options = nil
@@ -541,13 +560,26 @@ module Polars
         storage_options = nil
       end
 
-      lf.sink_ipc(
+      sink_options = {
+        "sync_on_close" => sync_on_close || "none",
+        "maintain_order" => maintain_order,
+        "mkdir" => mkdir
+      }
+
+      lf = lf.sink_ipc(
         path,
         compression,
-        maintain_order,
         storage_options,
-        retries
+        retries,
+        sink_options
       )
+      lf = LazyFrame._from_rbldf(lf)
+
+      if !lazy
+        lf.collect
+        return nil
+      end
+      lf
     end
 
     # Evaluate the query in streaming mode and write to a CSV file.
@@ -619,6 +651,10 @@ module Polars
     #   Slice pushdown optimization.
     # @param no_optimization [Boolean]
     #   Turn off (certain) optimizations.
+    # @param storage_options [Object]
+    #   Options that indicate how to connect to a cloud provider.
+    # @param retries [Integer]
+    #   Number of retries if accessing a cloud instance fails.
     #
     # @return [DataFrame]
     #
@@ -646,7 +682,12 @@ module Polars
       projection_pushdown: true,
       simplify_expression: true,
       slice_pushdown: true,
-      no_optimization: false
+      no_optimization: false,
+      storage_options: nil,
+      retries: 2,
+      sync_on_close: nil,
+      mkdir: false,
+      lazy: false
     )
       Utils._check_arg_is_1byte("separator", separator, false)
       Utils._check_arg_is_1byte("quote_char", quote_char, false)
@@ -660,7 +701,19 @@ module Polars
         no_optimization: no_optimization
       )
 
-      lf.sink_csv(
+      if storage_options&.any?
+        storage_options = storage_options.to_a
+      else
+        storage_options = nil
+      end
+
+      sink_options = {
+        "sync_on_close" => sync_on_close || "none",
+        "maintain_order" => maintain_order,
+        "mkdir" => mkdir
+      }
+
+      lf = lf.sink_csv(
         path,
         include_bom,
         include_header,
@@ -675,8 +728,17 @@ module Polars
         float_precision,
         null_value,
         quote_style,
-        maintain_order
+        storage_options,
+        retries,
+        sink_options
       )
+      lf = LazyFrame._from_rbldf(lf)
+
+      if !lazy
+        lf.collect
+        return nil
+      end
+      lf
     end
 
     # Evaluate the query in streaming mode and write to an NDJSON file.
@@ -716,7 +778,10 @@ module Polars
       slice_pushdown: true,
       no_optimization: false,
       storage_options: nil,
-      retries: 2
+      retries: 2,
+      sync_on_close: nil,
+      mkdir: false,
+      lazy: false
     )
       lf = _set_sink_optimizations(
         type_coercion: type_coercion,
@@ -733,7 +798,20 @@ module Polars
         storage_options = nil
       end
 
-      lf.sink_json(path, maintain_order, storage_options, retries)
+      sink_options = {
+        "sync_on_close" => sync_on_close || "none",
+        "maintain_order" => maintain_order,
+        "mkdir" => mkdir
+      }
+
+      lf = lf.sink_json(path, storage_options, retries, sink_options)
+      lf = LazyFrame._from_rbldf(lf)
+
+      if !lazy
+        lf.collect
+        return nil
+      end
+      lf
     end
 
     # @private
@@ -1923,6 +2001,24 @@ module Polars
     #     - true: -> Always coalesce join columns.
     #     - false: -> Never coalesce join columns.
     #   Note that joining on any other expressions than `col` will turn off coalescing.
+    # @param maintain_order ['none', 'left', 'right', 'left_right', 'right_left']
+    #   Which DataFrame row order to preserve, if any.
+    #   Do not rely on any observed ordering without explicitly
+    #   setting this parameter, as your code may break in a future release.
+    #   Not specifying any ordering can improve performance
+    #   Supported for inner, left, right and full joins
+    #
+    #   * *none*
+    #       No specific ordering is desired. The ordering might differ across
+    #       Polars versions or even between different runs.
+    #   * *left*
+    #       Preserves the order of the left DataFrame.
+    #   * *right*
+    #       Preserves the order of the right DataFrame.
+    #   * *left_right*
+    #       First preserves the order of the left DataFrame, then the right.
+    #   * *right_left*
+    #       First preserves the order of the right DataFrame, then the left.
     #
     # @return [LazyFrame]
     #
@@ -2016,18 +2112,33 @@ module Polars
       join_nulls: false,
       allow_parallel: true,
       force_parallel: false,
-      coalesce: nil
+      coalesce: nil,
+      maintain_order: nil
     )
       if !other.is_a?(LazyFrame)
         raise ArgumentError, "Expected a `LazyFrame` as join table, got #{other.class.name}"
       end
 
+      if maintain_order.nil?
+        maintain_order = "none"
+      end
+
       if how == "outer"
         how = "full"
       elsif how == "cross"
         return _from_rbldf(
           _ldf.join(
-            other._ldf, [], [], allow_parallel, join_nulls, force_parallel, how, suffix, validate, coalesce
+            other._ldf,
+            [],
+            [],
+            allow_parallel,
+            join_nulls,
+            force_parallel,
+            how,
+            suffix,
+            validate,
+            maintain_order,
+            coalesce
           )
         )
       end
@@ -2054,6 +2165,7 @@ module Polars
           how,
           suffix,
           validate,
+          maintain_order,
           coalesce
         )
       )
@@ -3347,12 +3459,17 @@ module Polars
       _from_rbldf(_ldf.merge_sorted(other._ldf, key))
     end
 
-    # Indicate that one or multiple columns are sorted.
+    # Flag a column as sorted.
+    #
+    # This can speed up future operations.
+    #
+    # @note
+    #   This can lead to incorrect results if the data is NOT sorted! Use with care!
     #
     # @param column [Object]
-    #   Columns that are sorted
+    #   Column that is sorted.
     # @param descending [Boolean]
-    #   Whether the columns are sorted in descending order.
+    #   Whether the column is sorted in descending order.
     #
     # @return [LazyFrame]
     def set_sorted(

data/lib/polars/list_expr.rb

@@ -403,7 +403,7 @@ module Polars
     # The indices may be defined in a single column, or by sublists in another
     # column of dtype `List`.
     #
-    # @param index [Object]
+    # @param indices [Object]
     #   Indices to return per sublist
     # @param null_on_oob [Boolean]
     #   Behavior if an index is out of bounds:
@@ -427,12 +427,9 @@ module Polars
     #   # │ []          ┆ [null, null] │
     #   # │ [1, 2, … 5] ┆ [1, 5]       │
     #   # └─────────────┴──────────────┘
-    def gather(index, null_on_oob: false)
-      if index.is_a?(::Array)
-        index = Series.new(index)
-      end
-      index = Utils.parse_into_expression(index, str_as_lit: false)
-      Utils.wrap_expr(_rbexpr.list_gather(index, null_on_oob))
+    def gather(indices, null_on_oob: false)
+      indices = Utils.parse_into_expression(indices)
+      Utils.wrap_expr(_rbexpr.list_gather(indices, null_on_oob))
     end
     alias_method :take, :gather
 

data/lib/polars/schema.rb

@@ -0,0 +1,29 @@
+module Polars
+  class Schema
+    def initialize(schema, check_dtypes: true)
+      raise Todo if check_dtypes
+      @schema = schema.to_h
+    end
+
+    def [](key)
+      @schema[key]
+    end
+
+    def names
+      @schema.keys
+    end
+
+    def dtypes
+      @schema.values
+    end
+
+    def length
+      @schema.length
+    end
+
+    def to_s
+      "#{self.class.name}(#{@schema})"
+    end
+    alias_method :inspect, :to_s
+  end
+end

data/lib/polars/series.rb

@@ -2554,29 +2554,7 @@ module Polars
     #   # Numo::Int64#shape=[3]
     #   # [1, 2, 3]
     def to_numo
-      if !has_validity
-        if is_datelike
-          Numo::RObject.cast(to_a)
-        elsif is_numeric
-          # TODO make more efficient
-          {
-            UInt8 => Numo::UInt8,
-            UInt16 => Numo::UInt16,
-            UInt32 => Numo::UInt32,
-            UInt64 => Numo::UInt64,
-            Int8 => Numo::Int8,
-            Int16 => Numo::Int16,
-            Int32 => Numo::Int32,
-            Int64 => Numo::Int64,
-            Float32 => Numo::SFloat,
-            Float64 => Numo::DFloat
-          }.fetch(dtype.class).cast(to_a)
-        elsif is_boolean
-          Numo::Bit.cast(to_a)
-        else
-          _s.to_numo
-        end
-      elsif is_datelike
+      if is_datelike
         Numo::RObject.cast(to_a)
       else
         _s.to_numo
@@ -3599,24 +3577,26 @@ module Polars
     #   Integer size of the rolling window.
     # @param bias [Boolean]
     #   If false, the calculations are corrected for statistical bias.
+    # @param min_samples [Integer]
+    #   The number of values in the window that should be non-null before computing
+    #   a result. If set to `nil` (default), it will be set equal to `window_size`.
+    # @param center [Boolean]
+    #   Set the labels at the center of the window.
     #
     # @return [Series]
     #
     # @example
-    #   s = Polars::Series.new("a", [1.0, 2.0, 3.0, 4.0, 6.0, 8.0])
-    #   s.rolling_skew(3)
+    #   Polars::Series.new([1, 4, 2, 9]).rolling_skew(3)
     #   # =>
-    #   # shape: (6,)
-    #   # Series: 'a' [f64]
+    #   # shape: (4,)
+    #   # Series: '' [f64]
     #   # [
     #   #         null
     #   #         null
-    #   #         0.0
-    #   #         0.0
     #   #         0.381802
-    #   #         0.0
+    #   #         0.47033
     #   # ]
-    def rolling_skew(window_size, bias: true)
+    def rolling_skew(window_size, bias: true, min_samples: nil, center: false)
       super
     end
 
@@ -3815,6 +3795,30 @@ module Polars
       super
     end
 
+    # Fill null values using interpolation based on another column.
+    #
+    # @param by [Expr]
+    #   Column to interpolate values based on.
+    #
+    # @return [Series]
+    #
+    # @example Fill null values using linear interpolation.
+    #   s = Polars::Series.new("a", [1, nil, nil, 3])
+    #   by = Polars::Series.new("b", [1, 2, 7, 8])
+    #   s.interpolate_by(by)
+    #   # =>
+    #   # shape: (4,)
+    #   # Series: 'a' [f64]
+    #   # [
+    #   #         1.0
+    #   #         1.285714
+    #   #         2.714286
+    #   #         3.0
+    #   # ]
+    def interpolate_by(by)
+      super
+    end
+
     # Compute absolute values.
     #
     # @return [Series]
@@ -4041,7 +4045,7 @@ module Polars
     #
     # @example
     #   s.kurtosis(fisher: false, bias: false)
-    #   # => 2.1040361802642726
+    #   # => 2.1040361802642717
     def kurtosis(fisher: true, bias: true)
       _s.kurtosis(fisher, bias)
     end

data/lib/polars/version.rb

@@ -1,4 +1,4 @@
 module Polars
   # @private
-  VERSION = "0.17.0"
+  VERSION = "0.18.0"
 end

data/lib/polars.rb

@@ -61,6 +61,7 @@ require_relative "polars/list_name_space"
 require_relative "polars/meta_expr"
 require_relative "polars/name_expr"
 require_relative "polars/rolling_group_by"
+require_relative "polars/schema"
 require_relative "polars/selectors"
 require_relative "polars/series"
 require_relative "polars/slice"