polars-df 0.16.0-arm64-darwin → 0.17.1-arm64-darwin

data/lib/polars/functions/eager.rb

@@ -6,8 +6,7 @@ module Polars
     #   DataFrames/Series/LazyFrames to concatenate.
     # @param rechunk [Boolean]
     #   Make sure that all data is in contiguous memory.
-    # @param how ["vertical", "vertical_relaxed", "diagonal", "horizontal"]
-    #   LazyFrames do not support the `horizontal` strategy.
+    # @param how ["vertical", "vertical_relaxed", "diagonal", "diagonal_relaxed", "horizontal"]
     #
     #   - Vertical: applies multiple `vstack` operations.
     #   - Diagonal: finds a union between the column schemas and fills missing column values with null.
@@ -21,7 +20,7 @@ module Polars
     # @example
     #   df1 = Polars::DataFrame.new({"a" => [1], "b" => [3]})
     #   df2 = Polars::DataFrame.new({"a" => [2], "b" => [4]})
-    #   Polars.concat([df1, df2])
+    #   Polars.concat([df1, df2])  # default is 'vertical' strategy
     #   # =>
     #   # shape: (2, 2)
     #   # ┌─────┬─────┐
@@ -32,38 +31,168 @@ module Polars
     #   # │ 1   ┆ 3   │
     #   # │ 2   ┆ 4   │
     #   # └─────┴─────┘
+    #
+    # @example
+    #   df1 = Polars::DataFrame.new({"a" => [1], "b" => [3]})
+    #   df2 = Polars::DataFrame.new({"a" => [2.5], "b" => [4]})
+    #   Polars.concat([df1, df2], how: "vertical_relaxed")  # 'a' coerced into f64
+    #   # =>
+    #   # shape: (2, 2)
+    #   # ┌─────┬─────┐
+    #   # │ a   ┆ b   │
+    #   # │ --- ┆ --- │
+    #   # │ f64 ┆ i64 │
+    #   # ╞═════╪═════╡
+    #   # │ 1.0 ┆ 3   │
+    #   # │ 2.5 ┆ 4   │
+    #   # └─────┴─────┘
+    #
+    # @example
+    #   df_h1 = Polars::DataFrame.new({"l1" => [1, 2], "l2" => [3, 4]})
+    #   df_h2 = Polars::DataFrame.new({"r1" => [5, 6], "r2" => [7, 8], "r3" => [9, 10]})
+    #   Polars.concat([df_h1, df_h2], how: "horizontal")
+    #   # =>
+    #   # shape: (2, 5)
+    #   # ┌─────┬─────┬─────┬─────┬─────┐
+    #   # │ l1  ┆ l2  ┆ r1  ┆ r2  ┆ r3  │
+    #   # │ --- ┆ --- ┆ --- ┆ --- ┆ --- │
+    #   # │ i64 ┆ i64 ┆ i64 ┆ i64 ┆ i64 │
+    #   # ╞═════╪═════╪═════╪═════╪═════╡
+    #   # │ 1   ┆ 3   ┆ 5   ┆ 7   ┆ 9   │
+    #   # │ 2   ┆ 4   ┆ 6   ┆ 8   ┆ 10  │
+    #   # └─────┴─────┴─────┴─────┴─────┘
+    #
+    # @example
+    #   df_d1 = Polars::DataFrame.new({"a" => [1], "b" => [3]})
+    #   df_d2 = Polars::DataFrame.new({"a" => [2], "c" => [4]})
+    #   Polars.concat([df_d1, df_d2], how: "diagonal")
+    #   # =>
+    #   # shape: (2, 3)
+    #   # ┌─────┬──────┬──────┐
+    #   # │ a   ┆ b    ┆ c    │
+    #   # │ --- ┆ ---  ┆ ---  │
+    #   # │ i64 ┆ i64  ┆ i64  │
+    #   # ╞═════╪══════╪══════╡
+    #   # │ 1   ┆ 3    ┆ null │
+    #   # │ 2   ┆ null ┆ 4    │
+    #   # └─────┴──────┴──────┘
+    #
+    # @example
+    #   df_a1 = Polars::DataFrame.new({"id" => [1, 2], "x" => [3, 4]})
+    #   df_a2 = Polars::DataFrame.new({"id" => [2, 3], "y" => [5, 6]})
+    #   df_a3 = Polars::DataFrame.new({"id" => [1, 3], "z" => [7, 8]})
+    #   Polars.concat([df_a1, df_a2, df_a3], how: "align")
+    #   # =>
+    #   # shape: (3, 4)
+    #   # ┌─────┬──────┬──────┬──────┐
+    #   # │ id  ┆ x    ┆ y    ┆ z    │
+    #   # │ --- ┆ ---  ┆ ---  ┆ ---  │
+    #   # │ i64 ┆ i64  ┆ i64  ┆ i64  │
+    #   # ╞═════╪══════╪══════╪══════╡
+    #   # │ 1   ┆ 3    ┆ null ┆ 7    │
+    #   # │ 2   ┆ 4    ┆ 5    ┆ null │
+    #   # │ 3   ┆ null ┆ 6    ┆ 8    │
+    #   # └─────┴──────┴──────┴──────┘
     def concat(items, rechunk: true, how: "vertical", parallel: true)
-      if items.empty?
+      elems = items.to_a
+
+      if elems.empty?
         raise ArgumentError, "cannot concat empty list"
       end
 
-      first = items[0]
+      if how == "align"
+        if !elems[0].is_a?(DataFrame) && !elems[0].is_a?(LazyFrame)
+          msg = "'align' strategy is not supported for #{elems[0].class.name}"
+          raise TypeError, msg
+        end
+
+        # establish common columns, maintaining the order in which they appear
+        all_columns = elems.flat_map { |e| e.collect_schema.names }
+        key = all_columns.uniq.map.with_index.to_h
+        common_cols = elems.map { |e| e.collect_schema.names }
+          .reduce { |x, y| Set.new(x) & Set.new(y) }
+          .sort_by { |k| key[k] }
+        # we require at least one key column for 'align'
+        if common_cols.empty?
+          msg = "'align' strategy requires at least one common column"
+          raise InvalidOperationError, msg
+        end
+
+        # align the frame data using a full outer join with no suffix-resolution
+        # (so we raise an error in case of column collision, like "horizontal")
+        lf = elems.map { |df| df.lazy }.reduce do |x, y|
+          x.join(
+            y,
+            how: "full",
+            on: common_cols,
+            suffix: "_PL_CONCAT_RIGHT",
+            maintain_order: "right_left"
+          )
+          # Coalesce full outer join columns
+          .with_columns(
+            common_cols.map { |name| F.coalesce([name, "#{name}_PL_CONCAT_RIGHT"]) }
+          )
+          .drop(common_cols.map { |name| "#{name}_PL_CONCAT_RIGHT" })
+        end.sort(common_cols)
+
+        eager = elems[0].is_a?(DataFrame)
+        return eager ? lf.collect : lf
+      end
+
+      first = elems[0]
+
       if first.is_a?(DataFrame)
         if how == "vertical"
-          out = Utils.wrap_df(Plr.concat_df(items))
+          out = Utils.wrap_df(Plr.concat_df(elems))
+        elsif how == "vertical_relaxed"
+          out = Utils.wrap_ldf(
+            Plr.concat_lf(
+              elems.map { |df| df.lazy },
+              rechunk,
+              parallel,
+              true
+            )
+          ).collect(no_optimization: true)
         elsif how == "diagonal"
-          out = Utils.wrap_df(Plr.concat_df_diagonal(items))
+          out = Utils.wrap_df(Plr.concat_df_diagonal(elems))
+        elsif how == "diagonal_relaxed"
+          out = Utils.wrap_ldf(
+            Plr.concat_lf_diagonal(
+              elems.map { |df| df.lazy },
+              rechunk,
+              parallel,
+              true
+            )
+          ).collect(no_optimization: true)
         elsif how == "horizontal"
-          out = Utils.wrap_df(Plr.concat_df_horizontal(items))
+          out = Utils.wrap_df(Plr.concat_df_horizontal(elems))
         else
-          raise ArgumentError, "how must be one of {{'vertical', 'diagonal', 'horizontal'}}, got #{how}"
+          raise ArgumentError, "how must be one of {{'vertical', 'vertical_relaxed', 'diagonal', 'diagonal_relaxed', 'horizontal'}}, got #{how}"
         end
       elsif first.is_a?(LazyFrame)
         if how == "vertical"
-          return Utils.wrap_ldf(Plr.concat_lf(items, rechunk, parallel, false))
+          return Utils.wrap_ldf(Plr.concat_lf(elems, rechunk, parallel, false))
         elsif how == "vertical_relaxed"
-          return Utils.wrap_ldf(Plr.concat_lf(items, rechunk, parallel, true))
+          return Utils.wrap_ldf(Plr.concat_lf(elems, rechunk, parallel, true))
         elsif how == "diagonal"
-          return Utils.wrap_ldf(Plr.concat_lf_diagonal(items, rechunk, parallel, false))
+          return Utils.wrap_ldf(Plr.concat_lf_diagonal(elems, rechunk, parallel, false))
+        elsif how == "diagonal_relaxed"
+          return Utils.wrap_ldf(Plr.concat_lf_diagonal(elems, rechunk, parallel, true))
+        elsif how == "horizontal"
+          return Utils.wrap_ldf(Plr.concat_lf_horizontal(elems, parallel))
         else
-          raise ArgumentError, "Lazy only allows 'vertical', 'vertical_relaxed', and 'diagonal' concat strategy."
+          raise ArgumentError, "Lazy only allows 'vertical', 'vertical_relaxed', 'diagonal', and 'diagonal_relaxed' concat strategy."
         end
       elsif first.is_a?(Series)
-        # TODO
-        out = Utils.wrap_s(Plr.concat_series(items))
+        if how == "vertical"
+          out = Utils.wrap_s(Plr.concat_series(elems))
+        else
+          msg = "Series only supports 'vertical' concat strategy"
+          raise ArgumentError, msg
+        end
       elsif first.is_a?(Expr)
         out = first
-        items[1..-1].each do |e|
+        elems[1..-1].each do |e|
           out = out.append(e)
         end
       else

data/lib/polars/io/database.rb

@@ -51,8 +51,25 @@ module Polars
           when :decimal
             Decimal
           when :float
+            # TODO uncomment in 0.18.0
+            # if column_type.limit && column_type.limit <= 24
+            #   Float32
+            # else
+            #   Float64
+            # end
             Float64
           when :integer
+            # TODO uncomment in 0.18.0
+            # case column_type.limit
+            # when 1
+            #   Int8
+            # when 2
+            #   Int16
+            # when 4
+            #   Int32
+            # else
+            #   Int64
+            # end
             Int64
           when :string, :text
             String

data/lib/polars/lazy_frame.rb

@@ -619,6 +619,10 @@ module Polars
     #   Slice pushdown optimization.
     # @param no_optimization [Boolean]
     #   Turn off (certain) optimizations.
+    # @param storage_options [Object]
+    #   Options that indicate how to connect to a cloud provider.
+    # @param retries [Integer]
+    #   Number of retries if accessing a cloud instance fails.
     #
     # @return [DataFrame]
     #
@@ -646,7 +650,9 @@ module Polars
       projection_pushdown: true,
       simplify_expression: true,
       slice_pushdown: true,
-      no_optimization: false
+      no_optimization: false,
+      storage_options: nil,
+      retries: 2
     )
       Utils._check_arg_is_1byte("separator", separator, false)
       Utils._check_arg_is_1byte("quote_char", quote_char, false)
@@ -660,6 +666,12 @@ module Polars
         no_optimization: no_optimization
       )
 
+      if storage_options&.any?
+        storage_options = storage_options.to_a
+      else
+        storage_options = nil
+      end
+
       lf.sink_csv(
         path,
         include_bom,
@@ -675,7 +687,9 @@ module Polars
         float_precision,
         null_value,
         quote_style,
-        maintain_order
+        maintain_order,
+        storage_options,
+        retries
       )
     end
 
@@ -1616,6 +1630,14 @@ module Polars
     #     - true: -> Always coalesce join columns.
     #     - false: -> Never coalesce join columns.
     #   Note that joining on any other expressions than `col` will turn off coalescing.
+    # @param allow_exact_matches [Boolean]
+    #   Whether exact matches are valid join predicates.
+    #     - If true, allow matching with the same `on` value (i.e. less-than-or-equal-to / greater-than-or-equal-to).
+    #     - If false, don't match the same `on` value (i.e., strictly less-than / strictly greater-than).
+    # @param check_sortedness [Boolean]
+    #   Check the sortedness of the asof keys. If the keys are not sorted Polars
+    #   will error, or in case of 'by' argument raise a warning. This might become
+    #   a hard error in the future.
     #
     # @return [LazyFrame]
     #
@@ -1815,7 +1837,9 @@ module Polars
       tolerance: nil,
       allow_parallel: true,
       force_parallel: false,
-      coalesce: true
+      coalesce: true,
+      allow_exact_matches: true,
+      check_sortedness: true
     )
       if !other.is_a?(LazyFrame)
         raise ArgumentError, "Expected a `LazyFrame` as join table, got #{other.class.name}"
@@ -1871,7 +1895,9 @@ module Polars
           strategy,
           tolerance_num,
           tolerance_str,
-          coalesce
+          coalesce,
+          allow_exact_matches,
+          check_sortedness
         )
       )
     end
@@ -1911,6 +1937,24 @@ module Polars
     #     - true: -> Always coalesce join columns.
     #     - false: -> Never coalesce join columns.
     #   Note that joining on any other expressions than `col` will turn off coalescing.
+    # @param maintain_order ['none', 'left', 'right', 'left_right', 'right_left']
+    #   Which DataFrame row order to preserve, if any.
+    #   Do not rely on any observed ordering without explicitly
+    #   setting this parameter, as your code may break in a future release.
+    #   Not specifying any ordering can improve performance
+    #   Supported for inner, left, right and full joins
+    #
+    #   * *none*
+    #       No specific ordering is desired. The ordering might differ across
+    #       Polars versions or even between different runs.
+    #   * *left*
+    #       Preserves the order of the left DataFrame.
+    #   * *right*
+    #       Preserves the order of the right DataFrame.
+    #   * *left_right*
+    #       First preserves the order of the left DataFrame, then the right.
+    #   * *right_left*
+    #       First preserves the order of the right DataFrame, then the left.
     #
     # @return [LazyFrame]
     #
@@ -2004,18 +2048,33 @@ module Polars
       join_nulls: false,
       allow_parallel: true,
       force_parallel: false,
-      coalesce: nil
+      coalesce: nil,
+      maintain_order: nil
     )
       if !other.is_a?(LazyFrame)
         raise ArgumentError, "Expected a `LazyFrame` as join table, got #{other.class.name}"
       end
 
+      if maintain_order.nil?
+        maintain_order = "none"
+      end
+
       if how == "outer"
         how = "full"
       elsif how == "cross"
         return _from_rbldf(
           _ldf.join(
-            other._ldf, [], [], allow_parallel, join_nulls, force_parallel, how, suffix, validate, coalesce
+            other._ldf,
+            [],
+            [],
+            allow_parallel,
+            join_nulls,
+            force_parallel,
+            how,
+            suffix,
+            validate,
+            maintain_order,
+            coalesce
           )
         )
       end
@@ -2042,6 +2101,7 @@ module Polars
           how,
           suffix,
           validate,
+          maintain_order,
           coalesce
         )
       )
@@ -3335,12 +3395,17 @@ module Polars
       _from_rbldf(_ldf.merge_sorted(other._ldf, key))
     end
 
-    # Indicate that one or multiple columns are sorted.
+    # Flag a column as sorted.
+    #
+    # This can speed up future operations.
+    #
+    # @note
+    #   This can lead to incorrect results if the data is NOT sorted! Use with care!
     #
     # @param column [Object]
-    #   Columns that are sorted
+    #   Column that is sorted.
     # @param descending [Boolean]
-    #   Whether the columns are sorted in descending order.
+    #   Whether the column is sorted in descending order.
     #
     # @return [LazyFrame]
     def set_sorted(

data/lib/polars/schema.rb

@@ -0,0 +1,29 @@
+module Polars
+  class Schema
+    def initialize(schema, check_dtypes: true)
+      raise Todo if check_dtypes
+      @schema = schema.to_h
+    end
+
+    def [](key)
+      @schema[key]
+    end
+
+    def names
+      @schema.keys
+    end
+
+    def dtypes
+      @schema.values
+    end
+
+    def length
+      @schema.length
+    end
+
+    def to_s
+      "#{self.class.name}(#{@schema})"
+    end
+    alias_method :inspect, :to_s
+  end
+end

data/lib/polars/series.rb

@@ -2554,29 +2554,7 @@ module Polars
     #   # Numo::Int64#shape=[3]
     #   # [1, 2, 3]
     def to_numo
-      if !has_validity
-        if is_datelike
-          Numo::RObject.cast(to_a)
-        elsif is_numeric
-          # TODO make more efficient
-          {
-            UInt8 => Numo::UInt8,
-            UInt16 => Numo::UInt16,
-            UInt32 => Numo::UInt32,
-            UInt64 => Numo::UInt64,
-            Int8 => Numo::Int8,
-            Int16 => Numo::Int16,
-            Int32 => Numo::Int32,
-            Int64 => Numo::Int64,
-            Float32 => Numo::SFloat,
-            Float64 => Numo::DFloat
-          }.fetch(dtype.class).cast(to_a)
-        elsif is_boolean
-          Numo::Bit.cast(to_a)
-        else
-          _s.to_numo
-        end
-      elsif is_datelike
+      if is_datelike
         Numo::RObject.cast(to_a)
       else
         _s.to_numo
@@ -3815,6 +3793,30 @@ module Polars
       super
     end
 
+    # Fill null values using interpolation based on another column.
+    #
+    # @param by [Expr]
+    #   Column to interpolate values based on.
+    #
+    # @return [Series]
+    #
+    # @example Fill null values using linear interpolation.
+    #   s = Polars::Series.new("a", [1, nil, nil, 3])
+    #   by = Polars::Series.new("b", [1, 2, 7, 8])
+    #   s.interpolate_by(by)
+    #   # =>
+    #   # shape: (4,)
+    #   # Series: 'a' [f64]
+    #   # [
+    #   #         1.0
+    #   #         1.285714
+    #   #         2.714286
+    #   #         3.0
+    #   # ]
+    def interpolate_by(by)
+      super
+    end
+
     # Compute absolute values.
     #
     # @return [Series]
@@ -4696,7 +4698,12 @@ module Polars
         end
 
         constructor = polars_type_to_constructor(dtype)
-        rbseries = constructor.call(name, values, strict)
+        rbseries =
+          if dtype == Array
+            constructor.call(name, values, strict)
+          else
+            construct_series_with_fallbacks(constructor, name, values, dtype, strict: strict)
+          end
 
         base_type = dtype.is_a?(DataType) ? dtype.class : dtype
         if [Date, Datetime, Duration, Time, Categorical, Boolean, Enum, Decimal].include?(base_type)

data/lib/polars/version.rb

@@ -1,4 +1,4 @@
 module Polars
   # @private
-  VERSION = "0.16.0"
+  VERSION = "0.17.1"
 end

data/lib/polars.rb

@@ -61,6 +61,7 @@ require_relative "polars/list_name_space"
 require_relative "polars/meta_expr"
 require_relative "polars/name_expr"
 require_relative "polars/rolling_group_by"
+require_relative "polars/schema"
 require_relative "polars/selectors"
 require_relative "polars/series"
 require_relative "polars/slice"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: polars-df
 version: !ruby/object:Gem::Version
-  version: 0.16.0
+  version: 0.17.1
 platform: arm64-darwin
 authors:
 - Andrew Kane
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2024-12-29 00:00:00.000000000 Z
+date: 2025-04-13 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bigdecimal
@@ -92,6 +92,7 @@ files:
 - lib/polars/name_expr.rb
 - lib/polars/plot.rb
 - lib/polars/rolling_group_by.rb
+- lib/polars/schema.rb
 - lib/polars/selectors.rb
 - lib/polars/series.rb
 - lib/polars/slice.rb