polars-df 0.15.0 → 0.17.0

data/lib/polars/data_types.rb

@@ -167,6 +167,10 @@ module Polars
   class Int64 < SignedIntegerType
   end
 
+  # 128-bit signed integer type.
+  class Int128 < SignedIntegerType
+  end
+
   # 8-bit unsigned integer type.
   class UInt8 < UnsignedIntegerType
   end
@@ -311,7 +315,7 @@ module Polars
       end
 
       if categories.empty?
-        self.categories = Series.new("category", [], dtype: String)
+        @categories = Series.new("category", [], dtype: String)
         return
       end
 

data/lib/polars/functions/aggregation/horizontal.rb

@@ -143,6 +143,9 @@ module Polars
     # @param exprs [Array]
     #   Column(s) to use in the aggregation. Accepts expression input. Strings are
     #   parsed as column names, other non-expression inputs are parsed as literals.
+    # @param ignore_nulls [Boolean]
+    #   Ignore null values (default).
+    #   If set to `false`, any null value in the input will lead to a null output.
     #
     # @return [Expr]
     #
@@ -166,9 +169,9 @@ module Polars
     #   # │ 8   ┆ 5    ┆ y   ┆ 13  │
     #   # │ 3   ┆ null ┆ z   ┆ 3   │
     #   # └─────┴──────┴─────┴─────┘
-    def sum_horizontal(*exprs)
+    def sum_horizontal(*exprs, ignore_nulls: true)
       rbexprs = Utils.parse_into_list_of_expressions(*exprs)
-      Utils.wrap_expr(Plr.sum_horizontal(rbexprs))
+      Utils.wrap_expr(Plr.sum_horizontal(rbexprs, ignore_nulls))
     end
 
     # Compute the mean of all values horizontally across columns.
@@ -176,6 +179,9 @@ module Polars
     # @param exprs [Array]
     #   Column(s) to use in the aggregation. Accepts expression input. Strings are
     #   parsed as column names, other non-expression inputs are parsed as literals.
+    # @param ignore_nulls [Boolean]
+    #   Ignore null values (default).
+    #   If set to `false`, any null value in the input will lead to a null output.
     #
     # @return [Expr]
     #
@@ -199,9 +205,9 @@ module Polars
     #   # │ 8   ┆ 5    ┆ y   ┆ 6.5  │
     #   # │ 3   ┆ null ┆ z   ┆ 3.0  │
     #   # └─────┴──────┴─────┴──────┘
-    def mean_horizontal(*exprs)
+    def mean_horizontal(*exprs, ignore_nulls: true)
       rbexprs = Utils.parse_into_list_of_expressions(*exprs)
-      Utils.wrap_expr(Plr.mean_horizontal(rbexprs))
+      Utils.wrap_expr(Plr.mean_horizontal(rbexprs, ignore_nulls))
     end
 
     # Cumulatively sum all values horizontally across columns.

data/lib/polars/functions/lazy.rb

@@ -729,16 +729,20 @@ module Polars
       a,
       b,
       method: "pearson",
-      ddof: 1,
+      ddof: nil,
       propagate_nans: false
     )
+      if !ddof.nil?
+        warn "The `ddof` parameter has no effect. Do not use it."
+      end
+
       a = Utils.parse_into_expression(a)
       b = Utils.parse_into_expression(b)
 
       if method == "pearson"
-        Utils.wrap_expr(Plr.pearson_corr(a, b, ddof))
+        Utils.wrap_expr(Plr.pearson_corr(a, b))
       elsif method == "spearman"
-        Utils.wrap_expr(Plr.spearman_rank_corr(a, b, ddof, propagate_nans))
+        Utils.wrap_expr(Plr.spearman_rank_corr(a, b, propagate_nans))
       else
         msg = "method must be one of {{'pearson', 'spearman'}}, got #{method}"
         raise ArgumentError, msg

data/lib/polars/io/delta.rb

@@ -0,0 +1,126 @@
+module Polars
+  module IO
+    # Reads into a DataFrame from a Delta lake table.
+    #
+    # @param source [Object]
+    #   DeltaTable or a Path or URI to the root of the Delta lake table.
+    # @param version [Object]
+    #   Numerical version or timestamp version of the Delta lake table.
+    # @param columns [Array]
+    #   Columns to select. Accepts a list of column names.
+    # @param rechunk [Boolean]
+    #   Make sure that all columns are contiguous in memory by
+    #   aggregating the chunks into a single array.
+    # @param storage_options [Hash]
+    #   Extra options for the storage backends supported by `deltalake-rb`.
+    # @param delta_table_options [Hash]
+    #   Additional keyword arguments while reading a Delta lake Table.
+    #
+    # @return [DataFrame]
+    def read_delta(
+      source,
+      version: nil,
+      columns: nil,
+      rechunk: false,
+      storage_options: nil,
+      delta_table_options: nil
+    )
+      dl_tbl =
+        _get_delta_lake_table(
+          source,
+          version: version,
+          storage_options: storage_options,
+          delta_table_options: delta_table_options
+        )
+
+      dl_tbl.to_polars(columns: columns, rechunk: rechunk)
+    end
+
+    # Lazily read from a Delta lake table.
+    #
+    # @param source [Object]
+    #   DeltaTable or a Path or URI to the root of the Delta lake table.
+    # @param version [Object]
+    #   Numerical version or timestamp version of the Delta lake table.
+    # @param storage_options [Hash]
+    #   Extra options for the storage backends supported by `deltalake-rb`.
+    # @param delta_table_options [Hash]
+    #   Additional keyword arguments while reading a Delta lake Table.
+    #
+    # @return [LazyFrame]
+    def scan_delta(
+      source,
+      version: nil,
+      storage_options: nil,
+      delta_table_options: nil
+    )
+      dl_tbl =
+        _get_delta_lake_table(
+          source,
+          version: version,
+          storage_options: storage_options,
+          delta_table_options: delta_table_options
+        )
+
+      dl_tbl.to_polars(eager: false)
+    end
+
+    private
+
+    def _resolve_delta_lake_uri(table_uri, strict: true)
+      require "uri"
+
+      parsed_result = URI(table_uri)
+
+      resolved_uri =
+        if parsed_result.scheme == ""
+          Utils.normalize_filepath(table_uri)
+        else
+          table_uri
+        end
+
+      resolved_uri
+    end
+
+    def _get_delta_lake_table(
+      table_path,
+      version: nil,
+      storage_options: nil,
+      delta_table_options: nil
+    )
+      _check_if_delta_available
+
+      if table_path.is_a?(DeltaLake::Table)
+        return table_path
+      end
+      delta_table_options ||= {}
+      resolved_uri = _resolve_delta_lake_uri(table_path)
+      if !version.is_a?(::String) && !version.is_a?(::Time)
+        dl_tbl =
+          DeltaLake::Table.new(
+            resolved_uri,
+            version: version,
+            storage_options: storage_options,
+            **delta_table_options
+          )
+      else
+        dl_tbl =
+          DeltaLake::Table.new(
+            resolved_uri,
+            storage_options: storage_options,
+            **delta_table_options
+          )
+        dl_tbl.load_as_version(version)
+      end
+
+      dl_tbl = DeltaLake::Table.new(table_path)
+      dl_tbl
+    end
+
+    def _check_if_delta_available
+      if !defined?(DeltaLake)
+        raise Error, "Delta Lake not available"
+      end
+    end
+  end
+end

data/lib/polars/lazy_frame.rb

@@ -431,7 +431,9 @@ module Polars
       projection_pushdown: true,
       simplify_expression: true,
       no_optimization: false,
-      slice_pushdown: true
+      slice_pushdown: true,
+      storage_options: nil,
+      retries: 2
     )
       lf = _set_sink_optimizations(
         type_coercion: type_coercion,
@@ -460,6 +462,12 @@ module Polars
         }
       end
 
+      if storage_options&.any?
+        storage_options = storage_options.to_a
+      else
+        storage_options = nil
+      end
+
       lf.sink_parquet(
         path,
         compression,
@@ -467,7 +475,9 @@ module Polars
         statistics,
         row_group_size,
         data_pagesize_limit,
-        maintain_order
+        maintain_order,
+        storage_options,
+        retries
       )
     end
 
@@ -512,6 +522,10 @@ module Polars
       slice_pushdown: true,
       no_optimization: false
     )
+      # TODO support storage options in Rust
+      storage_options = nil
+      retries = 2
+
       lf = _set_sink_optimizations(
         type_coercion: type_coercion,
         predicate_pushdown: predicate_pushdown,
@@ -521,10 +535,18 @@ module Polars
         no_optimization: no_optimization
       )
 
+      if storage_options&.any?
+        storage_options = storage_options.to_a
+      else
+        storage_options = nil
+      end
+
       lf.sink_ipc(
         path,
         compression,
-        maintain_order
+        maintain_order,
+        storage_options,
+        retries
       )
     end
 
@@ -692,7 +714,9 @@ module Polars
       projection_pushdown: true,
       simplify_expression: true,
       slice_pushdown: true,
-      no_optimization: false
+      no_optimization: false,
+      storage_options: nil,
+      retries: 2
     )
       lf = _set_sink_optimizations(
         type_coercion: type_coercion,
@@ -703,7 +727,13 @@ module Polars
         no_optimization: no_optimization
       )
 
-      lf.sink_json(path, maintain_order)
+      if storage_options&.any?
+        storage_options = storage_options.to_a
+      else
+        storage_options = nil
+      end
+
+      lf.sink_json(path, maintain_order, storage_options, retries)
     end
 
     # @private
@@ -1586,6 +1616,14 @@ module Polars
     #     - true: -> Always coalesce join columns.
     #     - false: -> Never coalesce join columns.
     #   Note that joining on any other expressions than `col` will turn off coalescing.
+    # @param allow_exact_matches [Boolean]
+    #   Whether exact matches are valid join predicates.
+    #     - If true, allow matching with the same `on` value (i.e. less-than-or-equal-to / greater-than-or-equal-to).
+    #     - If false, don't match the same `on` value (i.e., strictly less-than / strictly greater-than).
+    # @param check_sortedness [Boolean]
+    #   Check the sortedness of the asof keys. If the keys are not sorted Polars
+    #   will error, or in case of 'by' argument raise a warning. This might become
+    #   a hard error in the future.
     #
     # @return [LazyFrame]
     #
@@ -1785,7 +1823,9 @@ module Polars
       tolerance: nil,
       allow_parallel: true,
       force_parallel: false,
-      coalesce: true
+      coalesce: true,
+      allow_exact_matches: true,
+      check_sortedness: true
     )
       if !other.is_a?(LazyFrame)
         raise ArgumentError, "Expected a `LazyFrame` as join table, got #{other.class.name}"
@@ -1841,7 +1881,9 @@ module Polars
           strategy,
           tolerance_num,
           tolerance_str,
-          coalesce
+          coalesce,
+          allow_exact_matches,
+          check_sortedness
         )
       )
     end

data/lib/polars/selectors.rb

@@ -372,9 +372,91 @@ module Polars
     # def by_index
     # end
 
-    # TODO
-    # def by_name
-    # end
+    # Select all columns matching the given names.
+    #
+    # @param names [Array]
+    #   One or more names of columns to select.
+    # @param require_all [Boolean]
+    #   Whether to match *all* names (the default) or *any* of the names.
+    #
+    # @return [SelectorProxy]
+    #
+    # @note
+    #   Matching columns are returned in the order in which they are declared in
+    #   the selector, not the underlying schema order.
+    #
+    # @example
+    #   df = Polars::DataFrame.new(
+    #     {
+    #       "foo" => ["x", "y"],
+    #       "bar" => [123, 456],
+    #       "baz" => [2.0, 5.5],
+    #       "zap" => [false, true]
+    #     }
+    #   )
+    #
+    # @example Select columns by name:
+    #   df.select(Polars.cs.by_name("foo", "bar"))
+    #   # =>
+    #   # shape: (2, 2)
+    #   # ┌─────┬─────┐
+    #   # │ foo ┆ bar │
+    #   # │ --- ┆ --- │
+    #   # │ str ┆ i64 │
+    #   # ╞═════╪═════╡
+    #   # │ x   ┆ 123 │
+    #   # │ y   ┆ 456 │
+    #   # └─────┴─────┘
+    #
+    # @example Match *any* of the given columns by name:
+    #   df.select(Polars.cs.by_name("baz", "moose", "foo", "bear", require_all: false))
+    #   # =>
+    #   # shape: (2, 2)
+    #   # ┌─────┬─────┐
+    #   # │ foo ┆ baz │
+    #   # │ --- ┆ --- │
+    #   # │ str ┆ f64 │
+    #   # ╞═════╪═════╡
+    #   # │ x   ┆ 2.0 │
+    #   # │ y   ┆ 5.5 │
+    #   # └─────┴─────┘
+    #
+    # @example Match all columns *except* for those given:
+    #   df.select(~Polars.cs.by_name("foo", "bar"))
+    #   # =>
+    #   # shape: (2, 2)
+    #   # ┌─────┬───────┐
+    #   # │ baz ┆ zap   │
+    #   # │ --- ┆ ---   │
+    #   # │ f64 ┆ bool  │
+    #   # ╞═════╪═══════╡
+    #   # │ 2.0 ┆ false │
+    #   # │ 5.5 ┆ true  │
+    #   # └─────┴───────┘
+    def self.by_name(*names, require_all: true)
+      all_names = []
+      names.each do |nm|
+        if nm.is_a?(::String)
+          all_names << nm
+        else
+          msg = "invalid name: #{nm.inspect}"
+          raise TypeError, msg
+        end
+      end
+
+      selector_params = {"*names" => all_names}
+      match_cols = all_names
+      if !require_all
+        match_cols = "^(#{all_names.map { |nm| Utils.re_escape(nm) }.join("|")})$"
+        selector_params["require_all"] = require_all
+      end
+
+      _selector_proxy_(
+        F.col(match_cols),
+        name: "by_name",
+        parameters: selector_params
+      )
+    end
 
     # Select all categorical columns.
     #

data/lib/polars/series.rb

@@ -4696,7 +4696,12 @@ module Polars
         end
 
         constructor = polars_type_to_constructor(dtype)
-        rbseries = constructor.call(name, values, strict)
+        rbseries =
+          if dtype == Array
+            constructor.call(name, values, strict)
+          else
+            construct_series_with_fallbacks(constructor, name, values, dtype, strict: strict)
+          end
 
         base_type = dtype.is_a?(DataType) ? dtype.class : dtype
         if [Date, Datetime, Duration, Time, Categorical, Boolean, Enum, Decimal].include?(base_type)

data/lib/polars/version.rb

@@ -1,4 +1,4 @@
 module Polars
   # @private
-  VERSION = "0.15.0"
+  VERSION = "0.17.0"
 end

data/lib/polars.rb

@@ -49,6 +49,7 @@ require_relative "polars/group_by"
 require_relative "polars/io/avro"
 require_relative "polars/io/csv"
 require_relative "polars/io/database"
+require_relative "polars/io/delta"
 require_relative "polars/io/ipc"
 require_relative "polars/io/json"
 require_relative "polars/io/ndjson"
@@ -89,4 +90,18 @@ module Polars
 
   # @private
   N_INFER_DEFAULT = 100
+
+  # @private
+  class ArrowArrayStream
+    def arrow_c_stream
+      self
+    end
+  end
+
+  # Return the number of threads in the Polars thread pool.
+  #
+  # @return [Integer]
+  def self.thread_pool_size
+    Plr.thread_pool_size
+  end
 end

metadata

@@ -1,14 +1,13 @@
 --- !ruby/object:Gem::Specification
 name: polars-df
 version: !ruby/object:Gem::Version
-  version: 0.15.0
+  version: 0.17.0
 platform: ruby
 authors:
 - Andrew Kane
-autorequire:
 bindir: bin
 cert_chain: []
-date: 2024-11-20 00:00:00.000000000 Z
+date: 2025-01-28 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bigdecimal
@@ -38,7 +37,6 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-description:
 email: andrew@ankane.org
 executables: []
 extensions:
@@ -160,6 +158,7 @@ files:
 - lib/polars/io/avro.rb
 - lib/polars/io/csv.rb
 - lib/polars/io/database.rb
+- lib/polars/io/delta.rb
 - lib/polars/io/ipc.rb
 - lib/polars/io/json.rb
 - lib/polars/io/ndjson.rb
@@ -194,7 +193,6 @@ homepage: https://github.com/ankane/ruby-polars
 licenses:
 - MIT
 metadata: {}
-post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -202,15 +200,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '3.1'
+      version: '3.2'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.5.22
-signing_key:
+rubygems_version: 3.6.2
 specification_version: 4
 summary: Blazingly fast DataFrames for Ruby
 test_files: []