polars-df 0.25.1 → 0.26.1

data/lib/polars/list_expr.rb

@@ -11,6 +11,13 @@ module Polars
 
     # Evaluate whether all boolean values in a list are true.
     #
+    # @param ignore_nulls [Boolean]
+    #   * If set to `true` (default), null values are ignored. If there
+    #     are no non-null values, the output is `true`.
+    #   * If set to `false`, [Kleene logic](https://en.wikipedia.org/wiki/Three-valued_logic) is used to deal with nulls:
+    #     if the column contains any null values and no `false` values,
+    #     the output is null.
+    #
     # @return [Expr]
     #
     # @example
@@ -32,12 +39,19 @@ module Polars
     #   # │ []             ┆ true  │
     #   # │ null           ┆ null  │
     #   # └────────────────┴───────┘
-    def all
-      Utils.wrap_expr(_rbexpr.list_all)
+    def all(ignore_nulls: true)
+      agg(F.element.all(ignore_nulls: ignore_nulls))
     end
 
     # Evaluate whether any boolean value in a list is true.
     #
+    # @param ignore_nulls [Boolean]
+    #   * If set to `true` (default), null values are ignored. If there
+    #     are no non-null values, the output is `false`.
+    #   * If set to `false`, [Kleene logic](https://en.wikipedia.org/wiki/Three-valued_logic) is used to deal with nulls:
+    #     if the column contains any null values and no `true` values,
+    #     the output is null.
+    #
     # @return [Expr]
     #
     # @example
@@ -59,8 +73,8 @@ module Polars
     #   # │ []             ┆ false │
     #   # │ null           ┆ null  │
     #   # └────────────────┴───────┘
-    def any
-      Utils.wrap_expr(_rbexpr.list_any)
+    def any(ignore_nulls: true)
+      agg(F.element.any(ignore_nulls: ignore_nulls))
     end
 
     # Get the length of the arrays as `:u32`.
@@ -367,7 +381,7 @@ module Polars
     #   # │ [2, 1, 9] │
     #   # └───────────┘
     def reverse
-      Utils.wrap_expr(_rbexpr.list_reverse)
+      eval(F.element.reverse)
     end
 
     # Get the unique/distinct values in the list.
@@ -391,7 +405,7 @@ module Polars
     #   # │ [1, 2]    │
     #   # └───────────┘
     def unique(maintain_order: false)
-      Utils.wrap_expr(_rbexpr.list_unique(maintain_order))
+      eval(F.element.unique(maintain_order: maintain_order))
     end
 
     # Count the number of unique values in every sub-lists.
@@ -416,7 +430,7 @@ module Polars
     #   # │ [2, 3, 4] ┆ 3        │
     #   # └───────────┴──────────┘
     def n_unique
-      Utils.wrap_expr(_rbexpr.list_n_unique)
+      agg(F.element.n_unique)
     end
 
     # Concat the arrays in a Series dtype List in linear time.

data/lib/polars/list_name_space.rb

@@ -12,6 +12,13 @@ module Polars
 
     # Evaluate whether all boolean values in a list are true.
     #
+    # @param ignore_nulls [Boolean]
+    #   * If set to `true` (default), null values are ignored. If there
+    #     are no non-null values, the output is `true`.
+    #   * If set to `false`, [Kleene logic](https://en.wikipedia.org/wiki/Three-valued_logic) is used to deal with nulls:
+    #     if the column contains any null values and no `false` values,
+    #     the output is null.
+    #
     # @return [Series]
     #
     # @example
@@ -31,12 +38,19 @@ module Polars
     #   #         true
     #   #         null
     #   # ]
-    def all
+    def all(ignore_nulls: true)
       super
     end
 
     # Evaluate whether any boolean value in a list is true.
     #
+    # @param ignore_nulls [Boolean]
+    #   * If set to `true` (default), null values are ignored. If there
+    #     are no non-null values, the output is `false`.
+    #   * If set to `false`, [Kleene logic](https://en.wikipedia.org/wiki/Three-valued_logic) is used to deal with nulls:
+    #     if the column contains any null values and no `true` values,
+    #     the output is null.
+    #
     # @return [Series]
     #
     # @example
@@ -56,7 +70,7 @@ module Polars
     #   #         false
     #   #         null
     #   # ]
-    def any
+    def any(ignore_nulls: true)
       super
     end
 

data/lib/polars/query_opt_flags.rb

@@ -18,7 +18,8 @@ module Polars
       cluster_with_columns: nil,
       collapse_joins: nil,
       check_order_observe: nil,
-      fast_projection: nil
+      fast_projection: nil,
+      sort_collapse: nil
     )
       self._rboptflags = RbOptFlags.default
       update(
@@ -31,7 +32,8 @@ module Polars
         cluster_with_columns: cluster_with_columns,
         collapse_joins: collapse_joins,
         check_order_observe: check_order_observe,
-        fast_projection: fast_projection
+        fast_projection: fast_projection,
+        sort_collapse: sort_collapse
       )
     end
 
@@ -46,7 +48,8 @@ module Polars
       cluster_with_columns: nil,
       collapse_joins: nil,
       check_order_observe: nil,
-      fast_projection: nil
+      fast_projection: nil,
+      sort_collapse: nil
     )
       optflags = QueryOptFlags.new
       optflags.no_optimizations
@@ -60,7 +63,8 @@ module Polars
         cluster_with_columns: cluster_with_columns,
         collapse_joins: collapse_joins,
         check_order_observe: check_order_observe,
-        fast_projection: fast_projection
+        fast_projection: fast_projection,
+        sort_collapse: sort_collapse
       )
     end
 
@@ -74,7 +78,8 @@ module Polars
       cluster_with_columns: nil,
       collapse_joins: nil,
       check_order_observe: nil,
-      fast_projection: nil
+      fast_projection: nil,
+      sort_collapse: nil
     )
       if !predicate_pushdown.nil?
         self.predicate_pushdown = predicate_pushdown
@@ -112,6 +117,9 @@ module Polars
       if !fast_projection.nil?
         self.fast_projection = fast_projection
       end
+      if !sort_collapse.nil?
+        self.sort_collapse = sort_collapse
+      end
 
       self
     end
@@ -211,6 +219,15 @@ module Polars
       _rboptflags.fast_projection = value
     end
 
+    # Collapse sequential sort nodes into a single sort node.
+    def sort_collapse
+      _rboptflags.sort_collapse
+    end
+
+    def sort_collapse=(value)
+      _rboptflags.sort_collapse = value
+    end
+
     def to_s
       <<~STR
         QueryOptFlags {

data/lib/polars/schema.rb

@@ -71,6 +71,15 @@ module Polars
       @schema.values
     end
 
+    # Convert the schema to a nanoarrow schema.
+    #
+    # @return [Nanoarrow::Schema]
+    def to_arrow
+      require "nanoarrow"
+
+      Nanoarrow::Schema.new(self)
+    end
+
     # Get the number of schema entries.
     #
     # @return [Integer]

data/lib/polars/selectors.rb

@@ -264,7 +264,7 @@ module Polars
       re_alpha = ascii_only ? "a-zA-Z" : "\\p{Alphabetic}"
       re_digit = ascii_only ? "0-9" : "\\d"
       re_space = ignore_spaces ? " " : ""
-      return Selector._from_rbselector(
+      Selector._from_rbselector(
         RbSelector.matches("^[#{re_alpha}#{re_digit}#{re_space}]+$")
       )
     end

data/lib/polars/series.rb

@@ -73,6 +73,8 @@ module Polars
         self._s = Utils.dataframe_to_rbseries(
           original_name, values, dtype: dtype, strict: strict
         )
+      elsif values.respond_to?(:arrow_c_stream)
+        self._s = RbSeries.from_arrow_c_stream(values)
       else
         raise TypeError, "Series constructor called with unsupported type; got #{values.class.name}"
       end
@@ -496,6 +498,11 @@ module Polars
       end
     end
 
+    # @private
+    def arrow_c_stream
+      _s.arrow_c_stream
+    end
+
     # Return the Series as a scalar, or return the element at the given index.
     #
     # If no index is provided, this is equivalent to `s[0]`, with a check
@@ -1184,7 +1191,7 @@ module Polars
     #   s.cut([-1, 1], labels: ["a", "b", "c"])
     #   # =>
     #   # shape: (5,)
-    #   # Series: 'foo' [cat]
+    #   # Series: 'foo' [enum]
     #   # [
     #   #         "a"
     #   #         "a"
@@ -1201,7 +1208,7 @@ module Polars
     #   # ┌─────┬─────────────┬────────────┐
     #   # │ foo ┆ break_point ┆ category   │
     #   # │ --- ┆ ---         ┆ ---        │
-    #   # │ i64 ┆ f64         ┆ cat        │
+    #   # │ i64 ┆ f64         ┆ enum       │
     #   # ╞═════╪═════════════╪════════════╡
     #   # │ -2  ┆ -1.0        ┆ (-inf, -1] │
     #   # │ -1  ┆ -1.0        ┆ (-inf, -1] │
@@ -1515,13 +1522,7 @@ module Polars
     #   b = Polars::Series.new([0.65, 0.10, 0.25])
     #   b.entropy(normalize: true)
     #   # => 0.8568409950394724
-    def entropy(base: Math::E, normalize: nil)
-      # TODO update
-      if normalize.nil?
-        warn "The default `normalize` for `entropy` method will change from `false` to `true` in a future version"
-        normalize = false
-      end
-
+    def entropy(base: Math::E, normalize: true)
       Polars.select(Polars.lit(self).entropy(base: base, normalize: normalize)).to_series[0]
     end
 
@@ -2391,6 +2392,11 @@ module Polars
     #
     # @param indices [Array]
     #   Index location used for selection.
+    # @param null_on_oob [Boolean]
+    #   Behavior if an index is out of bounds:
+    #
+    #   - true  -> set the result to null
+    #   - false -> raise an error
     #
     # @return [Series]
     #
@@ -2404,7 +2410,7 @@ module Polars
     #   #         2
     #   #         4
     #   # ]
-    def gather(indices)
+    def gather(indices, null_on_oob: false)
       super
     end
 
@@ -2441,14 +2447,23 @@ module Polars
 
     # Check if the Series is empty.
     #
+    # @param ignore_nulls [Boolean]
+    #   If true a series containing only nulls will also be considered empty.
+    #   The default is false.
+    #
     # @return [Boolean]
     #
     # @example
     #   s = Polars::Series.new("a", [])
     #   s.is_empty
     #   # => true
-    def is_empty
-      len == 0
+    def is_empty(ignore_nulls: false)
+      if ignore_nulls
+        msg = "the `ignore_nulls` parameter of `Series.is_empty()` is considered unstable."
+        Utils.issue_unstable_warning(msg)
+      end
+
+      _s.is_empty(ignore_nulls: ignore_nulls)
     end
     alias_method :empty?, :is_empty
 
@@ -3106,6 +3121,8 @@ module Polars
     #   # Numo::Int64#shape=[3]
     #   # [1, 2, 3]
     def to_numo
+      require "numo/narray"
+
       if dtype.temporal?
         Numo::RObject.cast(to_a)
       else
@@ -3113,6 +3130,15 @@ module Polars
       end
     end
 
+    # Return the underlying Arrow array.
+    #
+    # @return [Nanoarrow::Array]
+    def to_arrow
+      require "nanoarrow"
+
+      Nanoarrow::Array.new(self)
+    end
+
     # Set masked values.
     #
     # @param filter [Series]
@@ -3391,6 +3417,51 @@ module Polars
       super
     end
 
+    # Truncate numeric data toward zero to `decimals` number of decimal places.
+    #
+    # @param decimals [Integer]
+    #   Number of decimal places to truncate to.
+    #
+    # @return [Series]
+    #
+    # @note
+    #   Truncation discards the fractional part beyond the given number of decimals.
+    #   For example, when rounding to 0 decimals 0.25, -0.25, 0.99, and -0.99 will
+    #   all round to 0. When rounding to 1 decimal 1.9999 rounds to 1.9 and -1.9999
+    #   rounds to -1.9. There is no tiebreak behaviour at midpoint values as there
+    #   is with :meth:`round` so 0.5 and -0.5 will also round to 0 when decimals=1.
+    #
+    # @note
+    #   This method performs numeric truncation. For truncating temporal
+    #   data (dates/datetimes), use :func:`Series.dt.truncate` instead.
+    #
+    # @example
+    #   s = Polars::Series.new("a", [1.12345, 2.56789, 3.991234])
+    #   s.truncate(2)
+    #   # =>
+    #   # shape: (3,)
+    #   # Series: 'a' [f64]
+    #   # [
+    #   #         1.12
+    #   #         2.56
+    #   #         3.99
+    #   # ]
+    #
+    # @example
+    #   s = Polars::Series.new("a", [-1.78, 2.56, -3.99])
+    #   s.truncate(0)
+    #   # =>
+    #   # shape: (3,)
+    #   # Series: 'a' [f64]
+    #   # [
+    #   #         -1.0
+    #   #         2.0
+    #   #         -3.0
+    #   # ]
+    def truncate(decimals = 0)
+      super
+    end
+
     # Round underlying floating point data by `decimals` digits.
     #
     # The default rounding mode is "half to even" (also known as "bankers' rounding").
@@ -4327,7 +4398,7 @@ module Polars
     def rolling_sum_by(
       by,
       window_size,
-      min_samples: 1,
+      min_samples: 0,
       closed: "right"
     )
       super
@@ -5185,8 +5256,8 @@ module Polars
     #   # shape: (2,)
     #   # Series: 'a' [i64]
     #   # [
-    #   #         5
     #   #         2
+    #   #         5
     #   # ]
     def sample(
       n: nil,
@@ -5299,13 +5370,18 @@ module Polars
       super
     end
 
-    # Reinterpret the underlying bits as a signed/unsigned integer.
+    # Reinterpret the underlying bits as a signed/unsigned integer or float.
     #
-    # This operation is only allowed for 64bit integers. For lower bits integers,
-    # you can safely use that cast operation.
+    # This operation is only allowed for numeric types of the same size.
+    # For lower bits numbers, you can safely use the cast operation.
+    #
+    # Either `signed` or `dtype` can be specified.
     #
     # @param signed [Boolean]
-    #   If true, reinterpret as `Polars::Int64`. Otherwise, reinterpret as `Polars::UInt64`.
+    #   If true, reinterpret as signed integer. Otherwise, reinterpret
+    #   as unsigned integer.
+    # @param dtype [Object]
+    #   DataType to reinterpret to.
     #
     # @return [Series]
     #
@@ -5320,7 +5396,18 @@ module Polars
     #   #         18446744073709551614
     #   #         3
     #   # ]
-    def reinterpret(signed: true)
+    #
+    # @example
+    #   s.reinterpret(dtype: Polars::Int64)
+    #   # =>
+    #   # shape: (3,)
+    #   # Series: 'a' [i64]
+    #   # [
+    #   #         -1152921504606846976
+    #   #         -2
+    #   #         3
+    #   # ]
+    def reinterpret(signed: nil, dtype: nil)
       super
     end
 

data/lib/polars/sql_context.rb

@@ -73,11 +73,11 @@ module Polars
     #   ctx.execute(
     #     "
     #     SELECT
-    #         MAX(release_year / 10) * 10 AS decade,
+    #         MAX(release_year // 10) * 10 AS decade,
     #         SUM(gross) AS total_gross,
     #         COUNT(title) AS n_films,
     #     FROM films
-    #     GROUP BY (release_year / 10) -- decade
+    #     GROUP BY (release_year // 10) -- decade
     #     ORDER BY total_gross DESC
     #     ",
     #     eager: true

data/lib/polars/string_cache.rb

@@ -1,30 +1,11 @@
 module Polars
-  # Class for enabling and disabling the global string cache.
+  # Does nothing.
   #
-  # @example Construct two Series using the same global string cache.
-  #   s1 = nil
-  #   s2 = nil
-  #   Polars::StringCache.new do
-  #     s1 = Polars::Series.new("color", ["red", "green", "red"], dtype: Polars::Categorical)
-  #     s2 = Polars::Series.new("color", ["blue", "red", "green"], dtype: Polars::Categorical)
-  #   end
-  #
-  # @example As both Series are constructed under the same global string cache, they can be concatenated.
-  #   Polars.concat([s1, s2])
-  #   # =>
-  #   # shape: (6,)
-  #   # Series: 'color' [cat]
-  #   # [
-  #   #         "red"
-  #   #         "green"
-  #   #         "red"
-  #   #         "blue"
-  #   #         "red"
-  #   #         "green"
-  #   # ]
+  # @deprecated
+  #   The string cache was used to maintain the mapping for the Categorical
+  #   dtype, this is now done through `Polars::Categories`.
   class StringCache
     def initialize(&block)
-      RbStringCacheHolder.hold(&block)
     end
   end
 
@@ -33,69 +14,35 @@ module Polars
   end
 
   module Functions
-    # Enable the global string cache.
+    # Does nothing.
     #
-    # `Categorical` columns created under the same global string cache have
-    # the same underlying physical value when string values are equal. This allows the
-    # columns to be concatenated or used in a join operation, for example.
+    # @deprecated
+    #   The string cache was used to maintain the mapping for the Categorical
+    #   dtype, this is now done through `Polars::Categories`.
     #
     # @return [nil]
-    #
-    # @example Construct two Series using the same global string cache.
-    #   Polars.enable_string_cache
-    #   s1 = Polars::Series.new("color", ["red", "green", "red"], dtype: Polars::Categorical)
-    #   s2 = Polars::Series.new("color", ["blue", "red", "green"], dtype: Polars::Categorical)
-    #   Polars.disable_string_cache
-    #
-    # @example As both Series are constructed under the same global string cache, they can be concatenated.
-    #   Polars.concat([s1, s2])
-    #   # =>
-    #   # shape: (6,)
-    #   # Series: 'color' [cat]
-    #   # [
-    #   #         "red"
-    #   #         "green"
-    #   #         "red"
-    #   #         "blue"
-    #   #         "red"
-    #   #         "green"
-    #   # ]
     def enable_string_cache
-      Plr.enable_string_cache
     end
 
-    # Disable and clear the global string cache.
+    # Does nothing.
     #
-    # @return [nil]
+    # @deprecated
+    #     The string cache was used to maintain the mapping for the Categorical
+    #     dtype, this is now done through `Polars::Categories`.
     #
-    # @example Construct two Series using the same global string cache.
-    #   Polars.enable_string_cache
-    #   s1 = Polars::Series.new("color", ["red", "green", "red"], dtype: Polars::Categorical)
-    #   s2 = Polars::Series.new("color", ["blue", "red", "green"], dtype: Polars::Categorical)
-    #   Polars.disable_string_cache
-    #
-    # @example As both Series are constructed under the same global string cache, they can be concatenated.
-    #   Polars.concat([s1, s2])
-    #   # =>
-    #   # shape: (6,)
-    #   # Series: 'color' [cat]
-    #   # [
-    #   #         "red"
-    #   #         "green"
-    #   #         "red"
-    #   #         "blue"
-    #   #         "red"
-    #   #         "green"
-    #   # ]
+    # @return [nil]
     def disable_string_cache
-      Plr.disable_string_cache
     end
 
-    # Check whether the global string cache is enabled.
+    # Always returns true.
+    #
+    # @deprecated
+    #   The string cache was used to maintain the mapping for the Categorical
+    #   dtype, this is now done through `Polars::Categories`.
     #
     # @return [Boolean]
     def using_string_cache
-      Plr.using_string_cache
+      true
     end
   end
 end

data/lib/polars/string_expr.rb

@@ -378,13 +378,7 @@ module Polars
     #   # ╞══════╡
     #   # │ null │
     #   # └──────┘
-    def join(delimiter = nil, ignore_nulls: true)
-      # TODO update
-      if delimiter.nil?
-        warn "The default `delimiter` for `join` method will change from `-` to empty string in a future version"
-        delimiter = "-"
-      end
-
+    def join(delimiter = "", ignore_nulls: true)
       Utils.wrap_expr(_rbexpr.str_join(delimiter, ignore_nulls))
     end
 

data/lib/polars/string_name_space.rb

@@ -1494,13 +1494,7 @@ module Polars
     #   # [
     #   #         null
     #   # ]
-    def join(delimiter = nil, ignore_nulls: true)
-      # TODO update
-      if delimiter.nil?
-        warn "The default `delimiter` for `join` method will change from `-` to empty string in a future version"
-        delimiter = "-"
-      end
-
+    def join(delimiter = "", ignore_nulls: true)
       super
     end
 

data/lib/polars/utils/construction/series.rb

@@ -313,12 +313,17 @@ module Polars
       TrueClass => RbSeries.method(:new_opt_bool),
       FalseClass => RbSeries.method(:new_opt_bool),
       Integer => RbSeries.method(:new_opt_i64),
-      String => RbSeries.method(:new_str),
-      BigDecimal => RbSeries.method(:new_decimal)
+      ::String => RbSeries.method(:new_str)
     }
 
     def self.rb_type_to_constructor(dtype)
-      RB_TYPE_TO_CONSTRUCTOR.fetch(dtype, RbSeries.method(:new_object))
+      RB_TYPE_TO_CONSTRUCTOR.fetch(dtype) do
+        if defined?(::BigDecimal) && dtype == ::BigDecimal
+          RbSeries.method(:new_decimal)
+        else
+          RbSeries.method(:new_object)
+        end
+      end
     end
 
     def self.numo_values_and_dtype(values)

data/lib/polars/utils/convert.rb

@@ -63,19 +63,24 @@ module Polars
     end
 
     def self._to_ruby_datetime(value, time_unit = "ns", time_zone = nil)
-      if time_zone.nil? || time_zone == "" || time_zone == "UTC"
+      utc_time =
         if time_unit == "ns"
-          ::Time.at(value / 1000000000, value % 1000000000, :nsec).utc
+          ::Time.at(value / 1_000_000_000, value % 1_000_000_000, :nsec).utc
         elsif time_unit == "us"
-          ::Time.at(value / 1000000, value % 1000000, :usec).utc
+          ::Time.at(value / 1_000_000, value % 1_000_000, :usec).utc
         elsif time_unit == "ms"
-          ::Time.at(value / 1000, value % 1000, :millisecond).utc
+          ::Time.at(value / 1_000, value % 1_000, :millisecond).utc
         else
           raise ArgumentError, "time_unit must be one of {{'ns', 'us', 'ms'}}, got #{time_unit}"
         end
-      else
-        raise Todo
+      return utc_time if time_zone.nil?
+
+      unless defined?(::TZInfo)
+        # TODO improve error message
+        require "tzinfo"
       end
+
+      utc_time.getlocal(::TZInfo::Timezone.get(time_zone))
     end
 
     def self._to_ruby_duration(value, time_unit = "ns")
@@ -91,6 +96,11 @@ module Polars
     end
 
     def self._to_ruby_decimal(prec, value)
+      unless defined?(::BigDecimal)
+        # TODO improve error message
+        require "bigdecimal"
+      end
+
       BigDecimal(value, prec)
     end
   end