polars-df 0.21.1 → 0.23.0

data/lib/polars/data_frame.rb

@@ -72,6 +72,43 @@ module Polars
       end
     end
 
+    # Read a serialized DataFrame from a file.
+    #
+    # @param source [Object]
+    #   Path to a file or a file-like object (by file-like object, we refer to
+    #   objects that have a `read` method, such as a file handler or `StringIO`).
+    #
+    # @return [DataFrame]
+    #
+    # @note
+    #   Serialization is not stable across Polars versions: a LazyFrame serialized
+    #   in one Polars version may not be deserializable in another Polars version.
+    #
+    # @example
+    #   df = Polars::DataFrame.new({"a" => [1, 2, 3], "b" => [4.0, 5.0, 6.0]})
+    #   bytes = df.serialize
+    #   Polars::DataFrame.deserialize(StringIO.new(bytes))
+    #   # =>
+    #   # shape: (3, 2)
+    #   # ┌─────┬─────┐
+    #   # │ a   ┆ b   │
+    #   # │ --- ┆ --- │
+    #   # │ i64 ┆ f64 │
+    #   # ╞═════╪═════╡
+    #   # │ 1   ┆ 4.0 │
+    #   # │ 2   ┆ 5.0 │
+    #   # │ 3   ┆ 6.0 │
+    #   # └─────┴─────┘
+    def self.deserialize(source)
+      if Utils.pathlike?(source)
+        source = Utils.normalize_filepath(source)
+      end
+
+      deserializer = RbDataFrame.method(:deserialize_binary)
+
+      _from_rbdf(deserializer.(source))
+    end
+
     # @private
     def self._from_rbdf(rb_df)
       df = DataFrame.allocate
@@ -562,8 +599,6 @@ module Polars
 
     # Convert every row to a hash.
     #
-    # Note that this is slow.
-    #
     # @return [Array]
     #
     # @example
@@ -572,12 +607,7 @@ module Polars
     #   # =>
     #   # [{"foo"=>1, "bar"=>4}, {"foo"=>2, "bar"=>5}, {"foo"=>3, "bar"=>6}]
     def to_hashes
-      rbdf = _df
-      names = columns
-
-      height.times.map do |i|
-        names.zip(rbdf.row_tuple(i)).to_h
-      end
+      rows(named: true)
     end
 
     # Convert DataFrame to a 2D Numo array.
@@ -634,6 +664,44 @@ module Polars
       Utils.wrap_s(_df.select_at_idx(index))
     end
 
+    # Serialize this DataFrame to a file or string.
+    #
+    # @param file [Object]
+    #   File path or writable file-like object to which the result will be written.
+    #   If set to `nil` (default), the output is returned as a string instead.
+    #
+    # @return [Object]
+    #
+    # @note
+    #   Serialization is not stable across Polars versions: a LazyFrame serialized
+    #   in one Polars version may not be deserializable in another Polars version.
+    #
+    # @example
+    #   df = Polars::DataFrame.new(
+    #     {
+    #       "foo" => [1, 2, 3],
+    #       "bar" => [6, 7, 8]
+    #     }
+    #   )
+    #   bytes = df.serialize
+    #   Polars::DataFrame.deserialize(StringIO.new(bytes))
+    #   # =>
+    #   # shape: (3, 2)
+    #   # ┌─────┬─────┐
+    #   # │ foo ┆ bar │
+    #   # │ --- ┆ --- │
+    #   # │ i64 ┆ i64 │
+    #   # ╞═════╪═════╡
+    #   # │ 1   ┆ 6   │
+    #   # │ 2   ┆ 7   │
+    #   # │ 3   ┆ 8   │
+    #   # └─────┴─────┘
+    def serialize(file = nil)
+      serializer = _df.method(:serialize_binary)
+
+      Utils.serialize_polars_object(serializer, file)
+    end
+
     # Serialize to JSON representation.
     #
     # @param file [String]
@@ -1148,6 +1216,40 @@ module Polars
       end
     end
 
+    # Write DataFrame to an Iceberg table.
+    #
+    # @note
+    #   This functionality is currently considered **unstable**. It may be
+    #   changed at any point without it being considered a breaking change.
+    #
+    # @param target [Object]
+    #   Name of the table or the Table object representing an Iceberg table.
+    # @param mode ['append', 'overwrite']
+    #   How to handle existing data.
+    #
+    #   - If 'append', will add new data.
+    #   - If 'overwrite', will replace table with new data.
+    #
+    # @return [nil]
+    def write_iceberg(target, mode:)
+      require "iceberg"
+
+      table =
+        if target.is_a?(Iceberg::Table)
+          target
+        else
+          raise Todo
+        end
+
+      data = self
+
+      if mode == "append"
+        table.append(data)
+      else
+        raise Todo
+      end
+    end
+
     # Write DataFrame as delta table.
     #
     # @param target [Object]
@@ -5957,8 +6059,13 @@ module Polars
     # The fields will be inserted into the `DataFrame` on the location of the
     # `struct` type.
     #
-    # @param names [Object]
-    #  Names of the struct columns that will be decomposed by its fields
+    # @param columns [Object]
+    #   Name of the struct column(s) that should be unnested.
+    # @param more_columns [Array]
+    #   Additional columns to unnest, specified as positional arguments.
+    # @param separator [String]
+    #   Rename output column names as combination of the struct column name,
+    #   name separator and field name.
     #
     # @return [DataFrame]
     #
@@ -5984,11 +6091,8 @@ module Polars
     #   # │ foo    ┆ 1   ┆ a   ┆ true ┆ [1, 2]    ┆ baz   │
     #   # │ bar    ┆ 2   ┆ b   ┆ null ┆ [3]       ┆ womp  │
     #   # └────────┴─────┴─────┴──────┴───────────┴───────┘
-    def unnest(names)
-      if names.is_a?(::String)
-        names = [names]
-      end
-      _from_rbdf(_df.unnest(names))
+    def unnest(columns, *more_columns, separator: nil)
+      lazy.unnest(columns, *more_columns, separator: separator).collect(_eager: true)
     end
 
     # Requires NumPy

data/lib/polars/data_types.rb

@@ -110,12 +110,23 @@ module Polars
       DataTypeExpr._from_rbdatatype_expr(RbDataTypeExpr.from_dtype(self))
     end
 
-    [:numeric?, :decimal?, :integer?, :signed_integer?, :unsigned_integer?, :float?, :temporal?, :nested?, :to_dtype_expr].each do |v|
+    [:numeric?, :decimal?, :integer?, :signed_integer?, :unsigned_integer?, :float?, :temporal?, :nested?].each do |v|
       define_method(v) do
         self.class.public_send(v)
       end
     end
 
+    # Return a `DataTypeExpr` with a static `DataType`.
+    #
+    # @return [Expr]
+    #
+    # @example
+    #   Polars::Int16.new.to_dtype_expr.collect_dtype({})
+    #   # => Polars::Int16
+    def to_dtype_expr
+      DataTypeExpr._from_rbdatatype_expr(RbDataTypeExpr.from_dtype(self))
+    end
+
     # Returns a string representing the data type.
     #
     # @return [String]
@@ -198,6 +209,10 @@ module Polars
   class UInt64 < UnsignedIntegerType
   end
 
+  # 128-bit unsigned integer type.
+  class UInt128 < UnsignedIntegerType
+  end
+
   # 32-bit floating point type.
   class Float32 < FloatType
   end
@@ -212,7 +227,11 @@ module Polars
   class Decimal < NumericType
     attr_reader :precision, :scale
 
-    def initialize(precision, scale)
+    def initialize(precision = nil, scale = 0)
+      if precision.nil?
+        precision = 38
+      end
+
       @precision = precision
       @scale = scale
     end
@@ -317,11 +336,9 @@ module Polars
   class Categories
     attr_accessor :_categories
 
-    def initialize
-      # TODO fix
-      name = nil
+    def initialize(name = nil)
       if name.nil? || name == ""
-        @_categories = RbCategories.global_categories
+        self._categories = RbCategories.global_categories
         return
       end
 

data/lib/polars/date_time_expr.rb

@@ -1188,7 +1188,7 @@ module Polars
       if Utils::DTYPE_TEMPORAL_UNITS.include?(time_unit)
         timestamp(time_unit)
       elsif time_unit == "s"
-        Utils.wrap_expr(_rbexpr.dt_epoch_seconds)
+        timestamp("ms").floordiv(F.lit(1000, dtype: Int64))
       elsif time_unit == "d"
         Utils.wrap_expr(_rbexpr).cast(:date).cast(:i32)
       else
@@ -1435,6 +1435,9 @@ module Polars
 
     # Extract the days from a Duration type.
     #
+    # @param fractional [Boolean]
+    #   Whether to include the fractional component of the second.
+    #
     # @return [Expr]
     #
     # @example
@@ -1462,13 +1465,16 @@ module Polars
     #   # │ 2020-04-01 00:00:00 ┆ 31        │
     #   # │ 2020-05-01 00:00:00 ┆ 30        │
     #   # └─────────────────────┴───────────┘
-    def total_days
-      Utils.wrap_expr(_rbexpr.dt_total_days)
+    def total_days(fractional: false)
+      Utils.wrap_expr(_rbexpr.dt_total_days(fractional))
     end
     alias_method :days, :total_days
 
     # Extract the hours from a Duration type.
     #
+    # @param fractional [Boolean]
+    #   Whether to include the fractional component of the second.
+    #
     # @return [Expr]
     #
     # @example
@@ -1497,13 +1503,16 @@ module Polars
     #   # │ 2020-01-03 00:00:00 ┆ 24         │
     #   # │ 2020-01-04 00:00:00 ┆ 24         │
     #   # └─────────────────────┴────────────┘
-    def total_hours
-      Utils.wrap_expr(_rbexpr.dt_total_hours)
+    def total_hours(fractional: false)
+      Utils.wrap_expr(_rbexpr.dt_total_hours(fractional))
     end
     alias_method :hours, :total_hours
 
     # Extract the minutes from a Duration type.
     #
+    # @param fractional [Boolean]
+    #   Whether to include the fractional component of the second.
+    #
     # @return [Expr]
     #
     # @example
@@ -1532,13 +1541,16 @@ module Polars
     #   # │ 2020-01-03 00:00:00 ┆ 1440         │
     #   # │ 2020-01-04 00:00:00 ┆ 1440         │
     #   # └─────────────────────┴──────────────┘
-    def total_minutes
-      Utils.wrap_expr(_rbexpr.dt_total_minutes)
+    def total_minutes(fractional: false)
+      Utils.wrap_expr(_rbexpr.dt_total_minutes(fractional))
     end
     alias_method :minutes, :total_minutes
 
     # Extract the seconds from a Duration type.
     #
+    # @param fractional [Boolean]
+    #   Whether to include the fractional component of the second.
+    #
     # @return [Expr]
     #
     # @example
@@ -1568,13 +1580,16 @@ module Polars
     #   # │ 2020-01-01 00:03:00 ┆ 60           │
     #   # │ 2020-01-01 00:04:00 ┆ 60           │
     #   # └─────────────────────┴──────────────┘
-    def total_seconds
-      Utils.wrap_expr(_rbexpr.dt_total_seconds)
+    def total_seconds(fractional: false)
+      Utils.wrap_expr(_rbexpr.dt_total_seconds(fractional))
     end
     alias_method :seconds, :total_seconds
 
     # Extract the milliseconds from a Duration type.
     #
+    # @param fractional [Boolean]
+    #   Whether to include the fractional component of the second.
+    #
     # @return [Expr]
     #
     # @example
@@ -1610,13 +1625,16 @@ module Polars
     #   # │ 2020-01-01 00:00:00.999 ┆ 1                 │
     #   # │ 2020-01-01 00:00:01     ┆ 1                 │
     #   # └─────────────────────────┴───────────────────┘
-    def total_milliseconds
-      Utils.wrap_expr(_rbexpr.dt_total_milliseconds)
+    def total_milliseconds(fractional: false)
+      Utils.wrap_expr(_rbexpr.dt_total_milliseconds(fractional))
     end
     alias_method :milliseconds, :total_milliseconds
 
     # Extract the microseconds from a Duration type.
     #
+    # @param fractional [Boolean]
+    #   Whether to include the fractional component of the second.
+    #
     # @return [Expr]
     #
     # @example
@@ -1652,13 +1670,16 @@ module Polars
     #   # │ 2020-01-01 00:00:00.999 ┆ 1000              │
     #   # │ 2020-01-01 00:00:01     ┆ 1000              │
     #   # └─────────────────────────┴───────────────────┘
-    def total_microseconds
-      Utils.wrap_expr(_rbexpr.dt_total_microseconds)
+    def total_microseconds(fractional: false)
+      Utils.wrap_expr(_rbexpr.dt_total_microseconds(fractional))
     end
     alias_method :microseconds, :total_microseconds
 
     # Extract the nanoseconds from a Duration type.
     #
+    # @param fractional [Boolean]
+    #   Whether to include the fractional component of the second.
+    #
     # @return [Expr]
     #
     # @example
@@ -1694,8 +1715,8 @@ module Polars
     #   # │ 2020-01-01 00:00:00.999 ┆ 1000000          │
     #   # │ 2020-01-01 00:00:01     ┆ 1000000          │
     #   # └─────────────────────────┴──────────────────┘
-    def total_nanoseconds
-      Utils.wrap_expr(_rbexpr.dt_total_nanoseconds)
+    def total_nanoseconds(fractional: false)
+      Utils.wrap_expr(_rbexpr.dt_total_nanoseconds(fractional))
     end
     alias_method :nanoseconds, :total_nanoseconds
 

data/lib/polars/expr.rb

@@ -146,6 +146,40 @@ module Polars
       wrap_expr(_rbexpr.neg)
     end
 
+    # Read a serialized expression from a file.
+    #
+    # @param source [Object]
+    #   Path to a file or a file-like object (by file-like object, we refer to
+    #   objects that have a `read` method, such as a file handler or `StringIO`).
+    #
+    # @return [Expr]
+    #
+    # @note
+    #   This function uses  marshaling if the logical plan contains Ruby UDFs,
+    #   and as such inherits the security implications. Deserializing can execute
+    #   arbitrary code, so it should only be attempted on trusted data.
+    #
+    # @note
+    #   Serialization is not stable across Polars versions: a LazyFrame serialized
+    #   in one Polars version may not be deserializable in another Polars version.
+    #
+    # @example
+    #   expr = Polars.col("foo").sum.over("bar")
+    #   bytes = expr.meta.serialize
+    #   Polars::Expr.deserialize(StringIO.new(bytes))
+    #   # => col("foo").sum().over([col("bar")])
+    def self.deserialize(source)
+      raise Todo unless RbExpr.respond_to?(:deserialize_binary)
+
+      if Utils.pathlike?(source)
+        source = Utils.normalize_filepath(source)
+      end
+
+      deserializer = RbExpr.method(:deserialize_binary)
+
+      _from_rbexpr(deserializer.(source))
+    end
+
     # Cast to physical representation of the logical dtype.
     #
     # - `:date` -> `:i32`
@@ -377,8 +411,6 @@ module Polars
       wrap_expr(_rbexpr._alias(name))
     end
 
-    # TODO support symbols for exclude
-
     # Exclude certain columns from a wildcard/regex selection.
     #
     # You may also use regexes in the exclude list. They must start with `^` and end
@@ -1787,7 +1819,7 @@ module Polars
       wrap_expr(_rbexpr.arg_min)
     end
 
-    # Get the index of the first occurrence of a value, or ``None`` if it's not found.
+    # Get the index of the first occurrence of a value, or `nil` if it's not found.
     #
     # @param element [Object]
     #   Value to find.
@@ -6580,8 +6612,8 @@ module Polars
     #   # │ 10   ┆ null       │
     #   # │ 11   ┆ 0.1        │
     #   # │ 12   ┆ 0.090909   │
-    #   # │ null ┆ 0.0        │
-    #   # │ 12   ┆ 0.0        │
+    #   # │ null ┆ null       │
+    #   # │ 12   ┆ null       │
     #   # └──────┴────────────┘
     def pct_change(n: 1)
       n = Utils.parse_into_expression(n)
@@ -7571,7 +7603,8 @@ module Polars
     #   # │ 1.584963 │
     #   # └──────────┘
     def log(base = Math::E)
-      wrap_expr(_rbexpr.log(base))
+      base_rbexpr = Utils.parse_into_expression(base)
+      wrap_expr(_rbexpr.log(base_rbexpr))
     end
 
     # Compute the natural logarithm of each element plus one.
@@ -7743,33 +7776,9 @@ module Polars
     # This can be used to reduce memory pressure.
     #
     # @return [Expr]
-    #
-    # @example
-    #   Polars::DataFrame.new(
-    #     {
-    #       "a" => [1, 2, 3],
-    #       "b" => [1, 2, 2 << 32],
-    #       "c" => [-1, 2, 1 << 30],
-    #       "d" => [-112, 2, 112],
-    #       "e" => [-112, 2, 129],
-    #       "f" => ["a", "b", "c"],
-    #       "g" => [0.1, 1.32, 0.12],
-    #       "h" => [true, nil, false]
-    #     }
-    #   ).select(Polars.all.shrink_dtype)
-    #   # =>
-    #   # shape: (3, 8)
-    #   # ┌─────┬────────────┬────────────┬──────┬──────┬─────┬──────┬───────┐
-    #   # │ a   ┆ b          ┆ c          ┆ d    ┆ e    ┆ f   ┆ g    ┆ h     │
-    #   # │ --- ┆ ---        ┆ ---        ┆ ---  ┆ ---  ┆ --- ┆ ---  ┆ ---   │
-    #   # │ i8  ┆ i64        ┆ i32        ┆ i8   ┆ i16  ┆ str ┆ f32  ┆ bool  │
-    #   # ╞═════╪════════════╪════════════╪══════╪══════╪═════╪══════╪═══════╡
-    #   # │ 1   ┆ 1          ┆ -1         ┆ -112 ┆ -112 ┆ a   ┆ 0.1  ┆ true  │
-    #   # │ 2   ┆ 2          ┆ 2          ┆ 2    ┆ 2    ┆ b   ┆ 1.32 ┆ null  │
-    #   # │ 3   ┆ 8589934592 ┆ 1073741824 ┆ 112  ┆ 129  ┆ c   ┆ 0.12 ┆ false │
-    #   # └─────┴────────────┴────────────┴──────┴──────┴─────┴──────┴───────┘
     def shrink_dtype
-      wrap_expr(_rbexpr.shrink_dtype)
+      warn "`Expr.shrink_dtype` is deprecated and is a no-op; use `Series.shrink_dtype` instead."
+      self
     end
 
     # Bin values into buckets and count their occurrences.

data/lib/polars/functions/business.rb

@@ -0,0 +1,95 @@
+module Polars
+  module Functions
+    # Count the number of business days between `start` and `end` (not including `end`).
+    #
+    # @note
+    #   This functionality is considered **unstable**. It may be changed
+    #   at any point without it being considered a breaking change.
+    #
+    # @param start [Object]
+    #   Start dates.
+    # @param stop [Object]
+    #   End dates.
+    # @param week_mask [Array]
+    #   Which days of the week to count. The default is Monday to Friday.
+    #   If you wanted to count only Monday to Thursday, you would pass
+    #   `[true, true, true, true, false, false, false]`.
+    # @param holidays [Array]
+    #   Holidays to exclude from the count.
+    #
+    # @return [Expr]
+    #
+    # @example
+    #   df = Polars::DataFrame.new(
+    #     {
+    #       "start" => [Date.new(2020, 1, 1), Date.new(2020, 1, 2)],
+    #       "end" => [Date.new(2020, 1, 2), Date.new(2020, 1, 10)]
+    #     }
+    #   )
+    #   df.with_columns(
+    #     business_day_count: Polars.business_day_count("start", "end")
+    #   )
+    #   # =>
+    #   # shape: (2, 3)
+    #   # ┌────────────┬────────────┬────────────────────┐
+    #   # │ start      ┆ end        ┆ business_day_count │
+    #   # │ ---        ┆ ---        ┆ ---                │
+    #   # │ date       ┆ date       ┆ i32                │
+    #   # ╞════════════╪════════════╪════════════════════╡
+    #   # │ 2020-01-01 ┆ 2020-01-02 ┆ 1                  │
+    #   # │ 2020-01-02 ┆ 2020-01-10 ┆ 6                  │
+    #   # └────────────┴────────────┴────────────────────┘
+    #
+    # @example You can pass a custom weekend - for example, if you only take Sunday off:
+    #   week_mask = [true, true, true, true, true, true, false]
+    #   df.with_columns(
+    #     business_day_count: Polars.business_day_count(
+    #       "start", "end", week_mask: week_mask
+    #     )
+    #   )
+    #   # =>
+    #   # shape: (2, 3)
+    #   # ┌────────────┬────────────┬────────────────────┐
+    #   # │ start      ┆ end        ┆ business_day_count │
+    #   # │ ---        ┆ ---        ┆ ---                │
+    #   # │ date       ┆ date       ┆ i32                │
+    #   # ╞════════════╪════════════╪════════════════════╡
+    #   # │ 2020-01-01 ┆ 2020-01-02 ┆ 1                  │
+    #   # │ 2020-01-02 ┆ 2020-01-10 ┆ 7                  │
+    #   # └────────────┴────────────┴────────────────────┘
+    #
+    # @example You can also pass a list of holidays to exclude from the count:
+    #   holidays = [Date.new(2020, 1, 1), Date.new(2020, 1, 2)]
+    #   df.with_columns(
+    #     business_day_count: Polars.business_day_count("start", "end", holidays: holidays)
+    #   )
+    #   # =>
+    #   # shape: (2, 3)
+    #   # ┌────────────┬────────────┬────────────────────┐
+    #   # │ start      ┆ end        ┆ business_day_count │
+    #   # │ ---        ┆ ---        ┆ ---                │
+    #   # │ date       ┆ date       ┆ i32                │
+    #   # ╞════════════╪════════════╪════════════════════╡
+    #   # │ 2020-01-01 ┆ 2020-01-02 ┆ 0                  │
+    #   # │ 2020-01-02 ┆ 2020-01-10 ┆ 5                  │
+    #   # └────────────┴────────────┴────────────────────┘
+    def business_day_count(
+      start,
+      stop,
+      week_mask: [true, true, true, true, true, false, false],
+      holidays: []
+    )
+      start_rbexpr = Utils.parse_into_expression(start)
+      end_rbexpr = Utils.parse_into_expression(stop)
+      unix_epoch = ::Date.new(1970, 1, 1)
+      Utils.wrap_expr(
+        Plr.business_day_count(
+          start_rbexpr,
+          end_rbexpr,
+          week_mask,
+          holidays.map { |holiday| holiday - unix_epoch }
+        )
+      )
+    end
+  end
+end

data/lib/polars/functions/lazy.rb

@@ -823,7 +823,7 @@ module Polars
     #   # ┌─────┐
     #   # │ sum │
     #   # │ --- │
-    #   # │ i64 │
+    #   # │ i32 │
     #   # ╞═════╡
     #   # │ 10  │
     #   # │ 13  │