polars-df 0.15.0-x86_64-linux-musl → 0.17.0-x86_64-linux-musl

data/LICENSE.txt

@@ -1,5 +1,5 @@
 Copyright (c) 2020 Ritchie Vink
-Copyright (c) 2022-2024 Andrew Kane
+Copyright (c) 2022-2025 Andrew Kane
 Some portions Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
 
 Permission is hereby granted, free of charge, to any person obtaining a copy

data/README.md

@@ -14,7 +14,7 @@ gem "polars-df"
 
 ## Getting Started
 
-This library follows the [Polars Python API](https://pola-rs.github.io/polars/py-polars/html/reference/index.html).
+This library follows the [Polars Python API](https://docs.pola.rs/api/python/stable/reference/index.html).
 
 ```ruby
 Polars.scan_csv("iris.csv")
@@ -24,7 +24,7 @@ Polars.scan_csv("iris.csv")
   .collect
 ```
 
-You can follow [Polars tutorials](https://pola-rs.github.io/polars-book/user-guide/) and convert the code to Ruby in many cases. Feel free to open an issue if you run into problems.
+You can follow [Polars tutorials](https://docs.pola.rs/user-guide/getting-started/) and convert the code to Ruby in many cases. Feel free to open an issue if you run into problems.
 
 ## Reference
 
@@ -88,6 +88,15 @@ From Avro
 Polars.read_avro("file.avro")
 ```
 
+From Delta Lake (requires [deltalake-rb](https://github.com/ankane/delta-ruby)) [experimental]
+
+```ruby
+Polars.read_delta("./table")
+
+# or lazily with
+Polars.scan_delta("./table")
+```
+
 From a hash
 
 ```ruby
@@ -336,6 +345,32 @@ Parquet
 df.write_parquet("file.parquet")
 ```
 
+JSON
+
+```ruby
+df.write_json("file.json")
+# or
+df.write_ndjson("file.ndjson")
+```
+
+Feather / Arrow IPC
+
+```ruby
+df.write_ipc("file.arrow")
+```
+
+Avro
+
+```ruby
+df.write_avro("file.avro")
+```
+
+Delta Lake [experimental]
+
+```ruby
+df.write_delta("./table")
+```
+
 Numo array
 
 ```ruby

data/lib/polars/data_frame.rb

@@ -604,10 +604,6 @@ module Polars
     #
     # @param file [String]
     #   File path to which the result should be written.
-    # @param pretty [Boolean]
-    #   Pretty serialize json.
-    # @param row_oriented [Boolean]
-    #   Write to row oriented json. This is slower, but more common.
     #
     # @return [nil]
     #
@@ -619,16 +615,8 @@ module Polars
     #     }
     #   )
     #   df.write_json
-    #   # => "{\"columns\":[{\"name\":\"foo\",\"datatype\":\"Int64\",\"bit_settings\":\"\",\"values\":[1,2,3]},{\"name\":\"bar\",\"datatype\":\"Int64\",\"bit_settings\":\"\",\"values\":[6,7,8]}]}"
-    #
-    # @example
-    #   df.write_json(row_oriented: true)
     #   # => "[{\"foo\":1,\"bar\":6},{\"foo\":2,\"bar\":7},{\"foo\":3,\"bar\":8}]"
-    def write_json(
-      file = nil,
-      pretty: false,
-      row_oriented: false
-    )
+    def write_json(file = nil)
       if Utils.pathlike?(file)
         file = Utils.normalize_filepath(file)
       end
@@ -636,7 +624,7 @@ module Polars
       if file.nil? || to_string_io
         buf = StringIO.new
         buf.set_encoding(Encoding::BINARY)
-        _df.write_json(buf, pretty, row_oriented)
+        _df.write_json(buf)
         json_bytes = buf.string
 
         json_str = json_bytes.force_encoding(Encoding::UTF_8)
@@ -646,7 +634,7 @@ module Polars
           return json_str
         end
       else
-        _df.write_json(file, pretty, row_oriented)
+        _df.write_json(file)
       end
       nil
     end
@@ -831,7 +819,13 @@ module Polars
     #   Compression method. Defaults to "uncompressed".
     #
     # @return [nil]
-    def write_ipc(file, compression: "uncompressed", compat_level: nil)
+    def write_ipc(
+      file,
+      compression: "uncompressed",
+      compat_level: nil,
+      storage_options: nil,
+      retries: 2
+    )
       return_bytes = file.nil?
       if return_bytes
         file = StringIO.new
@@ -849,7 +843,13 @@ module Polars
         compression = "uncompressed"
       end
 
-      _df.write_ipc(file, compression, compat_level)
+      if storage_options&.any?
+        storage_options = storage_options.to_a
+      else
+        storage_options = nil
+      end
+
+      _df.write_ipc(file, compression, compat_level, storage_options, retries)
       return_bytes ? file.string : nil
     end
 
@@ -961,6 +961,61 @@ module Polars
       )
     end
 
+    # Write DataFrame as delta table.
+    #
+    # @param target [Object]
+    #   URI of a table or a DeltaTable object.
+    # @param mode ["error", "append", "overwrite", "ignore", "merge"]
+    #   How to handle existing data.
+    # @param storage_options [Hash]
+    #   Extra options for the storage backends supported by `deltalake-rb`.
+    # @param delta_write_options [Hash]
+    #   Additional keyword arguments while writing a Delta lake Table.
+    # @param delta_merge_options [Hash]
+    #   Keyword arguments which are required to `MERGE` a Delta lake Table.
+    #
+    # @return [nil]
+    def write_delta(
+      target,
+      mode: "error",
+      storage_options: nil,
+      delta_write_options: nil,
+      delta_merge_options: nil
+    )
+      Polars.send(:_check_if_delta_available)
+
+      if Utils.pathlike?(target)
+        target = Polars.send(:_resolve_delta_lake_uri, target.to_s, strict: false)
+      end
+
+      data = self
+
+      if mode == "merge"
+        if delta_merge_options.nil?
+          msg = "You need to pass delta_merge_options with at least a given predicate for `MERGE` to work."
+          raise ArgumentError, msg
+        end
+        if target.is_a?(::String)
+          dt = DeltaLake::Table.new(target, storage_options: storage_options)
+        else
+          dt = target
+        end
+
+        predicate = delta_merge_options.delete(:predicate)
+        dt.merge(data, predicate, **delta_merge_options)
+      else
+        delta_write_options ||= {}
+
+        DeltaLake.write(
+          target,
+          data,
+          mode: mode,
+          storage_options: storage_options,
+          **delta_write_options
+        )
+      end
+    end
+
     # Return an estimation of the total (heap) allocated size of the DataFrame.
     #
     # Estimated size is given in the specified unit (bytes by default).
@@ -2227,6 +2282,14 @@ module Polars
     #   keys are within this distance. If an asof join is done on columns of dtype
     #   "Date", "Datetime", "Duration" or "Time" you use the following string
     #   language:
+    # @param allow_exact_matches [Boolean]
+    #   Whether exact matches are valid join predicates.
+    #     - If true, allow matching with the same `on` value (i.e. less-than-or-equal-to / greater-than-or-equal-to).
+    #     - If false, don't match the same `on` value (i.e., strictly less-than / strictly greater-than).
+    # @param check_sortedness [Boolean]
+    #   Check the sortedness of the asof keys. If the keys are not sorted Polars
+    #   will error, or in case of 'by' argument raise a warning. This might become
+    #   a hard error in the future.
     #
     #    - 1ns   (1 nanosecond)
     #    - 1us   (1 microsecond)
@@ -2308,7 +2371,9 @@ module Polars
       tolerance: nil,
       allow_parallel: true,
       force_parallel: false,
-      coalesce: true
+      coalesce: true,
+      allow_exact_matches: true,
+      check_sortedness: true
     )
       lazy
         .join_asof(
@@ -2324,7 +2389,9 @@ module Polars
           tolerance: tolerance,
           allow_parallel: allow_parallel,
           force_parallel: force_parallel,
-          coalesce: coalesce
+          coalesce: coalesce,
+          allow_exact_matches: allow_exact_matches,
+          check_sortedness: check_sortedness
         )
         .collect(no_optimization: true)
     end
@@ -3939,14 +4006,32 @@ module Polars
     #   # ╞═════╪═════╪═════╡
     #   # │ 3   ┆ 8   ┆ c   │
     #   # └─────┴─────┴─────┘
-    def max(axis: 0)
-      if axis == 0
-        lazy.max.collect(_eager: true)
-      elsif axis == 1
-        Utils.wrap_s(_df.max_horizontal)
-      else
-        raise ArgumentError, "Axis should be 0 or 1."
-      end
+    def max
+      lazy.max.collect(_eager: true)
+    end
+
+    # Get the maximum value horizontally across columns.
+    #
+    # @return [Series]
+    #
+    # @example
+    #   df = Polars::DataFrame.new(
+    #     {
+    #       "foo" => [1, 2, 3],
+    #       "bar" => [4.0, 5.0, 6.0]
+    #     }
+    #   )
+    #   df.max_horizontal
+    #   # =>
+    #   # shape: (3,)
+    #   # Series: 'max' [f64]
+    #   # [
+    #   #         4.0
+    #   #         5.0
+    #   #         6.0
+    #   # ]
+    def max_horizontal
+      select(max: F.max_horizontal(F.all)).to_series
     end
 
     # Aggregate the columns of this DataFrame to their minimum value.
@@ -3971,22 +4056,35 @@ module Polars
     #   # ╞═════╪═════╪═════╡
     #   # │ 1   ┆ 6   ┆ a   │
     #   # └─────┴─────┴─────┘
-    def min(axis: 0)
-      if axis == 0
-        lazy.min.collect(_eager: true)
-      elsif axis == 1
-        Utils.wrap_s(_df.min_horizontal)
-      else
-        raise ArgumentError, "Axis should be 0 or 1."
-      end
+    def min
+      lazy.min.collect(_eager: true)
     end
 
-    # Aggregate the columns of this DataFrame to their sum value.
+    # Get the minimum value horizontally across columns.
     #
-    # @param axis [Integer]
-    #   Either 0 or 1.
-    # @param null_strategy ["ignore", "propagate"]
-    #   This argument is only used if axis == 1.
+    # @return [Series]
+    #
+    # @example
+    #   df = Polars::DataFrame.new(
+    #     {
+    #       "foo" => [1, 2, 3],
+    #       "bar" => [4.0, 5.0, 6.0]
+    #     }
+    #   )
+    #   df.min_horizontal
+    #   # =>
+    #   # shape: (3,)
+    #   # Series: 'min' [f64]
+    #   # [
+    #   #         1.0
+    #   #         2.0
+    #   #         3.0
+    #   # ]
+    def min_horizontal
+      select(min: F.min_horizontal(F.all)).to_series
+    end
+
+    # Aggregate the columns of this DataFrame to their sum value.
     #
     # @return [DataFrame]
     #
@@ -4008,35 +4106,42 @@ module Polars
     #   # ╞═════╪═════╪══════╡
     #   # │ 6   ┆ 21  ┆ null │
     #   # └─────┴─────┴──────┘
+    def sum
+      lazy.sum.collect(_eager: true)
+    end
+
+    # Sum all values horizontally across columns.
+    #
+    # @param ignore_nulls [Boolean]
+    #   Ignore null values (default).
+    #   If set to `false`, any null value in the input will lead to a null output.
+    #
+    # @return [Series]
     #
     # @example
-    #   df.sum(axis: 1)
+    #   df = Polars::DataFrame.new(
+    #     {
+    #       "foo" => [1, 2, 3],
+    #       "bar" => [4.0, 5.0, 6.0]
+    #     }
+    #   )
+    #   df.sum_horizontal
     #   # =>
     #   # shape: (3,)
-    #   # Series: 'foo' [str]
+    #   # Series: 'sum' [f64]
     #   # [
-    #   #         "16a"
-    #   #         "27b"
-    #   #         "38c"
+    #   #         5.0
+    #   #         7.0
+    #   #         9.0
     #   # ]
-    def sum(axis: 0, null_strategy: "ignore")
-      case axis
-      when 0
-        lazy.sum.collect(_eager: true)
-      when 1
-        Utils.wrap_s(_df.sum_horizontal(null_strategy))
-      else
-        raise ArgumentError, "Axis should be 0 or 1."
-      end
+    def sum_horizontal(ignore_nulls: true)
+      select(
+        sum: F.sum_horizontal(F.all, ignore_nulls: ignore_nulls)
+      ).to_series
     end
 
     # Aggregate the columns of this DataFrame to their mean value.
     #
-    # @param axis [Integer]
-    #   Either 0 or 1.
-    # @param null_strategy ["ignore", "propagate"]
-    #   This argument is only used if axis == 1.
-    #
     # @return [DataFrame]
     #
     # @example
@@ -4057,15 +4162,38 @@ module Polars
     #   # ╞═════╪═════╪══════╡
     #   # │ 2.0 ┆ 7.0 ┆ null │
     #   # └─────┴─────┴──────┘
-    def mean(axis: 0, null_strategy: "ignore")
-      case axis
-      when 0
-        lazy.mean.collect(_eager: true)
-      when 1
-        Utils.wrap_s(_df.mean_horizontal(null_strategy))
-      else
-        raise ArgumentError, "Axis should be 0 or 1."
-      end
+    def mean
+      lazy.mean.collect(_eager: true)
+    end
+
+    # Take the mean of all values horizontally across columns.
+    #
+    # @param ignore_nulls [Boolean]
+    #   Ignore null values (default).
+    #   If set to `false`, any null value in the input will lead to a null output.
+    #
+    # @return [Series]
+    #
+    # @example
+    #   df = Polars::DataFrame.new(
+    #     {
+    #       "foo" => [1, 2, 3],
+    #       "bar" => [4.0, 5.0, 6.0]
+    #     }
+    #   )
+    #   df.mean_horizontal
+    #   # =>
+    #   # shape: (3,)
+    #   # Series: 'mean' [f64]
+    #   # [
+    #   #         2.5
+    #   #         3.5
+    #   #         4.5
+    #   # ]
+    def mean_horizontal(ignore_nulls: true)
+      select(
+        mean: F.mean_horizontal(F.all, ignore_nulls: ignore_nulls)
+      ).to_series
     end
 
     # Aggregate the columns of this DataFrame to their standard deviation value.

data/lib/polars/data_types.rb

@@ -167,6 +167,10 @@ module Polars
   class Int64 < SignedIntegerType
   end
 
+  # 128-bit signed integer type.
+  class Int128 < SignedIntegerType
+  end
+
   # 8-bit unsigned integer type.
   class UInt8 < UnsignedIntegerType
   end
@@ -311,7 +315,7 @@ module Polars
       end
 
       if categories.empty?
-        self.categories = Series.new("category", [], dtype: String)
+        @categories = Series.new("category", [], dtype: String)
         return
       end
 

data/lib/polars/functions/aggregation/horizontal.rb

@@ -143,6 +143,9 @@ module Polars
     # @param exprs [Array]
     #   Column(s) to use in the aggregation. Accepts expression input. Strings are
     #   parsed as column names, other non-expression inputs are parsed as literals.
+    # @param ignore_nulls [Boolean]
+    #   Ignore null values (default).
+    #   If set to `false`, any null value in the input will lead to a null output.
     #
     # @return [Expr]
     #
@@ -166,9 +169,9 @@ module Polars
     #   # │ 8   ┆ 5    ┆ y   ┆ 13  │
     #   # │ 3   ┆ null ┆ z   ┆ 3   │
     #   # └─────┴──────┴─────┴─────┘
-    def sum_horizontal(*exprs)
+    def sum_horizontal(*exprs, ignore_nulls: true)
       rbexprs = Utils.parse_into_list_of_expressions(*exprs)
-      Utils.wrap_expr(Plr.sum_horizontal(rbexprs))
+      Utils.wrap_expr(Plr.sum_horizontal(rbexprs, ignore_nulls))
     end
 
     # Compute the mean of all values horizontally across columns.
@@ -176,6 +179,9 @@ module Polars
     # @param exprs [Array]
     #   Column(s) to use in the aggregation. Accepts expression input. Strings are
     #   parsed as column names, other non-expression inputs are parsed as literals.
+    # @param ignore_nulls [Boolean]
+    #   Ignore null values (default).
+    #   If set to `false`, any null value in the input will lead to a null output.
     #
     # @return [Expr]
     #
@@ -199,9 +205,9 @@ module Polars
     #   # │ 8   ┆ 5    ┆ y   ┆ 6.5  │
     #   # │ 3   ┆ null ┆ z   ┆ 3.0  │
     #   # └─────┴──────┴─────┴──────┘
-    def mean_horizontal(*exprs)
+    def mean_horizontal(*exprs, ignore_nulls: true)
       rbexprs = Utils.parse_into_list_of_expressions(*exprs)
-      Utils.wrap_expr(Plr.mean_horizontal(rbexprs))
+      Utils.wrap_expr(Plr.mean_horizontal(rbexprs, ignore_nulls))
     end
 
     # Cumulatively sum all values horizontally across columns.

data/lib/polars/functions/lazy.rb

@@ -729,16 +729,20 @@ module Polars
       a,
       b,
       method: "pearson",
-      ddof: 1,
+      ddof: nil,
       propagate_nans: false
     )
+      if !ddof.nil?
+        warn "The `ddof` parameter has no effect. Do not use it."
+      end
+
       a = Utils.parse_into_expression(a)
       b = Utils.parse_into_expression(b)
 
       if method == "pearson"
-        Utils.wrap_expr(Plr.pearson_corr(a, b, ddof))
+        Utils.wrap_expr(Plr.pearson_corr(a, b))
       elsif method == "spearman"
-        Utils.wrap_expr(Plr.spearman_rank_corr(a, b, ddof, propagate_nans))
+        Utils.wrap_expr(Plr.spearman_rank_corr(a, b, propagate_nans))
       else
         msg = "method must be one of {{'pearson', 'spearman'}}, got #{method}"
         raise ArgumentError, msg

data/lib/polars/io/delta.rb

@@ -0,0 +1,126 @@
+module Polars
+  module IO
+    # Reads into a DataFrame from a Delta lake table.
+    #
+    # @param source [Object]
+    #   DeltaTable or a Path or URI to the root of the Delta lake table.
+    # @param version [Object]
+    #   Numerical version or timestamp version of the Delta lake table.
+    # @param columns [Array]
+    #   Columns to select. Accepts a list of column names.
+    # @param rechunk [Boolean]
+    #   Make sure that all columns are contiguous in memory by
+    #   aggregating the chunks into a single array.
+    # @param storage_options [Hash]
+    #   Extra options for the storage backends supported by `deltalake-rb`.
+    # @param delta_table_options [Hash]
+    #   Additional keyword arguments while reading a Delta lake Table.
+    #
+    # @return [DataFrame]
+    def read_delta(
+      source,
+      version: nil,
+      columns: nil,
+      rechunk: false,
+      storage_options: nil,
+      delta_table_options: nil
+    )
+      dl_tbl =
+        _get_delta_lake_table(
+          source,
+          version: version,
+          storage_options: storage_options,
+          delta_table_options: delta_table_options
+        )
+
+      dl_tbl.to_polars(columns: columns, rechunk: rechunk)
+    end
+
+    # Lazily read from a Delta lake table.
+    #
+    # @param source [Object]
+    #   DeltaTable or a Path or URI to the root of the Delta lake table.
+    # @param version [Object]
+    #   Numerical version or timestamp version of the Delta lake table.
+    # @param storage_options [Hash]
+    #   Extra options for the storage backends supported by `deltalake-rb`.
+    # @param delta_table_options [Hash]
+    #   Additional keyword arguments while reading a Delta lake Table.
+    #
+    # @return [LazyFrame]
+    def scan_delta(
+      source,
+      version: nil,
+      storage_options: nil,
+      delta_table_options: nil
+    )
+      dl_tbl =
+        _get_delta_lake_table(
+          source,
+          version: version,
+          storage_options: storage_options,
+          delta_table_options: delta_table_options
+        )
+
+      dl_tbl.to_polars(eager: false)
+    end
+
+    private
+
+    def _resolve_delta_lake_uri(table_uri, strict: true)
+      require "uri"
+
+      parsed_result = URI(table_uri)
+
+      resolved_uri =
+        if parsed_result.scheme == ""
+          Utils.normalize_filepath(table_uri)
+        else
+          table_uri
+        end
+
+      resolved_uri
+    end
+
+    def _get_delta_lake_table(
+      table_path,
+      version: nil,
+      storage_options: nil,
+      delta_table_options: nil
+    )
+      _check_if_delta_available
+
+      if table_path.is_a?(DeltaLake::Table)
+        return table_path
+      end
+      delta_table_options ||= {}
+      resolved_uri = _resolve_delta_lake_uri(table_path)
+      if !version.is_a?(::String) && !version.is_a?(::Time)
+        dl_tbl =
+          DeltaLake::Table.new(
+            resolved_uri,
+            version: version,
+            storage_options: storage_options,
+            **delta_table_options
+          )
+      else
+        dl_tbl =
+          DeltaLake::Table.new(
+            resolved_uri,
+            storage_options: storage_options,
+            **delta_table_options
+          )
+        dl_tbl.load_as_version(version)
+      end
+
+      dl_tbl = DeltaLake::Table.new(table_path)
+      dl_tbl
+    end
+
+    def _check_if_delta_available
+      if !defined?(DeltaLake)
+        raise Error, "Delta Lake not available"
+      end
+    end
+  end
+end