polars-df 0.15.0 → 0.16.0

data/lib/polars/io/delta.rb

@@ -0,0 +1,126 @@
+module Polars
+  module IO
+    # Reads into a DataFrame from a Delta lake table.
+    #
+    # @param source [Object]
+    #   DeltaTable or a Path or URI to the root of the Delta lake table.
+    # @param version [Object]
+    #   Numerical version or timestamp version of the Delta lake table.
+    # @param columns [Array]
+    #   Columns to select. Accepts a list of column names.
+    # @param rechunk [Boolean]
+    #   Make sure that all columns are contiguous in memory by
+    #   aggregating the chunks into a single array.
+    # @param storage_options [Hash]
+    #   Extra options for the storage backends supported by `deltalake-rb`.
+    # @param delta_table_options [Hash]
+    #   Additional keyword arguments while reading a Delta lake Table.
+    #
+    # @return [DataFrame]
+    def read_delta(
+      source,
+      version: nil,
+      columns: nil,
+      rechunk: false,
+      storage_options: nil,
+      delta_table_options: nil
+    )
+      dl_tbl =
+        _get_delta_lake_table(
+          source,
+          version: version,
+          storage_options: storage_options,
+          delta_table_options: delta_table_options
+        )
+
+      dl_tbl.to_polars(columns: columns, rechunk: rechunk)
+    end
+
+    # Lazily read from a Delta lake table.
+    #
+    # @param source [Object]
+    #   DeltaTable or a Path or URI to the root of the Delta lake table.
+    # @param version [Object]
+    #   Numerical version or timestamp version of the Delta lake table.
+    # @param storage_options [Hash]
+    #   Extra options for the storage backends supported by `deltalake-rb`.
+    # @param delta_table_options [Hash]
+    #   Additional keyword arguments while reading a Delta lake Table.
+    #
+    # @return [LazyFrame]
+    def scan_delta(
+      source,
+      version: nil,
+      storage_options: nil,
+      delta_table_options: nil
+    )
+      dl_tbl =
+        _get_delta_lake_table(
+          source,
+          version: version,
+          storage_options: storage_options,
+          delta_table_options: delta_table_options
+        )
+
+      dl_tbl.to_polars(eager: false)
+    end
+
+    private
+
+    def _resolve_delta_lake_uri(table_uri, strict: true)
+      require "uri"
+
+      parsed_result = URI(table_uri)
+
+      resolved_uri =
+        if parsed_result.scheme == ""
+          Utils.normalize_filepath(table_uri)
+        else
+          table_uri
+        end
+
+      resolved_uri
+    end
+
+    def _get_delta_lake_table(
+      table_path,
+      version: nil,
+      storage_options: nil,
+      delta_table_options: nil
+    )
+      _check_if_delta_available
+
+      if table_path.is_a?(DeltaLake::Table)
+        return table_path
+      end
+      delta_table_options ||= {}
+      resolved_uri = _resolve_delta_lake_uri(table_path)
+      if !version.is_a?(::String) && !version.is_a?(::Time)
+        dl_tbl =
+          DeltaLake::Table.new(
+            resolved_uri,
+            version: version,
+            storage_options: storage_options,
+            **delta_table_options
+          )
+      else
+        dl_tbl =
+          DeltaLake::Table.new(
+            resolved_uri,
+            storage_options: storage_options,
+            **delta_table_options
+          )
+        dl_tbl.load_as_version(version)
+      end
+
+      dl_tbl = DeltaLake::Table.new(table_path)
+      dl_tbl
+    end
+
+    def _check_if_delta_available
+      if !defined?(DeltaLake)
+        raise Error, "Delta Lake not available"
+      end
+    end
+  end
+end

data/lib/polars/lazy_frame.rb

@@ -431,7 +431,9 @@ module Polars
       projection_pushdown: true,
       simplify_expression: true,
       no_optimization: false,
-      slice_pushdown: true
+      slice_pushdown: true,
+      storage_options: nil,
+      retries: 2
     )
       lf = _set_sink_optimizations(
         type_coercion: type_coercion,
@@ -460,6 +462,12 @@ module Polars
         }
       end
 
+      if storage_options&.any?
+        storage_options = storage_options.to_a
+      else
+        storage_options = nil
+      end
+
       lf.sink_parquet(
         path,
         compression,
@@ -467,7 +475,9 @@ module Polars
         statistics,
         row_group_size,
         data_pagesize_limit,
-        maintain_order
+        maintain_order,
+        storage_options,
+        retries
       )
     end
 
@@ -512,6 +522,10 @@ module Polars
       slice_pushdown: true,
       no_optimization: false
     )
+      # TODO support storage options in Rust
+      storage_options = nil
+      retries = 2
+
       lf = _set_sink_optimizations(
         type_coercion: type_coercion,
         predicate_pushdown: predicate_pushdown,
@@ -521,10 +535,18 @@ module Polars
         no_optimization: no_optimization
       )
 
+      if storage_options&.any?
+        storage_options = storage_options.to_a
+      else
+        storage_options = nil
+      end
+
       lf.sink_ipc(
         path,
         compression,
-        maintain_order
+        maintain_order,
+        storage_options,
+        retries
       )
     end
 
@@ -692,7 +714,9 @@ module Polars
       projection_pushdown: true,
       simplify_expression: true,
       slice_pushdown: true,
-      no_optimization: false
+      no_optimization: false,
+      storage_options: nil,
+      retries: 2
     )
       lf = _set_sink_optimizations(
         type_coercion: type_coercion,
@@ -703,7 +727,13 @@ module Polars
         no_optimization: no_optimization
       )
 
-      lf.sink_json(path, maintain_order)
+      if storage_options&.any?
+        storage_options = storage_options.to_a
+      else
+        storage_options = nil
+      end
+
+      lf.sink_json(path, maintain_order, storage_options, retries)
     end
 
     # @private

data/lib/polars/selectors.rb

@@ -372,9 +372,91 @@ module Polars
     # def by_index
     # end
 
-    # TODO
-    # def by_name
-    # end
+    # Select all columns matching the given names.
+    #
+    # @param names [Array]
+    #   One or more names of columns to select.
+    # @param require_all [Boolean]
+    #   Whether to match *all* names (the default) or *any* of the names.
+    #
+    # @return [SelectorProxy]
+    #
+    # @note
+    #   Matching columns are returned in the order in which they are declared in
+    #   the selector, not the underlying schema order.
+    #
+    # @example
+    #   df = Polars::DataFrame.new(
+    #     {
+    #       "foo" => ["x", "y"],
+    #       "bar" => [123, 456],
+    #       "baz" => [2.0, 5.5],
+    #       "zap" => [false, true]
+    #     }
+    #   )
+    #
+    # @example Select columns by name:
+    #   df.select(Polars.cs.by_name("foo", "bar"))
+    #   # =>
+    #   # shape: (2, 2)
+    #   # ┌─────┬─────┐
+    #   # │ foo ┆ bar │
+    #   # │ --- ┆ --- │
+    #   # │ str ┆ i64 │
+    #   # ╞═════╪═════╡
+    #   # │ x   ┆ 123 │
+    #   # │ y   ┆ 456 │
+    #   # └─────┴─────┘
+    #
+    # @example Match *any* of the given columns by name:
+    #   df.select(Polars.cs.by_name("baz", "moose", "foo", "bear", require_all: false))
+    #   # =>
+    #   # shape: (2, 2)
+    #   # ┌─────┬─────┐
+    #   # │ foo ┆ baz │
+    #   # │ --- ┆ --- │
+    #   # │ str ┆ f64 │
+    #   # ╞═════╪═════╡
+    #   # │ x   ┆ 2.0 │
+    #   # │ y   ┆ 5.5 │
+    #   # └─────┴─────┘
+    #
+    # @example Match all columns *except* for those given:
+    #   df.select(~Polars.cs.by_name("foo", "bar"))
+    #   # =>
+    #   # shape: (2, 2)
+    #   # ┌─────┬───────┐
+    #   # │ baz ┆ zap   │
+    #   # │ --- ┆ ---   │
+    #   # │ f64 ┆ bool  │
+    #   # ╞═════╪═══════╡
+    #   # │ 2.0 ┆ false │
+    #   # │ 5.5 ┆ true  │
+    #   # └─────┴───────┘
+    def self.by_name(*names, require_all: true)
+      all_names = []
+      names.each do |nm|
+        if nm.is_a?(::String)
+          all_names << nm
+        else
+          msg = "invalid name: #{nm.inspect}"
+          raise TypeError, msg
+        end
+      end
+
+      selector_params = {"*names" => all_names}
+      match_cols = all_names
+      if !require_all
+        match_cols = "^(#{all_names.map { |nm| Utils.re_escape(nm) }.join("|")})$"
+        selector_params["require_all"] = require_all
+      end
+
+      _selector_proxy_(
+        F.col(match_cols),
+        name: "by_name",
+        parameters: selector_params
+      )
+    end
 
     # Select all categorical columns.
     #

data/lib/polars/version.rb

@@ -1,4 +1,4 @@
 module Polars
   # @private
-  VERSION = "0.15.0"
+  VERSION = "0.16.0"
 end

data/lib/polars.rb

@@ -49,6 +49,7 @@ require_relative "polars/group_by"
 require_relative "polars/io/avro"
 require_relative "polars/io/csv"
 require_relative "polars/io/database"
+require_relative "polars/io/delta"
 require_relative "polars/io/ipc"
 require_relative "polars/io/json"
 require_relative "polars/io/ndjson"
@@ -89,4 +90,18 @@ module Polars
 
   # @private
   N_INFER_DEFAULT = 100
+
+  # @private
+  class ArrowArrayStream
+    def arrow_c_stream
+      self
+    end
+  end
+
+  # Return the number of threads in the Polars thread pool.
+  #
+  # @return [Integer]
+  def self.thread_pool_size
+    Plr.thread_pool_size
+  end
 end

metadata

@@ -1,14 +1,13 @@
 --- !ruby/object:Gem::Specification
 name: polars-df
 version: !ruby/object:Gem::Version
-  version: 0.15.0
+  version: 0.16.0
 platform: ruby
 authors:
 - Andrew Kane
-autorequire:
 bindir: bin
 cert_chain: []
-date: 2024-11-20 00:00:00.000000000 Z
+date: 2024-12-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bigdecimal
@@ -38,7 +37,6 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-description:
 email: andrew@ankane.org
 executables: []
 extensions:
@@ -160,6 +158,7 @@ files:
 - lib/polars/io/avro.rb
 - lib/polars/io/csv.rb
 - lib/polars/io/database.rb
+- lib/polars/io/delta.rb
 - lib/polars/io/ipc.rb
 - lib/polars/io/json.rb
 - lib/polars/io/ndjson.rb
@@ -194,7 +193,6 @@ homepage: https://github.com/ankane/ruby-polars
 licenses:
 - MIT
 metadata: {}
-post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -202,15 +200,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '3.1'
+      version: '3.2'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.5.22
-signing_key:
+rubygems_version: 3.6.2
 specification_version: 4
 summary: Blazingly fast DataFrames for Ruby
 test_files: []