polars-df 0.26.0 → 0.27.0

data/lib/polars/iceberg_dataset.rb

@@ -17,92 +17,33 @@ module Polars
 
       table = scan.table
       snapshot = scan.snapshot
-      schema = snapshot ? table.schema_by_id(snapshot[:schema_id]) : table.current_schema
-
-      if files.empty?
-        # TODO improve
-        schema =
-          schema.fields.to_h do |field|
-            dtype =
-              case field[:type]
-              when "int"
-                Polars::Int32
-              when "long"
-                Polars::Int64
-              when "double"
-                Polars::Float64
-              when "string"
-                Polars::String
-              when "timestamp"
-                Polars::Datetime
-              else
-                raise Todo
-              end
-
-            [field[:name], dtype]
-          end
-
-        LazyFrame.new(schema: schema)
-      else
-        sources = files.map { |v| v[:data_file_path] }
-
-        column_mapping = [
-          "iceberg-column-mapping",
-          arrow_schema(schema)
-        ]
-
-        deletion_files = [
-          "iceberg-position-delete",
-          files.map.with_index
-            .select { |v, i| v[:deletes].any? }
-            .to_h { |v, i| [i, v[:deletes].map { |d| d[:file_path] }] }
-        ]
-
-        scan_options = {
-          storage_options: @storage_options,
-          cast_options: Polars::ScanCastOptions._default_iceberg,
-          missing_columns: "insert",
-          extra_columns: "ignore",
-          _column_mapping: column_mapping,
-          _deletion_files: deletion_files
-        }
-
-        Polars.scan_parquet(sources, **scan_options)
-      end
-    end
-
-    private
-
-    def arrow_schema(schema)
-      fields =
-        schema.fields.map do |field|
-          type =
-            case field[:type]
-            when "boolean"
-              "boolean"
-            when "int"
-              "int32"
-            when "long"
-              "int64"
-            when "float"
-              "float32"
-            when "double"
-              "float64"
-            else
-              raise Todo
-            end
-
-          {
-            name: field[:name],
-            type: type,
-            nullable: !field[:required],
-            metadata: {
-              "PARQUET:field_id" => field[:id].to_s
-            }
-          }
-        end
-
-      {fields: fields}
+      schema = snapshot ? table.schema_by_id(snapshot.schema_id) : table.current_schema
+
+      sources = files.map { |v| v.file.file_path }
+
+      column_mapping = [
+        "iceberg-column-mapping",
+        schema
+      ]
+
+      deletion_files = [
+        "iceberg-position-delete",
+        files.map.with_index
+          .select { |v, i| v.delete_files.any? }
+          .to_h { |v, i| [i, v.delete_files.map(&:file_path)] }
+      ]
+
+      scan_options = {
+        schema: Schema.new(schema),
+        cast_options: Polars::ScanCastOptions._default_iceberg,
+        missing_columns: "insert",
+        extra_columns: "ignore",
+        storage_options: @storage_options,
+        _column_mapping: column_mapping,
+        _deletion_files: deletion_files
+      }
+
+      Polars.scan_parquet(sources, **scan_options)
     end
   end
 end

data/lib/polars/io/delta.rb

@@ -25,6 +25,8 @@ module Polars
       storage_options: nil,
       delta_table_options: nil
     )
+      require "deltalake-rb"
+
       df =
         scan_delta(
           source,
@@ -62,6 +64,8 @@ module Polars
       delta_table_options: nil,
       rechunk: nil
     )
+      require "deltalake-rb"
+
       dl_tbl =
         _get_delta_lake_table(
           source,

data/lib/polars/lazy_frame.rb

@@ -4490,7 +4490,7 @@ module Polars
     #       "numbers" => [[1], [2, 3], [4, 5], [6, 7, 8]],
     #     }
     #   ).lazy
-    #   df.explode("numbers").collect
+    #   df.explode("numbers", empty_as_null: false).collect
     #   # =>
     #   # shape: (8, 2)
     #   # ┌─────────┬─────────┐
@@ -4510,9 +4510,16 @@ module Polars
     def explode(
       columns,
       *more_columns,
-      empty_as_null: true,
+      empty_as_null: OMITTED,
       keep_nulls: true
     )
+      if empty_as_null == OMITTED
+        Utils.issue_deprecation_warning(
+          "The default behavior for `empty_as_null` will change to `false`. " +
+          "To keep the current behavior, explicitly set `empty_as_null: true`."
+        )
+        empty_as_null = true
+      end
       subset = Utils.parse_list_into_selector(columns) | Utils.parse_list_into_selector(
         more_columns
       )
@@ -5149,7 +5156,8 @@ module Polars
     #   # │ elise  ┆ 44  │
     #   # └────────┴─────┘
     def merge_sorted(other, key, maintain_order: false)
-      _from_rbldf(_ldf.merge_sorted(other._ldf, key, maintain_order))
+      keys = Array(key)
+      _from_rbldf(_ldf.merge_sorted(other._ldf, keys, maintain_order))
     end
 
     # Flag a column as sorted.

data/lib/polars/list_expr.rb

@@ -907,7 +907,7 @@ module Polars
     #
     # @example
     #   df = Polars::DataFrame.new({"a" => [[1, 2, 3], [4, 5, 6]]})
-    #   df.select(Polars.col("a").list.explode)
+    #   df.select(Polars.col("a").list.explode(empty_as_null: false))
     #   # =>
     #   # shape: (6, 1)
     #   # ┌─────┐
@@ -922,7 +922,15 @@ module Polars
     #   # │ 5   │
     #   # │ 6   │
     #   # └─────┘
-    def explode(empty_as_null: true, keep_nulls: true)
+    def explode(empty_as_null: OMITTED, keep_nulls: true)
+      if empty_as_null == OMITTED
+        Utils.issue_deprecation_warning(
+          "The default behavior for `empty_as_null` will change to `false`. " +
+          "To keep the current behavior, explicitly set `empty_as_null: true`."
+        )
+        empty_as_null = true
+      end
+
       Utils.wrap_expr(_rbexpr.explode(empty_as_null, keep_nulls))
     end
 

data/lib/polars/list_name_space.rb

@@ -744,7 +744,7 @@ module Polars
     #
     # @example
     #   s = Polars::Series.new("a", [[1, 2, 3], [4, 5, 6]])
-    #   s.list.explode
+    #   s.list.explode(empty_as_null: false)
     #   # =>
     #   # shape: (6,)
     #   # Series: 'a' [i64]
@@ -756,7 +756,7 @@ module Polars
     #   #         5
     #   #         6
     #   # ]
-    def explode(empty_as_null: true, keep_nulls: true)
+    def explode(empty_as_null: nil, keep_nulls: true)
       super
     end
 

data/lib/polars/schema.rb

@@ -71,6 +71,15 @@ module Polars
       @schema.values
     end
 
+    # Convert the schema to a nanoarrow schema.
+    #
+    # @return [Nanoarrow::Schema]
+    def to_arrow
+      require "nanoarrow"
+
+      Nanoarrow::Schema.new(self)
+    end
+
     # Get the number of schema entries.
     #
     # @return [Integer]

data/lib/polars/series.rb

@@ -73,6 +73,8 @@ module Polars
         self._s = Utils.dataframe_to_rbseries(
           original_name, values, dtype: dtype, strict: strict
         )
+      elsif values.respond_to?(:arrow_c_stream)
+        self._s = RbSeries.from_arrow_c_stream(values)
       else
         raise TypeError, "Series constructor called with unsupported type; got #{values.class.name}"
       end
@@ -496,6 +498,11 @@ module Polars
       end
     end
 
+    # @private
+    def arrow_c_stream
+      _s.arrow_c_stream
+    end
+
     # Return the Series as a scalar, or return the element at the given index.
     #
     # If no index is provided, this is equivalent to `s[0]`, with a check
@@ -2800,7 +2807,7 @@ module Polars
     #
     # @example
     #   s = Polars::Series.new("a", [[1, 2], [3, 4], [9, 10]])
-    #   s.explode
+    #   s.explode(empty_as_null: false)
     #   # =>
     #   # shape: (6,)
     #   # Series: 'a' [i64]
@@ -3114,6 +3121,8 @@ module Polars
     #   # Numo::Int64#shape=[3]
     #   # [1, 2, 3]
     def to_numo
+      require "numo/narray"
+
       if dtype.temporal?
         Numo::RObject.cast(to_a)
       else
@@ -3121,6 +3130,15 @@ module Polars
       end
     end
 
+    # Return the underlying Arrow array.
+    #
+    # @return [Nanoarrow::Array]
+    def to_arrow
+      require "nanoarrow"
+
+      Nanoarrow::Array.new(self)
+    end
+
     # Set masked values.
     #
     # @param filter [Series]
@@ -6100,7 +6118,7 @@ module Polars
     #   # shape: (3,)
     #   # Series: 'a' [f64]
     #   # [
-    #   #         0.0
+    #   #         null
     #   #         0.707107
     #   #         0.963624
     #   # ]
@@ -6128,7 +6146,7 @@ module Polars
     #   # shape: (3,)
     #   # Series: 'a' [f64]
     #   # [
-    #   #         0.0
+    #   #         null
     #   #         0.5
     #   #         0.928571
     #   # ]

data/lib/polars/version.rb

@@ -1,4 +1,4 @@
 module Polars
   # @private
-  VERSION = "0.26.0"
+  VERSION = "0.27.0"
 end

data/lib/polars.rb

@@ -124,14 +124,10 @@ module Polars
   NO_DEFAULT = Object.new
 
   # @private
-  N_INFER_DEFAULT = 100
+  OMITTED = Object.new
 
   # @private
-  class ArrowArrayStream
-    def arrow_c_stream
-      self
-    end
-  end
+  N_INFER_DEFAULT = 100
 
   # Return the number of threads in the Polars thread pool.
   #

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: polars-df
 version: !ruby/object:Gem::Version
-  version: 0.26.0
+  version: 0.27.0
 platform: ruby
 authors:
 - Andrew Kane
@@ -113,6 +113,7 @@ files:
 - ext/polars/src/on_startup.rs
 - ext/polars/src/prelude.rs
 - ext/polars/src/rb_modules.rs
+- ext/polars/src/ruby/capsule.rs
 - ext/polars/src/ruby/exceptions.rs
 - ext/polars/src/ruby/gvl.rs
 - ext/polars/src/ruby/lazy.rs
@@ -262,7 +263,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 4.0.13
+rubygems_version: 4.0.14
 specification_version: 4
 summary: Blazingly fast DataFrames for Ruby
 test_files: []