polars-df 0.26.1 → 0.27.0

data/lib/polars/iceberg_dataset.rb

@@ -17,159 +17,33 @@ module Polars
 
       table = scan.table
       snapshot = scan.snapshot
-      schema = snapshot ? table.schema_by_id(snapshot.is_a?(Hash) ? snapshot[:schema_id] : snapshot.schema_id) : table.current_schema
-
-      if files.empty? && !schema.respond_to?(:arrow_c_schema)
-        # TODO remove in 0.27.0
-        schema =
-          schema.fields.to_h do |field|
-            dtype =
-              case field[:type]
-              when "unknown"
-                Unknown
-              when "boolean"
-                Boolean
-              when "int"
-                Int32
-              when "long"
-                Int64
-              when "float"
-                Float32
-              when "double"
-                Float64
-              when "decimal"
-                Decimal.new(field[:precision], field[:scale])
-              when "string"
-                String
-              when "uuid"
-                Binary
-              when "fixed"
-                Binary
-              when "binary"
-                Binary
-              when "date"
-                Date
-              when "time"
-                Time # ns instead of us since does not support time unit
-              when "timestamp"
-                Datetime.new("us")
-              when "timestamp_ns"
-                Datetime.new("ns")
-              when "timestamptz"
-                Datetime.new("us", "+00:00")
-              when "timestamptz_ns"
-                Datetime.new("ns", "+00:00")
-              else
-                raise Todo
-              end
-
-            [field[:name], dtype]
-          end
-
-        LazyFrame.new(schema: schema)
-      else
-        sources = files.map { |v| v.is_a?(Hash) ? v[:data_file_path] : v.file.file_path }
-
-        column_mapping = [
-          "iceberg-column-mapping",
-          schema.respond_to?(:arrow_c_schema) ? schema : arrow_schema(schema)
-        ]
-
-        deletion_files = [
-          "iceberg-position-delete",
-          files.map.with_index
-            .select { |v, i| (v.is_a?(Hash) ? v[:deletes] : v.delete_files).any? }
-            .to_h { |v, i| [i, v.is_a?(Hash) ? v[:deletes].map { |d| d[:file_path] } : v.delete_files.map(&:file_path)] }
-        ]
-
-        scan_options = {
-          schema: schema.respond_to?(:arrow_c_schema) ? Schema.new(schema) : nil,
-          cast_options: Polars::ScanCastOptions._default_iceberg,
-          missing_columns: "insert",
-          extra_columns: "ignore",
-          storage_options: @storage_options,
-          _column_mapping: column_mapping,
-          _deletion_files: deletion_files
-        }
-
-        Polars.scan_parquet(sources, **scan_options)
-      end
-    end
-
-    private
-
-    # TODO remove in 0.27.0
-    def arrow_schema(schema)
-      fields =
-        schema.fields.map do |field|
-          type =
-            case field[:type]
-            when "unknown"
-              "unknown"
-            when "boolean"
-              "boolean"
-            when "int"
-              "int32"
-            when "long"
-              "int64"
-            when "float"
-              "float32"
-            when "double"
-              "float64"
-            when "decimal"
-              "decimal"
-            when "string"
-              "string"
-            when "uuid"
-              limit = 16
-              "fixed_size_binary"
-            when "fixed"
-              limit = field[:limit]
-              "fixed_size_binary"
-            when "binary"
-              "large_binary"
-            when "date"
-              "date32"
-            when "time"
-              time_unit = "us"
-              "time64"
-            when "timestamp"
-              time_unit = "us"
-              "timestamp"
-            when "timestamp_ns"
-              time_unit = "ns"
-              "timestamp"
-            when "timestamptz"
-              time_unit = "us"
-              time_zone = "+00:00"
-              "timestamp"
-            when "timestamptz_ns"
-              time_unit = "ns"
-              time_zone = "+00:00"
-              "timestamp"
-            else
-              raise Todo
-            end
-
-          arrow_field = {
-            name: field[:name],
-            type: type,
-            nullable: !field[:required],
-            metadata: {
-              "PARQUET:field_id" => field[:id].to_s
-            }
-          }
-          arrow_field[:limit] = limit if limit
-          if type == "decimal"
-            arrow_field[:precision] = field[:precision]
-            arrow_field[:scale] = field[:scale]
-          end
-          arrow_field[:time_unit] = time_unit if time_unit
-          arrow_field[:time_zone] = time_zone if time_zone
-          arrow_field
-        end
-
-      {fields: fields}
+      schema = snapshot ? table.schema_by_id(snapshot.schema_id) : table.current_schema
+
+      sources = files.map { |v| v.file.file_path }
+
+      column_mapping = [
+        "iceberg-column-mapping",
+        schema
+      ]
+
+      deletion_files = [
+        "iceberg-position-delete",
+        files.map.with_index
+          .select { |v, i| v.delete_files.any? }
+          .to_h { |v, i| [i, v.delete_files.map(&:file_path)] }
+      ]
+
+      scan_options = {
+        schema: Schema.new(schema),
+        cast_options: Polars::ScanCastOptions._default_iceberg,
+        missing_columns: "insert",
+        extra_columns: "ignore",
+        storage_options: @storage_options,
+        _column_mapping: column_mapping,
+        _deletion_files: deletion_files
+      }
+
+      Polars.scan_parquet(sources, **scan_options)
     end
   end
 end

data/lib/polars/lazy_frame.rb

@@ -4490,7 +4490,7 @@ module Polars
     #       "numbers" => [[1], [2, 3], [4, 5], [6, 7, 8]],
     #     }
     #   ).lazy
-    #   df.explode("numbers").collect
+    #   df.explode("numbers", empty_as_null: false).collect
     #   # =>
     #   # shape: (8, 2)
     #   # ┌─────────┬─────────┐
@@ -4510,9 +4510,16 @@ module Polars
     def explode(
       columns,
       *more_columns,
-      empty_as_null: true,
+      empty_as_null: OMITTED,
       keep_nulls: true
     )
+      if empty_as_null == OMITTED
+        Utils.issue_deprecation_warning(
+          "The default behavior for `empty_as_null` will change to `false`. " +
+          "To keep the current behavior, explicitly set `empty_as_null: true`."
+        )
+        empty_as_null = true
+      end
       subset = Utils.parse_list_into_selector(columns) | Utils.parse_list_into_selector(
         more_columns
       )
@@ -5149,7 +5156,8 @@ module Polars
     #   # │ elise  ┆ 44  │
     #   # └────────┴─────┘
     def merge_sorted(other, key, maintain_order: false)
-      _from_rbldf(_ldf.merge_sorted(other._ldf, key, maintain_order))
+      keys = Array(key)
+      _from_rbldf(_ldf.merge_sorted(other._ldf, keys, maintain_order))
     end
 
     # Flag a column as sorted.

data/lib/polars/list_expr.rb

@@ -907,7 +907,7 @@ module Polars
     #
     # @example
     #   df = Polars::DataFrame.new({"a" => [[1, 2, 3], [4, 5, 6]]})
-    #   df.select(Polars.col("a").list.explode)
+    #   df.select(Polars.col("a").list.explode(empty_as_null: false))
     #   # =>
     #   # shape: (6, 1)
     #   # ┌─────┐
@@ -922,7 +922,15 @@ module Polars
     #   # │ 5   │
     #   # │ 6   │
     #   # └─────┘
-    def explode(empty_as_null: true, keep_nulls: true)
+    def explode(empty_as_null: OMITTED, keep_nulls: true)
+      if empty_as_null == OMITTED
+        Utils.issue_deprecation_warning(
+          "The default behavior for `empty_as_null` will change to `false`. " +
+          "To keep the current behavior, explicitly set `empty_as_null: true`."
+        )
+        empty_as_null = true
+      end
+
       Utils.wrap_expr(_rbexpr.explode(empty_as_null, keep_nulls))
     end
 

data/lib/polars/list_name_space.rb

@@ -744,7 +744,7 @@ module Polars
     #
     # @example
     #   s = Polars::Series.new("a", [[1, 2, 3], [4, 5, 6]])
-    #   s.list.explode
+    #   s.list.explode(empty_as_null: false)
     #   # =>
     #   # shape: (6,)
     #   # Series: 'a' [i64]
@@ -756,7 +756,7 @@ module Polars
     #   #         5
     #   #         6
     #   # ]
-    def explode(empty_as_null: true, keep_nulls: true)
+    def explode(empty_as_null: nil, keep_nulls: true)
       super
     end
 

data/lib/polars/series.rb

@@ -2807,7 +2807,7 @@ module Polars
     #
     # @example
     #   s = Polars::Series.new("a", [[1, 2], [3, 4], [9, 10]])
-    #   s.explode
+    #   s.explode(empty_as_null: false)
     #   # =>
     #   # shape: (6,)
     #   # Series: 'a' [i64]
@@ -6118,7 +6118,7 @@ module Polars
     #   # shape: (3,)
     #   # Series: 'a' [f64]
     #   # [
-    #   #         0.0
+    #   #         null
     #   #         0.707107
     #   #         0.963624
     #   # ]
@@ -6146,7 +6146,7 @@ module Polars
     #   # shape: (3,)
     #   # Series: 'a' [f64]
     #   # [
-    #   #         0.0
+    #   #         null
     #   #         0.5
     #   #         0.928571
     #   # ]

data/lib/polars/version.rb

@@ -1,4 +1,4 @@
 module Polars
   # @private
-  VERSION = "0.26.1"
+  VERSION = "0.27.0"
 end

data/lib/polars.rb

@@ -124,18 +124,10 @@ module Polars
   NO_DEFAULT = Object.new
 
   # @private
-  N_INFER_DEFAULT = 100
-
-  # @private
-  # TODO remove in 0.27.0
-  ArrowArrayStream = Capsule
+  OMITTED = Object.new
 
   # @private
-  class ArrowArrayStream
-    def arrow_c_stream
-      self
-    end
-  end
+  N_INFER_DEFAULT = 100
 
   # Return the number of threads in the Polars thread pool.
   #

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: polars-df
 version: !ruby/object:Gem::Version
-  version: 0.26.1
+  version: 0.27.0
 platform: ruby
 authors:
 - Andrew Kane