polars-df 0.22.0-x86_64-linux-musl → 0.23.0-x86_64-linux-musl

data/README.md

@@ -400,13 +400,14 @@ Polars::DataFrame.new(data, schema: {"a" => Polars::Int32, "b" => Polars::Float3
 Supported types are:
 
 - boolean - `Boolean`
-- float - `Float64`, `Float32`
-- integer - `Int64`, `Int32`, `Int16`, `Int8`
-- unsigned integer - `UInt64`, `UInt32`, `UInt16`, `UInt8`
-- string - `String`, `Binary`, `Categorical`
-- temporal - `Date`, `Datetime`, `Time`, `Duration`
-- nested -  `List`, `Struct`, `Array`
-- other - `Object`, `Null`
+- decimal - `Decimal`
+- float - `Float32`, `Float64`
+- integer - `Int8`, `Int16`, `Int32`, `Int64`, `Int128`
+- unsigned integer - `UInt8`, `UInt16`, `UInt32`, `UInt64`, `UInt128`
+- string - `String`, `Categorical`, `Enum`
+- temporal - `Date`, `Datetime`, `Duration`, `Time`
+- nested - `Array`, `List`, `Struct`
+- other - `Binary`, `Object`, `Null`, `Unknown`
 
 Get column types
 

data/lib/polars/data_frame.rb

@@ -75,8 +75,8 @@ module Polars
     # Read a serialized DataFrame from a file.
     #
     # @param source [Object]
-    #     Path to a file or a file-like object (by file-like object, we refer to
-    #     objects that have a `read` method, such as a file handler or `StringIO`).
+    #   Path to a file or a file-like object (by file-like object, we refer to
+    #   objects that have a `read` method, such as a file handler or `StringIO`).
     #
     # @return [DataFrame]
     #
@@ -6059,8 +6059,13 @@ module Polars
     # The fields will be inserted into the `DataFrame` on the location of the
     # `struct` type.
     #
-    # @param names [Object]
-    #  Names of the struct columns that will be decomposed by its fields
+    # @param columns [Object]
+    #   Name of the struct column(s) that should be unnested.
+    # @param more_columns [Array]
+    #   Additional columns to unnest, specified as positional arguments.
+    # @param separator [String]
+    #   Rename output column names as combination of the struct column name,
+    #   name separator and field name.
     #
     # @return [DataFrame]
     #
@@ -6086,11 +6091,8 @@ module Polars
     #   # │ foo    ┆ 1   ┆ a   ┆ true ┆ [1, 2]    ┆ baz   │
     #   # │ bar    ┆ 2   ┆ b   ┆ null ┆ [3]       ┆ womp  │
     #   # └────────┴─────┴─────┴──────┴───────────┴───────┘
-    def unnest(names)
-      if names.is_a?(::String)
-        names = [names]
-      end
-      _from_rbdf(_df.unnest(names))
+    def unnest(columns, *more_columns, separator: nil)
+      lazy.unnest(columns, *more_columns, separator: separator).collect(_eager: true)
     end
 
     # Requires NumPy

data/lib/polars/data_types.rb

@@ -209,6 +209,10 @@ module Polars
   class UInt64 < UnsignedIntegerType
   end
 
+  # 128-bit unsigned integer type.
+  class UInt128 < UnsignedIntegerType
+  end
+
   # 32-bit floating point type.
   class Float32 < FloatType
   end
@@ -223,7 +227,11 @@ module Polars
   class Decimal < NumericType
     attr_reader :precision, :scale
 
-    def initialize(precision, scale)
+    def initialize(precision = nil, scale = 0)
+      if precision.nil?
+        precision = 38
+      end
+
       @precision = precision
       @scale = scale
     end

data/lib/polars/date_time_expr.rb

@@ -1435,6 +1435,9 @@ module Polars
 
     # Extract the days from a Duration type.
     #
+    # @param fractional [Boolean]
+    #   Whether to include the fractional component of the second.
+    #
     # @return [Expr]
     #
     # @example
@@ -1462,13 +1465,16 @@ module Polars
     #   # │ 2020-04-01 00:00:00 ┆ 31        │
     #   # │ 2020-05-01 00:00:00 ┆ 30        │
     #   # └─────────────────────┴───────────┘
-    def total_days
-      Utils.wrap_expr(_rbexpr.dt_total_days)
+    def total_days(fractional: false)
+      Utils.wrap_expr(_rbexpr.dt_total_days(fractional))
     end
     alias_method :days, :total_days
 
     # Extract the hours from a Duration type.
     #
+    # @param fractional [Boolean]
+    #   Whether to include the fractional component of the second.
+    #
     # @return [Expr]
     #
     # @example
@@ -1497,13 +1503,16 @@ module Polars
     #   # │ 2020-01-03 00:00:00 ┆ 24         │
     #   # │ 2020-01-04 00:00:00 ┆ 24         │
     #   # └─────────────────────┴────────────┘
-    def total_hours
-      Utils.wrap_expr(_rbexpr.dt_total_hours)
+    def total_hours(fractional: false)
+      Utils.wrap_expr(_rbexpr.dt_total_hours(fractional))
     end
     alias_method :hours, :total_hours
 
     # Extract the minutes from a Duration type.
     #
+    # @param fractional [Boolean]
+    #   Whether to include the fractional component of the second.
+    #
     # @return [Expr]
     #
     # @example
@@ -1532,13 +1541,16 @@ module Polars
     #   # │ 2020-01-03 00:00:00 ┆ 1440         │
     #   # │ 2020-01-04 00:00:00 ┆ 1440         │
     #   # └─────────────────────┴──────────────┘
-    def total_minutes
-      Utils.wrap_expr(_rbexpr.dt_total_minutes)
+    def total_minutes(fractional: false)
+      Utils.wrap_expr(_rbexpr.dt_total_minutes(fractional))
     end
     alias_method :minutes, :total_minutes
 
     # Extract the seconds from a Duration type.
     #
+    # @param fractional [Boolean]
+    #   Whether to include the fractional component of the second.
+    #
     # @return [Expr]
     #
     # @example
@@ -1568,13 +1580,16 @@ module Polars
     #   # │ 2020-01-01 00:03:00 ┆ 60           │
     #   # │ 2020-01-01 00:04:00 ┆ 60           │
     #   # └─────────────────────┴──────────────┘
-    def total_seconds
-      Utils.wrap_expr(_rbexpr.dt_total_seconds)
+    def total_seconds(fractional: false)
+      Utils.wrap_expr(_rbexpr.dt_total_seconds(fractional))
     end
     alias_method :seconds, :total_seconds
 
     # Extract the milliseconds from a Duration type.
     #
+    # @param fractional [Boolean]
+    #   Whether to include the fractional component of the second.
+    #
     # @return [Expr]
     #
     # @example
@@ -1610,13 +1625,16 @@ module Polars
     #   # │ 2020-01-01 00:00:00.999 ┆ 1                 │
     #   # │ 2020-01-01 00:00:01     ┆ 1                 │
     #   # └─────────────────────────┴───────────────────┘
-    def total_milliseconds
-      Utils.wrap_expr(_rbexpr.dt_total_milliseconds)
+    def total_milliseconds(fractional: false)
+      Utils.wrap_expr(_rbexpr.dt_total_milliseconds(fractional))
     end
     alias_method :milliseconds, :total_milliseconds
 
     # Extract the microseconds from a Duration type.
     #
+    # @param fractional [Boolean]
+    #   Whether to include the fractional component of the second.
+    #
     # @return [Expr]
     #
     # @example
@@ -1652,13 +1670,16 @@ module Polars
     #   # │ 2020-01-01 00:00:00.999 ┆ 1000              │
     #   # │ 2020-01-01 00:00:01     ┆ 1000              │
     #   # └─────────────────────────┴───────────────────┘
-    def total_microseconds
-      Utils.wrap_expr(_rbexpr.dt_total_microseconds)
+    def total_microseconds(fractional: false)
+      Utils.wrap_expr(_rbexpr.dt_total_microseconds(fractional))
     end
     alias_method :microseconds, :total_microseconds
 
     # Extract the nanoseconds from a Duration type.
     #
+    # @param fractional [Boolean]
+    #   Whether to include the fractional component of the second.
+    #
     # @return [Expr]
     #
     # @example
@@ -1694,8 +1715,8 @@ module Polars
     #   # │ 2020-01-01 00:00:00.999 ┆ 1000000          │
     #   # │ 2020-01-01 00:00:01     ┆ 1000000          │
     #   # └─────────────────────────┴──────────────────┘
-    def total_nanoseconds
-      Utils.wrap_expr(_rbexpr.dt_total_nanoseconds)
+    def total_nanoseconds(fractional: false)
+      Utils.wrap_expr(_rbexpr.dt_total_nanoseconds(fractional))
     end
     alias_method :nanoseconds, :total_nanoseconds
 

data/lib/polars/expr.rb

@@ -6612,8 +6612,8 @@ module Polars
     #   # │ 10   ┆ null       │
     #   # │ 11   ┆ 0.1        │
     #   # │ 12   ┆ 0.090909   │
-    #   # │ null ┆ 0.0        │
-    #   # │ 12   ┆ 0.0        │
+    #   # │ null ┆ null       │
+    #   # │ 12   ┆ null       │
     #   # └──────┴────────────┘
     def pct_change(n: 1)
       n = Utils.parse_into_expression(n)

data/lib/polars/iceberg_dataset.rb

@@ -0,0 +1,113 @@
+module Polars
+  # @private
+  class IcebergDataset
+    def initialize(
+      source,
+      snapshot_id:,
+      storage_options:
+    )
+      @source = source
+      @snapshot_id = snapshot_id
+      @storage_options = storage_options
+    end
+
+    def to_lazyframe
+      # for iceberg < 0.1.3
+      if !@source.respond_to?(:scan)
+        return @source.to_polars(snapshot_id: @snapshot_id, storage_options: @storage_options)
+      end
+
+      scan = @source.scan(snapshot_id: @snapshot_id)
+      files = scan.plan_files
+
+      table = scan.table
+      snapshot = scan.snapshot
+      schema = snapshot ? table.schema_by_id(snapshot[:schema_id]) : table.current_schema
+
+      if files.empty?
+        # TODO improve
+        schema =
+          schema.fields.to_h do |field|
+            dtype =
+              case field[:type]
+              when "int"
+                Polars::Int32
+              when "long"
+                Polars::Int64
+              when "double"
+                Polars::Float64
+              when "string"
+                Polars::String
+              when "timestamp"
+                Polars::Datetime
+              else
+                raise Todo
+              end
+
+            [field[:name], dtype]
+          end
+
+        LazyFrame.new(schema: schema)
+      else
+        sources = files.map { |v| v[:data_file_path] }
+
+        column_mapping = [
+          "iceberg-column-mapping",
+          arrow_schema(schema)
+        ]
+
+        deletion_files = [
+          "iceberg-position-delete",
+          files.map.with_index
+            .select { |v, i| v[:deletes].any? }
+            .to_h { |v, i| [i, v[:deletes].map { |d| d[:file_path] }] }
+        ]
+
+        scan_options = {
+          storage_options: @storage_options,
+          cast_options: Polars::ScanCastOptions._default_iceberg,
+          allow_missing_columns: true,
+          extra_columns: "ignore",
+          _column_mapping: column_mapping,
+          _deletion_files: deletion_files
+        }
+
+        Polars.scan_parquet(sources, **scan_options)
+      end
+    end
+
+    private
+
+    def arrow_schema(schema)
+      fields =
+        schema.fields.map do |field|
+          type =
+            case field[:type]
+            when "boolean"
+              "boolean"
+            when "int"
+              "int32"
+            when "long"
+              "int64"
+            when "float"
+              "float32"
+            when "double"
+              "float64"
+            else
+              raise Todo
+            end
+
+          {
+            name: field[:name],
+            type: type,
+            nullable: !field[:required],
+            metadata: {
+              "PARQUET:field_id" => field[:id].to_s
+            }
+          }
+        end
+
+      {fields: fields}
+    end
+  end
+end

data/lib/polars/io/iceberg.rb

@@ -21,7 +21,14 @@ module Polars
         raise Todo
       end
 
-      source.to_polars(snapshot_id:, storage_options:)
+      dataset =
+        IcebergDataset.new(
+          source,
+          snapshot_id:,
+          storage_options:
+        )
+
+      dataset.to_lazyframe
     end
   end
 end

data/lib/polars/io/ipc.rb

@@ -187,8 +187,16 @@ module Polars
     #   DataFrame.
     # @param row_count_offset [Integer]
     #   Offset to start the row_count column (only use if the name is set).
+    # @param glob [Boolean]
+    #   Expand path given via globbing rules.
     # @param storage_options [Hash]
     #   Extra options that make sense for a particular storage connection.
+    # @param retries [Integer]
+    #   Number of retries if accessing a cloud instance fails.
+    # @param file_cache_ttl [Integer]
+    #   Amount of time to keep downloaded cloud files since their last access time,
+    #   in seconds. Uses the `POLARS_FILE_CACHE_TTL` environment variable
+    #   (which defaults to 1 hour) if not given.
     # @param hive_partitioning [Boolean]
     #   Infer statistics and schema from Hive partitioned URL and use them
     #   to prune reads. This is unset by default (i.e. `nil`), meaning it is
@@ -210,66 +218,37 @@ module Polars
       rechunk: true,
       row_count_name: nil,
       row_count_offset: 0,
+      glob: true,
       storage_options: nil,
+      retries: 2,
+      file_cache_ttl: nil,
       hive_partitioning: nil,
       hive_schema: nil,
       try_parse_hive_dates: true,
       include_file_paths: nil
     )
-      _scan_ipc_impl(
-        source,
-        n_rows: n_rows,
-        cache: cache,
-        rechunk: rechunk,
-        row_count_name: row_count_name,
-        row_count_offset: row_count_offset,
-        storage_options: storage_options,
-        hive_partitioning: hive_partitioning,
-        hive_schema: hive_schema,
-        try_parse_hive_dates: try_parse_hive_dates,
-        include_file_paths: include_file_paths
-      )
-    end
-
-    # @private
-    def _scan_ipc_impl(
-      source,
-      n_rows: nil,
-      cache: true,
-      rechunk: true,
-      row_count_name: nil,
-      row_count_offset: 0,
-      storage_options: nil,
-      hive_partitioning: nil,
-      hive_schema: nil,
-      try_parse_hive_dates: true,
-      include_file_paths: nil
-    )
-      sources = []
-      if Utils.pathlike?(source)
-        source = Utils.normalize_filepath(source)
-      elsif source.is_a?(::Array)
-        if Utils.is_path_or_str_sequence(source)
-          sources = source.map { |s| Utils.normalize_filepath(s) }
-        else
-          sources = source
-        end
+      row_index_name = row_count_name
+      row_index_offset = row_count_offset
 
-        source = nil
-      end
+      sources = get_sources(source)
 
       rblf =
         RbLazyFrame.new_from_ipc(
-          source,
           sources,
-          n_rows,
-          cache,
-          rechunk,
-          Utils.parse_row_index_args(row_count_name, row_count_offset),
-          hive_partitioning,
-          hive_schema,
-          try_parse_hive_dates,
-          include_file_paths
+          ScanOptions.new(
+            row_index: !row_index_name.nil? ? [row_index_name, row_index_offset] : nil,
+            pre_slice: !n_rows.nil? ? [0, n_rows] : nil,
+            include_file_paths: include_file_paths,
+            glob: glob,
+            hive_partitioning: hive_partitioning,
+            hive_schema: hive_schema,
+            try_parse_hive_dates: try_parse_hive_dates,
+            rechunk: rechunk,
+            cache: cache,
+            storage_options: !storage_options.nil? ? storage_options.to_a : nil,
+            retries: retries
+          ),
+          file_cache_ttl
         )
       Utils.wrap_ldf(rblf)
     end

data/lib/polars/io/scan_options.rb

@@ -2,9 +2,9 @@ module Polars
   module IO
     class ScanOptions
       attr_reader :row_index, :pre_slice, :cast_options, :extra_columns, :missing_columns,
-        :include_file_paths, :glob, :hive_partitioning, :hive_schema, :try_parse_hive_dates,
+        :include_file_paths, :glob, :hidden_file_prefix, :hive_partitioning, :hive_schema, :try_parse_hive_dates,
         :rechunk, :cache, :storage_options, :credential_provider, :retries, :column_mapping,
-        :default_values, :deletion_files
+        :default_values, :deletion_files, :table_statistics, :row_count
 
       def initialize(
         row_index: nil,
@@ -14,6 +14,7 @@ module Polars
         missing_columns: "raise",
         include_file_paths: nil,
         glob: true,
+        hidden_file_prefix: nil,
         hive_partitioning: nil,
         hive_schema: nil,
         try_parse_hive_dates: true,
@@ -24,7 +25,9 @@ module Polars
         retries: 2,
         column_mapping: nil,
         default_values: nil,
-        deletion_files: nil
+        deletion_files: nil,
+        table_statistics: nil,
+        row_count: nil
       )
         @row_index = row_index
         @pre_slice = pre_slice
@@ -33,6 +36,7 @@ module Polars
         @missing_columns = missing_columns
         @include_file_paths = include_file_paths
         @glob = glob
+        @hidden_file_prefix = hidden_file_prefix
         @hive_partitioning = hive_partitioning
         @hive_schema = hive_schema
         @try_parse_hive_dates = try_parse_hive_dates
@@ -44,6 +48,8 @@ module Polars
         @column_mapping = column_mapping
         @default_values = default_values
         @deletion_files = deletion_files
+        @table_statistics = table_statistics
+        @row_count = row_count
       end
     end
   end

data/lib/polars/io/utils.rb

@@ -0,0 +1,17 @@
+module Polars
+  module IO
+    private
+
+    def get_sources(source)
+      if Utils.pathlike?(source)
+        source = Utils.normalize_filepath(source, check_not_directory: false)
+      elsif Utils.is_path_or_str_sequence(source)
+        source = source.map { |s| Utils.normalize_filepath(s, check_not_directory: false) }
+      end
+      unless source.is_a?(::Array)
+        source = [source]
+      end
+      source
+    end
+  end
+end

data/lib/polars/lazy_frame.rb

@@ -4143,6 +4143,9 @@ module Polars
     #   Names of the struct columns that will be decomposed by its fields
     # @param more_columns [Array]
     #   Additional columns to unnest, specified as positional arguments.
+    # @param separator [String]
+    #   Rename output column names as combination of the struct column name,
+    #   name separator and field name.
     #
     # @return [LazyFrame]
     #
@@ -4187,11 +4190,11 @@ module Polars
     #   # │ foo    ┆ 1   ┆ a   ┆ true ┆ [1, 2]    ┆ baz   │
     #   # │ bar    ┆ 2   ┆ b   ┆ null ┆ [3]       ┆ womp  │
     #   # └────────┴─────┴─────┴──────┴───────────┴───────┘
-    def unnest(columns, *more_columns)
+    def unnest(columns, *more_columns, separator: nil)
       subset = Utils.parse_list_into_selector(columns) | Utils.parse_list_into_selector(
         more_columns
       )
-      _from_rbldf(_ldf.unnest(subset._rbselector))
+      _from_rbldf(_ldf.unnest(subset._rbselector, separator))
     end
 
     # Take two sorted DataFrames and merge them by the sorted key.

data/lib/polars/scan_cast_options.rb

@@ -1,7 +1,7 @@
 module Polars
   # Options for scanning files.
   class ScanCastOptions
-    attr_reader :integer_cast, :float_cast, :datetime_cast, :missing_struct_fields, :extra_struct_fields
+    attr_reader :integer_cast, :float_cast, :datetime_cast, :missing_struct_fields, :extra_struct_fields, :categorical_to_string
 
     # Common configuration for scanning files.
     #
@@ -50,6 +50,7 @@ module Polars
       datetime_cast: "forbid",
       missing_struct_fields: "raise",
       extra_struct_fields: "raise",
+      categorical_to_string: "forbid",
       _internal_call: false
     )
       if !_internal_call
@@ -61,6 +62,7 @@ module Polars
       @datetime_cast = datetime_cast
       @missing_struct_fields = missing_struct_fields
       @extra_struct_fields = extra_struct_fields
+      @categorical_to_string = categorical_to_string
     end
 
     def self._default
@@ -75,6 +77,7 @@ module Polars
           datetime_cast: ["nanosecond-downcast", "convert-timezone"],
           missing_struct_fields: "insert",
           extra_struct_fields: "ignore",
+          categorical_to_string: "allow",
           _internal_call: true
         )
       end

data/lib/polars/selectors.rb

@@ -1063,14 +1063,14 @@ module Polars
     #   df.select(Polars.cs.decimal)
     #   # =>
     #   # shape: (2, 2)
-    #   # ┌──────────────┬───────────────┐
-    #   # │ bar          ┆ baz           │
-    #   # │ ---          ┆ ---           │
-    #   # │ decimal[*,0] ┆ decimal[10,5] │
-    #   # ╞══════════════╪═══════════════╡
-    #   # │ 123          ┆ 2.00050       │
-    #   # │ 456          ┆ -50.55550     │
-    #   # └──────────────┴───────────────┘
+    #   # ┌───────────────┬───────────────┐
+    #   # │ bar           ┆ baz           │
+    #   # │ ---           ┆ ---           │
+    #   # │ decimal[38,0] ┆ decimal[10,5] │
+    #   # ╞═══════════════╪═══════════════╡
+    #   # │ 123           ┆ 2.00050       │
+    #   # │ 456           ┆ -50.55550     │
+    #   # └───────────────┴───────────────┘
     #
     # @example Select all columns *except* the decimal ones:
     #

data/lib/polars/series.rb

@@ -5747,11 +5747,31 @@ module Polars
           end
 
         base_type = dtype.is_a?(DataType) ? dtype.class : dtype
-        if [Date, Datetime, Duration, Time, Categorical, Boolean, Enum, Decimal].include?(base_type)
+        if [Date, Datetime, Duration, Time, Categorical, Boolean, Enum].include?(base_type) || dtype.is_a?(Decimal)
           if rbseries.dtype != dtype
             rbseries = rbseries.cast(dtype, true)
           end
         end
+
+        # Uninstanced Decimal is a bit special and has various inference paths
+        if dtype == Decimal
+          if rbseries.dtype == String
+            rbseries = rbseries.str_to_decimal_infer(0)
+          elsif rbseries.dtype.float?
+            # Go through string so we infer an appropriate scale.
+            rbseries = rbseries.cast(
+              String, strict: strict, wrap_numerical: false
+            ).str_to_decimal_infer(0)
+          elsif rbseries.dtype.integer? || rbseries.dtype == Null
+            rbseries = rbseries.cast(
+              Decimal.new(nil, 0), strict: strict, wrap_numerical: false
+            )
+          elsif !rbseries.dtype.is_a?(Decimal)
+            msg = "can't convert #{rbseries.dtype} to Decimal"
+            raise TypeError, msg
+          end
+        end
+
         rbseries
       elsif dtype == Struct
         struct_schema = dtype.is_a?(Struct) ? dtype.to_schema : nil
@@ -5856,6 +5876,7 @@ module Polars
       UInt16 => RbSeries.method(:new_opt_u16),
       UInt32 => RbSeries.method(:new_opt_u32),
       UInt64 => RbSeries.method(:new_opt_u64),
+      UInt128 => RbSeries.method(:new_opt_u128),
       Decimal => RbSeries.method(:new_decimal),
       Date => RbSeries.method(:new_from_any_values),
       Datetime => RbSeries.method(:new_from_any_values),
@@ -5882,6 +5903,7 @@ module Polars
       u16: RbSeries.method(:new_opt_u16),
       u32: RbSeries.method(:new_opt_u32),
       u64: RbSeries.method(:new_opt_u64),
+      u128: RbSeries.method(:new_opt_u128),
       bool: RbSeries.method(:new_opt_bool),
       str: RbSeries.method(:new_str)
     }

data/lib/polars/string_expr.rb

@@ -247,7 +247,7 @@ module Polars
     #   # ┌───────────┬─────────────────┐
     #   # │ numbers   ┆ numbers_decimal │
     #   # │ ---       ┆ ---             │
-    #   # │ str       ┆ decimal[*,2]    │
+    #   # │ str       ┆ decimal[38,2]   │
     #   # ╞═══════════╪═════════════════╡
     #   # │ 40.12     ┆ 40.12           │
     #   # │ 3420.13   ┆ 3420.13         │