polars-df 0.21.1 → 0.23.0

data/lib/polars/query_opt_flags.rb

@@ -0,0 +1,50 @@
+module Polars
+  # The set of the optimizations considered during query optimization.
+  #
+  # @note
+  #   This functionality is considered **unstable**. It may be changed
+  #   at any point without it being considered a breaking change.
+  class QueryOptFlags
+    def initialize(
+      predicate_pushdown: nil,
+      projection_pushdown: nil,
+      simplify_expression: nil,
+      slice_pushdown: nil,
+      comm_subplan_elim: nil,
+      comm_subexpr_elim: nil,
+      cluster_with_columns: nil,
+      collapse_joins: nil,
+      check_order_observe: nil,
+      fast_projection: nil
+    )
+      @_rboptflags = RbOptFlags.default
+      update(
+        predicate_pushdown: predicate_pushdown,
+        projection_pushdown: projection_pushdown,
+        simplify_expression: simplify_expression,
+        slice_pushdown: slice_pushdown,
+        comm_subplan_elim: comm_subplan_elim,
+        comm_subexpr_elim: comm_subexpr_elim,
+        cluster_with_columns: cluster_with_columns,
+        collapse_joins: collapse_joins,
+        check_order_observe: check_order_observe,
+        fast_projection: fast_projection
+      )
+    end
+
+    def update(
+      predicate_pushdown: nil,
+      projection_pushdown: nil,
+      simplify_expression: nil,
+      slice_pushdown: nil,
+      comm_subplan_elim: nil,
+      comm_subexpr_elim: nil,
+      cluster_with_columns: nil,
+      collapse_joins: nil,
+      check_order_observe: nil,
+      fast_projection: nil
+    )
+      raise Todo
+    end
+  end
+end

data/lib/polars/scan_cast_options.rb

@@ -1,6 +1,8 @@
 module Polars
   # Options for scanning files.
   class ScanCastOptions
+    attr_reader :integer_cast, :float_cast, :datetime_cast, :missing_struct_fields, :extra_struct_fields, :categorical_to_string
+
     # Common configuration for scanning files.
     #
     # @note
@@ -48,17 +50,37 @@ module Polars
       datetime_cast: "forbid",
       missing_struct_fields: "raise",
       extra_struct_fields: "raise",
+      categorical_to_string: "forbid",
       _internal_call: false
     )
+      if !_internal_call
+        warn "ScanCastOptions is considered unstable."
+      end
+
       @integer_cast = integer_cast
       @float_cast = float_cast
       @datetime_cast = datetime_cast
       @missing_struct_fields = missing_struct_fields
       @extra_struct_fields = extra_struct_fields
+      @categorical_to_string = categorical_to_string
     end
 
-    def self.default
+    def self._default
       new(_internal_call: true)
     end
+
+    def self._default_iceberg
+      @_default_cast_options_iceberg ||= begin
+        ScanCastOptions.new(
+          integer_cast: "upcast",
+          float_cast: ["upcast", "downcast"],
+          datetime_cast: ["nanosecond-downcast", "convert-timezone"],
+          missing_struct_fields: "insert",
+          extra_struct_fields: "ignore",
+          categorical_to_string: "allow",
+          _internal_call: true
+        )
+      end
+    end
   end
 end

data/lib/polars/schema.rb

@@ -30,7 +30,7 @@ module Polars
     #
     # @return [Object]
     def []=(name, dtype)
-      # TODO check dtype
+      _check_dtype(dtype)
       @schema[name] = dtype
     end
 

data/lib/polars/selectors.rb

@@ -1063,14 +1063,14 @@ module Polars
     #   df.select(Polars.cs.decimal)
     #   # =>
     #   # shape: (2, 2)
-    #   # ┌──────────────┬───────────────┐
-    #   # │ bar          ┆ baz           │
-    #   # │ ---          ┆ ---           │
-    #   # │ decimal[*,0] ┆ decimal[10,5] │
-    #   # ╞══════════════╪═══════════════╡
-    #   # │ 123          ┆ 2.00050       │
-    #   # │ 456          ┆ -50.55550     │
-    #   # └──────────────┴───────────────┘
+    #   # ┌───────────────┬───────────────┐
+    #   # │ bar           ┆ baz           │
+    #   # │ ---           ┆ ---           │
+    #   # │ decimal[38,0] ┆ decimal[10,5] │
+    #   # ╞═══════════════╪═══════════════╡
+    #   # │ 123           ┆ 2.00050       │
+    #   # │ 456           ┆ -50.55550     │
+    #   # └───────────────┴───────────────┘
     #
     # @example Select all columns *except* the decimal ones:
     #

data/lib/polars/series.rb

@@ -5307,7 +5307,7 @@ module Polars
     #   #         6
     #   # ]
     def shrink_dtype
-      super
+      Utils.wrap_s(_s.shrink_dtype)
     end
 
     # Get the chunks of this Series as a list of Series.
@@ -5747,11 +5747,31 @@ module Polars
           end
 
         base_type = dtype.is_a?(DataType) ? dtype.class : dtype
-        if [Date, Datetime, Duration, Time, Categorical, Boolean, Enum, Decimal].include?(base_type)
+        if [Date, Datetime, Duration, Time, Categorical, Boolean, Enum].include?(base_type) || dtype.is_a?(Decimal)
           if rbseries.dtype != dtype
             rbseries = rbseries.cast(dtype, true)
           end
         end
+
+        # Uninstanced Decimal is a bit special and has various inference paths
+        if dtype == Decimal
+          if rbseries.dtype == String
+            rbseries = rbseries.str_to_decimal_infer(0)
+          elsif rbseries.dtype.float?
+            # Go through string so we infer an appropriate scale.
+            rbseries = rbseries.cast(
+              String, strict: strict, wrap_numerical: false
+            ).str_to_decimal_infer(0)
+          elsif rbseries.dtype.integer? || rbseries.dtype == Null
+            rbseries = rbseries.cast(
+              Decimal.new(nil, 0), strict: strict, wrap_numerical: false
+            )
+          elsif !rbseries.dtype.is_a?(Decimal)
+            msg = "can't convert #{rbseries.dtype} to Decimal"
+            raise TypeError, msg
+          end
+        end
+
         rbseries
       elsif dtype == Struct
         struct_schema = dtype.is_a?(Struct) ? dtype.to_schema : nil
@@ -5851,10 +5871,12 @@ module Polars
       Int16 => RbSeries.method(:new_opt_i16),
       Int32 => RbSeries.method(:new_opt_i32),
       Int64 => RbSeries.method(:new_opt_i64),
+      Int128 => RbSeries.method(:new_opt_i128),
       UInt8 => RbSeries.method(:new_opt_u8),
       UInt16 => RbSeries.method(:new_opt_u16),
       UInt32 => RbSeries.method(:new_opt_u32),
       UInt64 => RbSeries.method(:new_opt_u64),
+      UInt128 => RbSeries.method(:new_opt_u128),
       Decimal => RbSeries.method(:new_decimal),
       Date => RbSeries.method(:new_from_any_values),
       Datetime => RbSeries.method(:new_from_any_values),
@@ -5876,10 +5898,12 @@ module Polars
       i16: RbSeries.method(:new_opt_i16),
       i32: RbSeries.method(:new_opt_i32),
       i64: RbSeries.method(:new_opt_i64),
+      i128: RbSeries.method(:new_opt_i128),
       u8: RbSeries.method(:new_opt_u8),
       u16: RbSeries.method(:new_opt_u16),
       u32: RbSeries.method(:new_opt_u32),
       u64: RbSeries.method(:new_opt_u64),
+      u128: RbSeries.method(:new_opt_u128),
       bool: RbSeries.method(:new_opt_bool),
       str: RbSeries.method(:new_str)
     }

data/lib/polars/string_expr.rb

@@ -222,10 +222,8 @@ module Polars
 
     # Convert a String column into a Decimal column.
     #
-    # This method infers the needed parameters `precision` and `scale`.
-    #
-    # @param inference_length [Integer]
-    #   Number of elements to parse to determine the `precision` and `scale`.
+    # @param scale [Integer]
+    #   Number of digits after the comma to use for the decimals.
     #
     # @return [Expr]
     #
@@ -243,13 +241,13 @@ module Polars
     #       ]
     #     }
     #   )
-    #   df.with_columns(numbers_decimal: Polars.col("numbers").str.to_decimal)
+    #   df.with_columns(numbers_decimal: Polars.col("numbers").str.to_decimal(scale: 2))
     #   # =>
     #   # shape: (7, 2)
     #   # ┌───────────┬─────────────────┐
     #   # │ numbers   ┆ numbers_decimal │
     #   # │ ---       ┆ ---             │
-    #   # │ str       ┆ decimal[*,2]    │
+    #   # │ str       ┆ decimal[38,2]   │
     #   # ╞═══════════╪═════════════════╡
     #   # │ 40.12     ┆ 40.12           │
     #   # │ 3420.13   ┆ 3420.13         │
@@ -259,8 +257,8 @@ module Polars
     #   # │ 143.09    ┆ 143.09          │
     #   # │ 143.9     ┆ 143.90          │
     #   # └───────────┴─────────────────┘
-    def to_decimal(inference_length = 100)
-      Utils.wrap_expr(_rbexpr.str_to_decimal(inference_length))
+    def to_decimal(scale:)
+      Utils.wrap_expr(_rbexpr.str_to_decimal(scale))
     end
 
     # Get length of the strings as `:u32` (as number of bytes).
@@ -774,7 +772,7 @@ module Polars
 
     # Return the bytes offset of the first substring matching a pattern.
     #
-    # If the pattern is not found, returns None.
+    # If the pattern is not found, returns nil.
     #
     # @param pattern [String]
     #   A valid regular expression pattern, compatible with the [regex crate](https://docs.rs/regex/latest/regex/).
@@ -919,11 +917,9 @@ module Polars
     # Throw errors if encounter invalid JSON strings.
     #
     # @param dtype [Object]
-    #   The dtype to cast the extracted value to. If nil, the dtype will be
-    #   inferred from the JSON value.
+    #   The dtype to cast the extracted value to.
     # @param infer_schema_length [Integer]
-    #   The maximum number of rows to scan for schema inference.
-    #   If set to `nil`, the full data may be scanned *(this is slow)*.
+    #   Deprecated and ignored.
     #
     # @return [Expr]
     #
@@ -932,23 +928,26 @@ module Polars
     #     {"json" => ['{"a":1, "b": true}', nil, '{"a":2, "b": false}']}
     #   )
     #   dtype = Polars::Struct.new([Polars::Field.new("a", Polars::Int64), Polars::Field.new("b", Polars::Boolean)])
-    #   df.select(Polars.col("json").str.json_decode(dtype))
+    #   df.with_columns(decoded: Polars.col("json").str.json_decode(dtype))
     #   # =>
-    #   # shape: (3, 1)
-    #   # ┌───────────┐
-    #   # │ json      │
-    #   # │ ---       │
-    #   # │ struct[2] │
-    #   # ╞═══════════╡
-    #   # │ {1,true}  │
-    #   # │ null      │
-    #   # │ {2,false} │
-    #   # └───────────┘
-    def json_decode(dtype = nil, infer_schema_length: 100)
-      if !dtype.nil?
-        dtype = Utils.rb_type_to_dtype(dtype)
+    #   # shape: (3, 2)
+    #   # ┌─────────────────────┬───────────┐
+    #   # │ json                ┆ decoded   │
+    #   # │ ---                 ┆ ---       │
+    #   # │ str                 ┆ struct[2] │
+    #   # ╞═════════════════════╪═══════════╡
+    #   # │ {"a":1, "b": true}  ┆ {1,true}  │
+    #   # │ null                ┆ null      │
+    #   # │ {"a":2, "b": false} ┆ {2,false} │
+    #   # └─────────────────────┴───────────┘
+    def json_decode(dtype, infer_schema_length: nil)
+      if dtype.nil?
+        msg = "`Expr.str.json_decode` needs an explicitly given `dtype` otherwise Polars is not able to determine the output type. If you want to eagerly infer datatype you can use `Series.str.json_decode`."
+        raise TypeError, msg
       end
-      Utils.wrap_expr(_rbexpr.str_json_decode(dtype, infer_schema_length))
+
+      dtype_expr = Utils.parse_into_datatype_expr(dtype)._rbdatatype_expr
+      Utils.wrap_expr(_rbexpr.str_json_decode(dtype_expr))
     end
     alias_method :json_extract, :json_decode
 

data/lib/polars/string_name_space.rb

@@ -200,7 +200,7 @@ module Polars
     #   s.str.to_decimal
     #   # =>
     #   # shape: (7,)
-    #   # Series: '' [decimal[*,2]]
+    #   # Series: '' [decimal[8,2]]
     #   # [
     #   #         40.12
     #   #         3420.13
@@ -210,8 +210,12 @@ module Polars
     #   #         143.09
     #   #         143.90
     #   # ]
-    def to_decimal(inference_length = 100)
-      super
+    def to_decimal(inference_length = 100, scale: nil)
+      if !scale.nil?
+        raise Todo
+      end
+
+      Utils.wrap_s(_s.str_to_decimal_infer(inference_length))
     end
 
     # Return the length of each string as the number of bytes.
@@ -450,7 +454,7 @@ module Polars
     # Throws an error if invalid JSON strings are encountered.
     #
     # @param dtype [Object]
-    #   The dtype to cast the extracted value to. If None, the dtype will be
+    #   The dtype to cast the extracted value to. If nil, the dtype will be
     #   inferred from the JSON value.
     # @param infer_schema_length [Integer]
     #   The maximum number of rows to scan for schema inference.
@@ -470,7 +474,16 @@ module Polars
     #   #         {2,false}
     #   # ]
     def json_decode(dtype = nil, infer_schema_length: 100)
-      super
+      if !dtype.nil?
+        s = Utils.wrap_s(_s)
+        return (
+          s.to_frame
+          .select_seq(F.col(s.name).str.json_decode(dtype))
+          .to_series
+        )
+      end
+
+      Utils.wrap_s(_s.str_json_decode(infer_schema_length))
     end
 
     # Extract the first match of json string with provided JSONPath expression.

data/lib/polars/utils/convert.rb

@@ -90,8 +90,8 @@ module Polars
       end
     end
 
-    def self._to_ruby_decimal(digits, scale)
-      BigDecimal("#{digits}e#{scale}")
+    def self._to_ruby_decimal(prec, value)
+      BigDecimal(value, prec)
     end
   end
 end

data/lib/polars/utils/serde.rb

@@ -0,0 +1,17 @@
+module Polars
+  module Utils
+    def self.serialize_polars_object(serializer, file)
+      serialize_to_bytes = lambda do
+        buf = StringIO.new
+        serializer.(buf)
+        buf.string
+      end
+
+      if file.nil?
+        return serialize_to_bytes.call
+      end
+
+      raise Todo
+    end
+  end
+end

data/lib/polars/utils/various.rb

@@ -75,6 +75,10 @@ module Polars
       end
     end
 
+    def self._polars_warn(msg)
+      warn msg
+    end
+
     def self.extend_bool(value, n_match, value_name, match_name)
       values = bool?(value) ? [value] * n_match : value
       if n_match != values.length

data/lib/polars/version.rb

@@ -1,4 +1,4 @@
 module Polars
   # @private
-  VERSION = "0.21.1"
+  VERSION = "0.23.0"
 end

data/lib/polars.rb

@@ -37,6 +37,7 @@ require_relative "polars/dynamic_group_by"
 require_relative "polars/exceptions"
 require_relative "polars/expr"
 require_relative "polars/functions/as_datatype"
+require_relative "polars/functions/business"
 require_relative "polars/functions/col"
 require_relative "polars/functions/datatype"
 require_relative "polars/functions/eager"
@@ -53,21 +54,25 @@ require_relative "polars/functions/range/datetime_range"
 require_relative "polars/functions/range/int_range"
 require_relative "polars/functions/range/time_range"
 require_relative "polars/group_by"
+require_relative "polars/iceberg_dataset"
 require_relative "polars/io/avro"
 require_relative "polars/io/csv"
 require_relative "polars/io/database"
 require_relative "polars/io/delta"
+require_relative "polars/io/iceberg"
 require_relative "polars/io/ipc"
 require_relative "polars/io/json"
 require_relative "polars/io/ndjson"
 require_relative "polars/io/parquet"
 require_relative "polars/io/scan_options"
+require_relative "polars/io/utils"
 require_relative "polars/lazy_frame"
 require_relative "polars/lazy_group_by"
 require_relative "polars/list_expr"
 require_relative "polars/list_name_space"
 require_relative "polars/meta_expr"
 require_relative "polars/name_expr"
+require_relative "polars/query_opt_flags"
 require_relative "polars/rolling_group_by"
 require_relative "polars/scan_cast_options"
 require_relative "polars/schema"
@@ -86,6 +91,7 @@ require_relative "polars/utils"
 require_relative "polars/utils/constants"
 require_relative "polars/utils/convert"
 require_relative "polars/utils/parse"
+require_relative "polars/utils/serde"
 require_relative "polars/utils/various"
 require_relative "polars/utils/wrap"
 require_relative "polars/version"

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: polars-df
 version: !ruby/object:Gem::Version
-  version: 0.21.1
+  version: 0.23.0
 platform: ruby
 authors:
 - Andrew Kane
@@ -58,6 +58,7 @@ files:
 - ext/polars/src/conversion/any_value.rs
 - ext/polars/src/conversion/categorical.rs
 - ext/polars/src/conversion/chunked_array.rs
+- ext/polars/src/conversion/datetime.rs
 - ext/polars/src/conversion/mod.rs
 - ext/polars/src/dataframe/construction.rs
 - ext/polars/src/dataframe/export.rs
@@ -80,6 +81,7 @@ files:
 - ext/polars/src/expr/name.rs
 - ext/polars/src/expr/rolling.rs
 - ext/polars/src/expr/selector.rs
+- ext/polars/src/expr/serde.rs
 - ext/polars/src/expr/string.rs
 - ext/polars/src/expr/struct.rs
 - ext/polars/src/file.rs
@@ -105,6 +107,7 @@ files:
 - ext/polars/src/io/mod.rs
 - ext/polars/src/lazyframe/general.rs
 - ext/polars/src/lazyframe/mod.rs
+- ext/polars/src/lazyframe/optflags.rs
 - ext/polars/src/lazyframe/serde.rs
 - ext/polars/src/lazyframe/sink.rs
 - ext/polars/src/lazygroupby.rs
@@ -158,6 +161,7 @@ files:
 - lib/polars/functions/aggregation/horizontal.rb
 - lib/polars/functions/aggregation/vertical.rb
 - lib/polars/functions/as_datatype.rb
+- lib/polars/functions/business.rb
 - lib/polars/functions/col.rb
 - lib/polars/functions/datatype.rb
 - lib/polars/functions/eager.rb
@@ -172,15 +176,18 @@ files:
 - lib/polars/functions/repeat.rb
 - lib/polars/functions/whenthen.rb
 - lib/polars/group_by.rb
+- lib/polars/iceberg_dataset.rb
 - lib/polars/io/avro.rb
 - lib/polars/io/csv.rb
 - lib/polars/io/database.rb
 - lib/polars/io/delta.rb
+- lib/polars/io/iceberg.rb
 - lib/polars/io/ipc.rb
 - lib/polars/io/json.rb
 - lib/polars/io/ndjson.rb
 - lib/polars/io/parquet.rb
 - lib/polars/io/scan_options.rb
+- lib/polars/io/utils.rb
 - lib/polars/lazy_frame.rb
 - lib/polars/lazy_group_by.rb
 - lib/polars/list_expr.rb
@@ -188,6 +195,7 @@ files:
 - lib/polars/meta_expr.rb
 - lib/polars/name_expr.rb
 - lib/polars/plot.rb
+- lib/polars/query_opt_flags.rb
 - lib/polars/rolling_group_by.rb
 - lib/polars/scan_cast_options.rb
 - lib/polars/schema.rb
@@ -206,6 +214,7 @@ files:
 - lib/polars/utils/constants.rb
 - lib/polars/utils/convert.rb
 - lib/polars/utils/parse.rb
+- lib/polars/utils/serde.rb
 - lib/polars/utils/various.rb
 - lib/polars/utils/wrap.rb
 - lib/polars/version.rb