polars-df 0.21.1-x86_64-linux → 0.22.0-x86_64-linux

data/lib/polars/list_expr.rb

@@ -925,7 +925,7 @@ module Polars
     # Convert the series of type `List` to a series of type `Struct`.
     #
     # @param n_field_strategy ["first_non_null", "max_width"]
-    #   Strategy to determine the number of fields of the struct.
+    #   Deprecated and ignored.
     # @param fields pArray
     #   If the name and number of the desired fields is known in advance
     #   a list of field names can be given, which will be assigned by index.
@@ -945,20 +945,28 @@ module Polars
     # @return [Expr]
     #
     # @example
-    #   df = Polars::DataFrame.new({"a" => [[1, 2, 3], [1, 2]]})
-    #   df.select([Polars.col("a").list.to_struct])
+    #   df = Polars::DataFrame.new({"n" => [[0, 1], [0, 1, 2]]})
+    #   df.with_columns(struct: Polars.col("n").list.to_struct(upper_bound: 2))
     #   # =>
-    #   # shape: (2, 1)
-    #   # ┌────────────┐
-    #   # │ a          │
-    #   # │ ---        │
-    #   # │ struct[3]  │
-    #   # ╞════════════╡
-    #   # │ {1,2,3}    │
-    #   # │ {1,2,null} │
-    #   # └────────────┘
+    #   # shape: (2, 2)
+    #   # ┌───────────┬───────────┐
+    #   # │ n         ┆ struct    │
+    #   # │ ---       ┆ ---       │
+    #   # │ list[i64] ┆ struct[2] │
+    #   # ╞═══════════╪═══════════╡
+    #   # │ [0, 1]    ┆ {0,1}     │
+    #   # │ [0, 1, 2] ┆ {0,1}     │
+    #   # └───────────┴───────────┘
     def to_struct(n_field_strategy: "first_non_null", fields: nil, upper_bound: nil)
-      Utils.wrap_expr(_rbexpr.list_to_struct(n_field_strategy, fields, nil))
+      if !fields.is_a?(::Array)
+        if fields.nil?
+          fields = upper_bound.times.map { |i| "field_#{i}" }
+        else
+          fields = upper_bound.times.map { |i| fields.(i) }
+        end
+      end
+
+      Utils.wrap_expr(_rbexpr.list_to_struct(fields))
     end
 
     # Run any polars expression against the lists' elements.

data/lib/polars/list_name_space.rb

@@ -755,27 +755,39 @@ module Polars
     #
     # @param n_field_strategy ["first_non_null", "max_width"]
     #   Strategy to determine the number of fields of the struct.
-    # @param name_generator [Object]
-    #   A custom function that can be used to generate the field names.
-    #   Default field names are `field_0, field_1 .. field_n`
-    #
-    # @return [Series]
-    #
-    # @example
-    #   df = Polars::DataFrame.new({"a" => [[1, 2, 3], [1, 2]]})
-    #   df.select([Polars.col("a").list.to_struct])
-    #   # =>
-    #   # shape: (2, 1)
-    #   # ┌────────────┐
-    #   # │ a          │
-    #   # │ ---        │
-    #   # │ struct[3]  │
-    #   # ╞════════════╡
-    #   # │ {1,2,3}    │
-    #   # │ {1,2,null} │
-    #   # └────────────┘
-    def to_struct(n_field_strategy: "first_non_null", name_generator: nil)
-      super
+    # @param fields [Object]
+    #   If the name and number of the desired fields is known in advance
+    #   a list of field names can be given, which will be assigned by index.
+    #   Otherwise, to dynamically assign field names, a custom function can be
+    #   used; if neither are set, fields will be `field_0, field_1 .. field_n`.
+    #
+    # @return [Series]
+    #
+    # @example Convert list to struct with field name assignment by index from a list of names:
+    #   s1 = Polars::Series.new("n", [[0, 1, 2], [0, 1]])
+    #   s1.list.to_struct(fields: ["one", "two", "three"]).struct.unnest
+    #   # =>
+    #   # shape: (2, 3)
+    #   # ┌─────┬─────┬───────┐
+    #   # │ one ┆ two ┆ three │
+    #   # │ --- ┆ --- ┆ ---   │
+    #   # │ i64 ┆ i64 ┆ i64   │
+    #   # ╞═════╪═════╪═══════╡
+    #   # │ 0   ┆ 1   ┆ 2     │
+    #   # │ 0   ┆ 1   ┆ null  │
+    #   # └─────┴─────┴───────┘
+    def to_struct(n_field_strategy: "first_non_null", fields: nil)
+      if fields.is_a?(::Array)
+        s = Utils.wrap_s(_s)
+        return (
+          s.to_frame
+          .select_seq(F.col(s.name).list.to_struct(fields: fields))
+          .to_series
+        )
+      end
+
+      raise Todo
+      # Utils.wrap_s(_s.list_to_struct(n_field_strategy, fields))
     end
 
     # Run any polars expression against the lists' elements.

data/lib/polars/meta_expr.rb

@@ -248,6 +248,31 @@ module Polars
       Selector._from_rbselector(_rbexpr.into_selector)
     end
 
+    # Serialize this expression to a file or string.
+    #
+    # @param file [Object]
+    #   File path to which the result should be written. If set to `nil`
+    #   (default), the output is returned as a string instead.
+    #
+    # @return [Object]
+    #
+    # @note
+    #   Serialization is not stable across Polars versions: a LazyFrame serialized
+    #   in one Polars version may not be deserializable in another Polars version.
+    #
+    # @example Serialize the expression into a binary representation.
+    #   expr = Polars.col("foo").sum.over("bar")
+    #   bytes = expr.meta.serialize
+    #   Polars::Expr.deserialize(StringIO.new(bytes))
+    #   # => col("foo").sum().over([col("bar")])
+    def serialize(file = nil)
+      raise Todo unless _rbexpr.respond_to?(:serialize_binary)
+
+      serializer = _rbexpr.method(:serialize_binary)
+
+      Utils.serialize_polars_object(serializer, file)
+    end
+
     # Format the expression as a tree.
     #
     # @param return_as_string [Boolean]

data/lib/polars/query_opt_flags.rb

@@ -0,0 +1,50 @@
+module Polars
+  # The set of the optimizations considered during query optimization.
+  #
+  # @note
+  #   This functionality is considered **unstable**. It may be changed
+  #   at any point without it being considered a breaking change.
+  class QueryOptFlags
+    def initialize(
+      predicate_pushdown: nil,
+      projection_pushdown: nil,
+      simplify_expression: nil,
+      slice_pushdown: nil,
+      comm_subplan_elim: nil,
+      comm_subexpr_elim: nil,
+      cluster_with_columns: nil,
+      collapse_joins: nil,
+      check_order_observe: nil,
+      fast_projection: nil
+    )
+      @_rboptflags = RbOptFlags.default
+      update(
+        predicate_pushdown: predicate_pushdown,
+        projection_pushdown: projection_pushdown,
+        simplify_expression: simplify_expression,
+        slice_pushdown: slice_pushdown,
+        comm_subplan_elim: comm_subplan_elim,
+        comm_subexpr_elim: comm_subexpr_elim,
+        cluster_with_columns: cluster_with_columns,
+        collapse_joins: collapse_joins,
+        check_order_observe: check_order_observe,
+        fast_projection: fast_projection
+      )
+    end
+
+    def update(
+      predicate_pushdown: nil,
+      projection_pushdown: nil,
+      simplify_expression: nil,
+      slice_pushdown: nil,
+      comm_subplan_elim: nil,
+      comm_subexpr_elim: nil,
+      cluster_with_columns: nil,
+      collapse_joins: nil,
+      check_order_observe: nil,
+      fast_projection: nil
+    )
+      raise Todo
+    end
+  end
+end

data/lib/polars/scan_cast_options.rb

@@ -1,6 +1,8 @@
 module Polars
   # Options for scanning files.
   class ScanCastOptions
+    attr_reader :integer_cast, :float_cast, :datetime_cast, :missing_struct_fields, :extra_struct_fields
+
     # Common configuration for scanning files.
     #
     # @note
@@ -50,6 +52,10 @@ module Polars
       extra_struct_fields: "raise",
       _internal_call: false
     )
+      if !_internal_call
+        warn "ScanCastOptions is considered unstable."
+      end
+
       @integer_cast = integer_cast
       @float_cast = float_cast
       @datetime_cast = datetime_cast
@@ -57,8 +63,21 @@ module Polars
       @extra_struct_fields = extra_struct_fields
     end
 
-    def self.default
+    def self._default
       new(_internal_call: true)
     end
+
+    def self._default_iceberg
+      @_default_cast_options_iceberg ||= begin
+        ScanCastOptions.new(
+          integer_cast: "upcast",
+          float_cast: ["upcast", "downcast"],
+          datetime_cast: ["nanosecond-downcast", "convert-timezone"],
+          missing_struct_fields: "insert",
+          extra_struct_fields: "ignore",
+          _internal_call: true
+        )
+      end
+    end
   end
 end

data/lib/polars/schema.rb

@@ -30,7 +30,7 @@ module Polars
     #
     # @return [Object]
     def []=(name, dtype)
-      # TODO check dtype
+      _check_dtype(dtype)
       @schema[name] = dtype
     end
 

data/lib/polars/series.rb

@@ -5307,7 +5307,7 @@ module Polars
     #   #         6
     #   # ]
     def shrink_dtype
-      super
+      Utils.wrap_s(_s.shrink_dtype)
     end
 
     # Get the chunks of this Series as a list of Series.
@@ -5851,6 +5851,7 @@ module Polars
       Int16 => RbSeries.method(:new_opt_i16),
       Int32 => RbSeries.method(:new_opt_i32),
       Int64 => RbSeries.method(:new_opt_i64),
+      Int128 => RbSeries.method(:new_opt_i128),
       UInt8 => RbSeries.method(:new_opt_u8),
       UInt16 => RbSeries.method(:new_opt_u16),
       UInt32 => RbSeries.method(:new_opt_u32),
@@ -5876,6 +5877,7 @@ module Polars
       i16: RbSeries.method(:new_opt_i16),
       i32: RbSeries.method(:new_opt_i32),
       i64: RbSeries.method(:new_opt_i64),
+      i128: RbSeries.method(:new_opt_i128),
       u8: RbSeries.method(:new_opt_u8),
       u16: RbSeries.method(:new_opt_u16),
       u32: RbSeries.method(:new_opt_u32),

data/lib/polars/string_expr.rb

@@ -222,10 +222,8 @@ module Polars
 
     # Convert a String column into a Decimal column.
     #
-    # This method infers the needed parameters `precision` and `scale`.
-    #
-    # @param inference_length [Integer]
-    #   Number of elements to parse to determine the `precision` and `scale`.
+    # @param scale [Integer]
+    #   Number of digits after the comma to use for the decimals.
     #
     # @return [Expr]
     #
@@ -243,7 +241,7 @@ module Polars
     #       ]
     #     }
     #   )
-    #   df.with_columns(numbers_decimal: Polars.col("numbers").str.to_decimal)
+    #   df.with_columns(numbers_decimal: Polars.col("numbers").str.to_decimal(scale: 2))
     #   # =>
     #   # shape: (7, 2)
     #   # ┌───────────┬─────────────────┐
@@ -259,8 +257,8 @@ module Polars
     #   # │ 143.09    ┆ 143.09          │
     #   # │ 143.9     ┆ 143.90          │
     #   # └───────────┴─────────────────┘
-    def to_decimal(inference_length = 100)
-      Utils.wrap_expr(_rbexpr.str_to_decimal(inference_length))
+    def to_decimal(scale:)
+      Utils.wrap_expr(_rbexpr.str_to_decimal(scale))
     end
 
     # Get length of the strings as `:u32` (as number of bytes).
@@ -774,7 +772,7 @@ module Polars
 
     # Return the bytes offset of the first substring matching a pattern.
     #
-    # If the pattern is not found, returns None.
+    # If the pattern is not found, returns nil.
     #
     # @param pattern [String]
     #   A valid regular expression pattern, compatible with the [regex crate](https://docs.rs/regex/latest/regex/).
@@ -919,11 +917,9 @@ module Polars
     # Throw errors if encounter invalid JSON strings.
     #
     # @param dtype [Object]
-    #   The dtype to cast the extracted value to. If nil, the dtype will be
-    #   inferred from the JSON value.
+    #   The dtype to cast the extracted value to.
     # @param infer_schema_length [Integer]
-    #   The maximum number of rows to scan for schema inference.
-    #   If set to `nil`, the full data may be scanned *(this is slow)*.
+    #   Deprecated and ignored.
     #
     # @return [Expr]
     #
@@ -932,23 +928,26 @@ module Polars
     #     {"json" => ['{"a":1, "b": true}', nil, '{"a":2, "b": false}']}
     #   )
     #   dtype = Polars::Struct.new([Polars::Field.new("a", Polars::Int64), Polars::Field.new("b", Polars::Boolean)])
-    #   df.select(Polars.col("json").str.json_decode(dtype))
+    #   df.with_columns(decoded: Polars.col("json").str.json_decode(dtype))
     #   # =>
-    #   # shape: (3, 1)
-    #   # ┌───────────┐
-    #   # │ json      │
-    #   # │ ---       │
-    #   # │ struct[2] │
-    #   # ╞═══════════╡
-    #   # │ {1,true}  │
-    #   # │ null      │
-    #   # │ {2,false} │
-    #   # └───────────┘
-    def json_decode(dtype = nil, infer_schema_length: 100)
-      if !dtype.nil?
-        dtype = Utils.rb_type_to_dtype(dtype)
+    #   # shape: (3, 2)
+    #   # ┌─────────────────────┬───────────┐
+    #   # │ json                ┆ decoded   │
+    #   # │ ---                 ┆ ---       │
+    #   # │ str                 ┆ struct[2] │
+    #   # ╞═════════════════════╪═══════════╡
+    #   # │ {"a":1, "b": true}  ┆ {1,true}  │
+    #   # │ null                ┆ null      │
+    #   # │ {"a":2, "b": false} ┆ {2,false} │
+    #   # └─────────────────────┴───────────┘
+    def json_decode(dtype, infer_schema_length: nil)
+      if dtype.nil?
+        msg = "`Expr.str.json_decode` needs an explicitly given `dtype` otherwise Polars is not able to determine the output type. If you want to eagerly infer datatype you can use `Series.str.json_decode`."
+        raise TypeError, msg
       end
-      Utils.wrap_expr(_rbexpr.str_json_decode(dtype, infer_schema_length))
+
+      dtype_expr = Utils.parse_into_datatype_expr(dtype)._rbdatatype_expr
+      Utils.wrap_expr(_rbexpr.str_json_decode(dtype_expr))
     end
     alias_method :json_extract, :json_decode
 

data/lib/polars/string_name_space.rb

@@ -210,8 +210,12 @@ module Polars
     #   #         143.09
     #   #         143.90
     #   # ]
-    def to_decimal(inference_length = 100)
-      super
+    def to_decimal(inference_length = 100, scale: nil)
+      if !scale.nil?
+        raise Todo
+      end
+
+      Utils.wrap_s(_s.str_to_decimal_infer(inference_length))
     end
 
     # Return the length of each string as the number of bytes.
@@ -450,7 +454,7 @@ module Polars
     # Throws an error if invalid JSON strings are encountered.
     #
     # @param dtype [Object]
-    #   The dtype to cast the extracted value to. If None, the dtype will be
+    #   The dtype to cast the extracted value to. If nil, the dtype will be
     #   inferred from the JSON value.
     # @param infer_schema_length [Integer]
     #   The maximum number of rows to scan for schema inference.
@@ -470,7 +474,16 @@ module Polars
     #   #         {2,false}
     #   # ]
     def json_decode(dtype = nil, infer_schema_length: 100)
-      super
+      if !dtype.nil?
+        s = Utils.wrap_s(_s)
+        return (
+          s.to_frame
+          .select_seq(F.col(s.name).str.json_decode(dtype))
+          .to_series
+        )
+      end
+
+      Utils.wrap_s(_s.str_json_decode(infer_schema_length))
     end
 
     # Extract the first match of json string with provided JSONPath expression.

data/lib/polars/utils/serde.rb

@@ -0,0 +1,17 @@
+module Polars
+  module Utils
+    def self.serialize_polars_object(serializer, file)
+      serialize_to_bytes = lambda do
+        buf = StringIO.new
+        serializer.(buf)
+        buf.string
+      end
+
+      if file.nil?
+        return serialize_to_bytes.call
+      end
+
+      raise Todo
+    end
+  end
+end

data/lib/polars/utils/various.rb

@@ -75,6 +75,10 @@ module Polars
       end
     end
 
+    def self._polars_warn(msg)
+      warn msg
+    end
+
     def self.extend_bool(value, n_match, value_name, match_name)
       values = bool?(value) ? [value] * n_match : value
       if n_match != values.length

data/lib/polars/version.rb

@@ -1,4 +1,4 @@
 module Polars
   # @private
-  VERSION = "0.21.1"
+  VERSION = "0.22.0"
 end

data/lib/polars.rb

@@ -37,6 +37,7 @@ require_relative "polars/dynamic_group_by"
 require_relative "polars/exceptions"
 require_relative "polars/expr"
 require_relative "polars/functions/as_datatype"
+require_relative "polars/functions/business"
 require_relative "polars/functions/col"
 require_relative "polars/functions/datatype"
 require_relative "polars/functions/eager"
@@ -57,6 +58,7 @@ require_relative "polars/io/avro"
 require_relative "polars/io/csv"
 require_relative "polars/io/database"
 require_relative "polars/io/delta"
+require_relative "polars/io/iceberg"
 require_relative "polars/io/ipc"
 require_relative "polars/io/json"
 require_relative "polars/io/ndjson"
@@ -68,6 +70,7 @@ require_relative "polars/list_expr"
 require_relative "polars/list_name_space"
 require_relative "polars/meta_expr"
 require_relative "polars/name_expr"
+require_relative "polars/query_opt_flags"
 require_relative "polars/rolling_group_by"
 require_relative "polars/scan_cast_options"
 require_relative "polars/schema"
@@ -86,6 +89,7 @@ require_relative "polars/utils"
 require_relative "polars/utils/constants"
 require_relative "polars/utils/convert"
 require_relative "polars/utils/parse"
+require_relative "polars/utils/serde"
 require_relative "polars/utils/various"
 require_relative "polars/utils/wrap"
 require_relative "polars/version"

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: polars-df
 version: !ruby/object:Gem::Version
-  version: 0.21.1
+  version: 0.22.0
 platform: x86_64-linux
 authors:
 - Andrew Kane
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2025-08-19 00:00:00.000000000 Z
+date: 2025-09-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bigdecimal
@@ -69,6 +69,7 @@ files:
 - lib/polars/functions/aggregation/horizontal.rb
 - lib/polars/functions/aggregation/vertical.rb
 - lib/polars/functions/as_datatype.rb
+- lib/polars/functions/business.rb
 - lib/polars/functions/col.rb
 - lib/polars/functions/datatype.rb
 - lib/polars/functions/eager.rb
@@ -87,6 +88,7 @@ files:
 - lib/polars/io/csv.rb
 - lib/polars/io/database.rb
 - lib/polars/io/delta.rb
+- lib/polars/io/iceberg.rb
 - lib/polars/io/ipc.rb
 - lib/polars/io/json.rb
 - lib/polars/io/ndjson.rb
@@ -99,6 +101,7 @@ files:
 - lib/polars/meta_expr.rb
 - lib/polars/name_expr.rb
 - lib/polars/plot.rb
+- lib/polars/query_opt_flags.rb
 - lib/polars/rolling_group_by.rb
 - lib/polars/scan_cast_options.rb
 - lib/polars/schema.rb
@@ -117,6 +120,7 @@ files:
 - lib/polars/utils/constants.rb
 - lib/polars/utils/convert.rb
 - lib/polars/utils/parse.rb
+- lib/polars/utils/serde.rb
 - lib/polars/utils/various.rb
 - lib/polars/utils/wrap.rb
 - lib/polars/version.rb