polars-df 0.15.0 → 0.26.0

data/lib/polars/data_frame.rb

@@ -1,8 +1,6 @@
 module Polars
   # Two-dimensional data structure representing data as a table with rows and columns.
   class DataFrame
-    include Plot
-
     # @private
     attr_accessor :_df
 
@@ -15,11 +13,11 @@ module Polars
     #   The schema of the resulting DataFrame. The schema may be declared in several
     #   ways:
     #
-    #   * As a hash of name:type pairs; if type is nil, it will be auto-inferred.
+    #   * As a hash of \\\\{name:type} pairs; if type is nil, it will be auto-inferred.
     #   * As an array of column names; in this case types are automatically inferred.
-    #   * As an array of (name,type) pairs; this is equivalent to the dictionary form.
+    #   * As an array of (name,type) pairs; this is equivalent to the hash form.
     #
-    #   If you supply a list of column names that does not match the names in the
+    #   If you supply an array of column names that does not match the names in the
     #   underlying data, the names given here will overwrite them. The number
     #   of names given in the schema should match the underlying data dimensions.
     #
@@ -43,29 +41,40 @@ module Polars
     # @param infer_schema_length [Integer]
     #   The maximum number of rows to scan for schema inference. If set to `nil`, the
     #   full data may be scanned *(this can be slow)*. This parameter only applies if
-    #   the input data is a sequence or generator of rows; other input is read as-is.
+    #   the input data is an array or generator of rows; other input is read as-is.
     # @param nan_to_null [Boolean]
     #   If the data comes from one or more Numo arrays, can optionally convert input
     #   data NaN values to null instead. This is a no-op for all other input data.
-    def initialize(data = nil, schema: nil, columns: nil, schema_overrides: nil, strict: true, orient: nil, infer_schema_length: 100, nan_to_null: false)
-      if schema && columns
-        warn "columns is ignored when schema is passed"
-      end
-      schema ||= columns
-
+    def initialize(
+      data = nil,
+      schema: nil,
+      schema_overrides: nil,
+      strict: true,
+      orient: nil,
+      infer_schema_length: N_INFER_DEFAULT,
+      nan_to_null: false,
+      height: nil
+    )
       if defined?(ActiveRecord) && (data.is_a?(ActiveRecord::Relation) || data.is_a?(ActiveRecord::Result))
         raise ArgumentError, "Use read_database instead"
       end
 
+      if !height.nil?
+        msg = "the `height` parameter of `DataFrame` is considered unstable."
+        Utils.issue_unstable_warning(msg)
+
+        raise Todo
+      end
+
       if data.nil?
-        self._df = self.class.hash_to_rbdf({}, schema: schema, schema_overrides: schema_overrides)
+        self._df = Utils.hash_to_rbdf({}, schema: schema, schema_overrides: schema_overrides)
       elsif data.is_a?(Hash)
         data = data.transform_keys { |v| v.is_a?(Symbol) ? v.to_s : v }
-        self._df = self.class.hash_to_rbdf(data, schema: schema, schema_overrides: schema_overrides, strict: strict, nan_to_null: nan_to_null)
+        self._df = Utils.hash_to_rbdf(data, schema: schema, schema_overrides: schema_overrides, strict: strict, nan_to_null: nan_to_null)
       elsif data.is_a?(::Array)
-        self._df = self.class.sequence_to_rbdf(data, schema: schema, schema_overrides: schema_overrides, strict: strict, orient: orient, infer_schema_length: infer_schema_length)
+        self._df = Utils.sequence_to_rbdf(data, schema: schema, schema_overrides: schema_overrides, strict: strict, orient: orient, infer_schema_length: infer_schema_length)
       elsif data.is_a?(Series)
-        self._df = self.class.series_to_rbdf(data, schema: schema, schema_overrides: schema_overrides, strict: strict)
+        self._df = Utils.series_to_rbdf(data, schema: schema, schema_overrides: schema_overrides, strict: strict)
       elsif data.respond_to?(:arrow_c_stream)
         # This uses the fact that RbSeries.from_arrow_c_stream will create a
         # struct-typed Series. Then we unpack that to a DataFrame.
@@ -77,6 +86,43 @@ module Polars
       end
     end
 
+    # Read a serialized DataFrame from a file.
+    #
+    # @param source [Object]
+    #   Path to a file or a file-like object (by file-like object, we refer to
+    #   objects that have a `read` method, such as a file handler or `StringIO`).
+    #
+    # @return [DataFrame]
+    #
+    # @note
+    #   Serialization is not stable across Polars versions: a LazyFrame serialized
+    #   in one Polars version may not be deserializable in another Polars version.
+    #
+    # @example
+    #   df = Polars::DataFrame.new({"a" => [1, 2, 3], "b" => [4.0, 5.0, 6.0]})
+    #   bytes = df.serialize
+    #   Polars::DataFrame.deserialize(StringIO.new(bytes))
+    #   # =>
+    #   # shape: (3, 2)
+    #   # ┌─────┬─────┐
+    #   # │ a   ┆ b   │
+    #   # │ --- ┆ --- │
+    #   # │ i64 ┆ f64 │
+    #   # ╞═════╪═════╡
+    #   # │ 1   ┆ 4.0 │
+    #   # │ 2   ┆ 5.0 │
+    #   # │ 3   ┆ 6.0 │
+    #   # └─────┴─────┘
+    def self.deserialize(source)
+      if Utils.pathlike?(source)
+        source = Utils.normalize_filepath(source)
+      end
+
+      deserializer = RbDataFrame.method(:deserialize_binary)
+
+      _from_rbdf(deserializer.(source))
+    end
+
     # @private
     def self._from_rbdf(rb_df)
       df = DataFrame.allocate
@@ -84,6 +130,45 @@ module Polars
       df
     end
 
+    # Plot data.
+    #
+    # @return [Object]
+    def plot(x = nil, y = nil, type: nil, group: nil, stacked: nil)
+      plot = DataFramePlot.new(self)
+      return plot if x.nil? && y.nil?
+
+      raise ArgumentError, "Must specify columns" if x.nil? || y.nil?
+      type ||= begin
+        if self[x].dtype.numeric? && self[y].dtype.numeric?
+          "scatter"
+        elsif self[x].dtype == String && self[y].dtype.numeric?
+          "column"
+        elsif (self[x].dtype == Date || self[x].dtype == Datetime) && self[y].dtype.numeric?
+          "line"
+        else
+          raise "Cannot determine type. Use the type option."
+        end
+      end
+
+      case type
+      when "line"
+        plot.line(x, y, color: group)
+      when "area"
+        plot.area(x, y, color: group)
+      when "pie"
+        raise ArgumentError, "Cannot use group option with pie chart" unless group.nil?
+        plot.pie(x, y)
+      when "column"
+        plot.column(x, y, color: group, stacked: stacked)
+      when "bar"
+        plot.bar(x, y, color: group, stacked: stacked)
+      when "scatter"
+        plot.scatter(x, y, color: group)
+      else
+        raise ArgumentError, "Invalid type: #{type}"
+      end
+    end
+
     # Get the shape of the DataFrame.
     #
     # @return [Array]
@@ -212,9 +297,9 @@ module Polars
     #     }
     #   )
     #   df.schema
-    #   # => {"foo"=>Polars::Int64, "bar"=>Polars::Float64, "ham"=>Polars::String}
+    #   # => Polars::Schema({"foo"=>Polars::Int64, "bar"=>Polars::Float64, "ham"=>Polars::String})
     def schema
-      columns.zip(dtypes).to_h
+      Schema.new(columns.zip(dtypes).to_h)
     end
 
     # Equal.
@@ -351,142 +436,243 @@ module Polars
     # Returns subset of the DataFrame.
     #
     # @return [Object]
-    def [](*args)
-      if args.size == 2
-        row_selection, col_selection = args
-
-        # df[.., unknown]
-        if row_selection.is_a?(Range)
-
-          # multiple slices
-          # df[.., ..]
-          if col_selection.is_a?(Range)
-            raise Todo
-          end
-        end
-
-        # df[2, ..] (select row as df)
-        if row_selection.is_a?(Integer)
-          if col_selection.is_a?(::Array)
-            df = self[0.., col_selection]
-            return df.slice(row_selection, 1)
-          end
-          # df[2, "a"]
-          if col_selection.is_a?(::String) || col_selection.is_a?(Symbol)
-            return self[col_selection][row_selection]
-          end
-        end
-
-        # column selection can be "a" and ["a", "b"]
-        if col_selection.is_a?(::String) || col_selection.is_a?(Symbol)
-          col_selection = [col_selection]
-        end
-
-        # df[.., 1]
-        if col_selection.is_a?(Integer)
-          series = to_series(col_selection)
-          return series[row_selection]
-        end
-
-        if col_selection.is_a?(::Array)
-          # df[.., [1, 2]]
-          if Utils.is_int_sequence(col_selection)
-            series_list = col_selection.map { |i| to_series(i) }
-            df = self.class.new(series_list)
-            return df[row_selection]
-          end
-        end
-
-        df = self[col_selection]
-        return df[row_selection]
-      elsif args.size == 1
-        item = args[0]
-
-        # select single column
-        # df["foo"]
-        if item.is_a?(::String) || item.is_a?(Symbol)
-          return Utils.wrap_s(_df.get_column(item.to_s))
-        end
-
-        # df[idx]
-        if item.is_a?(Integer)
-          return slice(_pos_idx(item, 0), 1)
-        end
-
-        # df[..]
-        if item.is_a?(Range)
-          return Slice.new(self).apply(item)
-        end
-
-        if item.is_a?(::Array) && item.all? { |v| Utils.strlike?(v) }
-          # select multiple columns
-          # df[["foo", "bar"]]
-          return _from_rbdf(_df.select(item.map(&:to_s)))
-        end
-
-        if Utils.is_int_sequence(item)
-          item = Series.new("", item)
-        end
-
-        if item.is_a?(Series)
-          dtype = item.dtype
-          if dtype == String
-            return _from_rbdf(_df.select(item))
-          elsif dtype == UInt32
-            return _from_rbdf(_df.take_with_series(item._s))
-          elsif [UInt8, UInt16, UInt64, Int8, Int16, Int32, Int64].include?(dtype)
-            return _from_rbdf(
-              _df.take_with_series(_pos_idxs(item, 0)._s)
-            )
-          end
-        end
-      end
-
-      # Ruby-specific
-      if item.is_a?(Expr) || item.is_a?(Series)
-        return filter(item)
-      end
-
-      raise ArgumentError, "Cannot get item of type: #{item.class.name}"
+    #
+    # @example
+    #   df = Polars::DataFrame.new(
+    #     {"a" => [1, 2, 3], "d" => [4, 5, 6], "c" => [1, 3, 2], "b" => [7, 8, 9]}
+    #   )
+    #   df[0]
+    #   # =>
+    #   # shape: (1, 4)
+    #   # ┌─────┬─────┬─────┬─────┐
+    #   # │ a   ┆ d   ┆ c   ┆ b   │
+    #   # │ --- ┆ --- ┆ --- ┆ --- │
+    #   # │ i64 ┆ i64 ┆ i64 ┆ i64 │
+    #   # ╞═════╪═════╪═════╪═════╡
+    #   # │ 1   ┆ 4   ┆ 1   ┆ 7   │
+    #   # └─────┴─────┴─────┴─────┘
+    #
+    # @example
+    #   df[0, "a"]
+    #   # => 1
+    #
+    # @example
+    #   df["a"]
+    #   # =>
+    #   # shape: (3,)
+    #   # Series: 'a' [i64]
+    #   # [
+    #   #         1
+    #   #         2
+    #   #         3
+    #   # ]
+    #
+    # @example
+    #   df[0..1]
+    #   # =>
+    #   # shape: (2, 4)
+    #   # ┌─────┬─────┬─────┬─────┐
+    #   # │ a   ┆ d   ┆ c   ┆ b   │
+    #   # │ --- ┆ --- ┆ --- ┆ --- │
+    #   # │ i64 ┆ i64 ┆ i64 ┆ i64 │
+    #   # ╞═════╪═════╪═════╪═════╡
+    #   # │ 1   ┆ 4   ┆ 1   ┆ 7   │
+    #   # │ 2   ┆ 5   ┆ 3   ┆ 8   │
+    #   # └─────┴─────┴─────┴─────┘
+    #
+    # @example
+    #   df[0..1, "a"]
+    #   # =>
+    #   # shape: (2,)
+    #   # Series: 'a' [i64]
+    #   # [
+    #   #         1
+    #   #         2
+    #   # ]
+    #
+    # @example
+    #   df[0..1, 0]
+    #   # =>
+    #   # shape: (2,)
+    #   # Series: 'a' [i64]
+    #   # [
+    #   #         1
+    #   #         2
+    #   # ]
+    #
+    # @example
+    #   df[[0, 1], [0, 1, 2]]
+    #   # =>
+    #   # shape: (2, 3)
+    #   # ┌─────┬─────┬─────┐
+    #   # │ a   ┆ d   ┆ c   │
+    #   # │ --- ┆ --- ┆ --- │
+    #   # │ i64 ┆ i64 ┆ i64 │
+    #   # ╞═════╪═════╪═════╡
+    #   # │ 1   ┆ 4   ┆ 1   │
+    #   # │ 2   ┆ 5   ┆ 3   │
+    #   # └─────┴─────┴─────┘
+    #
+    # @example
+    #   df[0..1, ["a", "c"]]
+    #   # =>
+    #   # shape: (2, 2)
+    #   # ┌─────┬─────┐
+    #   # │ a   ┆ c   │
+    #   # │ --- ┆ --- │
+    #   # │ i64 ┆ i64 │
+    #   # ╞═════╪═════╡
+    #   # │ 1   ┆ 1   │
+    #   # │ 2   ┆ 3   │
+    #   # └─────┴─────┘
+    #
+    # @example
+    #   df[0.., 0..1]
+    #   # =>
+    #   # shape: (3, 2)
+    #   # ┌─────┬─────┐
+    #   # │ a   ┆ d   │
+    #   # │ --- ┆ --- │
+    #   # │ i64 ┆ i64 │
+    #   # ╞═════╪═════╡
+    #   # │ 1   ┆ 4   │
+    #   # │ 2   ┆ 5   │
+    #   # │ 3   ┆ 6   │
+    #   # └─────┴─────┘
+    #
+    # @example
+    #   df[0.., "a".."c"]
+    #   # =>
+    #   # shape: (3, 3)
+    #   # ┌─────┬─────┬─────┐
+    #   # │ a   ┆ d   ┆ c   │
+    #   # │ --- ┆ --- ┆ --- │
+    #   # │ i64 ┆ i64 ┆ i64 │
+    #   # ╞═════╪═════╪═════╡
+    #   # │ 1   ┆ 4   ┆ 1   │
+    #   # │ 2   ┆ 5   ┆ 3   │
+    #   # │ 3   ┆ 6   ┆ 2   │
+    #   # └─────┴─────┴─────┘
+    def [](*key)
+      get_df_item_by_key(self, key)
     end
 
     # Set item.
     #
     # @return [Object]
+    #
+    # @example `df[["a", "b"]] = value`:
+    #   df = Polars::DataFrame.new({"a" => [1, 2, 3], "b" => [4, 5, 6]})
+    #   df[["a", "b"]] = [[10, 40], [20, 50], [30, 60]]
+    #   df
+    #   # =>
+    #   # shape: (3, 2)
+    #   # ┌─────┬─────┐
+    #   # │ a   ┆ b   │
+    #   # │ --- ┆ --- │
+    #   # │ i64 ┆ i64 │
+    #   # ╞═════╪═════╡
+    #   # │ 10  ┆ 40  │
+    #   # │ 20  ┆ 50  │
+    #   # │ 30  ┆ 60  │
+    #   # └─────┴─────┘
+    #
+    # @example `df[row_idx, "a"] = value`:
+    #   df[1, "a"] = 100
+    #   df
+    #   # =>
+    #   # shape: (3, 2)
+    #   # ┌─────┬─────┐
+    #   # │ a   ┆ b   │
+    #   # │ --- ┆ --- │
+    #   # │ i64 ┆ i64 │
+    #   # ╞═════╪═════╡
+    #   # │ 10  ┆ 40  │
+    #   # │ 100 ┆ 50  │
+    #   # │ 30  ┆ 60  │
+    #   # └─────┴─────┘
+    #
+    # @example `df[row_idx, col_idx] = value`:
+    #   df[0, 1] = 30
+    #   df
+    #   # =>
+    #   # shape: (3, 2)
+    #   # ┌─────┬─────┐
+    #   # │ a   ┆ b   │
+    #   # │ --- ┆ --- │
+    #   # │ i64 ┆ i64 │
+    #   # ╞═════╪═════╡
+    #   # │ 10  ┆ 30  │
+    #   # │ 100 ┆ 50  │
+    #   # │ 30  ┆ 60  │
+    #   # └─────┴─────┘
     def []=(*key, value)
-      if key.length == 1
-        key = key.first
-      elsif key.length != 2
+      if key.empty? || key.length > 2
         raise ArgumentError, "wrong number of arguments (given #{key.length + 1}, expected 2..3)"
       end
 
-      if Utils.strlike?(key)
+      if key.length == 1 && Utils.strlike?(key[0])
+        key = key[0]
+
         if value.is_a?(::Array) || (defined?(Numo::NArray) && value.is_a?(Numo::NArray))
           value = Series.new(value)
         elsif !value.is_a?(Series)
           value = Polars.lit(value)
         end
-        self._df = with_column(value.alias(key.to_s))._df
-      elsif key.is_a?(::Array)
+        self._df = with_columns(value.alias(key.to_s))._df
+
+      # df[["C", "D"]]
+      elsif key.length == 1 && key[0].is_a?(::Array)
+        key = key[0]
+
+        if !value.is_a?(::Array) || !value.all? { |v| v.is_a?(::Array) }
+          msg = "can only set multiple columns with 2D matrix"
+          raise ArgumentError, msg
+        end
+        if value.any? { |v| v.size != key.length }
+          msg = "matrix columns should be equal to list used to determine column names"
+          raise ArgumentError, msg
+        end
+
+        columns = []
+        key.each_with_index do |name, i|
+          columns << Series.new(name, value.map { |v| v[i] })
+        end
+        self._df = with_columns(columns)._df
+
+      # df[a, b]
+      else
         row_selection, col_selection = key
 
+        if (row_selection.is_a?(Series) && row_selection.dtype == Boolean) || Utils.is_bool_sequence(row_selection)
+          msg = (
+            "not allowed to set DataFrame by boolean mask in the row position" +
+            "\n\nConsider using `DataFrame.with_columns`."
+          )
+          raise TypeError, msg
+        end
+
+        # get series column selection
         if Utils.strlike?(col_selection)
           s = self[col_selection]
         elsif col_selection.is_a?(Integer)
-          raise Todo
+          s = self[0.., col_selection]
         else
-          raise ArgumentError, "column selection not understood: #{col_selection}"
+          msg = "unexpected column selection #{col_selection.inspect}"
+          raise TypeError, msg
         end
 
+        # dispatch to []= of Series to do modification
         s[row_selection] = value
 
+        # now find the location to place series
+        # df[idx]
         if col_selection.is_a?(Integer)
           replace_column(col_selection, s)
+        # df["foo"]
         elsif Utils.strlike?(col_selection)
-          replace(col_selection, s)
+          _replace(col_selection.to_s, s)
         end
-      else
-        raise Todo
       end
     end
 
@@ -495,22 +681,94 @@ module Polars
       _df.arrow_c_stream
     end
 
-    # Return the dataframe as a scalar.
+    # Get an ordered mapping of column names to their data type.
+    #
+    # @return [Schema]
+    #
+    # @note
+    #   This method is included to facilitate writing code that is generic for both
+    #   DataFrame and LazyFrame.
+    #
+    # @example Determine the schema.
+    #   df = Polars::DataFrame.new(
+    #     {
+    #       "foo" => [1, 2, 3],
+    #       "bar" => [6.0, 7.0, 8.0],
+    #       "ham" => ["a", "b", "c"]
+    #     }
+    #   )
+    #   df.collect_schema
+    #   # => Polars::Schema({"foo"=>Polars::Int64, "bar"=>Polars::Float64, "ham"=>Polars::String})
+    #
+    # @example Access various properties of the schema using the `Schema` object.
+    #   schema = df.collect_schema
+    #   schema["bar"]
+    #   # => Polars::Float64
+    #
+    # @example
+    #   schema.names
+    #   # => ["foo", "bar", "ham"]
+    #
+    # @example
+    #   schema.dtypes
+    #   # => [Polars::Int64, Polars::Float64, Polars::String]
+    #
+    # @example
+    #   schema.length
+    #   # => 3
+    def collect_schema
+      Schema.new(columns.zip(dtypes), check_dtypes: false)
+    end
+
+    # Return the DataFrame as a scalar, or return the element at the given row/column.
     #
-    # Equivalent to `df[0,0]`, with a check that the shape is (1,1).
+    # @param row [Integer]
+    #   Optional row index.
+    # @param column [Integer, String]
+    #   Optional column index or name.
     #
     # @return [Object]
     #
+    # @note
+    #   If row/col not provided, this is equivalent to `df[0,0]`, with a check that
+    #   the shape is (1,1). With row/col, this is equivalent to `df[row,col]`.
+    #
     # @example
     #   df = Polars::DataFrame.new({"a" => [1, 2, 3], "b" => [4, 5, 6]})
-    #   result = df.select((Polars.col("a") * Polars.col("b")).sum)
-    #   result.item
+    #   df.select((Polars.col("a") * Polars.col("b")).sum).item
     #   # => 32
-    def item
-      if shape != [1, 1]
-        raise ArgumentError, "Can only call .item if the dataframe is of shape (1,1), dataframe is of shape #{shape}"
+    #
+    # @example
+    #   df.item(1, 1)
+    #   # => 5
+    #
+    # @example
+    #   df.item(2, "b")
+    #   # => 6
+    def item(row = nil, column = nil)
+      if row.nil? && column.nil?
+        if shape != [1, 1]
+          msg = (
+            "can only call `.item()` if the dataframe is of shape (1, 1)," +
+            " or if explicit row/col values are provided;" +
+            " frame has shape #{shape.inspect}"
+          )
+          raise ArgumentError, msg
+        end
+        return _df.to_series(0).get_index(0)
+
+      elsif row.nil? || column.nil?
+        msg = "cannot call `.item()` with only one of `row` or `column`"
+        raise ArgumentError, msg
       end
-      self[0, 0]
+
+      s =
+        if column.is_a?(Integer)
+          _df.to_series(column)
+        else
+          _df.get_column(column)
+        end
+      s.get_index_signed(row)
     end
 
     # no to_arrow
@@ -526,9 +784,7 @@ module Polars
       end
     end
 
-    # Convert every row to a dictionary.
-    #
-    # Note that this is slow.
+    # Convert every row to a hash.
     #
     # @return [Array]
     #
@@ -538,13 +794,8 @@ module Polars
     #   # =>
     #   # [{"foo"=>1, "bar"=>4}, {"foo"=>2, "bar"=>5}, {"foo"=>3, "bar"=>6}]
     def to_hashes
-      rbdf = _df
-      names = columns
-
-      height.times.map do |i|
-        names.zip(rbdf.row_tuple(i)).to_h
-      end
-    end
+      rows(named: true)
+    end
 
     # Convert DataFrame to a 2D Numo array.
     #
@@ -597,17 +848,51 @@ module Polars
       if index < 0
         index = columns.length + index
       end
-      Utils.wrap_s(_df.select_at_idx(index))
+      Utils.wrap_s(_df.to_series(index))
+    end
+
+    # Serialize this DataFrame to a file or string.
+    #
+    # @param file [Object]
+    #   File path or writable file-like object to which the result will be written.
+    #   If set to `nil` (default), the output is returned as a string instead.
+    #
+    # @return [Object]
+    #
+    # @note
+    #   Serialization is not stable across Polars versions: a LazyFrame serialized
+    #   in one Polars version may not be deserializable in another Polars version.
+    #
+    # @example
+    #   df = Polars::DataFrame.new(
+    #     {
+    #       "foo" => [1, 2, 3],
+    #       "bar" => [6, 7, 8]
+    #     }
+    #   )
+    #   bytes = df.serialize
+    #   Polars::DataFrame.deserialize(StringIO.new(bytes))
+    #   # =>
+    #   # shape: (3, 2)
+    #   # ┌─────┬─────┐
+    #   # │ foo ┆ bar │
+    #   # │ --- ┆ --- │
+    #   # │ i64 ┆ i64 │
+    #   # ╞═════╪═════╡
+    #   # │ 1   ┆ 6   │
+    #   # │ 2   ┆ 7   │
+    #   # │ 3   ┆ 8   │
+    #   # └─────┴─────┘
+    def serialize(file = nil)
+      serializer = _df.method(:serialize_binary)
+
+      Utils.serialize_polars_object(serializer, file)
     end
 
     # Serialize to JSON representation.
     #
     # @param file [String]
     #   File path to which the result should be written.
-    # @param pretty [Boolean]
-    #   Pretty serialize json.
-    # @param row_oriented [Boolean]
-    #   Write to row oriented json. This is slower, but more common.
     #
     # @return [nil]
     #
@@ -619,16 +904,8 @@ module Polars
     #     }
     #   )
     #   df.write_json
-    #   # => "{\"columns\":[{\"name\":\"foo\",\"datatype\":\"Int64\",\"bit_settings\":\"\",\"values\":[1,2,3]},{\"name\":\"bar\",\"datatype\":\"Int64\",\"bit_settings\":\"\",\"values\":[6,7,8]}]}"
-    #
-    # @example
-    #   df.write_json(row_oriented: true)
     #   # => "[{\"foo\":1,\"bar\":6},{\"foo\":2,\"bar\":7},{\"foo\":3,\"bar\":8}]"
-    def write_json(
-      file = nil,
-      pretty: false,
-      row_oriented: false
-    )
+    def write_json(file = nil)
       if Utils.pathlike?(file)
         file = Utils.normalize_filepath(file)
       end
@@ -636,7 +913,7 @@ module Polars
       if file.nil? || to_string_io
         buf = StringIO.new
         buf.set_encoding(Encoding::BINARY)
-        _df.write_json(buf, pretty, row_oriented)
+        _df.write_json(buf)
         json_bytes = buf.string
 
         json_str = json_bytes.force_encoding(Encoding::UTF_8)
@@ -646,7 +923,7 @@ module Polars
           return json_str
         end
       else
-        _df.write_json(file, pretty, row_oriented)
+        _df.write_json(file)
       end
       nil
     end
@@ -667,26 +944,38 @@ module Polars
     #   )
     #   df.write_ndjson
     #   # => "{\"foo\":1,\"bar\":6}\n{\"foo\":2,\"bar\":7}\n{\"foo\":3,\"bar\":8}\n"
-    def write_ndjson(file = nil)
-      if Utils.pathlike?(file)
-        file = Utils.normalize_filepath(file)
+    def write_ndjson(
+      file = nil,
+      compression: "uncompressed",
+      compression_level: nil,
+      check_extension: true
+    )
+      should_return_buffer = false
+      if file.nil?
+        target = StringIO.new
+        target.set_encoding(Encoding::BINARY)
+        should_return_buffer = true
+      elsif Utils.pathlike?(file)
+        target = Utils.normalize_filepath(file)
+      else
+        target = file
       end
-      to_string_io = !file.nil? && file.is_a?(StringIO)
-      if file.nil? || to_string_io
-        buf = StringIO.new
-        buf.set_encoding(Encoding::BINARY)
-        _df.write_ndjson(buf)
-        json_bytes = buf.string
 
-        json_str = json_bytes.force_encoding(Encoding::UTF_8)
-        if to_string_io
-          file.write(json_str)
-        else
-          return json_str
-        end
-      else
-        _df.write_ndjson(file)
+      engine = "in-memory"
+
+      lazy.sink_ndjson(
+        target,
+        compression: compression,
+        compression_level: compression_level,
+        check_extension: check_extension,
+        optimizations: QueryOptFlags._eager,
+        engine: engine
+      )
+
+      if should_return_buffer
+        return target.string.force_encoding(Encoding::UTF_8)
       end
+
       nil
     end
 
@@ -695,11 +984,11 @@ module Polars
     # @param file [String, nil]
     #   File path to which the result should be written. If set to `nil`
     #   (default), the output is returned as a string instead.
-    # @param has_header [Boolean]
+    # @param include_header [Boolean]
     #   Whether to include header in the CSV output.
-    # @param sep [String]
+    # @param separator [String]
     #   Separate CSV fields with this symbol.
-    # @param quote [String]
+    # @param quote_char [String]
     #   Byte to use as quoting character.
     # @param batch_size [Integer]
     #   Number of rows that will be processed per thread.
@@ -718,8 +1007,8 @@ module Polars
     #   [chrono](https://docs.rs/chrono/latest/chrono/format/strftime/index.html)
     #   Rust crate.
     # @param float_precision [Integer, nil]
-    #   Number of decimal places to write, applied to both `:f32` and
-    #   `:f64` datatypes.
+    #   Number of decimal places to write, applied to both `Float32` and
+    #   `Float64` datatypes.
     # @param null_value [String, nil]
     #   A string representing null values (defaulting to the empty string).
     #
@@ -736,61 +1025,71 @@ module Polars
     #   df.write_csv("file.csv")
     def write_csv(
       file = nil,
-      has_header: true,
-      include_header: nil,
-      sep: ",",
-      quote: '"',
+      include_bom: false,
+      compression: "uncompressed",
+      compression_level: nil,
+      check_extension: true,
+      include_header: true,
+      separator: ",",
+      line_terminator: "\n",
+      quote_char: '"',
       batch_size: 1024,
       datetime_format: nil,
       date_format: nil,
       time_format: nil,
+      float_scientific: nil,
       float_precision: nil,
-      null_value: nil
+      decimal_comma: false,
+      null_value: nil,
+      quote_style: nil,
+      storage_options: nil,
+      credential_provider: "auto",
+      retries: nil
     )
-      include_header = has_header if include_header.nil?
-
-      if sep.length > 1
-        raise ArgumentError, "only single byte separator is allowed"
-      elsif quote.length > 1
-        raise ArgumentError, "only single byte quote char is allowed"
-      elsif null_value == ""
+      Utils._check_arg_is_1byte("separator", separator, false)
+      Utils._check_arg_is_1byte("quote_char", quote_char, true)
+      if null_value == ""
         null_value = nil
       end
 
+      should_return_buffer = false
       if file.nil?
-        buffer = StringIO.new
-        buffer.set_encoding(Encoding::BINARY)
-        _df.write_csv(
-          buffer,
-          include_header,
-          sep.ord,
-          quote.ord,
-          batch_size,
-          datetime_format,
-          date_format,
-          time_format,
-          float_precision,
-          null_value
-        )
-        return buffer.string.force_encoding(Encoding::UTF_8)
+        target = StringIO.new
+        target.set_encoding(Encoding::BINARY)
+        should_return_buffer = true
+      else
+        target = file
       end
 
-      if Utils.pathlike?(file)
-        file = Utils.normalize_filepath(file)
+      engine = "in-memory"
+
+      lazy.sink_csv(
+        target,
+        include_bom: include_bom,
+        include_header: include_header,
+        separator: separator,
+        line_terminator: line_terminator,
+        quote_char: quote_char,
+        batch_size: batch_size,
+        datetime_format: datetime_format,
+        date_format: date_format,
+        time_format: time_format,
+        float_scientific: float_scientific,
+        float_precision: float_precision,
+        decimal_comma: decimal_comma,
+        null_value: null_value,
+        quote_style: quote_style,
+        storage_options: storage_options,
+        credential_provider: credential_provider,
+        retries: retries,
+        optimizations: QueryOptFlags._eager,
+        engine: engine
+      )
+
+      if should_return_buffer
+        return target.string.force_encoding(Encoding::UTF_8)
       end
 
-      _df.write_csv(
-        file,
-        include_header,
-        sep.ord,
-        quote.ord,
-        batch_size,
-        datetime_format,
-        date_format,
-        time_format,
-        float_precision,
-        null_value,
-      )
       nil
     end
 
@@ -807,6 +1106,8 @@ module Polars
     #   File path to which the file should be written.
     # @param compression ["uncompressed", "snappy", "deflate"]
     #   Compression method. Defaults to "uncompressed".
+    # @param name [String]
+    #   Schema name. Defaults to empty string.
     #
     # @return [nil]
     def write_avro(file, compression = "uncompressed", name: "")
@@ -829,28 +1130,59 @@ module Polars
     #   File path to which the file should be written.
     # @param compression ["uncompressed", "lz4", "zstd"]
     #   Compression method. Defaults to "uncompressed".
+    # @param compat_level [Object]
+    #   Use a specific compatibility level
+    #   when exporting Polars' internal data structures.
+    # @param storage_options [Hash]
+    #   Options that indicate how to connect to a cloud provider.
+    #
+    #   The cloud providers currently supported are AWS, GCP, and Azure.
+    #   See supported keys here:
+    #
+    #   * [aws](https://docs.rs/object_store/latest/object_store/aws/enum.AmazonS3ConfigKey.html)
+    #   * [gcp](https://docs.rs/object_store/latest/object_store/gcp/enum.GoogleConfigKey.html)
+    #   * [azure](https://docs.rs/object_store/latest/object_store/azure/enum.AzureConfigKey.html)
+    #   * Hugging Face (`hf://`): Accepts an API key under the `token` parameter: `{'token': '...'}`, or by setting the `HF_TOKEN` environment variable.
+    #
+    #   If `storage_options` is not provided, Polars will try to infer the
+    #   information from environment variables.
+    # @param credential_provider [Object]
+    #   Provide a function that can be called to provide cloud storage
+    #   credentials. The function is expected to return a hash of
+    #   credential keys along with an optional credential expiry time.
+    # @param retries [Integer]
+    #   Number of retries if accessing a cloud instance fails.
     #
     # @return [nil]
-    def write_ipc(file, compression: "uncompressed", compat_level: nil)
+    def write_ipc(
+      file,
+      compression: "uncompressed",
+      compat_level: nil,
+      record_batch_size: nil,
+      storage_options: nil,
+      credential_provider: "auto",
+      retries: nil
+    )
       return_bytes = file.nil?
-      if return_bytes
-        file = StringIO.new
-        file.set_encoding(Encoding::BINARY)
-      end
-      if Utils.pathlike?(file)
-        file = Utils.normalize_filepath(file)
-      end
-
-      if compat_level.nil?
-        compat_level = true
-      end
-
-      if compression.nil?
-        compression = "uncompressed"
+      if file.nil?
+        target = StringIO.new
+        target.set_encoding(Encoding::BINARY)
+      else
+        target = file
       end
 
-      _df.write_ipc(file, compression, compat_level)
-      return_bytes ? file.string : nil
+      lazy.sink_ipc(
+        target,
+        compression: compression,
+        compat_level: compat_level,
+        record_batch_size: record_batch_size,
+        storage_options: storage_options,
+        credential_provider: credential_provider,
+        retries: retries,
+        optimizations: QueryOptFlags._eager,
+        engine: "streaming"
+      )
+      return_bytes ? target.string : nil
     end
 
     # Write to Arrow IPC record batch stream.
@@ -859,9 +1191,12 @@ module Polars
     #
     # @param file [Object]
     #   Path or writable file-like object to which the IPC record batch data will
-    #   be written. If set to `None`, the output is returned as a BytesIO object.
+    #   be written. If set to `nil`, the output is returned as a BytesIO object.
     # @param compression ['uncompressed', 'lz4', 'zstd']
     #   Compression method. Defaults to "uncompressed".
+    # @param compat_level [Object]
+    #   Use a specific compatibility level
+    #   when exporting Polars' internal data structures.
     #
     # @return [Object]
     #
@@ -927,9 +1262,17 @@ module Polars
       file,
       compression: "zstd",
       compression_level: nil,
-      statistics: false,
+      statistics: true,
       row_group_size: nil,
-      data_page_size: nil
+      data_page_size: nil,
+      partition_by: nil,
+      partition_chunk_size_bytes: 4_294_967_296,
+      storage_options: nil,
+      credential_provider: "auto",
+      retries: nil,
+      metadata: nil,
+      arrow_schema: nil,
+      mkdir: false
     )
       if compression.nil?
         compression = "uncompressed"
@@ -938,29 +1281,252 @@ module Polars
         file = Utils.normalize_filepath(file)
       end
 
-      if statistics == true
-        statistics = {
-          min: true,
-          max: true,
-          distinct_count: false,
-          null_count: true
-        }
-      elsif statistics == false
-        statistics = {}
-      elsif statistics == "full"
-        statistics = {
-          min: true,
-          max: true,
-          distinct_count: true,
-          null_count: true
-        }
+      target = file
+      engine = "streaming"
+      if !partition_by.nil?
+        raise Todo
       end
 
-      _df.write_parquet(
-        file, compression, compression_level, statistics, row_group_size, data_page_size
+      lazy.sink_parquet(
+        target,
+        compression: compression,
+        compression_level: compression_level,
+        statistics: statistics,
+        row_group_size: row_group_size,
+        data_page_size: data_page_size,
+        storage_options: storage_options,
+        credential_provider: credential_provider,
+        retries: retries,
+        metadata: metadata,
+        arrow_schema: arrow_schema,
+        engine: engine,
+        mkdir: mkdir,
+        optimizations: QueryOptFlags._eager
       )
     end
 
+    # Write the data in a Polars DataFrame to a database.
+    #
+    # @param table_name [String]
+    #   Schema-qualified name of the table to create or append to in the target
+    #   SQL database.
+    # @param connection [Object]
+    #   An existing Active Record connection against the target database.
+    # @param if_table_exists ['append', 'replace', 'fail']
+    #   The insert mode:
+    #
+    #   * 'replace' will create a new database table, overwriting an existing one.
+    #   * 'append' will append to an existing table.
+    #   * 'fail' will fail if table already exists.
+    #
+    # @return [Integer]
+    #
+    # @note
+    #   This functionality is experimental. It may be changed at any point without it being considered a breaking change.
+    def write_database(table_name, connection = nil, if_table_exists: "fail")
+      if !defined?(ActiveRecord)
+        raise Error, "Active Record not available"
+      elsif ActiveRecord::VERSION::MAJOR < 7
+        raise Error, "Requires Active Record 7+"
+      end
+
+      valid_write_modes = ["append", "replace", "fail"]
+      if !valid_write_modes.include?(if_table_exists)
+        msg = "write_database `if_table_exists` must be one of #{valid_write_modes.inspect}, got #{if_table_exists.inspect}"
+        raise ArgumentError, msg
+      end
+
+      with_connection(connection) do |connection|
+        table_exists = connection.table_exists?(table_name)
+        if table_exists && if_table_exists == "fail"
+          raise ArgumentError, "Table already exists"
+        end
+
+        create_table = !table_exists || if_table_exists == "replace"
+        maybe_transaction(connection, create_table) do
+          if create_table
+            mysql = connection.adapter_name.match?(/mysql|trilogy/i)
+            force = if_table_exists == "replace"
+            connection.create_table(table_name, id: false, force: force) do |t|
+              schema.each do |c, dtype|
+                options = {}
+                column_type =
+                  case dtype
+                  when Binary
+                    :binary
+                  when Boolean
+                    :boolean
+                  when Date
+                    :date
+                  when Datetime
+                    :datetime
+                  when Decimal
+                    if mysql
+                      options[:precision] = dtype.precision || 65
+                      options[:scale] = dtype.scale || 30
+                    end
+                    :decimal
+                  when Float32
+                    options[:limit] = 24
+                    :float
+                  when Float64
+                    options[:limit] = 53
+                    :float
+                  when Int8
+                    options[:limit] = 1
+                    :integer
+                  when Int16
+                    options[:limit] = 2
+                    :integer
+                  when Int32
+                    options[:limit] = 4
+                    :integer
+                  when Int64
+                    options[:limit] = 8
+                    :integer
+                  when UInt8
+                    if mysql
+                      options[:limit] = 1
+                      options[:unsigned] = true
+                    else
+                      options[:limit] = 2
+                    end
+                    :integer
+                  when UInt16
+                    if mysql
+                      options[:limit] = 2
+                      options[:unsigned] = true
+                    else
+                      options[:limit] = 4
+                    end
+                    :integer
+                  when UInt32
+                    if mysql
+                      options[:limit] = 4
+                      options[:unsigned] = true
+                    else
+                      options[:limit] = 8
+                    end
+                    :integer
+                  when UInt64
+                    if mysql
+                      options[:limit] = 8
+                      options[:unsigned] = true
+                      :integer
+                    else
+                      options[:precision] = 20
+                      options[:scale] = 0
+                      :decimal
+                    end
+                  when String
+                    :text
+                  when Time
+                    :time
+                  else
+                    raise ArgumentError, "column type not supported yet: #{dtype}"
+                  end
+                t.column c, column_type, **options
+              end
+            end
+          end
+
+          quoted_table = connection.quote_table_name(table_name)
+          quoted_columns = columns.map { |c| connection.quote_column_name(c) }
+          rows = cast({Polars::UInt64 => Polars::String}).rows(named: false).map { |row| "(#{row.map { |v| connection.quote(v) }.join(", ")})" }
+          connection.exec_update("INSERT INTO #{quoted_table} (#{quoted_columns.join(", ")}) VALUES #{rows.join(", ")}")
+        end
+      end
+    end
+
+    # Write DataFrame to an Iceberg table.
+    #
+    # @note
+    #   This functionality is currently considered **unstable**. It may be
+    #   changed at any point without it being considered a breaking change.
+    #
+    # @param target [Object]
+    #   Name of the table or the Table object representing an Iceberg table.
+    # @param mode ['append', 'overwrite']
+    #   How to handle existing data.
+    #
+    #   - If 'append', will add new data.
+    #   - If 'overwrite', will replace table with new data.
+    #
+    # @return [nil]
+    def write_iceberg(target, mode:)
+      require "iceberg"
+
+      table =
+        if target.is_a?(Iceberg::Table)
+          target
+        else
+          raise Todo
+        end
+
+      data = self
+
+      if mode == "append"
+        table.append(data)
+      else
+        raise Todo
+      end
+    end
+
+    # Write DataFrame as delta table.
+    #
+    # @param target [Object]
+    #   URI of a table or a DeltaTable object.
+    # @param mode ["error", "append", "overwrite", "ignore", "merge"]
+    #   How to handle existing data.
+    # @param storage_options [Hash]
+    #   Extra options for the storage backends supported by `deltalake-rb`.
+    # @param delta_write_options [Hash]
+    #   Additional keyword arguments while writing a Delta lake Table.
+    # @param delta_merge_options [Hash]
+    #   Keyword arguments which are required to `MERGE` a Delta lake Table.
+    #
+    # @return [nil]
+    def write_delta(
+      target,
+      mode: "error",
+      storage_options: nil,
+      delta_write_options: nil,
+      delta_merge_options: nil
+    )
+      Polars.send(:_check_if_delta_available)
+
+      if Utils.pathlike?(target)
+        target = Polars.send(:_resolve_delta_lake_uri, target.to_s, strict: false)
+      end
+
+      data = self
+
+      if mode == "merge"
+        if delta_merge_options.nil?
+          msg = "You need to pass delta_merge_options with at least a given predicate for `MERGE` to work."
+          raise ArgumentError, msg
+        end
+        if target.is_a?(::String)
+          dt = DeltaLake::Table.new(target, storage_options: storage_options)
+        else
+          dt = target
+        end
+
+        predicate = delta_merge_options.delete(:predicate)
+        dt.merge(data, predicate, **delta_merge_options)
+      else
+        delta_write_options ||= {}
+
+        DeltaLake.write(
+          target,
+          data,
+          mode: mode,
+          storage_options: storage_options,
+          **delta_write_options
+        )
+      end
+    end
+
     # Return an estimation of the total (heap) allocated size of the DataFrame.
     #
     # Estimated size is given in the specified unit (bytes by default).
@@ -988,7 +1554,7 @@ module Polars
     #       "y" => 1_000_000.times.map { |v| v / 1000.0 },
     #       "z" => 1_000_000.times.map(&:to_s)
     #     },
-    #     columns: {"x" => :u32, "y" => :f64, "z" => :str}
+    #     schema: {"x" => Polars::UInt32, "y" => Polars::Float64, "z" => Polars::String}
     #   )
     #   df.estimated_size
     #   # => 25888898
@@ -1120,14 +1686,14 @@ module Polars
     #   # │ 3     ┆ 8   ┆ c   │
     #   # └───────┴─────┴─────┘
     def rename(mapping, strict: true)
-      lazy.rename(mapping, strict: strict).collect(no_optimization: true)
+      lazy.rename(mapping, strict: strict).collect(optimizations: QueryOptFlags._eager)
     end
 
     # Insert a Series at a certain column index. This operation is in place.
     #
     # @param index [Integer]
     #   Column to insert the new `Series` column.
-    # @param series [Series]
+    # @param column [Series]
     #   `Series` to insert.
     #
     # @return [DataFrame]
@@ -1170,19 +1736,22 @@ module Polars
     #   # │ 3   ┆ 10.0 ┆ false ┆ 20.5 │
     #   # │ 4   ┆ 13.0 ┆ true  ┆ 0.0  │
     #   # └─────┴──────┴───────┴──────┘
-    def insert_column(index, series)
+    def insert_column(index, column)
       if index < 0
-        index = columns.length + index
+        index = width + index
       end
-      _df.insert_column(index, series._s)
+      _df.insert_column(index, column._s)
       self
     end
-    alias_method :insert_at_idx, :insert_column
 
     # Filter the rows in the DataFrame based on a predicate expression.
     #
-    # @param predicate [Expr]
-    #   Expression that evaluates to a boolean Series.
+    # @param predicates [Array]
+    #   Expression(s) that evaluate to a boolean Series.
+    # @param constraints [Hash]
+    #   Column filters; use `name = value` to filter columns by the supplied value.
+    #   Each constraint will behave the same as `Polars.col(name).eq(value)`, and
+    #   be implicitly joined with the other filter conditions using `&`.
     #
     # @return [DataFrame]
     #
@@ -1217,86 +1786,307 @@ module Polars
     #   # ╞═════╪═════╪═════╡
     #   # │ 1   ┆ 6   ┆ a   │
     #   # └─────┴─────┴─────┘
-    def filter(predicate)
-      lazy.filter(predicate).collect
+    def filter(*predicates, **constraints)
+      lazy.filter(*predicates, **constraints).collect(optimizations: QueryOptFlags._eager)
     end
 
-    # Summary statistics for a DataFrame.
+    # Remove rows, dropping those that match the given predicate expression(s).
+    #
+    # The original order of the remaining rows is preserved.
+    #
+    # Rows where the filter predicate does not evaluate to true are retained
+    # (this includes rows where the predicate evaluates as `null`).
+    #
+    # @param predicates [Array]
+    #   Expression that evaluates to a boolean Series.
+    # @param constraints [Hash]
+    #   Column filters; use `name = value` to filter columns using the supplied
+    #   value. Each constraint behaves the same as `Polars.col(name).eq(value)`,
+    #   and is implicitly joined with the other filter conditions using `&`.
     #
     # @return [DataFrame]
     #
+    # @example Remove rows matching a condition:
+    #   df = Polars::DataFrame.new(
+    #     {
+    #       "foo" => [2, 3, nil, 4, 0],
+    #       "bar" => [5, 6, nil, nil, 0],
+    #       "ham" => ["a", "b", nil, "c", "d"]
+    #     }
+    #   )
+    #   df.remove(Polars.col("bar") >= 5)
+    #   # =>
+    #   # shape: (3, 3)
+    #   # ┌──────┬──────┬──────┐
+    #   # │ foo  ┆ bar  ┆ ham  │
+    #   # │ ---  ┆ ---  ┆ ---  │
+    #   # │ i64  ┆ i64  ┆ str  │
+    #   # ╞══════╪══════╪══════╡
+    #   # │ null ┆ null ┆ null │
+    #   # │ 4    ┆ null ┆ c    │
+    #   # │ 0    ┆ 0    ┆ d    │
+    #   # └──────┴──────┴──────┘
+    #
+    # @example Discard rows based on multiple conditions, combined with and/or operators:
+    #   df.remove(
+    #     (Polars.col("foo") >= 0) & (Polars.col("bar") >= 0),
+    #   )
+    #   # =>
+    #   # shape: (2, 3)
+    #   # ┌──────┬──────┬──────┐
+    #   # │ foo  ┆ bar  ┆ ham  │
+    #   # │ ---  ┆ ---  ┆ ---  │
+    #   # │ i64  ┆ i64  ┆ str  │
+    #   # ╞══════╪══════╪══════╡
+    #   # │ null ┆ null ┆ null │
+    #   # │ 4    ┆ null ┆ c    │
+    #   # └──────┴──────┴──────┘
+    #
     # @example
+    #   df.remove(
+    #     (Polars.col("foo") >= 0) | (Polars.col("bar") >= 0),
+    #   )
+    #   # =>
+    #   # shape: (1, 3)
+    #   # ┌──────┬──────┬──────┐
+    #   # │ foo  ┆ bar  ┆ ham  │
+    #   # │ ---  ┆ ---  ┆ ---  │
+    #   # │ i64  ┆ i64  ┆ str  │
+    #   # ╞══════╪══════╪══════╡
+    #   # │ null ┆ null ┆ null │
+    #   # └──────┴──────┴──────┘
+    #
+    # @example Provide multiple constraints using `*args` syntax:
+    #   df.remove(
+    #     Polars.col("ham").is_not_null,
+    #     Polars.col("bar") >= 0
+    #   )
+    #   # =>
+    #   # shape: (2, 3)
+    #   # ┌──────┬──────┬──────┐
+    #   # │ foo  ┆ bar  ┆ ham  │
+    #   # │ ---  ┆ ---  ┆ ---  │
+    #   # │ i64  ┆ i64  ┆ str  │
+    #   # ╞══════╪══════╪══════╡
+    #   # │ null ┆ null ┆ null │
+    #   # │ 4    ┆ null ┆ c    │
+    #   # └──────┴──────┴──────┘
+    #
+    # @example Provide constraints(s) using `**kwargs` syntax:
+    #   df.remove(foo: 0, bar: 0)
+    #   # =>
+    #   # shape: (4, 3)
+    #   # ┌──────┬──────┬──────┐
+    #   # │ foo  ┆ bar  ┆ ham  │
+    #   # │ ---  ┆ ---  ┆ ---  │
+    #   # │ i64  ┆ i64  ┆ str  │
+    #   # ╞══════╪══════╪══════╡
+    #   # │ 2    ┆ 5    ┆ a    │
+    #   # │ 3    ┆ 6    ┆ b    │
+    #   # │ null ┆ null ┆ null │
+    #   # │ 4    ┆ null ┆ c    │
+    #   # └──────┴──────┴──────┘
+    #
+    # @example Remove rows by comparing two columns against each other:
+    #   df.remove(
+    #     Polars.col("foo").ne_missing(Polars.col("bar"))
+    #   )
+    #   # =>
+    #   # shape: (2, 3)
+    #   # ┌──────┬──────┬──────┐
+    #   # │ foo  ┆ bar  ┆ ham  │
+    #   # │ ---  ┆ ---  ┆ ---  │
+    #   # │ i64  ┆ i64  ┆ str  │
+    #   # ╞══════╪══════╪══════╡
+    #   # │ null ┆ null ┆ null │
+    #   # │ 0    ┆ 0    ┆ d    │
+    #   # └──────┴──────┴──────┘
+    def remove(
+      *predicates,
+      **constraints
+    )
+      lazy
+      .remove(*predicates, **constraints)
+      .collect(optimizations: QueryOptFlags._eager)
+    end
+
+    # Return a dense preview of the DataFrame.
+    #
+    # The formatting shows one line per column so that wide dataframes display
+    # cleanly. Each line shows the column name, the data type, and the first
+    # few values.
+    #
+    # @param max_items_per_column [Integer]
+    #   Maximum number of items to show per column.
+    # @param max_colname_length [Integer]
+    #   Maximum length of the displayed column names; values that exceed
+    #   this value are truncated with a trailing ellipsis.
+    # @param return_type [nil, 'self', 'frame', 'string']
+    #   Modify the return format:
+    #
+    #   - `nil` (default): Print the glimpse output to stdout, returning `nil`.
+    #   - `"self"`: Print the glimpse output to stdout, returning the *original* frame.
+    #   - `"frame"`: Return the glimpse output as a new DataFrame.
+    #   - `"string"`: Return the glimpse output as a string.
+    #
+    # @return [Object]
+    #
+    # @example Return the glimpse output as a DataFrame:
     #   df = Polars::DataFrame.new(
     #     {
     #       "a" => [1.0, 2.8, 3.0],
     #       "b" => [4, 5, nil],
     #       "c" => [true, false, true],
     #       "d" => [nil, "b", "c"],
-    #       "e" => ["usd", "eur", nil]
+    #       "e" => ["usd", "eur", nil],
+    #       "f" => [Date.new(2020, 1, 1), Date.new(2021, 1, 2), Date.new(2022, 1, 1)]
     #     }
     #   )
-    #   df.describe
+    #   df.glimpse(return_type: "frame")
     #   # =>
-    #   # shape: (7, 6)
-    #   # ┌────────────┬──────────┬──────────┬──────────┬──────┬──────┐
-    #   # │ describe   ┆ a        ┆ b        ┆ c        ┆ d    ┆ e    │
-    #   # │ ---        ┆ ---      ┆ ---      ┆ ---      ┆ ---  ┆ ---  │
-    #   # │ str        ┆ f64      ┆ f64      ┆ f64      ┆ str  ┆ str  │
-    #   # ╞════════════╪══════════╪══════════╪══════════╪══════╪══════╡
-    #   # │ count      ┆ 3.0      ┆ 3.0      ┆ 3.0      ┆ 3    ┆ 3    │
-    #   # │ null_count ┆ 0.0      ┆ 1.0      ┆ 0.0      ┆ 1    ┆ 1    │
-    #   # │ mean       ┆ 2.266667 ┆ 4.5      ┆ 0.666667 ┆ null ┆ null │
-    #   # │ std        ┆ 1.101514 ┆ 0.707107 ┆ 0.57735  ┆ null ┆ null │
-    #   # │ min        ┆ 1.0      ┆ 4.0      ┆ 0.0      ┆ b    ┆ eur  │
-    #   # │ max        ┆ 3.0      ┆ 5.0      ┆ 1.0      ┆ c    ┆ usd  │
-    #   # │ median     ┆ 2.8      ┆ 4.5      ┆ 1.0      ┆ null ┆ null │
-    #   # └────────────┴──────────┴──────────┴──────────┴──────┴──────┘
-    def describe
-      describe_cast = lambda do |stat|
-        columns = []
-        self.columns.each_with_index do |s, i|
-          if self[s].is_numeric || self[s].is_boolean
-            columns << stat[0.., i].cast(:f64)
-          else
-            # for dates, strings, etc, we cast to string so that all
-            # statistics can be shown
-            columns << stat[0.., i].cast(:str)
-          end
+    #   # shape: (6, 3)
+    #   # ┌────────┬───────┬─────────────────────────────────┐
+    #   # │ column ┆ dtype ┆ values                          │
+    #   # │ ---    ┆ ---   ┆ ---                             │
+    #   # │ str    ┆ str   ┆ list[str]                       │
+    #   # ╞════════╪═══════╪═════════════════════════════════╡
+    #   # │ a      ┆ f64   ┆ ["1.0", "2.8", "3.0"]           │
+    #   # │ b      ┆ i64   ┆ ["4", "5", null]                │
+    #   # │ c      ┆ bool  ┆ ["true", "false", "true"]       │
+    #   # │ d      ┆ str   ┆ [null, ""b"", ""c""]            │
+    #   # │ e      ┆ str   ┆ [""usd"", ""eur"", null]        │
+    #   # │ f      ┆ date  ┆ ["2020-01-01", "2021-01-02", "… │
+    #   # └────────┴───────┴─────────────────────────────────┘
+    def glimpse(
+      max_items_per_column: 10,
+      max_colname_length: 50,
+      return_type: nil
+    )
+      if return_type.nil?
+        return_frame = false
+      else
+        return_frame = return_type == "frame"
+        if !return_frame && !["self", "string"].include?(return_type)
+          msg = "invalid `return_type`; found #{return_type.inspect}, expected one of 'string', 'frame', 'self', or nil"
+          raise ArgumentError, msg
         end
-        self.class.new(columns)
       end
 
-      summary = _from_rbdf(
-        Polars.concat(
-          [
-            describe_cast.(
-              self.class.new(columns.to_h { |c| [c, [height]] })
-            ),
-            describe_cast.(null_count),
-            describe_cast.(mean),
-            describe_cast.(std),
-            describe_cast.(min),
-            describe_cast.(max),
-            describe_cast.(median)
-          ]
-        )._df
-      )
-      summary.insert_column(
-        0,
-        Polars::Series.new(
-          "describe",
-          ["count", "null_count", "mean", "std", "min", "max", "median"],
+      # always print at most this number of values (mainly ensures that
+      # we do not cast long arrays to strings, which would be slow)
+      max_n_values = [max_items_per_column, height].min
+      schema = self.schema
+
+      _column_to_row_output = lambda do |col_name, dtype|
+        fn = schema[col_name] == String ? :inspect : :to_s
+        values = self[0...max_n_values, col_name].to_a
+        if col_name.length > max_colname_length
+          col_name = col_name[0...(max_colname_length - 1)] + "…"
+        end
+        dtype_str = Plr.dtype_str_repr(dtype)
+        if !return_frame
+          dtype_str = "<#{dtype_str}>"
+        end
+        [col_name, dtype_str, values.map { |v| !v.nil? ? v.send(fn) : nil }]
+      end
+
+      data = self.schema.map { |s, dtype| _column_to_row_output.(s, dtype) }
+
+      # output one row per column
+      if return_frame
+        DataFrame.new(
+          data,
+          orient: "row",
+          schema: {"column" => String, "dtype" => String, "values" => List.new(String)}
         )
-      )
-      summary
+      else
+        raise Todo
+      end
     end
 
-    # Find the index of a column by name.
+    # Summary statistics for a DataFrame.
     #
-    # @param name [String]
-    #   Name of the column to find.
+    # @param percentiles [Array]
+    #   One or more percentiles to include in the summary statistics.
+    #   All values must be in the range `[0, 1]`.
+    # @param interpolation ['nearest', 'higher', 'lower', 'midpoint', 'linear', 'equiprobable']
+    #   Interpolation method used when calculating percentiles.
     #
-    # @return [Series]
+    # @return [DataFrame]
+    #
+    # @example Show default frame statistics:
+    #   df = Polars::DataFrame.new(
+    #     {
+    #       "float" => [1.0, 2.8, 3.0],
+    #       "int" => [40, 50, nil],
+    #       "bool" => [true, false, true],
+    #       "str" => ["zz", "xx", "yy"],
+    #       "date" => [Date.new(2020, 1, 1), Date.new(2021, 7, 5), Date.new(2022, 12, 31)]
+    #     }
+    #   )
+    #   df.describe
+    #   # =>
+    #   # shape: (9, 6)
+    #   # ┌────────────┬──────────┬──────────┬──────────┬──────┬─────────────────────────┐
+    #   # │ statistic  ┆ float    ┆ int      ┆ bool     ┆ str  ┆ date                    │
+    #   # │ ---        ┆ ---      ┆ ---      ┆ ---      ┆ ---  ┆ ---                     │
+    #   # │ str        ┆ f64      ┆ f64      ┆ f64      ┆ str  ┆ str                     │
+    #   # ╞════════════╪══════════╪══════════╪══════════╪══════╪═════════════════════════╡
+    #   # │ count      ┆ 3.0      ┆ 2.0      ┆ 3.0      ┆ 3    ┆ 3                       │
+    #   # │ null_count ┆ 0.0      ┆ 1.0      ┆ 0.0      ┆ 0    ┆ 0                       │
+    #   # │ mean       ┆ 2.266667 ┆ 45.0     ┆ 0.666667 ┆ null ┆ 2021-07-02 16:00:00 UTC │
+    #   # │ std        ┆ 1.101514 ┆ 7.071068 ┆ null     ┆ null ┆ null                    │
+    #   # │ min        ┆ 1.0      ┆ 40.0     ┆ 0.0      ┆ xx   ┆ 2020-01-01              │
+    #   # │ 25%        ┆ 2.8      ┆ 40.0     ┆ null     ┆ null ┆ 2021-07-05              │
+    #   # │ 50%        ┆ 2.8      ┆ 50.0     ┆ null     ┆ null ┆ 2021-07-05              │
+    #   # │ 75%        ┆ 3.0      ┆ 50.0     ┆ null     ┆ null ┆ 2022-12-31              │
+    #   # │ max        ┆ 3.0      ┆ 50.0     ┆ 1.0      ┆ zz   ┆ 2022-12-31              │
+    #   # └────────────┴──────────┴──────────┴──────────┴──────┴─────────────────────────┘
+    #
+    # @example Customize which percentiles are displayed, applying linear interpolation:
+    #   df.describe(
+    #     percentiles: [0.1, 0.3, 0.5, 0.7, 0.9],
+    #     interpolation: "linear"
+    #   )
+    #   # =>
+    #   # shape: (11, 6)
+    #   # ┌────────────┬──────────┬──────────┬──────────┬──────┬─────────────────────────┐
+    #   # │ statistic  ┆ float    ┆ int      ┆ bool     ┆ str  ┆ date                    │
+    #   # │ ---        ┆ ---      ┆ ---      ┆ ---      ┆ ---  ┆ ---                     │
+    #   # │ str        ┆ f64      ┆ f64      ┆ f64      ┆ str  ┆ str                     │
+    #   # ╞════════════╪══════════╪══════════╪══════════╪══════╪═════════════════════════╡
+    #   # │ count      ┆ 3.0      ┆ 2.0      ┆ 3.0      ┆ 3    ┆ 3                       │
+    #   # │ null_count ┆ 0.0      ┆ 1.0      ┆ 0.0      ┆ 0    ┆ 0                       │
+    #   # │ mean       ┆ 2.266667 ┆ 45.0     ┆ 0.666667 ┆ null ┆ 2021-07-02 16:00:00 UTC │
+    #   # │ std        ┆ 1.101514 ┆ 7.071068 ┆ null     ┆ null ┆ null                    │
+    #   # │ min        ┆ 1.0      ┆ 40.0     ┆ 0.0      ┆ xx   ┆ 2020-01-01              │
+    #   # │ …          ┆ …        ┆ …        ┆ …        ┆ …    ┆ …                       │
+    #   # │ 30%        ┆ 2.08     ┆ 43.0     ┆ null     ┆ null ┆ 2020-11-26              │
+    #   # │ 50%        ┆ 2.8      ┆ 45.0     ┆ null     ┆ null ┆ 2021-07-05              │
+    #   # │ 70%        ┆ 2.88     ┆ 47.0     ┆ null     ┆ null ┆ 2022-02-07              │
+    #   # │ 90%        ┆ 2.96     ┆ 49.0     ┆ null     ┆ null ┆ 2022-09-13              │
+    #   # │ max        ┆ 3.0      ┆ 50.0     ┆ 1.0      ┆ zz   ┆ 2022-12-31              │
+    #   # └────────────┴──────────┴──────────┴──────────┴──────┴─────────────────────────┘
+    def describe(
+      percentiles: [0.25, 0.5, 0.75],
+      interpolation: "nearest"
+    )
+      if columns.empty?
+        msg = "cannot describe a DataFrame that has no columns"
+        raise TypeError, msg
+      end
+
+      lazy.describe(
+        percentiles: percentiles, interpolation: interpolation
+      )
+    end
+
+    # Find the index of a column by name.
+    #
+    # @param name [String]
+    #   Name of the column to find.
+    #
+    # @return [Series]
     #
     # @example
     #   df = Polars::DataFrame.new(
@@ -1307,13 +2097,12 @@ module Polars
     def get_column_index(name)
       _df.get_column_index(name)
     end
-    alias_method :find_idx_by_name, :get_column_index
 
     # Replace a column at an index location.
     #
     # @param index [Integer]
     #   Column index.
-    # @param series [Series]
+    # @param column [Series]
     #   Series that will replace the column.
     #
     # @return [DataFrame]
@@ -1339,23 +2128,31 @@ module Polars
     #   # │ 20    ┆ 7   ┆ b   │
     #   # │ 30    ┆ 8   ┆ c   │
     #   # └───────┴─────┴─────┘
-    def replace_column(index, series)
+    def replace_column(index, column)
       if index < 0
-        index = columns.length + index
+        index = width + index
       end
-      _df.replace_column(index, series._s)
+      _df.replace_column(index, column._s)
       self
     end
-    alias_method :replace_at_idx, :replace_column
 
-    # Sort the DataFrame by column.
+    # Sort the dataframe by the given columns.
     #
-    # @param by [String]
-    #   By which column to sort.
-    # @param reverse [Boolean]
-    #   Reverse/descending sort.
+    # @param by [Object]
+    #   Column(s) to sort by. Accepts expression input, including selectors. Strings
+    #   are parsed as column names.
+    # @param more_by [Array]
+    #   Additional columns to sort by, specified as positional arguments.
+    # @param descending [Boolean]
+    #   Sort in descending order. When sorting by multiple columns, can be specified
+    #   per column by passing an array of booleans.
     # @param nulls_last [Boolean]
-    #   Place null values last. Can only be used if sorted by a single column.
+    #   Place null values last; can specify a single boolean applying to all columns
+    #   or an array of booleans for per-column control.
+    # @param multithreaded [Boolean]
+    #   Sort using multiple threads.
+    # @param maintain_order [Boolean]
+    #   Whether the order should be maintained if elements are equal.
     #
     # @return [DataFrame]
     #
@@ -1367,7 +2164,7 @@ module Polars
     #       "ham" => ["a", "b", "c"]
     #     }
     #   )
-    #   df.sort("foo", reverse: true)
+    #   df.sort("foo", descending: true)
     #   # =>
     #   # shape: (3, 3)
     #   # ┌─────┬─────┬─────┐
@@ -1383,7 +2180,7 @@ module Polars
     # @example Sort by multiple columns.
     #   df.sort(
     #     [Polars.col("foo"), Polars.col("bar")**2],
-    #     reverse: [true, false]
+    #     descending: [true, false]
     #   )
     #   # =>
     #   # shape: (3, 3)
@@ -1396,24 +2193,255 @@ module Polars
     #   # │ 2   ┆ 7.0 ┆ b   │
     #   # │ 1   ┆ 6.0 ┆ a   │
     #   # └─────┴─────┴─────┘
-    def sort(by, reverse: false, nulls_last: false)
+    def sort(
+      by,
+      *more_by,
+      descending: false,
+      nulls_last: false,
+      multithreaded: true,
+      maintain_order: false
+    )
       lazy
-        .sort(by, reverse: reverse, nulls_last: nulls_last)
-        .collect(no_optimization: true)
+        .sort(
+          by,
+          *more_by,
+          descending: descending,
+          nulls_last: nulls_last,
+          multithreaded: multithreaded,
+          maintain_order: maintain_order
+        )
+        .collect(optimizations: QueryOptFlags._eager)
     end
 
     # Sort the DataFrame by column in-place.
     #
     # @param by [String]
     #   By which column to sort.
-    # @param reverse [Boolean]
+    # @param descending [Boolean]
     #   Reverse/descending sort.
     # @param nulls_last [Boolean]
     #   Place null values last. Can only be used if sorted by a single column.
     #
     # @return [DataFrame]
-    def sort!(by, reverse: false, nulls_last: false)
-      self._df = sort(by, reverse: reverse, nulls_last: nulls_last)._df
+    def sort!(by, descending: false, nulls_last: false)
+      self._df = sort(by, descending: descending, nulls_last: nulls_last)._df
+    end
+
+    # Execute a SQL query against the DataFrame.
+    #
+    # @note
+    #   This functionality is considered **unstable**, although it is close to
+    #   being considered stable. It may be changed at any point without it being
+    #   considered a breaking change.
+    #
+    # @param query [String]
+    #   SQL query to execute.
+    # @param table_name [String]
+    #   Optionally provide an explicit name for the table that represents the
+    #   calling frame (defaults to "self").
+    #
+    # @return [DataFrame]
+    #
+    # @note
+    #   * The calling frame is automatically registered as a table in the SQL context
+    #     under the name "self". If you want access to the DataFrames and LazyFrames
+    #     found in the current globals, use the top-level :meth:`pl.sql <polars.sql>`.
+    #   * More control over registration and execution behaviour is available by
+    #     using the :class:`SQLContext` object.
+    #   * The SQL query executes in lazy mode before being collected and returned
+    #     as a DataFrame.
+    #
+    # @example Query the DataFrame using SQL:
+    #   df1 = Polars::DataFrame.new(
+    #     {
+    #       "a" => [1, 2, 3],
+    #       "b" => ["zz", "yy", "xx"],
+    #       "c" => [Date.new(1999, 12, 31), Date.new(2010, 10, 10), Date.new(2077, 8, 8)]
+    #     }
+    #   )
+    #   df1.sql("SELECT c, b FROM self WHERE a > 1")
+    #   # =>
+    #   # shape: (2, 2)
+    #   # ┌────────────┬─────┐
+    #   # │ c          ┆ b   │
+    #   # │ ---        ┆ --- │
+    #   # │ date       ┆ str │
+    #   # ╞════════════╪═════╡
+    #   # │ 2010-10-10 ┆ yy  │
+    #   # │ 2077-08-08 ┆ xx  │
+    #   # └────────────┴─────┘
+    #
+    # @example Apply transformations to a DataFrame using SQL, aliasing "self" to "frame".
+    #   df1.sql(
+    #     "
+    #       SELECT
+    #           a,
+    #           (a % 2 == 0) AS a_is_even,
+    #           CONCAT_WS(':', b, b) AS b_b,
+    #           EXTRACT(year FROM c) AS year,
+    #           0::float4 AS \"zero\",
+    #       FROM frame
+    #     ",
+    #     table_name: "frame"
+    #   )
+    #   # =>
+    #   # shape: (3, 5)
+    #   # ┌─────┬───────────┬───────┬──────┬──────┐
+    #   # │ a   ┆ a_is_even ┆ b_b   ┆ year ┆ zero │
+    #   # │ --- ┆ ---       ┆ ---   ┆ ---  ┆ ---  │
+    #   # │ i64 ┆ bool      ┆ str   ┆ i32  ┆ f32  │
+    #   # ╞═════╪═══════════╪═══════╪══════╪══════╡
+    #   # │ 1   ┆ false     ┆ zz:zz ┆ 1999 ┆ 0.0  │
+    #   # │ 2   ┆ true      ┆ yy:yy ┆ 2010 ┆ 0.0  │
+    #   # │ 3   ┆ false     ┆ xx:xx ┆ 2077 ┆ 0.0  │
+    #   # └─────┴───────────┴───────┴──────┴──────┘
+    def sql(query, table_name: "self")
+      ctx = SQLContext.new(eager: true)
+      name = table_name || "self"
+      ctx.register(name, self)
+      ctx.execute(query)
+    end
+
+    # Return the `k` largest rows.
+    #
+    # Non-null elements are always preferred over null elements, regardless of
+    # the value of `reverse`. The output is not guaranteed to be in any
+    # particular order, call `sort` after this function if you wish the
+    # output to be sorted.
+    #
+    # @param k [Integer]
+    #   Number of rows to return.
+    # @param by [Object]
+    #   Column(s) used to determine the top rows.
+    #   Accepts expression input. Strings are parsed as column names.
+    # @param reverse [Object]
+    #   Consider the `k` smallest elements of the `by` column(s) (instead of the `k`
+    #   largest). This can be specified per column by passing an array of
+    #   booleans.
+    #
+    # @return [DataFrame]
+    #
+    # @example Get the rows which contain the 4 largest values in column b.
+    #   df = Polars::DataFrame.new(
+    #     {
+    #       "a" => ["a", "b", "a", "b", "b", "c"],
+    #       "b" => [2, 1, 1, 3, 2, 1]
+    #     }
+    #   )
+    #   df.top_k(4, by: "b")
+    #   # =>
+    #   # shape: (4, 2)
+    #   # ┌─────┬─────┐
+    #   # │ a   ┆ b   │
+    #   # │ --- ┆ --- │
+    #   # │ str ┆ i64 │
+    #   # ╞═════╪═════╡
+    #   # │ b   ┆ 3   │
+    #   # │ a   ┆ 2   │
+    #   # │ b   ┆ 2   │
+    #   # │ b   ┆ 1   │
+    #   # └─────┴─────┘
+    #
+    # @example Get the rows which contain the 4 largest values when sorting on column b and a.
+    #   df.top_k(4, by: ["b", "a"])
+    #   # =>
+    #   # shape: (4, 2)
+    #   # ┌─────┬─────┐
+    #   # │ a   ┆ b   │
+    #   # │ --- ┆ --- │
+    #   # │ str ┆ i64 │
+    #   # ╞═════╪═════╡
+    #   # │ b   ┆ 3   │
+    #   # │ b   ┆ 2   │
+    #   # │ a   ┆ 2   │
+    #   # │ c   ┆ 1   │
+    #   # └─────┴─────┘
+    def top_k(
+      k,
+      by:,
+      reverse: false
+    )
+      lazy
+      .top_k(k, by: by, reverse: reverse)
+      .collect(
+        optimizations: QueryOptFlags.new(
+          projection_pushdown: false,
+          predicate_pushdown: false,
+          comm_subplan_elim: false,
+          slice_pushdown: true
+        )
+      )
+    end
+
+    # Return the `k` smallest rows.
+    #
+    # Non-null elements are always preferred over null elements, regardless of
+    # the value of `reverse`. The output is not guaranteed to be in any
+    # particular order, call `sort` after this function if you wish the
+    # output to be sorted.
+    #
+    # @param k [Integer]
+    #   Number of rows to return.
+    # @param by [Object]
+    #   Column(s) used to determine the bottom rows.
+    #   Accepts expression input. Strings are parsed as column names.
+    # @param reverse [Object]
+    #   Consider the `k` largest elements of the `by` column(s) (instead of the `k`
+    #   smallest). This can be specified per column by passing an array of
+    #   booleans.
+    #
+    # @return [DataFrame]
+    #
+    # @example Get the rows which contain the 4 smallest values in column b.
+    #   df = Polars::DataFrame.new(
+    #     {
+    #       "a" => ["a", "b", "a", "b", "b", "c"],
+    #       "b" => [2, 1, 1, 3, 2, 1]
+    #     }
+    #   )
+    #   df.bottom_k(4, by: "b")
+    #   # =>
+    #   # shape: (4, 2)
+    #   # ┌─────┬─────┐
+    #   # │ a   ┆ b   │
+    #   # │ --- ┆ --- │
+    #   # │ str ┆ i64 │
+    #   # ╞═════╪═════╡
+    #   # │ b   ┆ 1   │
+    #   # │ a   ┆ 1   │
+    #   # │ c   ┆ 1   │
+    #   # │ a   ┆ 2   │
+    #   # └─────┴─────┘
+    #
+    # @example Get the rows which contain the 4 smallest values when sorting on column a and b.
+    #   df.bottom_k(4, by: ["a", "b"])
+    #   # =>
+    #   # shape: (4, 2)
+    #   # ┌─────┬─────┐
+    #   # │ a   ┆ b   │
+    #   # │ --- ┆ --- │
+    #   # │ str ┆ i64 │
+    #   # ╞═════╪═════╡
+    #   # │ a   ┆ 1   │
+    #   # │ a   ┆ 2   │
+    #   # │ b   ┆ 1   │
+    #   # │ b   ┆ 2   │
+    #   # └─────┴─────┘
+    def bottom_k(
+      k,
+      by:,
+      reverse: false
+    )
+      lazy
+      .bottom_k(k, by: by, reverse: reverse)
+      .collect(
+        optimizations: QueryOptFlags.new(
+          projection_pushdown: false,
+          predicate_pushdown: false,
+          comm_subplan_elim: false,
+          slice_pushdown: true
+        )
+      )
     end
 
     # Check if DataFrame is equal to other.
@@ -1447,36 +2475,6 @@ module Polars
     def equals(other, null_equal: true)
       _df.equals(other._df, null_equal)
     end
-    alias_method :frame_equal, :equals
-
-    # Replace a column by a new Series.
-    #
-    # @param column [String]
-    #   Column to replace.
-    # @param new_col [Series]
-    #   New column to insert.
-    #
-    # @return [DataFrame]
-    #
-    # @example
-    #   df = Polars::DataFrame.new({"foo" => [1, 2, 3], "bar" => [4, 5, 6]})
-    #   s = Polars::Series.new([10, 20, 30])
-    #   df.replace("foo", s)
-    #   # =>
-    #   # shape: (3, 2)
-    #   # ┌─────┬─────┐
-    #   # │ foo ┆ bar │
-    #   # │ --- ┆ --- │
-    #   # │ i64 ┆ i64 │
-    #   # ╞═════╪═════╡
-    #   # │ 10  ┆ 4   │
-    #   # │ 20  ┆ 5   │
-    #   # │ 30  ┆ 6   │
-    #   # └─────┴─────┘
-    def replace(column, new_col)
-      _df.replace(column.to_s, new_col._s)
-      self
-    end
 
     # Get a slice of this DataFrame.
     #
@@ -1606,10 +2604,59 @@ module Polars
       _from_rbdf(_df.tail(n))
     end
 
-    # Return a new DataFrame where the null values are dropped.
+    # Drop all rows that contain one or more NaN values.
+    #
+    # The original order of the remaining rows is preserved.
+    #
+    # @param subset [Object]
+    #   Column name(s) for which NaN values are considered; if set to `nil`
+    #   (default), use all columns (note that only floating-point columns
+    #   can contain NaNs).
+    #
+    # @return [DataFrame]
+    #
+    # @example
+    #   df = Polars::DataFrame.new(
+    #     {
+    #       "foo" => [-20.5, Float::NAN, 80.0],
+    #       "bar" => [Float::NAN, 110.0, 25.5],
+    #       "ham" => ["xxx", "yyy", nil]
+    #     }
+    #   )
+    #   df.drop_nans
+    #   # =>
+    #   # shape: (1, 3)
+    #   # ┌──────┬──────┬──────┐
+    #   # │ foo  ┆ bar  ┆ ham  │
+    #   # │ ---  ┆ ---  ┆ ---  │
+    #   # │ f64  ┆ f64  ┆ str  │
+    #   # ╞══════╪══════╪══════╡
+    #   # │ 80.0 ┆ 25.5 ┆ null │
+    #   # └──────┴──────┴──────┘
+    #
+    # @example
+    #   df.drop_nans(subset: ["bar"])
+    #   # =>
+    #   # shape: (2, 3)
+    #   # ┌──────┬───────┬──────┐
+    #   # │ foo  ┆ bar   ┆ ham  │
+    #   # │ ---  ┆ ---   ┆ ---  │
+    #   # │ f64  ┆ f64   ┆ str  │
+    #   # ╞══════╪═══════╪══════╡
+    #   # │ NaN  ┆ 110.0 ┆ yyy  │
+    #   # │ 80.0 ┆ 25.5  ┆ null │
+    #   # └──────┴───────┴──────┘
+    def drop_nans(subset: nil)
+      lazy.drop_nans(subset: subset).collect(optimizations: QueryOptFlags._eager)
+    end
+
+    # Drop all rows that contain one or more null values.
+    #
+    # The original order of the remaining rows is preserved.
     #
     # @param subset [Object]
-    #   Subset of column(s) on which `drop_nulls` will be applied.
+    #   Column name(s) for which null values are considered.
+    #   If set to `nil` (default), use all columns.
     #
     # @return [DataFrame]
     #
@@ -1618,27 +2665,39 @@ module Polars
     #     {
     #       "foo" => [1, 2, 3],
     #       "bar" => [6, nil, 8],
-    #       "ham" => ["a", "b", "c"]
+    #       "ham" => ["a", "b", nil]
     #     }
     #   )
     #   df.drop_nulls
     #   # =>
-    #   # shape: (2, 3)
+    #   # shape: (1, 3)
     #   # ┌─────┬─────┬─────┐
     #   # │ foo ┆ bar ┆ ham │
     #   # │ --- ┆ --- ┆ --- │
     #   # │ i64 ┆ i64 ┆ str │
     #   # ╞═════╪═════╪═════╡
     #   # │ 1   ┆ 6   ┆ a   │
-    #   # │ 3   ┆ 8   ┆ c   │
     #   # └─────┴─────┴─────┘
+    #
+    # @example
+    #   df.drop_nulls(subset: Polars.cs.integer)
+    #   # =>
+    #   # shape: (2, 3)
+    #   # ┌─────┬─────┬──────┐
+    #   # │ foo ┆ bar ┆ ham  │
+    #   # │ --- ┆ --- ┆ ---  │
+    #   # │ i64 ┆ i64 ┆ str  │
+    #   # ╞═════╪═════╪══════╡
+    #   # │ 1   ┆ 6   ┆ a    │
+    #   # │ 3   ┆ 8   ┆ null │
+    #   # └─────┴─────┴──────┘
     def drop_nulls(subset: nil)
-      lazy.drop_nulls(subset: subset).collect(_eager: true)
+      lazy.drop_nulls(subset: subset).collect(optimizations: QueryOptFlags._eager)
     end
 
     # Offers a structured way to apply a sequence of user-defined functions (UDFs).
     #
-    # @param func [Object]
+    # @param function [Object]
     #   Callable; will receive the frame as the first parameter,
     #   followed by any given args/kwargs.
     # @param args [Object]
@@ -1655,7 +2714,7 @@ module Polars
     #
     # @example
     #   cast_str_to_int = lambda do |data, col_name:|
-    #     data.with_column(Polars.col(col_name).cast(:i64))
+    #     data.with_columns(Polars.col(col_name).cast(Polars::Int64))
     #   end
     #
     #   df = Polars::DataFrame.new({"a" => [1, 2, 3, 4], "b" => ["10", "20", "30", "40"]})
@@ -1672,8 +2731,94 @@ module Polars
     #   # │ 3   ┆ 30  │
     #   # │ 4   ┆ 40  │
     #   # └─────┴─────┘
-    def pipe(func, *args, **kwargs, &block)
-      func.call(self, *args, **kwargs, &block)
+    def pipe(function, *args, **kwargs, &block)
+      function.(self, *args, **kwargs, &block)
+    end
+
+    # Apply eager functions to columns of a DataFrame.
+    #
+    # Users should always prefer :meth:`with_columns` unless they are using
+    # expressions that are only possible on `Series` and not on `Expr`. This is almost
+    # never the case, except for a very select few functions that cannot know the
+    # output datatype without looking at the data.
+    #
+    # @param column_names [Object]
+    #   The columns to apply the UDF to.
+    # @param args [Array]
+    #   Arguments to pass to the UDF.
+    # @param kwargs [Hash]
+    #   Keyword arguments to pass to the UDF.
+    #
+    # @return [DataFrame]
+    #
+    # @example
+    #   df = Polars::DataFrame.new({"a" => [1, 2, 3, 4], "b" => ["10", "20", "30", "40"]})
+    #   df.map_columns("a") { |s| s.shrink_dtype }
+    #   # =>
+    #   # shape: (4, 2)
+    #   # ┌─────┬─────┐
+    #   # │ a   ┆ b   │
+    #   # │ --- ┆ --- │
+    #   # │ i8  ┆ str │
+    #   # ╞═════╪═════╡
+    #   # │ 1   ┆ 10  │
+    #   # │ 2   ┆ 20  │
+    #   # │ 3   ┆ 30  │
+    #   # │ 4   ┆ 40  │
+    #   # └─────┴─────┘
+    #
+    # @example
+    #   df = Polars::DataFrame.new(
+    #     {
+    #       "a" => ['{"x":"a"}', nil, '{"x":"b"}', nil],
+    #       "b" => ['{"a":1, "b": true}', nil, '{"a":2, "b": false}', nil]
+    #     }
+    #   )
+    #   df.map_columns(["a", "b"]) { |s| s.str.json_decode }
+    #   # =>
+    #   # shape: (4, 2)
+    #   # ┌───────────┬───────────┐
+    #   # │ a         ┆ b         │
+    #   # │ ---       ┆ ---       │
+    #   # │ struct[1] ┆ struct[2] │
+    #   # ╞═══════════╪═══════════╡
+    #   # │ {"a"}     ┆ {1,true}  │
+    #   # │ null      ┆ null      │
+    #   # │ {"b"}     ┆ {2,false} │
+    #   # │ null      ┆ null      │
+    #   # └───────────┴───────────┘
+    #
+    # @example
+    #   df.map_columns(Polars.cs.all) { |s| s.str.json_decode }
+    #   # =>
+    #   # shape: (4, 2)
+    #   # ┌───────────┬───────────┐
+    #   # │ a         ┆ b         │
+    #   # │ ---       ┆ ---       │
+    #   # │ struct[1] ┆ struct[2] │
+    #   # ╞═══════════╪═══════════╡
+    #   # │ {"a"}     ┆ {1,true}  │
+    #   # │ null      ┆ null      │
+    #   # │ {"b"}     ┆ {2,false} │
+    #   # │ null      ┆ null      │
+    #   # └───────────┴───────────┘
+    def map_columns(
+      column_names,
+      *args,
+      **kwargs,
+      &function
+    )
+      if column_names.is_a?(Selector) || column_names.is_a?(Expr)
+        c_names = Array(Utils.expand_selector(self, column_names))
+      elsif Utils.strlike?(column_names)
+        c_names = [column_names]
+      else
+        c_names = Array(column_names)
+      end
+
+      with_columns(
+        c_names.map { |c| function.call(self[c], *args, **kwargs) }
+      )
     end
 
     # Add a column at index 0 that counts the rows.
@@ -1707,28 +2852,30 @@ module Polars
     def with_row_index(name: "index", offset: 0)
       _from_rbdf(_df.with_row_index(name, offset))
     end
-    alias_method :with_row_count, :with_row_index
 
     # Start a group by operation.
     #
-    # @param by [Object]
+    # @param by [Array]
     #   Column(s) to group by.
     # @param maintain_order [Boolean]
     #   Make sure that the order of the groups remain consistent. This is more
     #   expensive than a default group by. Note that this only works in expression
     #   aggregations.
+    # @param named_by [Hash]
+    #   Additional columns to group by, specified as keyword arguments.
+    #   The columns will be renamed to the keyword used.
     #
     # @return [GroupBy]
     #
-    # @example
+    # @example Group by one column and call `agg` to compute the grouped sum of another column.
     #   df = Polars::DataFrame.new(
     #     {
-    #       "a" => ["a", "b", "a", "b", "b", "c"],
-    #       "b" => [1, 2, 3, 4, 5, 6],
-    #       "c" => [6, 5, 4, 3, 2, 1]
+    #       "a" => ["a", "b", "a", "b", "c"],
+    #       "b" => [1, 2, 1, 3, 3],
+    #       "c" => [5, 4, 3, 2, 1]
     #     }
     #   )
-    #   df.group_by("a").agg(Polars.col("b").sum).sort("a")
+    #   df.group_by("a").agg(Polars.col("b").sum)
     #   # =>
     #   # shape: (3, 2)
     #   # ┌─────┬─────┐
@@ -1736,27 +2883,76 @@ module Polars
     #   # │ --- ┆ --- │
     #   # │ str ┆ i64 │
     #   # ╞═════╪═════╡
-    #   # │ a   ┆ 4   │
-    #   # │ b   ┆ 11  │
-    #   # │ c   ┆ 6   │
+    #   # │ a   ┆ 2   │
+    #   # │ b   ┆ 5   │
+    #   # │ c   ┆ 3   │
     #   # └─────┴─────┘
-    def group_by(by, maintain_order: false)
-      if !Utils.bool?(maintain_order)
-        raise TypeError, "invalid input for group_by arg `maintain_order`: #{maintain_order}."
+    #
+    # @example Set `maintain_order: true` to ensure the order of the groups is consistent with the input.
+    #   df.group_by("a", maintain_order: true).agg(Polars.col("c"))
+    #   # =>
+    #   # shape: (3, 2)
+    #   # ┌─────┬───────────┐
+    #   # │ a   ┆ c         │
+    #   # │ --- ┆ ---       │
+    #   # │ str ┆ list[i64] │
+    #   # ╞═════╪═══════════╡
+    #   # │ a   ┆ [5, 3]    │
+    #   # │ b   ┆ [4, 2]    │
+    #   # │ c   ┆ [1]       │
+    #   # └─────┴───────────┘
+    #
+    # @example Group by multiple columns by passing a list of column names.
+    #   df.group_by(["a", "b"]).agg(Polars.max("c"))
+    #   # =>
+    #   # shape: (4, 3)
+    #   # ┌─────┬─────┬─────┐
+    #   # │ a   ┆ b   ┆ c   │
+    #   # │ --- ┆ --- ┆ --- │
+    #   # │ str ┆ i64 ┆ i64 │
+    #   # ╞═════╪═════╪═════╡
+    #   # │ a   ┆ 1   ┆ 5   │
+    #   # │ b   ┆ 2   ┆ 4   │
+    #   # │ b   ┆ 3   ┆ 2   │
+    #   # │ c   ┆ 3   ┆ 1   │
+    #   # └─────┴─────┴─────┘
+    #
+    # @example Or use positional arguments to group by multiple columns in the same way. Expressions are also accepted.
+    #   df.group_by("a", Polars.col("b") / 2).agg(Polars.col("c").mean)
+    #   # =>
+    #   # shape: (3, 3)
+    #   # ┌─────┬─────┬─────┐
+    #   # │ a   ┆ b   ┆ c   │
+    #   # │ --- ┆ --- ┆ --- │
+    #   # │ str ┆ i64 ┆ f64 │
+    #   # ╞═════╪═════╪═════╡
+    #   # │ a   ┆ 0   ┆ 4.0 │
+    #   # │ b   ┆ 1   ┆ 3.0 │
+    #   # │ c   ┆ 1   ┆ 1.0 │
+    #   # └─────┴─────┴─────┘
+    #
+    # @example The `GroupBy` object returned by this method is iterable, returning the name and data of each group.
+    #   df.group_by("a").each do |name, data|
+    #      # ...
+    #   end
+    def group_by(*by, maintain_order: false, **named_by)
+      named_by.each do |_, value|
+        if !(value.is_a?(::String) || value.is_a?(Expr) || value.is_a?(Series))
+          msg = "Expected Polars expression or object convertible to one, got #{value.class.name}."
+          raise TypeError, msg
+        end
       end
       GroupBy.new(
         self,
-        by,
-        maintain_order: maintain_order
+        *by,
+        **named_by,
+        maintain_order: maintain_order,
+        predicates: nil
       )
     end
-    alias_method :groupby, :group_by
-    alias_method :group, :group_by
 
     # Create rolling groups based on a time column.
     #
-    # Also works for index values of type `:i32` or `:i64`.
-    #
     # Different from a `dynamic_group_by` the windows are now determined by the
     # individual values and are not of constant intervals. For constant intervals use
     # *group_by_dynamic*
@@ -1790,16 +2986,16 @@ module Polars
     #   This column must be sorted in ascending order. If not the output will not
     #   make sense.
     #
-    #   In case of a rolling group by on indices, dtype needs to be one of
-    #   `:i32`, `:i64`. Note that `:i32` gets temporarily cast to `:i64`, so if
-    #   performance matters use an `:i64` column.
+    #   In case of a rolling operation on indices, dtype needs to be one of
+    #   \\\\{UInt32, UInt64, Int32, Int64}. Note that the first three get temporarily
+    #   cast to Int64, so if performance matters use an Int64 column.
     # @param period [Object]
     #   Length of the window.
     # @param offset [Object]
     #   Offset of the window. Default is -period.
     # @param closed ["right", "left", "both", "none"]
     #   Define whether the temporal window interval is closed or not.
-    # @param by [Object]
+    # @param group_by [Object]
     #   Also group by this column/these columns.
     #
     # @return [RollingGroupBy]
@@ -1813,7 +3009,7 @@ module Polars
     #     "2020-01-03 19:45:32",
     #     "2020-01-08 23:16:43"
     #   ]
-    #   df = Polars::DataFrame.new({"dt" => dates, "a" => [3, 7, 5, 9, 2, 1]}).with_column(
+    #   df = Polars::DataFrame.new({"dt" => dates, "a" => [3, 7, 5, 9, 2, 1]}).with_columns(
     #     Polars.col("dt").str.strptime(Polars::Datetime).set_sorted
     #   )
     #   df.rolling(index_column: "dt", period: "2d").agg(
@@ -1842,14 +3038,12 @@ module Polars
       period:,
       offset: nil,
       closed: "right",
-      by: nil
+      group_by: nil
     )
-      RollingGroupBy.new(self, index_column, period, offset, closed, by)
+      RollingGroupBy.new(self, index_column, period, offset, closed, group_by, nil)
     end
-    alias_method :groupby_rolling, :rolling
-    alias_method :group_by_rolling, :rolling
 
-    # Group based on a time value (or index value of type `:i32`, `:i64`).
+    # Group based on a time value (or index value of type Int32, Int64).
     #
     # Time windows are calculated and rows are assigned to windows. Different from a
     # normal group by is that a row can be member of multiple groups. The time/index
@@ -1892,25 +3086,47 @@ module Polars
     #   make sense.
     #
     #   In case of a dynamic group by on indices, dtype needs to be one of
-    #   `:i32`, `:i64`. Note that `:i32` gets temporarily cast to `:i64`, so if
-    #   performance matters use an `:i64` column.
+    #   \\\\{Int32, Int64}. Note that Int32 gets temporarily cast to Int64, so if
+    #   performance matters use an Int64 column.
     # @param every
     #   Interval of the window.
     # @param period
-    #   Length of the window, if None it is equal to 'every'.
+    #   Length of the window, if nil it is equal to 'every'.
     # @param offset
-    #   Offset of the window if None and period is None it will be equal to negative
+    #   Offset of the window if nil and period is nil it will be equal to negative
     #   `every`.
-    # @param truncate
-    #   Truncate the time value to the window lower bound.
     # @param include_boundaries
     #   Add the lower and upper bound of the window to the "_lower_bound" and
     #   "_upper_bound" columns. This will impact performance because it's harder to
     #   parallelize
     # @param closed ["right", "left", "both", "none"]
     #   Define whether the temporal window interval is closed or not.
-    # @param by
+    # @param label ['left', 'right', 'datapoint']
+    #   Define which label to use for the window:
+    #
+    #   - 'left': lower boundary of the window
+    #   - 'right': upper boundary of the window
+    #   - 'datapoint': the first value of the index column in the given window.
+    #     If you don't need the label to be at one of the boundaries, choose this
+    #     option for maximum performance
+    # @param group_by
     #   Also group by this column/these columns
+    # @param start_by ['window', 'datapoint', 'monday', 'tuesday', 'wednesday', 'thursday', 'friday', 'saturday', 'sunday']
+    #   The strategy to determine the start of the first window by.
+    #
+    #   * 'window': Start by taking the earliest timestamp, truncating it with
+    #     `every`, and then adding `offset`.
+    #     Note that weekly windows start on Monday.
+    #   * 'datapoint': Start from the first encountered data point.
+    #   * a day of the week (only takes effect if `every` contains `'w'`):
+    #
+    #     * 'monday': Start the window on the Monday before the first data point.
+    #     * 'tuesday': Start the window on the Tuesday before the first data point.
+    #     * ...
+    #     * 'sunday': Start the window on the Sunday before the first data point.
+    #
+    #     The resulting window is then shifted back until the earliest datapoint
+    #     is in or in front of it.
     #
     # @return [DataFrame]
     #
@@ -2035,7 +3251,7 @@ module Polars
     #     "time",
     #     every: "1h",
     #     closed: "both",
-    #     by: "groups",
+    #     group_by: "groups",
     #     include_boundaries: true
     #   ).agg([Polars.col("time").count.alias("time_count")])
     #   # =>
@@ -2085,10 +3301,10 @@ module Polars
       every:,
       period: nil,
       offset: nil,
-      truncate: true,
       include_boundaries: false,
       closed: "left",
-      by: nil,
+      label: "left",
+      group_by: nil,
       start_by: "window"
     )
       DynamicGroupBy.new(
@@ -2097,14 +3313,14 @@ module Polars
         every,
         period,
         offset,
-        truncate,
         include_boundaries,
         closed,
-        by,
-        start_by
+        label,
+        group_by,
+        start_by,
+        nil
       )
     end
-    alias_method :groupby_dynamic, :group_by_dynamic
 
     # Upsample a DataFrame at a regular frequency.
     #
@@ -2113,7 +3329,7 @@ module Polars
     #   Note that this column has to be sorted for the output to make sense.
     # @param every [String]
     #   interval will start 'every' duration
-    # @param by [Object]
+    # @param group_by [Object]
     #   First group by these columns and then upsample for every group
     # @param maintain_order [Boolean]
     #   Keep the ordering predictable. This is slower.
@@ -2152,7 +3368,7 @@ module Polars
     #     }
     #   ).set_sorted("time")
     #   df.upsample(
-    #     time_column: "time", every: "1mo", by: "groups", maintain_order: true
+    #     time_column: "time", every: "1mo", group_by: "groups", maintain_order: true
     #   ).select(Polars.all.forward_fill)
     #   # =>
     #   # shape: (7, 3)
@@ -2172,20 +3388,20 @@ module Polars
     def upsample(
       time_column:,
       every:,
-      by: nil,
+      group_by: nil,
       maintain_order: false
     )
-      if by.nil?
-        by = []
+      if group_by.nil?
+        group_by = []
       end
-      if by.is_a?(::String)
-        by = [by]
+      if group_by.is_a?(::String)
+        group_by = [group_by]
       end
 
       every = Utils.parse_as_duration_string(every)
 
       _from_rbdf(
-        _df.upsample(by, time_column, every, maintain_order)
+        _df.upsample(group_by, time_column, every, maintain_order)
       )
     end
 
@@ -2211,14 +3427,14 @@ module Polars
     #   Join column of the right DataFrame.
     # @param on [String]
     #   Join column of both DataFrames. If set, `left_on` and `right_on` should be
-    #   None.
-    # @param by [Object]
-    #   join on these columns before doing asof join
+    #   nil.
     # @param by_left [Object]
     #   join on these columns before doing asof join
     # @param by_right [Object]
     #   join on these columns before doing asof join
-    # @param strategy ["backward", "forward"]
+    # @param by [Object]
+    #   join on these columns before doing asof join
+    # @param strategy ["backward", "forward"]
     #   Join strategy.
     # @param suffix [String]
     #   Suffix to append to columns with a duplicate name.
@@ -2254,6 +3470,14 @@ module Polars
     #     - true: -> Always coalesce join columns.
     #     - false: -> Never coalesce join columns.
     #   Note that joining on any other expressions than `col` will turn off coalescing.
+    # @param allow_exact_matches [Boolean]
+    #   Whether exact matches are valid join predicates.
+    #     - If true, allow matching with the same `on` value (i.e. less-than-or-equal-to / greater-than-or-equal-to).
+    #     - If false, don't match the same `on` value (i.e., strictly less-than / strictly greater-than).
+    # @param check_sortedness [Boolean]
+    #   Check the sortedness of the asof keys. If the keys are not sorted Polars
+    #   will error, or in case of 'by' argument raise a warning. This might become
+    #   a hard error in the future.
     #
     # @return [DataFrame]
     #
@@ -2308,7 +3532,9 @@ module Polars
       tolerance: nil,
       allow_parallel: true,
       force_parallel: false,
-      coalesce: true
+      coalesce: true,
+      allow_exact_matches: true,
+      check_sortedness: true
     )
       lazy
         .join_asof(
@@ -2324,9 +3550,11 @@ module Polars
           tolerance: tolerance,
           allow_parallel: allow_parallel,
           force_parallel: force_parallel,
-          coalesce: coalesce
+          coalesce: coalesce,
+          allow_exact_matches: allow_exact_matches,
+          check_sortedness: check_sortedness
         )
-        .collect(no_optimization: true)
+        .collect(optimizations: QueryOptFlags._eager)
     end
 
     # Join in SQL-like fashion.
@@ -2349,7 +3577,7 @@ module Polars
     #     * *one_to_one* - “1:1”: check if join keys are unique in both left and right datasets
     #     * *one_to_many* - “1:m”: check if join keys are unique in left dataset
     #     * *many_to_one* - “m:1”: check if join keys are unique in right dataset
-    # @param join_nulls [Boolean]
+    # @param nulls_equal [Boolean]
     #   Join on null values. By default null values will never produce matches.
     # @param coalesce [Boolean]
     #   Coalescing behavior (merging of join columns).
@@ -2357,6 +3585,24 @@ module Polars
     #     - true: -> Always coalesce join columns.
     #     - false: -> Never coalesce join columns.
     #   Note that joining on any other expressions than `col` will turn off coalescing.
+    # @param maintain_order ['none', 'left', 'right', 'left_right', 'right_left']
+    #   Which DataFrame row order to preserve, if any.
+    #   Do not rely on any observed ordering without explicitly
+    #   setting this parameter, as your code may break in a future release.
+    #   Not specifying any ordering can improve performance
+    #   Supported for inner, left, right and full joins
+    #
+    #   * *none*
+    #       No specific ordering is desired. The ordering might differ across
+    #       Polars versions or even between different runs.
+    #   * *left*
+    #       Preserves the order of the left DataFrame.
+    #   * *right*
+    #       Preserves the order of the right DataFrame.
+    #   * *left_right*
+    #       First preserves the order of the left DataFrame, then the right.
+    #   * *right_left*
+    #       First preserves the order of the right DataFrame, then the left.
     #
     # @return [DataFrame]
     #
@@ -2439,15 +3685,17 @@ module Polars
     #   # ╞═════╪═════╪═════╡
     #   # │ 3   ┆ 8.0 ┆ c   │
     #   # └─────┴─────┴─────┘
-    def join(other,
+    def join(
+      other,
       left_on: nil,
       right_on: nil,
       on: nil,
       how: "inner",
       suffix: "_right",
       validate: "m:m",
-      join_nulls: false,
-      coalesce: nil
+      nulls_equal: false,
+      coalesce: nil,
+      maintain_order: nil
     )
       lazy
         .join(
@@ -2458,10 +3706,157 @@ module Polars
           how: how,
           suffix: suffix,
           validate: validate,
-          join_nulls: join_nulls,
-          coalesce: coalesce
+          nulls_equal: nulls_equal,
+          coalesce: coalesce,
+          maintain_order: maintain_order
         )
-        .collect(no_optimization: true)
+        .collect(optimizations: QueryOptFlags._eager)
+    end
+
+    # Perform a join based on one or multiple (in)equality predicates.
+    #
+    # This performs an inner join, so only rows where all predicates are true
+    # are included in the result, and a row from either DataFrame may be included
+    # multiple times in the result.
+    #
+    # @note
+    #   The row order of the input DataFrames is not preserved.
+    #
+    # @note
+    #   This functionality is experimental. It may be
+    #   changed at any point without it being considered a breaking change.
+    #
+    # @param other [DataFrame]
+    #   DataFrame to join with.
+    # @param predicates [Array]
+    #   (In)Equality condition to join the two tables on.
+    #   When a column name occurs in both tables, the proper suffix must
+    #   be applied in the predicate.
+    # @param suffix [String]
+    #   Suffix to append to columns with a duplicate name.
+    #
+    # @return [DataFrame]
+    #
+    # @example Join two dataframes together based on two predicates which get AND-ed together.
+    #   east = Polars::DataFrame.new(
+    #     {
+    #       "id": [100, 101, 102],
+    #       "dur": [120, 140, 160],
+    #       "rev": [12, 14, 16],
+    #       "cores": [2, 8, 4]
+    #     }
+    #   )
+    #   west = Polars::DataFrame.new(
+    #     {
+    #       "t_id": [404, 498, 676, 742],
+    #       "time": [90, 130, 150, 170],
+    #       "cost": [9, 13, 15, 16],
+    #       "cores": [4, 2, 1, 4]
+    #     }
+    #   )
+    #   east.join_where(
+    #     west,
+    #     Polars.col("dur") < Polars.col("time"),
+    #     Polars.col("rev") < Polars.col("cost")
+    #   )
+    #   # =>
+    #   # shape: (5, 8)
+    #   # ┌─────┬─────┬─────┬───────┬──────┬──────┬──────┬─────────────┐
+    #   # │ id  ┆ dur ┆ rev ┆ cores ┆ t_id ┆ time ┆ cost ┆ cores_right │
+    #   # │ --- ┆ --- ┆ --- ┆ ---   ┆ ---  ┆ ---  ┆ ---  ┆ ---         │
+    #   # │ i64 ┆ i64 ┆ i64 ┆ i64   ┆ i64  ┆ i64  ┆ i64  ┆ i64         │
+    #   # ╞═════╪═════╪═════╪═══════╪══════╪══════╪══════╪═════════════╡
+    #   # │ 100 ┆ 120 ┆ 12  ┆ 2     ┆ 498  ┆ 130  ┆ 13   ┆ 2           │
+    #   # │ 100 ┆ 120 ┆ 12  ┆ 2     ┆ 676  ┆ 150  ┆ 15   ┆ 1           │
+    #   # │ 100 ┆ 120 ┆ 12  ┆ 2     ┆ 742  ┆ 170  ┆ 16   ┆ 4           │
+    #   # │ 101 ┆ 140 ┆ 14  ┆ 8     ┆ 676  ┆ 150  ┆ 15   ┆ 1           │
+    #   # │ 101 ┆ 140 ┆ 14  ┆ 8     ┆ 742  ┆ 170  ┆ 16   ┆ 4           │
+    #   # └─────┴─────┴─────┴───────┴──────┴──────┴──────┴─────────────┘
+    #
+    # @example To OR them together, use a single expression and the `|` operator.
+    #   east.join_where(
+    #     west,
+    #     (Polars.col("dur") < Polars.col("time")) | (Polars.col("rev") < Polars.col("cost"))
+    #   )
+    #   # =>
+    #   # shape: (6, 8)
+    #   # ┌─────┬─────┬─────┬───────┬──────┬──────┬──────┬─────────────┐
+    #   # │ id  ┆ dur ┆ rev ┆ cores ┆ t_id ┆ time ┆ cost ┆ cores_right │
+    #   # │ --- ┆ --- ┆ --- ┆ ---   ┆ ---  ┆ ---  ┆ ---  ┆ ---         │
+    #   # │ i64 ┆ i64 ┆ i64 ┆ i64   ┆ i64  ┆ i64  ┆ i64  ┆ i64         │
+    #   # ╞═════╪═════╪═════╪═══════╪══════╪══════╪══════╪═════════════╡
+    #   # │ 100 ┆ 120 ┆ 12  ┆ 2     ┆ 498  ┆ 130  ┆ 13   ┆ 2           │
+    #   # │ 100 ┆ 120 ┆ 12  ┆ 2     ┆ 676  ┆ 150  ┆ 15   ┆ 1           │
+    #   # │ 100 ┆ 120 ┆ 12  ┆ 2     ┆ 742  ┆ 170  ┆ 16   ┆ 4           │
+    #   # │ 101 ┆ 140 ┆ 14  ┆ 8     ┆ 676  ┆ 150  ┆ 15   ┆ 1           │
+    #   # │ 101 ┆ 140 ┆ 14  ┆ 8     ┆ 742  ┆ 170  ┆ 16   ┆ 4           │
+    #   # │ 102 ┆ 160 ┆ 16  ┆ 4     ┆ 742  ┆ 170  ┆ 16   ┆ 4           │
+    #   # └─────┴─────┴─────┴───────┴──────┴──────┴──────┴─────────────┘
+    def join_where(
+      other,
+      *predicates,
+      suffix: "_right"
+    )
+      Utils.require_same_type(self, other)
+
+      lazy
+      .join_where(
+        other.lazy,
+        *predicates,
+        suffix: suffix
+      )
+      .collect(optimizations: QueryOptFlags._eager)
+    end
+
+    # Selects rows from this DataFrame at the given indices.
+    #
+    # @note
+    #   This functionality is experimental. It may be
+    #   changed at any point without it being considered a breaking change.
+    #
+    # @param indices [Object]
+    #   The indices of the rows to select.
+    # @param null_on_oob [Boolean]
+    #   If true when an index is out-of-bounds a null row will be generated
+    #   instead of raising an error.
+    #
+    # @return [DataFrame]
+    #
+    # @example
+    #   df = Polars::DataFrame.new({"x" => [2, 1, 0], "s" => ["foo", "bar", "baz"]})
+    #   df.gather([2, 0, 0])
+    #   # =>
+    #   # shape: (3, 2)
+    #   # ┌─────┬─────┐
+    #   # │ x   ┆ s   │
+    #   # │ --- ┆ --- │
+    #   # │ i64 ┆ str │
+    #   # ╞═════╪═════╡
+    #   # │ 0   ┆ baz │
+    #   # │ 2   ┆ foo │
+    #   # │ 2   ┆ foo │
+    #   # └─────┴─────┘
+    #
+    # @example
+    #   df.gather([0, 10, 1], null_on_oob: true)
+    #   # =>
+    #   # shape: (3, 2)
+    #   # ┌──────┬──────┐
+    #   # │ x    ┆ s    │
+    #   # │ ---  ┆ ---  │
+    #   # │ i64  ┆ str  │
+    #   # ╞══════╪══════╡
+    #   # │ 2    ┆ foo  │
+    #   # │ null ┆ null │
+    #   # │ 1    ┆ bar  │
+    #   # └──────┴──────┘
+    def gather(
+      indices,
+      null_on_oob: false
+    )
+      lazy
+      .gather(indices, null_on_oob: null_on_oob)
+      .collect(optimizations: QueryOptFlags._eager)
     end
 
     # Apply a custom/user-defined function (UDF) over the rows of the DataFrame.
@@ -2524,61 +3919,14 @@ module Polars
     #   # │ 9   │
     #   # │ 14  │
     #   # └─────┘
-    def map_rows(return_dtype: nil, inference_size: 256, &f)
-      out, is_df = _df.map_rows(f, return_dtype, inference_size)
+    def map_rows(return_dtype: nil, inference_size: 256, &function)
+      out, is_df = _df.map_rows(function, return_dtype, inference_size)
       if is_df
         _from_rbdf(out)
       else
         _from_rbdf(Utils.wrap_s(out).to_frame._df)
       end
     end
-    alias_method :apply, :map_rows
-
-    # Return a new DataFrame with the column added or replaced.
-    #
-    # @param column [Object]
-    #   Series, where the name of the Series refers to the column in the DataFrame.
-    #
-    # @return [DataFrame]
-    #
-    # @example Added
-    #   df = Polars::DataFrame.new(
-    #     {
-    #       "a" => [1, 3, 5],
-    #       "b" => [2, 4, 6]
-    #     }
-    #   )
-    #   df.with_column((Polars.col("b") ** 2).alias("b_squared"))
-    #   # =>
-    #   # shape: (3, 3)
-    #   # ┌─────┬─────┬───────────┐
-    #   # │ a   ┆ b   ┆ b_squared │
-    #   # │ --- ┆ --- ┆ ---       │
-    #   # │ i64 ┆ i64 ┆ i64       │
-    #   # ╞═════╪═════╪═══════════╡
-    #   # │ 1   ┆ 2   ┆ 4         │
-    #   # │ 3   ┆ 4   ┆ 16        │
-    #   # │ 5   ┆ 6   ┆ 36        │
-    #   # └─────┴─────┴───────────┘
-    #
-    # @example Replaced
-    #   df.with_column(Polars.col("a") ** 2)
-    #   # =>
-    #   # shape: (3, 2)
-    #   # ┌─────┬─────┐
-    #   # │ a   ┆ b   │
-    #   # │ --- ┆ --- │
-    #   # │ i64 ┆ i64 │
-    #   # ╞═════╪═════╡
-    #   # │ 1   ┆ 2   │
-    #   # │ 9   ┆ 4   │
-    #   # │ 25  ┆ 6   │
-    #   # └─────┴─────┘
-    def with_column(column)
-      lazy
-        .with_column(column)
-        .collect(no_optimization: true, string_cache: false)
-    end
 
     # Return a new DataFrame grown horizontally by stacking multiple Series to it.
     #
@@ -2624,7 +3972,7 @@ module Polars
 
     # Grow this DataFrame vertically by stacking a DataFrame to it.
     #
-    # @param df [DataFrame]
+    # @param other [DataFrame]
     #   DataFrame to stack.
     # @param in_place [Boolean]
     #   Modify in place
@@ -2659,12 +4007,12 @@ module Polars
     #   # │ 3   ┆ 8   ┆ c   │
     #   # │ 4   ┆ 9   ┆ d   │
     #   # └─────┴─────┴─────┘
-    def vstack(df, in_place: false)
+    def vstack(other, in_place: false)
       if in_place
-        _df.vstack_mut(df._df)
+        _df.vstack_mut(other._df)
         self
       else
-        _from_rbdf(_df.vstack(df._df))
+        _from_rbdf(_df.vstack(other._df))
       end
     end
 
@@ -2717,6 +4065,9 @@ module Polars
     #
     # @param columns [Object]
     #   Column(s) to drop.
+    # @param strict [Boolean]
+    #   Validate that all column names exist in the current schema,
+    #   and throw an exception if any do not.
     #
     # @return [DataFrame]
     #
@@ -2768,8 +4119,8 @@ module Polars
     #   # │ 7.0 │
     #   # │ 8.0 │
     #   # └─────┘
-    def drop(*columns)
-      lazy.drop(*columns).collect(_eager: true)
+    def drop(*columns, strict: true)
+      lazy.drop(*columns, strict: strict).collect(optimizations: QueryOptFlags._eager)
     end
 
     # Drop in place.
@@ -2882,7 +4233,7 @@ module Polars
     #   df.cast(Polars::String).to_h(as_series: false)
     #   # => {"foo"=>["1", "2", "3"], "bar"=>["6.0", "7.0", "8.0"], "ham"=>["2020-01-02", "2021-03-04", "2022-05-06"]}
     def cast(dtypes, strict: true)
-      lazy.cast(dtypes, strict: strict).collect(_eager: true)
+      lazy.cast(dtypes, strict: strict).collect(optimizations: QueryOptFlags._eager)
     end
 
     # Create an empty copy of the current DataFrame.
@@ -2932,7 +4283,6 @@ module Polars
         clone
       end
     end
-    alias_method :cleared, :clear
 
     # clone handled by initialize_copy
 
@@ -2994,10 +4344,13 @@ module Polars
       _df.get_columns.map { |s| Utils.wrap_s(s) }
     end
 
-    # Get a single column as Series by name.
+    # Get a single column by name.
     #
     # @param name [String]
     #   Name of the column to retrieve.
+    # @param default [Object]
+    #   Value to return if the column does not exist; if not explicitly set and
+    #   the column is not present a `ColumnNotFoundError` exception is raised.
     #
     # @return [Series]
     #
@@ -3012,8 +4365,22 @@ module Polars
     #   #         2
     #   #         3
     #   # ]
-    def get_column(name)
-      self[name]
+    #
+    # @example
+    #   df.get_column("baz", default: Polars::Series.new("baz", ["?", "?", "?"]))
+    #   # =>
+    #   # shape: (3,)
+    #   # Series: 'baz' [str]
+    #   # [
+    #   #         "?"
+    #   #         "?"
+    #   #         "?"
+    #   # ]
+    def get_column(name, default: NO_DEFAULT)
+      Utils.wrap_s(_df.get_column(name.to_s))
+    rescue ColumnNotFoundError
+      raise if default.eql?(NO_DEFAULT)
+      default
     end
 
     # Fill null values using the specified value or strategy.
@@ -3099,14 +4466,14 @@ module Polars
       _from_rbdf(
         lazy
           .fill_null(value, strategy: strategy, limit: limit, matches_supertype: matches_supertype)
-          .collect(no_optimization: true)
+          .collect(optimizations: QueryOptFlags._eager)
           ._df
       )
     end
 
     # Fill floating point NaN values by an Expression evaluation.
     #
-    # @param fill_value [Object]
+    # @param value [Object]
     #   Value to fill NaN with.
     #
     # @return [DataFrame]
@@ -3135,14 +4502,20 @@ module Polars
     #   # │ 99.0 ┆ 99.0 │
     #   # │ 4.0  ┆ 13.0 │
     #   # └──────┴──────┘
-    def fill_nan(fill_value)
-      lazy.fill_nan(fill_value).collect(no_optimization: true)
+    def fill_nan(value)
+      lazy.fill_nan(value).collect(optimizations: QueryOptFlags._eager)
     end
 
     # Explode `DataFrame` to long format by exploding a column with Lists.
     #
     # @param columns [Object]
     #   Column of LargeList type.
+    # @param more_columns [Array]
+    #   Additional names of columns to explode, specified as positional arguments.
+    # @param empty_as_null [Boolean]
+    #   Explode an empty list/array into a `null`.
+    # @param keep_nulls [Boolean]
+    #   Explode a `null` list/array into a `null`.
     #
     # @return [DataFrame]
     #
@@ -3170,25 +4543,43 @@ module Polars
     #   # │ c       ┆ 7       │
     #   # │ c       ┆ 8       │
     #   # └─────────┴─────────┘
-    def explode(columns)
-      lazy.explode(columns).collect(no_optimization: true)
+    def explode(
+      columns,
+      *more_columns,
+      empty_as_null: true,
+      keep_nulls: true
+    )
+      lazy
+      .explode(
+        columns,
+        *more_columns,
+        empty_as_null: empty_as_null,
+        keep_nulls: keep_nulls
+      ).collect(optimizations: QueryOptFlags._eager)
     end
 
     # Create a spreadsheet-style pivot table as a DataFrame.
     #
+    # @param on [Object]
+    #   Columns whose values will be used as the header of the output DataFrame
+    # @param on_columns [Object]
+    #  What value combinations will be considered for the output table.
+    # @param index [Object]
+    #   One or multiple keys to group by
     # @param values [Object]
     #   Column values to aggregate. Can be multiple columns if the *columns*
     #   arguments contains multiple columns as well
-    # @param index [Object]
-    #   One or multiple keys to group by
-    # @param on [Object]
-    #   Columns whose values will be used as the header of the output DataFrame
     # @param aggregate_function ["first", "sum", "max", "min", "mean", "median", "last", "count"]
     #   A predefined aggregate function str or an expression.
     # @param maintain_order [Object]
     #   Sort the grouped keys so that the output order is predictable.
     # @param sort_columns [Object]
     #   Sort the transposed columns by name. Default is by order of discovery.
+    # @param separator [String]
+    #   Used as separator/delimiter in generated column names in case of multiple
+    #   `values` columns.
+    # @param column_naming ['auto', 'combine']
+    #   How resulting column names will be constructed.
     #
     # @return [DataFrame]
     #
@@ -3213,60 +4604,37 @@ module Polars
     #   # └─────┴─────┴─────┘
     def pivot(
       on,
+      on_columns: nil,
       index: nil,
       values: nil,
       aggregate_function: nil,
       maintain_order: true,
       sort_columns: false,
-      separator: "_"
+      separator: "_",
+      column_naming: "auto"
     )
-      index = Utils._expand_selectors(self, index)
-      on = Utils._expand_selectors(self, on)
-      if !values.nil?
-        values = Utils._expand_selectors(self, values)
-      end
-
-      if aggregate_function.is_a?(::String)
-        case aggregate_function
-        when "first"
-          aggregate_expr = F.element.first._rbexpr
-        when "sum"
-          aggregate_expr = F.element.sum._rbexpr
-        when "max"
-          aggregate_expr = F.element.max._rbexpr
-        when "min"
-          aggregate_expr = F.element.min._rbexpr
-        when "mean"
-          aggregate_expr = F.element.mean._rbexpr
-        when "median"
-          aggregate_expr = F.element.median._rbexpr
-        when "last"
-          aggregate_expr = F.element.last._rbexpr
-        when "len"
-          aggregate_expr = F.len._rbexpr
-        when "count"
-          warn "`aggregate_function: \"count\"` input for `pivot` is deprecated. Use `aggregate_function: \"len\"` instead."
-          aggregate_expr = F.len._rbexpr
-        else
-          raise ArgumentError, "Argument aggregate fn: '#{aggregate_fn}' was not expected."
+      if on_columns.nil?
+        cols = select(on).unique(maintain_order: true)
+        if sort_columns
+          cols = cols.sort(on)
         end
-      elsif aggregate_function.nil?
-        aggregate_expr = nil
+        on_cols = cols
       else
-        aggregate_expr = aggregate_function._rbexpr
+        on_cols = on_columns
       end
 
-      _from_rbdf(
-        _df.pivot_expr(
-          on,
-          index,
-          values,
-          maintain_order,
-          sort_columns,
-          aggregate_expr,
-          separator
-        )
+      lazy
+      .pivot(
+        on,
+        on_columns: on_cols,
+        index: index,
+        values: values,
+        aggregate_function: aggregate_function,
+        maintain_order: maintain_order,
+        separator: separator,
+        column_naming: column_naming
       )
+      .collect(optimizations: QueryOptFlags._eager)
     end
 
     # Unpivot a DataFrame from wide to long format.
@@ -3280,7 +4648,8 @@ module Polars
     #
     # @param on [Object]
     #   Column(s) or selector(s) to use as values variables; if `on`
-    #   is empty all columns that are not in `index` will be used.
+    #   is empty no columns will be used. If set to `nil` (default)
+    #   all columns that are not in `index` will be used.
     # @param index [Object]
     #   Column(s) or selector(s) to use as identifier variables.
     # @param variable_name [Object]
@@ -3313,13 +4682,12 @@ module Polars
     #   # │ y   ┆ c        ┆ 4     │
     #   # │ z   ┆ c        ┆ 6     │
     #   # └─────┴──────────┴───────┘
-    def unpivot(on, index: nil, variable_name: nil, value_name: nil)
-      on = on.nil? ? [] : Utils._expand_selectors(self, on)
+    def unpivot(on = nil, index: nil, variable_name: nil, value_name: nil)
+      on = on.nil? ? nil : Utils._expand_selectors(self, on)
       index = index.nil? ? [] : Utils._expand_selectors(self, index)
 
       _from_rbdf(_df.unpivot(on, index, value_name, variable_name))
     end
-    alias_method :melt, :unpivot
 
     # Unstack a long table to a wide form without doing an aggregation.
     #
@@ -3424,7 +4792,7 @@ module Polars
 
       if how == "horizontal"
         df = (
-          df.with_column(
+          df.with_columns(
             (Polars.arange(0, n_cols * n_rows, eager: true) % n_cols).alias(
               "__sort_order"
             )
@@ -3447,14 +4815,18 @@ module Polars
 
     # Split into multiple DataFrames partitioned by groups.
     #
-    # @param groups [Object]
+    # @param by [Object]
     #   Groups to partition by.
+    # @param more_by [Array]
+    #   Additional names of columns to group by, specified as positional arguments.
     # @param maintain_order [Boolean]
     #   Keep predictable output order. This is slower as it requires an extra sort
     #   operation.
+    # @param include_key [Boolean]
+    #   Include the columns used to partition the DataFrame in the output.
     # @param as_dict [Boolean]
-    #   If true, return the partitions in a dictionary keyed by the distinct group
-    #   values instead of a list.
+    #   If true, return the partitions in a hash keyed by the distinct group
+    #   values instead of an array.
     #
     # @return [Object]
     #
@@ -3496,7 +4868,7 @@ module Polars
     # @example
     #   df.partition_by("foo", maintain_order: true, as_dict: true)
     #   # =>
-    #   # {"A"=>shape: (2, 3)
+    #   # {["A"]=>shape: (2, 3)
     #   # ┌─────┬─────┬─────┐
     #   # │ foo ┆ N   ┆ bar │
     #   # │ --- ┆ --- ┆ --- │
@@ -3504,7 +4876,7 @@ module Polars
     #   # ╞═════╪═════╪═════╡
     #   # │ A   ┆ 1   ┆ k   │
     #   # │ A   ┆ 2   ┆ l   │
-    #   # └─────┴─────┴─────┘, "B"=>shape: (2, 3)
+    #   # └─────┴─────┴─────┘, ["B"]=>shape: (2, 3)
     #   # ┌─────┬─────┬─────┐
     #   # │ foo ┆ N   ┆ bar │
     #   # │ --- ┆ --- ┆ --- │
@@ -3512,7 +4884,7 @@ module Polars
     #   # ╞═════╪═════╪═════╡
     #   # │ B   ┆ 2   ┆ m   │
     #   # │ B   ┆ 4   ┆ m   │
-    #   # └─────┴─────┴─────┘, "C"=>shape: (1, 3)
+    #   # └─────┴─────┴─────┘, ["C"]=>shape: (1, 3)
     #   # ┌─────┬─────┬─────┐
     #   # │ foo ┆ N   ┆ bar │
     #   # │ --- ┆ --- ┆ --- │
@@ -3520,30 +4892,26 @@ module Polars
     #   # ╞═════╪═════╪═════╡
     #   # │ C   ┆ 2   ┆ l   │
     #   # └─────┴─────┴─────┘}
-    def partition_by(groups, maintain_order: true, include_key: true, as_dict: false)
-      if groups.is_a?(::String)
-        groups = [groups]
-      elsif !groups.is_a?(::Array)
-        groups = Array(groups)
-      end
+    def partition_by(by, *more_by, maintain_order: true, include_key: true, as_dict: false)
+      by_parsed = Utils._expand_selectors(self, by, *more_by)
+
+      partitions = _df.partition_by(by_parsed, maintain_order, include_key).map { |df| _from_rbdf(df) }
 
       if as_dict
-        out = {}
-        if groups.length == 1
-          _df.partition_by(groups, maintain_order, include_key).each do |df|
-            df = _from_rbdf(df)
-            out[df[groups][0, 0]] = df
-          end
+        if include_key
+          names = partitions.map { |p| p.select(by_parsed).row(0) }
         else
-          _df.partition_by(groups, maintain_order, include_key).each do |df|
-            df = _from_rbdf(df)
-            out[df[groups].row(0)] = df
+          if !maintain_order
+            msg = "cannot use `partition_by` with `maintain_order: false, include_key: false, as_dict: true`"
+            raise ArgumentError, msg
           end
+          names = select(by_parsed).unique(maintain_order: true).rows
         end
-        out
-      else
-        _df.partition_by(groups, maintain_order, include_key).map { |df| _from_rbdf(df) }
+
+        return names.zip(partitions).to_h
       end
+
+      partitions
     end
 
     # Shift values by the given period.
@@ -3589,41 +4957,8 @@ module Polars
     #   # │ 3    ┆ 8    ┆ c    │
     #   # │ null ┆ null ┆ null │
     #   # └──────┴──────┴──────┘
-    def shift(n, fill_value: nil)
-      lazy.shift(n, fill_value: fill_value).collect(_eager: true)
-    end
-
-    # Shift the values by a given period and fill the resulting null values.
-    #
-    # @param periods [Integer]
-    #   Number of places to shift (may be negative).
-    # @param fill_value [Object]
-    #   fill nil values with this value.
-    #
-    # @return [DataFrame]
-    #
-    # @example
-    #   df = Polars::DataFrame.new(
-    #     {
-    #       "foo" => [1, 2, 3],
-    #       "bar" => [6, 7, 8],
-    #       "ham" => ["a", "b", "c"]
-    #     }
-    #   )
-    #   df.shift_and_fill(1, 0)
-    #   # =>
-    #   # shape: (3, 3)
-    #   # ┌─────┬─────┬─────┐
-    #   # │ foo ┆ bar ┆ ham │
-    #   # │ --- ┆ --- ┆ --- │
-    #   # │ i64 ┆ i64 ┆ str │
-    #   # ╞═════╪═════╪═════╡
-    #   # │ 0   ┆ 0   ┆ 0   │
-    #   # │ 1   ┆ 6   ┆ a   │
-    #   # │ 2   ┆ 7   ┆ b   │
-    #   # └─────┴─────┴─────┘
-    def shift_and_fill(periods, fill_value)
-      shift(periods, fill_value: fill_value)
+    def shift(n = 1, fill_value: nil)
+      lazy.shift(n, fill_value: fill_value).collect(optimizations: QueryOptFlags._eager)
     end
 
     # Get a mask of all duplicated rows in this DataFrame.
@@ -3679,6 +5014,16 @@ module Polars
     # Start a lazy query from this point.
     #
     # @return [LazyFrame]
+    #
+    # @example
+    #   df = Polars::DataFrame.new(
+    #     {
+    #       "a" => [nil, 2, 3, 4],
+    #       "b" => [0.5, nil, 2.5, 13],
+    #       "c" => [true, true, false, nil]
+    #     }
+    #   )
+    #   df.lazy
     def lazy
       wrap_ldf(_df.lazy)
     end
@@ -3772,7 +5117,27 @@ module Polars
     #   # │ 10      │
     #   # └─────────┘
     def select(*exprs, **named_exprs)
-      lazy.select(*exprs, **named_exprs).collect(_eager: true)
+      lazy.select(*exprs, **named_exprs).collect(optimizations: QueryOptFlags._eager)
+    end
+
+    # Select columns from this DataFrame.
+    #
+    # This will run all expression sequentially instead of in parallel.
+    # Use this when the work per expression is cheap.
+    #
+    # @param exprs [Array]
+    #   Column(s) to select, specified as positional arguments.
+    #   Accepts expression input. Strings are parsed as column names,
+    #   other non-expression inputs are parsed as literals.
+    # @param named_exprs [Hash]
+    #   Additional columns to select, specified as keyword arguments.
+    #   The columns will be renamed to the keyword used.
+    #
+    # @return [DataFrame]
+    def select_seq(*exprs, **named_exprs)
+      lazy
+      .select_seq(*exprs, **named_exprs)
+      .collect(optimizations: QueryOptFlags._eager)
     end
 
     # Add columns to this DataFrame.
@@ -3884,7 +5249,32 @@ module Polars
     #   # │ 4   ┆ 13.0 ┆ true  ┆ 52.0 ┆ false │
     #   # └─────┴──────┴───────┴──────┴───────┘
     def with_columns(*exprs, **named_exprs)
-      lazy.with_columns(*exprs, **named_exprs).collect(_eager: true)
+      lazy.with_columns(*exprs, **named_exprs).collect(optimizations: QueryOptFlags._eager)
+    end
+
+    # Add columns to this DataFrame.
+    #
+    # Added columns will replace existing columns with the same name.
+    #
+    # This will run all expression sequentially instead of in parallel.
+    # Use this when the work per expression is cheap.
+    #
+    # @param exprs [Array]
+    #   Column(s) to add, specified as positional arguments.
+    #   Accepts expression input. Strings are parsed as column names, other
+    #   non-expression inputs are parsed as literals.
+    # @param named_exprs [Hash]
+    #   Additional columns to add, specified as keyword arguments.
+    #   The columns will be renamed to the keyword used.
+    #
+    # @return [DataFrame]
+    def with_columns_seq(
+      *exprs,
+      **named_exprs
+    )
+      lazy
+      .with_columns_seq(*exprs, **named_exprs)
+      .collect(optimizations: QueryOptFlags._eager)
     end
 
     # Get number of chunks used by the ChunkedArrays of this DataFrame.
@@ -3939,14 +5329,32 @@ module Polars
     #   # ╞═════╪═════╪═════╡
     #   # │ 3   ┆ 8   ┆ c   │
     #   # └─────┴─────┴─────┘
-    def max(axis: 0)
-      if axis == 0
-        lazy.max.collect(_eager: true)
-      elsif axis == 1
-        Utils.wrap_s(_df.max_horizontal)
-      else
-        raise ArgumentError, "Axis should be 0 or 1."
-      end
+    def max
+      lazy.max.collect(optimizations: QueryOptFlags._eager)
+    end
+
+    # Get the maximum value horizontally across columns.
+    #
+    # @return [Series]
+    #
+    # @example
+    #   df = Polars::DataFrame.new(
+    #     {
+    #       "foo" => [1, 2, 3],
+    #       "bar" => [4.0, 5.0, 6.0]
+    #     }
+    #   )
+    #   df.max_horizontal
+    #   # =>
+    #   # shape: (3,)
+    #   # Series: 'max' [f64]
+    #   # [
+    #   #         4.0
+    #   #         5.0
+    #   #         6.0
+    #   # ]
+    def max_horizontal
+      select(max: F.max_horizontal(F.all)).to_series
     end
 
     # Aggregate the columns of this DataFrame to their minimum value.
@@ -3971,22 +5379,35 @@ module Polars
     #   # ╞═════╪═════╪═════╡
     #   # │ 1   ┆ 6   ┆ a   │
     #   # └─────┴─────┴─────┘
-    def min(axis: 0)
-      if axis == 0
-        lazy.min.collect(_eager: true)
-      elsif axis == 1
-        Utils.wrap_s(_df.min_horizontal)
-      else
-        raise ArgumentError, "Axis should be 0 or 1."
-      end
+    def min
+      lazy.min.collect(optimizations: QueryOptFlags._eager)
     end
 
-    # Aggregate the columns of this DataFrame to their sum value.
+    # Get the minimum value horizontally across columns.
     #
-    # @param axis [Integer]
-    #   Either 0 or 1.
-    # @param null_strategy ["ignore", "propagate"]
-    #   This argument is only used if axis == 1.
+    # @return [Series]
+    #
+    # @example
+    #   df = Polars::DataFrame.new(
+    #     {
+    #       "foo" => [1, 2, 3],
+    #       "bar" => [4.0, 5.0, 6.0]
+    #     }
+    #   )
+    #   df.min_horizontal
+    #   # =>
+    #   # shape: (3,)
+    #   # Series: 'min' [f64]
+    #   # [
+    #   #         1.0
+    #   #         2.0
+    #   #         3.0
+    #   # ]
+    def min_horizontal
+      select(min: F.min_horizontal(F.all)).to_series
+    end
+
+    # Aggregate the columns of this DataFrame to their sum value.
     #
     # @return [DataFrame]
     #
@@ -4008,35 +5429,42 @@ module Polars
     #   # ╞═════╪═════╪══════╡
     #   # │ 6   ┆ 21  ┆ null │
     #   # └─────┴─────┴──────┘
+    def sum
+      lazy.sum.collect(optimizations: QueryOptFlags._eager)
+    end
+
+    # Sum all values horizontally across columns.
+    #
+    # @param ignore_nulls [Boolean]
+    #   Ignore null values (default).
+    #   If set to `false`, any null value in the input will lead to a null output.
+    #
+    # @return [Series]
     #
     # @example
-    #   df.sum(axis: 1)
+    #   df = Polars::DataFrame.new(
+    #     {
+    #       "foo" => [1, 2, 3],
+    #       "bar" => [4.0, 5.0, 6.0]
+    #     }
+    #   )
+    #   df.sum_horizontal
     #   # =>
     #   # shape: (3,)
-    #   # Series: 'foo' [str]
+    #   # Series: 'sum' [f64]
     #   # [
-    #   #         "16a"
-    #   #         "27b"
-    #   #         "38c"
+    #   #         5.0
+    #   #         7.0
+    #   #         9.0
     #   # ]
-    def sum(axis: 0, null_strategy: "ignore")
-      case axis
-      when 0
-        lazy.sum.collect(_eager: true)
-      when 1
-        Utils.wrap_s(_df.sum_horizontal(null_strategy))
-      else
-        raise ArgumentError, "Axis should be 0 or 1."
-      end
+    def sum_horizontal(ignore_nulls: true)
+      select(
+        sum: F.sum_horizontal(F.all, ignore_nulls: ignore_nulls)
+      ).to_series
     end
 
     # Aggregate the columns of this DataFrame to their mean value.
     #
-    # @param axis [Integer]
-    #   Either 0 or 1.
-    # @param null_strategy ["ignore", "propagate"]
-    #   This argument is only used if axis == 1.
-    #
     # @return [DataFrame]
     #
     # @example
@@ -4057,40 +5485,63 @@ module Polars
     #   # ╞═════╪═════╪══════╡
     #   # │ 2.0 ┆ 7.0 ┆ null │
     #   # └─────┴─────┴──────┘
-    def mean(axis: 0, null_strategy: "ignore")
-      case axis
-      when 0
-        lazy.mean.collect(_eager: true)
-      when 1
-        Utils.wrap_s(_df.mean_horizontal(null_strategy))
-      else
-        raise ArgumentError, "Axis should be 0 or 1."
-      end
+    def mean
+      lazy.mean.collect(optimizations: QueryOptFlags._eager)
     end
 
-    # Aggregate the columns of this DataFrame to their standard deviation value.
+    # Take the mean of all values horizontally across columns.
     #
-    # @param ddof [Integer]
-    #   Degrees of freedom
+    # @param ignore_nulls [Boolean]
+    #   Ignore null values (default).
+    #   If set to `false`, any null value in the input will lead to a null output.
     #
-    # @return [DataFrame]
+    # @return [Series]
     #
     # @example
     #   df = Polars::DataFrame.new(
     #     {
     #       "foo" => [1, 2, 3],
-    #       "bar" => [6, 7, 8],
-    #       "ham" => ["a", "b", "c"]
+    #       "bar" => [4.0, 5.0, 6.0]
     #     }
     #   )
-    #   df.std
+    #   df.mean_horizontal
     #   # =>
-    #   # shape: (1, 3)
-    #   # ┌─────┬─────┬──────┐
-    #   # │ foo ┆ bar ┆ ham  │
-    #   # │ --- ┆ --- ┆ ---  │
-    #   # │ f64 ┆ f64 ┆ str  │
-    #   # ╞═════╪═════╪══════╡
+    #   # shape: (3,)
+    #   # Series: 'mean' [f64]
+    #   # [
+    #   #         2.5
+    #   #         3.5
+    #   #         4.5
+    #   # ]
+    def mean_horizontal(ignore_nulls: true)
+      select(
+        mean: F.mean_horizontal(F.all, ignore_nulls: ignore_nulls)
+      ).to_series
+    end
+
+    # Aggregate the columns of this DataFrame to their standard deviation value.
+    #
+    # @param ddof [Integer]
+    #   Degrees of freedom
+    #
+    # @return [DataFrame]
+    #
+    # @example
+    #   df = Polars::DataFrame.new(
+    #     {
+    #       "foo" => [1, 2, 3],
+    #       "bar" => [6, 7, 8],
+    #       "ham" => ["a", "b", "c"]
+    #     }
+    #   )
+    #   df.std
+    #   # =>
+    #   # shape: (1, 3)
+    #   # ┌─────┬─────┬──────┐
+    #   # │ foo ┆ bar ┆ ham  │
+    #   # │ --- ┆ --- ┆ ---  │
+    #   # │ f64 ┆ f64 ┆ str  │
+    #   # ╞═════╪═════╪══════╡
     #   # │ 1.0 ┆ 1.0 ┆ null │
     #   # └─────┴─────┴──────┘
     #
@@ -4106,7 +5557,7 @@ module Polars
     #   # │ 0.816497 ┆ 0.816497 ┆ null │
     #   # └──────────┴──────────┴──────┘
     def std(ddof: 1)
-      lazy.std(ddof: ddof).collect(_eager: true)
+      lazy.std(ddof: ddof).collect(optimizations: QueryOptFlags._eager)
     end
 
     # Aggregate the columns of this DataFrame to their variance value.
@@ -4147,7 +5598,7 @@ module Polars
     #   # │ 0.666667 ┆ 0.666667 ┆ null │
     #   # └──────────┴──────────┴──────┘
     def var(ddof: 1)
-      lazy.var(ddof: ddof).collect(_eager: true)
+      lazy.var(ddof: ddof).collect(optimizations: QueryOptFlags._eager)
     end
 
     # Aggregate the columns of this DataFrame to their median value.
@@ -4173,7 +5624,7 @@ module Polars
     #   # │ 2.0 ┆ 7.0 ┆ null │
     #   # └─────┴─────┴──────┘
     def median
-      lazy.median.collect(_eager: true)
+      lazy.median.collect(optimizations: QueryOptFlags._eager)
     end
 
     # Aggregate the columns of this DataFrame to their product values.
@@ -4230,14 +5681,20 @@ module Polars
     #   # │ 2.0 ┆ 7.0 ┆ null │
     #   # └─────┴─────┴──────┘
     def quantile(quantile, interpolation: "nearest")
-      lazy.quantile(quantile, interpolation: interpolation).collect(_eager: true)
+      lazy.quantile(quantile, interpolation: interpolation).collect(optimizations: QueryOptFlags._eager)
     end
 
     # Get one hot encoded dummy variables.
     #
-    # @param columns
+    # @param columns [Array]
     #   A subset of columns to convert to dummy variables. `nil` means
     #   "all columns".
+    # @param separator [String]
+    #   Separator/delimiter used when generating column names.
+    # @param drop_first [Boolean]
+    #   Remove the first category from the variables being encoded.
+    # @param drop_nulls [Boolean]
+    #   If there are `nil` values in the series, a `null` column is not generated
     #
     # @return [DataFrame]
     #
@@ -4260,11 +5717,11 @@ module Polars
     #   # │ 1     ┆ 0     ┆ 1     ┆ 0     ┆ 1     ┆ 0     │
     #   # │ 0     ┆ 1     ┆ 0     ┆ 1     ┆ 0     ┆ 1     │
     #   # └───────┴───────┴───────┴───────┴───────┴───────┘
-    def to_dummies(columns: nil, separator: "_", drop_first: false)
+    def to_dummies(columns: nil, separator: "_", drop_first: false, drop_nulls: false)
       if columns.is_a?(::String)
         columns = [columns]
       end
-      _from_rbdf(_df.to_dummies(columns, separator, drop_first))
+      _from_rbdf(_df.to_dummies(columns, separator, drop_first, drop_nulls))
     end
 
     # Drop duplicate rows from this DataFrame.
@@ -4291,7 +5748,7 @@ module Polars
     #       "c" => [true, true, true, false, true, true]
     #     }
     #   )
-    #   df.unique
+    #   df.unique(maintain_order: true)
     #   # =>
     #   # shape: (5, 3)
     #   # ┌─────┬─────┬───────┐
@@ -4305,11 +5762,11 @@ module Polars
     #   # │ 4   ┆ 3.0 ┆ true  │
     #   # │ 5   ┆ 3.0 ┆ true  │
     #   # └─────┴─────┴───────┘
-    def unique(maintain_order: true, subset: nil, keep: "first")
+    def unique(maintain_order: false, subset: nil, keep: "any")
       self._from_rbdf(
         lazy
           .unique(maintain_order: maintain_order, subset: subset, keep: keep)
-          .collect(no_optimization: true)
+          .collect(optimizations: QueryOptFlags._eager)
           ._df
       )
     end
@@ -4402,9 +5859,9 @@ module Polars
     # Sample from this DataFrame.
     #
     # @param n [Integer]
-    #   Number of items to return. Cannot be used with `frac`. Defaults to 1 if
-    #   `frac` is nil.
-    # @param frac [Float]
+    #   Number of items to return. Cannot be used with `fraction`. Defaults to 1 if
+    #   `fraction` is nil.
+    # @param fraction [Float]
     #   Fraction of items to return. Cannot be used with `n`.
     # @param with_replacement [Boolean]
     #   Allow values to be sampled more than once.
@@ -4432,25 +5889,25 @@ module Polars
     #   # │ --- ┆ --- ┆ --- │
     #   # │ i64 ┆ i64 ┆ str │
     #   # ╞═════╪═════╪═════╡
-    #   # │ 3   ┆ 8   ┆ c   │
+    #   # │ 1   ┆ 6   ┆ a   │
     #   # │ 2   ┆ 7   ┆ b   │
     #   # └─────┴─────┴─────┘
     def sample(
       n: nil,
-      frac: nil,
+      fraction: nil,
       with_replacement: false,
       shuffle: false,
       seed: nil
     )
-      if !n.nil? && !frac.nil?
-        raise ArgumentError, "cannot specify both `n` and `frac`"
+      if !n.nil? && !fraction.nil?
+        raise ArgumentError, "cannot specify both `n` and `fraction`"
       end
 
-      if n.nil? && !frac.nil?
-        frac = Series.new("frac", [frac]) unless frac.is_a?(Series)
+      if n.nil? && !fraction.nil?
+        fraction = Series.new("fraction", [fraction]) unless fraction.is_a?(Series)
 
         return _from_rbdf(
-          _df.sample_frac(frac._s, with_replacement, shuffle, seed)
+          _df.sample_frac(fraction._s, with_replacement, shuffle, seed)
         )
       end
 
@@ -4658,6 +6115,85 @@ module Polars
       end
     end
 
+    # Convert columnar data to rows as Ruby arrays in a hash keyed by some column.
+    #
+    # This method is like `rows`, but instead of returning rows in a flat list, rows
+    # are grouped by the values in the `key` column(s) and returned as a hash.
+    #
+    # Note that this method should not be used in place of native operations, due to
+    # the high cost of materializing all frame data out into a hash; it should
+    # be used only when you need to move the values out into a Ruby data structure
+    # or other object that cannot operate directly with Polars/Arrow.
+    #
+    # @param key [Object]
+    #   The column(s) to use as the key for the returned hash. If multiple
+    #   columns are specified, the key will be a tuple of those values, otherwise
+    #   it will be a string.
+    # @param named [Boolean]
+    #   Return hashes instead of arrays. The hashes are a mapping of
+    #   column name to row value. This is more expensive than returning an
+    #   array, but allows for accessing values by column name.
+    # @param include_key [Boolean]
+    #   Include key values inline with the associated data (by default the key
+    #   values are omitted as a memory/performance optimisation, as they can be
+    #   reoconstructed from the key).
+    # @param unique [Boolean]
+    #   Indicate that the key is unique; this will result in a 1:1 mapping from
+    #   key to a single associated row. Note that if the key is *not* actually
+    #   unique the last row with the given key will be returned.
+    #
+    # @return [Hash]
+    #
+    # @example Group rows by the given key column(s):
+    #   df = Polars::DataFrame.new(
+    #     {
+    #       "w" => ["a", "b", "b", "a"],
+    #       "x" => ["q", "q", "q", "k"],
+    #       "y" => [1.0, 2.5, 3.0, 4.5],
+    #       "z" => [9, 8, 7, 6]
+    #     }
+    #   )
+    #   df.rows_by_key(["w"])
+    #   # => {"a"=>[["q", 1.0, 9], ["k", 4.5, 6]], "b"=>[["q", 2.5, 8], ["q", 3.0, 7]]}
+    #
+    # @example Return the same row groupings as hashes:
+    #   df.rows_by_key(["w"], named: true)
+    #   # => {"a"=>[{"x"=>"q", "y"=>1.0, "z"=>9}, {"x"=>"k", "y"=>4.5, "z"=>6}], "b"=>[{"x"=>"q", "y"=>2.5, "z"=>8}, {"x"=>"q", "y"=>3.0, "z"=>7}]}
+    #
+    # @example Return row groupings, assuming keys are unique:
+    #   df.rows_by_key(["z"], unique: true)
+    #   # => {9=>["a", "q", 1.0], 8=>["b", "q", 2.5], 7=>["b", "q", 3.0], 6=>["a", "k", 4.5]}
+    #
+    # @example Return row groupings as hashes, assuming keys are unique:
+    #   df.rows_by_key(["z"], named: true, unique: true)
+    #   # => {9=>{"w"=>"a", "x"=>"q", "y"=>1.0}, 8=>{"w"=>"b", "x"=>"q", "y"=>2.5}, 7=>{"w"=>"b", "x"=>"q", "y"=>3.0}, 6=>{"w"=>"a", "x"=>"k", "y"=>4.5}}
+    #
+    # @example Return hash rows grouped by a compound key, including key values:
+    #   df.rows_by_key(["w", "x"], named: true, include_key: true)
+    #   # => {["a", "q"]=>[{"w"=>"a", "x"=>"q", "y"=>1.0, "z"=>9}], ["b", "q"]=>[{"w"=>"b", "x"=>"q", "y"=>2.5, "z"=>8}, {"w"=>"b", "x"=>"q", "y"=>3.0, "z"=>7}], ["a", "k"]=>[{"w"=>"a", "x"=>"k", "y"=>4.5, "z"=>6}]}
+    def rows_by_key(key, named: false, include_key: false, unique: false)
+      key = Utils._expand_selectors(self, key)
+
+      keys = key.size == 1 ? get_column(key[0]) : select(key).iter_rows
+
+      if include_key
+        values = self
+      else
+        data_cols = schema.names - key
+        values = select(data_cols)
+      end
+
+      zipped = keys.each.zip(values.iter_rows(named: named))
+
+      # if unique, we expect to write just one entry per key; otherwise, we're
+      # returning a list of rows for each key, so append into a hash of arrays.
+      if unique
+        zipped.to_h
+      else
+        zipped.each_with_object({}) { |(key, data), h| (h[key] ||= []) << data }
+      end
+    end
+
     # Returns an iterator over the DataFrame of rows of Ruby-native values.
     #
     # @param named [Boolean]
@@ -4686,7 +6222,7 @@ module Polars
     # @example
     #   df.iter_rows(named: true).map { |row| row["b"] }
     #   # => [2, 4, 6]
-    def iter_rows(named: false, buffer_size: 500, &block)
+    def iter_rows(named: false, buffer_size: 512, &block)
       return to_enum(:iter_rows, named: named, buffer_size: buffer_size) unless block_given?
 
       # load into the local namespace for a modest performance boost in the hot loops
@@ -4737,6 +6273,90 @@ module Polars
       iter_rows(named: named, buffer_size: buffer_size, &block)
     end
 
+    # Returns an iterator over the columns of this DataFrame.
+    #
+    # @return [Object]
+    #
+    # @note
+    #   Consider whether you can use `all` instead.
+    #   If you can, it will be more efficient.
+    #
+    # @example
+    #   df = Polars::DataFrame.new(
+    #     {
+    #       "a" => [1, 3, 5],
+    #       "b" => [2, 4, 6]
+    #     }
+    #   )
+    #   df.iter_columns.map { |s| s.name }
+    #   # => ["a", "b"]
+    #
+    # @example If you're using this to modify a dataframe's columns, e.g.
+    #   # Do NOT do this
+    #   Polars::DataFrame.new(df.iter_columns.map { |column| column * 2 })
+    #   # =>
+    #   # shape: (3, 2)
+    #   # ┌─────┬─────┐
+    #   # │ a   ┆ b   │
+    #   # │ --- ┆ --- │
+    #   # │ i64 ┆ i64 │
+    #   # ╞═════╪═════╡
+    #   # │ 2   ┆ 4   │
+    #   # │ 6   ┆ 8   │
+    #   # │ 10  ┆ 12  │
+    #   # └─────┴─────┘
+    #
+    # @example then consider whether you can use `all` instead:
+    #   df.select(Polars.all * 2)
+    #   # =>
+    #   # shape: (3, 2)
+    #   # ┌─────┬─────┐
+    #   # │ a   ┆ b   │
+    #   # │ --- ┆ --- │
+    #   # │ i64 ┆ i64 │
+    #   # ╞═════╪═════╡
+    #   # │ 2   ┆ 4   │
+    #   # │ 6   ┆ 8   │
+    #   # │ 10  ┆ 12  │
+    #   # └─────┴─────┘
+    def iter_columns
+      return to_enum(:iter_columns) unless block_given?
+
+      _df.get_columns.each do |s|
+        yield Utils.wrap_s(s)
+      end
+    end
+
+    # Returns a non-copying iterator of slices over the underlying DataFrame.
+    #
+    # @param n_rows [Integer]
+    #   Determines the number of rows contained in each DataFrame slice.
+    #
+    # @return [Object]
+    #
+    # @example
+    #   df = Polars::DataFrame.new(
+    #     {
+    #       "a" => 0...17_500,
+    #       "b" => Date.new(2023, 1, 1),
+    #       "c" => "klmnoopqrstuvwxyz"
+    #     },
+    #     schema_overrides: {"a" => Polars::Int32}
+    #   )
+    #   df.iter_slices.map.with_index do |frame, idx|
+    #     "#{frame.class.name}:[#{idx}]:#{frame.length}"
+    #   end
+    #   # => ["Polars::DataFrame:[0]:10000", "Polars::DataFrame:[1]:7500"]
+    def iter_slices(n_rows: 10_000)
+      return to_enum(:iter_slices, n_rows: n_rows) unless block_given?
+
+      offset = 0
+      while offset < height
+        yield slice(offset, n_rows)
+        offset += n_rows
+      end
+    end
+
     # Shrink DataFrame memory usage.
     #
     # Shrinks to fit the exact capacity needed to hold the data.
@@ -4773,11 +6393,10 @@ module Polars
     def gather_every(n, offset = 0)
       select(F.col("*").gather_every(n, offset))
     end
-    alias_method :take_every, :gather_every
 
     # Hash and combine the rows in this DataFrame.
     #
-    # The hash value is of type `:u64`.
+    # The hash value is of type `UInt64`.
     #
     # @param seed [Integer]
     #   Random seed parameter. Defaults to 0.
@@ -4884,8 +6503,8 @@ module Polars
     #   #         {4,"four"}
     #   #         {5,"five"}
     #   # ]
-    def to_struct(name)
-      Utils.wrap_s(_df.to_struct(name))
+    def to_struct(name = "")
+      Utils.wrap_s(_df.to_struct(name, []))
     end
 
     # Decompose a struct into its fields.
@@ -4893,8 +6512,13 @@ module Polars
     # The fields will be inserted into the `DataFrame` on the location of the
     # `struct` type.
     #
-    # @param names [Object]
-    #  Names of the struct columns that will be decomposed by its fields
+    # @param columns [Object]
+    #   Name of the struct column(s) that should be unnested.
+    # @param more_columns [Array]
+    #   Additional columns to unnest, specified as positional arguments.
+    # @param separator [String]
+    #   Rename output column names as combination of the struct column name,
+    #   name separator and field name.
     #
     # @return [DataFrame]
     #
@@ -4920,17 +6544,10 @@ module Polars
     #   # │ foo    ┆ 1   ┆ a   ┆ true ┆ [1, 2]    ┆ baz   │
     #   # │ bar    ┆ 2   ┆ b   ┆ null ┆ [3]       ┆ womp  │
     #   # └────────┴─────┴─────┴──────┴───────────┴───────┘
-    def unnest(names)
-      if names.is_a?(::String)
-        names = [names]
-      end
-      _from_rbdf(_df.unnest(names))
+    def unnest(columns = nil, *more_columns, separator: nil)
+      lazy.unnest(columns, *more_columns, separator: separator).collect(optimizations: QueryOptFlags._eager)
     end
 
-    # Requires NumPy
-    # def corr
-    # end
-
     # Take two sorted DataFrames and merge them by the sorted key.
     #
     # The output of this operation will also be sorted.
@@ -4943,6 +6560,10 @@ module Polars
     #   Other DataFrame that must be merged
     # @param key [String]
     #   Key that is sorted.
+    # @param maintain_order [Boolean]
+    #   If `true`, the output is guaranteed to have left-biased ordering
+    #   for equal keys: rows from the left frame appear before rows from
+    #   the right frame when their keys are equal.
     #
     # @return [DataFrame]
     #
@@ -4969,30 +6590,235 @@ module Polars
     #   # │ steve  ┆ 42  │
     #   # │ elise  ┆ 44  │
     #   # └────────┴─────┘
-    def merge_sorted(other, key)
-      lazy.merge_sorted(other.lazy, key).collect(_eager: true)
+    def merge_sorted(other, key, maintain_order: false)
+      lazy.merge_sorted(other.lazy, key, maintain_order: maintain_order).collect(optimizations: QueryOptFlags._eager)
     end
 
-    # Indicate that one or multiple columns are sorted.
+    # Flag a column as sorted.
+    #
+    # This can speed up future operations.
+    #
+    # @note
+    #   This can lead to incorrect results if the data is NOT sorted! Use with care!
     #
     # @param column [Object]
-    #   Columns that are sorted
+    #   Column that is sorted.
     # @param descending [Boolean]
-    #   Whether the columns are sorted in descending order.
+    #   Whether the column is sorted in descending order.
+    # @param nulls_last [Boolean]
+    #   Whether the nulls are at the end.
     #
     # @return [DataFrame]
     def set_sorted(
       column,
-      descending: false
+      descending: false,
+      nulls_last: false
     )
       lazy
-        .set_sorted(column, descending: descending)
-        .collect(no_optimization: true)
+        .set_sorted(column, descending: descending, nulls_last: nulls_last)
+        .collect(optimizations: QueryOptFlags._eager)
     end
 
-    # TODO
-    # def update
-    # end
+    # Update the values in this `DataFrame` with the values in `other`.
+    #
+    # @note
+    #   This functionality is considered **unstable**. It may be changed
+    #   at any point without it being considered a breaking change.
+    #
+    # @param other [DataFrame]
+    #   DataFrame that will be used to update the values
+    # @param on [Object]
+    #   Column names that will be joined on. If set to `nil` (default),
+    #   the implicit row index of each frame is used as a join key.
+    # @param how ['left', 'inner', 'full']
+    #   * 'left' will keep all rows from the left table; rows may be duplicated
+    #     if multiple rows in the right frame match the left row's key.
+    #   * 'inner' keeps only those rows where the key exists in both frames.
+    #   * 'full' will update existing rows where the key matches while also
+    #     adding any new rows contained in the given frame.
+    # @param left_on [Object]
+    #   Join column(s) of the left DataFrame.
+    # @param right_on [Object]
+    #   Join column(s) of the right DataFrame.
+    # @param include_nulls [Boolean]
+    #   Overwrite values in the left frame with null values from the right frame.
+    #   If set to `false` (default), null values in the right frame are ignored.
+    # @param maintain_order ['none', 'left', 'right', 'left_right', 'right_left']
+    #   Which order of rows from the inputs to preserve. See `DataFrame.join`
+    #   for details. Unlike `join` this function preserves the left order by
+    #   default.
+    #
+    # @return [DataFrame]
+    #
+    # @note
+    #   This is syntactic sugar for a left/inner join that preserves the order
+    #   of the left `DataFrame` by default, with an optional coalesce when
+    #   `include_nulls: false`.
+    #
+    # @example Update `df` values with the non-null values in `new_df`, by row index:
+    #   df = Polars::DataFrame.new(
+    #     {
+    #       "A" => [1, 2, 3, 4],
+    #       "B" => [400, 500, 600, 700]
+    #     }
+    #   )
+    #   new_df = Polars::DataFrame.new(
+    #     {
+    #       "B" => [-66, nil, -99],
+    #       "C" => [5, 3, 1]
+    #     }
+    #   )
+    #   df.update(new_df)
+    #   # =>
+    #   # shape: (4, 2)
+    #   # ┌─────┬─────┐
+    #   # │ A   ┆ B   │
+    #   # │ --- ┆ --- │
+    #   # │ i64 ┆ i64 │
+    #   # ╞═════╪═════╡
+    #   # │ 1   ┆ -66 │
+    #   # │ 2   ┆ 500 │
+    #   # │ 3   ┆ -99 │
+    #   # │ 4   ┆ 700 │
+    #   # └─────┴─────┘
+    #
+    # @example Update `df` values with the non-null values in `new_df`, by row index, but only keeping those rows that are common to both frames:
+    #   df.update(new_df, how: "inner")
+    #   # =>
+    #   # shape: (3, 2)
+    #   # ┌─────┬─────┐
+    #   # │ A   ┆ B   │
+    #   # │ --- ┆ --- │
+    #   # │ i64 ┆ i64 │
+    #   # ╞═════╪═════╡
+    #   # │ 1   ┆ -66 │
+    #   # │ 2   ┆ 500 │
+    #   # │ 3   ┆ -99 │
+    #   # └─────┴─────┘
+    #
+    # @example Update `df` values with the non-null values in `new_df`, using a full outer join strategy that defines explicit join columns in each frame:
+    #   df.update(new_df, left_on: ["A"], right_on: ["C"], how: "full")
+    #   # =>
+    #   # shape: (5, 2)
+    #   # ┌─────┬─────┐
+    #   # │ A   ┆ B   │
+    #   # │ --- ┆ --- │
+    #   # │ i64 ┆ i64 │
+    #   # ╞═════╪═════╡
+    #   # │ 1   ┆ -99 │
+    #   # │ 2   ┆ 500 │
+    #   # │ 3   ┆ 600 │
+    #   # │ 4   ┆ 700 │
+    #   # │ 5   ┆ -66 │
+    #   # └─────┴─────┘
+    #
+    # @example Update `df` values including null values in `new_df`, using a full outer join strategy that defines explicit join columns in each frame:
+    #   df.update(new_df, left_on: "A", right_on: "C", how: "full", include_nulls: true)
+    #   # =>
+    #   # shape: (5, 2)
+    #   # ┌─────┬──────┐
+    #   # │ A   ┆ B    │
+    #   # │ --- ┆ ---  │
+    #   # │ i64 ┆ i64  │
+    #   # ╞═════╪══════╡
+    #   # │ 1   ┆ -99  │
+    #   # │ 2   ┆ 500  │
+    #   # │ 3   ┆ null │
+    #   # │ 4   ┆ 700  │
+    #   # │ 5   ┆ -66  │
+    #   # └─────┴──────┘
+    def update(
+      other,
+      on: nil,
+      how: "left",
+      left_on: nil,
+      right_on: nil,
+      include_nulls: false,
+      maintain_order: "left"
+    )
+      Utils.require_same_type(self, other)
+      lazy
+      .update(
+        other.lazy,
+        on: on,
+        how: how,
+        left_on: left_on,
+        right_on: right_on,
+        include_nulls: include_nulls,
+        maintain_order: maintain_order
+      )
+      .collect(optimizations: QueryOptFlags._eager)
+    end
+
+    # Match or evolve the schema of a LazyFrame into a specific schema.
+    #
+    # By default, match_to_schema returns an error if the input schema does not
+    # exactly match the target schema. It also allows columns to be freely reordered,
+    # with additional coercion rules available through optional parameters.
+    #
+    # @note
+    #   This functionality is considered **unstable**. It may be changed
+    #   at any point without it being considered a breaking change.
+    #
+    # @param schema [Object]
+    #   Target schema to match or evolve to.
+    # @param missing_columns [Object]
+    #   Raise of insert missing columns from the input with respect to the `schema`.
+    #
+    #   This can also be an expression per column with what to insert if it is
+    #   missing.
+    # @param missing_struct_fields [Object]
+    #   Raise of insert missing struct fields from the input with respect to the
+    #   `schema`.
+    # @param extra_columns [Object]
+    #   Raise of ignore extra columns from the input with respect to the `schema`.
+    # @param extra_struct_fields [Object]
+    #   Raise of ignore extra struct fields from the input with respect to the
+    #   `schema`.
+    # @param integer_cast [Object]
+    #   Forbid of upcast for integer columns from the input to the respective column
+    #   in `schema`.
+    # @param float_cast [Object]
+    #   Forbid of upcast for float columns from the input to the respective column
+    #   in `schema`.
+    #
+    # @return [DataFrame]
+    #
+    # @example Ensuring the schema matches
+    #   df = Polars::DataFrame.new({"a" => [1, 2, 3], "b" => ["A", "B", "C"]})
+    #   df.match_to_schema({"a" => Polars::Int64, "b" => Polars::String})
+    #   # =>
+    #   # shape: (3, 2)
+    #   # ┌─────┬─────┐
+    #   # │ a   ┆ b   │
+    #   # │ --- ┆ --- │
+    #   # │ i64 ┆ str │
+    #   # ╞═════╪═════╡
+    #   # │ 1   ┆ A   │
+    #   # │ 2   ┆ B   │
+    #   # │ 3   ┆ C   │
+    #   # └─────┴─────┘
+    def match_to_schema(
+      schema,
+      missing_columns: "raise",
+      missing_struct_fields: "raise",
+      extra_columns: "raise",
+      extra_struct_fields: "raise",
+      integer_cast: "forbid",
+      float_cast: "forbid"
+    )
+      lazy
+      .match_to_schema(
+        schema,
+        missing_columns: missing_columns,
+        missing_struct_fields: missing_struct_fields,
+        extra_columns: extra_columns,
+        extra_struct_fields: extra_struct_fields,
+        integer_cast: integer_cast,
+        float_cast: float_cast
+      )
+      .collect(optimizations: QueryOptFlags._eager)
+    end
 
     private
 
@@ -5057,282 +6883,6 @@ module Polars
       raise ArgumentError, "Unsupported idxs datatype."
     end
 
-    # @private
-    def self.expand_hash_scalars(data, schema_overrides: nil, strict: true, order: nil, nan_to_null: false)
-      updated_data = {}
-      unless data.empty?
-        dtypes = schema_overrides || {}
-        array_len = data.values.map { |val| Utils.arrlen(val) || 0 }.max
-        if array_len > 0
-          data.each do |name, val|
-            dtype = dtypes[name]
-            if val.is_a?(Hash) && dtype != Struct
-              updated_data[name] = DataFrame.new(val, strict: strict).to_struct(name)
-            elsif !Utils.arrlen(val).nil?
-              updated_data[name] = Series.new(::String.new(name), val, dtype: dtype, strict: strict)
-            elsif val.nil? || [Integer, Float, TrueClass, FalseClass, ::String, ::Date, ::DateTime, ::Time].any? { |cls| val.is_a?(cls) }
-              dtype = Polars::Float64 if val.nil? && dtype.nil?
-              updated_data[name] = Series.new(::String.new(name), [val], dtype: dtype, strict: strict).extend_constant(val, array_len - 1)
-            else
-              raise Todo
-            end
-          end
-        elsif data.values.all? { |val| Utils.arrlen(val) == 0 }
-          data.each do |name, val|
-            updated_data[name] = Series.new(name, val, dtype: dtypes[name], strict: strict)
-          end
-        elsif data.values.all? { |val| Utils.arrlen(val).nil? }
-          data.each do |name, val|
-            updated_data[name] = Series.new(name, [val], dtype: dtypes[name], strict: strict)
-          end
-        end
-      end
-      updated_data
-    end
-
-    # @private
-    def self.hash_to_rbdf(data, schema: nil, schema_overrides: nil, strict: true, nan_to_null: nil)
-      if schema.is_a?(Hash) && !data.empty?
-        if !data.all? { |col, _| schema[col] }
-          raise ArgumentError, "The given column-schema names do not match the data dictionary"
-        end
-
-        data = schema.to_h { |col| [col, data[col]] }
-      end
-
-      column_names, schema_overrides = _unpack_schema(
-        schema, lookup_names: data.keys, schema_overrides: schema_overrides
-      )
-      if column_names.empty?
-        column_names = data.keys
-      end
-
-      if data.empty? && !schema_overrides.empty?
-        data_series = column_names.map { |name| Series.new(name, [], dtype: schema_overrides[name], strict: strict, nan_to_null: nan_to_null)._s }
-      else
-        data_series = expand_hash_scalars(data, schema_overrides: schema_overrides, strict: strict, nan_to_null: nan_to_null).values.map(&:_s)
-      end
-
-      data_series = _handle_columns_arg(data_series, columns: column_names, from_hash: true)
-      RbDataFrame.new(data_series)
-    end
-
-    # @private
-    def self.include_unknowns(schema, cols)
-      cols.to_h { |col| [col, schema.fetch(col, Unknown)] }
-    end
-
-    # @private
-    def self._unpack_schema(schema, schema_overrides: nil, n_expected: nil, lookup_names: nil, include_overrides_in_columns: false)
-      if schema.is_a?(Hash)
-        schema = schema.to_a
-      end
-      column_names =
-        (schema || []).map.with_index do |col, i|
-          if col.is_a?(::String)
-            col || "column_#{i}"
-          else
-            col[0]
-          end
-        end
-      if column_names.empty? && n_expected
-        column_names = n_expected.times.map { |i| "column_#{i}" }
-      end
-      # TODO zip_longest
-      lookup = column_names.zip(lookup_names || []).to_h
-
-      column_dtypes =
-        (schema || []).select { |col| !col.is_a?(::String) && col[1] }.to_h do |col|
-          [lookup[col[0]] || col[0], col[1]]
-        end
-
-      if schema_overrides && schema_overrides.any?
-        column_dtypes.merge!(schema_overrides)
-      end
-
-      column_dtypes.each do |col, dtype|
-        if !Utils.is_polars_dtype(dtype, include_unknown: true) && !dtype.nil?
-          column_dtypes[col] = Utils.rb_type_to_dtype(dtype)
-        end
-      end
-
-      [column_names, column_dtypes]
-    end
-
-    def self._handle_columns_arg(data, columns: nil, from_hash: false)
-      if columns.nil? || columns.empty?
-        data
-      else
-        if data.empty?
-          columns.map { |c| Series.new(c, nil)._s }
-        elsif data.length == columns.length
-          if from_hash
-            series_map = data.to_h { |s| [s.name, s] }
-            if columns.all? { |col| series_map.key?(col) }
-              return columns.map { |col| series_map[col] }
-            end
-          end
-
-          columns.each_with_index do |c, i|
-            # not in-place?
-            data[i].rename(c)
-          end
-          data
-        else
-          raise ArgumentError, "Dimensions of columns arg must match data dimensions."
-        end
-      end
-    end
-
-    def self._post_apply_columns(rbdf, columns, structs: nil, schema_overrides: nil, strict: true)
-      rbdf_columns = rbdf.columns
-      rbdf_dtypes = rbdf.dtypes
-      columns, dtypes = _unpack_schema(
-        (columns || rbdf_columns), schema_overrides: schema_overrides
-      )
-      column_subset = []
-      if columns != rbdf_columns
-        if columns.length < rbdf_columns.length && columns == rbdf_columns.first(columns.length)
-          column_subset = columns
-        else
-          rbdf.set_column_names(columns)
-        end
-      end
-
-      column_casts = []
-      columns.each_with_index do |col, i|
-        if dtypes[col] == Categorical # != rbdf_dtypes[i]
-          column_casts << Polars.col(col).cast(Categorical, strict: strict)._rbexpr
-        elsif structs&.any? && structs.include?(col) && structs[col] != rbdf_dtypes[i]
-          column_casts << Polars.col(col).cast(structs[col], strict: strict)._rbexpr
-        elsif dtypes.include?(col) && dtypes[col] != rbdf_dtypes[i]
-          column_casts << Polars.col(col).cast(dtypes[col], strict: strict)._rbexpr
-        end
-      end
-
-      if column_casts.any? || column_subset.any?
-        rbdf = rbdf.lazy
-        if column_casts.any?
-          rbdf = rbdf.with_columns(column_casts)
-        end
-        if column_subset.any?
-          rbdf = rbdf.select(column_subset.map { |col| Polars.col(col)._rbexpr })
-        end
-        rbdf = rbdf.collect
-      end
-
-      rbdf
-    end
-
-    # @private
-    def self.sequence_to_rbdf(data, schema: nil, schema_overrides: nil, strict: true, orient: nil, infer_schema_length: 50)
-      columns = schema
-
-      if data.length == 0
-        return hash_to_rbdf({}, schema: schema, schema_overrides: schema_overrides, strict: strict)
-      end
-
-      if data[0].is_a?(Series)
-        # series_names = data.map(&:name)
-        # columns, dtypes = _unpack_schema(columns || series_names, n_expected: data.length)
-        data_series = []
-        data.each do |s|
-          data_series << s._s
-        end
-      elsif data[0].is_a?(Hash)
-        column_names, dtypes = _unpack_schema(columns)
-        schema_overrides = dtypes ? include_unknowns(dtypes, column_names) : nil
-        rbdf = RbDataFrame.from_hashes(data, schema, schema_overrides, strict, infer_schema_length)
-        if column_names
-          rbdf = _post_apply_columns(rbdf, column_names)
-        end
-        return rbdf
-      elsif data[0].is_a?(::Array)
-        first_element = data[0]
-        if orient.nil? && !columns.nil?
-          row_types = first_element.filter_map { |value| value.class }.uniq
-          if row_types.include?(Integer) && row_types.include?(Float)
-            row_types.delete(Integer)
-          end
-          orient = row_types.length == 1 ? "col" : "row"
-        end
-
-        if orient == "row"
-          column_names, schema_overrides = _unpack_schema(
-            schema, schema_overrides: schema_overrides, n_expected: first_element.length
-          )
-          local_schema_override = (
-            schema_overrides.any? ? _include_unknowns(schema_overrides, column_names) : {}
-          )
-          if column_names.any? && first_element.length > 0 && first_element.length != column_names.length
-            raise ArgumentError, "the row data does not match the number of columns"
-          end
-
-          unpack_nested = false
-          local_schema_override.each do |col, tp|
-            if [Categorical, Enum].include?(tp)
-              local_schema_override[col] = String
-            elsif !unpack_nested && [Unknown, Struct].include?(tp.base_type)
-              raise Todo
-            end
-          end
-
-          if unpack_nested
-            raise Todo
-          else
-            rbdf = RbDataFrame.from_rows(
-              data,
-              infer_schema_length,
-              local_schema_override.any? ? local_schema_override : nil
-            )
-          end
-          if column_names.any? || schema_overrides.any?
-            rbdf = _post_apply_columns(
-              rbdf, column_names, schema_overrides: schema_overrides, strict: strict
-            )
-          end
-          return rbdf
-        elsif orient == "col" || orient.nil?
-          column_names, schema_overrides = _unpack_schema(
-            schema, schema_overrides: schema_overrides, n_expected: data.length
-          )
-          data_series =
-            data.map.with_index do |element, i|
-              Series.new(column_names[i], element, dtype: schema_overrides[column_names[i]], strict: strict)._s
-            end
-          return RbDataFrame.new(data_series)
-        else
-          raise ArgumentError, "orient must be one of {{'col', 'row', nil}}, got #{orient} instead."
-        end
-      end
-
-      data_series = _handle_columns_arg(data_series, columns: columns)
-      RbDataFrame.new(data_series)
-    end
-
-    # @private
-    def self._include_unknowns(schema, cols)
-      cols.to_h { |col| [col, schema[col] || Unknown] }
-    end
-
-    # @private
-    def self.series_to_rbdf(data, schema: nil, schema_overrides: nil, strict: true)
-      data_series = [data._s]
-      series_name = data_series.map(&:name)
-      column_names, schema_overrides = _unpack_schema(
-        schema || series_name, schema_overrides: schema_overrides, n_expected: 1
-      )
-      if schema_overrides.any?
-        new_dtype = schema_overrides.values[0]
-        if new_dtype != data.dtype
-          data_series[0] = data_series[0].cast(new_dtype, strict)
-        end
-      end
-
-      data_series = _handle_columns_arg(data_series, columns: column_names)
-      RbDataFrame.new(data_series)
-    end
-
     def wrap_ldf(ldf)
       LazyFrame._from_rbldf(ldf)
     end
@@ -5341,6 +6891,11 @@ module Polars
       self.class._from_rbdf(rb_df)
     end
 
+    def _replace(column, new_column)
+      self._df.replace(column, new_column._s)
+      self
+    end
+
     def _comp(other, op)
       if other.is_a?(DataFrame)
         _compare_to_other_df(other, op)
@@ -5358,7 +6913,7 @@ module Polars
       end
 
       suffix = "__POLARS_CMP_OTHER"
-      other_renamed = other.select(Polars.all.suffix(suffix))
+      other_renamed = other.select(Polars.all.name.suffix(suffix))
       combined = Polars.concat([self, other_renamed], how: "horizontal")
 
       expr = case op
@@ -5410,5 +6965,284 @@ module Polars
       end
       other
     end
+
+    def with_connection(connection, &block)
+      if !connection.nil?
+        yield connection
+      else
+        ActiveRecord::Base.connection_pool.with_connection(&block)
+      end
+    end
+
+    def maybe_transaction(connection, create_table, &block)
+      if create_table && connection.adapter_name.match?(/postg|sqlite/i) && connection.open_transactions == 0
+        connection.transaction(&block)
+      else
+        yield
+      end
+    end
+
+    def get_series_item_by_key(s, key)
+      if key.is_a?(Integer)
+        return s._s.get_index_signed(key)
+
+      elsif key.is_a?(Range)
+          return _select_elements_by_slice(s, key)
+
+      elsif key.is_a?(::Array)
+        if key.empty?
+          return s.clear
+        end
+
+        first = key[0]
+        if Utils.bool?(first)
+          _raise_on_boolean_mask
+        end
+
+        begin
+          indices = Series.new("", key, dtype: Int64)
+        rescue TypeError
+          msg = "cannot select elements using Sequence with elements of type #{first.class.name.inspect}"
+          raise TypeError, msg
+        end
+
+        indices = _convert_series_to_indices(indices, s.len)
+        return _select_elements_by_index(s, indices)
+
+      elsif key.is_a?(Series)
+        indices = _convert_series_to_indices(key, s.len)
+        return _select_elements_by_index(s, indices)
+      end
+
+      msg = "cannot select elements using key of type #{key.class.name.inspect}: #{key.inspect}"
+      raise TypeError, msg
+    end
+
+    def _select_elements_by_slice(s, key)
+      Slice.new(s).apply(key)
+    end
+
+    def _select_elements_by_index(s, key)
+      s.send(:_from_rbseries, s._s.gather_with_series(key._s))
+    end
+
+    def get_df_item_by_key(df, key)
+      if key.size == 2
+        row_key, col_key = key
+
+        # Support df[True, False] and df["a", "b"] as these are not ambiguous
+        if Utils.bool?(row_key) || Utils.strlike?(row_key)
+          return _select_columns(df, key)
+        end
+
+        selection = _select_columns(df, col_key)
+
+        if selection.is_empty
+          return selection
+        elsif selection.is_a?(Series)
+          return get_series_item_by_key(selection, row_key)
+        else
+          return _select_rows(selection, row_key)
+        end
+      end
+
+      key = key[0] if key.size == 1
+
+      # Single string input, e.g. df["a"]
+      if Utils.strlike?(key)
+        # This case is required because empty strings are otherwise treated
+        # as an empty Sequence in `_select_rows`
+        return df.get_column(key)
+      end
+
+      # Single input - df[1] - or multiple inputs - df["a", "b", "c"]
+      begin
+         _select_rows(df, key)
+      rescue TypeError
+        _select_columns(df, key)
+      end
+    end
+
+    def _select_columns(df, key)
+      if key.is_a?(Integer)
+        return df.to_series(key)
+
+      elsif Utils.strlike?(key)
+        return df.get_column(key)
+
+      elsif key.is_a?(Range)
+        start, stop = key.begin, key.end
+        if start.is_a?(::String)
+          start = df.get_column_index(start)
+          stop = df.get_column_index(stop)
+          rng = Range.new(start, stop, key.exclude_end?)
+          return _select_columns_by_index(df, rng)
+        else
+          return _select_columns_by_index(df, key)
+        end
+
+      elsif key.is_a?(::Array)
+        if key.empty?
+          return df.class.new
+        end
+        first = key[0]
+        if Utils.bool?(first)
+          return _select_columns_by_mask(df, key)
+        elsif first.is_a?(Integer)
+          return _select_columns_by_index(df, key)
+        elsif Utils.strlike?(first)
+          return _select_columns_by_name(df, key)
+        else
+          msg = "cannot select columns using Sequence with elements of type #{first.class.name.inspect}"
+          raise TypeError, msg
+        end
+
+      elsif key.is_a?(Series)
+        if key.is_empty
+          return df.class.new
+        end
+        dtype = key.dtype
+        if dtype == String
+          return _select_columns_by_name(df, key)
+        elsif dtype.integer?
+          return _select_columns_by_index(df, key)
+        elsif dtype == Boolean
+          return _select_columns_by_mask(df, key)
+        else
+          msg = "cannot select columns using Series of type #{dtype}"
+          raise TypeError, msg
+        end
+      end
+
+      msg = (
+        "cannot select columns using key of type #{key.class.name.inspect}: #{key.inspect}"
+      )
+      raise TypeError, msg
+    end
+
+    def _select_columns_by_index(df, key)
+      series = key.map { |i| df.to_series(i) }
+      df.class.new(series)
+    end
+
+    def _select_columns_by_name(df, key)
+      df.send(:_from_rbdf, df._df.select(Array(key)))
+    end
+
+    def _select_columns_by_mask(df, key)
+      if key.length != df.width
+        msg = "expected #{df.width} values when selecting columns by boolean mask, got #{key.length}"
+        raise ArgumentError, msg
+      end
+
+      indices = key.each_with_index.filter_map { |val, i| i if val }
+      _select_columns_by_index(df, indices)
+    end
+
+    def _select_rows(df, key)
+      if key.is_a?(Integer)
+        num_rows = df.height
+        if key >= num_rows || key < -num_rows
+          msg = "index #{key} is out of bounds for DataFrame of height #{num_rows}"
+          raise IndexError, msg
+        end
+        return df.slice(key, 1)
+      end
+
+      if key.is_a?(Range)
+        return _select_rows_by_slice(df, key)
+
+      elsif key.is_a?(::Array)
+        if key.empty?
+          return df.clear
+        end
+        if Utils.bool?(key[0])
+          _raise_on_boolean_mask
+        end
+        s = Series.new("", key, dtype: Int64)
+        indices = _convert_series_to_indices(s, df.height)
+        return _select_rows_by_index(df, indices)
+
+      elsif key.is_a?(Series)
+        indices = _convert_series_to_indices(key, df.height)
+        return _select_rows_by_index(df, indices)
+
+      else
+        msg = "cannot select rows using key of type #{key.class.name.inspect}: #{key.inspect}"
+        raise TypeError, msg
+      end
+    end
+
+    def _select_rows_by_slice(df, key)
+      Slice.new(df).apply(key)
+    end
+
+    def _select_rows_by_index(df, key)
+      df.send(:_from_rbdf, df._df.gather_with_series(key._s))
+    end
+
+    def _convert_series_to_indices(s, size)
+      idx_type = Plr.get_index_type
+
+      if s.dtype == idx_type
+        return s
+      end
+
+      if !s.dtype.integer?
+        if s.dtype == Boolean
+          _raise_on_boolean_mask
+        else
+          msg = "cannot treat Series of type #{s.dtype} as indices"
+          raise TypeError, msg
+        end
+      end
+
+      if s.len == 0
+        return Series.new(s.name, [], dtype: idx_type)
+      end
+
+      if idx_type == UInt32
+        if [Int64, UInt64].include?(s.dtype) && s.max >= Utils::U32_MAX
+          msg = "index positions should be smaller than 2^32"
+          raise ArgumentError, msg
+        end
+        if s.dtype == Int64 && s.min < -Utils::U32_MAX
+          msg = "index positions should be greater than or equal to -2^32"
+          raise ArgumentError, msg
+        end
+      end
+
+      if s.dtype.signed_integer?
+        if s.min < 0
+          if idx_type == UInt32
+            idxs = [Int8, Int16].include?(s.dtype) ? s.cast(Int32) : s
+          else
+            idxs = [Int8, Int16, Int32].include?(s.dtype) ? s.cast(Int64) : s
+          end
+
+          # Update negative indexes to absolute indexes.
+          return (
+            idxs.to_frame
+            .select(
+              F.when(F.col(idxs.name) < 0)
+              .then(size + F.col(idxs.name))
+              .otherwise(F.col(idxs.name))
+              .cast(idx_type)
+            )
+            .to_series(0)
+          )
+        end
+      end
+
+      s.cast(idx_type)
+    end
+
+    def _raise_on_boolean_mask
+      msg = (
+        "selecting rows by passing a boolean mask to `[]` is not supported" +
+        "\n\nHint: Use the `filter` method instead."
+      )
+      raise TypeError, msg
+    end
   end
 end