polars-df 0.23.0 → 0.24.0

data/lib/polars/lazy_frame.rb

@@ -5,12 +5,13 @@ module Polars
     attr_accessor :_ldf
 
     # Create a new LazyFrame.
-    def initialize(data = nil, schema: nil, schema_overrides: nil, orient: nil, infer_schema_length: 100, nan_to_null: false)
+    def initialize(data = nil, schema: nil, schema_overrides: nil, strict: true, orient: nil, infer_schema_length: N_INFER_DEFAULT, nan_to_null: false)
       self._ldf = (
         DataFrame.new(
           data,
           schema: schema,
           schema_overrides: schema_overrides,
+          strict: strict,
           orient: orient,
           infer_schema_length: infer_schema_length,
           nan_to_null: nan_to_null
@@ -27,25 +28,16 @@ module Polars
       ldf
     end
 
-    # Read a logical plan from a JSON file to construct a LazyFrame.
-    #
-    # @param file [String]
-    #   Path to a file or a file-like object.
-    #
-    # @return [LazyFrame]
-    def self.read_json(file)
-      if Utils.pathlike?(file)
-        file = Utils.normalize_filepath(file)
-      end
-
-      Utils.wrap_ldf(RbLazyFrame.deserialize_json(file))
-    end
-
     # Read a logical plan from a file to construct a LazyFrame.
     #
     # @param source [Object]
     #   Path to a file or a file-like object (by file-like object, we refer to
     #   objects that have a `read` method, such as a file handler or `StringIO`).
+    # @param format ['binary', 'json']
+    #   The format with which the LazyFrame was serialized. Options:
+    #
+    #   - `"binary"`: Deserialize from binary format (bytes). This is the default.
+    #   - `"json"`: Deserialize from JSON format (string).
     #
     # @return [LazyFrame]
     #
@@ -71,14 +63,20 @@ module Polars
     #   # ╞═════╡
     #   # │ 6   │
     #   # └─────┘
-    def self.deserialize(source)
-      raise Todo unless RbLazyFrame.respond_to?(:deserialize_binary)
-
+    def self.deserialize(source, format: "binary")
       if Utils.pathlike?(source)
         source = Utils.normalize_filepath(source)
       end
 
-      deserializer = RbLazyFrame.method(:deserialize_binary)
+      if format == "binary"
+        raise Todo unless RbLazyFrame.respond_to?(:deserialize_binary)
+        deserializer = RbLazyFrame.method(:deserialize_binary)
+      elsif format == "json"
+        deserializer = RbLazyFrame.method(:deserialize_json)
+      else
+        msg = "`format` must be one of {{'binary', 'json'}}, got #{format.inspect}"
+        raise ArgumentError, msg
+      end
 
       _from_rbldf(deserializer.(source))
     end
@@ -170,31 +168,22 @@ module Polars
     # @return [String]
     def to_s
       <<~EOS
-        naive plan: (run LazyFrame#describe_optimized_plan to see the optimized plan)
+        naive plan: (run LazyFrame#explain(optimized: true) to see the optimized plan)
 
-        #{describe_plan}
+        #{explain(optimized: false)}
       EOS
     end
 
-    # Write the logical plan of this LazyFrame to a file or string in JSON format.
-    #
-    # @param file [String]
-    #   File path to which the result should be written.
-    #
-    # @return [nil]
-    def write_json(file)
-      if Utils.pathlike?(file)
-        file = Utils.normalize_filepath(file)
-      end
-      _ldf.write_json(file)
-      nil
-    end
-
     # Serialize the logical plan of this LazyFrame to a file or string.
     #
     # @param file [Object]
     #   File path to which the result should be written. If set to `nil`
     #   (default), the output is returned as a string instead.
+    # @param format ['binary', 'json']
+    #   The format in which to serialize. Options:
+    #
+    #   - `"binary"`: Serialize to binary format (bytes). This is the default.
+    #   - `"json"`: Serialize to JSON format (string) (deprecated).
     #
     # @return [Object]
     #
@@ -215,16 +204,25 @@ module Polars
     #   # ╞═════╡
     #   # │ 6   │
     #   # └─────┘
-    def serialize(file = nil)
-      raise Todo unless _ldf.respond_to?(:serialize_binary)
+    def serialize(file = nil, format: "binary")
+      if format == "binary"
+        raise Todo unless _ldf.respond_to?(:serialize_binary)
+        serializer = _ldf.method(:serialize_binary)
+      elsif format == "json"
+        msg = "'json' serialization format of LazyFrame is deprecated"
+        warn msg
+        serializer = _ldf.method(:serialize_json)
+      else
+        msg = "`format` must be one of {{'binary', 'json'}}, got #{format.inspect}"
+        raise ArgumentError, msg
+      end
 
-      serializer = _ldf.method(:serialize_binary)
       Utils.serialize_polars_object(serializer, file)
     end
 
     # Offers a structured way to apply a sequence of user-defined functions (UDFs).
     #
-    # @param func [Object]
+    # @param function [Object]
     #   Callable; will receive the frame as the first parameter,
     #   followed by any given args/kwargs.
     # @param args [Object]
@@ -236,7 +234,7 @@ module Polars
     #
     # @example
     #   cast_str_to_int = lambda do |data, col_name:|
-    #     data.with_column(Polars.col(col_name).cast(:i64))
+    #     data.with_columns(Polars.col(col_name).cast(Polars::Int64))
     #   end
     #
     #   df = Polars::DataFrame.new({"a" => [1, 2, 3, 4], "b" => ["10", "20", "30", "40"]}).lazy
@@ -253,47 +251,344 @@ module Polars
     #   # │ 3   ┆ 30  │
     #   # │ 4   ┆ 40  │
     #   # └─────┴─────┘
-    def pipe(func, *args, **kwargs, &block)
-      func.call(self, *args, **kwargs, &block)
+    def pipe(function, *args, **kwargs, &block)
+      function.(self, *args, **kwargs, &block)
     end
 
-    # Create a string representation of the unoptimized query plan.
+    # Creates a summary of statistics for a LazyFrame, returning a DataFrame.
     #
-    # @return [String]
-    def describe_plan
-      _ldf.describe_plan
+    # @param percentiles [Array]
+    #   One or more percentiles to include in the summary statistics.
+    #   All values must be in the range `[0, 1]`.
+    # @param interpolation ['nearest', 'higher', 'lower', 'midpoint', 'linear', 'equiprobable']
+    #   Interpolation method used when calculating percentiles.
+    #
+    # @return [DataFrame]
+    #
+    # @note
+    #   The median is included by default as the 50% percentile.
+    #
+    # @note
+    #   This method does *not* maintain the laziness of the frame, and will `collect`
+    #   the final result. This could potentially be an expensive operation.
+    #
+    # @note
+    #   We do not guarantee the output of `describe` to be stable. It will show
+    #   statistics that we deem informative, and may be updated in the future.
+    #   Using `describe` programmatically (versus interactive exploration) is
+    #   not recommended for this reason.
+    #
+    # @example Show default frame statistics:
+    #   lf = Polars::LazyFrame.new(
+    #     {
+    #       "float" => [1.0, 2.8, 3.0],
+    #       "int" => [40, 50, nil],
+    #       "bool" => [true, false, true],
+    #       "str" => ["zz", "xx", "yy"],
+    #       "date" => [Date.new(2020, 1, 1), Date.new(2021, 7, 5), Date.new(2022, 12, 31)]
+    #     }
+    #   )
+    #   lf.describe
+    #   # =>
+    #   # shape: (9, 6)
+    #   # ┌────────────┬──────────┬──────────┬──────────┬──────┬─────────────────────────┐
+    #   # │ statistic  ┆ float    ┆ int      ┆ bool     ┆ str  ┆ date                    │
+    #   # │ ---        ┆ ---      ┆ ---      ┆ ---      ┆ ---  ┆ ---                     │
+    #   # │ str        ┆ f64      ┆ f64      ┆ f64      ┆ str  ┆ str                     │
+    #   # ╞════════════╪══════════╪══════════╪══════════╪══════╪═════════════════════════╡
+    #   # │ count      ┆ 3.0      ┆ 2.0      ┆ 3.0      ┆ 3    ┆ 3                       │
+    #   # │ null_count ┆ 0.0      ┆ 1.0      ┆ 0.0      ┆ 0    ┆ 0                       │
+    #   # │ mean       ┆ 2.266667 ┆ 45.0     ┆ 0.666667 ┆ null ┆ 2021-07-02 16:00:00 UTC │
+    #   # │ std        ┆ 1.101514 ┆ 7.071068 ┆ null     ┆ null ┆ null                    │
+    #   # │ min        ┆ 1.0      ┆ 40.0     ┆ 0.0      ┆ xx   ┆ 2020-01-01              │
+    #   # │ 25%        ┆ 2.8      ┆ 40.0     ┆ null     ┆ null ┆ 2021-07-05              │
+    #   # │ 50%        ┆ 2.8      ┆ 50.0     ┆ null     ┆ null ┆ 2021-07-05              │
+    #   # │ 75%        ┆ 3.0      ┆ 50.0     ┆ null     ┆ null ┆ 2022-12-31              │
+    #   # │ max        ┆ 3.0      ┆ 50.0     ┆ 1.0      ┆ zz   ┆ 2022-12-31              │
+    #   # └────────────┴──────────┴──────────┴──────────┴──────┴─────────────────────────┘
+    #
+    # @example Customize which percentiles are displayed, applying linear interpolation:
+    #   lf.describe(
+    #     percentiles: [0.1, 0.3, 0.5, 0.7, 0.9],
+    #     interpolation: "linear"
+    #   )
+    #   # =>
+    #   # shape: (11, 6)
+    #   # ┌────────────┬──────────┬──────────┬──────────┬──────┬─────────────────────────┐
+    #   # │ statistic  ┆ float    ┆ int      ┆ bool     ┆ str  ┆ date                    │
+    #   # │ ---        ┆ ---      ┆ ---      ┆ ---      ┆ ---  ┆ ---                     │
+    #   # │ str        ┆ f64      ┆ f64      ┆ f64      ┆ str  ┆ str                     │
+    #   # ╞════════════╪══════════╪══════════╪══════════╪══════╪═════════════════════════╡
+    #   # │ count      ┆ 3.0      ┆ 2.0      ┆ 3.0      ┆ 3    ┆ 3                       │
+    #   # │ null_count ┆ 0.0      ┆ 1.0      ┆ 0.0      ┆ 0    ┆ 0                       │
+    #   # │ mean       ┆ 2.266667 ┆ 45.0     ┆ 0.666667 ┆ null ┆ 2021-07-02 16:00:00 UTC │
+    #   # │ std        ┆ 1.101514 ┆ 7.071068 ┆ null     ┆ null ┆ null                    │
+    #   # │ min        ┆ 1.0      ┆ 40.0     ┆ 0.0      ┆ xx   ┆ 2020-01-01              │
+    #   # │ …          ┆ …        ┆ …        ┆ …        ┆ …    ┆ …                       │
+    #   # │ 30%        ┆ 2.08     ┆ 43.0     ┆ null     ┆ null ┆ 2020-11-26              │
+    #   # │ 50%        ┆ 2.8      ┆ 45.0     ┆ null     ┆ null ┆ 2021-07-05              │
+    #   # │ 70%        ┆ 2.88     ┆ 47.0     ┆ null     ┆ null ┆ 2022-02-07              │
+    #   # │ 90%        ┆ 2.96     ┆ 49.0     ┆ null     ┆ null ┆ 2022-09-13              │
+    #   # │ max        ┆ 3.0      ┆ 50.0     ┆ 1.0      ┆ zz   ┆ 2022-12-31              │
+    #   # └────────────┴──────────┴──────────┴──────────┴──────┴─────────────────────────┘
+    def describe(
+      percentiles: [0.25, 0.5, 0.75],
+      interpolation: "nearest"
+    )
+      schema = collect_schema.to_h
+
+      if schema.empty?
+        msg = "cannot describe a LazyFrame that has no columns"
+        raise TypeError, msg
+      end
+
+      # create list of metrics
+      metrics = ["count", "null_count", "mean", "std", "min"]
+      if (quantiles = Utils.parse_percentiles(percentiles)).any?
+        metrics.concat(quantiles.map { |q| "%g%%" % [q * 100] })
+      end
+      metrics.append("max")
+
+      skip_minmax = lambda do |dt|
+        dt.nested? || [Categorical, Enum, Null, Object, Unknown].include?(dt)
+      end
+
+      # determine which columns will produce std/mean/percentile/etc
+      # statistics in a single pass over the frame schema
+      has_numeric_result, sort_cols = Set.new, Set.new
+      metric_exprs = []
+      null = F.lit(nil)
+
+      schema.each do |c, dtype|
+        is_numeric = dtype.numeric?
+        is_temporal = !is_numeric && dtype.temporal?
+
+        # counts
+        count_exprs = [
+          F.col(c).count.name.prefix("count:"),
+          F.col(c).null_count.name.prefix("null_count:")
+        ]
+        # mean
+        mean_expr =
+          if is_temporal || is_numeric || dtype == Boolean
+            F.col(c).mean
+          else
+            null
+          end
+
+        # standard deviation, min, max
+        expr_std = is_numeric ? F.col(c).std : null
+        min_expr = !skip_minmax.(dtype) ? F.col(c).min : null
+        max_expr = !skip_minmax.(dtype) ? F.col(c).max : null
+
+        # percentiles
+        pct_exprs = []
+        quantiles.each do |p|
+          if is_numeric || is_temporal
+            pct_expr =
+              if is_temporal
+                F.col(c).to_physical.quantile(p, interpolation: interpolation).cast(dtype)
+              else
+                F.col(c).quantile(p, interpolation: interpolation)
+              end
+            sort_cols.add(c)
+          else
+            pct_expr = null
+          end
+          pct_exprs << pct_expr.alias("#{p}:#{c}")
+        end
+
+        if is_numeric || dtype.nested? || [Null, Boolean].include?(dtype)
+          has_numeric_result.add(c)
+        end
+
+        # add column expressions (in end-state 'metrics' list order)
+        metric_exprs.concat(
+          [
+            *count_exprs,
+            mean_expr.alias("mean:#{c}"),
+            expr_std.alias("std:#{c}"),
+            min_expr.alias("min:#{c}"),
+            *pct_exprs,
+            max_expr.alias("max:#{c}")
+          ]
+        )
+      end
+
+      # calculate requested metrics in parallel, then collect the result
+      df_metrics = (
+        (
+          # if more than one quantile, sort the relevant columns to make them O(1)
+          # TODO: drop sort once we have efficient retrieval of multiple quantiles
+          sort_cols ? with_columns(sort_cols.map { |c| F.col(c).sort }) : self
+        )
+        .select(*metric_exprs)
+        .collect
+      )
+
+      # reshape wide result
+      n_metrics = metrics.length
+      column_metrics =
+        schema.length.times.map do |n|
+          df_metrics.row(0)[(n * n_metrics)...((n + 1) * n_metrics)]
+        end
+
+      summary = schema.keys.zip(column_metrics).to_h
+
+      # cast by column type (numeric/bool -> float), (other -> string)
+      schema.each_key do |c|
+        summary[c] =
+          summary[c].map do |v|
+            if v.nil? || v.is_a?(Hash)
+              nil
+            else
+              if has_numeric_result.include?(c)
+                if v == true
+                  1.0
+                elsif v == false
+                  0.0
+                else
+                  v.to_f
+                end
+              else
+                "#{v}"
+              end
+            end
+          end
+      end
+
+      # return results as a DataFrame
+      df_summary = Polars.from_hash(summary)
+      df_summary.insert_column(0, Polars::Series.new("statistic", metrics))
+      df_summary
     end
 
-    # Create a string representation of the optimized query plan.
+    # Create a string representation of the query plan.
+    #
+    # Different optimizations can be turned on or off.
     #
     # @return [String]
-    def describe_optimized_plan(
-      type_coercion: true,
-      predicate_pushdown: true,
-      projection_pushdown: true,
-      simplify_expression: true,
-      slice_pushdown: true,
-      common_subplan_elimination: true,
-      comm_subexpr_elim: true,
-      allow_streaming: false
+    #
+    # @example
+    #   lf = Polars::LazyFrame.new(
+    #     {
+    #       "a" => ["a", "b", "a", "b", "b", "c"],
+    #       "b" => [1, 2, 3, 4, 5, 6],
+    #       "c" => [6, 5, 4, 3, 2, 1]
+    #     }
+    #   )
+    #   lf.group_by("a", maintain_order: true).agg(Polars.all.sum).sort(
+    #     "a"
+    #   ).explain
+    def explain(
+      format: "plain",
+      optimized: true,
+      engine: "auto",
+      optimizations: DEFAULT_QUERY_OPT_FLAGS
     )
-      ldf = _ldf.optimization_toggle(
-        type_coercion,
-        predicate_pushdown,
-        projection_pushdown,
-        simplify_expression,
-        slice_pushdown,
-        common_subplan_elimination,
-        comm_subexpr_elim,
-        allow_streaming,
-        false
-      )
+      engine = _select_engine(engine)
+
+      if engine == "streaming"
+        Utils.issue_unstable_warning("streaming mode is considered unstable.")
+      end
 
-      ldf.describe_optimized_plan
+      if optimized
+        ldf = _ldf.with_optimizations(optimizations._rboptflags)
+        if format == "tree"
+          return ldf.describe_optimized_plan_tree
+        else
+          return ldf.describe_optimized_plan
+        end
+      end
+
+      if format == "tree"
+        _ldf.describe_plan_tree
+      else
+        _ldf.describe_plan
+      end
     end
 
-    # def show_graph
-    # end
+    # Show a plot of the query plan.
+    #
+    # Note that Graphviz must be installed to render the visualization (if not
+    # already present, you can download it here: https://graphviz.org/download.
+    #
+    # @param optimized [Boolean]
+    #   Optimize the query plan.
+    # @param show [Boolean]
+    #   Show the figure.
+    # @param output_path [String]
+    #   Write the figure to disk.
+    # @param raw_output [Boolean]
+    #   Return dot syntax. This cannot be combined with `show` and/or `output_path`.
+    # @param engine [String]
+    #   Select the engine used to process the query, optional.
+    #   At the moment, if set to `"auto"` (default), the query
+    #   is run using the polars in-memory engine. Polars will also
+    #   attempt to use the engine set by the `POLARS_ENGINE_AFFINITY`
+    #   environment variable. If it cannot run the query using the
+    #   selected engine, the query is run using the polars in-memory
+    #   engine.
+    # @param plan_stage ['ir', 'physical']
+    #   Select the stage to display. Currently only the streaming engine has a
+    #   separate physical stage, for the other engines both IR and physical are the
+    #   same.
+    # @param optimizations [Object]
+    #   The set of the optimizations considered during query optimization.
+    #
+    # @return [Object]
+    #
+    # @example
+    #   lf = Polars::LazyFrame.new(
+    #     {
+    #       "a" => ["a", "b", "a", "b", "b", "c"],
+    #       "b" => [1, 2, 3, 4, 5, 6],
+    #       "c" => [6, 5, 4, 3, 2, 1]
+    #     }
+    #   )
+    #   lf.group_by("a", maintain_order: true).agg(Polars.all.sum).sort(
+    #     "a"
+    #   ).show_graph
+    def show_graph(
+      optimized: true,
+      show: true,
+      output_path: nil,
+      raw_output: false,
+      figsize: [16.0, 12.0],
+      engine: "auto",
+      plan_stage: "ir",
+      optimizations: DEFAULT_QUERY_OPT_FLAGS
+    )
+      engine = _select_engine(engine)
+
+      if engine == "streaming"
+        issue_unstable_warning("streaming mode is considered unstable.")
+      end
+
+      optimizations = optimizations.dup
+      optimizations._rboptflags.streaming = engine == "streaming"
+      _ldf = self._ldf.with_optimizations(optimizations._rboptflags)
+
+      if plan_stage == "ir"
+        dot = _ldf.to_dot(optimized)
+      elsif plan_stage == "physical"
+        if engine == "streaming"
+          dot = _ldf.to_dot_streaming_phys(optimized)
+        else
+          dot = _ldf.to_dot(optimized)
+        end
+      else
+        error_msg = "invalid plan stage '#{plan_stage}'"
+        raise TypeError, error_msg
+      end
+
+      Utils.display_dot_graph(
+        dot: dot,
+        show: show,
+        output_path: output_path,
+        raw_output: raw_output
+      )
+    end
 
     # Sort the DataFrame.
     #
@@ -307,7 +602,7 @@ module Polars
     #   Column (expressions) to sort by.
     # @param more_by [Array]
     #   Additional columns to sort by, specified as positional arguments.
-    # @param reverse [Boolean]
+    # @param descending [Boolean]
     #   Sort in descending order.
     # @param nulls_last [Boolean]
     #   Place null values last. Can only be used if sorted by a single column.
@@ -328,7 +623,7 @@ module Polars
     #       "ham" => ["a", "b", "c"]
     #     }
     #   ).lazy
-    #   df.sort("foo", reverse: true).collect
+    #   df.sort("foo", descending: true).collect
     #   # =>
     #   # shape: (3, 3)
     #   # ┌─────┬─────┬─────┐
@@ -340,21 +635,21 @@ module Polars
     #   # │ 2   ┆ 7.0 ┆ b   │
     #   # │ 1   ┆ 6.0 ┆ a   │
     #   # └─────┴─────┴─────┘
-    def sort(by, *more_by, reverse: false, nulls_last: false, maintain_order: false, multithreaded: true)
+    def sort(by, *more_by, descending: false, nulls_last: false, maintain_order: false, multithreaded: true)
       if by.is_a?(::String) && more_by.empty?
         return _from_rbldf(
           _ldf.sort(
-            by, reverse, nulls_last, maintain_order, multithreaded
+            by, descending, nulls_last, maintain_order, multithreaded
           )
         )
       end
 
       by = Utils.parse_into_list_of_expressions(by, *more_by)
-      reverse = Utils.extend_bool(reverse, by.length, "reverse", "by")
+      descending = Utils.extend_bool(descending, by.length, "descending", "by")
       nulls_last = Utils.extend_bool(nulls_last, by.length, "nulls_last", "by")
       _from_rbldf(
         _ldf.sort_by_exprs(
-          by, reverse, nulls_last, maintain_order, multithreaded
+          by, descending, nulls_last, maintain_order, multithreaded
         )
       )
     end
@@ -440,7 +735,7 @@ module Polars
     #   Accepts expression input. Strings are parsed as column names.
     # @param reverse [Object]
     #   Consider the `k` smallest elements of the `by` column(s) (instead of the `k`
-    #   largest). This can be specified per column by passing a sequence of
+    #   largest). This can be specified per column by passing an array of
     #   booleans.
     #
     # @return [LazyFrame]
@@ -504,7 +799,7 @@ module Polars
     #   Accepts expression input. Strings are parsed as column names.
     # @param reverse [Object]
     #   Consider the `k` largest elements of the `by` column(s) (instead of the `k`
-    #   smallest). This can be specified per column by passing a sequence of
+    #   smallest). This can be specified per column by passing an array of
     #   booleans.
     #
     # @return [LazyFrame]
@@ -554,35 +849,90 @@ module Polars
       _from_rbldf(_ldf.bottom_k(k, by, reverse))
     end
 
-    # def profile
-    # end
+    # Profile a LazyFrame.
+    #
+    # This will run the query and return a tuple
+    # containing the materialized DataFrame and a DataFrame that
+    # contains profiling information of each node that is executed.
+    #
+    # The units of the timings are microseconds.
+    #
+    # @param engine [String]
+    #     Select the engine used to process the query, optional.
+    #     At the moment, if set to `"auto"` (default), the query
+    #     is run using the polars in-memory engine. Polars will also
+    #     attempt to use the engine set by the `POLARS_ENGINE_AFFINITY`
+    #     environment variable. If it cannot run the query using the
+    #     selected engine, the query is run using the polars in-memory
+    #     engine.
+    # @param optimizations [Object]
+    #     The optimization passes done during query optimization.
+    #
+    # @return [Array]
+    #
+    # @example
+    #   lf = Polars::LazyFrame.new(
+    #     {
+    #       "a" => ["a", "b", "a", "b", "b", "c"],
+    #       "b" => [1, 2, 3, 4, 5, 6],
+    #       "c" => [6, 5, 4, 3, 2, 1]
+    #     }
+    #   )
+    #   lf.group_by("a", maintain_order: true).agg(Polars.all.sum).sort(
+    #     "a"
+    #   ).profile
+    #   # =>
+    #   # [shape: (3, 3)
+    #   # ┌─────┬─────┬─────┐
+    #   # │ a   ┆ b   ┆ c   │
+    #   # │ --- ┆ --- ┆ --- │
+    #   # │ str ┆ i64 ┆ i64 │
+    #   # ╞═════╪═════╪═════╡
+    #   # │ a   ┆ 4   ┆ 10  │
+    #   # │ b   ┆ 11  ┆ 10  │
+    #   # │ c   ┆ 6   ┆ 1   │
+    #   # └─────┴─────┴─────┘,
+    #   # shape: (3, 3)
+    #   # ┌──────────────┬───────┬─────┐
+    #   # │ node         ┆ start ┆ end │
+    #   # │ ---          ┆ ---   ┆ --- │
+    #   # │ str          ┆ u64   ┆ u64 │
+    #   # ╞══════════════╪═══════╪═════╡
+    #   # │ optimization ┆ 0     ┆ 67  │
+    #   # │ sort(a)      ┆ 67    ┆ 79  │
+    #   # └──────────────┴───────┴─────┘]
+    def profile(
+      engine: "auto",
+      optimizations: DEFAULT_QUERY_OPT_FLAGS
+    )
+      engine = _select_engine(engine)
+
+      ldf = _ldf.with_optimizations(optimizations._rboptflags)
+
+      df_rb, timings_rb = ldf.profile
+      [Utils.wrap_df(df_rb), Utils.wrap_df(timings_rb)]
+    end
 
-    # Collect into a DataFrame.
-    #
-    # Note: use {#fetch} if you want to run your query on the first `n` rows
-    # only. This can be a huge time saver in debugging queries.
-    #
-    # @param type_coercion [Boolean]
-    #   Do type coercion optimization.
-    # @param predicate_pushdown [Boolean]
-    #   Do predicate pushdown optimization.
-    # @param projection_pushdown [Boolean]
-    #   Do projection pushdown optimization.
-    # @param simplify_expression [Boolean]
-    #   Run simplify expressions optimization.
-    # @param string_cache [Boolean]
-    #   This argument is deprecated. Please set the string cache globally.
-    #   The argument will be ignored
-    # @param no_optimization [Boolean]
-    #   Turn off (certain) optimizations.
-    # @param slice_pushdown [Boolean]
-    #   Slice pushdown optimization.
-    # @param common_subplan_elimination [Boolean]
-    #   Will try to cache branching subplans that occur on self-joins or unions.
-    # @param comm_subexpr_elim [Boolean]
-    #   Common subexpressions will be cached and reused.
-    # @param allow_streaming [Boolean]
-    #   Run parts of the query in a streaming fashion (this is in an alpha state)
+    # Materialize this LazyFrame into a DataFrame.
+    #
+    # By default, all query optimizations are enabled. Individual optimizations may
+    # be disabled by setting the corresponding parameter to `false`.
+    #
+    # @param engine
+    #   Select the engine used to process the query, optional.
+    #   At the moment, if set to `"auto"` (default), the query is run
+    #   using the polars streaming engine. Polars will also
+    #   attempt to use the engine set by the `POLARS_ENGINE_AFFINITY`
+    #   environment variable. If it cannot run the query using the
+    #   selected engine, the query is run using the polars streaming
+    #   engine.
+    # @param background [Boolean]
+    #   Run the query in the background and get a handle to the query.
+    #   This handle can be used to fetch the result or cancel the query.
+    # @param optimizations
+    #   The optimization passes done during query optimization.
+    #
+    #   This has no effect if `lazy` is set to `true`.
     #
     # @return [DataFrame]
     #
@@ -607,42 +957,23 @@ module Polars
     #   # │ c   ┆ 6   ┆ 1   │
     #   # └─────┴─────┴─────┘
     def collect(
-      type_coercion: true,
-      predicate_pushdown: true,
-      projection_pushdown: true,
-      simplify_expression: true,
-      string_cache: false,
-      no_optimization: false,
-      slice_pushdown: true,
-      common_subplan_elimination: true,
-      comm_subexpr_elim: true,
-      allow_streaming: false,
-      _eager: false
+      engine: "auto",
+      background: false,
+      optimizations: DEFAULT_QUERY_OPT_FLAGS
     )
-      if no_optimization
-        predicate_pushdown = false
-        projection_pushdown = false
-        slice_pushdown = false
-        common_subplan_elimination = false
-        comm_subexpr_elim = false
+      engine = _select_engine(engine)
+
+      if engine == "streaming"
+        Utils.issue_unstable_warning("streaming mode is considered unstable.")
       end
 
-      if allow_streaming
-        common_subplan_elimination = false
+      ldf = _ldf.with_optimizations(optimizations._rboptflags)
+      if background
+        Utils.issue_unstable_warning("background mode is considered unstable.")
+        return InProcessQuery.new(ldf.collect_concurrently)
       end
 
-      ldf = _ldf.optimization_toggle(
-        type_coercion,
-        predicate_pushdown,
-        projection_pushdown,
-        simplify_expression,
-        slice_pushdown,
-        common_subplan_elimination,
-        comm_subexpr_elim,
-        allow_streaming,
-        _eager
-      )
-      Utils.wrap_df(ldf.collect)
+      Utils.wrap_df(ldf.collect(engine))
     end
 
     # Resolve the schema of this LazyFrame.
@@ -705,24 +1036,12 @@ module Polars
     #   If `nil` (default), the chunks of the `DataFrame` are
     #   used. Writing in smaller chunks may reduce memory pressure and improve
     #   writing speeds.
-    # @param data_pagesize_limit [Integer]
+    # @param data_page_size [Integer]
     #   Size limit of individual data pages.
     #   If not set defaults to 1024 * 1024 bytes
     # @param maintain_order [Boolean]
     #   Maintain the order in which data is processed.
     #   Setting this to `false` will  be slightly faster.
-    # @param type_coercion [Boolean]
-    #   Do type coercion optimization.
-    # @param predicate_pushdown [Boolean]
-    #   Do predicate pushdown optimization.
-    # @param projection_pushdown [Boolean]
-    #   Do projection pushdown optimization.
-    # @param simplify_expression [Boolean]
-    #   Run simplify expressions optimization.
-    # @param no_optimization [Boolean]
-    #   Turn off (certain) optimizations.
-    # @param slice_pushdown [Boolean]
-    #   Slice pushdown optimization.
     # @param storage_options [String]
     #   Options that indicate how to connect to a cloud provider.
     #
@@ -748,6 +1067,18 @@ module Polars
     #   Recursively create all the directories in the path.
     # @param lazy [Boolean]
     #     Wait to start execution until `collect` is called.
+    # @param engine
+    #   Select the engine used to process the query, optional.
+    #   At the moment, if set to `"auto"` (default), the query is run
+    #   using the polars streaming engine. Polars will also
+    #   attempt to use the engine set by the `POLARS_ENGINE_AFFINITY`
+    #   environment variable. If it cannot run the query using the
+    #   selected engine, the query is run using the polars streaming
+    #   engine.
+    # @param optimizations
+    #   The optimization passes done during query optimization.
+    #
+    #   This has no effect if `lazy` is set to `true`.
     #
     # @return [DataFrame]
     #
@@ -760,28 +1091,20 @@ module Polars
       compression_level: nil,
       statistics: true,
       row_group_size: nil,
-      data_pagesize_limit: nil,
+      data_page_size: nil,
       maintain_order: true,
-      type_coercion: true,
-      predicate_pushdown: true,
-      projection_pushdown: true,
-      simplify_expression: true,
-      no_optimization: false,
-      slice_pushdown: true,
       storage_options: nil,
+      credential_provider: "auto",
       retries: 2,
       sync_on_close: nil,
+      metadata: nil,
       mkdir: false,
-      lazy: false
+      lazy: false,
+      field_overwrites: nil,
+      engine: "auto",
+      optimizations: DEFAULT_QUERY_OPT_FLAGS
     )
-      lf = _set_sink_optimizations(
-        type_coercion: type_coercion,
-        predicate_pushdown: predicate_pushdown,
-        projection_pushdown: projection_pushdown,
-        simplify_expression: simplify_expression,
-        slice_pushdown: slice_pushdown,
-        no_optimization: no_optimization
-      )
+      engine = _select_engine(engine, path)
 
       if statistics == true
         statistics = {
@@ -801,6 +1124,12 @@ module Polars
         }
       end
 
+      _init_credential_provider_builder = Polars.method(:_init_credential_provider_builder)
+
+      credential_provider_builder = _init_credential_provider_builder.(
+        credential_provider, path, storage_options, "sink_parquet"
+      )
+
       if storage_options&.any?
         storage_options = storage_options.to_a
       else
@@ -813,24 +1142,33 @@ module Polars
         "mkdir" => mkdir
       }
 
-      lf = lf.sink_parquet(
+      field_overwrites_dicts = []
+      if !field_overwrites.nil?
+        raise Todo
+      end
+
+      ldf_rb = _ldf.sink_parquet(
         path,
         compression,
         compression_level,
         statistics,
         row_group_size,
-        data_pagesize_limit,
+        data_page_size,
         storage_options,
+        credential_provider_builder,
         retries,
-        sink_options
+        sink_options,
+        metadata,
+        field_overwrites_dicts
       )
-      lf = LazyFrame._from_rbldf(lf)
 
       if !lazy
-        lf.collect
+        ldf_rb = ldf_rb.with_optimizations(optimizations._rboptflags)
+        ldf = LazyFrame._from_rbldf(ldf_rb)
+        ldf.collect(engine: engine)
         return nil
       end
-      lf
+      LazyFrame._from_rbldf(ldf_rb)
     end
 
     # Evaluate the query in streaming mode and write to an IPC file.
@@ -860,18 +1198,6 @@ module Polars
     #   information from environment variables.
     # @param retries [Integer]
     #   Number of retries if accessing a cloud instance fails.
-    # @param type_coercion [Boolean]
-    #   Do type coercion optimization.
-    # @param predicate_pushdown [Boolean]
-    #   Do predicate pushdown optimization.
-    # @param projection_pushdown [Boolean]
-    #   Do projection pushdown optimization.
-    # @param simplify_expression [Boolean]
-    #   Run simplify expressions optimization.
-    # @param slice_pushdown [Boolean]
-    #   Slice pushdown optimization.
-    # @param no_optimization [Boolean]
-    #   Turn off (certain) optimizations.
     # @param sync_on_close ['data', 'all']
     #     Sync to disk when before closing a file.
     #
@@ -882,6 +1208,18 @@ module Polars
     #     Recursively create all the directories in the path.
     # @param lazy [Boolean]
     #     Wait to start execution until `collect` is called.
+    # @param engine
+    #   Select the engine used to process the query, optional.
+    #   At the moment, if set to `"auto"` (default), the query is run
+    #   using the polars streaming engine. Polars will also
+    #   attempt to use the engine set by the `POLARS_ENGINE_AFFINITY`
+    #   environment variable. If it cannot run the query using the
+    #   selected engine, the query is run using the polars streaming
+    #   engine.
+    # @param optimizations
+    #   The optimization passes done during query optimization.
+    #
+    #   This has no effect if `lazy` is set to `true`.
     #
     # @return [DataFrame]
     #
@@ -890,27 +1228,24 @@ module Polars
     #   lf.sink_ipc("out.arrow")
     def sink_ipc(
       path,
-      compression: "zstd",
+      compression: "uncompressed",
+      compat_level: nil,
       maintain_order: true,
       storage_options: nil,
+      credential_provider: "auto",
       retries: 2,
-      type_coercion: true,
-      predicate_pushdown: true,
-      projection_pushdown: true,
-      simplify_expression: true,
-      slice_pushdown: true,
-      no_optimization: false,
       sync_on_close: nil,
       mkdir: false,
-      lazy: false
+      lazy: false,
+      engine: "auto",
+      optimizations: DEFAULT_QUERY_OPT_FLAGS
     )
-      lf = _set_sink_optimizations(
-        type_coercion: type_coercion,
-        predicate_pushdown: predicate_pushdown,
-        projection_pushdown: projection_pushdown,
-        simplify_expression: simplify_expression,
-        slice_pushdown: slice_pushdown,
-        no_optimization: no_optimization
+      engine = _select_engine(engine, path)
+
+      _init_credential_provider_builder = Polars.method(:_init_credential_provider_builder)
+
+      credential_provider_builder = _init_credential_provider_builder.(
+        credential_provider, path, storage_options, "sink_ipc"
       )
 
       if storage_options&.any?
@@ -925,20 +1260,34 @@ module Polars
         "mkdir" => mkdir
       }
 
-      lf = lf.sink_ipc(
+      compat_level_rb = nil
+      if compat_level.nil?
+        compat_level_rb = true
+      else
+        raise Todo
+      end
+
+      if compression.nil?
+        compression = "uncompressed"
+      end
+
+      ldf_rb = _ldf.sink_ipc(
         path,
         compression,
+        compat_level_rb,
         storage_options,
+        credential_provider_builder,
         retries,
         sink_options
       )
-      lf = LazyFrame._from_rbldf(lf)
 
       if !lazy
-        lf.collect
+        ldf_rb = ldf_rb.with_optimizations(optimizations._rboptflags)
+        ldf = LazyFrame._from_rbldf(ldf_rb)
+        ldf.collect(engine: engine)
         return nil
       end
-      lf
+      LazyFrame._from_rbldf(ldf_rb)
     end
 
     # Evaluate the query in streaming mode and write to a CSV file.
@@ -1004,32 +1353,32 @@ module Polars
     # @param maintain_order [Boolean]
     #   Maintain the order in which data is processed.
     #   Setting this to `false` will  be slightly faster.
-    # @param type_coercion [Boolean]
-    #   Do type coercion optimization.
-    # @param predicate_pushdown [Boolean]
-    #   Do predicate pushdown optimization.
-    # @param projection_pushdown [Boolean]
-    #   Do projection pushdown optimization.
-    # @param simplify_expression [Boolean]
-    #   Run simplify expressions optimization.
-    # @param slice_pushdown [Boolean]
-    #   Slice pushdown optimization.
-    # @param no_optimization [Boolean]
-    #   Turn off (certain) optimizations.
     # @param storage_options [Object]
     #   Options that indicate how to connect to a cloud provider.
     # @param retries [Integer]
     #   Number of retries if accessing a cloud instance fails.
     # @param sync_on_close ['data', 'all']
-    #     Sync to disk when before closing a file.
+    #   Sync to disk when before closing a file.
     #
-    #     * `nil` does not sync.
-    #     * `data` syncs the file contents.
-    #     * `all` syncs the file contents and metadata.
+    #   * `nil` does not sync.
+    #   * `data` syncs the file contents.
+    #   * `all` syncs the file contents and metadata.
     # @param mkdir [Boolean]
-    #     Recursively create all the directories in the path.
+    #   Recursively create all the directories in the path.
     # @param lazy [Boolean]
-    #     Wait to start execution until `collect` is called.
+    #   Wait to start execution until `collect` is called.
+    # @param engine
+    #   Select the engine used to process the query, optional.
+    #   At the moment, if set to `"auto"` (default), the query is run
+    #   using the polars streaming engine. Polars will also
+    #   attempt to use the engine set by the `POLARS_ENGINE_AFFINITY`
+    #   environment variable. If it cannot run the query using the
+    #   selected engine, the query is run using the polars streaming
+    #   engine.
+    # @param optimizations
+    #   The optimization passes done during query optimization.
+    #
+    #   This has no effect if `lazy` is set to `true`.
     #
     # @return [DataFrame]
     #
@@ -1053,28 +1402,23 @@ module Polars
       null_value: nil,
       quote_style: nil,
       maintain_order: true,
-      type_coercion: true,
-      predicate_pushdown: true,
-      projection_pushdown: true,
-      simplify_expression: true,
-      slice_pushdown: true,
-      no_optimization: false,
       storage_options: nil,
+      credential_provider: "auto",
       retries: 2,
       sync_on_close: nil,
       mkdir: false,
-      lazy: false
+      lazy: false,
+      engine: "auto",
+      optimizations: DEFAULT_QUERY_OPT_FLAGS
     )
       Utils._check_arg_is_1byte("separator", separator, false)
       Utils._check_arg_is_1byte("quote_char", quote_char, false)
+      engine = _select_engine(engine, path)
 
-      lf = _set_sink_optimizations(
-        type_coercion: type_coercion,
-        predicate_pushdown: predicate_pushdown,
-        projection_pushdown: projection_pushdown,
-        simplify_expression: simplify_expression,
-        slice_pushdown: slice_pushdown,
-        no_optimization: no_optimization
+      _init_credential_provider_builder = Polars.method(:_init_credential_provider_builder)
+
+      credential_provider_builder = _init_credential_provider_builder.(
+        credential_provider, path, storage_options, "sink_csv"
       )
 
       if storage_options&.any?
@@ -1089,7 +1433,7 @@ module Polars
         "mkdir" => mkdir
       }
 
-      lf = lf.sink_csv(
+      ldf_rb = _ldf.sink_csv(
         path,
         include_bom,
         include_header,
@@ -1106,16 +1450,18 @@ module Polars
         null_value,
         quote_style,
         storage_options,
+        credential_provider_builder,
         retries,
         sink_options
       )
-      lf = LazyFrame._from_rbldf(lf)
 
       if !lazy
-        lf.collect
+        ldf_rb = ldf_rb.with_optimizations(optimizations._rboptflags)
+        ldf = LazyFrame._from_rbldf(ldf_rb)
+        ldf.collect(engine: engine)
         return nil
       end
-      lf
+      LazyFrame._from_rbldf(ldf_rb)
     end
 
     # Evaluate the query in streaming mode and write to an NDJSON file.
@@ -1127,18 +1473,6 @@ module Polars
     # @param maintain_order [Boolean]
     #   Maintain the order in which data is processed.
     #   Setting this to `false` will be slightly faster.
-    # @param type_coercion [Boolean]
-    #   Do type coercion optimization.
-    # @param predicate_pushdown [Boolean]
-    #   Do predicate pushdown optimization.
-    # @param projection_pushdown [Boolean]
-    #   Do projection pushdown optimization.
-    # @param simplify_expression [Boolean]
-    #   Run simplify expressions optimization.
-    # @param slice_pushdown [Boolean]
-    #   Slice pushdown optimization.
-    # @param no_optimization [Boolean]
-    #   Turn off (certain) optimizations.
     # @param storage_options [String]
     #   Options that indicate how to connect to a cloud provider.
     #
@@ -1164,6 +1498,18 @@ module Polars
     #   Recursively create all the directories in the path.
     # @param lazy [Boolean]
     #     Wait to start execution until `collect` is called.
+    # @param engine
+    #   Select the engine used to process the query, optional.
+    #   At the moment, if set to `"auto"` (default), the query is run
+    #   using the polars streaming engine. Polars will also
+    #   attempt to use the engine set by the `POLARS_ENGINE_AFFINITY`
+    #   environment variable. If it cannot run the query using the
+    #   selected engine, the query is run using the polars streaming
+    #   engine.
+    # @param optimizations
+    #   The optimization passes done during query optimization.
+    #
+    #   This has no effect if `lazy` is set to `true`.
     #
     # @return [DataFrame]
     #
@@ -1173,25 +1519,21 @@ module Polars
     def sink_ndjson(
       path,
       maintain_order: true,
-      type_coercion: true,
-      predicate_pushdown: true,
-      projection_pushdown: true,
-      simplify_expression: true,
-      slice_pushdown: true,
-      no_optimization: false,
       storage_options: nil,
+      credential_provider: "auto",
       retries: 2,
       sync_on_close: nil,
       mkdir: false,
-      lazy: false
+      lazy: false,
+      engine: "auto",
+      optimizations: DEFAULT_QUERY_OPT_FLAGS
     )
-      lf = _set_sink_optimizations(
-        type_coercion: type_coercion,
-        predicate_pushdown: predicate_pushdown,
-        projection_pushdown: projection_pushdown,
-        simplify_expression: simplify_expression,
-        slice_pushdown: slice_pushdown,
-        no_optimization: no_optimization
+      engine = _select_engine(engine, path)
+
+      _init_credential_provider_builder = Polars.method(:_init_credential_provider_builder)
+
+      credential_provider_builder = _init_credential_provider_builder.(
+        credential_provider, path, storage_options, "sink_ndjson"
       )
 
       if storage_options&.any?
@@ -1206,80 +1548,21 @@ module Polars
         "mkdir" => mkdir
       }
 
-      lf = lf.sink_json(path, storage_options, retries, sink_options)
-      lf = LazyFrame._from_rbldf(lf)
+      ldf_rb = _ldf.sink_json(
+        path,
+        storage_options,
+        credential_provider_builder,
+        retries,
+        sink_options
+      )
 
       if !lazy
-        lf.collect
+        ldf_rb = ldf_rb.with_optimizations(optimizations._rboptflags)
+        ldf = LazyFrame._from_rbldf(ldf_rb)
+        ldf.collect(engine: engine)
         return nil
       end
-      lf
-    end
-
-    # @private
-    def _set_sink_optimizations(
-      type_coercion: true,
-      predicate_pushdown: true,
-      projection_pushdown: true,
-      simplify_expression: true,
-      slice_pushdown: true,
-      no_optimization: false
-    )
-      if no_optimization
-        predicate_pushdown = false
-        projection_pushdown = false
-        slice_pushdown = false
-      end
-
-      _ldf.optimization_toggle(
-        type_coercion,
-        predicate_pushdown,
-        projection_pushdown,
-        simplify_expression,
-        slice_pushdown,
-        false,
-        false,
-        true,
-        false
-      )
-    end
-
-    # Collect a small number of rows for debugging purposes.
-    #
-    # Fetch is like a {#collect} operation, but it overwrites the number of rows
-    # read by every scan operation. This is a utility that helps debug a query on a
-    # smaller number of rows.
-    #
-    # Note that the fetch does not guarantee the final number of rows in the
-    # DataFrame. Filter, join operations and a lower number of rows available in the
-    # scanned file influence the final number of rows.
-    #
-    # @param n_rows [Integer]
-    #   Collect n_rows from the data sources.
-    #
-    # @return [DataFrame]
-    #
-    # @example
-    #   df = Polars::DataFrame.new(
-    #     {
-    #       "a" => ["a", "b", "a", "b", "b", "c"],
-    #       "b" => [1, 2, 3, 4, 5, 6],
-    #       "c" => [6, 5, 4, 3, 2, 1]
-    #     }
-    #   ).lazy
-    #   df.group_by("a", maintain_order: true).agg(Polars.all.sum).fetch(2)
-    #   # =>
-    #   # shape: (2, 3)
-    #   # ┌─────┬─────┬─────┐
-    #   # │ a   ┆ b   ┆ c   │
-    #   # │ --- ┆ --- ┆ --- │
-    #   # │ str ┆ i64 ┆ i64 │
-    #   # ╞═════╪═════╪═════╡
-    #   # │ a   ┆ 4   ┆ 10  │
-    #   # │ b   ┆ 11  ┆ 10  │
-    #   # └─────┴─────┴─────┘
-    def fetch(n_rows = 500, **kwargs)
-      head(n_rows).collect(**kwargs)
+      LazyFrame._from_rbldf(ldf_rb)
     end
 
     # Return lazy representation, i.e. itself.
@@ -1388,7 +1671,7 @@ module Polars
     #       "c" => [true, true, false, nil],
     #     }
     #   ).lazy
-    #   lf.clear.fetch
+    #   lf.clear.collect
     #   # =>
     #   # shape: (0, 3)
     #   # ┌─────┬─────┬──────┐
@@ -1399,7 +1682,7 @@ module Polars
     #   # └─────┴─────┴──────┘
     #
     # @example
-    #   lf.clear(2).fetch
+    #   lf.clear(2).collect
     #   # =>
     #   # shape: (2, 3)
     #   # ┌──────┬──────┬──────┐
@@ -1413,24 +1696,82 @@ module Polars
     def clear(n = 0)
       DataFrame.new(schema: schema).clear(n).lazy
     end
-    alias_method :cleared, :clear
 
     # Filter the rows in the DataFrame based on a predicate expression.
     #
-    # @param predicate [Object]
-    #   Expression that evaluates to a boolean Series.
+    # @param predicates [Array]
+    #   Expression(s) that evaluate to a boolean Series.
+    # @param constraints [Hash]
+    #   Column filters; use `name = value` to filter columns using the supplied
+    #   value. Each constraint behaves the same as `Polars.col(name).eq(value)`,
+    #   and is implicitly joined with the other filter conditions using `&`.
     #
     # @return [LazyFrame]
     #
     # @example Filter on one condition:
-    #   lf = Polars::DataFrame.new(
+    #   lf = Polars::LazyFrame.new(
     #     {
-    #       "foo" => [1, 2, 3],
-    #       "bar" => [6, 7, 8],
-    #       "ham" => ["a", "b", "c"]
+    #       "foo" => [1, 2, 3, nil, 4, nil, 0],
+    #       "bar" => [6, 7, 8, nil, nil, 9, 0],
+    #       "ham" => ["a", "b", "c", nil, "d", "e", "f"]
     #     }
-    #   ).lazy
-    #   lf.filter(Polars.col("foo") < 3).collect
+    #   )
+    #   lf.filter(Polars.col("foo") > 1).collect
+    #   # =>
+    #   # shape: (3, 3)
+    #   # ┌─────┬──────┬─────┐
+    #   # │ foo ┆ bar  ┆ ham │
+    #   # │ --- ┆ ---  ┆ --- │
+    #   # │ i64 ┆ i64  ┆ str │
+    #   # ╞═════╪══════╪═════╡
+    #   # │ 2   ┆ 7    ┆ b   │
+    #   # │ 3   ┆ 8    ┆ c   │
+    #   # │ 4   ┆ null ┆ d   │
+    #   # └─────┴──────┴─────┘
+    #
+    # @example Filter on multiple conditions:
+    #   lf.filter((Polars.col("foo") < 3) & (Polars.col("ham") == "a")).collect
+    #   # =>
+    #   # shape: (1, 3)
+    #   # ┌─────┬─────┬─────┐
+    #   # │ foo ┆ bar ┆ ham │
+    #   # │ --- ┆ --- ┆ --- │
+    #   # │ i64 ┆ i64 ┆ str │
+    #   # ╞═════╪═════╪═════╡
+    #   # │ 1   ┆ 6   ┆ a   │
+    #   # └─────┴─────┴─────┘
+    #
+    # @example Provide multiple filters using `*args` syntax:
+    #   lf.filter(
+    #     Polars.col("foo") == 1,
+    #     Polars.col("ham") == "a"
+    #   ).collect
+    #   # =>
+    #   # shape: (1, 3)
+    #   # ┌─────┬─────┬─────┐
+    #   # │ foo ┆ bar ┆ ham │
+    #   # │ --- ┆ --- ┆ --- │
+    #   # │ i64 ┆ i64 ┆ str │
+    #   # ╞═════╪═════╪═════╡
+    #   # │ 1   ┆ 6   ┆ a   │
+    #   # └─────┴─────┴─────┘
+    #
+    # @example Provide multiple filters using `**kwargs` syntax:
+    #   lf.filter(foo: 1, ham: "a").collect
+    #   # =>
+    #   # shape: (1, 3)
+    #   # ┌─────┬─────┬─────┐
+    #   # │ foo ┆ bar ┆ ham │
+    #   # │ --- ┆ --- ┆ --- │
+    #   # │ i64 ┆ i64 ┆ str │
+    #   # ╞═════╪═════╪═════╡
+    #   # │ 1   ┆ 6   ┆ a   │
+    #   # └─────┴─────┴─────┘
+    #
+    # @example Filter on an OR condition:
+    #   lf.filter(
+    #     (Polars.col("foo") == 1) | (Polars.col("ham") == "c")
+    #   ).collect
     #   # =>
     #   # shape: (2, 3)
     #   # ┌─────┬─────┬─────┐
@@ -1439,11 +1780,13 @@ module Polars
     #   # │ i64 ┆ i64 ┆ str │
     #   # ╞═════╪═════╪═════╡
     #   # │ 1   ┆ 6   ┆ a   │
-    #   # │ 2   ┆ 7   ┆ b   │
+    #   # │ 3   ┆ 8   ┆ c   │
     #   # └─────┴─────┴─────┘
     #
-    # @example Filter on multiple conditions:
-    #   lf.filter((Polars.col("foo") < 3) & (Polars.col("ham") == "a")).collect
+    # @example Filter by comparing two columns against each other
+    #   lf.filter(
+    #     Polars.col("foo") == Polars.col("bar")
+    #   ).collect
     #   # =>
     #   # shape: (1, 3)
     #   # ┌─────┬─────┬─────┐
@@ -1451,13 +1794,39 @@ module Polars
     #   # │ --- ┆ --- ┆ --- │
     #   # │ i64 ┆ i64 ┆ str │
     #   # ╞═════╪═════╪═════╡
+    #   # │ 0   ┆ 0   ┆ f   │
+    #   # └─────┴─────┴─────┘
+    #
+    # @example
+    #   lf.filter(
+    #     Polars.col("foo") != Polars.col("bar")
+    #   ).collect
+    #   # =>
+    #   # shape: (3, 3)
+    #   # ┌─────┬─────┬─────┐
+    #   # │ foo ┆ bar ┆ ham │
+    #   # │ --- ┆ --- ┆ --- │
+    #   # │ i64 ┆ i64 ┆ str │
+    #   # ╞═════╪═════╪═════╡
     #   # │ 1   ┆ 6   ┆ a   │
+    #   # │ 2   ┆ 7   ┆ b   │
+    #   # │ 3   ┆ 8   ┆ c   │
     #   # └─────┴─────┴─────┘
-    def filter(predicate)
-      _from_rbldf(
-        _ldf.filter(
-          Utils.parse_into_expression(predicate, str_as_lit: false)
-        )
+    def filter(*predicates, **constraints)
+      if constraints.empty?
+        # early-exit conditions (exclude/include all rows)
+        if predicates.empty? || (predicates.length == 1 && predicates[0].is_a?(TrueClass))
+          return dup
+        end
+        if predicates.length == 1 && predicates[0].is_a?(FalseClass)
+          return clear
+        end
+      end
+
+      _filter(
+        predicates: predicates,
+        constraints: constraints,
+        invert: false
       )
     end
 
@@ -1752,13 +2121,9 @@ module Polars
       lgb = _ldf.group_by(exprs, maintain_order)
       LazyGroupBy.new(lgb)
     end
-    alias_method :groupby, :group_by
-    alias_method :group, :group_by
 
     # Create rolling groups based on a time column.
     #
-    # Also works for index values of type `:i32` or `:i64`.
-    #
     # Different from a `dynamic_group_by` the windows are now determined by the
     # individual values and are not of constant intervals. For constant intervals
     # use *group_by_dynamic*.
@@ -1793,15 +2158,15 @@ module Polars
     #   make sense.
     #
     #   In case of a rolling group by on indices, dtype needs to be one of
-    #   `:i32`, `:i64`. Note that `:i32` gets temporarily cast to `:i64`, so if
-    #   performance matters use an `:i64` column.
+    #   \\\\{UInt32, UInt64, Int32, Int64}. Note that the first three get temporarily
+    #   cast to Int64, so if performance matters use an Int64 column.
     # @param period [Object]
     #   Length of the window.
     # @param offset [Object]
     #   Offset of the window. Default is -period.
     # @param closed ["right", "left", "both", "none"]
     #   Define whether the temporal window interval is closed or not.
-    # @param by [Object]
+    # @param group_by [Object]
     #   Also group by this column/these columns.
     #
     # @return [LazyFrame]
@@ -1815,7 +2180,7 @@ module Polars
     #     "2020-01-03 19:45:32",
     #     "2020-01-08 23:16:43"
     #   ]
-    #   df = Polars::LazyFrame.new({"dt" => dates, "a" => [3, 7, 5, 9, 2, 1]}).with_column(
+    #   df = Polars::LazyFrame.new({"dt" => dates, "a" => [3, 7, 5, 9, 2, 1]}).with_columns(
     #     Polars.col("dt").str.strptime(Polars::Datetime).set_sorted
     #   )
     #   df.rolling(index_column: "dt", period: "2d").agg(
@@ -1844,7 +2209,7 @@ module Polars
       period:,
       offset: nil,
       closed: "right",
-      by: nil
+      group_by: nil
     )
       index_column = Utils.parse_into_expression(index_column)
       if offset.nil?
@@ -1852,7 +2217,7 @@ module Polars
       end
 
       rbexprs_by = (
-        !by.nil? ? Utils.parse_into_list_of_expressions(by) : []
+        !group_by.nil? ? Utils.parse_into_list_of_expressions(group_by) : []
       )
       period = Utils.parse_as_duration_string(period)
       offset = Utils.parse_as_duration_string(offset)
@@ -1860,10 +2225,8 @@ module Polars
       lgb = _ldf.rolling(index_column, period, offset, closed, rbexprs_by)
       LazyGroupBy.new(lgb)
     end
-    alias_method :group_by_rolling, :rolling
-    alias_method :groupby_rolling, :rolling
 
-    # Group based on a time value (or index value of type `:i32`, `:i64`).
+    # Group based on a time value (or index value of type Int32, Int64).
     #
     # Time windows are calculated and rows are assigned to windows. Different from a
     # normal group by is that a row can be member of multiple groups. The time/index
@@ -1906,8 +2269,8 @@ module Polars
     #   make sense.
     #
     #   In case of a dynamic group by on indices, dtype needs to be one of
-    #   `:i32`, `:i64`. Note that `:i32` gets temporarily cast to `:i64`, so if
-    #   performance matters use an `:i64` column.
+    #   \\\\{Int32, Int64}. Note that Int32 gets temporarily cast to Int64, so if
+    #   performance matters use an Int64 column.
     # @param every [Object]
     #   Interval of the window.
     # @param period [Object]
@@ -1915,8 +2278,6 @@ module Polars
     # @param offset [Object]
     #   Offset of the window if nil and period is nil it will be equal to negative
     #   `every`.
-    # @param truncate [Boolean]
-    #   Truncate the time value to the window lower bound.
     # @param include_boundaries [Boolean]
     #   Add the lower and upper bound of the window to the "_lower_bound" and
     #   "_upper_bound" columns. This will impact performance because it's harder to
@@ -1931,7 +2292,7 @@ module Polars
     #   - 'datapoint': the first value of the index column in the given window.
     #     If you don't need the label to be at one of the boundaries, choose this
     #     option for maximum performance
-    # @param by [Object]
+    # @param group_by [Object]
     #   Also group by this column/these columns
     # @param start_by ['window', 'datapoint', 'monday', 'tuesday', 'wednesday', 'thursday', 'friday', 'saturday', 'sunday']
     #   The strategy to determine the start of the first window by.
@@ -2073,7 +2434,7 @@ module Polars
     #     "time",
     #     every: "1h",
     #     closed: "both",
-    #     by: "groups",
+    #     group_by: "groups",
     #     include_boundaries: true
     #   ).agg([Polars.col("time").count.alias("time_count")])
     #   # =>
@@ -2123,17 +2484,12 @@ module Polars
       every:,
       period: nil,
       offset: nil,
-      truncate: nil,
       include_boundaries: false,
       closed: "left",
       label: "left",
-      by: nil,
+      group_by: nil,
       start_by: "window"
     )
-      if !truncate.nil?
-        label = truncate ? "left" : "datapoint"
-      end
-
       index_column = Utils.parse_into_expression(index_column, str_as_lit: false)
       if offset.nil?
         offset = period.nil? ? "-#{every}" : "0ns"
@@ -2147,7 +2503,7 @@ module Polars
       offset = Utils.parse_as_duration_string(offset)
       every = Utils.parse_as_duration_string(every)
 
-      rbexprs_by = by.nil? ? [] : Utils.parse_into_list_of_expressions(by)
+      rbexprs_by = group_by.nil? ? [] : Utils.parse_into_list_of_expressions(group_by)
       lgb = _ldf.group_by_dynamic(
         index_column,
         every,
@@ -2161,7 +2517,6 @@ module Polars
       )
       LazyGroupBy.new(lgb)
     end
-    alias_method :groupby_dynamic, :group_by_dynamic
 
     # Perform an asof join.
     #
@@ -2521,7 +2876,7 @@ module Polars
     #     * *one_to_one* - “1:1”: check if join keys are unique in both left and right datasets
     #     * *one_to_many* - “1:m”: check if join keys are unique in left dataset
     #     * *many_to_one* - “m:1”: check if join keys are unique in right dataset
-    # @param join_nulls [Boolean]
+    # @param nulls_equal [Boolean]
     #   Join on null values. By default null values will never produce matches.
     # @param allow_parallel [Boolean]
     #   Allow the physical plan to optionally evaluate the computation of both
@@ -2643,7 +2998,7 @@ module Polars
       how: "inner",
       suffix: "_right",
       validate: "m:m",
-      join_nulls: false,
+      nulls_equal: false,
       allow_parallel: true,
       force_parallel: false,
       coalesce: nil,
@@ -2666,7 +3021,7 @@ module Polars
             [],
             [],
             allow_parallel,
-            join_nulls,
+            nulls_equal,
             force_parallel,
             how,
             suffix,
@@ -2695,7 +3050,7 @@ module Polars
           rbexprs_right,
           allow_parallel,
           force_parallel,
-          join_nulls,
+          nulls_equal,
           how,
           suffix,
           validate,
@@ -2875,87 +3230,6 @@ module Polars
       _from_rbldf(_ldf.with_columns_seq(rbexprs))
     end
 
-    # Add an external context to the computation graph.
-    #
-    # This allows expressions to also access columns from DataFrames
-    # that are not part of this one.
-    #
-    # @param other [Object]
-    #   Lazy DataFrame to join with.
-    #
-    # @return [LazyFrame]
-    #
-    # @example
-    #   df_a = Polars::DataFrame.new({"a" => [1, 2, 3], "b" => ["a", "c", nil]}).lazy
-    #   df_other = Polars::DataFrame.new({"c" => ["foo", "ham"]})
-    #   (
-    #     df_a.with_context(df_other.lazy).select(
-    #       [Polars.col("b") + Polars.col("c").first]
-    #     )
-    #   ).collect
-    #   # =>
-    #   # shape: (3, 1)
-    #   # ┌──────┐
-    #   # │ b    │
-    #   # │ ---  │
-    #   # │ str  │
-    #   # ╞══════╡
-    #   # │ afoo │
-    #   # │ cfoo │
-    #   # │ null │
-    #   # └──────┘
-    def with_context(other)
-      if !other.is_a?(::Array)
-        other = [other]
-      end
-
-      _from_rbldf(_ldf.with_context(other.map(&:_ldf)))
-    end
-
-    # Add or overwrite column in a DataFrame.
-    #
-    # @param column [Object]
-    #   Expression that evaluates to column or a Series to use.
-    #
-    # @return [LazyFrame]
-    #
-    # @example
-    #   df = Polars::DataFrame.new(
-    #     {
-    #       "a" => [1, 3, 5],
-    #       "b" => [2, 4, 6]
-    #     }
-    #   ).lazy
-    #   df.with_column((Polars.col("b") ** 2).alias("b_squared")).collect
-    #   # =>
-    #   # shape: (3, 3)
-    #   # ┌─────┬─────┬───────────┐
-    #   # │ a   ┆ b   ┆ b_squared │
-    #   # │ --- ┆ --- ┆ ---       │
-    #   # │ i64 ┆ i64 ┆ i64       │
-    #   # ╞═════╪═════╪═══════════╡
-    #   # │ 1   ┆ 2   ┆ 4         │
-    #   # │ 3   ┆ 4   ┆ 16        │
-    #   # │ 5   ┆ 6   ┆ 36        │
-    #   # └─────┴─────┴───────────┘
-    #
-    # @example
-    #   df.with_column(Polars.col("a") ** 2).collect
-    #   # =>
-    #   # shape: (3, 2)
-    #   # ┌─────┬─────┐
-    #   # │ a   ┆ b   │
-    #   # │ --- ┆ --- │
-    #   # │ i64 ┆ i64 │
-    #   # ╞═════╪═════╡
-    #   # │ 1   ┆ 2   │
-    #   # │ 9   ┆ 4   │
-    #   # │ 25  ┆ 6   │
-    #   # └─────┴─────┘
-    def with_column(column)
-      with_columns([column])
-    end
-
     # Remove one or multiple columns from a DataFrame.
     #
     # @param columns [Object]
@@ -3060,20 +3334,6 @@ module Polars
     #   # │ 2     ┆ 7   ┆ b   │
     #   # │ 3     ┆ 8   ┆ c   │
     #   # └───────┴─────┴─────┘
-    #
-    # @example
-    #   lf.rename(->(column_name) { "c" + column_name[1..] }).collect
-    #   # =>
-    #   # shape: (3, 3)
-    #   # ┌─────┬─────┬─────┐
-    #   # │ coo ┆ car ┆ cam │
-    #   # │ --- ┆ --- ┆ --- │
-    #   # │ i64 ┆ i64 ┆ str │
-    #   # ╞═════╪═════╪═════╡
-    #   # │ 1   ┆ 6   ┆ a   │
-    #   # │ 2   ┆ 7   ┆ b   │
-    #   # │ 3   ┆ 8   ┆ c   │
-    #   # └─────┴─────┴─────┘
     def rename(mapping, strict: true)
       if mapping.respond_to?(:call)
         select(F.all.name.map(&mapping))
@@ -3153,7 +3413,7 @@ module Polars
     #   # │ 5    ┆ 6    │
     #   # │ null ┆ null │
     #   # └──────┴──────┘
-    def shift(n, fill_value: nil)
+    def shift(n = 1, fill_value: nil)
       if !fill_value.nil?
         fill_value = Utils.parse_into_expression(fill_value, str_as_lit: true)
       end
@@ -3161,52 +3421,6 @@ module Polars
       _from_rbldf(_ldf.shift(n, fill_value))
     end
 
-    # Shift the values by a given period and fill the resulting null values.
-    #
-    # @param periods [Integer]
-    #   Number of places to shift (may be negative).
-    # @param fill_value [Object]
-    #   Fill `nil` values with the result of this expression.
-    #
-    # @return [LazyFrame]
-    #
-    # @example
-    #   df = Polars::DataFrame.new(
-    #     {
-    #       "a" => [1, 3, 5],
-    #       "b" => [2, 4, 6]
-    #     }
-    #   ).lazy
-    #   df.shift_and_fill(1, 0).collect
-    #   # =>
-    #   # shape: (3, 2)
-    #   # ┌─────┬─────┐
-    #   # │ a   ┆ b   │
-    #   # │ --- ┆ --- │
-    #   # │ i64 ┆ i64 │
-    #   # ╞═════╪═════╡
-    #   # │ 0   ┆ 0   │
-    #   # │ 1   ┆ 2   │
-    #   # │ 3   ┆ 4   │
-    #   # └─────┴─────┘
-    #
-    # @example
-    #   df.shift_and_fill(-1, 0).collect
-    #   # =>
-    #   # shape: (3, 2)
-    #   # ┌─────┬─────┐
-    #   # │ a   ┆ b   │
-    #   # │ --- ┆ --- │
-    #   # │ i64 ┆ i64 │
-    #   # ╞═════╪═════╡
-    #   # │ 3   ┆ 4   │
-    #   # │ 5   ┆ 6   │
-    #   # │ 0   ┆ 0   │
-    #   # └─────┴─────┘
-    def shift_and_fill(periods, fill_value)
-      shift(periods, fill_value: fill_value)
-    end
-
     # Get a slice of this DataFrame.
     #
     # @param offset [Integer]
@@ -3252,11 +3466,6 @@ module Polars
     #
     # @return [LazyFrame]
     #
-    # @note
-    #   Consider using the {#fetch} operation if you only want to test your
-    #   query. The {#fetch} operation will load the first `n` rows at the scan
-    #   level, whereas the {#head}/{#limit} are applied at the end.
-    #
     # @example
     #   lf = Polars::LazyFrame.new(
     #     {
@@ -3302,11 +3511,6 @@ module Polars
     #
     # @return [LazyFrame]
     #
-    # @note
-    #   Consider using the {#fetch} operation if you only want to test your
-    #   query. The {#fetch} operation will load the first `n` rows at the scan
-    #   level, whereas the {#head}/{#limit} are applied at the end.
-    #
     # @example
     #   lf = Polars::LazyFrame.new(
     #     {
@@ -3475,10 +3679,14 @@ module Polars
     def with_row_index(name: "index", offset: 0)
       _from_rbldf(_ldf.with_row_index(name, offset))
     end
-    alias_method :with_row_count, :with_row_index
 
     # Take every nth row in the LazyFrame and return as a new LazyFrame.
     #
+    # @param n [Integer]
+    #   Gather every *n*-th row.
+    # @param offset [Integer]
+    #   Starting index.
+    #
     # @return [LazyFrame]
     #
     # @example
@@ -3494,10 +3702,9 @@ module Polars
     #   # │ 1   ┆ 5   │
     #   # │ 3   ┆ 7   │
     #   # └─────┴─────┘
-    def gather_every(n)
-      select(F.col("*").gather_every(n))
+    def gather_every(n, offset: 0)
+      select(F.col("*").gather_every(n, offset))
     end
-    alias_method :take_every, :gather_every
 
     # Fill null values using the specified value or strategy.
     #
@@ -3568,13 +3775,53 @@ module Polars
     #   # │ 0   ┆ 0.0  │
     #   # │ 4   ┆ 13.0 │
     #   # └─────┴──────┘
-    def fill_null(value = nil, strategy: nil, limit: nil, matches_supertype: nil)
+    def fill_null(value = nil, strategy: nil, limit: nil, matches_supertype: true)
+      if !value.nil?
+        if value.is_a?(Expr)
+          dtypes = nil
+        elsif value.is_a?(TrueClass) || value.is_a?(FalseClass)
+          dtypes = [Boolean]
+        elsif matches_supertype && (value.is_a?(Integer) || value.is_a?(Float))
+          dtypes = [
+            Int8,
+            Int16,
+            Int32,
+            Int64,
+            Int128,
+            UInt8,
+            UInt16,
+            UInt32,
+            UInt64,
+            Float32,
+            Float64,
+            Decimal.new
+          ]
+        elsif value.is_a?(Integer)
+          dtypes = [Int64]
+        elsif value.is_a?(Float)
+          dtypes = [Float64]
+        elsif value.is_a?(::Date)
+          dtypes = [Date]
+        elsif value.is_a?(::String)
+          dtypes = [String, Categorical]
+        else
+          # fallback; anything not explicitly handled above
+          dtypes = nil
+        end
+
+        if dtypes
+          return with_columns(
+            F.col(dtypes).fill_null(value, strategy: strategy, limit: limit)
+          )
+        end
+      end
+
       select(Polars.all.fill_null(value, strategy: strategy, limit: limit))
     end
 
     # Fill floating point NaN values.
     #
-    # @param fill_value [Object]
+    # @param value [Object]
     #   Value to fill the NaN values with.
     #
     # @return [LazyFrame]
@@ -3603,11 +3850,11 @@ module Polars
     #   # │ 99.0 ┆ 99.0 │
     #   # │ 4.0  ┆ 13.0 │
     #   # └──────┴──────┘
-    def fill_nan(fill_value)
-      if !fill_value.is_a?(Expr)
-        fill_value = F.lit(fill_value)
+    def fill_nan(value)
+      if !value.is_a?(Expr)
+        value = F.lit(value)
       end
-      _from_rbldf(_ldf.fill_nan(fill_value._rbexpr))
+      _from_rbldf(_ldf.fill_nan(value._rbexpr))
     end
 
     # Aggregate the columns in the DataFrame to their standard deviation value.
@@ -3922,7 +4169,7 @@ module Polars
     #   # │ 3   ┆ a   ┆ b   │
     #   # │ 1   ┆ a   ┆ b   │
     #   # └─────┴─────┴─────┘
-    def unique(maintain_order: true, subset: nil, keep: "first")
+    def unique(maintain_order: false, subset: nil, keep: "any")
       selector_subset = nil
       if !subset.nil?
         selector_subset = Utils.parse_list_into_selector(subset)._rbselector
@@ -4078,7 +4325,7 @@ module Polars
     #   # │ z   ┆ c        ┆ 6     │
     #   # └─────┴──────────┴───────┘
     def unpivot(
-      on,
+      on = nil,
       index: nil,
       variable_name: nil,
       value_name: nil,
@@ -4100,7 +4347,6 @@ module Polars
         )
       )
     end
-    alias_method :melt, :unpivot
 
     # def map
     # end
@@ -4166,7 +4412,7 @@ module Polars
     #       ["before", Polars.struct(Polars.col("^t_.$")).alias("t_struct"), "after"]
     #     )
     #   )
-    #   df.fetch
+    #   df.collect
     #   # =>
     #   # shape: (2, 3)
     #   # ┌────────┬─────────────────────┬───────┐
@@ -4179,7 +4425,7 @@ module Polars
     #   # └────────┴─────────────────────┴───────┘
     #
     # @example
-    #   df.unnest("t_struct").fetch
+    #   df.unnest("t_struct").collect
     #   # =>
     #   # shape: (2, 6)
     #   # ┌────────┬─────┬─────┬──────┬───────────┬───────┐
@@ -4248,19 +4494,41 @@ module Polars
     #
     # @param column [Object]
     #   Column that is sorted.
+    # @param more_columns [Array]
+    #   Columns that are sorted over after `column`.
     # @param descending [Boolean]
     #   Whether the column is sorted in descending order.
+    # @param nulls_last [Boolean]
+    #   Whether the nulls are at the end.
     #
     # @return [LazyFrame]
     def set_sorted(
       column,
-      descending: false
+      *more_columns,
+      descending: false,
+      nulls_last: false
     )
       if !Utils.strlike?(column)
         msg = "expected a 'str' for argument 'column' in 'set_sorted'"
         raise TypeError, msg
       end
-      with_columns(F.col(column).set_sorted(descending: descending))
+
+      if Utils.bool?(descending)
+        ds = [descending]
+      else
+        ds = descending
+      end
+      if Utils.bool?(nulls_last)
+        nl = [nulls_last]
+      else
+        nl = nulls_last
+      end
+
+      _from_rbldf(
+        _ldf.hint_sorted(
+          [column] + more_columns, ds, nl
+        )
+      )
     end
 
     # Update the values in this `LazyFrame` with the values in `other`.
@@ -4297,7 +4565,7 @@ module Polars
     # @note
     #   This is syntactic sugar for a left/inner join that preserves the order
     #   of the left `DataFrame` by default, with an optional coalesce when
-    #   `include_nulls: False`.
+    #   `include_nulls: false`.
     #
     # @example Update `df` values with the non-null values in `new_df`, by row index:
     #   lf = Polars::LazyFrame.new(
@@ -4451,7 +4719,7 @@ module Polars
       # only use non-idx right columns present in left frame
       right_other = Set.new(right_schema.to_h.keys).intersection(left_schema.to_h.keys) - Set.new(right_on)
 
-      # When include_nulls is True, we need to distinguish records after the join that
+      # When include_nulls is true, we need to distinguish records after the join that
       # were originally null in the right frame, as opposed to records that were null
       # because the key was missing from the right frame.
       # Add a validity column to track whether row was matched or not.
@@ -4574,11 +4842,29 @@ module Polars
       end
 
       # if multiple predicates, combine as 'horizontal' expression
-      combined_predicate = all_predicates ? (all_predicates.length > 1 ? F.all_horizontal(*all_predicates) : all_predicates[0]) : nil
+      combined_predicate =
+        if all_predicates.any?
+          if all_predicates.length > 1
+            F.all_horizontal(*all_predicates)
+          else
+            all_predicates[0]
+          end
+        else
+          nil
+        end
 
       # apply reduced boolean mask first, if applicable, then predicates
       if boolean_masks.any?
-        raise Todo
+        if boolean_masks.length > 1
+          raise Todo
+        end
+        mask_expr = F.lit(boolean_masks[0])
+        combined_predicate =
+          if combined_predicate.nil?
+            mask_expr
+          else
+            mask_expr & combined_predicate
+          end
       end
 
       if combined_predicate.nil?
@@ -4588,5 +4874,10 @@ module Polars
       filter_method = invert ? _ldf.method(:remove) : _ldf.method(:filter)
       _from_rbldf(filter_method.(combined_predicate._rbexpr))
     end
+
+    def _select_engine(engine, path = nil)
+      engine = Plr.get_engine_affinity if engine == "auto"
+      engine == "auto" && !path.is_a?(::String) && !path.nil? ? "in-memory" : engine
+    end
   end
 end