polars-df 0.9.0 → 0.11.0

data/lib/polars/lazy_frame.rb

@@ -27,145 +27,6 @@ module Polars
       ldf
     end
 
-    # @private
-    def self._scan_csv(
-      file,
-      has_header: true,
-      sep: ",",
-      comment_char: nil,
-      quote_char: '"',
-      skip_rows: 0,
-      dtypes: nil,
-      null_values: nil,
-      ignore_errors: false,
-      cache: true,
-      with_column_names: nil,
-      infer_schema_length: 100,
-      n_rows: nil,
-      encoding: "utf8",
-      low_memory: false,
-      rechunk: true,
-      skip_rows_after_header: 0,
-      row_count_name: nil,
-      row_count_offset: 0,
-      parse_dates: false,
-      eol_char: "\n"
-    )
-      dtype_list = nil
-      if !dtypes.nil?
-        dtype_list = []
-        dtypes.each do |k, v|
-          dtype_list << [k, Utils.rb_type_to_dtype(v)]
-        end
-      end
-      processed_null_values = Utils._process_null_values(null_values)
-
-      _from_rbldf(
-        RbLazyFrame.new_from_csv(
-          file,
-          sep,
-          has_header,
-          ignore_errors,
-          skip_rows,
-          n_rows,
-          cache,
-          dtype_list,
-          low_memory,
-          comment_char,
-          quote_char,
-          processed_null_values,
-          infer_schema_length,
-          with_column_names,
-          rechunk,
-          skip_rows_after_header,
-          encoding,
-          Utils._prepare_row_count_args(row_count_name, row_count_offset),
-          parse_dates,
-          eol_char
-        )
-      )
-    end
-
-    # @private
-    def self._scan_parquet(
-      file,
-      n_rows: nil,
-      cache: true,
-      parallel: "auto",
-      rechunk: true,
-      row_count_name: nil,
-      row_count_offset: 0,
-      storage_options: nil,
-      low_memory: false,
-      use_statistics: true,
-      hive_partitioning: true
-    )
-      _from_rbldf(
-        RbLazyFrame.new_from_parquet(
-          file,
-          n_rows,
-          cache,
-          parallel,
-          rechunk,
-          Utils._prepare_row_count_args(row_count_name, row_count_offset),
-          low_memory,
-          use_statistics,
-          hive_partitioning
-        )
-      )
-    end
-
-    # @private
-    def self._scan_ipc(
-      file,
-      n_rows: nil,
-      cache: true,
-      rechunk: true,
-      row_count_name: nil,
-      row_count_offset: 0,
-      storage_options: nil,
-      memory_map: true
-    )
-      if Utils.pathlike?(file)
-        file = Utils.normalise_filepath(file)
-      end
-
-      _from_rbldf(
-        RbLazyFrame.new_from_ipc(
-          file,
-          n_rows,
-          cache,
-          rechunk,
-          Utils._prepare_row_count_args(row_count_name, row_count_offset),
-          memory_map
-        )
-      )
-    end
-
-    # @private
-    def self._scan_ndjson(
-      file,
-      infer_schema_length: nil,
-      batch_size: nil,
-      n_rows: nil,
-      low_memory: false,
-      rechunk: true,
-      row_count_name: nil,
-      row_count_offset: 0
-    )
-      _from_rbldf(
-        RbLazyFrame.new_from_ndjson(
-          file,
-          infer_schema_length,
-          batch_size,
-          n_rows,
-          low_memory,
-          rechunk,
-          Utils._prepare_row_count_args(row_count_name, row_count_offset)
-        )
-      )
-    end
-
     # def self.from_json
     # end
 
@@ -177,7 +38,7 @@ module Polars
     # @return [LazyFrame]
     def self.read_json(file)
       if Utils.pathlike?(file)
-        file = Utils.normalise_filepath(file)
+        file = Utils.normalize_filepath(file)
       end
 
       Utils.wrap_ldf(RbLazyFrame.read_json(file))
@@ -284,7 +145,7 @@ module Polars
     # @return [nil]
     def write_json(file)
       if Utils.pathlike?(file)
-        file = Utils.normalise_filepath(file)
+        file = Utils.normalize_filepath(file)
       end
       _ldf.write_json(file)
       nil
@@ -400,16 +261,16 @@ module Polars
     #   # │ 2   ┆ 7.0 ┆ b   │
     #   # │ 1   ┆ 6.0 ┆ a   │
     #   # └─────┴─────┴─────┘
-    def sort(by, reverse: false, nulls_last: false, maintain_order: false)
+    def sort(by, reverse: false, nulls_last: false, maintain_order: false, multithreaded: true)
       if by.is_a?(::String)
-        return _from_rbldf(_ldf.sort(by, reverse, nulls_last, maintain_order))
+        return _from_rbldf(_ldf.sort(by, reverse, nulls_last, maintain_order, multithreaded))
       end
       if Utils.bool?(reverse)
         reverse = [reverse]
       end
 
       by = Utils.selection_to_rbexpr_list(by)
-      _from_rbldf(_ldf.sort_by_exprs(by, reverse, nulls_last, maintain_order))
+      _from_rbldf(_ldf.sort_by_exprs(by, reverse, nulls_last, maintain_order, multithreaded))
     end
 
     # def profile
@@ -1133,7 +994,7 @@ module Polars
     #   # ┌─────────┐
     #   # │ literal │
     #   # │ ---     │
-    #   # │ i64     │
+    #   # │ i32     │
     #   # ╞═════════╡
     #   # │ 0       │
     #   # │ 0       │
@@ -1445,16 +1306,16 @@ module Polars
     #   )
     #   # =>
     #   # shape: (4, 3)
-    #   # ┌─────────────────────┬────────────┬───────────────────────────────────┐
-    #   # │ time                ┆ time_count ┆ time_agg_list                     │
-    #   # │ ---                 ┆ ---        ┆ ---                               │
-    #   # │ datetime[μs]        ┆ u32        ┆ list[datetime[μs]]                │
-    #   # ╞═════════════════════╪════════════╪═══════════════════════════════════╡
-    #   # │ 2021-12-16 00:00:00 ┆ 2          ┆ [2021-12-16 00:00:00, 2021-12-16… │
-    #   # │ 2021-12-16 01:00:00 ┆ 2          ┆ [2021-12-16 01:00:00, 2021-12-16… │
-    #   # │ 2021-12-16 02:00:00 ┆ 2          ┆ [2021-12-16 02:00:00, 2021-12-16… │
-    #   # │ 2021-12-16 03:00:00 ┆ 1          ┆ [2021-12-16 03:00:00]             │
-    #   # └─────────────────────┴────────────┴───────────────────────────────────┘
+    #   # ┌─────────────────────┬────────────┬─────────────────────────────────┐
+    #   # │ time                ┆ time_count ┆ time_agg_list                   │
+    #   # │ ---                 ┆ ---        ┆ ---                             │
+    #   # │ datetime[μs]        ┆ u32        ┆ list[datetime[μs]]              │
+    #   # ╞═════════════════════╪════════════╪═════════════════════════════════╡
+    #   # │ 2021-12-16 00:00:00 ┆ 2          ┆ [2021-12-16 00:00:00, 2021-12-… │
+    #   # │ 2021-12-16 01:00:00 ┆ 2          ┆ [2021-12-16 01:00:00, 2021-12-… │
+    #   # │ 2021-12-16 02:00:00 ┆ 2          ┆ [2021-12-16 02:00:00, 2021-12-… │
+    #   # │ 2021-12-16 03:00:00 ┆ 1          ┆ [2021-12-16 03:00:00]           │
+    #   # └─────────────────────┴────────────┴─────────────────────────────────┘
     #
     # @example When closed="both" the time values at the window boundaries belong to 2 groups.
     #   df.group_by_dynamic("time", every: "1h", closed: "both").agg(
@@ -1523,12 +1384,13 @@ module Polars
     #     closed: "right"
     #   ).agg(Polars.col("A").alias("A_agg_list"))
     #   # =>
-    #   # shape: (3, 4)
+    #   # shape: (4, 4)
     #   # ┌─────────────────┬─────────────────┬─────┬─────────────────┐
     #   # │ _lower_boundary ┆ _upper_boundary ┆ idx ┆ A_agg_list      │
     #   # │ ---             ┆ ---             ┆ --- ┆ ---             │
     #   # │ i64             ┆ i64             ┆ i64 ┆ list[str]       │
     #   # ╞═════════════════╪═════════════════╪═════╪═════════════════╡
+    #   # │ -2              ┆ 1               ┆ -2  ┆ ["A", "A"]      │
     #   # │ 0               ┆ 3               ┆ 0   ┆ ["A", "B", "B"] │
     #   # │ 2               ┆ 5               ┆ 2   ┆ ["B", "B", "C"] │
     #   # │ 4               ┆ 7               ┆ 4   ┆ ["C"]           │
@@ -1837,7 +1699,7 @@ module Polars
       if how == "cross"
         return _from_rbldf(
           _ldf.join(
-            other._ldf, [], [], allow_parallel, force_parallel, how, suffix
+            other._ldf, [], [], allow_parallel, join_nulls, force_parallel, how, suffix
           )
         )
       end
@@ -1891,16 +1753,16 @@ module Polars
     #   ).collect
     #   # =>
     #   # shape: (4, 6)
-    #   # ┌─────┬──────┬───────┬──────┬──────┬───────┐
-    #   # │ a   ┆ b    ┆ c     ┆ a^2  ┆ b/2  ┆ not c │
-    #   # │ --- ┆ ---  ┆ ---   ┆ ---  ┆ ---  ┆ ---   │
-    #   # │ i64 ┆ f64  ┆ bool  ┆ f64  ┆ f64  ┆ bool  │
-    #   # ╞═════╪══════╪═══════╪══════╪══════╪═══════╡
-    #   # │ 1   ┆ 0.5  ┆ true  ┆ 1.0  ┆ 0.25 ┆ false │
-    #   # │ 2   ┆ 4.0  ┆ true  ┆ 4.0  ┆ 2.0  ┆ false │
-    #   # │ 3   ┆ 10.0 ┆ false ┆ 9.0  ┆ 5.0  ┆ true  │
-    #   # │ 4   ┆ 13.0 ┆ true  ┆ 16.0 ┆ 6.5  ┆ false │
-    #   # └─────┴──────┴───────┴──────┴──────┴───────┘
+    #   # ┌─────┬──────┬───────┬─────┬──────┬───────┐
+    #   # │ a   ┆ b    ┆ c     ┆ a^2 ┆ b/2  ┆ not c │
+    #   # │ --- ┆ ---  ┆ ---   ┆ --- ┆ ---  ┆ ---   │
+    #   # │ i64 ┆ f64  ┆ bool  ┆ i64 ┆ f64  ┆ bool  │
+    #   # ╞═════╪══════╪═══════╪═════╪══════╪═══════╡
+    #   # │ 1   ┆ 0.5  ┆ true  ┆ 1   ┆ 0.25 ┆ false │
+    #   # │ 2   ┆ 4.0  ┆ true  ┆ 4   ┆ 2.0  ┆ false │
+    #   # │ 3   ┆ 10.0 ┆ false ┆ 9   ┆ 5.0  ┆ true  │
+    #   # │ 4   ┆ 13.0 ┆ true  ┆ 16  ┆ 6.5  ┆ false │
+    #   # └─────┴──────┴───────┴─────┴──────┴───────┘
     def with_columns(*exprs, **named_exprs)
       structify = ENV.fetch("POLARS_AUTO_STRUCTIFY", "0") != "0"
       rbexprs = Utils.parse_as_list_of_expressions(*exprs, **named_exprs, __structify: structify)
@@ -1965,26 +1827,26 @@ module Polars
     #   # ┌─────┬─────┬───────────┐
     #   # │ a   ┆ b   ┆ b_squared │
     #   # │ --- ┆ --- ┆ ---       │
-    #   # │ i64 ┆ i64 ┆ f64       │
+    #   # │ i64 ┆ i64 ┆ i64       │
     #   # ╞═════╪═════╪═══════════╡
-    #   # │ 1   ┆ 2   ┆ 4.0       │
-    #   # │ 3   ┆ 4   ┆ 16.0      │
-    #   # │ 5   ┆ 6   ┆ 36.0      │
+    #   # │ 1   ┆ 2   ┆ 4         │
+    #   # │ 3   ┆ 4   ┆ 16        │
+    #   # │ 5   ┆ 6   ┆ 36        │
     #   # └─────┴─────┴───────────┘
     #
     # @example
     #   df.with_column(Polars.col("a") ** 2).collect
     #   # =>
     #   # shape: (3, 2)
-    #   # ┌──────┬─────┐
-    #   # │ a    ┆ b   │
-    #   # │ ---  ┆ --- │
-    #   # │ f64  ┆ i64 │
-    #   # ╞══════╪═════╡
-    #   # │ 1.0  ┆ 2   │
-    #   # │ 9.0  ┆ 4   │
-    #   # │ 25.0 ┆ 6   │
-    #   # └──────┴─────┘
+    #   # ┌─────┬─────┐
+    #   # │ a   ┆ b   │
+    #   # │ --- ┆ --- │
+    #   # │ i64 ┆ i64 │
+    #   # ╞═════╪═════╡
+    #   # │ 1   ┆ 2   │
+    #   # │ 9   ┆ 4   │
+    #   # │ 25  ┆ 6   │
+    #   # └─────┴─────┘
     def with_column(column)
       with_columns([column])
     end
@@ -1996,11 +1858,9 @@ module Polars
     #   - List of column names.
     #
     # @return [LazyFrame]
-    def drop(columns)
-      if columns.is_a?(::String)
-        columns = [columns]
-      end
-      _from_rbldf(_ldf.drop(columns))
+    def drop(*columns)
+      drop_cols = Utils._expand_selectors(self, *columns)
+      _from_rbldf(_ldf.drop(drop_cols))
     end
 
     # Rename column names.
@@ -2233,16 +2093,16 @@ module Polars
     #   df.with_row_index.collect
     #   # =>
     #   # shape: (3, 3)
-    #   # ┌────────┬─────┬─────┐
-    #   # │ row_nr ┆ a   ┆ b   │
-    #   # │ ---    ┆ --- ┆ --- │
-    #   # │ u32    ┆ i64 ┆ i64 │
-    #   # ╞════════╪═════╪═════╡
-    #   # │ 0      ┆ 1   ┆ 2   │
-    #   # │ 1      ┆ 3   ┆ 4   │
-    #   # │ 2      ┆ 5   ┆ 6   │
-    #   # └────────┴─────┴─────┘
-    def with_row_index(name: "row_nr", offset: 0)
+    #   # ┌───────┬─────┬─────┐
+    #   # │ index ┆ a   ┆ b   │
+    #   # │ ---   ┆ --- ┆ --- │
+    #   # │ u32   ┆ i64 ┆ i64 │
+    #   # ╞═══════╪═════╪═════╡
+    #   # │ 0     ┆ 1   ┆ 2   │
+    #   # │ 1     ┆ 3   ┆ 4   │
+    #   # │ 2     ┆ 5   ┆ 6   │
+    #   # └───────┴─────┴─────┘
+    def with_row_index(name: "index", offset: 0)
       _from_rbldf(_ldf.with_row_index(name, offset))
     end
     alias_method :with_row_count, :with_row_index

data/lib/polars/lazy_group_by.rb

@@ -6,11 +6,108 @@ module Polars
       @lgb = lgb
     end
 
-    # Describe the aggregation that need to be done on a group.
+    # Compute aggregations for each group of a group by operation.
+    #
+    # @param aggs [Array]
+    #   Aggregations to compute for each group of the group by operation,
+    #   specified as positional arguments.
+    #   Accepts expression input. Strings are parsed as column names.
+    # @param named_aggs [Hash]
+    #   Additional aggregations, specified as keyword arguments.
+    #   The resulting columns will be renamed to the keyword used.
     #
     # @return [LazyFrame]
-    def agg(aggs)
-      rbexprs = Utils.selection_to_rbexpr_list(aggs)
+    #
+    # @example Compute the aggregation of the columns for each group.
+    #   ldf = Polars::DataFrame.new(
+    #     {
+    #       "a" => ["a", "b", "a", "b", "c"],
+    #       "b" => [1, 2, 1, 3, 3],
+    #       "c" => [5, 4, 3, 2, 1]
+    #     }
+    #   ).lazy
+    #   ldf.group_by("a").agg(
+    #     [Polars.col("b"), Polars.col("c")]
+    #   ).collect
+    #   # =>
+    #   # shape: (3, 3)
+    #   # ┌─────┬───────────┬───────────┐
+    #   # │ a   ┆ b         ┆ c         │
+    #   # │ --- ┆ ---       ┆ ---       │
+    #   # │ str ┆ list[i64] ┆ list[i64] │
+    #   # ╞═════╪═══════════╪═══════════╡
+    #   # │ a   ┆ [1, 1]    ┆ [5, 3]    │
+    #   # │ b   ┆ [2, 3]    ┆ [4, 2]    │
+    #   # │ c   ┆ [3]       ┆ [1]       │
+    #   # └─────┴───────────┴───────────┘
+    #
+    # @example Compute the sum of a column for each group.
+    #   ldf.group_by("a").agg(
+    #     Polars.col("b").sum
+    #   ).collect
+    #   # =>
+    #   # shape: (3, 2)
+    #   # ┌─────┬─────┐
+    #   # │ a   ┆ b   │
+    #   # │ --- ┆ --- │
+    #   # │ str ┆ i64 │
+    #   # ╞═════╪═════╡
+    #   # │ a   ┆ 2   │
+    #   # │ b   ┆ 5   │
+    #   # │ c   ┆ 3   │
+    #   # └─────┴─────┘
+    #
+    # @example Compute multiple aggregates at once by passing a list of expressions.
+    #   ldf.group_by("a").agg(
+    #     [Polars.sum("b"), Polars.mean("c")]
+    #   ).collect
+    #   # =>
+    #   # shape: (3, 3)
+    #   # ┌─────┬─────┬─────┐
+    #   # │ a   ┆ b   ┆ c   │
+    #   # │ --- ┆ --- ┆ --- │
+    #   # │ str ┆ i64 ┆ f64 │
+    #   # ╞═════╪═════╪═════╡
+    #   # │ c   ┆ 3   ┆ 1.0 │
+    #   # │ a   ┆ 2   ┆ 4.0 │
+    #   # │ b   ┆ 5   ┆ 3.0 │
+    #   # └─────┴─────┴─────┘
+    #
+    # @example Or use positional arguments to compute multiple aggregations in the same way.
+    #   ldf.group_by("a").agg(
+    #     Polars.sum("b").name.suffix("_sum"),
+    #     (Polars.col("c") ** 2).mean.name.suffix("_mean_squared")
+    #   ).collect
+    #   # =>
+    #   # shape: (3, 3)
+    #   # ┌─────┬───────┬────────────────┐
+    #   # │ a   ┆ b_sum ┆ c_mean_squared │
+    #   # │ --- ┆ ---   ┆ ---            │
+    #   # │ str ┆ i64   ┆ f64            │
+    #   # ╞═════╪═══════╪════════════════╡
+    #   # │ a   ┆ 2     ┆ 17.0           │
+    #   # │ c   ┆ 3     ┆ 1.0            │
+    #   # │ b   ┆ 5     ┆ 10.0           │
+    #   # └─────┴───────┴────────────────┘
+    #
+    # @example Use keyword arguments to easily name your expression inputs.
+    #   ldf.group_by("a").agg(
+    #     b_sum: Polars.sum("b"),
+    #     c_mean_squared: (Polars.col("c") ** 2).mean
+    #   ).collect
+    #   # =>
+    #   # shape: (3, 3)
+    #   # ┌─────┬───────┬────────────────┐
+    #   # │ a   ┆ b_sum ┆ c_mean_squared │
+    #   # │ --- ┆ ---   ┆ ---            │
+    #   # │ str ┆ i64   ┆ f64            │
+    #   # ╞═════╪═══════╪════════════════╡
+    #   # │ a   ┆ 2     ┆ 17.0           │
+    #   # │ c   ┆ 3     ┆ 1.0            │
+    #   # │ b   ┆ 5     ┆ 10.0           │
+    #   # └─────┴───────┴────────────────┘
+    def agg(*aggs, **named_aggs)
+      rbexprs = Utils.parse_as_list_of_expressions(*aggs, **named_aggs)
       Utils.wrap_ldf(@lgb.agg(rbexprs))
     end
 

data/lib/polars/list_expr.rb

@@ -365,6 +365,10 @@ module Polars
     #
     # @param index [Integer]
     #   Index to return per sublist
+    # @param null_on_oob [Boolean]
+    #   Behavior if an index is out of bounds:
+    #   true -> set as null
+    #   false -> raise an error
     #
     # @return [Expr]
     #
@@ -382,9 +386,9 @@ module Polars
     #   # │ null │
     #   # │ 1    │
     #   # └──────┘
-    def get(index)
+    def get(index, null_on_oob: true)
       index = Utils.parse_as_expression(index)
-      Utils.wrap_expr(_rbexpr.list_get(index))
+      Utils.wrap_expr(_rbexpr.list_get(index, null_on_oob))
     end
 
     # Get the value by index in the sublists.

data/lib/polars/rolling_group_by.rb

@@ -25,12 +25,12 @@ module Polars
       @check_sorted = check_sorted
     end
 
-    def agg(aggs)
+    def agg(*aggs, **named_aggs)
       @df.lazy
         .group_by_rolling(
           index_column: @time_column, period: @period, offset: @offset, closed: @closed, by: @by, check_sorted: @check_sorted
         )
-        .agg(aggs)
+        .agg(*aggs, **named_aggs)
         .collect(no_optimization: true, string_cache: false)
     end
   end

data/lib/polars/series.rb

@@ -1155,13 +1155,13 @@ module Polars
     #   s.cumulative_eval(Polars.element.first - Polars.element.last ** 2)
     #   # =>
     #   # shape: (5,)
-    #   # Series: 'values' [f64]
+    #   # Series: 'values' [i64]
     #   # [
-    #   #         0.0
-    #   #         -3.0
-    #   #         -8.0
-    #   #         -15.0
-    #   #         -24.0
+    #   #         0
+    #   #         -3
+    #   #         -8
+    #   #         -15
+    #   #         -24
     #   # ]
     def cumulative_eval(expr, min_periods: 1, parallel: false)
       super
@@ -1567,12 +1567,12 @@ module Polars
     #   #         2
     #   #         1
     #   # ]
-    def sort(reverse: false, nulls_last: false, in_place: false)
+    def sort(reverse: false, nulls_last: false, multithreaded: true, in_place: false)
       if in_place
-        self._s = _s.sort(reverse, nulls_last)
+        self._s = _s.sort(reverse, nulls_last, multithreaded)
         self
       else
-        Utils.wrap_s(_s.sort(reverse, nulls_last))
+        Utils.wrap_s(_s.sort(reverse, nulls_last, multithreaded))
       end
     end
 
@@ -1594,7 +1594,7 @@ module Polars
     #   #         4
     #   #         3
     #   # ]
-    def top_k(k: 5)
+    def top_k(k: 5, nulls_last: false, multithreaded: true)
       super
     end
 
@@ -1616,7 +1616,7 @@ module Polars
     #   #         2
     #   #         3
     #   # ]
-    def bottom_k(k: 5)
+    def bottom_k(k: 5, nulls_last: false, multithreaded: true)
       super
     end
 
@@ -3646,6 +3646,8 @@ module Polars
     #     on the order that the values occur in the Series.
     # @param reverse [Boolean]
     #   Reverse the operation.
+    # @param seed [Integer]
+    #   If `method: "random"`, use this as seed.
     #
     # @return [Series]
     #
@@ -3676,7 +3678,7 @@ module Polars
     #   #         2
     #   #         5
     #   # ]
-    def rank(method: "average", reverse: false)
+    def rank(method: "average", reverse: false, seed: nil)
       super
     end