polars-df 0.20.0-x64-mingw-ucrt → 0.21.1-x64-mingw-ucrt

data/lib/polars/functions/col.rb

@@ -8,11 +8,11 @@ module Polars
         if Utils.strlike?(name)
           names_str = [name]
           names_str.concat(more_names)
-          return Utils.wrap_expr(Plr.cols(names_str.map(&:to_s)))
+          return Selector._by_name(names_str.map(&:to_s), strict: true).as_expr
         elsif Utils.is_polars_dtype(name)
           dtypes = [name]
           dtypes.concat(more_names)
-          return Utils.wrap_expr(Plr.dtype_cols(dtypes))
+          return Selector._by_type(dtypes).as_expr
         else
           msg = "invalid input for `col`\n\nExpected `str` or `DataType`, got #{name.class.name}."
           raise TypeError, msg
@@ -22,7 +22,8 @@ module Polars
       if Utils.strlike?(name)
         Utils.wrap_expr(Plr.col(name.to_s))
       elsif Utils.is_polars_dtype(name)
-        Utils.wrap_expr(Plr.dtype_cols([name]))
+        dtypes = [name]
+        Selector._by_dtype(dtypes).as_expr
       elsif name.is_a?(::Array) || name.is_a?(::Set)
         names = Array(name)
         if names.empty?
@@ -31,9 +32,9 @@ module Polars
 
         item = names[0]
         if Utils.strlike?(item)
-          Utils.wrap_expr(Plr.cols(names.map(&:to_s)))
+          Selector._by_name(names.map(&:to_s), strict: true).as_expr
         elsif Utils.is_polars_dtype(item)
-          Utils.wrap_expr(Plr.dtype_cols(names))
+          Selector._by_dtype(names).as_expr
         else
           msg = "invalid input for `col`\n\nExpected iterable of type `str` or `DataType`, got iterable of type #{item.class.name}."
           raise TypeError, msg

data/lib/polars/functions/datatype.rb

@@ -0,0 +1,21 @@
+module Polars
+  module Functions
+    # Get a lazily evaluated :class:`DataType` of a column or expression.
+    #
+    # @note
+    #   This functionality is considered **unstable**. It may be changed
+    #   at any point without it being considered a breaking change.
+    #
+    # @return [DataTypeExpr]
+    def dtype_of(col_or_expr)
+      e = nil
+      if col_or_expr.is_a?(::String)
+        e = F.col(col_or_expr)
+      else
+        e = col_or_expr
+      end
+
+      DataTypeExpr._from_rbdatatype_expr(RbDataTypeExpr.of_expr(e._rbexpr))
+    end
+  end
+end

data/lib/polars/functions/lazy.rb

@@ -1,5 +1,18 @@
 module Polars
   module Functions
+    # Select a field in the current `struct.with_fields` scope.
+    #
+    # @param name [Object]
+    #   Name of the field(s) to select.
+    #
+    # @return [Expr]
+    def field(name)
+      if name.is_a?(::String)
+        name = [name]
+      end
+      Utils.wrap_expr(Plr.field(name))
+    end
+
     # Alias for an element in evaluated in an `eval` expression.
     #
     # @return [Expr]
@@ -458,7 +471,7 @@ module Polars
     #   # └─────┴─────┘
     def first(*columns)
       if columns.empty?
-        return Utils.wrap_expr(Plr.first)
+        return cs.first.as_expr
       end
 
       col(*columns).first
@@ -518,7 +531,7 @@ module Polars
     #   # └─────┴─────┘
     def last(*columns)
       if columns.empty?
-        return Utils.wrap_expr(Plr.last)
+        return cs.last.as_expr
       end
 
       col(*columns).last
@@ -565,12 +578,8 @@ module Polars
     #   # │ bar ┆ 8   │
     #   # │ baz ┆ 3   │
     #   # └─────┴─────┘
-    def nth(*indices)
-      if indices.length == 1 && indices[0].is_a?(Array)
-        indices = indices[0]
-      end
-
-      Utils.wrap_expr(Plr.index_cols(indices))
+    def nth(*indices, strict: true)
+      cs.by_index(*indices, require_all: strict).as_expr
     end
 
     # Get the first `n` rows.
@@ -675,12 +684,12 @@ module Polars
     #   Column name or Expression.
     # @param b [Object]
     #   Column name or Expression.
+    # @param method ["pearson", "spearman"]
+    #   Correlation method.
     # @param ddof [Integer]
     #   "Delta Degrees of Freedom": the divisor used in the calculation is N - ddof,
     #   where N represents the number of elements.
     #   By default ddof is 1.
-    # @param method ["pearson", "spearman"]
-    #   Correlation method.
     # @param propagate_nans [Boolean]
     #   If `true` any `NaN` encountered will lead to `NaN` in the output.
     #   Defaults to `False` where `NaN` are regarded as larger than any finite number
@@ -795,14 +804,82 @@ module Polars
     # Accumulate over multiple columns horizontally/row wise with a left fold.
     #
     # @return [Expr]
-    def fold(acc, f, exprs)
+    #
+    # @example Horizontally sum over all columns and add 1.
+    #   df = Polars::DataFrame.new(
+    #    {
+    #      "a" => [1, 2, 3],
+    #      "b" => [3, 4, 5],
+    #      "c" => [5, 6, 7]
+    #    }
+    #   )
+    #   df.select(
+    #     Polars.fold(
+    #       Polars.lit(1), ->(acc, x) { acc + x }, Polars.col("*")
+    #     ).alias("sum")
+    #   )
+    #   # =>
+    #   # shape: (3, 1)
+    #   # ┌─────┐
+    #   # │ sum │
+    #   # │ --- │
+    #   # │ i64 │
+    #   # ╞═════╡
+    #   # │ 10  │
+    #   # │ 13  │
+    #   # │ 16  │
+    #   # └─────┘
+    #
+    # @example You can also apply a condition/predicate on all columns:
+    #   df = Polars::DataFrame.new(
+    #     {
+    #       "a" => [1, 2, 3],
+    #       "b" => [0, 1, 2]
+    #     }
+    #   )
+    #   df.filter(
+    #     Polars.fold(
+    #       Polars.lit(true),
+    #       ->(acc, x) { acc & x },
+    #       Polars.col("*") > 1
+    #     )
+    #   )
+    #   # =>
+    #   # shape: (1, 2)
+    #   # ┌─────┬─────┐
+    #   # │ a   ┆ b   │
+    #   # │ --- ┆ --- │
+    #   # │ i64 ┆ i64 │
+    #   # ╞═════╪═════╡
+    #   # │ 3   ┆ 2   │
+    #   # └─────┴─────┘
+    def fold(
+      acc,
+      function,
+      exprs,
+      returns_scalar: false,
+      return_dtype: nil
+    )
       acc = Utils.parse_into_expression(acc, str_as_lit: true)
       if exprs.is_a?(Expr)
         exprs = [exprs]
       end
 
+      rt = nil
+      if !return_dtype.nil?
+        rt = Utils.parse_into_datatype_expr(return_dtype)._rbdatatype_expr
+      end
+
       exprs = Utils.parse_into_list_of_expressions(exprs)
-      Utils.wrap_expr(Plr.fold(acc, f, exprs))
+      Utils.wrap_expr(
+        Plr.fold(
+          acc,
+          function,
+          exprs,
+          returns_scalar,
+          rt
+        )
+      )
     end
 
     # def reduce
@@ -815,11 +892,17 @@ module Polars
     # @param acc [Object]
     #   Accumulator Expression. This is the value that will be initialized when the fold
     #   starts. For a sum this could for instance be lit(0).
-    # @param f [Object]
+    # @param function [Object]
     #   Function to apply over the accumulator and the value.
     #   Fn(acc, value) -> new_value
     # @param exprs [Object]
     #   Expressions to aggregate over. May also be a wildcard expression.
+    # @param returns_scalar [Boolean]
+    #   Whether or not `function` applied returns a scalar. This must be set correctly
+    #   by the user.
+    # @param return_dtype [Object]
+    #   Output datatype.
+    #   If not set, the dtype will be inferred based on the dtype of the accumulator.
     # @param include_init [Boolean]
     #   Include the initial accumulator state as struct field.
     #
@@ -851,14 +934,35 @@ module Polars
     #   # │ 2   ┆ 4   ┆ 6   ┆ {3,7,13}  │
     #   # │ 3   ┆ 5   ┆ 7   ┆ {4,9,16}  │
     #   # └─────┴─────┴─────┴───────────┘
-    def cum_fold(acc, f, exprs, include_init: false)
+    def cum_fold(
+      acc,
+      function,
+      exprs,
+      returns_scalar: false,
+      return_dtype: nil,
+      include_init: false
+    )
       acc = Utils.parse_into_expression(acc, str_as_lit: true)
       if exprs.is_a?(Expr)
         exprs = [exprs]
       end
 
+      rt = nil
+      if !return_dtype.nil?
+        rt = Utils.parse_into_datatype_expr(return_dtype)._rbdatatype_expr
+      end
+
       exprs = Utils.parse_into_list_of_expressions(exprs)
-      Utils.wrap_expr(Plr.cum_fold(acc, f, exprs, include_init)._alias("cum_fold"))
+      Utils.wrap_expr(
+        Plr.cum_fold(
+          acc,
+          function,
+          exprs,
+          returns_scalar,
+          rt,
+          include_init
+        )._alias("cum_fold")
+      )
     end
     alias_method :cumfold, :cum_fold
 
@@ -1047,8 +1151,16 @@ module Polars
     #
     # @param exprs [Object]
     #   Columns use to determine the ordering.
+    # @param more_exprs [Array]
+    #   Additional columns to arg sort by, specified as positional arguments.
     # @param reverse [Boolean]
     #   Default is ascending.
+    # @param nulls_last [Boolean]
+    #   Place null values last.
+    # @param multithreaded [Boolean]
+    #   Sort using multiple threads.
+    # @param maintain_order [Boolean]
+    #   Whether the order should be maintained if elements are equal.
     #
     # @return [Expr]
     #

data/lib/polars/functions/repeat.rb

@@ -6,6 +6,10 @@ module Polars
     #   Value to repeat.
     # @param n [Integer]
     #   Repeat `n` times.
+    # @param dtype [Object]
+    #   Data type of the resulting column. If set to `nil` (default), data type is
+    #   inferred from the given value. Defaults to Int32 for integer values, unless
+    #   Int64 is required to fit the given value. Defaults to Float64 for float values.
     # @param eager [Boolean]
     #   Run eagerly and collect into a `Series`.
     # @param name [String]

data/lib/polars/io/csv.rb

@@ -347,6 +347,9 @@ module Polars
     #   - `String`: All values equal to this string will be null.
     #   - `Array`: All values equal to any string in this array will be null.
     #   - `Hash`: A hash that maps column name to a null value string.
+    # @param missing_utf8_is_empty_string [Boolean]
+    #   By default a missing value is considered to be null; if you would prefer missing
+    #   utf8 values to be treated as the empty string you can set this param true.
     # @param ignore_errors [Boolean]
     #   Try to keep reading lines if some lines yield errors.
     #   First try `infer_schema_length: 0` to read all columns as
@@ -387,8 +390,13 @@ module Polars
     #   Offset to start the row_count column (only used if the name is set).
     # @param eol_char [String]
     #   Single byte end of line character.
+    # @param raise_if_empty [Boolean]
+    #   When there is no data in the source,`NoDataError` is raised. If this parameter
+    #   is set to false, `nil` will be returned from `next_batches(n)` instead.
     # @param truncate_ragged_lines [Boolean]
     #   Truncate lines that are longer than the schema.
+    # @param decimal_comma [Boolean]
+    #   Parse floats using a comma as the decimal separator instead of a period.
     #
     # @return [BatchedCsvReader]
     #
@@ -491,7 +499,7 @@ module Polars
     #   for instance `#`.
     # @param quote_char [String]
     #   Single byte character used for csv quoting.
-    #   Set to None to turn off special handling and escaping of quotes.
+    #   Set to nil to turn off special handling and escaping of quotes.
     # @param skip_rows [Integer]
     #   Start reading after `skip_rows` lines. The header will be parsed at this
     #   offset.
@@ -503,6 +511,9 @@ module Polars
     #   - `String`: All values equal to this string will be null.
     #   - `Array`: All values equal to any string in this array will be null.
     #   - `Hash`: A hash that maps column name to a null value string.
+    # @param missing_utf8_is_empty_string [Boolean]
+    #   By default a missing value is considered to be null; if you would prefer missing
+    #   utf8 values to be treated as the empty string you can set this param true.
     # @param ignore_errors [Boolean]
     #   Try to keep reading lines if some lines yield errors.
     #   First try `infer_schema_length: 0` to read all columns as
@@ -538,8 +549,15 @@ module Polars
     #   the column remains of data type `:str`.
     # @param eol_char [String]
     #   Single byte end of line character.
+    # @param raise_if_empty [Boolean]
+    #   When there is no data in the source, `NoDataError` is raised. If this parameter
+    #   is set to false, an empty LazyFrame (with no columns) is returned instead.
     # @param truncate_ragged_lines [Boolean]
     #   Truncate lines that are longer than the schema.
+    # @param decimal_comma [Boolean]
+    #   Parse floats using a comma as the decimal separator instead of a period.
+    # @param glob [Boolean]
+    #   Expand path given via globbing rules.
     #
     # @return [LazyFrame]
     def scan_csv(

data/lib/polars/io/json.rb

@@ -4,6 +4,22 @@ module Polars
     #
     # @param source [Object]
     #   Path to a file or a file-like object.
+    # @param schema [Object]
+    #   The DataFrame schema may be declared in several ways:
+    #
+    #   * As a hash of \\\\{name:type} pairs; if type is nil, it will be auto-inferred.
+    #   * As an array of column names; in this case types are automatically inferred.
+    #   * As an array of [name,type] pairs; this is equivalent to the hash form.
+    #
+    #   If you supply an array of column names that does not match the names in the
+    #   underlying data, the names given here will overwrite them. The number
+    #   of names given in the schema should match the underlying data dimensions.
+    # @param schema_overrides [Hash]
+    #   Support type specification or override of one or more columns; note that
+    #   any dtypes inferred from the schema param will be overridden.
+    # @param infer_schema_length [Integer]
+    #   The maximum number of rows to scan for schema inference.
+    #   If set to `nil`, the full data may be scanned *(this is slow)*.
     #
     # @return [DataFrame]
     def read_json(

data/lib/polars/io/ndjson.rb

@@ -4,6 +4,19 @@ module Polars
     #
     # @param source [Object]
     #   Path to a file or a file-like object.
+    # @param schema [Object]
+    #   The DataFrame schema may be declared in several ways:
+    #
+    #   * As a hash of \\\\{name:type} pairs; if type is nil, it will be auto-inferred.
+    #   * As an array of column names; in this case types are automatically inferred.
+    #   * As an array of [name,type] pairs; this is equivalent to the hash form.
+    #
+    #   If you supply an array of column names that does not match the names in the
+    #   underlying data, the names given here will overwrite them. The number
+    #   of names given in the schema should match the underlying data dimensions.
+    # @param schema_overrides [Hash]
+    #   Support type specification or override of one or more columns; note that
+    #   any dtypes inferred from the schema param will be overridden.
     #
     # @return [DataFrame]
     def read_ndjson(

data/lib/polars/io/parquet.rb

@@ -43,12 +43,18 @@ module Polars
     #   Extra options that make sense for a particular storage connection.
     # @param credential_provider [Object]
     #   Provide a function that can be called to provide cloud storage
-    #   credentials. The function is expected to return a dictionary of
+    #   credentials. The function is expected to return a hash of
     #   credential keys along with an optional credential expiry time.
     # @param retries [Integer]
     #   Number of retries if accessing a cloud instance fails.
     # @param include_file_paths [String]
     #   Include the path of the source file(s) as a column with this name.
+    # @param allow_missing_columns [Boolean]
+    #   When reading a list of parquet files, if a column existing in the first
+    #   file cannot be found in subsequent files, the default behavior is to
+    #   raise an error. However, if `allow_missing_columns` is set to
+    #   `true`, a full-NULL column is returned instead of erroring for the files
+    #   that do not contain the column.
     #
     # @return [DataFrame]
     def read_parquet(
@@ -117,7 +123,26 @@ module Polars
         source = Utils.normalize_filepath(source)
       end
 
-      Plr.parquet_schema(source)
+      # TODO return Schema
+      scan_parquet(source).collect_schema.to_h
+    end
+
+    # Get file-level custom metadata of a Parquet file without reading data.
+    #
+    # @note
+    #   This functionality is considered **experimental**. It may be removed or
+    #   changed at any point without it being considered a breaking change.
+    #
+    # @param source [Object]
+    #   Path to a file or a file-like object.
+    #
+    # @return [Hash]
+    def read_parquet_metadata(source)
+      if Utils.pathlike?(source)
+        source = Utils.normalize_filepath(source, check_not_directory: false)
+      end
+
+      Plr.read_parquet_metadata(source)
     end
 
     # Lazily read from a parquet file or multiple files via glob patterns.
@@ -165,12 +190,23 @@ module Polars
     #   Extra options that make sense for a particular storage connection.
     # @param credential_provider [Object]
     #   Provide a function that can be called to provide cloud storage
-    #   credentials. The function is expected to return a dictionary of
+    #   credentials. The function is expected to return a hash of
     #   credential keys along with an optional credential expiry time.
     # @param retries [Integer]
     #   Number of retries if accessing a cloud instance fails.
     # @param include_file_paths [String]
     #   Include the path of the source file(s) as a column with this name.
+    # @param allow_missing_columns [Boolean]
+    #   When reading a list of parquet files, if a column existing in the first
+    #   file cannot be found in subsequent files, the default behavior is to
+    #   raise an error. However, if `allow_missing_columns` is set to
+    #   `true`, a full-NULL column is returned instead of erroring for the files
+    #   that do not contain the column.
+    # @param extra_columns ['ignore', 'raise']
+    #   Configuration for behavior when extra columns outside of the
+    #   defined schema are encountered in the data:
+    #     * `ignore`: Silently ignores.
+    #     * `raise`: Raises an error.
     #
     # @return [LazyFrame]
     def scan_parquet(
@@ -192,8 +228,13 @@ module Polars
       credential_provider: nil,
       retries: 2,
       include_file_paths: nil,
-      allow_missing_columns: false
+      allow_missing_columns: false,
+      extra_columns: "raise",
+      _column_mapping: nil,
+      _deletion_files: nil
     )
+      missing_columns = allow_missing_columns ? "insert" : "raise"
+
       if Utils.pathlike?(source)
         source = Utils.normalize_filepath(source, check_not_directory: false)
       elsif Utils.is_path_or_str_sequence(source)
@@ -204,56 +245,11 @@ module Polars
         raise Todo
       end
 
-      _scan_parquet_impl(
-        source,
-        n_rows: n_rows,
-        cache: cache,
-        parallel: parallel,
-        rechunk: rechunk,
-        row_index_name: row_count_name,
-        row_index_offset: row_count_offset,
-        storage_options: storage_options,
-        credential_provider: credential_provider,
-        low_memory: low_memory,
-        use_statistics: use_statistics,
-        hive_partitioning: hive_partitioning,
-        schema: schema,
-        hive_schema: hive_schema,
-        try_parse_hive_dates: try_parse_hive_dates,
-        retries: retries,
-        glob: glob,
-        include_file_paths: include_file_paths,
-        allow_missing_columns: allow_missing_columns
-      )
-    end
-
-    # @private
-    def _scan_parquet_impl(
-      source,
-      n_rows: nil,
-      cache: true,
-      parallel: "auto",
-      rechunk: true,
-      row_index_name: nil,
-      row_index_offset: 0,
-      storage_options: nil,
-      credential_provider: nil,
-      low_memory: false,
-      use_statistics: true,
-      hive_partitioning: nil,
-      glob: true,
-      schema: nil,
-      hive_schema: nil,
-      try_parse_hive_dates: true,
-      retries: 2,
-      include_file_paths: nil,
-      allow_missing_columns: false
-    )
       if source.is_a?(::Array)
         sources = source
         source = nil
       else
-        sources = []
+        sources = [source]
       end
 
       if storage_options
@@ -262,27 +258,35 @@ module Polars
         storage_options = nil
       end
 
+      row_index_name = row_count_name
+      row_index_offset = row_count_offset
+
       rblf =
         RbLazyFrame.new_from_parquet(
-          source,
           sources,
-          n_rows,
-          cache,
+          schema,
+          ScanOptions.new(
+            row_index: !row_index_name.nil? ? [row_index_name, row_index_offset] : nil,
+            pre_slice: !n_rows.nil? ? [0, n_rows] : nil,
+            # cast_options: cast_options,
+            extra_columns: extra_columns,
+            missing_columns: missing_columns,
+            include_file_paths: include_file_paths,
+            glob: glob,
+            hive_partitioning: hive_partitioning,
+            hive_schema: hive_schema,
+            try_parse_hive_dates: try_parse_hive_dates,
+            rechunk: rechunk,
+            cache: cache,
+            storage_options: storage_options,
+            # credential_provider: credential_provider_builder,
+            retries: retries,
+            deletion_files: _deletion_files,
+            column_mapping: _column_mapping
+          ),
           parallel,
-          rechunk,
-          Utils.parse_row_index_args(row_index_name, row_index_offset),
           low_memory,
-          storage_options,
-          credential_provider,
-          use_statistics,
-          hive_partitioning,
-          schema,
-          hive_schema,
-          try_parse_hive_dates,
-          retries,
-          glob,
-          include_file_paths,
-          allow_missing_columns
+          use_statistics
         )
       Utils.wrap_ldf(rblf)
     end

data/lib/polars/io/scan_options.rb

@@ -0,0 +1,47 @@
+module Polars
+  module IO
+    class ScanOptions
+      attr_reader :row_index, :pre_slice, :cast_options, :extra_columns, :missing_columns,
+        :include_file_paths, :glob, :hive_partitioning, :hive_schema, :try_parse_hive_dates,
+        :rechunk, :cache, :storage_options, :credential_provider, :retries, :column_mapping, :deletion_files
+
+      def initialize(
+        row_index: nil,
+        pre_slice: nil,
+        cast_options: nil,
+        extra_columns: "raise",
+        missing_columns: "raise",
+        include_file_paths: nil,
+        glob: true,
+        hive_partitioning: nil,
+        hive_schema: nil,
+        try_parse_hive_dates: true,
+        rechunk: false,
+        cache: true,
+        storage_options: nil,
+        credential_provider: nil,
+        retries: 2,
+        column_mapping: nil,
+        deletion_files: nil
+      )
+        @row_index = row_index
+        @pre_slice = pre_slice
+        @cast_options = cast_options
+        @extra_columns = extra_columns
+        @missing_columns = missing_columns
+        @include_file_paths = include_file_paths
+        @glob = glob
+        @hive_partitioning = hive_partitioning
+        @hive_schema = hive_schema
+        @try_parse_hive_dates = try_parse_hive_dates
+        @rechunk = rechunk
+        @cache = cache
+        @storage_options = storage_options
+        @credential_provider = credential_provider
+        @retries = retries
+        @column_mapping = column_mapping
+        @deletion_files = deletion_files
+      end
+    end
+  end
+end