polars-df 0.23.0 → 0.24.0

data/lib/polars/io/parquet.rb

@@ -9,10 +9,10 @@ module Polars
     #   of column names.
     # @param n_rows [Integer]
     #   Stop reading from parquet file after reading `n_rows`.
-    # @param row_count_name [String]
+    # @param row_index_name [String]
     #   If not nil, this will insert a row count column with give name into the
     #   DataFrame.
-    # @param row_count_offset [Integer]
+    # @param row_index_offset [Integer]
     #   Offset to start the row_count column (only use if the name is set).
     # @param parallel ["auto", "columns", "row_groups", "none"]
     #   This determines the direction of parallelism. 'auto' will try to determine the
@@ -49,6 +49,12 @@ module Polars
     #   Number of retries if accessing a cloud instance fails.
     # @param include_file_paths [String]
     #   Include the path of the source file(s) as a column with this name.
+    # @param missing_columns ['insert', 'raise']
+    #   Configuration for behavior when columns defined in the schema
+    #   are missing from the data:
+    #
+    #   * `insert`: Inserts the missing columns using NULLs as the row values.
+    #   * `raise`: Raises an error.
     # @param allow_missing_columns [Boolean]
     #   When reading a list of parquet files, if a column existing in the first
     #   file cannot be found in subsequent files, the default behavior is to
@@ -61,8 +67,8 @@ module Polars
       source,
       columns: nil,
       n_rows: nil,
-      row_count_name: nil,
-      row_count_offset: 0,
+      row_index_name: nil,
+      row_index_offset: 0,
       parallel: "auto",
       use_statistics: true,
       hive_partitioning: nil,
@@ -73,17 +79,18 @@ module Polars
       rechunk: false,
       low_memory: false,
       storage_options: nil,
-      credential_provider: nil,
+      credential_provider: "auto",
       retries: 2,
       include_file_paths: nil,
-      allow_missing_columns: false
+      missing_columns: "raise",
+      allow_missing_columns: nil
     )
       lf =
         scan_parquet(
           source,
           n_rows: n_rows,
-          row_count_name: row_count_name,
-          row_count_offset: row_count_offset,
+          row_index_name: row_index_name,
+          row_index_offset: row_index_offset,
           parallel: parallel,
           use_statistics: use_statistics,
           hive_partitioning: hive_partitioning,
@@ -98,6 +105,7 @@ module Polars
           retries: retries,
           glob: glob,
           include_file_paths: include_file_paths,
+          missing_columns: missing_columns,
           allow_missing_columns: allow_missing_columns
         )
 
@@ -134,14 +142,40 @@ module Polars
     #
     # @param source [Object]
     #   Path to a file or a file-like object.
+    # @param storage_options [Hash]
+    #   Extra options that make sense for a particular storage connection.
+    # @param credential_provider [Object]
+    #   Provide a function that can be called to provide cloud storage
+    #   credentials. The function is expected to return a hash of
+    #   credential keys along with an optional credential expiry time.
+    # @param retries [Integer]
+    #   Number of retries if accessing a cloud instance fails.
     #
     # @return [Hash]
-    def read_parquet_metadata(source)
+    def read_parquet_metadata(
+      source,
+      storage_options: nil,
+      credential_provider: "auto",
+      retries: 2
+    )
+      if storage_options
+        raise Todo
+      end
+
       if Utils.pathlike?(source)
         source = Utils.normalize_filepath(source, check_not_directory: false)
       end
 
-      Plr.read_parquet_metadata(source)
+      credential_provider_builder = _init_credential_provider_builder(
+        credential_provider, source, storage_options, "scan_parquet"
+      )
+
+      Plr.read_parquet_metadata(
+        source,
+        storage_options&.any? ? storage_options.map { |k, v| [k.to_s, v.to_s] } : nil,
+        credential_provider_builder,
+        retries
+      )
     end
 
     # Lazily read from a parquet file or multiple files via glob patterns.
@@ -153,10 +187,10 @@ module Polars
     #   Path to a file or a file-like object.
     # @param n_rows [Integer]
     #   Stop reading from parquet file after reading `n_rows`.
-    # @param row_count_name [String]
+    # @param row_index_name [String]
     #   If not nil, this will insert a row count column with give name into the
     #   DataFrame.
-    # @param row_count_offset [Integer]
+    # @param row_index_offset [Integer]
     #   Offset to start the row_count column (only use if the name is set).
     # @param parallel ["auto", "columns", "row_groups", "none"]
     #   This determines the direction of parallelism. 'auto' will try to determine the
@@ -169,6 +203,8 @@ module Polars
     #   to prune reads.
     # @param glob [Boolean]
     #   Expand path given via globbing rules.
+    # @param hidden_file_prefix [Boolean]
+    #   Skip reading files whose names begin with the specified prefixes.
     # @param schema [Object]
     #   Specify the datatypes of the columns. The datatypes must match the
     #   datatypes in the file(s). If there are extra columns that are not in the
@@ -195,6 +231,12 @@ module Polars
     #   Number of retries if accessing a cloud instance fails.
     # @param include_file_paths [String]
     #   Include the path of the source file(s) as a column with this name.
+    # @param missing_columns ['insert', 'raise']
+    #   Configuration for behavior when columns defined in the schema
+    #   are missing from the data:
+    #
+    #   * `insert`: Inserts the missing columns using NULLs as the row values.
+    #   * `raise`: Raises an error.
     # @param allow_missing_columns [Boolean]
     #   When reading a list of parquet files, if a column existing in the first
     #   file cannot be found in subsequent files, the default behavior is to
@@ -214,12 +256,13 @@ module Polars
     def scan_parquet(
       source,
       n_rows: nil,
-      row_count_name: nil,
-      row_count_offset: 0,
+      row_index_name: nil,
+      row_index_offset: 0,
       parallel: "auto",
       use_statistics: true,
       hive_partitioning: nil,
       glob: true,
+      hidden_file_prefix: nil,
       schema: nil,
       hive_schema: nil,
       try_parse_hive_dates: true,
@@ -227,42 +270,58 @@ module Polars
       low_memory: false,
       cache: true,
       storage_options: nil,
-      credential_provider: nil,
+      credential_provider: "auto",
       retries: 2,
       include_file_paths: nil,
-      allow_missing_columns: false,
+      missing_columns: "raise",
+      allow_missing_columns: nil,
       extra_columns: "raise",
       cast_options: nil,
       _column_mapping: nil,
-      _deletion_files: nil
+      _default_values: nil,
+      _deletion_files: nil,
+      _table_statistics: nil,
+      _row_count: nil
     )
-      missing_columns = allow_missing_columns ? "insert" : "raise"
+      if !schema.nil?
+        msg = "the `schema` parameter of `scan_parquet` is considered unstable."
+        Utils.issue_unstable_warning(msg)
+      end
 
-      if Utils.pathlike?(source)
-        source = Utils.normalize_filepath(source, check_not_directory: false)
-      elsif Utils.is_path_or_str_sequence(source)
-        source = source.map { |s| Utils.normalize_filepath(s, check_not_directory: false) }
+      if !hive_schema.nil?
+        msg = "the `hive_schema` parameter of `scan_parquet` is considered unstable."
+        Utils.issue_unstable_warning(msg)
       end
 
-      if credential_provider
-        raise Todo
+      if !cast_options.nil?
+        msg = "The `cast_options` parameter of `scan_parquet` is considered unstable."
+        Utils.issue_unstable_warning(msg)
       end
 
-      if source.is_a?(::Array)
-        sources = source
-        source = nil
-      else
-        sources = [source]
+      if !hidden_file_prefix.nil?
+        msg = "The `hidden_file_prefix` parameter of `scan_parquet` is considered unstable."
+        Utils.issue_unstable_warning(msg)
       end
 
-      if storage_options
-        storage_options = storage_options.map { |k, v| [k.to_s, v.to_s] }
-      else
-        storage_options = nil
+      if !allow_missing_columns.nil?
+        Utils.issue_deprecation_warning(
+          "the parameter `allow_missing_columns` for `scan_parquet` is deprecated. " +
+          "Use the parameter `missing_columns` instead and pass one of " +
+          "`('insert', 'raise')`."
+        )
+
+        missing_columns = allow_missing_columns ? "insert" : "raise"
       end
 
-      row_index_name = row_count_name
-      row_index_offset = row_count_offset
+      sources = get_sources(source)
+
+      credential_provider_builder =
+        _init_credential_provider_builder(
+          credential_provider,
+          sources,
+          storage_options,
+          "scan_parquet"
+        )
 
       rblf =
         RbLazyFrame.new_from_parquet(
@@ -276,16 +335,20 @@ module Polars
             missing_columns: missing_columns,
             include_file_paths: include_file_paths,
             glob: glob,
+            hidden_file_prefix: hidden_file_prefix.is_a?(::String) ? [hidden_file_prefix] : hidden_file_prefix,
             hive_partitioning: hive_partitioning,
             hive_schema: hive_schema,
             try_parse_hive_dates: try_parse_hive_dates,
             rechunk: rechunk,
             cache: cache,
-            storage_options: storage_options,
-            # credential_provider: credential_provider_builder,
+            storage_options: storage_options ? storage_options.map { |k, v| [k.to_s, v.to_s] } : nil,
+            credential_provider: credential_provider_builder,
             retries: retries,
+            column_mapping: _column_mapping,
+            default_values: _default_values,
             deletion_files: _deletion_files,
-            column_mapping: _column_mapping
+            table_statistics: _table_statistics,
+            row_count: _row_count
           ),
           parallel,
           low_memory,