polars-df 0.15.0 → 0.26.0

data/lib/polars/io/ndjson.rb

@@ -2,28 +2,106 @@ module Polars
   module IO
     # Read into a DataFrame from a newline delimited JSON file.
     #
-    # @param source [Object]
-    #   Path to a file or a file-like object.
+    # @param source [String]
+    #   Path to a file.
+    # @param schema [Object]
+    #   The DataFrame schema may be declared in several ways:
+    #
+    #   * As a dict of \\\\{name:type} pairs; if type is nil, it will be auto-inferred.
+    #   * As a list of column names; in this case types are automatically inferred.
+    #   * As a list of (name,type) pairs; this is equivalent to the hash form.
+    #
+    #   If you supply a list of column names that does not match the names in the
+    #   underlying data, the names given here will overwrite them. The number
+    #   of names given in the schema should match the underlying data dimensions.
+    # @param schema_overrides [Hash]
+    #   Support type specification or override of one or more columns; note that
+    #   any dtypes inferred from the schema param will be overridden.
+    # @param infer_schema_length [Integer]
+    #   Infer the schema length from the first `infer_schema_length` rows.
+    # @param batch_size [Integer]
+    #   Number of rows to read in each batch.
+    # @param n_rows [Integer]
+    #   Stop reading from JSON file after reading `n_rows`.
+    # @param low_memory [Boolean]
+    #   Reduce memory pressure at the expense of performance.
+    # @param rechunk [Boolean]
+    #   Reallocate to contiguous memory when all chunks/ files are parsed.
+    # @param row_index_name [String]
+    #   If not nil, this will insert a row count column with give name into the
+    #   DataFrame.
+    # @param row_index_offset [Integer]
+    #   Offset to start the row_count column (only use if the name is set).
+    # @param ignore_errors [Boolean]
+    #   Return `Null` if parsing fails because of schema mismatches.
+    # @param storage_options [Hash]
+    #   Options that indicate how to connect to a cloud provider.
+    #
+    #   The cloud providers currently supported are AWS, GCP, and Azure.
+    #   See supported keys here:
+    #
+    #   * [aws](https://docs.rs/object_store/latest/object_store/aws/enum.AmazonS3ConfigKey.html)
+    #   * [gcp](https://docs.rs/object_store/latest/object_store/gcp/enum.GoogleConfigKey.html)
+    #   * [azure](https://docs.rs/object_store/latest/object_store/azure/enum.AzureConfigKey.html)
+    #   * Hugging Face (`hf://`): Accepts an API key under the `token` parameter: \
+    #     `{'token': '...'}`, or by setting the `HF_TOKEN` environment variable.
+    #
+    #   If `storage_options` is not provided, Polars will try to infer the information
+    #   from environment variables.
+    # @param credential_provider [Object]
+    #   Provide a function that can be called to provide cloud storage
+    #   credentials. The function is expected to return a hash of
+    #   credential keys along with an optional credential expiry time.
+    # @param retries [Integer]
+    #   Number of retries if accessing a cloud instance fails.
+    # @param file_cache_ttl [Integer]
+    #   Amount of time to keep downloaded cloud files since their last access time,
+    #   in seconds. Uses the `POLARS_FILE_CACHE_TTL` environment variable
+    #   (which defaults to 1 hour) if not given.
+    # @param include_file_paths [String]
+    #   Include the path of the source file(s) as a column with this name.
     #
     # @return [DataFrame]
     def read_ndjson(
       source,
       schema: nil,
       schema_overrides: nil,
-      ignore_errors: false
+      infer_schema_length: N_INFER_DEFAULT,
+      batch_size: 1024,
+      n_rows: nil,
+      low_memory: false,
+      rechunk: false,
+      row_index_name: nil,
+      row_index_offset: 0,
+      ignore_errors: false,
+      storage_options: nil,
+      credential_provider: "auto",
+      retries: nil,
+      file_cache_ttl: nil,
+      include_file_paths: nil
     )
-      if Utils.pathlike?(source)
-        source = Utils.normalize_filepath(source)
-      end
+      credential_provider_builder = _init_credential_provider_builder(
+        credential_provider, source, storage_options, "read_ndjson"
+      )
 
-      rbdf =
-        RbDataFrame.read_ndjson(
-          source,
-          ignore_errors,
-          schema,
-          schema_overrides
-        )
-      Utils.wrap_df(rbdf)
+      scan_ndjson(
+        source,
+        schema: schema,
+        schema_overrides: schema_overrides,
+        infer_schema_length: infer_schema_length,
+        batch_size: batch_size,
+        n_rows: n_rows,
+        low_memory: low_memory,
+        rechunk: rechunk,
+        row_index_name: row_index_name,
+        row_index_offset: row_index_offset,
+        ignore_errors: ignore_errors,
+        include_file_paths: include_file_paths,
+        retries: retries,
+        storage_options: storage_options,
+        credential_provider: credential_provider_builder,
+        file_cache_ttl: file_cache_ttl,
+      ).collect
     end
 
     # Lazily read from a newline delimited JSON file.
@@ -33,6 +111,19 @@ module Polars
     #
     # @param source [String]
     #   Path to a file.
+    # @param schema [Object]
+    #   The DataFrame schema may be declared in several ways:
+    #
+    #   * As a dict of \\\\{name:type} pairs; if type is nil, it will be auto-inferred.
+    #   * As a list of column names; in this case types are automatically inferred.
+    #   * As a list of (name,type) pairs; this is equivalent to the hash form.
+    #
+    #   If you supply a list of column names that does not match the names in the
+    #   underlying data, the names given here will overwrite them. The number
+    #   of names given in the schema should match the underlying data dimensions.
+    # @param schema_overrides [Hash]
+    #   Support type specification or override of one or more columns; note that
+    #   any dtypes inferred from the schema param will be overridden.
     # @param infer_schema_length [Integer]
     #   Infer the schema length from the first `infer_schema_length` rows.
     # @param batch_size [Integer]
@@ -43,22 +134,58 @@ module Polars
     #   Reduce memory pressure at the expense of performance.
     # @param rechunk [Boolean]
     #   Reallocate to contiguous memory when all chunks/ files are parsed.
-    # @param row_count_name [String]
+    # @param row_index_name [String]
     #   If not nil, this will insert a row count column with give name into the
     #   DataFrame.
-    # @param row_count_offset [Integer]
+    # @param row_index_offset [Integer]
     #   Offset to start the row_count column (only use if the name is set).
+    # @param ignore_errors [Boolean]
+    #   Return `Null` if parsing fails because of schema mismatches.
+    # @param storage_options [Hash]
+    #   Options that indicate how to connect to a cloud provider.
+    #
+    #   The cloud providers currently supported are AWS, GCP, and Azure.
+    #   See supported keys here:
+    #
+    #   * [aws](https://docs.rs/object_store/latest/object_store/aws/enum.AmazonS3ConfigKey.html)
+    #   * [gcp](https://docs.rs/object_store/latest/object_store/gcp/enum.GoogleConfigKey.html)
+    #   * [azure](https://docs.rs/object_store/latest/object_store/azure/enum.AzureConfigKey.html)
+    #   * Hugging Face (`hf://`): Accepts an API key under the `token` parameter: \
+    #     `{'token': '...'}`, or by setting the `HF_TOKEN` environment variable.
+    #
+    #   If `storage_options` is not provided, Polars will try to infer the information
+    #   from environment variables.
+    # @param credential_provider [Object]
+    #   Provide a function that can be called to provide cloud storage
+    #   credentials. The function is expected to return a hash of
+    #   credential keys along with an optional credential expiry time.
+    # @param retries [Integer]
+    #   Number of retries if accessing a cloud instance fails.
+    # @param file_cache_ttl [Integer]
+    #   Amount of time to keep downloaded cloud files since their last access time,
+    #   in seconds. Uses the `POLARS_FILE_CACHE_TTL` environment variable
+    #   (which defaults to 1 hour) if not given.
+    # @param include_file_paths [String]
+    #   Include the path of the source file(s) as a column with this name.
     #
     # @return [LazyFrame]
     def scan_ndjson(
       source,
+      schema: nil,
+      schema_overrides: nil,
       infer_schema_length: N_INFER_DEFAULT,
       batch_size: 1024,
       n_rows: nil,
       low_memory: false,
-      rechunk: true,
-      row_count_name: nil,
-      row_count_offset: 0
+      rechunk: false,
+      row_index_name: nil,
+      row_index_offset: 0,
+      ignore_errors: false,
+      storage_options: nil,
+      credential_provider: "auto",
+      retries: nil,
+      file_cache_ttl: nil,
+      include_file_paths: nil
     )
       sources = []
       if Utils.pathlike?(source)
@@ -73,16 +200,45 @@ module Polars
         source = nil
       end
 
+      if infer_schema_length == 0
+        msg = "'infer_schema_length' should be positive"
+        raise ArgumentError, msg
+      end
+
+      if !retries.nil?
+        msg = "the `retries` parameter was deprecated in 0.25.0; specify 'max_retries' in `storage_options` instead."
+        Utils.issue_deprecation_warning(msg)
+        storage_options = storage_options || {}
+        storage_options["max_retries"] = retries
+      end
+
+      if !file_cache_ttl.nil?
+        msg = "the `file_cache_ttl` parameter was deprecated in 0.25.0; specify 'file_cache_ttl' in `storage_options` instead."
+        Utils.issue_deprecation_warning(msg)
+        storage_options = storage_options || {}
+        storage_options["file_cache_ttl"] = file_cache_ttl
+      end
+
+      credential_provider_builder = _init_credential_provider_builder(
+        credential_provider, source, storage_options, "scan_ndjson"
+      )
+
       rblf =
         RbLazyFrame.new_from_ndjson(
           source,
           sources,
           infer_schema_length,
+          schema,
+          schema_overrides,
           batch_size,
           n_rows,
           low_memory,
           rechunk,
-          Utils.parse_row_index_args(row_count_name, row_count_offset)
+          Utils.parse_row_index_args(row_index_name, row_index_offset),
+          ignore_errors,
+          include_file_paths,
+          storage_options,
+          credential_provider_builder
         )
       Utils.wrap_ldf(rblf)
     end

data/lib/polars/io/parquet.rb

@@ -9,10 +9,10 @@ module Polars
     #   of column names.
     # @param n_rows [Integer]
     #   Stop reading from parquet file after reading `n_rows`.
-    # @param row_count_name [String]
+    # @param row_index_name [String]
     #   If not nil, this will insert a row count column with give name into the
     #   DataFrame.
-    # @param row_count_offset [Integer]
+    # @param row_index_offset [Integer]
     #   Offset to start the row_count column (only use if the name is set).
     # @param parallel ["auto", "columns", "row_groups", "none"]
     #   This determines the direction of parallelism. 'auto' will try to determine the
@@ -43,20 +43,32 @@ module Polars
     #   Extra options that make sense for a particular storage connection.
     # @param credential_provider [Object]
     #   Provide a function that can be called to provide cloud storage
-    #   credentials. The function is expected to return a dictionary of
+    #   credentials. The function is expected to return a hash of
     #   credential keys along with an optional credential expiry time.
     # @param retries [Integer]
     #   Number of retries if accessing a cloud instance fails.
     # @param include_file_paths [String]
     #   Include the path of the source file(s) as a column with this name.
+    # @param missing_columns ['insert', 'raise']
+    #   Configuration for behavior when columns defined in the schema
+    #   are missing from the data:
+    #
+    #   * `insert`: Inserts the missing columns using NULLs as the row values.
+    #   * `raise`: Raises an error.
+    # @param allow_missing_columns [Boolean]
+    #   When reading a list of parquet files, if a column existing in the first
+    #   file cannot be found in subsequent files, the default behavior is to
+    #   raise an error. However, if `allow_missing_columns` is set to
+    #   `true`, a full-NULL column is returned instead of erroring for the files
+    #   that do not contain the column.
     #
     # @return [DataFrame]
     def read_parquet(
       source,
       columns: nil,
       n_rows: nil,
-      row_count_name: nil,
-      row_count_offset: 0,
+      row_index_name: nil,
+      row_index_offset: 0,
       parallel: "auto",
       use_statistics: true,
       hive_partitioning: nil,
@@ -67,17 +79,18 @@ module Polars
       rechunk: false,
       low_memory: false,
       storage_options: nil,
-      credential_provider: nil,
-      retries: 2,
+      credential_provider: "auto",
+      retries: nil,
       include_file_paths: nil,
-      allow_missing_columns: false
+      missing_columns: "raise",
+      allow_missing_columns: nil
     )
       lf =
         scan_parquet(
           source,
           n_rows: n_rows,
-          row_count_name: row_count_name,
-          row_count_offset: row_count_offset,
+          row_index_name: row_index_name,
+          row_index_offset: row_index_offset,
           parallel: parallel,
           use_statistics: use_statistics,
           hive_partitioning: hive_partitioning,
@@ -92,6 +105,7 @@ module Polars
           retries: retries,
           glob: glob,
           include_file_paths: include_file_paths,
+          missing_columns: missing_columns,
           allow_missing_columns: allow_missing_columns
         )
 
@@ -111,13 +125,59 @@ module Polars
     # @param source [Object]
     #   Path to a file or a file-like object.
     #
-    # @return [Hash]
+    # @return [Schema]
     def read_parquet_schema(source)
       if Utils.pathlike?(source)
         source = Utils.normalize_filepath(source)
       end
 
-      Plr.parquet_schema(source)
+      scan_parquet(source).collect_schema
+    end
+
+    # Get file-level custom metadata of a Parquet file without reading data.
+    #
+    # @note
+    #   This functionality is considered **experimental**. It may be removed or
+    #   changed at any point without it being considered a breaking change.
+    #
+    # @param source [Object]
+    #   Path to a file or a file-like object.
+    # @param storage_options [Hash]
+    #   Extra options that make sense for a particular storage connection.
+    # @param credential_provider [Object]
+    #   Provide a function that can be called to provide cloud storage
+    #   credentials. The function is expected to return a hash of
+    #   credential keys along with an optional credential expiry time.
+    # @param retries [Integer]
+    #   Number of retries if accessing a cloud instance fails.
+    #
+    # @return [Hash]
+    def read_parquet_metadata(
+      source,
+      storage_options: nil,
+      credential_provider: "auto",
+      retries: nil
+    )
+      if Utils.pathlike?(source)
+        source = Utils.normalize_filepath(source, check_not_directory: false)
+      end
+
+      if !retries.nil?
+        msg = "the `retries` parameter was deprecated in 0.25.0; specify 'max_retries' in `storage_options` instead."
+        Utils.issue_deprecation_warning(msg)
+        storage_options = storage_options || {}
+        storage_options["max_retries"] = retries
+      end
+
+      credential_provider_builder = _init_credential_provider_builder(
+        credential_provider, source, storage_options, "scan_parquet"
+      )
+
+      Plr.read_parquet_metadata(
+        source,
+        storage_options,
+        credential_provider_builder
+      )
     end
 
     # Lazily read from a parquet file or multiple files via glob patterns.
@@ -129,10 +189,10 @@ module Polars
     #   Path to a file or a file-like object.
     # @param n_rows [Integer]
     #   Stop reading from parquet file after reading `n_rows`.
-    # @param row_count_name [String]
+    # @param row_index_name [String]
     #   If not nil, this will insert a row count column with give name into the
     #   DataFrame.
-    # @param row_count_offset [Integer]
+    # @param row_index_offset [Integer]
     #   Offset to start the row_count column (only use if the name is set).
     # @param parallel ["auto", "columns", "row_groups", "none"]
     #   This determines the direction of parallelism. 'auto' will try to determine the
@@ -145,6 +205,8 @@ module Polars
     #   to prune reads.
     # @param glob [Boolean]
     #   Expand path given via globbing rules.
+    # @param hidden_file_prefix [Boolean]
+    #   Skip reading files whose names begin with the specified prefixes.
     # @param schema [Object]
     #   Specify the datatypes of the columns. The datatypes must match the
     #   datatypes in the file(s). If there are extra columns that are not in the
@@ -165,23 +227,44 @@ module Polars
     #   Extra options that make sense for a particular storage connection.
     # @param credential_provider [Object]
     #   Provide a function that can be called to provide cloud storage
-    #   credentials. The function is expected to return a dictionary of
+    #   credentials. The function is expected to return a hash of
     #   credential keys along with an optional credential expiry time.
     # @param retries [Integer]
     #   Number of retries if accessing a cloud instance fails.
     # @param include_file_paths [String]
     #   Include the path of the source file(s) as a column with this name.
+    # @param missing_columns ['insert', 'raise']
+    #   Configuration for behavior when columns defined in the schema
+    #   are missing from the data:
+    #
+    #   * `insert`: Inserts the missing columns using NULLs as the row values.
+    #   * `raise`: Raises an error.
+    # @param allow_missing_columns [Boolean]
+    #   When reading a list of parquet files, if a column existing in the first
+    #   file cannot be found in subsequent files, the default behavior is to
+    #   raise an error. However, if `allow_missing_columns` is set to
+    #   `true`, a full-NULL column is returned instead of erroring for the files
+    #   that do not contain the column.
+    # @param extra_columns ['ignore', 'raise']
+    #   Configuration for behavior when extra columns outside of the
+    #   defined schema are encountered in the data:
+    #     * `ignore`: Silently ignores.
+    #     * `raise`: Raises an error.
+    # @param cast_options [Object]
+    #   Configuration for column type-casting during scans. Useful for datasets
+    #   containing files that have differing schemas.
     #
     # @return [LazyFrame]
     def scan_parquet(
       source,
       n_rows: nil,
-      row_count_name: nil,
-      row_count_offset: 0,
+      row_index_name: nil,
+      row_index_offset: 0,
       parallel: "auto",
       use_statistics: true,
       hive_partitioning: nil,
       glob: true,
+      hidden_file_prefix: nil,
       schema: nil,
       hive_schema: nil,
       try_parse_hive_dates: true,
@@ -189,100 +272,95 @@ module Polars
       low_memory: false,
       cache: true,
       storage_options: nil,
-      credential_provider: nil,
-      retries: 2,
+      credential_provider: "auto",
+      retries: nil,
       include_file_paths: nil,
-      allow_missing_columns: false
+      missing_columns: "raise",
+      allow_missing_columns: nil,
+      extra_columns: "raise",
+      cast_options: nil,
+      _column_mapping: nil,
+      _default_values: nil,
+      _deletion_files: nil,
+      _table_statistics: nil,
+      _row_count: nil
     )
-      if Utils.pathlike?(source)
-        source = Utils.normalize_filepath(source, check_not_directory: false)
-      elsif Utils.is_path_or_str_sequence(source)
-        source = source.map { |s| Utils.normalize_filepath(s, check_not_directory: false) }
+      if !schema.nil?
+        msg = "the `schema` parameter of `scan_parquet` is considered unstable."
+        Utils.issue_unstable_warning(msg)
       end
 
-      if credential_provider
-        raise Todo
+      if !hive_schema.nil?
+        msg = "the `hive_schema` parameter of `scan_parquet` is considered unstable."
+        Utils.issue_unstable_warning(msg)
       end
 
-      _scan_parquet_impl(
-        source,
-        n_rows: n_rows,
-        cache: cache,
-        parallel: parallel,
-        rechunk: rechunk,
-        row_index_name: row_count_name,
-        row_index_offset: row_count_offset,
-        storage_options: storage_options,
-        credential_provider: credential_provider,
-        low_memory: low_memory,
-        use_statistics: use_statistics,
-        hive_partitioning: hive_partitioning,
-        schema: schema,
-        hive_schema: hive_schema,
-        try_parse_hive_dates: try_parse_hive_dates,
-        retries: retries,
-        glob: glob,
-        include_file_paths: include_file_paths,
-        allow_missing_columns: allow_missing_columns
-      )
-    end
+      if !cast_options.nil?
+        msg = "The `cast_options` parameter of `scan_parquet` is considered unstable."
+        Utils.issue_unstable_warning(msg)
+      end
 
-    # @private
-    def _scan_parquet_impl(
-      source,
-      n_rows: nil,
-      cache: true,
-      parallel: "auto",
-      rechunk: true,
-      row_index_name: nil,
-      row_index_offset: 0,
-      storage_options: nil,
-      credential_provider: nil,
-      low_memory: false,
-      use_statistics: true,
-      hive_partitioning: nil,
-      glob: true,
-      schema: nil,
-      hive_schema: nil,
-      try_parse_hive_dates: true,
-      retries: 2,
-      include_file_paths: nil,
-      allow_missing_columns: false
-    )
-      if source.is_a?(::Array)
-        sources = source
-        source = nil
-      else
-        sources = []
+      if !hidden_file_prefix.nil?
+        msg = "The `hidden_file_prefix` parameter of `scan_parquet` is considered unstable."
+        Utils.issue_unstable_warning(msg)
       end
 
-      if storage_options
-        storage_options = storage_options.map { |k, v| [k.to_s, v.to_s] }
-      else
-        storage_options = nil
+      if !allow_missing_columns.nil?
+        Utils.issue_deprecation_warning(
+          "the parameter `allow_missing_columns` for `scan_parquet` is deprecated. " +
+          "Use the parameter `missing_columns` instead and pass one of " +
+          "`('insert', 'raise')`."
+        )
+
+        missing_columns = allow_missing_columns ? "insert" : "raise"
       end
 
+      if !retries.nil?
+        msg = "the `retries` parameter was deprecated in 0.25.0; specify 'max_retries' in `storage_options` instead."
+        Utils.issue_deprecation_warning(msg)
+        storage_options = storage_options || {}
+        storage_options["max_retries"] = retries
+      end
+
+      sources = get_sources(source)
+
+      credential_provider_builder =
+        _init_credential_provider_builder(
+          credential_provider,
+          sources,
+          storage_options,
+          "scan_parquet"
+        )
+
       rblf =
         RbLazyFrame.new_from_parquet(
-          source,
           sources,
-          n_rows,
-          cache,
+          schema,
+          ScanOptions.new(
+            row_index: !row_index_name.nil? ? [row_index_name, row_index_offset] : nil,
+            pre_slice: !n_rows.nil? ? [0, n_rows] : nil,
+            cast_options: cast_options,
+            extra_columns: extra_columns,
+            missing_columns: missing_columns,
+            include_file_paths: include_file_paths,
+            glob: glob,
+            hidden_file_prefix: hidden_file_prefix.is_a?(::String) ? [hidden_file_prefix] : hidden_file_prefix,
+            hive_partitioning: hive_partitioning,
+            hive_schema: hive_schema,
+            try_parse_hive_dates: try_parse_hive_dates,
+            rechunk: rechunk,
+            cache: cache,
+            storage_options: storage_options,
+            credential_provider: credential_provider_builder,
+            column_mapping: _column_mapping,
+            default_values: _default_values,
+            deletion_files: _deletion_files,
+            table_statistics: _table_statistics,
+            row_count: _row_count
+          ),
           parallel,
-          rechunk,
-          Utils.parse_row_index_args(row_index_name, row_index_offset),
           low_memory,
-          storage_options,
-          credential_provider,
-          use_statistics,
-          hive_partitioning,
-          schema,
-          hive_schema,
-          try_parse_hive_dates,
-          retries,
-          glob,
-          include_file_paths,
-          allow_missing_columns
+          use_statistics
         )
       Utils.wrap_ldf(rblf)
     end

data/lib/polars/io/scan_options.rb

@@ -0,0 +1,55 @@
+module Polars
+  module IO
+    # @private
+    class ScanOptions
+      attr_reader :row_index, :pre_slice, :cast_options, :extra_columns, :missing_columns,
+        :include_file_paths, :glob, :hidden_file_prefix, :hive_partitioning, :hive_schema, :try_parse_hive_dates,
+        :rechunk, :cache, :storage_options, :credential_provider, :retries, :column_mapping,
+        :default_values, :deletion_files, :table_statistics, :row_count
+
+      def initialize(
+        row_index: nil,
+        pre_slice: nil,
+        cast_options: nil,
+        extra_columns: "raise",
+        missing_columns: "raise",
+        include_file_paths: nil,
+        glob: true,
+        hidden_file_prefix: nil,
+        hive_partitioning: nil,
+        hive_schema: nil,
+        try_parse_hive_dates: true,
+        rechunk: false,
+        cache: true,
+        storage_options: nil,
+        credential_provider: nil,
+        column_mapping: nil,
+        default_values: nil,
+        deletion_files: nil,
+        table_statistics: nil,
+        row_count: nil
+      )
+        @row_index = row_index
+        @pre_slice = pre_slice
+        @cast_options = cast_options
+        @extra_columns = extra_columns
+        @missing_columns = missing_columns
+        @include_file_paths = include_file_paths
+        @glob = glob
+        @hidden_file_prefix = hidden_file_prefix
+        @hive_partitioning = hive_partitioning
+        @hive_schema = hive_schema
+        @try_parse_hive_dates = try_parse_hive_dates
+        @rechunk = rechunk
+        @cache = cache
+        @storage_options = storage_options
+        @credential_provider = credential_provider
+        @column_mapping = column_mapping
+        @default_values = default_values
+        @deletion_files = deletion_files
+        @table_statistics = table_statistics
+        @row_count = row_count
+      end
+    end
+  end
+end