polars-df 0.15.0 → 0.26.0

data/lib/polars/io/database.rb

@@ -51,16 +51,32 @@ module Polars
           when :decimal
             Decimal
           when :float
+            # TODO uncomment in future release
+            # if column_type.limit && column_type.limit <= 24
+            #   Float32
+            # else
+            #   Float64
+            # end
             Float64
           when :integer
+            # TODO uncomment in future release
+            # case column_type.limit
+            # when 1
+            #   Int8
+            # when 2
+            #   Int16
+            # when 4
+            #   Int32
+            # else
+            #   Int64
+            # end
             Int64
           when :string, :text
             String
           when :time
             Time
-          # TODO fix issue with null
-          # when :json, :jsonb
-          #   Struct
+          when :json, :jsonb
+            Struct
           end
 
         schema_overrides[k] ||= polars_type if polars_type
@@ -68,6 +84,5 @@ module Polars
 
       DataFrame.new(data, schema_overrides: schema_overrides)
     end
-    alias_method :read_sql, :read_database
   end
 end

data/lib/polars/io/delta.rb

@@ -0,0 +1,134 @@
+module Polars
+  module IO
+    # Reads into a DataFrame from a Delta lake table.
+    #
+    # @param source [Object]
+    #   DeltaTable or a Path or URI to the root of the Delta lake table.
+    # @param version [Object]
+    #   Numerical version or timestamp version of the Delta lake table.
+    # @param columns [Array]
+    #   Columns to select. Accepts a list of column names.
+    # @param rechunk [Boolean]
+    #   Make sure that all columns are contiguous in memory by
+    #   aggregating the chunks into a single array.
+    # @param storage_options [Hash]
+    #   Extra options for the storage backends supported by `deltalake-rb`.
+    # @param delta_table_options [Hash]
+    #   Additional keyword arguments while reading a Delta lake Table.
+    #
+    # @return [DataFrame]
+    def read_delta(
+      source,
+      version: nil,
+      columns: nil,
+      rechunk: nil,
+      storage_options: nil,
+      delta_table_options: nil
+    )
+      df =
+        scan_delta(
+          source,
+          version: version,
+          storage_options: storage_options,
+          delta_table_options: delta_table_options,
+          rechunk: rechunk
+        )
+
+      if !columns.nil?
+        df = df.select(columns)
+      end
+      df.collect
+    end
+
+    # Lazily read from a Delta lake table.
+    #
+    # @param source [Object]
+    #   DeltaTable or a Path or URI to the root of the Delta lake table.
+    # @param version [Object]
+    #   Numerical version or timestamp version of the Delta lake table.
+    # @param storage_options [Hash]
+    #   Extra options for the storage backends supported by `deltalake-rb`.
+    # @param delta_table_options [Hash]
+    #   Additional keyword arguments while reading a Delta lake Table.
+    # @param rechunk [Boolean]
+    #   Make sure that all columns are contiguous in memory by
+    #   aggregating the chunks into a single array.
+    #
+    # @return [LazyFrame]
+    def scan_delta(
+      source,
+      version: nil,
+      storage_options: nil,
+      delta_table_options: nil,
+      rechunk: nil
+    )
+      dl_tbl =
+        _get_delta_lake_table(
+          source,
+          version: version,
+          storage_options: storage_options,
+          delta_table_options: delta_table_options
+        )
+
+      dl_tbl.to_polars(eager: false, rechunk: rechunk || false)
+    end
+
+    private
+
+    def _resolve_delta_lake_uri(table_uri, strict: true)
+      require "uri"
+
+      parsed_result = URI(table_uri)
+
+      resolved_uri =
+        if parsed_result.scheme == ""
+          Utils.normalize_filepath(table_uri)
+        else
+          table_uri
+        end
+
+      resolved_uri
+    end
+
+    def _get_delta_lake_table(
+      table_path,
+      version: nil,
+      storage_options: nil,
+      delta_table_options: nil
+    )
+      _check_if_delta_available
+
+      if table_path.is_a?(DeltaLake::Table)
+        return table_path
+      end
+      delta_table_options ||= {}
+      resolved_uri = _resolve_delta_lake_uri(table_path)
+      if !version.is_a?(::String) && !version.is_a?(::Time)
+        dl_tbl =
+          DeltaLake::Table.new(
+            resolved_uri,
+            version: version,
+            storage_options: storage_options,
+            **delta_table_options
+          )
+      else
+        dl_tbl =
+          DeltaLake::Table.new(
+            resolved_uri,
+            storage_options: storage_options,
+            **delta_table_options
+          )
+        dl_tbl.load_as_version(version)
+      end
+
+      dl_tbl = DeltaLake::Table.new(table_path)
+      dl_tbl
+    end
+
+    def _check_if_delta_available
+      if !defined?(DeltaLake)
+        raise Error, "Delta Lake not available"
+      end
+    end
+  end
+end

data/lib/polars/io/iceberg.rb

@@ -0,0 +1,34 @@
+module Polars
+  module IO
+    # Lazily read from an Apache Iceberg table.
+    #
+    # @param source [Object]
+    #   A Iceberg Ruby table, or a direct path to the metadata.
+    # @param snapshot_id [Integer]
+    #   The snapshot ID to scan from.
+    # @param storage_options [Hash]
+    #   Extra options for the storage backends.
+    #
+    # @return [LazyFrame]
+    def scan_iceberg(
+      source,
+      snapshot_id: nil,
+      storage_options: nil
+    )
+      require "iceberg"
+
+      unless source.is_a?(Iceberg::Table)
+        raise Todo
+      end
+
+      dataset =
+        IcebergDataset.new(
+          source,
+          snapshot_id:,
+          storage_options:
+        )
+
+      dataset.to_lazyframe
+    end
+  end
+end

data/lib/polars/io/ipc.rb

@@ -15,10 +15,10 @@ module Polars
     #   Only uncompressed IPC files can be memory mapped.
     # @param storage_options [Hash]
     #   Extra options that make sense for a particular storage connection.
-    # @param row_count_name [String]
+    # @param row_index_name [String]
     #   If not nil, this will insert a row count column with give name into the
     #   DataFrame.
-    # @param row_count_offset [Integer]
+    # @param row_index_offset [Integer]
     #   Offset to start the row_count column (only use if the name is set).
     # @param rechunk [Boolean]
     #   Make sure that all data is contiguous.
@@ -30,8 +30,8 @@ module Polars
       n_rows: nil,
       memory_map: true,
       storage_options: nil,
-      row_count_name: nil,
-      row_count_offset: 0,
+      row_index_name: nil,
+      row_index_offset: 0,
       rechunk: true
     )
       storage_options ||= {}
@@ -40,8 +40,8 @@ module Polars
           data,
           columns: columns,
           n_rows: n_rows,
-          row_count_name: row_count_name,
-          row_count_offset: row_count_offset,
+          row_index_name: row_index_name,
+          row_index_offset: row_index_offset,
           rechunk: rechunk,
           memory_map: memory_map
         )
@@ -53,8 +53,8 @@ module Polars
       file,
       columns: nil,
       n_rows: nil,
-      row_count_name: nil,
-      row_count_offset: 0,
+      row_index_name: nil,
+      row_index_offset: 0,
       rechunk: true,
       memory_map: true
     )
@@ -76,7 +76,7 @@ module Polars
           columns,
           projection,
           n_rows,
-          Utils.parse_row_index_args(row_count_name, row_count_offset),
+          Utils.parse_row_index_args(row_index_name, row_index_offset),
           memory_map
         )
       Utils.wrap_df(rbdf)
@@ -182,13 +182,25 @@ module Polars
     #   Cache the result after reading.
     # @param rechunk [Boolean]
     #   Reallocate to contiguous memory when all chunks/ files are parsed.
-    # @param row_count_name [String]
+    # @param row_index_name [String]
     #   If not nil, this will insert a row count column with give name into the
     #   DataFrame.
-    # @param row_count_offset [Integer]
+    # @param row_index_offset [Integer]
     #   Offset to start the row_count column (only use if the name is set).
+    # @param glob [Boolean]
+    #   Expand path given via globbing rules.
     # @param storage_options [Hash]
     #   Extra options that make sense for a particular storage connection.
+    # @param credential_provider [Object]
+    #   Provide a function that can be called to provide cloud storage
+    #   credentials. The function is expected to return a hash of
+    #   credential keys along with an optional credential expiry time.
+    # @param retries [Integer]
+    #   Number of retries if accessing a cloud instance fails.
+    # @param file_cache_ttl [Integer]
+    #   Amount of time to keep downloaded cloud files since their last access time,
+    #   in seconds. Uses the `POLARS_FILE_CACHE_TTL` environment variable
+    #   (which defaults to 1 hour) if not given.
     # @param hive_partitioning [Boolean]
     #   Infer statistics and schema from Hive partitioned URL and use them
     #   to prune reads. This is unset by default (i.e. `nil`), meaning it is
@@ -206,70 +218,58 @@ module Polars
     def scan_ipc(
       source,
       n_rows: nil,
-      cache: true,
-      rechunk: true,
-      row_count_name: nil,
-      row_count_offset: 0,
+      cache: nil,
+      rechunk: false,
+      row_index_name: nil,
+      row_index_offset: 0,
+      glob: true,
       storage_options: nil,
+      credential_provider: "auto",
+      retries: nil,
+      file_cache_ttl: nil,
       hive_partitioning: nil,
       hive_schema: nil,
       try_parse_hive_dates: true,
-      include_file_paths: nil
+      include_file_paths: nil,
+      _record_batch_statistics: false
     )
-      _scan_ipc_impl(
-        source,
-        n_rows: n_rows,
-        cache: cache,
-        rechunk: rechunk,
-        row_count_name: row_count_name,
-        row_count_offset: row_count_offset,
-        storage_options: storage_options,
-        hive_partitioning: hive_partitioning,
-        hive_schema: hive_schema,
-        try_parse_hive_dates: try_parse_hive_dates,
-        include_file_paths: include_file_paths
-      )
-    end
+      sources = get_sources(source)
 
-    # @private
-    def _scan_ipc_impl(
-      source,
-      n_rows: nil,
-      cache: true,
-      rechunk: true,
-      row_count_name: nil,
-      row_count_offset: 0,
-      storage_options: nil,
-      hive_partitioning: nil,
-      hive_schema: nil,
-      try_parse_hive_dates: true,
-      include_file_paths: nil
-    )
-      sources = []
-      if Utils.pathlike?(source)
-        source = Utils.normalize_filepath(source)
-      elsif source.is_a?(::Array)
-        if Utils.is_path_or_str_sequence(source)
-          sources = source.map { |s| Utils.normalize_filepath(s) }
-        else
-          sources = source
-        end
+      if !retries.nil?
+        msg = "the `retries` parameter was deprecated in 0.25.0; specify 'max_retries' in `storage_options` instead."
+        Utils.issue_deprecation_warning(msg)
+        storage_options = storage_options || {}
+        storage_options["max_retries"] = retries
+      end
 
-        source = nil
+      if !file_cache_ttl.nil?
+        msg = "the `file_cache_ttl` parameter was deprecated in 0.25.0; specify 'file_cache_ttl' in `storage_options` instead."
+        Utils.issue_deprecation_warning(msg)
+        storage_options = storage_options || {}
+        storage_options["file_cache_ttl"] = file_cache_ttl
       end
 
+      credential_provider_builder = _init_credential_provider_builder(
+        credential_provider, sources, storage_options, "scan_parquet"
+      )
+
       rblf =
         RbLazyFrame.new_from_ipc(
-          source,
           sources,
-          n_rows,
-          cache,
-          rechunk,
-          Utils.parse_row_index_args(row_count_name, row_count_offset),
-          hive_partitioning,
-          hive_schema,
-          try_parse_hive_dates,
-          include_file_paths
+          _record_batch_statistics,
+          ScanOptions.new(
+            row_index: !row_index_name.nil? ? [row_index_name, row_index_offset] : nil,
+            pre_slice: !n_rows.nil? ? [0, n_rows] : nil,
+            include_file_paths: include_file_paths,
+            glob: glob,
+            hive_partitioning: hive_partitioning,
+            hive_schema: hive_schema,
+            try_parse_hive_dates: try_parse_hive_dates,
+            rechunk: rechunk,
+            cache: cache,
+            storage_options: storage_options,
+            credential_provider: credential_provider_builder
+          )
         )
       Utils.wrap_ldf(rblf)
     end

data/lib/polars/io/json.rb

@@ -4,6 +4,22 @@ module Polars
     #
     # @param source [Object]
     #   Path to a file or a file-like object.
+    # @param schema [Object]
+    #   The DataFrame schema may be declared in several ways:
+    #
+    #   * As a hash of \\\\{name:type} pairs; if type is nil, it will be auto-inferred.
+    #   * As an array of column names; in this case types are automatically inferred.
+    #   * As an array of [name,type] pairs; this is equivalent to the hash form.
+    #
+    #   If you supply an array of column names that does not match the names in the
+    #   underlying data, the names given here will overwrite them. The number
+    #   of names given in the schema should match the underlying data dimensions.
+    # @param schema_overrides [Hash]
+    #   Support type specification or override of one or more columns; note that
+    #   any dtypes inferred from the schema param will be overridden.
+    # @param infer_schema_length [Integer]
+    #   The maximum number of rows to scan for schema inference.
+    #   If set to `nil`, the full data may be scanned *(this is slow)*.
     #
     # @return [DataFrame]
     def read_json(

data/lib/polars/io/lines.rb

@@ -0,0 +1,172 @@
+module Polars
+  module IO
+    # Read lines into a string column from a file.
+    #
+    # @note
+    #   This functionality is considered **unstable**. It may be changed
+    #   at any point without it being considered a breaking change.
+    #
+    # @param source [Object]
+    #   Path(s) to a file or directory
+    #   When needing to authenticate for scanning cloud locations, see the
+    #   `storage_options` parameter.
+    # @param name [String]
+    #   Name to use for the output column.
+    # @param n_rows [Integer]
+    #   Stop reading from parquet file after reading `n_rows`.
+    # @param row_index_name [String]
+    #   If not nil, this will insert a row index column with the given name into the
+    #   DataFrame
+    # @param row_index_offset [Integer]
+    #   Offset to start the row index column (only used if the name is set)
+    # @param glob [Boolean]
+    #   Expand path given via globbing rules.
+    # @param storage_options [Hash]
+    #   Options that indicate how to connect to a cloud provider.
+    #
+    #   The cloud providers currently supported are AWS, GCP, and Azure.
+    #   See supported keys here:
+    #
+    #   * [aws](https://docs.rs/object_store/latest/object_store/aws/enum.AmazonS3ConfigKey.html)
+    #   * [gcp](https://docs.rs/object_store/latest/object_store/gcp/enum.GoogleConfigKey.html)
+    #   * [azure](https://docs.rs/object_store/latest/object_store/azure/enum.AzureConfigKey.html)
+    #   * Hugging Face (`hf://`): Accepts an API key under the `token` parameter: \
+    #     `{'token': '...'}`, or by setting the `HF_TOKEN` environment variable.
+    #
+    #   If `storage_options` is not provided, Polars will try to infer the information
+    #   from environment variables.
+    # @param credential_provider [Object]
+    #   Provide a function that can be called to provide cloud storage
+    #   credentials. The function is expected to return a dictionary of
+    #   credential keys along with an optional credential expiry time.
+    # @param include_file_paths [String]
+    #   Include the path of the source file(s) as a column with this name.
+    #
+    # @return [DataFrame]
+    #
+    # @example
+    #   Polars.read_lines(StringIO.new("Hello\nworld"))
+    #   # =>
+    #   # shape: (2, 1)
+    #   # ┌───────┐
+    #   # │ line  │
+    #   # │ ---   │
+    #   # │ str   │
+    #   # ╞═══════╡
+    #   # │ Hello │
+    #   # │ world │
+    #   # └───────┘
+    def read_lines(
+      source,
+      name: "line",
+      n_rows: nil,
+      row_index_name: nil,
+      row_index_offset: 0,
+      glob: true,
+      storage_options: nil,
+      credential_provider: "auto",
+      include_file_paths: nil
+    )
+      scan_lines(
+        source,
+        name: name,
+        n_rows: n_rows,
+        row_index_name: row_index_name,
+        row_index_offset: row_index_offset,
+        glob: glob,
+        storage_options: storage_options,
+        credential_provider: credential_provider,
+        include_file_paths: include_file_paths
+      ).collect
+    end
+
+    # Construct a LazyFrame which scans lines into a string column from a file.
+    #
+    # @note
+    #   This functionality is considered **unstable**. It may be changed
+    #   at any point without it being considered a breaking change.
+    #
+    # @param source [Object]
+    #   Path(s) to a file or directory
+    #   When needing to authenticate for scanning cloud locations, see the
+    #   `storage_options` parameter.
+    # @param name [String]
+    #   Name to use for the output column.
+    # @param n_rows [Integer]
+    #   Stop reading from parquet file after reading `n_rows`.
+    # @param row_index_name [String]
+    #   If not nil, this will insert a row index column with the given name into the
+    #   DataFrame
+    # @param row_index_offset
+    #   Offset to start the row index column (only used if the name is set)
+    # @param glob [Boolean]
+    #   Expand path given via globbing rules.
+    # @param storage_options [Hash]
+    #   Options that indicate how to connect to a cloud provider.
+    #
+    #   The cloud providers currently supported are AWS, GCP, and Azure.
+    #   See supported keys here:
+    #
+    #   * [aws](https://docs.rs/object_store/latest/object_store/aws/enum.AmazonS3ConfigKey.html)
+    #   * [gcp](https://docs.rs/object_store/latest/object_store/gcp/enum.GoogleConfigKey.html)
+    #   * [azure](https://docs.rs/object_store/latest/object_store/azure/enum.AzureConfigKey.html)
+    #   * Hugging Face (`hf://`): Accepts an API key under the `token` parameter: \
+    #     `{'token': '...'}`, or by setting the `HF_TOKEN` environment variable.
+    #
+    #   If `storage_options` is not provided, Polars will try to infer the information
+    #   from environment variables.
+    # @param credential_provider [Object]
+    #   Provide a function that can be called to provide cloud storage
+    #   credentials. The function is expected to return a dictionary of
+    #   credential keys along with an optional credential expiry time.
+    # @param include_file_paths [String]
+    #   Include the path of the source file(s) as a column with this name.
+    #
+    # @return [LazyFrame]
+    #
+    # @example
+    #   Polars.scan_lines(StringIO.new("Hello\nworld")).collect
+    #   # =>
+    #   # shape: (2, 1)
+    #   # ┌───────┐
+    #   # │ line  │
+    #   # │ ---   │
+    #   # │ str   │
+    #   # ╞═══════╡
+    #   # │ Hello │
+    #   # │ world │
+    #   # └───────┘
+    def scan_lines(
+      source,
+      name: "line",
+      n_rows: nil,
+      row_index_name: nil,
+      row_index_offset: 0,
+      glob: true,
+      storage_options: nil,
+      credential_provider: "auto",
+      include_file_paths: nil
+    )
+      sources = get_sources(source)
+
+      credential_provider_builder = _init_credential_provider_builder(
+        credential_provider, sources, storage_options, "scan_lines"
+      )
+
+      rblf = RbLazyFrame.new_from_scan_lines(
+        sources,
+        ScanOptions.new(
+          row_index: !row_index_name.nil? ? [row_index_name, row_index_offset] : nil,
+          pre_slice: !n_rows.nil? ? [0, n_rows] : nil,
+          include_file_paths: include_file_paths,
+          glob: glob,
+          storage_options: storage_options,
+          credential_provider: credential_provider_builder
+        ),
+        name
+      )
+
+      Utils.wrap_ldf(rblf)
+    end
+  end
+end