polars-df 0.23.0 → 0.24.0

data/lib/polars/utils/construction/series.rb

@@ -0,0 +1,364 @@
+module Polars
+  module Utils
+    def self.sequence_to_rbseries(name, values, dtype: nil, strict: true, nan_to_null: false)
+      ruby_dtype = nil
+
+      if values.is_a?(Range)
+        if values.begin.is_a?(::String)
+          values = values.to_a
+        else
+          return range_to_series(name, values, dtype: dtype)._s
+        end
+      end
+
+      if values.length == 0 && dtype.nil?
+        dtype = Null
+      elsif [List, Array].include?(dtype)
+        ruby_dtype = ::Array
+      end
+
+      rb_temporal_types = [::Date, ::DateTime, ::Time]
+      rb_temporal_types << ActiveSupport::TimeWithZone if defined?(ActiveSupport::TimeWithZone)
+
+      value = get_first_non_none(values)
+      if !value.nil?
+        if value.is_a?(Hash) && dtype != Object
+          return DataFrame.new(values).to_struct(name)._s
+        end
+      end
+
+      if !dtype.nil? && is_polars_dtype(dtype) && !dtype.nested? && dtype != Unknown && ruby_dtype.nil?
+        constructor = polars_type_to_constructor(dtype)
+        rbseries = _construct_series_with_fallbacks(
+          constructor, name, values, dtype, strict: strict
+        )
+
+        if [Date, Datetime, Duration, Time, Boolean, Categorical, Enum].include?(dtype) || dtype.is_a?(Decimal) || dtype.is_a?(Categorical)
+          if rbseries.dtype != dtype
+            rbseries = rbseries.cast(dtype, true, false)
+          end
+        end
+
+        # Uninstanced Decimal is a bit special and has various inference paths
+        if dtype == Decimal
+          if rbseries.dtype == String
+            rbseries = rbseries.str_to_decimal_infer(0)
+          elsif rbseries.dtype.float?
+            # Go through string so we infer an appropriate scale.
+            rbseries = rbseries.cast(
+              String, strict, false
+            ).str_to_decimal_infer(0)
+          elsif rbseries.dtype.integer? || rbseries.dtype == Null
+            rbseries = rbseries.cast(
+              Decimal.new(nil, 0), strict, false
+            )
+          elsif !rbseries.dtype.is_a?(Decimal)
+            msg = "can't convert #{rbseries.dtype} to Decimal"
+            raise TypeError, msg
+          end
+        end
+
+        return rbseries
+
+      elsif dtype == Struct
+        struct_schema = dtype.is_a?(Struct) ? dtype.to_schema : nil
+        empty = {}
+
+        data = []
+        invalid = []
+        values.each_with_index do |v, i|
+          if v.nil?
+            invalid << i
+            data << empty
+          else
+            data << v
+          end
+        end
+
+        return sequence_to_rbdf(
+          values.map { |v| v.nil? ? empty : v },
+          schema: struct_schema,
+          orient: "row",
+        ).to_struct(name, invalid)
+      end
+
+      if ruby_dtype.nil?
+        if value.nil?
+          constructor = polars_type_to_constructor(Null)
+          return constructor.(name, values, strict)
+        end
+
+        ruby_dtype = value.class
+      end
+
+      # temporal branch
+      if rb_temporal_types.include?(ruby_dtype)
+        if dtype.nil?
+          dtype = parse_into_dtype(ruby_dtype)
+        elsif rb_temporal_types.include?(dtype)
+          dtype = parse_into_dtype(dtype)
+        end
+
+        values_dtype = value.nil? ? nil : try_parse_into_dtype(value.class)
+        if !values_dtype.nil? && values_dtype.float?
+          msg = "'float' object cannot be interpreted as a #{ruby_dtype.name.inspect}"
+          raise TypeError, msg
+        end
+
+        rb_series = RbSeries.new_from_any_values(name, values, strict)
+
+        time_unit = dtype.respond_to?(:time_unit) ? dtype.time_unit : nil
+        time_zone = dtype.respond_to?(:time_zone) ? dtype.time_zone : nil
+
+        if dtype.temporal? && values_dtype == String && dtype != Duration
+          s = wrap_s(rb_series).str.strptime(dtype, strict: strict)
+        elsif !time_unit.nil? && values_dtype != Date
+          s = wrap_s(rb_series).dt.cast_time_unit(time_unit)
+        else
+          s = wrap_s(rb_series)
+        end
+
+        if dtype == Datetime && !time_zone.nil?
+          return s.dt.convert_time_zone(time_zone)._s
+        end
+        s._s
+
+      elsif defined?(Numo::NArray) && value.is_a?(Numo::NArray) && value.shape.length == 1
+        raise Todo
+
+      elsif ruby_dtype == ::Array
+        if dtype.nil?
+          RbSeries.new_from_any_values(name, values, strict)
+        elsif dtype.is_a?(Object)
+          RbSeries.new_object(name, values, strict)
+        else
+          inner_dtype = dtype.respond_to?(:inner) ? dtype.inner : nil
+          if !inner_dtype.nil?
+            rbseries_list =
+              values.map do |value|
+                if value.nil?
+                  nil
+                else
+                  sequence_to_rbseries(
+                    "",
+                    value,
+                    dtype: inner_dtype,
+                    strict: strict,
+                    nan_to_null: nan_to_null,
+                  )
+                end
+              end
+            rbseries = RbSeries.new_series_list(name, rbseries_list, strict)
+          else
+            # panics in Python
+            raise Todo if dtype.eql?(Array)
+
+            rbseries = RbSeries.new_from_any_values_and_dtype(
+              name, values, dtype, strict
+            )
+          end
+          if dtype != rbseries.dtype
+            rbseries = rbseries.cast(dtype, false, false)
+          end
+          rbseries
+        end
+
+      elsif ruby_dtype == Series
+        RbSeries.new_series_list(
+          name, values.map { |v| !v.nil? ? v._s : v }, strict
+        )
+
+      elsif ruby_dtype == RbSeries
+        RbSeries.new_series_list(name, values, strict)
+      else
+        constructor =
+          if value.is_a?(::String)
+            if value.encoding == Encoding::UTF_8
+              RbSeries.method(:new_str)
+            else
+              RbSeries.method(:new_binary)
+            end
+          elsif value.is_a?(Integer) && values.any? { |v| v.is_a?(Float) }
+            # TODO improve performance
+            RbSeries.method(:new_opt_f64)
+          else
+            rb_type_to_constructor(value.class)
+          end
+
+        _construct_series_with_fallbacks(constructor, name, values, dtype, strict: strict)
+      end
+    end
+
+    def self._construct_series_with_fallbacks(constructor, name, values, dtype, strict:)
+      begin
+        constructor.(name, values, strict)
+      rescue
+        if dtype.nil?
+          RbSeries.new_from_any_values(name, values, strict)
+        else
+          RbSeries.new_from_any_values_and_dtype(name, values, dtype, strict)
+        end
+      end
+    end
+
+    def self.numo_to_rbseries(name, values, strict: true, nan_to_null: false)
+      # not needed yet
+      # if !values.contiguous?
+      # end
+
+      if values.shape.length == 1
+        values, dtype = numo_values_and_dtype(values)
+        constructor = numo_type_to_constructor(dtype)
+        constructor.(
+          name, values.to_a, [Numo::SFloat, Numo::DFloat].include?(dtype) ? nan_to_null : strict
+        )
+      else
+        original_shape = values.shape
+        values_1d = values.reshape(original_shape.inject(&:*))
+
+        rb_s = numo_to_rbseries(
+          name,
+          values_1d,
+          strict: strict,
+          nan_to_null: nan_to_null
+        )
+        Utils.wrap_s(rb_s).reshape(original_shape)._s
+      end
+    end
+
+    def self.series_to_rbseries(name, values, dtype: nil, strict: true)
+      s = values.clone
+      if !dtype.nil? && dtype != s.dtype
+        s = s.cast(dtype, strict: strict)
+      end
+      if !name.nil?
+        s = s.alias(name)
+      end
+      s._s
+    end
+
+    def self.dataframe_to_rbseries(
+      name,
+      values,
+      dtype: nil,
+      strict: true
+    )
+      if values.width > 1
+        name ||= ""
+        s = values.to_struct(name)
+      elsif values.width == 1
+        s = values.to_series
+        if !name.nil?
+          s = s.alias(name)
+        end
+      else
+        msg = "cannot initialize Series from DataFrame without any columns"
+        raise TypeError, msg
+      end
+
+      if !dtype.nil? && dtype != s.dtype
+        s = s.cast(dtype, strict: strict)
+      end
+
+      s._s
+    end
+
+    # TODO move rest
+
+    POLARS_TYPE_TO_CONSTRUCTOR = {
+      Float32 => RbSeries.method(:new_opt_f32),
+      Float64 => RbSeries.method(:new_opt_f64),
+      Int8 => RbSeries.method(:new_opt_i8),
+      Int16 => RbSeries.method(:new_opt_i16),
+      Int32 => RbSeries.method(:new_opt_i32),
+      Int64 => RbSeries.method(:new_opt_i64),
+      Int128 => RbSeries.method(:new_opt_i128),
+      UInt8 => RbSeries.method(:new_opt_u8),
+      UInt16 => RbSeries.method(:new_opt_u16),
+      UInt32 => RbSeries.method(:new_opt_u32),
+      UInt64 => RbSeries.method(:new_opt_u64),
+      UInt128 => RbSeries.method(:new_opt_u128),
+      Decimal => RbSeries.method(:new_decimal),
+      Date => RbSeries.method(:new_from_any_values),
+      Datetime => RbSeries.method(:new_from_any_values),
+      Duration => RbSeries.method(:new_from_any_values),
+      Time => RbSeries.method(:new_from_any_values),
+      Boolean => RbSeries.method(:new_opt_bool),
+      Utf8 => RbSeries.method(:new_str),
+      Object => RbSeries.method(:new_object),
+      Categorical => RbSeries.method(:new_str),
+      Enum => RbSeries.method(:new_str),
+      Binary => RbSeries.method(:new_binary),
+      Null => RbSeries.method(:new_null)
+    }
+
+    SYM_TYPE_TO_CONSTRUCTOR = {
+      f32: RbSeries.method(:new_opt_f32),
+      f64: RbSeries.method(:new_opt_f64),
+      i8: RbSeries.method(:new_opt_i8),
+      i16: RbSeries.method(:new_opt_i16),
+      i32: RbSeries.method(:new_opt_i32),
+      i64: RbSeries.method(:new_opt_i64),
+      i128: RbSeries.method(:new_opt_i128),
+      u8: RbSeries.method(:new_opt_u8),
+      u16: RbSeries.method(:new_opt_u16),
+      u32: RbSeries.method(:new_opt_u32),
+      u64: RbSeries.method(:new_opt_u64),
+      u128: RbSeries.method(:new_opt_u128),
+      bool: RbSeries.method(:new_opt_bool),
+      str: RbSeries.method(:new_str)
+    }
+
+    def self.polars_type_to_constructor(dtype)
+      if dtype.is_a?(Array)
+        lambda do |name, values, strict|
+          RbSeries.new_array(dtype.width, dtype.inner, name, values, strict)
+        end
+      elsif dtype.is_a?(Class) && dtype < DataType
+        POLARS_TYPE_TO_CONSTRUCTOR.fetch(dtype)
+      elsif dtype.is_a?(DataType)
+        POLARS_TYPE_TO_CONSTRUCTOR.fetch(dtype.class)
+      else
+        SYM_TYPE_TO_CONSTRUCTOR.fetch(dtype.to_sym)
+      end
+    rescue KeyError
+      raise ArgumentError, "Cannot construct RbSeries for type #{dtype}."
+    end
+
+    RB_TYPE_TO_CONSTRUCTOR = {
+      Float => RbSeries.method(:new_opt_f64),
+      Integer => RbSeries.method(:new_opt_i64),
+      TrueClass => RbSeries.method(:new_opt_bool),
+      FalseClass => RbSeries.method(:new_opt_bool),
+      BigDecimal => RbSeries.method(:new_decimal),
+      NilClass => RbSeries.method(:new_null)
+    }
+
+    def self.rb_type_to_constructor(dtype)
+      RB_TYPE_TO_CONSTRUCTOR.fetch(dtype)
+    rescue KeyError
+      RbSeries.method(:new_object)
+    end
+
+    def self.numo_values_and_dtype(values)
+      [values, values.class]
+    end
+
+    def self.numo_type_to_constructor(dtype)
+      {
+        Numo::Float32 => RbSeries.method(:new_opt_f32),
+        Numo::Float64 => RbSeries.method(:new_opt_f64),
+        Numo::Int8 => RbSeries.method(:new_opt_i8),
+        Numo::Int16 => RbSeries.method(:new_opt_i16),
+        Numo::Int32 => RbSeries.method(:new_opt_i32),
+        Numo::Int64 => RbSeries.method(:new_opt_i64),
+        Numo::UInt8 => RbSeries.method(:new_opt_u8),
+        Numo::UInt16 => RbSeries.method(:new_opt_u16),
+        Numo::UInt32 => RbSeries.method(:new_opt_u32),
+        Numo::UInt64 => RbSeries.method(:new_opt_u64)
+      }.fetch(dtype)
+    rescue KeyError
+      RbSeries.method(:new_object)
+    end
+  end
+end

data/lib/polars/utils/construction/utils.rb

@@ -0,0 +1,9 @@
+module Polars
+  module Utils
+    def self.get_first_non_none(values)
+      if !values.nil?
+        values.find { |v| !v.nil? }
+      end
+    end
+  end
+end

data/lib/polars/utils/deprecation.rb

@@ -0,0 +1,11 @@
+module Polars
+  module Utils
+    def self.issue_deprecation_warning(message)
+      warn message
+    end
+
+    def self.deprecated(message)
+      issue_deprecation_warning(message)
+    end
+  end
+end

data/lib/polars/utils/serde.rb

@@ -8,10 +8,15 @@ module Polars
       end
 
       if file.nil?
-        return serialize_to_bytes.call
+        serialize_to_bytes.call
+      elsif pathlike?(file)
+        file = normalize_filepath(file)
+        serializer.(file)
+        nil
+      else
+        serializer.(file)
+        nil
       end
-
-      raise Todo
     end
   end
 end

data/lib/polars/utils/unstable.rb

@@ -0,0 +1,19 @@
+module Polars
+  module Utils
+    def self.issue_unstable_warning(message = nil)
+      warnings_enabled = ENV.fetch("POLARS_WARN_UNSTABLE", 0).to_i != 0
+      if !warnings_enabled
+        return
+      end
+
+      if message.nil?
+        message = "this functionality is considered unstable."
+      end
+      message += (
+        " It may be changed at any point without it being considered a breaking change."
+      )
+
+      warn message
+    end
+  end
+end

data/lib/polars/utils/various.rb

@@ -39,6 +39,20 @@ module Polars
       end
     end
 
+    def self.range_to_series(name, rng, dtype: nil)
+      dtype ||= Int64
+      if dtype.integer?
+        range = F.int_range(
+          rng.first, rng.last + (rng.exclude_end? ? 0 : 1), step: 1, dtype: dtype, eager: true
+        )
+      else
+        range = F.int_range(
+          rng.first, rng.last + (rng.exclude_end? ? 0 : 1), step: 1, eager: true
+        ).cast(dtype)
+      end
+      range.alias(name)
+    end
+
     def self.arrlen(obj)
       if obj.is_a?(Range)
         # size only works for numeric ranges
@@ -97,5 +111,50 @@ module Polars
         raise TypeError, msg
       end
     end
+
+    def self._update_columns(df, new_columns)
+      if df.width > new_columns.length
+        cols = df.columns
+        new_columns.each_with_index do |name, i|
+          cols[i] = name
+        end
+        new_columns = cols
+      end
+      df.columns = new_columns.to_a
+      df
+    end
+
+    def self.parse_percentiles(
+      percentiles, inject_median: false
+    )
+      if percentiles.is_a?(Float)
+        percentiles = [percentiles]
+      elsif percentiles.nil?
+        percentiles = []
+      end
+      if !percentiles.all? { |p| p >= 0 && p <= 1 }
+        msg = "`percentiles` must all be in the range [0, 1]"
+        raise ArgumentError, msg
+      end
+
+      sub_50_percentiles = percentiles.select { |p| p < 0.5 }.sort
+      at_or_above_50_percentiles = percentiles.select { |p| p >= 0.5 }.sort
+
+      if inject_median && (!at_or_above_50_percentiles || at_or_above_50_percentiles[0] != 0.5)
+        at_or_above_50_percentiles = [0.5, *at_or_above_50_percentiles]
+      end
+
+      [*sub_50_percentiles, *at_or_above_50_percentiles]
+    end
+
+    def self.display_dot_graph(
+      dot:,
+      show: true,
+      output_path: nil,
+      raw_output: false,
+      figsize: [16.0, 12.0]
+    )
+      raise Todo
+    end
   end
 end

data/lib/polars/utils.rb

@@ -4,7 +4,7 @@ module Polars
     DTYPE_TEMPORAL_UNITS = ["ns", "us", "ms"]
 
     def self.is_polars_dtype(dtype, include_unknown: false)
-      is_dtype = dtype.is_a?(Symbol) || dtype.is_a?(::String) || dtype.is_a?(DataType) || (dtype.is_a?(Class) && dtype < DataType)
+      is_dtype = dtype.is_a?(DataType) || (dtype.is_a?(Class) && dtype < DataType)
 
       if !include_unknown
         is_dtype && dtype != Unknown
@@ -17,42 +17,6 @@ module Polars
       obj.is_a?(Expr) && obj.meta.is_column
     end
 
-    def self.map_rb_type_to_dtype(ruby_dtype)
-      if ruby_dtype == Float
-        Float64
-      elsif ruby_dtype == Integer
-        Int64
-      elsif ruby_dtype == ::String
-        Utf8
-      elsif ruby_dtype == TrueClass || ruby_dtype == FalseClass
-        Boolean
-      elsif ruby_dtype == DateTime || ruby_dtype == ::Time || (defined?(ActiveSupport::TimeWithZone) && ruby_dtype == ActiveSupport::TimeWithZone)
-        Datetime.new("ns")
-      elsif ruby_dtype == ::Date
-        Date
-      elsif ruby_dtype == ::Array
-        List
-      elsif ruby_dtype == NilClass
-        Null
-      else
-        raise TypeError, "Invalid type"
-      end
-    end
-
-    # TODO fix
-    def self.rb_type_to_dtype(data_type)
-      if is_polars_dtype(data_type)
-        data_type = data_type.to_s if data_type.is_a?(Symbol)
-        return data_type
-      end
-
-      begin
-        map_rb_type_to_dtype(data_type)
-      rescue TypeError
-        raise ArgumentError, "Conversion of Ruby data type #{data_type.inspect} to Polars data type not implemented."
-      end
-    end
-
     def self.parse_row_index_args(row_index_name = nil, row_index_offset = 0)
       if row_index_name.nil?
         nil
@@ -64,15 +28,14 @@ module Polars
     def self.handle_projection_columns(columns)
       projection = nil
       if columns
-        raise Todo
-        # if columns.is_a?(::String) || columns.is_a?(Symbol)
-        #   columns = [columns]
-        # elsif is_int_sequence(columns)
-        #   projection = columns.to_a
-        #   columns = nil
-        # elsif !is_str_sequence(columns)
-        #   raise ArgumentError, "columns arg should contain a list of all integers or all strings values."
-        # end
+        if columns.is_a?(::String)
+          columns = [columns]
+        elsif is_int_sequence(columns)
+          projection = columns.to_a
+          columns = nil
+        elsif !is_str_sequence(columns)
+          raise ArgumentError, "columns arg should contain a list of all integers or all strings values."
+        end
       end
       [projection, columns]
     end
@@ -153,10 +116,46 @@ module Polars
       if is_polars_dtype(input)
         input
       else
-        raise Todo
+        parse_rb_type_into_dtype(input)
       end
     end
 
+    def self.try_parse_into_dtype(input)
+      parse_into_dtype(input)
+    rescue TypeError
+      nil
+    end
+
+    def self.parse_rb_type_into_dtype(input)
+      if input == Integer
+        Int64.new
+      elsif input == Float
+        Float64.new
+      elsif input == ::String
+        String.new
+      elsif input == ::Time || input == ::DateTime || (defined?(ActiveSupport::TimeWithZone) && input == ActiveSupport::TimeWithZone)
+        Datetime.new("ns")
+      elsif input == ::Date
+        Date.new
+      elsif input == NilClass
+        Null.new
+      elsif input == ::Array
+        List
+      # this is required as pass through. Don't remove
+      elsif input == Unknown
+        Unknown
+      else
+        _raise_on_invalid_dtype(input)
+      end
+    end
+
+    def self._raise_on_invalid_dtype(input)
+      # TODO improve
+      input_type = input.inspect
+      msg = "cannot parse input #{input_type} into Polars data type"
+      raise TypeError, msg
+    end
+
     def self.re_escape(s)
       # escapes _only_ those metachars with meaning to the rust regex crate
       Plr.re_escape(s)

data/lib/polars/version.rb

@@ -1,4 +1,4 @@
 module Polars
   # @private
-  VERSION = "0.23.0"
+  VERSION = "0.24.0"
 end