polars-df 0.15.0 → 0.26.0

data/lib/polars/utils/construction/series.rb

@@ -0,0 +1,350 @@
+module Polars
+  module Utils
+    def self.sequence_to_rbseries(name, values, dtype: nil, strict: true, nan_to_null: false)
+      ruby_dtype = nil
+
+      if values.is_a?(Range)
+        if values.begin.is_a?(::String)
+          values = values.to_a
+        else
+          return range_to_series(name, values, dtype: dtype)._s
+        end
+      end
+
+      if values.length == 0 && dtype.nil?
+        dtype = Null
+      elsif [List, Array].include?(dtype)
+        ruby_dtype = ::Array
+      end
+
+      rb_temporal_types = [::Date, ::DateTime, ::Time]
+      rb_temporal_types << ActiveSupport::TimeWithZone if defined?(ActiveSupport::TimeWithZone)
+
+      value = get_first_non_none(values)
+
+      if !dtype.nil? && is_polars_dtype(dtype) && !dtype.nested? && dtype != Unknown && ruby_dtype.nil?
+        constructor = polars_type_to_constructor(dtype)
+        rbseries = _construct_series_with_fallbacks(
+          constructor, name, values, dtype, strict: strict
+        )
+
+        if [Date, Datetime, Duration, Time, Boolean, Categorical, Enum].include?(dtype) || dtype.is_a?(Decimal) || dtype.is_a?(Categorical)
+          if rbseries.dtype != dtype
+            rbseries = rbseries.cast(dtype, true, false)
+          end
+        end
+
+        # Uninstanced Decimal is a bit special and has various inference paths
+        if dtype == Decimal
+          if rbseries.dtype == String
+            rbseries = rbseries.str_to_decimal_infer(0)
+          elsif rbseries.dtype.float?
+            # Go through string so we infer an appropriate scale.
+            rbseries = rbseries.cast(
+              String, strict, false
+            ).str_to_decimal_infer(0)
+          elsif rbseries.dtype.integer? || rbseries.dtype == Null
+            rbseries = rbseries.cast(
+              Decimal.new(nil, 0), strict, false
+            )
+          elsif !rbseries.dtype.is_a?(Decimal)
+            msg = "can't convert #{rbseries.dtype} to Decimal"
+            raise TypeError, msg
+          end
+        end
+
+        return rbseries
+
+      elsif dtype == Struct
+        struct_schema = dtype.is_a?(Struct) ? dtype.to_schema : nil
+        empty = {}
+
+        data = []
+        invalid = []
+        values.each_with_index do |v, i|
+          if v.nil?
+            invalid << i
+            data << empty
+          else
+            data << v
+          end
+        end
+
+        return sequence_to_rbdf(
+          data,
+          schema: struct_schema,
+          orient: "row",
+        ).to_struct(name, invalid)
+      end
+
+      if ruby_dtype.nil?
+        if value.nil?
+          constructor = polars_type_to_constructor(Null)
+          return constructor.(name, values, strict)
+        end
+
+        ruby_dtype = value.class
+      end
+
+      # temporal branch
+      if rb_temporal_types.include?(ruby_dtype)
+        if dtype.nil?
+          dtype = parse_into_dtype(ruby_dtype)
+        elsif rb_temporal_types.include?(dtype)
+          dtype = parse_into_dtype(dtype)
+        end
+
+        values_dtype = value.nil? ? nil : try_parse_into_dtype(value.class)
+        if !values_dtype.nil? && values_dtype.float?
+          msg = "'float' object cannot be interpreted as a #{ruby_dtype.name.inspect}"
+          raise TypeError, msg
+        end
+
+        rb_series = RbSeries.new_from_any_values(name, values, strict)
+
+        time_unit = dtype.respond_to?(:time_unit) ? dtype.time_unit : nil
+        time_zone = dtype.respond_to?(:time_zone) ? dtype.time_zone : nil
+
+        if dtype.temporal? && values_dtype == String && dtype != Duration
+          s = wrap_s(rb_series).str.strptime(dtype, strict: strict)
+        elsif !time_unit.nil? && values_dtype != Date
+          s = wrap_s(rb_series).dt.cast_time_unit(time_unit)
+        else
+          s = wrap_s(rb_series)
+        end
+
+        if dtype == Datetime && !time_zone.nil?
+          return s.dt.convert_time_zone(time_zone)._s
+        end
+        s._s
+
+      elsif defined?(Numo::NArray) && value.is_a?(Numo::NArray) && value.shape.length == 1
+        raise Todo
+
+      elsif ruby_dtype == ::Array
+        if dtype.nil?
+          RbSeries.new_from_any_values(name, values, strict)
+        elsif dtype.is_a?(Object)
+          RbSeries.new_object(name, values, strict)
+        else
+          inner_dtype = dtype.respond_to?(:inner) ? dtype.inner : nil
+          if !inner_dtype.nil?
+            rbseries_list =
+              values.map do |value|
+                if value.nil?
+                  nil
+                else
+                  sequence_to_rbseries(
+                    "",
+                    value,
+                    dtype: inner_dtype,
+                    strict: strict,
+                    nan_to_null: nan_to_null,
+                  )
+                end
+              end
+            rbseries = RbSeries.new_series_list(name, rbseries_list, strict)
+          else
+            # panics in Python
+            raise Todo if dtype.eql?(Array)
+
+            rbseries = RbSeries.new_from_any_values_and_dtype(
+              name, values, dtype, strict
+            )
+          end
+          if dtype != rbseries.dtype
+            rbseries = rbseries.cast(dtype, false, false)
+          end
+          rbseries
+        end
+
+      elsif ruby_dtype == Series
+        RbSeries.new_series_list(
+          name, values.map { |v| !v.nil? ? v._s : v }, strict
+        )
+
+      elsif ruby_dtype == RbSeries
+        RbSeries.new_series_list(name, values, strict)
+      else
+        constructor =
+          if value.is_a?(::String)
+            if value.encoding == Encoding::UTF_8
+              RbSeries.method(:new_str)
+            else
+              RbSeries.method(:new_binary)
+            end
+          elsif value.is_a?(Integer) && values.any? { |v| v.is_a?(Float) }
+            # TODO improve performance
+            RbSeries.method(:new_opt_f64)
+          else
+            rb_type_to_constructor(value.class)
+          end
+
+        if constructor == RbSeries.method(:new_object)
+          srs = RbSeries.new_from_any_values(name, values, strict)
+          return srs
+        end
+
+        _construct_series_with_fallbacks(constructor, name, values, dtype, strict: strict)
+      end
+    end
+
+    def self._construct_series_with_fallbacks(constructor, name, values, dtype, strict:)
+      begin
+        constructor.(name, values, strict)
+      rescue
+        if dtype.nil?
+          RbSeries.new_from_any_values(name, values, strict)
+        else
+          RbSeries.new_from_any_values_and_dtype(name, values, dtype, strict)
+        end
+      end
+    end
+
+    def self.numo_to_rbseries(name, values, strict: true, nan_to_null: false)
+      # not needed yet
+      # if !values.contiguous?
+      # end
+
+      if values.shape.length == 1
+        values, dtype = numo_values_and_dtype(values)
+        constructor = numo_type_to_constructor(dtype)
+        constructor.(
+          name, values.to_a, [Numo::SFloat, Numo::DFloat].include?(dtype) ? nan_to_null : strict
+        )
+      else
+        original_shape = values.shape
+        values_1d = values.reshape(original_shape.inject(&:*))
+
+        rb_s = numo_to_rbseries(
+          name,
+          values_1d,
+          strict: strict,
+          nan_to_null: nan_to_null
+        )
+        Utils.wrap_s(rb_s).reshape(original_shape)._s
+      end
+    end
+
+    def self.series_to_rbseries(name, values, dtype: nil, strict: true)
+      s = values.clone
+      if !dtype.nil? && dtype != s.dtype
+        s = s.cast(dtype, strict: strict)
+      end
+      if !name.nil?
+        s = s.alias(name)
+      end
+      s._s
+    end
+
+    def self.dataframe_to_rbseries(
+      name,
+      values,
+      dtype: nil,
+      strict: true
+    )
+      if values.width > 1
+        name ||= ""
+        s = values.to_struct(name)
+      elsif values.width == 1
+        s = values.to_series
+        if !name.nil?
+          s = s.alias(name)
+        end
+      else
+        msg = "cannot initialize Series from DataFrame without any columns"
+        raise TypeError, msg
+      end
+
+      if !dtype.nil? && dtype != s.dtype
+        s = s.cast(dtype, strict: strict)
+      end
+
+      s._s
+    end
+
+    # TODO move rest
+
+    POLARS_TYPE_TO_CONSTRUCTOR = {
+      Float16 => RbSeries.method(:new_opt_f16),
+      Float32 => RbSeries.method(:new_opt_f32),
+      Float64 => RbSeries.method(:new_opt_f64),
+      Int8 => RbSeries.method(:new_opt_i8),
+      Int16 => RbSeries.method(:new_opt_i16),
+      Int32 => RbSeries.method(:new_opt_i32),
+      Int64 => RbSeries.method(:new_opt_i64),
+      Int128 => RbSeries.method(:new_opt_i128),
+      UInt8 => RbSeries.method(:new_opt_u8),
+      UInt16 => RbSeries.method(:new_opt_u16),
+      UInt32 => RbSeries.method(:new_opt_u32),
+      UInt64 => RbSeries.method(:new_opt_u64),
+      UInt128 => RbSeries.method(:new_opt_u128),
+      Decimal => RbSeries.method(:new_decimal),
+      Date => RbSeries.method(:new_from_any_values),
+      Datetime => RbSeries.method(:new_from_any_values),
+      Duration => RbSeries.method(:new_from_any_values),
+      Time => RbSeries.method(:new_from_any_values),
+      Boolean => RbSeries.method(:new_opt_bool),
+      Utf8 => RbSeries.method(:new_str),
+      Object => RbSeries.method(:new_object),
+      Categorical => RbSeries.method(:new_str),
+      Enum => RbSeries.method(:new_str),
+      Binary => RbSeries.method(:new_binary),
+      Null => RbSeries.method(:new_null)
+    }
+
+    def self.polars_type_to_constructor(dtype)
+      if dtype.is_a?(Array)
+        return lambda do |name, values, strict|
+          RbSeries.new_array(name, values, strict, dtype)
+        end
+      end
+
+      begin
+        base_type = dtype.base_type
+        POLARS_TYPE_TO_CONSTRUCTOR.fetch(base_type)
+      rescue KeyError
+        raise ArgumentError, "Cannot construct RbSeries for type #{dtype}."
+      end
+    end
+
+    RB_TYPE_TO_CONSTRUCTOR = {
+      Float => RbSeries.method(:new_opt_f64),
+      TrueClass => RbSeries.method(:new_opt_bool),
+      FalseClass => RbSeries.method(:new_opt_bool),
+      Integer => RbSeries.method(:new_opt_i64),
+      ::String => RbSeries.method(:new_str)
+    }
+
+    def self.rb_type_to_constructor(dtype)
+      RB_TYPE_TO_CONSTRUCTOR.fetch(dtype) do
+        if defined?(::BigDecimal) && dtype == ::BigDecimal
+          RbSeries.method(:new_decimal)
+        else
+          RbSeries.method(:new_object)
+        end
+      end
+    end
+
+    def self.numo_values_and_dtype(values)
+      [values, values.class]
+    end
+
+    def self.numo_type_to_constructor(dtype)
+      {
+        Numo::Float32 => RbSeries.method(:new_opt_f32),
+        Numo::Float64 => RbSeries.method(:new_opt_f64),
+        Numo::Int8 => RbSeries.method(:new_opt_i8),
+        Numo::Int16 => RbSeries.method(:new_opt_i16),
+        Numo::Int32 => RbSeries.method(:new_opt_i32),
+        Numo::Int64 => RbSeries.method(:new_opt_i64),
+        Numo::UInt8 => RbSeries.method(:new_opt_u8),
+        Numo::UInt16 => RbSeries.method(:new_opt_u16),
+        Numo::UInt32 => RbSeries.method(:new_opt_u32),
+        Numo::UInt64 => RbSeries.method(:new_opt_u64)
+      }.fetch(dtype)
+    rescue KeyError
+      RbSeries.method(:new_object)
+    end
+  end
+end

data/lib/polars/utils/construction/utils.rb

@@ -0,0 +1,9 @@
+module Polars
+  module Utils
+    def self.get_first_non_none(values)
+      if !values.nil?
+        values.find { |v| !v.nil? }
+      end
+    end
+  end
+end

data/lib/polars/utils/convert.rb

@@ -63,19 +63,24 @@ module Polars
     end
 
     def self._to_ruby_datetime(value, time_unit = "ns", time_zone = nil)
-      if time_zone.nil? || time_zone == "" || time_zone == "UTC"
+      utc_time =
         if time_unit == "ns"
-          ::Time.at(value / 1000000000, value % 1000000000, :nsec).utc
+          ::Time.at(value / 1_000_000_000, value % 1_000_000_000, :nsec).utc
         elsif time_unit == "us"
-          ::Time.at(value / 1000000, value % 1000000, :usec).utc
+          ::Time.at(value / 1_000_000, value % 1_000_000, :usec).utc
         elsif time_unit == "ms"
-          ::Time.at(value / 1000, value % 1000, :millisecond).utc
+          ::Time.at(value / 1_000, value % 1_000, :millisecond).utc
         else
           raise ArgumentError, "time_unit must be one of {{'ns', 'us', 'ms'}}, got #{time_unit}"
         end
-      else
-        raise Todo
+      return utc_time if time_zone.nil?
+
+      unless defined?(::TZInfo)
+        # TODO improve error message
+        require "tzinfo"
       end
+
+      utc_time.getlocal(::TZInfo::Timezone.get(time_zone))
     end
 
     def self._to_ruby_duration(value, time_unit = "ns")
@@ -90,8 +95,13 @@ module Polars
       end
     end
 
-    def self._to_ruby_decimal(digits, scale)
-      BigDecimal("#{digits}e#{scale}")
+    def self._to_ruby_decimal(prec, value)
+      unless defined?(::BigDecimal)
+        # TODO improve error message
+        require "bigdecimal"
+      end
+
+      BigDecimal(value, prec)
     end
   end
 end

data/lib/polars/utils/deprecation.rb

@@ -0,0 +1,11 @@
+module Polars
+  module Utils
+    def self.issue_deprecation_warning(message)
+      warn message
+    end
+
+    def self.deprecated(message)
+      issue_deprecation_warning(message)
+    end
+  end
+end

data/lib/polars/utils/parse.rb

@@ -5,7 +5,8 @@ module Polars
       str_as_lit: false,
       list_as_series: false,
       structify: false,
-      dtype: nil
+      dtype: nil,
+      require_selector: false
     )
       if input.is_a?(Expr)
         expr = input
@@ -14,17 +15,22 @@ module Polars
         end
       elsif (input.is_a?(::String) || input.is_a?(Symbol)) && !str_as_lit
         expr = F.col(input)
-      elsif input.is_a?(::Array) && list_as_series
-        expr = F.lit(Series.new(input), dtype: dtype)
       else
-        expr = F.lit(input, dtype: dtype)
+        if require_selector
+          msg = "cannot turn #{input.inspect} into selector"
+          raise TypeError, msg
+        elsif input.is_a?(::Array) && list_as_series
+          expr = F.lit(Series.new(input), dtype: dtype)
+        else
+          expr = F.lit(input, dtype: dtype)
+        end
       end
 
       expr._rbexpr
     end
 
-    def self.parse_into_list_of_expressions(*inputs, __structify: false, **named_inputs)
-      exprs = _parse_positional_inputs(inputs, structify: __structify)
+    def self.parse_into_list_of_expressions(*inputs, __structify: false, __require_selectors: false, **named_inputs)
+      exprs = _parse_positional_inputs(inputs, structify: __structify, require_selectors: __require_selectors)
       if named_inputs.any?
         named_exprs = _parse_named_inputs(named_inputs, structify: __structify)
         exprs.concat(named_exprs)
@@ -33,9 +39,49 @@ module Polars
       exprs
     end
 
-    def self._parse_positional_inputs(inputs, structify: false)
+    def self.parse_into_selector(i, strict: true)
+      if i.is_a?(::String)
+        cs = Selectors
+
+        cs.by_name([i], require_all: strict)
+      elsif i.is_a?(Selector)
+        i
+      elsif i.is_a?(Expr)
+        i.meta.as_selector
+      else
+        msg = "cannot turn #{i.inspect} into selector"
+        raise TypeError, msg
+      end
+    end
+
+    def self.parse_list_into_selector(inputs, strict: true)
+      if inputs.is_a?(::Array)
+        cs = Selectors
+
+        columns = inputs.select { |i| i.is_a?(::String) }
+        selector = cs.by_name(columns, require_all: strict)
+
+        if columns.length == inputs.length
+          return selector
+        end
+
+        # A bit cleaner
+        if columns.length == 0
+          selector = cs.empty
+        end
+
+        inputs.each do |i|
+          selector |= parse_into_selector(i, strict: strict)
+        end
+        selector
+      else
+        parse_into_selector(inputs, strict: strict)
+      end
+    end
+
+    def self._parse_positional_inputs(inputs, structify: false, require_selectors: false)
       inputs_iter = _parse_inputs_as_iterable(inputs)
-      inputs_iter.map { |e| parse_into_expression(e, structify: structify) }
+      inputs_iter.map { |e| parse_into_expression(e, structify: structify, require_selector: require_selectors) }
     end
 
     def self._parse_inputs_as_iterable(inputs)
@@ -52,7 +98,7 @@ module Polars
 
     def self._parse_named_inputs(named_inputs, structify: false)
       named_inputs.map do |name, input|
-        parse_into_expression(input, structify: structify)._alias(name.to_s)
+        parse_into_expression(input, structify: structify).alias(name.to_s)
       end
     end
 
@@ -85,5 +131,12 @@ module Polars
 
       Plr.all_horizontal(predicates)
     end
+
+    def self._holidays_to_expr(holidays)
+      if !(holidays.is_a?(Expr) || holidays.is_a?(Series))
+        holidays = Series.new("", [holidays], strict: false, dtype: List.new(Date))
+      end
+      parse_into_expression(holidays, dtype: Date)
+    end
   end
 end

data/lib/polars/utils/reduce_balanced.rb

@@ -0,0 +1,43 @@
+module Polars
+  module Utils
+    def self.reduce_balanced(function, iterable)
+      values = iterable.to_a
+
+      if values.empty?
+        msg = "reduce_balanced() of empty iterable"
+        raise TypeError, msg
+      end
+
+      if values.length == 1
+        return values.shift
+      end
+
+      stack = [[0, values.length]]
+
+      i = 0
+
+      while i < stack.length
+        offset, length = stack[i]
+        half = -(length / -2)
+
+        if length > 3
+          stack << [offset + half, length - half]
+        end
+
+        if length > 2
+          stack << [offset, half]
+        end
+
+        stack[i] = [offset, offset + half]
+
+        i += 1
+      end
+
+      stack.reverse.each do |idx_l, idx_r|
+        values[idx_l] = function.(values[idx_l], values[idx_r])
+      end
+
+      values[0]
+    end
+  end
+end

data/lib/polars/utils/serde.rb

@@ -0,0 +1,22 @@
+module Polars
+  module Utils
+    def self.serialize_polars_object(serializer, file)
+      serialize_to_bytes = lambda do
+        buf = StringIO.new
+        serializer.(buf)
+        buf.string
+      end
+
+      if file.nil?
+        serialize_to_bytes.call
+      elsif pathlike?(file)
+        file = normalize_filepath(file)
+        serializer.(file)
+        nil
+      else
+        serializer.(file)
+        nil
+      end
+    end
+  end
+end

data/lib/polars/utils/unstable.rb

@@ -0,0 +1,19 @@
+module Polars
+  module Utils
+    def self.issue_unstable_warning(message = nil)
+      warnings_enabled = ENV.fetch("POLARS_WARN_UNSTABLE", 0).to_i != 0
+      if !warnings_enabled
+        return
+      end
+
+      if message.nil?
+        message = "this functionality is considered unstable."
+      end
+      message += (
+        " It may be changed at any point without it being considered a breaking change."
+      )
+
+      warn message
+    end
+  end
+end