rover-df 0.2.6 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 01e2a90ba133ae07ad6ad482bdca985df806d6a073fa2d93029b2b7e1b55dc49
-  data.tar.gz: 96f4171420dea68b38cffdd5a365657bc464f6d1f0c4f6bf1aefb20377c56179
+  metadata.gz: 171a06a05afa4ec4bb09efe7fef53b49220a3d4fa5352621112e29f2b70812b9
+  data.tar.gz: 435d8f3d4781f1960236f3c2b7f9fa2c4e38dfc987b53cb9fbe6351a9e8db4e9
 SHA512:
-  metadata.gz: 2451d6844c7ece459e61c8e1499047f8efd6472a0d57317b7e2e1110527d843e8177c16ccbb1aeb0fd61e647fdd4291ebf73d4bfe008560eff7b963b1ac22ee6
-  data.tar.gz: 18ad0cfb8fc22aeb63d2e1b11333b1a5989c7bcc0f2b5fbebedb11acf3d3dc26e7235109e1501d0e4b9a06b5aa7e47b71bdb23de6d6fcd87c9fb53d2bf0be330
+  metadata.gz: 9e5baa8cb051f7c7f06bbe0025ac4d923947b34768461a814e533aa78ec5d2d391a12edc6a9a64abc2fa9b1147255211ff26e8094cfaef67c9d70e393e57bcc0
+  data.tar.gz: 55f3438d438326c324c612a92b39a54698123889aeb28bafd93a196f2659208b4b72a1ac02efd407166fc56c8c9f3abff9472804ddfb231220b6adf41ff38df1

data/CHANGELOG.md

@@ -1,3 +1,24 @@
+## 0.3.0 (2022-04-03)
+
+- Added `deep_dup` method to data frames
+- Changed `:int` to `:int64`, `:uint` to `:uint64`, and `:float` to `:float64` for type methods
+- Changed missing column to raise `KeyError` instead of `ArgumentError` for aggregate methods
+- Changed passing too many headers to `read_csv` and `parse_csv` to raise `ArgumentError`
+- Changed empty string in CSV headers to match behavior of `nil`
+- Fixed `clone` and `dup` method for vectors
+- Dropped support for Ruby < 2.7
+
+## 0.2.8 (2022-03-15)
+
+- Added `group` and `stacked` options to `plot`
+- Improved performance of `read_csv` and `parse_csv`
+
+## 0.2.7 (2022-01-16)
+
+- Added support for booleans to Parquet methods
+- Added support for creating data frames from `ActiveRecord::Result`
+- Added `types` option to `read_parquet` and `parse_parquet` methods
+
 ## 0.2.6 (2021-10-27)
 
 - Added support for `nil` headers to `read_csv` and `parse_csv`

data/LICENSE.txt

@@ -1,4 +1,4 @@
-Copyright (c) 2020-2021 Andrew Kane
+Copyright (c) 2020-2022 Andrew Kane
 
 MIT License
 

data/README.md

@@ -13,7 +13,7 @@ Simple, powerful data frames for Ruby
 Add this line to your application’s Gemfile:
 
 ```ruby
-gem 'rover-df'
+gem "rover-df"
 ```
 
 ## Intro
@@ -61,7 +61,7 @@ Rover.read_csv("file.csv")
 Rover.parse_csv("CSV,data,string")
 ```
 
-From Parquet (requires the [red-parquet](https://github.com/apache/arrow/tree/master/ruby/red-parquet) gem) [unreleased]
+From Parquet (requires the [red-parquet](https://github.com/apache/arrow/tree/master/ruby/red-parquet) gem)
 
 ```ruby
 Rover.read_parquet("file.parquet")
@@ -236,7 +236,7 @@ df.group(:a).max(:b)
 Multiple groups
 
 ```ruby
-df.group([:a, :b]).count
+df.group(:a, :b).count
 ```
 
 ## Visualization
@@ -244,7 +244,7 @@ df.group([:a, :b]).count
 Add [Vega](https://github.com/ankane/vega) to your application’s Gemfile:
 
 ```ruby
-gem 'vega'
+gem "vega"
 ```
 
 And use:
@@ -259,6 +259,18 @@ Specify the chart type (`line`, `pie`, `column`, `bar`, `area`, or `scatter`)
 df.plot(:a, :b, type: "pie")
 ```
 
+Group data
+
+```ruby
+df.plot(:a, :b, group: :c)
+```
+
+Stacked columns or bars
+
+```ruby
+df.plot(:a, :b, group: :c, stacked: true)
+```
+
 ## Updating Data
 
 Add a new column
@@ -401,7 +413,7 @@ CSV
 df.to_csv
 ```
 
-Parquet (requires the [red-parquet](https://github.com/apache/arrow/tree/master/ruby/red-parquet) gem) [unreleased]
+Parquet (requires the [red-parquet](https://github.com/apache/arrow/tree/master/ruby/red-parquet) gem)
 
 ```ruby
 df.to_parquet
@@ -412,22 +424,22 @@ df.to_parquet
 You can specify column types when creating a data frame
 
 ```ruby
-Rover::DataFrame.new(data, types: {"a" => :int, "b" => :float})
+Rover::DataFrame.new(data, types: {"a" => :int64, "b" => :float64})
 ```
 
 Or
 
 ```ruby
-Rover.read_csv("data.csv", types: {"a" => :int, "b" => :float})
+Rover.read_csv("data.csv", types: {"a" => :int64, "b" => :float64})
 ```
 
 Supported types are:
 
-- boolean - `bool`
-- float - `float`, `float32`
-- integer - `int`, `int32`, `int16`, `int8`
-- unsigned integer - `uint`, `uint32`, `uint16`, `uint8`
-- object - `object`
+- boolean - `:bool`
+- float - `:float64`, `:float32`
+- integer - `:int64`, `:int32`, `:int16`, `:int8`
+- unsigned integer - `:uint64`, `:uint32`, `:uint16`, `:uint8`
+- object - `:object`
 
 Get column types
 
@@ -444,7 +456,7 @@ df[:a].type
 Change the type of a column
 
 ```ruby
-df[:a] = df[:a].to(:int)
+df[:a] = df[:a].to(:int32)
 ```
 
 ## History

data/lib/rover/data_frame.rb

@@ -40,8 +40,8 @@ module Rover
         vectors.each do |k, v|
           @vectors[k] = to_vector(v, type: types[k])
         end
-      elsif defined?(ActiveRecord) && (data.is_a?(ActiveRecord::Relation) || (data.is_a?(Class) && data < ActiveRecord::Base))
-        result = data.connection.select_all(data.all.to_sql)
+      elsif defined?(ActiveRecord) && (data.is_a?(ActiveRecord::Relation) || (data.is_a?(Class) && data < ActiveRecord::Base) || data.is_a?(ActiveRecord::Result))
+        result = data.is_a?(ActiveRecord::Result) ? data : data.connection.select_all(data.all.to_sql)
         result.columns.each_with_index do |k, i|
           @vectors[k] = to_vector(result.rows.map { |r| r[i] }, type: types[k])
         end
@@ -72,7 +72,7 @@ module Rover
         # multiple columns
         df = DataFrame.new
         where.each do |k|
-          check_column(k, true)
+          check_column(k)
           df[k] = @vectors[k]
         end
         df
@@ -102,7 +102,7 @@ module Rover
     def []=(k, v)
       check_key(k)
       v = to_vector(v, size: size)
-      raise ArgumentError, "Size mismatch: expected #{size}, got #{v.size}" if @vectors.any? && v.size != size
+      raise ArgumentError, "Size mismatch (given #{v.size}, expected #{size})" if @vectors.any? && v.size != size
       @vectors[k] = v
     end
 
@@ -242,14 +242,16 @@ module Rover
       types.each do |name, type|
         schema[name] =
           case type
-          when :int
+          when :int64
             :int64
-          when :uint
+          when :uint64
             :uint64
-          when :float
+          when :float64
             :double
           when :float32
             :float
+          when :bool
+            :boolean
           when :object
             if @vectors[name].all? { |v| v.is_a?(String) }
               :string
@@ -344,10 +346,10 @@ module Rover
       end
     end
 
-    def dup
+    def deep_dup
       df = DataFrame.new
       @vectors.each do |k, v|
-        df[k] = v
+        df[k] = v.dup
       end
       df
     end
@@ -399,7 +401,7 @@ module Rover
       keys.all? { |k| self[k].to_numo == other[k].to_numo }
     end
 
-    def plot(x = nil, y = nil, type: nil)
+    def plot(x = nil, y = nil, type: nil, group: nil, stacked: nil)
       require "vega"
 
       raise ArgumentError, "Must specify columns" if keys.size != 2 && (!x || !y)
@@ -414,7 +416,7 @@ module Rover
           raise "Cannot determine type. Use the type option."
         end
       end
-      data = self[[x, y]]
+      data = self[group.nil? ? [x, y] : [x, y, group]]
 
       case type
       when "line", "area"
@@ -428,16 +430,20 @@ module Rover
           end
 
         scale = x_type == "temporal" ? {type: "utc"} : {}
+        encoding = {
+          x: {field: x, type: x_type, scale: scale},
+          y: {field: y, type: "quantitative"}
+        }
+        encoding[:color] = {field: group} if group
 
         Vega.lite
           .data(data)
           .mark(type: type, tooltip: true, interpolate: "cardinal", point: {size: 60})
-          .encoding(
-            x: {field: x, type: x_type, scale: scale},
-            y: {field: y, type: "quantitative"}
-          )
+          .encoding(encoding)
           .config(axis: {labelFontSize: 12})
       when "pie"
+        raise ArgumentError, "Cannot use group option with pie chart" unless group.nil?
+
         Vega.lite
           .data(data)
           .mark(type: "arc", tooltip: true)
@@ -447,34 +453,48 @@ module Rover
           )
           .view(stroke: nil)
       when "column"
+        encoding = {
+          x: {field: x, type: "nominal", sort: "none", axis: {labelAngle: 0}},
+          y: {field: y, type: "quantitative"}
+        }
+        if group
+          encoding[:color] = {field: group}
+          encoding[:xOffset] = {field: group} unless stacked
+        end
+
         Vega.lite
           .data(data)
           .mark(type: "bar", tooltip: true)
-          .encoding(
-            # TODO determine label angle
-            x: {field: x, type: "nominal", sort: "none", axis: {labelAngle: 0}},
-            y: {field: y, type: "quantitative"}
-          )
+          .encoding(encoding)
           .config(axis: {labelFontSize: 12})
       when "bar"
+        encoding = {
+          # TODO determine label angle
+          y: {field: x, type: "nominal", sort: "none", axis: {labelAngle: 0}},
+          x: {field: y, type: "quantitative"}
+        }
+        if group
+          encoding[:color] = {field: group}
+          encoding[:yOffset] = {field: group} unless stacked
+        end
+
         Vega.lite
           .data(data)
           .mark(type: "bar", tooltip: true)
-          .encoding(
-            # TODO determine label angle
-            y: {field: x, type: "nominal", sort: "none", axis: {labelAngle: 0}},
-            x: {field: y, type: "quantitative"}
-          )
+          .encoding(encoding)
           .config(axis: {labelFontSize: 12})
       when "scatter"
+        encoding = {
+          x: {field: x, type: "quantitative", scale: {zero: false}},
+          y: {field: y, type: "quantitative", scale: {zero: false}},
+          size: {value: 60}
+        }
+        encoding[:color] = {field: group} if group
+
         Vega.lite
           .data(data)
           .mark(type: "circle", tooltip: true)
-          .encoding(
-            x: {field: x, type: "quantitative", scale: {zero: false}},
-            y: {field: y, type: "quantitative", scale: {zero: false}},
-            size: {value: 60}
-          )
+          .encoding(encoding)
           .config(axis: {labelFontSize: 12})
       else
         raise ArgumentError, "Invalid type: #{type}"
@@ -483,8 +503,20 @@ module Rover
 
     private
 
+    # for clone
+    def initialize_clone(_)
+      @vectors = @vectors.clone
+      super
+    end
+
+    # for dup
+    def initialize_dup(_)
+      @vectors = @vectors.dup
+      super
+    end
+
     def check_key(key)
-      raise ArgumentError, "Key must be a string or symbol, got #{key.inspect}" unless key.is_a?(String) || key.is_a?(Symbol)
+      raise ArgumentError, "Key must be a String or Symbol, given #{key.class.name}" unless key.is_a?(String) || key.is_a?(Symbol)
     end
 
     # TODO make more efficient
@@ -545,19 +577,9 @@ module Rover
       raise ArgumentError, "Missing keys: #{missing_keys.join(", ")}" if missing_keys.any?
     end
 
-    # TODO in 0.3.0
-    # always use did_you_mean
-    def check_column(key, did_you_mean = false)
+    def check_column(key)
       unless include?(key)
-        if did_you_mean
-          if RUBY_VERSION.to_f >= 2.6
-            raise KeyError.new("Missing column: #{key}", receiver: self, key: key)
-          else
-            raise KeyError.new("Missing column: #{key}")
-          end
-        else
-          raise ArgumentError, "Missing column: #{key}"
-        end
+        raise KeyError.new("Missing column: #{key}", receiver: self, key: key)
       end
     end
 

data/lib/rover/group.rb

@@ -3,6 +3,7 @@ module Rover
     def initialize(df, columns)
       @df = df
       @columns = columns
+      check_columns
     end
 
     def group(*columns)
@@ -22,16 +23,20 @@ module Rover
       end
     end
 
+    def plot(*args, **options)
+      raise ArgumentError, "Multiple groups not supported" if @columns.size > 1
+      # same message as Ruby
+      raise ArgumentError, "unknown keyword: :group" if options.key?(:group)
+
+      @df.plot(*args, **options, group: @columns.first)
+    end
+
     private
 
     # TODO make more efficient
     def grouped_dfs
       # cache here so we can reuse for multiple calcuations if needed
       @grouped_dfs ||= begin
-        raise ArgumentError, "No columns given" if @columns.empty?
-        missing_keys = @columns - @df.keys
-        raise ArgumentError, "Missing keys: #{missing_keys.join(", ")}" if missing_keys.any?
-
         groups = Hash.new { |hash, key| hash[key] = [] }
         i = 0
         @df.each_row do |row|
@@ -46,5 +51,12 @@ module Rover
         result
       end
     end
+
+    def check_columns
+      raise ArgumentError, "No columns given" if @columns.empty?
+
+      missing_keys = @columns - @df.keys
+      raise ArgumentError, "Missing keys: #{missing_keys.join(", ")}" if missing_keys.any?
+    end
   end
 end

data/lib/rover/vector.rb

@@ -1,23 +1,23 @@
 module Rover
   class Vector
     # if a user never specifies types,
-    # the defaults are bool, float, int, and object
-    # keep these simple
-    #
-    # we could create aliases for float64, int64, uint64
-    # if so, type should still return the simple type
+    # the defaults are bool, float64, int64, and object
     TYPE_CAST_MAPPING = {
       bool: Numo::Bit,
       float32: Numo::SFloat,
-      float: Numo::DFloat,
+      float64: Numo::DFloat,
       int8: Numo::Int8,
       int16: Numo::Int16,
       int32: Numo::Int32,
-      int: Numo::Int64,
+      int64: Numo::Int64,
       object: Numo::RObject,
       uint8: Numo::UInt8,
       uint16: Numo::UInt16,
       uint32: Numo::UInt32,
+      uint64: Numo::UInt64,
+      # legacy - must come last
+      float: Numo::DFloat,
+      int: Numo::Int64,
       uint: Numo::UInt64
     }
 
@@ -333,6 +333,18 @@ module Rover
 
     private
 
+    # for clone
+    def initialize_clone(_)
+      @data = @data.clone
+      super
+    end
+
+    # for dup
+    def initialize_dup(_)
+      @data = @data.dup
+      super
+    end
+
     def cast_data(data, type: nil)
       numo_type = numo_type(type) if type
 
@@ -359,6 +371,7 @@ module Rover
         data = data.to_a
 
         if type
+          data = data.map { |v| v || Float::NAN } if [:float, :float32].include?(type)
           data = numo_type.cast(data)
         else
           data =

data/lib/rover/version.rb

@@ -1,3 +1,3 @@
 module Rover
-  VERSION = "0.2.6"
+  VERSION = "0.3.0"
 end

data/lib/rover.rb

@@ -9,47 +9,58 @@ require "rover/version"
 
 module Rover
   class << self
-    def read_csv(path, types: nil, **options)
-      require "csv"
-      csv_to_df(CSV.read(path, **csv_options(options)), types: types, headers: options[:headers])
+    def read_csv(path, **options)
+      csv_to_df(**options) do |csv_options|
+        CSV.read(path, **csv_options)
+      end
     end
 
-    def parse_csv(str, types: nil, **options)
-      require "csv"
-      csv_to_df(CSV.parse(str, **csv_options(options)), types: types, headers: options[:headers])
+    def parse_csv(str, **options)
+      csv_to_df(**options) do |csv_options|
+        CSV.parse(str, **csv_options)
+      end
     end
 
-    def read_parquet(path)
-      require "parquet"
-      parquet_to_df(Arrow::Table.load(path))
+    def read_parquet(path, **options)
+      parquet_to_df(**options) do
+        Arrow::Table.load(path)
+      end
     end
 
-    def parse_parquet(str)
-      require "parquet"
-      parquet_to_df(Arrow::Table.load(Arrow::Buffer.new(str), format: :parquet))
+    def parse_parquet(str, **options)
+      parquet_to_df(**options) do
+        Arrow::Table.load(Arrow::Buffer.new(str), format: :parquet)
+      end
     end
 
     private
 
-    # TODO use date converter
-    def csv_options(options)
-      options = {headers: true, converters: :numeric}.merge(options)
-      raise ArgumentError, "Must specify headers" unless options[:headers]
-      options
-    end
+    def csv_to_df(types: nil, headers: nil, **csv_options)
+      require "csv"
 
-    def csv_to_df(table, types: nil, headers: nil)
-      if headers && headers.size < table.headers.size
-        raise ArgumentError, "Expected #{table.headers.size} headers, got #{headers.size}"
+      raise ArgumentError, "Must specify headers" if headers == false
+
+      # TODO use date converter in 0.4.0 - need to test performance
+      table = yield({converters: :numeric}.merge(csv_options))
+
+      headers = nil if headers == true
+      if headers && table.first && headers.size != table.first.size
+        raise ArgumentError, "Expected #{table.first.size} headers, given #{headers.size}"
+      end
+
+      table_headers = (headers || table.shift || []).dup
+      # keep same behavior as headers: true
+      if table.first
+        while table_headers.size < table.first.size
+          table_headers << nil
+        end
       end
 
-      table.by_col!
       data = {}
-      keys = table.map { |k, _| [k, true] }.to_h
+      keys = table_headers.map { |k| [k, true] }.to_h
       unnamed_suffix = 1
-      table.each do |k, v|
-        # TODO do same for empty string in 0.3.0
-        if k.nil?
+      table_headers.each_with_index do |k, i|
+        if k.nil? || k.empty?
           k = "unnamed"
           while keys.include?(k)
             unnamed_suffix += 1
@@ -57,13 +68,25 @@ module Rover
           end
           keys[k] = true
         end
-        data[k] = v
+        table_headers[i] = k
+      end
+
+      table_headers.each_with_index do |k, i|
+        # use first value for duplicate headers like headers: true
+        next if data[k]
+
+        values = []
+        table.each do |row|
+          values << row[i]
+        end
+        data[k] = values
       end
 
       DataFrame.new(data, types: types)
     end
 
     PARQUET_TYPE_MAPPING = {
+      "bool" => Numo::Bit,
       "float" => Numo::SFloat,
       "double" => Numo::DFloat,
       "int8" => Numo::Int8,
@@ -77,15 +100,31 @@ module Rover
       "uint64" => Numo::UInt64
     }
 
-    def parquet_to_df(table)
+    def parquet_to_df(types: nil)
+      require "parquet"
+
+      table = yield
       data = {}
+      types ||= {}
       table.each_column do |column|
         k = column.field.name
-        type = column.field.data_type.to_s
-        numo_type = PARQUET_TYPE_MAPPING[type]
-        raise "Unknown type: #{type}" unless numo_type
-        # TODO improve performance
-        data[k] = numo_type.cast(column.data.values)
+        if types[k]
+          data[k] = Vector.new(column.data.values, type: types[k])
+        else
+          type = column.field.data_type.to_s
+          numo_type = PARQUET_TYPE_MAPPING[type]
+          raise "Unknown type: #{type}" unless numo_type
+
+          # TODO automatic conversion?
+          # int => float
+          # bool => object
+          if (type.include?("int") || type == "bool") && column.n_nulls > 0
+            raise "Nulls not supported for #{type} column: #{k}"
+          end
+
+          # TODO improve performance
+          data[k] = numo_type.cast(column.data.values)
+        end
       end
       DataFrame.new(data)
     end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rover-df
 version: !ruby/object:Gem::Version
-  version: 0.2.6
+  version: 0.3.0
 platform: ruby
 authors:
 - Andrew Kane
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-10-27 00:00:00.000000000 Z
+date: 2022-04-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: numo-narray
@@ -51,14 +51,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '2.4'
+      version: '2.7'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.2.22
+rubygems_version: 3.3.7
 signing_key:
 specification_version: 4
 summary: Simple, powerful data frames for Ruby