rover-df 0.2.3 → 0.2.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b8ac8c0dda5ee8ea5482b5d52927446e52a60151c05959324970b6b420c6b825
-  data.tar.gz: cbabf42c40195303fa62a85b40c3d516dff7cb56a4059c2ab6867921fae62bb9
+  metadata.gz: c75bed3989211e806e54c296290e5f7b3af236a15742daac876e211e3ca5a76f
+  data.tar.gz: 5865ff8f1d0036423f18cfee867da63214ee50f79d373b0f0f244853d8efbefa
 SHA512:
-  metadata.gz: 2b906f49a0accbbf4682216808faf3113c3f31c24e9e434a03f996d8e8e9b4db1c8ca0ccfb3f604e798261f97d88b26a5376bace349f230b5eda5949b492fb88
-  data.tar.gz: 8f3d590c6df3d588f92c6c84b327211a3dce6b27452b4a1161492ca90dc87cfd6aad02a3c7ef038a9c6cb69155558f2a332acbfd65a9bb4ba1d220333b051872
+  metadata.gz: 11718bc8ade75a605e92cabe05c29e55c6d4dfe427cd5ada0a8a216db678b32a88f4a43843d1e7dcda7b7a64adb63b76969f1d958e91ca57c4f71989632e14aa
+  data.tar.gz: 16940236090625bef69cb14d6d9f9f50720314edea1b5892f60443799e5389700ddfb0d79a29ee1e193168097add9d7195799e7f049d85f9c9dc9c443843a678

data/CHANGELOG.md

@@ -1,3 +1,25 @@
+## 0.2.7 (2022-01-16)
+
+- Added support for booleans to Parquet methods
+- Added support for creating data frames from `ActiveRecord::Result`
+- Added `types` option to `read_parquet` and `parse_parquet` methods
+
+## 0.2.6 (2021-10-27)
+
+- Added support for `nil` headers to `read_csv` and `parse_csv`
+- Added `read_parquet`, `parse_parquet`, and `to_parquet` methods
+
+## 0.2.5 (2021-09-25)
+
+- Fixed column types with joins
+
+## 0.2.4 (2021-06-03)
+
+- Added grouping for `std` and `var`
+- Fixed `==` for data frames
+- Fixed error with `first` and `last` for data frames
+- Fixed error with `last` when vector size is smaller than `n`
+
 ## 0.2.3 (2021-02-08)
 
 - Added `select`, `reject`, and `map!` methods to vectors

data/LICENSE.txt

@@ -1,4 +1,4 @@
-Copyright (c) 2020-2021 Andrew Kane
+Copyright (c) 2020-2022 Andrew Kane
 
 MIT License
 

data/README.md

@@ -20,7 +20,7 @@ gem 'rover-df'
 
 A data frame is an in-memory table. It’s a useful data structure for data analysis and machine learning. It uses columnar storage for fast operations on columns.
 
-Try it out for forecasting by clicking the button below:
+Try it out for forecasting by clicking the button below (it can take a few minutes to start):
 
 [![Binder](https://mybinder.org/badge_logo.svg)](https://mybinder.org/v2/gh/ankane/ml-stack/master?filepath=Forecasting.ipynb)
 
@@ -61,6 +61,14 @@ Rover.read_csv("file.csv")
 Rover.parse_csv("CSV,data,string")
 ```
 
+From Parquet (requires the [red-parquet](https://github.com/apache/arrow/tree/master/ruby/red-parquet) gem)
+
+```ruby
+Rover.read_parquet("file.parquet")
+# or
+Rover.parse_parquet("PAR1...")
+```
+
 ## Attributes
 
 Get number of rows
@@ -89,7 +97,7 @@ Select a column
 df[:a]
 ```
 
-> Note that strings and symbols are different keys, just like hashes
+> Note that strings and symbols are different keys, just like hashes. Creating a data frame from Active Record, a CSV, or Parquet uses strings.
 
 Select multiple columns
 
@@ -123,6 +131,20 @@ df[1..3]
 df[[1, 4, 5]]
 ```
 
+Iterate over rows
+
+```ruby
+df.each_row { |row| ... }
+```
+
+Iterate over a column
+
+```ruby
+df[:a].each { |item| ... }
+# or
+df[:a].each_with_index { |item, index| ... }
+```
+
 ## Filtering
 
 Filter on a condition
@@ -181,6 +203,8 @@ df[:a].median
 df[:a].percentile(90)
 df[:a].min
 df[:a].max
+df[:a].std
+df[:a].var
 ```
 
 Count occurrences
@@ -259,6 +283,14 @@ df[:a][0..2] = 1
 df[:a][0..2] = [1, 2, 3]
 ```
 
+Update all elements
+
+```ruby
+df[:a] = df[:a].map { |v| v.gsub("a", "b") }
+# or
+df[:a].map! { |v| v.gsub("a", "b") }
+```
+
 Update elements matching a condition
 
 ```ruby
@@ -369,6 +401,12 @@ CSV
 df.to_csv
 ```
 
+Parquet (requires the [red-parquet](https://github.com/apache/arrow/tree/master/ruby/red-parquet) gem)
+
+```ruby
+df.to_parquet
+```
+
 ## Types
 
 You can specify column types when creating a data frame

data/lib/rover/data_frame.rb

@@ -40,8 +40,8 @@ module Rover
         vectors.each do |k, v|
           @vectors[k] = to_vector(v, type: types[k])
         end
-      elsif defined?(ActiveRecord) && (data.is_a?(ActiveRecord::Relation) || (data.is_a?(Class) && data < ActiveRecord::Base))
-        result = data.connection.select_all(data.all.to_sql)
+      elsif defined?(ActiveRecord) && (data.is_a?(ActiveRecord::Relation) || (data.is_a?(Class) && data < ActiveRecord::Base) || data.is_a?(ActiveRecord::Result))
+        result = data.is_a?(ActiveRecord::Result) ? data : data.connection.select_all(data.all.to_sql)
         result.columns.each_with_index do |k, i|
           @vectors[k] = to_vector(result.rows.map { |r| r[i] }, type: types[k])
         end
@@ -163,7 +163,7 @@ module Rover
       last(n)
     end
 
-    def first(n = nil)
+    def first(n = 1)
       new_vectors = {}
       @vectors.each do |k, v|
         new_vectors[k] = v.first(n)
@@ -171,7 +171,7 @@ module Rover
       DataFrame.new(new_vectors)
     end
 
-    def last(n = nil)
+    def last(n = 1)
       new_vectors = {}
       @vectors.each do |k, v|
         new_vectors[k] = v.last(n)
@@ -235,6 +235,44 @@ module Rover
       end
     end
 
+    def to_parquet
+      require "parquet"
+
+      schema = {}
+      types.each do |name, type|
+        schema[name] =
+          case type
+          when :int
+            :int64
+          when :uint
+            :uint64
+          when :float
+            :double
+          when :float32
+            :float
+          when :bool
+            :boolean
+          when :object
+            if @vectors[name].all? { |v| v.is_a?(String) }
+              :string
+            else
+              raise "Unknown type"
+            end
+          else
+            type
+          end
+      end
+      # TODO improve performance
+      raw_records = []
+      size.times do |i|
+        raw_records << @vectors.map { |_, v| v[i] }
+      end
+      table = Arrow::Table.new(schema, raw_records)
+      buffer = Arrow::ResizableBuffer.new(1024)
+      table.save(buffer, format: :parquet)
+      buffer.data.to_s
+    end
+
     # for IRuby
     def to_html
       require "iruby"
@@ -301,7 +339,7 @@ module Rover
       Group.new(self, columns.flatten)
     end
 
-    [:max, :min, :median, :mean, :percentile, :sum].each do |name|
+    [:max, :min, :median, :mean, :percentile, :sum, :std, :var].each do |name|
       define_method(name) do |column, *args|
         check_column(column)
         self[column].send(name, *args)
@@ -360,7 +398,7 @@ module Rover
     def ==(other)
       size == other.size &&
       keys == other.keys &&
-      keys.all? { |k| self[k] == other[k] }
+      keys.all? { |k| self[k].to_numo == other[k].to_numo }
     end
 
     def plot(x = nil, y = nil, type: nil)
@@ -475,10 +513,12 @@ module Rover
 
       left = how == "left"
 
+      types = {}
       vectors = {}
       keys = (self.keys + other.keys).uniq
       keys.each do |k|
         vectors[k] = []
+        types[k] = join_type(self.types[k], other.types[k])
       end
 
       each_row do |r|
@@ -498,7 +538,7 @@ module Rover
         end
       end
 
-      DataFrame.new(vectors)
+      DataFrame.new(vectors, types: types)
     end
 
     def check_join_keys(df, keys)
@@ -523,6 +563,19 @@ module Rover
       end
     end
 
+    def join_type(a, b)
+      if a.nil?
+        b
+      elsif b.nil?
+        a
+      elsif a == b
+        a
+      else
+        # TODO specify
+        nil
+      end
+    end
+
     def to_vector(v, size: nil, type: nil)
       if v.is_a?(Vector)
         v = v.to(type) if type && v.type != type

data/lib/rover/group.rb

@@ -9,7 +9,7 @@ module Rover
       Group.new(@df, @columns + columns.flatten)
     end
 
-    [:count, :max, :min, :mean, :median, :percentile, :sum].each do |name|
+    [:count, :max, :min, :mean, :median, :percentile, :sum, :std, :var].each do |name|
       define_method(name) do |*args|
         n = [name, args.first].compact.join("_")
 

data/lib/rover/vector.rb

@@ -263,7 +263,11 @@ module Rover
     end
 
     def last(n = 1)
-      Vector.new(@data[-n..-1])
+      if n >= size
+        Vector.new(@data)
+      else
+        Vector.new(@data[-n..-1])
+      end
     end
 
     def take(n)
@@ -355,6 +359,7 @@ module Rover
         data = data.to_a
 
         if type
+          data = data.map { |v| v || Float::NAN } if [:float, :float32].include?(type)
           data = numo_type.cast(data)
         else
           data =

data/lib/rover/version.rb

@@ -1,3 +1,3 @@
 module Rover
-  VERSION = "0.2.3"
+  VERSION = "0.2.7"
 end

data/lib/rover.rb

@@ -19,6 +19,16 @@ module Rover
       csv_to_df(CSV.parse(str, **csv_options(options)), types: types, headers: options[:headers])
     end
 
+    def read_parquet(path, types: nil)
+      require "parquet"
+      parquet_to_df(Arrow::Table.load(path), types: types)
+    end
+
+    def parse_parquet(str, types: nil)
+      require "parquet"
+      parquet_to_df(Arrow::Table.load(Arrow::Buffer.new(str), format: :parquet), types: types)
+    end
+
     private
 
     # TODO use date converter
@@ -35,10 +45,63 @@ module Rover
 
       table.by_col!
       data = {}
+      keys = table.map { |k, _| [k, true] }.to_h
+      unnamed_suffix = 1
       table.each do |k, v|
+        # TODO do same for empty string in 0.3.0
+        if k.nil?
+          k = "unnamed"
+          while keys.include?(k)
+            unnamed_suffix += 1
+            k = "unnamed#{unnamed_suffix}"
+          end
+          keys[k] = true
+        end
         data[k] = v
       end
+
       DataFrame.new(data, types: types)
     end
+
+    PARQUET_TYPE_MAPPING = {
+      "bool" => Numo::Bit,
+      "float" => Numo::SFloat,
+      "double" => Numo::DFloat,
+      "int8" => Numo::Int8,
+      "int16" => Numo::Int16,
+      "int32" => Numo::Int32,
+      "int64" => Numo::Int64,
+      "string" => Numo::RObject,
+      "uint8" => Numo::UInt8,
+      "uint16" => Numo::UInt16,
+      "uint32" => Numo::UInt32,
+      "uint64" => Numo::UInt64
+    }
+
+    def parquet_to_df(table, types: nil)
+      data = {}
+      types ||= {}
+      table.each_column do |column|
+        k = column.field.name
+        if types[k]
+          data[k] = Vector.new(column.data.values, type: types[k])
+        else
+          type = column.field.data_type.to_s
+          numo_type = PARQUET_TYPE_MAPPING[type]
+          raise "Unknown type: #{type}" unless numo_type
+
+          # TODO automatic conversion?
+          # int => float
+          # bool => object
+          if (type.include?("int") || type == "bool") && column.n_nulls > 0
+            raise "Nulls not supported for #{type} column: #{k}"
+          end
+
+          # TODO improve performance
+          data[k] = numo_type.cast(column.data.values)
+        end
+      end
+      DataFrame.new(data)
+    end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rover-df
 version: !ruby/object:Gem::Version
-  version: 0.2.3
+  version: 0.2.7
 platform: ruby
 authors:
 - Andrew Kane
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-02-08 00:00:00.000000000 Z
+date: 2022-01-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: numo-narray
@@ -58,7 +58,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.2.3
+rubygems_version: 3.3.3
 signing_key:
 specification_version: 4
 summary: Simple, powerful data frames for Ruby