rover-df 0.2.6 → 0.2.7

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 01e2a90ba133ae07ad6ad482bdca985df806d6a073fa2d93029b2b7e1b55dc49
-  data.tar.gz: 96f4171420dea68b38cffdd5a365657bc464f6d1f0c4f6bf1aefb20377c56179
+  metadata.gz: c75bed3989211e806e54c296290e5f7b3af236a15742daac876e211e3ca5a76f
+  data.tar.gz: 5865ff8f1d0036423f18cfee867da63214ee50f79d373b0f0f244853d8efbefa
 SHA512:
-  metadata.gz: 2451d6844c7ece459e61c8e1499047f8efd6472a0d57317b7e2e1110527d843e8177c16ccbb1aeb0fd61e647fdd4291ebf73d4bfe008560eff7b963b1ac22ee6
-  data.tar.gz: 18ad0cfb8fc22aeb63d2e1b11333b1a5989c7bcc0f2b5fbebedb11acf3d3dc26e7235109e1501d0e4b9a06b5aa7e47b71bdb23de6d6fcd87c9fb53d2bf0be330
+  metadata.gz: 11718bc8ade75a605e92cabe05c29e55c6d4dfe427cd5ada0a8a216db678b32a88f4a43843d1e7dcda7b7a64adb63b76969f1d958e91ca57c4f71989632e14aa
+  data.tar.gz: 16940236090625bef69cb14d6d9f9f50720314edea1b5892f60443799e5389700ddfb0d79a29ee1e193168097add9d7195799e7f049d85f9c9dc9c443843a678

data/CHANGELOG.md

@@ -1,3 +1,9 @@
+## 0.2.7 (2022-01-16)
+
+- Added support for booleans to Parquet methods
+- Added support for creating data frames from `ActiveRecord::Result`
+- Added `types` option to `read_parquet` and `parse_parquet` methods
+
 ## 0.2.6 (2021-10-27)
 
 - Added support for `nil` headers to `read_csv` and `parse_csv`

data/LICENSE.txt

@@ -1,4 +1,4 @@
-Copyright (c) 2020-2021 Andrew Kane
+Copyright (c) 2020-2022 Andrew Kane
 
 MIT License
 

data/README.md

@@ -61,7 +61,7 @@ Rover.read_csv("file.csv")
 Rover.parse_csv("CSV,data,string")
 ```
 
-From Parquet (requires the [red-parquet](https://github.com/apache/arrow/tree/master/ruby/red-parquet) gem) [unreleased]
+From Parquet (requires the [red-parquet](https://github.com/apache/arrow/tree/master/ruby/red-parquet) gem)
 
 ```ruby
 Rover.read_parquet("file.parquet")
@@ -401,7 +401,7 @@ CSV
 df.to_csv
 ```
 
-Parquet (requires the [red-parquet](https://github.com/apache/arrow/tree/master/ruby/red-parquet) gem) [unreleased]
+Parquet (requires the [red-parquet](https://github.com/apache/arrow/tree/master/ruby/red-parquet) gem)
 
 ```ruby
 df.to_parquet

data/lib/rover/data_frame.rb

@@ -40,8 +40,8 @@ module Rover
         vectors.each do |k, v|
           @vectors[k] = to_vector(v, type: types[k])
         end
-      elsif defined?(ActiveRecord) && (data.is_a?(ActiveRecord::Relation) || (data.is_a?(Class) && data < ActiveRecord::Base))
-        result = data.connection.select_all(data.all.to_sql)
+      elsif defined?(ActiveRecord) && (data.is_a?(ActiveRecord::Relation) || (data.is_a?(Class) && data < ActiveRecord::Base) || data.is_a?(ActiveRecord::Result))
+        result = data.is_a?(ActiveRecord::Result) ? data : data.connection.select_all(data.all.to_sql)
         result.columns.each_with_index do |k, i|
           @vectors[k] = to_vector(result.rows.map { |r| r[i] }, type: types[k])
         end
@@ -250,6 +250,8 @@ module Rover
             :double
           when :float32
             :float
+          when :bool
+            :boolean
           when :object
             if @vectors[name].all? { |v| v.is_a?(String) }
               :string

data/lib/rover/vector.rb

@@ -359,6 +359,7 @@ module Rover
         data = data.to_a
 
         if type
+          data = data.map { |v| v || Float::NAN } if [:float, :float32].include?(type)
           data = numo_type.cast(data)
         else
           data =

data/lib/rover/version.rb

@@ -1,3 +1,3 @@
 module Rover
-  VERSION = "0.2.6"
+  VERSION = "0.2.7"
 end

data/lib/rover.rb

@@ -19,14 +19,14 @@ module Rover
       csv_to_df(CSV.parse(str, **csv_options(options)), types: types, headers: options[:headers])
     end
 
-    def read_parquet(path)
+    def read_parquet(path, types: nil)
       require "parquet"
-      parquet_to_df(Arrow::Table.load(path))
+      parquet_to_df(Arrow::Table.load(path), types: types)
     end
 
-    def parse_parquet(str)
+    def parse_parquet(str, types: nil)
       require "parquet"
-      parquet_to_df(Arrow::Table.load(Arrow::Buffer.new(str), format: :parquet))
+      parquet_to_df(Arrow::Table.load(Arrow::Buffer.new(str), format: :parquet), types: types)
     end
 
     private
@@ -64,6 +64,7 @@ module Rover
     end
 
     PARQUET_TYPE_MAPPING = {
+      "bool" => Numo::Bit,
       "float" => Numo::SFloat,
       "double" => Numo::DFloat,
       "int8" => Numo::Int8,
@@ -77,15 +78,28 @@ module Rover
       "uint64" => Numo::UInt64
     }
 
-    def parquet_to_df(table)
+    def parquet_to_df(table, types: nil)
       data = {}
+      types ||= {}
       table.each_column do |column|
         k = column.field.name
-        type = column.field.data_type.to_s
-        numo_type = PARQUET_TYPE_MAPPING[type]
-        raise "Unknown type: #{type}" unless numo_type
-        # TODO improve performance
-        data[k] = numo_type.cast(column.data.values)
+        if types[k]
+          data[k] = Vector.new(column.data.values, type: types[k])
+        else
+          type = column.field.data_type.to_s
+          numo_type = PARQUET_TYPE_MAPPING[type]
+          raise "Unknown type: #{type}" unless numo_type
+
+          # TODO automatic conversion?
+          # int => float
+          # bool => object
+          if (type.include?("int") || type == "bool") && column.n_nulls > 0
+            raise "Nulls not supported for #{type} column: #{k}"
+          end
+
+          # TODO improve performance
+          data[k] = numo_type.cast(column.data.values)
+        end
       end
       DataFrame.new(data)
     end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rover-df
 version: !ruby/object:Gem::Version
-  version: 0.2.6
+  version: 0.2.7
 platform: ruby
 authors:
 - Andrew Kane
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-10-27 00:00:00.000000000 Z
+date: 2022-01-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: numo-narray
@@ -58,7 +58,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.2.22
+rubygems_version: 3.3.3
 signing_key:
 specification_version: 4
 summary: Simple, powerful data frames for Ruby