rover-df 0.2.5 → 0.2.6

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: ca39a558c3c12103f03fed4cb8f007fbd00a1f8e84b839916fd0010aae4613ba
-  data.tar.gz: 43df8cdc415cc036ac383f30b7c91a35b644067a3cb8ea199abd7452b98298d5
+  metadata.gz: 01e2a90ba133ae07ad6ad482bdca985df806d6a073fa2d93029b2b7e1b55dc49
+  data.tar.gz: 96f4171420dea68b38cffdd5a365657bc464f6d1f0c4f6bf1aefb20377c56179
 SHA512:
-  metadata.gz: 2724c7e85ee7921f277be833cf89be638c14cbb37a44411bba86c42cacffe7c0e4b82ea04d4dfb3d694c6429ba41bc8e8c10f7cb40e5d34bf59d14755858735f
-  data.tar.gz: fa860158decbca0a0b35ccb82e6f73d9a513c37b483eca52d140842d5dd255899a2e1ded3ec4375a492b86d3ec09ffa53d4871e05f1fdad39f3d2630215417dc
+  metadata.gz: 2451d6844c7ece459e61c8e1499047f8efd6472a0d57317b7e2e1110527d843e8177c16ccbb1aeb0fd61e647fdd4291ebf73d4bfe008560eff7b963b1ac22ee6
+  data.tar.gz: 18ad0cfb8fc22aeb63d2e1b11333b1a5989c7bcc0f2b5fbebedb11acf3d3dc26e7235109e1501d0e4b9a06b5aa7e47b71bdb23de6d6fcd87c9fb53d2bf0be330

data/CHANGELOG.md

@@ -1,3 +1,8 @@
+## 0.2.6 (2021-10-27)
+
+- Added support for `nil` headers to `read_csv` and `parse_csv`
+- Added `read_parquet`, `parse_parquet`, and `to_parquet` methods
+
 ## 0.2.5 (2021-09-25)
 
 - Fixed column types with joins

data/README.md

@@ -61,6 +61,14 @@ Rover.read_csv("file.csv")
 Rover.parse_csv("CSV,data,string")
 ```
 
+From Parquet (requires the [red-parquet](https://github.com/apache/arrow/tree/master/ruby/red-parquet) gem) [unreleased]
+
+```ruby
+Rover.read_parquet("file.parquet")
+# or
+Rover.parse_parquet("PAR1...")
+```
+
 ## Attributes
 
 Get number of rows
@@ -89,7 +97,7 @@ Select a column
 df[:a]
 ```
 
-> Note that strings and symbols are different keys, just like hashes
+> Note that strings and symbols are different keys, just like hashes. Creating a data frame from Active Record, a CSV, or Parquet uses strings.
 
 Select multiple columns
 
@@ -393,6 +401,12 @@ CSV
 df.to_csv
 ```
 
+Parquet (requires the [red-parquet](https://github.com/apache/arrow/tree/master/ruby/red-parquet) gem) [unreleased]
+
+```ruby
+df.to_parquet
+```
+
 ## Types
 
 You can specify column types when creating a data frame

data/lib/rover/data_frame.rb

@@ -235,6 +235,42 @@ module Rover
       end
     end
 
+    def to_parquet
+      require "parquet"
+
+      schema = {}
+      types.each do |name, type|
+        schema[name] =
+          case type
+          when :int
+            :int64
+          when :uint
+            :uint64
+          when :float
+            :double
+          when :float32
+            :float
+          when :object
+            if @vectors[name].all? { |v| v.is_a?(String) }
+              :string
+            else
+              raise "Unknown type"
+            end
+          else
+            type
+          end
+      end
+      # TODO improve performance
+      raw_records = []
+      size.times do |i|
+        raw_records << @vectors.map { |_, v| v[i] }
+      end
+      table = Arrow::Table.new(schema, raw_records)
+      buffer = Arrow::ResizableBuffer.new(1024)
+      table.save(buffer, format: :parquet)
+      buffer.data.to_s
+    end
+
     # for IRuby
     def to_html
       require "iruby"

data/lib/rover/version.rb

@@ -1,3 +1,3 @@
 module Rover
-  VERSION = "0.2.5"
+  VERSION = "0.2.6"
 end

data/lib/rover.rb

@@ -19,6 +19,16 @@ module Rover
       csv_to_df(CSV.parse(str, **csv_options(options)), types: types, headers: options[:headers])
     end
 
+    def read_parquet(path)
+      require "parquet"
+      parquet_to_df(Arrow::Table.load(path))
+    end
+
+    def parse_parquet(str)
+      require "parquet"
+      parquet_to_df(Arrow::Table.load(Arrow::Buffer.new(str), format: :parquet))
+    end
+
     private
 
     # TODO use date converter
@@ -35,10 +45,49 @@ module Rover
 
       table.by_col!
       data = {}
+      keys = table.map { |k, _| [k, true] }.to_h
+      unnamed_suffix = 1
       table.each do |k, v|
+        # TODO do same for empty string in 0.3.0
+        if k.nil?
+          k = "unnamed"
+          while keys.include?(k)
+            unnamed_suffix += 1
+            k = "unnamed#{unnamed_suffix}"
+          end
+          keys[k] = true
+        end
         data[k] = v
       end
+
       DataFrame.new(data, types: types)
     end
+
+    PARQUET_TYPE_MAPPING = {
+      "float" => Numo::SFloat,
+      "double" => Numo::DFloat,
+      "int8" => Numo::Int8,
+      "int16" => Numo::Int16,
+      "int32" => Numo::Int32,
+      "int64" => Numo::Int64,
+      "string" => Numo::RObject,
+      "uint8" => Numo::UInt8,
+      "uint16" => Numo::UInt16,
+      "uint32" => Numo::UInt32,
+      "uint64" => Numo::UInt64
+    }
+
+    def parquet_to_df(table)
+      data = {}
+      table.each_column do |column|
+        k = column.field.name
+        type = column.field.data_type.to_s
+        numo_type = PARQUET_TYPE_MAPPING[type]
+        raise "Unknown type: #{type}" unless numo_type
+        # TODO improve performance
+        data[k] = numo_type.cast(column.data.values)
+      end
+      DataFrame.new(data)
+    end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rover-df
 version: !ruby/object:Gem::Version
-  version: 0.2.5
+  version: 0.2.6
 platform: ruby
 authors:
 - Andrew Kane
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2021-09-25 00:00:00.000000000 Z
+date: 2021-10-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: numo-narray