red-arrow 0.4.1 → 0.8.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 244f843a8c061c6853ae9325eb442adada478f30
-  data.tar.gz: 304840d2642191a717b21695db9012b8c8037fe3
+  metadata.gz: add6b81f8b6fe0d623d022d16b358b756bf0a559
+  data.tar.gz: ffa5678bd352df22212c7dd3d3b6beea6491bbde
 SHA512:
-  metadata.gz: c8cb38809f2a5532569bd07fa1c39471887f79ce330faf5bc0c1543912cce125926d6e0d042c3a302c91ebbfe85a4f0cc56e639b109d04a656152dd0bbd733fe
-  data.tar.gz: 7e0c529708ae390dbd3a275308c7e14f5c7c1484ca630951c415ad0102b09a0aa5decc32875cbc74554f291aab2eed1e144ca14d07ef58d8b6121e0bd9db86ba
+  metadata.gz: da4959b8123e205d5fd16d17704ecc1e93d29659294575ac6c5797e7f578b8ce0c43710f344c909a70b0b61def35a5d1f9d6c1d1de59679d3ad6bb037df2671f
+  data.tar.gz: 11c7f909073eaa609aa2402a629016a27ecac19a0ac85a54468fba5d5ea433b23c41cfa6179ff176363d17c61f86b2233812440c3bb26d32459a145d14d7d06a

data/README.md

@@ -20,7 +20,7 @@ gobject-introspection gem is a Ruby bindings of GObject Introspection. Red Arrow
 
 ## Install
 
-Install Arrow GLib before install Red Arrow. Use [Apache Arrow packages](https://github.com/red-data-tools/arrow-packages) for installing Arrow GLib.
+Install Arrow GLib before install Red Arrow. Use [packages.red-data-tools.org](https://github.com/red-data-tools/packages.red-data-tools.org) for installing Arrow GLib.
 
 Install Red Arrow after you install Arrow GLib:
 
@@ -50,7 +50,8 @@ require "arrow"
 
 ## License
 
-Apache License 2.0. See doc/text/apache-2.0.txt for details.
+Apache License 2.0. See `doc/text/apache-2.0.txt` and `NOTICE` for
+details.
 
 (Kouhei Sutou has a right to change the license including contributed
 patches.)

data/dependency-check/Rakefile

@@ -30,7 +30,7 @@ end
 namespace :dependency do
   desc "Check dependency"
   task :check do
-    unless PKGConfig.check_version?("arrow-glib")
+    unless PKGConfig.check_version?("arrow-glib", 0, 8, 0)
       unless NativePackageInstaller.install(:debian => "libarrow-glib-dev",
                                             :redhat => "arrow-glib-devel")
         exit(false)

data/doc/text/news.md

@@ -1,6 +1,42 @@
 # News
 
-## 0.7.0 - 2017-09-19
+## 0.8.0 - 2018-01-04
+
+### Improvements
+
+  * Required Apache Arrow 0.8.0.
+
+  * Update README. [GitHub#5][Patch by mikisou]
+
+  * `Arrow::Table#each_record_batch`: Added.
+
+  * `Arrow::ArrayBuilder#build`: Added.
+
+  * `Arrow::CSVReader`: Added.
+
+  * `Arrow::Array#[]`: Added `NULL` support.
+
+  * `Arrow::TimestampArray`: Added.
+
+  * `Arrow::Table#to_s`: Added table style format.
+
+  * `Arrow::Table#slice`: Added.
+
+  * `Arrow::Table#[]`: Added.
+
+  * `Arrow::Table`: Added dynamic column name reader.
+
+  * `Arrow::Table#merge`: Added.
+
+  * `Arrow::Table#remove_column`: Added column name support.
+
+  * `Arrow::Table#select_columns`: Added.
+
+### Thanks
+
+  * mikisou
+
+## 0.4.1 - 2017-09-19
 
 ### Improvements
 

data/lib/arrow/array-builder.rb

@@ -17,50 +17,54 @@ module Arrow
     class << self
       def build(values)
         builder = new
-        if builder.respond_to?(:append_values)
-          start_index = 0
-          current_index = 0
-          status = :value
-          values.each do |value|
-            if value.nil?
-              if status == :value
-                if start_index != current_index
-                  builder.append_values(values[start_index...current_index])
-                  start_index = current_index
-                end
-                status = :null
-              end
-            else
-              if status == :null
-                builder.append_nulls(current_index - start_index)
+        builder.build(values)
+      end
+    end
+
+    def build(values)
+      if respond_to?(:append_values)
+        start_index = 0
+        current_index = 0
+        status = :value
+        values.each do |value|
+          if value.nil?
+            if status == :value
+              if start_index != current_index
+                append_values(values[start_index...current_index])
                 start_index = current_index
-                status = :value
               end
+              status = :null
             end
-            current_index += 1
-          end
-          if start_index != current_index
-            if status == :value
-              if start_index == 0 and current_index == values.size
-                builder.append_values(values)
-              else
-                builder.append_values(values[start_index...current_index])
-              end
-            else
-              builder.append_nulls(current_index - start_index)
+          else
+            if status == :null
+              append_nulls(current_index - start_index)
+              start_index = current_index
+              status = :value
             end
           end
-        else
-          values.each do |value|
-            if value.nil?
-              builder.append_null
+          current_index += 1
+        end
+        if start_index != current_index
+          if status == :value
+            if start_index == 0 and current_index == values.size
+              append_values(values)
             else
-              builder.append(value)
+              append_values(values[start_index...current_index])
             end
+          else
+            append_nulls(current_index - start_index)
+          end
+        end
+      else
+        values.each do |value|
+          if value.nil?
+            append_null
+          else
+            append(value)
           end
         end
-        builder.finish
       end
+      finish
     end
   end
 end

data/lib/arrow/array.rb

@@ -28,13 +28,19 @@ module Arrow
       end
     end
 
+    def [](i)
+      if null?(i)
+        nil
+      else
+        get_value(i)
+      end
+    end
+
     def each
+      return to_enum(__method__) unless block_given?
+
       length.times do |i|
-        if null?(i)
-          yield(nil)
-        else
-          yield(self[i])
-        end
+        yield(self[i])
       end
     end
 

data/lib/arrow/chunked-array.rb

@@ -16,13 +16,29 @@ module Arrow
   class ChunkedArray
     include Enumerable
 
+    def [](i)
+      n_chunks.times do |j|
+        array = get_chunk(j)
+        return array[i] if i < array.length
+        i -= array.length
+      end
+      nil
+    end
+
     def each(&block)
       return to_enum(__method__) unless block_given?
 
-      n_chunks.times do |i|
-        array = get_chunk(i)
+      each_chunk do |array|
         array.each(&block)
       end
     end
+
+    def each_chunk
+      return to_enum(__method__) unless block_given?
+
+      n_chunks.times do |i|
+        yield(get_chunk(i))
+      end
+    end
   end
 end

data/lib/arrow/column.rb

@@ -16,6 +16,10 @@ module Arrow
   class Column
     include Enumerable
 
+    def [](i)
+      data[i]
+    end
+
     def each(&block)
       return to_enum(__method__) unless block_given?
 

data/lib/arrow/csv-reader.rb

@@ -0,0 +1,162 @@
+# Copyright 2017 Kouhei Sutou <kou@clear-code.com>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require "csv"
+require "pathname"
+require "time"
+
+module Arrow
+  class CSVReader
+    class << self
+      def read(csv, **options)
+        case csv
+        when Pathname
+          path = csv.to_path
+          options = update_csv_parse_options(options, :open_csv, path)
+          open_csv(path, **options) do |_csv|
+            read(_csv)
+          end
+        when /\A.+\.csv\z/i
+          read(Pathname.new(csv), **options)
+        when String
+          options = update_csv_parse_options(options, :parse_csv_data, csv)
+          parse_csv_data(csv, **options) do |_csv|
+            read(_csv)
+          end
+        else
+          new(csv).read
+        end
+      end
+
+      private
+      def open_csv(path, **options)
+        CSV.open(path, **options) do |csv|
+          yield(csv)
+        end
+      end
+
+      def parse_csv_data(data, **options)
+        csv = CSV.new(data, **options)
+        begin
+          yield(csv)
+        ensure
+          csv.close
+        end
+      end
+
+      ISO8601_CONVERTER = lambda do |field|
+        begin
+          encoded_field = field.encode(CSV::ConverterEncoding)
+        rescue EncodingError
+          field
+        else
+          begin
+            Time.iso8601(encoded_field)
+          rescue ArgumentError
+            field
+          end
+        end
+      end
+
+      def update_csv_parse_options(options, create_csv, *args)
+        return options unless options.empty?
+
+        new_options = options.merge(converters: [:all, ISO8601_CONVERTER])
+        __send__(create_csv, *args, **new_options) do |csv|
+          row1 = csv.shift
+          if row1.nil?
+            new_options[:headers] = false
+            return new_options
+          end
+          if row1.any?(&:nil?)
+            new_options[:headers] = false
+            return new_options
+          end
+
+          row2 = csv.shift
+          return new_options if row2.nil?
+          if row2.any?(&:nil?)
+            new_options[:headers] = true
+            return new_options
+          end
+
+          if row1.collect(&:class) != row2.collect(&:class)
+            new_options[:headers] = true
+            return new_options
+          end
+
+          new_options
+        end
+      end
+    end
+
+    def initialize(csv)
+      @csv = csv
+    end
+
+    def read
+      builders = []
+      values_set = []
+      @csv.each do |row|
+        if row.is_a?(CSV::Row)
+          row = row.collect(&:last)
+        end
+        row.each_with_index do |value, i|
+          builders[i] ||= create_builder(value)
+          values = (values_set[i] ||= [])
+          case value
+          when Time
+            value = value.to_i * (10 ** 9) + value.nsec
+          end
+          values << value
+        end
+      end
+      return nil if values_set.empty?
+
+      arrays = values_set.collect.with_index do |values, i|
+        builders[i].build(values)
+      end
+      if @csv.headers
+        names = @csv.headers
+      else
+        names = builders.size.times.collect(&:to_s)
+      end
+      fields = names.collect.with_index do |name, i|
+        Arrow::Field.new(name, arrays[i].value_data_type)
+      end
+      schema = Schema.new(fields)
+      columns = arrays.collect.with_index do |array, i|
+        Column.new(fields[i], array)
+      end
+      Table.new(schema, columns)
+    end
+
+    private
+    def create_builder(sample_value)
+      case sample_value
+      when Integer
+        IntArrayBuilder.new
+      when Float
+        DoubleArrayBuilder.new
+      when String
+        StringArrayBuilder.new
+      when Time
+        data_type = TimestampDataType.new(:nano)
+        TimestampArrayBuilder.new(data_type)
+      else
+        nil
+      end
+    end
+  end
+end

data/lib/arrow/loader.rb

@@ -33,10 +33,14 @@ module Arrow
       require "arrow/buffer"
       require "arrow/chunked-array"
       require "arrow/column"
+      require "arrow/csv-reader"
       require "arrow/field"
       require "arrow/record-batch"
+      require "arrow/slicer"
       require "arrow/table"
+      require "arrow/table-formatter"
       require "arrow/tensor"
+      require "arrow/timestamp-array"
 
       require "arrow/record-batch-file-reader"
       require "arrow/record-batch-stream-reader"
@@ -57,22 +61,20 @@ module Arrow
       case klass.name
       when "Arrow::StringArray"
         case method_name
-        when "[]"
-          method_name = "get_value"
+        when "get_value"
+          method_name = "get_raw_value"
         when "get_string"
-          method_name = "[]"
+          method_name = "get_value"
+        end
+        super(info, klass, method_name)
+      when "Arrow::TimestampArray"
+        case method_name
+        when "get_value"
+          method_name = "get_raw_value"
         end
         super(info, klass, method_name)
       else
-        super
-      end
-    end
-
-    def rubyish_method_name(function_info, options={})
-      if function_info.n_in_args == 1 and function_info.name == "get_value"
-        "[]"
-      else
-        super
+       super
       end
     end
   end