red-arrow 0.8.0 → 0.8.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: add6b81f8b6fe0d623d022d16b358b756bf0a559
-  data.tar.gz: ffa5678bd352df22212c7dd3d3b6beea6491bbde
+  metadata.gz: 59cce19c00fbc436f48f29c2d58aca2685330a63
+  data.tar.gz: 280b1305e60c39d74d4fd101dc43387cd398e770
 SHA512:
-  metadata.gz: da4959b8123e205d5fd16d17704ecc1e93d29659294575ac6c5797e7f578b8ce0c43710f344c909a70b0b61def35a5d1f9d6c1d1de59679d3ad6bb037df2671f
-  data.tar.gz: 11c7f909073eaa609aa2402a629016a27ecac19a0ac85a54468fba5d5ea433b23c41cfa6179ff176363d17c61f86b2233812440c3bb26d32459a145d14d7d06a
+  metadata.gz: 3ecba86d0d061186def935c54a999f1c0adf1f109a1687eefd0375792a87f34333e7a46ee7f78fe51b01fbe294166491f300d9e46a8315e19ff47a1cb3f45102
+  data.tar.gz: 49d09794b0d007bf9e65675e133c45cd1a90a3eec748b9c747d2e7d78e4c21d682ba503c696cb61d128fec362fa138ec50b04ab780a0bf554557417ccaf02afd

data/doc/text/development.md

@@ -0,0 +1,15 @@
+# Development
+
+## Naming convention
+
+### Reader and Writer
+
+Reader and Writer require an opened IO stream.
+
+### Loader and Saver
+
+Loader and Saver require a path. They are convenient classes.
+
+Loader opens the path and reads data by Reader.
+
+Writer opens the path and writes data by Writer.

data/doc/text/news.md

@@ -1,5 +1,19 @@
 # News
 
+## 0.8.1 - 2018-01-05
+
+### Improvements
+
+  * `Arrow::ArrayBuilder.build`: Added generic array build support.
+
+  * `Arrow::Table#save`: Added.
+
+  * `Arrow::Table.load`: Added.
+
+  * `Arrow::CSVLoader`: Added.
+
+  * `Arrow::CSVReader.read`: Removed.
+
 ## 0.8.0 - 2018-01-04
 
 ### Improvements

data/lib/arrow/array-builder.rb

@@ -12,16 +12,58 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+require "date"
+
 module Arrow
   class ArrayBuilder
     class << self
       def build(values)
-        builder = new
-        builder.build(values)
+        if self != ArrayBuilder
+          builder = new
+          return builder.build(values)
+        end
+
+        builder_class = nil
+        values.each do |value|
+          case value
+          when nil
+            # Ignore
+            nil
+          when true, false
+            return BooleanArray.new(values)
+          when String
+            return StringArray.new(values)
+          when Float
+            return DoubleArray.new(values)
+          when Integer
+            if value.negative?
+              builder = IntArrayBuilder.new
+              return builder.build(values)
+            else
+              builder_class = UIntArrayBuilder
+            end
+          when Time
+            data_type = TimestampDataType.new(:nano)
+            builder = TimestampArrayBuilder.new(data_type)
+            return builder.build(values)
+          when DateTime
+            return Date64Array.new(values)
+          when Date
+            return Date32Array.new(values)
+          else
+            return StringArray.new(values)
+          end
+        end
+        if builder_class
+          builder_class.new.build(values)
+        else
+          Arrow::StringArray.new(values)
+        end
       end
     end
 
     def build(values)
+      value_convertable = respond_to?(:convert_to_arrow_value, true)
       if respond_to?(:append_values)
         start_index = 0
         current_index = 0
@@ -30,7 +72,13 @@ module Arrow
           if value.nil?
             if status == :value
               if start_index != current_index
-                append_values(values[start_index...current_index])
+                target_values = values[start_index...current_index]
+                if value_convertable
+                  target_values = target_values.collect do |v|
+                    convert_to_arrow_value(v)
+                  end
+                end
+                append_values(target_values)
                 start_index = current_index
               end
               status = :null
@@ -47,10 +95,16 @@ module Arrow
         if start_index != current_index
           if status == :value
             if start_index == 0 and current_index == values.size
-              append_values(values)
+              target_values = values
             else
-              append_values(values[start_index...current_index])
+              target_values = values[start_index...current_index]
+            end
+            if value_convertable
+              target_values = target_values.collect do |v|
+                convert_to_arrow_value(v)
+              end
             end
+            append_values(target_values)
           else
             append_nulls(current_index - start_index)
           end
@@ -60,6 +114,7 @@ module Arrow
           if value.nil?
             append_null
           else
+            value = convert_to_arrow_value(value) if value_convertable
             append(value)
           end
         end

data/lib/arrow/csv-loader.rb

@@ -0,0 +1,206 @@
+# Copyright 2017-2018 Kouhei Sutou <kou@clear-code.com>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require "csv"
+require "pathname"
+require "time"
+
+module Arrow
+  class CSVLoader
+    class << self
+      def load(path_or_data, **options)
+        new(path_or_data, **options).load
+      end
+    end
+
+    def initialize(path_or_data, **options)
+      @path_or_data = path_or_data
+      @options = options
+    end
+
+    def load
+      case @path_or_data
+      when Pathname
+        load_from_path(@path_or_data.to_path)
+      when /\A.+\.csv\z/i
+        load_from_path(@path_or_data)
+      else
+        load_data(@path_or_data)
+      end
+    end
+
+    private
+    def open_csv(path, **options)
+      CSV.open(path, **options) do |csv|
+        yield(csv)
+      end
+    end
+
+    def parse_csv_data(data, **options)
+      csv = CSV.new(data, **options)
+      begin
+        yield(csv)
+      ensure
+        csv.close
+      end
+    end
+
+    def read_csv(csv)
+      reader = CSVReader.new(csv)
+      reader.read
+    end
+
+    def load_from_path(path)
+      options = update_csv_parse_options(@options, :open_csv, path)
+      open_csv(path, **options) do |csv|
+        read_csv(csv)
+      end
+    end
+
+    def load_data(data)
+      options = update_csv_parse_options(@options, :parse_csv_data, data)
+      parse_csv_data(data, **options) do |csv|
+        read_csv(csv)
+      end
+    end
+
+    def selective_converter(target_index)
+      lambda do |field, field_info|
+        if target_index.nil? or field_info.index == target_index
+          yield(field)
+        else
+          field
+        end
+      end
+    end
+
+    BOOLEAN_CONVERTER = lambda do |field|
+      begin
+        encoded_field = field.encode(CSV::ConverterEncoding)
+      rescue EncodingError
+        field
+      else
+        case encoded_field
+        when "true"
+          true
+        when "false"
+          false
+        else
+          field
+        end
+      end
+    end
+
+    ISO8601_CONVERTER = lambda do |field|
+      begin
+        encoded_field = field.encode(CSV::ConverterEncoding)
+      rescue EncodingError
+        field
+      else
+        begin
+          Time.iso8601(encoded_field)
+        rescue ArgumentError
+          field
+        end
+      end
+    end
+
+    def update_csv_parse_options(options, create_csv, *args)
+      return options unless options.empty?
+
+      converters = [:all, BOOLEAN_CONVERTER, ISO8601_CONVERTER]
+      new_options = options.merge(converters: converters)
+      __send__(create_csv, *args, **new_options) do |csv|
+        new_options[:headers] = have_header?(csv)
+      end
+      __send__(create_csv, *args, **new_options) do |csv|
+        new_options[:converters] = detect_robust_converters(csv)
+      end
+      return new_options
+    end
+
+    def have_header?(csv)
+      row1 = csv.shift
+      return false if row1.nil?
+      return false if row1.any?(&:nil?)
+
+      row2 = csv.shift
+      return nil if row2.nil?
+      return true if row2.any?(&:nil?)
+
+      if row1.collect(&:class) != row2.collect(&:class)
+        return true
+      end
+
+      nil
+    end
+
+    def detect_robust_converters(csv)
+      column_types = []
+      csv.each do |row|
+        row.each_with_index do |(_name, value), i|
+          current_column_type = column_types[i]
+          next if current_column_type == :string
+
+          candidate_type = nil
+          case value
+          when nil
+            next
+          when "true", "false", true, false
+            candidate_type = :boolean
+          when Integer
+            candidate_type = :integer
+          when Float
+            candidate_type = :float
+            if current_column_type == :integer
+              column_types[i] = candidate_type
+            end
+          when Time
+            candidate_type = :time
+          when DateTime
+            candidate_type = :date_time
+          when Date
+            candidate_type = :date
+          else
+            candidate_type = :string
+          end
+
+          column_types[i] ||= candidate_type
+          if column_types[i] != candidate_type
+            column_types[i] = :string
+          end
+        end
+      end
+
+      converters = []
+      column_types.each_with_index do |type, i|
+        case type
+        when :boolean
+          converters << selective_converter(i, &BOOLEAN_CONVERTER)
+        when :integer
+          converters << selective_converter(i, &CSV::Converters[:integer])
+        when :float
+          converters << selective_converter(i, &CSV::Converters[:float])
+        when :time
+          converters << selective_converter(i, &ISO8601_CONVERTER)
+        when :date_time
+          converters << selective_converter(i, &CSV::Converters[:date_time])
+        when :date
+          converters << selective_converter(i, &CSV::Converters[:date])
+        end
+      end
+      converters
+    end
+  end
+end

data/lib/arrow/csv-reader.rb

@@ -13,150 +13,39 @@
 # limitations under the License.
 
 require "csv"
-require "pathname"
-require "time"
 
 module Arrow
   class CSVReader
-    class << self
-      def read(csv, **options)
-        case csv
-        when Pathname
-          path = csv.to_path
-          options = update_csv_parse_options(options, :open_csv, path)
-          open_csv(path, **options) do |_csv|
-            read(_csv)
-          end
-        when /\A.+\.csv\z/i
-          read(Pathname.new(csv), **options)
-        when String
-          options = update_csv_parse_options(options, :parse_csv_data, csv)
-          parse_csv_data(csv, **options) do |_csv|
-            read(_csv)
-          end
-        else
-          new(csv).read
-        end
-      end
-
-      private
-      def open_csv(path, **options)
-        CSV.open(path, **options) do |csv|
-          yield(csv)
-        end
-      end
-
-      def parse_csv_data(data, **options)
-        csv = CSV.new(data, **options)
-        begin
-          yield(csv)
-        ensure
-          csv.close
-        end
-      end
-
-      ISO8601_CONVERTER = lambda do |field|
-        begin
-          encoded_field = field.encode(CSV::ConverterEncoding)
-        rescue EncodingError
-          field
-        else
-          begin
-            Time.iso8601(encoded_field)
-          rescue ArgumentError
-            field
-          end
-        end
-      end
-
-      def update_csv_parse_options(options, create_csv, *args)
-        return options unless options.empty?
-
-        new_options = options.merge(converters: [:all, ISO8601_CONVERTER])
-        __send__(create_csv, *args, **new_options) do |csv|
-          row1 = csv.shift
-          if row1.nil?
-            new_options[:headers] = false
-            return new_options
-          end
-          if row1.any?(&:nil?)
-            new_options[:headers] = false
-            return new_options
-          end
-
-          row2 = csv.shift
-          return new_options if row2.nil?
-          if row2.any?(&:nil?)
-            new_options[:headers] = true
-            return new_options
-          end
-
-          if row1.collect(&:class) != row2.collect(&:class)
-            new_options[:headers] = true
-            return new_options
-          end
-
-          new_options
-        end
-      end
-    end
-
     def initialize(csv)
       @csv = csv
     end
 
     def read
-      builders = []
       values_set = []
       @csv.each do |row|
         if row.is_a?(CSV::Row)
           row = row.collect(&:last)
         end
         row.each_with_index do |value, i|
-          builders[i] ||= create_builder(value)
           values = (values_set[i] ||= [])
-          case value
-          when Time
-            value = value.to_i * (10 ** 9) + value.nsec
-          end
           values << value
         end
       end
       return nil if values_set.empty?
 
       arrays = values_set.collect.with_index do |values, i|
-        builders[i].build(values)
+        ArrayBuilder.build(values)
       end
       if @csv.headers
         names = @csv.headers
       else
-        names = builders.size.times.collect(&:to_s)
-      end
-      fields = names.collect.with_index do |name, i|
-        Arrow::Field.new(name, arrays[i].value_data_type)
-      end
-      schema = Schema.new(fields)
-      columns = arrays.collect.with_index do |array, i|
-        Column.new(fields[i], array)
+        names = arrays.size.times.collect(&:to_s)
       end
-      Table.new(schema, columns)
-    end
-
-    private
-    def create_builder(sample_value)
-      case sample_value
-      when Integer
-        IntArrayBuilder.new
-      when Float
-        DoubleArrayBuilder.new
-      when String
-        StringArrayBuilder.new
-      when Time
-        data_type = TimestampDataType.new(:nano)
-        TimestampArrayBuilder.new(data_type)
-      else
-        nil
+      raw_table = {}
+      names.each_with_index do |name, i|
+        raw_table[name] = arrays[i]
       end
+      Table.new(raw_table)
     end
   end
 end