red-arrow 0.8.1 → 0.8.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 59cce19c00fbc436f48f29c2d58aca2685330a63
-  data.tar.gz: 280b1305e60c39d74d4fd101dc43387cd398e770
+  metadata.gz: d5c8c88ef78b27019b54dca4de8c914f17b80d9e
+  data.tar.gz: 85a527d72ea710c9f4c55e4c2db36110693a9f2c
 SHA512:
-  metadata.gz: 3ecba86d0d061186def935c54a999f1c0adf1f109a1687eefd0375792a87f34333e7a46ee7f78fe51b01fbe294166491f300d9e46a8315e19ff47a1cb3f45102
-  data.tar.gz: 49d09794b0d007bf9e65675e133c45cd1a90a3eec748b9c747d2e7d78e4c21d682ba503c696cb61d128fec362fa138ec50b04ab780a0bf554557417ccaf02afd
+  metadata.gz: 7b4635c2517a7b750c7f46341bd659504e4d2cbbba6f1d2ef4f32c47c1870de891a7bf009fcd4783284f72b515eaeb2e4846121bf0b66e8eeb017d9ab5f1727c
+  data.tar.gz: 21a10ffe298a91ec56b5354bcc6693348fdb815db04861fc04124849b111ffadc5ddbac2582cf6d87d8f96ec2060179fb2e19866f8f7be2699d547a2e2d861dc

data/doc/text/news.md

@@ -1,5 +1,49 @@
 # News
 
+## 0.8.2 - 2018-02-04
+
+### Improvements
+
+  * `Arrow::Table#size`: Added.
+
+  * `Arrow::Table#length`: Added.
+
+  * `Arrow::Table#pack`: Added.
+
+  * `Arrow::Column#pack`: Added.
+
+  * `Arrow::ChunkedArray#pack`: Added.
+
+  * `Arrow::Column#reverse_each`: Added.
+
+  * `Arrow::Table#slice`: Added negative integer support.
+
+  * `Arrow::Slicer::ColumnCondition#in?`: Added.
+
+  * `Arrow::Table#group`: Added.
+
+  * `Arrow::ChunkedArray#null?`: Added.
+
+  * `Arrow::Column#null?`: Added.
+
+  * `Arrow::Group`: Added.
+
+  * `Arrow::CSVLoader`: Changed to treat `""` as a null value instead
+    of empty string.
+
+  * `Arrow::Table#[]`: Stopped to accept multiple column name.
+
+  * `Arrow::ChunkedArray#valid?`: Added.
+
+  * `Arrow::Column#valid?`: Added.
+
+  * `Arrow::Slicer::ColumnCondition#valid?`: Added.
+
+### Fixes
+
+  * `Arrow::TableFormatter`: Fixed a bug that too much records are
+    formatted.
+
 ## 0.8.1 - 2018-01-05
 
 ### Improvements

data/lib/arrow/array-builder.rb

@@ -28,7 +28,6 @@ module Arrow
           case value
           when nil
             # Ignore
-            nil
           when true, false
             return BooleanArray.new(values)
           when String

data/lib/arrow/array.rb

@@ -1,4 +1,4 @@
-# Copyright 2017 Kouhei Sutou <kou@clear-code.com>
+# Copyright 2017-2018 Kouhei Sutou <kou@clear-code.com>
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -44,6 +44,14 @@ module Arrow
       end
     end
 
+    def reverse_each
+      return to_enum(__method__) unless block_given?
+
+      (length - 1).downto(0) do |i|
+        yield(self[i])
+      end
+    end
+
     def to_arrow
       self
     end

data/lib/arrow/chunked-array.rb

@@ -1,4 +1,4 @@
-# Copyright 2017 Kouhei Sutou <kou@clear-code.com>
+# Copyright 2017-2018 Kouhei Sutou <kou@clear-code.com>
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -16,9 +16,29 @@ module Arrow
   class ChunkedArray
     include Enumerable
 
+    alias_method :chunks_raw, :chunks
+    def chunks
+      @chunks ||= chunks_raw
+    end
+
+    def null?(i)
+      chunks.each do |array|
+        return array.null?(i) if i < array.length
+        i -= array.length
+      end
+      nil
+    end
+
+    def valid?(i)
+      chunks.each do |array|
+        return array.valid?(i) if i < array.length
+        i -= array.length
+      end
+      nil
+    end
+
     def [](i)
-      n_chunks.times do |j|
-        array = get_chunk(j)
+      chunks.each do |array|
         return array[i] if i < array.length
         i -= array.length
       end
@@ -28,16 +48,32 @@ module Arrow
     def each(&block)
       return to_enum(__method__) unless block_given?
 
-      each_chunk do |array|
+      chunks.each do |array|
         array.each(&block)
       end
     end
 
-    def each_chunk
+    def reverse_each(&block)
       return to_enum(__method__) unless block_given?
 
-      n_chunks.times do |i|
-        yield(get_chunk(i))
+      chunks.reverse_each do |array|
+        array.reverse_each(&block)
+      end
+    end
+
+    def each_chunk(&block)
+      chunks.each(&block)
+    end
+
+    def pack
+      first_chunk = chunks.first
+      data_type = first_chunk.value_data_type
+      case data_type
+      when TimestampDataType
+        builder = TimestampArrayBuilder.new(data_type)
+        builder.build(to_a)
+      else
+        first_chunk.class.new(to_a)
       end
     end
   end

data/lib/arrow/column.rb

@@ -16,6 +16,14 @@ module Arrow
   class Column
     include Enumerable
 
+    def null?(i)
+      data.null?(i)
+    end
+
+    def valid?(i)
+      data.valid?(i)
+    end
+
     def [](i)
       data[i]
     end
@@ -25,5 +33,15 @@ module Arrow
 
       data.each(&block)
     end
+
+    def reverse_each(&block)
+      return to_enum(__method__) unless block_given?
+
+      data.reverse_each(&block)
+    end
+
+    def pack
+      self.class.new(field, data.pack)
+    end
   end
 end

data/lib/arrow/csv-loader.rb

@@ -172,6 +172,9 @@ module Arrow
             candidate_type = :date_time
           when Date
             candidate_type = :date
+          when String
+            next if value.empty?
+            candidate_type = :string
           else
             candidate_type = :string
           end
@@ -189,9 +192,21 @@ module Arrow
         when :boolean
           converters << selective_converter(i, &BOOLEAN_CONVERTER)
         when :integer
-          converters << selective_converter(i, &CSV::Converters[:integer])
+          converters << selective_converter(i) do |field|
+            if field.nil? or field.empty?
+              nil
+            else
+              CSV::Converters[:integer].call(field)
+            end
+          end
         when :float
-          converters << selective_converter(i, &CSV::Converters[:float])
+          converters << selective_converter(i) do |field|
+            if field.nil? or field.empty?
+              nil
+            else
+              CSV::Converters[:float].call(field)
+            end
+          end
         when :time
           converters << selective_converter(i, &ISO8601_CONVERTER)
         when :date_time

data/lib/arrow/data-type.rb

@@ -0,0 +1,81 @@
+# Copyright 2018 Kouhei Sutou <kou@clear-code.com>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+module Arrow
+  class DataType
+    def numeric?
+      false
+    end
+  end
+
+  class Int8DataType
+    def numeric?
+      true
+    end
+  end
+
+  class Int16DataType
+    def numeric?
+      true
+    end
+  end
+
+  class Int32DataType
+    def numeric?
+      true
+    end
+  end
+
+  class Int64DataType
+    def numeric?
+      true
+    end
+  end
+
+  class UInt8DataType
+    def numeric?
+      true
+    end
+  end
+
+  class UInt16DataType
+    def numeric?
+      true
+    end
+  end
+
+  class UInt32DataType
+    def numeric?
+      true
+    end
+  end
+
+  class UInt64DataType
+    def numeric?
+      true
+    end
+  end
+
+  class FloatDataType
+    def numeric?
+      true
+    end
+  end
+
+  class DoubleDataType
+    def numeric?
+      true
+    end
+  end
+end

data/lib/arrow/group.rb

@@ -0,0 +1,133 @@
+# Copyright 2018 Kouhei Sutou <kou@clear-code.com>
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+module Arrow
+  class Group
+    def initialize(table, keys)
+      @table = table
+      @keys = keys
+    end
+
+    def count
+      key_names = @keys.collect(&:to_s)
+      target_columns = @table.columns.reject do |column|
+        key_names.include?(column.name)
+      end
+      aggregate(target_columns) do |column, indexes|
+        n = 0
+        indexes.each do |index|
+          n += 1 unless column.null?(index)
+        end
+        n
+      end
+    end
+
+    def sum
+      key_names = @keys.collect(&:to_s)
+      target_columns = @table.columns.reject do |column|
+        key_names.include?(column.name) or
+          not column.data_type.numeric?
+      end
+      aggregate(target_columns) do |column, indexes|
+        n = 0
+        indexes.each do |index|
+          value = column[index]
+          n += value unless value.nil?
+        end
+        n
+      end
+    end
+
+    def average
+      key_names = @keys.collect(&:to_s)
+      target_columns = @table.columns.reject do |column|
+        key_names.include?(column.name) or
+          not column.data_type.numeric?
+      end
+      aggregate(target_columns) do |column, indexes|
+        average = 0.0
+        n = 0
+        indexes.each do |index|
+          value = column[index]
+          unless value.nil?
+            n += 1
+            average += (value - average) / n
+          end
+        end
+        average
+      end
+    end
+
+    private
+    def aggregate(target_columns)
+      sort_values = @table.n_rows.times.collect do |i|
+        key_values = @keys.collect do |key|
+          @table[key][i]
+        end
+        [key_values, i]
+      end
+      sorted = sort_values.sort_by do |key_values, i|
+        key_values
+      end
+
+      grouped_keys = []
+      aggregated_arrays_raw = []
+      target_columns.size.times do
+        aggregated_arrays_raw << []
+      end
+      indexes = []
+      sorted.each do |key_values, i|
+        if grouped_keys.empty?
+          grouped_keys << key_values
+          indexes.clear
+          indexes << i
+        else
+          if key_values == grouped_keys.last
+            indexes << i
+          else
+            grouped_keys << key_values
+            target_columns.each_with_index do |column, j|
+              aggregated_arrays_raw[j] << yield(column, indexes)
+            end
+            indexes.clear
+            indexes << i
+          end
+        end
+      end
+      target_columns.each_with_index do |column, j|
+        aggregated_arrays_raw[j] << yield(column, indexes)
+      end
+
+      grouped_key_arrays_raw = grouped_keys.transpose
+      columns = @keys.collect.with_index do |key, i|
+        key_column = @table[key]
+        key_column_array_class = key_column.data.chunks.first.class
+        if key_column_array_class == TimestampArray
+          builder = TimestampArrayBuilder.new(key_column.data_type)
+          key_column_array = builder.build(grouped_key_arrays_raw[i])
+        else
+          key_column_array =
+            key_column_array_class.new(grouped_key_arrays_raw[i])
+        end
+        Column.new(key_column.field, key_column_array)
+      end
+      target_columns.each_with_index do |column, i|
+        array = ArrayBuilder.build(aggregated_arrays_raw[i])
+        field = Field.new(column.name, array.value_data_type)
+        columns << Column.new(field, array)
+      end
+      Table.new(columns)
+    end
+  end
+end