red-arrow 0.8.2 → 0.10.0

data/lib/arrow/block-closable.rb

@@ -1,16 +1,19 @@
-# Copyright 2017 Kouhei Sutou <kou@clear-code.com>
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
 #
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
+#   http://www.apache.org/licenses/LICENSE-2.0
 #
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
 
 module Arrow
   module BlockClosable

data/lib/arrow/chunked-array.rb

@@ -1,16 +1,19 @@
-# Copyright 2017-2018 Kouhei Sutou <kou@clear-code.com>
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
 #
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
+#   http://www.apache.org/licenses/LICENSE-2.0
 #
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
 
 module Arrow
   class ChunkedArray
@@ -38,6 +41,7 @@ module Arrow
     end
 
     def [](i)
+      i += length if i < 0
       chunks.each do |array|
         return array[i] if i < array.length
         i -= array.length

data/lib/arrow/column.rb

@@ -1,16 +1,19 @@
-# Copyright 2017 Kouhei Sutou <kou@clear-code.com>
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
 #
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
+#   http://www.apache.org/licenses/LICENSE-2.0
 #
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
 
 module Arrow
   class Column

data/lib/arrow/csv-loader.rb

@@ -1,16 +1,19 @@
-# Copyright 2017-2018 Kouhei Sutou <kou@clear-code.com>
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
 #
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
+#   http://www.apache.org/licenses/LICENSE-2.0
 #
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
 
 require "csv"
 require "pathname"
@@ -117,20 +120,32 @@ module Arrow
     end
 
     def update_csv_parse_options(options, create_csv, *args)
-      return options unless options.empty?
+      if options.key?(:converters)
+        new_options = options.dup
+      else
+        converters = [:all, BOOLEAN_CONVERTER, ISO8601_CONVERTER]
+        new_options = options.merge(converters: converters)
+      end
 
-      converters = [:all, BOOLEAN_CONVERTER, ISO8601_CONVERTER]
-      new_options = options.merge(converters: converters)
-      __send__(create_csv, *args, **new_options) do |csv|
-        new_options[:headers] = have_header?(csv)
+      unless options.key?(:headers)
+        __send__(create_csv, *args, **new_options) do |csv|
+          new_options[:headers] = have_header?(csv)
+        end
       end
-      __send__(create_csv, *args, **new_options) do |csv|
-        new_options[:converters] = detect_robust_converters(csv)
+      unless options.key?(:converters)
+        __send__(create_csv, *args, **new_options) do |csv|
+          new_options[:converters] = detect_robust_converters(csv)
+        end
       end
-      return new_options
+
+      new_options
     end
 
     def have_header?(csv)
+      if @options.key?(:headers)
+        return @options[:headers]
+      end
+
       row1 = csv.shift
       return false if row1.nil?
       return false if row1.any?(&:nil?)
@@ -139,6 +154,8 @@ module Arrow
       return nil if row2.nil?
       return true if row2.any?(&:nil?)
 
+      return false if row1.any? {|value| not value.is_a?(String)}
+
       if row1.collect(&:class) != row2.collect(&:class)
         return true
       end
@@ -149,7 +166,16 @@ module Arrow
     def detect_robust_converters(csv)
       column_types = []
       csv.each do |row|
-        row.each_with_index do |(_name, value), i|
+        if row.is_a?(CSV::Row)
+          each_value = Enumerator.new do |yielder|
+            row.each do |_name, value|
+              yielder << value
+            end
+          end
+        else
+          each_value = row.each
+        end
+        each_value.with_index do |value, i|
           current_column_type = column_types[i]
           next if current_column_type == :string
 
@@ -161,6 +187,9 @@ module Arrow
             candidate_type = :boolean
           when Integer
             candidate_type = :integer
+            if current_column_type == :float
+              candidate_type = :float
+            end
           when Float
             candidate_type = :float
             if current_column_type == :integer

data/lib/arrow/csv-reader.rb

@@ -1,16 +1,19 @@
-# Copyright 2017 Kouhei Sutou <kou@clear-code.com>
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
 #
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
+#   http://www.apache.org/licenses/LICENSE-2.0
 #
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
 
 require "csv"
 

data/lib/arrow/date32-array-builder.rb

@@ -1,16 +1,19 @@
-# Copyright 2018 Kouhei Sutou <kou@clear-code.com>
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
 #
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
+#   http://www.apache.org/licenses/LICENSE-2.0
 #
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
 
 module Arrow
   class Date32ArrayBuilder

data/lib/arrow/date32-array.rb

@@ -1,16 +1,19 @@
-# Copyright 2017 Kouhei Sutou <kou@clear-code.com>
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
 #
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
+#   http://www.apache.org/licenses/LICENSE-2.0
 #
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
 
 module Arrow
   class Date32Array

data/lib/arrow/date64-array-builder.rb

@@ -1,16 +1,19 @@
-# Copyright 2018 Kouhei Sutou <kou@clear-code.com>
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
 #
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
+#   http://www.apache.org/licenses/LICENSE-2.0
 #
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
 
 module Arrow
   class Date64ArrayBuilder
@@ -21,7 +24,7 @@ module Arrow
       end
 
       if value.is_a?(Time)
-        value.to_i * 1_000 + value.usec / 1000
+        value.to_i * 1_000 + value.usec / 1_000
       else
         value
       end

data/lib/arrow/date64-array.rb

@@ -1,16 +1,19 @@
-# Copyright 2017 Kouhei Sutou <kou@clear-code.com>
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
 #
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
+#   http://www.apache.org/licenses/LICENSE-2.0
 #
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
 
 module Arrow
   class Date64Array

data/lib/arrow/field.rb

@@ -1,16 +1,19 @@
-# Copyright 2017 Kouhei Sutou <kou@clear-code.com>
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
 #
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
+#   http://www.apache.org/licenses/LICENSE-2.0
 #
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
 
 module Arrow
   class Field

data/lib/arrow/group.rb

@@ -1,18 +1,24 @@
-# Copyright 2018 Kouhei Sutou <kou@clear-code.com>
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
 #
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
+#   http://www.apache.org/licenses/LICENSE-2.0
 #
-#     http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
 
 module Arrow
+  # Experimental
+  #
+  # TODO: Almost codes should be implemented in Apache Arrow C++.
   class Group
     def initialize(table, keys)
       @table = table
@@ -37,7 +43,7 @@ module Arrow
       key_names = @keys.collect(&:to_s)
       target_columns = @table.columns.reject do |column|
         key_names.include?(column.name) or
-          not column.data_type.numeric?
+          not column.data_type.is_a?(NumericDataType)
       end
       aggregate(target_columns) do |column, indexes|
         n = 0
@@ -53,7 +59,7 @@ module Arrow
       key_names = @keys.collect(&:to_s)
       target_columns = @table.columns.reject do |column|
         key_names.include?(column.name) or
-          not column.data_type.numeric?
+          not column.data_type.is_a?(NumericDataType)
       end
       aggregate(target_columns) do |column, indexes|
         average = 0.0
@@ -69,6 +75,42 @@ module Arrow
       end
     end
 
+    def min
+      key_names = @keys.collect(&:to_s)
+      target_columns = @table.columns.reject do |column|
+        key_names.include?(column.name) or
+          not column.data_type.is_a?(NumericDataType)
+      end
+      aggregate(target_columns) do |column, indexes|
+        n = nil
+        indexes.each do |index|
+          value = column[index]
+          next if value.nil?
+          n ||= value
+          n = value if value < n
+        end
+        n
+      end
+    end
+
+    def max
+      key_names = @keys.collect(&:to_s)
+      target_columns = @table.columns.reject do |column|
+        key_names.include?(column.name) or
+          not column.data_type.is_a?(NumericDataType)
+      end
+      aggregate(target_columns) do |column, indexes|
+        n = nil
+        indexes.each do |index|
+          value = column[index]
+          next if value.nil?
+          n ||= value
+          n = value if value > n
+        end
+        n
+      end
+    end
+
     private
     def aggregate(target_columns)
       sort_values = @table.n_rows.times.collect do |i|