red-arrow 0.12.0 → 0.13.0

data/ext/arrow/red-arrow.hpp

@@ -0,0 +1,60 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+#pragma once
+
+#include <arrow/api.h>
+
+#ifdef _WIN32
+#  define gmtime_r gmtime_r_ruby_win32
+#  define localtime_r localtime_r_ruby_win32
+#  include <ruby.h>
+#  undef gmtime_r
+#  undef localtime_r
+#endif
+
+#include <arrow-glib/arrow-glib.hpp>
+#include <rbgobject.h>
+
+namespace red_arrow {
+  extern VALUE cDate;
+
+  extern ID id_BigDecimal;
+  extern ID id_jd;
+  extern ID id_to_datetime;
+
+  VALUE record_batch_raw_records(VALUE obj);
+
+  inline VALUE time_unit_to_scale(arrow::TimeUnit::type unit) {
+    switch (unit) {
+    case arrow::TimeUnit::SECOND:
+      return INT2FIX(1);
+    case arrow::TimeUnit::MILLI:
+      return INT2FIX(1000);
+    case arrow::TimeUnit::MICRO:
+      return INT2FIX(1000 * 1000);
+    case arrow::TimeUnit::NANO:
+      // NOTE: INT2FIX works for 1e+9 because: FIXNUM_MAX >= (1<<30) - 1 > 1e+9
+      return INT2FIX(1000 * 1000 * 1000);
+    default:
+      break; // NOT REACHED
+    }
+    return Qnil;
+  }
+}

data/lib/arrow.rb

@@ -15,7 +15,8 @@
 # specific language governing permissions and limitations
 # under the License.
 
-require "gobject-introspection"
+require "extpp/setup"
+require "gio2"
 
 require "arrow/version"
 

data/lib/arrow/array-builder.rb

@@ -62,6 +62,10 @@ module Arrow
           Arrow::StringArray.new(values)
         end
       end
+
+      def buildable?(args)
+        args.size == method(:build).arity
+      end
     end
 
     def build(values)

data/lib/arrow/array.rb

@@ -24,7 +24,7 @@ module Arrow
         builder_class_name = "#{name}Builder"
         if const_defined?(builder_class_name)
           builder_class = const_get(builder_class_name)
-          if args.size == builder_class.method(:build).arity
+          if builder_class.buildable?(args)
             builder_class.build(*args)
           else
             super
@@ -35,8 +35,18 @@ module Arrow
       end
     end
 
+    # @param i [Integer]
+    #   The index of the value to be gotten.
+    #
+    #   You can specify negative index like for `::Array#[]`.
+    #
+    # @return [Object, nil]
+    #   The `i`-th value.
+    #
+    #   `nil` for NULL value or out of range `i`.
     def [](i)
       i += length if i < 0
+      return nil if i < 0 or i >= length
       if null?(i)
         nil
       else

data/lib/arrow/bigdecimal-extension.rb

@@ -0,0 +1,24 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+require "bigdecimal"
+
+class BigDecimal
+  def to_arrow
+    Arrow::Decimal128.new(to_s)
+  end
+end

data/lib/arrow/binary-array-builder.rb

@@ -0,0 +1,36 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+module Arrow
+  class BinaryArrayBuilder
+    def append_values(values, is_valids=nil)
+      if is_valids
+        is_valids.each_with_index do |is_valid, i|
+          if is_valid
+            append_value(values[i])
+          else
+            append_null
+          end
+        end
+      else
+        values.each do |value|
+          append_value(value)
+        end
+      end
+    end
+  end
+end

data/lib/arrow/block-closable.rb

@@ -24,7 +24,11 @@ module Arrow
       begin
         yield(io)
       ensure
-        io.close
+        if io.respond_to?(:closed?)
+          io.close unless io.closed?
+        else
+          io.close
+        end
       end
     end
   end

data/lib/arrow/csv-loader.rb

@@ -104,6 +104,8 @@ module Arrow
           end
         when :schema
           options.add_schema(value)
+        when :encoding
+          # process encoding on opening input
         else
           setter = "#{key}="
           if options.respond_to?(setter)
@@ -116,7 +118,7 @@ module Arrow
       options
     end
 
-    def open_input(raw_input)
+    def open_decompress_input(raw_input)
       if @compression
         codec = Codec.new(@compression)
         CompressedInputStream.open(codec, raw_input) do |input|
@@ -127,16 +129,36 @@ module Arrow
       end
     end
 
+    def open_encoding_convert_stream(raw_input, &block)
+      encoding = @options[:encoding]
+      if encoding
+        converter = Gio::CharsetConverter.new("UTF-8", encoding)
+        convert_input_stream =
+          Gio::ConverterInputStream.new(raw_input, converter)
+        GIOInputStream.open(convert_input_stream, &block)
+      else
+        yield(raw_input)
+      end
+    end
+
+    def wrap_input(raw_input)
+      open_decompress_input(raw_input) do |input_|
+        open_encoding_convert_stream(input_) do |input__|
+          yield(input__)
+        end
+      end
+    end
+
     def load_from_path(path)
       options = reader_options
       if options
         begin
-          MemoryMappedInputStream.open(path.to_s) do |raw_input|
-            open_input(raw_input) do |input|
+          MemoryMappedInputStream.open(path) do |raw_input|
+            wrap_input(raw_input) do |input|
               return CSVReader.new(input, options).read
             end
           end
-        rescue Arrow::Error::Invalid
+        rescue Arrow::Error::Invalid, Gio::Error
         end
       end
 
@@ -151,11 +173,11 @@ module Arrow
       if options
         begin
           BufferInputStream.open(Buffer.new(data)) do |raw_input|
-            open_input(raw_input) do |input|
+            wrap_input(raw_input) do |input|
               return CSVReader.new(input, options).read
             end
           end
-        rescue Arrow::Error::Invalid
+        rescue Arrow::Error::Invalid, Gio::Error
         end
       end
 

data/lib/arrow/data-type.rb

@@ -114,14 +114,18 @@ module Arrow
 
       private
       def resolve_class(data_type)
-        data_type_name = data_type.to_s.capitalize.gsub(/\AUint/, "UInt")
+        components = data_type.to_s.split("_").collect(&:capitalize)
+        data_type_name = components.join.gsub(/\AUint/, "UInt")
         data_type_class_name = "#{data_type_name}DataType"
         unless Arrow.const_defined?(data_type_class_name)
           available_types = []
           Arrow.constants.each do |name|
-            if name.to_s.end_with?("DataType")
-              available_types << name.to_s.gsub(/DataType\z/, "").downcase.to_sym
-            end
+            name = name.to_s
+            next if name == "DataType"
+            next unless name.end_with?("DataType")
+            name = name.gsub(/DataType\z/, "")
+            components = name.scan(/(UInt[0-9]+|[A-Z][a-z\d]+)/).flatten
+            available_types << components.collect(&:downcase).join("_").to_sym
           end
           message =
             "unknown type: #{data_type.inspect}: " +

data/lib/arrow/decimal128-array-builder.rb

@@ -15,7 +15,7 @@
 # specific language governing permissions and limitations
 # under the License.
 
-require "bigdecimal"
+require "arrow/bigdecimal-extension"
 
 module Arrow
   class Decimal128ArrayBuilder
@@ -36,7 +36,7 @@ module Arrow
       when Float
         value = Decimal128.new(value.to_s)
       when BigDecimal
-        value = Decimal128.new(value.to_s)
+        value = value.to_arrow
       end
       append_value_raw(value)
     end

data/lib/arrow/decimal128.rb

@@ -0,0 +1,42 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+module Arrow
+  class Decimal128
+    alias_method :to_s_raw, :to_s
+
+    # @overload to_s
+    #
+    #   @return [String]
+    #     The string representation of the decimal.
+    #
+    # @overload to_s(scale)
+    #
+    #   @param scale [Integer] The scale of the decimal.
+    #   @return [String]
+    #      The string representation of the decimal including the scale.
+    #
+    # @since 0.13.0
+    def to_s(scale=nil)
+      if scale
+        to_string_scale(scale)
+      else
+        to_s_raw
+      end
+    end
+  end
+end

data/lib/arrow/list-array-builder.rb

@@ -56,7 +56,7 @@ module Arrow
         when ::Array
           append_value_raw
           @value_builder ||= value_builder
-          @value_builder.append_values(value, nil)
+          @value_builder.append(*value)
         else
           message = "list value must be nil or Array: #{value.inspect}"
           raise ArgumentError, message

data/lib/arrow/loader.rb

@@ -28,11 +28,13 @@ module Arrow
     private
     def post_load(repository, namespace)
       require_libraries
+      require_extension_library
     end
 
     def require_libraries
       require "arrow/array"
       require "arrow/array-builder"
+      require "arrow/binary-array-builder"
       require "arrow/chunked-array"
       require "arrow/column"
       require "arrow/compression-type"
@@ -43,6 +45,7 @@ module Arrow
       require "arrow/date32-array-builder"
       require "arrow/date64-array"
       require "arrow/date64-array-builder"
+      require "arrow/decimal128"
       require "arrow/decimal128-array-builder"
       require "arrow/decimal128-data-type"
       require "arrow/dense-union-data-type"
@@ -51,6 +54,7 @@ module Arrow
       require "arrow/file-output-stream"
       require "arrow/list-array-builder"
       require "arrow/list-data-type"
+      require "arrow/null-array-builder"
       require "arrow/path-extension"
       require "arrow/record"
       require "arrow/record-batch"
@@ -79,6 +83,10 @@ module Arrow
       require "arrow/writable"
     end
 
+    def require_extension_library
+      require "arrow.so"
+    end
+
     def load_object_info(info)
       super
 

data/lib/arrow/null-array-builder.rb

@@ -0,0 +1,26 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+module Arrow
+  class NullArrayBuilder
+    class << self
+      def buildable?(args)
+        super and args.collect(&:class) != [Integer]
+      end
+    end
+  end
+end

data/lib/arrow/record-batch-builder.rb

@@ -65,7 +65,7 @@ module Arrow
 
     # @since 0.12.0
     def append_records(records)
-      n = n_fields
+      n = n_columns
       columns = n.times.collect do
         []
       end
@@ -99,17 +99,16 @@ module Arrow
       end
     end
 
+    # @since 0.13.0
+    def column_builders
+      @column_builders ||= n_columns.times.collect do |i|
+        get_column_builder(i)
+      end
+    end
+
     private
     def resolve_name(name)
       @name_to_index[name.to_s]
     end
-
-    # TODO: Make public with good name. Is column_builders good enough?
-    # builders? sub_builders?
-    def column_builders
-      @column_builders ||= n_fields.times.collect do |i|
-        get_field(i)
-      end
-    end
   end
 end