red-arrow 8.0.0 → 24.0.0

data/lib/arrow/scalar.rb

@@ -17,6 +17,73 @@
 
 module Arrow
   class Scalar
+    class << self
+      # @api private
+      def try_convert(value)
+        case value
+        when self
+          value
+        when true, false
+          BooleanScalar.new(value)
+        when Symbol, String
+          StringScalar.new(value.to_s)
+        when Integer
+          Int64Scalar.new(value)
+        when Float
+          DoubleScalar.new(value)
+        else
+          nil
+        end
+      end
+
+      # Ensure returning suitable {Arrow::Scalar}.
+      #
+      # @overload resolve(scalar)
+      #
+      #   Returns the given scalar itself. This is convenient to
+      #   use this method as {Arrow::Scalar} converter.
+      #
+      #   @param scalar [Arrow::Scalar] The scalar.
+      #
+      #   @return [Arrow::Scalar] The given scalar itself.
+      #
+      # @overload resolve(value)
+      #
+      #   Creates a suitable scalar from the given value. For example,
+      #   you can create {Arrow::BooleanScalar} from `true`.
+      #
+      #   @param value [Object] The value.
+      #
+      #   @return [Arrow::Scalar] A suitable {Arrow::Scalar} for `value`.
+      #
+      # @overload resolve(value, data_type)
+      #
+      #   Creates a scalar of `data_type.scalar_class` from the given
+      #   value. For example, you can create {Arrow::Int32Scalar} from
+      #   `29` and {Arrow::Int32DataType}.
+      #
+      #   @param value [Object] The value.
+      #
+      #   @param data_type [Arrow::DataType] The {Arrow::DataType} to
+      #     decide the returned scalar class.
+      #
+      #   @return [Arrow::Scalar] A suitable {Arrow::Scalar} for `value`.
+      #
+      # @since 12.0.0
+      def resolve(value, data_type=nil)
+        return try_convert(value) if data_type.nil?
+
+        data_type = DataType.resolve(data_type)
+        scalar_class = data_type.scalar_class
+        case value
+        when Scalar
+          return value if value.class == scalar_class
+          value = value.value
+        end
+        scalar_class.new(value)
+      end
+    end
+
     # @param other [Arrow::Scalar] The scalar to be compared.
     # @param options [Arrow::EqualOptions, Hash] (nil)
     #   The options to custom how to compare.

data/lib/arrow/slicer.rb

@@ -162,6 +162,40 @@ module Arrow
       def reject(&block)
         RejectCondition.new(@column, block)
       end
+
+      def end_with?(substring, ignore_case: false)
+        MatchSubstringFamilyCondition.new("ends_with",
+                                          @column, substring, ignore_case)
+      end
+
+      def match_like?(pattern, ignore_case: false)
+        MatchSubstringFamilyCondition.new("match_like",
+                                          @column, pattern, ignore_case)
+      end
+
+      def match_substring?(pattern, ignore_case: nil)
+        case pattern
+        when String
+          ignore_case = false if ignore_case.nil?
+          MatchSubstringFamilyCondition.new("match_substring",
+                                            @column, pattern, ignore_case)
+        when Regexp
+          ignore_case = pattern.casefold? if ignore_case.nil?
+          MatchSubstringFamilyCondition.new("match_substring_regex",
+                                            @column,
+                                            pattern.source,
+                                            ignore_case)
+        else
+          message =
+             "pattern must be either String or Regexp: #{pattern.inspect}"
+          raise ArgumentError, message
+        end
+      end
+
+      def start_with?(substring, ignore_case: false)
+        MatchSubstringFamilyCondition.new("starts_with",
+                                          @column, substring, ignore_case)
+      end
     end
 
     class NotColumnCondition < Condition
@@ -351,5 +385,32 @@ module Arrow
         BooleanArray.new(raw_array)
       end
     end
+
+    class MatchSubstringFamilyCondition < Condition
+      def initialize(function, column, pattern, ignore_case, invert: false)
+        @function = function
+        @column = column
+        @options = MatchSubstringOptions.new
+        @options.pattern = pattern
+        @options.ignore_case = ignore_case
+        @invert = invert
+      end
+
+      def !@
+        MatchSubstringFamilyCondition.new(@function,
+                                          @column,
+                                          @options.pattern,
+                                          @options.ignore_case?,
+                                          invert: !@invert)
+      end
+
+      def evaluate
+        datum = Function.find(@function).execute([@column.data], @options)
+        if @invert
+          datum = Function.find("invert").execute([datum])
+        end
+        datum.value
+      end
+    end
   end
 end

data/lib/arrow/sort-key.rb

@@ -79,9 +79,9 @@ module Arrow
     #     target and corresponding order is used. `"+"` uses ascending
     #     order and `"-"` uses ascending order.
     #
-    #     If `target` is not a String nor `target` doesn't start with the
-    #     leading order mark, sort column target is `target` as-is and
-    #     ascending order is used.
+    #     If `target` is either not a String or `target` doesn't start
+    #     with the leading order mark, sort column is `target` as-is
+    #     and ascending order is used.
     #
     #   @example String without the leading order mark
     #     key = Arrow::SortKey.new("count")

data/lib/arrow/sparse-union-array-builder.rb

@@ -0,0 +1,56 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+module Arrow
+  class SparseUnionArrayBuilder
+    alias_method :append_value_raw, :append_value
+
+    # @overload append_value
+    #
+    #   Starts appending an union record. You need to append values of
+    #   fields.
+    #
+    # @overload append_value(value)
+    #
+    #   Appends an union record including values of fields.
+    #
+    #   @param value [nil, Hash] The union record value.
+    #
+    #     If this is `nil`, the union record is null.
+    #
+    #     If this is `Hash`, it's  values of fields.
+    #
+    # @since 12.0.0
+    def append_value(value)
+      if value.nil?
+        append_null
+      else
+        key = value.keys[0]
+        child_info = child_infos[key]
+        append_value_raw(child_info[:id])
+        child_infos.each do |child_key, child_info|
+          builder = child_info[:builder]
+          if child_key == key
+            builder.append(value.values[0])
+          else
+            builder.append_null
+          end
+        end
+      end
+    end
+  end
+end

data/lib/arrow/sparse-union-array.rb

@@ -0,0 +1,26 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+module Arrow
+  class SparseUnionArray
+    def get_value(i)
+      child_id = get_child_id(i)
+      field = fields[child_id]
+      field[i]
+    end
+  end
+end

data/lib/arrow/stream-decoder.rb

@@ -0,0 +1,29 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+module Arrow
+  class StreamDecoder
+    def consume(data)
+      case data
+      when Buffer
+        consume_buffer(data)
+      else
+        consume_bytes(data)
+      end
+    end
+  end
+end

data/{test/test-decimal256-data-type.rb → lib/arrow/stream-listener.rb}

@@ -15,17 +15,33 @@
 # specific language governing permissions and limitations
 # under the License.
 
-class Decimal256DataTypeTest < Test::Unit::TestCase
-  sub_test_case(".new") do
-    test("ordered arguments") do
-      assert_equal("decimal256(8, 2)",
-                   Arrow::Decimal256DataType.new(8, 2).to_s)
+module Arrow
+  class StreamListener < StreamListenerRaw
+    type_register
+
+    def on_eos
+    end
+
+    def on_record_batch_decoded(record_batch, metadata)
+    end
+
+    def on_schema(schema, filtered_schema)
+    end
+
+    private
+    def virtual_do_on_eos
+      on_eos
+      true
+    end
+
+    def virtual_do_on_record_batch_decoded(record_batch, metadata)
+      on_record_batch_decoded(record_batch, metadata)
+      true
     end
 
-    test("description") do
-      assert_equal("decimal256(8, 2)",
-                   Arrow::Decimal256DataType.new(precision: 8,
-                                                 scale: 2).to_s)
+    def virtual_do_on_schema_decoded(schema, filtered_schema)
+      on_schema_decoded(schema, filtered_schema)
+      true
     end
   end
 end

data/lib/arrow/string-array-builder.rb

@@ -0,0 +1,30 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+module Arrow
+  class StringArrayBuilder
+    private
+    def convert_to_arrow_value(value)
+      case value
+      when GLib::Bytes, String
+        value
+      else
+        value.to_s
+      end
+    end
+  end
+end

data/lib/arrow/struct-array-builder.rb

@@ -110,11 +110,6 @@ module Arrow
       end
     end
 
-    alias_method :append_null_raw, :append_null
-    def append_null
-      append_null_raw
-    end
-
     # @since 0.12.0
     def append(*values)
       if values.empty?

data/lib/arrow/table-formatter.rb

@@ -24,7 +24,8 @@ module Arrow
       attr_reader :head_values
       attr_reader :tail_values
       attr_reader :sample_values
-      def initialize(column, head_values, tail_values)
+      def initialize(table_formatter, column, head_values, tail_values)
+        @table_formatter = table_formatter
         @column = column
         @head_values = head_values
         @tail_values = tail_values
@@ -36,6 +37,15 @@ module Arrow
         @data_type ||= @column.data_type
       end
 
+      def formatted_data_type_name
+        @formatted_data_type_name ||= "(#{data_type.name})"
+      end
+
+      def aligned_data_type_name
+        @aligned_data_type_name ||=
+          "%*s" % [aligned_name.size, formatted_data_type_name]
+      end
+
       def name
         @name ||= @column.name
       end
@@ -63,14 +73,18 @@ module Arrow
             formatted_value = format_value(value[field_name], field_value_width)
             "#{formatted_name}: #{formatted_value}"
           end
-          formatted = "{"
+          formatted = +"{"
           formatted << formatted_values.join(", ")
           formatted << "}"
           "%-*s" % [width, formatted]
         when nil
           "%*s" % [width, FORMATTED_NULL]
         else
-          "%-*s" % [width, value.to_s]
+          value = value.to_s
+          if value.encoding == Encoding::ASCII_8BIT
+            value = value.each_byte.collect {|byte| "%X" % byte}.join
+          end
+          "%-*s" % [width, value]
         end
       end
 
@@ -90,9 +104,16 @@ module Arrow
       end
 
       def format_aligned_name(name, data_type, sample_values)
+        if @table_formatter.show_column_type?
+          min_width = formatted_data_type_name.size
+        else
+          min_width = 0
+        end
         case data_type
         when TimestampDataType
-          "%*s" % [::Time.now.iso8601.size, name]
+          width = ::Time.now.iso8601.size
+          width = min_width if width < min_width
+          "%*s" % [width, name]
         when IntegerDataType
           have_null = false
           have_negative = false
@@ -118,9 +139,12 @@ module Arrow
           end
           width += 1 if have_negative # Need "-"
           width = [width, FORMATTED_NULL.size].max if have_null
+          width = min_width if width < min_width
           "%*s" % [width, name]
         when FloatDataType, DoubleDataType
-          "%*s" % [FLOAT_N_DIGITS, name]
+          width = FLOAT_N_DIGITS
+          width = min_width if width < min_width
+          "%*s" % [width, name]
         when StructDataType
           field_widths = data_type.fields.collect do |field|
             field_value_width = compute_field_value_width(field, sample_values)
@@ -130,9 +154,11 @@ module Arrow
           if field_widths.size > 0
             width += (", ".size * (field_widths.size - 1))
           end
+          width = min_width if width < min_width
           "%*s" % [width, name]
         else
-          name
+          width = min_width
+          "%*s" % [width, name]
         end
       end
     end
@@ -143,7 +169,7 @@ module Arrow
     end
 
     def format
-      text = ""
+      text = +""
       n_rows = @table.n_rows
       border = @options[:border] || 10
 
@@ -159,7 +185,7 @@ module Arrow
         else
           tail_values = []
         end
-        ColumnFormatter.new(column, head_values, tail_values)
+        ColumnFormatter.new(self, column, head_values, tail_values)
       end
 
       format_header(text, column_formatters)
@@ -186,5 +212,9 @@ module Arrow
 
       text
     end
+
+    def show_column_type?
+      @options.fetch(:show_column_type, true)
+    end
   end
 end

data/lib/arrow/table-list-formatter.rb

@@ -27,9 +27,9 @@ module Arrow
         text << ("=" * 20 + " #{start_offset + nth_row} " + "=" * 20 + "\n")
         row.each_with_index do |column_value, nth_column|
           column_formatter = column_formatters[nth_column]
-          formatted_name = column_formatter.name
-          formatted_value = column_formatter.format_value(column_value)
-          text << "#{formatted_name}: #{formatted_value}\n"
+          text << column_formatter.name
+          text << "(#{column_formatter.data_type.name})" if show_column_type?
+          text << ": #{column_formatter.format_value(column_value)}\n"
         end
       end
     end

data/lib/arrow/table-loader.rb

@@ -161,7 +161,7 @@ module Arrow
         record_batches << record_batch
       end
       table = Table.new(schema, record_batches)
-      table.instance_variable_set(:@input, input)
+      table.refer_input(input)
       table
     end
 
@@ -211,7 +211,7 @@ module Arrow
           field_indexes = @options[:field_indexes]
           reader.set_field_indexes(field_indexes) if field_indexes
           table = reader.read_stripes
-          table.instance_variable_set(:@input, input)
+          table.refer_input(input)
           table
         end
       end
@@ -245,16 +245,22 @@ module Arrow
       open_input_stream do |input|
         reader = FeatherFileReader.new(input)
         table = reader.read
-        table.instance_variable_set(:@input, input)
+        table.refer_input(input)
         table
       end
     end
 
     def load_as_json
       open_input_stream do |input|
-        reader = JSONReader.new(input)
+        options = JSONReadOptions.new
+        @options.each do |key, value|
+          next if value.nil?
+          setter = :"#{key}="
+          options.__send__(setter, value) if options.respond_to?(setter)
+        end
+        reader = JSONReader.new(input, options)
         table = reader.read
-        table.instance_variable_set(:@input, input)
+        table.refer_input(input)
         table
       end
     end

data/lib/arrow/table-saver.rb

@@ -51,6 +51,7 @@ module Arrow
         raise ArgumentError, message
       end
       __send__(custom_save_method)
+      @table
     end
 
     private
@@ -129,9 +130,9 @@ module Arrow
       end
     end
 
-    def save_raw(writer_class)
+    def save_raw(writer_class, *args)
       open_output_stream do |output|
-        writer_class.open(output, @table.schema) do |writer|
+        writer_class.open(output, @table.schema, *args) do |writer|
           writer.write_table(@table)
         end
       end
@@ -143,7 +144,7 @@ module Arrow
 
     # @since 1.0.0
     def save_as_arrow_file
-      save_raw(RecordBatchFileWriter)
+      save_raw(RecordBatchFileWriter, nil, @options[:metadata])
     end
 
     # @deprecated Use `format: :arrow_batch` instead.

data/lib/arrow/table-table-formatter.rb

@@ -26,6 +26,13 @@ module Arrow
         text << "\t"
         text << column_formatter.aligned_name
       end
+      if show_column_type?
+        text << "\n"
+        column_formatters.each do |column_formatter|
+          text << "\t"
+          text << column_formatter.aligned_data_type_name
+        end
+      end
       text << "\n"
     end