red-arrow 0.17.1 → 1.0.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 661c88599c8cb32f10fbea1e03994a861bb0a14da3617a0124c614428dc8e016
-  data.tar.gz: b478f0588467345bdce0da3c8a10453767bb0ce9cf64586bbd38411363f4e423
+  metadata.gz: a45e003f0a453f175b1dbfc81c1fcf092cbfe964dd43e44d7b16e2087834ef5d
+  data.tar.gz: 5d1dc1d87a821d1ac4c49603d1d92d723230eb1f8a7cdd540dafe3715a53c12e
 SHA512:
-  metadata.gz: e2c72551f2211db790cb0627754290a5d1ddf6583d0b14b830ee5922786a5096d1ad0e2b78849f8213b2164a4c2e6803fd6317d65d5a3f36cbf71b3c660bbd51
-  data.tar.gz: b1271385c4e346627dd58340a369742307fb432ce29a0f2acab501c385c1677c50ed885e3528f5d895bc8a35d5e5a99ddc2ee79357c902cd2236186d6da647d5
+  metadata.gz: 85329e05ae20268d44a9ffff5fb3278263e5b77deda6f03fda31595b117252b7970fd17328ce2650c8650895a1813a1fb95799d3331a312bce2c276f3e5d55a4
+  data.tar.gz: fe90b0ff0dfb9b9126765818d54f7d4d203e6c72ab53dfb2680a48dc474fef09b8ed3f0553f3c1fe110d500d625ab2300563c3e4169403326c63c6bf27f33a5f

data/ext/arrow/converters.hpp

@@ -285,7 +285,8 @@ namespace red_arrow {
     // VISIT(Interval)
     VISIT(List)
     VISIT(Struct)
-    VISIT(Union)
+    VISIT(SparseUnion)
+    VISIT(DenseUnion)
     VISIT(Dictionary)
     VISIT(Decimal128)
     // TODO
@@ -339,9 +340,9 @@ namespace red_arrow {
       index_ = index;
       result_ = rb_hash_new();
       const auto struct_type = array.struct_type();
-      const auto n = struct_type->num_children();
+      const auto n = struct_type->num_fields();
       for (int i = 0; i < n; ++i) {
-        const auto field_type = struct_type->child(i).get();
+        const auto field_type = struct_type->field(i).get();
         const auto& field_name = field_type->name();
         auto key_keep = key_;
         key_ = rb_utf8_str_new(field_name.data(), field_name.length());
@@ -388,7 +389,8 @@ namespace red_arrow {
     // VISIT(Interval)
     VISIT(List)
     VISIT(Struct)
-    VISIT(Union)
+    VISIT(SparseUnion)
+    VISIT(DenseUnion)
     VISIT(Dictionary)
     VISIT(Decimal128)
     // TODO
@@ -432,10 +434,10 @@ namespace red_arrow {
       index_ = index;
       switch (array.mode()) {
       case arrow::UnionMode::SPARSE:
-        convert_sparse(array);
+        convert_sparse(static_cast<const arrow::SparseUnionArray&>(array));
         break;
       case arrow::UnionMode::DENSE:
-        convert_dense(array);
+        convert_dense(static_cast<const arrow::DenseUnionArray&>(array));
         break;
       default:
         rb_raise(rb_eArgError, "Invalid union mode");
@@ -479,7 +481,8 @@ namespace red_arrow {
     // VISIT(Interval)
     VISIT(List)
     VISIT(Struct)
-    VISIT(Union)
+    VISIT(SparseUnion)
+    VISIT(DenseUnion)
     VISIT(Dictionary)
     VISIT(Decimal128)
     // TODO
@@ -501,14 +504,14 @@ namespace red_arrow {
       result_ = result;
     }
 
-    uint8_t compute_child_index(const arrow::UnionArray& array,
+    uint8_t compute_field_index(const arrow::UnionArray& array,
                                 arrow::UnionType* type,
                                 const char* tag) {
       const auto type_code = array.raw_type_codes()[index_];
       if (type_code >= 0 && type_code <= arrow::UnionType::kMaxTypeCode) {
-        const auto child_id = type->child_ids()[type_code];
-        if (child_id >= 0) {
-          return child_id;
+        const auto field_id = type->child_ids()[type_code];
+        if (field_id >= 0) {
+          return field_id;
         }
       }
       check_status(arrow::Status::Invalid("Unknown type ID: ", type_code),
@@ -516,33 +519,33 @@ namespace red_arrow {
       return 0;
     }
 
-    void convert_sparse(const arrow::UnionArray& array) {
+    void convert_sparse(const arrow::SparseUnionArray& array) {
       const auto type =
         std::static_pointer_cast<arrow::UnionType>(array.type()).get();
       const auto tag = "[raw-records][union-sparse-array]";
-      const auto child_index = compute_child_index(array, type, tag);
-      const auto child_field = type->child(child_index).get();
-      const auto& field_name = child_field->name();
+      const auto index = compute_field_index(array, type, tag);
+      const auto field = type->field(index).get();
+      const auto& field_name = field->name();
       const auto field_name_keep = field_name_;
       field_name_ = rb_utf8_str_new(field_name.data(), field_name.length());
-      const auto child_array = array.child(child_index).get();
-      check_status(child_array->Accept(this), tag);
+      const auto field_array = array.field(index).get();
+      check_status(field_array->Accept(this), tag);
       field_name_ = field_name_keep;
     }
 
-    void convert_dense(const arrow::UnionArray& array) {
+    void convert_dense(const arrow::DenseUnionArray& array) {
       const auto type =
         std::static_pointer_cast<arrow::UnionType>(array.type()).get();
       const auto tag = "[raw-records][union-dense-array]";
-      const auto child_index = compute_child_index(array, type, tag);
-      const auto child_field = type->child(child_index).get();
-      const auto& field_name = child_field->name();
+      const auto index = compute_field_index(array, type, tag);
+      const auto field = type->field(index).get();
+      const auto& field_name = field->name();
       const auto field_name_keep = field_name_;
       field_name_ = rb_utf8_str_new(field_name.data(), field_name.length());
-      const auto child_array = array.child(child_index);
+      const auto field_array = array.field(index);
       const auto index_keep = index_;
       index_ = array.value_offset(index_);
-      check_status(child_array->Accept(this), tag);
+      check_status(field_array->Accept(this), tag);
       index_ = index_keep;
       field_name_ = field_name_keep;
     }
@@ -557,30 +560,57 @@ namespace red_arrow {
   public:
     explicit DictionaryArrayValueConverter(ArrayValueConverter* converter)
       : array_value_converter_(converter),
-        index_(0),
+        value_index_(0),
         result_(Qnil) {
     }
 
     VALUE convert(const arrow::DictionaryArray& array,
                   const int64_t index) {
-      index_ = index;
-      auto indices = array.indices().get();
-      check_status(indices->Accept(this),
+      value_index_ = array.GetValueIndex(index);
+      auto dictionary = array.dictionary().get();
+      check_status(dictionary->Accept(this),
                    "[raw-records][dictionary-array]");
       return result_;
     }
 
-    // TODO: Convert to real value.
 #define VISIT(TYPE)                                                     \
     arrow::Status Visit(const arrow::TYPE ## Array& array) override {   \
-      result_ = convert_value(array, index_);                           \
+      result_ = convert_value(array, value_index_);                     \
       return arrow::Status::OK();                                       \
       }
 
+    VISIT(Null)
+    VISIT(Boolean)
     VISIT(Int8)
     VISIT(Int16)
     VISIT(Int32)
     VISIT(Int64)
+    VISIT(UInt8)
+    VISIT(UInt16)
+    VISIT(UInt32)
+    VISIT(UInt64)
+    // TODO
+    // VISIT(HalfFloat)
+    VISIT(Float)
+    VISIT(Double)
+    VISIT(Binary)
+    VISIT(String)
+    VISIT(FixedSizeBinary)
+    VISIT(Date32)
+    VISIT(Date64)
+    VISIT(Time32)
+    VISIT(Time64)
+    VISIT(Timestamp)
+    // TODO
+    // VISIT(Interval)
+    VISIT(List)
+    VISIT(Struct)
+    VISIT(SparseUnion)
+    VISIT(DenseUnion)
+    VISIT(Dictionary)
+    VISIT(Decimal128)
+    // TODO
+    // VISIT(Extension)
 
 #undef VISIT
 
@@ -592,7 +622,7 @@ namespace red_arrow {
     }
 
     ArrayValueConverter* array_value_converter_;
-    int64_t index_;
+    int64_t value_index_;
     VALUE result_;
   };
 

data/ext/arrow/raw-records.cpp

@@ -100,7 +100,8 @@ namespace red_arrow {
       // VISIT(Interval)
       VISIT(List)
       VISIT(Struct)
-      VISIT(Union)
+      VISIT(SparseUnion)
+      VISIT(DenseUnion)
       VISIT(Dictionary)
       VISIT(Decimal128)
       // TODO

data/ext/arrow/values.cpp

@@ -81,7 +81,8 @@ namespace red_arrow {
       // VISIT(Interval)
       VISIT(List)
       VISIT(Struct)
-      VISIT(Union)
+      VISIT(SparseUnion)
+      VISIT(DenseUnion)
       VISIT(Dictionary)
       VISIT(Decimal128)
       // TODO

data/lib/arrow/buffer.rb

@@ -0,0 +1,28 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+module Arrow
+  class Buffer
+    alias_method :initialize_raw, :initialize
+    private :initialize_raw
+
+    def initialize(data)
+      @data = data
+      initialize_raw(data)
+    end
+  end
+end

data/lib/arrow/dictionary-array.rb

@@ -0,0 +1,24 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+module Arrow
+  class DictionaryArray
+    def get_value(i)
+      dictionary[indices[i]]
+    end
+  end
+end

data/lib/arrow/loader.rb

@@ -41,6 +41,7 @@ module Arrow
       require "arrow/array"
       require "arrow/array-builder"
       require "arrow/bigdecimal-extension"
+      require "arrow/buffer"
       require "arrow/chunked-array"
       require "arrow/column"
       require "arrow/compression-type"
@@ -56,6 +57,7 @@ module Arrow
       require "arrow/decimal128-array-builder"
       require "arrow/decimal128-data-type"
       require "arrow/dense-union-data-type"
+      require "arrow/dictionary-array"
       require "arrow/dictionary-data-type"
       require "arrow/field"
       require "arrow/file-output-stream"
@@ -69,6 +71,7 @@ module Arrow
       require "arrow/record-batch"
       require "arrow/record-batch-builder"
       require "arrow/record-batch-file-reader"
+      require "arrow/record-batch-iterator"
       require "arrow/record-batch-stream-reader"
       require "arrow/rolling-window"
       require "arrow/schema"

data/lib/arrow/raw-table-converter.rb

@@ -0,0 +1,47 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+module Arrow
+  class RawTableConverter
+    attr_reader :n_rows
+    attr_reader :schema
+    attr_reader :values
+    def initialize(raw_table)
+      @raw_table = raw_table
+      convert
+    end
+
+    private
+    def convert
+      if @raw_table.is_a?(::Array) and @raw_table[0].is_a?(Column)
+        fields = @raw_table.collect(&:field)
+        @schema = Schema.new(fields)
+        @values = @raw_table.collect(&:data)
+      else
+        fields = []
+        @values = []
+        @raw_table.each do |name, array|
+          array = ArrayBuilder.build(array) if array.is_a?(::Array)
+          fields << Field.new(name.to_s, array.value_data_type)
+          @values << array
+        end
+        @schema = Schema.new(fields)
+      end
+      @n_rows = @values[0].length
+    end
+  end
+end

data/lib/arrow/record-batch-iterator.rb

@@ -0,0 +1,22 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements.  See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership.  The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License.  You may obtain a copy of the License at
+#
+#   http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied.  See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+module Arrow
+  class RecordBatchIterator
+    alias_method :to_a, :to_list
+  end
+end

data/lib/arrow/record-batch.rb

@@ -15,6 +15,8 @@
 # specific language governing permissions and limitations
 # under the License.
 
+require "arrow/raw-table-converter"
+
 module Arrow
   class RecordBatch
     include ColumnContainable
@@ -25,13 +27,19 @@ module Arrow
       def new(*args)
         n_args = args.size
         case n_args
+        when 1
+          raw_table_converter = RawTableConverter.new(args[0])
+          n_rows = raw_table_converter.n_rows
+          schema = raw_table_converter.schema
+          values = raw_table_converter.values
+          super(schema, n_rows, values)
         when 2
           schema, data = args
           RecordBatchBuilder.build(schema, data)
         when 3
           super
         else
-          message = "wrong number of arguments (given #{n_args}, expected 2..3)"
+          message = "wrong number of arguments (given #{n_args}, expected 1..3)"
           raise ArgumentError, message
         end
       end

data/lib/arrow/struct-array-builder.rb

@@ -32,7 +32,7 @@ module Arrow
       case index_or_name
       when String, Symbol
         name = index_or_name
-        (@name_to_builder ||= build_name_to_builder)[name.to_s]
+        cached_name_to_builder[name.to_s]
       else
         index = index_or_name
         cached_field_builders[index]
@@ -70,13 +70,18 @@ module Arrow
           append_null
         when ::Array
           append_value_raw
-          value.each_with_index do |sub_value, i|
-            self[i].append(sub_value)
+          cached_field_builders.zip(value) do |builder, sub_value|
+            builder.append(sub_value)
           end
         when Hash
           append_value_raw
+          local_name_to_builder = cached_name_to_builder.dup
           value.each do |name, sub_value|
-            self[name].append(sub_value)
+            builder = local_name_to_builder.delete(name.to_s)
+            builder.append(sub_value)
+          end
+          local_name_to_builder.each do |_, builder|
+            builder.append_null
           end
         else
           message =
@@ -108,9 +113,6 @@ module Arrow
     alias_method :append_null_raw, :append_null
     def append_null
       append_null_raw
-      cached_field_builders.each do |builder|
-        builder.append_null
-      end
     end
 
     # @since 0.12.0
@@ -136,5 +138,9 @@ module Arrow
       end
       name_to_builder
     end
+
+    def cached_name_to_builder
+      @name_to_builder ||= build_name_to_builder
+    end
   end
 end