RubyGems - red-arrow - Versions diffs - 8.0.0 → 24.0.0 - Mend

red-arrow 8.0.0 → 24.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (178) hide show

checksums.yaml +4 -4
data/README.md +15 -7
data/ext/arrow/arrow.cpp +67 -0
data/ext/arrow/converters.cpp +10 -0
data/ext/arrow/converters.hpp +310 -46
data/ext/arrow/extconf.rb +41 -22
data/ext/arrow/raw-records.cpp +165 -2
data/ext/arrow/red-arrow.hpp +2 -0
data/ext/arrow/values.cpp +6 -2
data/lib/arrow/array-builder.rb +89 -14
data/{test/test-time32-data-type.rb → lib/arrow/array-computable.rb} +24 -16
data/{test/test-buffer.rb → lib/arrow/array-statistics.rb} +19 -24
data/lib/arrow/array.rb +40 -4
data/lib/arrow/chunked-array.rb +56 -1
data/lib/arrow/column-containable.rb +9 -0
data/lib/arrow/column.rb +49 -4
data/{test/test-tensor.rb → lib/arrow/csv-write-options.rb} +28 -31
data/lib/arrow/data-type.rb +17 -3
data/lib/arrow/decimal128-array-builder.rb +16 -6
data/lib/arrow/decimal128.rb +14 -0
data/lib/arrow/decimal256-array-builder.rb +16 -6
data/lib/arrow/decimal256.rb +14 -0
data/{test/test-float-scalar.rb → lib/arrow/dense-union-array-builder.rb} +27 -24
data/{test/test-boolean-scalar.rb → lib/arrow/dense-union-array.rb} +7 -7
data/lib/arrow/duration-array-builder.rb +27 -0
data/lib/arrow/duration-array.rb +24 -0
data/lib/arrow/duration-data-type.rb +32 -0
data/lib/arrow/expression.rb +6 -2
data/lib/arrow/field-containable.rb +1 -1
data/lib/arrow/field.rb +44 -3
data/lib/arrow/fixed-size-list-array-builder.rb +29 -0
data/lib/arrow/fixed-size-list-data-type.rb +118 -0
data/lib/arrow/function.rb +0 -1
data/lib/arrow/half-float-array-builder.rb +32 -0
data/lib/arrow/half-float-array.rb +24 -0
data/lib/arrow/half-float.rb +118 -0
data/{test/helper/fixture.rb → lib/arrow/input-referable.rb} +7 -6
data/lib/arrow/jruby/array-builder.rb +114 -0
data/lib/arrow/jruby/array.rb +109 -0
data/lib/arrow/jruby/chunked-array.rb +36 -0
data/lib/arrow/jruby/compression-type.rb +26 -0
data/lib/arrow/jruby/csv-read-options.rb +32 -0
data/{test/test-map-data-type.rb → lib/arrow/jruby/data-type.rb} +24 -12
data/lib/arrow/jruby/decimal128.rb +28 -0
data/lib/arrow/jruby/decimal256.rb +28 -0
data/{test/fixture/float-integer.csv → lib/arrow/jruby/error.rb} +7 -4
data/lib/arrow/jruby/file-system.rb +24 -0
data/{test/test-null-array.rb → lib/arrow/jruby/function.rb} +5 -4
data/lib/arrow/jruby/record-batch-iterator.rb +24 -0
data/{test/fixture/null-with-double-quote.csv → lib/arrow/jruby/record-batch.rb} +8 -4
data/{test/fixture/integer-float.csv → lib/arrow/jruby/sort-key.rb} +8 -4
data/lib/arrow/jruby/sort-options.rb +24 -0
data/lib/arrow/jruby/stream-listener-raw.rb +25 -0
data/{test/test-rolling-window.rb → lib/arrow/jruby/table.rb} +19 -19
data/lib/arrow/jruby/writable.rb +24 -0
data/lib/arrow/jruby.rb +52 -0
data/{test/test-date32-array.rb → lib/arrow/large-list-array-builder.rb} +10 -5
data/lib/arrow/large-list-data-type.rb +83 -0
data/lib/arrow/libraries.rb +140 -0
data/lib/arrow/list-array-builder.rb +1 -68
data/lib/arrow/list-data-type.rb +3 -38
data/{test/test-dictionary-array.rb → lib/arrow/list-field-resolvable.rb} +26 -17
data/lib/arrow/list-slice-options.rb +76 -0
data/lib/arrow/list-values-appendable.rb +88 -0
data/lib/arrow/loader.rb +15 -96
data/{test/test-decimal128-array.rb → lib/arrow/make-struct-options.rb} +18 -18
data/lib/arrow/raw-table-converter.rb +10 -3
data/lib/arrow/raw-tensor-converter.rb +89 -0
data/lib/arrow/record-batch-file-reader.rb +2 -0
data/lib/arrow/record-batch-stream-reader.rb +2 -0
data/lib/arrow/record-batch.rb +6 -2
data/{test/fixture/null-without-double-quote.csv → lib/arrow/ruby.rb} +5 -4
data/lib/arrow/scalar.rb +67 -0
data/lib/arrow/slicer.rb +61 -0
data/lib/arrow/sort-key.rb +3 -3
data/lib/arrow/sparse-union-array-builder.rb +56 -0
data/lib/arrow/sparse-union-array.rb +26 -0
data/lib/arrow/stream-decoder.rb +29 -0
data/{test/test-decimal256-data-type.rb → lib/arrow/stream-listener.rb} +25 -9
data/lib/arrow/string-array-builder.rb +30 -0
data/lib/arrow/struct-array-builder.rb +0 -5
data/lib/arrow/table-formatter.rb +38 -8
data/lib/arrow/table-list-formatter.rb +3 -3
data/lib/arrow/table-loader.rb +11 -5
data/lib/arrow/table-saver.rb +4 -3
data/lib/arrow/table-table-formatter.rb +7 -0
data/lib/arrow/table.rb +180 -33
data/lib/arrow/tensor.rb +144 -0
data/lib/arrow/time-unit.rb +31 -0
data/lib/arrow/time32-array-builder.rb +2 -14
data/lib/arrow/time32-data-type.rb +9 -38
data/lib/arrow/time64-array-builder.rb +2 -14
data/lib/arrow/time64-data-type.rb +9 -38
data/lib/arrow/timestamp-array-builder.rb +3 -15
data/lib/arrow/timestamp-data-type.rb +9 -34
data/{test/test-date64-array.rb → lib/arrow/timestamp-parser.rb} +14 -6
data/lib/arrow/union-array-builder.rb +59 -0
data/lib/arrow/union-array.rb +26 -0
data/lib/arrow/version.rb +1 -1
data/lib/arrow.rb +2 -7
data/red-arrow.gemspec +74 -11
metadata +85 -210
data/test/fixture/TestOrcFile.test1.orc +0 -0
data/test/fixture/with-header-float.csv +0 -20
data/test/fixture/with-header.csv +0 -20
data/test/fixture/without-header-float.csv +0 -19
data/test/fixture/without-header.csv +0 -19
data/test/helper/omittable.rb +0 -36
data/test/helper.rb +0 -30
data/test/raw-records/test-basic-arrays.rb +0 -395
data/test/raw-records/test-dense-union-array.rb +0 -521
data/test/raw-records/test-list-array.rb +0 -610
data/test/raw-records/test-map-array.rb +0 -478
data/test/raw-records/test-multiple-columns.rb +0 -65
data/test/raw-records/test-sparse-union-array.rb +0 -511
data/test/raw-records/test-struct-array.rb +0 -515
data/test/raw-records/test-table.rb +0 -47
data/test/run-test.rb +0 -71
data/test/test-array-builder.rb +0 -136
data/test/test-array.rb +0 -325
data/test/test-bigdecimal.rb +0 -40
data/test/test-binary-dictionary-array-builder.rb +0 -103
data/test/test-chunked-array.rb +0 -183
data/test/test-column.rb +0 -92
data/test/test-csv-loader.rb +0 -250
data/test/test-data-type.rb +0 -83
data/test/test-decimal128-array-builder.rb +0 -112
data/test/test-decimal128-data-type.rb +0 -31
data/test/test-decimal128.rb +0 -102
data/test/test-decimal256-array-builder.rb +0 -112
data/test/test-decimal256-array.rb +0 -38
data/test/test-decimal256.rb +0 -102
data/test/test-dense-union-data-type.rb +0 -41
data/test/test-dictionary-data-type.rb +0 -40
data/test/test-expression.rb +0 -40
data/test/test-feather.rb +0 -49
data/test/test-field.rb +0 -91
data/test/test-file-output-stream.rb +0 -54
data/test/test-fixed-size-binary-array-builder.rb +0 -92
data/test/test-fixed-size-binary-array.rb +0 -36
data/test/test-function.rb +0 -210
data/test/test-group.rb +0 -180
data/test/test-list-array-builder.rb +0 -79
data/test/test-list-array.rb +0 -32
data/test/test-list-data-type.rb +0 -69
data/test/test-map-array-builder.rb +0 -110
data/test/test-map-array.rb +0 -33
data/test/test-memory-view.rb +0 -434
data/test/test-orc.rb +0 -173
data/test/test-record-batch-builder.rb +0 -125
data/test/test-record-batch-file-reader.rb +0 -115
data/test/test-record-batch-iterator.rb +0 -37
data/test/test-record-batch-reader.rb +0 -46
data/test/test-record-batch.rb +0 -182
data/test/test-schema.rb +0 -134
data/test/test-slicer.rb +0 -487
data/test/test-sort-indices.rb +0 -40
data/test/test-sort-key.rb +0 -81
data/test/test-sort-options.rb +0 -58
data/test/test-sparse-union-data-type.rb +0 -41
data/test/test-string-dictionary-array-builder.rb +0 -103
data/test/test-struct-array-builder.rb +0 -184
data/test/test-struct-array.rb +0 -94
data/test/test-struct-data-type.rb +0 -112
data/test/test-table.rb +0 -1123
data/test/test-time.rb +0 -288
data/test/test-time32-array.rb +0 -81
data/test/test-time64-array.rb +0 -81
data/test/test-time64-data-type.rb +0 -42
data/test/test-timestamp-array.rb +0 -45
data/test/test-timestamp-data-type.rb +0 -42
data/test/values/test-basic-arrays.rb +0 -325
data/test/values/test-dense-union-array.rb +0 -509
data/test/values/test-dictionary-array.rb +0 -295
data/test/values/test-list-array.rb +0 -571
data/test/values/test-map-array.rb +0 -466
data/test/values/test-sparse-union-array.rb +0 -500
data/test/values/test-struct-array.rb +0 -512

data/test/test-string-dictionary-array-builder.rb DELETED Viewed

@@ -1,103 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-class StringDictionaryArrayBuilderTest < Test::Unit::TestCase
-  def setup
-    @builder = Arrow::StringDictionaryArrayBuilder.new
-  end
-  sub_test_case("#append_values") do
-    test("[nil]") do
-      @builder.append_values([nil])
-      array = @builder.finish
-      assert_equal([
-                     [],
-                     [nil],
-                   ],
-                   [
-                     array.dictionary.to_a,
-                     array.indices.to_a,
-                   ])
-    end
-    test("[String]") do
-      @builder.append_values(["hello"])
-      array = @builder.finish
-      assert_equal([
-                     ["hello"],
-                     [0],
-                   ],
-                   [
-                     array.dictionary.to_a,
-                     array.indices.to_a,
-                   ])
-    end
-    test("[Symbol]") do
-      @builder.append_values([:hello])
-      array = @builder.finish
-      assert_equal([
-                     ["hello"],
-                     [0],
-                   ],
-                   [
-                     array.dictionary.to_a,
-                     array.indices.to_a,
-                   ])
-    end
-    test("[nil, String, Symbol]") do
-      @builder.append_values([
-                               nil,
-                               "Hello",
-                               :world,
-                               "world",
-                             ])
-      array = @builder.finish
-      assert_equal([
-                     ["Hello", "world"],
-                     [nil, 0, 1, 1],
-                   ],
-                   [
-                     array.dictionary.to_a,
-                     array.indices.to_a,
-                   ])
-    end
-    test("is_valids") do
-      @builder.append_values([
-                               "Hello",
-                               :world,
-                               :goodbye,
-                             ],
-                             [
-                               true,
-                               false,
-                               true,
-                             ])
-      array = @builder.finish
-      assert_equal([
-                     ["Hello", "goodbye"],
-                     [0, nil, 1],
-                   ],
-                   [
-                     array.dictionary.to_a,
-                     array.indices.to_a,
-                   ])
-    end
-  end
-end

data/test/test-struct-array-builder.rb DELETED Viewed

@@ -1,184 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-class StructArrayBuilderTest < Test::Unit::TestCase
-  def setup
-    @data_type = Arrow::StructDataType.new(visible: {type: :boolean},
-                                           count: {type: :uint64})
-    @builder = Arrow::StructArrayBuilder.new(@data_type)
-  end
-  sub_test_case("#append_value") do
-    test("nil") do
-      @builder.append_value(nil)
-      array = @builder.finish
-      assert_equal([
-                     [false],
-                     [0],
-                   ],
-                   [
-                     array.find_field(0).to_a,
-                     array.find_field(1).to_a,
-                   ])
-    end
-    test("Array") do
-      @builder.append_value([true, 1])
-      @builder.append_value([])
-      @builder.append_value([false])
-      array = @builder.finish
-      assert_equal([
-                     [true, nil, false],
-                     [1, nil, nil],
-                   ],
-                   [
-                     array.find_field(0).to_a,
-                     array.find_field(1).to_a,
-                   ])
-    end
-    test("Arrow::Struct") do
-      source_array = Arrow::StructArray.new(@data_type, [[true, 1]])
-      struct = source_array.get_value(0)
-      @builder.append_value(struct)
-      array = @builder.finish
-      assert_equal([
-                     [true],
-                     [1],
-                   ],
-                   [
-                     array.find_field(0).to_a,
-                     array.find_field(1).to_a,
-                   ])
-    end
-    test("Hash") do
-      @builder.append_value(count: 1, visible: true)
-      @builder.append_value(visible: false)
-      @builder.append_value(count: 2)
-      array = @builder.finish
-      assert_equal([
-                     [true, false, nil],
-                     [1, nil, 2],
-                   ],
-                   [
-                     array.find_field(0).to_a,
-                     array.find_field(1).to_a,
-                   ])
-    end
-  end
-  sub_test_case("#append_values") do
-    test("[nil]") do
-      @builder.append_values([nil])
-      array = @builder.finish
-      assert_equal([
-                     [false],
-                     [0],
-                   ],
-                   [
-                     array.find_field(0).to_a,
-                     array.find_field(1).to_a,
-                   ])
-    end
-    test("[Array]") do
-      @builder.append_values([[true, 1]])
-      array = @builder.finish
-      assert_equal([
-                     [true],
-                     [1],
-                   ],
-                   [
-                     array.find_field(0).to_a,
-                     array.find_field(1).to_a,
-                   ])
-    end
-    test("[Hash]") do
-      @builder.append_values([{count: 1, visible: true}])
-      array = @builder.finish
-      assert_equal([
-                     [true],
-                     [1],
-                   ],
-                   [
-                     array.find_field(0).to_a,
-                     array.find_field(1).to_a,
-                   ])
-    end
-    test("[nil, Array, Hash]") do
-      @builder.append_values([
-                               nil,
-                               [true, 1],
-                               {count: 2, visible: false},
-                             ])
-      array = @builder.finish
-      assert_equal([
-                     [false, true, false],
-                     [0, 1, 2],
-                   ],
-                   [
-                     array.find_field(0).to_a,
-                     array.find_field(1).to_a,
-                   ])
-    end
-    test("is_valids") do
-      @builder.append_values([
-                               [true, 1],
-                               [false, 2],
-                               [true, 3],
-                             ],
-                             [
-                               true,
-                               false,
-                               true,
-                             ])
-      array = @builder.finish
-      assert_equal([
-                     [true, false, true],
-                     [1, 0, 3],
-                   ],
-                   [
-                     array.find_field(0).to_a,
-                     array.find_field(1).to_a,
-                   ])
-    end
-  end
-  sub_test_case("#append") do
-    test("backward compatibility") do
-      @builder.append
-      @builder.get_field_builder(0).append(true)
-      @builder.get_field_builder(1).append(1)
-      @builder.append
-      @builder.get_field_builder(0).append(false)
-      @builder.get_field_builder(1).append(2)
-      array = @builder.finish
-      assert_equal([
-                     {"visible" => true,  "count" => 1},
-                     {"visible" => false, "count" => 2},
-                   ],
-                   [
-                     array.get_value(0),
-                     array.get_value(1),
-                   ])
-    end
-  end
-end

data/test/test-struct-array.rb DELETED Viewed

@@ -1,94 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-class StructArrayTest < Test::Unit::TestCase
-  sub_test_case(".new") do
-    test("build") do
-      data_type = Arrow::StructDataType.new(visible: :boolean,
-                                            count: :uint64)
-      values = [
-        [true, 1],
-        nil,
-        [false, 2],
-      ]
-      array = Arrow::StructArray.new(data_type, values)
-      assert_equal([
-                     [true, false, false],
-                     [1, 0, 2],
-                   ],
-                   [
-                     array.find_field(0).to_a,
-                     array.find_field(1).to_a,
-                   ])
-    end
-  end
-  sub_test_case("instance methods") do
-    def setup
-      @data_type = Arrow::StructDataType.new(visible: {type: :boolean},
-                                             count: {type: :uint64})
-      @values = [
-        [true, 1],
-        [false, 2],
-      ]
-      @array = Arrow::StructArray.new(@data_type, @values)
-    end
-    test("#[]") do
-      assert_equal([
-                     {"visible" => true,  "count" => 1},
-                     {"visible" => false, "count" => 2},
-                   ],
-                   @array.to_a)
-    end
-    test("#get_value") do
-      assert_equal([
-                     {"visible" => true,  "count" => 1},
-                     {"visible" => false, "count" => 2},
-                   ],
-                   [
-                     @array.get_value(0),
-                     @array.get_value(1),
-                   ])
-    end
-    sub_test_case("#find_field") do
-      test("Integer") do
-        assert_equal([
-                       [true, false],
-                       [1, 2],
-                     ],
-                     [
-                       @array.find_field(0).to_a,
-                       @array.find_field(1).to_a,
-                     ])
-      end
-      test("String, Symbol") do
-        assert_equal([
-                       [true, false],
-                       [1, 2],
-                     ],
-                     [
-                       @array.find_field("visible").to_a,
-                       @array.find_field(:count).to_a,
-                     ])
-      end
-    end
-  end
-end

data/test/test-struct-data-type.rb DELETED Viewed

@@ -1,112 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-class StructDataTypeTest < Test::Unit::TestCase
-  def setup
-    @count_field = Arrow::Field.new("count", :uint32)
-    @visible_field = Arrow::Field.new("visible", :boolean)
-  end
-  sub_test_case(".new") do
-    test("[Arrow::Field]") do
-      fields = [
-        @count_field,
-        @visible_field,
-      ]
-      assert_equal("struct<count: uint32, visible: bool>",
-                   Arrow::StructDataType.new(fields).to_s)
-    end
-    test("[Hash]") do
-      fields = [
-        {name: "count", data_type: :uint32},
-        {name: "visible", data_type: :boolean},
-      ]
-      assert_equal("struct<count: uint32, visible: bool>",
-                   Arrow::StructDataType.new(fields).to_s)
-    end
-    test("[Arrow::Field, Hash]") do
-      fields = [
-        @count_field,
-        {name: "visible", data_type: :boolean},
-      ]
-      assert_equal("struct<count: uint32, visible: bool>",
-                   Arrow::StructDataType.new(fields).to_s)
-    end
-    test("{Arrow::DataType}") do
-      fields = {
-        "count" => Arrow::UInt32DataType.new,
-        "visible" => Arrow::BooleanDataType.new,
-      }
-      assert_equal("struct<count: uint32, visible: bool>",
-                   Arrow::StructDataType.new(fields).to_s)
-    end
-    test("{Hash}") do
-      fields = {
-        "count" => {type: :uint32},
-        "visible" => {type: :boolean},
-      }
-      assert_equal("struct<count: uint32, visible: bool>",
-                   Arrow::StructDataType.new(fields).to_s)
-    end
-    test("{String, Symbol}") do
-      fields = {
-        "count" => "uint32",
-        "visible" => :boolean,
-      }
-      assert_equal("struct<count: uint32, visible: bool>",
-                   Arrow::StructDataType.new(fields).to_s)
-    end
-  end
-  sub_test_case("instance methods") do
-    def setup
-      super
-      @data_type = Arrow::StructDataType.new([@count_field, @visible_field])
-    end
-    sub_test_case("#[]") do
-      test("[String]") do
-        assert_equal([@count_field, @visible_field],
-                     [@data_type["count"], @data_type["visible"]])
-      end
-      test("[Symbol]") do
-        assert_equal([@count_field, @visible_field],
-                     [@data_type[:count], @data_type[:visible]])
-      end
-      test("[Integer]") do
-        assert_equal([@count_field, @visible_field],
-                     [@data_type[0], @data_type[1]])
-      end
-      test("[invalid]") do
-        invalid = []
-        message = "field name or index must be String, Symbol or Integer"
-        message << ": <#{invalid.inspect}>"
-        assert_raise(ArgumentError.new(message)) do
-          @data_type[invalid]
-        end
-      end
-    end
-  end
-end