RubyGems - red-arrow - Versions diffs - 18.1.0 → 19.0.0 - Mend

red-arrow 18.1.0 → 19.0.0

Files changed (133) hide show

checksums.yaml +4 -4
data/lib/arrow/array.rb +7 -4
data/lib/arrow/column.rb +4 -4
data/lib/arrow/jruby/array-builder.rb +114 -0
data/lib/arrow/jruby/array.rb +109 -0
data/{test/helper/fixture.rb → lib/arrow/jruby/chunked-array.rb} +14 -6
data/lib/arrow/jruby/compression-type.rb +26 -0
data/{test/test-boolean-scalar.rb → lib/arrow/jruby/csv-read-options.rb} +12 -6
data/{test/test-map-data-type.rb → lib/arrow/jruby/data-type.rb} +24 -12
data/lib/arrow/jruby/decimal128.rb +28 -0
data/lib/arrow/jruby/decimal256.rb +28 -0
data/{test/fixture/null-with-double-quote.csv → lib/arrow/jruby/error.rb} +7 -4
data/lib/arrow/jruby/file-system.rb +24 -0
data/{test/test-null-array.rb → lib/arrow/jruby/function.rb} +5 -4
data/lib/arrow/jruby/record-batch-iterator.rb +24 -0
data/{test/fixture/integer-float.csv → lib/arrow/jruby/record-batch.rb} +8 -4
data/{test/fixture/float-integer.csv → lib/arrow/jruby/sort-key.rb} +8 -4
data/lib/arrow/jruby/sort-options.rb +24 -0
data/lib/arrow/jruby/stream-listener-raw.rb +25 -0
data/{test/test-rolling-window.rb → lib/arrow/jruby/table.rb} +19 -19
data/lib/arrow/jruby/writable.rb +24 -0
data/lib/arrow/jruby.rb +52 -0
data/lib/arrow/libraries.rb +126 -0
data/lib/arrow/list-array-builder.rb +1 -0
data/lib/arrow/loader.rb +3 -111
data/{test/fixture/null-without-double-quote.csv → lib/arrow/ruby.rb} +6 -4
data/lib/arrow/version.rb +1 -1
data/lib/arrow.rb +2 -7
data/red-arrow.gemspec +15 -6
metadata +26 -229
data/test/each-raw-record/test-basic-arrays.rb +0 -411
data/test/each-raw-record/test-dense-union-array.rb +0 -566
data/test/each-raw-record/test-dictionary-array.rb +0 -341
data/test/each-raw-record/test-list-array.rb +0 -628
data/test/each-raw-record/test-map-array.rb +0 -507
data/test/each-raw-record/test-multiple-columns.rb +0 -72
data/test/each-raw-record/test-sparse-union-array.rb +0 -528
data/test/each-raw-record/test-struct-array.rb +0 -529
data/test/each-raw-record/test-table.rb +0 -47
data/test/fixture/TestOrcFile.test1.orc +0 -0
data/test/fixture/with-header-float.csv +0 -20
data/test/fixture/with-header.csv +0 -20
data/test/fixture/without-header-float.csv +0 -19
data/test/fixture/without-header.csv +0 -19
data/test/helper/omittable.rb +0 -49
data/test/helper.rb +0 -31
data/test/raw-records/test-basic-arrays.rb +0 -405
data/test/raw-records/test-dense-union-array.rb +0 -566
data/test/raw-records/test-dictionary-array.rb +0 -341
data/test/raw-records/test-list-array.rb +0 -628
data/test/raw-records/test-map-array.rb +0 -507
data/test/raw-records/test-multiple-columns.rb +0 -65
data/test/raw-records/test-sparse-union-array.rb +0 -556
data/test/raw-records/test-struct-array.rb +0 -529
data/test/raw-records/test-table.rb +0 -47
data/test/run-test.rb +0 -71
data/test/test-array-builder.rb +0 -198
data/test/test-array.rb +0 -332
data/test/test-bigdecimal.rb +0 -40
data/test/test-binary-dictionary-array-builder.rb +0 -103
data/test/test-buffer.rb +0 -49
data/test/test-chunked-array.rb +0 -198
data/test/test-column.rb +0 -123
data/test/test-csv-loader.rb +0 -297
data/test/test-data-type.rb +0 -84
data/test/test-date32-array.rb +0 -24
data/test/test-date64-array.rb +0 -25
data/test/test-decimal128-array-builder.rb +0 -126
data/test/test-decimal128-array.rb +0 -47
data/test/test-decimal128-data-type.rb +0 -31
data/test/test-decimal128.rb +0 -126
data/test/test-decimal256-array-builder.rb +0 -126
data/test/test-decimal256-array.rb +0 -47
data/test/test-decimal256-data-type.rb +0 -31
data/test/test-decimal256.rb +0 -126
data/test/test-dense-union-array.rb +0 -42
data/test/test-dense-union-data-type.rb +0 -41
data/test/test-dictionary-array.rb +0 -41
data/test/test-dictionary-data-type.rb +0 -40
data/test/test-expression.rb +0 -51
data/test/test-feather.rb +0 -49
data/test/test-field.rb +0 -117
data/test/test-file-output-stream.rb +0 -54
data/test/test-fixed-size-binary-array-builder.rb +0 -92
data/test/test-fixed-size-binary-array.rb +0 -36
data/test/test-float-scalar.rb +0 -46
data/test/test-function.rb +0 -210
data/test/test-group.rb +0 -193
data/test/test-half-float-array.rb +0 -43
data/test/test-half-float.rb +0 -130
data/test/test-list-array-builder.rb +0 -79
data/test/test-list-array.rb +0 -32
data/test/test-list-data-type.rb +0 -69
data/test/test-map-array-builder.rb +0 -110
data/test/test-map-array.rb +0 -33
data/test/test-memory-view.rb +0 -434
data/test/test-orc.rb +0 -173
data/test/test-ractor.rb +0 -34
data/test/test-record-batch-builder.rb +0 -125
data/test/test-record-batch-file-reader.rb +0 -136
data/test/test-record-batch-iterator.rb +0 -37
data/test/test-record-batch-reader.rb +0 -46
data/test/test-record-batch-stream-reader.rb +0 -129
data/test/test-record-batch.rb +0 -182
data/test/test-scalar.rb +0 -65
data/test/test-schema.rb +0 -134
data/test/test-slicer.rb +0 -589
data/test/test-sort-indices.rb +0 -40
data/test/test-sort-key.rb +0 -81
data/test/test-sort-options.rb +0 -58
data/test/test-sparse-union-array.rb +0 -38
data/test/test-sparse-union-data-type.rb +0 -41
data/test/test-stream-listener.rb +0 -60
data/test/test-string-dictionary-array-builder.rb +0 -103
data/test/test-struct-array-builder.rb +0 -184
data/test/test-struct-array.rb +0 -94
data/test/test-struct-data-type.rb +0 -112
data/test/test-table.rb +0 -1530
data/test/test-tensor.rb +0 -297
data/test/test-time.rb +0 -288
data/test/test-time32-array.rb +0 -81
data/test/test-time32-data-type.rb +0 -42
data/test/test-time64-array.rb +0 -81
data/test/test-time64-data-type.rb +0 -42
data/test/test-timestamp-array.rb +0 -45
data/test/test-timestamp-data-type.rb +0 -42
data/test/values/test-basic-arrays.rb +0 -335
data/test/values/test-dense-union-array.rb +0 -552
data/test/values/test-dictionary-array.rb +0 -325
data/test/values/test-list-array.rb +0 -587
data/test/values/test-map-array.rb +0 -489
data/test/values/test-sparse-union-array.rb +0 -543
data/test/values/test-struct-array.rb +0 -524

data/test/raw-records/test-list-array.rb DELETED Viewed

@@ -1,628 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-module RawRecordsListArrayTests
-  def build_schema(type)
-    field_description = {
-      name: :element,
-    }
-    if type.is_a?(Hash)
-      field_description = field_description.merge(type)
-    else
-      field_description[:type] = type
-    end
-    {
-      column: {
-        type: :list,
-        field: field_description,
-      },
-    }
-  end
-  def test_null
-    records = [
-      [[nil, nil, nil]],
-      [nil],
-    ]
-    target = build(:null, records)
-    assert_equal(records, target.raw_records)
-  end
-  def test_boolean
-    records = [
-      [[true, nil, false]],
-      [nil],
-    ]
-    target = build(:boolean, records)
-    assert_equal(records, target.raw_records)
-  end
-  def test_int8
-    records = [
-      [[-(2 ** 7), nil, (2 ** 7) - 1]],
-      [nil],
-    ]
-    target = build(:int8, records)
-    assert_equal(records, target.raw_records)
-  end
-  def test_uint8
-    records = [
-      [[0, nil, (2 ** 8) - 1]],
-      [nil],
-    ]
-    target = build(:uint8, records)
-    assert_equal(records, target.raw_records)
-  end
-  def test_int16
-    records = [
-      [[-(2 ** 15), nil, (2 ** 15) - 1]],
-      [nil],
-    ]
-    target = build(:int16, records)
-    assert_equal(records, target.raw_records)
-  end
-  def test_uint16
-    records = [
-      [[0, nil, (2 ** 16) - 1]],
-      [nil],
-    ]
-    target = build(:uint16, records)
-    assert_equal(records, target.raw_records)
-  end
-  def test_int32
-    records = [
-      [[-(2 ** 31), nil, (2 ** 31) - 1]],
-      [nil],
-    ]
-    target = build(:int32, records)
-    assert_equal(records, target.raw_records)
-  end
-  def test_uint32
-    records = [
-      [[0, nil, (2 ** 32) - 1]],
-      [nil],
-    ]
-    target = build(:uint32, records)
-    assert_equal(records, target.raw_records)
-  end
-  def test_int64
-    records = [
-      [[-(2 ** 63), nil, (2 ** 63) - 1]],
-      [nil],
-    ]
-    target = build(:int64, records)
-    assert_equal(records, target.raw_records)
-  end
-  def test_uint64
-    records = [
-      [[0, nil, (2 ** 64) - 1]],
-      [nil],
-    ]
-    target = build(:uint64, records)
-    assert_equal(records, target.raw_records)
-  end
-  def test_float
-    records = [
-      [[-1.0, nil, 1.0]],
-      [nil],
-    ]
-    target = build(:float, records)
-    assert_equal(records, target.raw_records)
-  end
-  def test_double
-    records = [
-      [[-1.0, nil, 1.0]],
-      [nil],
-    ]
-    target = build(:double, records)
-    assert_equal(records, target.raw_records)
-  end
-  def test_binary
-    records = [
-      [["\x00".b, nil, "\xff".b]],
-      [nil],
-    ]
-    target = build(:binary, records)
-    assert_equal(records, target.raw_records)
-  end
-  def test_string
-    records = [
-      [
-        [
-          "Ruby",
-          nil,
-          "\u3042", # U+3042 HIRAGANA LETTER A
-        ],
-      ],
-      [nil],
-    ]
-    target = build(:string, records)
-    assert_equal(records, target.raw_records)
-  end
-  def test_date32
-    records = [
-      [
-        [
-          Date.new(1960, 1, 1),
-          nil,
-          Date.new(2017, 8, 23),
-        ],
-      ],
-      [nil],
-    ]
-    target = build(:date32, records)
-    assert_equal(records, target.raw_records)
-  end
-  def test_date64
-    records = [
-      [
-        [
-          DateTime.new(1960, 1, 1, 2, 9, 30),
-          nil,
-          DateTime.new(2017, 8, 23, 14, 57, 2),
-        ],
-      ],
-      [nil],
-    ]
-    target = build(:date64, records)
-    assert_equal(records, target.raw_records)
-  end
-  def test_timestamp_second
-    records = [
-      [
-        [
-          Time.parse("1960-01-01T02:09:30Z"),
-          nil,
-          Time.parse("2017-08-23T14:57:02Z"),
-        ],
-      ],
-      [nil],
-    ]
-    target = build({
-                     type: :timestamp,
-                     unit: :second,
-                   },
-                   records)
-    assert_equal(records, target.raw_records)
-  end
-  def test_timestamp_milli
-    records = [
-      [
-        [
-          Time.parse("1960-01-01T02:09:30.123Z"),
-          nil,
-          Time.parse("2017-08-23T14:57:02.987Z"),
-        ],
-      ],
-      [nil],
-    ]
-    target = build({
-                     type: :timestamp,
-                     unit: :milli,
-                   },
-                   records)
-    assert_equal(records, target.raw_records)
-  end
-  def test_timestamp_micro
-    records = [
-      [
-        [
-          Time.parse("1960-01-01T02:09:30.123456Z"),
-          nil,
-          Time.parse("2017-08-23T14:57:02.987654Z"),
-        ],
-      ],
-      [nil],
-    ]
-    target = build({
-                     type: :timestamp,
-                     unit: :micro,
-                   },
-                   records)
-    assert_equal(records, target.raw_records)
-  end
-  def test_timestamp_nano
-    records = [
-      [
-        [
-          Time.parse("1960-01-01T02:09:30.123456789Z"),
-          nil,
-          Time.parse("2017-08-23T14:57:02.987654321Z"),
-        ],
-      ],
-      [nil],
-    ]
-    target = build({
-                     type: :timestamp,
-                     unit: :nano,
-                   },
-                   records)
-    assert_equal(records, target.raw_records)
-  end
-  def test_time32_second
-    unit = Arrow::TimeUnit::SECOND
-    records = [
-      [
-        [
-          # 00:10:00
-          Arrow::Time.new(unit, 60 * 10),
-          nil,
-          # 02:00:09
-          Arrow::Time.new(unit, 60 * 60 * 2 + 9),
-        ],
-      ],
-      [nil],
-    ]
-    target = build({
-                     type: :time32,
-                     unit: :second,
-                   },
-                   records)
-    assert_equal(records, target.raw_records)
-  end
-  def test_time32_milli
-    unit = Arrow::TimeUnit::MILLI
-    records = [
-      [
-        [
-          # 00:10:00.123
-          Arrow::Time.new(unit, (60 * 10) * 1000 + 123),
-          nil,
-          # 02:00:09.987
-          Arrow::Time.new(unit, (60 * 60 * 2 + 9) * 1000 + 987),
-        ],
-      ],
-      [nil],
-    ]
-    target = build({
-                     type: :time32,
-                     unit: :milli,
-                   },
-                   records)
-    assert_equal(records, target.raw_records)
-  end
-  def test_time64_micro
-    unit = Arrow::TimeUnit::MICRO
-    records = [
-      [
-        [
-          # 00:10:00.123456
-          Arrow::Time.new(unit, (60 * 10) * 1_000_000 + 123_456),
-          nil,
-          # 02:00:09.987654
-          Arrow::Time.new(unit, (60 * 60 * 2 + 9) * 1_000_000 + 987_654),
-        ],
-      ],
-      [nil],
-    ]
-    target = build({
-                     type: :time64,
-                     unit: :micro,
-                   },
-                   records)
-    assert_equal(records, target.raw_records)
-  end
-  def test_time64_nano
-    unit = Arrow::TimeUnit::NANO
-    records = [
-      [
-        [
-          # 00:10:00.123456789
-          Arrow::Time.new(unit, (60 * 10) * 1_000_000_000 + 123_456_789),
-          nil,
-          # 02:00:09.987654321
-          Arrow::Time.new(unit, (60 * 60 * 2 + 9) * 1_000_000_000 + 987_654_321),
-        ],
-      ],
-      [nil],
-    ]
-    target = build({
-                     type: :time64,
-                     unit: :nano,
-                   },
-                   records)
-    assert_equal(records, target.raw_records)
-  end
-  def test_decimal128
-    records = [
-      [
-        [
-          BigDecimal("92.92"),
-          nil,
-          BigDecimal("29.29"),
-        ],
-      ],
-      [nil],
-    ]
-    target = build({
-                     type: :decimal128,
-                     precision: 8,
-                     scale: 2,
-                   },
-                   records)
-    assert_equal(records, target.raw_records)
-  end
-  def test_decimal256
-    records = [
-      [
-        [
-          BigDecimal("92.92"),
-          nil,
-          BigDecimal("29.29"),
-        ],
-      ],
-      [nil],
-    ]
-    target = build({
-                     type: :decimal256,
-                     precision: 38,
-                     scale: 2,
-                   },
-                   records)
-    assert_equal(records, target.raw_records)
-  end
-  def test_month_interval
-    records = [
-      [[1, nil, 12]],
-      [nil],
-    ]
-    target = build(:month_interval, records)
-    assert_equal(records, target.raw_records)
-  end
-  def test_day_time_interval
-    records = [
-      [
-        [
-          {day: 1, millisecond: 100},
-          nil,
-          {day: 2, millisecond: 300},
-        ]
-      ],
-      [nil],
-    ]
-    target = build(:day_time_interval, records)
-    assert_equal(records, target.raw_records)
-  end
-  def test_month_day_nano_interval
-    records = [
-      [
-        [
-          {month: 1, day: 1, nanosecond: 100},
-          nil,
-          {month: 2, day: 3, nanosecond: 400},
-        ]
-      ],
-      [nil],
-    ]
-    target = build(:month_day_nano_interval, records)
-    assert_equal(records, target.raw_records)
-  end
-  def test_list
-    records = [
-      [
-        [
-          [
-            true,
-            nil,
-          ],
-          nil,
-          [
-            nil,
-            false,
-          ],
-        ],
-      ],
-      [nil],
-    ]
-    target = build({
-                     type: :list,
-                     field: {
-                       name: :sub_element,
-                       type: :boolean,
-                     },
-                   },
-                   records)
-    assert_equal(records, target.raw_records)
-  end
-  def test_struct
-    records = [
-      [
-        [
-          {"field" => true},
-          nil,
-          {"field" => nil},
-        ],
-      ],
-      [nil],
-    ]
-    target = build({
-                     type: :struct,
-                     fields: [
-                       {
-                         name: :field,
-                         type: :boolean,
-                       },
-                     ],
-                   },
-                   records)
-    assert_equal(records, target.raw_records)
-  end
-  def test_map
-    records = [
-      [
-        [
-          {"key1" => true, "key2" => nil},
-          nil,
-        ],
-      ],
-      [nil],
-    ]
-    target = build({
-                     type: :map,
-                     key: :string,
-                     item: :boolean,
-                   },
-                   records)
-    assert_equal(records, target.raw_records)
-  end
-  def remove_union_field_names(records)
-    records.collect do |record|
-      record.collect do |column|
-        if column.nil?
-          column
-        else
-          column.collect do |value|
-            if value.nil?
-              value
-            else
-              value.values[0]
-            end
-          end
-        end
-      end
-    end
-  end
-  def test_sparse_union
-    records = [
-      [
-        [
-          {"field1" => true},
-          nil,
-          {"field2" => 29},
-          {"field2" => nil},
-        ],
-      ],
-      [nil],
-    ]
-    target = build({
-                     type: :sparse_union,
-                     fields: [
-                       {
-                         name: :field1,
-                         type: :boolean,
-                       },
-                       {
-                         name: :field2,
-                         type: :uint8,
-                       },
-                     ],
-                     type_codes: [0, 1],
-                   },
-                   records)
-    assert_equal(remove_union_field_names(records),
-                 target.raw_records)
-  end
-  def test_dense_union
-    records = [
-      [
-        [
-          {"field1" => true},
-          nil,
-          {"field2" => 29},
-          {"field2" => nil},
-        ],
-      ],
-      [nil],
-    ]
-    target = build({
-                     type: :dense_union,
-                     fields: [
-                       {
-                         name: :field1,
-                         type: :boolean,
-                       },
-                       {
-                         name: :field2,
-                         type: :uint8,
-                       },
-                     ],
-                     type_codes: [0, 1],
-                   },
-                   records)
-    assert_equal(remove_union_field_names(records),
-                 target.raw_records)
-  end
-  def test_dictionary
-    records = [
-      [
-        [
-          "Ruby",
-          nil,
-          "GLib",
-        ],
-      ],
-      [nil],
-    ]
-    target = build({
-                     type: :dictionary,
-                     index_data_type: :int8,
-                     value_data_type: :string,
-                     ordered: false,
-                   },
-                   records)
-    assert_equal(records, target.raw_records)
-  end
-end
-class RawRecordsRecordBatchListArrayTest < Test::Unit::TestCase
-  include RawRecordsListArrayTests
-  def build(type, records)
-    Arrow::RecordBatch.new(build_schema(type), records)
-  end
-end
-class RawRecordsTableListArrayTest < Test::Unit::TestCase
-  include RawRecordsListArrayTests
-  def build(type, records)
-    Arrow::Table.new(build_schema(type), records)
-  end
-end