RubyGems - red-arrow - Versions diffs - 18.1.0 → 19.0.0 - Mend

red-arrow 18.1.0 → 19.0.0

Files changed (133) hide show

checksums.yaml +4 -4
data/lib/arrow/array.rb +7 -4
data/lib/arrow/column.rb +4 -4
data/lib/arrow/jruby/array-builder.rb +114 -0
data/lib/arrow/jruby/array.rb +109 -0
data/{test/helper/fixture.rb → lib/arrow/jruby/chunked-array.rb} +14 -6
data/lib/arrow/jruby/compression-type.rb +26 -0
data/{test/test-boolean-scalar.rb → lib/arrow/jruby/csv-read-options.rb} +12 -6
data/{test/test-map-data-type.rb → lib/arrow/jruby/data-type.rb} +24 -12
data/lib/arrow/jruby/decimal128.rb +28 -0
data/lib/arrow/jruby/decimal256.rb +28 -0
data/{test/fixture/null-with-double-quote.csv → lib/arrow/jruby/error.rb} +7 -4
data/lib/arrow/jruby/file-system.rb +24 -0
data/{test/test-null-array.rb → lib/arrow/jruby/function.rb} +5 -4
data/lib/arrow/jruby/record-batch-iterator.rb +24 -0
data/{test/fixture/integer-float.csv → lib/arrow/jruby/record-batch.rb} +8 -4
data/{test/fixture/float-integer.csv → lib/arrow/jruby/sort-key.rb} +8 -4
data/lib/arrow/jruby/sort-options.rb +24 -0
data/lib/arrow/jruby/stream-listener-raw.rb +25 -0
data/{test/test-rolling-window.rb → lib/arrow/jruby/table.rb} +19 -19
data/lib/arrow/jruby/writable.rb +24 -0
data/lib/arrow/jruby.rb +52 -0
data/lib/arrow/libraries.rb +126 -0
data/lib/arrow/list-array-builder.rb +1 -0
data/lib/arrow/loader.rb +3 -111
data/{test/fixture/null-without-double-quote.csv → lib/arrow/ruby.rb} +6 -4
data/lib/arrow/version.rb +1 -1
data/lib/arrow.rb +2 -7
data/red-arrow.gemspec +15 -6
metadata +26 -229
data/test/each-raw-record/test-basic-arrays.rb +0 -411
data/test/each-raw-record/test-dense-union-array.rb +0 -566
data/test/each-raw-record/test-dictionary-array.rb +0 -341
data/test/each-raw-record/test-list-array.rb +0 -628
data/test/each-raw-record/test-map-array.rb +0 -507
data/test/each-raw-record/test-multiple-columns.rb +0 -72
data/test/each-raw-record/test-sparse-union-array.rb +0 -528
data/test/each-raw-record/test-struct-array.rb +0 -529
data/test/each-raw-record/test-table.rb +0 -47
data/test/fixture/TestOrcFile.test1.orc +0 -0
data/test/fixture/with-header-float.csv +0 -20
data/test/fixture/with-header.csv +0 -20
data/test/fixture/without-header-float.csv +0 -19
data/test/fixture/without-header.csv +0 -19
data/test/helper/omittable.rb +0 -49
data/test/helper.rb +0 -31
data/test/raw-records/test-basic-arrays.rb +0 -405
data/test/raw-records/test-dense-union-array.rb +0 -566
data/test/raw-records/test-dictionary-array.rb +0 -341
data/test/raw-records/test-list-array.rb +0 -628
data/test/raw-records/test-map-array.rb +0 -507
data/test/raw-records/test-multiple-columns.rb +0 -65
data/test/raw-records/test-sparse-union-array.rb +0 -556
data/test/raw-records/test-struct-array.rb +0 -529
data/test/raw-records/test-table.rb +0 -47
data/test/run-test.rb +0 -71
data/test/test-array-builder.rb +0 -198
data/test/test-array.rb +0 -332
data/test/test-bigdecimal.rb +0 -40
data/test/test-binary-dictionary-array-builder.rb +0 -103
data/test/test-buffer.rb +0 -49
data/test/test-chunked-array.rb +0 -198
data/test/test-column.rb +0 -123
data/test/test-csv-loader.rb +0 -297
data/test/test-data-type.rb +0 -84
data/test/test-date32-array.rb +0 -24
data/test/test-date64-array.rb +0 -25
data/test/test-decimal128-array-builder.rb +0 -126
data/test/test-decimal128-array.rb +0 -47
data/test/test-decimal128-data-type.rb +0 -31
data/test/test-decimal128.rb +0 -126
data/test/test-decimal256-array-builder.rb +0 -126
data/test/test-decimal256-array.rb +0 -47
data/test/test-decimal256-data-type.rb +0 -31
data/test/test-decimal256.rb +0 -126
data/test/test-dense-union-array.rb +0 -42
data/test/test-dense-union-data-type.rb +0 -41
data/test/test-dictionary-array.rb +0 -41
data/test/test-dictionary-data-type.rb +0 -40
data/test/test-expression.rb +0 -51
data/test/test-feather.rb +0 -49
data/test/test-field.rb +0 -117
data/test/test-file-output-stream.rb +0 -54
data/test/test-fixed-size-binary-array-builder.rb +0 -92
data/test/test-fixed-size-binary-array.rb +0 -36
data/test/test-float-scalar.rb +0 -46
data/test/test-function.rb +0 -210
data/test/test-group.rb +0 -193
data/test/test-half-float-array.rb +0 -43
data/test/test-half-float.rb +0 -130
data/test/test-list-array-builder.rb +0 -79
data/test/test-list-array.rb +0 -32
data/test/test-list-data-type.rb +0 -69
data/test/test-map-array-builder.rb +0 -110
data/test/test-map-array.rb +0 -33
data/test/test-memory-view.rb +0 -434
data/test/test-orc.rb +0 -173
data/test/test-ractor.rb +0 -34
data/test/test-record-batch-builder.rb +0 -125
data/test/test-record-batch-file-reader.rb +0 -136
data/test/test-record-batch-iterator.rb +0 -37
data/test/test-record-batch-reader.rb +0 -46
data/test/test-record-batch-stream-reader.rb +0 -129
data/test/test-record-batch.rb +0 -182
data/test/test-scalar.rb +0 -65
data/test/test-schema.rb +0 -134
data/test/test-slicer.rb +0 -589
data/test/test-sort-indices.rb +0 -40
data/test/test-sort-key.rb +0 -81
data/test/test-sort-options.rb +0 -58
data/test/test-sparse-union-array.rb +0 -38
data/test/test-sparse-union-data-type.rb +0 -41
data/test/test-stream-listener.rb +0 -60
data/test/test-string-dictionary-array-builder.rb +0 -103
data/test/test-struct-array-builder.rb +0 -184
data/test/test-struct-array.rb +0 -94
data/test/test-struct-data-type.rb +0 -112
data/test/test-table.rb +0 -1530
data/test/test-tensor.rb +0 -297
data/test/test-time.rb +0 -288
data/test/test-time32-array.rb +0 -81
data/test/test-time32-data-type.rb +0 -42
data/test/test-time64-array.rb +0 -81
data/test/test-time64-data-type.rb +0 -42
data/test/test-timestamp-array.rb +0 -45
data/test/test-timestamp-data-type.rb +0 -42
data/test/values/test-basic-arrays.rb +0 -335
data/test/values/test-dense-union-array.rb +0 -552
data/test/values/test-dictionary-array.rb +0 -325
data/test/values/test-list-array.rb +0 -587
data/test/values/test-map-array.rb +0 -489
data/test/values/test-sparse-union-array.rb +0 -543
data/test/values/test-struct-array.rb +0 -524

data/test/each-raw-record/test-dense-union-array.rb DELETED Viewed

@@ -1,566 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements.  See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership.  The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License.  You may obtain a copy of the License at
-#
-#   http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied.  See the License for the
-# specific language governing permissions and limitations
-# under the License.
-module EachRawRecordDenseUnionArrayTests
-  def build_schema(type, type_codes)
-    field_description = {}
-    if type.is_a?(Hash)
-      field_description = field_description.merge(type)
-    else
-      field_description[:type] = type
-    end
-    {
-      column: {
-        type: :dense_union,
-        fields: [
-          field_description.merge(name: "0"),
-          field_description.merge(name: "1"),
-        ],
-        type_codes: type_codes,
-      },
-    }
-  end
-  # TODO: Use Arrow::RecordBatch.new(build_schema(type, type_codes), records)
-  def build_record_batch(type, records)
-    type_codes = [0, 1]
-    schema = Arrow::Schema.new(build_schema(type, type_codes))
-    type_ids = []
-    offsets = []
-    arrays = schema.fields[0].data_type.fields.collect do |field|
-      sub_schema = Arrow::Schema.new([field])
-      sub_records = []
-      records.each do |record|
-        column = record[0]
-        next if column.nil?
-        next unless column.key?(field.name)
-        sub_records << [column[field.name]]
-      end
-      sub_record_batch = Arrow::RecordBatch.new(sub_schema,
-                                                sub_records)
-      sub_record_batch.columns[0].data
-    end
-    records.each do |record|
-      column = record[0]
-      if column.key?("0")
-        type_id = type_codes[0]
-        type_ids << type_id
-        offsets << (type_ids.count(type_id) - 1)
-      elsif column.key?("1")
-        type_id = type_codes[1]
-        type_ids << type_id
-        offsets << (type_ids.count(type_id) - 1)
-      end
-    end
-    union_array = Arrow::DenseUnionArray.new(schema.fields[0].data_type,
-                                             Arrow::Int8Array.new(type_ids),
-                                             Arrow::Int32Array.new(offsets),
-                                             arrays)
-    schema = Arrow::Schema.new(column: union_array.value_data_type)
-    Arrow::RecordBatch.new(schema,
-                           records.size,
-                           [union_array])
-  end
-  def remove_field_names(records)
-    records.collect do |record|
-      record.collect do |column|
-        if column.nil?
-          column
-        else
-          column.values[0]
-        end
-      end
-    end
-  end
-  def test_null
-    records = [
-      [{"0" => nil}],
-    ]
-    target = build(:null, records)
-    assert_equal(remove_field_names(records),
-                 target.each_raw_record.to_a)
-  end
-  def test_boolean
-    records = [
-      [{"0" => true}],
-      [{"1" => nil}],
-    ]
-    target = build(:boolean, records)
-    assert_equal(remove_field_names(records),
-                 target.each_raw_record.to_a)
-  end
-  def test_int8
-    records = [
-      [{"0" => -(2 ** 7)}],
-      [{"1" => nil}],
-    ]
-    target = build(:int8, records)
-    assert_equal(remove_field_names(records),
-                 target.each_raw_record.to_a)
-  end
-  def test_uint8
-    records = [
-      [{"0" => (2 ** 8) - 1}],
-      [{"1" => nil}],
-    ]
-    target = build(:uint8, records)
-    assert_equal(remove_field_names(records),
-                 target.each_raw_record.to_a)
-  end
-  def test_int16
-    records = [
-      [{"0" => -(2 ** 15)}],
-      [{"1" => nil}],
-    ]
-    target = build(:int16, records)
-    assert_equal(remove_field_names(records),
-                 target.each_raw_record.to_a)
-  end
-  def test_uint16
-    records = [
-      [{"0" => (2 ** 16) - 1}],
-      [{"1" => nil}],
-    ]
-    target = build(:uint16, records)
-    assert_equal(remove_field_names(records),
-                 target.each_raw_record.to_a)
-  end
-  def test_int32
-    records = [
-      [{"0" => -(2 ** 31)}],
-      [{"1" => nil}],
-    ]
-    target = build(:int32, records)
-    assert_equal(remove_field_names(records),
-                 target.each_raw_record.to_a)
-  end
-  def test_uint32
-    records = [
-      [{"0" => (2 ** 32) - 1}],
-      [{"1" => nil}],
-    ]
-    target = build(:uint32, records)
-    assert_equal(remove_field_names(records),
-                 target.each_raw_record.to_a)
-  end
-  def test_int64
-    records = [
-      [{"0" => -(2 ** 63)}],
-      [{"1" => nil}],
-    ]
-    target = build(:int64, records)
-    assert_equal(remove_field_names(records),
-                 target.each_raw_record.to_a)
-  end
-  def test_uint64
-    records = [
-      [{"0" => (2 ** 64) - 1}],
-      [{"1" => nil}],
-    ]
-    target = build(:uint64, records)
-    assert_equal(remove_field_names(records),
-                 target.each_raw_record.to_a)
-  end
-  def test_float
-    records = [
-      [{"0" => -1.0}],
-      [{"1" => nil}],
-    ]
-    target = build(:float, records)
-    assert_equal(remove_field_names(records),
-                 target.each_raw_record.to_a)
-  end
-  def test_double
-    records = [
-      [{"0" => -1.0}],
-      [{"1" => nil}],
-    ]
-    target = build(:double, records)
-    assert_equal(remove_field_names(records),
-                 target.each_raw_record.to_a)
-  end
-  def test_binary
-    records = [
-      [{"0" => "\xff".b}],
-      [{"1" => nil}],
-    ]
-    target = build(:binary, records)
-    assert_equal(remove_field_names(records),
-                 target.each_raw_record.to_a)
-  end
-  def test_string
-    records = [
-      [{"0" => "Ruby"}],
-      [{"1" => nil}],
-    ]
-    target = build(:string, records)
-    assert_equal(remove_field_names(records),
-                 target.each_raw_record.to_a)
-  end
-  def test_date32
-    records = [
-      [{"0" => Date.new(1960, 1, 1)}],
-      [{"1" => nil}],
-    ]
-    target = build(:date32, records)
-    assert_equal(remove_field_names(records),
-                 target.each_raw_record.to_a)
-  end
-  def test_date64
-    records = [
-      [{"0" => DateTime.new(1960, 1, 1, 2, 9, 30)}],
-      [{"1" => nil}],
-    ]
-    target = build(:date64, records)
-    assert_equal(remove_field_names(records),
-                 target.each_raw_record.to_a)
-  end
-  def test_timestamp_second
-    records = [
-      [{"0" => Time.parse("1960-01-01T02:09:30Z")}],
-      [{"1" => nil}],
-    ]
-    target = build({
-                     type: :timestamp,
-                     unit: :second,
-                   },
-                   records)
-    assert_equal(remove_field_names(records),
-                 target.each_raw_record.to_a)
-  end
-  def test_timestamp_milli
-    records = [
-      [{"0" => Time.parse("1960-01-01T02:09:30.123Z")}],
-      [{"1" => nil}],
-    ]
-    target = build({
-                     type: :timestamp,
-                     unit: :milli,
-                   },
-                   records)
-    assert_equal(remove_field_names(records),
-                 target.each_raw_record.to_a)
-  end
-  def test_timestamp_micro
-    records = [
-      [{"0" => Time.parse("1960-01-01T02:09:30.123456Z")}],
-      [{"1" => nil}],
-    ]
-    target = build({
-                     type: :timestamp,
-                     unit: :micro,
-                   },
-                   records)
-    assert_equal(remove_field_names(records),
-                 target.each_raw_record.to_a)
-  end
-  def test_timestamp_nano
-    records = [
-      [{"0" => Time.parse("1960-01-01T02:09:30.123456789Z")}],
-      [{"1" => nil}],
-    ]
-    target = build({
-                     type: :timestamp,
-                     unit: :nano,
-                   },
-                   records)
-    assert_equal(remove_field_names(records),
-                 target.each_raw_record.to_a)
-  end
-  def test_time32_second
-    unit = Arrow::TimeUnit::SECOND
-    records = [
-      # 00:10:00
-      [{"0" => Arrow::Time.new(unit, 60 * 10)}],
-      [{"1" => nil}],
-    ]
-    target = build({
-                     type: :time32,
-                     unit: :second,
-                   },
-                   records)
-    assert_equal(remove_field_names(records),
-                 target.each_raw_record.to_a)
-  end
-  def test_time32_milli
-    unit = Arrow::TimeUnit::MILLI
-    records = [
-      # 00:10:00.123
-      [{"0" => Arrow::Time.new(unit, (60 * 10) * 1000 + 123)}],
-      [{"1" => nil}],
-    ]
-    target = build({
-                     type: :time32,
-                     unit: :milli,
-                   },
-                   records)
-    assert_equal(remove_field_names(records),
-                 target.each_raw_record.to_a)
-  end
-  def test_time64_micro
-    unit = Arrow::TimeUnit::MICRO
-    records = [
-      # 00:10:00.123456
-      [{"0" => Arrow::Time.new(unit, (60 * 10) * 1_000_000 + 123_456)}],
-      [{"1" => nil}],
-    ]
-    target = build({
-                     type: :time64,
-                     unit: :micro,
-                   },
-                   records)
-    assert_equal(remove_field_names(records),
-                 target.each_raw_record.to_a)
-  end
-  def test_time64_nano
-    unit = Arrow::TimeUnit::NANO
-    records = [
-      # 00:10:00.123456789
-      [{"0" => Arrow::Time.new(unit, (60 * 10) * 1_000_000_000 + 123_456_789)}],
-      [{"1" => nil}],
-    ]
-    target = build({
-                     type: :time64,
-                     unit: :nano,
-                   },
-                   records)
-    assert_equal(remove_field_names(records),
-                 target.each_raw_record.to_a)
-  end
-  def test_decimal128
-    records = [
-      [{"0" => BigDecimal("92.92")}],
-      [{"1" => nil}],
-    ]
-    target = build({
-                     type: :decimal128,
-                     precision: 8,
-                     scale: 2,
-                   },
-                   records)
-    assert_equal(remove_field_names(records),
-                 target.each_raw_record.to_a)
-  end
-  def test_decimal256
-    records = [
-      [{"0" => BigDecimal("92.92")}],
-      [{"1" => nil}],
-    ]
-    target = build({
-                     type: :decimal256,
-                     precision: 38,
-                     scale: 2,
-                   },
-                   records)
-    assert_equal(remove_field_names(records),
-                 target.each_raw_record.to_a)
-  end
-  def test_month_interval
-    records = [
-      [{"0" => 1}],
-      [{"1" => nil}],
-    ]
-    target = build(:month_interval, records)
-    assert_equal(remove_field_names(records),
-                 target.each_raw_record.to_a)
-  end
-  def test_day_time_interval
-    records = [
-      [{"0" => {day: 1, millisecond: 100}}],
-      [{"1" => nil}],
-    ]
-    target = build(:day_time_interval, records)
-    assert_equal(remove_field_names(records),
-                 target.each_raw_record.to_a)
-  end
-  def test_month_day_nano_interval
-    records = [
-      [{"0" => {month: 1, day: 1, nanosecond: 100}}],
-      [{"1" => nil}],
-    ]
-    target = build(:month_day_nano_interval, records)
-    assert_equal(remove_field_names(records),
-                 target.each_raw_record.to_a)
-  end
-  def test_list
-    records = [
-      [{"0" => [true, nil, false]}],
-      [{"1" => nil}],
-    ]
-    target = build({
-                     type: :list,
-                     field: {
-                       name: :sub_element,
-                       type: :boolean,
-                     },
-                   },
-                   records)
-    assert_equal(remove_field_names(records),
-                 target.each_raw_record.to_a)
-  end
-  def test_struct
-    records = [
-      [{"0" => {"sub_field" => true}}],
-      [{"1" => nil}],
-      [{"0" => {"sub_field" => nil}}],
-    ]
-    target = build({
-                     type: :struct,
-                     fields: [
-                       {
-                         name: :sub_field,
-                         type: :boolean,
-                       },
-                     ],
-                   },
-                   records)
-    assert_equal(remove_field_names(records),
-                 target.each_raw_record.to_a)
-  end
-  def test_map
-    records = [
-      [{"0" => {"key1" => true, "key2" => nil}}],
-      [{"1" => nil}],
-    ]
-    target = build({
-                     type: :map,
-                     key: :string,
-                     item: :boolean,
-                   },
-                   records)
-    assert_equal(remove_field_names(records),
-                 target.each_raw_record.to_a)
-  end
-  def test_sparse_union
-    records = [
-      [{"0" => {"field1" => true}}],
-      [{"1" => nil}],
-      [{"0" => {"field2" => 29}}],
-      [{"0" => {"field2" => nil}}],
-    ]
-    target = build({
-                     type: :sparse_union,
-                     fields: [
-                       {
-                         name: :field1,
-                         type: :boolean,
-                       },
-                       {
-                         name: :field2,
-                         type: :uint8,
-                       },
-                     ],
-                     type_codes: [0, 1],
-                   },
-                   records)
-    assert_equal(remove_field_names(remove_field_names(records)),
-                 target.each_raw_record.to_a)
-  end
-  def test_dense_union
-    records = [
-      [{"0" => {"field1" => true}}],
-      [{"1" => nil}],
-      [{"0" => {"field2" => 29}}],
-      [{"0" => {"field2" => nil}}],
-    ]
-    target = build({
-                     type: :dense_union,
-                     fields: [
-                       {
-                         name: :field1,
-                         type: :boolean,
-                       },
-                       {
-                         name: :field2,
-                         type: :uint8,
-                       },
-                     ],
-                     type_codes: [0, 1],
-                   },
-                   records)
-    assert_equal(remove_field_names(remove_field_names(records)),
-                 target.each_raw_record.to_a)
-  end
-  def test_dictionary
-    records = [
-      [{"0" => "Ruby"}],
-      [{"1" => nil}],
-      [{"0" => "GLib"}],
-    ]
-    target = build({
-                     type: :dictionary,
-                     index_data_type: :int8,
-                     value_data_type: :string,
-                     ordered: false,
-                   },
-                   records)
-    assert_equal(remove_field_names(records),
-                 target.each_raw_record.to_a)
-  end
-end
-class EachRawRecordRecordBatchDenseUnionArrayTest < Test::Unit::TestCase
-  include EachRawRecordDenseUnionArrayTests
-  def build(type, records)
-    build_record_batch(type, records)
-  end
-end
-class EachRawRecordTableDenseUnionArrayTest < Test::Unit::TestCase
-  include EachRawRecordDenseUnionArrayTests
-  def build(type, records)
-    build_record_batch(type, records).to_table
-  end
-end