red-arrow 10.0.1 → 12.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +3 -3
- data/ext/arrow/converters.hpp +45 -41
- data/ext/arrow/extconf.rb +14 -2
- data/ext/arrow/raw-records.cpp +1 -2
- data/ext/arrow/values.cpp +1 -2
- data/lib/arrow/array-computable.rb +13 -0
- data/lib/arrow/array.rb +5 -0
- data/lib/arrow/chunked-array.rb +23 -1
- data/lib/arrow/column-containable.rb +9 -0
- data/lib/arrow/column.rb +1 -0
- data/lib/arrow/data-type.rb +9 -0
- data/lib/arrow/dense-union-array-builder.rb +49 -0
- data/lib/arrow/dense-union-array.rb +26 -0
- data/lib/arrow/half-float-array-builder.rb +32 -0
- data/lib/arrow/half-float-array.rb +24 -0
- data/lib/arrow/half-float.rb +118 -0
- data/lib/arrow/input-referable.rb +29 -0
- data/lib/arrow/loader.rb +10 -0
- data/lib/arrow/raw-table-converter.rb +7 -5
- data/lib/arrow/record-batch-file-reader.rb +2 -0
- data/lib/arrow/record-batch-stream-reader.rb +2 -0
- data/lib/arrow/record-batch.rb +6 -2
- data/lib/arrow/scalar.rb +67 -0
- data/lib/arrow/slicer.rb +61 -0
- data/lib/arrow/sparse-union-array-builder.rb +56 -0
- data/lib/arrow/sparse-union-array.rb +26 -0
- data/lib/arrow/struct-array-builder.rb +0 -5
- data/lib/arrow/table-loader.rb +4 -4
- data/lib/arrow/table-saver.rb +1 -0
- data/lib/arrow/table.rb +178 -31
- data/lib/arrow/tensor.rb +4 -0
- data/lib/arrow/union-array-builder.rb +59 -0
- data/lib/arrow/version.rb +1 -1
- data/red-arrow.gemspec +1 -1
- data/test/raw-records/test-basic-arrays.rb +10 -0
- data/test/raw-records/test-dense-union-array.rb +90 -45
- data/test/raw-records/test-list-array.rb +28 -10
- data/test/raw-records/test-map-array.rb +39 -10
- data/test/raw-records/test-sparse-union-array.rb +86 -41
- data/test/raw-records/test-struct-array.rb +22 -8
- data/test/test-array.rb +7 -0
- data/test/test-chunked-array.rb +9 -0
- data/test/test-data-type.rb +2 -1
- data/test/test-dense-union-array.rb +42 -0
- data/test/test-dense-union-data-type.rb +1 -1
- data/test/test-function.rb +7 -7
- data/test/test-group.rb +58 -58
- data/test/test-half-float-array.rb +43 -0
- data/test/test-half-float.rb +130 -0
- data/test/test-record-batch-file-reader.rb +21 -0
- data/test/test-record-batch-stream-reader.rb +129 -0
- data/test/test-scalar.rb +65 -0
- data/test/test-slicer.rb +194 -129
- data/test/test-sparse-union-array.rb +38 -0
- data/test/test-table.rb +324 -40
- data/test/values/test-basic-arrays.rb +10 -0
- data/test/values/test-dense-union-array.rb +88 -45
- data/test/values/test-list-array.rb +26 -10
- data/test/values/test-map-array.rb +33 -10
- data/test/values/test-sparse-union-array.rb +84 -41
- data/test/values/test-struct-array.rb +20 -8
- metadata +30 -9
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 583b384182a9906a4fcd723cb67b0600b07dd962a1d45e114619014a396800c6
|
4
|
+
data.tar.gz: 97b7a63c52c678b7255a520d768f59fde993ffd2fd9014f24b0b946940f656b8
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: ab37d28349e4a3b23b629c4a2431a48e6b0a0b4884fb0274a484965e5c2ed551cc5b7b998433940c8e992135803f65a4c3b0555abebd6478fdaeb665f70e3f25
|
7
|
+
data.tar.gz: cfb8de7ce8d9b9cabb76f016e51bde489f9c58fc42253612395082a9ccab79001f56978d102ef18ac93f31e696406980304f2d4e40af46e8b2ba041363308920
|
data/README.md
CHANGED
@@ -25,9 +25,9 @@ Red Arrow is the Ruby bindings of Apache Arrow. Red Arrow is based on GObject In
|
|
25
25
|
|
26
26
|
[GObject Introspection](https://wiki.gnome.org/action/show/Projects/GObjectIntrospection) is a middleware for language bindings of C library. GObject Introspection can generate language bindings automatically at runtime.
|
27
27
|
|
28
|
-
Red Arrow uses [Apache Arrow GLib](https://github.com/apache/arrow/tree/
|
28
|
+
Red Arrow uses [Apache Arrow GLib](https://github.com/apache/arrow/tree/main/c_glib) and [gobject-introspection gem](https://rubygems.org/gems/gobject-introspection) to generate Ruby bindings of Apache Arrow.
|
29
29
|
|
30
|
-
Apache Arrow GLib is a C wrapper for [Apache Arrow C++](https://github.com/apache/arrow/tree/
|
30
|
+
Apache Arrow GLib is a C wrapper for [Apache Arrow C++](https://github.com/apache/arrow/tree/main/cpp). GObject Introspection can't use Apache Arrow C++ directly. Apache Arrow GLib is a bridge between Apache Arrow C++ and GObject Introspection.
|
31
31
|
|
32
32
|
gobject-introspection gem is a Ruby bindings of GObject Introspection. Red Arrow uses GObject Introspection via gobject-introspection gem.
|
33
33
|
|
@@ -56,7 +56,7 @@ table.save("/dev/shm/data-processed.arrow")
|
|
56
56
|
Note that you need to install Apache Arrow C++/GLib at master before preparing Red Arrow. See also:
|
57
57
|
|
58
58
|
* For Apache Arrow C++: https://arrow.apache.org/docs/developers/cpp/building.html
|
59
|
-
* For Apache Arrow GLib: https://github.com/apache/arrow/blob/
|
59
|
+
* For Apache Arrow GLib: https://github.com/apache/arrow/blob/main/c_glib/README.md
|
60
60
|
|
61
61
|
```console
|
62
62
|
$ cd ruby/red-arrow
|
data/ext/arrow/converters.hpp
CHANGED
@@ -106,10 +106,34 @@ namespace red_arrow {
|
|
106
106
|
return ULL2NUM(array.Value(i));
|
107
107
|
}
|
108
108
|
|
109
|
-
|
110
|
-
|
111
|
-
|
112
|
-
|
109
|
+
inline VALUE convert(const arrow::HalfFloatArray& array,
|
110
|
+
const int64_t i) {
|
111
|
+
const auto value = array.Value(i);
|
112
|
+
// | sign (1 bit) | exponent (5 bit) | fraction (10 bit) |
|
113
|
+
constexpr auto exponent_n_bits = 5;
|
114
|
+
static const auto exponent_mask =
|
115
|
+
static_cast<uint32_t>(std::pow(2.0, exponent_n_bits) - 1);
|
116
|
+
constexpr auto exponent_bias = 15;
|
117
|
+
constexpr auto fraction_n_bits = 10;
|
118
|
+
static const auto fraction_mask =
|
119
|
+
static_cast<uint32_t>(std::pow(2.0, fraction_n_bits)) - 1;
|
120
|
+
static const auto fraction_denominator = std::pow(2.0, fraction_n_bits);
|
121
|
+
const auto sign = value >> (exponent_n_bits + fraction_n_bits);
|
122
|
+
const auto exponent = (value >> fraction_n_bits) & exponent_mask;
|
123
|
+
const auto fraction = value & fraction_mask;
|
124
|
+
if (exponent == exponent_mask) {
|
125
|
+
if (sign == 0) {
|
126
|
+
return DBL2NUM(HUGE_VAL);
|
127
|
+
} else {
|
128
|
+
return DBL2NUM(-HUGE_VAL);
|
129
|
+
}
|
130
|
+
} else {
|
131
|
+
const auto implicit_fraction = (exponent == 0) ? 0 : 1;
|
132
|
+
return DBL2NUM(((sign == 0) ? 1 : -1) *
|
133
|
+
std::pow(2.0, exponent - exponent_bias) *
|
134
|
+
(implicit_fraction + fraction / fraction_denominator));
|
135
|
+
}
|
136
|
+
}
|
113
137
|
|
114
138
|
inline VALUE convert(const arrow::FloatArray& array,
|
115
139
|
const int64_t i) {
|
@@ -320,8 +344,7 @@ namespace red_arrow {
|
|
320
344
|
VISIT(UInt16)
|
321
345
|
VISIT(UInt32)
|
322
346
|
VISIT(UInt64)
|
323
|
-
|
324
|
-
// VISIT(HalfFloat)
|
347
|
+
VISIT(HalfFloat)
|
325
348
|
VISIT(Float)
|
326
349
|
VISIT(Double)
|
327
350
|
VISIT(Binary)
|
@@ -427,8 +450,7 @@ namespace red_arrow {
|
|
427
450
|
VISIT(UInt16)
|
428
451
|
VISIT(UInt32)
|
429
452
|
VISIT(UInt64)
|
430
|
-
|
431
|
-
// VISIT(HalfFloat)
|
453
|
+
VISIT(HalfFloat)
|
432
454
|
VISIT(Float)
|
433
455
|
VISIT(Double)
|
434
456
|
VISIT(Binary)
|
@@ -530,8 +552,7 @@ namespace red_arrow {
|
|
530
552
|
VISIT(UInt16)
|
531
553
|
VISIT(UInt32)
|
532
554
|
VISIT(UInt64)
|
533
|
-
|
534
|
-
// VISIT(HalfFloat)
|
555
|
+
VISIT(HalfFloat)
|
535
556
|
VISIT(Float)
|
536
557
|
VISIT(Double)
|
537
558
|
VISIT(Binary)
|
@@ -634,8 +655,7 @@ namespace red_arrow {
|
|
634
655
|
VISIT(UInt16)
|
635
656
|
VISIT(UInt32)
|
636
657
|
VISIT(UInt64)
|
637
|
-
|
638
|
-
// VISIT(HalfFloat)
|
658
|
+
VISIT(HalfFloat)
|
639
659
|
VISIT(Float)
|
640
660
|
VISIT(Double)
|
641
661
|
VISIT(Binary)
|
@@ -665,25 +685,21 @@ namespace red_arrow {
|
|
665
685
|
private:
|
666
686
|
template <typename ArrayType>
|
667
687
|
inline void convert_value(const ArrayType& array) {
|
668
|
-
auto result = rb_hash_new();
|
669
688
|
if (array.IsNull(index_)) {
|
670
|
-
|
689
|
+
result_ = RUBY_Qnil;
|
671
690
|
} else {
|
672
|
-
|
673
|
-
field_name_,
|
674
|
-
array_value_converter_->convert(array, index_));
|
691
|
+
result_ = array_value_converter_->convert(array, index_);
|
675
692
|
}
|
676
|
-
result_ = result;
|
677
693
|
}
|
678
694
|
|
679
|
-
|
680
|
-
|
681
|
-
|
695
|
+
int8_t compute_child_id(const arrow::UnionArray& array,
|
696
|
+
arrow::UnionType* type,
|
697
|
+
const char* tag) {
|
682
698
|
const auto type_code = array.raw_type_codes()[index_];
|
683
699
|
if (type_code >= 0 && type_code <= arrow::UnionType::kMaxTypeCode) {
|
684
|
-
const auto
|
685
|
-
if (
|
686
|
-
return
|
700
|
+
const auto child_id = type->child_ids()[type_code];
|
701
|
+
if (child_id >= 0) {
|
702
|
+
return child_id;
|
687
703
|
}
|
688
704
|
}
|
689
705
|
check_status(arrow::Status::Invalid("Unknown type ID: ", type_code),
|
@@ -695,36 +711,25 @@ namespace red_arrow {
|
|
695
711
|
const auto type =
|
696
712
|
std::static_pointer_cast<arrow::UnionType>(array.type()).get();
|
697
713
|
const auto tag = "[raw-records][union-sparse-array]";
|
698
|
-
const auto
|
699
|
-
const auto
|
700
|
-
const auto& field_name = field->name();
|
701
|
-
const auto field_name_keep = field_name_;
|
702
|
-
field_name_ = rb_utf8_str_new(field_name.data(), field_name.length());
|
703
|
-
const auto field_array = array.field(index).get();
|
714
|
+
const auto child_id = compute_child_id(array, type, tag);
|
715
|
+
const auto field_array = array.field(child_id).get();
|
704
716
|
check_status(field_array->Accept(this), tag);
|
705
|
-
field_name_ = field_name_keep;
|
706
717
|
}
|
707
718
|
|
708
719
|
void convert_dense(const arrow::DenseUnionArray& array) {
|
709
720
|
const auto type =
|
710
721
|
std::static_pointer_cast<arrow::UnionType>(array.type()).get();
|
711
722
|
const auto tag = "[raw-records][union-dense-array]";
|
712
|
-
const auto
|
713
|
-
const auto
|
714
|
-
const auto& field_name = field->name();
|
715
|
-
const auto field_name_keep = field_name_;
|
716
|
-
field_name_ = rb_utf8_str_new(field_name.data(), field_name.length());
|
717
|
-
const auto field_array = array.field(index);
|
723
|
+
const auto child_id = compute_child_id(array, type, tag);
|
724
|
+
const auto field_array = array.field(child_id);
|
718
725
|
const auto index_keep = index_;
|
719
726
|
index_ = array.value_offset(index_);
|
720
727
|
check_status(field_array->Accept(this), tag);
|
721
728
|
index_ = index_keep;
|
722
|
-
field_name_ = field_name_keep;
|
723
729
|
}
|
724
730
|
|
725
731
|
ArrayValueConverter* array_value_converter_;
|
726
732
|
int64_t index_;
|
727
|
-
VALUE field_name_;
|
728
733
|
VALUE result_;
|
729
734
|
};
|
730
735
|
|
@@ -761,8 +766,7 @@ namespace red_arrow {
|
|
761
766
|
VISIT(UInt16)
|
762
767
|
VISIT(UInt32)
|
763
768
|
VISIT(UInt64)
|
764
|
-
|
765
|
-
// VISIT(HalfFloat)
|
769
|
+
VISIT(HalfFloat)
|
766
770
|
VISIT(Float)
|
767
771
|
VISIT(Double)
|
768
772
|
VISIT(Binary)
|
data/ext/arrow/extconf.rb
CHANGED
@@ -38,8 +38,6 @@ checking_for(checking_message("Homebrew")) do
|
|
38
38
|
end
|
39
39
|
end
|
40
40
|
|
41
|
-
$CXXFLAGS += " -std=c++17 "
|
42
|
-
|
43
41
|
unless required_pkg_config_package([
|
44
42
|
"arrow",
|
45
43
|
Arrow::Version::MAJOR,
|
@@ -77,4 +75,18 @@ end
|
|
77
75
|
add_depend_package_path(name, source_dir, build_dir)
|
78
76
|
end
|
79
77
|
|
78
|
+
case RUBY_PLATFORM
|
79
|
+
when /darwin/
|
80
|
+
symbols_in_external_bundles = [
|
81
|
+
"_rbgerr_gerror2exception",
|
82
|
+
"_rbgobj_instance_from_ruby_object",
|
83
|
+
]
|
84
|
+
symbols_in_external_bundles.each do |symbol|
|
85
|
+
$DLDFLAGS << " -Wl,-U,#{symbol}"
|
86
|
+
end
|
87
|
+
mmacosx_version_min = "-mmacosx-version-min=10.14"
|
88
|
+
$CFLAGS << " #{mmacosx_version_min}"
|
89
|
+
$CXXFLAGS << " #{mmacosx_version_min}"
|
90
|
+
end
|
91
|
+
|
80
92
|
create_makefile("arrow")
|
data/ext/arrow/raw-records.cpp
CHANGED
data/ext/arrow/values.cpp
CHANGED
@@ -29,6 +29,19 @@ module Arrow
|
|
29
29
|
unique.values
|
30
30
|
end
|
31
31
|
|
32
|
+
# Finds the index of the first occurrence of a given value.
|
33
|
+
#
|
34
|
+
# @param value [Object] The value to be compared.
|
35
|
+
#
|
36
|
+
# @return [Integer] The index of the first occurrence of a given
|
37
|
+
# value on found, -1 on not found.
|
38
|
+
#
|
39
|
+
# @since 12.0.0
|
40
|
+
def index(value)
|
41
|
+
value = Scalar.resolve(value, value_data_type)
|
42
|
+
compute("index", options: {value: value}).value
|
43
|
+
end
|
44
|
+
|
32
45
|
private
|
33
46
|
def compute(name, options: nil)
|
34
47
|
Function.find(name).execute([self], options).value
|
data/lib/arrow/array.rb
CHANGED
@@ -22,6 +22,7 @@ module Arrow
|
|
22
22
|
include ArrayComputable
|
23
23
|
include GenericFilterable
|
24
24
|
include GenericTakeable
|
25
|
+
include InputReferable
|
25
26
|
|
26
27
|
class << self
|
27
28
|
def new(*args)
|
@@ -115,6 +116,10 @@ module Arrow
|
|
115
116
|
self
|
116
117
|
end
|
117
118
|
|
119
|
+
def to_arrow_chunked_array
|
120
|
+
ChunkedArray.new([self])
|
121
|
+
end
|
122
|
+
|
118
123
|
alias_method :value_data_type_raw, :value_data_type
|
119
124
|
def value_data_type
|
120
125
|
@value_data_type ||= value_data_type_raw
|
data/lib/arrow/chunked-array.rb
CHANGED
@@ -22,6 +22,19 @@ module Arrow
|
|
22
22
|
include ArrayComputable
|
23
23
|
include GenericFilterable
|
24
24
|
include GenericTakeable
|
25
|
+
include InputReferable
|
26
|
+
|
27
|
+
def to_arrow
|
28
|
+
self
|
29
|
+
end
|
30
|
+
|
31
|
+
def to_arrow_array
|
32
|
+
combine
|
33
|
+
end
|
34
|
+
|
35
|
+
def to_arrow_chunked_array
|
36
|
+
self
|
37
|
+
end
|
25
38
|
|
26
39
|
alias_method :size, :n_rows
|
27
40
|
unless method_defined?(:length)
|
@@ -30,7 +43,16 @@ module Arrow
|
|
30
43
|
|
31
44
|
alias_method :chunks_raw, :chunks
|
32
45
|
def chunks
|
33
|
-
@chunks ||= chunks_raw
|
46
|
+
@chunks ||= chunks_raw.tap do |_chunks|
|
47
|
+
_chunks.each do |chunk|
|
48
|
+
share_input(chunk)
|
49
|
+
end
|
50
|
+
end
|
51
|
+
end
|
52
|
+
|
53
|
+
alias_method :get_chunk_raw, :get_chunk
|
54
|
+
def get_chunk(i)
|
55
|
+
chunks[i]
|
34
56
|
end
|
35
57
|
|
36
58
|
def null?(i)
|
@@ -143,5 +143,14 @@ module Arrow
|
|
143
143
|
find_column(selector)
|
144
144
|
end
|
145
145
|
end
|
146
|
+
|
147
|
+
# Return column names in this object.
|
148
|
+
#
|
149
|
+
# @return [::Array<String>] column names.
|
150
|
+
#
|
151
|
+
# @since 11.0.0
|
152
|
+
def column_names
|
153
|
+
@column_names ||= columns.collect(&:name)
|
154
|
+
end
|
146
155
|
end
|
147
156
|
end
|
data/lib/arrow/column.rb
CHANGED
data/lib/arrow/data-type.rb
CHANGED
@@ -199,5 +199,14 @@ module Arrow
|
|
199
199
|
args.unshift(self) unless builder_class.buildable?(args)
|
200
200
|
builder_class.build(*args)
|
201
201
|
end
|
202
|
+
|
203
|
+
# @return [Arrow::Scalar} A corresponding {Arrow::Scalar} class
|
204
|
+
# for this data type.
|
205
|
+
#
|
206
|
+
# @since 12.0.0
|
207
|
+
def scalar_class
|
208
|
+
base_name = self.class.name.gsub(/DataType\z/, "")
|
209
|
+
::Arrow.const_get("#{base_name}Scalar")
|
210
|
+
end
|
202
211
|
end
|
203
212
|
end
|
@@ -0,0 +1,49 @@
|
|
1
|
+
# Licensed to the Apache Software Foundation (ASF) under one
|
2
|
+
# or more contributor license agreements. See the NOTICE file
|
3
|
+
# distributed with this work for additional information
|
4
|
+
# regarding copyright ownership. The ASF licenses this file
|
5
|
+
# to you under the Apache License, Version 2.0 (the
|
6
|
+
# "License"); you may not use this file except in compliance
|
7
|
+
# with the License. You may obtain a copy of the License at
|
8
|
+
#
|
9
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
10
|
+
#
|
11
|
+
# Unless required by applicable law or agreed to in writing,
|
12
|
+
# software distributed under the License is distributed on an
|
13
|
+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
14
|
+
# KIND, either express or implied. See the License for the
|
15
|
+
# specific language governing permissions and limitations
|
16
|
+
# under the License.
|
17
|
+
|
18
|
+
module Arrow
|
19
|
+
class DenseUnionArrayBuilder
|
20
|
+
alias_method :append_value_raw, :append_value
|
21
|
+
|
22
|
+
# @overload append_value
|
23
|
+
#
|
24
|
+
# Starts appending an union record. You need to append values of
|
25
|
+
# fields.
|
26
|
+
#
|
27
|
+
# @overload append_value(value)
|
28
|
+
#
|
29
|
+
# Appends an union record including values of fields.
|
30
|
+
#
|
31
|
+
# @param value [nil, Hash] The union record value.
|
32
|
+
#
|
33
|
+
# If this is `nil`, the union record is null.
|
34
|
+
#
|
35
|
+
# If this is `Hash`, it's values of fields.
|
36
|
+
#
|
37
|
+
# @since 12.0.0
|
38
|
+
def append_value(value)
|
39
|
+
if value.nil?
|
40
|
+
append_null
|
41
|
+
else
|
42
|
+
key = value.keys[0]
|
43
|
+
child_info = child_infos[key]
|
44
|
+
append_value_raw(child_info[:id])
|
45
|
+
child_info[:builder].append(value.values[0])
|
46
|
+
end
|
47
|
+
end
|
48
|
+
end
|
49
|
+
end
|
@@ -0,0 +1,26 @@
|
|
1
|
+
# Licensed to the Apache Software Foundation (ASF) under one
|
2
|
+
# or more contributor license agreements. See the NOTICE file
|
3
|
+
# distributed with this work for additional information
|
4
|
+
# regarding copyright ownership. The ASF licenses this file
|
5
|
+
# to you under the Apache License, Version 2.0 (the
|
6
|
+
# "License"); you may not use this file except in compliance
|
7
|
+
# with the License. You may obtain a copy of the License at
|
8
|
+
#
|
9
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
10
|
+
#
|
11
|
+
# Unless required by applicable law or agreed to in writing,
|
12
|
+
# software distributed under the License is distributed on an
|
13
|
+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
14
|
+
# KIND, either express or implied. See the License for the
|
15
|
+
# specific language governing permissions and limitations
|
16
|
+
# under the License.
|
17
|
+
|
18
|
+
module Arrow
|
19
|
+
class DenseUnionArray
|
20
|
+
def get_value(i)
|
21
|
+
child_id = get_child_id(i)
|
22
|
+
field = get_field(child_id)
|
23
|
+
field[get_value_offset(i)]
|
24
|
+
end
|
25
|
+
end
|
26
|
+
end
|
@@ -0,0 +1,32 @@
|
|
1
|
+
# Licensed to the Apache Software Foundation (ASF) under one
|
2
|
+
# or more contributor license agreements. See the NOTICE file
|
3
|
+
# distributed with this work for additional information
|
4
|
+
# regarding copyright ownership. The ASF licenses this file
|
5
|
+
# to you under the Apache License, Version 2.0 (the
|
6
|
+
# "License"); you may not use this file except in compliance
|
7
|
+
# with the License. You may obtain a copy of the License at
|
8
|
+
#
|
9
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
10
|
+
#
|
11
|
+
# Unless required by applicable law or agreed to in writing,
|
12
|
+
# software distributed under the License is distributed on an
|
13
|
+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
14
|
+
# KIND, either express or implied. See the License for the
|
15
|
+
# specific language governing permissions and limitations
|
16
|
+
# under the License.
|
17
|
+
|
18
|
+
module Arrow
|
19
|
+
class HalfFloatArrayBuilder
|
20
|
+
private
|
21
|
+
def convert_to_arrow_value(value)
|
22
|
+
case value
|
23
|
+
when Float
|
24
|
+
HalfFloat.new(value).to_uint16
|
25
|
+
when HalfFloat
|
26
|
+
value.to_uint16
|
27
|
+
else
|
28
|
+
value
|
29
|
+
end
|
30
|
+
end
|
31
|
+
end
|
32
|
+
end
|
@@ -0,0 +1,24 @@
|
|
1
|
+
# Licensed to the Apache Software Foundation (ASF) under one
|
2
|
+
# or more contributor license agreements. See the NOTICE file
|
3
|
+
# distributed with this work for additional information
|
4
|
+
# regarding copyright ownership. The ASF licenses this file
|
5
|
+
# to you under the Apache License, Version 2.0 (the
|
6
|
+
# "License"); you may not use this file except in compliance
|
7
|
+
# with the License. You may obtain a copy of the License at
|
8
|
+
#
|
9
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
10
|
+
#
|
11
|
+
# Unless required by applicable law or agreed to in writing,
|
12
|
+
# software distributed under the License is distributed on an
|
13
|
+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
14
|
+
# KIND, either express or implied. See the License for the
|
15
|
+
# specific language governing permissions and limitations
|
16
|
+
# under the License.
|
17
|
+
|
18
|
+
module Arrow
|
19
|
+
class HalfFloatArray
|
20
|
+
def get_value(i)
|
21
|
+
HalfFloat.new(get_raw_value(i)).to_f
|
22
|
+
end
|
23
|
+
end
|
24
|
+
end
|
@@ -0,0 +1,118 @@
|
|
1
|
+
# Licensed to the Apache Software Foundation (ASF) under one
|
2
|
+
# or more contributor license agreements. See the NOTICE file
|
3
|
+
# distributed with this work for additional information
|
4
|
+
# regarding copyright ownership. The ASF licenses this file
|
5
|
+
# to you under the Apache License, Version 2.0 (the
|
6
|
+
# "License"); you may not use this file except in compliance
|
7
|
+
# with the License. You may obtain a copy of the License at
|
8
|
+
#
|
9
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
10
|
+
#
|
11
|
+
# Unless required by applicable law or agreed to in writing,
|
12
|
+
# software distributed under the License is distributed on an
|
13
|
+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
14
|
+
# KIND, either express or implied. See the License for the
|
15
|
+
# specific language governing permissions and limitations
|
16
|
+
# under the License.
|
17
|
+
|
18
|
+
module Arrow
|
19
|
+
class HalfFloat
|
20
|
+
MAX = 65504
|
21
|
+
MIN = -65504
|
22
|
+
EXPONENT_N_BITS = 5
|
23
|
+
EXPONENT_MASK = (2 ** EXPONENT_N_BITS) - 1
|
24
|
+
EXPONENT_BIAS = 15
|
25
|
+
FRACTION_N_BITS = 10
|
26
|
+
FRACTION_MASK = (2 ** FRACTION_N_BITS) - 1
|
27
|
+
FRACTION_DENOMINATOR = 2.0 ** FRACTION_N_BITS
|
28
|
+
|
29
|
+
attr_reader :sign
|
30
|
+
attr_reader :exponent
|
31
|
+
attr_reader :fraction
|
32
|
+
def initialize(*args)
|
33
|
+
n_args = args.size
|
34
|
+
case n_args
|
35
|
+
when 1
|
36
|
+
if args[0].is_a?(Float)
|
37
|
+
@sign, @exponent, @fraction = deconstruct_float(args[0])
|
38
|
+
else
|
39
|
+
@sign, @exponent, @fraction = deconstruct_uint16(args[0])
|
40
|
+
end
|
41
|
+
when 3
|
42
|
+
@sign, @exponent, @fraction = *args
|
43
|
+
else
|
44
|
+
message = "wrong number of arguments (given #{n_args}, expected 1 or 3)"
|
45
|
+
raise ArgumentError, message
|
46
|
+
end
|
47
|
+
end
|
48
|
+
|
49
|
+
def to_f
|
50
|
+
if @exponent == EXPONENT_MASK
|
51
|
+
if @sign.zero?
|
52
|
+
Float::INFINITY
|
53
|
+
else
|
54
|
+
-Float::INFINITY
|
55
|
+
end
|
56
|
+
else
|
57
|
+
if @exponent.zero?
|
58
|
+
implicit_fraction = 0
|
59
|
+
else
|
60
|
+
implicit_fraction = 1
|
61
|
+
end
|
62
|
+
((-1) ** @sign) *
|
63
|
+
(2 ** (@exponent - EXPONENT_BIAS)) *
|
64
|
+
(implicit_fraction + @fraction / FRACTION_DENOMINATOR)
|
65
|
+
end
|
66
|
+
end
|
67
|
+
|
68
|
+
def to_uint16
|
69
|
+
(@sign << (EXPONENT_N_BITS + FRACTION_N_BITS)) ^
|
70
|
+
(@exponent << FRACTION_N_BITS) ^
|
71
|
+
@fraction
|
72
|
+
end
|
73
|
+
|
74
|
+
def pack
|
75
|
+
[to_uint16].pack("S")
|
76
|
+
end
|
77
|
+
|
78
|
+
private
|
79
|
+
def deconstruct_float(float)
|
80
|
+
if float > MAX
|
81
|
+
float = Float::INFINITY
|
82
|
+
elsif float < MIN
|
83
|
+
float = -Float::INFINITY
|
84
|
+
end
|
85
|
+
is_infinite = float.infinite?
|
86
|
+
if is_infinite
|
87
|
+
sign = (is_infinite == 1) ? 0 : 1
|
88
|
+
exponent = EXPONENT_MASK
|
89
|
+
fraction = 0
|
90
|
+
elsif float.zero?
|
91
|
+
sign = 0
|
92
|
+
exponent = 0
|
93
|
+
fraction = 0
|
94
|
+
else
|
95
|
+
sign = (float.positive? ? 0 : 1)
|
96
|
+
float_abs = float.abs
|
97
|
+
1.upto(EXPONENT_MASK) do |e|
|
98
|
+
next_exponent_value = 2 ** (e + 1 - EXPONENT_BIAS)
|
99
|
+
next if float_abs > next_exponent_value
|
100
|
+
exponent = e
|
101
|
+
exponent_value = 2 ** (e - EXPONENT_BIAS)
|
102
|
+
fraction =
|
103
|
+
((float_abs / exponent_value - 1) * FRACTION_DENOMINATOR).round
|
104
|
+
break
|
105
|
+
end
|
106
|
+
end
|
107
|
+
[sign, exponent, fraction]
|
108
|
+
end
|
109
|
+
|
110
|
+
def deconstruct_uint16(uint16)
|
111
|
+
# | sign (1 bit) | exponent (5 bit) | fraction (10 bit) |
|
112
|
+
sign = (uint16 >> (EXPONENT_N_BITS + FRACTION_N_BITS))
|
113
|
+
exponent = ((uint16 >> FRACTION_N_BITS) & EXPONENT_MASK)
|
114
|
+
fraction = (uint16 & FRACTION_MASK)
|
115
|
+
[sign, exponent, fraction]
|
116
|
+
end
|
117
|
+
end
|
118
|
+
end
|
@@ -0,0 +1,29 @@
|
|
1
|
+
# Licensed to the Apache Software Foundation (ASF) under one
|
2
|
+
# or more contributor license agreements. See the NOTICE file
|
3
|
+
# distributed with this work for additional information
|
4
|
+
# regarding copyright ownership. The ASF licenses this file
|
5
|
+
# to you under the Apache License, Version 2.0 (the
|
6
|
+
# "License"); you may not use this file except in compliance
|
7
|
+
# with the License. You may obtain a copy of the License at
|
8
|
+
#
|
9
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
10
|
+
#
|
11
|
+
# Unless required by applicable law or agreed to in writing,
|
12
|
+
# software distributed under the License is distributed on an
|
13
|
+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
14
|
+
# KIND, either express or implied. See the License for the
|
15
|
+
# specific language governing permissions and limitations
|
16
|
+
# under the License.
|
17
|
+
|
18
|
+
module Arrow
|
19
|
+
module InputReferable
|
20
|
+
def refer_input(input)
|
21
|
+
@input = input
|
22
|
+
end
|
23
|
+
|
24
|
+
def share_input(other)
|
25
|
+
return unless defined?(@input)
|
26
|
+
other.refer_input(@input)
|
27
|
+
end
|
28
|
+
end
|
29
|
+
end
|