red-arrow 3.0.0 → 6.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +23 -0
- data/ext/arrow/arrow.cpp +3 -0
- data/ext/arrow/converters.cpp +5 -0
- data/ext/arrow/converters.hpp +126 -0
- data/ext/arrow/extconf.rb +13 -0
- data/ext/arrow/memory-view.cpp +311 -0
- data/ext/arrow/memory-view.hpp +26 -0
- data/ext/arrow/raw-records.cpp +1 -0
- data/ext/arrow/values.cpp +1 -0
- data/lib/arrow/aggregate-node-options.rb +35 -0
- data/lib/arrow/aggregation.rb +46 -0
- data/lib/arrow/array-builder.rb +5 -0
- data/lib/arrow/array.rb +130 -0
- data/lib/arrow/binary-dictionary-array-builder.rb +27 -0
- data/lib/arrow/buffer.rb +10 -6
- data/lib/arrow/column-containable.rb +100 -1
- data/lib/arrow/constructor-arguments-gc-guardable.rb +25 -0
- data/lib/arrow/data-type.rb +14 -5
- data/lib/arrow/datum.rb +100 -0
- data/lib/arrow/dense-union-data-type.rb +2 -2
- data/lib/arrow/dictionary-data-type.rb +2 -2
- data/lib/arrow/equal-options.rb +38 -0
- data/lib/arrow/expression.rb +48 -0
- data/lib/arrow/file-system.rb +34 -0
- data/lib/arrow/group.rb +116 -124
- data/lib/arrow/loader.rb +46 -0
- data/lib/arrow/map-array-builder.rb +109 -0
- data/lib/arrow/map-array.rb +26 -0
- data/lib/arrow/map-data-type.rb +89 -0
- data/lib/arrow/path-extension.rb +1 -1
- data/lib/arrow/record-batch-reader.rb +41 -0
- data/lib/arrow/record-batch.rb +0 -2
- data/lib/arrow/scalar.rb +32 -0
- data/lib/arrow/slicer.rb +44 -143
- data/lib/arrow/sort-key.rb +193 -0
- data/lib/arrow/sort-options.rb +109 -0
- data/lib/arrow/source-node-options.rb +32 -0
- data/lib/arrow/sparse-union-data-type.rb +2 -2
- data/lib/arrow/string-dictionary-array-builder.rb +27 -0
- data/lib/arrow/symbol-values-appendable.rb +34 -0
- data/lib/arrow/table-concatenate-options.rb +36 -0
- data/lib/arrow/table-formatter.rb +141 -17
- data/lib/arrow/table-list-formatter.rb +5 -3
- data/lib/arrow/table-loader.rb +41 -3
- data/lib/arrow/table-saver.rb +29 -3
- data/lib/arrow/table-table-formatter.rb +7 -31
- data/lib/arrow/table.rb +34 -40
- data/lib/arrow/time32-data-type.rb +2 -2
- data/lib/arrow/time64-data-type.rb +2 -2
- data/lib/arrow/timestamp-data-type.rb +2 -2
- data/lib/arrow/version.rb +1 -1
- data/red-arrow.gemspec +2 -1
- data/test/helper.rb +1 -0
- data/test/raw-records/test-dense-union-array.rb +14 -0
- data/test/raw-records/test-list-array.rb +19 -0
- data/test/raw-records/test-map-array.rb +441 -0
- data/test/raw-records/test-sparse-union-array.rb +14 -0
- data/test/raw-records/test-struct-array.rb +15 -0
- data/test/test-array-builder.rb +7 -0
- data/test/test-array.rb +154 -0
- data/test/test-binary-dictionary-array-builder.rb +103 -0
- data/test/test-boolean-scalar.rb +26 -0
- data/test/test-csv-loader.rb +8 -8
- data/test/test-decimal128-data-type.rb +2 -2
- data/test/test-expression.rb +40 -0
- data/test/test-float-scalar.rb +46 -0
- data/test/test-function.rb +176 -0
- data/test/test-group.rb +75 -51
- data/test/test-map-array-builder.rb +110 -0
- data/test/test-map-array.rb +33 -0
- data/test/test-map-data-type.rb +36 -0
- data/test/test-memory-view.rb +434 -0
- data/test/test-orc.rb +19 -23
- data/test/test-record-batch-reader.rb +46 -0
- data/test/test-record-batch.rb +42 -0
- data/test/test-slicer.rb +166 -167
- data/test/test-sort-indices.rb +40 -0
- data/test/test-sort-key.rb +81 -0
- data/test/test-sort-options.rb +58 -0
- data/test/test-string-dictionary-array-builder.rb +103 -0
- data/test/test-table.rb +190 -53
- data/test/values/test-dense-union-array.rb +14 -0
- data/test/values/test-list-array.rb +17 -0
- data/test/values/test-map-array.rb +433 -0
- data/test/values/test-sparse-union-array.rb +14 -0
- data/test/values/test-struct-array.rb +15 -0
- metadata +73 -6
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: c553e573f87398da7d492e8339684bb86b73d1fff02b56cff3277bb87d96a4cd
|
4
|
+
data.tar.gz: 9b97be2878369da0bed9ac8621330926b8a95144d833de6798858a210f88c92a
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 0bc3526645457a551e25621ba3efaa52a3ad7398e8216c34440651beab58886ec8b52804b7afe227f953b727871980a2ac9a9b1153776499b48cd790ae476304
|
7
|
+
data.tar.gz: 9f3032d5a54a9b0b911edd273916ce68f140c50753d13cb88b950166ea771138f432d738a3a694f44bfff985fcf8329457e97712dca10a7f8a8430d3d8cb82f6
|
data/README.md
CHANGED
@@ -50,3 +50,26 @@ table = Arrow::Table.load("/dev/shm/data.arrow")
|
|
50
50
|
# Process data in table
|
51
51
|
table.save("/dev/shm/data-processed.arrow")
|
52
52
|
```
|
53
|
+
|
54
|
+
## Development
|
55
|
+
|
56
|
+
Note that you need to install Apache Arrow C++/GLib at master before preparing Red Arrow. See also:
|
57
|
+
|
58
|
+
* For Apache Arrow C++: https://arrow.apache.org/docs/developers/cpp/building.html
|
59
|
+
* For Apache Arrow GLib: https://github.com/apache/arrow/blob/master/c_glib/README.md
|
60
|
+
|
61
|
+
```console
|
62
|
+
$ cd ruby/red-arrow
|
63
|
+
$ bundle install
|
64
|
+
$ bundle exec rake test
|
65
|
+
```
|
66
|
+
|
67
|
+
### For macOS with Homebrew
|
68
|
+
|
69
|
+
```console
|
70
|
+
$ cd ruby/red-arrow
|
71
|
+
$ bundle install
|
72
|
+
$ brew install apache-arrow --head
|
73
|
+
$ brew install apache-arrow-glib --head
|
74
|
+
$ bundle exec rake test
|
75
|
+
```
|
data/ext/arrow/arrow.cpp
CHANGED
@@ -18,6 +18,7 @@
|
|
18
18
|
*/
|
19
19
|
|
20
20
|
#include "red-arrow.hpp"
|
21
|
+
#include "memory-view.hpp"
|
21
22
|
|
22
23
|
#include <ruby.hpp>
|
23
24
|
|
@@ -78,4 +79,6 @@ extern "C" void Init_arrow() {
|
|
78
79
|
red_arrow::id_jd = rb_intern("jd");
|
79
80
|
red_arrow::id_new = rb_intern("new");
|
80
81
|
red_arrow::id_to_datetime = rb_intern("to_datetime");
|
82
|
+
|
83
|
+
red_arrow::memory_view::init(mArrow);
|
81
84
|
}
|
data/ext/arrow/converters.cpp
CHANGED
@@ -30,6 +30,11 @@ namespace red_arrow {
|
|
30
30
|
return struct_array_value_converter_->convert(array, i);
|
31
31
|
}
|
32
32
|
|
33
|
+
VALUE ArrayValueConverter::convert(const arrow::MapArray& array,
|
34
|
+
const int64_t i) {
|
35
|
+
return map_array_value_converter_->convert(array, i);
|
36
|
+
}
|
37
|
+
|
33
38
|
VALUE ArrayValueConverter::convert(const arrow::UnionArray& array,
|
34
39
|
const int64_t i) {
|
35
40
|
return union_array_value_converter_->convert(array, i);
|
data/ext/arrow/converters.hpp
CHANGED
@@ -29,6 +29,7 @@
|
|
29
29
|
namespace red_arrow {
|
30
30
|
class ListArrayValueConverter;
|
31
31
|
class StructArrayValueConverter;
|
32
|
+
class MapArrayValueConverter;
|
32
33
|
class UnionArrayValueConverter;
|
33
34
|
class DictionaryArrayValueConverter;
|
34
35
|
|
@@ -38,16 +39,19 @@ namespace red_arrow {
|
|
38
39
|
: decimal_buffer_(),
|
39
40
|
list_array_value_converter_(nullptr),
|
40
41
|
struct_array_value_converter_(nullptr),
|
42
|
+
map_array_value_converter_(nullptr),
|
41
43
|
union_array_value_converter_(nullptr),
|
42
44
|
dictionary_array_value_converter_(nullptr) {
|
43
45
|
}
|
44
46
|
|
45
47
|
inline void set_sub_value_converters(ListArrayValueConverter* list_array_value_converter,
|
46
48
|
StructArrayValueConverter* struct_array_value_converter,
|
49
|
+
MapArrayValueConverter* map_array_value_converter,
|
47
50
|
UnionArrayValueConverter* union_array_value_converter,
|
48
51
|
DictionaryArrayValueConverter* dictionary_array_value_converter) {
|
49
52
|
list_array_value_converter_ = list_array_value_converter;
|
50
53
|
struct_array_value_converter_ = struct_array_value_converter;
|
54
|
+
map_array_value_converter_ = map_array_value_converter;
|
51
55
|
union_array_value_converter_ = union_array_value_converter;
|
52
56
|
dictionary_array_value_converter_ = dictionary_array_value_converter;
|
53
57
|
}
|
@@ -204,6 +208,9 @@ namespace red_arrow {
|
|
204
208
|
VALUE convert(const arrow::StructArray& array,
|
205
209
|
const int64_t i);
|
206
210
|
|
211
|
+
VALUE convert(const arrow::MapArray& array,
|
212
|
+
const int64_t i);
|
213
|
+
|
207
214
|
VALUE convert(const arrow::UnionArray& array,
|
208
215
|
const int64_t i);
|
209
216
|
|
@@ -234,6 +241,7 @@ namespace red_arrow {
|
|
234
241
|
std::string decimal_buffer_;
|
235
242
|
ListArrayValueConverter* list_array_value_converter_;
|
236
243
|
StructArrayValueConverter* struct_array_value_converter_;
|
244
|
+
MapArrayValueConverter* map_array_value_converter_;
|
237
245
|
UnionArrayValueConverter* union_array_value_converter_;
|
238
246
|
DictionaryArrayValueConverter* dictionary_array_value_converter_;
|
239
247
|
};
|
@@ -294,6 +302,7 @@ namespace red_arrow {
|
|
294
302
|
// VISIT(Interval)
|
295
303
|
VISIT(List)
|
296
304
|
VISIT(Struct)
|
305
|
+
VISIT(Map)
|
297
306
|
VISIT(SparseUnion)
|
298
307
|
VISIT(DenseUnion)
|
299
308
|
VISIT(Dictionary)
|
@@ -399,6 +408,7 @@ namespace red_arrow {
|
|
399
408
|
// VISIT(Interval)
|
400
409
|
VISIT(List)
|
401
410
|
VISIT(Struct)
|
411
|
+
VISIT(Map)
|
402
412
|
VISIT(SparseUnion)
|
403
413
|
VISIT(DenseUnion)
|
404
414
|
VISIT(Dictionary)
|
@@ -431,6 +441,117 @@ namespace red_arrow {
|
|
431
441
|
VALUE result_;
|
432
442
|
};
|
433
443
|
|
444
|
+
class MapArrayValueConverter : public arrow::ArrayVisitor {
|
445
|
+
public:
|
446
|
+
explicit MapArrayValueConverter(ArrayValueConverter* converter)
|
447
|
+
: array_value_converter_(converter),
|
448
|
+
offset_(0),
|
449
|
+
length_(0),
|
450
|
+
values_(Qnil) {}
|
451
|
+
|
452
|
+
VALUE convert(const arrow::MapArray& array,
|
453
|
+
const int64_t index) {
|
454
|
+
auto key_array = array.keys().get();
|
455
|
+
auto item_array = array.items().get();
|
456
|
+
auto offset_keep = offset_;
|
457
|
+
auto length_keep = length_;
|
458
|
+
auto values_keep = values_;
|
459
|
+
offset_ = array.value_offset(index);
|
460
|
+
length_ = array.value_length(index);
|
461
|
+
auto keys = rb_ary_new_capa(length_);
|
462
|
+
values_ = keys;
|
463
|
+
check_status(key_array->Accept(this),
|
464
|
+
"[raw-records][map-array][keys]");
|
465
|
+
auto items = rb_ary_new_capa(length_);
|
466
|
+
values_ = items;
|
467
|
+
check_status(item_array->Accept(this),
|
468
|
+
"[raw-records][map-array][items]");
|
469
|
+
auto map = rb_hash_new();
|
470
|
+
auto n = RARRAY_LEN(keys);
|
471
|
+
auto raw_keys = RARRAY_CONST_PTR(keys);
|
472
|
+
auto raw_items = RARRAY_CONST_PTR(items);
|
473
|
+
for (long i = 0; i < n; ++i) {
|
474
|
+
rb_hash_aset(map, raw_keys[i], raw_items[i]);
|
475
|
+
}
|
476
|
+
offset_ = offset_keep;
|
477
|
+
length_ = length_keep;
|
478
|
+
values_ = values_keep;
|
479
|
+
return map;
|
480
|
+
}
|
481
|
+
|
482
|
+
#define VISIT(TYPE) \
|
483
|
+
arrow::Status Visit(const arrow::TYPE ## Array& array) override { \
|
484
|
+
return visit_value(array); \
|
485
|
+
}
|
486
|
+
|
487
|
+
VISIT(Null)
|
488
|
+
VISIT(Boolean)
|
489
|
+
VISIT(Int8)
|
490
|
+
VISIT(Int16)
|
491
|
+
VISIT(Int32)
|
492
|
+
VISIT(Int64)
|
493
|
+
VISIT(UInt8)
|
494
|
+
VISIT(UInt16)
|
495
|
+
VISIT(UInt32)
|
496
|
+
VISIT(UInt64)
|
497
|
+
// TODO
|
498
|
+
// VISIT(HalfFloat)
|
499
|
+
VISIT(Float)
|
500
|
+
VISIT(Double)
|
501
|
+
VISIT(Binary)
|
502
|
+
VISIT(String)
|
503
|
+
VISIT(FixedSizeBinary)
|
504
|
+
VISIT(Date32)
|
505
|
+
VISIT(Date64)
|
506
|
+
VISIT(Time32)
|
507
|
+
VISIT(Time64)
|
508
|
+
VISIT(Timestamp)
|
509
|
+
// TODO
|
510
|
+
// VISIT(Interval)
|
511
|
+
VISIT(List)
|
512
|
+
VISIT(Struct)
|
513
|
+
VISIT(Map)
|
514
|
+
VISIT(SparseUnion)
|
515
|
+
VISIT(DenseUnion)
|
516
|
+
VISIT(Dictionary)
|
517
|
+
VISIT(Decimal128)
|
518
|
+
VISIT(Decimal256)
|
519
|
+
// TODO
|
520
|
+
// VISIT(Extension)
|
521
|
+
|
522
|
+
#undef VISIT
|
523
|
+
|
524
|
+
private:
|
525
|
+
template <typename ArrayType>
|
526
|
+
inline VALUE convert_value(const ArrayType& array,
|
527
|
+
const int64_t i) {
|
528
|
+
return array_value_converter_->convert(array, i);
|
529
|
+
}
|
530
|
+
|
531
|
+
template <typename ArrayType>
|
532
|
+
arrow::Status visit_value(const ArrayType& array) {
|
533
|
+
if (array.null_count() > 0) {
|
534
|
+
for (int64_t i = 0; i < length_; ++i) {
|
535
|
+
auto value = Qnil;
|
536
|
+
if (!array.IsNull(i + offset_)) {
|
537
|
+
value = convert_value(array, i + offset_);
|
538
|
+
}
|
539
|
+
rb_ary_push(values_, value);
|
540
|
+
}
|
541
|
+
} else {
|
542
|
+
for (int64_t i = 0; i < length_; ++i) {
|
543
|
+
rb_ary_push(values_, convert_value(array, i + offset_));
|
544
|
+
}
|
545
|
+
}
|
546
|
+
return arrow::Status::OK();
|
547
|
+
}
|
548
|
+
|
549
|
+
ArrayValueConverter* array_value_converter_;
|
550
|
+
int32_t offset_;
|
551
|
+
int32_t length_;
|
552
|
+
VALUE values_;
|
553
|
+
};
|
554
|
+
|
434
555
|
class UnionArrayValueConverter : public arrow::ArrayVisitor {
|
435
556
|
public:
|
436
557
|
explicit UnionArrayValueConverter(ArrayValueConverter* converter)
|
@@ -492,6 +613,7 @@ namespace red_arrow {
|
|
492
613
|
// VISIT(Interval)
|
493
614
|
VISIT(List)
|
494
615
|
VISIT(Struct)
|
616
|
+
VISIT(Map)
|
495
617
|
VISIT(SparseUnion)
|
496
618
|
VISIT(DenseUnion)
|
497
619
|
VISIT(Dictionary)
|
@@ -617,6 +739,7 @@ namespace red_arrow {
|
|
617
739
|
// VISIT(Interval)
|
618
740
|
VISIT(List)
|
619
741
|
VISIT(Struct)
|
742
|
+
VISIT(Map)
|
620
743
|
VISIT(SparseUnion)
|
621
744
|
VISIT(DenseUnion)
|
622
745
|
VISIT(Dictionary)
|
@@ -645,11 +768,13 @@ namespace red_arrow {
|
|
645
768
|
: array_value_converter_(),
|
646
769
|
list_array_value_converter_(&array_value_converter_),
|
647
770
|
struct_array_value_converter_(&array_value_converter_),
|
771
|
+
map_array_value_converter_(&array_value_converter_),
|
648
772
|
union_array_value_converter_(&array_value_converter_),
|
649
773
|
dictionary_array_value_converter_(&array_value_converter_) {
|
650
774
|
array_value_converter_.
|
651
775
|
set_sub_value_converters(&list_array_value_converter_,
|
652
776
|
&struct_array_value_converter_,
|
777
|
+
&map_array_value_converter_,
|
653
778
|
&union_array_value_converter_,
|
654
779
|
&dictionary_array_value_converter_);
|
655
780
|
}
|
@@ -663,6 +788,7 @@ namespace red_arrow {
|
|
663
788
|
ArrayValueConverter array_value_converter_;
|
664
789
|
ListArrayValueConverter list_array_value_converter_;
|
665
790
|
StructArrayValueConverter struct_array_value_converter_;
|
791
|
+
MapArrayValueConverter map_array_value_converter_;
|
666
792
|
UnionArrayValueConverter union_array_value_converter_;
|
667
793
|
DictionaryArrayValueConverter dictionary_array_value_converter_;
|
668
794
|
};
|
data/ext/arrow/extconf.rb
CHANGED
@@ -25,6 +25,19 @@ if arrow_pkg_config_path
|
|
25
25
|
ENV["PKG_CONFIG_PATH"] = pkg_config_paths.join(File::PATH_SEPARATOR)
|
26
26
|
end
|
27
27
|
|
28
|
+
checking_for(checking_message("Homebrew")) do
|
29
|
+
platform = NativePackageInstaller::Platform.detect
|
30
|
+
if platform.is_a?(NativePackageInstaller::Platform::Homebrew)
|
31
|
+
openssl_prefix = `brew --prefix openssl@1.1`.chomp
|
32
|
+
unless openssl_prefix.empty?
|
33
|
+
PKGConfig.add_path("#{openssl_prefix}/lib/pkgconfig")
|
34
|
+
end
|
35
|
+
true
|
36
|
+
else
|
37
|
+
false
|
38
|
+
end
|
39
|
+
end
|
40
|
+
|
28
41
|
unless required_pkg_config_package([
|
29
42
|
"arrow",
|
30
43
|
Arrow::Version::MAJOR,
|
@@ -0,0 +1,311 @@
|
|
1
|
+
/*
|
2
|
+
* Licensed to the Apache Software Foundation (ASF) under one
|
3
|
+
* or more contributor license agreements. See the NOTICE file
|
4
|
+
* distributed with this work for additional information
|
5
|
+
* regarding copyright ownership. The ASF licenses this file
|
6
|
+
* to you under the Apache License, Version 2.0 (the
|
7
|
+
* "License"); you may not use this file except in compliance
|
8
|
+
* with the License. You may obtain a copy of the License at
|
9
|
+
*
|
10
|
+
* http://www.apache.org/licenses/LICENSE-2.0
|
11
|
+
*
|
12
|
+
* Unless required by applicable law or agreed to in writing,
|
13
|
+
* software distributed under the License is distributed on an
|
14
|
+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
15
|
+
* KIND, either express or implied. See the License for the
|
16
|
+
* specific language governing permissions and limitations
|
17
|
+
* under the License.
|
18
|
+
*/
|
19
|
+
|
20
|
+
#include "memory-view.hpp"
|
21
|
+
|
22
|
+
#include <arrow-glib/arrow-glib.hpp>
|
23
|
+
#include <rbgobject.h>
|
24
|
+
|
25
|
+
#include <ruby/version.h>
|
26
|
+
|
27
|
+
#if RUBY_API_VERSION_MAJOR >= 3
|
28
|
+
# define HAVE_MEMORY_VIEW
|
29
|
+
# define private memory_view_private
|
30
|
+
# include <ruby/memory_view.h>
|
31
|
+
# undef private
|
32
|
+
#endif
|
33
|
+
|
34
|
+
#include <sstream>
|
35
|
+
|
36
|
+
namespace red_arrow {
|
37
|
+
namespace memory_view {
|
38
|
+
#ifdef HAVE_MEMORY_VIEW
|
39
|
+
// This is workaround for the following rb_memory_view_t problems
|
40
|
+
// in C++:
|
41
|
+
//
|
42
|
+
// * Can't use "private" as member name
|
43
|
+
// * Can't assign a value to "rb_memory_view_t::private"
|
44
|
+
//
|
45
|
+
// This has compatible layout with rb_memory_view_t.
|
46
|
+
struct memory_view {
|
47
|
+
VALUE obj;
|
48
|
+
void *data;
|
49
|
+
ssize_t byte_size;
|
50
|
+
bool readonly;
|
51
|
+
const char *format;
|
52
|
+
ssize_t item_size;
|
53
|
+
struct {
|
54
|
+
const rb_memory_view_item_component_t *components;
|
55
|
+
size_t length;
|
56
|
+
} item_desc;
|
57
|
+
ssize_t ndim;
|
58
|
+
const ssize_t *shape;
|
59
|
+
const ssize_t *strides;
|
60
|
+
const ssize_t *sub_offsets;
|
61
|
+
void *private_data;
|
62
|
+
};
|
63
|
+
|
64
|
+
struct PrivateData {
|
65
|
+
std::string format;
|
66
|
+
};
|
67
|
+
|
68
|
+
class PrimitiveArrayGetter : public arrow::ArrayVisitor {
|
69
|
+
public:
|
70
|
+
explicit PrimitiveArrayGetter(memory_view *view)
|
71
|
+
: view_(view) {
|
72
|
+
}
|
73
|
+
|
74
|
+
arrow::Status Visit(const arrow::BooleanArray& array) override {
|
75
|
+
fill(static_cast<const arrow::Array&>(array));
|
76
|
+
// Memory view doesn't support bit stream. We use one byte
|
77
|
+
// for 8 elements. Users can't calculate the number of
|
78
|
+
// elements from memory view but it's limitation of memory view.
|
79
|
+
#ifdef ARROW_LITTLE_ENDIAN
|
80
|
+
view_->format = "b8";
|
81
|
+
#else
|
82
|
+
view_->format = "B8";
|
83
|
+
#endif
|
84
|
+
view_->item_size = 1;
|
85
|
+
view_->byte_size = (array.length() + 7) / 8;
|
86
|
+
return arrow::Status::OK();
|
87
|
+
}
|
88
|
+
|
89
|
+
arrow::Status Visit(const arrow::Int8Array& array) override {
|
90
|
+
fill(static_cast<const arrow::Array&>(array));
|
91
|
+
view_->format = "c";
|
92
|
+
return arrow::Status::OK();
|
93
|
+
}
|
94
|
+
|
95
|
+
arrow::Status Visit(const arrow::Int16Array& array) override {
|
96
|
+
fill(static_cast<const arrow::Array&>(array));
|
97
|
+
view_->format = "s";
|
98
|
+
return arrow::Status::OK();
|
99
|
+
}
|
100
|
+
|
101
|
+
arrow::Status Visit(const arrow::Int32Array& array) override {
|
102
|
+
fill(static_cast<const arrow::Array&>(array));
|
103
|
+
view_->format = "l";
|
104
|
+
return arrow::Status::OK();
|
105
|
+
}
|
106
|
+
|
107
|
+
arrow::Status Visit(const arrow::Int64Array& array) override {
|
108
|
+
fill(static_cast<const arrow::Array&>(array));
|
109
|
+
view_->format = "q";
|
110
|
+
return arrow::Status::OK();
|
111
|
+
}
|
112
|
+
|
113
|
+
arrow::Status Visit(const arrow::UInt8Array& array) override {
|
114
|
+
fill(static_cast<const arrow::Array&>(array));
|
115
|
+
view_->format = "C";
|
116
|
+
return arrow::Status::OK();
|
117
|
+
}
|
118
|
+
|
119
|
+
arrow::Status Visit(const arrow::UInt16Array& array) override {
|
120
|
+
fill(static_cast<const arrow::Array&>(array));
|
121
|
+
view_->format = "S";
|
122
|
+
return arrow::Status::OK();
|
123
|
+
}
|
124
|
+
|
125
|
+
arrow::Status Visit(const arrow::UInt32Array& array) override {
|
126
|
+
fill(static_cast<const arrow::Array&>(array));
|
127
|
+
view_->format = "L";
|
128
|
+
return arrow::Status::OK();
|
129
|
+
}
|
130
|
+
|
131
|
+
arrow::Status Visit(const arrow::UInt64Array& array) override {
|
132
|
+
fill(static_cast<const arrow::Array&>(array));
|
133
|
+
view_->format = "Q";
|
134
|
+
return arrow::Status::OK();
|
135
|
+
}
|
136
|
+
|
137
|
+
arrow::Status Visit(const arrow::FloatArray& array) override {
|
138
|
+
fill(static_cast<const arrow::Array&>(array));
|
139
|
+
view_->format = "f";
|
140
|
+
return arrow::Status::OK();
|
141
|
+
}
|
142
|
+
|
143
|
+
arrow::Status Visit(const arrow::DoubleArray& array) override {
|
144
|
+
fill(static_cast<const arrow::Array&>(array));
|
145
|
+
view_->format = "d";
|
146
|
+
return arrow::Status::OK();
|
147
|
+
}
|
148
|
+
|
149
|
+
arrow::Status Visit(const arrow::FixedSizeBinaryArray& array) override {
|
150
|
+
fill(static_cast<const arrow::Array&>(array));
|
151
|
+
auto priv = static_cast<PrivateData *>(view_->private_data);
|
152
|
+
const auto type =
|
153
|
+
std::static_pointer_cast<const arrow::FixedSizeBinaryType>(
|
154
|
+
array.type());
|
155
|
+
std::ostringstream output;
|
156
|
+
output << "C" << type->byte_width();
|
157
|
+
priv->format = output.str();
|
158
|
+
view_->format = priv->format.c_str();
|
159
|
+
return arrow::Status::OK();
|
160
|
+
}
|
161
|
+
|
162
|
+
arrow::Status Visit(const arrow::Date32Array& array) override {
|
163
|
+
fill(static_cast<const arrow::Array&>(array));
|
164
|
+
view_->format = "l";
|
165
|
+
return arrow::Status::OK();
|
166
|
+
}
|
167
|
+
|
168
|
+
arrow::Status Visit(const arrow::Date64Array& array) override {
|
169
|
+
fill(static_cast<const arrow::Array&>(array));
|
170
|
+
view_->format = "q";
|
171
|
+
return arrow::Status::OK();
|
172
|
+
}
|
173
|
+
|
174
|
+
arrow::Status Visit(const arrow::Time32Array& array) override {
|
175
|
+
fill(static_cast<const arrow::Array&>(array));
|
176
|
+
view_->format = "l";
|
177
|
+
return arrow::Status::OK();
|
178
|
+
}
|
179
|
+
|
180
|
+
arrow::Status Visit(const arrow::Time64Array& array) override {
|
181
|
+
fill(static_cast<const arrow::Array&>(array));
|
182
|
+
view_->format = "q";
|
183
|
+
return arrow::Status::OK();
|
184
|
+
}
|
185
|
+
|
186
|
+
arrow::Status Visit(const arrow::TimestampArray& array) override {
|
187
|
+
fill(static_cast<const arrow::Array&>(array));
|
188
|
+
view_->format = "q";
|
189
|
+
return arrow::Status::OK();
|
190
|
+
}
|
191
|
+
|
192
|
+
arrow::Status Visit(const arrow::Decimal128Array& array) override {
|
193
|
+
fill(static_cast<const arrow::Array&>(array));
|
194
|
+
view_->format = "q2";
|
195
|
+
return arrow::Status::OK();
|
196
|
+
}
|
197
|
+
|
198
|
+
arrow::Status Visit(const arrow::Decimal256Array& array) override {
|
199
|
+
fill(static_cast<const arrow::Array&>(array));
|
200
|
+
view_->format = "q4";
|
201
|
+
return arrow::Status::OK();
|
202
|
+
}
|
203
|
+
|
204
|
+
private:
|
205
|
+
void fill(const arrow::Array& array) {
|
206
|
+
const auto array_data = array.data();
|
207
|
+
const auto data = array_data->GetValuesSafe<uint8_t>(1);
|
208
|
+
view_->data = const_cast<void *>(reinterpret_cast<const void *>(data));
|
209
|
+
const auto type =
|
210
|
+
std::static_pointer_cast<const arrow::FixedWidthType>(array.type());
|
211
|
+
view_->item_size = type->bit_width() / 8;
|
212
|
+
view_->byte_size = view_->item_size * array.length();
|
213
|
+
}
|
214
|
+
|
215
|
+
memory_view *view_;
|
216
|
+
};
|
217
|
+
|
218
|
+
bool primitive_array_get(VALUE obj, rb_memory_view_t *view, int flags) {
|
219
|
+
if (flags != RUBY_MEMORY_VIEW_SIMPLE) {
|
220
|
+
return false;
|
221
|
+
}
|
222
|
+
auto view_ = reinterpret_cast<memory_view *>(view);
|
223
|
+
view_->obj = obj;
|
224
|
+
view_->private_data = new PrivateData();
|
225
|
+
auto array = GARROW_ARRAY(RVAL2GOBJ(obj));
|
226
|
+
auto arrow_array = garrow_array_get_raw(array);
|
227
|
+
PrimitiveArrayGetter getter(view_);
|
228
|
+
auto status = arrow_array->Accept(&getter);
|
229
|
+
if (!status.ok()) {
|
230
|
+
return false;
|
231
|
+
}
|
232
|
+
view_->readonly = true;
|
233
|
+
view_->ndim = 1;
|
234
|
+
view_->shape = NULL;
|
235
|
+
view_->strides = NULL;
|
236
|
+
view_->sub_offsets = NULL;
|
237
|
+
return true;
|
238
|
+
}
|
239
|
+
|
240
|
+
bool primitive_array_release(VALUE obj, rb_memory_view_t *view) {
|
241
|
+
auto view_ = reinterpret_cast<memory_view *>(view);
|
242
|
+
delete static_cast<PrivateData *>(view_->private_data);
|
243
|
+
return true;
|
244
|
+
}
|
245
|
+
|
246
|
+
bool primitive_array_available_p(VALUE obj) {
|
247
|
+
return true;
|
248
|
+
}
|
249
|
+
|
250
|
+
rb_memory_view_entry_t primitive_array_entry = {
|
251
|
+
primitive_array_get,
|
252
|
+
primitive_array_release,
|
253
|
+
primitive_array_available_p,
|
254
|
+
};
|
255
|
+
|
256
|
+
bool buffer_get(VALUE obj, rb_memory_view_t *view, int flags) {
|
257
|
+
if (flags != RUBY_MEMORY_VIEW_SIMPLE) {
|
258
|
+
return false;
|
259
|
+
}
|
260
|
+
auto view_ = reinterpret_cast<memory_view *>(view);
|
261
|
+
view_->obj = obj;
|
262
|
+
auto buffer = GARROW_BUFFER(RVAL2GOBJ(obj));
|
263
|
+
auto arrow_buffer = garrow_buffer_get_raw(buffer);
|
264
|
+
view_->data =
|
265
|
+
const_cast<void *>(reinterpret_cast<const void *>(arrow_buffer->data()));
|
266
|
+
// Memory view doesn't support bit stream. We use one byte
|
267
|
+
// for 8 elements. Users can't calculate the number of
|
268
|
+
// elements from memory view but it's limitation of memory view.
|
269
|
+
#ifdef ARROW_LITTLE_ENDIAN
|
270
|
+
view_->format = "b8";
|
271
|
+
#else
|
272
|
+
view_->format = "B8";
|
273
|
+
#endif
|
274
|
+
view_->item_size = 1;
|
275
|
+
view_->byte_size = arrow_buffer->size();
|
276
|
+
view_->readonly = true;
|
277
|
+
view_->ndim = 1;
|
278
|
+
view_->shape = NULL;
|
279
|
+
view_->strides = NULL;
|
280
|
+
view_->sub_offsets = NULL;
|
281
|
+
return true;
|
282
|
+
}
|
283
|
+
|
284
|
+
bool buffer_release(VALUE obj, rb_memory_view_t *view) {
|
285
|
+
return true;
|
286
|
+
}
|
287
|
+
|
288
|
+
bool buffer_available_p(VALUE obj) {
|
289
|
+
return true;
|
290
|
+
}
|
291
|
+
|
292
|
+
rb_memory_view_entry_t buffer_entry = {
|
293
|
+
buffer_get,
|
294
|
+
buffer_release,
|
295
|
+
buffer_available_p,
|
296
|
+
};
|
297
|
+
#endif
|
298
|
+
|
299
|
+
void init(VALUE mArrow) {
|
300
|
+
#ifdef HAVE_MEMORY_VIEW
|
301
|
+
auto cPrimitiveArray =
|
302
|
+
rb_const_get_at(mArrow, rb_intern("PrimitiveArray"));
|
303
|
+
rb_memory_view_register(cPrimitiveArray,
|
304
|
+
&(red_arrow::memory_view::primitive_array_entry));
|
305
|
+
|
306
|
+
auto cBuffer = rb_const_get_at(mArrow, rb_intern("Buffer"));
|
307
|
+
rb_memory_view_register(cBuffer, &(red_arrow::memory_view::buffer_entry));
|
308
|
+
#endif
|
309
|
+
}
|
310
|
+
}
|
311
|
+
}
|
@@ -0,0 +1,26 @@
|
|
1
|
+
/*
|
2
|
+
* Licensed to the Apache Software Foundation (ASF) under one
|
3
|
+
* or more contributor license agreements. See the NOTICE file
|
4
|
+
* distributed with this work for additional information
|
5
|
+
* regarding copyright ownership. The ASF licenses this file
|
6
|
+
* to you under the Apache License, Version 2.0 (the
|
7
|
+
* "License"); you may not use this file except in compliance
|
8
|
+
* with the License. You may obtain a copy of the License at
|
9
|
+
*
|
10
|
+
* http://www.apache.org/licenses/LICENSE-2.0
|
11
|
+
*
|
12
|
+
* Unless required by applicable law or agreed to in writing,
|
13
|
+
* software distributed under the License is distributed on an
|
14
|
+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
15
|
+
* KIND, either express or implied. See the License for the
|
16
|
+
* specific language governing permissions and limitations
|
17
|
+
* under the License.
|
18
|
+
*/
|
19
|
+
|
20
|
+
#include <ruby.hpp>
|
21
|
+
|
22
|
+
namespace red_arrow {
|
23
|
+
namespace memory_view {
|
24
|
+
void init(VALUE mArrow);
|
25
|
+
}
|
26
|
+
}
|
data/ext/arrow/raw-records.cpp
CHANGED
data/ext/arrow/values.cpp
CHANGED
@@ -0,0 +1,35 @@
|
|
1
|
+
# Licensed to the Apache Software Foundation (ASF) under one
|
2
|
+
# or more contributor license agreements. See the NOTICE file
|
3
|
+
# distributed with this work for additional information
|
4
|
+
# regarding copyright ownership. The ASF licenses this file
|
5
|
+
# to you under the Apache License, Version 2.0 (the
|
6
|
+
# "License"); you may not use this file except in compliance
|
7
|
+
# with the License. You may obtain a copy of the License at
|
8
|
+
#
|
9
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
10
|
+
#
|
11
|
+
# Unless required by applicable law or agreed to in writing,
|
12
|
+
# software distributed under the License is distributed on an
|
13
|
+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
14
|
+
# KIND, either express or implied. See the License for the
|
15
|
+
# specific language governing permissions and limitations
|
16
|
+
# under the License.
|
17
|
+
|
18
|
+
module Arrow
|
19
|
+
class AggregateNodeOptions
|
20
|
+
class << self
|
21
|
+
# @api private
|
22
|
+
def try_convert(value)
|
23
|
+
case value
|
24
|
+
when Hash
|
25
|
+
aggregations = value[:aggregations]
|
26
|
+
return nil if aggregations.nil?
|
27
|
+
keys = value[:keys]
|
28
|
+
new(aggregations, keys)
|
29
|
+
else
|
30
|
+
nil
|
31
|
+
end
|
32
|
+
end
|
33
|
+
end
|
34
|
+
end
|
35
|
+
end
|