red-arrow 4.0.0 → 6.0.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/README.md +23 -0
- data/ext/arrow/arrow.cpp +3 -0
- data/ext/arrow/converters.cpp +5 -0
- data/ext/arrow/converters.hpp +126 -0
- data/ext/arrow/extconf.rb +13 -0
- data/ext/arrow/memory-view.cpp +311 -0
- data/ext/arrow/memory-view.hpp +26 -0
- data/ext/arrow/raw-records.cpp +1 -0
- data/ext/arrow/values.cpp +1 -0
- data/lib/arrow/aggregate-node-options.rb +35 -0
- data/lib/arrow/aggregation.rb +46 -0
- data/lib/arrow/array-builder.rb +5 -0
- data/lib/arrow/array.rb +12 -0
- data/lib/arrow/binary-dictionary-array-builder.rb +27 -0
- data/lib/arrow/buffer.rb +10 -6
- data/lib/arrow/column-containable.rb +100 -1
- data/lib/arrow/constructor-arguments-gc-guardable.rb +25 -0
- data/lib/arrow/datum.rb +100 -0
- data/lib/arrow/equal-options.rb +38 -0
- data/lib/arrow/expression.rb +48 -0
- data/lib/arrow/file-system.rb +34 -0
- data/lib/arrow/group.rb +116 -124
- data/lib/arrow/loader.rb +44 -0
- data/lib/arrow/map-array-builder.rb +109 -0
- data/lib/arrow/map-array.rb +26 -0
- data/lib/arrow/map-data-type.rb +89 -0
- data/lib/arrow/path-extension.rb +1 -1
- data/lib/arrow/record-batch-reader.rb +41 -0
- data/lib/arrow/record-batch.rb +0 -2
- data/lib/arrow/scalar.rb +32 -0
- data/lib/arrow/slicer.rb +44 -143
- data/lib/arrow/source-node-options.rb +32 -0
- data/lib/arrow/string-dictionary-array-builder.rb +27 -0
- data/lib/arrow/symbol-values-appendable.rb +34 -0
- data/lib/arrow/table-concatenate-options.rb +36 -0
- data/lib/arrow/table-formatter.rb +141 -17
- data/lib/arrow/table-list-formatter.rb +5 -3
- data/lib/arrow/table-loader.rb +41 -3
- data/lib/arrow/table-saver.rb +29 -3
- data/lib/arrow/table-table-formatter.rb +7 -31
- data/lib/arrow/table.rb +34 -40
- data/lib/arrow/version.rb +1 -1
- data/red-arrow.gemspec +2 -1
- data/test/helper.rb +1 -0
- data/test/raw-records/test-dense-union-array.rb +14 -0
- data/test/raw-records/test-list-array.rb +19 -0
- data/test/raw-records/test-map-array.rb +441 -0
- data/test/raw-records/test-sparse-union-array.rb +14 -0
- data/test/raw-records/test-struct-array.rb +15 -0
- data/test/test-array-builder.rb +7 -0
- data/test/test-array.rb +34 -0
- data/test/test-binary-dictionary-array-builder.rb +103 -0
- data/test/test-boolean-scalar.rb +26 -0
- data/test/test-csv-loader.rb +8 -8
- data/test/test-expression.rb +40 -0
- data/test/test-float-scalar.rb +46 -0
- data/test/test-function.rb +176 -0
- data/test/test-group.rb +75 -51
- data/test/test-map-array-builder.rb +110 -0
- data/test/test-map-array.rb +33 -0
- data/test/test-map-data-type.rb +36 -0
- data/test/test-memory-view.rb +434 -0
- data/test/test-record-batch-reader.rb +46 -0
- data/test/test-record-batch.rb +42 -0
- data/test/test-slicer.rb +166 -167
- data/test/test-string-dictionary-array-builder.rb +103 -0
- data/test/test-table.rb +190 -53
- data/test/values/test-dense-union-array.rb +14 -0
- data/test/values/test-list-array.rb +17 -0
- data/test/values/test-map-array.rb +433 -0
- data/test/values/test-sparse-union-array.rb +14 -0
- data/test/values/test-struct-array.rb +15 -0
- metadata +132 -73
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 853f00543077ecc6ecbc5dcca0f535eda624ece16e2f74bb5ec966f435d60137
|
4
|
+
data.tar.gz: 8e6ea3a343fb7977729a004f61789c5f579b0d2a3c0823faedc4c10425eaec56
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: cb6de71e011620648821b4b13081386f5ad5a7e77ca7be26554c956a2f1d97ac488ad7f14bf289656fa15504600243fd4e3ba16068d355d6e1559a02a8e1c56f
|
7
|
+
data.tar.gz: 5e1ec7c8e1d306d5c72c282447a23c0b58674ee64f74c6d230834ba495265560a22953e489009bcecff9baa8c49816e5c8e431a1c1c5557cd2bac39a3696df02
|
data/README.md
CHANGED
@@ -50,3 +50,26 @@ table = Arrow::Table.load("/dev/shm/data.arrow")
|
|
50
50
|
# Process data in table
|
51
51
|
table.save("/dev/shm/data-processed.arrow")
|
52
52
|
```
|
53
|
+
|
54
|
+
## Development
|
55
|
+
|
56
|
+
Note that you need to install Apache Arrow C++/GLib at master before preparing Red Arrow. See also:
|
57
|
+
|
58
|
+
* For Apache Arrow C++: https://arrow.apache.org/docs/developers/cpp/building.html
|
59
|
+
* For Apache Arrow GLib: https://github.com/apache/arrow/blob/master/c_glib/README.md
|
60
|
+
|
61
|
+
```console
|
62
|
+
$ cd ruby/red-arrow
|
63
|
+
$ bundle install
|
64
|
+
$ bundle exec rake test
|
65
|
+
```
|
66
|
+
|
67
|
+
### For macOS with Homebrew
|
68
|
+
|
69
|
+
```console
|
70
|
+
$ cd ruby/red-arrow
|
71
|
+
$ bundle install
|
72
|
+
$ brew install apache-arrow --head
|
73
|
+
$ brew install apache-arrow-glib --head
|
74
|
+
$ bundle exec rake test
|
75
|
+
```
|
data/ext/arrow/arrow.cpp
CHANGED
@@ -18,6 +18,7 @@
|
|
18
18
|
*/
|
19
19
|
|
20
20
|
#include "red-arrow.hpp"
|
21
|
+
#include "memory-view.hpp"
|
21
22
|
|
22
23
|
#include <ruby.hpp>
|
23
24
|
|
@@ -78,4 +79,6 @@ extern "C" void Init_arrow() {
|
|
78
79
|
red_arrow::id_jd = rb_intern("jd");
|
79
80
|
red_arrow::id_new = rb_intern("new");
|
80
81
|
red_arrow::id_to_datetime = rb_intern("to_datetime");
|
82
|
+
|
83
|
+
red_arrow::memory_view::init(mArrow);
|
81
84
|
}
|
data/ext/arrow/converters.cpp
CHANGED
@@ -30,6 +30,11 @@ namespace red_arrow {
|
|
30
30
|
return struct_array_value_converter_->convert(array, i);
|
31
31
|
}
|
32
32
|
|
33
|
+
VALUE ArrayValueConverter::convert(const arrow::MapArray& array,
|
34
|
+
const int64_t i) {
|
35
|
+
return map_array_value_converter_->convert(array, i);
|
36
|
+
}
|
37
|
+
|
33
38
|
VALUE ArrayValueConverter::convert(const arrow::UnionArray& array,
|
34
39
|
const int64_t i) {
|
35
40
|
return union_array_value_converter_->convert(array, i);
|
data/ext/arrow/converters.hpp
CHANGED
@@ -29,6 +29,7 @@
|
|
29
29
|
namespace red_arrow {
|
30
30
|
class ListArrayValueConverter;
|
31
31
|
class StructArrayValueConverter;
|
32
|
+
class MapArrayValueConverter;
|
32
33
|
class UnionArrayValueConverter;
|
33
34
|
class DictionaryArrayValueConverter;
|
34
35
|
|
@@ -38,16 +39,19 @@ namespace red_arrow {
|
|
38
39
|
: decimal_buffer_(),
|
39
40
|
list_array_value_converter_(nullptr),
|
40
41
|
struct_array_value_converter_(nullptr),
|
42
|
+
map_array_value_converter_(nullptr),
|
41
43
|
union_array_value_converter_(nullptr),
|
42
44
|
dictionary_array_value_converter_(nullptr) {
|
43
45
|
}
|
44
46
|
|
45
47
|
inline void set_sub_value_converters(ListArrayValueConverter* list_array_value_converter,
|
46
48
|
StructArrayValueConverter* struct_array_value_converter,
|
49
|
+
MapArrayValueConverter* map_array_value_converter,
|
47
50
|
UnionArrayValueConverter* union_array_value_converter,
|
48
51
|
DictionaryArrayValueConverter* dictionary_array_value_converter) {
|
49
52
|
list_array_value_converter_ = list_array_value_converter;
|
50
53
|
struct_array_value_converter_ = struct_array_value_converter;
|
54
|
+
map_array_value_converter_ = map_array_value_converter;
|
51
55
|
union_array_value_converter_ = union_array_value_converter;
|
52
56
|
dictionary_array_value_converter_ = dictionary_array_value_converter;
|
53
57
|
}
|
@@ -204,6 +208,9 @@ namespace red_arrow {
|
|
204
208
|
VALUE convert(const arrow::StructArray& array,
|
205
209
|
const int64_t i);
|
206
210
|
|
211
|
+
VALUE convert(const arrow::MapArray& array,
|
212
|
+
const int64_t i);
|
213
|
+
|
207
214
|
VALUE convert(const arrow::UnionArray& array,
|
208
215
|
const int64_t i);
|
209
216
|
|
@@ -234,6 +241,7 @@ namespace red_arrow {
|
|
234
241
|
std::string decimal_buffer_;
|
235
242
|
ListArrayValueConverter* list_array_value_converter_;
|
236
243
|
StructArrayValueConverter* struct_array_value_converter_;
|
244
|
+
MapArrayValueConverter* map_array_value_converter_;
|
237
245
|
UnionArrayValueConverter* union_array_value_converter_;
|
238
246
|
DictionaryArrayValueConverter* dictionary_array_value_converter_;
|
239
247
|
};
|
@@ -294,6 +302,7 @@ namespace red_arrow {
|
|
294
302
|
// VISIT(Interval)
|
295
303
|
VISIT(List)
|
296
304
|
VISIT(Struct)
|
305
|
+
VISIT(Map)
|
297
306
|
VISIT(SparseUnion)
|
298
307
|
VISIT(DenseUnion)
|
299
308
|
VISIT(Dictionary)
|
@@ -399,6 +408,7 @@ namespace red_arrow {
|
|
399
408
|
// VISIT(Interval)
|
400
409
|
VISIT(List)
|
401
410
|
VISIT(Struct)
|
411
|
+
VISIT(Map)
|
402
412
|
VISIT(SparseUnion)
|
403
413
|
VISIT(DenseUnion)
|
404
414
|
VISIT(Dictionary)
|
@@ -431,6 +441,117 @@ namespace red_arrow {
|
|
431
441
|
VALUE result_;
|
432
442
|
};
|
433
443
|
|
444
|
+
class MapArrayValueConverter : public arrow::ArrayVisitor {
|
445
|
+
public:
|
446
|
+
explicit MapArrayValueConverter(ArrayValueConverter* converter)
|
447
|
+
: array_value_converter_(converter),
|
448
|
+
offset_(0),
|
449
|
+
length_(0),
|
450
|
+
values_(Qnil) {}
|
451
|
+
|
452
|
+
VALUE convert(const arrow::MapArray& array,
|
453
|
+
const int64_t index) {
|
454
|
+
auto key_array = array.keys().get();
|
455
|
+
auto item_array = array.items().get();
|
456
|
+
auto offset_keep = offset_;
|
457
|
+
auto length_keep = length_;
|
458
|
+
auto values_keep = values_;
|
459
|
+
offset_ = array.value_offset(index);
|
460
|
+
length_ = array.value_length(index);
|
461
|
+
auto keys = rb_ary_new_capa(length_);
|
462
|
+
values_ = keys;
|
463
|
+
check_status(key_array->Accept(this),
|
464
|
+
"[raw-records][map-array][keys]");
|
465
|
+
auto items = rb_ary_new_capa(length_);
|
466
|
+
values_ = items;
|
467
|
+
check_status(item_array->Accept(this),
|
468
|
+
"[raw-records][map-array][items]");
|
469
|
+
auto map = rb_hash_new();
|
470
|
+
auto n = RARRAY_LEN(keys);
|
471
|
+
auto raw_keys = RARRAY_CONST_PTR(keys);
|
472
|
+
auto raw_items = RARRAY_CONST_PTR(items);
|
473
|
+
for (long i = 0; i < n; ++i) {
|
474
|
+
rb_hash_aset(map, raw_keys[i], raw_items[i]);
|
475
|
+
}
|
476
|
+
offset_ = offset_keep;
|
477
|
+
length_ = length_keep;
|
478
|
+
values_ = values_keep;
|
479
|
+
return map;
|
480
|
+
}
|
481
|
+
|
482
|
+
#define VISIT(TYPE) \
|
483
|
+
arrow::Status Visit(const arrow::TYPE ## Array& array) override { \
|
484
|
+
return visit_value(array); \
|
485
|
+
}
|
486
|
+
|
487
|
+
VISIT(Null)
|
488
|
+
VISIT(Boolean)
|
489
|
+
VISIT(Int8)
|
490
|
+
VISIT(Int16)
|
491
|
+
VISIT(Int32)
|
492
|
+
VISIT(Int64)
|
493
|
+
VISIT(UInt8)
|
494
|
+
VISIT(UInt16)
|
495
|
+
VISIT(UInt32)
|
496
|
+
VISIT(UInt64)
|
497
|
+
// TODO
|
498
|
+
// VISIT(HalfFloat)
|
499
|
+
VISIT(Float)
|
500
|
+
VISIT(Double)
|
501
|
+
VISIT(Binary)
|
502
|
+
VISIT(String)
|
503
|
+
VISIT(FixedSizeBinary)
|
504
|
+
VISIT(Date32)
|
505
|
+
VISIT(Date64)
|
506
|
+
VISIT(Time32)
|
507
|
+
VISIT(Time64)
|
508
|
+
VISIT(Timestamp)
|
509
|
+
// TODO
|
510
|
+
// VISIT(Interval)
|
511
|
+
VISIT(List)
|
512
|
+
VISIT(Struct)
|
513
|
+
VISIT(Map)
|
514
|
+
VISIT(SparseUnion)
|
515
|
+
VISIT(DenseUnion)
|
516
|
+
VISIT(Dictionary)
|
517
|
+
VISIT(Decimal128)
|
518
|
+
VISIT(Decimal256)
|
519
|
+
// TODO
|
520
|
+
// VISIT(Extension)
|
521
|
+
|
522
|
+
#undef VISIT
|
523
|
+
|
524
|
+
private:
|
525
|
+
template <typename ArrayType>
|
526
|
+
inline VALUE convert_value(const ArrayType& array,
|
527
|
+
const int64_t i) {
|
528
|
+
return array_value_converter_->convert(array, i);
|
529
|
+
}
|
530
|
+
|
531
|
+
template <typename ArrayType>
|
532
|
+
arrow::Status visit_value(const ArrayType& array) {
|
533
|
+
if (array.null_count() > 0) {
|
534
|
+
for (int64_t i = 0; i < length_; ++i) {
|
535
|
+
auto value = Qnil;
|
536
|
+
if (!array.IsNull(i + offset_)) {
|
537
|
+
value = convert_value(array, i + offset_);
|
538
|
+
}
|
539
|
+
rb_ary_push(values_, value);
|
540
|
+
}
|
541
|
+
} else {
|
542
|
+
for (int64_t i = 0; i < length_; ++i) {
|
543
|
+
rb_ary_push(values_, convert_value(array, i + offset_));
|
544
|
+
}
|
545
|
+
}
|
546
|
+
return arrow::Status::OK();
|
547
|
+
}
|
548
|
+
|
549
|
+
ArrayValueConverter* array_value_converter_;
|
550
|
+
int32_t offset_;
|
551
|
+
int32_t length_;
|
552
|
+
VALUE values_;
|
553
|
+
};
|
554
|
+
|
434
555
|
class UnionArrayValueConverter : public arrow::ArrayVisitor {
|
435
556
|
public:
|
436
557
|
explicit UnionArrayValueConverter(ArrayValueConverter* converter)
|
@@ -492,6 +613,7 @@ namespace red_arrow {
|
|
492
613
|
// VISIT(Interval)
|
493
614
|
VISIT(List)
|
494
615
|
VISIT(Struct)
|
616
|
+
VISIT(Map)
|
495
617
|
VISIT(SparseUnion)
|
496
618
|
VISIT(DenseUnion)
|
497
619
|
VISIT(Dictionary)
|
@@ -617,6 +739,7 @@ namespace red_arrow {
|
|
617
739
|
// VISIT(Interval)
|
618
740
|
VISIT(List)
|
619
741
|
VISIT(Struct)
|
742
|
+
VISIT(Map)
|
620
743
|
VISIT(SparseUnion)
|
621
744
|
VISIT(DenseUnion)
|
622
745
|
VISIT(Dictionary)
|
@@ -645,11 +768,13 @@ namespace red_arrow {
|
|
645
768
|
: array_value_converter_(),
|
646
769
|
list_array_value_converter_(&array_value_converter_),
|
647
770
|
struct_array_value_converter_(&array_value_converter_),
|
771
|
+
map_array_value_converter_(&array_value_converter_),
|
648
772
|
union_array_value_converter_(&array_value_converter_),
|
649
773
|
dictionary_array_value_converter_(&array_value_converter_) {
|
650
774
|
array_value_converter_.
|
651
775
|
set_sub_value_converters(&list_array_value_converter_,
|
652
776
|
&struct_array_value_converter_,
|
777
|
+
&map_array_value_converter_,
|
653
778
|
&union_array_value_converter_,
|
654
779
|
&dictionary_array_value_converter_);
|
655
780
|
}
|
@@ -663,6 +788,7 @@ namespace red_arrow {
|
|
663
788
|
ArrayValueConverter array_value_converter_;
|
664
789
|
ListArrayValueConverter list_array_value_converter_;
|
665
790
|
StructArrayValueConverter struct_array_value_converter_;
|
791
|
+
MapArrayValueConverter map_array_value_converter_;
|
666
792
|
UnionArrayValueConverter union_array_value_converter_;
|
667
793
|
DictionaryArrayValueConverter dictionary_array_value_converter_;
|
668
794
|
};
|
data/ext/arrow/extconf.rb
CHANGED
@@ -25,6 +25,19 @@ if arrow_pkg_config_path
|
|
25
25
|
ENV["PKG_CONFIG_PATH"] = pkg_config_paths.join(File::PATH_SEPARATOR)
|
26
26
|
end
|
27
27
|
|
28
|
+
checking_for(checking_message("Homebrew")) do
|
29
|
+
platform = NativePackageInstaller::Platform.detect
|
30
|
+
if platform.is_a?(NativePackageInstaller::Platform::Homebrew)
|
31
|
+
openssl_prefix = `brew --prefix openssl@1.1`.chomp
|
32
|
+
unless openssl_prefix.empty?
|
33
|
+
PKGConfig.add_path("#{openssl_prefix}/lib/pkgconfig")
|
34
|
+
end
|
35
|
+
true
|
36
|
+
else
|
37
|
+
false
|
38
|
+
end
|
39
|
+
end
|
40
|
+
|
28
41
|
unless required_pkg_config_package([
|
29
42
|
"arrow",
|
30
43
|
Arrow::Version::MAJOR,
|
@@ -0,0 +1,311 @@
|
|
1
|
+
/*
|
2
|
+
* Licensed to the Apache Software Foundation (ASF) under one
|
3
|
+
* or more contributor license agreements. See the NOTICE file
|
4
|
+
* distributed with this work for additional information
|
5
|
+
* regarding copyright ownership. The ASF licenses this file
|
6
|
+
* to you under the Apache License, Version 2.0 (the
|
7
|
+
* "License"); you may not use this file except in compliance
|
8
|
+
* with the License. You may obtain a copy of the License at
|
9
|
+
*
|
10
|
+
* http://www.apache.org/licenses/LICENSE-2.0
|
11
|
+
*
|
12
|
+
* Unless required by applicable law or agreed to in writing,
|
13
|
+
* software distributed under the License is distributed on an
|
14
|
+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
15
|
+
* KIND, either express or implied. See the License for the
|
16
|
+
* specific language governing permissions and limitations
|
17
|
+
* under the License.
|
18
|
+
*/
|
19
|
+
|
20
|
+
#include "memory-view.hpp"
|
21
|
+
|
22
|
+
#include <arrow-glib/arrow-glib.hpp>
|
23
|
+
#include <rbgobject.h>
|
24
|
+
|
25
|
+
#include <ruby/version.h>
|
26
|
+
|
27
|
+
#if RUBY_API_VERSION_MAJOR >= 3
|
28
|
+
# define HAVE_MEMORY_VIEW
|
29
|
+
# define private memory_view_private
|
30
|
+
# include <ruby/memory_view.h>
|
31
|
+
# undef private
|
32
|
+
#endif
|
33
|
+
|
34
|
+
#include <sstream>
|
35
|
+
|
36
|
+
namespace red_arrow {
|
37
|
+
namespace memory_view {
|
38
|
+
#ifdef HAVE_MEMORY_VIEW
|
39
|
+
// This is workaround for the following rb_memory_view_t problems
|
40
|
+
// in C++:
|
41
|
+
//
|
42
|
+
// * Can't use "private" as member name
|
43
|
+
// * Can't assign a value to "rb_memory_view_t::private"
|
44
|
+
//
|
45
|
+
// This has compatible layout with rb_memory_view_t.
|
46
|
+
struct memory_view {
|
47
|
+
VALUE obj;
|
48
|
+
void *data;
|
49
|
+
ssize_t byte_size;
|
50
|
+
bool readonly;
|
51
|
+
const char *format;
|
52
|
+
ssize_t item_size;
|
53
|
+
struct {
|
54
|
+
const rb_memory_view_item_component_t *components;
|
55
|
+
size_t length;
|
56
|
+
} item_desc;
|
57
|
+
ssize_t ndim;
|
58
|
+
const ssize_t *shape;
|
59
|
+
const ssize_t *strides;
|
60
|
+
const ssize_t *sub_offsets;
|
61
|
+
void *private_data;
|
62
|
+
};
|
63
|
+
|
64
|
+
struct PrivateData {
|
65
|
+
std::string format;
|
66
|
+
};
|
67
|
+
|
68
|
+
class PrimitiveArrayGetter : public arrow::ArrayVisitor {
|
69
|
+
public:
|
70
|
+
explicit PrimitiveArrayGetter(memory_view *view)
|
71
|
+
: view_(view) {
|
72
|
+
}
|
73
|
+
|
74
|
+
arrow::Status Visit(const arrow::BooleanArray& array) override {
|
75
|
+
fill(static_cast<const arrow::Array&>(array));
|
76
|
+
// Memory view doesn't support bit stream. We use one byte
|
77
|
+
// for 8 elements. Users can't calculate the number of
|
78
|
+
// elements from memory view but it's limitation of memory view.
|
79
|
+
#ifdef ARROW_LITTLE_ENDIAN
|
80
|
+
view_->format = "b8";
|
81
|
+
#else
|
82
|
+
view_->format = "B8";
|
83
|
+
#endif
|
84
|
+
view_->item_size = 1;
|
85
|
+
view_->byte_size = (array.length() + 7) / 8;
|
86
|
+
return arrow::Status::OK();
|
87
|
+
}
|
88
|
+
|
89
|
+
arrow::Status Visit(const arrow::Int8Array& array) override {
|
90
|
+
fill(static_cast<const arrow::Array&>(array));
|
91
|
+
view_->format = "c";
|
92
|
+
return arrow::Status::OK();
|
93
|
+
}
|
94
|
+
|
95
|
+
arrow::Status Visit(const arrow::Int16Array& array) override {
|
96
|
+
fill(static_cast<const arrow::Array&>(array));
|
97
|
+
view_->format = "s";
|
98
|
+
return arrow::Status::OK();
|
99
|
+
}
|
100
|
+
|
101
|
+
arrow::Status Visit(const arrow::Int32Array& array) override {
|
102
|
+
fill(static_cast<const arrow::Array&>(array));
|
103
|
+
view_->format = "l";
|
104
|
+
return arrow::Status::OK();
|
105
|
+
}
|
106
|
+
|
107
|
+
arrow::Status Visit(const arrow::Int64Array& array) override {
|
108
|
+
fill(static_cast<const arrow::Array&>(array));
|
109
|
+
view_->format = "q";
|
110
|
+
return arrow::Status::OK();
|
111
|
+
}
|
112
|
+
|
113
|
+
arrow::Status Visit(const arrow::UInt8Array& array) override {
|
114
|
+
fill(static_cast<const arrow::Array&>(array));
|
115
|
+
view_->format = "C";
|
116
|
+
return arrow::Status::OK();
|
117
|
+
}
|
118
|
+
|
119
|
+
arrow::Status Visit(const arrow::UInt16Array& array) override {
|
120
|
+
fill(static_cast<const arrow::Array&>(array));
|
121
|
+
view_->format = "S";
|
122
|
+
return arrow::Status::OK();
|
123
|
+
}
|
124
|
+
|
125
|
+
arrow::Status Visit(const arrow::UInt32Array& array) override {
|
126
|
+
fill(static_cast<const arrow::Array&>(array));
|
127
|
+
view_->format = "L";
|
128
|
+
return arrow::Status::OK();
|
129
|
+
}
|
130
|
+
|
131
|
+
arrow::Status Visit(const arrow::UInt64Array& array) override {
|
132
|
+
fill(static_cast<const arrow::Array&>(array));
|
133
|
+
view_->format = "Q";
|
134
|
+
return arrow::Status::OK();
|
135
|
+
}
|
136
|
+
|
137
|
+
arrow::Status Visit(const arrow::FloatArray& array) override {
|
138
|
+
fill(static_cast<const arrow::Array&>(array));
|
139
|
+
view_->format = "f";
|
140
|
+
return arrow::Status::OK();
|
141
|
+
}
|
142
|
+
|
143
|
+
arrow::Status Visit(const arrow::DoubleArray& array) override {
|
144
|
+
fill(static_cast<const arrow::Array&>(array));
|
145
|
+
view_->format = "d";
|
146
|
+
return arrow::Status::OK();
|
147
|
+
}
|
148
|
+
|
149
|
+
arrow::Status Visit(const arrow::FixedSizeBinaryArray& array) override {
|
150
|
+
fill(static_cast<const arrow::Array&>(array));
|
151
|
+
auto priv = static_cast<PrivateData *>(view_->private_data);
|
152
|
+
const auto type =
|
153
|
+
std::static_pointer_cast<const arrow::FixedSizeBinaryType>(
|
154
|
+
array.type());
|
155
|
+
std::ostringstream output;
|
156
|
+
output << "C" << type->byte_width();
|
157
|
+
priv->format = output.str();
|
158
|
+
view_->format = priv->format.c_str();
|
159
|
+
return arrow::Status::OK();
|
160
|
+
}
|
161
|
+
|
162
|
+
arrow::Status Visit(const arrow::Date32Array& array) override {
|
163
|
+
fill(static_cast<const arrow::Array&>(array));
|
164
|
+
view_->format = "l";
|
165
|
+
return arrow::Status::OK();
|
166
|
+
}
|
167
|
+
|
168
|
+
arrow::Status Visit(const arrow::Date64Array& array) override {
|
169
|
+
fill(static_cast<const arrow::Array&>(array));
|
170
|
+
view_->format = "q";
|
171
|
+
return arrow::Status::OK();
|
172
|
+
}
|
173
|
+
|
174
|
+
arrow::Status Visit(const arrow::Time32Array& array) override {
|
175
|
+
fill(static_cast<const arrow::Array&>(array));
|
176
|
+
view_->format = "l";
|
177
|
+
return arrow::Status::OK();
|
178
|
+
}
|
179
|
+
|
180
|
+
arrow::Status Visit(const arrow::Time64Array& array) override {
|
181
|
+
fill(static_cast<const arrow::Array&>(array));
|
182
|
+
view_->format = "q";
|
183
|
+
return arrow::Status::OK();
|
184
|
+
}
|
185
|
+
|
186
|
+
arrow::Status Visit(const arrow::TimestampArray& array) override {
|
187
|
+
fill(static_cast<const arrow::Array&>(array));
|
188
|
+
view_->format = "q";
|
189
|
+
return arrow::Status::OK();
|
190
|
+
}
|
191
|
+
|
192
|
+
arrow::Status Visit(const arrow::Decimal128Array& array) override {
|
193
|
+
fill(static_cast<const arrow::Array&>(array));
|
194
|
+
view_->format = "q2";
|
195
|
+
return arrow::Status::OK();
|
196
|
+
}
|
197
|
+
|
198
|
+
arrow::Status Visit(const arrow::Decimal256Array& array) override {
|
199
|
+
fill(static_cast<const arrow::Array&>(array));
|
200
|
+
view_->format = "q4";
|
201
|
+
return arrow::Status::OK();
|
202
|
+
}
|
203
|
+
|
204
|
+
private:
|
205
|
+
void fill(const arrow::Array& array) {
|
206
|
+
const auto array_data = array.data();
|
207
|
+
const auto data = array_data->GetValuesSafe<uint8_t>(1);
|
208
|
+
view_->data = const_cast<void *>(reinterpret_cast<const void *>(data));
|
209
|
+
const auto type =
|
210
|
+
std::static_pointer_cast<const arrow::FixedWidthType>(array.type());
|
211
|
+
view_->item_size = type->bit_width() / 8;
|
212
|
+
view_->byte_size = view_->item_size * array.length();
|
213
|
+
}
|
214
|
+
|
215
|
+
memory_view *view_;
|
216
|
+
};
|
217
|
+
|
218
|
+
bool primitive_array_get(VALUE obj, rb_memory_view_t *view, int flags) {
|
219
|
+
if (flags != RUBY_MEMORY_VIEW_SIMPLE) {
|
220
|
+
return false;
|
221
|
+
}
|
222
|
+
auto view_ = reinterpret_cast<memory_view *>(view);
|
223
|
+
view_->obj = obj;
|
224
|
+
view_->private_data = new PrivateData();
|
225
|
+
auto array = GARROW_ARRAY(RVAL2GOBJ(obj));
|
226
|
+
auto arrow_array = garrow_array_get_raw(array);
|
227
|
+
PrimitiveArrayGetter getter(view_);
|
228
|
+
auto status = arrow_array->Accept(&getter);
|
229
|
+
if (!status.ok()) {
|
230
|
+
return false;
|
231
|
+
}
|
232
|
+
view_->readonly = true;
|
233
|
+
view_->ndim = 1;
|
234
|
+
view_->shape = NULL;
|
235
|
+
view_->strides = NULL;
|
236
|
+
view_->sub_offsets = NULL;
|
237
|
+
return true;
|
238
|
+
}
|
239
|
+
|
240
|
+
bool primitive_array_release(VALUE obj, rb_memory_view_t *view) {
|
241
|
+
auto view_ = reinterpret_cast<memory_view *>(view);
|
242
|
+
delete static_cast<PrivateData *>(view_->private_data);
|
243
|
+
return true;
|
244
|
+
}
|
245
|
+
|
246
|
+
bool primitive_array_available_p(VALUE obj) {
|
247
|
+
return true;
|
248
|
+
}
|
249
|
+
|
250
|
+
rb_memory_view_entry_t primitive_array_entry = {
|
251
|
+
primitive_array_get,
|
252
|
+
primitive_array_release,
|
253
|
+
primitive_array_available_p,
|
254
|
+
};
|
255
|
+
|
256
|
+
bool buffer_get(VALUE obj, rb_memory_view_t *view, int flags) {
|
257
|
+
if (flags != RUBY_MEMORY_VIEW_SIMPLE) {
|
258
|
+
return false;
|
259
|
+
}
|
260
|
+
auto view_ = reinterpret_cast<memory_view *>(view);
|
261
|
+
view_->obj = obj;
|
262
|
+
auto buffer = GARROW_BUFFER(RVAL2GOBJ(obj));
|
263
|
+
auto arrow_buffer = garrow_buffer_get_raw(buffer);
|
264
|
+
view_->data =
|
265
|
+
const_cast<void *>(reinterpret_cast<const void *>(arrow_buffer->data()));
|
266
|
+
// Memory view doesn't support bit stream. We use one byte
|
267
|
+
// for 8 elements. Users can't calculate the number of
|
268
|
+
// elements from memory view but it's limitation of memory view.
|
269
|
+
#ifdef ARROW_LITTLE_ENDIAN
|
270
|
+
view_->format = "b8";
|
271
|
+
#else
|
272
|
+
view_->format = "B8";
|
273
|
+
#endif
|
274
|
+
view_->item_size = 1;
|
275
|
+
view_->byte_size = arrow_buffer->size();
|
276
|
+
view_->readonly = true;
|
277
|
+
view_->ndim = 1;
|
278
|
+
view_->shape = NULL;
|
279
|
+
view_->strides = NULL;
|
280
|
+
view_->sub_offsets = NULL;
|
281
|
+
return true;
|
282
|
+
}
|
283
|
+
|
284
|
+
bool buffer_release(VALUE obj, rb_memory_view_t *view) {
|
285
|
+
return true;
|
286
|
+
}
|
287
|
+
|
288
|
+
bool buffer_available_p(VALUE obj) {
|
289
|
+
return true;
|
290
|
+
}
|
291
|
+
|
292
|
+
rb_memory_view_entry_t buffer_entry = {
|
293
|
+
buffer_get,
|
294
|
+
buffer_release,
|
295
|
+
buffer_available_p,
|
296
|
+
};
|
297
|
+
#endif
|
298
|
+
|
299
|
+
void init(VALUE mArrow) {
|
300
|
+
#ifdef HAVE_MEMORY_VIEW
|
301
|
+
auto cPrimitiveArray =
|
302
|
+
rb_const_get_at(mArrow, rb_intern("PrimitiveArray"));
|
303
|
+
rb_memory_view_register(cPrimitiveArray,
|
304
|
+
&(red_arrow::memory_view::primitive_array_entry));
|
305
|
+
|
306
|
+
auto cBuffer = rb_const_get_at(mArrow, rb_intern("Buffer"));
|
307
|
+
rb_memory_view_register(cBuffer, &(red_arrow::memory_view::buffer_entry));
|
308
|
+
#endif
|
309
|
+
}
|
310
|
+
}
|
311
|
+
}
|
@@ -0,0 +1,26 @@
|
|
1
|
+
/*
|
2
|
+
* Licensed to the Apache Software Foundation (ASF) under one
|
3
|
+
* or more contributor license agreements. See the NOTICE file
|
4
|
+
* distributed with this work for additional information
|
5
|
+
* regarding copyright ownership. The ASF licenses this file
|
6
|
+
* to you under the Apache License, Version 2.0 (the
|
7
|
+
* "License"); you may not use this file except in compliance
|
8
|
+
* with the License. You may obtain a copy of the License at
|
9
|
+
*
|
10
|
+
* http://www.apache.org/licenses/LICENSE-2.0
|
11
|
+
*
|
12
|
+
* Unless required by applicable law or agreed to in writing,
|
13
|
+
* software distributed under the License is distributed on an
|
14
|
+
* "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
15
|
+
* KIND, either express or implied. See the License for the
|
16
|
+
* specific language governing permissions and limitations
|
17
|
+
* under the License.
|
18
|
+
*/
|
19
|
+
|
20
|
+
#include <ruby.hpp>
|
21
|
+
|
22
|
+
namespace red_arrow {
|
23
|
+
namespace memory_view {
|
24
|
+
void init(VALUE mArrow);
|
25
|
+
}
|
26
|
+
}
|
data/ext/arrow/raw-records.cpp
CHANGED
data/ext/arrow/values.cpp
CHANGED
@@ -0,0 +1,35 @@
|
|
1
|
+
# Licensed to the Apache Software Foundation (ASF) under one
|
2
|
+
# or more contributor license agreements. See the NOTICE file
|
3
|
+
# distributed with this work for additional information
|
4
|
+
# regarding copyright ownership. The ASF licenses this file
|
5
|
+
# to you under the Apache License, Version 2.0 (the
|
6
|
+
# "License"); you may not use this file except in compliance
|
7
|
+
# with the License. You may obtain a copy of the License at
|
8
|
+
#
|
9
|
+
# http://www.apache.org/licenses/LICENSE-2.0
|
10
|
+
#
|
11
|
+
# Unless required by applicable law or agreed to in writing,
|
12
|
+
# software distributed under the License is distributed on an
|
13
|
+
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
|
14
|
+
# KIND, either express or implied. See the License for the
|
15
|
+
# specific language governing permissions and limitations
|
16
|
+
# under the License.
|
17
|
+
|
18
|
+
module Arrow
|
19
|
+
class AggregateNodeOptions
|
20
|
+
class << self
|
21
|
+
# @api private
|
22
|
+
def try_convert(value)
|
23
|
+
case value
|
24
|
+
when Hash
|
25
|
+
aggregations = value[:aggregations]
|
26
|
+
return nil if aggregations.nil?
|
27
|
+
keys = value[:keys]
|
28
|
+
new(aggregations, keys)
|
29
|
+
else
|
30
|
+
nil
|
31
|
+
end
|
32
|
+
end
|
33
|
+
end
|
34
|
+
end
|
35
|
+
end
|