red-arrow 3.0.0 → 6.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (88) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +23 -0
  3. data/ext/arrow/arrow.cpp +3 -0
  4. data/ext/arrow/converters.cpp +5 -0
  5. data/ext/arrow/converters.hpp +126 -0
  6. data/ext/arrow/extconf.rb +13 -0
  7. data/ext/arrow/memory-view.cpp +311 -0
  8. data/ext/arrow/memory-view.hpp +26 -0
  9. data/ext/arrow/raw-records.cpp +1 -0
  10. data/ext/arrow/values.cpp +1 -0
  11. data/lib/arrow/aggregate-node-options.rb +35 -0
  12. data/lib/arrow/aggregation.rb +46 -0
  13. data/lib/arrow/array-builder.rb +5 -0
  14. data/lib/arrow/array.rb +130 -0
  15. data/lib/arrow/binary-dictionary-array-builder.rb +27 -0
  16. data/lib/arrow/buffer.rb +10 -6
  17. data/lib/arrow/column-containable.rb +100 -1
  18. data/lib/arrow/constructor-arguments-gc-guardable.rb +25 -0
  19. data/lib/arrow/data-type.rb +14 -5
  20. data/lib/arrow/datum.rb +100 -0
  21. data/lib/arrow/dense-union-data-type.rb +2 -2
  22. data/lib/arrow/dictionary-data-type.rb +2 -2
  23. data/lib/arrow/equal-options.rb +38 -0
  24. data/lib/arrow/expression.rb +48 -0
  25. data/lib/arrow/file-system.rb +34 -0
  26. data/lib/arrow/group.rb +116 -124
  27. data/lib/arrow/loader.rb +46 -0
  28. data/lib/arrow/map-array-builder.rb +109 -0
  29. data/lib/arrow/map-array.rb +26 -0
  30. data/lib/arrow/map-data-type.rb +89 -0
  31. data/lib/arrow/path-extension.rb +1 -1
  32. data/lib/arrow/record-batch-reader.rb +41 -0
  33. data/lib/arrow/record-batch.rb +0 -2
  34. data/lib/arrow/scalar.rb +32 -0
  35. data/lib/arrow/slicer.rb +44 -143
  36. data/lib/arrow/sort-key.rb +193 -0
  37. data/lib/arrow/sort-options.rb +109 -0
  38. data/lib/arrow/source-node-options.rb +32 -0
  39. data/lib/arrow/sparse-union-data-type.rb +2 -2
  40. data/lib/arrow/string-dictionary-array-builder.rb +27 -0
  41. data/lib/arrow/symbol-values-appendable.rb +34 -0
  42. data/lib/arrow/table-concatenate-options.rb +36 -0
  43. data/lib/arrow/table-formatter.rb +141 -17
  44. data/lib/arrow/table-list-formatter.rb +5 -3
  45. data/lib/arrow/table-loader.rb +41 -3
  46. data/lib/arrow/table-saver.rb +29 -3
  47. data/lib/arrow/table-table-formatter.rb +7 -31
  48. data/lib/arrow/table.rb +34 -40
  49. data/lib/arrow/time32-data-type.rb +2 -2
  50. data/lib/arrow/time64-data-type.rb +2 -2
  51. data/lib/arrow/timestamp-data-type.rb +2 -2
  52. data/lib/arrow/version.rb +1 -1
  53. data/red-arrow.gemspec +2 -1
  54. data/test/helper.rb +1 -0
  55. data/test/raw-records/test-dense-union-array.rb +14 -0
  56. data/test/raw-records/test-list-array.rb +19 -0
  57. data/test/raw-records/test-map-array.rb +441 -0
  58. data/test/raw-records/test-sparse-union-array.rb +14 -0
  59. data/test/raw-records/test-struct-array.rb +15 -0
  60. data/test/test-array-builder.rb +7 -0
  61. data/test/test-array.rb +154 -0
  62. data/test/test-binary-dictionary-array-builder.rb +103 -0
  63. data/test/test-boolean-scalar.rb +26 -0
  64. data/test/test-csv-loader.rb +8 -8
  65. data/test/test-decimal128-data-type.rb +2 -2
  66. data/test/test-expression.rb +40 -0
  67. data/test/test-float-scalar.rb +46 -0
  68. data/test/test-function.rb +176 -0
  69. data/test/test-group.rb +75 -51
  70. data/test/test-map-array-builder.rb +110 -0
  71. data/test/test-map-array.rb +33 -0
  72. data/test/test-map-data-type.rb +36 -0
  73. data/test/test-memory-view.rb +434 -0
  74. data/test/test-orc.rb +19 -23
  75. data/test/test-record-batch-reader.rb +46 -0
  76. data/test/test-record-batch.rb +42 -0
  77. data/test/test-slicer.rb +166 -167
  78. data/test/test-sort-indices.rb +40 -0
  79. data/test/test-sort-key.rb +81 -0
  80. data/test/test-sort-options.rb +58 -0
  81. data/test/test-string-dictionary-array-builder.rb +103 -0
  82. data/test/test-table.rb +190 -53
  83. data/test/values/test-dense-union-array.rb +14 -0
  84. data/test/values/test-list-array.rb +17 -0
  85. data/test/values/test-map-array.rb +433 -0
  86. data/test/values/test-sparse-union-array.rb +14 -0
  87. data/test/values/test-struct-array.rb +15 -0
  88. metadata +73 -6
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: a70c1505c294a8f74c992ec0f2ab5d651647e45dd178db8523f5b5c01e64a541
4
- data.tar.gz: dad979599033104a25d5be3d05c57e552c803f9c8002d66b757866c425937ce9
3
+ metadata.gz: c553e573f87398da7d492e8339684bb86b73d1fff02b56cff3277bb87d96a4cd
4
+ data.tar.gz: 9b97be2878369da0bed9ac8621330926b8a95144d833de6798858a210f88c92a
5
5
  SHA512:
6
- metadata.gz: b2363ba6468985a3d237cbb9cec8b2ccb5031aae94153b8f263916e811524e8a5326f408867a843b7077f624050fabcde93520b26e6f8ceb1dd094aaa0068ed6
7
- data.tar.gz: 797b89062dfd212d92ca957b8e51c6dc8acf16ab51e5a60ecb0cbf31dd979c288f2afb71274b2235a24d31da344cfce93b40830d6dc6edd000ce0aa607ce8cde
6
+ metadata.gz: 0bc3526645457a551e25621ba3efaa52a3ad7398e8216c34440651beab58886ec8b52804b7afe227f953b727871980a2ac9a9b1153776499b48cd790ae476304
7
+ data.tar.gz: 9f3032d5a54a9b0b911edd273916ce68f140c50753d13cb88b950166ea771138f432d738a3a694f44bfff985fcf8329457e97712dca10a7f8a8430d3d8cb82f6
data/README.md CHANGED
@@ -50,3 +50,26 @@ table = Arrow::Table.load("/dev/shm/data.arrow")
50
50
  # Process data in table
51
51
  table.save("/dev/shm/data-processed.arrow")
52
52
  ```
53
+
54
+ ## Development
55
+
56
+ Note that you need to install Apache Arrow C++/GLib at master before preparing Red Arrow. See also:
57
+
58
+ * For Apache Arrow C++: https://arrow.apache.org/docs/developers/cpp/building.html
59
+ * For Apache Arrow GLib: https://github.com/apache/arrow/blob/master/c_glib/README.md
60
+
61
+ ```console
62
+ $ cd ruby/red-arrow
63
+ $ bundle install
64
+ $ bundle exec rake test
65
+ ```
66
+
67
+ ### For macOS with Homebrew
68
+
69
+ ```console
70
+ $ cd ruby/red-arrow
71
+ $ bundle install
72
+ $ brew install apache-arrow --head
73
+ $ brew install apache-arrow-glib --head
74
+ $ bundle exec rake test
75
+ ```
data/ext/arrow/arrow.cpp CHANGED
@@ -18,6 +18,7 @@
18
18
  */
19
19
 
20
20
  #include "red-arrow.hpp"
21
+ #include "memory-view.hpp"
21
22
 
22
23
  #include <ruby.hpp>
23
24
 
@@ -78,4 +79,6 @@ extern "C" void Init_arrow() {
78
79
  red_arrow::id_jd = rb_intern("jd");
79
80
  red_arrow::id_new = rb_intern("new");
80
81
  red_arrow::id_to_datetime = rb_intern("to_datetime");
82
+
83
+ red_arrow::memory_view::init(mArrow);
81
84
  }
@@ -30,6 +30,11 @@ namespace red_arrow {
30
30
  return struct_array_value_converter_->convert(array, i);
31
31
  }
32
32
 
33
+ VALUE ArrayValueConverter::convert(const arrow::MapArray& array,
34
+ const int64_t i) {
35
+ return map_array_value_converter_->convert(array, i);
36
+ }
37
+
33
38
  VALUE ArrayValueConverter::convert(const arrow::UnionArray& array,
34
39
  const int64_t i) {
35
40
  return union_array_value_converter_->convert(array, i);
@@ -29,6 +29,7 @@
29
29
  namespace red_arrow {
30
30
  class ListArrayValueConverter;
31
31
  class StructArrayValueConverter;
32
+ class MapArrayValueConverter;
32
33
  class UnionArrayValueConverter;
33
34
  class DictionaryArrayValueConverter;
34
35
 
@@ -38,16 +39,19 @@ namespace red_arrow {
38
39
  : decimal_buffer_(),
39
40
  list_array_value_converter_(nullptr),
40
41
  struct_array_value_converter_(nullptr),
42
+ map_array_value_converter_(nullptr),
41
43
  union_array_value_converter_(nullptr),
42
44
  dictionary_array_value_converter_(nullptr) {
43
45
  }
44
46
 
45
47
  inline void set_sub_value_converters(ListArrayValueConverter* list_array_value_converter,
46
48
  StructArrayValueConverter* struct_array_value_converter,
49
+ MapArrayValueConverter* map_array_value_converter,
47
50
  UnionArrayValueConverter* union_array_value_converter,
48
51
  DictionaryArrayValueConverter* dictionary_array_value_converter) {
49
52
  list_array_value_converter_ = list_array_value_converter;
50
53
  struct_array_value_converter_ = struct_array_value_converter;
54
+ map_array_value_converter_ = map_array_value_converter;
51
55
  union_array_value_converter_ = union_array_value_converter;
52
56
  dictionary_array_value_converter_ = dictionary_array_value_converter;
53
57
  }
@@ -204,6 +208,9 @@ namespace red_arrow {
204
208
  VALUE convert(const arrow::StructArray& array,
205
209
  const int64_t i);
206
210
 
211
+ VALUE convert(const arrow::MapArray& array,
212
+ const int64_t i);
213
+
207
214
  VALUE convert(const arrow::UnionArray& array,
208
215
  const int64_t i);
209
216
 
@@ -234,6 +241,7 @@ namespace red_arrow {
234
241
  std::string decimal_buffer_;
235
242
  ListArrayValueConverter* list_array_value_converter_;
236
243
  StructArrayValueConverter* struct_array_value_converter_;
244
+ MapArrayValueConverter* map_array_value_converter_;
237
245
  UnionArrayValueConverter* union_array_value_converter_;
238
246
  DictionaryArrayValueConverter* dictionary_array_value_converter_;
239
247
  };
@@ -294,6 +302,7 @@ namespace red_arrow {
294
302
  // VISIT(Interval)
295
303
  VISIT(List)
296
304
  VISIT(Struct)
305
+ VISIT(Map)
297
306
  VISIT(SparseUnion)
298
307
  VISIT(DenseUnion)
299
308
  VISIT(Dictionary)
@@ -399,6 +408,7 @@ namespace red_arrow {
399
408
  // VISIT(Interval)
400
409
  VISIT(List)
401
410
  VISIT(Struct)
411
+ VISIT(Map)
402
412
  VISIT(SparseUnion)
403
413
  VISIT(DenseUnion)
404
414
  VISIT(Dictionary)
@@ -431,6 +441,117 @@ namespace red_arrow {
431
441
  VALUE result_;
432
442
  };
433
443
 
444
+ class MapArrayValueConverter : public arrow::ArrayVisitor {
445
+ public:
446
+ explicit MapArrayValueConverter(ArrayValueConverter* converter)
447
+ : array_value_converter_(converter),
448
+ offset_(0),
449
+ length_(0),
450
+ values_(Qnil) {}
451
+
452
+ VALUE convert(const arrow::MapArray& array,
453
+ const int64_t index) {
454
+ auto key_array = array.keys().get();
455
+ auto item_array = array.items().get();
456
+ auto offset_keep = offset_;
457
+ auto length_keep = length_;
458
+ auto values_keep = values_;
459
+ offset_ = array.value_offset(index);
460
+ length_ = array.value_length(index);
461
+ auto keys = rb_ary_new_capa(length_);
462
+ values_ = keys;
463
+ check_status(key_array->Accept(this),
464
+ "[raw-records][map-array][keys]");
465
+ auto items = rb_ary_new_capa(length_);
466
+ values_ = items;
467
+ check_status(item_array->Accept(this),
468
+ "[raw-records][map-array][items]");
469
+ auto map = rb_hash_new();
470
+ auto n = RARRAY_LEN(keys);
471
+ auto raw_keys = RARRAY_CONST_PTR(keys);
472
+ auto raw_items = RARRAY_CONST_PTR(items);
473
+ for (long i = 0; i < n; ++i) {
474
+ rb_hash_aset(map, raw_keys[i], raw_items[i]);
475
+ }
476
+ offset_ = offset_keep;
477
+ length_ = length_keep;
478
+ values_ = values_keep;
479
+ return map;
480
+ }
481
+
482
+ #define VISIT(TYPE) \
483
+ arrow::Status Visit(const arrow::TYPE ## Array& array) override { \
484
+ return visit_value(array); \
485
+ }
486
+
487
+ VISIT(Null)
488
+ VISIT(Boolean)
489
+ VISIT(Int8)
490
+ VISIT(Int16)
491
+ VISIT(Int32)
492
+ VISIT(Int64)
493
+ VISIT(UInt8)
494
+ VISIT(UInt16)
495
+ VISIT(UInt32)
496
+ VISIT(UInt64)
497
+ // TODO
498
+ // VISIT(HalfFloat)
499
+ VISIT(Float)
500
+ VISIT(Double)
501
+ VISIT(Binary)
502
+ VISIT(String)
503
+ VISIT(FixedSizeBinary)
504
+ VISIT(Date32)
505
+ VISIT(Date64)
506
+ VISIT(Time32)
507
+ VISIT(Time64)
508
+ VISIT(Timestamp)
509
+ // TODO
510
+ // VISIT(Interval)
511
+ VISIT(List)
512
+ VISIT(Struct)
513
+ VISIT(Map)
514
+ VISIT(SparseUnion)
515
+ VISIT(DenseUnion)
516
+ VISIT(Dictionary)
517
+ VISIT(Decimal128)
518
+ VISIT(Decimal256)
519
+ // TODO
520
+ // VISIT(Extension)
521
+
522
+ #undef VISIT
523
+
524
+ private:
525
+ template <typename ArrayType>
526
+ inline VALUE convert_value(const ArrayType& array,
527
+ const int64_t i) {
528
+ return array_value_converter_->convert(array, i);
529
+ }
530
+
531
+ template <typename ArrayType>
532
+ arrow::Status visit_value(const ArrayType& array) {
533
+ if (array.null_count() > 0) {
534
+ for (int64_t i = 0; i < length_; ++i) {
535
+ auto value = Qnil;
536
+ if (!array.IsNull(i + offset_)) {
537
+ value = convert_value(array, i + offset_);
538
+ }
539
+ rb_ary_push(values_, value);
540
+ }
541
+ } else {
542
+ for (int64_t i = 0; i < length_; ++i) {
543
+ rb_ary_push(values_, convert_value(array, i + offset_));
544
+ }
545
+ }
546
+ return arrow::Status::OK();
547
+ }
548
+
549
+ ArrayValueConverter* array_value_converter_;
550
+ int32_t offset_;
551
+ int32_t length_;
552
+ VALUE values_;
553
+ };
554
+
434
555
  class UnionArrayValueConverter : public arrow::ArrayVisitor {
435
556
  public:
436
557
  explicit UnionArrayValueConverter(ArrayValueConverter* converter)
@@ -492,6 +613,7 @@ namespace red_arrow {
492
613
  // VISIT(Interval)
493
614
  VISIT(List)
494
615
  VISIT(Struct)
616
+ VISIT(Map)
495
617
  VISIT(SparseUnion)
496
618
  VISIT(DenseUnion)
497
619
  VISIT(Dictionary)
@@ -617,6 +739,7 @@ namespace red_arrow {
617
739
  // VISIT(Interval)
618
740
  VISIT(List)
619
741
  VISIT(Struct)
742
+ VISIT(Map)
620
743
  VISIT(SparseUnion)
621
744
  VISIT(DenseUnion)
622
745
  VISIT(Dictionary)
@@ -645,11 +768,13 @@ namespace red_arrow {
645
768
  : array_value_converter_(),
646
769
  list_array_value_converter_(&array_value_converter_),
647
770
  struct_array_value_converter_(&array_value_converter_),
771
+ map_array_value_converter_(&array_value_converter_),
648
772
  union_array_value_converter_(&array_value_converter_),
649
773
  dictionary_array_value_converter_(&array_value_converter_) {
650
774
  array_value_converter_.
651
775
  set_sub_value_converters(&list_array_value_converter_,
652
776
  &struct_array_value_converter_,
777
+ &map_array_value_converter_,
653
778
  &union_array_value_converter_,
654
779
  &dictionary_array_value_converter_);
655
780
  }
@@ -663,6 +788,7 @@ namespace red_arrow {
663
788
  ArrayValueConverter array_value_converter_;
664
789
  ListArrayValueConverter list_array_value_converter_;
665
790
  StructArrayValueConverter struct_array_value_converter_;
791
+ MapArrayValueConverter map_array_value_converter_;
666
792
  UnionArrayValueConverter union_array_value_converter_;
667
793
  DictionaryArrayValueConverter dictionary_array_value_converter_;
668
794
  };
data/ext/arrow/extconf.rb CHANGED
@@ -25,6 +25,19 @@ if arrow_pkg_config_path
25
25
  ENV["PKG_CONFIG_PATH"] = pkg_config_paths.join(File::PATH_SEPARATOR)
26
26
  end
27
27
 
28
+ checking_for(checking_message("Homebrew")) do
29
+ platform = NativePackageInstaller::Platform.detect
30
+ if platform.is_a?(NativePackageInstaller::Platform::Homebrew)
31
+ openssl_prefix = `brew --prefix openssl@1.1`.chomp
32
+ unless openssl_prefix.empty?
33
+ PKGConfig.add_path("#{openssl_prefix}/lib/pkgconfig")
34
+ end
35
+ true
36
+ else
37
+ false
38
+ end
39
+ end
40
+
28
41
  unless required_pkg_config_package([
29
42
  "arrow",
30
43
  Arrow::Version::MAJOR,
@@ -0,0 +1,311 @@
1
+ /*
2
+ * Licensed to the Apache Software Foundation (ASF) under one
3
+ * or more contributor license agreements. See the NOTICE file
4
+ * distributed with this work for additional information
5
+ * regarding copyright ownership. The ASF licenses this file
6
+ * to you under the Apache License, Version 2.0 (the
7
+ * "License"); you may not use this file except in compliance
8
+ * with the License. You may obtain a copy of the License at
9
+ *
10
+ * http://www.apache.org/licenses/LICENSE-2.0
11
+ *
12
+ * Unless required by applicable law or agreed to in writing,
13
+ * software distributed under the License is distributed on an
14
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15
+ * KIND, either express or implied. See the License for the
16
+ * specific language governing permissions and limitations
17
+ * under the License.
18
+ */
19
+
20
+ #include "memory-view.hpp"
21
+
22
+ #include <arrow-glib/arrow-glib.hpp>
23
+ #include <rbgobject.h>
24
+
25
+ #include <ruby/version.h>
26
+
27
+ #if RUBY_API_VERSION_MAJOR >= 3
28
+ # define HAVE_MEMORY_VIEW
29
+ # define private memory_view_private
30
+ # include <ruby/memory_view.h>
31
+ # undef private
32
+ #endif
33
+
34
+ #include <sstream>
35
+
36
+ namespace red_arrow {
37
+ namespace memory_view {
38
+ #ifdef HAVE_MEMORY_VIEW
39
+ // This is workaround for the following rb_memory_view_t problems
40
+ // in C++:
41
+ //
42
+ // * Can't use "private" as member name
43
+ // * Can't assign a value to "rb_memory_view_t::private"
44
+ //
45
+ // This has compatible layout with rb_memory_view_t.
46
+ struct memory_view {
47
+ VALUE obj;
48
+ void *data;
49
+ ssize_t byte_size;
50
+ bool readonly;
51
+ const char *format;
52
+ ssize_t item_size;
53
+ struct {
54
+ const rb_memory_view_item_component_t *components;
55
+ size_t length;
56
+ } item_desc;
57
+ ssize_t ndim;
58
+ const ssize_t *shape;
59
+ const ssize_t *strides;
60
+ const ssize_t *sub_offsets;
61
+ void *private_data;
62
+ };
63
+
64
+ struct PrivateData {
65
+ std::string format;
66
+ };
67
+
68
+ class PrimitiveArrayGetter : public arrow::ArrayVisitor {
69
+ public:
70
+ explicit PrimitiveArrayGetter(memory_view *view)
71
+ : view_(view) {
72
+ }
73
+
74
+ arrow::Status Visit(const arrow::BooleanArray& array) override {
75
+ fill(static_cast<const arrow::Array&>(array));
76
+ // Memory view doesn't support bit stream. We use one byte
77
+ // for 8 elements. Users can't calculate the number of
78
+ // elements from memory view but it's limitation of memory view.
79
+ #ifdef ARROW_LITTLE_ENDIAN
80
+ view_->format = "b8";
81
+ #else
82
+ view_->format = "B8";
83
+ #endif
84
+ view_->item_size = 1;
85
+ view_->byte_size = (array.length() + 7) / 8;
86
+ return arrow::Status::OK();
87
+ }
88
+
89
+ arrow::Status Visit(const arrow::Int8Array& array) override {
90
+ fill(static_cast<const arrow::Array&>(array));
91
+ view_->format = "c";
92
+ return arrow::Status::OK();
93
+ }
94
+
95
+ arrow::Status Visit(const arrow::Int16Array& array) override {
96
+ fill(static_cast<const arrow::Array&>(array));
97
+ view_->format = "s";
98
+ return arrow::Status::OK();
99
+ }
100
+
101
+ arrow::Status Visit(const arrow::Int32Array& array) override {
102
+ fill(static_cast<const arrow::Array&>(array));
103
+ view_->format = "l";
104
+ return arrow::Status::OK();
105
+ }
106
+
107
+ arrow::Status Visit(const arrow::Int64Array& array) override {
108
+ fill(static_cast<const arrow::Array&>(array));
109
+ view_->format = "q";
110
+ return arrow::Status::OK();
111
+ }
112
+
113
+ arrow::Status Visit(const arrow::UInt8Array& array) override {
114
+ fill(static_cast<const arrow::Array&>(array));
115
+ view_->format = "C";
116
+ return arrow::Status::OK();
117
+ }
118
+
119
+ arrow::Status Visit(const arrow::UInt16Array& array) override {
120
+ fill(static_cast<const arrow::Array&>(array));
121
+ view_->format = "S";
122
+ return arrow::Status::OK();
123
+ }
124
+
125
+ arrow::Status Visit(const arrow::UInt32Array& array) override {
126
+ fill(static_cast<const arrow::Array&>(array));
127
+ view_->format = "L";
128
+ return arrow::Status::OK();
129
+ }
130
+
131
+ arrow::Status Visit(const arrow::UInt64Array& array) override {
132
+ fill(static_cast<const arrow::Array&>(array));
133
+ view_->format = "Q";
134
+ return arrow::Status::OK();
135
+ }
136
+
137
+ arrow::Status Visit(const arrow::FloatArray& array) override {
138
+ fill(static_cast<const arrow::Array&>(array));
139
+ view_->format = "f";
140
+ return arrow::Status::OK();
141
+ }
142
+
143
+ arrow::Status Visit(const arrow::DoubleArray& array) override {
144
+ fill(static_cast<const arrow::Array&>(array));
145
+ view_->format = "d";
146
+ return arrow::Status::OK();
147
+ }
148
+
149
+ arrow::Status Visit(const arrow::FixedSizeBinaryArray& array) override {
150
+ fill(static_cast<const arrow::Array&>(array));
151
+ auto priv = static_cast<PrivateData *>(view_->private_data);
152
+ const auto type =
153
+ std::static_pointer_cast<const arrow::FixedSizeBinaryType>(
154
+ array.type());
155
+ std::ostringstream output;
156
+ output << "C" << type->byte_width();
157
+ priv->format = output.str();
158
+ view_->format = priv->format.c_str();
159
+ return arrow::Status::OK();
160
+ }
161
+
162
+ arrow::Status Visit(const arrow::Date32Array& array) override {
163
+ fill(static_cast<const arrow::Array&>(array));
164
+ view_->format = "l";
165
+ return arrow::Status::OK();
166
+ }
167
+
168
+ arrow::Status Visit(const arrow::Date64Array& array) override {
169
+ fill(static_cast<const arrow::Array&>(array));
170
+ view_->format = "q";
171
+ return arrow::Status::OK();
172
+ }
173
+
174
+ arrow::Status Visit(const arrow::Time32Array& array) override {
175
+ fill(static_cast<const arrow::Array&>(array));
176
+ view_->format = "l";
177
+ return arrow::Status::OK();
178
+ }
179
+
180
+ arrow::Status Visit(const arrow::Time64Array& array) override {
181
+ fill(static_cast<const arrow::Array&>(array));
182
+ view_->format = "q";
183
+ return arrow::Status::OK();
184
+ }
185
+
186
+ arrow::Status Visit(const arrow::TimestampArray& array) override {
187
+ fill(static_cast<const arrow::Array&>(array));
188
+ view_->format = "q";
189
+ return arrow::Status::OK();
190
+ }
191
+
192
+ arrow::Status Visit(const arrow::Decimal128Array& array) override {
193
+ fill(static_cast<const arrow::Array&>(array));
194
+ view_->format = "q2";
195
+ return arrow::Status::OK();
196
+ }
197
+
198
+ arrow::Status Visit(const arrow::Decimal256Array& array) override {
199
+ fill(static_cast<const arrow::Array&>(array));
200
+ view_->format = "q4";
201
+ return arrow::Status::OK();
202
+ }
203
+
204
+ private:
205
+ void fill(const arrow::Array& array) {
206
+ const auto array_data = array.data();
207
+ const auto data = array_data->GetValuesSafe<uint8_t>(1);
208
+ view_->data = const_cast<void *>(reinterpret_cast<const void *>(data));
209
+ const auto type =
210
+ std::static_pointer_cast<const arrow::FixedWidthType>(array.type());
211
+ view_->item_size = type->bit_width() / 8;
212
+ view_->byte_size = view_->item_size * array.length();
213
+ }
214
+
215
+ memory_view *view_;
216
+ };
217
+
218
+ bool primitive_array_get(VALUE obj, rb_memory_view_t *view, int flags) {
219
+ if (flags != RUBY_MEMORY_VIEW_SIMPLE) {
220
+ return false;
221
+ }
222
+ auto view_ = reinterpret_cast<memory_view *>(view);
223
+ view_->obj = obj;
224
+ view_->private_data = new PrivateData();
225
+ auto array = GARROW_ARRAY(RVAL2GOBJ(obj));
226
+ auto arrow_array = garrow_array_get_raw(array);
227
+ PrimitiveArrayGetter getter(view_);
228
+ auto status = arrow_array->Accept(&getter);
229
+ if (!status.ok()) {
230
+ return false;
231
+ }
232
+ view_->readonly = true;
233
+ view_->ndim = 1;
234
+ view_->shape = NULL;
235
+ view_->strides = NULL;
236
+ view_->sub_offsets = NULL;
237
+ return true;
238
+ }
239
+
240
+ bool primitive_array_release(VALUE obj, rb_memory_view_t *view) {
241
+ auto view_ = reinterpret_cast<memory_view *>(view);
242
+ delete static_cast<PrivateData *>(view_->private_data);
243
+ return true;
244
+ }
245
+
246
+ bool primitive_array_available_p(VALUE obj) {
247
+ return true;
248
+ }
249
+
250
+ rb_memory_view_entry_t primitive_array_entry = {
251
+ primitive_array_get,
252
+ primitive_array_release,
253
+ primitive_array_available_p,
254
+ };
255
+
256
+ bool buffer_get(VALUE obj, rb_memory_view_t *view, int flags) {
257
+ if (flags != RUBY_MEMORY_VIEW_SIMPLE) {
258
+ return false;
259
+ }
260
+ auto view_ = reinterpret_cast<memory_view *>(view);
261
+ view_->obj = obj;
262
+ auto buffer = GARROW_BUFFER(RVAL2GOBJ(obj));
263
+ auto arrow_buffer = garrow_buffer_get_raw(buffer);
264
+ view_->data =
265
+ const_cast<void *>(reinterpret_cast<const void *>(arrow_buffer->data()));
266
+ // Memory view doesn't support bit stream. We use one byte
267
+ // for 8 elements. Users can't calculate the number of
268
+ // elements from memory view but it's limitation of memory view.
269
+ #ifdef ARROW_LITTLE_ENDIAN
270
+ view_->format = "b8";
271
+ #else
272
+ view_->format = "B8";
273
+ #endif
274
+ view_->item_size = 1;
275
+ view_->byte_size = arrow_buffer->size();
276
+ view_->readonly = true;
277
+ view_->ndim = 1;
278
+ view_->shape = NULL;
279
+ view_->strides = NULL;
280
+ view_->sub_offsets = NULL;
281
+ return true;
282
+ }
283
+
284
+ bool buffer_release(VALUE obj, rb_memory_view_t *view) {
285
+ return true;
286
+ }
287
+
288
+ bool buffer_available_p(VALUE obj) {
289
+ return true;
290
+ }
291
+
292
+ rb_memory_view_entry_t buffer_entry = {
293
+ buffer_get,
294
+ buffer_release,
295
+ buffer_available_p,
296
+ };
297
+ #endif
298
+
299
+ void init(VALUE mArrow) {
300
+ #ifdef HAVE_MEMORY_VIEW
301
+ auto cPrimitiveArray =
302
+ rb_const_get_at(mArrow, rb_intern("PrimitiveArray"));
303
+ rb_memory_view_register(cPrimitiveArray,
304
+ &(red_arrow::memory_view::primitive_array_entry));
305
+
306
+ auto cBuffer = rb_const_get_at(mArrow, rb_intern("Buffer"));
307
+ rb_memory_view_register(cBuffer, &(red_arrow::memory_view::buffer_entry));
308
+ #endif
309
+ }
310
+ }
311
+ }
@@ -0,0 +1,26 @@
1
+ /*
2
+ * Licensed to the Apache Software Foundation (ASF) under one
3
+ * or more contributor license agreements. See the NOTICE file
4
+ * distributed with this work for additional information
5
+ * regarding copyright ownership. The ASF licenses this file
6
+ * to you under the Apache License, Version 2.0 (the
7
+ * "License"); you may not use this file except in compliance
8
+ * with the License. You may obtain a copy of the License at
9
+ *
10
+ * http://www.apache.org/licenses/LICENSE-2.0
11
+ *
12
+ * Unless required by applicable law or agreed to in writing,
13
+ * software distributed under the License is distributed on an
14
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15
+ * KIND, either express or implied. See the License for the
16
+ * specific language governing permissions and limitations
17
+ * under the License.
18
+ */
19
+
20
+ #include <ruby.hpp>
21
+
22
+ namespace red_arrow {
23
+ namespace memory_view {
24
+ void init(VALUE mArrow);
25
+ }
26
+ }
@@ -100,6 +100,7 @@ namespace red_arrow {
100
100
  // VISIT(Interval)
101
101
  VISIT(List)
102
102
  VISIT(Struct)
103
+ VISIT(Map)
103
104
  VISIT(SparseUnion)
104
105
  VISIT(DenseUnion)
105
106
  VISIT(Dictionary)
data/ext/arrow/values.cpp CHANGED
@@ -81,6 +81,7 @@ namespace red_arrow {
81
81
  // VISIT(Interval)
82
82
  VISIT(List)
83
83
  VISIT(Struct)
84
+ VISIT(Map)
84
85
  VISIT(SparseUnion)
85
86
  VISIT(DenseUnion)
86
87
  VISIT(Dictionary)
@@ -0,0 +1,35 @@
1
+ # Licensed to the Apache Software Foundation (ASF) under one
2
+ # or more contributor license agreements. See the NOTICE file
3
+ # distributed with this work for additional information
4
+ # regarding copyright ownership. The ASF licenses this file
5
+ # to you under the Apache License, Version 2.0 (the
6
+ # "License"); you may not use this file except in compliance
7
+ # with the License. You may obtain a copy of the License at
8
+ #
9
+ # http://www.apache.org/licenses/LICENSE-2.0
10
+ #
11
+ # Unless required by applicable law or agreed to in writing,
12
+ # software distributed under the License is distributed on an
13
+ # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
+ # KIND, either express or implied. See the License for the
15
+ # specific language governing permissions and limitations
16
+ # under the License.
17
+
18
+ module Arrow
19
+ class AggregateNodeOptions
20
+ class << self
21
+ # @api private
22
+ def try_convert(value)
23
+ case value
24
+ when Hash
25
+ aggregations = value[:aggregations]
26
+ return nil if aggregations.nil?
27
+ keys = value[:keys]
28
+ new(aggregations, keys)
29
+ else
30
+ nil
31
+ end
32
+ end
33
+ end
34
+ end
35
+ end