red-arrow 3.0.0 → 6.0.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (88) hide show
  1. checksums.yaml +4 -4
  2. data/README.md +23 -0
  3. data/ext/arrow/arrow.cpp +3 -0
  4. data/ext/arrow/converters.cpp +5 -0
  5. data/ext/arrow/converters.hpp +126 -0
  6. data/ext/arrow/extconf.rb +13 -0
  7. data/ext/arrow/memory-view.cpp +311 -0
  8. data/ext/arrow/memory-view.hpp +26 -0
  9. data/ext/arrow/raw-records.cpp +1 -0
  10. data/ext/arrow/values.cpp +1 -0
  11. data/lib/arrow/aggregate-node-options.rb +35 -0
  12. data/lib/arrow/aggregation.rb +46 -0
  13. data/lib/arrow/array-builder.rb +5 -0
  14. data/lib/arrow/array.rb +130 -0
  15. data/lib/arrow/binary-dictionary-array-builder.rb +27 -0
  16. data/lib/arrow/buffer.rb +10 -6
  17. data/lib/arrow/column-containable.rb +100 -1
  18. data/lib/arrow/constructor-arguments-gc-guardable.rb +25 -0
  19. data/lib/arrow/data-type.rb +14 -5
  20. data/lib/arrow/datum.rb +100 -0
  21. data/lib/arrow/dense-union-data-type.rb +2 -2
  22. data/lib/arrow/dictionary-data-type.rb +2 -2
  23. data/lib/arrow/equal-options.rb +38 -0
  24. data/lib/arrow/expression.rb +48 -0
  25. data/lib/arrow/file-system.rb +34 -0
  26. data/lib/arrow/group.rb +116 -124
  27. data/lib/arrow/loader.rb +46 -0
  28. data/lib/arrow/map-array-builder.rb +109 -0
  29. data/lib/arrow/map-array.rb +26 -0
  30. data/lib/arrow/map-data-type.rb +89 -0
  31. data/lib/arrow/path-extension.rb +1 -1
  32. data/lib/arrow/record-batch-reader.rb +41 -0
  33. data/lib/arrow/record-batch.rb +0 -2
  34. data/lib/arrow/scalar.rb +32 -0
  35. data/lib/arrow/slicer.rb +44 -143
  36. data/lib/arrow/sort-key.rb +193 -0
  37. data/lib/arrow/sort-options.rb +109 -0
  38. data/lib/arrow/source-node-options.rb +32 -0
  39. data/lib/arrow/sparse-union-data-type.rb +2 -2
  40. data/lib/arrow/string-dictionary-array-builder.rb +27 -0
  41. data/lib/arrow/symbol-values-appendable.rb +34 -0
  42. data/lib/arrow/table-concatenate-options.rb +36 -0
  43. data/lib/arrow/table-formatter.rb +141 -17
  44. data/lib/arrow/table-list-formatter.rb +5 -3
  45. data/lib/arrow/table-loader.rb +41 -3
  46. data/lib/arrow/table-saver.rb +29 -3
  47. data/lib/arrow/table-table-formatter.rb +7 -31
  48. data/lib/arrow/table.rb +34 -40
  49. data/lib/arrow/time32-data-type.rb +2 -2
  50. data/lib/arrow/time64-data-type.rb +2 -2
  51. data/lib/arrow/timestamp-data-type.rb +2 -2
  52. data/lib/arrow/version.rb +1 -1
  53. data/red-arrow.gemspec +2 -1
  54. data/test/helper.rb +1 -0
  55. data/test/raw-records/test-dense-union-array.rb +14 -0
  56. data/test/raw-records/test-list-array.rb +19 -0
  57. data/test/raw-records/test-map-array.rb +441 -0
  58. data/test/raw-records/test-sparse-union-array.rb +14 -0
  59. data/test/raw-records/test-struct-array.rb +15 -0
  60. data/test/test-array-builder.rb +7 -0
  61. data/test/test-array.rb +154 -0
  62. data/test/test-binary-dictionary-array-builder.rb +103 -0
  63. data/test/test-boolean-scalar.rb +26 -0
  64. data/test/test-csv-loader.rb +8 -8
  65. data/test/test-decimal128-data-type.rb +2 -2
  66. data/test/test-expression.rb +40 -0
  67. data/test/test-float-scalar.rb +46 -0
  68. data/test/test-function.rb +176 -0
  69. data/test/test-group.rb +75 -51
  70. data/test/test-map-array-builder.rb +110 -0
  71. data/test/test-map-array.rb +33 -0
  72. data/test/test-map-data-type.rb +36 -0
  73. data/test/test-memory-view.rb +434 -0
  74. data/test/test-orc.rb +19 -23
  75. data/test/test-record-batch-reader.rb +46 -0
  76. data/test/test-record-batch.rb +42 -0
  77. data/test/test-slicer.rb +166 -167
  78. data/test/test-sort-indices.rb +40 -0
  79. data/test/test-sort-key.rb +81 -0
  80. data/test/test-sort-options.rb +58 -0
  81. data/test/test-string-dictionary-array-builder.rb +103 -0
  82. data/test/test-table.rb +190 -53
  83. data/test/values/test-dense-union-array.rb +14 -0
  84. data/test/values/test-list-array.rb +17 -0
  85. data/test/values/test-map-array.rb +433 -0
  86. data/test/values/test-sparse-union-array.rb +14 -0
  87. data/test/values/test-struct-array.rb +15 -0
  88. metadata +73 -6
checksums.yaml CHANGED
@@ -1,7 +1,7 @@
1
1
  ---
2
2
  SHA256:
3
- metadata.gz: a70c1505c294a8f74c992ec0f2ab5d651647e45dd178db8523f5b5c01e64a541
4
- data.tar.gz: dad979599033104a25d5be3d05c57e552c803f9c8002d66b757866c425937ce9
3
+ metadata.gz: c553e573f87398da7d492e8339684bb86b73d1fff02b56cff3277bb87d96a4cd
4
+ data.tar.gz: 9b97be2878369da0bed9ac8621330926b8a95144d833de6798858a210f88c92a
5
5
  SHA512:
6
- metadata.gz: b2363ba6468985a3d237cbb9cec8b2ccb5031aae94153b8f263916e811524e8a5326f408867a843b7077f624050fabcde93520b26e6f8ceb1dd094aaa0068ed6
7
- data.tar.gz: 797b89062dfd212d92ca957b8e51c6dc8acf16ab51e5a60ecb0cbf31dd979c288f2afb71274b2235a24d31da344cfce93b40830d6dc6edd000ce0aa607ce8cde
6
+ metadata.gz: 0bc3526645457a551e25621ba3efaa52a3ad7398e8216c34440651beab58886ec8b52804b7afe227f953b727871980a2ac9a9b1153776499b48cd790ae476304
7
+ data.tar.gz: 9f3032d5a54a9b0b911edd273916ce68f140c50753d13cb88b950166ea771138f432d738a3a694f44bfff985fcf8329457e97712dca10a7f8a8430d3d8cb82f6
data/README.md CHANGED
@@ -50,3 +50,26 @@ table = Arrow::Table.load("/dev/shm/data.arrow")
50
50
  # Process data in table
51
51
  table.save("/dev/shm/data-processed.arrow")
52
52
  ```
53
+
54
+ ## Development
55
+
56
+ Note that you need to install Apache Arrow C++/GLib at master before preparing Red Arrow. See also:
57
+
58
+ * For Apache Arrow C++: https://arrow.apache.org/docs/developers/cpp/building.html
59
+ * For Apache Arrow GLib: https://github.com/apache/arrow/blob/master/c_glib/README.md
60
+
61
+ ```console
62
+ $ cd ruby/red-arrow
63
+ $ bundle install
64
+ $ bundle exec rake test
65
+ ```
66
+
67
+ ### For macOS with Homebrew
68
+
69
+ ```console
70
+ $ cd ruby/red-arrow
71
+ $ bundle install
72
+ $ brew install apache-arrow --head
73
+ $ brew install apache-arrow-glib --head
74
+ $ bundle exec rake test
75
+ ```
data/ext/arrow/arrow.cpp CHANGED
@@ -18,6 +18,7 @@
18
18
  */
19
19
 
20
20
  #include "red-arrow.hpp"
21
+ #include "memory-view.hpp"
21
22
 
22
23
  #include <ruby.hpp>
23
24
 
@@ -78,4 +79,6 @@ extern "C" void Init_arrow() {
78
79
  red_arrow::id_jd = rb_intern("jd");
79
80
  red_arrow::id_new = rb_intern("new");
80
81
  red_arrow::id_to_datetime = rb_intern("to_datetime");
82
+
83
+ red_arrow::memory_view::init(mArrow);
81
84
  }
@@ -30,6 +30,11 @@ namespace red_arrow {
30
30
  return struct_array_value_converter_->convert(array, i);
31
31
  }
32
32
 
33
+ VALUE ArrayValueConverter::convert(const arrow::MapArray& array,
34
+ const int64_t i) {
35
+ return map_array_value_converter_->convert(array, i);
36
+ }
37
+
33
38
  VALUE ArrayValueConverter::convert(const arrow::UnionArray& array,
34
39
  const int64_t i) {
35
40
  return union_array_value_converter_->convert(array, i);
@@ -29,6 +29,7 @@
29
29
  namespace red_arrow {
30
30
  class ListArrayValueConverter;
31
31
  class StructArrayValueConverter;
32
+ class MapArrayValueConverter;
32
33
  class UnionArrayValueConverter;
33
34
  class DictionaryArrayValueConverter;
34
35
 
@@ -38,16 +39,19 @@ namespace red_arrow {
38
39
  : decimal_buffer_(),
39
40
  list_array_value_converter_(nullptr),
40
41
  struct_array_value_converter_(nullptr),
42
+ map_array_value_converter_(nullptr),
41
43
  union_array_value_converter_(nullptr),
42
44
  dictionary_array_value_converter_(nullptr) {
43
45
  }
44
46
 
45
47
  inline void set_sub_value_converters(ListArrayValueConverter* list_array_value_converter,
46
48
  StructArrayValueConverter* struct_array_value_converter,
49
+ MapArrayValueConverter* map_array_value_converter,
47
50
  UnionArrayValueConverter* union_array_value_converter,
48
51
  DictionaryArrayValueConverter* dictionary_array_value_converter) {
49
52
  list_array_value_converter_ = list_array_value_converter;
50
53
  struct_array_value_converter_ = struct_array_value_converter;
54
+ map_array_value_converter_ = map_array_value_converter;
51
55
  union_array_value_converter_ = union_array_value_converter;
52
56
  dictionary_array_value_converter_ = dictionary_array_value_converter;
53
57
  }
@@ -204,6 +208,9 @@ namespace red_arrow {
204
208
  VALUE convert(const arrow::StructArray& array,
205
209
  const int64_t i);
206
210
 
211
+ VALUE convert(const arrow::MapArray& array,
212
+ const int64_t i);
213
+
207
214
  VALUE convert(const arrow::UnionArray& array,
208
215
  const int64_t i);
209
216
 
@@ -234,6 +241,7 @@ namespace red_arrow {
234
241
  std::string decimal_buffer_;
235
242
  ListArrayValueConverter* list_array_value_converter_;
236
243
  StructArrayValueConverter* struct_array_value_converter_;
244
+ MapArrayValueConverter* map_array_value_converter_;
237
245
  UnionArrayValueConverter* union_array_value_converter_;
238
246
  DictionaryArrayValueConverter* dictionary_array_value_converter_;
239
247
  };
@@ -294,6 +302,7 @@ namespace red_arrow {
294
302
  // VISIT(Interval)
295
303
  VISIT(List)
296
304
  VISIT(Struct)
305
+ VISIT(Map)
297
306
  VISIT(SparseUnion)
298
307
  VISIT(DenseUnion)
299
308
  VISIT(Dictionary)
@@ -399,6 +408,7 @@ namespace red_arrow {
399
408
  // VISIT(Interval)
400
409
  VISIT(List)
401
410
  VISIT(Struct)
411
+ VISIT(Map)
402
412
  VISIT(SparseUnion)
403
413
  VISIT(DenseUnion)
404
414
  VISIT(Dictionary)
@@ -431,6 +441,117 @@ namespace red_arrow {
431
441
  VALUE result_;
432
442
  };
433
443
 
444
+ class MapArrayValueConverter : public arrow::ArrayVisitor {
445
+ public:
446
+ explicit MapArrayValueConverter(ArrayValueConverter* converter)
447
+ : array_value_converter_(converter),
448
+ offset_(0),
449
+ length_(0),
450
+ values_(Qnil) {}
451
+
452
+ VALUE convert(const arrow::MapArray& array,
453
+ const int64_t index) {
454
+ auto key_array = array.keys().get();
455
+ auto item_array = array.items().get();
456
+ auto offset_keep = offset_;
457
+ auto length_keep = length_;
458
+ auto values_keep = values_;
459
+ offset_ = array.value_offset(index);
460
+ length_ = array.value_length(index);
461
+ auto keys = rb_ary_new_capa(length_);
462
+ values_ = keys;
463
+ check_status(key_array->Accept(this),
464
+ "[raw-records][map-array][keys]");
465
+ auto items = rb_ary_new_capa(length_);
466
+ values_ = items;
467
+ check_status(item_array->Accept(this),
468
+ "[raw-records][map-array][items]");
469
+ auto map = rb_hash_new();
470
+ auto n = RARRAY_LEN(keys);
471
+ auto raw_keys = RARRAY_CONST_PTR(keys);
472
+ auto raw_items = RARRAY_CONST_PTR(items);
473
+ for (long i = 0; i < n; ++i) {
474
+ rb_hash_aset(map, raw_keys[i], raw_items[i]);
475
+ }
476
+ offset_ = offset_keep;
477
+ length_ = length_keep;
478
+ values_ = values_keep;
479
+ return map;
480
+ }
481
+
482
+ #define VISIT(TYPE) \
483
+ arrow::Status Visit(const arrow::TYPE ## Array& array) override { \
484
+ return visit_value(array); \
485
+ }
486
+
487
+ VISIT(Null)
488
+ VISIT(Boolean)
489
+ VISIT(Int8)
490
+ VISIT(Int16)
491
+ VISIT(Int32)
492
+ VISIT(Int64)
493
+ VISIT(UInt8)
494
+ VISIT(UInt16)
495
+ VISIT(UInt32)
496
+ VISIT(UInt64)
497
+ // TODO
498
+ // VISIT(HalfFloat)
499
+ VISIT(Float)
500
+ VISIT(Double)
501
+ VISIT(Binary)
502
+ VISIT(String)
503
+ VISIT(FixedSizeBinary)
504
+ VISIT(Date32)
505
+ VISIT(Date64)
506
+ VISIT(Time32)
507
+ VISIT(Time64)
508
+ VISIT(Timestamp)
509
+ // TODO
510
+ // VISIT(Interval)
511
+ VISIT(List)
512
+ VISIT(Struct)
513
+ VISIT(Map)
514
+ VISIT(SparseUnion)
515
+ VISIT(DenseUnion)
516
+ VISIT(Dictionary)
517
+ VISIT(Decimal128)
518
+ VISIT(Decimal256)
519
+ // TODO
520
+ // VISIT(Extension)
521
+
522
+ #undef VISIT
523
+
524
+ private:
525
+ template <typename ArrayType>
526
+ inline VALUE convert_value(const ArrayType& array,
527
+ const int64_t i) {
528
+ return array_value_converter_->convert(array, i);
529
+ }
530
+
531
+ template <typename ArrayType>
532
+ arrow::Status visit_value(const ArrayType& array) {
533
+ if (array.null_count() > 0) {
534
+ for (int64_t i = 0; i < length_; ++i) {
535
+ auto value = Qnil;
536
+ if (!array.IsNull(i + offset_)) {
537
+ value = convert_value(array, i + offset_);
538
+ }
539
+ rb_ary_push(values_, value);
540
+ }
541
+ } else {
542
+ for (int64_t i = 0; i < length_; ++i) {
543
+ rb_ary_push(values_, convert_value(array, i + offset_));
544
+ }
545
+ }
546
+ return arrow::Status::OK();
547
+ }
548
+
549
+ ArrayValueConverter* array_value_converter_;
550
+ int32_t offset_;
551
+ int32_t length_;
552
+ VALUE values_;
553
+ };
554
+
434
555
  class UnionArrayValueConverter : public arrow::ArrayVisitor {
435
556
  public:
436
557
  explicit UnionArrayValueConverter(ArrayValueConverter* converter)
@@ -492,6 +613,7 @@ namespace red_arrow {
492
613
  // VISIT(Interval)
493
614
  VISIT(List)
494
615
  VISIT(Struct)
616
+ VISIT(Map)
495
617
  VISIT(SparseUnion)
496
618
  VISIT(DenseUnion)
497
619
  VISIT(Dictionary)
@@ -617,6 +739,7 @@ namespace red_arrow {
617
739
  // VISIT(Interval)
618
740
  VISIT(List)
619
741
  VISIT(Struct)
742
+ VISIT(Map)
620
743
  VISIT(SparseUnion)
621
744
  VISIT(DenseUnion)
622
745
  VISIT(Dictionary)
@@ -645,11 +768,13 @@ namespace red_arrow {
645
768
  : array_value_converter_(),
646
769
  list_array_value_converter_(&array_value_converter_),
647
770
  struct_array_value_converter_(&array_value_converter_),
771
+ map_array_value_converter_(&array_value_converter_),
648
772
  union_array_value_converter_(&array_value_converter_),
649
773
  dictionary_array_value_converter_(&array_value_converter_) {
650
774
  array_value_converter_.
651
775
  set_sub_value_converters(&list_array_value_converter_,
652
776
  &struct_array_value_converter_,
777
+ &map_array_value_converter_,
653
778
  &union_array_value_converter_,
654
779
  &dictionary_array_value_converter_);
655
780
  }
@@ -663,6 +788,7 @@ namespace red_arrow {
663
788
  ArrayValueConverter array_value_converter_;
664
789
  ListArrayValueConverter list_array_value_converter_;
665
790
  StructArrayValueConverter struct_array_value_converter_;
791
+ MapArrayValueConverter map_array_value_converter_;
666
792
  UnionArrayValueConverter union_array_value_converter_;
667
793
  DictionaryArrayValueConverter dictionary_array_value_converter_;
668
794
  };
data/ext/arrow/extconf.rb CHANGED
@@ -25,6 +25,19 @@ if arrow_pkg_config_path
25
25
  ENV["PKG_CONFIG_PATH"] = pkg_config_paths.join(File::PATH_SEPARATOR)
26
26
  end
27
27
 
28
+ checking_for(checking_message("Homebrew")) do
29
+ platform = NativePackageInstaller::Platform.detect
30
+ if platform.is_a?(NativePackageInstaller::Platform::Homebrew)
31
+ openssl_prefix = `brew --prefix openssl@1.1`.chomp
32
+ unless openssl_prefix.empty?
33
+ PKGConfig.add_path("#{openssl_prefix}/lib/pkgconfig")
34
+ end
35
+ true
36
+ else
37
+ false
38
+ end
39
+ end
40
+
28
41
  unless required_pkg_config_package([
29
42
  "arrow",
30
43
  Arrow::Version::MAJOR,
@@ -0,0 +1,311 @@
1
+ /*
2
+ * Licensed to the Apache Software Foundation (ASF) under one
3
+ * or more contributor license agreements. See the NOTICE file
4
+ * distributed with this work for additional information
5
+ * regarding copyright ownership. The ASF licenses this file
6
+ * to you under the Apache License, Version 2.0 (the
7
+ * "License"); you may not use this file except in compliance
8
+ * with the License. You may obtain a copy of the License at
9
+ *
10
+ * http://www.apache.org/licenses/LICENSE-2.0
11
+ *
12
+ * Unless required by applicable law or agreed to in writing,
13
+ * software distributed under the License is distributed on an
14
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15
+ * KIND, either express or implied. See the License for the
16
+ * specific language governing permissions and limitations
17
+ * under the License.
18
+ */
19
+
20
+ #include "memory-view.hpp"
21
+
22
+ #include <arrow-glib/arrow-glib.hpp>
23
+ #include <rbgobject.h>
24
+
25
+ #include <ruby/version.h>
26
+
27
+ #if RUBY_API_VERSION_MAJOR >= 3
28
+ # define HAVE_MEMORY_VIEW
29
+ # define private memory_view_private
30
+ # include <ruby/memory_view.h>
31
+ # undef private
32
+ #endif
33
+
34
+ #include <sstream>
35
+
36
+ namespace red_arrow {
37
+ namespace memory_view {
38
+ #ifdef HAVE_MEMORY_VIEW
39
+ // This is workaround for the following rb_memory_view_t problems
40
+ // in C++:
41
+ //
42
+ // * Can't use "private" as member name
43
+ // * Can't assign a value to "rb_memory_view_t::private"
44
+ //
45
+ // This has compatible layout with rb_memory_view_t.
46
+ struct memory_view {
47
+ VALUE obj;
48
+ void *data;
49
+ ssize_t byte_size;
50
+ bool readonly;
51
+ const char *format;
52
+ ssize_t item_size;
53
+ struct {
54
+ const rb_memory_view_item_component_t *components;
55
+ size_t length;
56
+ } item_desc;
57
+ ssize_t ndim;
58
+ const ssize_t *shape;
59
+ const ssize_t *strides;
60
+ const ssize_t *sub_offsets;
61
+ void *private_data;
62
+ };
63
+
64
+ struct PrivateData {
65
+ std::string format;
66
+ };
67
+
68
+ class PrimitiveArrayGetter : public arrow::ArrayVisitor {
69
+ public:
70
+ explicit PrimitiveArrayGetter(memory_view *view)
71
+ : view_(view) {
72
+ }
73
+
74
+ arrow::Status Visit(const arrow::BooleanArray& array) override {
75
+ fill(static_cast<const arrow::Array&>(array));
76
+ // Memory view doesn't support bit stream. We use one byte
77
+ // for 8 elements. Users can't calculate the number of
78
+ // elements from memory view but it's limitation of memory view.
79
+ #ifdef ARROW_LITTLE_ENDIAN
80
+ view_->format = "b8";
81
+ #else
82
+ view_->format = "B8";
83
+ #endif
84
+ view_->item_size = 1;
85
+ view_->byte_size = (array.length() + 7) / 8;
86
+ return arrow::Status::OK();
87
+ }
88
+
89
+ arrow::Status Visit(const arrow::Int8Array& array) override {
90
+ fill(static_cast<const arrow::Array&>(array));
91
+ view_->format = "c";
92
+ return arrow::Status::OK();
93
+ }
94
+
95
+ arrow::Status Visit(const arrow::Int16Array& array) override {
96
+ fill(static_cast<const arrow::Array&>(array));
97
+ view_->format = "s";
98
+ return arrow::Status::OK();
99
+ }
100
+
101
+ arrow::Status Visit(const arrow::Int32Array& array) override {
102
+ fill(static_cast<const arrow::Array&>(array));
103
+ view_->format = "l";
104
+ return arrow::Status::OK();
105
+ }
106
+
107
+ arrow::Status Visit(const arrow::Int64Array& array) override {
108
+ fill(static_cast<const arrow::Array&>(array));
109
+ view_->format = "q";
110
+ return arrow::Status::OK();
111
+ }
112
+
113
+ arrow::Status Visit(const arrow::UInt8Array& array) override {
114
+ fill(static_cast<const arrow::Array&>(array));
115
+ view_->format = "C";
116
+ return arrow::Status::OK();
117
+ }
118
+
119
+ arrow::Status Visit(const arrow::UInt16Array& array) override {
120
+ fill(static_cast<const arrow::Array&>(array));
121
+ view_->format = "S";
122
+ return arrow::Status::OK();
123
+ }
124
+
125
+ arrow::Status Visit(const arrow::UInt32Array& array) override {
126
+ fill(static_cast<const arrow::Array&>(array));
127
+ view_->format = "L";
128
+ return arrow::Status::OK();
129
+ }
130
+
131
+ arrow::Status Visit(const arrow::UInt64Array& array) override {
132
+ fill(static_cast<const arrow::Array&>(array));
133
+ view_->format = "Q";
134
+ return arrow::Status::OK();
135
+ }
136
+
137
+ arrow::Status Visit(const arrow::FloatArray& array) override {
138
+ fill(static_cast<const arrow::Array&>(array));
139
+ view_->format = "f";
140
+ return arrow::Status::OK();
141
+ }
142
+
143
+ arrow::Status Visit(const arrow::DoubleArray& array) override {
144
+ fill(static_cast<const arrow::Array&>(array));
145
+ view_->format = "d";
146
+ return arrow::Status::OK();
147
+ }
148
+
149
+ arrow::Status Visit(const arrow::FixedSizeBinaryArray& array) override {
150
+ fill(static_cast<const arrow::Array&>(array));
151
+ auto priv = static_cast<PrivateData *>(view_->private_data);
152
+ const auto type =
153
+ std::static_pointer_cast<const arrow::FixedSizeBinaryType>(
154
+ array.type());
155
+ std::ostringstream output;
156
+ output << "C" << type->byte_width();
157
+ priv->format = output.str();
158
+ view_->format = priv->format.c_str();
159
+ return arrow::Status::OK();
160
+ }
161
+
162
+ arrow::Status Visit(const arrow::Date32Array& array) override {
163
+ fill(static_cast<const arrow::Array&>(array));
164
+ view_->format = "l";
165
+ return arrow::Status::OK();
166
+ }
167
+
168
+ arrow::Status Visit(const arrow::Date64Array& array) override {
169
+ fill(static_cast<const arrow::Array&>(array));
170
+ view_->format = "q";
171
+ return arrow::Status::OK();
172
+ }
173
+
174
+ arrow::Status Visit(const arrow::Time32Array& array) override {
175
+ fill(static_cast<const arrow::Array&>(array));
176
+ view_->format = "l";
177
+ return arrow::Status::OK();
178
+ }
179
+
180
+ arrow::Status Visit(const arrow::Time64Array& array) override {
181
+ fill(static_cast<const arrow::Array&>(array));
182
+ view_->format = "q";
183
+ return arrow::Status::OK();
184
+ }
185
+
186
+ arrow::Status Visit(const arrow::TimestampArray& array) override {
187
+ fill(static_cast<const arrow::Array&>(array));
188
+ view_->format = "q";
189
+ return arrow::Status::OK();
190
+ }
191
+
192
+ arrow::Status Visit(const arrow::Decimal128Array& array) override {
193
+ fill(static_cast<const arrow::Array&>(array));
194
+ view_->format = "q2";
195
+ return arrow::Status::OK();
196
+ }
197
+
198
+ arrow::Status Visit(const arrow::Decimal256Array& array) override {
199
+ fill(static_cast<const arrow::Array&>(array));
200
+ view_->format = "q4";
201
+ return arrow::Status::OK();
202
+ }
203
+
204
+ private:
205
+ void fill(const arrow::Array& array) {
206
+ const auto array_data = array.data();
207
+ const auto data = array_data->GetValuesSafe<uint8_t>(1);
208
+ view_->data = const_cast<void *>(reinterpret_cast<const void *>(data));
209
+ const auto type =
210
+ std::static_pointer_cast<const arrow::FixedWidthType>(array.type());
211
+ view_->item_size = type->bit_width() / 8;
212
+ view_->byte_size = view_->item_size * array.length();
213
+ }
214
+
215
+ memory_view *view_;
216
+ };
217
+
218
+ bool primitive_array_get(VALUE obj, rb_memory_view_t *view, int flags) {
219
+ if (flags != RUBY_MEMORY_VIEW_SIMPLE) {
220
+ return false;
221
+ }
222
+ auto view_ = reinterpret_cast<memory_view *>(view);
223
+ view_->obj = obj;
224
+ view_->private_data = new PrivateData();
225
+ auto array = GARROW_ARRAY(RVAL2GOBJ(obj));
226
+ auto arrow_array = garrow_array_get_raw(array);
227
+ PrimitiveArrayGetter getter(view_);
228
+ auto status = arrow_array->Accept(&getter);
229
+ if (!status.ok()) {
230
+ return false;
231
+ }
232
+ view_->readonly = true;
233
+ view_->ndim = 1;
234
+ view_->shape = NULL;
235
+ view_->strides = NULL;
236
+ view_->sub_offsets = NULL;
237
+ return true;
238
+ }
239
+
240
+ bool primitive_array_release(VALUE obj, rb_memory_view_t *view) {
241
+ auto view_ = reinterpret_cast<memory_view *>(view);
242
+ delete static_cast<PrivateData *>(view_->private_data);
243
+ return true;
244
+ }
245
+
246
+ bool primitive_array_available_p(VALUE obj) {
247
+ return true;
248
+ }
249
+
250
+ rb_memory_view_entry_t primitive_array_entry = {
251
+ primitive_array_get,
252
+ primitive_array_release,
253
+ primitive_array_available_p,
254
+ };
255
+
256
+ bool buffer_get(VALUE obj, rb_memory_view_t *view, int flags) {
257
+ if (flags != RUBY_MEMORY_VIEW_SIMPLE) {
258
+ return false;
259
+ }
260
+ auto view_ = reinterpret_cast<memory_view *>(view);
261
+ view_->obj = obj;
262
+ auto buffer = GARROW_BUFFER(RVAL2GOBJ(obj));
263
+ auto arrow_buffer = garrow_buffer_get_raw(buffer);
264
+ view_->data =
265
+ const_cast<void *>(reinterpret_cast<const void *>(arrow_buffer->data()));
266
+ // Memory view doesn't support bit stream. We use one byte
267
+ // for 8 elements. Users can't calculate the number of
268
+ // elements from memory view but it's limitation of memory view.
269
+ #ifdef ARROW_LITTLE_ENDIAN
270
+ view_->format = "b8";
271
+ #else
272
+ view_->format = "B8";
273
+ #endif
274
+ view_->item_size = 1;
275
+ view_->byte_size = arrow_buffer->size();
276
+ view_->readonly = true;
277
+ view_->ndim = 1;
278
+ view_->shape = NULL;
279
+ view_->strides = NULL;
280
+ view_->sub_offsets = NULL;
281
+ return true;
282
+ }
283
+
284
+ bool buffer_release(VALUE obj, rb_memory_view_t *view) {
285
+ return true;
286
+ }
287
+
288
+ bool buffer_available_p(VALUE obj) {
289
+ return true;
290
+ }
291
+
292
+ rb_memory_view_entry_t buffer_entry = {
293
+ buffer_get,
294
+ buffer_release,
295
+ buffer_available_p,
296
+ };
297
+ #endif
298
+
299
+ void init(VALUE mArrow) {
300
+ #ifdef HAVE_MEMORY_VIEW
301
+ auto cPrimitiveArray =
302
+ rb_const_get_at(mArrow, rb_intern("PrimitiveArray"));
303
+ rb_memory_view_register(cPrimitiveArray,
304
+ &(red_arrow::memory_view::primitive_array_entry));
305
+
306
+ auto cBuffer = rb_const_get_at(mArrow, rb_intern("Buffer"));
307
+ rb_memory_view_register(cBuffer, &(red_arrow::memory_view::buffer_entry));
308
+ #endif
309
+ }
310
+ }
311
+ }
@@ -0,0 +1,26 @@
1
+ /*
2
+ * Licensed to the Apache Software Foundation (ASF) under one
3
+ * or more contributor license agreements. See the NOTICE file
4
+ * distributed with this work for additional information
5
+ * regarding copyright ownership. The ASF licenses this file
6
+ * to you under the Apache License, Version 2.0 (the
7
+ * "License"); you may not use this file except in compliance
8
+ * with the License. You may obtain a copy of the License at
9
+ *
10
+ * http://www.apache.org/licenses/LICENSE-2.0
11
+ *
12
+ * Unless required by applicable law or agreed to in writing,
13
+ * software distributed under the License is distributed on an
14
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
15
+ * KIND, either express or implied. See the License for the
16
+ * specific language governing permissions and limitations
17
+ * under the License.
18
+ */
19
+
20
+ #include <ruby.hpp>
21
+
22
+ namespace red_arrow {
23
+ namespace memory_view {
24
+ void init(VALUE mArrow);
25
+ }
26
+ }
@@ -100,6 +100,7 @@ namespace red_arrow {
100
100
  // VISIT(Interval)
101
101
  VISIT(List)
102
102
  VISIT(Struct)
103
+ VISIT(Map)
103
104
  VISIT(SparseUnion)
104
105
  VISIT(DenseUnion)
105
106
  VISIT(Dictionary)
data/ext/arrow/values.cpp CHANGED
@@ -81,6 +81,7 @@ namespace red_arrow {
81
81
  // VISIT(Interval)
82
82
  VISIT(List)
83
83
  VISIT(Struct)
84
+ VISIT(Map)
84
85
  VISIT(SparseUnion)
85
86
  VISIT(DenseUnion)
86
87
  VISIT(Dictionary)
@@ -0,0 +1,35 @@
1
+ # Licensed to the Apache Software Foundation (ASF) under one
2
+ # or more contributor license agreements. See the NOTICE file
3
+ # distributed with this work for additional information
4
+ # regarding copyright ownership. The ASF licenses this file
5
+ # to you under the Apache License, Version 2.0 (the
6
+ # "License"); you may not use this file except in compliance
7
+ # with the License. You may obtain a copy of the License at
8
+ #
9
+ # http://www.apache.org/licenses/LICENSE-2.0
10
+ #
11
+ # Unless required by applicable law or agreed to in writing,
12
+ # software distributed under the License is distributed on an
13
+ # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14
+ # KIND, either express or implied. See the License for the
15
+ # specific language governing permissions and limitations
16
+ # under the License.
17
+
18
+ module Arrow
19
+ class AggregateNodeOptions
20
+ class << self
21
+ # @api private
22
+ def try_convert(value)
23
+ case value
24
+ when Hash
25
+ aggregations = value[:aggregations]
26
+ return nil if aggregations.nil?
27
+ keys = value[:keys]
28
+ new(aggregations, keys)
29
+ else
30
+ nil
31
+ end
32
+ end
33
+ end
34
+ end
35
+ end