red-arrow-format 23.0.1 → 25.0.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/Gemfile +7 -2
- data/Rakefile +23 -1
- data/lib/arrow-format/array.rb +486 -117
- data/lib/arrow-format/bitmap.rb +25 -6
- data/lib/arrow-format/buffer-alignable.rb +35 -0
- data/lib/arrow-format/dictionary.rb +29 -0
- data/lib/arrow-format/error.rb +1 -1
- data/lib/arrow-format/field.rb +24 -3
- data/lib/arrow-format/file-reader.rb +47 -25
- data/lib/arrow-format/file-writer.rb +56 -0
- data/lib/arrow-format/flatbuffers.rb +67 -0
- data/lib/arrow-format/org/apache/arrow/flatbuf/binary.rb +5 -1
- data/lib/arrow-format/org/apache/arrow/flatbuf/binary_view.rb +5 -1
- data/lib/arrow-format/org/apache/arrow/flatbuf/block.rb +10 -2
- data/lib/arrow-format/org/apache/arrow/flatbuf/body_compression.rb +8 -1
- data/lib/arrow-format/org/apache/arrow/flatbuf/body_compression_method.rb +3 -1
- data/lib/arrow-format/org/apache/arrow/flatbuf/bool.rb +5 -1
- data/lib/arrow-format/org/apache/arrow/flatbuf/buffer.rb +8 -1
- data/lib/arrow-format/org/apache/arrow/flatbuf/compression_type.rb +1 -1
- data/lib/arrow-format/org/apache/arrow/flatbuf/date.rb +7 -1
- data/lib/arrow-format/org/apache/arrow/flatbuf/date_unit.rb +1 -1
- data/lib/arrow-format/org/apache/arrow/flatbuf/decimal.rb +9 -1
- data/lib/arrow-format/org/apache/arrow/flatbuf/dictionary_batch.rb +9 -1
- data/lib/arrow-format/org/apache/arrow/flatbuf/dictionary_encoding.rb +11 -2
- data/lib/arrow-format/org/apache/arrow/flatbuf/dictionary_kind.rb +1 -1
- data/lib/arrow-format/org/apache/arrow/flatbuf/duration.rb +7 -1
- data/lib/arrow-format/org/apache/arrow/flatbuf/endianness.rb +1 -1
- data/lib/arrow-format/org/apache/arrow/flatbuf/feature.rb +1 -1
- data/lib/arrow-format/org/apache/arrow/flatbuf/field.rb +13 -1
- data/lib/arrow-format/org/apache/arrow/flatbuf/field_node.rb +8 -1
- data/lib/arrow-format/org/apache/arrow/flatbuf/fixed_size_binary.rb +7 -1
- data/lib/arrow-format/org/apache/arrow/flatbuf/fixed_size_list.rb +7 -1
- data/lib/arrow-format/org/apache/arrow/flatbuf/floating_point.rb +7 -1
- data/lib/arrow-format/org/apache/arrow/flatbuf/footer.rb +22 -2
- data/lib/arrow-format/org/apache/arrow/flatbuf/int.rb +8 -1
- data/lib/arrow-format/org/apache/arrow/flatbuf/interval.rb +7 -1
- data/lib/arrow-format/org/apache/arrow/flatbuf/interval_unit.rb +1 -1
- data/lib/arrow-format/org/apache/arrow/flatbuf/key_value.rb +8 -1
- data/lib/arrow-format/org/apache/arrow/flatbuf/large_binary.rb +5 -1
- data/lib/arrow-format/org/apache/arrow/flatbuf/large_list.rb +5 -1
- data/lib/arrow-format/org/apache/arrow/flatbuf/large_list_view.rb +5 -1
- data/lib/arrow-format/org/apache/arrow/flatbuf/large_utf8.rb +5 -1
- data/lib/arrow-format/org/apache/arrow/flatbuf/list.rb +5 -1
- data/lib/arrow-format/org/apache/arrow/flatbuf/list_view.rb +5 -1
- data/lib/arrow-format/org/apache/arrow/flatbuf/map.rb +8 -2
- data/lib/arrow-format/org/apache/arrow/flatbuf/message.rb +22 -2
- data/lib/arrow-format/org/apache/arrow/flatbuf/message_header.rb +1 -1
- data/lib/arrow-format/org/apache/arrow/flatbuf/metadata_version.rb +1 -1
- data/lib/arrow-format/org/apache/arrow/flatbuf/null.rb +5 -1
- data/lib/arrow-format/org/apache/arrow/flatbuf/precision.rb +1 -1
- data/lib/arrow-format/org/apache/arrow/flatbuf/record_batch.rb +12 -2
- data/lib/arrow-format/org/apache/arrow/flatbuf/run_end_encoded.rb +5 -1
- data/lib/arrow-format/org/apache/arrow/flatbuf/schema.rb +10 -1
- data/lib/arrow-format/org/apache/arrow/flatbuf/sparse_matrix_compressed_axis.rb +1 -1
- data/lib/arrow-format/org/apache/arrow/flatbuf/sparse_matrix_index_csx.rb +12 -2
- data/lib/arrow-format/org/apache/arrow/flatbuf/sparse_tensor.rb +13 -1
- data/lib/arrow-format/org/apache/arrow/flatbuf/sparse_tensor_index.rb +1 -1
- data/lib/arrow-format/org/apache/arrow/flatbuf/sparse_tensor_index_coo.rb +13 -4
- data/lib/arrow-format/org/apache/arrow/flatbuf/sparse_tensor_index_csf.rb +14 -4
- data/lib/arrow-format/org/apache/arrow/flatbuf/struct_.rb +5 -1
- data/lib/arrow-format/org/apache/arrow/flatbuf/tensor.rb +11 -1
- data/lib/arrow-format/org/apache/arrow/flatbuf/tensor_dim.rb +8 -1
- data/lib/arrow-format/org/apache/arrow/flatbuf/time.rb +8 -1
- data/lib/arrow-format/org/apache/arrow/flatbuf/time_unit.rb +1 -1
- data/lib/arrow-format/org/apache/arrow/flatbuf/timestamp.rb +8 -1
- data/lib/arrow-format/org/apache/arrow/flatbuf/type.rb +1 -1
- data/lib/arrow-format/org/apache/arrow/flatbuf/union.rb +8 -1
- data/lib/arrow-format/org/apache/arrow/flatbuf/union_mode.rb +1 -1
- data/lib/arrow-format/org/apache/arrow/flatbuf/utf8.rb +5 -1
- data/lib/arrow-format/org/apache/arrow/flatbuf/utf8view.rb +5 -1
- data/lib/arrow-format/readable.rb +151 -96
- data/lib/arrow-format/record-batch.rb +70 -1
- data/lib/arrow-format/schema.rb +14 -1
- data/lib/arrow-format/streaming-pull-reader.rb +49 -23
- data/lib/arrow-format/streaming-reader.rb +51 -14
- data/lib/arrow-format/streaming-writer.rb +156 -0
- data/lib/arrow-format/type.rb +397 -34
- data/lib/arrow-format/version.rb +1 -1
- data/lib/arrow-format.rb +2 -0
- data/red-arrow-format.gemspec +2 -1
- metadata +9 -4
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
# Automatically generated. Don't modify manually.
|
|
2
2
|
#
|
|
3
|
-
# Red FlatBuffers version: 0.0.
|
|
3
|
+
# Red FlatBuffers version: 0.0.4
|
|
4
4
|
# Declared by: //Message.fbs
|
|
5
5
|
# Rooting type: org.apache.arrow.flatbuf.Message (//Message.fbs)
|
|
6
6
|
|
|
@@ -19,6 +19,14 @@ module ArrowFormat
|
|
|
19
19
|
# may be spread across multiple dictionary batches by using the isDelta
|
|
20
20
|
# flag
|
|
21
21
|
class DictionaryBatch < ::FlatBuffers::Table
|
|
22
|
+
FIELDS = {
|
|
23
|
+
id: ::FlatBuffers::Field.new(:id, 0, 4, :long, 0),
|
|
24
|
+
data: ::FlatBuffers::Field.new(:data, 1, 6, "::ArrowFormat::Org::Apache::Arrow::Flatbuf::RecordBatch", 0),
|
|
25
|
+
delta?: ::FlatBuffers::Field.new(:delta?, 2, 8, :bool, 0),
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
Data = define_data_class
|
|
29
|
+
|
|
22
30
|
def data
|
|
23
31
|
field_offset = @view.unpack_virtual_offset(6)
|
|
24
32
|
return nil if field_offset.zero?
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
# Automatically generated. Don't modify manually.
|
|
2
2
|
#
|
|
3
|
-
# Red FlatBuffers version: 0.0.
|
|
3
|
+
# Red FlatBuffers version: 0.0.4
|
|
4
4
|
# Declared by: //Schema.fbs
|
|
5
5
|
# Rooting type: org.apache.arrow.flatbuf.Message (//Message.fbs)
|
|
6
6
|
|
|
@@ -14,6 +14,15 @@ module ArrowFormat
|
|
|
14
14
|
module Arrow
|
|
15
15
|
module Flatbuf
|
|
16
16
|
class DictionaryEncoding < ::FlatBuffers::Table
|
|
17
|
+
FIELDS = {
|
|
18
|
+
id: ::FlatBuffers::Field.new(:id, 0, 4, :long, 0),
|
|
19
|
+
index_type: ::FlatBuffers::Field.new(:index_type, 1, 6, "::ArrowFormat::Org::Apache::Arrow::Flatbuf::Int", 0),
|
|
20
|
+
ordered?: ::FlatBuffers::Field.new(:ordered?, 2, 8, :bool, 0),
|
|
21
|
+
dictionary_kind: ::FlatBuffers::Field.new(:dictionary_kind, 3, 10, :short, 0),
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
Data = define_data_class
|
|
25
|
+
|
|
17
26
|
def dictionary_kind
|
|
18
27
|
field_offset = @view.unpack_virtual_offset(10)
|
|
19
28
|
if field_offset.zero?
|
|
@@ -47,7 +56,7 @@ module ArrowFormat
|
|
|
47
56
|
end
|
|
48
57
|
|
|
49
58
|
# By default, dictionaries are not ordered, or the order does not have
|
|
50
|
-
# semantic meaning. In some statistical
|
|
59
|
+
# semantic meaning. In some statistical applications, dictionary-encoding
|
|
51
60
|
# is used to represent ordered categorical data, and we provide a way to
|
|
52
61
|
# preserve that metadata here
|
|
53
62
|
def ordered?
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
# Automatically generated. Don't modify manually.
|
|
2
2
|
#
|
|
3
|
-
# Red FlatBuffers version: 0.0.
|
|
3
|
+
# Red FlatBuffers version: 0.0.4
|
|
4
4
|
# Declared by: //Schema.fbs
|
|
5
5
|
# Rooting type: org.apache.arrow.flatbuf.Message (//Message.fbs)
|
|
6
6
|
|
|
@@ -13,6 +13,12 @@ module ArrowFormat
|
|
|
13
13
|
module Arrow
|
|
14
14
|
module Flatbuf
|
|
15
15
|
class Duration < ::FlatBuffers::Table
|
|
16
|
+
FIELDS = {
|
|
17
|
+
unit: ::FlatBuffers::Field.new(:unit, 0, 4, :short, 0),
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
Data = define_data_class
|
|
21
|
+
|
|
16
22
|
def unit
|
|
17
23
|
field_offset = @view.unpack_virtual_offset(4)
|
|
18
24
|
if field_offset.zero?
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
# Automatically generated. Don't modify manually.
|
|
2
2
|
#
|
|
3
|
-
# Red FlatBuffers version: 0.0.
|
|
3
|
+
# Red FlatBuffers version: 0.0.4
|
|
4
4
|
# Declared by: //Schema.fbs
|
|
5
5
|
# Rooting type: org.apache.arrow.flatbuf.Message (//Message.fbs)
|
|
6
6
|
|
|
@@ -18,6 +18,18 @@ module ArrowFormat
|
|
|
18
18
|
# A field represents a named column in a record / row batch or child of a
|
|
19
19
|
# nested type.
|
|
20
20
|
class Field < ::FlatBuffers::Table
|
|
21
|
+
FIELDS = {
|
|
22
|
+
name: ::FlatBuffers::Field.new(:name, 0, 4, :string, 0),
|
|
23
|
+
nullable?: ::FlatBuffers::Field.new(:nullable?, 1, 6, :bool, 0),
|
|
24
|
+
type_type: ::FlatBuffers::Field.new(:type_type, 2, 8, :utype, 0),
|
|
25
|
+
type: ::FlatBuffers::Field.new(:type, 3, 10, "::ArrowFormat::Org::Apache::Arrow::Flatbuf::Type", 0),
|
|
26
|
+
dictionary: ::FlatBuffers::Field.new(:dictionary, 4, 12, "::ArrowFormat::Org::Apache::Arrow::Flatbuf::DictionaryEncoding", 0),
|
|
27
|
+
children: ::FlatBuffers::Field.new(:children, 5, 14, ["::ArrowFormat::Org::Apache::Arrow::Flatbuf::Field"], 0),
|
|
28
|
+
custom_metadata: ::FlatBuffers::Field.new(:custom_metadata, 6, 16, ["::ArrowFormat::Org::Apache::Arrow::Flatbuf::KeyValue"], 0),
|
|
29
|
+
}
|
|
30
|
+
|
|
31
|
+
Data = define_data_class
|
|
32
|
+
|
|
21
33
|
# children apply only to nested data types like Struct, List and Union. For
|
|
22
34
|
# primitive types children will have length 0.
|
|
23
35
|
def children
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
# Automatically generated. Don't modify manually.
|
|
2
2
|
#
|
|
3
|
-
# Red FlatBuffers version: 0.0.
|
|
3
|
+
# Red FlatBuffers version: 0.0.4
|
|
4
4
|
# Declared by: //Message.fbs
|
|
5
5
|
# Rooting type: org.apache.arrow.flatbuf.Message (//Message.fbs)
|
|
6
6
|
|
|
@@ -21,6 +21,13 @@ module ArrowFormat
|
|
|
21
21
|
# would have {length: 5, null_count: 2} for its List node, and {length: 6,
|
|
22
22
|
# null_count: 0} for its Int16 node, as separate FieldNode structs
|
|
23
23
|
class FieldNode < ::FlatBuffers::Struct
|
|
24
|
+
FIELDS = {
|
|
25
|
+
length: ::FlatBuffers::Field.new(:length, 0, 0, :long, 0),
|
|
26
|
+
null_count: ::FlatBuffers::Field.new(:null_count, 1, 8, :long, 0),
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
Data = define_data_class
|
|
30
|
+
|
|
24
31
|
# The number of value slots in the Arrow array at this level of a nested
|
|
25
32
|
# tree
|
|
26
33
|
def length
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
# Automatically generated. Don't modify manually.
|
|
2
2
|
#
|
|
3
|
-
# Red FlatBuffers version: 0.0.
|
|
3
|
+
# Red FlatBuffers version: 0.0.4
|
|
4
4
|
# Declared by: //Schema.fbs
|
|
5
5
|
# Rooting type: org.apache.arrow.flatbuf.Message (//Message.fbs)
|
|
6
6
|
|
|
@@ -12,6 +12,12 @@ module ArrowFormat
|
|
|
12
12
|
module Arrow
|
|
13
13
|
module Flatbuf
|
|
14
14
|
class FixedSizeBinary < ::FlatBuffers::Table
|
|
15
|
+
FIELDS = {
|
|
16
|
+
byte_width: ::FlatBuffers::Field.new(:byte_width, 0, 4, :int, 0),
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
Data = define_data_class
|
|
20
|
+
|
|
15
21
|
# Number of bytes per value
|
|
16
22
|
def byte_width
|
|
17
23
|
field_offset = @view.unpack_virtual_offset(4)
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
# Automatically generated. Don't modify manually.
|
|
2
2
|
#
|
|
3
|
-
# Red FlatBuffers version: 0.0.
|
|
3
|
+
# Red FlatBuffers version: 0.0.4
|
|
4
4
|
# Declared by: //Schema.fbs
|
|
5
5
|
# Rooting type: org.apache.arrow.flatbuf.Message (//Message.fbs)
|
|
6
6
|
|
|
@@ -12,6 +12,12 @@ module ArrowFormat
|
|
|
12
12
|
module Arrow
|
|
13
13
|
module Flatbuf
|
|
14
14
|
class FixedSizeList < ::FlatBuffers::Table
|
|
15
|
+
FIELDS = {
|
|
16
|
+
list_size: ::FlatBuffers::Field.new(:list_size, 0, 4, :int, 0),
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
Data = define_data_class
|
|
20
|
+
|
|
15
21
|
# Number of list items per value
|
|
16
22
|
def list_size
|
|
17
23
|
field_offset = @view.unpack_virtual_offset(4)
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
# Automatically generated. Don't modify manually.
|
|
2
2
|
#
|
|
3
|
-
# Red FlatBuffers version: 0.0.
|
|
3
|
+
# Red FlatBuffers version: 0.0.4
|
|
4
4
|
# Declared by: //Schema.fbs
|
|
5
5
|
# Rooting type: org.apache.arrow.flatbuf.Message (//Message.fbs)
|
|
6
6
|
|
|
@@ -13,6 +13,12 @@ module ArrowFormat
|
|
|
13
13
|
module Arrow
|
|
14
14
|
module Flatbuf
|
|
15
15
|
class FloatingPoint < ::FlatBuffers::Table
|
|
16
|
+
FIELDS = {
|
|
17
|
+
precision: ::FlatBuffers::Field.new(:precision, 0, 4, :short, 0),
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
Data = define_data_class
|
|
21
|
+
|
|
16
22
|
def precision
|
|
17
23
|
field_offset = @view.unpack_virtual_offset(4)
|
|
18
24
|
if field_offset.zero?
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
# Automatically generated. Don't modify manually.
|
|
2
2
|
#
|
|
3
|
-
# Red FlatBuffers version: 0.0.
|
|
3
|
+
# Red FlatBuffers version: 0.0.4
|
|
4
4
|
# Declared by: //File.fbs
|
|
5
5
|
# Rooting type: org.apache.arrow.flatbuf.Footer (//File.fbs)
|
|
6
6
|
|
|
@@ -18,7 +18,27 @@ module ArrowFormat
|
|
|
18
18
|
# ----------------------------------------------------------------------
|
|
19
19
|
# Arrow File metadata
|
|
20
20
|
#
|
|
21
|
-
class Footer < ::FlatBuffers::
|
|
21
|
+
class Footer < ::FlatBuffers::RootTable
|
|
22
|
+
class << self
|
|
23
|
+
def file_identifier
|
|
24
|
+
""
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
def file_extension
|
|
28
|
+
""
|
|
29
|
+
end
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
FIELDS = {
|
|
33
|
+
version: ::FlatBuffers::Field.new(:version, 0, 4, :short, 0),
|
|
34
|
+
schema: ::FlatBuffers::Field.new(:schema, 1, 6, "::ArrowFormat::Org::Apache::Arrow::Flatbuf::Schema", 0),
|
|
35
|
+
dictionaries: ::FlatBuffers::Field.new(:dictionaries, 2, 8, ["::ArrowFormat::Org::Apache::Arrow::Flatbuf::Block"], 0),
|
|
36
|
+
record_batches: ::FlatBuffers::Field.new(:record_batches, 3, 10, ["::ArrowFormat::Org::Apache::Arrow::Flatbuf::Block"], 0),
|
|
37
|
+
custom_metadata: ::FlatBuffers::Field.new(:custom_metadata, 4, 12, ["::ArrowFormat::Org::Apache::Arrow::Flatbuf::KeyValue"], 0),
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
Data = define_data_class
|
|
41
|
+
|
|
22
42
|
# User-defined metadata
|
|
23
43
|
def custom_metadata
|
|
24
44
|
field_offset = @view.unpack_virtual_offset(12)
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
# Automatically generated. Don't modify manually.
|
|
2
2
|
#
|
|
3
|
-
# Red FlatBuffers version: 0.0.
|
|
3
|
+
# Red FlatBuffers version: 0.0.4
|
|
4
4
|
# Declared by: //Schema.fbs
|
|
5
5
|
# Rooting type: org.apache.arrow.flatbuf.Message (//Message.fbs)
|
|
6
6
|
|
|
@@ -12,6 +12,13 @@ module ArrowFormat
|
|
|
12
12
|
module Arrow
|
|
13
13
|
module Flatbuf
|
|
14
14
|
class Int < ::FlatBuffers::Table
|
|
15
|
+
FIELDS = {
|
|
16
|
+
bit_width: ::FlatBuffers::Field.new(:bit_width, 0, 4, :int, 0),
|
|
17
|
+
signed?: ::FlatBuffers::Field.new(:signed?, 1, 6, :bool, 0),
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
Data = define_data_class
|
|
21
|
+
|
|
15
22
|
def bit_width
|
|
16
23
|
field_offset = @view.unpack_virtual_offset(4)
|
|
17
24
|
return 0 if field_offset.zero?
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
# Automatically generated. Don't modify manually.
|
|
2
2
|
#
|
|
3
|
-
# Red FlatBuffers version: 0.0.
|
|
3
|
+
# Red FlatBuffers version: 0.0.4
|
|
4
4
|
# Declared by: //Schema.fbs
|
|
5
5
|
# Rooting type: org.apache.arrow.flatbuf.Message (//Message.fbs)
|
|
6
6
|
|
|
@@ -13,6 +13,12 @@ module ArrowFormat
|
|
|
13
13
|
module Arrow
|
|
14
14
|
module Flatbuf
|
|
15
15
|
class Interval < ::FlatBuffers::Table
|
|
16
|
+
FIELDS = {
|
|
17
|
+
unit: ::FlatBuffers::Field.new(:unit, 0, 4, :short, 0),
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
Data = define_data_class
|
|
21
|
+
|
|
16
22
|
def unit
|
|
17
23
|
field_offset = @view.unpack_virtual_offset(4)
|
|
18
24
|
if field_offset.zero?
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
# Automatically generated. Don't modify manually.
|
|
2
2
|
#
|
|
3
|
-
# Red FlatBuffers version: 0.0.
|
|
3
|
+
# Red FlatBuffers version: 0.0.4
|
|
4
4
|
# Declared by: //Schema.fbs
|
|
5
5
|
# Rooting type: org.apache.arrow.flatbuf.Message (//Message.fbs)
|
|
6
6
|
|
|
@@ -15,6 +15,13 @@ module ArrowFormat
|
|
|
15
15
|
# user defined key value pairs to add custom metadata to arrow
|
|
16
16
|
# key namespacing is the responsibility of the user
|
|
17
17
|
class KeyValue < ::FlatBuffers::Table
|
|
18
|
+
FIELDS = {
|
|
19
|
+
key: ::FlatBuffers::Field.new(:key, 0, 4, :string, 0),
|
|
20
|
+
value: ::FlatBuffers::Field.new(:value, 1, 6, :string, 0),
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
Data = define_data_class
|
|
24
|
+
|
|
18
25
|
def key
|
|
19
26
|
field_offset = @view.unpack_virtual_offset(4)
|
|
20
27
|
return nil if field_offset.zero?
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
# Automatically generated. Don't modify manually.
|
|
2
2
|
#
|
|
3
|
-
# Red FlatBuffers version: 0.0.
|
|
3
|
+
# Red FlatBuffers version: 0.0.4
|
|
4
4
|
# Declared by: //Schema.fbs
|
|
5
5
|
# Rooting type: org.apache.arrow.flatbuf.Message (//Message.fbs)
|
|
6
6
|
|
|
@@ -14,6 +14,10 @@ module ArrowFormat
|
|
|
14
14
|
# Same as Binary, but with 64-bit offsets, allowing to represent
|
|
15
15
|
# extremely large data values.
|
|
16
16
|
class LargeBinary < ::FlatBuffers::Table
|
|
17
|
+
FIELDS = {
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
Data = define_data_class
|
|
17
21
|
end
|
|
18
22
|
end
|
|
19
23
|
end
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
# Automatically generated. Don't modify manually.
|
|
2
2
|
#
|
|
3
|
-
# Red FlatBuffers version: 0.0.
|
|
3
|
+
# Red FlatBuffers version: 0.0.4
|
|
4
4
|
# Declared by: //Schema.fbs
|
|
5
5
|
# Rooting type: org.apache.arrow.flatbuf.Message (//Message.fbs)
|
|
6
6
|
|
|
@@ -14,6 +14,10 @@ module ArrowFormat
|
|
|
14
14
|
# Same as List, but with 64-bit offsets, allowing to represent
|
|
15
15
|
# extremely large data values.
|
|
16
16
|
class LargeList < ::FlatBuffers::Table
|
|
17
|
+
FIELDS = {
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
Data = define_data_class
|
|
17
21
|
end
|
|
18
22
|
end
|
|
19
23
|
end
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
# Automatically generated. Don't modify manually.
|
|
2
2
|
#
|
|
3
|
-
# Red FlatBuffers version: 0.0.
|
|
3
|
+
# Red FlatBuffers version: 0.0.4
|
|
4
4
|
# Declared by: //Schema.fbs
|
|
5
5
|
# Rooting type: org.apache.arrow.flatbuf.Message (//Message.fbs)
|
|
6
6
|
|
|
@@ -14,6 +14,10 @@ module ArrowFormat
|
|
|
14
14
|
# Same as ListView, but with 64-bit offsets and sizes, allowing to represent
|
|
15
15
|
# extremely large data values.
|
|
16
16
|
class LargeListView < ::FlatBuffers::Table
|
|
17
|
+
FIELDS = {
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
Data = define_data_class
|
|
17
21
|
end
|
|
18
22
|
end
|
|
19
23
|
end
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
# Automatically generated. Don't modify manually.
|
|
2
2
|
#
|
|
3
|
-
# Red FlatBuffers version: 0.0.
|
|
3
|
+
# Red FlatBuffers version: 0.0.4
|
|
4
4
|
# Declared by: //Schema.fbs
|
|
5
5
|
# Rooting type: org.apache.arrow.flatbuf.Message (//Message.fbs)
|
|
6
6
|
|
|
@@ -14,6 +14,10 @@ module ArrowFormat
|
|
|
14
14
|
# Same as Utf8, but with 64-bit offsets, allowing to represent
|
|
15
15
|
# extremely large data values.
|
|
16
16
|
class LargeUtf8 < ::FlatBuffers::Table
|
|
17
|
+
FIELDS = {
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
Data = define_data_class
|
|
17
21
|
end
|
|
18
22
|
end
|
|
19
23
|
end
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
# Automatically generated. Don't modify manually.
|
|
2
2
|
#
|
|
3
|
-
# Red FlatBuffers version: 0.0.
|
|
3
|
+
# Red FlatBuffers version: 0.0.4
|
|
4
4
|
# Declared by: //Schema.fbs
|
|
5
5
|
# Rooting type: org.apache.arrow.flatbuf.Message (//Message.fbs)
|
|
6
6
|
|
|
@@ -12,6 +12,10 @@ module ArrowFormat
|
|
|
12
12
|
module Arrow
|
|
13
13
|
module Flatbuf
|
|
14
14
|
class List < ::FlatBuffers::Table
|
|
15
|
+
FIELDS = {
|
|
16
|
+
}
|
|
17
|
+
|
|
18
|
+
Data = define_data_class
|
|
15
19
|
end
|
|
16
20
|
end
|
|
17
21
|
end
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
# Automatically generated. Don't modify manually.
|
|
2
2
|
#
|
|
3
|
-
# Red FlatBuffers version: 0.0.
|
|
3
|
+
# Red FlatBuffers version: 0.0.4
|
|
4
4
|
# Declared by: //Schema.fbs
|
|
5
5
|
# Rooting type: org.apache.arrow.flatbuf.Message (//Message.fbs)
|
|
6
6
|
|
|
@@ -15,6 +15,10 @@ module ArrowFormat
|
|
|
15
15
|
# sizes allowing for writes in any order and sharing of child values among
|
|
16
16
|
# list values.
|
|
17
17
|
class ListView < ::FlatBuffers::Table
|
|
18
|
+
FIELDS = {
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
Data = define_data_class
|
|
18
22
|
end
|
|
19
23
|
end
|
|
20
24
|
end
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
# Automatically generated. Don't modify manually.
|
|
2
2
|
#
|
|
3
|
-
# Red FlatBuffers version: 0.0.
|
|
3
|
+
# Red FlatBuffers version: 0.0.4
|
|
4
4
|
# Declared by: //Schema.fbs
|
|
5
5
|
# Rooting type: org.apache.arrow.flatbuf.Message (//Message.fbs)
|
|
6
6
|
|
|
@@ -21,7 +21,7 @@ module ArrowFormat
|
|
|
21
21
|
# may be set in the metadata for this field.
|
|
22
22
|
#
|
|
23
23
|
# In a field with Map type, the field has a child Struct field, which then
|
|
24
|
-
# has two children: key type and the
|
|
24
|
+
# has two children: the key type and the value type. The names of the
|
|
25
25
|
# child fields may be respectively "entries", "key", and "value", but this is
|
|
26
26
|
# not enforced.
|
|
27
27
|
#
|
|
@@ -37,6 +37,12 @@ module ArrowFormat
|
|
|
37
37
|
# for Map can make Map an alias for List. The "layout" attribute for the Map
|
|
38
38
|
# field must have the same contents as a List.
|
|
39
39
|
class Map < ::FlatBuffers::Table
|
|
40
|
+
FIELDS = {
|
|
41
|
+
keys_sorted?: ::FlatBuffers::Field.new(:keys_sorted?, 0, 4, :bool, 0),
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
Data = define_data_class
|
|
45
|
+
|
|
40
46
|
# Set to true if the keys within each value are sorted
|
|
41
47
|
def keys_sorted?
|
|
42
48
|
field_offset = @view.unpack_virtual_offset(4)
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
# Automatically generated. Don't modify manually.
|
|
2
2
|
#
|
|
3
|
-
# Red FlatBuffers version: 0.0.
|
|
3
|
+
# Red FlatBuffers version: 0.0.4
|
|
4
4
|
# Declared by: //Message.fbs
|
|
5
5
|
# Rooting type: org.apache.arrow.flatbuf.Message (//Message.fbs)
|
|
6
6
|
|
|
@@ -14,7 +14,27 @@ module ArrowFormat
|
|
|
14
14
|
module Apache
|
|
15
15
|
module Arrow
|
|
16
16
|
module Flatbuf
|
|
17
|
-
class Message < ::FlatBuffers::
|
|
17
|
+
class Message < ::FlatBuffers::RootTable
|
|
18
|
+
class << self
|
|
19
|
+
def file_identifier
|
|
20
|
+
""
|
|
21
|
+
end
|
|
22
|
+
|
|
23
|
+
def file_extension
|
|
24
|
+
""
|
|
25
|
+
end
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
FIELDS = {
|
|
29
|
+
version: ::FlatBuffers::Field.new(:version, 0, 4, :short, 0),
|
|
30
|
+
header_type: ::FlatBuffers::Field.new(:header_type, 1, 6, :utype, 0),
|
|
31
|
+
header: ::FlatBuffers::Field.new(:header, 2, 8, "::ArrowFormat::Org::Apache::Arrow::Flatbuf::MessageHeader", 0),
|
|
32
|
+
body_length: ::FlatBuffers::Field.new(:body_length, 3, 10, :long, 0),
|
|
33
|
+
custom_metadata: ::FlatBuffers::Field.new(:custom_metadata, 4, 12, ["::ArrowFormat::Org::Apache::Arrow::Flatbuf::KeyValue"], 0),
|
|
34
|
+
}
|
|
35
|
+
|
|
36
|
+
Data = define_data_class
|
|
37
|
+
|
|
18
38
|
def body_length
|
|
19
39
|
field_offset = @view.unpack_virtual_offset(10)
|
|
20
40
|
return 0 if field_offset.zero?
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
# Automatically generated. Don't modify manually.
|
|
2
2
|
#
|
|
3
|
-
# Red FlatBuffers version: 0.0.
|
|
3
|
+
# Red FlatBuffers version: 0.0.4
|
|
4
4
|
# Declared by: //Schema.fbs
|
|
5
5
|
# Rooting type: org.apache.arrow.flatbuf.Message (//Message.fbs)
|
|
6
6
|
|
|
@@ -13,6 +13,10 @@ module ArrowFormat
|
|
|
13
13
|
module Flatbuf
|
|
14
14
|
# These are stored in the flatbuffer in the Type union below
|
|
15
15
|
class Null < ::FlatBuffers::Table
|
|
16
|
+
FIELDS = {
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
Data = define_data_class
|
|
16
20
|
end
|
|
17
21
|
end
|
|
18
22
|
end
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
# Automatically generated. Don't modify manually.
|
|
2
2
|
#
|
|
3
|
-
# Red FlatBuffers version: 0.0.
|
|
3
|
+
# Red FlatBuffers version: 0.0.4
|
|
4
4
|
# Declared by: //Message.fbs
|
|
5
5
|
# Rooting type: org.apache.arrow.flatbuf.Message (//Message.fbs)
|
|
6
6
|
|
|
@@ -18,6 +18,16 @@ module ArrowFormat
|
|
|
18
18
|
# batch. Some systems call this a "row batch" internally and others a "record
|
|
19
19
|
# batch".
|
|
20
20
|
class RecordBatch < ::FlatBuffers::Table
|
|
21
|
+
FIELDS = {
|
|
22
|
+
length: ::FlatBuffers::Field.new(:length, 0, 4, :long, 0),
|
|
23
|
+
nodes: ::FlatBuffers::Field.new(:nodes, 1, 6, ["::ArrowFormat::Org::Apache::Arrow::Flatbuf::FieldNode"], 0),
|
|
24
|
+
buffers: ::FlatBuffers::Field.new(:buffers, 2, 8, ["::ArrowFormat::Org::Apache::Arrow::Flatbuf::Buffer"], 0),
|
|
25
|
+
compression: ::FlatBuffers::Field.new(:compression, 3, 10, "::ArrowFormat::Org::Apache::Arrow::Flatbuf::BodyCompression", 0),
|
|
26
|
+
variadic_buffer_counts: ::FlatBuffers::Field.new(:variadic_buffer_counts, 4, 12, [:long], 0),
|
|
27
|
+
}
|
|
28
|
+
|
|
29
|
+
Data = define_data_class
|
|
30
|
+
|
|
21
31
|
# Buffers correspond to the pre-ordered flattened buffer tree
|
|
22
32
|
#
|
|
23
33
|
# The number of buffers appended to this list depends on the schema. For
|
|
@@ -64,7 +74,7 @@ module ArrowFormat
|
|
|
64
74
|
|
|
65
75
|
# Some types such as Utf8View are represented using a variable number of buffers.
|
|
66
76
|
# For each such Field in the pre-ordered flattened logical schema, there will be
|
|
67
|
-
# an entry in variadicBufferCounts to indicate the number of
|
|
77
|
+
# an entry in variadicBufferCounts to indicate the number of variadic
|
|
68
78
|
# buffers which belong to that Field in the current RecordBatch.
|
|
69
79
|
#
|
|
70
80
|
# For example, the schema
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
# Automatically generated. Don't modify manually.
|
|
2
2
|
#
|
|
3
|
-
# Red FlatBuffers version: 0.0.
|
|
3
|
+
# Red FlatBuffers version: 0.0.4
|
|
4
4
|
# Declared by: //Schema.fbs
|
|
5
5
|
# Rooting type: org.apache.arrow.flatbuf.Message (//Message.fbs)
|
|
6
6
|
|
|
@@ -17,6 +17,10 @@ module ArrowFormat
|
|
|
17
17
|
# each corresponding index in the values child array ends.
|
|
18
18
|
# Like list/struct types, the value array can be of any type.
|
|
19
19
|
class RunEndEncoded < ::FlatBuffers::Table
|
|
20
|
+
FIELDS = {
|
|
21
|
+
}
|
|
22
|
+
|
|
23
|
+
Data = define_data_class
|
|
20
24
|
end
|
|
21
25
|
end
|
|
22
26
|
end
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
# Automatically generated. Don't modify manually.
|
|
2
2
|
#
|
|
3
|
-
# Red FlatBuffers version: 0.0.
|
|
3
|
+
# Red FlatBuffers version: 0.0.4
|
|
4
4
|
# Declared by: //Schema.fbs
|
|
5
5
|
# Rooting type: org.apache.arrow.flatbuf.Message (//Message.fbs)
|
|
6
6
|
|
|
@@ -17,6 +17,15 @@ module ArrowFormat
|
|
|
17
17
|
# ----------------------------------------------------------------------
|
|
18
18
|
# A Schema describes the columns in a row batch
|
|
19
19
|
class Schema < ::FlatBuffers::Table
|
|
20
|
+
FIELDS = {
|
|
21
|
+
endianness: ::FlatBuffers::Field.new(:endianness, 0, 4, :short, 0),
|
|
22
|
+
fields: ::FlatBuffers::Field.new(:fields, 1, 6, ["::ArrowFormat::Org::Apache::Arrow::Flatbuf::Field"], 0),
|
|
23
|
+
custom_metadata: ::FlatBuffers::Field.new(:custom_metadata, 2, 8, ["::ArrowFormat::Org::Apache::Arrow::Flatbuf::KeyValue"], 0),
|
|
24
|
+
features: ::FlatBuffers::Field.new(:features, 3, 10, [:long], 0),
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
Data = define_data_class
|
|
28
|
+
|
|
20
29
|
def custom_metadata
|
|
21
30
|
field_offset = @view.unpack_virtual_offset(8)
|
|
22
31
|
return nil if field_offset.zero?
|