red-arrow-format 23.0.1 → 25.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (82) hide show
  1. checksums.yaml +4 -4
  2. data/Gemfile +7 -2
  3. data/Rakefile +23 -1
  4. data/lib/arrow-format/array.rb +486 -117
  5. data/lib/arrow-format/bitmap.rb +25 -6
  6. data/lib/arrow-format/buffer-alignable.rb +35 -0
  7. data/lib/arrow-format/dictionary.rb +29 -0
  8. data/lib/arrow-format/error.rb +1 -1
  9. data/lib/arrow-format/field.rb +24 -3
  10. data/lib/arrow-format/file-reader.rb +47 -25
  11. data/lib/arrow-format/file-writer.rb +56 -0
  12. data/lib/arrow-format/flatbuffers.rb +67 -0
  13. data/lib/arrow-format/org/apache/arrow/flatbuf/binary.rb +5 -1
  14. data/lib/arrow-format/org/apache/arrow/flatbuf/binary_view.rb +5 -1
  15. data/lib/arrow-format/org/apache/arrow/flatbuf/block.rb +10 -2
  16. data/lib/arrow-format/org/apache/arrow/flatbuf/body_compression.rb +8 -1
  17. data/lib/arrow-format/org/apache/arrow/flatbuf/body_compression_method.rb +3 -1
  18. data/lib/arrow-format/org/apache/arrow/flatbuf/bool.rb +5 -1
  19. data/lib/arrow-format/org/apache/arrow/flatbuf/buffer.rb +8 -1
  20. data/lib/arrow-format/org/apache/arrow/flatbuf/compression_type.rb +1 -1
  21. data/lib/arrow-format/org/apache/arrow/flatbuf/date.rb +7 -1
  22. data/lib/arrow-format/org/apache/arrow/flatbuf/date_unit.rb +1 -1
  23. data/lib/arrow-format/org/apache/arrow/flatbuf/decimal.rb +9 -1
  24. data/lib/arrow-format/org/apache/arrow/flatbuf/dictionary_batch.rb +9 -1
  25. data/lib/arrow-format/org/apache/arrow/flatbuf/dictionary_encoding.rb +11 -2
  26. data/lib/arrow-format/org/apache/arrow/flatbuf/dictionary_kind.rb +1 -1
  27. data/lib/arrow-format/org/apache/arrow/flatbuf/duration.rb +7 -1
  28. data/lib/arrow-format/org/apache/arrow/flatbuf/endianness.rb +1 -1
  29. data/lib/arrow-format/org/apache/arrow/flatbuf/feature.rb +1 -1
  30. data/lib/arrow-format/org/apache/arrow/flatbuf/field.rb +13 -1
  31. data/lib/arrow-format/org/apache/arrow/flatbuf/field_node.rb +8 -1
  32. data/lib/arrow-format/org/apache/arrow/flatbuf/fixed_size_binary.rb +7 -1
  33. data/lib/arrow-format/org/apache/arrow/flatbuf/fixed_size_list.rb +7 -1
  34. data/lib/arrow-format/org/apache/arrow/flatbuf/floating_point.rb +7 -1
  35. data/lib/arrow-format/org/apache/arrow/flatbuf/footer.rb +22 -2
  36. data/lib/arrow-format/org/apache/arrow/flatbuf/int.rb +8 -1
  37. data/lib/arrow-format/org/apache/arrow/flatbuf/interval.rb +7 -1
  38. data/lib/arrow-format/org/apache/arrow/flatbuf/interval_unit.rb +1 -1
  39. data/lib/arrow-format/org/apache/arrow/flatbuf/key_value.rb +8 -1
  40. data/lib/arrow-format/org/apache/arrow/flatbuf/large_binary.rb +5 -1
  41. data/lib/arrow-format/org/apache/arrow/flatbuf/large_list.rb +5 -1
  42. data/lib/arrow-format/org/apache/arrow/flatbuf/large_list_view.rb +5 -1
  43. data/lib/arrow-format/org/apache/arrow/flatbuf/large_utf8.rb +5 -1
  44. data/lib/arrow-format/org/apache/arrow/flatbuf/list.rb +5 -1
  45. data/lib/arrow-format/org/apache/arrow/flatbuf/list_view.rb +5 -1
  46. data/lib/arrow-format/org/apache/arrow/flatbuf/map.rb +8 -2
  47. data/lib/arrow-format/org/apache/arrow/flatbuf/message.rb +22 -2
  48. data/lib/arrow-format/org/apache/arrow/flatbuf/message_header.rb +1 -1
  49. data/lib/arrow-format/org/apache/arrow/flatbuf/metadata_version.rb +1 -1
  50. data/lib/arrow-format/org/apache/arrow/flatbuf/null.rb +5 -1
  51. data/lib/arrow-format/org/apache/arrow/flatbuf/precision.rb +1 -1
  52. data/lib/arrow-format/org/apache/arrow/flatbuf/record_batch.rb +12 -2
  53. data/lib/arrow-format/org/apache/arrow/flatbuf/run_end_encoded.rb +5 -1
  54. data/lib/arrow-format/org/apache/arrow/flatbuf/schema.rb +10 -1
  55. data/lib/arrow-format/org/apache/arrow/flatbuf/sparse_matrix_compressed_axis.rb +1 -1
  56. data/lib/arrow-format/org/apache/arrow/flatbuf/sparse_matrix_index_csx.rb +12 -2
  57. data/lib/arrow-format/org/apache/arrow/flatbuf/sparse_tensor.rb +13 -1
  58. data/lib/arrow-format/org/apache/arrow/flatbuf/sparse_tensor_index.rb +1 -1
  59. data/lib/arrow-format/org/apache/arrow/flatbuf/sparse_tensor_index_coo.rb +13 -4
  60. data/lib/arrow-format/org/apache/arrow/flatbuf/sparse_tensor_index_csf.rb +14 -4
  61. data/lib/arrow-format/org/apache/arrow/flatbuf/struct_.rb +5 -1
  62. data/lib/arrow-format/org/apache/arrow/flatbuf/tensor.rb +11 -1
  63. data/lib/arrow-format/org/apache/arrow/flatbuf/tensor_dim.rb +8 -1
  64. data/lib/arrow-format/org/apache/arrow/flatbuf/time.rb +8 -1
  65. data/lib/arrow-format/org/apache/arrow/flatbuf/time_unit.rb +1 -1
  66. data/lib/arrow-format/org/apache/arrow/flatbuf/timestamp.rb +8 -1
  67. data/lib/arrow-format/org/apache/arrow/flatbuf/type.rb +1 -1
  68. data/lib/arrow-format/org/apache/arrow/flatbuf/union.rb +8 -1
  69. data/lib/arrow-format/org/apache/arrow/flatbuf/union_mode.rb +1 -1
  70. data/lib/arrow-format/org/apache/arrow/flatbuf/utf8.rb +5 -1
  71. data/lib/arrow-format/org/apache/arrow/flatbuf/utf8view.rb +5 -1
  72. data/lib/arrow-format/readable.rb +151 -96
  73. data/lib/arrow-format/record-batch.rb +70 -1
  74. data/lib/arrow-format/schema.rb +14 -1
  75. data/lib/arrow-format/streaming-pull-reader.rb +49 -23
  76. data/lib/arrow-format/streaming-reader.rb +51 -14
  77. data/lib/arrow-format/streaming-writer.rb +156 -0
  78. data/lib/arrow-format/type.rb +397 -34
  79. data/lib/arrow-format/version.rb +1 -1
  80. data/lib/arrow-format.rb +2 -0
  81. data/red-arrow-format.gemspec +2 -1
  82. metadata +9 -4
@@ -1,6 +1,6 @@
1
1
  # Automatically generated. Don't modify manually.
2
2
  #
3
- # Red FlatBuffers version: 0.0.3
3
+ # Red FlatBuffers version: 0.0.4
4
4
  # Declared by: //Message.fbs
5
5
  # Rooting type: org.apache.arrow.flatbuf.Message (//Message.fbs)
6
6
 
@@ -19,6 +19,14 @@ module ArrowFormat
19
19
  # may be spread across multiple dictionary batches by using the isDelta
20
20
  # flag
21
21
  class DictionaryBatch < ::FlatBuffers::Table
22
+ FIELDS = {
23
+ id: ::FlatBuffers::Field.new(:id, 0, 4, :long, 0),
24
+ data: ::FlatBuffers::Field.new(:data, 1, 6, "::ArrowFormat::Org::Apache::Arrow::Flatbuf::RecordBatch", 0),
25
+ delta?: ::FlatBuffers::Field.new(:delta?, 2, 8, :bool, 0),
26
+ }
27
+
28
+ Data = define_data_class
29
+
22
30
  def data
23
31
  field_offset = @view.unpack_virtual_offset(6)
24
32
  return nil if field_offset.zero?
@@ -1,6 +1,6 @@
1
1
  # Automatically generated. Don't modify manually.
2
2
  #
3
- # Red FlatBuffers version: 0.0.3
3
+ # Red FlatBuffers version: 0.0.4
4
4
  # Declared by: //Schema.fbs
5
5
  # Rooting type: org.apache.arrow.flatbuf.Message (//Message.fbs)
6
6
 
@@ -14,6 +14,15 @@ module ArrowFormat
14
14
  module Arrow
15
15
  module Flatbuf
16
16
  class DictionaryEncoding < ::FlatBuffers::Table
17
+ FIELDS = {
18
+ id: ::FlatBuffers::Field.new(:id, 0, 4, :long, 0),
19
+ index_type: ::FlatBuffers::Field.new(:index_type, 1, 6, "::ArrowFormat::Org::Apache::Arrow::Flatbuf::Int", 0),
20
+ ordered?: ::FlatBuffers::Field.new(:ordered?, 2, 8, :bool, 0),
21
+ dictionary_kind: ::FlatBuffers::Field.new(:dictionary_kind, 3, 10, :short, 0),
22
+ }
23
+
24
+ Data = define_data_class
25
+
17
26
  def dictionary_kind
18
27
  field_offset = @view.unpack_virtual_offset(10)
19
28
  if field_offset.zero?
@@ -47,7 +56,7 @@ module ArrowFormat
47
56
  end
48
57
 
49
58
  # By default, dictionaries are not ordered, or the order does not have
50
- # semantic meaning. In some statistical, applications, dictionary-encoding
59
+ # semantic meaning. In some statistical applications, dictionary-encoding
51
60
  # is used to represent ordered categorical data, and we provide a way to
52
61
  # preserve that metadata here
53
62
  def ordered?
@@ -1,6 +1,6 @@
1
1
  # Automatically generated. Don't modify manually.
2
2
  #
3
- # Red FlatBuffers version: 0.0.3
3
+ # Red FlatBuffers version: 0.0.4
4
4
  # Declared by: //Schema.fbs
5
5
  # Rooting type: org.apache.arrow.flatbuf.Message (//Message.fbs)
6
6
 
@@ -1,6 +1,6 @@
1
1
  # Automatically generated. Don't modify manually.
2
2
  #
3
- # Red FlatBuffers version: 0.0.3
3
+ # Red FlatBuffers version: 0.0.4
4
4
  # Declared by: //Schema.fbs
5
5
  # Rooting type: org.apache.arrow.flatbuf.Message (//Message.fbs)
6
6
 
@@ -13,6 +13,12 @@ module ArrowFormat
13
13
  module Arrow
14
14
  module Flatbuf
15
15
  class Duration < ::FlatBuffers::Table
16
+ FIELDS = {
17
+ unit: ::FlatBuffers::Field.new(:unit, 0, 4, :short, 0),
18
+ }
19
+
20
+ Data = define_data_class
21
+
16
22
  def unit
17
23
  field_offset = @view.unpack_virtual_offset(4)
18
24
  if field_offset.zero?
@@ -1,6 +1,6 @@
1
1
  # Automatically generated. Don't modify manually.
2
2
  #
3
- # Red FlatBuffers version: 0.0.3
3
+ # Red FlatBuffers version: 0.0.4
4
4
  # Declared by: //Schema.fbs
5
5
  # Rooting type: org.apache.arrow.flatbuf.Message (//Message.fbs)
6
6
 
@@ -1,6 +1,6 @@
1
1
  # Automatically generated. Don't modify manually.
2
2
  #
3
- # Red FlatBuffers version: 0.0.3
3
+ # Red FlatBuffers version: 0.0.4
4
4
  # Declared by: //Schema.fbs
5
5
  # Rooting type: org.apache.arrow.flatbuf.Message (//Message.fbs)
6
6
 
@@ -1,6 +1,6 @@
1
1
  # Automatically generated. Don't modify manually.
2
2
  #
3
- # Red FlatBuffers version: 0.0.3
3
+ # Red FlatBuffers version: 0.0.4
4
4
  # Declared by: //Schema.fbs
5
5
  # Rooting type: org.apache.arrow.flatbuf.Message (//Message.fbs)
6
6
 
@@ -18,6 +18,18 @@ module ArrowFormat
18
18
  # A field represents a named column in a record / row batch or child of a
19
19
  # nested type.
20
20
  class Field < ::FlatBuffers::Table
21
+ FIELDS = {
22
+ name: ::FlatBuffers::Field.new(:name, 0, 4, :string, 0),
23
+ nullable?: ::FlatBuffers::Field.new(:nullable?, 1, 6, :bool, 0),
24
+ type_type: ::FlatBuffers::Field.new(:type_type, 2, 8, :utype, 0),
25
+ type: ::FlatBuffers::Field.new(:type, 3, 10, "::ArrowFormat::Org::Apache::Arrow::Flatbuf::Type", 0),
26
+ dictionary: ::FlatBuffers::Field.new(:dictionary, 4, 12, "::ArrowFormat::Org::Apache::Arrow::Flatbuf::DictionaryEncoding", 0),
27
+ children: ::FlatBuffers::Field.new(:children, 5, 14, ["::ArrowFormat::Org::Apache::Arrow::Flatbuf::Field"], 0),
28
+ custom_metadata: ::FlatBuffers::Field.new(:custom_metadata, 6, 16, ["::ArrowFormat::Org::Apache::Arrow::Flatbuf::KeyValue"], 0),
29
+ }
30
+
31
+ Data = define_data_class
32
+
21
33
  # children apply only to nested data types like Struct, List and Union. For
22
34
  # primitive types children will have length 0.
23
35
  def children
@@ -1,6 +1,6 @@
1
1
  # Automatically generated. Don't modify manually.
2
2
  #
3
- # Red FlatBuffers version: 0.0.3
3
+ # Red FlatBuffers version: 0.0.4
4
4
  # Declared by: //Message.fbs
5
5
  # Rooting type: org.apache.arrow.flatbuf.Message (//Message.fbs)
6
6
 
@@ -21,6 +21,13 @@ module ArrowFormat
21
21
  # would have {length: 5, null_count: 2} for its List node, and {length: 6,
22
22
  # null_count: 0} for its Int16 node, as separate FieldNode structs
23
23
  class FieldNode < ::FlatBuffers::Struct
24
+ FIELDS = {
25
+ length: ::FlatBuffers::Field.new(:length, 0, 0, :long, 0),
26
+ null_count: ::FlatBuffers::Field.new(:null_count, 1, 8, :long, 0),
27
+ }
28
+
29
+ Data = define_data_class
30
+
24
31
  # The number of value slots in the Arrow array at this level of a nested
25
32
  # tree
26
33
  def length
@@ -1,6 +1,6 @@
1
1
  # Automatically generated. Don't modify manually.
2
2
  #
3
- # Red FlatBuffers version: 0.0.3
3
+ # Red FlatBuffers version: 0.0.4
4
4
  # Declared by: //Schema.fbs
5
5
  # Rooting type: org.apache.arrow.flatbuf.Message (//Message.fbs)
6
6
 
@@ -12,6 +12,12 @@ module ArrowFormat
12
12
  module Arrow
13
13
  module Flatbuf
14
14
  class FixedSizeBinary < ::FlatBuffers::Table
15
+ FIELDS = {
16
+ byte_width: ::FlatBuffers::Field.new(:byte_width, 0, 4, :int, 0),
17
+ }
18
+
19
+ Data = define_data_class
20
+
15
21
  # Number of bytes per value
16
22
  def byte_width
17
23
  field_offset = @view.unpack_virtual_offset(4)
@@ -1,6 +1,6 @@
1
1
  # Automatically generated. Don't modify manually.
2
2
  #
3
- # Red FlatBuffers version: 0.0.3
3
+ # Red FlatBuffers version: 0.0.4
4
4
  # Declared by: //Schema.fbs
5
5
  # Rooting type: org.apache.arrow.flatbuf.Message (//Message.fbs)
6
6
 
@@ -12,6 +12,12 @@ module ArrowFormat
12
12
  module Arrow
13
13
  module Flatbuf
14
14
  class FixedSizeList < ::FlatBuffers::Table
15
+ FIELDS = {
16
+ list_size: ::FlatBuffers::Field.new(:list_size, 0, 4, :int, 0),
17
+ }
18
+
19
+ Data = define_data_class
20
+
15
21
  # Number of list items per value
16
22
  def list_size
17
23
  field_offset = @view.unpack_virtual_offset(4)
@@ -1,6 +1,6 @@
1
1
  # Automatically generated. Don't modify manually.
2
2
  #
3
- # Red FlatBuffers version: 0.0.3
3
+ # Red FlatBuffers version: 0.0.4
4
4
  # Declared by: //Schema.fbs
5
5
  # Rooting type: org.apache.arrow.flatbuf.Message (//Message.fbs)
6
6
 
@@ -13,6 +13,12 @@ module ArrowFormat
13
13
  module Arrow
14
14
  module Flatbuf
15
15
  class FloatingPoint < ::FlatBuffers::Table
16
+ FIELDS = {
17
+ precision: ::FlatBuffers::Field.new(:precision, 0, 4, :short, 0),
18
+ }
19
+
20
+ Data = define_data_class
21
+
16
22
  def precision
17
23
  field_offset = @view.unpack_virtual_offset(4)
18
24
  if field_offset.zero?
@@ -1,6 +1,6 @@
1
1
  # Automatically generated. Don't modify manually.
2
2
  #
3
- # Red FlatBuffers version: 0.0.3
3
+ # Red FlatBuffers version: 0.0.4
4
4
  # Declared by: //File.fbs
5
5
  # Rooting type: org.apache.arrow.flatbuf.Footer (//File.fbs)
6
6
 
@@ -18,7 +18,27 @@ module ArrowFormat
18
18
  # ----------------------------------------------------------------------
19
19
  # Arrow File metadata
20
20
  #
21
- class Footer < ::FlatBuffers::Table
21
+ class Footer < ::FlatBuffers::RootTable
22
+ class << self
23
+ def file_identifier
24
+ ""
25
+ end
26
+
27
+ def file_extension
28
+ ""
29
+ end
30
+ end
31
+
32
+ FIELDS = {
33
+ version: ::FlatBuffers::Field.new(:version, 0, 4, :short, 0),
34
+ schema: ::FlatBuffers::Field.new(:schema, 1, 6, "::ArrowFormat::Org::Apache::Arrow::Flatbuf::Schema", 0),
35
+ dictionaries: ::FlatBuffers::Field.new(:dictionaries, 2, 8, ["::ArrowFormat::Org::Apache::Arrow::Flatbuf::Block"], 0),
36
+ record_batches: ::FlatBuffers::Field.new(:record_batches, 3, 10, ["::ArrowFormat::Org::Apache::Arrow::Flatbuf::Block"], 0),
37
+ custom_metadata: ::FlatBuffers::Field.new(:custom_metadata, 4, 12, ["::ArrowFormat::Org::Apache::Arrow::Flatbuf::KeyValue"], 0),
38
+ }
39
+
40
+ Data = define_data_class
41
+
22
42
  # User-defined metadata
23
43
  def custom_metadata
24
44
  field_offset = @view.unpack_virtual_offset(12)
@@ -1,6 +1,6 @@
1
1
  # Automatically generated. Don't modify manually.
2
2
  #
3
- # Red FlatBuffers version: 0.0.3
3
+ # Red FlatBuffers version: 0.0.4
4
4
  # Declared by: //Schema.fbs
5
5
  # Rooting type: org.apache.arrow.flatbuf.Message (//Message.fbs)
6
6
 
@@ -12,6 +12,13 @@ module ArrowFormat
12
12
  module Arrow
13
13
  module Flatbuf
14
14
  class Int < ::FlatBuffers::Table
15
+ FIELDS = {
16
+ bit_width: ::FlatBuffers::Field.new(:bit_width, 0, 4, :int, 0),
17
+ signed?: ::FlatBuffers::Field.new(:signed?, 1, 6, :bool, 0),
18
+ }
19
+
20
+ Data = define_data_class
21
+
15
22
  def bit_width
16
23
  field_offset = @view.unpack_virtual_offset(4)
17
24
  return 0 if field_offset.zero?
@@ -1,6 +1,6 @@
1
1
  # Automatically generated. Don't modify manually.
2
2
  #
3
- # Red FlatBuffers version: 0.0.3
3
+ # Red FlatBuffers version: 0.0.4
4
4
  # Declared by: //Schema.fbs
5
5
  # Rooting type: org.apache.arrow.flatbuf.Message (//Message.fbs)
6
6
 
@@ -13,6 +13,12 @@ module ArrowFormat
13
13
  module Arrow
14
14
  module Flatbuf
15
15
  class Interval < ::FlatBuffers::Table
16
+ FIELDS = {
17
+ unit: ::FlatBuffers::Field.new(:unit, 0, 4, :short, 0),
18
+ }
19
+
20
+ Data = define_data_class
21
+
16
22
  def unit
17
23
  field_offset = @view.unpack_virtual_offset(4)
18
24
  if field_offset.zero?
@@ -1,6 +1,6 @@
1
1
  # Automatically generated. Don't modify manually.
2
2
  #
3
- # Red FlatBuffers version: 0.0.3
3
+ # Red FlatBuffers version: 0.0.4
4
4
  # Declared by: //Schema.fbs
5
5
  # Rooting type: org.apache.arrow.flatbuf.Message (//Message.fbs)
6
6
 
@@ -1,6 +1,6 @@
1
1
  # Automatically generated. Don't modify manually.
2
2
  #
3
- # Red FlatBuffers version: 0.0.3
3
+ # Red FlatBuffers version: 0.0.4
4
4
  # Declared by: //Schema.fbs
5
5
  # Rooting type: org.apache.arrow.flatbuf.Message (//Message.fbs)
6
6
 
@@ -15,6 +15,13 @@ module ArrowFormat
15
15
  # user defined key value pairs to add custom metadata to arrow
16
16
  # key namespacing is the responsibility of the user
17
17
  class KeyValue < ::FlatBuffers::Table
18
+ FIELDS = {
19
+ key: ::FlatBuffers::Field.new(:key, 0, 4, :string, 0),
20
+ value: ::FlatBuffers::Field.new(:value, 1, 6, :string, 0),
21
+ }
22
+
23
+ Data = define_data_class
24
+
18
25
  def key
19
26
  field_offset = @view.unpack_virtual_offset(4)
20
27
  return nil if field_offset.zero?
@@ -1,6 +1,6 @@
1
1
  # Automatically generated. Don't modify manually.
2
2
  #
3
- # Red FlatBuffers version: 0.0.3
3
+ # Red FlatBuffers version: 0.0.4
4
4
  # Declared by: //Schema.fbs
5
5
  # Rooting type: org.apache.arrow.flatbuf.Message (//Message.fbs)
6
6
 
@@ -14,6 +14,10 @@ module ArrowFormat
14
14
  # Same as Binary, but with 64-bit offsets, allowing to represent
15
15
  # extremely large data values.
16
16
  class LargeBinary < ::FlatBuffers::Table
17
+ FIELDS = {
18
+ }
19
+
20
+ Data = define_data_class
17
21
  end
18
22
  end
19
23
  end
@@ -1,6 +1,6 @@
1
1
  # Automatically generated. Don't modify manually.
2
2
  #
3
- # Red FlatBuffers version: 0.0.3
3
+ # Red FlatBuffers version: 0.0.4
4
4
  # Declared by: //Schema.fbs
5
5
  # Rooting type: org.apache.arrow.flatbuf.Message (//Message.fbs)
6
6
 
@@ -14,6 +14,10 @@ module ArrowFormat
14
14
  # Same as List, but with 64-bit offsets, allowing to represent
15
15
  # extremely large data values.
16
16
  class LargeList < ::FlatBuffers::Table
17
+ FIELDS = {
18
+ }
19
+
20
+ Data = define_data_class
17
21
  end
18
22
  end
19
23
  end
@@ -1,6 +1,6 @@
1
1
  # Automatically generated. Don't modify manually.
2
2
  #
3
- # Red FlatBuffers version: 0.0.3
3
+ # Red FlatBuffers version: 0.0.4
4
4
  # Declared by: //Schema.fbs
5
5
  # Rooting type: org.apache.arrow.flatbuf.Message (//Message.fbs)
6
6
 
@@ -14,6 +14,10 @@ module ArrowFormat
14
14
  # Same as ListView, but with 64-bit offsets and sizes, allowing to represent
15
15
  # extremely large data values.
16
16
  class LargeListView < ::FlatBuffers::Table
17
+ FIELDS = {
18
+ }
19
+
20
+ Data = define_data_class
17
21
  end
18
22
  end
19
23
  end
@@ -1,6 +1,6 @@
1
1
  # Automatically generated. Don't modify manually.
2
2
  #
3
- # Red FlatBuffers version: 0.0.3
3
+ # Red FlatBuffers version: 0.0.4
4
4
  # Declared by: //Schema.fbs
5
5
  # Rooting type: org.apache.arrow.flatbuf.Message (//Message.fbs)
6
6
 
@@ -14,6 +14,10 @@ module ArrowFormat
14
14
  # Same as Utf8, but with 64-bit offsets, allowing to represent
15
15
  # extremely large data values.
16
16
  class LargeUtf8 < ::FlatBuffers::Table
17
+ FIELDS = {
18
+ }
19
+
20
+ Data = define_data_class
17
21
  end
18
22
  end
19
23
  end
@@ -1,6 +1,6 @@
1
1
  # Automatically generated. Don't modify manually.
2
2
  #
3
- # Red FlatBuffers version: 0.0.3
3
+ # Red FlatBuffers version: 0.0.4
4
4
  # Declared by: //Schema.fbs
5
5
  # Rooting type: org.apache.arrow.flatbuf.Message (//Message.fbs)
6
6
 
@@ -12,6 +12,10 @@ module ArrowFormat
12
12
  module Arrow
13
13
  module Flatbuf
14
14
  class List < ::FlatBuffers::Table
15
+ FIELDS = {
16
+ }
17
+
18
+ Data = define_data_class
15
19
  end
16
20
  end
17
21
  end
@@ -1,6 +1,6 @@
1
1
  # Automatically generated. Don't modify manually.
2
2
  #
3
- # Red FlatBuffers version: 0.0.3
3
+ # Red FlatBuffers version: 0.0.4
4
4
  # Declared by: //Schema.fbs
5
5
  # Rooting type: org.apache.arrow.flatbuf.Message (//Message.fbs)
6
6
 
@@ -15,6 +15,10 @@ module ArrowFormat
15
15
  # sizes allowing for writes in any order and sharing of child values among
16
16
  # list values.
17
17
  class ListView < ::FlatBuffers::Table
18
+ FIELDS = {
19
+ }
20
+
21
+ Data = define_data_class
18
22
  end
19
23
  end
20
24
  end
@@ -1,6 +1,6 @@
1
1
  # Automatically generated. Don't modify manually.
2
2
  #
3
- # Red FlatBuffers version: 0.0.3
3
+ # Red FlatBuffers version: 0.0.4
4
4
  # Declared by: //Schema.fbs
5
5
  # Rooting type: org.apache.arrow.flatbuf.Message (//Message.fbs)
6
6
 
@@ -21,7 +21,7 @@ module ArrowFormat
21
21
  # may be set in the metadata for this field.
22
22
  #
23
23
  # In a field with Map type, the field has a child Struct field, which then
24
- # has two children: key type and the second the value type. The names of the
24
+ # has two children: the key type and the value type. The names of the
25
25
  # child fields may be respectively "entries", "key", and "value", but this is
26
26
  # not enforced.
27
27
  #
@@ -37,6 +37,12 @@ module ArrowFormat
37
37
  # for Map can make Map an alias for List. The "layout" attribute for the Map
38
38
  # field must have the same contents as a List.
39
39
  class Map < ::FlatBuffers::Table
40
+ FIELDS = {
41
+ keys_sorted?: ::FlatBuffers::Field.new(:keys_sorted?, 0, 4, :bool, 0),
42
+ }
43
+
44
+ Data = define_data_class
45
+
40
46
  # Set to true if the keys within each value are sorted
41
47
  def keys_sorted?
42
48
  field_offset = @view.unpack_virtual_offset(4)
@@ -1,6 +1,6 @@
1
1
  # Automatically generated. Don't modify manually.
2
2
  #
3
- # Red FlatBuffers version: 0.0.3
3
+ # Red FlatBuffers version: 0.0.4
4
4
  # Declared by: //Message.fbs
5
5
  # Rooting type: org.apache.arrow.flatbuf.Message (//Message.fbs)
6
6
 
@@ -14,7 +14,27 @@ module ArrowFormat
14
14
  module Apache
15
15
  module Arrow
16
16
  module Flatbuf
17
- class Message < ::FlatBuffers::Table
17
+ class Message < ::FlatBuffers::RootTable
18
+ class << self
19
+ def file_identifier
20
+ ""
21
+ end
22
+
23
+ def file_extension
24
+ ""
25
+ end
26
+ end
27
+
28
+ FIELDS = {
29
+ version: ::FlatBuffers::Field.new(:version, 0, 4, :short, 0),
30
+ header_type: ::FlatBuffers::Field.new(:header_type, 1, 6, :utype, 0),
31
+ header: ::FlatBuffers::Field.new(:header, 2, 8, "::ArrowFormat::Org::Apache::Arrow::Flatbuf::MessageHeader", 0),
32
+ body_length: ::FlatBuffers::Field.new(:body_length, 3, 10, :long, 0),
33
+ custom_metadata: ::FlatBuffers::Field.new(:custom_metadata, 4, 12, ["::ArrowFormat::Org::Apache::Arrow::Flatbuf::KeyValue"], 0),
34
+ }
35
+
36
+ Data = define_data_class
37
+
18
38
  def body_length
19
39
  field_offset = @view.unpack_virtual_offset(10)
20
40
  return 0 if field_offset.zero?
@@ -1,6 +1,6 @@
1
1
  # Automatically generated. Don't modify manually.
2
2
  #
3
- # Red FlatBuffers version: 0.0.3
3
+ # Red FlatBuffers version: 0.0.4
4
4
  # Declared by: //Message.fbs
5
5
  # Rooting type: org.apache.arrow.flatbuf.Message (//Message.fbs)
6
6
 
@@ -1,6 +1,6 @@
1
1
  # Automatically generated. Don't modify manually.
2
2
  #
3
- # Red FlatBuffers version: 0.0.3
3
+ # Red FlatBuffers version: 0.0.4
4
4
  # Declared by: //Schema.fbs
5
5
  # Rooting type: org.apache.arrow.flatbuf.Message (//Message.fbs)
6
6
 
@@ -1,6 +1,6 @@
1
1
  # Automatically generated. Don't modify manually.
2
2
  #
3
- # Red FlatBuffers version: 0.0.3
3
+ # Red FlatBuffers version: 0.0.4
4
4
  # Declared by: //Schema.fbs
5
5
  # Rooting type: org.apache.arrow.flatbuf.Message (//Message.fbs)
6
6
 
@@ -13,6 +13,10 @@ module ArrowFormat
13
13
  module Flatbuf
14
14
  # These are stored in the flatbuffer in the Type union below
15
15
  class Null < ::FlatBuffers::Table
16
+ FIELDS = {
17
+ }
18
+
19
+ Data = define_data_class
16
20
  end
17
21
  end
18
22
  end
@@ -1,6 +1,6 @@
1
1
  # Automatically generated. Don't modify manually.
2
2
  #
3
- # Red FlatBuffers version: 0.0.3
3
+ # Red FlatBuffers version: 0.0.4
4
4
  # Declared by: //Schema.fbs
5
5
  # Rooting type: org.apache.arrow.flatbuf.Message (//Message.fbs)
6
6
 
@@ -1,6 +1,6 @@
1
1
  # Automatically generated. Don't modify manually.
2
2
  #
3
- # Red FlatBuffers version: 0.0.3
3
+ # Red FlatBuffers version: 0.0.4
4
4
  # Declared by: //Message.fbs
5
5
  # Rooting type: org.apache.arrow.flatbuf.Message (//Message.fbs)
6
6
 
@@ -18,6 +18,16 @@ module ArrowFormat
18
18
  # batch. Some systems call this a "row batch" internally and others a "record
19
19
  # batch".
20
20
  class RecordBatch < ::FlatBuffers::Table
21
+ FIELDS = {
22
+ length: ::FlatBuffers::Field.new(:length, 0, 4, :long, 0),
23
+ nodes: ::FlatBuffers::Field.new(:nodes, 1, 6, ["::ArrowFormat::Org::Apache::Arrow::Flatbuf::FieldNode"], 0),
24
+ buffers: ::FlatBuffers::Field.new(:buffers, 2, 8, ["::ArrowFormat::Org::Apache::Arrow::Flatbuf::Buffer"], 0),
25
+ compression: ::FlatBuffers::Field.new(:compression, 3, 10, "::ArrowFormat::Org::Apache::Arrow::Flatbuf::BodyCompression", 0),
26
+ variadic_buffer_counts: ::FlatBuffers::Field.new(:variadic_buffer_counts, 4, 12, [:long], 0),
27
+ }
28
+
29
+ Data = define_data_class
30
+
21
31
  # Buffers correspond to the pre-ordered flattened buffer tree
22
32
  #
23
33
  # The number of buffers appended to this list depends on the schema. For
@@ -64,7 +74,7 @@ module ArrowFormat
64
74
 
65
75
  # Some types such as Utf8View are represented using a variable number of buffers.
66
76
  # For each such Field in the pre-ordered flattened logical schema, there will be
67
- # an entry in variadicBufferCounts to indicate the number of number of variadic
77
+ # an entry in variadicBufferCounts to indicate the number of variadic
68
78
  # buffers which belong to that Field in the current RecordBatch.
69
79
  #
70
80
  # For example, the schema
@@ -1,6 +1,6 @@
1
1
  # Automatically generated. Don't modify manually.
2
2
  #
3
- # Red FlatBuffers version: 0.0.3
3
+ # Red FlatBuffers version: 0.0.4
4
4
  # Declared by: //Schema.fbs
5
5
  # Rooting type: org.apache.arrow.flatbuf.Message (//Message.fbs)
6
6
 
@@ -17,6 +17,10 @@ module ArrowFormat
17
17
  # each corresponding index in the values child array ends.
18
18
  # Like list/struct types, the value array can be of any type.
19
19
  class RunEndEncoded < ::FlatBuffers::Table
20
+ FIELDS = {
21
+ }
22
+
23
+ Data = define_data_class
20
24
  end
21
25
  end
22
26
  end
@@ -1,6 +1,6 @@
1
1
  # Automatically generated. Don't modify manually.
2
2
  #
3
- # Red FlatBuffers version: 0.0.3
3
+ # Red FlatBuffers version: 0.0.4
4
4
  # Declared by: //Schema.fbs
5
5
  # Rooting type: org.apache.arrow.flatbuf.Message (//Message.fbs)
6
6
 
@@ -17,6 +17,15 @@ module ArrowFormat
17
17
  # ----------------------------------------------------------------------
18
18
  # A Schema describes the columns in a row batch
19
19
  class Schema < ::FlatBuffers::Table
20
+ FIELDS = {
21
+ endianness: ::FlatBuffers::Field.new(:endianness, 0, 4, :short, 0),
22
+ fields: ::FlatBuffers::Field.new(:fields, 1, 6, ["::ArrowFormat::Org::Apache::Arrow::Flatbuf::Field"], 0),
23
+ custom_metadata: ::FlatBuffers::Field.new(:custom_metadata, 2, 8, ["::ArrowFormat::Org::Apache::Arrow::Flatbuf::KeyValue"], 0),
24
+ features: ::FlatBuffers::Field.new(:features, 3, 10, [:long], 0),
25
+ }
26
+
27
+ Data = define_data_class
28
+
20
29
  def custom_metadata
21
30
  field_offset = @view.unpack_virtual_offset(8)
22
31
  return nil if field_offset.zero?
@@ -1,6 +1,6 @@
1
1
  # Automatically generated. Don't modify manually.
2
2
  #
3
- # Red FlatBuffers version: 0.0.3
3
+ # Red FlatBuffers version: 0.0.4
4
4
  # Declared by: //SparseTensor.fbs
5
5
  # Rooting type: org.apache.arrow.flatbuf.Message (//Message.fbs)
6
6