red-arrow-format 23.0.0

data/lib/arrow-format/org/apache/arrow/flatbuf/list_view.rb

@@ -0,0 +1,23 @@
+# Automatically generated. Don't modify manually.
+#
+# Red FlatBuffers version: 0.0.3
+# Declared by:             //Schema.fbs
+# Rooting type:            org.apache.arrow.flatbuf.Message (//Message.fbs)
+
+require "flatbuffers"
+
+module ArrowFormat
+  module Org
+    module Apache
+      module Arrow
+        module Flatbuf
+          # Represents the same logical types that List can, but contains offsets and
+          # sizes allowing for writes in any order and sharing of child values among
+          # list values.
+          class ListView < ::FlatBuffers::Table
+          end
+        end
+      end
+    end
+  end
+end

data/lib/arrow-format/org/apache/arrow/flatbuf/map.rb

@@ -0,0 +1,52 @@
+# Automatically generated. Don't modify manually.
+#
+# Red FlatBuffers version: 0.0.3
+# Declared by:             //Schema.fbs
+# Rooting type:            org.apache.arrow.flatbuf.Message (//Message.fbs)
+
+require "flatbuffers"
+
+module ArrowFormat
+  module Org
+    module Apache
+      module Arrow
+        module Flatbuf
+          # A Map is a logical nested type that is represented as
+          #
+          # List<entries: Struct<key: K, value: V>>
+          #
+          # In this layout, the keys and values are each respectively contiguous. We do
+          # not constrain the key and value types, so the application is responsible
+          # for ensuring that the keys are hashable and unique. Whether the keys are sorted
+          # may be set in the metadata for this field.
+          #
+          # In a field with Map type, the field has a child Struct field, which then
+          # has two children: key type and the second the value type. The names of the
+          # child fields may be respectively "entries", "key", and "value", but this is
+          # not enforced.
+          #
+          # Map
+          # ```text
+          #   - child[0] entries: Struct
+          #     - child[0] key: K
+          #     - child[1] value: V
+          # ```
+          # Neither the "entries" field nor the "key" field may be nullable.
+          #
+          # The metadata is structured so that Arrow systems without special handling
+          # for Map can make Map an alias for List. The "layout" attribute for the Map
+          # field must have the same contents as a List.
+          class Map < ::FlatBuffers::Table
+            # Set to true if the keys within each value are sorted
+            def keys_sorted?
+              field_offset = @view.unpack_virtual_offset(4)
+              return false if field_offset.zero?
+
+              @view.unpack_bool(field_offset)
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/arrow-format/org/apache/arrow/flatbuf/message.rb

@@ -0,0 +1,68 @@
+# Automatically generated. Don't modify manually.
+#
+# Red FlatBuffers version: 0.0.3
+# Declared by:             //Message.fbs
+# Rooting type:            org.apache.arrow.flatbuf.Message (//Message.fbs)
+
+require "flatbuffers"
+require_relative "../../../apache/arrow/flatbuf/key_value"
+require_relative "../../../apache/arrow/flatbuf/message_header"
+require_relative "../../../apache/arrow/flatbuf/metadata_version"
+
+module ArrowFormat
+  module Org
+    module Apache
+      module Arrow
+        module Flatbuf
+          class Message < ::FlatBuffers::Table
+            def body_length
+              field_offset = @view.unpack_virtual_offset(10)
+              return 0 if field_offset.zero?
+
+              @view.unpack_long(field_offset)
+            end
+
+            def custom_metadata
+              field_offset = @view.unpack_virtual_offset(12)
+              return nil if field_offset.zero?
+
+              element_size = 4
+              @view.unpack_vector(field_offset, element_size) do |element_offset|
+                @view.unpack_table(::ArrowFormat::Org::Apache::Arrow::Flatbuf::KeyValue, element_offset)
+              end
+            end
+
+            def header
+              type = header_type
+              return nil if type.nil?
+
+              field_offset = @view.unpack_virtual_offset(8)
+              return nil if field_offset.zero?
+              @view.unpack_union(type.table_class, field_offset)
+            end
+
+            def header_type
+              field_offset = @view.unpack_virtual_offset(6)
+              if field_offset.zero?
+                enum_value = 0
+              else
+                enum_value = @view.unpack_utype(field_offset)
+              end
+              ::ArrowFormat::Org::Apache::Arrow::Flatbuf::MessageHeader.try_convert(enum_value) || enum_value
+            end
+
+            def version
+              field_offset = @view.unpack_virtual_offset(4)
+              if field_offset.zero?
+                enum_value = 0
+              else
+                enum_value = @view.unpack_short(field_offset)
+              end
+              ::ArrowFormat::Org::Apache::Arrow::Flatbuf::MetadataVersion.try_convert(enum_value) || enum_value
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/arrow-format/org/apache/arrow/flatbuf/message_header.rb

@@ -0,0 +1,39 @@
+# Automatically generated. Don't modify manually.
+#
+# Red FlatBuffers version: 0.0.3
+# Declared by:             //Message.fbs
+# Rooting type:            org.apache.arrow.flatbuf.Message (//Message.fbs)
+
+require "flatbuffers"
+
+module ArrowFormat
+  module Org
+    module Apache
+      module Arrow
+        module Flatbuf
+          # ----------------------------------------------------------------------
+          # The root Message type
+          # This union enables us to easily send different message types without
+          # redundant storage, and in the future we can easily add new message types.
+          #
+          # Arrow implementations do not need to implement all of the message types,
+          # which may include experimental metadata types. For maximum compatibility,
+          # it is best to send data using RecordBatch
+          class MessageHeader < ::FlatBuffers::Union
+            NONE = register("NONE", 0, "::ArrowFormat::Org::Apache::Arrow::Flatbuf::Utf8View", "../../../apache/arrow/flatbuf/utf8view")
+            SCHEMA = register("Schema", 1, "::ArrowFormat::Org::Apache::Arrow::Flatbuf::Schema", "../../../apache/arrow/flatbuf/schema")
+            DICTIONARY_BATCH = register("DictionaryBatch", 2, "::ArrowFormat::Org::Apache::Arrow::Flatbuf::DictionaryBatch", "../../../apache/arrow/flatbuf/dictionary_batch")
+            RECORD_BATCH = register("RecordBatch", 3, "::ArrowFormat::Org::Apache::Arrow::Flatbuf::RecordBatch", "../../../apache/arrow/flatbuf/record_batch")
+            TENSOR = register("Tensor", 4, "::ArrowFormat::Org::Apache::Arrow::Flatbuf::Tensor", "../../../apache/arrow/flatbuf/tensor")
+            SPARSE_TENSOR = register("SparseTensor", 5, "::ArrowFormat::Org::Apache::Arrow::Flatbuf::SparseTensor", "../../../apache/arrow/flatbuf/sparse_tensor")
+            
+
+            private def require_table_class
+              require_relative @require_path
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/arrow-format/org/apache/arrow/flatbuf/metadata_version.rb

@@ -0,0 +1,36 @@
+# Automatically generated. Don't modify manually.
+#
+# Red FlatBuffers version: 0.0.3
+# Declared by:             //Schema.fbs
+# Rooting type:            org.apache.arrow.flatbuf.Message (//Message.fbs)
+
+require "flatbuffers"
+
+module ArrowFormat
+  module Org
+    module Apache
+      module Arrow
+        module Flatbuf
+          class MetadataVersion < ::FlatBuffers::Enum
+            # 0.1.0 (October 2016).
+            V1 = register("V1", 0)
+            # 0.2.0 (February 2017). Non-backwards compatible with V1.
+            V2 = register("V2", 1)
+            # 0.3.0 -> 0.7.1 (May - December 2017). Non-backwards compatible with V2.
+            V3 = register("V3", 2)
+            # >= 0.8.0 (December 2017). Non-backwards compatible with V3.
+            V4 = register("V4", 3)
+            # >= 1.0.0 (July 2020). Backwards compatible with V4 (V5 readers can read V4
+            # metadata and IPC messages). Implementations are recommended to provide a
+            # V4 compatibility mode with V5 format changes disabled.
+            #
+            # Incompatible changes between V4 and V5:
+            # - Union buffer layout has changed. In V5, Unions don't have a validity
+            #   bitmap buffer.
+            V5 = register("V5", 4)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/arrow-format/org/apache/arrow/flatbuf/null.rb

@@ -0,0 +1,21 @@
+# Automatically generated. Don't modify manually.
+#
+# Red FlatBuffers version: 0.0.3
+# Declared by:             //Schema.fbs
+# Rooting type:            org.apache.arrow.flatbuf.Message (//Message.fbs)
+
+require "flatbuffers"
+
+module ArrowFormat
+  module Org
+    module Apache
+      module Arrow
+        module Flatbuf
+          # These are stored in the flatbuffer in the Type union below
+          class Null < ::FlatBuffers::Table
+          end
+        end
+      end
+    end
+  end
+end

data/lib/arrow-format/org/apache/arrow/flatbuf/precision.rb

@@ -0,0 +1,23 @@
+# Automatically generated. Don't modify manually.
+#
+# Red FlatBuffers version: 0.0.3
+# Declared by:             //Schema.fbs
+# Rooting type:            org.apache.arrow.flatbuf.Message (//Message.fbs)
+
+require "flatbuffers"
+
+module ArrowFormat
+  module Org
+    module Apache
+      module Arrow
+        module Flatbuf
+          class Precision < ::FlatBuffers::Enum
+            HALF = register("HALF", 0)
+            SINGLE = register("SINGLE", 1)
+            DOUBLE = register("DOUBLE", 2)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/arrow-format/org/apache/arrow/flatbuf/record_batch.rb

@@ -0,0 +1,93 @@
+# Automatically generated. Don't modify manually.
+#
+# Red FlatBuffers version: 0.0.3
+# Declared by:             //Message.fbs
+# Rooting type:            org.apache.arrow.flatbuf.Message (//Message.fbs)
+
+require "flatbuffers"
+require_relative "../../../apache/arrow/flatbuf/buffer"
+require_relative "../../../apache/arrow/flatbuf/body_compression"
+require_relative "../../../apache/arrow/flatbuf/field_node"
+
+module ArrowFormat
+  module Org
+    module Apache
+      module Arrow
+        module Flatbuf
+          # A data header describing the shared memory layout of a "record" or "row"
+          # batch. Some systems call this a "row batch" internally and others a "record
+          # batch".
+          class RecordBatch < ::FlatBuffers::Table
+            # Buffers correspond to the pre-ordered flattened buffer tree
+            #
+            # The number of buffers appended to this list depends on the schema. For
+            # example, most primitive arrays will have 2 buffers, 1 for the validity
+            # bitmap and 1 for the values. For struct arrays, there will only be a
+            # single buffer for the validity (nulls) bitmap
+            def buffers
+              field_offset = @view.unpack_virtual_offset(8)
+              return nil if field_offset.zero?
+
+              element_size = 16
+              @view.unpack_vector(field_offset, element_size) do |element_offset|
+                @view.unpack_struct(::ArrowFormat::Org::Apache::Arrow::Flatbuf::Buffer, element_offset)
+              end
+            end
+
+            # Optional compression of the message body
+            def compression
+              field_offset = @view.unpack_virtual_offset(10)
+              return nil if field_offset.zero?
+
+              @view.unpack_table(::ArrowFormat::Org::Apache::Arrow::Flatbuf::BodyCompression, field_offset)
+            end
+
+            # number of records / rows. The arrays in the batch should all have this
+            # length
+            def length
+              field_offset = @view.unpack_virtual_offset(4)
+              return 0 if field_offset.zero?
+
+              @view.unpack_long(field_offset)
+            end
+
+            # Nodes correspond to the pre-ordered flattened logical schema
+            def nodes
+              field_offset = @view.unpack_virtual_offset(6)
+              return nil if field_offset.zero?
+
+              element_size = 16
+              @view.unpack_vector(field_offset, element_size) do |element_offset|
+                @view.unpack_struct(::ArrowFormat::Org::Apache::Arrow::Flatbuf::FieldNode, element_offset)
+              end
+            end
+
+            # Some types such as Utf8View are represented using a variable number of buffers.
+            # For each such Field in the pre-ordered flattened logical schema, there will be
+            # an entry in variadicBufferCounts to indicate the number of number of variadic
+            # buffers which belong to that Field in the current RecordBatch.
+            #
+            # For example, the schema
+            #     col1: Struct<alpha: Int32, beta: BinaryView, gamma: Float64>
+            #     col2: Utf8View
+            # contains two Fields with variadic buffers so variadicBufferCounts will have
+            # two entries, the first counting the variadic buffers of `col1.beta` and the
+            # second counting `col2`'s.
+            #
+            # This field may be omitted if and only if the schema contains no Fields with
+            # a variable number of buffers, such as BinaryView and Utf8View.
+            def variadic_buffer_counts
+              field_offset = @view.unpack_virtual_offset(12)
+              return nil if field_offset.zero?
+
+              element_size = 8
+              @view.unpack_vector(field_offset, element_size) do |element_offset|
+                @view.unpack_long(element_offset)
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/arrow-format/org/apache/arrow/flatbuf/run_end_encoded.rb

@@ -0,0 +1,25 @@
+# Automatically generated. Don't modify manually.
+#
+# Red FlatBuffers version: 0.0.3
+# Declared by:             //Schema.fbs
+# Rooting type:            org.apache.arrow.flatbuf.Message (//Message.fbs)
+
+require "flatbuffers"
+
+module ArrowFormat
+  module Org
+    module Apache
+      module Arrow
+        module Flatbuf
+          # Contains two child arrays, run_ends and values.
+          # The run_ends child array must be a 16/32/64-bit integer array
+          # which encodes the indices at which the run with the value in 
+          # each corresponding index in the values child array ends.
+          # Like list/struct types, the value array can be of any type.
+          class RunEndEncoded < ::FlatBuffers::Table
+          end
+        end
+      end
+    end
+  end
+end

data/lib/arrow-format/org/apache/arrow/flatbuf/schema.rb

@@ -0,0 +1,68 @@
+# Automatically generated. Don't modify manually.
+#
+# Red FlatBuffers version: 0.0.3
+# Declared by:             //Schema.fbs
+# Rooting type:            org.apache.arrow.flatbuf.Message (//Message.fbs)
+
+require "flatbuffers"
+require_relative "../../../apache/arrow/flatbuf/key_value"
+require_relative "../../../apache/arrow/flatbuf/endianness"
+require_relative "../../../apache/arrow/flatbuf/field"
+
+module ArrowFormat
+  module Org
+    module Apache
+      module Arrow
+        module Flatbuf
+          # ----------------------------------------------------------------------
+          # A Schema describes the columns in a row batch
+          class Schema < ::FlatBuffers::Table
+            def custom_metadata
+              field_offset = @view.unpack_virtual_offset(8)
+              return nil if field_offset.zero?
+
+              element_size = 4
+              @view.unpack_vector(field_offset, element_size) do |element_offset|
+                @view.unpack_table(::ArrowFormat::Org::Apache::Arrow::Flatbuf::KeyValue, element_offset)
+              end
+            end
+
+            # endianness of the buffer
+            # it is Little Endian by default
+            # if endianness doesn't match the underlying system then the vectors need to be converted
+            def endianness
+              field_offset = @view.unpack_virtual_offset(4)
+              if field_offset.zero?
+                enum_value = 0
+              else
+                enum_value = @view.unpack_short(field_offset)
+              end
+              ::ArrowFormat::Org::Apache::Arrow::Flatbuf::Endianness.try_convert(enum_value) || enum_value
+            end
+
+            # Features used in the stream/file.
+            def features
+              field_offset = @view.unpack_virtual_offset(10)
+              return nil if field_offset.zero?
+
+              element_size = 8
+              @view.unpack_vector(field_offset, element_size) do |element_offset|
+                @view.unpack_long(element_offset)
+              end
+            end
+
+            def fields
+              field_offset = @view.unpack_virtual_offset(6)
+              return nil if field_offset.zero?
+
+              element_size = 4
+              @view.unpack_vector(field_offset, element_size) do |element_offset|
+                @view.unpack_table(::ArrowFormat::Org::Apache::Arrow::Flatbuf::Field, element_offset)
+              end
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/arrow-format/org/apache/arrow/flatbuf/sparse_matrix_compressed_axis.rb

@@ -0,0 +1,22 @@
+# Automatically generated. Don't modify manually.
+#
+# Red FlatBuffers version: 0.0.3
+# Declared by:             //SparseTensor.fbs
+# Rooting type:            org.apache.arrow.flatbuf.Message (//Message.fbs)
+
+require "flatbuffers"
+
+module ArrowFormat
+  module Org
+    module Apache
+      module Arrow
+        module Flatbuf
+          class SparseMatrixCompressedAxis < ::FlatBuffers::Enum
+            ROW = register("Row", 0)
+            COLUMN = register("Column", 1)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/arrow-format/org/apache/arrow/flatbuf/sparse_matrix_index_csx.rb

@@ -0,0 +1,96 @@
+# Automatically generated. Don't modify manually.
+#
+# Red FlatBuffers version: 0.0.3
+# Declared by:             //SparseTensor.fbs
+# Rooting type:            org.apache.arrow.flatbuf.Message (//Message.fbs)
+
+require "flatbuffers"
+require_relative "../../../apache/arrow/flatbuf/sparse_matrix_compressed_axis"
+require_relative "../../../apache/arrow/flatbuf/buffer"
+require_relative "../../../apache/arrow/flatbuf/int"
+
+module ArrowFormat
+  module Org
+    module Apache
+      module Arrow
+        module Flatbuf
+          # Compressed Sparse format, that is matrix-specific.
+          class SparseMatrixIndexCSX < ::FlatBuffers::Table
+            # Which axis, row or column, is compressed
+            def compressed_axis
+              field_offset = @view.unpack_virtual_offset(4)
+              if field_offset.zero?
+                enum_value = 0
+              else
+                enum_value = @view.unpack_short(field_offset)
+              end
+              ::ArrowFormat::Org::Apache::Arrow::Flatbuf::SparseMatrixCompressedAxis.try_convert(enum_value) || enum_value
+            end
+
+            # indicesBuffer stores the location and size of the array that
+            # contains the column indices of the corresponding non-zero values.
+            # The type of index value is long.
+            #
+            # For example, the indices of the above X is:
+            # ```text
+            #   indices(X) = [1, 2, 2, 1, 3, 0, 2, 3, 1].
+            # ```
+            # Note that the indices are sorted in lexicographical order for each row.
+            def indices_buffer
+              field_offset = @view.unpack_virtual_offset(12)
+              return nil if field_offset.zero?
+
+              @view.unpack_struct(::ArrowFormat::Org::Apache::Arrow::Flatbuf::Buffer, field_offset)
+            end
+
+            # The type of values in indicesBuffer
+            def indices_type
+              field_offset = @view.unpack_virtual_offset(10)
+              return nil if field_offset.zero?
+
+              @view.unpack_table(::ArrowFormat::Org::Apache::Arrow::Flatbuf::Int, field_offset)
+            end
+
+            # indptrBuffer stores the location and size of indptr array that
+            # represents the range of the rows.
+            # The i-th row spans from `indptr[i]` to `indptr[i+1]` in the data.
+            # The length of this array is 1 + (the number of rows), and the type
+            # of index value is long.
+            #
+            # For example, let X be the following 6x4 matrix:
+            # ```text
+            #   X := [[0, 1, 2, 0],
+            #         [0, 0, 3, 0],
+            #         [0, 4, 0, 5],
+            #         [0, 0, 0, 0],
+            #         [6, 0, 7, 8],
+            #         [0, 9, 0, 0]].
+            # ```
+            # The array of non-zero values in X is:
+            # ```text
+            #   values(X) = [1, 2, 3, 4, 5, 6, 7, 8, 9].
+            # ```
+            # And the indptr of X is:
+            # ```text
+            #   indptr(X) = [0, 2, 3, 5, 5, 8, 10].
+            # ```
+            def indptr_buffer
+              field_offset = @view.unpack_virtual_offset(8)
+              return nil if field_offset.zero?
+
+              @view.unpack_struct(::ArrowFormat::Org::Apache::Arrow::Flatbuf::Buffer, field_offset)
+            end
+
+            # The type of values in indptrBuffer
+            def indptr_type
+              field_offset = @view.unpack_virtual_offset(6)
+              return nil if field_offset.zero?
+
+              @view.unpack_table(::ArrowFormat::Org::Apache::Arrow::Flatbuf::Int, field_offset)
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/arrow-format/org/apache/arrow/flatbuf/sparse_tensor.rb

@@ -0,0 +1,92 @@
+# Automatically generated. Don't modify manually.
+#
+# Red FlatBuffers version: 0.0.3
+# Declared by:             //SparseTensor.fbs
+# Rooting type:            org.apache.arrow.flatbuf.Message (//Message.fbs)
+
+require "flatbuffers"
+require_relative "../../../apache/arrow/flatbuf/buffer"
+require_relative "../../../apache/arrow/flatbuf/tensor_dim"
+require_relative "../../../apache/arrow/flatbuf/sparse_tensor_index"
+require_relative "../../../apache/arrow/flatbuf/type"
+
+module ArrowFormat
+  module Org
+    module Apache
+      module Arrow
+        module Flatbuf
+          class SparseTensor < ::FlatBuffers::Table
+            # The location and size of the tensor's data
+            def data
+              field_offset = @view.unpack_virtual_offset(16)
+              return nil if field_offset.zero?
+
+              @view.unpack_struct(::ArrowFormat::Org::Apache::Arrow::Flatbuf::Buffer, field_offset)
+            end
+
+            # The number of non-zero values in a sparse tensor.
+            def non_zero_length
+              field_offset = @view.unpack_virtual_offset(10)
+              return 0 if field_offset.zero?
+
+              @view.unpack_long(field_offset)
+            end
+
+            # The dimensions of the tensor, optionally named.
+            def shape
+              field_offset = @view.unpack_virtual_offset(8)
+              return nil if field_offset.zero?
+
+              element_size = 4
+              @view.unpack_vector(field_offset, element_size) do |element_offset|
+                @view.unpack_table(::ArrowFormat::Org::Apache::Arrow::Flatbuf::TensorDim, element_offset)
+              end
+            end
+
+            # Sparse tensor index
+            def sparse_index
+              type = sparse_index_type
+              return nil if type.nil?
+
+              field_offset = @view.unpack_virtual_offset(14)
+              return nil if field_offset.zero?
+              @view.unpack_union(type.table_class, field_offset)
+            end
+
+            def sparse_index_type
+              field_offset = @view.unpack_virtual_offset(12)
+              if field_offset.zero?
+                enum_value = 0
+              else
+                enum_value = @view.unpack_utype(field_offset)
+              end
+              ::ArrowFormat::Org::Apache::Arrow::Flatbuf::SparseTensorIndex.try_convert(enum_value) || enum_value
+            end
+
+            # The type of data contained in a value cell.
+            # Currently only fixed-width value types are supported,
+            # no strings or nested types.
+            def type
+              type = type_type
+              return nil if type.nil?
+
+              field_offset = @view.unpack_virtual_offset(6)
+              return nil if field_offset.zero?
+              @view.unpack_union(type.table_class, field_offset)
+            end
+
+            def type_type
+              field_offset = @view.unpack_virtual_offset(4)
+              if field_offset.zero?
+                enum_value = 0
+              else
+                enum_value = @view.unpack_utype(field_offset)
+              end
+              ::ArrowFormat::Org::Apache::Arrow::Flatbuf::Type.try_convert(enum_value) || enum_value
+            end
+          end
+        end
+      end
+    end
+  end
+end