red-arrow-format 23.0.0

data/lib/arrow-format/org/apache/arrow/flatbuf/sparse_tensor_index.rb

@@ -0,0 +1,29 @@
+# Automatically generated. Don't modify manually.
+#
+# Red FlatBuffers version: 0.0.3
+# Declared by:             //SparseTensor.fbs
+# Rooting type:            org.apache.arrow.flatbuf.Message (//Message.fbs)
+
+require "flatbuffers"
+
+module ArrowFormat
+  module Org
+    module Apache
+      module Arrow
+        module Flatbuf
+          class SparseTensorIndex < ::FlatBuffers::Union
+            NONE = register("NONE", 0, "::ArrowFormat::Org::Apache::Arrow::Flatbuf::Utf8View", "../../../apache/arrow/flatbuf/utf8view")
+            SPARSE_TENSOR_INDEX_COO = register("SparseTensorIndexCOO", 1, "::ArrowFormat::Org::Apache::Arrow::Flatbuf::SparseTensorIndexCOO", "../../../apache/arrow/flatbuf/sparse_tensor_index_coo")
+            SPARSE_MATRIX_INDEX_CSX = register("SparseMatrixIndexCSX", 2, "::ArrowFormat::Org::Apache::Arrow::Flatbuf::SparseMatrixIndexCSX", "../../../apache/arrow/flatbuf/sparse_matrix_index_csx")
+            SPARSE_TENSOR_INDEX_CSF = register("SparseTensorIndexCSF", 3, "::ArrowFormat::Org::Apache::Arrow::Flatbuf::SparseTensorIndexCSF", "../../../apache/arrow/flatbuf/sparse_tensor_index_csf")
+            
+
+            private def require_table_class
+              require_relative @require_path
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/arrow-format/org/apache/arrow/flatbuf/sparse_tensor_index_coo.rb

@@ -0,0 +1,93 @@
+# Automatically generated. Don't modify manually.
+#
+# Red FlatBuffers version: 0.0.3
+# Declared by:             //SparseTensor.fbs
+# Rooting type:            org.apache.arrow.flatbuf.Message (//Message.fbs)
+
+require "flatbuffers"
+require_relative "../../../apache/arrow/flatbuf/buffer"
+require_relative "../../../apache/arrow/flatbuf/int"
+
+module ArrowFormat
+  module Org
+    module Apache
+      module Arrow
+        module Flatbuf
+          # ----------------------------------------------------------------------
+          # EXPERIMENTAL: Data structures for sparse tensors
+          # Coordinate (COO) format of sparse tensor index.
+          #
+          # COO's index list are represented as a NxM matrix,
+          # where N is the number of non-zero values,
+          # and M is the number of dimensions of a sparse tensor.
+          #
+          # indicesBuffer stores the location and size of the data of this indices
+          # matrix.  The value type and the stride of the indices matrix is
+          # specified in indicesType and indicesStrides fields.
+          #
+          # For example, let X be a 2x3x4x5 tensor, and it has the following
+          # 6 non-zero values:
+          # ```text
+          #   X[0, 1, 2, 0] := 1
+          #   X[1, 1, 2, 3] := 2
+          #   X[0, 2, 1, 0] := 3
+          #   X[0, 1, 3, 0] := 4
+          #   X[0, 1, 2, 1] := 5
+          #   X[1, 2, 0, 4] := 6
+          # ```
+          # In COO format, the index matrix of X is the following 4x6 matrix:
+          # ```text
+          #   [[0, 0, 0, 0, 1, 1],
+          #    [1, 1, 1, 2, 1, 2],
+          #    [2, 2, 3, 1, 2, 0],
+          #    [0, 1, 0, 0, 3, 4]]
+          # ```
+          # When isCanonical is true, the indices is sorted in lexicographical order
+          # (row-major order), and it does not have duplicated entries.  Otherwise,
+          # the indices may not be sorted, or may have duplicated entries.
+          class SparseTensorIndexCOO < ::FlatBuffers::Table
+            # The location and size of the indices matrix's data
+            def indices_buffer
+              field_offset = @view.unpack_virtual_offset(8)
+              return nil if field_offset.zero?
+
+              @view.unpack_struct(::ArrowFormat::Org::Apache::Arrow::Flatbuf::Buffer, field_offset)
+            end
+
+            # Non-negative byte offsets to advance one value cell along each dimension
+            # If omitted, default to row-major order (C-like).
+            def indices_strides
+              field_offset = @view.unpack_virtual_offset(6)
+              return nil if field_offset.zero?
+
+              element_size = 8
+              @view.unpack_vector(field_offset, element_size) do |element_offset|
+                @view.unpack_long(element_offset)
+              end
+            end
+
+            # The type of values in indicesBuffer
+            def indices_type
+              field_offset = @view.unpack_virtual_offset(4)
+              return nil if field_offset.zero?
+
+              @view.unpack_table(::ArrowFormat::Org::Apache::Arrow::Flatbuf::Int, field_offset)
+            end
+
+            # This flag is true if and only if the indices matrix is sorted in
+            # row-major order, and does not have duplicated entries.
+            # This sort order is the same as of Tensorflow's SparseTensor,
+            # but it is inverse order of SciPy's canonical coo_matrix
+            # (SciPy employs column-major order for its coo_matrix).
+            def canonical?
+              field_offset = @view.unpack_virtual_offset(10)
+              return false if field_offset.zero?
+
+              @view.unpack_bool(field_offset)
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/arrow-format/org/apache/arrow/flatbuf/sparse_tensor_index_csf.rb

@@ -0,0 +1,129 @@
+# Automatically generated. Don't modify manually.
+#
+# Red FlatBuffers version: 0.0.3
+# Declared by:             //SparseTensor.fbs
+# Rooting type:            org.apache.arrow.flatbuf.Message (//Message.fbs)
+
+require "flatbuffers"
+require_relative "../../../apache/arrow/flatbuf/buffer"
+require_relative "../../../apache/arrow/flatbuf/int"
+
+module ArrowFormat
+  module Org
+    module Apache
+      module Arrow
+        module Flatbuf
+          # Compressed Sparse Fiber (CSF) sparse tensor index.
+          class SparseTensorIndexCSF < ::FlatBuffers::Table
+            # axisOrder stores the sequence in which dimensions were traversed to
+            # produce the prefix tree.
+            # For example, the axisOrder for the above X is:
+            # ```text
+            #   axisOrder(X) = [0, 1, 2, 3].
+            # ```
+            def axis_order
+              field_offset = @view.unpack_virtual_offset(12)
+              return nil if field_offset.zero?
+
+              element_size = 4
+              @view.unpack_vector(field_offset, element_size) do |element_offset|
+                @view.unpack_int(element_offset)
+              end
+            end
+
+            # indicesBuffers stores values of nodes.
+            # Each tensor dimension corresponds to a buffer in indicesBuffers.
+            # For example, the indicesBuffers for the above X is:
+            # ```text
+            #   indicesBuffer(X) = [
+            #                        [0, 1],
+            #                        [0, 1, 1],
+            #                        [0, 0, 1, 1],
+            #                        [1, 2, 0, 2, 0, 0, 1, 2]
+            #                      ].
+            # ```
+            def indices_buffers
+              field_offset = @view.unpack_virtual_offset(10)
+              return nil if field_offset.zero?
+
+              element_size = 16
+              @view.unpack_vector(field_offset, element_size) do |element_offset|
+                @view.unpack_struct(::ArrowFormat::Org::Apache::Arrow::Flatbuf::Buffer, element_offset)
+              end
+            end
+
+            # The type of values in indicesBuffers
+            def indices_type
+              field_offset = @view.unpack_virtual_offset(8)
+              return nil if field_offset.zero?
+
+              @view.unpack_table(::ArrowFormat::Org::Apache::Arrow::Flatbuf::Int, field_offset)
+            end
+
+            # indptrBuffers stores the sparsity structure.
+            # Each two consecutive dimensions in a tensor correspond to a buffer in
+            # indptrBuffers. A pair of consecutive values at `indptrBuffers[dim][i]`
+            # and `indptrBuffers[dim][i + 1]` signify a range of nodes in
+            # `indicesBuffers[dim + 1]` who are children of `indicesBuffers[dim][i]` node.
+            #
+            # For example, the indptrBuffers for the above X is:
+            # ```text
+            #   indptrBuffer(X) = [
+            #                       [0, 2, 3],
+            #                       [0, 1, 3, 4],
+            #                       [0, 2, 4, 5, 8]
+            #                     ].
+            # ```
+            def indptr_buffers
+              field_offset = @view.unpack_virtual_offset(6)
+              return nil if field_offset.zero?
+
+              element_size = 16
+              @view.unpack_vector(field_offset, element_size) do |element_offset|
+                @view.unpack_struct(::ArrowFormat::Org::Apache::Arrow::Flatbuf::Buffer, element_offset)
+              end
+            end
+
+            # CSF is a generalization of compressed sparse row (CSR) index.
+            # See [smith2017knl](http://shaden.io/pub-files/smith2017knl.pdf)
+            #
+            # CSF index recursively compresses each dimension of a tensor into a set
+            # of prefix trees. Each path from a root to leaf forms one tensor
+            # non-zero index. CSF is implemented with two arrays of buffers and one
+            # arrays of integers.
+            #
+            # For example, let X be a 2x3x4x5 tensor and let it have the following
+            # 8 non-zero values:
+            # ```text
+            #   X[0, 0, 0, 1] := 1
+            #   X[0, 0, 0, 2] := 2
+            #   X[0, 1, 0, 0] := 3
+            #   X[0, 1, 0, 2] := 4
+            #   X[0, 1, 1, 0] := 5
+            #   X[1, 1, 1, 0] := 6
+            #   X[1, 1, 1, 1] := 7
+            #   X[1, 1, 1, 2] := 8
+            # ```
+            # As a prefix tree this would be represented as:
+            # ```text
+            #         0          1
+            #        / \         |
+            #       0   1        1
+            #      /   / \       |
+            #     0   0   1      1
+            #    /|  /|   |    /| |
+            #   1 2 0 2   0   0 1 2
+            # ```
+            # The type of values in indptrBuffers
+            def indptr_type
+              field_offset = @view.unpack_virtual_offset(4)
+              return nil if field_offset.zero?
+
+              @view.unpack_table(::ArrowFormat::Org::Apache::Arrow::Flatbuf::Int, field_offset)
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/arrow-format/org/apache/arrow/flatbuf/struct_.rb

@@ -0,0 +1,23 @@
+# Automatically generated. Don't modify manually.
+#
+# Red FlatBuffers version: 0.0.3
+# Declared by:             //Schema.fbs
+# Rooting type:            org.apache.arrow.flatbuf.Message (//Message.fbs)
+
+require "flatbuffers"
+
+module ArrowFormat
+  module Org
+    module Apache
+      module Arrow
+        module Flatbuf
+          # A Struct_ in the flatbuffer metadata is the same as an Arrow Struct
+          # (according to the physical memory layout). We used Struct_ here as
+          # Struct is a reserved word in Flatbuffers
+          class Struct < ::FlatBuffers::Table
+          end
+        end
+      end
+    end
+  end
+end

data/lib/arrow-format/org/apache/arrow/flatbuf/tensor.rb

@@ -0,0 +1,74 @@
+# Automatically generated. Don't modify manually.
+#
+# Red FlatBuffers version: 0.0.3
+# Declared by:             //Tensor.fbs
+# Rooting type:            org.apache.arrow.flatbuf.Message (//Message.fbs)
+
+require "flatbuffers"
+require_relative "../../../apache/arrow/flatbuf/buffer"
+require_relative "../../../apache/arrow/flatbuf/tensor_dim"
+require_relative "../../../apache/arrow/flatbuf/type"
+
+module ArrowFormat
+  module Org
+    module Apache
+      module Arrow
+        module Flatbuf
+          class Tensor < ::FlatBuffers::Table
+            # The location and size of the tensor's data
+            def data
+              field_offset = @view.unpack_virtual_offset(12)
+              return nil if field_offset.zero?
+
+              @view.unpack_struct(::ArrowFormat::Org::Apache::Arrow::Flatbuf::Buffer, field_offset)
+            end
+
+            # The dimensions of the tensor, optionally named
+            def shape
+              field_offset = @view.unpack_virtual_offset(8)
+              return nil if field_offset.zero?
+
+              element_size = 4
+              @view.unpack_vector(field_offset, element_size) do |element_offset|
+                @view.unpack_table(::ArrowFormat::Org::Apache::Arrow::Flatbuf::TensorDim, element_offset)
+              end
+            end
+
+            # Non-negative byte offsets to advance one value cell along each dimension
+            # If omitted, default to row-major order (C-like).
+            def strides
+              field_offset = @view.unpack_virtual_offset(10)
+              return nil if field_offset.zero?
+
+              element_size = 8
+              @view.unpack_vector(field_offset, element_size) do |element_offset|
+                @view.unpack_long(element_offset)
+              end
+            end
+
+            # The type of data contained in a value cell. Currently only fixed-width
+            # value types are supported, no strings or nested types
+            def type
+              type = type_type
+              return nil if type.nil?
+
+              field_offset = @view.unpack_virtual_offset(6)
+              return nil if field_offset.zero?
+              @view.unpack_union(type.table_class, field_offset)
+            end
+
+            def type_type
+              field_offset = @view.unpack_virtual_offset(4)
+              if field_offset.zero?
+                enum_value = 0
+              else
+                enum_value = @view.unpack_utype(field_offset)
+              end
+              ::ArrowFormat::Org::Apache::Arrow::Flatbuf::Type.try_convert(enum_value) || enum_value
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/arrow-format/org/apache/arrow/flatbuf/tensor_dim.rb

@@ -0,0 +1,38 @@
+# Automatically generated. Don't modify manually.
+#
+# Red FlatBuffers version: 0.0.3
+# Declared by:             //Tensor.fbs
+# Rooting type:            org.apache.arrow.flatbuf.Message (//Message.fbs)
+
+require "flatbuffers"
+
+module ArrowFormat
+  module Org
+    module Apache
+      module Arrow
+        module Flatbuf
+          # ----------------------------------------------------------------------
+          # Data structures for dense tensors
+          # Shape data for a single axis in a tensor
+          class TensorDim < ::FlatBuffers::Table
+            # Name of the dimension, optional
+            def name
+              field_offset = @view.unpack_virtual_offset(6)
+              return nil if field_offset.zero?
+
+              @view.unpack_string(field_offset)
+            end
+
+            # Length of dimension
+            def size
+              field_offset = @view.unpack_virtual_offset(4)
+              return 0 if field_offset.zero?
+
+              @view.unpack_long(field_offset)
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/arrow-format/org/apache/arrow/flatbuf/time.rb

@@ -0,0 +1,51 @@
+# Automatically generated. Don't modify manually.
+#
+# Red FlatBuffers version: 0.0.3
+# Declared by:             //Schema.fbs
+# Rooting type:            org.apache.arrow.flatbuf.Message (//Message.fbs)
+
+require "flatbuffers"
+require_relative "../../../apache/arrow/flatbuf/time_unit"
+
+module ArrowFormat
+  module Org
+    module Apache
+      module Arrow
+        module Flatbuf
+          # Time is either a 32-bit or 64-bit signed integer type representing an
+          # elapsed time since midnight, stored in either of four units: seconds,
+          # milliseconds, microseconds or nanoseconds.
+          #
+          # The integer `bitWidth` depends on the `unit` and must be one of the following:
+          # * SECOND and MILLISECOND: 32 bits
+          # * MICROSECOND and NANOSECOND: 64 bits
+          #
+          # The allowed values are between 0 (inclusive) and 86400 (=24*60*60) seconds
+          # (exclusive), adjusted for the time unit (for example, up to 86400000
+          # exclusive for the MILLISECOND unit).
+          # This definition doesn't allow for leap seconds. Time values from
+          # measurements with leap seconds will need to be corrected when ingesting
+          # into Arrow (for example by replacing the value 86400 with 86399).
+          class Time < ::FlatBuffers::Table
+            def bit_width
+              field_offset = @view.unpack_virtual_offset(6)
+              return 32 if field_offset.zero?
+
+              @view.unpack_int(field_offset)
+            end
+
+            def unit
+              field_offset = @view.unpack_virtual_offset(4)
+              if field_offset.zero?
+                enum_value = 1
+              else
+                enum_value = @view.unpack_short(field_offset)
+              end
+              ::ArrowFormat::Org::Apache::Arrow::Flatbuf::TimeUnit.try_convert(enum_value) || enum_value
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/arrow-format/org/apache/arrow/flatbuf/time_unit.rb

@@ -0,0 +1,24 @@
+# Automatically generated. Don't modify manually.
+#
+# Red FlatBuffers version: 0.0.3
+# Declared by:             //Schema.fbs
+# Rooting type:            org.apache.arrow.flatbuf.Message (//Message.fbs)
+
+require "flatbuffers"
+
+module ArrowFormat
+  module Org
+    module Apache
+      module Arrow
+        module Flatbuf
+          class TimeUnit < ::FlatBuffers::Enum
+            SECOND = register("SECOND", 0)
+            MILLISECOND = register("MILLISECOND", 1)
+            MICROSECOND = register("MICROSECOND", 2)
+            NANOSECOND = register("NANOSECOND", 3)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/arrow-format/org/apache/arrow/flatbuf/timestamp.rb

@@ -0,0 +1,152 @@
+# Automatically generated. Don't modify manually.
+#
+# Red FlatBuffers version: 0.0.3
+# Declared by:             //Schema.fbs
+# Rooting type:            org.apache.arrow.flatbuf.Message (//Message.fbs)
+
+require "flatbuffers"
+require_relative "../../../apache/arrow/flatbuf/time_unit"
+
+module ArrowFormat
+  module Org
+    module Apache
+      module Arrow
+        module Flatbuf
+          # Timestamp is a 64-bit signed integer representing an elapsed time since a
+          # fixed epoch, stored in either of four units: seconds, milliseconds,
+          # microseconds or nanoseconds, and is optionally annotated with a timezone.
+          #
+          # Timestamp values do not include any leap seconds (in other words, all
+          # days are considered 86400 seconds long).
+          #
+          # Timestamps with a non-empty timezone
+          # ------------------------------------
+          #
+          # If a Timestamp column has a non-empty timezone value, its epoch is
+          # 1970-01-01 00:00:00 (January 1st 1970, midnight) in the *UTC* timezone
+          # (the Unix epoch), regardless of the Timestamp's own timezone.
+          #
+          # Therefore, timestamp values with a non-empty timezone correspond to
+          # physical points in time together with some additional information about
+          # how the data was obtained and/or how to display it (the timezone).
+          #
+          #   For example, the timestamp value 0 with the timezone string "Europe/Paris"
+          #   corresponds to "January 1st 1970, 00h00" in the UTC timezone, but the
+          #   application may prefer to display it as "January 1st 1970, 01h00" in
+          #   the Europe/Paris timezone (which is the same physical point in time).
+          #
+          # One consequence is that timestamp values with a non-empty timezone
+          # can be compared and ordered directly, since they all share the same
+          # well-known point of reference (the Unix epoch).
+          #
+          # Timestamps with an unset / empty timezone
+          # -----------------------------------------
+          #
+          # If a Timestamp column has no timezone value, its epoch is
+          # 1970-01-01 00:00:00 (January 1st 1970, midnight) in an *unknown* timezone.
+          #
+          # Therefore, timestamp values without a timezone cannot be meaningfully
+          # interpreted as physical points in time, but only as calendar / clock
+          # indications ("wall clock time") in an unspecified timezone.
+          #
+          #   For example, the timestamp value 0 with an empty timezone string
+          #   corresponds to "January 1st 1970, 00h00" in an unknown timezone: there
+          #   is not enough information to interpret it as a well-defined physical
+          #   point in time.
+          #
+          # One consequence is that timestamp values without a timezone cannot
+          # be reliably compared or ordered, since they may have different points of
+          # reference.  In particular, it is *not* possible to interpret an unset
+          # or empty timezone as the same as "UTC".
+          #
+          # Conversion between timezones
+          # ----------------------------
+          #
+          # If a Timestamp column has a non-empty timezone, changing the timezone
+          # to a different non-empty value is a metadata-only operation:
+          # the timestamp values need not change as their point of reference remains
+          # the same (the Unix epoch).
+          #
+          # However, if a Timestamp column has no timezone value, changing it to a
+          # non-empty value requires to think about the desired semantics.
+          # One possibility is to assume that the original timestamp values are
+          # relative to the epoch of the timezone being set; timestamp values should
+          # then adjusted to the Unix epoch (for example, changing the timezone from
+          # empty to "Europe/Paris" would require converting the timestamp values
+          # from "Europe/Paris" to "UTC", which seems counter-intuitive but is
+          # nevertheless correct).
+          #
+          # Guidelines for encoding data from external libraries
+          # ----------------------------------------------------
+          #
+          # Date & time libraries often have multiple different data types for temporal
+          # data. In order to ease interoperability between different implementations the
+          # Arrow project has some recommendations for encoding these types into a Timestamp
+          # column.
+          #
+          # An "instant" represents a physical point in time that has no relevant timezone
+          # (for example, astronomical data). To encode an instant, use a Timestamp with
+          # the timezone string set to "UTC", and make sure the Timestamp values
+          # are relative to the UTC epoch (January 1st 1970, midnight).
+          #
+          # A "zoned date-time" represents a physical point in time annotated with an
+          # informative timezone (for example, the timezone in which the data was
+          # recorded).  To encode a zoned date-time, use a Timestamp with the timezone
+          # string set to the name of the timezone, and make sure the Timestamp values
+          # are relative to the UTC epoch (January 1st 1970, midnight).
+          #
+          #  (There is some ambiguity between an instant and a zoned date-time with the
+          #   UTC timezone.  Both of these are stored the same in Arrow.  Typically,
+          #   this distinction does not matter.  If it does, then an application should
+          #   use custom metadata or an extension type to distinguish between the two cases.)
+          #
+          # An "offset date-time" represents a physical point in time combined with an
+          # explicit offset from UTC.  To encode an offset date-time, use a Timestamp
+          # with the timezone string set to the numeric timezone offset string
+          # (e.g. "+03:00"), and make sure the Timestamp values are relative to
+          # the UTC epoch (January 1st 1970, midnight).
+          #
+          # A "naive date-time" (also called "local date-time" in some libraries)
+          # represents a wall clock time combined with a calendar date, but with
+          # no indication of how to map this information to a physical point in time.
+          # Naive date-times must be handled with care because of this missing
+          # information, and also because daylight saving time (DST) may make
+          # some values ambiguous or nonexistent. A naive date-time may be
+          # stored as a struct with Date and Time fields. However, it may also be
+          # encoded into a Timestamp column with an empty timezone. The timestamp
+          # values should be computed "as if" the timezone of the date-time values
+          # was UTC; for example, the naive date-time "January 1st 1970, 00h00" would
+          # be encoded as timestamp value 0.
+          class Timestamp < ::FlatBuffers::Table
+            # The timezone is an optional string indicating the name of a timezone,
+            # one of:
+            #
+            # * As used in the Olson timezone database (the "tz database" or
+            #   "tzdata"), such as "America/New_York".
+            # * An absolute timezone offset of the form "+XX:XX" or "-XX:XX",
+            #   such as "+07:30".
+            #
+            # Whether a timezone string is present indicates different semantics about
+            # the data (see above).
+            def timezone
+              field_offset = @view.unpack_virtual_offset(6)
+              return nil if field_offset.zero?
+
+              @view.unpack_string(field_offset)
+            end
+
+            def unit
+              field_offset = @view.unpack_virtual_offset(4)
+              if field_offset.zero?
+                enum_value = 0
+              else
+                enum_value = @view.unpack_short(field_offset)
+              end
+              ::ArrowFormat::Org::Apache::Arrow::Flatbuf::TimeUnit.try_convert(enum_value) || enum_value
+            end
+          end
+        end
+      end
+    end
+  end
+end