unibuf 0.1.1 → 0.1.2

data/lib/unibuf/serializers/capnproto/pointer_encoder.rb

@@ -0,0 +1,118 @@
+# frozen_string_literal: true
+
+module Unibuf
+  module Serializers
+    module Capnproto
+      # Encoder for Cap'n Proto pointer words
+      # Encodes pointer information into 64-bit words
+      class PointerEncoder
+        # Pointer type constants (bits 0-1)
+        POINTER_TYPE_STRUCT = 0
+        POINTER_TYPE_LIST = 1
+        POINTER_TYPE_FAR = 2
+        POINTER_TYPE_OTHER = 3
+
+        # List element size constants
+        ELEMENT_SIZE_VOID = 0
+        ELEMENT_SIZE_BIT = 1
+        ELEMENT_SIZE_BYTE = 2
+        ELEMENT_SIZE_TWO_BYTES = 3
+        ELEMENT_SIZE_FOUR_BYTES = 4
+        ELEMENT_SIZE_EIGHT_BYTES = 5
+        ELEMENT_SIZE_POINTER = 6
+        ELEMENT_SIZE_INLINE_COMPOSITE = 7
+
+        class << self
+          # Encode a null pointer
+          # @return [Integer] 64-bit word
+          def encode_null
+            0
+          end
+
+          # Encode a struct pointer
+          # @param offset [Integer] Signed word offset (relative to pointer position)
+          # @param data_words [Integer] Number of data words
+          # @param pointer_words [Integer] Number of pointer words
+          # @return [Integer] 64-bit pointer word
+          def encode_struct(offset, data_words, pointer_words)
+            # Bits: [pointer_words:16][data_words:16][offset:30][type:2]
+
+            # Convert signed offset to 30-bit representation
+            offset_bits = offset & 0x3FFFFFFF
+
+            word = POINTER_TYPE_STRUCT |
+              (offset_bits << 2) |
+              (data_words << 32) |
+              (pointer_words << 48)
+
+            word & 0xFFFFFFFFFFFFFFFF
+          end
+
+          # Encode a list pointer
+          # @param offset [Integer] Signed word offset
+          # @param element_size [Integer] Element size code (0-7)
+          # @param element_count [Integer] Number of elements
+          # @return [Integer] 64-bit pointer word
+          def encode_list(offset, element_size, element_count)
+            # Bits: [element_count:29][element_size:3][offset:30][type:2]
+
+            # Convert signed offset to 30-bit representation
+            offset_bits = offset & 0x3FFFFFFF
+
+            word = POINTER_TYPE_LIST |
+              (offset_bits << 2) |
+              (element_size << 32) |
+              (element_count << 35)
+
+            word & 0xFFFFFFFFFFFFFFFF
+          end
+
+          # Encode a far pointer
+          # @param segment_id [Integer] Target segment ID
+          # @param offset [Integer] Word offset within target segment
+          # @param double_far [Boolean] Landing pad flag
+          # @return [Integer] 64-bit pointer word
+          def encode_far(segment_id, offset, double_far: false)
+            # Bits: [segment_id:32][offset:29][double_far:1][type:2]
+
+            double_far_bit = double_far ? 1 : 0
+
+            word = POINTER_TYPE_FAR |
+              (double_far_bit << 2) |
+              (offset << 3) |
+              (segment_id << 32)
+
+            word & 0xFFFFFFFFFFFFFFFF
+          end
+
+          # Encode a capability pointer
+          # @param index [Integer] Capability index
+          # @return [Integer] 64-bit pointer word
+          def encode_capability(index)
+            # Bits: [index:32][reserved:30][type:2]
+
+            word = POINTER_TYPE_OTHER |
+              (index << 32)
+
+            word & 0xFFFFFFFFFFFFFFFF
+          end
+
+          # Get element size code for a type
+          # @param type_name [String, Symbol] Type name
+          # @return [Integer] Element size code
+          def element_size_for_type(type_name)
+            case type_name.to_s
+            when "Void" then ELEMENT_SIZE_VOID
+            when "Bool" then ELEMENT_SIZE_BIT
+            when "Int8", "UInt8" then ELEMENT_SIZE_BYTE
+            when "Int16", "UInt16" then ELEMENT_SIZE_TWO_BYTES
+            when "Int32", "UInt32", "Float32" then ELEMENT_SIZE_FOUR_BYTES
+            when "Int64", "UInt64", "Float64" then ELEMENT_SIZE_EIGHT_BYTES
+            else ELEMENT_SIZE_POINTER # For user types and Text/Data
+            end
+          end
+        end
+      end
+    end
+  end
+end

data/lib/unibuf/serializers/capnproto/segment_builder.rb

@@ -0,0 +1,124 @@
+# frozen_string_literal: true
+
+module Unibuf
+  module Serializers
+    module Capnproto
+      # Builder for Cap'n Proto binary segments
+      # Manages segment allocation and word-aligned writes
+      class SegmentBuilder
+        WORD_SIZE = 8 # Cap'n Proto uses 8-byte words
+
+        attr_reader :segments
+
+        def initialize
+          @segments = []
+          @current_segment = []
+          @current_segment_id = 0
+        end
+
+        # Get all segments including current
+        def segments
+          if @current_segment.empty?
+            @segments
+          else
+            @segments + [@current_segment]
+          end
+        end
+
+        # Allocate words in current segment
+        # @param word_count [Integer] Number of words to allocate
+        # @return [Array<Integer>] [segment_id, word_offset]
+        def allocate(word_count)
+          segment_id = @current_segment_id
+          word_offset = @current_segment.length
+
+          # Add placeholder words
+          word_count.times { @current_segment << 0 }
+
+          [segment_id, word_offset]
+        end
+
+        # Write a word at specific location
+        # @param segment_id [Integer] Segment index
+        # @param word_offset [Integer] Word offset within segment
+        # @param value [Integer] 64-bit word value
+        def write_word(segment_id, word_offset, value)
+          if segment_id == @current_segment_id
+            # Writing to current segment
+            @current_segment[word_offset] = value & 0xFFFFFFFFFFFFFFFF
+          elsif segment_id < @segments.length
+            # Writing to finalized segment
+            @segments[segment_id][word_offset] = value & 0xFFFFFFFFFFFFFFFF
+          else
+            raise "Invalid segment ID: #{segment_id}"
+          end
+        end
+
+        # Write multiple words
+        # @param segment_id [Integer] Segment index
+        # @param word_offset [Integer] Starting word offset
+        # @param values [Array<Integer>] Word values
+        def write_words(segment_id, word_offset, values)
+          values.each_with_index do |value, i|
+            write_word(segment_id, word_offset + i, value)
+          end
+        end
+
+        # Finalize current segment and start a new one
+        def finalize_segment
+          @segments << @current_segment unless @current_segment.empty?
+          @current_segment = []
+          @current_segment_id = @segments.length
+        end
+
+        # Build final binary output
+        # @return [String] Binary data
+        def build
+          # Finalize current segment if not empty
+          finalize_segment unless @current_segment.empty?
+
+          return "" if @segments.empty?
+
+          # Build segment table
+          segment_count = @segments.length
+          segment_sizes = @segments.map(&:length)
+
+          # Segment table header
+          output = [segment_count - 1].pack("L<") # segment count - 1
+
+          # Segment sizes
+          segment_sizes.each do |size|
+            output += [size].pack("L<")
+          end
+
+          # Add padding if needed (when segment count is even)
+          output += [0].pack("L<") unless segment_count.odd?
+
+          # Write segment data
+          @segments.each do |segment|
+            segment.each do |word|
+              output += [word].pack("Q<")
+            end
+          end
+
+          output
+        end
+
+        # Get current position for relative offset calculation
+        # @return [Array<Integer>] [segment_id, word_offset]
+        def current_position
+          [@current_segment_id, @current_segment.length]
+        end
+
+        private
+
+        def ensure_segment(segment_id)
+          return if segment_id == @current_segment_id
+          return if segment_id < @segments.length
+
+          raise "Invalid segment ID: #{segment_id}"
+        end
+      end
+    end
+  end
+end

data/lib/unibuf/serializers/capnproto/struct_writer.rb

@@ -0,0 +1,139 @@
+# frozen_string_literal: true
+
+require_relative "segment_builder"
+require_relative "pointer_encoder"
+
+module Unibuf
+  module Serializers
+    module Capnproto
+      # Writer for Cap'n Proto struct data
+      # Writes data section (primitives) and pointer section (references)
+      class StructWriter
+        attr_reader :segment_builder, :segment_id, :word_offset, :data_words,
+                    :pointer_words
+
+        # Initialize struct writer
+        # @param segment_builder [SegmentBuilder] Segment builder
+        # @param segment_id [Integer] Segment ID
+        # @param word_offset [Integer] Word offset in segment
+        # @param data_words [Integer] Number of data words
+        # @param pointer_words [Integer] Number of pointer words
+        def initialize(segment_builder, segment_id, word_offset, data_words,
+pointer_words)
+          @segment_builder = segment_builder
+          @segment_id = segment_id
+          @word_offset = word_offset
+          @data_words = data_words
+          @pointer_words = pointer_words
+        end
+
+        # Write a primitive value to data section
+        # @param word_index [Integer] Word index in data section
+        # @param bit_offset [Integer] Bit offset within word (0-63)
+        # @param bit_width [Integer] Width in bits
+        # @param value [Integer] Value to write
+        def write_data_field(word_index, bit_offset, bit_width, value)
+          return if word_index >= @data_words
+
+          # Read current word
+          all_segments = @segment_builder.segments
+          current = if @segment_id < all_segments.length
+                      all_segments[@segment_id][@word_offset + word_index] || 0
+                    else
+                      0
+                    end
+
+          # Create mask and insert value
+          mask = ((1 << bit_width) - 1) << bit_offset
+          cleared = current & ~mask
+          new_value = cleared | ((value << bit_offset) & mask)
+
+          @segment_builder.write_word(@segment_id, @word_offset + word_index,
+                                      new_value)
+        end
+
+        # Write unsigned 8-bit integer
+        def write_uint8(word_index, byte_offset, value)
+          write_data_field(word_index, byte_offset * 8, 8, value & 0xFF)
+        end
+
+        # Write signed 8-bit integer
+        def write_int8(word_index, byte_offset, value)
+          unsigned = value.negative? ? value + 256 : value
+          write_uint8(word_index, byte_offset, unsigned)
+        end
+
+        # Write unsigned 16-bit integer
+        def write_uint16(word_index, half_word_offset, value)
+          write_data_field(word_index, half_word_offset * 16, 16,
+                           value & 0xFFFF)
+        end
+
+        # Write signed 16-bit integer
+        def write_int16(word_index, half_word_offset, value)
+          unsigned = value.negative? ? value + 65536 : value
+          write_uint16(word_index, half_word_offset, unsigned)
+        end
+
+        # Write unsigned 32-bit integer
+        def write_uint32(word_index, dword_offset, value)
+          write_data_field(word_index, dword_offset * 32, 32,
+                           value & 0xFFFFFFFF)
+        end
+
+        # Write signed 32-bit integer
+        def write_int32(word_index, dword_offset, value)
+          unsigned = value.negative? ? value + 4294967296 : value
+          write_uint32(word_index, dword_offset, unsigned)
+        end
+
+        # Write unsigned 64-bit integer
+        def write_uint64(word_index, value)
+          @segment_builder.write_word(@segment_id, @word_offset + word_index,
+                                      value & 0xFFFFFFFFFFFFFFFF)
+        end
+
+        # Write signed 64-bit integer
+        def write_int64(word_index, value)
+          unsigned = value.negative? ? value + 18446744073709551616 : value
+          write_uint64(word_index, unsigned)
+        end
+
+        # Write 32-bit float
+        def write_float32(word_index, dword_offset, value)
+          bits = [value].pack("f").unpack1("L")
+          write_uint32(word_index, dword_offset, bits)
+        end
+
+        # Write 64-bit float
+        def write_float64(word_index, value)
+          bits = [value].pack("d").unpack1("Q")
+          write_uint64(word_index, bits)
+        end
+
+        # Write boolean
+        def write_bool(word_index, bit_offset, value)
+          write_data_field(word_index, bit_offset, 1, value ? 1 : 0)
+        end
+
+        # Write a pointer to pointer section
+        # @param pointer_index [Integer] Pointer index in pointer section
+        # @param pointer_word [Integer] Encoded pointer word
+        def write_pointer(pointer_index, pointer_word)
+          return if pointer_index >= @pointer_words
+
+          pointer_word_offset = @word_offset + @data_words + pointer_index
+          @segment_builder.write_word(@segment_id, pointer_word_offset,
+                                      pointer_word)
+        end
+
+        # Get pointer position for offset calculations
+        # @param pointer_index [Integer] Pointer index
+        # @return [Integer] Absolute word offset of pointer
+        def pointer_position(pointer_index)
+          @word_offset + @data_words + pointer_index
+        end
+      end
+    end
+  end
+end

data/lib/unibuf/serializers/flatbuffers/binary_serializer.rb

@@ -0,0 +1,167 @@
+# frozen_string_literal: true
+
+module Unibuf
+  module Serializers
+    module Flatbuffers
+      # FlatBuffers binary format serializer - Pure Ruby
+      class BinarySerializer
+        attr_reader :schema
+
+        def initialize(schema)
+          @schema = schema
+        end
+
+        def serialize(data)
+          raise ArgumentError, "Data cannot be nil" if data.nil?
+
+          root_table_def = schema.find_table(schema.root_type)
+          unless root_table_def
+            raise Unibuf::SerializationError,
+                  "Root type '#{schema.root_type}' not found in schema"
+          end
+
+          # Manual buffer construction
+          []
+
+          # Serialize table and collect parts
+          table_parts = serialize_table(data, root_table_def)
+
+          # Calculate positions
+          root_offset = 4 # After root offset itself
+
+          # Build final buffer
+          buffer = [root_offset].pack("L<") # Root offset
+          buffer += table_parts.pack("C*")
+
+          buffer
+        end
+
+        def serialize_to_file(data, path)
+          File.binwrite(path, serialize(data))
+        end
+
+        private
+
+        def serialize_table(data, table_def)
+          # PHASE 1: Write strings (out-of-line data)
+          string_data = {}
+          string_bytes_total = []
+
+          table_def.fields.each do |field_def|
+            value = data[field_def.name]
+            next unless value && field_def.type == "string"
+
+            # Serialize string
+            str_bytes = []
+            str_bytes.concat([value.bytesize].pack("L<").bytes) # length
+            str_bytes.concat(value.bytes) # content
+            str_bytes << 0 # null terminator
+
+            # Align to 4
+            while (str_bytes.size % 4) != 0
+              str_bytes << 0
+            end
+
+            string_data[field_def.name] = {
+              bytes: str_bytes,
+              start_in_string_section: string_bytes_total.size,
+            }
+            string_bytes_total.concat(str_bytes)
+          end
+
+          # PHASE 2: Build table
+          table_bytes = []
+
+          # Reserve vtable offset
+          table_bytes.push(0, 0, 0, 0)
+
+          # Calculate vtable size
+          vtable_size = 4 + (table_def.fields.size * 2)
+
+          # Write fields
+          field_offsets = []
+          current_pos = 4 # Start after vtable offset
+
+          table_def.fields.each do |field_def|
+            value = data[field_def.name]
+
+            if value.nil?
+              field_offsets << 0
+            elsif field_def.scalar?
+              field_offsets << current_pos
+              # Write scalar value
+              case field_def.type
+              when "byte"
+                table_bytes.concat([value].pack("c").bytes)
+                current_pos += 1
+              when "ubyte", "bool"
+                val = if value.is_a?(TrueClass)
+                        1
+                      else
+                        (value.is_a?(FalseClass) ? 0 : value)
+                      end
+                table_bytes << (val & 0xFF)
+                current_pos += 1
+              when "short"
+                table_bytes.concat([value].pack("s<").bytes)
+                current_pos += 2
+              when "ushort"
+                table_bytes.concat([value].pack("S<").bytes)
+                current_pos += 2
+              when "int"
+                table_bytes.concat([value].pack("l<").bytes)
+                current_pos += 4
+              when "uint"
+                table_bytes.concat([value].pack("L<").bytes)
+                current_pos += 4
+              when "long"
+                table_bytes.concat([value].pack("q<").bytes)
+                current_pos += 8
+              when "ulong"
+                table_bytes.concat([value].pack("Q<").bytes)
+                current_pos += 8
+              when "float"
+                table_bytes.concat([value].pack("e").bytes)
+                current_pos += 4
+              when "double"
+                table_bytes.concat([value].pack("E").bytes)
+                current_pos += 8
+              end
+            elsif field_def.type == "string" && string_data[field_def.name]
+              field_offsets << current_pos
+              # Calculate uoffset from current field position to string
+              # String will be at: 4 (root) + table_size + vtable_size + string_start
+              str_info = string_data[field_def.name]
+              table_size = table_bytes.size + 4 # +4 for remaining field
+              string_abs_pos = 4 + table_size + vtable_size + str_info[:start_in_string_section]
+              # uoffset from field position in final buffer
+              field_abs_pos = 4 + current_pos
+              uoffset = string_abs_pos - field_abs_pos
+              table_bytes.concat([uoffset].pack("L<").bytes)
+              current_pos += 4
+            else
+              field_offsets << 0
+            end
+          end
+
+          # PHASE 3: Build vtable
+          vtable_bytes = []
+          object_size = table_bytes.size
+
+          vtable_bytes.concat([vtable_size].pack("S<").bytes)
+          vtable_bytes.concat([object_size].pack("S<").bytes)
+          field_offsets.each do |off|
+            vtable_bytes.concat([off].pack("S<").bytes)
+          end
+
+          # Patch vtable offset
+          vtable_offset = -object_size
+          table_bytes[0..3] = [vtable_offset].pack("l<").bytes
+
+          # Return table + vtable + strings
+          table_bytes + vtable_bytes + string_bytes_total
+        end
+      end
+    end
+  end
+end

data/lib/unibuf/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Unibuf
-  VERSION = "0.1.1"
+  VERSION = "0.1.2"
 end

data/lib/unibuf.rb

@@ -21,6 +21,10 @@ module Unibuf
     # FlatBuffers schema parser
     module Flatbuffers
     end
+
+    # Cap'n Proto schema parser
+    module Capnproto
+    end
   end
 
   # Module for all models
@@ -31,6 +35,10 @@ module Unibuf
   module Validators
   end
 
+  # Module for serializers
+  module Serializers
+  end
+
   class << self
     # ===== TEXT FORMAT PARSING (no schema required) =====
 
@@ -123,6 +131,22 @@ module Unibuf
       Parsers::Flatbuffers::BinaryParser.new(schema).parse(content)
     end
 
+    # ===== CAP'N PROTO =====
+
+    # Parse Cap'n Proto schema file
+    # @param path [String] Path to .capnp file
+    # @return [Models::Capnproto::Schema] Cap'n Proto schema
+    def parse_capnproto_schema(path)
+      require_relative "unibuf/parsers/capnproto/grammar"
+      require_relative "unibuf/parsers/capnproto/processor"
+
+      grammar = Parsers::Capnproto::Grammar.new
+      content = File.read(path)
+      ast = grammar.parse(content)
+      Parsers::Capnproto::Processor.process(ast)
+    end
+    alias parse_capnp parse_capnproto_schema
+
     # ===== AUTO-DETECTION (convenience methods) =====
 
     # Auto-detect format and parse
@@ -158,6 +182,9 @@ module Unibuf
       when ".fbs"
         raise ArgumentError,
               ".fbs files are schemas, use parse_flatbuffers_schema()"
+      when ".capnp"
+        raise ArgumentError,
+              ".capnp files are schemas, use parse_capnproto_schema()"
       when ".pb"
         # Ambiguous extension - try to detect
         detect_and_parse_pb(path, schema)

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: unibuf
 version: !ruby/object:Gem::Version
-  version: 0.1.1
+  version: 0.1.2
 platform: ruby
 authors:
 - Ronald Tse
@@ -78,8 +78,13 @@ files:
 - ".rspec"
 - ".rubocop.yml"
 - ".rubocop_todo.yml"
+- CODE_OF_CONDUCT.md
 - README.adoc
 - Rakefile
+- docs/CAPNPROTO.adoc
+- docs/FLATBUFFERS.adoc
+- docs/PROTOBUF.adoc
+- docs/TXTPROTO.adoc
 - exe/unibuf
 - lib/unibuf.rb
 - lib/unibuf/cli.rb
@@ -88,9 +93,22 @@ files:
 - lib/unibuf/commands/schema.rb
 - lib/unibuf/commands/validate.rb
 - lib/unibuf/errors.rb
+- lib/unibuf/models/capnproto/enum_definition.rb
+- lib/unibuf/models/capnproto/field_definition.rb
+- lib/unibuf/models/capnproto/interface_definition.rb
+- lib/unibuf/models/capnproto/method_definition.rb
+- lib/unibuf/models/capnproto/schema.rb
+- lib/unibuf/models/capnproto/struct_definition.rb
+- lib/unibuf/models/capnproto/union_definition.rb
 - lib/unibuf/models/enum_definition.rb
 - lib/unibuf/models/field.rb
 - lib/unibuf/models/field_definition.rb
+- lib/unibuf/models/flatbuffers/enum_definition.rb
+- lib/unibuf/models/flatbuffers/field_definition.rb
+- lib/unibuf/models/flatbuffers/schema.rb
+- lib/unibuf/models/flatbuffers/struct_definition.rb
+- lib/unibuf/models/flatbuffers/table_definition.rb
+- lib/unibuf/models/flatbuffers/union_definition.rb
 - lib/unibuf/models/message.rb
 - lib/unibuf/models/message_definition.rb
 - lib/unibuf/models/schema.rb
@@ -100,11 +118,28 @@ files:
 - lib/unibuf/models/values/message_value.rb
 - lib/unibuf/models/values/scalar_value.rb
 - lib/unibuf/parsers/binary/wire_format_parser.rb
+- lib/unibuf/parsers/capnproto/binary_parser.rb
+- lib/unibuf/parsers/capnproto/grammar.rb
+- lib/unibuf/parsers/capnproto/list_reader.rb
+- lib/unibuf/parsers/capnproto/pointer_decoder.rb
+- lib/unibuf/parsers/capnproto/processor.rb
+- lib/unibuf/parsers/capnproto/segment_reader.rb
+- lib/unibuf/parsers/capnproto/struct_reader.rb
+- lib/unibuf/parsers/flatbuffers/binary_parser.rb
+- lib/unibuf/parsers/flatbuffers/grammar.rb
+- lib/unibuf/parsers/flatbuffers/processor.rb
 - lib/unibuf/parsers/proto3/grammar.rb
 - lib/unibuf/parsers/proto3/processor.rb
 - lib/unibuf/parsers/textproto/grammar.rb
 - lib/unibuf/parsers/textproto/parser.rb
 - lib/unibuf/parsers/textproto/processor.rb
+- lib/unibuf/serializers/binary_serializer.rb
+- lib/unibuf/serializers/capnproto/binary_serializer.rb
+- lib/unibuf/serializers/capnproto/list_writer.rb
+- lib/unibuf/serializers/capnproto/pointer_encoder.rb
+- lib/unibuf/serializers/capnproto/segment_builder.rb
+- lib/unibuf/serializers/capnproto/struct_writer.rb
+- lib/unibuf/serializers/flatbuffers/binary_serializer.rb
 - lib/unibuf/validators/schema_validator.rb
 - lib/unibuf/validators/type_validator.rb
 - lib/unibuf/version.rb