protostar 0.1.0.beta1

data/lib/protostar/view/scan.rb

@@ -0,0 +1,183 @@
+# frozen_string_literal: true
+
+module Protostar
+  class View
+    # What the generated index builders and the lazy containers share: the
+    # rare paths of the scan, and turning indexed positions into values.
+    # Every check mirrors the eager decoder's, with the same messages and offsets.
+    # @api private
+    module Scan
+      F = Message::FieldInfo
+      UTF_8 = Encoding::UTF_8
+      BINARY = Encoding::BINARY
+
+      module_function
+
+      # A tag the class has no branch for: an end-group tag, a malformed tag,
+      # an unknown field or a known one with the wrong wire type. The field is
+      # skipped and its range kept for {View#unknown_fields}.
+      def other(c, tag, at, ix, u, depth)
+        number = tag >> 3
+        wt = tag & 7
+        raise DecodeError.new("unmatched end of group #{number}", offset: at) if wt == Wire::END_GROUP
+        Wire::Reader.check_tag(tag, at)
+        start = c.pos
+        c.skip_field(number, wt, depth)
+        (ix[u] ||= []) << number << wt << start << c.pos
+      end
+
+      # Skips a group's contents and its end tag
+      # @return [Integer] the offset of the end tag, where the contents stop
+      def group_end(c, number, depth)
+        loop do
+          raise DecodeError.new("missing end of group #{number}", offset: c.pos) if c.pos >= c.limit
+          at = c.pos
+          tag = c.read_varint
+          n = tag >> 3
+          wt = tag & 7
+          if wt == Wire::END_GROUP
+            return at if n == number
+            raise DecodeError.new("unmatched end of group #{n}", offset: at)
+          end
+          Wire::Reader.check_tag(tag, at)
+          c.skip_field(n, wt, depth)
+        end
+      end
+
+      # Checks that a packed run is whole varints, as reading it would
+      def check_varints(buf, pos, limit)
+        while pos < limit
+          start = pos
+          shift = 0
+          loop do
+            raise DecodeError.new("truncated varint", offset: start) if pos >= limit
+            byte = buf.getbyte(pos)
+            pos += 1
+            break if byte < 0x80
+            shift += 7
+            raise DecodeError.new("varint longer than 10 bytes", offset: start) if shift >= 70
+          end
+        end
+      end
+
+      # A packed run of a closed enum: numbers without a name become unknown fields
+      def packed_closed_enum(c, pos, limit, ix, u, number, names)
+        c.pos = pos
+        old = c.push_limit(limit - pos)
+        while c.pos < limit
+          start = c.pos
+          n = c.read_int32
+          (ix[u] ||= []) << number << Wire::VARINT << start << c.pos unless names.key?(n)
+        end
+        c.pop_limit(old)
+      end
+
+      # @return [Boolean] whether a map entry's value is a closed enum's
+      #   unknown number, which sends the whole entry to the unknown fields
+      def map_unknown_enum?(buf, pos, length, info, depth)
+        Map.entry(buf, pos, length, info, depth)[2]
+      end
+
+      def string(buf, offset, length, info)
+        s = buf.byteslice(offset, length).force_encoding(UTF_8)
+        if info.utf8_validation && !s.valid_encoding?
+          raise DecodeError.new("invalid UTF-8 in #{info.descriptor.full_name}", offset: offset)
+        end
+        s.freeze
+      end
+
+      def bytes(buf, offset, length)
+        buf.byteslice(offset, length).force_encoding(BINARY).freeze
+      end
+
+      # The value of a varint-encoded kind from the raw unsigned varint;
+      # enums arrive already narrowed to int32
+      def varint_value(kind, v, info)
+        case kind
+        when F::INT32
+          v &= Wire::MASK32
+          (v >= 0x8000_0000) ? v - 0x1_0000_0000 : v
+        when F::INT64 then (v >= 0x8000_0000_0000_0000) ? v - 0x1_0000_0000_0000_0000 : v
+        when F::UINT32 then v & Wire::MASK32
+        when F::UINT64 then v
+        when F::SINT32
+          v &= Wire::MASK32
+          (v >> 1) ^ -(v & 1)
+        when F::SINT64 then (v >> 1) ^ -(v & 1)
+        when F::BOOL then v != 0
+        when F::ENUM then info.enum_names[v] || v
+        end
+      end
+
+      # Every element of a repeated scalar field. `entries` holds pairs: a
+      # value (varint kinds) or an offset (fixed kinds) and -1 for one
+      # element, or an offset and a length for a packed run.
+      # @return [Array] frozen
+      def scalars(buf, info, entries)
+        out = []
+        kind = info.kind
+        k = 0
+        size = entries.size
+        if (fixed = F::FIXED[kind])
+          width, directive = fixed
+          while k < size
+            a = entries[k]
+            b = entries[k + 1]
+            if b < 0
+              out << buf.unpack1(directive, offset: a)
+            else
+              stop = a + b
+              while a < stop
+                out << buf.unpack1(directive, offset: a)
+                a += width
+              end
+            end
+            k += 2
+          end
+        else
+          c = nil
+          enum = kind == F::ENUM
+          names = info.enum_names
+          while k < size
+            a = entries[k]
+            b = entries[k + 1]
+            if b < 0
+              out << varint_value(kind, a, info)
+            else
+              c ||= Wire::Reader.new(buf, 0, buf.bytesize)
+              c.pos = a
+              old = c.push_limit(b)
+              stop = a + b
+              while c.pos < stop
+                if enum
+                  n = c.read_int32
+                  if (name = names[n])
+                    out << name
+                  elsif !info.closed_enum
+                    out << n
+                  end
+                else
+                  out << varint_value(kind, c.read_varint, info)
+                end
+              end
+              c.pop_limit(old)
+            end
+            k += 2
+          end
+        end
+        out.freeze
+      end
+
+      # A nested view over the segments of a message value
+      def nested(info, buf, segments, depth)
+        raise DecodeError.new("recursion limit exceeded", offset: segments[0]) if depth <= 0
+        klass = info.message_class.__view_class
+        case segments.size
+        when 0 then klass.__new(buf, 0, 0, nil, depth - 1)
+        when 2 then klass.__new(buf, segments[0], segments[1], nil, depth - 1)
+        else klass.__new(buf, segments[0], segments[1], segments.freeze, depth - 1)
+        end
+      end
+    end
+  end
+end

data/lib/protostar/view.rb

@@ -0,0 +1,283 @@
+# frozen_string_literal: true
+
+module Protostar
+  # A read-only message over the bytes it was encoded in. Making a view
+  # scans the top level once and keeps where each field is; nothing is
+  # decoded until a reader asks, and what a reader decodes is kept. Nested
+  # messages are views over slices of the same buffer, so reading one
+  # field of a large message copies nothing but that field's value.
+  #
+  #   view = Person.view(bytes)          # a String or an IO::Buffer
+  #   view.name                          # => "Ada"
+  #   view.friends[2].name               # a nested view, same buffer
+  #   view.encode.equal?(bytes)          # => true
+  #   view.to_message                    # the mutable message, decoded eagerly
+  #
+  # Every message class has a view class, `Person.view_class`, with the
+  # same readers as the message: scalars, enums, `has_`, `which_`,
+  # repeated fields as {Repeated}, maps as {Map}. Views are frozen; writers
+  # raise FrozenError. A view reads the buffer it was given for as long as
+  # it lives, so that buffer must not change underneath it.
+  #
+  # Malformed input raises {DecodeError} when the view is made, for the top
+  # level; a nested message is checked when it is first read, and a string
+  # with invalid UTF-8 when the string is read.
+  class View
+    class << self
+      # @return [Class] the message class this view class reads
+      attr_reader :message_class
+
+      # @api private
+      attr_reader :__fields, :__by_name, :__readers, :__unknown_slot
+
+      # @return [Schema::Message]
+      def descriptor
+        @message_class.descriptor
+      end
+
+      def inspect
+        name || "#<Class #{@message_class ? "#{@message_class.descriptor.full_name}::View" : "Protostar::View"}>"
+      end
+
+      # @api private
+      def __define(message_class, fields)
+        @message_class = message_class
+        @__fields = message_class.__fields
+        @__by_name = message_class.__by_name
+        @__readers = Ractor.make_shareable(fields.map { |f| :"__f#{f.index}" })
+        @__unknown_slot = 3 * fields.size
+        self
+      end
+
+      # @api private
+      def __new(buf, offset, length, segments, depth)
+        new(buf, offset, length, segments, depth)
+      end
+
+      # @api private
+      def __open(bytes, offset, length, depth)
+        raise NotImplementedError, "#{self} is abstract; views come from message classes" unless @message_class
+        buf = if bytes.is_a?(String)
+          bytes
+        elsif defined?(IO::Buffer) && bytes.is_a?(IO::Buffer)
+          Wire::IOSource.new(bytes)
+        else
+          raise ::TypeError, "expected a String or IO::Buffer, got #{bytes.class}"
+        end
+        size = buf.bytesize
+        length ||= size - offset
+        if offset < 0 || length < 0 || offset + length > size
+          raise ArgumentError, "offset #{offset} and length #{length} do not fit a buffer of #{size} bytes"
+        end
+        new(buf, offset, length, nil, depth)
+      end
+
+      private :new
+    end
+
+    # @return [Schema::Message]
+    def descriptor
+      self.class.descriptor
+    end
+
+    # @param name [String, Symbol]
+    # @return [Object] what the field's reader returns
+    # @raise [ArgumentError] when there is no such field
+    def [](name)
+      info = self.class.__by_name[name] || raise(ArgumentError, "#{descriptor.full_name} has no field #{name.inspect}")
+      __send__(self.class.__readers[info.index])
+    end
+
+    # @raise [FrozenError] always
+    def []=(_name, _value)
+      __frozen!
+    end
+
+    # @raise [FrozenError] always
+    def merge!(_other)
+      __frozen!
+    end
+
+    # @raise [FrozenError] always
+    def clear!
+      __frozen!
+    end
+
+    # @return [Array<UnknownField>] frozen, in the order they occur
+    def unknown_fields
+      ix = @ix
+      u = self.class.__unknown_slot
+      cached = ix[u + 1]
+      return cached if cached
+      ranges = ix[u]
+      return Message::EMPTY_ARRAY if ranges.nil?
+      list = []
+      k = 0
+      while k < ranges.size
+        start = ranges[k + 2]
+        bytes = @buf.byteslice(start, ranges[k + 3] - start).force_encoding(Encoding::BINARY).freeze
+        list << UnknownField.new(ranges[k], ranges[k + 1], bytes)
+        k += 4
+      end
+      ix[u + 1] = list.freeze
+    end
+
+    # @return [String] the bytes this view reads: the very String it was
+    #   made from when it covers all of it, otherwise a copy of its part
+    def encode
+      buf = @buf
+      if @segs.nil?
+        return buf if @off == 0 && buf.is_a?(String) && @len == buf.bytesize
+        return buf.byteslice(@off, @len).force_encoding(Encoding::BINARY)
+      end
+      out = (+"").force_encoding(Encoding::BINARY)
+      __each_segment { |offset, length| out << buf.byteslice(offset, length) }
+      out
+    end
+    alias_method :to_proto, :encode
+
+    # Decodes the view's bytes into a message, as {Message.decode} would
+    # @param frozen [Boolean] return a deep-frozen, Ractor-shareable message
+    # @return [Message]
+    def to_message(frozen: false)
+      msg = self.class.message_class.__empty
+      buf = @buf
+      depth = @depth
+      __each_segment do |offset, length|
+        c = Wire::Reader.new(buf, offset, offset + length)
+        msg.__decode_fields(c, depth, 0, frozen)
+      end
+      msg.freeze if frozen
+      msg
+    end
+
+    # @return [Hash] what {Message#to_h} gives for the same bytes
+    def to_h
+      __to_h
+    end
+
+    # @api private
+    def __to_h
+      h = {}
+      ix = @ix
+      readers = self.class.__readers
+      self.class.__fields.each do |info|
+        next if ix[info.index].nil?
+        v = __send__(readers[info.index])
+        if info.map
+          next if v.empty?
+          h[info.symbol] = info.value.message? ? v.to_h.transform_values(&:__to_h) : v.to_h.dup
+        elsif info.repeated
+          next if v.empty?
+          h[info.symbol] = info.message? ? v.map(&:__to_h) : v.to_a.dup
+        elsif info.message?
+          h[info.symbol] = v.__to_h
+        elsif info.presence || !info.zero?(v)
+          h[info.symbol] = v
+        end
+      end
+      h
+    end
+
+    # Equal to a message of the view's class, or a view of it, with the
+    # same field values (unknown fields aside), as {Message#==} compares
+    def ==(other)
+      return true if equal?(other)
+      case other
+      when View then other.instance_of?(self.class) && to_message == other.to_message
+      when Message then other.instance_of?(self.class.message_class) && to_message == other
+      else false
+      end
+    end
+    alias_method :eql?, :==
+
+    def hash
+      to_message.hash
+    end
+
+    # Reads every field, nested views and containers included, and freezes
+    # what was read. A view over a frozen String is then Ractor-shareable.
+    # @return [self]
+    def freeze_index
+      ix = @ix
+      return self if ix.frozen?
+      readers = self.class.__readers
+      self.class.__fields.each do |info|
+        next if ix[info.index].nil?
+        v = __send__(readers[info.index])
+        v.freeze_index if v.is_a?(View) || v.is_a?(Repeated) || v.is_a?(Map)
+      end
+      unknown_fields
+      ix.each { |e| e.freeze if e.is_a?(Array) }
+      @segs&.freeze
+      ix.freeze
+      self
+    end
+
+    def inspect
+      parts = __shown.map { |name, v| "#{name}: #{v.inspect}" }
+      name = "view #{descriptor.full_name}"
+      parts.empty? ? "#<#{name}>" : "#<#{name} #{parts.join(", ")}>"
+    end
+    alias_method :to_s, :inspect
+
+    # For `pp`, laid out as {Message#pretty_print} lays out messages
+    def pretty_print(q)
+      Message.__pretty_print(q, "view #{descriptor.full_name}", __shown)
+    end
+
+    # @return [Array<Array(String, Object)>] the fields `inspect` shows
+    # @api private
+    def __shown
+      shown = []
+      ix = @ix
+      readers = self.class.__readers
+      self.class.__fields.each do |info|
+        next if ix[info.index].nil?
+        v = __send__(readers[info.index])
+        next if (info.repeated || info.map) && v.empty?
+        next if !info.presence && !info.repeated && !info.map && info.zero?(v)
+        shown << [info.name, v]
+      end
+      shown
+    end
+
+    # @return [Array<Array(Integer, Integer)>] offset and length of each
+    #   piece of the buffer this view reads, in order
+    # @api private
+    def __segments
+      list = []
+      __each_segment { |offset, length| list << [offset, length] }
+      list
+    end
+
+    private
+
+    def __each_segment
+      segs = @segs
+      if segs.nil?
+        yield @off, @len
+      else
+        k = 0
+        while k < segs.size
+          yield segs[k], segs[k + 1]
+          k += 2
+        end
+      end
+    end
+
+    def __nested(i, o, n)
+      info = self.class.__fields[i]
+      o.is_a?(Integer) ? Scan.nested(info, @buf, [o, n], @depth) : Scan.nested(info, @buf, o, @depth)
+    end
+
+    def __frozen!
+      raise FrozenError.new("can't modify a view of #{descriptor.full_name}", receiver: self)
+    end
+  end
+end
+
+require_relative "view/scan"
+require_relative "view/repeated"
+require_relative "view/map"
+require_relative "view/generator"

data/lib/protostar/well_known/any.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+module Protostar
+  module WellKnown
+    # Mixed into google.protobuf.Any
+    module Any
+      DEFAULT_PREFIX = "type.googleapis.com/"
+
+      def self.included(base)
+        base.extend(ClassMethods)
+      end
+
+      module ClassMethods
+        # @param message [Message]
+        # @param type_url_prefix [String]
+        # @return [Message] a new Any holding the message
+        def pack(message, type_url_prefix = DEFAULT_PREFIX)
+          new.pack(message, type_url_prefix)
+        end
+      end
+
+      # Stores the message's bytes and a type URL of the prefix and its full
+      # name; a `/` is put between them when the prefix lacks one
+      # @param message [Message]
+      # @param type_url_prefix [String]
+      # @return [self]
+      def pack(message, type_url_prefix = DEFAULT_PREFIX)
+        name = message.class.descriptor.full_name
+        self.type_url = type_url_prefix.end_with?("/") ? "#{type_url_prefix}#{name}" : "#{type_url_prefix}/#{name}"
+        self.value = message.encode
+        self
+      end
+
+      # @param klass [Class] the message class expected inside
+      # @return [Message, nil] the decoded message, or nil when the Any holds another type
+      def unpack(klass)
+        klass.decode(value) if is(klass)
+      end
+
+      # @return [String] the full name at the end of the type URL
+      def type_name
+        type_url.split("/")[-1]
+      end
+
+      # @param klass [Class]
+      # @return [Boolean] whether the Any holds a message of that class
+      def is(klass)
+        type_name == klass.descriptor.full_name
+      end
+    end
+  end
+end

data/lib/protostar/well_known/duration.rb

@@ -0,0 +1,53 @@
+# frozen_string_literal: true
+
+module Protostar
+  module WellKnown
+    # Mixed into google.protobuf.Duration
+    module Duration
+      # Duration's range in seconds, about 10,000 years
+      MAX_SECONDS = 315_576_000_000
+      NANOS = 1_000_000_000
+
+      def self.included(base)
+        base.extend(ClassMethods)
+      end
+
+      # Splits a number of seconds into whole seconds and nanoseconds of the
+      # same sign, rounding to the nearest nanosecond
+      # @param value [Object]
+      # @return [Array(Integer, Integer), nil] nil when the value is not a Numeric
+      # @raise [FloatDomainError] for NaN and Infinity
+      # @raise [RangeError] beyond {MAX_SECONDS}
+      def self.parts(value)
+        return nil unless value.is_a?(Numeric)
+        r = value.to_r
+        seconds = r.truncate
+        nanos = ((r - seconds) * NANOS).round
+        if nanos.abs >= NANOS
+          seconds += nanos <=> 0
+          nanos -= (nanos <=> 0) * NANOS
+        end
+        raise RangeError, "#{value.inspect} is out of Duration's range" if seconds.abs > MAX_SECONDS
+        [seconds, nanos]
+      end
+
+      module ClassMethods
+        # What a Duration field makes of a value that is neither a Duration
+        # nor a Hash: a Numeric of seconds (Integer, Float, Rational) becomes
+        # a Duration
+        # @return [Message, nil]
+        # @raise [RangeError] for NaN, Infinity, or beyond Duration's range
+        # @api private
+        def __coerce(value)
+          seconds, nanos = Duration.parts(value)
+          new(seconds: seconds, nanos: nanos) if seconds
+        end
+      end
+
+      # @return [Float] seconds
+      def to_f
+        seconds + nanos.to_f / 1_000_000_000
+      end
+    end
+  end
+end

data/lib/protostar/well_known/list_value.rb

@@ -0,0 +1,63 @@
+# frozen_string_literal: true
+
+module Protostar
+  module WellKnown
+    # Mixed into google.protobuf.ListValue: an Array of plain Ruby values
+    module ListValue
+      include Enumerable
+
+      def self.included(base)
+        base.extend(ClassMethods)
+      end
+
+      module ClassMethods
+        # @param array [Array]
+        # @return [Message] a new ListValue
+        # @raise [UnexpectedStructType]
+        def from_a(array)
+          list = new
+          array.each { |v| list << v }
+          list
+        end
+
+        # @api private
+        def __value_class
+          __by_name.fetch("values").message_class
+        end
+      end
+
+      # @return [Integer]
+      def length
+        values.length
+      end
+
+      # @return [Object] the element as Ruby, Structs and ListValues left as they are
+      def [](index)
+        values[index].to_ruby
+      end
+
+      def []=(index, value)
+        values[index].from_ruby(value)
+      end
+
+      # Appends a Ruby value
+      # @return [self]
+      def <<(value)
+        values << self.class.__value_class.new.from_ruby(value)
+        self
+      end
+
+      # Yields each element as Ruby, Structs and ListValues left as they are
+      def each
+        return enum_for(:each) { length } unless block_given?
+        values.each { |v| yield v.to_ruby }
+        self
+      end
+
+      # @return [Array] with nested Structs and ListValues converted too
+      def to_a
+        values.map { |v| v.to_ruby(true) }
+      end
+    end
+  end
+end

data/lib/protostar/well_known/struct.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+module Protostar
+  module WellKnown
+    # Mixed into google.protobuf.Struct: a Hash of String keys to plain Ruby
+    # values (nil, Numeric, String, true, false, Hash, Array)
+    module Struct
+      def self.included(base)
+        base.extend(ClassMethods)
+      end
+
+      module ClassMethods
+        # @param hash [Hash{String => Object}]
+        # @return [Message] a new Struct
+        # @raise [UnexpectedStructType]
+        def from_hash(hash)
+          struct = new
+          hash.each { |key, value| struct[key] = value }
+          struct
+        end
+
+        # @api private
+        def __value_class
+          __by_name.fetch("fields").value.message_class
+        end
+      end
+
+      # @param key [String]
+      # @return [Object, nil] the value as Ruby, Structs and ListValues
+      #   left as they are; nil for a missing key
+      def [](key)
+        fields[key]&.to_ruby
+      end
+
+      # @param key [String]
+      # @param value [Object]
+      # @raise [UnexpectedStructType] for a key that is not a String or a value Struct cannot hold
+      def []=(key, value)
+        raise UnexpectedStructType, "Struct keys must be strings." unless key.is_a?(String)
+        (fields[key] ||= self.class.__value_class.new).from_ruby(value)
+      end
+
+      # @return [Hash{String => Object}] with nested Structs and ListValues
+      #   converted too
+      def to_h
+        fields.transform_values { |v| v.to_ruby(true) }
+      end
+
+      # @param key [String]
+      def has_key?(key)
+        fields.key?(key)
+      end
+    end
+  end
+end