fast-protowire 0.1.0 → 0.3.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a0c2962e443067325c5f4a089b72e499b9e273f8f563464e9830ed9cdf6eccfa
-  data.tar.gz: cdc454d06627bf2d1f28ddc35127024e119e06e748120cfceac91fa7e4e40860
+  metadata.gz: 9d6f10862b8163b7b4d5210479a6b27b62298e0951b24425db97cdec94356a10
+  data.tar.gz: 5a882709766e557ae6a65b3107ec3cc8de6fba07f90de3293607518cf880b4f8
 SHA512:
-  metadata.gz: 5b6b5ea5c39ebc3fb9b25490fbea312dccdaaf450ab08b7a9be794056dbc3565e814f98890629664e405333e340e61d493760a57adc99b072b2cf59ff8399bc6
-  data.tar.gz: b788d96fe73458a33391b5339d35f7be57a37ec3bb203a2d645c590a5cb98d6ef4f9a5641788d8a12f20c7a51911c27287cba01cbac49e0d4b3c9e22f944be1d
+  metadata.gz: 8a54a74df181260136e00d9227f050c1ef785096515b4a91197f939af58df1d865bea3eb5e660db97335111a1728407edccfce11afd8b60a4335076e95b1a258
+  data.tar.gz: 5048f2b557f2e3c4f44b4a2cb95110b11142c6b7c94975dbca68f7a3fc1550ea2e9b120460b4e36bb2ed7f316bdb476b60b2b334f449728c2a8ba869242a8036

data/AGENTS.md

@@ -8,7 +8,9 @@ Standing context for agents working in this repository.
 bundle install
 bundle exec sus
 bundle exec rubocop
-protoc --proto_path=fixtures/proto --ruby_out=fixtures/pb fixtures/proto/*.proto   # regenerate the reference schemas
+bundle exec ruby benchmark/messages.rb   # encode, build and decode against google-protobuf; BENCH_QUICK=1 for a short run
+protoc --proto_path=fixtures/proto --proto_path=/opt/homebrew/include --ruby_out=fixtures/pb fixtures/proto/*.proto
+# regenerates the reference schemas; the second path is where protoc keeps google/protobuf/timestamp.proto
 ```
 
 ## What this is

data/CHANGELOG.md

@@ -1,5 +1,90 @@
 # Changelog
 
+## 0.3.0
+
+- A declared field arriving with a wire type it does not accept is kept as
+  an unknown field and written back verbatim, where it used to raise
+  `DecodeError`: schema drift is not corruption, and the reference keeps it.
+  So is a map entry carrying more than a key and a value — an undeclared
+  subfield, or a key or value with a wire type the entry does not accept —
+  kept whole among the parent message's unknown fields and left out of the
+  map. An entry that omits its key or its value decodes to that type's
+  default, an empty message for a message-typed value.
+- Hostile input is bounded rather than fatal. Nesting deeper than
+  `Reader::MAX_DEPTH` (100, the reference's limit; nested messages, map
+  entries and groups count, a packed field does not) raises `DecodeError`
+  instead of overflowing the VM stack with `SystemStackError`. Field number
+  0 anywhere in the input, a group closed by an `END_GROUP` carrying another
+  number, and a proto3 `string` whose bytes are not valid UTF-8 raise
+  `DecodeError` too. A varint carrying bits above 64 is truncated rather
+  than read wide, as every implementation does, and a `sint32` is truncated
+  to 32 bits before its zigzag is undone.
+- Decoding is about bytes, so a `Reader` reads its input whatever the String
+  is tagged: a non-binary one is read through a binary view made once on
+  construction, and the input is never modified. `encode` holds its buffer
+  to the same rule: an empty String of any encoding is retagged binary, and
+  one already holding text raises `ArgumentError` rather than widening the
+  bytes it appends into that encoding's characters.
+- `Message#==`, `eql?` and `hash` compare the declared fields only. Unknown
+  fields are no longer part of the comparison, as the reference does not
+  compare them; they are still kept, still readable through
+  `unknown_fields`, and still written back by `encode`. Byte identity is
+  `a.encode == b.encode`.
+- A `float` field narrows to single precision on assignment rather than on
+  encode, so what it reads back is what the wire carries and
+  `decode(encode(m)) == m` holds: `0.1` reads back `0.10000000149011612`,
+  `3.5e38` reads `Infinity`, `1e-50` reads `0.0`. A 32-bit NaN is still
+  written canonical (`7fc00000`) because Ruby's `pack("e")` drops the sign
+  and payload bits the reference preserves; that one is documented, not
+  fixed. `double` NaNs round-trip whole.
+- A declaration the wire format has no form for raises `ArgumentError` where
+  it is written rather than at the first encode: a field number outside
+  1 ... 2²⁹−1 or inside 19000 ... 19999, which the specification reserves for
+  the implementation, and `packed: true` on a `string`, `bytes` or message
+  field.
+- `Reader#read_packed` is back for packed fields, which are read in place
+  and cost no depth level (0.2.0's entry recorded it removed).
+  `Reader#read_key` returns a tag as the raw key holding number and wire
+  type together; `Reader#skip` takes the field number the value arrived
+  under, `skip(wire_type, number = nil)`, so a group is closed only by its
+  own number; `Field#accepts?(wire_type)`, `Wire.binary_buffer` and
+  `Wire::UINT64_MASK` are public.
+- Decoding costs what it did on 0.2.0, guards included: the 36,000-metric
+  family decodes in 0.801 s where 0.2.0 takes 0.784 s on the same machine
+  and Ruby (4.0.7, five timed calls, same session). A field's wire type is
+  computed once, when the field is declared, rather than on every read,
+  where the per-field check had been paying for it twice. What it allocates
+  is unchanged, 1,404,005 objects for that family and one object to encode
+  it, and so are the budgets in `test/fast/protowire/allocations.rb`; the
+  benchmarks page carries the re-measured tables.
+
+## 0.2.0
+
+- Encoding allocates nothing per nested message, packed field or map
+  entry: each is written into the parent buffer behind a length prefix
+  filled in afterwards. A family of 36,000 metrics x 12 labels encodes in
+  one object, its output, where it took 647,000, and 22% faster.
+  `Wire.append_length_delimited_from` now appends in place and returns the
+  width of the prefix it wrote, a hint to pass back for the next value;
+  `Wire.reserve_length` and `Wire.close_length` are its pieces, and
+  `Wire.varint_size` and `Wire.append_bytes` are public.
+- Fixed-width values pack with literal formats, so a double costs no Array
+  on Ruby 3.4+; text is appended with `String#append_as_bytes` there, tag
+  and one-byte size in the same call and no binary copy for UTF-8.
+  Zigzag no longer allocates for negative values.
+- Decoding reads nested messages, packed fields and map entries in place
+  (`Reader#read_nested`), unpacks fixed-width values at an offset
+  (`Reader#read_fixed`) and splits tags without an Array, so it allocates
+  only the messages, containers and Strings it returns: the same family
+  decodes in 1.4M objects where it took 7.1M, 1.7x faster.
+- `Message.new` no longer allocates an empty keyword Hash per message.
+- Removed: `Reader#read_packed` (use `read_nested`) and the interpretive
+  `Field#encode`, which the compiled encoder had superseded.
+- `test/fast/protowire/allocations.rb` holds the budgets; the Prometheus
+  client model (`fixtures/proto/metrics.proto`, upstream `client_model`)
+  joins the parity suite; `benchmark/messages.rb` measures encode, build
+  and decode against google-protobuf, published on the benchmarks page.
+
 ## 0.1.0
 
 - Initial release: `Wire`, `Reader`, `Enum` and the `Message` DSL covering

data/README.md

@@ -50,6 +50,23 @@ and the output is byte-identical to what `protoc`-generated code and `google-pro
 produce for the same values. The [tutorial](docs/tutorials/encode-a-message.md) walks
 through a full schema and proves that.
 
+## Performance
+
+Measured against `google-protobuf` on the Prometheus client model, a family of 36,000
+metrics with twelve labels each (Ruby 4.0.7; conditions and every table on the
+[benchmarks page](docs/explanation/benchmarks.md)):
+
+- Encoding allocates **one object, the output**, whatever the message's size or depth;
+  0.1.0 allocated 647,000 for this family. Decoding allocates only the messages,
+  containers and Strings it returns, 5x fewer than 0.1.0 and 1.3x faster.
+- Built a message at a time, the way an exposition builds series, `google-protobuf`
+  leaves **504,001 native arenas and 225 MiB** behind for an 11.76 MB body and spends
+  1.77 s of every ten builds in GC; fast-protowire leaves 16.5 MiB, no arenas, and 0.27 s.
+- `google-protobuf` is native, and 9 to 25x faster per operation on an existing tree
+  or one nested Hash. This gem trades that speed for memory that is roughly the size of
+  the output; fast-prometheus's scrape path goes further and writes series with `Wire`
+  directly, with no message per series at all.
+
 ## Documentation
 
 ### Tutorials
@@ -73,10 +90,12 @@ through a full schema and proves that.
 
 - [Design: the wire format and nothing else](docs/explanation/design.md) — what google-protobuf costs per message, what this gem does instead, and what it leaves out.
 - [How encoding works](docs/explanation/encoding.md) — compiled encoders, buffers, field order, presence, and decoding.
+- [Benchmarks](docs/explanation/benchmarks.md) — encode, build and decode against google-protobuf, what changed since 0.1.0, and how to reproduce them.
 
 ## Development
 
 ```bash
 bundle exec sus
 bundle exec rubocop
+BENCH_QUICK=1 bundle exec ruby benchmark/messages.rb   # encode, build and decode against google-protobuf
 ```

data/lib/fast/protowire/field.rb

@@ -30,9 +30,15 @@ module Fast
 
       MAP_KEY_TYPES = %i[int32 int64 uint32 uint64 sint32 sint64 fixed32 fixed64 sfixed32 sfixed64 bool string].freeze
 
-      private_constant :SCALAR_WIRE_TYPES, :INTEGER_RANGES, :FIXED_FORMATS, :PACKABLE, :MAP_KEY_TYPES
+      # The field numbers a tag can carry, less the range the specification
+      # reserves for the implementation.
+      NUMBERS = (1..(1 << 29) - 1)
+      RESERVED_NUMBERS = (19_000..19_999)
 
-      attr_reader :name, :number, :type, :rule, :oneof, :ivar, :enum
+      private_constant :SCALAR_WIRE_TYPES, :INTEGER_RANGES, :FIXED_FORMATS, :PACKABLE, :MAP_KEY_TYPES,
+                       :NUMBERS, :RESERVED_NUMBERS
+
+      attr_reader :name, :number, :type, :rule, :oneof, :ivar, :enum, :wire_type
 
       # +type+ is a scalar Symbol, an Enum module, a Message class, or a
       # String / Proc naming a Message class resolved on first use (for
@@ -40,13 +46,17 @@ module Fast
       # :repeated or :map.
       def initialize(name, type, number, rule:, owner:, packed: nil, default: nil, oneof: nil, key_type: nil)
         @name = name
-        @number = number
+        @number = validate_number(number)
         @rule = rule
         @owner = owner
         @oneof = oneof
         @ivar = :"@#{name}"
         resolve_type(type)
+        @wire_type = rule == :map ? Wire::LENGTH_DELIMITED : SCALAR_WIRE_TYPES.fetch(@type)
+        raise ArgumentError, "#{name}: a #{@type} field cannot be packed" if packed && !packable?
+
         @packed = rule == :repeated && (packed.nil? ? owner.syntax == :proto3 && packable? : packed)
+        @strict_utf8 = @type == :string && owner.syntax == :proto3
         @default = default
         @tag = Wire.tag(number, packed? ? Wire::LENGTH_DELIMITED : wire_type)
         return unless map?
@@ -55,6 +65,9 @@ module Fast
 
         @key_field = Field.new(:key, key_type, 1, rule: :optional, owner: owner)
         @value_field = Field.new(:value, type, 2, rule: :optional, owner: owner)
+        # The only two tags an entry accepts: number and wire type together.
+        @key_tag = (1 << 3) | @key_field.wire_type
+        @value_tag = (2 << 3) | @value_field.wire_type
       end
 
       def message_class
@@ -65,10 +78,6 @@ module Fast
                            end
       end
 
-      def wire_type
-        map? ? Wire::LENGTH_DELIMITED : SCALAR_WIRE_TYPES.fetch(type)
-      end
-
       def repeated?
         rule == :repeated
       end
@@ -85,6 +94,13 @@ module Fast
         PACKABLE.include?(type)
       end
 
+      # Whether a value of +wire_type+ can be read into this field: its own,
+      # plus the packed form of a packable repeated field however it was
+      # declared. Anything else is schema drift, kept as an unknown field.
+      def accepts?(wire_type)
+        wire_type == self.wire_type || (repeated? && packable? && wire_type == Wire::LENGTH_DELIMITED)
+      end
+
       # Whether an unset field is distinguishable from one set to its default.
       def explicit_presence?
         rule != :implicit || !oneof.nil? || type == :message
@@ -141,23 +157,17 @@ module Fast
         end
       end
 
-      def encode(buffer, value)
-        case rule
-        when :repeated then encode_repeated(buffer, value)
-        when :map then encode_map(buffer, value)
-        else encode_one(buffer, value)
-        end
-      end
-
       # Reads one occurrence of this field into +current+ (the value already
-      # held) and returns the value to store.
-      def decode(reader, wire_type, current)
+      # held) and returns the value to store. +message+ is the message being
+      # read into, for a map entry it cannot accept: that belongs among the
+      # message's unknown fields rather than in the map.
+      def decode(reader, wire_type, current, message)
         case rule
         when :repeated then decode_repeated(reader, wire_type, current)
-        when :map then decode_map_entry(reader, wire_type, current)
+        when :map then decode_map_entry(reader, wire_type, current, message)
         else
           if type == :message && current
-            current.merge_from(Reader.new(expect(reader, wire_type).read_length_delimited))
+            expect(reader, wire_type).read_nested { |nested| current.merge_from(nested) }
           else
             read_one(reader, wire_type)
           end
@@ -166,6 +176,15 @@ module Fast
 
       protected
 
+      # A second occurrence of a message field merges into the first; of
+      # anything else, replaces it. (A map's value field is read this way by
+      # the map field, which is why this is protected rather than private.)
+      def decode_singular(reader, wire_type, current)
+        return read_one(reader, wire_type) unless type == :message && current
+
+        expect(reader, wire_type).read_nested { |nested| current.merge_from(nested) }
+      end
+
       def coerce_one(value)
         case type
         when :string then coerce_string(value)
@@ -178,28 +197,55 @@ module Fast
         end
       end
 
-      def encode_one(buffer, value)
-        case type
-        when :message then Wire.append_length_delimited(buffer, @tag, value.encode)
-        when :string, :bytes then Wire.append_length_delimited(buffer, @tag, value)
-        else
-          buffer << @tag
-          append_scalar(buffer, value)
-        end
-      end
-
       def read_one(reader, wire_type)
         expect(reader, wire_type)
         case type
-        when :message then message_class.decode(reader.read_length_delimited)
-        when :string then reader.read_length_delimited.force_encoding(Encoding::UTF_8)
+        when :message then reader.read_nested { |nested| message_class.new.merge_from(nested) }
+        when :string
+          @strict_utf8 ? read_string(reader) : reader.read_length_delimited.force_encoding(Encoding::UTF_8)
         when :bytes then reader.read_length_delimited
         else read_scalar(reader)
         end
       end
 
+      # What a map entry holds for this field when the entry omits it: the
+      # type's default, except that a message value is an empty instance, as
+      # the reference materialises one (and nil would not encode).
+      def entry_default
+        type == :message ? message_class.new : default_value
+      end
+
+      # Appends the tag and one value. A nested message is encoded straight
+      # into the buffer behind a length prefix filled in after it, with no
+      # buffer of its own; +width+ carries the prefix width from one value
+      # to the next, a hint shared by every encode of the field. (A map's
+      # key and value fields build the entry writer together, which is why
+      # this is protected rather than private.)
+      def writer
+        tag = @tag
+        case type
+        when :message
+          width = 1
+          ->(buffer, value) { width = Wire.append_length_delimited_from(buffer, tag, width) { |b| value.encode(b) } }
+        when :string, :bytes then ->(buffer, value) { Wire.append_length_delimited(buffer, tag, value) }
+        else
+          scalar = scalar_writer
+          lambda do |buffer, value|
+            buffer << tag
+            scalar.call(buffer, value)
+          end
+        end
+      end
+
       private
 
+      def validate_number(number)
+        raise ArgumentError, "field number #{number} is reserved" if RESERVED_NUMBERS.cover?(number)
+        raise ArgumentError, "field number #{number} is outside #{NUMBERS}" unless NUMBERS.cover?(number)
+
+        number
+      end
+
       def resolve_type(type)
         case type
         when Symbol
@@ -265,17 +311,19 @@ module Fast
         end
       end
 
+      # Packed: every value bare, behind one tag and length.
       def repeated_step(ivar)
         if packed?
           tag = @tag
           scalar = scalar_writer
+          width = 1
           lambda do |message, buffer|
             values = message.instance_variable_get(ivar)
             next if values.empty?
 
-            payload = String.new
-            values.each { |value| scalar.call(payload, value) }
-            Wire.append_length_delimited(buffer, tag, payload)
+            width = Wire.append_length_delimited_from(buffer, tag, width) do |b|
+              values.each { |value| scalar.call(b, value) }
+            end
           end
         else
           write = writer
@@ -283,36 +331,26 @@ module Fast
         end
       end
 
+      # Each entry is a message { key = 1; value = 2 }.
       def map_step(ivar)
         tag = @tag
-        key = @key_field.writer
-        value = @value_field.writer
+        key, value = entry_writers
+        width = 1
         lambda do |message, buffer|
           message.instance_variable_get(ivar).each do |k, v|
-            entry = String.new
-            key.call(entry, k)
-            value.call(entry, v)
-            Wire.append_length_delimited(buffer, tag, entry)
+            width = Wire.append_length_delimited_from(buffer, tag, width) do |b|
+              key.call(b, k)
+              value.call(b, v)
+            end
           end
         end
       end
 
-      # Appends the tag and one value.
-      def writer
-        tag = @tag
-        case type
-        when :message then ->(buffer, value) { Wire.append_length_delimited(buffer, tag, value.encode) }
-        when :string, :bytes then ->(buffer, value) { Wire.append_length_delimited(buffer, tag, value) }
-        else
-          scalar = scalar_writer
-          lambda do |buffer, value|
-            buffer << tag
-            scalar.call(buffer, value)
-          end
-        end
-      end
+      # -- writers -----------------------------------------------------------
 
       # Appends one scalar value without its tag, as packed fields need.
+      # Fixed-width types get a lambda each so the pack format is a literal:
+      # Ruby elides the Array in [value].pack(literal, buffer:), and only then.
       def scalar_writer
         case type
         when :int32, :int64, :uint32, :uint64 then ->(buffer, value) { Wire.append_varint(buffer, value) }
@@ -322,9 +360,12 @@ module Fast
         when :enum
           enum = @enum
           ->(buffer, value) { Wire.append_varint(buffer, value.is_a?(Symbol) ? enum.resolve(value) : value) }
-        else
-          format = FIXED_FORMATS.fetch(type)
-          ->(buffer, value) { [value].pack(format, buffer: buffer) }
+        when :double then ->(buffer, value) { [value].pack("E", buffer: buffer) }
+        when :float then ->(buffer, value) { [value].pack("e", buffer: buffer) }
+        when :fixed32 then ->(buffer, value) { [value].pack("L<", buffer: buffer) }
+        when :sfixed32 then ->(buffer, value) { [value].pack("l<", buffer: buffer) }
+        when :fixed64 then ->(buffer, value) { [value].pack("Q<", buffer: buffer) }
+        when :sfixed64 then ->(buffer, value) { [value].pack("q<", buffer: buffer) }
         end
       end
 
@@ -365,10 +406,13 @@ module Fast
         value.encoding == Encoding::BINARY ? value : value.b
       end
 
+      # A +float+ holds what the wire holds: the value narrowed to single
+      # precision on assignment, as the reference narrows it, so what is
+      # read back is what a decode of the encoding reads.
       def coerce_float(value)
         raise ::TypeError, "#{name} expects a number, got #{value.class}" unless value.is_a?(Numeric)
 
-        value.to_f
+        type == :float ? [value.to_f].pack("e").unpack1("e") : value.to_f
       end
 
       def coerce_bool(value)
@@ -409,66 +453,68 @@ module Fast
         integer
       end
 
-      # -- encoding ----------------------------------------------------------
-
-      def encode_repeated(buffer, values)
-        return if values.empty?
-
-        if packed?
-          Wire.append_length_delimited_from(buffer, @tag) { |b| values.each { |v| append_scalar(b, v) } }
-        else
-          values.each { |v| encode_one(buffer, v) }
-        end
-      end
-
-      def encode_map(buffer, hash)
-        hash.each do |key, value|
-          Wire.append_length_delimited_from(buffer, @tag) do |entry|
-            @key_field.encode_one(entry, key)
-            @value_field.encode_one(entry, value)
-          end
-        end
-      end
-
-      def append_scalar(buffer, value)
-        case type
-        when :int32, :int64, :uint32, :uint64 then Wire.append_varint(buffer, value)
-        when :sint32 then Wire.append_varint(buffer, Wire.zigzag32(value))
-        when :sint64 then Wire.append_varint(buffer, Wire.zigzag64(value))
-        when :bool then buffer << (value ? 1 : 0)
-        when :enum then Wire.append_varint(buffer, enum_number(value))
-        else buffer << [value].pack(FIXED_FORMATS.fetch(type))
-        end
-      end
-
       # -- decoding ----------------------------------------------------------
 
       def decode_repeated(reader, wire_type, values)
         if packable? && wire_type == Wire::LENGTH_DELIMITED
-          packed = reader.read_packed
-          values << read_scalar(packed) until packed.eof?
+          reader.read_packed { |packed| values << read_scalar(packed) until packed.eof? }
         else
           values << read_one(reader, wire_type)
         end
         values
       end
 
-      def decode_map_entry(reader, wire_type, hash)
-        entry = Reader.new(expect(reader, wire_type).read_length_delimited)
-        key = @key_field.default_value
-        value = @value_field.default_value
-        until entry.eof?
-          number, entry_wire_type = entry.read_tag
-          case number
-          when 1 then key = @key_field.read_one(entry, entry_wire_type)
-          when 2 then value = @value_field.decode(entry, entry_wire_type, value)
-          else entry.skip(entry_wire_type)
+      # An entry the map accepts holds its key, its value, or both; one that
+      # carries anything else — an undeclared subfield, or the key or value
+      # with a wire type the entry does not accept — is not a map entry at
+      # all. The reference keeps such an entry among the parent message's
+      # unknown fields and leaves the map alone, so this does too.
+      def decode_map_entry(reader, wire_type, hash, message)
+        expect(reader, wire_type).read_nested do |entry|
+          key = nil
+          value = nil
+          unknown = nil
+          until entry.eof?
+            tag = entry.read_key
+            case tag
+            when @key_tag then key = @key_field.read_one(entry, tag & 0x7)
+            when @value_tag then value = @value_field.decode_singular(entry, tag & 0x7, value)
+            else
+              Wire.append_varint(unknown ||= String.new, tag)
+              unknown << entry.skip(tag & 0x7, tag >> 3)
+            end
+          end
+          if unknown
+            keep_entry(message, key, value, unknown)
+          else
+            # An absent key or value reads as its type's default, an empty
+            # message for a message-typed value, as the reference
+            # materialises one. (Only nil is absent: false is a bool key.)
+            hash[key.nil? ? @key_field.default_value : key] = value.nil? ? @value_field.entry_default : value
           end
         end
-        hash[key] = value
         hash
       end
 
+      # A rejected entry, into the message's unknown fields (its own buffer,
+      # made here when the message has none yet) and written the way the
+      # reference writes one: the subfields it did carry, in number order and
+      # omitted when they are at their default — a message value is written
+      # whenever it was there — then the bytes it carried besides.
+      def keep_entry(message, key, value, unknown)
+        buffer = message.unknown_fields || message.instance_variable_set(:@unknown_fields, String.new)
+        write_key, write_value = entry_writers
+        Wire.append_length_delimited_from(buffer, @tag) do |b|
+          write_key.call(b, key) unless key.nil? || @key_field.omit?(key)
+          write_value.call(b, value) unless value.nil? || @value_field.omit?(value)
+          b << unknown
+        end
+      end
+
+      def entry_writers
+        @entry_writers ||= [@key_field.writer, @value_field.writer]
+      end
+
       def expect(reader, wire_type)
         return reader if wire_type == self.wire_type
 
@@ -482,20 +528,27 @@ module Fast
         when :int64 then signed(reader.read_varint, 64)
         when :uint32 then reader.read_varint & 0xFFFF_FFFF
         when :uint64 then reader.read_varint
-        when :sint32, :sint64 then Wire.unzigzag(reader.read_varint)
+        when :sint32 then Wire.unzigzag(reader.read_varint & 0xFFFF_FFFF)
+        when :sint64 then Wire.unzigzag(reader.read_varint)
         when :bool then reader.read_varint != 0
-        when :double, :fixed64, :sfixed64 then reader.read_bytes(8).unpack1(FIXED_FORMATS.fetch(type))
-        else reader.read_bytes(4).unpack1(FIXED_FORMATS.fetch(type))
+        when :double, :fixed64, :sfixed64 then reader.read_fixed(FIXED_FORMATS.fetch(type), 8)
+        else reader.read_fixed(FIXED_FORMATS.fetch(type), 4)
         end
       end
 
+      # proto3 requires a parser to reject a string field that is not valid
+      # UTF-8; proto2 does not, and the reference keeps it.
+      def read_string(reader)
+        string = reader.read_length_delimited.force_encoding(Encoding::UTF_8)
+        raise DecodeError, "#{name}: string is not valid UTF-8" if @strict_utf8 && !string.valid_encoding?
+
+        string
+      end
+
       def signed(value, bits)
         value &= (1 << bits) - 1
         value >= (1 << (bits - 1)) ? value - (1 << bits) : value
       end
-
-      # A map's key and value fields build the entry writer together.
-      protected :writer
     end
   end
 end

data/lib/fast/protowire/message.rb

@@ -97,6 +97,7 @@ module Fast
         def compile_encoder
           steps = sorted_fields.map(&:encoder_step)
           define_method(:encode) do |buffer = String.new|
+            Wire.binary_buffer(buffer)
             steps.each { |step| step.call(self, buffer) }
             buffer << @unknown_fields if @unknown_fields
             buffer
@@ -143,11 +144,12 @@ module Fast
 
       # Scalar and message ivars stay unset until written (an unset ivar reads
       # as nil); repeated and map fields get their container up front so it
-      # can be mutated in place.
-      def initialize(attributes = nil, **keywords)
+      # can be mutated in place. +attributes+ is a Hash, or the keywords a
+      # caller writes instead of one.
+      def initialize(attributes = nil)
         self.class.container_fields.each { |field| instance_variable_set(field.ivar, field.default_value) }
         @unknown_fields = nil
-        (attributes || keywords).each do |name, value|
+        attributes&.each do |name, value|
           field = self.class.fields[name.to_sym]
           raise ArgumentError, "unknown field #{name.inspect} for #{self.class}" unless field
 
@@ -171,15 +173,23 @@ module Fast
 
       # Reads fields from +reader+ into this message (protobuf merge
       # semantics: later scalars win, repeated fields append, nested messages
-      # merge) and returns self.
+      # merge) and returns self. A declared field arriving with a wire type
+      # it does not accept is schema drift rather than corruption, so it is
+      # kept as an unknown field, as the reference does; so is a map entry
+      # carrying a subfield the entry does not accept, kept whole.
       def merge_from(reader)
         until reader.eof?
-          number, wire_type = reader.read_tag
+          key = reader.read_varint
+          wire_type = key & 0x7
+          number = key >> 3
+          raise DecodeError, "field number 0" if number.zero? # as Reader#read_key, inlined for the loop
+
           field = self.class.fields_by_number[number]
-          if field
-            write_field(field, field.decode(reader, wire_type, instance_variable_get(field.ivar)))
+          if field&.accepts?(wire_type)
+            write_field(field, field.decode(reader, wire_type, instance_variable_get(field.ivar), self))
           else
-            (@unknown_fields ||= String.new) << Wire.varint((number << 3) | wire_type) << reader.skip(wire_type)
+            Wire.append_varint(@unknown_fields ||= String.new, key)
+            @unknown_fields << reader.skip(wire_type, number)
           end
         end
         self
@@ -194,8 +204,12 @@ module Fast
         end
       end
 
-      # Two messages are equal when every field reads the same; an implicit
-      # field set to its default is the same as one never set.
+      # Two messages are equal when every declared field reads the same; an
+      # implicit field set to its default is the same as one never set.
+      # Unknown fields are not compared, as the reference does not compare
+      # them: two messages that differ only in what they carry for fields
+      # this class never declared are the same message. Byte identity is
+      # +a.encode == b.encode+.
       def ==(other)
         other.class == self.class && comparable_values == other.comparable_values
       end
@@ -225,11 +239,10 @@ module Fast
       protected
 
       def comparable_values
-        values = self.class.fields.each_value.map do |field|
+        self.class.fields.each_value.map do |field|
           value = instance_variable_get(field.ivar)
           value.nil? && !field.explicit_presence? ? field.default_value : value
         end
-        values << @unknown_fields
       end
 
       private

data/lib/fast/protowire/reader.rb

@@ -8,10 +8,18 @@ module Fast
     # A cursor over an encoded message. Reads the primitive wire values and
     # skips what it is not asked to interpret.
     class Reader
+      # How deep nested messages and groups may go before the input is taken
+      # to be hostile rather than deep; the reference's limit.
+      MAX_DEPTH = 100
+
+      # Decoding is about bytes, so the input is read as bytes: a String
+      # tagged anything else is copied into a binary view once, here, rather
+      # than leaking its encoding into every slice handed out below.
       def initialize(buffer, position = 0, limit = buffer.bytesize)
-        @buffer = buffer
+        @buffer = buffer.encoding == Encoding::BINARY ? buffer : buffer.b
         @position = position
         @limit = limit
+        @depth = 0
       end
 
       attr_reader :position
@@ -22,32 +30,56 @@ module Fast
 
       # Returns [field number, wire type].
       def read_tag
-        key = read_varint
+        key = read_key
         [key >> 3, key & 0x7]
       end
 
+      # The next field's key, the varint holding its number and wire type
+      # together. Field number 0 exists on no wire: the reference rejects it
+      # wherever it appears, and so does this.
+      def read_key
+        key = read_varint
+        raise DecodeError, "field number 0" if key < 8
+
+        key
+      end
+
+      # Most varints (tags, small lengths) are one byte; the loop is only
+      # entered past it, and is a bare while because Kernel#loop costs an
+      # object per call.
       def read_varint
-        result = 0
-        shift = 0
-        loop do
-          raise DecodeError, "truncated varint" if @position >= @limit
+        byte = read_byte
+        return byte if byte < 0x80
 
-          byte = @buffer.getbyte(@position)
-          @position += 1
-          result |= (byte & 0x7f) << shift
-          return result if byte < 0x80
+        result = byte & 0x7f
+        shift = 7
+        while byte >= 0x80
+          raise DecodeError, "varint too long" if shift > 63
 
+          byte = read_byte
+          result |= (byte & 0x7f) << shift
           shift += 7
-          raise DecodeError, "varint too long" if shift > 63
         end
+        # Only a tenth byte can carry bits above 64; every implementation
+        # truncates them rather than reading a wider value.
+        shift > 63 ? result & Wire::UINT64_MASK : result
       end
 
       def read_fixed32
-        read_bytes(4).unpack1("L<")
+        read_fixed("L<", 4)
       end
 
       def read_fixed64
-        read_bytes(8).unpack1("Q<")
+        read_fixed("Q<", 8)
+      end
+
+      # Reads +width+ bytes as one value of the pack +format+, in place.
+      def read_fixed(format, width)
+        raise DecodeError, "truncated field" if @position + width > @limit
+
+        value = @buffer.unpack1(format, offset: @position)
+        @position += width
+        value
       end
 
       def read_bytes(length)
@@ -62,25 +94,53 @@ module Fast
         read_bytes(read_varint)
       end
 
-      # A reader bounded to the next length-delimited value, for packed fields.
+      # Bounds the reader to the next length-delimited value for the block
+      # and returns the block's result, so a packed repeated field is read
+      # in place, with no copy of its bytes.
       def read_packed
         length = read_varint
-        raise DecodeError, "truncated packed field" if @position + length > @limit
+        limit = @limit
+        raise DecodeError, "truncated field" if @position + length > limit
 
-        reader = Reader.new(@buffer, @position, @position + length)
-        @position += length
-        reader
+        @limit = @position + length
+        result = yield self
+        @position = @limit
+        @limit = limit
+        result
+      end
+
+      # The same, for a value read by recursing into it — a nested message or
+      # a map entry — counting the nesting so deeply nested input raises
+      # rather than overflowing the VM stack. A packed field holds scalars
+      # and does not recurse, so it costs no level and reads through
+      # read_packed; the bounding is spelled out twice rather than delegated
+      # because every nested message pays for the call.
+      def read_nested
+        length = read_varint
+        limit = @limit
+        raise DecodeError, "truncated field" if @position + length > limit
+        raise DecodeError, "nested deeper than #{MAX_DEPTH}" if @depth >= MAX_DEPTH
+
+        @limit = @position + length
+        @depth += 1
+        result = yield self
+        @depth -= 1
+        @position = @limit
+        @limit = limit
+        result
       end
 
       # Skips one value of +wire_type+ and returns its raw bytes, so unknown
-      # fields survive a decode/encode round trip.
-      def skip(wire_type)
+      # fields survive a decode/encode round trip. +number+ is the field
+      # number the value arrived under: a group is closed by its own number
+      # and nothing else. Without one, any END_GROUP closes it.
+      def skip(wire_type, number = nil)
         start = @position
         case wire_type
         when Wire::VARINT then read_varint
         when Wire::FIXED64 then read_bytes(8)
         when Wire::LENGTH_DELIMITED then read_length_delimited
-        when Wire::START_GROUP then skip_group
+        when Wire::START_GROUP then skip_group(number)
         when Wire::FIXED32 then read_bytes(4)
         else raise DecodeError, "unknown wire type #{wire_type}"
         end
@@ -89,14 +149,28 @@ module Fast
 
       private
 
-      def skip_group
+      def read_byte
+        raise DecodeError, "truncated varint" if @position >= @limit
+
+        byte = @buffer.getbyte(@position)
+        @position += 1
+        byte
+      end
+
+      def skip_group(number)
+        raise DecodeError, "nested deeper than #{MAX_DEPTH}" if @depth >= MAX_DEPTH
+
+        @depth += 1
         loop do
-          number, wire_type = read_tag
-          return if wire_type == Wire::END_GROUP
-          raise DecodeError, "invalid group" if number.zero?
+          inner, wire_type = read_tag
+          if wire_type == Wire::END_GROUP
+            raise DecodeError, "group #{number} closed by #{inner}" unless number.nil? || inner == number
 
-          skip(wire_type)
+            break
+          end
+          skip(wire_type, inner)
         end
+        @depth -= 1
       end
     end
   end

data/lib/fast/protowire/version.rb

@@ -2,6 +2,6 @@
 
 module Fast
   module Protowire
-    VERSION = "0.1.0"
+    VERSION = "0.3.0"
   end
 end

data/lib/fast/protowire/wire.rb

@@ -3,8 +3,9 @@
 module Fast
   module Protowire
     # The wire format itself: tags, varints, zigzag, fixed-width values and
-    # length prefixes, appended to a binary String buffer. Everything above
-    # this module is bookkeeping about which field gets which of these.
+    # length prefixes, appended to a binary String buffer, and the size each
+    # takes so a length prefix can be written before its payload. Everything
+    # above this module is bookkeeping about which field gets which of these.
     module Wire
       VARINT = 0
       FIXED64 = 1
@@ -14,10 +15,23 @@ module Fast
       FIXED32 = 5
 
       UINT64_MASK = (1 << 64) - 1
-      private_constant :UINT64_MASK
+      # Zero bytes a length prefix is widened to, by varint width.
+      PLACEHOLDERS = Array.new(11) { |width| ("\0" * width).b.freeze }.freeze
+      private_constant :PLACEHOLDERS
 
       module_function
 
+      # A buffer written to here must be binary: appending a byte to a text
+      # String appends that encoding's character for it instead, silently
+      # writing something else. An empty buffer is simply retagged, so the
+      # usual ways of making one (+"", String.new("")) still work.
+      def binary_buffer(buffer)
+        return buffer if buffer.encoding == Encoding::BINARY
+        raise ArgumentError, "buffer must be a binary String, got #{buffer.encoding}" unless buffer.empty?
+
+        buffer.force_encoding(Encoding::BINARY)
+      end
+
       # The key for +number+ / +wire_type+ as frozen bytes.
       def tag(number, wire_type)
         varint((number << 3) | wire_type).freeze
@@ -38,34 +52,100 @@ module Fast
         buffer << value
       end
 
+      # Bytes append_varint writes for +value+.
+      def varint_size(value)
+        return 10 if value.negative?
+
+        size = 1
+        while value > 0x7f
+          value >>= 7
+          size += 1
+        end
+        size
+      end
+
+      # (* 2 rather than << 1: a left shift of a negative Fixnum allocates.)
       def zigzag32(value)
-        (value << 1) ^ (value >> 31)
+        (value * 2) ^ (value >> 31)
       end
 
       def zigzag64(value)
-        (value << 1) ^ (value >> 63)
+        (value * 2) ^ (value >> 63)
       end
 
       def unzigzag(value)
         (value >> 1) ^ -(value & 1)
       end
 
-      # A length-delimited field: +tag+ then the varint size of +bytes+ then
-      # +bytes+. Non-ASCII text is appended as its bytes so the binary buffer
-      # never trips Ruby's encoding compatibility check.
-      def append_length_delimited(buffer, tag, bytes)
+      # A length-delimited field whose payload the block appends to +buffer+
+      # itself, behind a length prefix filled in once the payload is there,
+      # so a nested message, packed field or map entry costs no buffer of
+      # its own. Returns the width of the prefix it wrote: a caller writing
+      # many alike values passes it back as +width+ and the prefix is
+      # rarely resized.
+      def append_length_delimited_from(buffer, tag, width = 1)
         buffer << tag
-        append_varint(buffer, bytes.bytesize)
-        buffer << (bytes.ascii_only? ? bytes : bytes.b)
+        start = reserve_length(buffer, width)
+        yield buffer
+        close_length(buffer, start, width)
+      end
+
+      # Appends +width+ placeholder bytes for a length prefix and returns the
+      # position after them, for close_length.
+      def reserve_length(buffer, width = 1)
+        buffer << PLACEHOLDERS[width]
+        buffer.bytesize
       end
 
-      # A length-delimited field whose payload the block writes into a fresh
-      # buffer. (Writing into the parent and inserting the size afterwards
-      # looks cheaper but String#insert costs O(size) on the parent.)
-      def append_length_delimited_from(buffer, tag)
-        payload = String.new
-        yield payload
-        append_length_delimited(buffer, tag, payload)
+      # Writes the varint size of everything appended since +start+ into the
+      # +width+ placeholder bytes before it, first resizing the placeholder
+      # when the size needs a different width, which moves only the payload.
+      # Returns the width written. (Growing a String in place reallocates it
+      # to exactly the new size, so on a large buffer a resize costs far more
+      # than the bytes moved; hence the width hint.)
+      def close_length(buffer, start, width = 1)
+        length = buffer.bytesize - start
+        needed = length < 0x80 ? 1 : varint_size(length)
+        buffer.bytesplice(start - width, width, PLACEHOLDERS[needed]) unless needed == width
+        position = start - width
+        while length > 0x7f
+          buffer.setbyte(position, (length & 0x7f) | 0x80)
+          position += 1
+          length >>= 7
+        end
+        buffer.setbyte(position, length)
+        needed
+      end
+
+      # Text of any encoding goes into the binary buffer as bytes. Ruby 3.4's
+      # String#append_as_bytes does that with no copy, and takes the tag and
+      # a one-byte size in the same call; before it, non-ASCII text is
+      # appended as a binary copy so the buffer never trips Ruby's encoding
+      # compatibility check.
+      if String.method_defined?(:append_as_bytes)
+        # A length-delimited field: +tag+, the varint size of +bytes+, +bytes+.
+        def append_length_delimited(buffer, tag, bytes)
+          size = bytes.bytesize
+          return buffer.append_as_bytes(tag, size, bytes) if size < 0x80
+
+          buffer << tag
+          append_varint(buffer, size)
+          buffer.append_as_bytes(bytes)
+        end
+
+        def append_bytes(buffer, bytes)
+          buffer.append_as_bytes(bytes)
+        end
+      else
+        def append_length_delimited(buffer, tag, bytes)
+          buffer << tag
+          append_varint(buffer, bytes.bytesize)
+          append_bytes(buffer, bytes)
+        end
+
+        def append_bytes(buffer, bytes)
+          buffer << (bytes.encoding == Encoding::BINARY || bytes.ascii_only? ? bytes : bytes.b)
+        end
       end
     end
   end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: fast-protowire
 version: !ruby/object:Gem::Version
-  version: 0.1.0
+  version: 0.3.0
 platform: ruby
 authors:
 - Eric Jacobs
@@ -46,7 +46,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 4.0.10
+rubygems_version: 4.0.20
 specification_version: 4
 summary: 'Protocol Buffers wire format for Ruby: declare messages, encode and decode
   bytes, no runtime.'