google-protobuf 3.25.8 → 4.31.1

data/lib/google/protobuf/ffi/ffi.rb

@@ -21,7 +21,7 @@ module Google
       Upb_Map_Begin = -1
 
       ## Encoding Status
-      Upb_Status_MaxMessage = 127
+      Upb_Status_MaxMessage = 511
       Upb_Encode_Deterministic = 1
       Upb_Encode_SkipUnknown = 2
 
@@ -36,6 +36,10 @@ module Google
       ## JSON Decoding options
       Upb_JsonDecode_IgnoreUnknown = 1
 
+      ## JSON Decoding results
+      Upb_JsonDecodeResult_Ok = 0
+      Upb_JsonDecodeResult_Error = 2
+
       typedef :pointer, :Array
       typedef :pointer, :DefPool
       typedef :pointer, :EnumValueDef
@@ -95,11 +99,6 @@ module Google
         :repeated
       )
 
-      Syntax = enum(
-        :Proto2, 2,
-        :Proto3
-      )
-
       # All the different kind of well known type messages. For simplicity of check,
       # number wrappers and string wrappers are grouped together. Make sure the
       # order and merber of these groups are not changed.
@@ -190,6 +189,8 @@ module Google
                :str_val, StringView
       end
 
+      Upb_Message_Begin = -1
+
       class MutableMessageValue < ::FFI::Union
         layout :map, :Map,
                :msg, :Message,

data/lib/google/protobuf/ffi/field_descriptor.rb

@@ -67,8 +67,9 @@ module Google
         @type ||= Google::Protobuf::FFI.get_type(self)
       end
 
+      # DEPRECATED: Use required? or repeated? instead.
       def label
-        @label ||= Google::Protobuf::FFI::Label[Google::Protobuf::FFI.get_label(self)]
+        @label ||= Google::Protobuf::FFI.get_label(self)
       end
 
       def default
@@ -156,6 +157,14 @@ module Google
         @has_presence ||= Google::Protobuf::FFI.get_has_presence(self)
       end
 
+      ##
+      # Tests if this is a repeated field that uses packed encoding.
+      #
+      # @return [Boolean] True iff this field is packed
+      def is_packed?
+        @is_packed ||= Google::Protobuf::FFI.get_is_packed(self)
+      end
+
       # @param msg [Google::Protobuf::Message]
       def clear(msg)
         if msg.class.descriptor != Google::Protobuf::FFI.get_containing_message_def(self)
@@ -189,6 +198,10 @@ module Google
         @map ||= Google::Protobuf::FFI.is_map self
       end
 
+      def required?
+        @required ||= Google::Protobuf::FFI.is_required self
+      end
+
       def repeated?
         @repeated ||= Google::Protobuf::FFI.is_repeated self
       end
@@ -211,7 +224,18 @@ module Google
           size_ptr = ::FFI::MemoryPointer.new(:size_t, 1)
           temporary_arena = Google::Protobuf::FFI.create_arena
           buffer = Google::Protobuf::FFI.field_options(self, size_ptr, temporary_arena)
-          Google::Protobuf::FieldOptions.decode(buffer.read_string_length(size_ptr.read(:size_t)).force_encoding("ASCII-8BIT").freeze).send(:internal_deep_freeze)
+          opts = Google::Protobuf::FieldOptions.decode(buffer.read_string_length(size_ptr.read(:size_t)).force_encoding("ASCII-8BIT").freeze)
+          opts.clear_features()
+          opts.freeze
+        end
+      end
+
+      def to_proto
+        @to_proto ||= begin
+          size_ptr = ::FFI::MemoryPointer.new(:size_t, 1)
+          temporary_arena = Google::Protobuf::FFI.create_arena
+          buffer = Google::Protobuf::FFI.field_to_proto(self, size_ptr, temporary_arena)
+          Google::Protobuf::FieldDescriptorProto.decode(buffer.read_string_length(size_ptr.read(:size_t)).force_encoding("ASCII-8BIT").freeze)
         end
       end
 
@@ -304,7 +328,9 @@ module Google
       attach_function :get_default,                :upb_FieldDef_Default,               [FieldDescriptor], MessageValue.by_value
       attach_function :get_subtype_as_enum,        :upb_FieldDef_EnumSubDef,            [FieldDescriptor], EnumDescriptor
       attach_function :get_has_presence,           :upb_FieldDef_HasPresence,           [FieldDescriptor], :bool
+      attach_function :get_is_packed,              :upb_FieldDef_IsPacked,              [FieldDescriptor], :bool
       attach_function :is_map,                     :upb_FieldDef_IsMap,                 [FieldDescriptor], :bool
+      attach_function :is_required,                :upb_FieldDef_IsRequired,            [FieldDescriptor], :bool
       attach_function :is_repeated,                :upb_FieldDef_IsRepeated,            [FieldDescriptor], :bool
       attach_function :is_sub_message,             :upb_FieldDef_IsSubMessage,          [FieldDescriptor], :bool
       attach_function :get_json_name,              :upb_FieldDef_JsonName,              [FieldDescriptor], :string
@@ -314,6 +340,7 @@ module Google
       attach_function :get_number,                 :upb_FieldDef_Number,                [FieldDescriptor], :uint32_t
       attach_function :get_type,                   :upb_FieldDef_Type,                  [FieldDescriptor], FieldType
       attach_function :file_def_by_raw_field_def,  :upb_FieldDef_File,                  [:pointer], :FileDef
+      attach_function :field_to_proto,             :FieldDescriptor_serialized_to_proto,[FieldDescriptor, :pointer, Internal::Arena], :pointer
     end
   end
 end

data/lib/google/protobuf/ffi/file_descriptor.rb

@@ -10,14 +10,40 @@ module Google
     class FFI
       # FileDescriptor
       attach_function :file_def_name,   :upb_FileDef_Name,   [:FileDef], :string
-      attach_function :file_def_syntax, :upb_FileDef_Syntax, [:FileDef], Syntax
       attach_function :file_def_pool,   :upb_FileDef_Pool,   [:FileDef], :DefPool
       attach_function :file_options,    :FileDescriptor_serialized_options,  [:FileDef, :pointer, Internal::Arena], :pointer
+      attach_function :file_to_proto,   :FileDescriptor_serialized_to_proto,  [:FileDef, :pointer, Internal::Arena], :pointer
     end
 
     class FileDescriptor
       attr :descriptor_pool, :file_def
 
+      # FFI Interface methods and setup
+      extend ::FFI::DataConverter
+      native_type ::FFI::Type::POINTER
+
+      class << self
+        prepend Google::Protobuf::Internal::TypeSafety
+        include Google::Protobuf::Internal::PointerHelper
+
+        # @param value [FileDescriptor] FileDescriptor to convert to an FFI native type
+        # @param _ [Object] Unused
+        def to_native(value, _)
+          file_def_ptr = value.nil? ? nil : value.instance_variable_get(:@file_def)
+          return ::FFI::Pointer::NULL if file_def_ptr.nil?
+          raise "Underlying file_def was null!" if file_def_ptr.null?
+          file_def_ptr
+        end
+
+        ##
+        # @param file_def [::FFI::Pointer] FileDef pointer to be wrapped
+        # @param _ [Object] Unused
+        def from_native(file_def, _ = nil)
+          return nil if file_def.nil? or file_def.null?
+          descriptor_from_file_def(file_def)
+        end
+      end
+
       def initialize(file_def, descriptor_pool)
         @descriptor_pool = descriptor_pool
         @file_def = file_def
@@ -31,17 +57,6 @@ module Google
         "#{self.class.name}: #{name}"
       end
 
-      def syntax
-        case Google::Protobuf::FFI.file_def_syntax(@file_def)
-        when :Proto3
-          :proto3
-        when :Proto2
-          :proto2
-        else
-          nil
-        end
-      end
-
       def name
         Google::Protobuf::FFI.file_def_name(@file_def)
       end
@@ -51,7 +66,18 @@ module Google
           size_ptr = ::FFI::MemoryPointer.new(:size_t, 1)
           temporary_arena = Google::Protobuf::FFI.create_arena
           buffer = Google::Protobuf::FFI.file_options(@file_def, size_ptr, temporary_arena)
-          Google::Protobuf::FileOptions.decode(buffer.read_string_length(size_ptr.read(:size_t)).force_encoding("ASCII-8BIT").freeze).send(:internal_deep_freeze)
+          opts = Google::Protobuf::FileOptions.decode(buffer.read_string_length(size_ptr.read(:size_t)).force_encoding("ASCII-8BIT").freeze)
+          opts.clear_features()
+          opts.freeze
+        end
+      end
+
+      def to_proto
+        @to_proto ||= begin
+          size_ptr = ::FFI::MemoryPointer.new(:size_t, 1)
+          temporary_arena = Google::Protobuf::FFI.create_arena
+          buffer = Google::Protobuf::FFI.file_to_proto(@file_def, size_ptr, temporary_arena)
+          Google::Protobuf::FileDescriptorProto.decode(buffer.read_string_length(size_ptr.read(:size_t)).force_encoding("ASCII-8BIT").freeze)
         end
       end
     end

data/lib/google/protobuf/ffi/internal/arena.rb

@@ -12,8 +12,6 @@ module Google
   module Protobuf
     module Internal
       class Arena
-        attr :pinned_messages
-
         # FFI Interface methods and setup
         extend ::FFI::DataConverter
         native_type ::FFI::Type::POINTER
@@ -46,10 +44,6 @@ module Google
             raise RuntimeError.new "Unable to fuse arenas. This should never happen since Ruby does not use initial blocks"
           end
         end
-
-        def pin(message)
-          pinned_messages.push message
-        end
       end
     end
 

data/lib/google/protobuf/ffi/internal/convert.rb

@@ -32,12 +32,15 @@ module Google
             raise TypeError.new "Invalid argument for boolean field '#{name}' (given #{value.class})." unless [TrueClass, FalseClass].include? value.class
             return_value[:bool_val] = value
           when :string
-            raise TypeError.new "Invalid argument for string field '#{name}' (given #{value.class})." unless [Symbol, String].include? value.class
-            begin
+            raise TypeError.new "Invalid argument for string field '#{name}' (given #{value.class})." unless value.is_a?(String) or value.is_a?(Symbol)
+            value = value.to_s if value.is_a?(Symbol)
+            if value.encoding == Encoding::UTF_8
+              unless value.valid_encoding?
+                raise Encoding::InvalidByteSequenceError.new "String is invalid UTF-8"
+              end
+              string_value = value
+            else
               string_value = value.to_s.encode("UTF-8")
-            rescue Encoding::UndefinedConversionError
-              # TODO - why not include the field name here?
-              raise Encoding::UndefinedConversionError.new "String is invalid UTF-8"
             end
             return_value[:str_val][:size] = string_value.bytesize
             return_value[:str_val][:data] = Google::Protobuf::FFI.arena_malloc(arena, string_value.bytesize)
@@ -70,7 +73,7 @@ module Google
               case wkt
               when :Timestamp
                 raise TypeError.new "Invalid type #{value.class} to assign to submessage field '#{name}'." unless value.kind_of? Time
-                new_message = Google::Protobuf::FFI.new_message_from_def msg_or_enum_def, arena
+                new_message = Google::Protobuf::FFI.new_message_from_def Google::Protobuf::FFI.get_mini_table(msg_or_enum_def), arena
                 sec = Google::Protobuf::FFI::MessageValue.new
                 sec[:int64_val] = value.tv_sec
                 sec_field_def = Google::Protobuf::FFI.get_field_by_number msg_or_enum_def, 1
@@ -82,7 +85,7 @@ module Google
                 return_value[:msg_val] = new_message
               when :Duration
                 raise TypeError.new "Invalid type #{value.class} to assign to submessage field '#{name}'." unless value.kind_of? Numeric
-                new_message = Google::Protobuf::FFI.new_message_from_def msg_or_enum_def, arena
+                new_message = Google::Protobuf::FFI.new_message_from_def Google::Protobuf::FFI.get_mini_table(msg_or_enum_def), arena
                 sec = Google::Protobuf::FFI::MessageValue.new
                 sec[:int64_val] = value
                 sec_field_def = Google::Protobuf::FFI.get_field_by_number msg_or_enum_def, 1
@@ -190,39 +193,23 @@ module Google
         def to_h_internal(msg, message_descriptor)
           return nil if msg.nil? or msg.null?
           hash = {}
-          is_proto2 = Google::Protobuf::FFI.message_def_syntax(message_descriptor) == :Proto2
-          message_descriptor.each do |field_descriptor|
-            # TODO: Legacy behavior, remove when we fix the is_proto2 differences.
-            if !is_proto2 and
-              field_descriptor.sub_message? and
-              !field_descriptor.repeated? and
-              !Google::Protobuf::FFI.get_message_has(msg, field_descriptor)
-              hash[field_descriptor.name.to_sym] = nil
-              next
-            end
-
-            # Do not include fields that are not present (oneof or optional fields).
-            if is_proto2 and field_descriptor.has_presence? and !Google::Protobuf::FFI.get_message_has(msg, field_descriptor)
-              next
-            end
+          iter = ::FFI::MemoryPointer.new(:size_t, 1)
+          iter.write(:size_t, Google::Protobuf::FFI::Upb_Message_Begin)
+          message_value = Google::Protobuf::FFI::MessageValue.new
+          field_def_ptr = ::FFI::MemoryPointer.new :pointer
 
-            message_value = Google::Protobuf::FFI.get_message_value msg, field_descriptor
+          while Google::Protobuf::FFI::message_next(msg, message_descriptor, nil, field_def_ptr, message_value, iter) do
+            field_descriptor = FieldDescriptor.from_native field_def_ptr.get_pointer(0)
 
-            # Proto2 omits empty map/repeated fields also.
             if field_descriptor.map?
               hash_entry = map_create_hash(message_value[:map_val], field_descriptor)
             elsif field_descriptor.repeated?
-              array = message_value[:array_val]
-              if is_proto2 and (array.null? || Google::Protobuf::FFI.array_size(array).zero?)
-                next
-              end
-              hash_entry = repeated_field_create_array(array, field_descriptor, field_descriptor.type)
+              hash_entry = repeated_field_create_array(message_value[:array_val], field_descriptor, field_descriptor.type)
             else
               hash_entry = scalar_create_hash(message_value, field_descriptor.type, field_descriptor: field_descriptor)
             end
 
             hash[field_descriptor.name.to_sym] = hash_entry
-
           end
 
           hash

data/lib/google/protobuf/ffi/internal/pointer_helper.rb

@@ -13,7 +13,8 @@ module Google
         # the pool, and either retrieve the wrapper object for the given pointer
         # or create one. Assumes that the caller is the wrapper class for the
         # given pointer and that it implements `private_constructor`.
-        def descriptor_from_file_def(file_def, pointer)
+        def descriptor_from_file_def(file_def, pointer = nil)
+          pointer = file_def if pointer.nil?
           raise RuntimeError.new "FileDef is nil" if file_def.nil?
           raise RuntimeError.new "FileDef is null" if file_def.null?
           pool_def = Google::Protobuf::FFI.file_def_pool file_def

data/lib/google/protobuf/ffi/map.rb

@@ -9,18 +9,20 @@ module Google
   module Protobuf
     class FFI
       # Map
-      attach_function :map_clear,  :upb_Map_Clear,                    [:Map], :void
-      attach_function :map_delete, :upb_Map_Delete,                   [:Map, MessageValue.by_value, MessageValue.by_ref], :bool
-      attach_function :map_get,    :upb_Map_Get,                      [:Map, MessageValue.by_value, MessageValue.by_ref], :bool
-      attach_function :create_map, :upb_Map_New,                      [Internal::Arena, CType, CType], :Map
-      attach_function :map_size,   :upb_Map_Size,                     [:Map], :size_t
-      attach_function :map_set,    :upb_Map_Set,                      [:Map, MessageValue.by_value, MessageValue.by_value, Internal::Arena], :bool
+      attach_function :map_clear,   :upb_Map_Clear,         [:Map], :void
+      attach_function :map_delete,  :upb_Map_Delete,        [:Map, MessageValue.by_value, MessageValue.by_ref], :bool
+      attach_function :map_get,     :upb_Map_Get,           [:Map, MessageValue.by_value, MessageValue.by_ref], :bool
+      attach_function :create_map,  :upb_Map_New,           [Internal::Arena, CType, CType], :Map
+      attach_function :map_size,    :upb_Map_Size,          [:Map], :size_t
+      attach_function :map_set,     :upb_Map_Set,           [:Map, MessageValue.by_value, MessageValue.by_value, Internal::Arena], :bool
+      attach_function :map_freeze,  :upb_Map_Freeze,        [:Map, MiniTable.by_ref], :void
+      attach_function :map_frozen?, :upb_Map_IsFrozen,      [:Map], :bool
 
       # MapIterator
-      attach_function :map_next,   :upb_MapIterator_Next,             [:Map, :pointer], :bool
-      attach_function :map_done,   :upb_MapIterator_Done,             [:Map, :size_t], :bool
-      attach_function :map_key,    :upb_MapIterator_Key,              [:Map, :size_t], MessageValue.by_value
-      attach_function :map_value,  :upb_MapIterator_Value,            [:Map, :size_t], MessageValue.by_value
+      attach_function :map_next,    :upb_MapIterator_Next,  [:Map, :pointer], :bool
+      attach_function :map_done,    :upb_MapIterator_Done,  [:Map, :size_t], :bool
+      attach_function :map_key,     :upb_MapIterator_Key,   [:Map, :size_t], MessageValue.by_value
+      attach_function :map_value,   :upb_MapIterator_Value, [:Map, :size_t], MessageValue.by_value
     end
     class Map
       include Enumerable
@@ -155,6 +157,37 @@ module Google
       end
       alias size length
 
+      ##
+      # Is this object frozen?
+      # Returns true if either this Ruby wrapper or the underlying
+      # representation are frozen. Freezes the wrapper if the underlying
+      # representation is already frozen but this wrapper isn't.
+      def frozen?
+        unless Google::Protobuf::FFI.map_frozen? @map_ptr
+          raise RuntimeError.new "Ruby frozen Map with mutable representation" if super
+          return false
+        end
+        method(:freeze).super_method.call unless super
+        true
+      end
+
+      ##
+      # Freezes the map object. We have to intercept this so we can freeze the
+      # underlying representation, not just the Ruby wrapper. Returns self.
+      def freeze
+        if method(:frozen?).super_method.call
+          unless Google::Protobuf::FFI.map_frozen? @map_ptr
+            raise RuntimeError.new "Underlying representation of map still mutable despite frozen wrapper"
+          end
+          return self
+        end
+        unless Google::Protobuf::FFI.map_frozen? @map_ptr
+          mini_table = (value_type == :message) ? Google::Protobuf::FFI.get_mini_table(@descriptor) : nil
+          Google::Protobuf::FFI.map_freeze(@map_ptr, mini_table)
+        end
+        super
+      end
+
       ##
       # call-seq:
       #    Map.dup => new_map
@@ -199,8 +232,8 @@ module Google
       def hash
         return_value = 0
         each_msg_val do |key_message_value, value_message_value|
-          return_value = Google::Protobuf::FFI.message_value_hash(key_message_value, key_type, nil, return_value)
-          return_value = Google::Protobuf::FFI.message_value_hash(value_message_value, value_type, descriptor, return_value)
+          return_value += Google::Protobuf::FFI.message_value_hash(key_message_value, key_type, nil, 0)
+          return_value += Google::Protobuf::FFI.message_value_hash(value_message_value, value_type, descriptor, 0)
         end
         return_value
       end
@@ -269,17 +302,6 @@ module Google
 
       include Google::Protobuf::Internal::Convert
 
-      def internal_deep_freeze
-        freeze
-        if value_type == :message
-          internal_iterator do |iterator|
-            value_message_value = Google::Protobuf::FFI.map_value(@map_ptr, iterator)
-            convert_upb_to_ruby(value_message_value, value_type, descriptor, arena).send :internal_deep_freeze
-          end
-        end
-        self
-      end
-
       def internal_iterator
         iter = ::FFI::MemoryPointer.new(:size_t, 1)
         iter.write(:size_t, Google::Protobuf::FFI::Upb_Map_Begin)
@@ -369,10 +391,14 @@ module Google
         OBJECT_CACHE.try_add(@map_ptr.address, self)
       end
 
-      # @param field [FieldDescriptor] Descriptor of the field where the RepeatedField will be assigned
-      # @param values [Hash|Map] Initial value; may be nil or empty
+      ##
+      # Constructor that uses the type information from the given
+      # FieldDescriptor to configure the new Map instance.
+      # @param field [FieldDescriptor] Type information for the new Map
       # @param arena [Arena] Owning message's arena
-      def self.construct_for_field(field, arena, value: nil, map: nil)
+      # @param value [Hash|Map] Initial value
+      # @param map [::FFI::Pointer] Existing upb_Map
+      def self.construct_for_field(field, arena: nil, value: nil, map: nil)
         raise ArgumentError.new "Expected Hash object as initializer value for map field '#{field.name}' (given #{value.class})." unless value.nil? or value.is_a? Hash
         instance = allocate
         raise ArgumentError.new "Expected field with type :message, instead got #{field.class}" unless field.type == :message