google-protobuf 3.7.1 → 3.17.3

data/ext/google/protobuf_c/third_party/wyhash/wyhash.h

@@ -0,0 +1,145 @@
+/* Copyright 2020 王一 Wang Yi <godspeed_china@yeah.net>
+   This is free and unencumbered software released into the public domain. http://unlicense.org/
+   See github.com/wangyi-fudan/wyhash/ LICENSE
+ */
+#ifndef wyhash_final_version
+#define wyhash_final_version
+//defines that change behavior
+#ifndef WYHASH_CONDOM
+#define WYHASH_CONDOM 1 //0: read 8 bytes before and after boundaries, dangerous but fastest. 1: normal valid behavior 2: extra protection against entropy loss (probability=2^-63), aka. "blind multiplication"
+#endif
+#define WYHASH_32BIT_MUM 0	//faster on 32 bit system
+//includes
+#include <stdint.h>
+#include <string.h>
+#if defined(_MSC_VER) && defined(_M_X64)
+  #include <intrin.h>
+  #pragma intrinsic(_umul128)
+#endif
+#if defined(__GNUC__) || defined(__INTEL_COMPILER) || defined(__clang__)
+  #define _likely_(x)	__builtin_expect(x,1)
+  #define _unlikely_(x)	__builtin_expect(x,0)
+#else
+  #define _likely_(x) (x)
+  #define _unlikely_(x) (x)
+#endif
+//mum function
+static inline uint64_t _wyrot(uint64_t x) { return (x>>32)|(x<<32); }
+static inline void _wymum(uint64_t *A, uint64_t *B){
+#if(WYHASH_32BIT_MUM)
+  uint64_t hh=(*A>>32)*(*B>>32), hl=(*A>>32)*(unsigned)*B, lh=(unsigned)*A*(*B>>32), ll=(uint64_t)(unsigned)*A*(unsigned)*B;
+  #if(WYHASH_CONDOM>1)
+  *A^=_wyrot(hl)^hh; *B^=_wyrot(lh)^ll;
+  #else
+  *A=_wyrot(hl)^hh; *B=_wyrot(lh)^ll;
+  #endif
+#elif defined(__SIZEOF_INT128__)
+  __uint128_t r=*A; r*=*B; 
+  #if(WYHASH_CONDOM>1)
+  *A^=(uint64_t)r; *B^=(uint64_t)(r>>64);
+  #else
+  *A=(uint64_t)r; *B=(uint64_t)(r>>64);
+  #endif
+#elif defined(_MSC_VER) && defined(_M_X64)
+  #if(WYHASH_CONDOM>1)
+  uint64_t  a,  b;
+  a=_umul128(*A,*B,&b);
+  *A^=a;  *B^=b;
+  #else
+  *A=_umul128(*A,*B,B);
+  #endif
+#else
+  uint64_t ha=*A>>32, hb=*B>>32, la=(uint32_t)*A, lb=(uint32_t)*B, hi, lo;
+  uint64_t rh=ha*hb, rm0=ha*lb, rm1=hb*la, rl=la*lb, t=rl+(rm0<<32), c=t<rl;
+  lo=t+(rm1<<32); c+=lo<t; hi=rh+(rm0>>32)+(rm1>>32)+c;
+  #if(WYHASH_CONDOM>1)
+  *A^=lo;  *B^=hi;
+  #else
+  *A=lo;  *B=hi;
+  #endif
+#endif
+}
+static inline uint64_t _wymix(uint64_t A, uint64_t B){ _wymum(&A,&B); return A^B; }
+//read functions
+#ifndef WYHASH_LITTLE_ENDIAN
+  #if defined(_WIN32) || defined(__LITTLE_ENDIAN__) || (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__)
+    #define WYHASH_LITTLE_ENDIAN 1
+  #elif defined(__BIG_ENDIAN__) || (defined(__BYTE_ORDER__) && __BYTE_ORDER__ == __ORDER_BIG_ENDIAN__)
+    #define WYHASH_LITTLE_ENDIAN 0
+  #endif
+#endif
+#if (WYHASH_LITTLE_ENDIAN)
+static inline uint64_t _wyr8(const uint8_t *p) { uint64_t v; memcpy(&v, p, 8); return v;}
+static inline uint64_t _wyr4(const uint8_t *p) { unsigned v; memcpy(&v, p, 4); return v;}
+#elif defined(__GNUC__) || defined(__INTEL_COMPILER) || defined(__clang__)
+static inline uint64_t _wyr8(const uint8_t *p) { uint64_t v; memcpy(&v, p, 8); return __builtin_bswap64(v);}
+static inline uint64_t _wyr4(const uint8_t *p) { unsigned v; memcpy(&v, p, 4); return __builtin_bswap32(v);}
+#elif defined(_MSC_VER)
+static inline uint64_t _wyr8(const uint8_t *p) { uint64_t v; memcpy(&v, p, 8); return _byteswap_uint64(v);}
+static inline uint64_t _wyr4(const uint8_t *p) { unsigned v; memcpy(&v, p, 4); return _byteswap_ulong(v);}
+#endif
+static inline uint64_t _wyr3(const uint8_t *p, unsigned k) { return (((uint64_t)p[0])<<16)|(((uint64_t)p[k>>1])<<8)|p[k-1];}
+//wyhash function
+static inline uint64_t _wyfinish16(const uint8_t *p, uint64_t len, uint64_t seed, const uint64_t *secret, uint64_t i){
+#if(WYHASH_CONDOM>0)
+  uint64_t a, b;
+  if(_likely_(i<=8)){
+    if(_likely_(i>=4)){ a=_wyr4(p); b=_wyr4(p+i-4); }
+    else if (_likely_(i)){ a=_wyr3(p,i); b=0; }
+    else a=b=0;
+  } 
+  else{ a=_wyr8(p); b=_wyr8(p+i-8); }
+  return _wymix(secret[1]^len,_wymix(a^secret[1], b^seed));
+#else
+  #define oneshot_shift ((i<8)*((8-i)<<3))
+  return _wymix(secret[1]^len,_wymix((_wyr8(p)<<oneshot_shift)^secret[1],(_wyr8(p+i-8)>>oneshot_shift)^seed));
+#endif
+}
+
+static inline uint64_t _wyfinish(const uint8_t *p, uint64_t len, uint64_t seed, const uint64_t *secret, uint64_t i){
+  if(_likely_(i<=16)) return _wyfinish16(p,len,seed,secret,i);
+  return _wyfinish(p+16,len,_wymix(_wyr8(p)^secret[1],_wyr8(p+8)^seed),secret,i-16);
+}
+
+static inline uint64_t wyhash(const void *key, uint64_t len, uint64_t seed, const uint64_t *secret){
+  const uint8_t *p=(const uint8_t *)key;
+  uint64_t i=len; seed^=*secret;
+  if(_unlikely_(i>64)){
+    uint64_t see1=seed;
+    do{
+      seed=_wymix(_wyr8(p)^secret[1],_wyr8(p+8)^seed)^_wymix(_wyr8(p+16)^secret[2],_wyr8(p+24)^seed);
+      see1=_wymix(_wyr8(p+32)^secret[3],_wyr8(p+40)^see1)^_wymix(_wyr8(p+48)^secret[4],_wyr8(p+56)^see1);
+      p+=64; i-=64;
+    }while(i>64);
+    seed^=see1;
+  }
+  return _wyfinish(p,len,seed,secret,i);
+}
+//utility functions
+static const uint64_t _wyp[5] = {0xa0761d6478bd642full, 0xe7037ed1a0b428dbull, 0x8ebc6af09c88c6e3ull, 0x589965cc75374cc3ull, 0x1d8e4e27c47d124full};
+static inline uint64_t wyhash64(uint64_t A, uint64_t B){  A^=_wyp[0]; B^=_wyp[1];  _wymum(&A,&B);  return _wymix(A^_wyp[0],B^_wyp[1]);}
+static inline uint64_t wyrand(uint64_t *seed){  *seed+=_wyp[0]; return _wymix(*seed,*seed^_wyp[1]);}
+static inline double wy2u01(uint64_t r){ const double _wynorm=1.0/(1ull<<52); return (r>>12)*_wynorm;}
+static inline double wy2gau(uint64_t r){ const double _wynorm=1.0/(1ull<<20); return ((r&0x1fffff)+((r>>21)&0x1fffff)+((r>>42)&0x1fffff))*_wynorm-3.0;}
+static inline uint64_t wy2u0k(uint64_t r, uint64_t k){ _wymum(&r,&k); return k; }
+
+static inline void make_secret(uint64_t seed, uint64_t *secret){
+  uint8_t c[] = {15, 23, 27, 29, 30, 39, 43, 45, 46, 51, 53, 54, 57, 58, 60, 71, 75, 77, 78, 83, 85, 86, 89, 90, 92, 99, 101, 102, 105, 106, 108, 113, 114, 116, 120, 135, 139, 141, 142, 147, 149, 150, 153, 154, 156, 163, 165, 166, 169, 170, 172, 177, 178, 180, 184, 195, 197, 198, 201, 202, 204, 209, 210, 212, 216, 225, 226, 228, 232, 240 };
+  for(size_t i=0;i<5;i++){
+    uint8_t ok;
+    do{
+      ok=1; secret[i]=0;
+      for(size_t j=0;j<64;j+=8) secret[i]|=((uint64_t)c[wyrand(&seed)%sizeof(c)])<<j;
+      if(secret[i]%2==0){ ok=0; continue; }
+      for(size_t j=0;j<i;j++)
+#if defined(__GNUC__) || defined(__INTEL_COMPILER) || defined(__clang__)
+        if(__builtin_popcountll(secret[j]^secret[i])!=32){ ok=0; break; }
+#elif defined(_MSC_VER) && defined(_M_X64)
+        if(_mm_popcnt_u64(secret[j]^secret[i])!=32){ ok=0; break; }
+#endif
+       if(!ok)continue;
+       for(uint64_t j=3;j<0x100000000ull;j+=2) if(secret[i]%j==0){ ok=0; break; }
+    }while(!ok);
+  }
+}
+#endif

data/lib/google/protobuf.rb

@@ -50,6 +50,76 @@ else
   rescue LoadError
     require 'google/protobuf_c'
   end
+
+  module Google
+    module Protobuf
+      module Internal
+        def self.infer_package(names)
+          # Package is longest common prefix ending in '.', if any.
+          if not names.empty?
+            min, max = names.minmax
+            last_common_dot = nil
+            min.size.times { |i|
+              if min[i] != max[i] then break end
+              if min[i] == ?. then last_common_dot = i end
+            }
+            if last_common_dot
+              return min.slice(0, last_common_dot)
+            end
+          end
+
+          nil
+        end
+
+        class NestingBuilder
+          def initialize(msg_names, enum_names)
+            @to_pos = {nil=>nil}
+            @msg_children = Hash.new { |hash, key| hash[key] = [] }
+            @enum_children = Hash.new { |hash, key| hash[key] = [] }
+
+            msg_names.each_with_index { |name, idx| @to_pos[name] = idx }
+            enum_names.each_with_index { |name, idx| @to_pos[name] = idx }
+
+            msg_names.each { |name| @msg_children[parent(name)] << name }
+            enum_names.each { |name| @enum_children[parent(name)] << name }
+          end
+
+          def build(package)
+            return build_msg(package)
+          end
+
+          private
+          def build_msg(msg)
+            return {
+              :pos => @to_pos[msg],
+              :msgs => @msg_children[msg].map { |child| build_msg(child) },
+              :enums => @enum_children[msg].map { |child| @to_pos[child] },
+            }
+          end
+
+          private
+          def parent(name)
+            idx = name.rindex(?.)
+            if idx
+              return name.slice(0, idx)
+            else
+              return nil
+            end
+          end
+        end
+
+        def self.fixup_descriptor(package, msg_names, enum_names)
+          if package.nil?
+            package = self.infer_package(msg_names + enum_names)
+          end
+
+          nesting = NestingBuilder.new(msg_names, enum_names).build(package)
+
+          return package, nesting
+        end
+      end
+    end
+  end
 end
 
 require 'google/protobuf/repeated_field'

data/lib/google/protobuf/any_pb.rb

@@ -14,6 +14,6 @@ end
 
 module Google
   module Protobuf
-    Any = Google::Protobuf::DescriptorPool.generated_pool.lookup("google.protobuf.Any").msgclass
+    Any = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("google.protobuf.Any").msgclass
   end
 end

data/lib/google/protobuf/api_pb.rb

@@ -34,8 +34,8 @@ end
 
 module Google
   module Protobuf
-    Api = Google::Protobuf::DescriptorPool.generated_pool.lookup("google.protobuf.Api").msgclass
-    Method = Google::Protobuf::DescriptorPool.generated_pool.lookup("google.protobuf.Method").msgclass
-    Mixin = Google::Protobuf::DescriptorPool.generated_pool.lookup("google.protobuf.Mixin").msgclass
+    Api = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("google.protobuf.Api").msgclass
+    Method = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("google.protobuf.Method").msgclass
+    Mixin = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("google.protobuf.Mixin").msgclass
   end
 end

data/lib/google/protobuf/duration_pb.rb

@@ -14,6 +14,6 @@ end
 
 module Google
   module Protobuf
-    Duration = Google::Protobuf::DescriptorPool.generated_pool.lookup("google.protobuf.Duration").msgclass
+    Duration = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("google.protobuf.Duration").msgclass
   end
 end

data/lib/google/protobuf/empty_pb.rb

@@ -12,6 +12,6 @@ end
 
 module Google
   module Protobuf
-    Empty = Google::Protobuf::DescriptorPool.generated_pool.lookup("google.protobuf.Empty").msgclass
+    Empty = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("google.protobuf.Empty").msgclass
   end
 end

data/lib/google/protobuf/field_mask_pb.rb

@@ -13,6 +13,6 @@ end
 
 module Google
   module Protobuf
-    FieldMask = Google::Protobuf::DescriptorPool.generated_pool.lookup("google.protobuf.FieldMask").msgclass
+    FieldMask = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("google.protobuf.FieldMask").msgclass
   end
 end

data/lib/google/protobuf/source_context_pb.rb

@@ -13,6 +13,6 @@ end
 
 module Google
   module Protobuf
-    SourceContext = Google::Protobuf::DescriptorPool.generated_pool.lookup("google.protobuf.SourceContext").msgclass
+    SourceContext = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("google.protobuf.SourceContext").msgclass
   end
 end

data/lib/google/protobuf/struct_pb.rb

@@ -29,9 +29,9 @@ end
 
 module Google
   module Protobuf
-    Struct = Google::Protobuf::DescriptorPool.generated_pool.lookup("google.protobuf.Struct").msgclass
-    Value = Google::Protobuf::DescriptorPool.generated_pool.lookup("google.protobuf.Value").msgclass
-    ListValue = Google::Protobuf::DescriptorPool.generated_pool.lookup("google.protobuf.ListValue").msgclass
-    NullValue = Google::Protobuf::DescriptorPool.generated_pool.lookup("google.protobuf.NullValue").enummodule
+    Struct = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("google.protobuf.Struct").msgclass
+    Value = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("google.protobuf.Value").msgclass
+    ListValue = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("google.protobuf.ListValue").msgclass
+    NullValue = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("google.protobuf.NullValue").enummodule
   end
 end

data/lib/google/protobuf/timestamp_pb.rb

@@ -14,6 +14,6 @@ end
 
 module Google
   module Protobuf
-    Timestamp = Google::Protobuf::DescriptorPool.generated_pool.lookup("google.protobuf.Timestamp").msgclass
+    Timestamp = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("google.protobuf.Timestamp").msgclass
   end
 end

data/lib/google/protobuf/type_pb.rb

@@ -79,13 +79,13 @@ end
 
 module Google
   module Protobuf
-    Type = Google::Protobuf::DescriptorPool.generated_pool.lookup("google.protobuf.Type").msgclass
-    Field = Google::Protobuf::DescriptorPool.generated_pool.lookup("google.protobuf.Field").msgclass
-    Field::Kind = Google::Protobuf::DescriptorPool.generated_pool.lookup("google.protobuf.Field.Kind").enummodule
-    Field::Cardinality = Google::Protobuf::DescriptorPool.generated_pool.lookup("google.protobuf.Field.Cardinality").enummodule
-    Enum = Google::Protobuf::DescriptorPool.generated_pool.lookup("google.protobuf.Enum").msgclass
-    EnumValue = Google::Protobuf::DescriptorPool.generated_pool.lookup("google.protobuf.EnumValue").msgclass
-    Option = Google::Protobuf::DescriptorPool.generated_pool.lookup("google.protobuf.Option").msgclass
-    Syntax = Google::Protobuf::DescriptorPool.generated_pool.lookup("google.protobuf.Syntax").enummodule
+    Type = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("google.protobuf.Type").msgclass
+    Field = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("google.protobuf.Field").msgclass
+    Field::Kind = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("google.protobuf.Field.Kind").enummodule
+    Field::Cardinality = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("google.protobuf.Field.Cardinality").enummodule
+    Enum = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("google.protobuf.Enum").msgclass
+    EnumValue = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("google.protobuf.EnumValue").msgclass
+    Option = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("google.protobuf.Option").msgclass
+    Syntax = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("google.protobuf.Syntax").enummodule
   end
 end

data/lib/google/protobuf/well_known_types.rb

@@ -72,8 +72,14 @@ module Google
     end
 
     Timestamp.class_eval do
-      def to_time
-        Time.at(self.to_f)
+      if RUBY_VERSION < "2.5"
+        def to_time
+          Time.at(self.to_f)
+        end
+      else
+        def to_time
+          Time.at(seconds, nanos, :nanosecond)
+        end
       end
 
       def from_time(time)

data/lib/google/protobuf/wrappers_pb.rb

@@ -37,14 +37,14 @@ end
 
 module Google
   module Protobuf
-    DoubleValue = Google::Protobuf::DescriptorPool.generated_pool.lookup("google.protobuf.DoubleValue").msgclass
-    FloatValue = Google::Protobuf::DescriptorPool.generated_pool.lookup("google.protobuf.FloatValue").msgclass
-    Int64Value = Google::Protobuf::DescriptorPool.generated_pool.lookup("google.protobuf.Int64Value").msgclass
-    UInt64Value = Google::Protobuf::DescriptorPool.generated_pool.lookup("google.protobuf.UInt64Value").msgclass
-    Int32Value = Google::Protobuf::DescriptorPool.generated_pool.lookup("google.protobuf.Int32Value").msgclass
-    UInt32Value = Google::Protobuf::DescriptorPool.generated_pool.lookup("google.protobuf.UInt32Value").msgclass
-    BoolValue = Google::Protobuf::DescriptorPool.generated_pool.lookup("google.protobuf.BoolValue").msgclass
-    StringValue = Google::Protobuf::DescriptorPool.generated_pool.lookup("google.protobuf.StringValue").msgclass
-    BytesValue = Google::Protobuf::DescriptorPool.generated_pool.lookup("google.protobuf.BytesValue").msgclass
+    DoubleValue = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("google.protobuf.DoubleValue").msgclass
+    FloatValue = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("google.protobuf.FloatValue").msgclass
+    Int64Value = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("google.protobuf.Int64Value").msgclass
+    UInt64Value = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("google.protobuf.UInt64Value").msgclass
+    Int32Value = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("google.protobuf.Int32Value").msgclass
+    UInt32Value = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("google.protobuf.UInt32Value").msgclass
+    BoolValue = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("google.protobuf.BoolValue").msgclass
+    StringValue = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("google.protobuf.StringValue").msgclass
+    BytesValue = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("google.protobuf.BytesValue").msgclass
   end
 end

data/tests/basic.rb

@@ -17,7 +17,6 @@ module BasicTest
     add_message "BadFieldNames" do
       optional :dup, :int32, 1
       optional :class, :int32, 2
-      optional :"a.b", :int32, 3
     end
   end
 
@@ -32,12 +31,51 @@ module BasicTest
     end
     include CommonTests
 
+    def test_issue_8311_crash
+      Google::Protobuf::DescriptorPool.generated_pool.build do
+        add_file("inner.proto", :syntax => :proto3) do
+          add_message "Inner" do
+            # Removing either of these fixes the segfault.
+            optional :foo, :string, 1
+            optional :bar, :string, 2
+          end
+        end
+      end
+
+      Google::Protobuf::DescriptorPool.generated_pool.build do
+        add_file("outer.proto", :syntax => :proto3) do
+          add_message "Outer" do
+            repeated :inners, :message, 1, "Inner"
+          end
+        end
+      end
+
+      outer = ::Google::Protobuf::DescriptorPool.generated_pool.lookup("Outer").msgclass
+
+      outer.new(
+          inners: []
+      )['inners'].to_s
+
+      assert_raise Google::Protobuf::TypeError do
+        outer.new(
+            inners: [nil]
+        ).to_s
+      end
+    end
+
+    def test_issue_8559_crash
+      msg = TestMessage.new
+      msg.repeated_int32 = ::Google::Protobuf::RepeatedField.new(:int32, [1, 2, 3])
+      GC.start(full_mark: true, immediate_sweep: true)
+      TestMessage.encode(msg)
+    end
+
     def test_has_field
-      m = TestMessage.new
-      assert !m.has_optional_msg?
-      m.optional_msg = TestMessage2.new
-      assert m.has_optional_msg?
-      assert TestMessage.descriptor.lookup('optional_msg').has?(m)
+      m = TestSingularFields.new
+      assert !m.has_singular_msg?
+      m.singular_msg = TestMessage2.new
+      assert m.has_singular_msg?
+      assert TestSingularFields.descriptor.lookup('singular_msg').has?(m)
 
       m = OneofMessage.new
       assert !m.has_my_oneof?
@@ -46,32 +84,31 @@ module BasicTest
       assert_raise NoMethodError do
         m.has_a?
       end
-      assert_raise ArgumentError do
-        OneofMessage.descriptor.lookup('a').has?(m)
-      end
+      assert_true OneofMessage.descriptor.lookup('a').has?(m)
 
-      m = TestMessage.new
+      m = TestSingularFields.new
       assert_raise NoMethodError do
-        m.has_optional_int32?
+        m.has_singular_int32?
       end
       assert_raise ArgumentError do
-        TestMessage.descriptor.lookup('optional_int32').has?(m)
+        TestSingularFields.descriptor.lookup('singular_int32').has?(m)
       end
 
       assert_raise NoMethodError do
-        m.has_optional_string?
+        m.has_singular_string?
       end
       assert_raise ArgumentError do
-        TestMessage.descriptor.lookup('optional_string').has?(m)
+        TestSingularFields.descriptor.lookup('singular_string').has?(m)
       end
 
       assert_raise NoMethodError do
-        m.has_optional_bool?
+        m.has_singular_bool?
       end
       assert_raise ArgumentError do
-        TestMessage.descriptor.lookup('optional_bool').has?(m)
+        TestSingularFields.descriptor.lookup('singular_bool').has?(m)
       end
 
+      m = TestMessage.new
       assert_raise NoMethodError do
         m.has_repeated_msg?
       end
@@ -80,40 +117,59 @@ module BasicTest
       end
     end
 
+    def test_no_presence
+      m = TestSingularFields.new
+
+      # Explicitly setting to zero does not cause anything to be serialized.
+      m.singular_int32 = 0
+      assert_equal "", TestSingularFields.encode(m)
+
+      # Explicitly setting to a non-zero value *does* cause serialization.
+      m.singular_int32 = 1
+      assert_not_equal "", TestSingularFields.encode(m)
+
+      m.singular_int32 = 0
+      assert_equal "", TestSingularFields.encode(m)
+    end
+
     def test_set_clear_defaults
-      m = TestMessage.new
+      m = TestSingularFields.new
+
+      m.singular_int32 = -42
+      assert_equal -42, m.singular_int32
+      m.clear_singular_int32
+      assert_equal 0, m.singular_int32
+
+      m.singular_int32 = 50
+      assert_equal 50, m.singular_int32
+      TestSingularFields.descriptor.lookup('singular_int32').clear(m)
+      assert_equal 0, m.singular_int32
+
+      m.singular_string = "foo bar"
+      assert_equal "foo bar", m.singular_string
+      m.clear_singular_string
+      assert_equal "", m.singular_string
+
+      m.singular_string = "foo"
+      assert_equal "foo", m.singular_string
+      TestSingularFields.descriptor.lookup('singular_string').clear(m)
+      assert_equal "", m.singular_string
+
+      m.singular_msg = TestMessage2.new(:foo => 42)
+      assert_equal TestMessage2.new(:foo => 42), m.singular_msg
+      assert m.has_singular_msg?
+      m.clear_singular_msg
+      assert_equal nil, m.singular_msg
+      assert !m.has_singular_msg?
+
+      m.singular_msg = TestMessage2.new(:foo => 42)
+      assert_equal TestMessage2.new(:foo => 42), m.singular_msg
+      TestSingularFields.descriptor.lookup('singular_msg').clear(m)
+      assert_equal nil, m.singular_msg
+    end
 
-      m.optional_int32 = -42
-      assert_equal -42, m.optional_int32
-      m.clear_optional_int32
-      assert_equal 0, m.optional_int32
-
-      m.optional_int32 = 50
-      assert_equal 50, m.optional_int32
-      TestMessage.descriptor.lookup('optional_int32').clear(m)
-      assert_equal 0, m.optional_int32
-
-      m.optional_string = "foo bar"
-      assert_equal "foo bar", m.optional_string
-      m.clear_optional_string
-      assert_equal "", m.optional_string
-
-      m.optional_string = "foo"
-      assert_equal "foo", m.optional_string
-      TestMessage.descriptor.lookup('optional_string').clear(m)
-      assert_equal "", m.optional_string
-
-      m.optional_msg = TestMessage2.new(:foo => 42)
-      assert_equal TestMessage2.new(:foo => 42), m.optional_msg
-      assert m.has_optional_msg?
-      m.clear_optional_msg
-      assert_equal nil, m.optional_msg
-      assert !m.has_optional_msg?
-
-      m.optional_msg = TestMessage2.new(:foo => 42)
-      assert_equal TestMessage2.new(:foo => 42), m.optional_msg
-      TestMessage.descriptor.lookup('optional_msg').clear(m)
-      assert_equal nil, m.optional_msg
+    def test_clear_repeated_fields
+      m = TestMessage.new
 
       m.repeated_int32.push(1)
       assert_equal [1], m.repeated_int32
@@ -129,6 +185,7 @@ module BasicTest
       m.a = "foo"
       assert_equal "foo", m.a
       assert m.has_my_oneof?
+      assert_equal :a, m.my_oneof
       m.clear_a
       assert !m.has_my_oneof?
 
@@ -144,7 +201,6 @@ module BasicTest
       assert !m.has_my_oneof?
     end
 
-
     def test_initialization_map_errors
       e = assert_raise ArgumentError do
         TestMessage.new(:hello => "world")
@@ -170,10 +226,12 @@ module BasicTest
       m = MapMessage.new(
         :map_string_int32 => {"a" => 1, "b" => 2},
         :map_string_msg => {"a" => TestMessage2.new(:foo => 1),
-                            "b" => TestMessage2.new(:foo => 2)})
+                            "b" => TestMessage2.new(:foo => 2)},
+        :map_string_enum => {"a" => :A, "b" => :B})
       assert m.map_string_int32.keys.sort == ["a", "b"]
       assert m.map_string_int32["a"] == 1
       assert m.map_string_msg["b"].foo == 2
+      assert m.map_string_enum["a"] == :A
 
       m.map_string_int32["c"] = 3
       assert m.map_string_int32["c"] == 3
@@ -197,18 +255,37 @@ module BasicTest
         m.map_string_int32 = {}
       end
 
-      assert_raise TypeError do
+      assert_raise Google::Protobuf::TypeError do
         m = MapMessage.new(:map_string_int32 => { 1 => "I am not a number" })
       end
     end
 
+    def test_map_field_with_symbol
+      m = MapMessage.new
+      assert m.map_string_int32 == {}
+      assert m.map_string_msg == {}
+
+      m = MapMessage.new(
+        :map_string_int32 => {a: 1, "b" => 2},
+        :map_string_msg => {a: TestMessage2.new(:foo => 1),
+                            b: TestMessage2.new(:foo => 10)})
+      assert_equal 1, m.map_string_int32[:a]
+      assert_equal 2, m.map_string_int32[:b]
+      assert_equal 10, m.map_string_msg[:b].foo
+    end
+
     def test_map_inspect
       m = MapMessage.new(
         :map_string_int32 => {"a" => 1, "b" => 2},
         :map_string_msg => {"a" => TestMessage2.new(:foo => 1),
-                            "b" => TestMessage2.new(:foo => 2)})
-      expected = "<BasicTest::MapMessage: map_string_int32: {\"b\"=>2, \"a\"=>1}, map_string_msg: {\"b\"=><BasicTest::TestMessage2: foo: 2>, \"a\"=><BasicTest::TestMessage2: foo: 1>}>"
-      assert_equal expected, m.inspect
+                            "b" => TestMessage2.new(:foo => 2)},
+        :map_string_enum => {"a" => :A, "b" => :B})
+
+      # JRuby doesn't keep consistent ordering so check for either version
+      expected_a = "<BasicTest::MapMessage: map_string_int32: {\"b\"=>2, \"a\"=>1}, map_string_msg: {\"b\"=><BasicTest::TestMessage2: foo: 2>, \"a\"=><BasicTest::TestMessage2: foo: 1>}, map_string_enum: {\"b\"=>:B, \"a\"=>:A}>"
+      expected_b = "<BasicTest::MapMessage: map_string_int32: {\"a\"=>1, \"b\"=>2}, map_string_msg: {\"a\"=><BasicTest::TestMessage2: foo: 1>, \"b\"=><BasicTest::TestMessage2: foo: 2>}, map_string_enum: {\"a\"=>:A, \"b\"=>:B}>"
+      inspect_result = m.inspect
+      assert expected_a == inspect_result || expected_b == inspect_result, "Incorrect inspect result: #{inspect_result}"
     end
 
     def test_map_corruption
@@ -218,6 +295,128 @@ module BasicTest
       m.map_string_int32['aaa'] = 3
     end
 
+    def test_map_wrappers
+      run_asserts = ->(m) {
+        assert_equal 2.0, m.map_double[0].value
+        assert_equal 4.0, m.map_float[0].value
+        assert_equal 3, m.map_int32[0].value
+        assert_equal 4, m.map_int64[0].value
+        assert_equal 5, m.map_uint32[0].value
+        assert_equal 6, m.map_uint64[0].value
+        assert_equal true, m.map_bool[0].value
+        assert_equal 'str', m.map_string[0].value
+        assert_equal 'fun', m.map_bytes[0].value
+      }
+
+      m = proto_module::Wrapper.new(
+        map_double: {0 => Google::Protobuf::DoubleValue.new(value: 2.0)},
+        map_float: {0 => Google::Protobuf::FloatValue.new(value: 4.0)},
+        map_int32: {0 => Google::Protobuf::Int32Value.new(value: 3)},
+        map_int64: {0 => Google::Protobuf::Int64Value.new(value: 4)},
+        map_uint32: {0 => Google::Protobuf::UInt32Value.new(value: 5)},
+        map_uint64: {0 => Google::Protobuf::UInt64Value.new(value: 6)},
+        map_bool: {0 => Google::Protobuf::BoolValue.new(value: true)},
+        map_string: {0 => Google::Protobuf::StringValue.new(value: 'str')},
+        map_bytes: {0 => Google::Protobuf::BytesValue.new(value: 'fun')},
+      )
+
+      run_asserts.call(m)
+      serialized = proto_module::Wrapper::encode(m)
+      m2 = proto_module::Wrapper::decode(serialized)
+      run_asserts.call(m2)
+
+      # Test the case where we are serializing directly from the parsed form
+      # (before anything lazy is materialized).
+      m3 = proto_module::Wrapper::decode(serialized)
+      serialized2 = proto_module::Wrapper::encode(m3)
+      m4 = proto_module::Wrapper::decode(serialized2)
+      run_asserts.call(m4)
+
+      # Test that the lazy form compares equal to the expanded form.
+      m5 = proto_module::Wrapper::decode(serialized2)
+      assert_equal m5, m
+    end
+
+    def test_map_wrappers_with_default_values
+      run_asserts = ->(m) {
+        assert_equal 0.0, m.map_double[0].value
+        assert_equal 0.0, m.map_float[0].value
+        assert_equal 0, m.map_int32[0].value
+        assert_equal 0, m.map_int64[0].value
+        assert_equal 0, m.map_uint32[0].value
+        assert_equal 0, m.map_uint64[0].value
+        assert_equal false, m.map_bool[0].value
+        assert_equal '', m.map_string[0].value
+        assert_equal '', m.map_bytes[0].value
+      }
+
+      m = proto_module::Wrapper.new(
+        map_double: {0 => Google::Protobuf::DoubleValue.new(value: 0.0)},
+        map_float: {0 => Google::Protobuf::FloatValue.new(value: 0.0)},
+        map_int32: {0 => Google::Protobuf::Int32Value.new(value: 0)},
+        map_int64: {0 => Google::Protobuf::Int64Value.new(value: 0)},
+        map_uint32: {0 => Google::Protobuf::UInt32Value.new(value: 0)},
+        map_uint64: {0 => Google::Protobuf::UInt64Value.new(value: 0)},
+        map_bool: {0 => Google::Protobuf::BoolValue.new(value: false)},
+        map_string: {0 => Google::Protobuf::StringValue.new(value: '')},
+        map_bytes: {0 => Google::Protobuf::BytesValue.new(value: '')},
+      )
+
+      run_asserts.call(m)
+      serialized = proto_module::Wrapper::encode(m)
+      m2 = proto_module::Wrapper::decode(serialized)
+      run_asserts.call(m2)
+
+      # Test the case where we are serializing directly from the parsed form
+      # (before anything lazy is materialized).
+      m3 = proto_module::Wrapper::decode(serialized)
+      serialized2 = proto_module::Wrapper::encode(m3)
+      m4 = proto_module::Wrapper::decode(serialized2)
+      run_asserts.call(m4)
+
+      # Test that the lazy form compares equal to the expanded form.
+      m5 = proto_module::Wrapper::decode(serialized2)
+      assert_equal m5, m
+    end
+
+    def test_map_wrappers_with_no_value
+      run_asserts = ->(m) {
+        assert_equal 0.0, m.map_double[0].value
+        assert_equal 0.0, m.map_float[0].value
+        assert_equal 0, m.map_int32[0].value
+        assert_equal 0, m.map_int64[0].value
+        assert_equal 0, m.map_uint32[0].value
+        assert_equal 0, m.map_uint64[0].value
+        assert_equal false, m.map_bool[0].value
+        assert_equal '', m.map_string[0].value
+        assert_equal '', m.map_bytes[0].value
+      }
+
+      m = proto_module::Wrapper.new(
+        map_double: {0 => Google::Protobuf::DoubleValue.new()},
+        map_float: {0 => Google::Protobuf::FloatValue.new()},
+        map_int32: {0 => Google::Protobuf::Int32Value.new()},
+        map_int64: {0 => Google::Protobuf::Int64Value.new()},
+        map_uint32: {0 => Google::Protobuf::UInt32Value.new()},
+        map_uint64: {0 => Google::Protobuf::UInt64Value.new()},
+        map_bool: {0 => Google::Protobuf::BoolValue.new()},
+        map_string: {0 => Google::Protobuf::StringValue.new()},
+        map_bytes: {0 => Google::Protobuf::BytesValue.new()},
+      )
+      run_asserts.call(m)
+
+      serialized = proto_module::Wrapper::encode(m)
+      m2 = proto_module::Wrapper::decode(serialized)
+      run_asserts.call(m2)
+
+      # Test the case where we are serializing directly from the parsed form
+      # (before anything lazy is materialized).
+      m3 = proto_module::Wrapper::decode(serialized)
+      serialized2 = proto_module::Wrapper::encode(m3)
+      m4 = proto_module::Wrapper::decode(serialized2)
+      run_asserts.call(m4)
+    end
+
     def test_concurrent_decoding
       o = Outer.new
       o.items[0] = Inner.new
@@ -237,7 +436,8 @@ module BasicTest
       m = MapMessage.new(
         :map_string_int32 => {"a" => 1, "b" => 2},
         :map_string_msg => {"a" => TestMessage2.new(:foo => 1),
-                            "b" => TestMessage2.new(:foo => 2)})
+                            "b" => TestMessage2.new(:foo => 2)},
+        :map_string_enum => {"a" => :A, "b" => :B})
       m2 = MapMessage.decode(MapMessage.encode(m))
       assert m == m2
 
@@ -267,6 +467,14 @@ module BasicTest
       assert_match(/No such field: not_in_message/, e.message)
     end
 
+    #def test_json_quoted_string
+    #  m = TestMessage.decode_json(%q(
+    #    "optionalInt64": "1",,
+    #  }))
+    #  puts(m)
+    #  assert_equal 1, m.optional_int32
+    #end
+
     def test_to_h
       m = TestMessage.new(:optional_bool => true, :optional_double => -10.100001, :optional_string => 'foo', :repeated_string => ['bar1', 'bar2'], :repeated_msg => [TestMessage2.new(:foo => 100)])
       expected_result = {
@@ -278,6 +486,7 @@ module BasicTest
         :optional_int32=>0,
         :optional_int64=>0,
         :optional_msg=>nil,
+        :optional_msg2=>nil,
         :optional_string=>"foo",
         :optional_uint32=>0,
         :optional_uint64=>0,
@@ -298,10 +507,12 @@ module BasicTest
       m = MapMessage.new(
         :map_string_int32 => {"a" => 1, "b" => 2},
         :map_string_msg => {"a" => TestMessage2.new(:foo => 1),
-                            "b" => TestMessage2.new(:foo => 2)})
+                            "b" => TestMessage2.new(:foo => 2)},
+        :map_string_enum => {"a" => :A, "b" => :B})
       expected_result = {
         :map_string_int32 => {"a" => 1, "b" => 2},
-        :map_string_msg => {"a" => {:foo => 1}, "b" => {:foo => 2}}
+        :map_string_msg => {"a" => {:foo => 1}, "b" => {:foo => 2}},
+        :map_string_enum => {"a" => :A, "b" => :B}
       }
       assert_equal expected_result, m.to_h
     end
@@ -311,26 +522,50 @@ module BasicTest
       # TODO: Fix JSON in JRuby version.
       return if RUBY_PLATFORM == "java"
       m = MapMessage.new(:map_string_int32 => {"a" => 1})
-      expected = {mapStringInt32: {a: 1}, mapStringMsg: {}}
-      expected_preserve = {map_string_int32: {a: 1}, map_string_msg: {}}
-      assert JSON.parse(MapMessage.encode_json(m), :symbolize_names => true) == expected
+      expected = {mapStringInt32: {a: 1}, mapStringMsg: {}, mapStringEnum: {}}
+      expected_preserve = {map_string_int32: {a: 1}, map_string_msg: {}, map_string_enum: {}}
+      assert_equal JSON.parse(MapMessage.encode_json(m, :emit_defaults=>true), :symbolize_names => true), expected
 
-      json = MapMessage.encode_json(m, :preserve_proto_fieldnames => true)
-      assert JSON.parse(json, :symbolize_names => true) == expected_preserve
+      json = MapMessage.encode_json(m, :preserve_proto_fieldnames => true, :emit_defaults=>true)
+      assert_equal JSON.parse(json, :symbolize_names => true), expected_preserve
 
       m2 = MapMessage.decode_json(MapMessage.encode_json(m))
-      assert m == m2
+      assert_equal m, m2
     end
 
     def test_json_maps_emit_defaults_submsg
       # TODO: Fix JSON in JRuby version.
       return if RUBY_PLATFORM == "java"
-      m = MapMessage.new(:map_string_msg => {"a" => TestMessage2.new})
-      expected = {mapStringInt32: {}, mapStringMsg: {a: {foo: 0}}}
+      m = MapMessage.new(:map_string_msg => {"a" => TestMessage2.new(foo: 0)})
+      expected = {mapStringInt32: {}, mapStringMsg: {a: {foo: 0}}, mapStringEnum: {}}
 
       actual = MapMessage.encode_json(m, :emit_defaults => true)
 
-      assert JSON.parse(actual, :symbolize_names => true) == expected
+      assert_equal JSON.parse(actual, :symbolize_names => true), expected
+    end
+
+    def test_json_emit_defaults_submsg
+      # TODO: Fix JSON in JRuby version.
+      return if RUBY_PLATFORM == "java"
+      m = TestSingularFields.new(singular_msg: proto_module::TestMessage2.new)
+
+      expected = {
+        singularInt32: 0,
+        singularInt64: "0",
+        singularUint32: 0,
+        singularUint64: "0",
+        singularBool: false,
+        singularFloat: 0,
+        singularDouble: 0,
+        singularString: "",
+        singularBytes: "",
+        singularMsg: {},
+        singularEnum: "Default",
+      }
+
+      actual = proto_module::TestMessage.encode_json(m, :emit_defaults => true)
+
+      assert_equal expected, JSON.parse(actual, :symbolize_names => true)
     end
 
     def test_respond_to
@@ -351,11 +586,26 @@ module BasicTest
       assert nil != file_descriptor
       assert_equal "tests/basic_test.proto", file_descriptor.name
       assert_equal :proto3, file_descriptor.syntax
+    end
 
-      file_descriptor = BadFieldNames.descriptor.file_descriptor
-      assert nil != file_descriptor
-      assert_equal nil, file_descriptor.name
-      assert_equal :proto3, file_descriptor.syntax
+    # Ruby 2.5 changed to raise FrozenError instead of RuntimeError
+    FrozenErrorType = Gem::Version.new(RUBY_VERSION) < Gem::Version.new('2.5') ? RuntimeError : FrozenError
+
+    def test_map_freeze
+      m = proto_module::MapMessage.new
+      m.map_string_int32['a'] = 5
+      m.map_string_msg['b'] = proto_module::TestMessage2.new
+
+      m.map_string_int32.freeze
+      m.map_string_msg.freeze
+
+      assert m.map_string_int32.frozen?
+      assert m.map_string_msg.frozen?
+
+      assert_raise(FrozenErrorType) { m.map_string_int32['foo'] = 1 }
+      assert_raise(FrozenErrorType) { m.map_string_msg['bar'] = proto_module::TestMessage2.new }
+      assert_raise(FrozenErrorType) { m.map_string_int32.delete('a') }
+      assert_raise(FrozenErrorType) { m.map_string_int32.clear }
     end
   end
 end