grpc 1.32.0-universal-darwin → 1.35.0-universal-darwin

data/src/ruby/pb/test/xds_client.rb

@@ -39,11 +39,38 @@ require_relative '../src/proto/grpc/testing/empty_pb'
 require_relative '../src/proto/grpc/testing/messages_pb'
 require_relative '../src/proto/grpc/testing/test_services_pb'
 
+class RpcConfig
+  def init(rpcs_to_send, metadata_to_send)
+    @rpcs_to_send = rpcs_to_send
+    @metadata_to_send = metadata_to_send
+  end
+  def rpcs_to_send
+    @rpcs_to_send
+  end
+  def metadata_to_send
+    @metadata_to_send
+  end
+end
+
+# Some global constant mappings
+$RPC_MAP = {
+  'UnaryCall' => :UNARY_CALL,
+  'EmptyCall' => :EMPTY_CALL,
+}
+
 # Some global variables to be shared by server and client
 $watchers = Array.new
 $watchers_mutex = Mutex.new
 $watchers_cv = ConditionVariable.new
 $shutdown = false
+# These can be configured by the test runner dynamically
+$rpc_config = RpcConfig.new
+$rpc_config.init([:UNARY_CALL], {})
+# These stats are shared across threads
+$accumulated_stats_mu = Mutex.new
+$num_rpcs_started_by_method = {}
+$num_rpcs_succeeded_by_method = {}
+$num_rpcs_failed_by_method = {}
 
 # RubyLogger defines a logger for gRPC based on the standard ruby logger.
 module RubyLogger
@@ -71,6 +98,31 @@ def create_stub(opts)
   )
 end
 
+class ConfigureTarget < Grpc::Testing::XdsUpdateClientConfigureService::Service
+  include Grpc::Testing
+
+  def configure(req, _call)
+    rpcs_to_send = req['types'];
+    metadata_to_send = {}
+    req['metadata'].each do |m|
+      rpc = m.type
+      if !metadata_to_send.key?(rpc)
+        metadata_to_send[rpc] = {}
+      end
+      metadata_key = m.key
+      metadata_value = m.value
+      metadata_to_send[rpc][metadata_key] = metadata_value
+    end
+    GRPC.logger.info("Configuring new rpcs_to_send and metadata_to_send...")
+    GRPC.logger.info(rpcs_to_send)
+    GRPC.logger.info(metadata_to_send)
+    new_rpc_config = RpcConfig.new
+    new_rpc_config.init(rpcs_to_send, metadata_to_send)
+    $rpc_config = new_rpc_config
+    ClientConfigureResponse.new();
+  end
+end
+
 # This implements LoadBalancerStatsService required by the test runner
 class TestTarget < Grpc::Testing::LoadBalancerStatsService::Service
   include Grpc::Testing
@@ -109,10 +161,20 @@ class TestTarget < Grpc::Testing::LoadBalancerStatsService::Service
       num_failures: watcher['no_remote_peer'] + watcher['rpcs_needed']
     );
   end
+
+  def get_client_accumulated_stats(req, _call)
+    $accumulated_stats_mu.synchronize do
+      LoadBalancerAccumulatedStatsResponse.new(
+        num_rpcs_started_by_method: $num_rpcs_started_by_method,
+        num_rpcs_succeeded_by_method: $num_rpcs_succeeded_by_method,
+        num_rpcs_failed_by_method: $num_rpcs_failed_by_method
+      )
+    end
+  end
 end
 
 # execute 1 RPC and return remote hostname
-def execute_rpc(op, fail_on_failed_rpcs)
+def execute_rpc(op, fail_on_failed_rpcs, rpc_stats_key)
   remote_peer = ""
   begin
     op.execute
@@ -120,60 +182,108 @@ def execute_rpc(op, fail_on_failed_rpcs)
       remote_peer = op.metadata['hostname']
     end
   rescue GRPC::BadStatus => e
-    GRPC.logger.info("ruby xds: rpc failed:|#{e.message}|, " \
-                     "this may or may not be expected")
     if fail_on_failed_rpcs
       raise e
     end
   end
+  $accumulated_stats_mu.synchronize do
+    if remote_peer.empty?
+      $num_rpcs_failed_by_method[rpc_stats_key] += 1
+    else
+      $num_rpcs_succeeded_by_method[rpc_stats_key] += 1
+    end
+  end
   remote_peer
 end
 
+def execute_rpc_in_thread(op, rpc_stats_key)
+  Thread.new {
+    begin
+      op.execute
+      # The following should _not_ happen with the current spec
+      # because we are only executing RPCs in a thread if we expect it
+      # to be kept open, or deadline_exceeded, or dropped by the load
+      # balancing policy. These RPCs should not complete successfully.
+      # Doing this for consistency
+      $accumulated_stats_mu.synchronize do
+        $num_rpcs_succeeded_by_method[rpc_stats_key] += 1
+      end
+    rescue GRPC::BadStatus => e
+      # Normal execution arrives here,
+      # either because of deadline_exceeded or "call dropped by load
+      # balancing policy"
+      $accumulated_stats_mu.synchronize do
+        $num_rpcs_failed_by_method[rpc_stats_key] += 1
+      end
+    end
+  }
+end
+
 # send 1 rpc every 1/qps second
-def run_test_loop(stub, target_seconds_between_rpcs, fail_on_failed_rpcs,
-                  rpcs_to_send, metadata_to_send)
+def run_test_loop(stub, target_seconds_between_rpcs, fail_on_failed_rpcs)
   include Grpc::Testing
   simple_req = SimpleRequest.new()
   empty_req = Empty.new()
   target_next_start = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+  # Some RPCs are meant to be "kept open". Since Ruby does not have an
+  # async API, we are executing those RPCs in a thread so that they don't
+  # block.
+  keep_open_threads = Array.new
   while !$shutdown
     now = Process.clock_gettime(Process::CLOCK_MONOTONIC)
     sleep_seconds = target_next_start - now
     if sleep_seconds < 0
       target_next_start = now + target_seconds_between_rpcs
-      GRPC.logger.info(
-        "ruby xds: warning, rpc takes too long to finish. " \
-        "Deficit = %.1fms. " \
-        "If you consistently see this, the qps is too high." \
-        % [(sleep_seconds * 1000).abs().round(1)])
     else
       target_next_start += target_seconds_between_rpcs
       sleep(sleep_seconds)
     end
     deadline = GRPC::Core::TimeConsts::from_relative_time(30) # 30 seconds
     results = {}
-    rpcs_to_send.each do |rpc|
-      metadata = metadata_to_send.key?(rpc) ? metadata_to_send[rpc] : {}
-      if rpc == 'UnaryCall'
+    $rpc_config.rpcs_to_send.each do |rpc|
+      # rpc is in the form of :UNARY_CALL or :EMPTY_CALL here
+      metadata = $rpc_config.metadata_to_send.key?(rpc) ?
+                   $rpc_config.metadata_to_send[rpc] : {}
+      $accumulated_stats_mu.synchronize do
+        $num_rpcs_started_by_method[rpc.to_s] += 1
+        num_started = $num_rpcs_started_by_method[rpc.to_s]
+        if num_started % 100 == 0
+          GRPC.logger.info("Started #{num_started} of #{rpc}")
+        end
+      end
+      if rpc == :UNARY_CALL
         op = stub.unary_call(simple_req,
                              metadata: metadata,
                              deadline: deadline,
                              return_op: true)
-      elsif rpc == 'EmptyCall'
+      elsif rpc == :EMPTY_CALL
         op = stub.empty_call(empty_req,
                              metadata: metadata,
                              deadline: deadline,
                              return_op: true)
       else
-        raise "Unsupported rpc %s" % [rpc]
+        raise "Unsupported rpc #{rpc}"
+      end
+      rpc_stats_key = rpc.to_s
+      if metadata.key?('rpc-behavior') and
+        (metadata['rpc-behavior'] == 'keep-open')
+        num_open_threads = keep_open_threads.size
+        if num_open_threads % 50 == 0
+          GRPC.logger.info("number of keep_open_threads = #{num_open_threads}")
+        end
+        keep_open_threads << execute_rpc_in_thread(op, rpc_stats_key)
+      else
+        results[rpc] = execute_rpc(op, fail_on_failed_rpcs, rpc_stats_key)
       end
-      results[rpc] = execute_rpc(op, fail_on_failed_rpcs)
     end
     $watchers_mutex.synchronize do
       $watchers.each do |watcher|
         # this is counted once when each group of all rpcs_to_send were done
         watcher['rpcs_needed'] -= 1
         results.each do |rpc_name, remote_peer|
+          # These stats expect rpc_name to be in the form of
+          # UnaryCall or EmptyCall, not the underscore-case all-caps form
+          rpc_name = $RPC_MAP.invert()[rpc_name]
           if remote_peer.strip.empty?
             # error is counted per individual RPC
             watcher['no_remote_peer'] += 1
@@ -191,6 +301,7 @@ def run_test_loop(stub, target_seconds_between_rpcs, fail_on_failed_rpcs,
       $watchers_cv.broadcast
     end
   end
+  keep_open_threads.each { |thd| thd.join }
 end
 
 # Args is used to hold the command line info.
@@ -242,18 +353,22 @@ def main
   s = GRPC::RpcServer.new
   s.add_http2_port(host, :this_port_is_insecure)
   s.handle(TestTarget)
+  s.handle(ConfigureTarget)
   server_thread = Thread.new {
     # run the server until the main test runner terminates this process
     s.run_till_terminated_or_interrupted(['TERM'])
   }
 
-  # The client just sends unary rpcs continuously in a regular interval
+  # Initialize stats
+  $RPC_MAP.values.each do |rpc|
+    $num_rpcs_started_by_method[rpc.to_s] = 0
+    $num_rpcs_succeeded_by_method[rpc.to_s] = 0
+    $num_rpcs_failed_by_method[rpc.to_s] = 0
+  end
+
+  # The client just sends rpcs continuously in a regular interval
   stub = create_stub(opts)
   target_seconds_between_rpcs = (1.0 / opts['qps'].to_f)
-  rpcs_to_send = []
-  if opts['rpc']
-    rpcs_to_send = opts['rpc'].split(',')
-  end
   # Convert 'metadata' input in the form of
   #   rpc1:k1:v1,rpc2:k2:v2,rpc1:k3:v3
   # into
@@ -266,11 +381,13 @@ def main
   #       'k2' => 'v2'
   #     },
   #   }
+  rpcs_to_send = []
   metadata_to_send = {}
   if opts['metadata']
     metadata_entries = opts['metadata'].split(',')
     metadata_entries.each do |e|
       (rpc_name, metadata_key, metadata_value) = e.split(':')
+      rpc_name = $RPC_MAP[rpc_name]
       # initialize if we haven't seen this rpc_name yet
       if !metadata_to_send.key?(rpc_name)
         metadata_to_send[rpc_name] = {}
@@ -278,12 +395,20 @@ def main
       metadata_to_send[rpc_name][metadata_key] = metadata_value
     end
   end
+  if opts['rpc']
+    rpcs_to_send = opts['rpc'].split(',')
+  end
+  if rpcs_to_send.size > 0
+    rpcs_to_send.map! { |rpc| $RPC_MAP[rpc] }
+    new_rpc_config = RpcConfig.new
+    new_rpc_config.init(rpcs_to_send, metadata_to_send)
+    $rpc_config = new_rpc_config
+  end
   client_threads = Array.new
   opts['num_channels'].times {
     client_threads << Thread.new {
       run_test_loop(stub, target_seconds_between_rpcs,
-                    opts['fail_on_failed_rpcs'],
-                    rpcs_to_send, metadata_to_send)
+                    opts['fail_on_failed_rpcs'])
     }
   }
 

data/src/ruby/spec/pb/codegen/package_option_spec.rb

@@ -71,14 +71,10 @@ describe 'Code Generation Options' do
 end
 
 def with_protos(file_paths)
-  fail 'CONFIG env variable unexpectedly unset' unless ENV['CONFIG']
-  bins_sub_dir = ENV['CONFIG']
-
   pb_dir = File.dirname(__FILE__)
-  bins_dir = File.join('..', '..', '..', '..', '..', 'bins', bins_sub_dir)
-
+  bins_dir = File.join('..', '..', '..', '..', '..', 'cmake', 'build')
   plugin = File.join(bins_dir, 'grpc_ruby_plugin')
-  protoc = File.join(bins_dir, 'protobuf', 'protoc')
+  protoc = File.join(bins_dir, 'third_party', 'protobuf', 'protoc')
 
   # Generate the service from the proto
   Dir.mktmpdir(nil, File.dirname(__FILE__)) do |tmp_dir|

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: grpc
 version: !ruby/object:Gem::Version
-  version: 1.32.0
+  version: 1.35.0
 platform: universal-darwin
 authors:
 - gRPC Authors
-autorequire:
+autorequire: 
 bindir: src/ruby/bin
 cert_chain: []
-date: 2020-09-08 00:00:00.000000000 Z
+date: 2021-01-19 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: google-protobuf
@@ -16,14 +16,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.13'
+        version: '3.14'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '3.13'
+        version: '3.14'
 - !ruby/object:Gem::Dependency
   name: googleapis-common-protos-types
   requirement: !ruby/object:Gem::Requirement
@@ -330,7 +330,7 @@ homepage: https://github.com/google/grpc/tree/master/src/ruby
 licenses:
 - Apache-2.0
 metadata: {}
-post_install_message:
+post_install_message: 
 rdoc_options: []
 require_paths:
 - src/ruby/lib
@@ -343,15 +343,15 @@ required_ruby_version: !ruby/object:Gem::Requirement
       version: '2.3'
   - - "<"
     - !ruby/object:Gem::Version
-      version: 2.8.dev
+      version: 3.1.dev
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.4
-signing_key:
+rubygems_version: 3.2.6
+signing_key: 
 specification_version: 4
 summary: GRPC system in Ruby
 test_files: