google-cloud-pubsub 0.36.0 → 0.37.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: e6bdfb3062337b8aaf70394ea5f2c35054ac9cbe83836631b188456b8801c39e
-  data.tar.gz: 6f51848602a6702095214030780ddeae82e715b364541d2c1b8014232610c6b7
+  metadata.gz: ff2481f99b8def0df9c9203a64569dc317907152518da6b522d63314524ebea5
+  data.tar.gz: 8773411fca063ab56c81d94951406866747f9137f4c9d9e9860b011412a0bae3
 SHA512:
-  metadata.gz: 693140e1add6261c55b61d3a9514db9c10afcf4dd04f0e0d1dd369aabd915aa8dce15eda2408e1d9be6cc9f54d9f546c638eb7d8aa66ca63319dd014e2382536
-  data.tar.gz: 8872761deb3b50239a4bee8d8ef9259492e5e60259e6a6508f632134e0521443910dc54dd4536c0dc4242a2d40c0fc9e4cb911109a5754aace8c8596f2c36138
+  metadata.gz: 47976e8e93c6a68707a24f26b92110fca18eef669c41c86f93cf9d746c55596abbddde9f067a42b47c68c6472d5655616380f0e334623872455d5932b97ffd9c
+  data.tar.gz: ebf63bd135e128d19f56c9ece11aad5f1fc8cf2489fa7b8c726ccc68e98632ee4751c28079d9dfc17379dbf40cf9871a44b9d1c03d50dc0b1a6c9215de45f25d

data/CHANGELOG.md

@@ -1,5 +1,21 @@
 # Release History
 
+### 0.37.0 / 2019-06-17
+
+* Add Topic#persistence_regions
+* Subscriber changes
+  * Fix potential inventory bug
+  * Messages are removed after callback
+    * This change prevents the Subscriber inventory from filling up
+      when messages are never acked or nacked in the user callback.
+      This might happen due to an error in the user callback code.
+      Removing a message from the inventory will cause the message to
+      be redelivered and reprocessed.
+  * Update concurrency implementation
+    * Use concurrent-ruby Promises framework.
+* Update network configuration
+* Enable grpc.service_config_disable_resolution
+
 ### 0.36.0 / 2019-05-21
 
 * Add Topic#kms_key

data/lib/google/cloud/pubsub/async_publisher.rb

@@ -193,10 +193,10 @@ module Google
 
         def init_resources!
           @first_published_at   ||= Time.now
-          @publish_thread_pool  ||= Concurrent::FixedThreadPool.new \
-            @publish_threads
-          @callback_thread_pool ||= Concurrent::FixedThreadPool.new \
-            @callback_threads
+          @publish_thread_pool  ||= Concurrent::CachedThreadPool.new \
+            max_threads: @publish_threads
+          @callback_thread_pool ||= Concurrent::CachedThreadPool.new \
+            max_threads: @callback_threads
           @thread ||= Thread.new { run_background }
         end
 
@@ -235,33 +235,39 @@ module Google
         def publish_batch_async topic_name, batch
           return unless @publish_thread_pool.running?
 
-          Concurrent::Future.new executor: @publish_thread_pool do
-            begin
-              grpc = @service.publish topic_name, batch.messages
-              batch.items.zip Array(grpc.message_ids) do |item, id|
-                next unless item.callback
+          Concurrent::Promises.future_on(
+            @publish_thread_pool, topic_name, batch
+          ) do |t_name, btch|
+            publish_batch_sync t_name, btch
+          end
+        end
 
-                item.msg.message_id = id
-                publish_result = PublishResult.from_grpc item.msg
-                execute_callback_async item.callback, publish_result
-              end
-            rescue StandardError => e
-              batch.items.each do |item|
-                next unless item.callback
+        def publish_batch_sync topic_name, batch
+          grpc = @service.publish topic_name, batch.messages
+          batch.items.zip Array(grpc.message_ids) do |item, id|
+            next unless item.callback
 
-                publish_result = PublishResult.from_error item.msg, e
-                execute_callback_async item.callback, publish_result
-              end
-            end
-          end.execute
+            item.msg.message_id = id
+            publish_result = PublishResult.from_grpc item.msg
+            execute_callback_async item.callback, publish_result
+          end
+        rescue StandardError => e
+          batch.items.each do |item|
+            next unless item.callback
+
+            publish_result = PublishResult.from_error item.msg, e
+            execute_callback_async item.callback, publish_result
+          end
         end
 
         def execute_callback_async callback, publish_result
           return unless @callback_thread_pool.running?
 
-          Concurrent::Future.new executor: @callback_thread_pool do
-            callback.call publish_result
-          end.execute
+          Concurrent::Promises.future_on(
+            @callback_thread_pool, callback, publish_result
+          ) do |cback, p_result|
+            cback.call p_result
+          end
         end
 
         def create_pubsub_message data, attributes

data/lib/google/cloud/pubsub/service.rb

@@ -46,9 +46,10 @@ module Google
         end
 
         def chan_args
-          { "grpc.max_send_message_length"    => -1,
-            "grpc.max_receive_message_length" => -1,
-            "grpc.keepalive_time_ms"          => 300000 }
+          { "grpc.max_send_message_length"           => -1,
+            "grpc.max_receive_message_length"        => -1,
+            "grpc.keepalive_time_ms"                 => 300000,
+            "grpc.service_config_disable_resolution" => 1 }
         end
 
         def chan_creds

data/lib/google/cloud/pubsub/subscriber/inventory.rb

@@ -40,7 +40,8 @@ module Google
           end
 
           def add *ack_ids
-            ack_ids.flatten!.compact!
+            ack_ids.flatten!
+            ack_ids.compact!
             return if ack_ids.empty?
 
             synchronize do
@@ -50,7 +51,8 @@ module Google
           end
 
           def remove *ack_ids
-            ack_ids.flatten!.compact!
+            ack_ids.flatten!
+            ack_ids.compact!
             return if ack_ids.empty?
 
             synchronize do

data/lib/google/cloud/pubsub/subscriber/stream.rb

@@ -48,8 +48,8 @@ module Google
             @pause_cond = new_cond
 
             @inventory = Inventory.new self, @subscriber.stream_inventory
-            @callback_thread_pool = Concurrent::FixedThreadPool.new \
-              @subscriber.callback_threads
+            @callback_thread_pool = Concurrent::CachedThreadPool.new \
+              max_threads: @subscriber.callback_threads
 
             @stream_keepalive_task = Concurrent::TimerTask.new(
               execution_interval: 30
@@ -181,21 +181,25 @@ module Google
           # rubocop:disable all
 
           def background_run
-            # Don't allow a stream to restart if already stopped
-            return if @stopped
+            synchronize do
+              # Don't allow a stream to restart if already stopped
+              return if @stopped
 
-            # signal to the previous queue to shut down
-            old_queue = []
-            old_queue = @request_queue.quit_and_dump_queue if @request_queue
+              @stopped = false
+              @paused  = false
 
-            # Always create a new request queue and enum
-            @request_queue = EnumeratorQueue.new self
-            @request_queue.push initial_input_request
-            old_queue.each { |obj| @request_queue.push obj }
-            enum = @subscriber.service.streaming_pull @request_queue.each
+              # signal to the previous queue to shut down
+              old_queue = []
+              old_queue = @request_queue.quit_and_dump_queue if @request_queue
 
-            @stopped = nil
-            @paused  = nil
+              # Always create a new request queue
+              @request_queue = EnumeratorQueue.new self
+              @request_queue.push initial_input_request
+              old_queue.each { |obj| @request_queue.push obj }
+            end
+
+            # Call the StreamingPull API to get the response enumerator
+            enum = @subscriber.service.streaming_pull @request_queue.each
 
             loop do
               synchronize do
@@ -242,7 +246,7 @@ module Google
             raise RestartStream unless synchronize { @stopped }
 
             # We must be stopped, tell the stream to quit.
-            @request_queue.push self
+            stop
           rescue GRPC::Cancelled, GRPC::DeadlineExceeded, GRPC::Internal,
                  GRPC::ResourceExhausted, GRPC::Unauthenticated,
                  GRPC::Unavailable, GRPC::Core::CallError
@@ -253,10 +257,7 @@ module Google
           rescue RestartStream
             retry
           rescue StandardError => e
-            synchronize do
-              @subscriber.error! e
-              start_streaming! unless @stopped
-            end
+            @subscriber.error! e
 
             retry
           end
@@ -266,13 +267,17 @@ module Google
           def perform_callback_async rec_msg
             return unless callback_thread_pool.running?
 
-            Concurrent::Future.new executor: callback_thread_pool do
+            Concurrent::Promises.future_on(
+              callback_thread_pool, @subscriber, @inventory, rec_msg
+            ) do |sub, inv, msg|
               begin
-                @subscriber.callback.call rec_msg
+                sub.callback.call msg
               rescue StandardError => callback_error
-                @subscriber.error! callback_error
+                sub.error! callback_error
+              ensure
+                inv.remove msg.ack_id
               end
-            end.execute
+            end
           end
 
           def start_streaming!
@@ -281,9 +286,6 @@ module Google
             # could not be recovered from, so be it.
             return if @background_thread
 
-            @stopped = false
-            @paused  = false
-
             # create new background thread to handle new enumerator
             @background_thread = Thread.new { background_run }
           end

data/lib/google/cloud/pubsub/subscriber/timed_unary_buffer.rb

@@ -187,7 +187,8 @@ module Google
           end
 
           def with_threadpool
-            pool = Concurrent::FixedThreadPool.new @subscriber.push_threads
+            pool = Concurrent::CachedThreadPool.new \
+              max_threads: @subscriber.push_threads
 
             yield pool
 
@@ -204,13 +205,13 @@ module Google
           end
 
           def add_future pool
-            Concurrent::Future.new executor: pool do
+            Concurrent::Promises.future_on pool do
               begin
                 yield
               rescue StandardError => error
                 error! error
               end
-            end.execute
+            end
           end
         end
       end

data/lib/google/cloud/pubsub/subscription.rb

@@ -501,8 +501,12 @@ module Google
         end
 
         ##
-        # Create a {Subscriber} object that receives  and processes messages
-        # using the code provided in the callback.
+        # Create a {Subscriber} object that receives and processes messages
+        # using the code provided in the callback. Messages passed to the
+        # callback should acknowledge ({ReceivedMessage#acknowledge!}) or reject
+        # ({ReceivedMessage#reject!}) the message. If no action is taken, the
+        # message will be removed from the subscriber and made available for
+        # redelivery after the callback is completed.
         #
         # @param [Numeric] deadline The default number of seconds the stream
         #   will hold received messages before modifying the message's ack

data/lib/google/cloud/pubsub/topic.rb

@@ -182,6 +182,36 @@ module Google
           @resource_name = nil
         end
 
+        ##
+        # The list of GCP region IDs where messages that are published to the
+        # topic may be persisted in storage.
+        #
+        # Messages published by publishers running in non-allowed GCP regions
+        # (or running outside of GCP altogether) will be routed for storage in
+        # one of the allowed regions. An empty list indicates a misconfiguration
+        # at the project or organization level, which will result in all publish
+        # operations failing.
+        #
+        # Makes an API call to retrieve the list of GCP region IDs values when
+        # called on a reference object. See {#reference?}.
+        #
+        # @return [Array<String>]
+        #
+        # @example
+        #   require "google/cloud/pubsub"
+        #
+        #   pubsub = Google::Cloud::PubSub.new
+        #
+        #   topic = pubsub.topic "my-topic"
+        #
+        #   topic.persistence_regions #=> ["us-central1", "us-central2"]
+        #
+        def persistence_regions
+          ensure_grpc!
+          return [] if @grpc.message_storage_policy.nil?
+          Array @grpc.message_storage_policy.allowed_persistence_regions
+        end
+
         ##
         # Permanently deletes the topic.
         #

data/lib/google/cloud/pubsub/v1/publisher_client.rb

@@ -28,6 +28,7 @@ require "google/gax"
 require "google/iam/v1/iam_policy_pb"
 require "google/pubsub/v1/pubsub_pb"
 require "google/cloud/pubsub/v1/credentials"
+require "google/cloud/pubsub/version"
 
 module Google
   module Cloud
@@ -182,7 +183,7 @@ module Google
               updater_proc = credentials.updater_proc
             end
 
-            package_version = Gem.loaded_specs['google-cloud-pubsub'].version.version
+            package_version = Google::Cloud::PubSub::VERSION
 
             google_api_client = "gl-ruby/#{RUBY_VERSION}"
             google_api_client << " #{lib_name}/#{lib_version}" if lib_name

data/lib/google/cloud/pubsub/v1/publisher_client_config.json

@@ -35,7 +35,7 @@
           "initial_retry_delay_millis": 100,
           "retry_delay_multiplier": 1.3,
           "max_retry_delay_millis": 60000,
-          "initial_rpc_timeout_millis": 12000,
+          "initial_rpc_timeout_millis": 25000,
           "rpc_timeout_multiplier": 1.0,
           "max_rpc_timeout_millis": 30000,
           "total_timeout_millis": 600000

data/lib/google/cloud/pubsub/v1/subscriber_client.rb

@@ -28,6 +28,7 @@ require "google/gax"
 require "google/iam/v1/iam_policy_pb"
 require "google/pubsub/v1/pubsub_pb"
 require "google/cloud/pubsub/v1/credentials"
+require "google/cloud/pubsub/version"
 
 module Google
   module Cloud
@@ -206,7 +207,7 @@ module Google
               updater_proc = credentials.updater_proc
             end
 
-            package_version = Gem.loaded_specs['google-cloud-pubsub'].version.version
+            package_version = Google::Cloud::PubSub::VERSION
 
             google_api_client = "gl-ruby/#{RUBY_VERSION}"
             google_api_client << " #{lib_name}/#{lib_version}" if lib_name

data/lib/google/cloud/pubsub/v1/subscriber_client_config.json

@@ -26,9 +26,9 @@
           "initial_retry_delay_millis": 100,
           "retry_delay_multiplier": 1.3,
           "max_retry_delay_millis": 60000,
-          "initial_rpc_timeout_millis": 12000,
+          "initial_rpc_timeout_millis": 25000,
           "rpc_timeout_multiplier": 1.0,
-          "max_rpc_timeout_millis": 12000,
+          "max_rpc_timeout_millis": 25000,
           "total_timeout_millis": 600000
         },
         "streaming_messaging": {

data/lib/google/cloud/pubsub/version.rb

@@ -16,7 +16,7 @@
 module Google
   module Cloud
     module PubSub
-      VERSION = "0.36.0".freeze
+      VERSION = "0.37.0".freeze
     end
 
     Pubsub = PubSub unless const_defined? :Pubsub

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: google-cloud-pubsub
 version: !ruby/object:Gem::Version
-  version: 0.36.0
+  version: 0.37.0
 platform: ruby
 authors:
 - Mike Moore
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-05-21 00:00:00.000000000 Z
+date: 2019-06-18 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: google-cloud-core
@@ -79,14 +79,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.0'
+        version: '1.1'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '1.0'
+        version: '1.1'
 - !ruby/object:Gem::Dependency
   name: minitest
   requirement: !ruby/object:Gem::Requirement