google-cloud-pubsub 1.4.0 → 1.7.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2ba8847f393a5fc464ab63f9d6bca1a7e501c8457afa477486a9bd9c44e6d126
-  data.tar.gz: '034684fa844d264673c7e79ea17deeea9edc329de16376863e736a5f0825c256'
+  metadata.gz: 25d3602668077c7dee90a868dcc01353710ac3001ece8ebe92b7acc6c7bc1bb3
+  data.tar.gz: 2119781ea548d6e87d201bdb759643c7ac78b7ffab0a4ee7505f731c3c812670
 SHA512:
-  metadata.gz: 8a0e87a57f3fbb1522fd4cf5d93d1c0e03e10ecc5cd2c65b2f0ecfc8aabda230370f6118e9f52c9aadc4763ada09ff6bd0128539fa6469f68d22a327292f3403
-  data.tar.gz: b18e4d310dbfa486b4b40ed4a6bd657c484c4416517b890bc65889644f928d523b8c447b9b0d3b15980cc80673a9fc2ec3927a6eaff9dd742038f999ac51cee0
+  metadata.gz: ed1de151bbd790feb886162024dbe20e75825c7313de7f0349468d8adbdf0400ce0616f53925315a1a2558b2683fd9ac8b478b842b2008dae67990676c163a4b
+  data.tar.gz: 2247b98b5f04c936bd908c7a517d396e03af0f07030e7939282f750d8d2b9f569f428961c793f929d5595da60445a55948fe17864a1f25f1ab7d1746b06be9c0

data/CHANGELOG.md

@@ -1,5 +1,48 @@
 # Release History
 
+### 1.7.1 / 2020-05-28
+
+#### Documentation
+
+* Fix a few broken links
+
+### 1.7.0 / 2020-05-21
+
+#### Features
+
+* Add Retry Policy support
+  * Add RetryPolicy
+  * Add retry_policy param to Topic#subscribe
+  * Add Subscription#retry_policy
+  * Add Subscription#retry_policy=
+* Set client-scoped UUID in initial StreamingPullRequest#client_id
+
+### 1.6.1 / 2020-05-06
+
+#### Documentation
+
+* Fix example in Emulator documentation
+* Remove experimental notice from ReceivedMessage#delivery_attempt
+* Wrap example URLs in backticks
+
+### 1.6.0 / 2020-04-06
+
+#### Features
+
+* Add list_topic_snapshots and get_snapshot
+  * Add PublisherClient#list_topic_snapshots
+  * Add SubscriberClient#get_snapshot
+
+#### Documentation
+
+* Remove a spurious link in the low-level interface documentation.
+
+### 1.5.0 / 2020-03-25
+
+#### Features
+
+* Add max_duration_per_lease_extension to Subscription#listen and Subscriber
+
 ### 1.4.0 / 2020-03-11
 
 #### Features

data/EMULATOR.md

@@ -17,7 +17,7 @@ require "google/cloud/pubsub"
 # Make Pub/Sub use the emulator
 ENV["PUBSUB_EMULATOR_HOST"] = "localhost:8918"
 
-pubsub = Google::Cloud::PubSub.new "emulator-project-id"
+pubsub = Google::Cloud::PubSub.new project_id:"emulator-project-id"
 
 # Get a topic in the current project
 my_topic = pubsub.new_topic "my-topic"

data/TROUBLESHOOTING.md

@@ -24,14 +24,8 @@ improved, *please* create a new issue on GitHub so we can talk about it.
 
   - [New issue][gh-ruby]
 
-Or, you can ask questions on the [Google Cloud Platform Slack][slack-ruby]. You
-can use the "ruby" channel for general Ruby questions, or use the
-"google-cloud-ruby" channel if you have questions about this gem in particular.
-
 [so-ruby]: http://stackoverflow.com/questions/tagged/google-cloud-platform+ruby+pubsub
 
-[gh-search-ruby]: https://github.com/googlecloudplatform/google-cloud-ruby/issues?q=label%3A%22api%3A+pubsub%22
-
-[gh-ruby]: https://github.com/googlecloudplatform/google-cloud-ruby/issues/new
+[gh-search-ruby]: https://github.com/googleapis/google-cloud-ruby/issues?q=label%3A%22api%3A+pubsub%22
 
-[slack-ruby]: https://gcp-slack.appspot.com/
+[gh-ruby]: https://github.com/googleapis/google-cloud-ruby/issues/new

data/lib/google/cloud/pubsub/received_message.rb

@@ -78,11 +78,6 @@ module Google
         # The first delivery of a given message will have this value as `1`. The value is calculated at best effort and
         # is approximate.
         #
-        # EXPERIMENTAL: This feature is part of a closed alpha release and is available only to whitelisted partners.
-        # This method will return `nil` if a dead letter policy is not set on the subscription. This API might be
-        # changed in backward-incompatible ways and is not recommended for production use. It is not subject to any SLA
-        # or deprecation policy.
-        #
         # @return [Integer, nil] A delivery attempt value of `1` or greater, or `nil` if a dead letter policy is not set
         #   on the subscription.
         #

data/lib/google/cloud/pubsub/retry_policy.rb

@@ -0,0 +1,90 @@
+# Copyright 2016 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+require "google/cloud/errors"
+
+module Google
+  module Cloud
+    module PubSub
+      ##
+      # # RetryPolicy
+      #
+      # An immutable Retry Policy value object that specifies how Cloud Pub/Sub retries message delivery.
+      #
+      # Retry delay will be exponential based on provided minimum and maximum backoffs. (See [Exponential
+      # backoff](https://en.wikipedia.org/wiki/Exponential_backoff).)
+      #
+      # Retry Policy will be triggered on NACKs or acknowledgement deadline exceeded events for a given message.
+      #
+      # Retry Policy is implemented on a best effort basis. At times, the delay between consecutive deliveries may not
+      # match the configuration. That is, delay can be more or less than configured backoff.
+      #
+      # **EXPERIMENTAL:** This API might be changed in backward-incompatible ways and is not recommended for production
+      # use. It is not subject to any SLA or deprecation policy.
+      #
+      # @attr [Numeric] minimum_backoff The minimum delay between consecutive deliveries of a given message. Value
+      #   should be between 0 and 600 seconds. The default value is 10 seconds.
+      # @attr [Numeric] maximum_backoff The maximum delay between consecutive deliveries of a given message. Value
+      #   should be between 0 and 600 seconds. The default value is 600 seconds.
+      #
+      # @example
+      #   require "google/cloud/pubsub"
+      #
+      #   pubsub = Google::Cloud::PubSub.new
+      #
+      #   sub = pubsub.subscription "my-topic-sub"
+      #
+      #   sub.retry_policy = Google::Cloud::PubSub::RetryPolicy.new minimum_backoff: 5, maximum_backoff: 300
+      #
+      #   sub.retry_policy.minimum_backoff #=> 5
+      #   sub.retry_policy.maximum_backoff #=> 300
+      #
+      class RetryPolicy
+        attr_reader :minimum_backoff, :maximum_backoff
+
+        ##
+        # Creates a new, immutable RetryPolicy value object.
+        #
+        # @attr [Numeric, nil] minimum_backoff The minimum delay between consecutive deliveries of a given message.
+        #   Value should be between 0 and 600 seconds. If `nil` is provided, the default value is 10 seconds.
+        # @attr [Numeric, nil] maximum_backoff The maximum delay between consecutive deliveries of a given message.
+        #   Value should be between 0 and 600 seconds. If `nil` is provided, the default value is 600 seconds.
+        #
+        def initialize minimum_backoff: nil, maximum_backoff: nil
+          @minimum_backoff = minimum_backoff
+          @maximum_backoff = maximum_backoff
+        end
+
+        ##
+        # @private Convert the RetryPolicy to a Google::Cloud::PubSub::V1::RetryPolicy object.
+        def to_grpc
+          Google::Cloud::PubSub::V1::RetryPolicy.new(
+            minimum_backoff: Convert.number_to_duration(minimum_backoff),
+            maximum_backoff: Convert.number_to_duration(maximum_backoff)
+          )
+        end
+
+        ##
+        # @private New RetryPolicy from a Google::Cloud::PubSub::V1::RetryPolicy object.
+        def self.from_grpc grpc
+          new(
+            minimum_backoff: Convert.duration_to_number(grpc.minimum_backoff),
+            maximum_backoff: Convert.duration_to_number(grpc.maximum_backoff)
+          )
+        end
+      end
+    end
+  end
+end

data/lib/google/cloud/pubsub/service.rb

@@ -19,6 +19,7 @@ require "google/cloud/pubsub/convert"
 require "google/cloud/pubsub/version"
 require "google/cloud/pubsub/v1"
 require "google/gax/errors"
+require "securerandom"
 
 module Google
   module Cloud
@@ -28,6 +29,12 @@ module Google
       # methods.
       class Service
         attr_accessor :project, :credentials, :host, :timeout, :client_config
+        ###
+        # The same client_id is used across all streaming pull connections that are created by this client. This is
+        # intentional, as it indicates to the server that any guarantees, such as message ordering, made for a stream
+        # that is disconnected will be made for the stream that is created to replace it. The attr_accessor allows the
+        # value to be replaced for unit testing.
+        attr_accessor :client_id
 
         ##
         # Creates a new Service instance.
@@ -38,6 +45,7 @@ module Google
           @host = host || V1::PublisherClient::SERVICE_ADDRESS
           @timeout = timeout
           @client_config = client_config || {}
+          @client_id = SecureRandom.uuid.freeze
         end
 
         def channel
@@ -248,6 +256,7 @@ module Google
               labels:                     labels,
               enable_message_ordering:    message_ordering,
               dead_letter_policy:         dead_letter_policy(options),
+              retry_policy:               options[:retry_policy],
               options:                    default_options
           end
         end

data/lib/google/cloud/pubsub/subscriber.rb

@@ -104,6 +104,7 @@ module Google
         # received messages.
         #
         # @return [Subscriber] returns self so calls can be chained.
+        #
         def start
           start_pool = synchronize do
             @started = true
@@ -129,6 +130,7 @@ module Google
         # received messages have been processed or released.
         #
         # @return [Subscriber] returns self so calls can be chained.
+        #
         def stop
           stop_pool = synchronize do
             @started = false
@@ -158,6 +160,7 @@ module Google
         #   subscriber is fully stopped. Default will block indefinitely.
         #
         # @return [Subscriber] returns self so calls can be chained.
+        #
         def wait! timeout = nil
           wait_pool = synchronize do
             @stream_pool.map do |stream|
@@ -190,6 +193,7 @@ module Google
         # Whether the subscriber has been started.
         #
         # @return [boolean] `true` when started, `false` otherwise.
+        #
         def started?
           synchronize { @started }
         end
@@ -198,6 +202,7 @@ module Google
         # Whether the subscriber has been stopped.
         #
         # @return [boolean] `true` when stopped, `false` otherwise.
+        #
         def stopped?
           synchronize { @stopped }
         end
@@ -279,6 +284,9 @@ module Google
 
         ##
         # The number of received messages to be collected by subscriber. Default is 1,000.
+        #
+        # @return [Integer] The maximum number of messages.
+        #
         def max_outstanding_messages
           @inventory[:max_outstanding_messages]
         end
@@ -289,6 +297,9 @@ module Google
 
         ##
         # The total byte size of received messages to be collected by subscriber. Default is 100,000,000 (100MB).
+        #
+        # @return [Integer] The maximum number of bytes.
+        #
         def max_outstanding_bytes
           @inventory[:max_outstanding_bytes]
         end
@@ -297,19 +308,33 @@ module Google
 
         ##
         # The number of seconds that received messages can be held awaiting processing. Default is 3,600 (1 hour).
+        #
+        # @return [Integer] The maximum number of seconds.
+        #
         def max_total_lease_duration
           @inventory[:max_total_lease_duration]
         end
         # @deprecated Use {#max_total_lease_duration}.
         alias inventory_extension max_total_lease_duration
 
+        ##
+        # The maximum amount of time in seconds for a single lease extension attempt. Bounds the delay before a message
+        # redelivery if the subscriber fails to extend the deadline. Default is 0 (disabled).
+        #
+        # @return [Integer] The maximum number of seconds.
+        #
+        def max_duration_per_lease_extension
+          @inventory[:max_duration_per_lease_extension]
+        end
+
         ##
         # @private
         def stream_inventory
           {
-            limit:     @inventory[:max_outstanding_messages].fdiv(@streams).ceil,
-            bytesize:  @inventory[:max_outstanding_bytes].fdiv(@streams).ceil,
-            extension: @inventory[:max_total_lease_duration]
+            limit:                            @inventory[:max_outstanding_messages].fdiv(@streams).ceil,
+            bytesize:                         @inventory[:max_outstanding_bytes].fdiv(@streams).ceil,
+            extension:                        @inventory[:max_total_lease_duration],
+            max_duration_per_lease_extension: @inventory[:max_duration_per_lease_extension]
           }
         end
 
@@ -351,6 +376,7 @@ module Google
           @inventory[:max_outstanding_messages] = Integer(@inventory[:max_outstanding_messages] || 1000)
           @inventory[:max_outstanding_bytes] = Integer(@inventory[:max_outstanding_bytes] || 100_000_000)
           @inventory[:max_total_lease_duration] = Integer(@inventory[:max_total_lease_duration] || 3600)
+          @inventory[:max_duration_per_lease_extension] = Integer(@inventory[:max_duration_per_lease_extension] || 0)
         end
 
         def default_error_callbacks

data/lib/google/cloud/pubsub/subscriber/inventory.rb

@@ -30,14 +30,15 @@ module Google
 
           include MonitorMixin
 
-          attr_reader :stream, :limit, :bytesize, :extension
+          attr_reader :stream, :limit, :bytesize, :extension, :max_duration_per_lease_extension
 
-          def initialize stream, limit:, bytesize:, extension:
+          def initialize stream, limit:, bytesize:, extension:, max_duration_per_lease_extension:
             super()
             @stream = stream
             @limit = limit
             @bytesize = bytesize
             @extension = extension
+            @max_duration_per_lease_extension = max_duration_per_lease_extension
             @inventory = {}
             @wait_cond = new_cond
           end
@@ -152,7 +153,9 @@ module Google
           end
 
           def calc_delay
-            (stream.subscriber.deadline - 3) * rand(0.8..0.9)
+            delay = (stream.subscriber.deadline - 3) * rand(0.8..0.9)
+            delay = [delay, max_duration_per_lease_extension].min if max_duration_per_lease_extension.positive?
+            delay
           end
         end
       end

data/lib/google/cloud/pubsub/subscriber/stream.rb

@@ -363,6 +363,7 @@ module Google
               req.stream_ack_deadline_seconds = @subscriber.deadline
               req.modify_deadline_ack_ids += @inventory.ack_ids
               req.modify_deadline_seconds += @inventory.ack_ids.map { @subscriber.deadline }
+              req.client_id = @subscriber.service.client_id
             end
           end
 

data/lib/google/cloud/pubsub/subscription.rb

@@ -18,6 +18,7 @@ require "google/cloud/errors"
 require "google/cloud/pubsub/subscription/list"
 require "google/cloud/pubsub/subscription/push_config"
 require "google/cloud/pubsub/received_message"
+require "google/cloud/pubsub/retry_policy"
 require "google/cloud/pubsub/snapshot"
 require "google/cloud/pubsub/subscriber"
 require "google/cloud/pubsub/v1"
@@ -188,7 +189,7 @@ module Google
         ##
         # Returns the URL locating the endpoint to which messages should be
         # pushed. For example, a Webhook endpoint might use
-        # "https://example.com/push".
+        # `https://example.com/push`.
         #
         # Makes an API call to retrieve the endpoint value when called on a
         # reference object. See {#reference?}.
@@ -202,7 +203,7 @@ module Google
 
         ##
         # Sets the URL locating the endpoint to which messages should be pushed.
-        # For example, a Webhook endpoint might use "https://example.com/push".
+        # For example, a Webhook endpoint might use `https://example.com/push`.
         #
         # @param [String] new_endpoint The new endpoint value.
         #
@@ -483,6 +484,65 @@ module Google
           @resource_name = nil
         end
 
+        ##
+        # A policy that specifies how Cloud Pub/Sub retries message delivery for this subscription. If `nil`, the
+        # default retry policy is applied. This generally implies that messages will be retried as soon as possible
+        # for healthy subscribers. Retry Policy will be triggered on NACKs or acknowledgement deadline exceeded events
+        # for a given message.
+        #
+        # **EXPERIMENTAL:** This API might be changed in backward-incompatible ways and is not recommended for
+        # production use. It is not subject to any SLA or deprecation policy.
+        #
+        # @return [RetryPolicy, nil] The retry policy for the subscription, or `nil`.
+        #
+        # @example
+        #   require "google/cloud/pubsub"
+        #
+        #   pubsub = Google::Cloud::PubSub.new
+        #
+        #   sub = pubsub.subscription "my-topic-sub"
+        #
+        #   sub.retry_policy = Google::Cloud::PubSub::RetryPolicy.new minimum_backoff: 5, maximum_backoff: 300
+        #
+        #   sub.retry_policy.minimum_backoff #=> 5
+        #   sub.retry_policy.maximum_backoff #=> 300
+        #
+        def retry_policy
+          ensure_grpc!
+          return nil unless @grpc.retry_policy
+          RetryPolicy.from_grpc @grpc.retry_policy
+        end
+
+        ##
+        # Sets a policy that specifies how Cloud Pub/Sub retries message delivery for this subscription. If `nil`, the
+        # default retry policy is applied. This generally implies that messages will be retried as soon as possible
+        # for healthy subscribers. Retry Policy will be triggered on NACKs or acknowledgement deadline exceeded events
+        # for a given message.
+        #
+        # **EXPERIMENTAL:** This API might be changed in backward-incompatible ways and is not recommended for
+        # production use. It is not subject to any SLA or deprecation policy.
+        #
+        # @param [RetryPolicy, nil] new_retry_policy A new retry policy for the subscription, or `nil`.
+        #
+        # @example
+        #   require "google/cloud/pubsub"
+        #
+        #   pubsub = Google::Cloud::PubSub.new
+        #
+        #   sub = pubsub.subscription "my-topic-sub"
+        #
+        #   sub.retry_policy = Google::Cloud::PubSub::RetryPolicy.new minimum_backoff: 5, maximum_backoff: 300
+        #
+        #   sub.retry_policy.minimum_backoff #=> 5
+        #   sub.retry_policy.maximum_backoff #=> 300
+        #
+        def retry_policy= new_retry_policy
+          ensure_grpc!
+          new_retry_policy = new_retry_policy.to_grpc if new_retry_policy
+          update_grpc = Google::Cloud::PubSub::V1::Subscription.new name: name, retry_policy: new_retry_policy
+          @grpc = service.update_subscription update_grpc, :retry_policy
+        end
+
         ##
         # Whether message ordering has been enabled. When enabled, messages
         # published with the same `ordering_key` will be delivered in the order
@@ -699,6 +759,9 @@ module Google
         #       subscriber. Default is 100,000,000 (100MB). (Note: replaces `:bytesize`, which is deprecated.)
         #     * `:max_total_lease_duration` [Integer] The number of seconds that received messages can be held awaiting
         #       processing. Default is 3,600 (1 hour). (Note: replaces `:extension`, which is deprecated.)
+        #     * `:max_duration_per_lease_extension` [Integer] The maximum amount of time in seconds for a single lease
+        #       extension attempt. Bounds the delay before a message redelivery if the subscriber fails to extend the
+        #       deadline. Default is 0 (disabled).
         # @param [Hash] threads The number of threads to create to handle
         #   concurrent calls by each stream opened by the subscriber. Optional.
         #
@@ -774,6 +837,25 @@ module Google
         #   # Shut down the subscriber when ready to stop receiving messages.
         #   subscriber.stop.wait!
         #
+        # @example Set the maximum amount of time before redelivery if the subscriber fails to extend the deadline:
+        #   require "google/cloud/pubsub"
+        #
+        #   pubsub = Google::Cloud::PubSub.new
+        #
+        #   sub = pubsub.subscription "my-topic-sub"
+        #
+        #   subscriber = sub.listen inventory: { max_duration_per_lease_extension: 20 } do |received_message|
+        #     # Process message very slowly with possibility of failure.
+        #     process rec_message.data # takes minutes
+        #     rec_message.acknowledge!
+        #   end
+        #
+        #   # Start background threads that will call block passed to listen.
+        #   subscriber.start
+        #
+        #   # Shut down the subscriber when ready to stop receiving messages.
+        #   subscriber.stop.wait!
+        #
         def listen deadline: nil, message_ordering: nil, streams: nil, inventory: nil, threads: {}, &block
           ensure_service!
           deadline ||= self.deadline