google-cloud-pubsub 1.0.2 → 2.19.0

data/lib/google/cloud/pubsub/subscriber/timed_unary_buffer.rb

@@ -14,7 +14,10 @@
 
 
 require "concurrent"
+require "google/cloud/errors"
+require "google/cloud/pubsub/acknowledge_result"
 require "monitor"
+require "retriable"
 
 module Google
   module Cloud
@@ -25,9 +28,25 @@ module Google
         class TimedUnaryBuffer
           include MonitorMixin
 
-          attr_reader :max_bytes, :interval
-
-          def initialize subscriber, max_bytes: 500000, interval: 1.0
+          attr_reader :max_bytes
+          attr_reader :interval
+          attr_reader :retry_thread_pool
+          attr_reader :callback_thread_pool
+
+          PERMANENT_FAILURE = "PERMANENT_FAILURE".freeze
+          # Google::Cloud::Unavailable error is already retried at gapic level
+          EXACTLY_ONCE_DELIVERY_POSSIBLE_RETRIABLE_ERRORS = [Google::Cloud::CanceledError,
+                                                             Google::Cloud::DeadlineExceededError,
+                                                             Google::Cloud::InternalError,
+                                                             Google::Cloud::ResourceExhaustedError,
+                                                             Google::Cloud::InvalidArgumentError].freeze
+          MAX_RETRY_DURATION = 600 # 600s since the server allows ack/modacks for 10 mins max
+          MAX_TRIES = 15
+          BASE_INTERVAL = 1
+          MAX_INTERVAL = 64
+          MULTIPLIER = 2
+
+          def initialize subscriber, max_bytes: 500_000, interval: 1.0
             super() # to init MonitorMixin
 
             @subscriber = subscriber
@@ -39,30 +58,37 @@ module Google
             # entry.
             @register = {}
 
+            @ack_callback_register = {}
+            @modack_callback_register = {}
+
+            @retry_thread_pool = Concurrent::ThreadPoolExecutor.new max_threads: @subscriber.callback_threads
+            @callback_thread_pool = Concurrent::ThreadPoolExecutor.new max_threads: @subscriber.callback_threads
             @task = Concurrent::TimerTask.new execution_interval: interval do
               flush!
             end
           end
 
-          def acknowledge ack_ids
+          def acknowledge ack_ids, callback = nil
             return if ack_ids.empty?
 
             synchronize do
               ack_ids.each do |ack_id|
                 # ack has no deadline set, use :ack indicate it is an ack
                 @register[ack_id] = :ack
+                @ack_callback_register[ack_id] = callback unless callback.nil?
               end
             end
 
             true
           end
 
-          def modify_ack_deadline deadline, ack_ids
+          def modify_ack_deadline deadline, ack_ids, callback = nil
             return if ack_ids.empty?
 
             synchronize do
               ack_ids.each do |ack_id|
                 @register[ack_id] = deadline
+                @modack_callback_register[ack_id] = callback unless callback.nil?
               end
             end
 
@@ -89,22 +115,46 @@ module Google
 
             # Perform the RCP calls concurrently
             with_threadpool do |pool|
-              requests[:acknowledge].each do |ack_req|
-                add_future pool do
-                  @subscriber.service.acknowledge \
-                    ack_req.subscription, *ack_req.ack_ids
+              make_acknowledge_request requests, pool
+              make_modack_request requests, pool
+            end
+
+            true
+          end
+
+          def make_acknowledge_request requests, pool
+            requests[:acknowledge].each do |ack_req|
+              add_future pool do
+                begin
+                  @subscriber.service.acknowledge ack_req.subscription, *ack_req.ack_ids
+                  handle_callback AcknowledgeResult.new(AcknowledgeResult::SUCCESS), ack_req.ack_ids
+                rescue *EXACTLY_ONCE_DELIVERY_POSSIBLE_RETRIABLE_ERRORS => e
+                  handle_failure e, ack_req.ack_ids if @subscriber.exactly_once_delivery_enabled
+                rescue StandardError => e
+                  handle_callback construct_result(e), ack_req.ack_ids
                 end
               end
-              requests[:modify_ack_deadline].each do |mod_ack_req|
-                add_future pool do
-                  @subscriber.service.modify_ack_deadline \
-                    mod_ack_req.subscription, mod_ack_req.ack_ids,
-                    mod_ack_req.ack_deadline_seconds
+            end
+          end
+
+          def make_modack_request requests, pool
+            requests[:modify_ack_deadline].each do |mod_ack_req|
+              add_future pool do
+                begin
+                  @subscriber.service.modify_ack_deadline mod_ack_req.subscription, mod_ack_req.ack_ids,
+                                                          mod_ack_req.ack_deadline_seconds
+                  handle_callback AcknowledgeResult.new(AcknowledgeResult::SUCCESS),
+                                  mod_ack_req.ack_ids,
+                                  modack: true
+                rescue *EXACTLY_ONCE_DELIVERY_POSSIBLE_RETRIABLE_ERRORS => e
+                  if @subscriber.exactly_once_delivery_enabled
+                    handle_failure e, mod_ack_req.ack_ids, mod_ack_req.ack_deadline_seconds
+                  end
+                rescue StandardError => e
+                  handle_callback construct_result(e), mod_ack_req.ack_ids, modack: true
                 end
               end
             end
-
-            true
           end
 
           def start
@@ -115,8 +165,9 @@ module Google
 
           def stop
             @task.shutdown
+            @retry_thread_pool.shutdown
+            @callback_thread_pool.shutdown
             flush!
-
             self
           end
 
@@ -130,6 +181,121 @@ module Google
 
           private
 
+          def handle_failure error, ack_ids, ack_deadline_seconds = nil
+            error_ack_ids = parse_error error, modack: ack_deadline_seconds.nil?
+            unless error_ack_ids.nil?
+              handle_callback AcknowledgeResult.new(AcknowledgeResult::SUCCESS),
+                              ack_ids - error_ack_ids,
+                              modack: ack_deadline_seconds.nil?
+              ack_ids = error_ack_ids
+            end
+            perform_retry_async ack_ids, ack_deadline_seconds
+          end
+
+          def parse_error error, modack: false
+            metadata = error.error_metadata
+            return if metadata.nil?
+            permanent_failures, temporary_failures = metadata.partition do |_, v|
+              v.include? PERMANENT_FAILURE
+            end.map(&:to_h)
+            unless permanent_failures.empty?
+              handle_callback construct_result(error),
+                              permanent_failures.keys.map(&:to_s),
+                              modack: modack
+            end
+            temporary_failures.keys.map(&:to_s) unless temporary_failures.empty?
+          end
+
+          def construct_result error
+            case error
+            when Google::Cloud::PermissionDeniedError
+              AcknowledgeResult.new AcknowledgeResult::PERMISSION_DENIED, error
+            when Google::Cloud::FailedPreconditionError
+              AcknowledgeResult.new AcknowledgeResult::FAILED_PRECONDITION, error
+            when Google::Cloud::InvalidArgumentError
+              AcknowledgeResult.new AcknowledgeResult::INVALID_ACK_ID, error
+            else
+              AcknowledgeResult.new AcknowledgeResult::OTHER, error
+            end
+          end
+
+          def handle_callback result, ack_ids, modack: false
+            ack_ids.each do |ack_id|
+              callback = modack ? @modack_callback_register[ack_id] : @ack_callback_register[ack_id]
+              perform_callback_async result, callback unless callback.nil?
+            end
+            synchronize do
+              if modack
+                @modack_callback_register.delete_if { |ack_id, _| ack_ids.include? ack_id }
+              else
+                @ack_callback_register.delete_if { |ack_id, _| ack_ids.include? ack_id }
+              end
+            end
+          end
+
+          def perform_callback_async result, callback
+            return unless retry_thread_pool.running?
+            Concurrent::Promises.future_on(
+              callback_thread_pool, result, callback, &method(:perform_callback_sync)
+            )
+          end
+
+          def perform_callback_sync result, callback
+            begin
+              callback.call result unless stopped?
+            rescue StandardError => e
+              @subscriber.error! e
+            end
+          end
+
+          def perform_retry_async ack_ids, ack_deadline_seconds = nil
+            return unless retry_thread_pool.running?
+            Concurrent::Promises.future_on(
+              retry_thread_pool, ack_ids, ack_deadline_seconds, &method(:retry_transient_error)
+            )
+          end
+
+          def retry_transient_error ack_ids, ack_deadline_seconds
+            if ack_deadline_seconds.nil?
+              retry_request ack_ids, ack_deadline_seconds.nil? do |retry_ack_ids|
+                @subscriber.service.acknowledge subscription_name, *retry_ack_ids
+                handle_callback AcknowledgeResult.new AcknowledgeResult::SUCCESS, retry_ack_ids
+              end
+            else
+              retry_request ack_ids, ack_deadline_seconds.nil? do |retry_ack_ids|
+                @subscriber.service.modify_ack_deadline subscription_name, retry_ack_ids, ack_deadline_seconds
+                handle_callback AcknowledgeResult.new AcknowledgeResult::SUCCESS, retry_ack_ids, modack: true
+              end
+            end
+          end
+
+          def retry_request ack_ids, modack
+            begin
+              Retriable.retriable tries: MAX_TRIES,
+                                  base_interval: BASE_INTERVAL,
+                                  max_interval: MAX_INTERVAL,
+                                  multiplier: MULTIPLIER,
+                                  max_elapsed_time: MAX_RETRY_DURATION,
+                                  on: EXACTLY_ONCE_DELIVERY_POSSIBLE_RETRIABLE_ERRORS do
+                return if ack_ids.nil?
+                begin
+                  yield ack_ids
+                rescue Google::Cloud::InvalidArgumentError => e
+                  error_ack_ids = parse_error e.error_metadata, modack: modack
+                  unless error_ack_ids.nil?
+                    handle_callback AcknowledgeResult.new(AcknowledgeResult::SUCCESS),
+                                    ack_ids - error_ack_ids,
+                                    modack: modack
+                  end
+                  ack_ids = error_ack_ids
+                  raise e
+                end
+              end
+            rescue StandardError => e
+              handle_callback e, ack_ids, modack: modack
+            end
+          end
+
           def flush_requests!
             prev_reg =
               synchronize do
@@ -140,13 +306,11 @@ module Google
               end
 
             groups = prev_reg.each_pair.group_by { |_ack_id, delay| delay }
-            req_hash = Hash[groups.map { |k, v| [k, v.map(&:first)] }]
+            req_hash = groups.transform_values { |v| v.map(&:first) }
 
             requests = { acknowledge: [] }
             ack_ids = Array(req_hash.delete(:ack)) # ack has no deadline set
-            if ack_ids.any?
-              requests[:acknowledge] = create_acknowledge_requests ack_ids
-            end
+            requests[:acknowledge] = create_acknowledge_requests ack_ids if ack_ids.any?
             requests[:modify_ack_deadline] =
               req_hash.map do |mod_deadline, mod_ack_ids|
                 create_modify_ack_deadline_requests mod_deadline, mod_ack_ids
@@ -201,8 +365,7 @@ module Google
           end
 
           def with_threadpool
-            pool = Concurrent::ThreadPoolExecutor.new \
-              max_threads: @subscriber.push_threads
+            pool = Concurrent::ThreadPoolExecutor.new max_threads: @subscriber.push_threads
 
             yield pool
 
@@ -213,18 +376,16 @@ module Google
             pool.kill
             begin
               raise "Timeout making subscriber API calls"
-            rescue StandardError => error
-              error! error
+            rescue StandardError => e
+              error! e
             end
           end
 
           def add_future pool
             Concurrent::Promises.future_on pool do
-              begin
-                yield
-              rescue StandardError => error
-                error! error
-              end
+              yield
+            rescue StandardError => e
+              error! e
             end
           end
         end

data/lib/google/cloud/pubsub/subscriber.rb

@@ -41,7 +41,7 @@ module Google
       #   subscriber.start
       #
       #   # Shut down the subscriber when ready to stop receiving messages.
-      #   subscriber.stop.wait!
+      #   subscriber.stop!
       #
       # @attr_reader [String] subscription_name The name of the subscription the
       #   messages are pulled from.
@@ -50,10 +50,10 @@ module Google
       # @attr_reader [Numeric] deadline The default number of seconds the stream
       #   will hold received messages before modifying the message's ack
       #   deadline. The minimum is 10, the maximum is 600. Default is 60.
+      # @attr_reader [Boolean] message_ordering Whether message ordering has
+      #   been enabled.
       # @attr_reader [Integer] streams The number of concurrent streams to open
-      #   to pull messages from the subscription. Default is 4.
-      # @attr_reader [Integer] inventory The number of received messages to be
-      #   collected by subscriber. Default is 1,000.
+      #   to pull messages from the subscription. Default is 2.
       # @attr_reader [Integer] callback_threads The number of threads used to
       #   handle the received messages. Default is 8.
       # @attr_reader [Integer] push_threads The number of threads to handle
@@ -64,32 +64,42 @@ module Google
       class Subscriber
         include MonitorMixin
 
-        attr_reader :subscription_name, :callback, :deadline, :streams,
-                    :inventory, :callback_threads, :push_threads
+        attr_reader :subscription_name
+        attr_reader :callback
+        attr_reader :deadline
+        attr_reader :streams
+        attr_reader :message_ordering
+        attr_reader :callback_threads
+        attr_reader :push_threads
 
         ##
         # @private Implementation attributes.
-        attr_reader :stream_inventory, :stream_pool, :thread_pool, :buffer,
-                    :service
+        attr_reader :stream_pool, :thread_pool, :buffer, :service
+
+        ##
+        # @private Implementation attributes.
+        attr_accessor :exactly_once_delivery_enabled
 
         ##
         # @private Create an empty {Subscriber} object.
-        def initialize subscription_name, callback, deadline: nil, streams: nil,
-                       inventory: nil, threads: {}, service: nil
+        def initialize subscription_name, callback, deadline: nil, message_ordering: nil, streams: nil, inventory: nil,
+                       threads: {}, service: nil
+          super() # to init MonitorMixin
+
           @callback = callback
           @error_callbacks = []
           @subscription_name = subscription_name
           @deadline = deadline || 60
-          @streams = streams || 4
-          @inventory = inventory || 1000
-          @callback_threads = (threads[:callback] || 8).to_i
-          @push_threads = (threads[:push] || 4).to_i
+          @streams = streams || 2
+          coerce_inventory inventory
+          @message_ordering = message_ordering
+          @callback_threads = Integer(threads[:callback] || 8)
+          @push_threads = Integer(threads[:push] || 4)
+          @exactly_once_delivery_enabled = nil
 
-          @stream_inventory = @inventory.fdiv(@streams).ceil
           @service = service
 
-          @started = nil
-          @stopped = nil
+          @started = @stopped = nil
 
           stream_pool = Array.new @streams do
             Thread.new { Stream.new self }
@@ -97,8 +107,6 @@ module Google
           @stream_pool = stream_pool.map(&:value)
 
           @buffer = TimedUnaryBuffer.new self
-
-          super() # to init MonitorMixin
         end
 
         ##
@@ -106,6 +114,7 @@ module Google
         # received messages.
         #
         # @return [Subscriber] returns self so calls can be chained.
+        #
         def start
           start_pool = synchronize do
             @started = true
@@ -124,27 +133,21 @@ module Google
 
         ##
         # Immediately stops the subscriber. No new messages will be pulled from
-        # the subscription. All actions taken on received messages that have not
-        # yet been sent to the API will be sent to the API. All received but
-        # unprocessed messages will be released back to the API and redelivered.
-        # Use {#wait!} to block until the subscriber is fully stopped and all
-        # received messages have been processed or released.
+        # the subscription. Use {#wait!} to block until all received messages have
+        # been processed or released: All actions taken on received messages that
+        # have not yet been sent to the API will be sent to the API. All received
+        # but unprocessed messages will be released back to the API and redelivered.
         #
         # @return [Subscriber] returns self so calls can be chained.
+        #
         def stop
-          stop_pool = synchronize do
+          synchronize do
             @started = false
             @stopped = true
-
-            @stream_pool.map do |stream|
-              Thread.new { stream.stop }
-            end
+            @stream_pool.map(&:stop)
+            wait_stop_buffer_thread!
+            self
           end
-          stop_pool.map(&:join)
-          # Stop the buffer after the streams are all stopped
-          synchronize { @buffer.stop }
-
-          self
         end
 
         ##
@@ -160,14 +163,10 @@ module Google
         #   subscriber is fully stopped. Default will block indefinitely.
         #
         # @return [Subscriber] returns self so calls can be chained.
+        #
         def wait! timeout = nil
-          wait_pool = synchronize do
-            @stream_pool.map do |stream|
-              Thread.new { stream.wait! timeout }
-            end
-          end
-          wait_pool.map(&:join)
-
+          wait_stop_buffer_thread!
+          @wait_stop_buffer_thread.join timeout
           self
         end
 
@@ -192,6 +191,7 @@ module Google
         # Whether the subscriber has been started.
         #
         # @return [boolean] `true` when started, `false` otherwise.
+        #
         def started?
           synchronize { @started }
         end
@@ -200,6 +200,7 @@ module Google
         # Whether the subscriber has been stopped.
         #
         # @return [boolean] `true` when stopped, `false` otherwise.
+        #
         def stopped?
           synchronize { @stopped }
         end
@@ -237,7 +238,7 @@ module Google
         #   subscriber.start
         #
         #   # Shut down the subscriber when ready to stop receiving messages.
-        #   subscriber.stop.wait!
+        #   subscriber.stop!
         #
         def on_error &block
           synchronize do
@@ -273,12 +274,91 @@ module Google
         #   subscriber.last_error #=> nil
         #
         #   # Shut down the subscriber when ready to stop receiving messages.
-        #   subscriber.stop.wait!
+        #   subscriber.stop!
         #
         def last_error
           synchronize { @last_error }
         end
 
+        ##
+        # The number of received messages to be collected by subscriber. Default is 1,000.
+        #
+        # @return [Integer] The maximum number of messages.
+        #
+        def max_outstanding_messages
+          @inventory[:max_outstanding_messages]
+        end
+        # @deprecated Use {#max_outstanding_messages}.
+        alias inventory_limit max_outstanding_messages
+        # @deprecated Use {#max_outstanding_messages}.
+        alias inventory max_outstanding_messages
+
+        ##
+        # The total byte size of received messages to be collected by subscriber. Default is 100,000,000 (100MB).
+        #
+        # @return [Integer] The maximum number of bytes.
+        #
+        def max_outstanding_bytes
+          @inventory[:max_outstanding_bytes]
+        end
+        # @deprecated Use {#max_outstanding_bytes}.
+        alias inventory_bytesize max_outstanding_bytes
+
+        ##
+        # Whether to enforce flow control at the client side only or to enforce it at both the client and
+        # the server. For more details about flow control see https://cloud.google.com/pubsub/docs/pull#config.
+        #
+        # @return [Boolean] `true` when only client side flow control is enforced, `false` when both client and
+        # server side flow control are enforced.
+        #
+        def use_legacy_flow_control?
+          @inventory[:use_legacy_flow_control]
+        end
+
+        ##
+        # The number of seconds that received messages can be held awaiting processing. Default is 3,600 (1 hour).
+        #
+        # @return [Integer] The maximum number of seconds.
+        #
+        def max_total_lease_duration
+          @inventory[:max_total_lease_duration]
+        end
+        # @deprecated Use {#max_total_lease_duration}.
+        alias inventory_extension max_total_lease_duration
+
+        ##
+        # The maximum amount of time in seconds for a single lease extension attempt. Bounds the delay before a message
+        # redelivery if the subscriber fails to extend the deadline. Default is 0 (disabled).
+        #
+        # @return [Integer] The maximum number of seconds.
+        #
+        def max_duration_per_lease_extension
+          @inventory[:max_duration_per_lease_extension]
+        end
+
+        ##
+        # The minimum amount of time in seconds for a single lease extension attempt. Bounds the delay before a message
+        # redelivery if the subscriber fails to extend the deadline. Default is 0 (disabled).
+        #
+        # @return [Integer] The minimum number of seconds.
+        #
+        def min_duration_per_lease_extension
+          @inventory[:min_duration_per_lease_extension]
+        end
+
+        ##
+        # @private
+        def stream_inventory
+          {
+            limit:                            @inventory[:max_outstanding_messages].fdiv(@streams).ceil,
+            bytesize:                         @inventory[:max_outstanding_bytes].fdiv(@streams).ceil,
+            extension:                        @inventory[:max_total_lease_duration],
+            max_duration_per_lease_extension: @inventory[:max_duration_per_lease_extension],
+            min_duration_per_lease_extension: @inventory[:min_duration_per_lease_extension],
+            use_legacy_flow_control:          @inventory[:use_legacy_flow_control]
+          }
+        end
+
         # @private returns error object from the stream thread.
         def error! error
           error_callbacks = synchronize do
@@ -292,8 +372,7 @@ module Google
         ##
         # @private
         def to_s
-          "(subscription: #{subscription_name}, " \
-            "streams: [#{stream_pool.map(&:to_s).join(', ')}])"
+          "(subscription: #{subscription_name}, streams: [#{stream_pool.map(&:to_s).join(', ')}])"
         end
 
         ##
@@ -304,6 +383,37 @@ module Google
 
         protected
 
+        ##
+        # Starts a new thread to call wait! (blocking) on each Stream and then stop the TimedUnaryBuffer.
+        def wait_stop_buffer_thread!
+          synchronize do
+            @wait_stop_buffer_thread ||= Thread.new do
+              @stream_pool.map(&:wait!)
+              # Shutdown the buffer TimerTask (and flush the buffer) after the streams are all stopped.
+              @buffer.stop
+            end
+          end
+        end
+
+        def coerce_inventory inventory
+          @inventory = inventory
+          if @inventory.is_a? Hash
+            @inventory = @inventory.dup
+            # Support deprecated field names
+            @inventory[:max_outstanding_messages] ||= @inventory.delete :limit
+            @inventory[:max_outstanding_bytes] ||= @inventory.delete :bytesize
+            @inventory[:max_total_lease_duration] ||= @inventory.delete :extension
+          else
+            @inventory = { max_outstanding_messages: @inventory }
+          end
+          @inventory[:max_outstanding_messages] = Integer(@inventory[:max_outstanding_messages] || 1000)
+          @inventory[:max_outstanding_bytes] = Integer(@inventory[:max_outstanding_bytes] || 100_000_000)
+          @inventory[:max_total_lease_duration] = Integer(@inventory[:max_total_lease_duration] || 3600)
+          @inventory[:max_duration_per_lease_extension] = Integer(@inventory[:max_duration_per_lease_extension] || 0)
+          @inventory[:min_duration_per_lease_extension] = Integer(@inventory[:min_duration_per_lease_extension] || 0)
+          @inventory[:use_legacy_flow_control] = @inventory[:use_legacy_flow_control] || false
+        end
+
         def default_error_callbacks
           # This is memoized to reduce calls to the configuration.
           @default_error_callbacks ||= begin

data/lib/google/cloud/pubsub/subscription/list.rb

@@ -130,17 +130,15 @@ module Google
           #     puts subscription.name
           #   end
           #
-          def all request_limit: nil
+          def all request_limit: nil, &block
             request_limit = request_limit.to_i if request_limit
-            unless block_given?
-              return enum_for :all, request_limit: request_limit
-            end
+            return enum_for :all, request_limit: request_limit unless block_given?
             results = self
             loop do
-              results.each { |r| yield r }
+              results.each(&block)
               if request_limit
                 request_limit -= 1
-                break if request_limit < 0
+                break if request_limit.negative?
               end
               break unless results.next?
               results = results.next