google-cloud-pubsub 1.0.2 → 2.19.0

data/lib/google/cloud/pubsub/snapshot/list.rb

@@ -125,17 +125,15 @@ module Google
           #     puts snapshot.name
           #   end
           #
-          def all request_limit: nil
+          def all request_limit: nil, &block
             request_limit = request_limit.to_i if request_limit
-            unless block_given?
-              return enum_for :all, request_limit: request_limit
-            end
+            return enum_for :all, request_limit: request_limit unless block_given?
             results = self
             loop do
-              results.each { |r| yield r }
+              results.each(&block)
               if request_limit
                 request_limit -= 1
-                break if request_limit < 0
+                break if request_limit.negative?
               end
               break unless results.next?
               results = results.next

data/lib/google/cloud/pubsub/snapshot.rb

@@ -15,6 +15,7 @@
 
 require "google/cloud/errors"
 require "google/cloud/pubsub/snapshot/list"
+require "google/cloud/pubsub/v1"
 
 module Google
   module Cloud
@@ -58,8 +59,10 @@ module Google
         end
 
         ##
-        # The name of the snapshot. Format is
-        # `projects/{project}/snapshots/{snap}`.
+        # The name of the snapshot.
+        #
+        # @return [String] A fully-qualified snapshot name in the form
+        #   `projects/{project_id}/snapshots/{snapshot_id}`.
         def name
           @grpc.name
         end

data/lib/google/cloud/pubsub/subscriber/inventory.rb

@@ -22,30 +22,49 @@ module Google
         ##
         # @private
         class Inventory
+          InventoryItem = Struct.new :bytesize, :pulled_at do
+            def self.from rec_msg
+              new rec_msg.to_proto.bytesize, Time.now
+            end
+          end
+
           include MonitorMixin
 
-          attr_reader :stream, :limit
+          attr_reader :stream
+          attr_reader :limit
+          attr_reader :bytesize
+          attr_reader :extension
+          attr_reader :max_duration_per_lease_extension
+          attr_accessor :min_duration_per_lease_extension
+          attr_reader :use_legacy_flow_control
 
-          def initialize stream, limit
+          def initialize stream, limit:, bytesize:, extension:, max_duration_per_lease_extension:,
+                         min_duration_per_lease_extension:, use_legacy_flow_control:
+            super()
             @stream = stream
             @limit = limit
-            @_ack_ids = []
+            @bytesize = bytesize
+            @extension = extension
+            @max_duration_per_lease_extension = max_duration_per_lease_extension
+            @min_duration_per_lease_extension = min_duration_per_lease_extension
+            @use_legacy_flow_control = use_legacy_flow_control
+            @inventory = {}
             @wait_cond = new_cond
-
-            super()
           end
 
           def ack_ids
-            @_ack_ids
+            @inventory.keys
           end
 
-          def add *ack_ids
-            ack_ids.flatten!
-            ack_ids.compact!
-            return if ack_ids.empty?
+          def add *rec_msgs
+            rec_msgs.flatten!
+            rec_msgs.compact!
+            return if rec_msgs.empty?
 
             synchronize do
-              @_ack_ids += ack_ids
+              rec_msgs.each do |rec_msg|
+                @inventory[rec_msg.ack_id] = InventoryItem.from rec_msg
+              end
               @wait_cond.broadcast
             end
           end
@@ -56,20 +75,34 @@ module Google
             return if ack_ids.empty?
 
             synchronize do
-              @_ack_ids -= ack_ids
+              @inventory.delete_if { |ack_id, _| ack_ids.include? ack_id }
+              @wait_cond.broadcast
+            end
+          end
+
+          def remove_expired!
+            synchronize do
+              extension_time = Time.new - extension
+              @inventory.delete_if { |_ack_id, item| item.pulled_at < extension_time }
               @wait_cond.broadcast
             end
           end
 
           def count
             synchronize do
-              @_ack_ids.count
+              @inventory.count
+            end
+          end
+
+          def total_bytesize
+            synchronize do
+              @inventory.values.sum(&:bytesize)
             end
           end
 
           def empty?
             synchronize do
-              @_ack_ids.empty?
+              @inventory.empty?
             end
           end
 
@@ -93,7 +126,9 @@ module Google
           end
 
           def full?
-            count >= limit
+            synchronize do
+              @inventory.count >= limit || @inventory.values.sum(&:bytesize) >= bytesize
+            end
           end
 
           protected
@@ -101,26 +136,24 @@ module Google
           def background_run
             delay_target = nil
 
-            synchronize do
-              until @stopped
-                if @_ack_ids.empty?
-                  delay_target = nil
-
-                  @wait_cond.wait # wait until broadcast
-                  next
-                end
+            until stopped?
+              if empty?
+                delay_target = nil
 
-                delay_target ||= calc_target
-                delay_gap = delay_target - Time.now
+                synchronize { @wait_cond.wait } # wait until broadcast
+                next
+              end
 
-                unless delay_gap.positive?
-                  delay_target = calc_target
-                  stream.renew_lease!
-                  next
-                end
+              delay_target ||= calc_target
+              delay_gap = delay_target - Time.now
 
-                @wait_cond.wait delay_gap
+              unless delay_gap.positive?
+                delay_target = calc_target
+                stream.renew_lease!
+                next
               end
+
+              synchronize { @wait_cond.wait delay_gap }
             end
           end
 
@@ -129,7 +162,11 @@ module Google
           end
 
           def calc_delay
-            (stream.subscriber.deadline - 3) * rand(0.8..0.9)
+            delay = (stream.subscriber.deadline - 3) * rand(0.8..0.9)
+            delay = [delay, max_duration_per_lease_extension].min if max_duration_per_lease_extension.positive?
+            delay = [delay, min_duration_per_lease_extension].max if min_duration_per_lease_extension.positive? &&
+                                                                     stream.exactly_once_delivery_enabled
+            delay
           end
         end
       end

data/lib/google/cloud/pubsub/subscriber/sequencer.rb

@@ -0,0 +1,115 @@
+# Copyright 2019 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+require "monitor"
+
+module Google
+  module Cloud
+    module PubSub
+      class Subscriber
+        ##
+        # @private The sequencer's job is simple, keep track of all the
+        # streams's recieved message and deliver the messages with an
+        # ordering_key in the order they were recieved. The sequencer ensures
+        # only one callback can be performed at a time per ordering_key.
+        class Sequencer
+          include MonitorMixin
+
+          ##
+          # @private Create an empty Subscriber::Sequencer object.
+          def initialize &block
+            raise ArgumentError if block.nil?
+
+            super() # to init MonitorMixin
+
+            @seq_hash = Hash.new { |hash, key| hash[key] = [] }
+            @process_callback = block
+          end
+
+          ##
+          # @private Add a ReceivedMessage to the sequencer.
+          def add message
+            # Messages without ordering_key are not managed by the sequencer
+            if message.ordering_key.empty?
+              @process_callback.call message
+              return
+            end
+
+            perform_callback = synchronize do
+              # The purpose of this block is to add the message to the
+              # sequencer, and to return whether the message should be processed
+              # immediately, or whether it will be processed later by #next. We
+              # want to ensure that these operations happen atomically.
+
+              @seq_hash[message.ordering_key].push message
+              @seq_hash[message.ordering_key].count == 1
+            end
+
+            @process_callback.call message if perform_callback
+          end
+
+          ##
+          # @private Indicate a ReceivedMessage was processed, and the next in
+          # the queue can now be processed.
+          def next message
+            # Messages without ordering_key are not managed by the sequencer
+            return if message.ordering_key.empty?
+
+            next_message = synchronize do
+              # The purpose of this block is to remove the message that was
+              # processed from the sequencer, and to return the next message to
+              # be processed. We want to ensure that these operations happen
+              # atomically.
+
+              # The message should be at index 0, so this should be a very quick
+              # operation.
+              if @seq_hash[message.ordering_key].first != message
+                # Raising this error will stop the other messages with this
+                # ordering key from being processed by the callback (delivered).
+                raise OrderedMessageDeliveryError, message
+              end
+
+              # Remove the message
+              @seq_hash[message.ordering_key].shift
+
+              # Retrieve the next message to be processed, or nil if empty
+              next_msg = @seq_hash[message.ordering_key].first
+
+              # Remove the ordering_key from hash when empty
+              @seq_hash.delete message.ordering_key if next_msg.nil?
+
+              # Return the next message to be processed, or nil if empty
+              next_msg
+            end
+
+            @process_callback.call next_message unless next_message.nil?
+          end
+
+          # @private
+          def to_s
+            "#{@seq_hash.count}/#{@seq_hash.values.sum(&:count)}"
+          end
+
+          # @private
+          def inspect
+            "#<#{self.class.name} (#{self})>"
+          end
+        end
+      end
+    end
+
+    Pubsub = PubSub unless const_defined? :Pubsub
+  end
+end

data/lib/google/cloud/pubsub/subscriber/stream.rb

@@ -13,6 +13,7 @@
 # limitations under the License.
 
 
+require "google/cloud/pubsub/subscriber/sequencer"
 require "google/cloud/pubsub/subscriber/enumerator_queue"
 require "google/cloud/pubsub/subscriber/inventory"
 require "google/cloud/pubsub/service"
@@ -34,33 +35,46 @@ module Google
           attr_reader :callback_thread_pool
 
           ##
-          # Subscriber attributes.
+          # @private Subscriber attributes.
           attr_reader :subscriber
 
+          ##
+          # @private Inventory.
+          attr_reader :inventory
+
+          ##
+          # @private Sequencer.
+          attr_reader :sequencer
+
+          ##
+          # @private exactly_once_delivery_enabled.
+          attr_reader :exactly_once_delivery_enabled
+
           ##
           # @private Create an empty Subscriber::Stream object.
           def initialize subscriber
+            super() # to init MonitorMixin
+
             @subscriber = subscriber
 
             @request_queue = nil
             @stopped = nil
             @paused  = nil
             @pause_cond = new_cond
+            @exactly_once_delivery_enabled = false
+
+            @inventory = Inventory.new self, **@subscriber.stream_inventory
+
+            @sequencer = Sequencer.new(&method(:perform_callback_async)) if subscriber.message_ordering
 
-            @inventory = Inventory.new self, @subscriber.stream_inventory
-            @callback_thread_pool = Concurrent::ThreadPoolExecutor.new \
-              max_threads: @subscriber.callback_threads
+            @callback_thread_pool = Concurrent::ThreadPoolExecutor.new max_threads: @subscriber.callback_threads
 
             @stream_keepalive_task = Concurrent::TimerTask.new(
               execution_interval: 30
             ) do
               # push empty request every 30 seconds to keep stream alive
-              unless inventory.empty?
-                push Google::Cloud::PubSub::V1::StreamingPullRequest.new
-              end
+              push Google::Cloud::PubSub::V1::StreamingPullRequest.new unless inventory.empty?
             end.execute
-
-            super() # to init MonitorMixin
           end
 
           def start
@@ -81,7 +95,7 @@ module Google
 
               # Close the stream by pushing the sentinel value.
               # The unary pusher does not use the stream, so it can close here.
-              @request_queue.push self unless @request_queue.nil?
+              @request_queue&.push self
 
               # Signal to the background thread that we are stopped.
               @stopped = true
@@ -107,6 +121,10 @@ module Google
             synchronize { @paused }
           end
 
+          def running?
+            !stopped?
+          end
+
           def wait! timeout = nil
             # Wait for all queued callbacks to be processed.
             @callback_thread_pool.wait_for_termination timeout
@@ -116,13 +134,13 @@ module Google
 
           ##
           # @private
-          def acknowledge *messages
+          def acknowledge *messages, &callback
             ack_ids = coerce_ack_ids messages
             return true if ack_ids.empty?
 
             synchronize do
               @inventory.remove ack_ids
-              @subscriber.buffer.acknowledge ack_ids
+              @subscriber.buffer.acknowledge ack_ids, callback
             end
 
             true
@@ -130,13 +148,13 @@ module Google
 
           ##
           # @private
-          def modify_ack_deadline deadline, *messages
+          def modify_ack_deadline deadline, *messages, &callback
             mod_ack_ids = coerce_ack_ids messages
             return true if mod_ack_ids.empty?
 
             synchronize do
               @inventory.remove mod_ack_ids
-              @subscriber.buffer.modify_ack_deadline deadline, mod_ack_ids
+              @subscriber.buffer.modify_ack_deadline deadline, mod_ack_ids, callback
             end
 
             true
@@ -162,18 +180,14 @@ module Google
             synchronize { @request_queue.push request }
           end
 
-          def inventory
-            synchronize { @inventory }
-          end
-
           ##
           # @private
           def renew_lease!
             synchronize do
               return true if @inventory.empty?
 
-              @subscriber.buffer.renew_lease @subscriber.deadline,
-                                             @inventory.ack_ids
+              @inventory.remove_expired!
+              @subscriber.buffer.renew_lease @subscriber.deadline, @inventory.ack_ids
               unpause_streaming!
             end
 
@@ -182,8 +196,8 @@ module Google
 
           # @private
           def to_s
-            "(inventory: #{@inventory.count}, " \
-              "status: #{status}, thread: #{thread_status})"
+            seq_str = "sequenced: #{sequencer}, " if sequencer
+            "(inventory: #{@inventory.count}, #{seq_str}status: #{status}, thread: #{thread_status})"
           end
 
           # @private
@@ -217,7 +231,8 @@ module Google
             end
 
             # Call the StreamingPull API to get the response enumerator
-            enum = @subscriber.service.streaming_pull @request_queue.each
+            options = { :"metadata" => { :"x-goog-request-params" =>  @subscriber.subscription_name } }
+            enum = @subscriber.service.streaming_pull @request_queue.each, options
 
             loop do
               synchronize do
@@ -231,28 +246,33 @@ module Google
               break if synchronize { @stopped }
 
               begin
-                # Cannot syncronize the enumerator, causes deadlock
+                # Cannot synchronize the enumerator, causes deadlock
                 response = enum.next
+                new_exactly_once_delivery_enabled = response&.subscription_properties&.exactly_once_delivery_enabled
+                received_messages = response.received_messages
 
-                # Create a list of all the received ack_id values
-                received_ack_ids = response.received_messages.map(&:ack_id)
-
+                # Use synchronize so changes happen atomically
                 synchronize do
-                  # Create receipt of received messages reception
-                  @subscriber.buffer.modify_ack_deadline @subscriber.deadline,
-                                                         received_ack_ids
+                  update_min_duration_per_lease_extension new_exactly_once_delivery_enabled
+                  @exactly_once_delivery_enabled = new_exactly_once_delivery_enabled unless new_exactly_once_delivery_enabled.nil?
+                  @subscriber.exactly_once_delivery_enabled = @exactly_once_delivery_enabled
 
-                  # Add received messages to inventory
-                  @inventory.add received_ack_ids
+                  # Create receipt of received messages reception
+                  if @exactly_once_delivery_enabled
+                    create_receipt_modack_for_eos received_messages
+                  else
+                    @subscriber.buffer.modify_ack_deadline @subscriber.deadline, received_messages.map(&:ack_id)
+                    # Add received messages to inventory
+                    @inventory.add received_messages
+                  end
                 end
 
-                response.received_messages.each do |rec_msg_grpc|
+                received_messages.each do |rec_msg_grpc|
                   rec_msg = ReceivedMessage.from_grpc(rec_msg_grpc, self)
-                  synchronize do
-                    # Call user provided code for received message
-                    perform_callback_async rec_msg
-                  end
-                end
+                  # No need to synchronize the callback future
+                  register_callback rec_msg
+                end if !@exactly_once_delivery_enabled # Exactly once delivery scenario is handled by callback
+
                 synchronize { pause_streaming! }
               rescue StopIteration
                 break
@@ -267,9 +287,8 @@ module Google
             stop
           rescue GRPC::Cancelled, GRPC::DeadlineExceeded, GRPC::Internal,
                  GRPC::ResourceExhausted, GRPC::Unauthenticated,
-                 GRPC::Unavailable, GRPC::Core::CallError
+                 GRPC::Unavailable
             # Restart the stream with an incremental back for a retriable error.
-            # Also when GRPC raises the internal CallError.
 
             retry
           rescue RestartStream
@@ -282,18 +301,56 @@ module Google
 
           # rubocop:enable all
 
+          def create_receipt_modack_for_eos received_messages
+            received_messages.each do |rec_msg_grpc|
+              callback = proc do |result|
+                if result.succeeded?
+                  synchronize { @inventory.add rec_msg_grpc }
+                  rec_msg = ReceivedMessage.from_grpc rec_msg_grpc, self
+                  register_callback rec_msg
+                end
+              end
+              @subscriber.buffer.modify_ack_deadline @subscriber.deadline, [rec_msg_grpc.ack_id], callback
+            end
+          end
+
+          # Updates min_duration_per_lease_extension to 60 when exactly_once_delivery_enabled
+          # and reverts back to default 0 when disabled.
+          # Skips if exactly_once_enabled is not modified.
+          def update_min_duration_per_lease_extension new_exactly_once_delivery_enabled
+            return if new_exactly_once_delivery_enabled == @exactly_once_delivery_enabled
+            @inventory.min_duration_per_lease_extension = new_exactly_once_delivery_enabled ? 60 : 0
+          end
+
+          def register_callback rec_msg
+            if @sequencer
+              # Add the message to the sequencer to invoke the callback.
+              @sequencer.add rec_msg
+            else
+              # Call user provided code for received message
+              perform_callback_async rec_msg
+            end
+          end
+
           def perform_callback_async rec_msg
             return unless callback_thread_pool.running?
 
             Concurrent::Promises.future_on(
-              callback_thread_pool, self, rec_msg
-            ) do |stream, msg|
+              callback_thread_pool, rec_msg, &method(:perform_callback_sync)
+            )
+          end
+
+          def perform_callback_sync rec_msg
+            @subscriber.callback.call rec_msg unless stopped?
+          rescue StandardError => e
+            @subscriber.error! e
+          ensure
+            release rec_msg
+            if @sequencer && running?
               begin
-                stream.subscriber.callback.call msg unless stream.stopped?
-              rescue StandardError => callback_error
-                stream.subscriber.error! callback_error
-              ensure
-                stream.release msg
+                @sequencer.next rec_msg
+              rescue OrderedMessageDeliveryError => e
+                @subscriber.error! e
               end
             end
           end
@@ -341,8 +398,10 @@ module Google
               req.subscription = @subscriber.subscription_name
               req.stream_ack_deadline_seconds = @subscriber.deadline
               req.modify_deadline_ack_ids += @inventory.ack_ids
-              req.modify_deadline_seconds += \
-                @inventory.ack_ids.map { @subscriber.deadline }
+              req.modify_deadline_seconds += @inventory.ack_ids.map { @subscriber.deadline }
+              req.client_id = @subscriber.service.client_id
+              req.max_outstanding_messages = @inventory.use_legacy_flow_control ? 0 : @inventory.limit
+              req.max_outstanding_bytes = @inventory.use_legacy_flow_control ? 0 : @inventory.bytesize
             end
           end