google-cloud-pubsub 1.9.0

data/lib/google/cloud/pubsub/subscriber/enumerator_queue.rb

@@ -0,0 +1,54 @@
+# Copyright 2017 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+module Google
+  module Cloud
+    module PubSub
+      class Subscriber
+        # @private
+        class EnumeratorQueue
+          def initialize sentinel = nil
+            @queue    = Queue.new
+            @sentinel = sentinel
+          end
+
+          def push obj
+            @queue.push obj
+          end
+
+          def quit_and_dump_queue
+            objs = []
+            objs << @queue.pop until @queue.empty?
+            # Signal that the enumerator is ready to end
+            @queue.push @sentinel
+            objs
+          end
+
+          def each
+            return enum_for :each unless block_given?
+
+            loop do
+              obj = @queue.pop
+              break if obj.equal? @sentinel
+              yield obj
+            end
+          end
+        end
+      end
+    end
+
+    Pubsub = PubSub unless const_defined? :Pubsub
+  end
+end

data/lib/google/cloud/pubsub/subscriber/inventory.rb

@@ -0,0 +1,166 @@
+# Copyright 2017 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+require "monitor"
+
+module Google
+  module Cloud
+    module PubSub
+      class Subscriber
+        ##
+        # @private
+        class Inventory
+          InventoryItem = Struct.new :bytesize, :pulled_at do
+            def self.from rec_msg
+              new rec_msg.to_proto.bytesize, Time.now
+            end
+          end
+
+          include MonitorMixin
+
+          attr_reader :stream, :limit, :bytesize, :extension, :max_duration_per_lease_extension
+
+          def initialize stream, limit:, bytesize:, extension:, max_duration_per_lease_extension:
+            super()
+            @stream = stream
+            @limit = limit
+            @bytesize = bytesize
+            @extension = extension
+            @max_duration_per_lease_extension = max_duration_per_lease_extension
+            @inventory = {}
+            @wait_cond = new_cond
+          end
+
+          def ack_ids
+            @inventory.keys
+          end
+
+          def add *rec_msgs
+            rec_msgs.flatten!
+            rec_msgs.compact!
+            return if rec_msgs.empty?
+
+            synchronize do
+              rec_msgs.each do |rec_msg|
+                @inventory[rec_msg.ack_id] = InventoryItem.from rec_msg
+              end
+              @wait_cond.broadcast
+            end
+          end
+
+          def remove *ack_ids
+            ack_ids.flatten!
+            ack_ids.compact!
+            return if ack_ids.empty?
+
+            synchronize do
+              @inventory.delete_if { |ack_id, _| ack_ids.include? ack_id }
+              @wait_cond.broadcast
+            end
+          end
+
+          def remove_expired!
+            synchronize do
+              extension_time = Time.new - extension
+              @inventory.delete_if { |_ack_id, item| item.pulled_at < extension_time }
+              @wait_cond.broadcast
+            end
+          end
+
+          def count
+            synchronize do
+              @inventory.count
+            end
+          end
+
+          def total_bytesize
+            synchronize do
+              @inventory.values.sum(&:bytesize)
+            end
+          end
+
+          def empty?
+            synchronize do
+              @inventory.empty?
+            end
+          end
+
+          def start
+            @background_thread ||= Thread.new { background_run }
+
+            self
+          end
+
+          def stop
+            synchronize do
+              @stopped = true
+              @wait_cond.broadcast
+            end
+
+            self
+          end
+
+          def stopped?
+            synchronize { @stopped }
+          end
+
+          def full?
+            synchronize do
+              @inventory.count >= limit || @inventory.values.sum(&:bytesize) >= bytesize
+            end
+          end
+
+          protected
+
+          def background_run
+            delay_target = nil
+
+            until stopped?
+              if empty?
+                delay_target = nil
+
+                synchronize { @wait_cond.wait } # wait until broadcast
+                next
+              end
+
+              delay_target ||= calc_target
+              delay_gap = delay_target - Time.now
+
+              unless delay_gap.positive?
+                delay_target = calc_target
+                stream.renew_lease!
+                next
+              end
+
+              synchronize { @wait_cond.wait delay_gap }
+            end
+          end
+
+          def calc_target
+            Time.now + calc_delay
+          end
+
+          def calc_delay
+            delay = (stream.subscriber.deadline - 3) * rand(0.8..0.9)
+            delay = [delay, max_duration_per_lease_extension].min if max_duration_per_lease_extension.positive?
+            delay
+          end
+        end
+      end
+    end
+
+    Pubsub = PubSub unless const_defined? :Pubsub
+  end
+end

data/lib/google/cloud/pubsub/subscriber/sequencer.rb

@@ -0,0 +1,115 @@
+# Copyright 2019 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+require "monitor"
+
+module Google
+  module Cloud
+    module PubSub
+      class Subscriber
+        ##
+        # @private The sequencer's job is simple, keep track of all the
+        # streams's recieved message and deliver the messages with an
+        # ordering_key in the order they were recieved. The sequencer ensures
+        # only one callback can be performed at a time per ordering_key.
+        class Sequencer
+          include MonitorMixin
+
+          ##
+          # @private Create an empty Subscriber::Sequencer object.
+          def initialize &block
+            raise ArgumentError if block.nil?
+
+            super() # to init MonitorMixin
+
+            @seq_hash = Hash.new { |hash, key| hash[key] = [] }
+            @process_callback = block
+          end
+
+          ##
+          # @private Add a ReceivedMessage to the sequencer.
+          def add message
+            # Messages without ordering_key are not managed by the sequencer
+            if message.ordering_key.empty?
+              @process_callback.call message
+              return
+            end
+
+            perform_callback = synchronize do
+              # The purpose of this block is to add the message to the
+              # sequencer, and to return whether the message should be processed
+              # immediately, or whether it will be processed later by #next. We
+              # want to ensure that these operations happen atomically.
+
+              @seq_hash[message.ordering_key].push message
+              @seq_hash[message.ordering_key].count == 1
+            end
+
+            @process_callback.call message if perform_callback
+          end
+
+          ##
+          # @private Indicate a ReceivedMessage was processed, and the next in
+          # the queue can now be processed.
+          def next message
+            # Messages without ordering_key are not managed by the sequencer
+            return if message.ordering_key.empty?
+
+            next_message = synchronize do
+              # The purpose of this block is to remove the message that was
+              # processed from the sequencer, and to return the next message to
+              # be processed. We want to ensure that these operations happen
+              # atomically.
+
+              # The message should be at index 0, so this should be a very quick
+              # operation.
+              if @seq_hash[message.ordering_key].first != message
+                # Raising this error will stop the other messages with this
+                # ordering key from being processed by the callback (delivered).
+                raise OrderedMessageDeliveryError, message
+              end
+
+              # Remove the message
+              @seq_hash[message.ordering_key].shift
+
+              # Retrieve the next message to be processed, or nil if empty
+              next_msg = @seq_hash[message.ordering_key].first
+
+              # Remove the ordering_key from hash when empty
+              @seq_hash.delete message.ordering_key if next_msg.nil?
+
+              # Return the next message to be processed, or nil if empty
+              next_msg
+            end
+
+            @process_callback.call next_message unless next_message.nil?
+          end
+
+          # @private
+          def to_s
+            "#{@seq_hash.count}/#{@seq_hash.values.sum(&:count)}"
+          end
+
+          # @private
+          def inspect
+            "#<#{self.class.name} (#{self})>"
+          end
+        end
+      end
+    end
+
+    Pubsub = PubSub unless const_defined? :Pubsub
+  end
+end

data/lib/google/cloud/pubsub/subscriber/stream.rb

@@ -0,0 +1,401 @@
+# Copyright 2017 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+require "google/cloud/pubsub/subscriber/sequencer"
+require "google/cloud/pubsub/subscriber/enumerator_queue"
+require "google/cloud/pubsub/subscriber/inventory"
+require "google/cloud/pubsub/service"
+require "google/cloud/errors"
+require "monitor"
+require "concurrent"
+
+module Google
+  module Cloud
+    module PubSub
+      class Subscriber
+        ##
+        # @private
+        class Stream
+          include MonitorMixin
+
+          ##
+          # @private Implementation attributes.
+          attr_reader :callback_thread_pool
+
+          ##
+          # @private Subscriber attributes.
+          attr_reader :subscriber
+
+          ##
+          # @private Inventory.
+          attr_reader :inventory
+
+          ##
+          # @private Sequencer.
+          attr_reader :sequencer
+
+          ##
+          # @private Create an empty Subscriber::Stream object.
+          def initialize subscriber
+            super() # to init MonitorMixin
+
+            @subscriber = subscriber
+
+            @request_queue = nil
+            @stopped = nil
+            @paused  = nil
+            @pause_cond = new_cond
+
+            @inventory = Inventory.new self, @subscriber.stream_inventory
+
+            @sequencer = Sequencer.new(&method(:perform_callback_async)) if subscriber.message_ordering
+
+            @callback_thread_pool = Concurrent::ThreadPoolExecutor.new max_threads: @subscriber.callback_threads
+
+            @stream_keepalive_task = Concurrent::TimerTask.new(
+              execution_interval: 30
+            ) do
+              # push empty request every 30 seconds to keep stream alive
+              push Google::Cloud::PubSub::V1::StreamingPullRequest.new unless inventory.empty?
+            end.execute
+          end
+
+          def start
+            synchronize do
+              break if @background_thread
+
+              @inventory.start
+
+              start_streaming!
+            end
+
+            self
+          end
+
+          def stop
+            synchronize do
+              break if @stopped
+
+              # Close the stream by pushing the sentinel value.
+              # The unary pusher does not use the stream, so it can close here.
+              @request_queue&.push self
+
+              # Signal to the background thread that we are stopped.
+              @stopped = true
+              @pause_cond.broadcast
+
+              # Now that the reception thread is stopped, immediately stop the
+              # callback thread pool. All queued callbacks will see the stream
+              # is stopped and perform a noop.
+              @callback_thread_pool.shutdown
+
+              # Once all the callbacks are stopped, we can stop the inventory.
+              @inventory.stop
+            end
+
+            self
+          end
+
+          def stopped?
+            synchronize { @stopped }
+          end
+
+          def paused?
+            synchronize { @paused }
+          end
+
+          def running?
+            !stopped?
+          end
+
+          def wait! timeout = nil
+            # Wait for all queued callbacks to be processed.
+            @callback_thread_pool.wait_for_termination timeout
+
+            self
+          end
+
+          ##
+          # @private
+          def acknowledge *messages
+            ack_ids = coerce_ack_ids messages
+            return true if ack_ids.empty?
+
+            synchronize do
+              @inventory.remove ack_ids
+              @subscriber.buffer.acknowledge ack_ids
+            end
+
+            true
+          end
+
+          ##
+          # @private
+          def modify_ack_deadline deadline, *messages
+            mod_ack_ids = coerce_ack_ids messages
+            return true if mod_ack_ids.empty?
+
+            synchronize do
+              @inventory.remove mod_ack_ids
+              @subscriber.buffer.modify_ack_deadline deadline, mod_ack_ids
+            end
+
+            true
+          end
+
+          ##
+          # @private
+          def release *messages
+            ack_ids = coerce_ack_ids messages
+            return if ack_ids.empty?
+
+            synchronize do
+              # Remove from inventory if the message was not explicitly acked or
+              # nacked in the callback
+              @inventory.remove ack_ids
+              # Check whether to unpause the stream only after the callback is
+              # completed and the thread is being reclaimed.
+              unpause_streaming!
+            end
+          end
+
+          def push request
+            synchronize { @request_queue.push request }
+          end
+
+          ##
+          # @private
+          def renew_lease!
+            synchronize do
+              return true if @inventory.empty?
+
+              @inventory.remove_expired!
+              @subscriber.buffer.renew_lease @subscriber.deadline, @inventory.ack_ids
+              unpause_streaming!
+            end
+
+            true
+          end
+
+          # @private
+          def to_s
+            seq_str = "sequenced: #{sequencer}, " if sequencer
+            "(inventory: #{@inventory.count}, #{seq_str}status: #{status}, thread: #{thread_status})"
+          end
+
+          # @private
+          def inspect
+            "#<#{self.class.name} #{self}>"
+          end
+
+          protected
+
+          # @private
+          class RestartStream < StandardError; end
+
+          # rubocop:disable all
+
+          def background_run
+            synchronize do
+              # Don't allow a stream to restart if already stopped
+              return if @stopped
+
+              @stopped = false
+              @paused  = false
+
+              # signal to the previous queue to shut down
+              old_queue = []
+              old_queue = @request_queue.quit_and_dump_queue if @request_queue
+
+              # Always create a new request queue
+              @request_queue = EnumeratorQueue.new self
+              @request_queue.push initial_input_request
+              old_queue.each { |obj| @request_queue.push obj }
+            end
+
+            # Call the StreamingPull API to get the response enumerator
+            enum = @subscriber.service.streaming_pull @request_queue.each
+
+            loop do
+              synchronize do
+                if @paused && !@stopped
+                  @pause_cond.wait
+                  next
+                end
+              end
+
+              # Break loop, close thread if stopped
+              break if synchronize { @stopped }
+
+              begin
+                # Cannot syncronize the enumerator, causes deadlock
+                response = enum.next
+
+                # Use synchronize so both changes happen atomically
+                synchronize do
+                  # Create receipt of received messages reception
+                  @subscriber.buffer.modify_ack_deadline @subscriber.deadline, response.received_messages.map(&:ack_id)
+
+                  # Add received messages to inventory
+                  @inventory.add response.received_messages
+                end
+
+                response.received_messages.each do |rec_msg_grpc|
+                  rec_msg = ReceivedMessage.from_grpc(rec_msg_grpc, self)
+                  # No need to synchronize the callback future
+                  register_callback rec_msg
+                end
+                synchronize { pause_streaming! }
+              rescue StopIteration
+                break
+              end
+            end
+
+            # Has the loop broken but we aren't stopped?
+            # Could be GRPC has thrown an internal error, so restart.
+            raise RestartStream unless synchronize { @stopped }
+
+            # We must be stopped, tell the stream to quit.
+            stop
+          rescue GRPC::Cancelled, GRPC::DeadlineExceeded, GRPC::Internal,
+                 GRPC::ResourceExhausted, GRPC::Unauthenticated,
+                 GRPC::Unavailable, GRPC::Core::CallError
+            # Restart the stream with an incremental back for a retriable error.
+            # Also when GRPC raises the internal CallError.
+
+            retry
+          rescue RestartStream
+            retry
+          rescue StandardError => e
+            @subscriber.error! e
+
+            retry
+          end
+
+          # rubocop:enable all
+
+          def register_callback rec_msg
+            if @sequencer
+              # Add the message to the sequencer to invoke the callback.
+              @sequencer.add rec_msg
+            else
+              # Call user provided code for received message
+              perform_callback_async rec_msg
+            end
+          end
+
+          def perform_callback_async rec_msg
+            return unless callback_thread_pool.running?
+
+            Concurrent::Promises.future_on(
+              callback_thread_pool, rec_msg, &method(:perform_callback_sync)
+            )
+          end
+
+          def perform_callback_sync rec_msg
+            @subscriber.callback.call rec_msg unless stopped?
+          rescue StandardError => e
+            @subscriber.error! e
+          ensure
+            release rec_msg
+            if @sequencer && running?
+              begin
+                @sequencer.next rec_msg
+              rescue OrderedMessageDeliveryError => e
+                @subscriber.error! e
+              end
+            end
+          end
+
+          def start_streaming!
+            # A Stream will only ever have one background thread. If the thread
+            # dies because it was stopped, or because of an unhandled error that
+            # could not be recovered from, so be it.
+            return if @background_thread
+
+            # create new background thread to handle new enumerator
+            @background_thread = Thread.new { background_run }
+          end
+
+          def pause_streaming!
+            return unless pause_streaming?
+
+            @paused = true
+          end
+
+          def pause_streaming?
+            return if @stopped
+            return if @paused
+
+            @inventory.full?
+          end
+
+          def unpause_streaming!
+            return unless unpause_streaming?
+
+            @paused = nil
+            # signal to the background thread that we are unpaused
+            @pause_cond.broadcast
+          end
+
+          def unpause_streaming?
+            return if @stopped
+            return if @paused.nil?
+
+            @inventory.count < @inventory.limit * 0.8
+          end
+
+          def initial_input_request
+            Google::Cloud::PubSub::V1::StreamingPullRequest.new.tap do |req|
+              req.subscription = @subscriber.subscription_name
+              req.stream_ack_deadline_seconds = @subscriber.deadline
+              req.modify_deadline_ack_ids += @inventory.ack_ids
+              req.modify_deadline_seconds += @inventory.ack_ids.map { @subscriber.deadline }
+              req.client_id = @subscriber.service.client_id
+              req.max_outstanding_messages = @inventory.limit
+              req.max_outstanding_bytes = @inventory.bytesize
+            end
+          end
+
+          ##
+          # Makes sure the values are the `ack_id`. If given several
+          # {ReceivedMessage} objects extract the `ack_id` values.
+          def coerce_ack_ids messages
+            Array(messages).flatten.map do |msg|
+              msg.respond_to?(:ack_id) ? msg.ack_id : msg.to_s
+            end
+          end
+
+          def status
+            return "stopped" if stopped?
+            return "paused" if paused?
+            "running"
+          end
+
+          def thread_status
+            return "not started" if @background_thread.nil?
+
+            status = @background_thread.status
+            return "error" if status.nil?
+            return "stopped" if status == false
+            status
+          end
+        end
+      end
+    end
+
+    Pubsub = PubSub unless const_defined? :Pubsub
+  end
+end