google-cloud-pubsub 0.31.1 → 0.32.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 3817e592071ccc0badbc1c511c848341de65d0e5fb4b9feba8c528074cd71724
-  data.tar.gz: 9e789ffcc98b3672040ff67f5e198f6f214cde83a205d6893982ca87b5e99e2b
+  metadata.gz: 47a9873e0c2c11d5625c00418517bf241b344dd45edf5efabace63b042e26137
+  data.tar.gz: 6cd96cfb644807699b787fa4502905923ca522f2ae5efefe937d8b08b04eeeeb
 SHA512:
-  metadata.gz: 52e115e0f6bf8968976a853064e30088374ea1836410785e3042fe00324f065e085d54e7b1502a34cd8fde8dc4395b901a95293856846f8b5b6f0af421c11586
-  data.tar.gz: 2cf2bf6b75e926fcb0fda9ba22cae4a49a75b0fc7d408faf61ecc90fbd5a424e496369780eadee1b3fa90a448b37052933d70ee6b790dbed81ecbaed18380bae
+  metadata.gz: 1bd5df7f2e73e44bcd51c60f00baf3097567c9703899b6bffe5d0384a59b30f9fb33ed48b396674d221872c39831697168f5a541604ec48a0e08d08ea20d7f4a
+  data.tar.gz: 1cba0d5355c7565ab5c6b3787c72d55a27b31ff5555a5e9995641f8d88e291c88476c473621d090f1618cff7023604ac5616fa5106b34cca0d49f507a6b63111

data/.yardopts

@@ -2,6 +2,7 @@
 --title=Google Cloud Pub/Sub
 --exclude lib/google/pubsub/v1
 --markup markdown
+--markup-provider redcarpet
 
 ./lib/**/*.rb
 -

data/README.md

@@ -74,14 +74,14 @@ end
 
 ## Supported Ruby Versions
 
-This library is supported on Ruby 2.0+.
-
-However, Ruby 2.3 or later is strongly recommended, as earlier releases have
-reached or are nearing end-of-life. After June 1, 2018, Google will provide
-official support only for Ruby versions that are considered current and
-supported by Ruby Core (that is, Ruby versions that are either in normal
-maintenance or in security maintenance).
-See https://www.ruby-lang.org/en/downloads/branches/ for further details.
+This library is supported on Ruby 2.3+.
+
+Google provides official support for Ruby versions that are actively supported
+by Ruby Core—that is, Ruby versions that are either in normal maintenance or
+in security maintenance, and not end of life. Currently, this means Ruby 2.3
+and later. Older versions of Ruby _may_ still work, but are unsupported and not
+recommended. See https://www.ruby-lang.org/en/downloads/branches/ for details
+about the Ruby support schedule.
 
 ## Versioning
 

data/lib/google/cloud/pubsub/async_publisher.rb

@@ -228,6 +228,8 @@ module Google
         end
 
         def publish_batch_async topic_name, batch
+          return unless @publish_thread_pool.running?
+
           Concurrent::Future.new(executor: @publish_thread_pool) do
             begin
               grpc = @service.publish topic_name, batch.messages
@@ -250,6 +252,8 @@ module Google
         end
 
         def execute_callback_async callback, publish_result
+          return unless @callback_thread_pool.running?
+
           Concurrent::Future.new(executor: @callback_thread_pool) do
             callback.call publish_result
           end.execute

data/lib/google/cloud/pubsub/credentials.rb

@@ -14,6 +14,7 @@
 
 
 require "googleauth"
+require "google/cloud/pubsub/v1/credentials.rb"
 
 module Google
   module Cloud
@@ -37,20 +38,7 @@ module Google
       #
       #   pubsub.project_id #=> "my-project"
       #
-      class Credentials < Google::Auth::Credentials
-        SCOPE = ["https://www.googleapis.com/auth/pubsub"].freeze
-        PATH_ENV_VARS = %w[PUBSUB_CREDENTIALS
-                           PUBSUB_KEYFILE
-                           GOOGLE_CLOUD_CREDENTIALS
-                           GOOGLE_CLOUD_KEYFILE
-                           GCLOUD_KEYFILE].freeze
-        JSON_ENV_VARS = %w[PUBSUB_CREDENTIALS_JSON
-                           PUBSUB_KEYFILE_JSON
-                           GOOGLE_CLOUD_CREDENTIALS_JSON
-                           GOOGLE_CLOUD_KEYFILE_JSON
-                           GCLOUD_KEYFILE_JSON].freeze
-        DEFAULT_PATHS = \
-          ["~/.config/gcloud/application_default_credentials.json"].freeze
+      class Credentials < Google::Cloud::Pubsub::V1::Credentials
       end
     end
   end

data/lib/google/cloud/pubsub/subscriber.rb

@@ -74,6 +74,7 @@ module Google
         def initialize subscription_name, callback, deadline: nil, streams: nil,
                        inventory: nil, threads: {}, service: nil
           @callback = callback
+          @error_callbacks = []
           @subscription_name = subscription_name
           @deadline = deadline || 60
           @streams = streams || 4
@@ -169,6 +170,90 @@ module Google
           synchronize { @stopped }
         end
 
+        ##
+        # Register to be notified of errors when raised.
+        #
+        # If an unhandled error has occurred the subscriber will attempt to
+        # recover from the error and resume listening.
+        #
+        # Multiple error handlers can be added.
+        #
+        # @yield [callback] The block to be called when an error is raised.
+        # @yieldparam [Exception] error The error raised.
+        #
+        # @example
+        #   require "google/cloud/pubsub"
+        #
+        #   pubsub = Google::Cloud::Pubsub.new
+        #
+        #   sub = pubsub.subscription "my-topic-sub"
+        #
+        #   subscriber = sub.listen do |received_message|
+        #     # process message
+        #     received_message.acknowledge!
+        #   end
+        #
+        #   # Register to be notified when unhandled errors occur.
+        #   subscriber.on_error do |error|
+        #     # log error
+        #     puts error
+        #   end
+        #
+        #   # Start listening for messages and errors.
+        #   subscriber.start
+        #
+        #   # Shut down the subscriber when ready to stop receiving messages.
+        #   subscriber.stop.wait!
+        #
+        def on_error &block
+          synchronize do
+            @error_callbacks << block
+          end
+        end
+
+        ##
+        # The most recent unhandled error to occur while listening to messages
+        # on the subscriber.
+        #
+        # If an unhandled error has occurred the subscriber will attempt to
+        # recover from the error and resume listening.
+        #
+        # @return [Exception, nil] error The most recent error raised.
+        #
+        # @example
+        #   require "google/cloud/pubsub"
+        #
+        #   pubsub = Google::Cloud::Pubsub.new
+        #
+        #   sub = pubsub.subscription "my-topic-sub"
+        #
+        #   subscriber = sub.listen do |received_message|
+        #     # process message
+        #     received_message.acknowledge!
+        #   end
+        #
+        #   # Start listening for messages and errors.
+        #   subscriber.start
+        #
+        #   # If an error was raised, it can be retrieved here:
+        #   subscriber.last_error #=> nil
+        #
+        #   # Shut down the subscriber when ready to stop receiving messages.
+        #   subscriber.stop.wait!
+        #
+        def last_error
+          synchronize { @last_error }
+        end
+
+        # @private returns error object from the stream thread.
+        def error! error
+          error_callbacks = synchronize do
+            @last_error = error
+            @error_callbacks
+          end
+          error_callbacks.each { |error_callback| error_callback.call error }
+        end
+
         ##
         # @private
         def to_s

data/lib/google/cloud/pubsub/subscriber/async_stream_pusher.rb

@@ -59,9 +59,6 @@ module Google
                   end
                 end
 
-                @batch_created_at ||= Time.now
-                @background_thread ||= Thread.new { run_background }
-
                 push_batch_request! if @batch.ready?
               end
 
@@ -88,9 +85,6 @@ module Google
                   end
                 end
 
-                @batch_created_at ||= Time.now
-                @background_thread ||= Thread.new { run_background }
-
                 push_batch_request! if @batch.ready?
               end
 
@@ -100,15 +94,25 @@ module Google
             nil
           end
 
+          def start
+            synchronize do
+              @stopped = false
+
+              @background_thread ||= Thread.new { run_background }
+            end
+
+            self
+          end
+
           def stop
             synchronize do
-              @stopped = true
+              push_batch_request!
 
-              # Stop any background activity, clean up happens in wait!
-              @background_thread.kill if @background_thread
+              @stopped = true
+              @cond.broadcast
             end
 
-            push_batch_request!
+            self
           end
 
           def started?
@@ -125,19 +129,20 @@ module Google
             synchronize do
               until @stopped
                 if @batch.nil?
-                  @cond.wait
+                  @cond.wait # wait until broadcast
                   next
                 end
 
-                time_since_batch_creation = Time.now - @batch_created_at
-                if time_since_batch_creation > @interval
+                time_from_batch_creation = Time.now - @batch.created_at
+                time_until_next_push = @interval - time_from_batch_creation
+
+                if time_until_next_push <= 0
                   # interval met, publish the batch...
                   push_batch_request!
-                  @cond.wait
-                else
-                  # still waiting for the interval to publish the batch...
-                  @cond.wait(@interval - time_since_batch_creation)
+                  time_until_next_push = nil # wait until broadcast
                 end
+
+                @cond.wait time_until_next_push
               end
             end
           end
@@ -148,16 +153,16 @@ module Google
             @stream.push @batch.request
 
             @batch = nil
-            @batch_created_at = nil
           end
 
           class Batch
-            attr_reader :max_bytes, :request
+            attr_reader :max_bytes, :request, :created_at
 
             def initialize max_bytes: 10000000
               @max_bytes = max_bytes
               @request = Google::Pubsub::V1::StreamingPullRequest.new
               @total_message_bytes = 0
+              @created_at = Time.now
             end
 
             def ack ack_id

data/lib/google/cloud/pubsub/subscriber/async_unary_pusher.rb

@@ -59,9 +59,6 @@ module Google
                   end
                 end
 
-                @batch_created_at ||= Time.now
-                @background_thread ||= Thread.new { run_background }
-
                 push_batch_request! if @batch.ready?
               end
 
@@ -88,9 +85,6 @@ module Google
                   end
                 end
 
-                @batch_created_at ||= Time.now
-                @background_thread ||= Thread.new { run_background }
-
                 push_batch_request! if @batch.ready?
               end
 
@@ -100,15 +94,25 @@ module Google
             nil
           end
 
+          def start
+            synchronize do
+              @stopped = false
+
+              @background_thread ||= Thread.new { run_background }
+            end
+
+            self
+          end
+
           def stop
             synchronize do
-              @stopped = true
+              push_batch_request!
 
-              # Stop any background activity, clean up happens in wait!
-              @background_thread.kill if @background_thread
+              @stopped = true
+              @cond.broadcast
             end
 
-            push_batch_request!
+            self
           end
 
           def started?
@@ -125,19 +129,20 @@ module Google
             synchronize do
               until @stopped
                 if @batch.nil?
-                  @cond.wait
+                  @cond.wait # wait until broadcast
                   next
                 end
 
-                time_since_batch_creation = Time.now - @batch_created_at
-                if time_since_batch_creation > @interval
+                time_from_batch_creation = Time.now - @batch.created_at
+                time_until_next_push = @interval - time_from_batch_creation
+
+                if time_until_next_push <= 0
                   # interval met, publish the batch...
                   push_batch_request!
-                  @cond.wait
-                else
-                  # still waiting for the interval to publish the batch...
-                  @cond.wait(@interval - time_since_batch_creation)
+                  time_until_next_push = nil # wait until broadcast
                 end
+
+                @cond.wait time_until_next_push
               end
             end
           end
@@ -145,34 +150,48 @@ module Google
           def push_batch_request!
             return unless @batch
 
+            sub_name = @stream.subscriber.subscription_name
             service = @stream.subscriber.service
-            name = @stream.subscriber.subscription_name
 
             if @batch.ack?
               ack_ids = @batch.ack_ids
-              Concurrent::Future.new(executor: @stream.push_thread_pool) do
-                service.acknowledge name, *ack_ids
-              end.execute
+              push_change_request do
+                service.acknowledge sub_name, *ack_ids
+              end
             end
             if @batch.delay?
               @batch.modify_deadline_hash.each do |delay_seconds, delay_ack_ids|
-                Concurrent::Future.new(executor: @stream.push_thread_pool) do
-                  service.modify_ack_deadline name, delay_ack_ids, delay_seconds
-                end.execute
+                push_change_request do
+                  service.modify_ack_deadline sub_name,
+                                              delay_ack_ids,
+                                              delay_seconds
+                end
               end
             end
 
             @batch = nil
-            @batch_created_at = nil
+          end
+
+          def push_change_request
+            return unless @stream.push_thread_pool.running?
+
+            Concurrent::Future.new(executor: @stream.push_thread_pool) do
+              begin
+                yield
+              rescue StandardError => push_error
+                @stream.subscriber.error! push_error
+              end
+            end.execute
           end
 
           class Batch
-            attr_reader :max_bytes, :request
+            attr_reader :max_bytes, :request, :created_at
 
             def initialize max_bytes: 10000000
               @max_bytes = max_bytes
               @request = Google::Pubsub::V1::StreamingPullRequest.new
               @total_message_bytes = 0
+              @created_at = Time.now
             end
 
             def ack ack_id

data/lib/google/cloud/pubsub/subscriber/inventory.rb

@@ -0,0 +1,136 @@
+# Copyright 2017 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+require "monitor"
+
+module Google
+  module Cloud
+    module Pubsub
+      class Subscriber
+        ##
+        # @private
+        class Inventory
+          include MonitorMixin
+
+          attr_reader :stream, :limit
+
+          def initialize stream, limit
+            @stream = stream
+            @limit = limit
+            @_ack_ids = []
+            @wait_cond = new_cond
+
+            super()
+          end
+
+          def ack_ids
+            @_ack_ids
+          end
+
+          def add *ack_ids
+            ack_ids.flatten!.compact!
+            return if ack_ids.empty?
+
+            synchronize do
+              @_ack_ids += ack_ids
+              @wait_cond.broadcast
+            end
+          end
+
+          def remove *ack_ids
+            ack_ids.flatten!.compact!
+            return if ack_ids.empty?
+
+            synchronize do
+              @_ack_ids -= ack_ids
+              @wait_cond.broadcast
+            end
+          end
+
+          def count
+            synchronize do
+              @_ack_ids.count
+            end
+          end
+
+          def empty?
+            synchronize do
+              @_ack_ids.empty?
+            end
+          end
+
+          def start
+            @background_thread ||= Thread.new { background_run }
+
+            self
+          end
+
+          def stop
+            synchronize do
+              @stopped = true
+              @wait_cond.broadcast
+            end
+
+            self
+          end
+
+          def stopped?
+            synchronize { @stopped }
+          end
+
+          def full?
+            count >= limit
+          end
+
+          protected
+
+          def background_run
+            delay_target = nil
+
+            synchronize do
+              until @stopped
+                if @_ack_ids.empty?
+                  delay_target = nil
+
+                  @wait_cond.wait # wait until broadcast
+                  next
+                end
+
+                delay_target ||= calc_target
+                delay_gap = delay_target - Time.now
+
+                unless delay_gap.positive?
+                  delay_target = nil
+                  delay_gap = nil # wait until broadcast
+                  stream.delay_inventory!
+                end
+
+                @wait_cond.wait delay_gap
+              end
+            end
+          end
+
+          def calc_target
+            Time.now + calc_delay
+          end
+
+          def calc_delay
+            (stream.subscriber.deadline - 3) * rand(0.8..0.9)
+          end
+        end
+      end
+    end
+  end
+end