google-cloud-pubsub 0.31.1 → 0.32.0

data/lib/google/cloud/pubsub/subscriber/stream.rb

@@ -15,6 +15,7 @@
 
 require "google/cloud/pubsub/subscriber/async_unary_pusher"
 require "google/cloud/pubsub/subscriber/enumerator_queue"
+require "google/cloud/pubsub/subscriber/inventory"
 require "google/cloud/pubsub/service"
 require "google/cloud/errors"
 require "monitor"
@@ -67,7 +68,9 @@ module Google
 
           def start
             synchronize do
-              break if @request_queue
+              break if @background_thread
+
+              @inventory.start
 
               start_streaming!
             end
@@ -79,12 +82,26 @@ module Google
             synchronize do
               break if @stopped
 
+              # Close the stream by pushing the sentinel value.
+              # The unary pusher does not use the stream, so it can close here.
+              @request_queue.push self unless @request_queue.nil?
+
+              # Signal to the background thread that we are stopped.
               @stopped = true
+              @pause_cond.broadcast
+
+              # Now that the reception thread is stopped, immediately stop the
+              # callback thread pool and purge all pending callbacks.
+              @callback_thread_pool.kill
 
+              # Once all the callbacks are stopped, we can stop the inventory.
               @inventory.stop
 
-              # signal to the background thread that we are unpaused
-              @pause_cond.broadcast
+              # Stop the publisher and send the final batch of changes.
+              @async_pusher.stop if @async_pusher # will push current batch
+
+              # Stop the push thread pool now that the pusher is stopped.
+              @push_thread_pool.shutdown
             end
 
             self
@@ -100,21 +117,10 @@ module Google
 
           def wait!
             synchronize do
-              # Now that the reception thread is dead, make sure all recieved
-              # messages have had the callback called.
-              @callback_thread_pool.shutdown
-              @callback_thread_pool.wait_for_termination
-
-              # Once all the callbacks are complete, we can stop the publisher
-              # and send the final request to the steeam.
-              @async_pusher.stop if @async_pusher # will push current batch
-
-              # Close the push thread pool now that the pusher is closed.
-              @push_thread_pool.shutdown
-              @push_thread_pool.wait_for_termination
-
-              # Close the stream now that all requests have been made.
-              @request_queue.push self unless @request_queue.nil?
+              # # Wait for the push thread pool to finish pushing all remaining
+              # changes. Do not wait indefinitely.
+              @push_thread_pool.wait_for_termination 60
+              @push_thread_pool.kill if @push_thread_pool.shuttingdown?
             end
 
             self
@@ -127,7 +133,7 @@ module Google
             return true if ack_ids.empty?
 
             synchronize do
-              @async_pusher ||= AsyncUnaryPusher.new self
+              @async_pusher ||= AsyncUnaryPusher.new(self).start
               @async_pusher.acknowledge ack_ids
               @inventory.remove ack_ids
               unpause_streaming!
@@ -143,7 +149,7 @@ module Google
             return true if mod_ack_ids.empty?
 
             synchronize do
-              @async_pusher ||= AsyncUnaryPusher.new self
+              @async_pusher ||= AsyncUnaryPusher.new(self).start
               @async_pusher.delay deadline, mod_ack_ids
               @inventory.remove mod_ack_ids
               unpause_streaming!
@@ -170,7 +176,7 @@ module Google
             synchronize do
               return true if @inventory.empty?
 
-              @async_pusher ||= AsyncUnaryPusher.new self
+              @async_pusher ||= AsyncUnaryPusher.new(self).start
               @async_pusher.delay subscriber.deadline, @inventory.ack_ids
             end
 
@@ -190,9 +196,28 @@ module Google
 
           protected
 
+          # @private
+          class RestartStream < StandardError; end
+
           # rubocop:disable all
 
-          def background_run enum
+          def background_run
+            # Don't allow a stream to restart if already stopped
+            return if @stopped
+
+            # signal to the previous queue to shut down
+            old_queue = []
+            old_queue = @request_queue.quit_and_dump_queue if @request_queue
+
+            # Always create a new request queue and enum
+            @request_queue = EnumeratorQueue.new self
+            @request_queue.push initial_input_request
+            old_queue.each { |obj| @request_queue.push obj }
+            enum = subscriber.service.streaming_pull @request_queue.each
+
+            @stopped = nil
+            @paused  = nil
+
             loop do
               synchronize do
                 if @paused && !@stopped
@@ -213,7 +238,7 @@ module Google
 
                 synchronize do
                   # Create receipt of received messages reception
-                  @async_pusher ||= AsyncUnaryPusher.new self
+                  @async_pusher ||= AsyncUnaryPusher.new(self).start
                   @async_pusher.delay subscriber.deadline, received_ack_ids
 
                   # Add received messages to inventory
@@ -232,57 +257,56 @@ module Google
                 break
               end
             end
+
             # Has the loop broken but we aren't stopped?
             # Could be GRPC has thrown an internal error, so restart.
-            synchronize { raise "restart thread" unless @stopped }
-          rescue GRPC::DeadlineExceeded, GRPC::Unavailable, GRPC::Cancelled,
-                 GRPC::ResourceExhausted, GRPC::Internal, GRPC::Core::CallError
-            # The GAPIC layer will raise DeadlineExceeded when stream is opened
-            # longer than the timeout value it is configured for. When this
-            # happends, restart the stream stealthly.
-            # Also stealthly restart the stream on Unavailable, Cancelled,
-            # ResourceExhausted, and Internal.
-            # Also, also stealthly restart the stream when GRPC raises the
-            # internal CallError.
-            synchronize { start_streaming! }
+            raise RestartStream unless synchronize { @stopped }
+
+            # We must be stopped, tell the stream to quit.
+            @request_queue.push self
+          rescue GRPC::Cancelled, GRPC::DeadlineExceeded, GRPC::Internal,
+                 GRPC::ResourceExhausted, GRPC::Unauthenticated,
+                 GRPC::Unavailable, GRPC::Core::CallError
+            # Restart the stream with an incremental back for a retriable error.
+            # Also when GRPC raises the internal CallError.
+
+            retry
+          rescue RestartStream
+            retry
           rescue StandardError => e
             synchronize do
-              if @stopped
-                raise Google::Cloud::Error.from_error(e)
-              else
-                start_streaming!
-              end
+              subscriber.error! e
+              start_streaming! unless @stopped
             end
+
+            retry
           end
 
           # rubocop:enable all
 
           def perform_callback_async rec_msg
+            return unless callback_thread_pool.running?
+
             Concurrent::Future.new(executor: callback_thread_pool) do
-              subscriber.callback.call rec_msg
+              begin
+                subscriber.callback.call rec_msg
+              rescue StandardError => callback_error
+                subscriber.error! callback_error
+              end
             end.execute
           end
 
           def start_streaming!
-            # Don't allow a stream to restart if already stopped
-            return if @stopped
-
-            # signal to the previous queue to shut down
-            old_queue = []
-            old_queue = @request_queue.quit_and_dump_queue if @request_queue
-
-            @request_queue = EnumeratorQueue.new self
-            @request_queue.push initial_input_request
-            old_queue.each { |obj| @request_queue.push obj }
-            output_enum = subscriber.service.streaming_pull @request_queue.each
+            # A Stream will only ever have one background thread. If the thread
+            # dies because it was stopped, or because of an unhandled error that
+            # could not be recovered from, so be it.
+            return if @background_thread
 
-            @stopped = nil
-            @paused  = nil
+            @stopped = false
+            @paused  = false
 
             # create new background thread to handle new enumerator
-            @background_thread = Thread.new(output_enum) do |enum|
-              background_run enum
-            end
+            @background_thread = Thread.new { background_run }
           end
 
           def pause_streaming!
@@ -340,88 +364,6 @@ module Google
             return "stopped" if status == false
             status
           end
-
-          ##
-          # @private
-          class Inventory
-            include MonitorMixin
-
-            attr_reader :stream, :limit
-
-            def initialize stream, limit
-              @stream = stream
-              @limit = limit
-              @_ack_ids = []
-              @wait_cond = new_cond
-
-              super()
-            end
-
-            def ack_ids
-              @_ack_ids
-            end
-
-            def add *ack_ids
-              ack_ids = Array(ack_ids).flatten
-              synchronize do
-                @_ack_ids += ack_ids
-                unless @stopped
-                  @background_thread ||= Thread.new { background_run }
-                end
-              end
-            end
-
-            def remove *ack_ids
-              ack_ids = Array(ack_ids).flatten
-              synchronize do
-                @_ack_ids -= ack_ids
-                if @_ack_ids.empty?
-                  if @background_thread
-                    @background_thread.kill
-                    @background_thread = nil
-                  end
-                end
-              end
-            end
-
-            def count
-              synchronize do
-                @_ack_ids.count
-              end
-            end
-
-            def empty?
-              synchronize do
-                @_ack_ids.empty?
-              end
-            end
-
-            def stop
-              synchronize do
-                @stopped = true
-                @background_thread.kill if @background_thread
-              end
-            end
-
-            def full?
-              count >= limit
-            end
-
-            protected
-
-            def background_run
-              until synchronize { @stopped }
-                delay = calc_delay
-                synchronize { @wait_cond.wait delay }
-
-                stream.delay_inventory!
-              end
-            end
-
-            def calc_delay
-              (stream.subscriber.deadline - 3) * rand(0.8..0.9)
-            end
-          end
         end
       end
     end

data/lib/google/cloud/pubsub/subscription.rb

@@ -108,8 +108,8 @@ module Google
         ##
         # Indicates whether to retain acknowledged messages. If `true`, then
         # messages are not expunged from the subscription's backlog, even if
-        # they are acknowledged, until they fall out of the
-        # {#retention_duration} window. Default is `false`.
+        # they are acknowledged, until they fall out of the {#retention} window.
+        # Default is `false`.
         #
         # @return [Boolean] Returns `true` if acknowledged messages are
         #   retained.

data/lib/google/cloud/pubsub/topic.rb

@@ -121,14 +121,14 @@ module Google
         # @param [Boolean] retain_acked Indicates whether to retain acknowledged
         #   messages. If `true`, then messages are not expunged from the
         #   subscription's backlog, even if they are acknowledged, until they
-        #   fall out of the `retention_duration` window. Default is `false`.
+        #   fall out of the `retention` window. Default is `false`.
         # @param [Numeric] retention How long to retain unacknowledged messages
         #   in the subscription's backlog, from the moment a message is
         #   published. If `retain_acked` is `true`, then this also configures
         #   the retention of acknowledged messages, and thus configures how far
-        #   back in time a {#seek} can be done. Cannot be more than 604,800
-        #   seconds (7 days) or less than 600 seconds (10 minutes). Default is
-        #   604,800 seconds (7 days).
+        #   back in time a {Subscription#seek} can be done. Cannot be more than
+        #   604,800 seconds (7 days) or less than 600 seconds (10 minutes).
+        #   Default is 604,800 seconds (7 days).
         # @param [String] endpoint A URL locating the endpoint to which messages
         #   should be pushed.
         #

data/lib/google/cloud/pubsub/v1/credentials.rb

@@ -0,0 +1,38 @@
+# Copyright 2018 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require "googleauth"
+
+module Google
+  module Cloud
+    module Pubsub
+      module V1
+        class Credentials < Google::Auth::Credentials
+          SCOPE = ["https://www.googleapis.com/auth/pubsub"].freeze
+          PATH_ENV_VARS = %w(PUBSUB_CREDENTIALS
+                             PUBSUB_KEYFILE
+                             GOOGLE_CLOUD_CREDENTIALS
+                             GOOGLE_CLOUD_KEYFILE
+                             GCLOUD_KEYFILE)
+          JSON_ENV_VARS = %w(PUBSUB_CREDENTIALS_JSON
+                             PUBSUB_KEYFILE_JSON
+                             GOOGLE_CLOUD_CREDENTIALS_JSON
+                             GOOGLE_CLOUD_KEYFILE_JSON
+                             GCLOUD_KEYFILE_JSON)
+          DEFAULT_PATHS = ["~/.config/gcloud/application_default_credentials.json"]
+        end
+      end
+    end
+  end
+end

data/lib/google/cloud/pubsub/v1/doc/google/iam/v1/iam_policy.rb

@@ -0,0 +1,62 @@
+# Copyright 2018 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+module Google
+  module Iam
+    module V1
+      # Request message for +SetIamPolicy+ method.
+      # @!attribute [rw] resource
+      #   @return [String]
+      #     REQUIRED: The resource for which the policy is being specified.
+      #     +resource+ is usually specified as a path. For example, a Project
+      #     resource is specified as +projects/{project}+.
+      # @!attribute [rw] policy
+      #   @return [Google::Iam::V1::Policy]
+      #     REQUIRED: The complete policy to be applied to the +resource+. The size of
+      #     the policy is limited to a few 10s of KB. An empty policy is a
+      #     valid policy but certain Cloud Platform services (such as Projects)
+      #     might reject them.
+      class SetIamPolicyRequest; end
+
+      # Request message for +GetIamPolicy+ method.
+      # @!attribute [rw] resource
+      #   @return [String]
+      #     REQUIRED: The resource for which the policy is being requested.
+      #     +resource+ is usually specified as a path. For example, a Project
+      #     resource is specified as +projects/{project}+.
+      class GetIamPolicyRequest; end
+
+      # Request message for +TestIamPermissions+ method.
+      # @!attribute [rw] resource
+      #   @return [String]
+      #     REQUIRED: The resource for which the policy detail is being requested.
+      #     +resource+ is usually specified as a path. For example, a Project
+      #     resource is specified as +projects/{project}+.
+      # @!attribute [rw] permissions
+      #   @return [Array<String>]
+      #     The set of permissions to check for the +resource+. Permissions with
+      #     wildcards (such as '*' or 'storage.*') are not allowed. For more
+      #     information see
+      #     [IAM Overview](https://cloud.google.com/iam/docs/overview#permissions).
+      class TestIamPermissionsRequest; end
+
+      # Response message for +TestIamPermissions+ method.
+      # @!attribute [rw] permissions
+      #   @return [Array<String>]
+      #     A subset of +TestPermissionsRequest.permissions+ that the caller is
+      #     allowed.
+      class TestIamPermissionsResponse; end
+    end
+  end
+end

data/lib/google/cloud/pubsub/v1/doc/google/iam/v1/policy.rb

@@ -0,0 +1,127 @@
+# Copyright 2018 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+module Google
+  module Iam
+    module V1
+      # Defines an Identity and Access Management (IAM) policy. It is used to
+      # specify access control policies for Cloud Platform resources.
+      #
+      #
+      # A +Policy+ consists of a list of +bindings+. A +Binding+ binds a list of
+      # +members+ to a +role+, where the members can be user accounts, Google groups,
+      # Google domains, and service accounts. A +role+ is a named list of permissions
+      # defined by IAM.
+      #
+      # **Example**
+      #
+      #     {
+      #       "bindings": [
+      #         {
+      #           "role": "roles/owner",
+      #           "members": [
+      #             "user:mike@example.com",
+      #             "group:admins@example.com",
+      #             "domain:google.com",
+      #             "serviceAccount:my-other-app@appspot.gserviceaccount.com",
+      #           ]
+      #         },
+      #         {
+      #           "role": "roles/viewer",
+      #           "members": ["user:sean@example.com"]
+      #         }
+      #       ]
+      #     }
+      #
+      # For a description of IAM and its features, see the
+      # [IAM developer's guide](https://cloud.google.com/iam).
+      # @!attribute [rw] version
+      #   @return [Integer]
+      #     Version of the +Policy+. The default version is 0.
+      # @!attribute [rw] bindings
+      #   @return [Array<Google::Iam::V1::Binding>]
+      #     Associates a list of +members+ to a +role+.
+      #     Multiple +bindings+ must not be specified for the same +role+.
+      #     +bindings+ with no members will result in an error.
+      # @!attribute [rw] etag
+      #   @return [String]
+      #     +etag+ is used for optimistic concurrency control as a way to help
+      #     prevent simultaneous updates of a policy from overwriting each other.
+      #     It is strongly suggested that systems make use of the +etag+ in the
+      #     read-modify-write cycle to perform policy updates in order to avoid race
+      #     conditions: An +etag+ is returned in the response to +getIamPolicy+, and
+      #     systems are expected to put that etag in the request to +setIamPolicy+ to
+      #     ensure that their change will be applied to the same version of the policy.
+      #
+      #     If no +etag+ is provided in the call to +setIamPolicy+, then the existing
+      #     policy is overwritten blindly.
+      class Policy; end
+
+      # Associates +members+ with a +role+.
+      # @!attribute [rw] role
+      #   @return [String]
+      #     Role that is assigned to +members+.
+      #     For example, +roles/viewer+, +roles/editor+, or +roles/owner+.
+      #     Required
+      # @!attribute [rw] members
+      #   @return [Array<String>]
+      #     Specifies the identities requesting access for a Cloud Platform resource.
+      #     +members+ can have the following values:
+      #
+      #     * +allUsers+: A special identifier that represents anyone who is
+      #       on the internet; with or without a Google account.
+      #
+      #     * +allAuthenticatedUsers+: A special identifier that represents anyone
+      #       who is authenticated with a Google account or a service account.
+      #
+      #     * +user:{emailid}+: An email address that represents a specific Google
+      #       account. For example, +alice@gmail.com+ or +joe@example.com+.
+      #
+      #
+      #     * +serviceAccount:{emailid}+: An email address that represents a service
+      #       account. For example, +my-other-app@appspot.gserviceaccount.com+.
+      #
+      #     * +group:{emailid}+: An email address that represents a Google group.
+      #       For example, +admins@example.com+.
+      #
+      #     * +domain:{domain}+: A Google Apps domain name that represents all the
+      #       users of that domain. For example, +google.com+ or +example.com+.
+      class Binding; end
+
+      # The difference delta between two policies.
+      # @!attribute [rw] binding_deltas
+      #   @return [Array<Google::Iam::V1::BindingDelta>]
+      #     The delta for Bindings between two policies.
+      class PolicyDelta; end
+
+      # One delta entry for Binding. Each individual change (only one member in each
+      # entry) to a binding will be a separate entry.
+      # @!attribute [rw] action
+      #   @return [Google::Iam::V1::BindingDelta::Action]
+      #     The action that was performed on a Binding.
+      #     Required
+      # @!attribute [rw] role
+      #   @return [String]
+      #     Role that is assigned to +members+.
+      #     For example, +roles/viewer+, +roles/editor+, or +roles/owner+.
+      #     Required
+      # @!attribute [rw] member
+      #   @return [String]
+      #     A single identity requesting access for a Cloud Platform resource.
+      #     Follows the same format of Binding.members.
+      #     Required
+      class BindingDelta; end
+    end
+  end
+end