google-cloud-pubsub 1.1.3 → 2.0.0

data/lib/google/cloud/pubsub/service.rb

@@ -18,56 +18,43 @@ require "google/cloud/pubsub/credentials"
 require "google/cloud/pubsub/convert"
 require "google/cloud/pubsub/version"
 require "google/cloud/pubsub/v1"
-require "google/gax/errors"
+require "securerandom"
 
 module Google
   module Cloud
     module PubSub
       ##
-      # @private Represents the GAX Pub/Sub service, including all the API
-      # methods.
+      # @private Represents the Pub/Sub service API, including IAM mixins.
       class Service
-        attr_accessor :project, :credentials, :host, :timeout, :client_config
+        attr_accessor :project, :credentials, :host, :timeout
+        ###
+        # The same client_id is used across all streaming pull connections that are created by this client. This is
+        # intentional, as it indicates to the server that any guarantees, such as message ordering, made for a stream
+        # that is disconnected will be made for the stream that is created to replace it. The attr_accessor allows the
+        # value to be replaced for unit testing.
+        attr_accessor :client_id
 
         ##
         # Creates a new Service instance.
-        def initialize project, credentials, host: nil, timeout: nil,
-                       client_config: nil
+        def initialize project, credentials, host: nil, timeout: nil
           @project = project
           @credentials = credentials
-          @host = host || V1::PublisherClient::SERVICE_ADDRESS
+          @host = host
           @timeout = timeout
-          @client_config = client_config || {}
-        end
-
-        def channel
-          require "grpc"
-          GRPC::Core::Channel.new host, chan_args, chan_creds
-        end
-
-        def chan_args
-          { "grpc.max_send_message_length"           => -1,
-            "grpc.max_receive_message_length"        => -1,
-            "grpc.keepalive_time_ms"                 => 300_000,
-            "grpc.service_config_disable_resolution" => 1 }
-        end
-
-        def chan_creds
-          return credentials if insecure?
-          require "grpc"
-          GRPC::Core::ChannelCredentials.new.compose GRPC::Core::CallCredentials.new credentials.client.updater_proc
+          @client_id = SecureRandom.uuid.freeze
         end
 
         def subscriber
           return mocked_subscriber if mocked_subscriber
           @subscriber ||= begin
-            V1::SubscriberClient.new(
-              credentials:   channel,
-              timeout:       timeout,
-              client_config: client_config,
-              lib_name:      "gccl",
-              lib_version:   Google::Cloud::PubSub::VERSION
-            )
+            V1::Subscriber::Client.new do |config|
+              config.credentials = credentials if credentials
+              config.timeout = timeout if timeout
+              config.endpoint = host if host
+              config.lib_name = "gccl"
+              config.lib_version = Google::Cloud::PubSub::VERSION
+              config.metadata = { "google-cloud-resource-prefix": "projects/#{@project}" }
+            end
           end
         end
         attr_accessor :mocked_subscriber
@@ -75,20 +62,32 @@ module Google
         def publisher
           return mocked_publisher if mocked_publisher
           @publisher ||= begin
-            V1::PublisherClient.new(
-              credentials:   channel,
-              timeout:       timeout,
-              client_config: client_config,
-              lib_name:      "gccl",
-              lib_version:   Google::Cloud::PubSub::VERSION
-            )
+            V1::Publisher::Client.new do |config|
+              config.credentials = credentials if credentials
+              config.timeout = timeout if timeout
+              config.endpoint = host if host
+              config.lib_name = "gccl"
+              config.lib_version = Google::Cloud::PubSub::VERSION
+              config.metadata = { "google-cloud-resource-prefix": "projects/#{@project}" }
+            end
           end
         end
         attr_accessor :mocked_publisher
 
-        def insecure?
-          credentials == :this_channel_is_insecure
+        def iam
+          return mocked_iam if mocked_iam
+          @iam ||= begin
+            V1::IAMPolicy::Client.new do |config|
+              config.credentials = credentials if credentials
+              config.timeout = timeout if timeout
+              config.endpoint = host if host
+              config.lib_name = "gccl"
+              config.lib_version = Google::Cloud::PubSub::VERSION
+              config.metadata = { "google-cloud-resource-prefix" => "projects/#{@project}" }
+            end
+          end
         end
+        attr_accessor :mocked_iam
 
         ##
         # Gets the configuration of a topic.
@@ -97,30 +96,17 @@ module Google
         # If other attributes are added in the future,
         # they will be returned here.
         def get_topic topic_name, options = {}
-          execute do
-            publisher.get_topic topic_path(topic_name, options),
-                                options: default_options
-          end
+          publisher.get_topic topic: topic_path(topic_name, options)
         end
 
         ##
         # Lists matching topics.
         def list_topics options = {}
-          call_options = default_options
-          if (token = options[:token])
-            call_options = Google::Gax::CallOptions.new(
-              kwargs:     default_headers,
-              page_token: token
-            )
-          end
+          paged_enum = publisher.list_topics project:    project_path(options),
+                                             page_size:  options[:max],
+                                             page_token: options[:token]
 
-          execute do
-            paged_enum = publisher.list_topics project_path(options),
-                                               page_size: options[:max],
-                                               options:   call_options
-
-            paged_enum.page.response
-          end
+          paged_enum.response
         end
 
         ##
@@ -132,21 +118,16 @@ module Google
             }
           end
 
-          execute do
-            publisher.create_topic \
-              topic_path(topic_name, options),
-              labels:                 labels,
-              kms_key_name:           kms_key_name,
-              message_storage_policy: message_storage_policy,
-              options:                default_options
-          end
+          publisher.create_topic \
+            name:                   topic_path(topic_name, options),
+            labels:                 labels,
+            kms_key_name:           kms_key_name,
+            message_storage_policy: message_storage_policy
         end
 
         def update_topic topic_obj, *fields
           mask = Google::Protobuf::FieldMask.new paths: fields.map(&:to_s)
-          execute do
-            publisher.update_topic topic_obj, mask, options: default_options
-          end
+          publisher.update_topic topic: topic_obj, update_mask: mask
         end
 
         ##
@@ -155,10 +136,7 @@ module Google
         # exist. After a topic is deleted, a new topic may be created with the
         # same name.
         def delete_topic topic_name
-          execute do
-            publisher.delete_topic topic_path(topic_name),
-                                   options: default_options
-          end
+          publisher.delete_topic topic: topic_path(topic_name)
         end
 
         ##
@@ -167,142 +145,98 @@ module Google
         # The messages parameter is an array of arrays.
         # The first element is the data, second is attributes hash.
         def publish topic, messages
-          execute do
-            publisher.publish topic_path(topic), messages,
-                              options: default_options
-          end
+          publisher.publish topic: topic_path(topic), messages: messages
         end
 
         ##
         # Gets the details of a subscription.
         def get_subscription subscription_name, options = {}
-          subscription = subscription_path subscription_name, options
-          execute do
-            subscriber.get_subscription subscription, options: default_options
-          end
+          subscriber.get_subscription subscription: subscription_path(subscription_name, options)
         end
 
         ##
         # Lists matching subscriptions by project and topic.
         def list_topics_subscriptions topic, options = {}
-          call_options = default_options
-          if (token = options[:token])
-            call_options = Google::Gax::CallOptions.new(
-              kwargs:     default_headers,
-              page_token: token
-            )
-          end
-
-          execute do
-            paged_enum = publisher.list_topic_subscriptions \
-              topic_path(topic, options),
-              page_size: options[:max],
-              options:   call_options
-
-            paged_enum.page.response
-          end
+          publisher.list_topic_subscriptions topic:      topic_path(topic, options),
+                                             page_size:  options[:max],
+                                             page_token: options[:token]
         end
 
         ##
         # Lists matching subscriptions by project.
         def list_subscriptions options = {}
-          call_options = default_options
-          if (token = options[:token])
-            call_options = Google::Gax::CallOptions.new(
-              kwargs:     default_headers,
-              page_token: token
-            )
-          end
-
-          execute do
-            paged_enum = subscriber.list_subscriptions project_path(options),
-                                                       page_size: options[:max],
-                                                       options:   call_options
+          paged_enum = subscriber.list_subscriptions project:    project_path(options),
+                                                     page_size:  options[:max],
+                                                     page_token: options[:token]
 
-            paged_enum.page.response
-          end
+          paged_enum.response
         end
 
         ##
         # Creates a subscription on a given topic for a given subscriber.
         def create_subscription topic, subscription_name, options = {}
-          name = subscription_path subscription_name, options
-          topic = topic_path topic
           push_config = if options[:endpoint]
                           Google::Cloud::PubSub::V1::PushConfig.new \
                             push_endpoint: options[:endpoint],
                             attributes:    (options[:attributes] || {}).to_h
                         end
-          deadline = options[:deadline]
-          retain_acked = options[:retain_acked]
-          mrd = Convert.number_to_duration options[:retention]
-          labels = options[:labels]
-          message_ordering = options[:message_ordering]
-
-          execute do
-            subscriber.create_subscription \
-              name, topic,
-              push_config:                push_config,
-              ack_deadline_seconds:       deadline,
-              retain_acked_messages:      retain_acked,
-              message_retention_duration: mrd,
-              labels:                     labels,
-              enable_message_ordering:    message_ordering,
-              options:                    default_options
-          end
+          subscriber.create_subscription \
+            name:                       subscription_path(subscription_name, options),
+            topic:                      topic_path(topic),
+            push_config:                push_config,
+            ack_deadline_seconds:       options[:deadline],
+            retain_acked_messages:      options[:retain_acked],
+            message_retention_duration: Convert.number_to_duration(options[:retention]),
+            labels:                     options[:labels],
+            enable_message_ordering:    options[:message_ordering],
+            filter:                     options[:filter],
+            dead_letter_policy:         dead_letter_policy(options),
+            retry_policy:               options[:retry_policy]
         end
 
         def update_subscription subscription_obj, *fields
           mask = Google::Protobuf::FieldMask.new paths: fields.map(&:to_s)
-          execute do
-            subscriber.update_subscription subscription_obj, mask, options: default_options
-          end
+          subscriber.update_subscription subscription: subscription_obj, update_mask: mask
         end
 
         ##
-        # Deletes an existing subscription.
-        # All pending messages in the subscription are immediately dropped.
+        # Deletes an existing subscription. All pending messages in the subscription are immediately dropped.
         def delete_subscription subscription
-          execute do
-            subscriber.delete_subscription subscription_path(subscription),
-                                           options: default_options
-          end
+          subscriber.delete_subscription subscription: subscription_path(subscription)
+        end
+
+        ##
+        # Detaches a subscription from its topic. All messages retained in the subscription are dropped. Subsequent
+        # `Pull` and `StreamingPull` requests will raise `FAILED_PRECONDITION`. If the subscription is a push
+        # subscription, pushes to the endpoint will stop.
+        def detach_subscription subscription
+          publisher.detach_subscription subscription: subscription_path(subscription)
         end
 
         ##
         # Pulls a single message from the server.
         def pull subscription, options = {}
-          subscription = subscription_path subscription, options
           max_messages = options.fetch(:max, 100).to_i
           return_immediately = !(!options.fetch(:immediate, true))
 
-          execute do
-            subscriber.pull subscription,
-                            max_messages,
-                            return_immediately: return_immediately,
-                            options:            default_options
-          end
+          subscriber.pull subscription:       subscription_path(subscription, options),
+                          max_messages:       max_messages,
+                          return_immediately: return_immediately
         end
 
         def streaming_pull request_enum
-          execute do
-            subscriber.streaming_pull request_enum, options: default_options
-          end
+          subscriber.streaming_pull request_enum
         end
 
         ##
         # Acknowledges receipt of a message.
         def acknowledge subscription, *ack_ids
-          execute do
-            subscriber.acknowledge subscription_path(subscription), ack_ids,
-                                   options: default_options
-          end
+          subscriber.acknowledge subscription: subscription_path(subscription), ack_ids: ack_ids
         end
 
         ##
         # Modifies the PushConfig for a specified subscription.
         def modify_push_config subscription, endpoint, attributes
-          subscription = subscription_path subscription
           # Convert attributes to strings to match the protobuf definition
           attributes = Hash[attributes.map { |k, v| [String(k), String(v)] }]
           push_config = Google::Cloud::PubSub::V1::PushConfig.new(
@@ -310,138 +244,83 @@ module Google
             attributes:    attributes
           )
 
-          execute do
-            subscriber.modify_push_config subscription, push_config,
-                                          options: default_options
-          end
+          subscriber.modify_push_config subscription: subscription_path(subscription),
+                                        push_config:  push_config
         end
 
         ##
         # Modifies the ack deadline for a specific message.
         def modify_ack_deadline subscription, ids, deadline
-          execute do
-            subscriber.modify_ack_deadline subscription_path(subscription),
-                                           Array(ids),
-                                           deadline, options: default_options
-          end
+          subscriber.modify_ack_deadline subscription:         subscription_path(subscription),
+                                         ack_ids:              Array(ids),
+                                         ack_deadline_seconds: deadline
         end
 
         ##
         # Lists snapshots by project.
         def list_snapshots options = {}
-          call_options = default_options
-          if (token = options[:token])
-            call_options = Google::Gax::CallOptions.new(
-              kwargs:     default_headers,
-              page_token: token
-            )
-          end
+          paged_enum = subscriber.list_snapshots project:    project_path(options),
+                                                 page_size:  options[:max],
+                                                 page_token: options[:token]
 
-          execute do
-            paged_enum = subscriber.list_snapshots project_path(options),
-                                                   page_size: options[:max],
-                                                   options:   call_options
-
-            paged_enum.page.response
-          end
+          paged_enum.response
         end
 
         ##
         # Creates a snapshot on a given subscription.
         def create_snapshot subscription, snapshot_name, labels: nil
-          name = snapshot_path snapshot_name
-          execute do
-            subscriber.create_snapshot name,
-                                       subscription_path(subscription),
-                                       labels:  labels,
-                                       options: default_options
-          end
+          subscriber.create_snapshot name:         snapshot_path(snapshot_name),
+                                     subscription: subscription_path(subscription),
+                                     labels:       labels
         end
 
         def update_snapshot snapshot_obj, *fields
           mask = Google::Protobuf::FieldMask.new paths: fields.map(&:to_s)
-          execute do
-            subscriber.update_snapshot snapshot_obj, mask, options: default_options
-          end
+          subscriber.update_snapshot snapshot: snapshot_obj, update_mask: mask
         end
 
         ##
         # Deletes an existing snapshot.
         # All pending messages in the snapshot are immediately dropped.
         def delete_snapshot snapshot
-          execute do
-            subscriber.delete_snapshot snapshot_path(snapshot),
-                                       options: default_options
-          end
+          subscriber.delete_snapshot snapshot: snapshot_path(snapshot)
         end
 
         ##
         # Adjusts the given subscription to a time or snapshot.
         def seek subscription, time_or_snapshot
-          subscription = subscription_path subscription
-          execute do
-            if a_time? time_or_snapshot
-              time = Convert.time_to_timestamp time_or_snapshot
-              subscriber.seek subscription, time: time, options: default_options
-            else
-              time_or_snapshot = time_or_snapshot.name if time_or_snapshot.is_a? Snapshot
-              subscriber.seek subscription,
-                              snapshot: snapshot_path(time_or_snapshot),
-                              options:  default_options
-            end
+          if a_time? time_or_snapshot
+            time = Convert.time_to_timestamp time_or_snapshot
+            subscriber.seek subscription: subscription, time: time
+          else
+            time_or_snapshot = time_or_snapshot.name if time_or_snapshot.is_a? Snapshot
+            subscriber.seek subscription: subscription_path(subscription),
+                            snapshot:     snapshot_path(time_or_snapshot)
           end
         end
 
         def get_topic_policy topic_name, options = {}
-          execute do
-            publisher.get_iam_policy topic_path(topic_name, options),
-                                     options: default_options
-          end
+          iam.get_iam_policy resource: topic_path(topic_name, options)
         end
 
         def set_topic_policy topic_name, new_policy, options = {}
-          resource = topic_path topic_name, options
-
-          execute do
-            publisher.set_iam_policy resource, new_policy,
-                                     options: default_options
-          end
+          iam.set_iam_policy resource: topic_path(topic_name, options), policy: new_policy
         end
 
         def test_topic_permissions topic_name, permissions, options = {}
-          resource = topic_path topic_name, options
-
-          execute do
-            publisher.test_iam_permissions resource, permissions,
-                                           options: default_options
-          end
+          iam.test_iam_permissions resource: topic_path(topic_name, options), permissions: permissions
         end
 
         def get_subscription_policy subscription_name, options = {}
-          resource = subscription_path subscription_name, options
-
-          execute do
-            subscriber.get_iam_policy resource, options: default_options
-          end
+          iam.get_iam_policy resource: subscription_path(subscription_name, options)
         end
 
         def set_subscription_policy subscription_name, new_policy, options = {}
-          resource = subscription_path subscription_name, options
-
-          execute do
-            subscriber.set_iam_policy resource, new_policy,
-                                      options: default_options
-          end
+          iam.set_iam_policy resource: subscription_path(subscription_name, options), policy: new_policy
         end
 
-        def test_subscription_permissions subscription_name,
-                                          permissions, options = {}
-          resource = subscription_path subscription_name, options
-
-          execute do
-            subscriber.test_iam_permissions resource, permissions,
-                                            options: default_options
-          end
+        def test_subscription_permissions subscription_name, permissions, options = {}
+          iam.test_iam_permissions resource: subscription_path(subscription_name, options), permissions: permissions
         end
 
         def project_path options = {}
@@ -477,19 +356,13 @@ module Google
           true
         end
 
-        def default_headers
-          { "google-cloud-resource-prefix" => "projects/#{@project}" }
-        end
-
-        def default_options
-          Google::Gax::CallOptions.new kwargs: default_headers
-        end
-
-        def execute
-          yield
-        rescue Google::Gax::GaxError => e
-          # GaxError wraps BadStatus, but exposes it as #cause
-          raise Google::Cloud::Error.from_error(e.cause)
+        def dead_letter_policy options
+          return nil unless options[:dead_letter_topic_name]
+          policy = Google::Cloud::PubSub::V1::DeadLetterPolicy.new dead_letter_topic: options[:dead_letter_topic_name]
+          if options[:dead_letter_max_delivery_attempts]
+            policy.max_delivery_attempts = options[:dead_letter_max_delivery_attempts]
+          end
+          policy
         end
       end
     end

data/lib/google/cloud/pubsub/subscriber/inventory.rb

@@ -22,30 +22,40 @@ module Google
         ##
         # @private
         class Inventory
+          InventoryItem = Struct.new :bytesize, :pulled_at do
+            def self.from rec_msg
+              new rec_msg.to_proto.bytesize, Time.now
+            end
+          end
+
           include MonitorMixin
 
-          attr_reader :stream, :limit
+          attr_reader :stream, :limit, :bytesize, :extension, :max_duration_per_lease_extension
 
-          def initialize stream, limit
+          def initialize stream, limit:, bytesize:, extension:, max_duration_per_lease_extension:
             super()
-
             @stream = stream
             @limit = limit
-            @_ack_ids = []
+            @bytesize = bytesize
+            @extension = extension
+            @max_duration_per_lease_extension = max_duration_per_lease_extension
+            @inventory = {}
             @wait_cond = new_cond
           end
 
           def ack_ids
-            @_ack_ids
+            @inventory.keys
           end
 
-          def add *ack_ids
-            ack_ids.flatten!
-            ack_ids.compact!
-            return if ack_ids.empty?
+          def add *rec_msgs
+            rec_msgs.flatten!
+            rec_msgs.compact!
+            return if rec_msgs.empty?
 
             synchronize do
-              @_ack_ids += ack_ids
+              rec_msgs.each do |rec_msg|
+                @inventory[rec_msg.ack_id] = InventoryItem.from rec_msg
+              end
               @wait_cond.broadcast
             end
           end
@@ -56,20 +66,34 @@ module Google
             return if ack_ids.empty?
 
             synchronize do
-              @_ack_ids -= ack_ids
+              @inventory.delete_if { |ack_id, _| ack_ids.include? ack_id }
+              @wait_cond.broadcast
+            end
+          end
+
+          def remove_expired!
+            synchronize do
+              extension_time = Time.new - extension
+              @inventory.delete_if { |_ack_id, item| item.pulled_at < extension_time }
               @wait_cond.broadcast
             end
           end
 
           def count
             synchronize do
-              @_ack_ids.count
+              @inventory.count
+            end
+          end
+
+          def total_bytesize
+            synchronize do
+              @inventory.values.sum(&:bytesize)
             end
           end
 
           def empty?
             synchronize do
-              @_ack_ids.empty?
+              @inventory.empty?
             end
           end
 
@@ -93,7 +117,9 @@ module Google
           end
 
           def full?
-            count >= limit
+            synchronize do
+              @inventory.count >= limit || @inventory.values.sum(&:bytesize) >= bytesize
+            end
           end
 
           protected
@@ -127,7 +153,9 @@ module Google
           end
 
           def calc_delay
-            (stream.subscriber.deadline - 3) * rand(0.8..0.9)
+            delay = (stream.subscriber.deadline - 3) * rand(0.8..0.9)
+            delay = [delay, max_duration_per_lease_extension].min if max_duration_per_lease_extension.positive?
+            delay
           end
         end
       end