google-cloud-pubsub 2.3.2 → 2.7.0

data/lib/google/cloud/pubsub/service.rb

@@ -26,7 +26,10 @@ module Google
       ##
       # @private Represents the Pub/Sub service API, including IAM mixins.
       class Service
-        attr_accessor :project, :credentials, :host, :timeout
+        attr_accessor :project
+        attr_accessor :credentials
+        attr_accessor :host
+        attr_accessor :timeout
         ###
         # The same client_id is used across all streaming pull connections that are created by this client. This is
         # intentional, as it indicates to the server that any guarantees, such as message ordering, made for a stream
@@ -46,49 +49,56 @@ module Google
 
         def subscriber
           return mocked_subscriber if mocked_subscriber
-          @subscriber ||= begin
-            V1::Subscriber::Client.new do |config|
-              config.credentials = credentials if credentials
-              config.timeout = timeout if timeout
-              config.endpoint = host if host
-              config.lib_name = "gccl"
-              config.lib_version = Google::Cloud::PubSub::VERSION
-              config.metadata = { "google-cloud-resource-prefix": "projects/#{@project}" }
-            end
+          @subscriber ||= V1::Subscriber::Client.new do |config|
+            config.credentials = credentials if credentials
+            config.timeout = timeout if timeout
+            config.endpoint = host if host
+            config.lib_name = "gccl"
+            config.lib_version = Google::Cloud::PubSub::VERSION
+            config.metadata = { "google-cloud-resource-prefix": "projects/#{@project}" }
           end
         end
         attr_accessor :mocked_subscriber
 
         def publisher
           return mocked_publisher if mocked_publisher
-          @publisher ||= begin
-            V1::Publisher::Client.new do |config|
-              config.credentials = credentials if credentials
-              config.timeout = timeout if timeout
-              config.endpoint = host if host
-              config.lib_name = "gccl"
-              config.lib_version = Google::Cloud::PubSub::VERSION
-              config.metadata = { "google-cloud-resource-prefix": "projects/#{@project}" }
-            end
+          @publisher ||= V1::Publisher::Client.new do |config|
+            config.credentials = credentials if credentials
+            config.timeout = timeout if timeout
+            config.endpoint = host if host
+            config.lib_name = "gccl"
+            config.lib_version = Google::Cloud::PubSub::VERSION
+            config.metadata = { "google-cloud-resource-prefix": "projects/#{@project}" }
           end
         end
         attr_accessor :mocked_publisher
 
         def iam
           return mocked_iam if mocked_iam
-          @iam ||= begin
-            V1::IAMPolicy::Client.new do |config|
-              config.credentials = credentials if credentials
-              config.timeout = timeout if timeout
-              config.endpoint = host if host
-              config.lib_name = "gccl"
-              config.lib_version = Google::Cloud::PubSub::VERSION
-              config.metadata = { "google-cloud-resource-prefix" => "projects/#{@project}" }
-            end
+          @iam ||= V1::IAMPolicy::Client.new do |config|
+            config.credentials = credentials if credentials
+            config.timeout = timeout if timeout
+            config.endpoint = host if host
+            config.lib_name = "gccl"
+            config.lib_version = Google::Cloud::PubSub::VERSION
+            config.metadata = { "google-cloud-resource-prefix" => "projects/#{@project}" }
           end
         end
         attr_accessor :mocked_iam
 
+        def schemas
+          return mocked_schemas if mocked_schemas
+          @schemas ||= V1::SchemaService::Client.new do |config|
+            config.credentials = credentials if credentials
+            config.timeout = timeout if timeout
+            config.endpoint = host if host
+            config.lib_name = "gccl"
+            config.lib_version = Google::Cloud::PubSub::VERSION
+            config.metadata = { "google-cloud-resource-prefix" => "projects/#{@project}" }
+          end
+        end
+        attr_accessor :mocked_schemas
+
         ##
         # Gets the configuration of a topic.
         # Since the topic only has the name attribute,
@@ -111,18 +121,35 @@ module Google
 
         ##
         # Creates the given topic with the given name.
-        def create_topic topic_name, labels: nil, kms_key_name: nil, persistence_regions: nil, options: {}
+        def create_topic topic_name,
+                         labels: nil,
+                         kms_key_name: nil,
+                         persistence_regions: nil,
+                         schema_name: nil,
+                         message_encoding: nil,
+                         options: {}
           if persistence_regions
-            message_storage_policy = {
+            message_storage_policy = Google::Cloud::PubSub::V1::MessageStoragePolicy.new(
               allowed_persistence_regions: Array(persistence_regions)
-            }
+            )
+          end
+
+          if schema_name || message_encoding
+            unless schema_name && message_encoding
+              raise ArgumentError, "Schema settings must include both schema_name and message_encoding."
+            end
+            schema_settings = Google::Cloud::PubSub::V1::SchemaSettings.new(
+              schema:   schema_path(schema_name),
+              encoding: message_encoding.to_s.upcase
+            )
           end
 
           publisher.create_topic \
             name:                   topic_path(topic_name, options),
             labels:                 labels,
             kms_key_name:           kms_key_name,
-            message_storage_policy: message_storage_policy
+            message_storage_policy: message_storage_policy,
+            schema_settings:        schema_settings
         end
 
         def update_topic topic_obj, *fields
@@ -294,6 +321,89 @@ module Google
           end
         end
 
+        ##
+        # Lists schemas in the current (or given) project.
+        # @param view [String, Symbol, nil] Possible values:
+        #   * `BASIC` - Include the name and type of the schema, but not the definition.
+        #   * `FULL` - Include all Schema object fields.
+        #
+        def list_schemas view, options = {}
+          schema_view = Google::Cloud::PubSub::V1::SchemaView.const_get view.to_s.upcase
+          paged_enum = schemas.list_schemas parent:     project_path(options),
+                                            view:       schema_view,
+                                            page_size:  options[:max],
+                                            page_token: options[:token]
+
+          paged_enum.response
+        end
+
+        ##
+        # Creates a schema in the current (or given) project.
+        def create_schema schema_id, type, definition, options = {}
+          schema = Google::Cloud::PubSub::V1::Schema.new(
+            type:       type,
+            definition: definition
+          )
+          schemas.create_schema parent:    project_path(options),
+                                schema:    schema,
+                                schema_id: schema_id
+        end
+
+        ##
+        # Gets the details of a schema.
+        # @param view [String, Symbol, nil] The set of fields to return in the response. Possible values:
+        #   * `BASIC` - Include the name and type of the schema, but not the definition.
+        #   * `FULL` - Include all Schema object fields.
+        #
+        def get_schema schema_name, view, options = {}
+          schema_view = Google::Cloud::PubSub::V1::SchemaView.const_get view.to_s.upcase
+          schemas.get_schema name: schema_path(schema_name, options),
+                             view: schema_view
+        end
+
+        ##
+        # Delete a schema.
+        def delete_schema schema_name
+          schemas.delete_schema name: schema_path(schema_name)
+        end
+
+        ##
+        # Validate the definition string intended for a schema.
+        def validate_schema type, definition, options = {}
+          schema = Google::Cloud::PubSub::V1::Schema.new(
+            type:       type,
+            definition: definition
+          )
+          schemas.validate_schema parent: project_path(options),
+                                  schema: schema
+        end
+
+        ##
+        # Validates a message against a schema.
+        #
+        # @param message_data [String] Message to validate against the provided `schema_spec`.
+        # @param message_encoding [Google::Cloud::PubSub::V1::Encoding] The encoding expected for messages.
+        # @param schema_name [String] Name of the schema against which to validate.
+        # @param project [String] Name of the project if not the default project.
+        # @param type [String] Ad-hoc schema type against which to validate.
+        # @param definition [String] Ad-hoc schema definition against which to validate.
+        #
+        def validate_message message_data, message_encoding, schema_name: nil, project: nil, type: nil, definition: nil
+          if type && definition
+            schema = Google::Cloud::PubSub::V1::Schema.new(
+              type:       type,
+              definition: definition
+            )
+          end
+          schemas.validate_message parent:   project_path(project: project),
+                                   name:     schema_path(schema_name),
+                                   schema:   schema,
+                                   message:  message_data,
+                                   encoding: message_encoding
+        end
+
+        # Helper methods
+
         def get_topic_policy topic_name, options = {}
           iam.get_iam_policy resource: topic_path(topic_name, options)
         end
@@ -338,6 +448,11 @@ module Google
           "#{project_path options}/snapshots/#{snapshot_name}"
         end
 
+        def schema_path schema_name, options = {}
+          return schema_name if schema_name.nil? || schema_name.to_s.include?("/")
+          "#{project_path options}/schemas/#{schema_name}"
+        end
+
         def inspect
           "#<#{self.class.name} (#{@project})>"
         end

data/lib/google/cloud/pubsub/snapshot.rb

@@ -15,6 +15,7 @@
 
 require "google/cloud/errors"
 require "google/cloud/pubsub/snapshot/list"
+require "google/cloud/pubsub/v1"
 
 module Google
   module Cloud

data/lib/google/cloud/pubsub/snapshot/list.rb

@@ -125,12 +125,12 @@ module Google
           #     puts snapshot.name
           #   end
           #
-          def all request_limit: nil
+          def all request_limit: nil, &block
             request_limit = request_limit.to_i if request_limit
             return enum_for :all, request_limit: request_limit unless block_given?
             results = self
             loop do
-              results.each { |r| yield r }
+              results.each(&block)
               if request_limit
                 request_limit -= 1
                 break if request_limit.negative?

data/lib/google/cloud/pubsub/subscriber.rb

@@ -64,8 +64,13 @@ module Google
       class Subscriber
         include MonitorMixin
 
-        attr_reader :subscription_name, :callback, :deadline, :streams, :message_ordering, :callback_threads,
-                    :push_threads
+        attr_reader :subscription_name
+        attr_reader :callback
+        attr_reader :deadline
+        attr_reader :streams
+        attr_reader :message_ordering
+        attr_reader :callback_threads
+        attr_reader :push_threads
 
         ##
         # @private Implementation attributes.
@@ -123,28 +128,21 @@ module Google
 
         ##
         # Immediately stops the subscriber. No new messages will be pulled from
-        # the subscription. All actions taken on received messages that have not
-        # yet been sent to the API will be sent to the API. All received but
-        # unprocessed messages will be released back to the API and redelivered.
-        # Use {#wait!} to block until the subscriber is fully stopped and all
-        # received messages have been processed or released.
+        # the subscription. Use {#wait!} to block until all received messages have
+        # been processed or released: All actions taken on received messages that
+        # have not yet been sent to the API will be sent to the API. All received
+        # but unprocessed messages will be released back to the API and redelivered.
         #
         # @return [Subscriber] returns self so calls can be chained.
         #
         def stop
-          stop_pool = synchronize do
+          synchronize do
             @started = false
             @stopped = true
-
-            @stream_pool.map do |stream|
-              Thread.new { stream.stop }
-            end
+            @stream_pool.map(&:stop)
+            wait_stop_buffer_thread!
+            self
           end
-          stop_pool.map(&:join)
-          # Stop the buffer after the streams are all stopped
-          synchronize { @buffer.stop }
-
-          self
         end
 
         ##
@@ -162,13 +160,8 @@ module Google
         # @return [Subscriber] returns self so calls can be chained.
         #
         def wait! timeout = nil
-          wait_pool = synchronize do
-            @stream_pool.map do |stream|
-              Thread.new { stream.wait! timeout }
-            end
-          end
-          wait_pool.map(&:join)
-
+          wait_stop_buffer_thread!
+          @wait_stop_buffer_thread.join timeout
           self
         end
 
@@ -374,6 +367,18 @@ module Google
 
         protected
 
+        ##
+        # Starts a new thread to call wait! (blocking) on each Stream and then stop the TimedUnaryBuffer.
+        def wait_stop_buffer_thread!
+          synchronize do
+            @wait_stop_buffer_thread ||= Thread.new do
+              @stream_pool.map(&:wait!)
+              # Shutdown the buffer TimerTask (and flush the buffer) after the streams are all stopped.
+              @buffer.stop
+            end
+          end
+        end
+
         def coerce_inventory inventory
           @inventory = inventory
           if @inventory.is_a? Hash

data/lib/google/cloud/pubsub/subscriber/inventory.rb

@@ -30,8 +30,12 @@ module Google
 
           include MonitorMixin
 
-          attr_reader :stream, :limit, :bytesize, :extension, :max_duration_per_lease_extension,
-                      :use_legacy_flow_control
+          attr_reader :stream
+          attr_reader :limit
+          attr_reader :bytesize
+          attr_reader :extension
+          attr_reader :max_duration_per_lease_extension
+          attr_reader :use_legacy_flow_control
 
           def initialize stream, limit:, bytesize:, extension:, max_duration_per_lease_extension:,
                          use_legacy_flow_control:

data/lib/google/cloud/pubsub/subscriber/stream.rb

@@ -58,7 +58,7 @@ module Google
             @paused  = nil
             @pause_cond = new_cond
 
-            @inventory = Inventory.new self, @subscriber.stream_inventory
+            @inventory = Inventory.new self, **@subscriber.stream_inventory
 
             @sequencer = Sequencer.new(&method(:perform_callback_async)) if subscriber.message_ordering
 

data/lib/google/cloud/pubsub/subscriber/timed_unary_buffer.rb

@@ -25,7 +25,8 @@ module Google
         class TimedUnaryBuffer
           include MonitorMixin
 
-          attr_reader :max_bytes, :interval
+          attr_reader :max_bytes
+          attr_reader :interval
 
           def initialize subscriber, max_bytes: 500_000, interval: 1.0
             super() # to init MonitorMixin
@@ -138,7 +139,7 @@ module Google
               end
 
             groups = prev_reg.each_pair.group_by { |_ack_id, delay| delay }
-            req_hash = Hash[groups.map { |k, v| [k, v.map(&:first)] }]
+            req_hash = groups.transform_values { |v| v.map(&:first) }
 
             requests = { acknowledge: [] }
             ack_ids = Array(req_hash.delete(:ack)) # ack has no deadline set
@@ -215,11 +216,9 @@ module Google
 
           def add_future pool
             Concurrent::Promises.future_on pool do
-              begin
-                yield
-              rescue StandardError => e
-                error! e
-              end
+              yield
+            rescue StandardError => e
+              error! e
             end
           end
         end

data/lib/google/cloud/pubsub/subscription.rb

@@ -733,19 +733,28 @@ module Google
         end
 
         ##
-        # Pulls messages from the server. Returns an empty list if there are no
-        # messages available in the backlog. Raises an ApiError with status
-        # `UNAVAILABLE` if there are too many concurrent pull requests pending
-        # for the given subscription.
+        # Pulls messages from the server, blocking until messages are available
+        # when called with the `immediate: false` option, which is recommended
+        # to avoid adverse impacts on the performance of pull operations.
+        #
+        # Raises an API error with status `UNAVAILABLE` if there are too many
+        # concurrent pull requests pending for the given subscription.
         #
         # See also {#listen} for the preferred way to process messages as they
         # become available.
         #
-        # @param [Boolean] immediate When `true` the system will respond
-        #   immediately even if it is not able to return messages. When `false`
-        #   the system is allowed to wait until it can return least one message.
-        #   No messages are returned when a request times out. The default value
-        #   is `true`.
+        # @param [Boolean] immediate Whether to return immediately or block until
+        #   messages are available.
+        #
+        #   **Warning:** The default value of this field is `true`. However, sending
+        #   `true` is discouraged because it adversely impacts the performance of
+        #   pull operations. We recommend that users always explicitly set this field
+        #   to `false`.
+        #
+        #   If this field set to `true`, the system will respond immediately
+        #   even if it there are no messages available to return in the pull
+        #   response. Otherwise, the system may wait (for a bounded amount of time)
+        #   until at least one message is available, rather than returning no messages.
         #
         #   See also {#listen} for the preferred way to process messages as they
         #   become available.
@@ -755,31 +764,24 @@ module Google
         #
         # @return [Array<Google::Cloud::PubSub::ReceivedMessage>]
         #
-        # @example
+        # @example The `immediate: false` option is now recommended to avoid adverse impacts on pull operations:
         #   require "google/cloud/pubsub"
         #
         #   pubsub = Google::Cloud::PubSub.new
         #
         #   sub = pubsub.subscription "my-topic-sub"
-        #   sub.pull.each { |received_message| received_message.acknowledge! }
-        #
-        # @example A maximum number of messages returned can also be specified:
-        #   require "google/cloud/pubsub"
-        #
-        #   pubsub = Google::Cloud::PubSub.new
-        #
-        #   sub = pubsub.subscription "my-topic-sub"
-        #   sub.pull(max: 10).each do |received_message|
+        #   received_messages = sub.pull immediate: false
+        #   received_messages.each do |received_message|
         #     received_message.acknowledge!
         #   end
         #
-        # @example The call can block until messages are available:
+        # @example A maximum number of messages returned can also be specified:
         #   require "google/cloud/pubsub"
         #
         #   pubsub = Google::Cloud::PubSub.new
         #
         #   sub = pubsub.subscription "my-topic-sub"
-        #   received_messages = sub.pull immediate: false
+        #   received_messages = sub.pull immediate: false, max: 10
         #   received_messages.each do |received_message|
         #     received_message.acknowledge!
         #   end
@@ -1010,7 +1012,7 @@ module Google
         #   pubsub = Google::Cloud::PubSub.new
         #
         #   sub = pubsub.subscription "my-topic-sub"
-        #   received_messages = sub.pull
+        #   received_messages = sub.pull immediate: false
         #   sub.acknowledge received_messages
         #
         def acknowledge *messages
@@ -1045,7 +1047,7 @@ module Google
         #   pubsub = Google::Cloud::PubSub.new
         #
         #   sub = pubsub.subscription "my-topic-sub"
-        #   received_messages = sub.pull
+        #   received_messages = sub.pull immediate: false
         #   sub.modify_ack_deadline 120, received_messages
         #
         def modify_ack_deadline new_deadline, *messages
@@ -1143,7 +1145,7 @@ module Google
         #
         #   snapshot = sub.create_snapshot
         #
-        #   received_messages = sub.pull
+        #   received_messages = sub.pull immediate: false
         #   sub.acknowledge received_messages
         #
         #   sub.seek snapshot
@@ -1156,7 +1158,7 @@ module Google
         #
         #   time = Time.now
         #
-        #   received_messages = sub.pull
+        #   received_messages = sub.pull immediate: false
         #   sub.acknowledge received_messages
         #
         #   sub.seek time