google-cloud-pubsub 2.2.0 → 2.5.0

data/lib/google/cloud/pubsub/service.rb

@@ -26,7 +26,10 @@ module Google
       ##
       # @private Represents the Pub/Sub service API, including IAM mixins.
       class Service
-        attr_accessor :project, :credentials, :host, :timeout
+        attr_accessor :project
+        attr_accessor :credentials
+        attr_accessor :host
+        attr_accessor :timeout
         ###
         # The same client_id is used across all streaming pull connections that are created by this client. This is
         # intentional, as it indicates to the server that any guarantees, such as message ordering, made for a stream
@@ -46,49 +49,56 @@ module Google
 
         def subscriber
           return mocked_subscriber if mocked_subscriber
-          @subscriber ||= begin
-            V1::Subscriber::Client.new do |config|
-              config.credentials = credentials if credentials
-              config.timeout = timeout if timeout
-              config.endpoint = host if host
-              config.lib_name = "gccl"
-              config.lib_version = Google::Cloud::PubSub::VERSION
-              config.metadata = { "google-cloud-resource-prefix": "projects/#{@project}" }
-            end
+          @subscriber ||= V1::Subscriber::Client.new do |config|
+            config.credentials = credentials if credentials
+            config.timeout = timeout if timeout
+            config.endpoint = host if host
+            config.lib_name = "gccl"
+            config.lib_version = Google::Cloud::PubSub::VERSION
+            config.metadata = { "google-cloud-resource-prefix": "projects/#{@project}" }
           end
         end
         attr_accessor :mocked_subscriber
 
         def publisher
           return mocked_publisher if mocked_publisher
-          @publisher ||= begin
-            V1::Publisher::Client.new do |config|
-              config.credentials = credentials if credentials
-              config.timeout = timeout if timeout
-              config.endpoint = host if host
-              config.lib_name = "gccl"
-              config.lib_version = Google::Cloud::PubSub::VERSION
-              config.metadata = { "google-cloud-resource-prefix": "projects/#{@project}" }
-            end
+          @publisher ||= V1::Publisher::Client.new do |config|
+            config.credentials = credentials if credentials
+            config.timeout = timeout if timeout
+            config.endpoint = host if host
+            config.lib_name = "gccl"
+            config.lib_version = Google::Cloud::PubSub::VERSION
+            config.metadata = { "google-cloud-resource-prefix": "projects/#{@project}" }
           end
         end
         attr_accessor :mocked_publisher
 
         def iam
           return mocked_iam if mocked_iam
-          @iam ||= begin
-            V1::IAMPolicy::Client.new do |config|
-              config.credentials = credentials if credentials
-              config.timeout = timeout if timeout
-              config.endpoint = host if host
-              config.lib_name = "gccl"
-              config.lib_version = Google::Cloud::PubSub::VERSION
-              config.metadata = { "google-cloud-resource-prefix" => "projects/#{@project}" }
-            end
+          @iam ||= V1::IAMPolicy::Client.new do |config|
+            config.credentials = credentials if credentials
+            config.timeout = timeout if timeout
+            config.endpoint = host if host
+            config.lib_name = "gccl"
+            config.lib_version = Google::Cloud::PubSub::VERSION
+            config.metadata = { "google-cloud-resource-prefix" => "projects/#{@project}" }
           end
         end
         attr_accessor :mocked_iam
 
+        def schemas
+          return mocked_schemas if mocked_schemas
+          @schemas ||= V1::SchemaService::Client.new do |config|
+            config.credentials = credentials if credentials
+            config.timeout = timeout if timeout
+            config.endpoint = host if host
+            config.lib_name = "gccl"
+            config.lib_version = Google::Cloud::PubSub::VERSION
+            config.metadata = { "google-cloud-resource-prefix" => "projects/#{@project}" }
+          end
+        end
+        attr_accessor :mocked_schemas
+
         ##
         # Gets the configuration of a topic.
         # Since the topic only has the name attribute,
@@ -111,18 +121,35 @@ module Google
 
         ##
         # Creates the given topic with the given name.
-        def create_topic topic_name, labels: nil, kms_key_name: nil, persistence_regions: nil, options: {}
+        def create_topic topic_name,
+                         labels: nil,
+                         kms_key_name: nil,
+                         persistence_regions: nil,
+                         schema_name: nil,
+                         message_encoding: nil,
+                         options: {}
           if persistence_regions
-            message_storage_policy = {
+            message_storage_policy = Google::Cloud::PubSub::V1::MessageStoragePolicy.new(
               allowed_persistence_regions: Array(persistence_regions)
-            }
+            )
+          end
+
+          if schema_name || message_encoding
+            unless schema_name && message_encoding
+              raise ArgumentError, "Schema settings must include both schema_name and message_encoding."
+            end
+            schema_settings = Google::Cloud::PubSub::V1::SchemaSettings.new(
+              schema:   schema_path(schema_name),
+              encoding: message_encoding.to_s.upcase
+            )
           end
 
           publisher.create_topic \
             name:                   topic_path(topic_name, options),
             labels:                 labels,
             kms_key_name:           kms_key_name,
-            message_storage_policy: message_storage_policy
+            message_storage_policy: message_storage_policy,
+            schema_settings:        schema_settings
         end
 
         def update_topic topic_obj, *fields
@@ -294,6 +321,89 @@ module Google
           end
         end
 
+        ##
+        # Lists schemas in the current (or given) project.
+        # @param view [String, Symbol, nil] Possible values:
+        #   * `BASIC` - Include the name and type of the schema, but not the definition.
+        #   * `FULL` - Include all Schema object fields.
+        #
+        def list_schemas view, options = {}
+          schema_view = Google::Cloud::PubSub::V1::SchemaView.const_get view.to_s.upcase
+          paged_enum = schemas.list_schemas parent:     project_path(options),
+                                            view:       schema_view,
+                                            page_size:  options[:max],
+                                            page_token: options[:token]
+
+          paged_enum.response
+        end
+
+        ##
+        # Creates a schema in the current (or given) project.
+        def create_schema schema_id, type, definition, options = {}
+          schema = Google::Cloud::PubSub::V1::Schema.new(
+            type:       type,
+            definition: definition
+          )
+          schemas.create_schema parent:    project_path(options),
+                                schema:    schema,
+                                schema_id: schema_id
+        end
+
+        ##
+        # Gets the details of a schema.
+        # @param view [String, Symbol, nil] The set of fields to return in the response. Possible values:
+        #   * `BASIC` - Include the name and type of the schema, but not the definition.
+        #   * `FULL` - Include all Schema object fields.
+        #
+        def get_schema schema_name, view, options = {}
+          schema_view = Google::Cloud::PubSub::V1::SchemaView.const_get view.to_s.upcase
+          schemas.get_schema name: schema_path(schema_name, options),
+                             view: schema_view
+        end
+
+        ##
+        # Delete a schema.
+        def delete_schema schema_name
+          schemas.delete_schema name: schema_path(schema_name)
+        end
+
+        ##
+        # Validate the definition string intended for a schema.
+        def validate_schema type, definition, options = {}
+          schema = Google::Cloud::PubSub::V1::Schema.new(
+            type:       type,
+            definition: definition
+          )
+          schemas.validate_schema parent: project_path(options),
+                                  schema: schema
+        end
+
+        ##
+        # Validates a message against a schema.
+        #
+        # @param message_data [String] Message to validate against the provided `schema_spec`.
+        # @param message_encoding [Google::Cloud::PubSub::V1::Encoding] The encoding expected for messages.
+        # @param schema_name [String] Name of the schema against which to validate.
+        # @param project [String] Name of the project if not the default project.
+        # @param type [String] Ad-hoc schema type against which to validate.
+        # @param definition [String] Ad-hoc schema definition against which to validate.
+        #
+        def validate_message message_data, message_encoding, schema_name: nil, project: nil, type: nil, definition: nil
+          if type && definition
+            schema = Google::Cloud::PubSub::V1::Schema.new(
+              type:       type,
+              definition: definition
+            )
+          end
+          schemas.validate_message parent:   project_path(project: project),
+                                   name:     schema_path(schema_name),
+                                   schema:   schema,
+                                   message:  message_data,
+                                   encoding: message_encoding
+        end
+
+        # Helper methods
+
         def get_topic_policy topic_name, options = {}
           iam.get_iam_policy resource: topic_path(topic_name, options)
         end
@@ -338,6 +448,11 @@ module Google
           "#{project_path options}/snapshots/#{snapshot_name}"
         end
 
+        def schema_path schema_name, options = {}
+          return schema_name if schema_name.nil? || schema_name.to_s.include?("/")
+          "#{project_path options}/schemas/#{schema_name}"
+        end
+
         def inspect
           "#<#{self.class.name} (#{@project})>"
         end

data/lib/google/cloud/pubsub/snapshot.rb

@@ -15,6 +15,7 @@
 
 require "google/cloud/errors"
 require "google/cloud/pubsub/snapshot/list"
+require "google/cloud/pubsub/v1"
 
 module Google
   module Cloud
@@ -58,8 +59,10 @@ module Google
         end
 
         ##
-        # The name of the snapshot. Format is
-        # `projects/{project}/snapshots/{snap}`.
+        # The name of the snapshot.
+        #
+        # @return [String] A fully-qualified snapshot name in the form
+        #   `projects/{project_id}/snapshots/{snapshot_id}`.
         def name
           @grpc.name
         end

data/lib/google/cloud/pubsub/snapshot/list.rb

@@ -125,12 +125,12 @@ module Google
           #     puts snapshot.name
           #   end
           #
-          def all request_limit: nil
+          def all request_limit: nil, &block
             request_limit = request_limit.to_i if request_limit
             return enum_for :all, request_limit: request_limit unless block_given?
             results = self
             loop do
-              results.each { |r| yield r }
+              results.each(&block)
               if request_limit
                 request_limit -= 1
                 break if request_limit.negative?

data/lib/google/cloud/pubsub/subscriber.rb

@@ -64,8 +64,13 @@ module Google
       class Subscriber
         include MonitorMixin
 
-        attr_reader :subscription_name, :callback, :deadline, :streams, :message_ordering, :callback_threads,
-                    :push_threads
+        attr_reader :subscription_name
+        attr_reader :callback
+        attr_reader :deadline
+        attr_reader :streams
+        attr_reader :message_ordering
+        attr_reader :callback_threads
+        attr_reader :push_threads
 
         ##
         # @private Implementation attributes.
@@ -306,6 +311,17 @@ module Google
         # @deprecated Use {#max_outstanding_bytes}.
         alias inventory_bytesize max_outstanding_bytes
 
+        ##
+        # Whether to enforce flow control at the client side only or to enforce it at both the client and
+        # the server. For more details about flow control see https://cloud.google.com/pubsub/docs/pull#config.
+        #
+        # @return [Boolean] `true` when only client side flow control is enforced, `false` when both client and
+        # server side flow control are enforced.
+        #
+        def use_legacy_flow_control?
+          @inventory[:use_legacy_flow_control]
+        end
+
         ##
         # The number of seconds that received messages can be held awaiting processing. Default is 3,600 (1 hour).
         #
@@ -334,7 +350,8 @@ module Google
             limit:                            @inventory[:max_outstanding_messages].fdiv(@streams).ceil,
             bytesize:                         @inventory[:max_outstanding_bytes].fdiv(@streams).ceil,
             extension:                        @inventory[:max_total_lease_duration],
-            max_duration_per_lease_extension: @inventory[:max_duration_per_lease_extension]
+            max_duration_per_lease_extension: @inventory[:max_duration_per_lease_extension],
+            use_legacy_flow_control:          @inventory[:use_legacy_flow_control]
           }
         end
 
@@ -377,6 +394,7 @@ module Google
           @inventory[:max_outstanding_bytes] = Integer(@inventory[:max_outstanding_bytes] || 100_000_000)
           @inventory[:max_total_lease_duration] = Integer(@inventory[:max_total_lease_duration] || 3600)
           @inventory[:max_duration_per_lease_extension] = Integer(@inventory[:max_duration_per_lease_extension] || 0)
+          @inventory[:use_legacy_flow_control] = @inventory[:use_legacy_flow_control] || false
         end
 
         def default_error_callbacks

data/lib/google/cloud/pubsub/subscriber/inventory.rb

@@ -30,15 +30,22 @@ module Google
 
           include MonitorMixin
 
-          attr_reader :stream, :limit, :bytesize, :extension, :max_duration_per_lease_extension
-
-          def initialize stream, limit:, bytesize:, extension:, max_duration_per_lease_extension:
+          attr_reader :stream
+          attr_reader :limit
+          attr_reader :bytesize
+          attr_reader :extension
+          attr_reader :max_duration_per_lease_extension
+          attr_reader :use_legacy_flow_control
+
+          def initialize stream, limit:, bytesize:, extension:, max_duration_per_lease_extension:,
+                         use_legacy_flow_control:
             super()
             @stream = stream
             @limit = limit
             @bytesize = bytesize
             @extension = extension
             @max_duration_per_lease_extension = max_duration_per_lease_extension
+            @use_legacy_flow_control = use_legacy_flow_control
             @inventory = {}
             @wait_cond = new_cond
           end

data/lib/google/cloud/pubsub/subscriber/stream.rb

@@ -58,7 +58,7 @@ module Google
             @paused  = nil
             @pause_cond = new_cond
 
-            @inventory = Inventory.new self, @subscriber.stream_inventory
+            @inventory = Inventory.new self, **@subscriber.stream_inventory
 
             @sequencer = Sequencer.new(&method(:perform_callback_async)) if subscriber.message_ordering
 
@@ -363,8 +363,8 @@ module Google
               req.modify_deadline_ack_ids += @inventory.ack_ids
               req.modify_deadline_seconds += @inventory.ack_ids.map { @subscriber.deadline }
               req.client_id = @subscriber.service.client_id
-              req.max_outstanding_messages = @inventory.limit
-              req.max_outstanding_bytes = @inventory.bytesize
+              req.max_outstanding_messages = @inventory.use_legacy_flow_control ? 0 : @inventory.limit
+              req.max_outstanding_bytes = @inventory.use_legacy_flow_control ? 0 : @inventory.bytesize
             end
           end
 

data/lib/google/cloud/pubsub/subscriber/timed_unary_buffer.rb

@@ -25,7 +25,8 @@ module Google
         class TimedUnaryBuffer
           include MonitorMixin
 
-          attr_reader :max_bytes, :interval
+          attr_reader :max_bytes
+          attr_reader :interval
 
           def initialize subscriber, max_bytes: 500_000, interval: 1.0
             super() # to init MonitorMixin
@@ -138,7 +139,7 @@ module Google
               end
 
             groups = prev_reg.each_pair.group_by { |_ack_id, delay| delay }
-            req_hash = Hash[groups.map { |k, v| [k, v.map(&:first)] }]
+            req_hash = groups.transform_values { |v| v.map(&:first) }
 
             requests = { acknowledge: [] }
             ack_ids = Array(req_hash.delete(:ack)) # ack has no deadline set
@@ -215,11 +216,9 @@ module Google
 
           def add_future pool
             Concurrent::Promises.future_on pool do
-              begin
-                yield
-              rescue StandardError => e
-                error! e
-              end
+              yield
+            rescue StandardError => e
+              error! e
             end
           end
         end

data/lib/google/cloud/pubsub/subscription.rb

@@ -77,7 +77,9 @@ module Google
         ##
         # The name of the subscription.
         #
-        # @return [String]
+        # @return [String] A fully-qualified subscription name in the form
+        #   `projects/{project_id}/subscriptions/{subscription_id}`.
+        #
         def name
           return @resource_name if reference?
           @grpc.name
@@ -330,7 +332,7 @@ module Google
         # If {#expires_in=} is not set, a *default* value of of 31 days will be
         # used. The minimum allowed value is 1 day.
         #
-        # Makes an API call to retrieve the value when called on a
+        # Makes an API call to retrieve the expires_in value when called on a
         # reference object. See {#reference?}.
         #
         # @return [Numeric, nil] The expiration duration, or `nil` if unset.
@@ -365,6 +367,9 @@ module Google
         # `attributes` field matches the filter are delivered on this subscription. If empty, then no messages are
         # filtered out.
         #
+        # Makes an API call to retrieve the filter value when called on a reference
+        # object. See {#reference?}.
+        #
         # @return [String] The frozen filter string.
         #
         def filter
@@ -409,6 +414,9 @@ module Google
         # The operation will fail if the topic does not exist. Users should ensure that there is a subscription attached
         # to this topic since messages published to a topic with no subscriptions are lost.
         #
+        # Makes an API call to retrieve the dead_letter_policy value when called on a
+        # reference object. See {#reference?}.
+        #
         # See also {#dead_letter_topic}, {#dead_letter_max_delivery_attempts=}, {#dead_letter_max_delivery_attempts}
         # and {#remove_dead_letter_policy}.
         #
@@ -448,7 +456,7 @@ module Google
         # See also {#dead_letter_max_delivery_attempts=}, {#dead_letter_topic=}, {#dead_letter_topic}
         # and {#remove_dead_letter_policy}.
         #
-        # Makes an API call to retrieve the value when called on a reference object. See {#reference?}.
+        # Makes an API call to retrieve the dead_letter_policy when called on a reference object. See {#reference?}.
         #
         # @return [Integer, nil] A value between `5` and `100`, or `nil` if no dead letter policy is configured.
         #
@@ -477,6 +485,8 @@ module Google
         #
         # This field will be honored on a best effort basis. If this parameter is 0, a default value of 5 is used.
         #
+        # Makes an API call to retrieve the dead_letter_policy when called on a reference object. See {#reference?}.
+        #
         # The dead letter topic must be set first. See {#dead_letter_topic=}, {#dead_letter_topic} and
         # {#remove_dead_letter_policy}.
         #
@@ -512,6 +522,8 @@ module Google
         # Removes an existing dead letter policy. A dead letter policy specifies the conditions for dead lettering
         # messages in the subscription. If a dead letter policy is not set, dead lettering is disabled.
         #
+        # Makes an API call to retrieve the dead_letter_policy when called on a reference object. See {#reference?}.
+        #
         # See {#dead_letter_topic}, {#dead_letter_topic=}, {#dead_letter_max_delivery_attempts} and
         # {#dead_letter_max_delivery_attempts=}.
         #
@@ -547,8 +559,7 @@ module Google
         # for healthy subscribers. Retry Policy will be triggered on NACKs or acknowledgement deadline exceeded events
         # for a given message.
         #
-        # **EXPERIMENTAL:** This API might be changed in backward-incompatible ways and is not recommended for
-        # production use. It is not subject to any SLA or deprecation policy.
+        # Makes an API call to retrieve the retry_policy when called on a reference object. See {#reference?}.
         #
         # @return [RetryPolicy, nil] The retry policy for the subscription, or `nil`.
         #
@@ -576,9 +587,6 @@ module Google
         # for healthy subscribers. Retry Policy will be triggered on NACKs or acknowledgement deadline exceeded events
         # for a given message.
         #
-        # **EXPERIMENTAL:** This API might be changed in backward-incompatible ways and is not recommended for
-        # production use. It is not subject to any SLA or deprecation policy.
-        #
         # @param [RetryPolicy, nil] new_retry_policy A new retry policy for the subscription, or `nil`.
         #
         # @example
@@ -594,10 +602,11 @@ module Google
         #   sub.retry_policy.maximum_backoff #=> 300
         #
         def retry_policy= new_retry_policy
-          ensure_grpc!
+          ensure_service!
           new_retry_policy = new_retry_policy.to_grpc if new_retry_policy
           update_grpc = Google::Cloud::PubSub::V1::Subscription.new name: name, retry_policy: new_retry_policy
           @grpc = service.update_subscription update_grpc, :retry_policy
+          @resource_name = nil
         end
 
         ##
@@ -629,7 +638,7 @@ module Google
         #
         # See {Topic#subscribe} and {#detach}.
         #
-        # Makes an API call to retrieve the value when called on a
+        # Makes an API call to retrieve the detached value when called on a
         # reference object. See {#reference?}.
         #
         # @return [Boolean]
@@ -870,6 +879,9 @@ module Google
         #       Default is 1,000. (Note: replaces `:limit`, which is deprecated.)
         #     * `:max_outstanding_bytes` [Integer] The total byte size of received messages to be collected by
         #       subscriber. Default is 100,000,000 (100MB). (Note: replaces `:bytesize`, which is deprecated.)
+        #     * `:use_legacy_flow_control` [Boolean] Disables enforcing flow control settings at the Cloud PubSub
+        #       server and the less accurate method of only enforcing flow control at the client side is used instead.
+        #       Default is false.
         #     * `:max_total_lease_duration` [Integer] The number of seconds that received messages can be held awaiting
         #       processing. Default is 3,600 (1 hour). (Note: replaces `:extension`, which is deprecated.)
         #     * `:max_duration_per_lease_extension` [Integer] The maximum amount of time in seconds for a single lease
@@ -1054,14 +1066,19 @@ module Google
         # * Any messages published to the subscription's topic following the
         #   successful completion of the `create_snapshot` operation.
         #
-        # @param [String, nil] snapshot_name Name of the new snapshot. If the
-        #   name is not provided, the server will assign a random name
-        #   for this snapshot on the same project as the subscription. The
-        #   format is `projects/{project}/snapshots/{snap}`. The name must start
-        #   with a letter, and contain only letters ([A-Za-z]), numbers
-        #   ([0-9], dashes (-), underscores (_), periods (.), tildes (~), plus
-        #   (+) or percent signs (%). It must be between 3 and 255 characters in
-        #   length, and it must not start with "goog". Optional.
+        # @param [String, nil] snapshot_name Name of the new snapshot. Optional.
+        #   If the name is not provided, the server will assign a random name
+        #   for this snapshot on the same project as the subscription.
+        #   The value can be a simple snapshot ID (relative name), in which
+        #   case the current project ID will be supplied, or a fully-qualified
+        #   snapshot name in the form
+        #   `projects/{project_id}/snapshots/{snapshot_id}`.
+        #
+        #   The snapshot ID (relative name) must start with a letter, and
+        #   contain only letters (`[A-Za-z]`), numbers (`[0-9]`), dashes (`-`),
+        #   underscores (`_`), periods (`.`), tildes (`~`), plus (`+`) or percent
+        #   signs (`%`). It must be between 3 and 255 characters in length, and
+        #   it must not start with `goog`.
         # @param [Hash] labels A hash of user-provided labels associated with
         #   the snapshot. You can use these to organize and group your
         #   snapshots. Label keys and values can be no longer than 63