google-cloud-pubsub 2.3.2 → 2.7.0

data/lib/google/cloud/pubsub/credentials.rb

@@ -14,7 +14,7 @@
 
 
 require "googleauth"
-require "google/cloud/pubsub/v1/publisher/credentials.rb"
+require "google/cloud/pubsub/v1/publisher/credentials"
 
 module Google
   module Cloud

data/lib/google/cloud/pubsub/errors.rb

@@ -78,6 +78,14 @@ module Google
           super "Can't publish message using #{ordering_key}."
         end
       end
+
+      ##
+      # Raised when the desired action is `error` and the message would exceed
+      # flow control limits, or when the desired action is `block` and the
+      # message would block forever against the flow control limits.
+      #
+      class FlowControlLimitError < Google::Cloud::Error
+      end
     end
 
     Pubsub = PubSub unless const_defined? :Pubsub

data/lib/google/cloud/pubsub/flow_controller.rb

@@ -0,0 +1,139 @@
+# Copyright 2021 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+require "google/cloud/pubsub/errors"
+require "concurrent/atomics"
+
+module Google
+  module Cloud
+    module PubSub
+      ##
+      # @private
+      #
+      # Used to control the flow of messages passing through it.
+      #
+      class FlowController
+        attr_reader :message_limit
+        attr_reader :byte_limit
+        attr_reader :limit_exceeded_behavior
+        ##
+        # @private Implementation accessors
+        attr_reader :outstanding_messages, :outstanding_bytes, :awaiting
+
+        def initialize message_limit: 1000, byte_limit: 10_000_000, limit_exceeded_behavior: :ignore
+          unless [:ignore, :error, :block].include? limit_exceeded_behavior
+            raise ArgumentError, "limit_exceeded_behavior must be one of :ignore, :error, :block"
+          end
+          if [:error, :block].include?(limit_exceeded_behavior) && message_limit < 1
+            raise ArgumentError,
+                  "Flow control message limit (#{message_limit}) exceeded by a single message, would block forever"
+          end
+          @mutex = Mutex.new
+          @message_limit = message_limit
+          @byte_limit = byte_limit
+          @limit_exceeded_behavior = limit_exceeded_behavior
+          @outstanding_messages = 0
+          @outstanding_bytes = 0
+
+          @awaiting = []
+        end
+
+        def acquire message_size
+          return if limit_exceeded_behavior == :ignore
+          @mutex.lock
+          if limit_exceeded_behavior == :error && would_exceed_message_limit?
+            raise FlowControlLimitError, "Flow control message limit (#{message_limit}) would be exceeded"
+          end
+          if limit_exceeded_behavior == :error && would_exceed_byte_limit?(message_size)
+            raise FlowControlLimitError,
+                  "Flow control byte limit (#{byte_limit}) would be exceeded, message_size: #{message_size}"
+          end
+          if limit_exceeded_behavior == :block && message_size > byte_limit
+            raise FlowControlLimitError,
+                  "Flow control byte limit (#{byte_limit}) exceeded by a single message, would block forever"
+          end
+
+          acquire_or_wait message_size
+        ensure
+          @mutex.unlock if @mutex.owned?
+        end
+
+        def release message_size
+          return if limit_exceeded_behavior == :ignore
+          @mutex.synchronize do
+            raise "Flow control messages count would be negative" if (@outstanding_messages - 1).negative?
+            raise "Flow control bytes count would be negative" if (@outstanding_bytes - message_size).negative?
+
+            @outstanding_messages -= 1
+            @outstanding_bytes -= message_size
+            @awaiting.first.set unless @awaiting.empty?
+          end
+        end
+
+        protected
+
+        # rubocop:disable Style/IdenticalConditionalBranches
+        # rubocop:disable Style/GuardClause
+
+        def acquire_or_wait message_size
+          waiter = nil
+          while is_new_and_others_wait?(waiter) ||
+                would_exceed_byte_limit?(message_size) ||
+                would_exceed_message_limit?
+
+            if waiter.nil?
+              waiter = Concurrent::Event.new
+              # This waiter gets added to the back of the line.
+              @awaiting << waiter
+            else
+              waiter = Concurrent::Event.new
+              # This waiter already in line stays at the head of the line.
+              @awaiting[0] = waiter
+            end
+            @mutex.unlock
+            waiter.wait
+            @mutex.lock
+          end
+          @outstanding_messages += 1
+          @outstanding_bytes += message_size
+
+          @awaiting.shift if waiter # Remove the newly released waiter from the head of the queue.
+
+          # There may be some surplus left; let the next message waiting try to acquire a permit.
+          if !@awaiting.empty? && @outstanding_bytes < byte_limit && @outstanding_messages < message_limit
+            @awaiting.first.set
+          end
+        end
+
+        # rubocop:enable Style/IdenticalConditionalBranches
+        # rubocop:enable Style/GuardClause
+
+        def is_new_and_others_wait? waiter
+          waiter.nil? && !@awaiting.empty?
+        end
+
+        def would_exceed_message_limit?
+          @outstanding_messages + 1 > message_limit
+        end
+
+        def would_exceed_byte_limit? bytes_requested
+          @outstanding_bytes + bytes_requested > byte_limit
+        end
+      end
+    end
+
+    Pubsub = PubSub unless const_defined? :Pubsub
+  end
+end

data/lib/google/cloud/pubsub/policy.rb

@@ -68,7 +68,8 @@ module Google
       #   end
       #
       class Policy
-        attr_reader :etag, :roles
+        attr_reader :etag
+        attr_reader :roles
 
         ##
         # @private Creates a Policy object.
@@ -167,7 +168,7 @@ module Google
                 role:    role_name,
                 members: roles[role_name]
               )
-            end
+            end.compact
           )
         end
 

data/lib/google/cloud/pubsub/project.rb

@@ -18,6 +18,7 @@ require "google/cloud/pubsub/service"
 require "google/cloud/pubsub/credentials"
 require "google/cloud/pubsub/topic"
 require "google/cloud/pubsub/batch_publisher"
+require "google/cloud/pubsub/schema"
 require "google/cloud/pubsub/snapshot"
 
 module Google
@@ -102,6 +103,16 @@ module Google
         #   * `:threads` (Hash) The number of threads to create to handle concurrent calls by the publisher:
         #     * `:publish` (Integer) The number of threads used to publish messages. Default is 2.
         #     * `:callback` (Integer) The number of threads to handle the published messages' callbacks. Default is 4.
+        #   * `:flow_control` (Hash) The client flow control settings for message publishing:
+        #     * `:message_limit` (Integer) The maximum number of messages allowed to wait to be published. Default is
+        #       `10 * max_messages`.
+        #     * `:byte_limit` (Integer) The maximum total size of messages allowed to wait to be published. Default is
+        #       `10 * max_bytes`.
+        #     * `:limit_exceeded_behavior` (Symbol) The action to take when publish flow control limits are exceeded.
+        #       Possible values include: `:ignore` - Flow control is disabled. `:error` - Calls to {Topic#publish_async}
+        #       will raise {FlowControlLimitError} when publish flow control limits are exceeded. `:block` - Calls to
+        #       {Topic#publish_async} will block until capacity is available when publish flow control limits are
+        #       exceeded. The default value is `:ignore`.
         #
         # @return [Google::Cloud::PubSub::Topic, nil] Returns `nil` if topic
         #   does not exist.
@@ -192,15 +203,41 @@ module Google
         #
         #   Hash keys and values may include the following:
         #
-        #   * `:max_bytes` (Integer) The maximum size of messages to be collected before the batch is published. Default
-        #     is 1,000,000 (1MB).
-        #   * `:max_messages` (Integer) The maximum number of messages to be collected before the batch is published.
-        #     Default is 100.
-        #   * `:interval` (Numeric) The number of seconds to collect messages before the batch is published. Default is
-        #     0.01.
-        #   * `:threads` (Hash) The number of threads to create to handle concurrent calls by the publisher:
-        #     * `:publish` (Integer) The number of threads used to publish messages. Default is 2.
-        #     * `:callback` (Integer) The number of threads to handle the published messages' callbacks. Default is 4.
+        #   * `:max_bytes` (Integer) The maximum size of messages to be collected
+        #     before the batch is published. Default is 1,000,000 (1MB).
+        #   * `:max_messages` (Integer) The maximum number of messages to be
+        #     collected before the batch is published. Default is 100.
+        #   * `:interval` (Numeric) The number of seconds to collect messages before
+        #     the batch is published. Default is 0.01.
+        #   * `:threads` (Hash) The number of threads to create to handle concurrent
+        #     calls by the publisher:
+        #
+        #     * `:publish` (Integer) The number of threads used to publish messages.
+        #       Default is 2.
+        #     * `:callback` (Integer) The number of threads to handle the published
+        #       messages' callbacks. Default is 4.
+        #   * `:flow_control` (Hash) The client flow control settings for message publishing:
+        #     * `:message_limit` (Integer) The maximum number of messages allowed to wait to be published. Default is
+        #       `10 * max_messages`.
+        #     * `:byte_limit` (Integer) The maximum total size of messages allowed to wait to be published. Default is
+        #       `10 * max_bytes`.
+        #     * `:limit_exceeded_behavior` (Symbol) The action to take when publish flow control limits are exceeded.
+        #       Possible values include: `:ignore` - Flow control is disabled. `:error` - Calls to {Topic#publish_async}
+        #       will raise {FlowControlLimitError} when publish flow control limits are exceeded. `:block` - Calls to
+        #       {Topic#publish_async} will block until capacity is available when publish flow control limits are
+        #       exceeded. The default value is `:ignore`.
+        # @param [String] schema_name The name of the schema that messages
+        #   published should be validated against. Optional. The value can be a
+        #   simple schema ID (relative name), in which case the current project
+        #   ID will be supplied, or a fully-qualified schema name in the form
+        #   `projects/{project_id}/schemas/{schema_id}`. If provided,
+        #   `message_encoding` must also be provided.
+        # @param [String, Symbol] message_encoding The encoding of messages validated
+        #   against the schema identified by `schema_name`. Optional. Values include:
+        #
+        #   * `JSON` - JSON encoding.
+        #   * `BINARY` - Binary encoding, as defined by the schema type. For some
+        #     schema types, binary encoding may not be available.
         #
         # @return [Google::Cloud::PubSub::Topic]
         #
@@ -210,12 +247,20 @@ module Google
         #   pubsub = Google::Cloud::PubSub.new
         #   topic = pubsub.create_topic "my-topic"
         #
-        def create_topic topic_name, labels: nil, kms_key: nil, persistence_regions: nil, async: nil
+        def create_topic topic_name,
+                         labels: nil,
+                         kms_key: nil,
+                         persistence_regions: nil,
+                         async: nil,
+                         schema_name: nil,
+                         message_encoding: nil
           ensure_service!
           grpc = service.create_topic topic_name,
                                       labels:              labels,
                                       kms_key_name:        kms_key,
-                                      persistence_regions: persistence_regions
+                                      persistence_regions: persistence_regions,
+                                      schema_name:         schema_name,
+                                      message_encoding:    message_encoding
           Topic.from_grpc grpc, service, async: async
         end
         alias new_topic create_topic
@@ -387,6 +432,198 @@ module Google
         alias find_snapshots snapshots
         alias list_snapshots snapshots
 
+        ##
+        # Retrieves schema by name.
+        #
+        # @param [String] schema_name Name of a schema. The value can
+        #   be a simple schema ID, in which case the current project ID
+        #   will be supplied, or a fully-qualified schema name in the form
+        #   `projects/{project_id}/schemas/{schema_id}`.
+        # @param view [Symbol, String, nil] Possible values:
+        #   * `BASIC` - Include the `name` and `type` of the schema, but not the `definition`.
+        #   * `FULL` - Include all Schema object fields.
+        #
+        #   The default value is `FULL`.
+        # @param [String] project If the schema belongs to a project other
+        #   than the one currently connected to, the alternate project ID can be
+        #   specified here. Not used if a fully-qualified schema name is
+        #   provided for `schema_name`.
+        # @param [Boolean] skip_lookup Optionally create a {Schema} object
+        #   without verifying the schema resource exists on the Pub/Sub
+        #   service. Calls made on this object will raise errors if the service
+        #   resource does not exist. Default is `false`.
+        #
+        # @return [Google::Cloud::PubSub::Schema, nil] Returns `nil` if
+        #   the schema does not exist.
+        #
+        # @example
+        #   require "google/cloud/pubsub"
+        #
+        #   pubsub = Google::Cloud::PubSub.new
+        #
+        #   schema = pubsub.schema "my-schema"
+        #   schema.name #=> "projects/my-project/schemas/my-schema"
+        #   schema.type #=> :PROTOCOL_BUFFER
+        #   schema.definition # The schema definition
+        #
+        # @example Skip the lookup against the service with `skip_lookup`:
+        #   require "google/cloud/pubsub"
+        #
+        #   pubsub = Google::Cloud::PubSub.new
+        #
+        #   # No API call is made to retrieve the schema information.
+        #   # The default project is used in the name.
+        #   schema = pubsub.schema "my-schema", skip_lookup: true
+        #   schema.name #=> "projects/my-project/schemas/my-schema"
+        #   schema.type #=> nil
+        #   schema.definition #=> nil
+        #
+        # @example Omit the schema definition with `view: :basic`:
+        #   require "google/cloud/pubsub"
+        #
+        #   pubsub = Google::Cloud::PubSub.new
+        #
+        #   schema = pubsub.schema "my-schema", view: :basic
+        #   schema.name #=> "projects/my-project/schemas/my-schema"
+        #   schema.type #=> :PROTOCOL_BUFFER
+        #   schema.definition #=> nil
+        #
+        def schema schema_name, view: nil, project: nil, skip_lookup: nil
+          ensure_service!
+          options = { project: project }
+          return Schema.from_name schema_name, view, service, options if skip_lookup
+          view ||= :FULL
+          grpc = service.get_schema schema_name, view, options
+          Schema.from_grpc grpc, service
+        rescue Google::Cloud::NotFoundError
+          nil
+        end
+        alias get_schema schema
+        alias find_schema schema
+
+        ##
+        # Creates a new schema.
+        #
+        # @param [String] schema_id The ID to use for the schema, which will
+        #   become the final component of the schema's resource name. Required.
+        #
+        #   The schema ID (relative name) must start with a letter, and
+        #   contain only letters (`[A-Za-z]`), numbers (`[0-9]`), dashes (`-`),
+        #   underscores (`_`), periods (`.`), tildes (`~`), plus (`+`) or percent
+        #   signs (`%`). It must be between 3 and 255 characters in length, and
+        #   it must not start with `goog`.
+        # @param [String, Symbol] type The type of the schema. Required. Possible
+        #   values are case-insensitive and include:
+        #
+        #     * `PROTOCOL_BUFFER` - A Protocol Buffer schema definition.
+        #     * `AVRO` - An Avro schema definition.
+        # @param [String] definition  The definition of the schema. Required. This
+        #   should be a string representing the full definition of the schema that
+        #   is a valid schema definition of the type specified in `type`.
+        # @param [String] project If the schema belongs to a project other
+        #   than the one currently connected to, the alternate project ID can be
+        #   specified here. Optional.
+        #
+        # @return [Google::Cloud::PubSub::Schema]
+        #
+        # @example
+        #   require "google/cloud/pubsub"
+        #
+        #   pubsub = Google::Cloud::PubSub.new
+        #
+        #   definition = "..."
+        #   schema = pubsub.create_schema "my-schema", :avro, definition
+        #   schema.name #=> "projects/my-project/schemas/my-schema"
+        #
+        def create_schema schema_id, type, definition, project: nil
+          ensure_service!
+          type = type.to_s.upcase
+          grpc = service.create_schema schema_id, type, definition, project: project
+          Schema.from_grpc grpc, service
+        end
+        alias new_schema create_schema
+
+        ##
+        # Retrieves a list of schemas for the given project.
+        #
+        # @param view [String, Symbol, nil] The set of fields to return in the response. Possible values:
+        #
+        #     * `BASIC` - Include the `name` and `type` of the schema, but not the `definition`.
+        #     * `FULL` - Include all Schema object fields.
+        #
+        #   The default value is `FULL`.
+        # @param [String] token A previously-returned page token representing
+        #   part of the larger set of results to view.
+        # @param [Integer] max Maximum number of schemas to return.
+        #
+        # @return [Array<Google::Cloud::PubSub::Schema>] (See
+        #   {Google::Cloud::PubSub::Schema::List})
+        #
+        # @example
+        #   require "google/cloud/pubsub"
+        #
+        #   pubsub = Google::Cloud::PubSub.new
+        #
+        #   schemas = pubsub.schemas
+        #   schemas.each do |schema|
+        #     puts schema.name
+        #   end
+        #
+        # @example Retrieve all schemas: (See {Schema::List#all})
+        #   require "google/cloud/pubsub"
+        #
+        #   pubsub = Google::Cloud::PubSub.new
+        #
+        #   schemas = pubsub.schemas
+        #   schemas.all do |schema|
+        #     puts schema.name
+        #   end
+        #
+        def schemas view: nil, token: nil, max: nil
+          ensure_service!
+          view ||= :FULL
+          options = { token: token, max: max }
+          grpc = service.list_schemas view, options
+          Schema::List.from_grpc grpc, service, view, max
+        end
+        alias find_schemas schemas
+        alias list_schemas schemas
+
+        ##
+        # Validates a schema type and definition.
+        #
+        # @param [String, Symbol] type The type of the schema. Required. Possible
+        #   values are case-insensitive and include:
+        #
+        #     * `PROTOCOL_BUFFER` - A Protocol Buffer schema definition.
+        #     * `AVRO` - An Avro schema definition.
+        # @param [String] definition  The definition of the schema. Required. This
+        #   should be a string representing the full definition of the schema that
+        #   is a valid schema definition of the type specified in `type`.
+        # @param [String] project If the schema belongs to a project other
+        #   than the one currently connected to, the alternate project ID can be
+        #   specified here. Optional.
+        #
+        # @return [Boolean] `true` if the schema is valid, `false` otherwise.
+        #
+        # @example
+        #   require "google/cloud/pubsub"
+        #
+        #   pubsub = Google::Cloud::PubSub.new
+        #
+        #   definition = "..."
+        #   pubsub.validate_schema :avro, definition #=> true
+        #
+        def valid_schema? type, definition, project: nil
+          ensure_service!
+          type = type.to_s.upcase
+          service.validate_schema type, definition, project: project # return type is empty
+          true
+        rescue Google::Cloud::InvalidArgumentError
+          false
+        end
+        alias validate_schema valid_schema?
+
         protected
 
         ##