google-cloud-pubsub 2.6.1 → 2.7.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 24eb8350951100fd325571412067861f44eab7575cadd16a8daee35ebf8f26ae
-  data.tar.gz: ae3c0aef696e531b2f5bfb61bbff09f4f50e4c42e62d94b027b38a620858aa88
+  metadata.gz: f6636ac65229c0dc438c63fa9aebda474396701364bfbea7e5da460ceea50a26
+  data.tar.gz: a0c0cafbbaf4083c1141d851dc6cecb8cb70bc01df940a139fe21d057aa2fb97
 SHA512:
-  metadata.gz: 7103a23c4129c14745401b84e41f6522518d6c64b0a89871c2af1bca4464d6883304a7e752411d35bf2bc2e3a877b3cbffc3afffed71df935cb98a3ee696a347
-  data.tar.gz: 425057b02a2ced62a0e55db13eddd0c67f377d4a34b98f90d225cc79a1261c99afb9bb7b67f96198c8d183af7708c9f15b409e5fa73c8cdbf96176e6efb441dd
+  metadata.gz: a187e9935196b66d4be65c7cf7b34148de36197f9e2c5e9144ccc9f78652370d7a441b33e14a002133b05aa6565cc38998912277e8896d81178bf5712ce9df8c
+  data.tar.gz: ee801f9d5959f5e2a38ccf13beaac930ca4629c81a268b627a8bacc7ad9ce430581fd690100fa5ef57b4fa5e20e3c5f40d72e905ce2535d978398f03112a48eb

data/CHANGELOG.md

@@ -1,5 +1,19 @@
 # Release History
 
+### 2.7.0 / 2021-06-15
+
+#### Features
+
+* Add Publisher Flow Control
+  * Add flow_control to async options in Project#create_topic and Project#topic
+  * Add FlowControlLimitError
+
+#### Bug Fixes
+
+* Fix Project#schema and #schemas to return full resource
+  * Include schema definition in default return values.
+  * Fix Schema#definition to return nil instead of empty string when not present.
+
 ### 2.6.1 / 2021-04-28
 
 #### Bug Fixes

data/lib/google/cloud/pubsub.rb

@@ -33,8 +33,6 @@ module Google
     # See {file:OVERVIEW.md Google Cloud Pub/Sub Overview}.
     #
     module PubSub
-      # rubocop:disable Metrics/AbcSize
-
       ##
       # Creates a new object for connecting to the Pub/Sub service.
       # Each call creates a new connection.
@@ -110,8 +108,6 @@ module Google
         PubSub::Project.new service
       end
 
-      # rubocop:enable Metrics/AbcSize
-
       ##
       # Configure the Google Cloud PubSub library.
       #

data/lib/google/cloud/pubsub/async_publisher.rb

@@ -16,6 +16,7 @@
 require "monitor"
 require "concurrent"
 require "google/cloud/pubsub/errors"
+require "google/cloud/pubsub/flow_controller"
 require "google/cloud/pubsub/async_publisher/batch"
 require "google/cloud/pubsub/publish_result"
 require "google/cloud/pubsub/service"
@@ -65,14 +66,21 @@ module Google
         attr_reader :interval
         attr_reader :publish_threads
         attr_reader :callback_threads
+        attr_reader :flow_control
         ##
         # @private Implementation accessors
         attr_reader :service, :batch, :publish_thread_pool,
-                    :callback_thread_pool
+                    :callback_thread_pool, :flow_controller
 
         ##
         # @private Create a new instance of the object.
-        def initialize topic_name, service, max_bytes: 1_000_000, max_messages: 100, interval: 0.01, threads: {}
+        def initialize topic_name,
+                       service,
+                       max_bytes: 1_000_000,
+                       max_messages: 100,
+                       interval: 0.01,
+                       threads: {},
+                       flow_control: {}
           # init MonitorMixin
           super()
           @topic_name = service.topic_path topic_name
@@ -83,6 +91,10 @@ module Google
           @interval         = interval
           @publish_threads  = (threads[:publish] || 2).to_i
           @callback_threads = (threads[:callback] || 4).to_i
+          @flow_control = {
+            message_limit: 10 * @max_messages,
+            byte_limit: 10 * @max_bytes
+          }.merge(flow_control).freeze
 
           @published_at = nil
           @publish_thread_pool = Concurrent::ThreadPoolExecutor.new max_threads: @publish_threads
@@ -91,7 +103,7 @@ module Google
           @ordered = false
           @batches = {}
           @cond = new_cond
-
+          @flow_controller = FlowController.new(**@flow_control)
           @thread = Thread.new { run_background }
         end
 
@@ -121,13 +133,22 @@ module Google
         #
         def publish data = nil, attributes = nil, ordering_key: nil, **extra_attrs, &callback
           msg = Convert.pubsub_message data, attributes, ordering_key, extra_attrs
+          begin
+            @flow_controller.acquire msg.to_proto.bytesize
+          rescue FlowControlLimitError => e
+            stop_publish ordering_key, e if ordering_key
+            raise
+          end
 
           synchronize do
             raise AsyncPublisherStopped if @stopped
             raise OrderedMessagesDisabled if !@ordered && !msg.ordering_key.empty? # default is empty string
 
             batch = resolve_batch_for_message msg
-            raise OrderingKeyError, batch.ordering_key if batch.canceled?
+            if batch.canceled?
+              @flow_controller.release msg.to_proto.bytesize
+              raise OrderingKeyError, batch.ordering_key
+            end
             batch_action = batch.add msg, callback
             if batch_action == :full
               publish_batches!
@@ -305,6 +326,21 @@ module Google
           @batches[ordering_key]
         end
 
+        def stop_publish ordering_key, err
+          synchronize do
+            batch = resolve_batch_for_ordering_key ordering_key
+            return if batch.nil?
+            items = batch.cancel!
+            items.each do |item|
+              @flow_controller.release item.bytesize
+              next unless item.callback
+
+              publish_result = PublishResult.from_error item.msg, err
+              execute_callback_async item.callback, publish_result
+            end
+          end
+        end
+
         def publish_batches! stop: nil
           @batches.reject! { |_ordering_key, batch| batch.empty? }
           @batches.each_value do |batch|
@@ -325,7 +361,6 @@ module Google
         end
 
         # rubocop:disable Metrics/AbcSize
-        # rubocop:disable Metrics/MethodLength
 
         def publish_batch_sync topic_name, batch
           # The only batch methods that are safe to call from the loop are
@@ -337,6 +372,7 @@ module Google
             unless items.empty?
               grpc = @service.publish topic_name, items.map(&:msg)
               items.zip Array(grpc.message_ids) do |item, id|
+                @flow_controller.release item.bytesize
                 next unless item.callback
 
                 item.msg.message_id = id
@@ -363,6 +399,7 @@ module Google
           end
 
           items.each do |item|
+            @flow_controller.release item.bytesize
             next unless item.callback
 
             publish_result = PublishResult.from_error item.msg, e
@@ -374,7 +411,6 @@ module Google
         end
 
         # rubocop:enable Metrics/AbcSize
-        # rubocop:enable Metrics/MethodLength
 
         PUBLISH_RETRY_ERRORS = [
           GRPC::Cancelled, GRPC::DeadlineExceeded, GRPC::Internal,

data/lib/google/cloud/pubsub/async_publisher/batch.rb

@@ -129,8 +129,8 @@ module Google
           end
 
           ##
-          # Rebalances the batch by moving any queued items that will fit into
-          # the active item list.
+          # Fills the batch by sequentially moving the queued items that will
+          # fit into the active item list.
           #
           # This method is only intended to be used by the active publishing
           # job.
@@ -152,8 +152,6 @@ module Google
             end
           end
 
-          # rubocop:disable Metrics/MethodLength
-
           ##
           # Resets the batch after a successful publish. This clears the active
           # item list and moves the queued items that will fit into the active
@@ -202,8 +200,6 @@ module Google
             true
           end
 
-          # rubocop:enable Metrics/MethodLength
-
           ##
           # Cancel the batch and hault futher batches until resumed. See
           # {#resume!} and {#canceled?}.

data/lib/google/cloud/pubsub/errors.rb

@@ -78,6 +78,14 @@ module Google
           super "Can't publish message using #{ordering_key}."
         end
       end
+
+      ##
+      # Raised when the desired action is `error` and the message would exceed
+      # flow control limits, or when the desired action is `block` and the
+      # message would block forever against the flow control limits.
+      #
+      class FlowControlLimitError < Google::Cloud::Error
+      end
     end
 
     Pubsub = PubSub unless const_defined? :Pubsub

data/lib/google/cloud/pubsub/flow_controller.rb

@@ -0,0 +1,139 @@
+# Copyright 2021 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+require "google/cloud/pubsub/errors"
+require "concurrent/atomics"
+
+module Google
+  module Cloud
+    module PubSub
+      ##
+      # @private
+      #
+      # Used to control the flow of messages passing through it.
+      #
+      class FlowController
+        attr_reader :message_limit
+        attr_reader :byte_limit
+        attr_reader :limit_exceeded_behavior
+        ##
+        # @private Implementation accessors
+        attr_reader :outstanding_messages, :outstanding_bytes, :awaiting
+
+        def initialize message_limit: 1000, byte_limit: 10_000_000, limit_exceeded_behavior: :ignore
+          unless [:ignore, :error, :block].include? limit_exceeded_behavior
+            raise ArgumentError, "limit_exceeded_behavior must be one of :ignore, :error, :block"
+          end
+          if [:error, :block].include?(limit_exceeded_behavior) && message_limit < 1
+            raise ArgumentError,
+                  "Flow control message limit (#{message_limit}) exceeded by a single message, would block forever"
+          end
+          @mutex = Mutex.new
+          @message_limit = message_limit
+          @byte_limit = byte_limit
+          @limit_exceeded_behavior = limit_exceeded_behavior
+          @outstanding_messages = 0
+          @outstanding_bytes = 0
+
+          @awaiting = []
+        end
+
+        def acquire message_size
+          return if limit_exceeded_behavior == :ignore
+          @mutex.lock
+          if limit_exceeded_behavior == :error && would_exceed_message_limit?
+            raise FlowControlLimitError, "Flow control message limit (#{message_limit}) would be exceeded"
+          end
+          if limit_exceeded_behavior == :error && would_exceed_byte_limit?(message_size)
+            raise FlowControlLimitError,
+                  "Flow control byte limit (#{byte_limit}) would be exceeded, message_size: #{message_size}"
+          end
+          if limit_exceeded_behavior == :block && message_size > byte_limit
+            raise FlowControlLimitError,
+                  "Flow control byte limit (#{byte_limit}) exceeded by a single message, would block forever"
+          end
+
+          acquire_or_wait message_size
+        ensure
+          @mutex.unlock if @mutex.owned?
+        end
+
+        def release message_size
+          return if limit_exceeded_behavior == :ignore
+          @mutex.synchronize do
+            raise "Flow control messages count would be negative" if (@outstanding_messages - 1).negative?
+            raise "Flow control bytes count would be negative" if (@outstanding_bytes - message_size).negative?
+
+            @outstanding_messages -= 1
+            @outstanding_bytes -= message_size
+            @awaiting.first.set unless @awaiting.empty?
+          end
+        end
+
+        protected
+
+        # rubocop:disable Style/IdenticalConditionalBranches
+        # rubocop:disable Style/GuardClause
+
+        def acquire_or_wait message_size
+          waiter = nil
+          while is_new_and_others_wait?(waiter) ||
+                would_exceed_byte_limit?(message_size) ||
+                would_exceed_message_limit?
+
+            if waiter.nil?
+              waiter = Concurrent::Event.new
+              # This waiter gets added to the back of the line.
+              @awaiting << waiter
+            else
+              waiter = Concurrent::Event.new
+              # This waiter already in line stays at the head of the line.
+              @awaiting[0] = waiter
+            end
+            @mutex.unlock
+            waiter.wait
+            @mutex.lock
+          end
+          @outstanding_messages += 1
+          @outstanding_bytes += message_size
+
+          @awaiting.shift if waiter # Remove the newly released waiter from the head of the queue.
+
+          # There may be some surplus left; let the next message waiting try to acquire a permit.
+          if !@awaiting.empty? && @outstanding_bytes < byte_limit && @outstanding_messages < message_limit
+            @awaiting.first.set
+          end
+        end
+
+        # rubocop:enable Style/IdenticalConditionalBranches
+        # rubocop:enable Style/GuardClause
+
+        def is_new_and_others_wait? waiter
+          waiter.nil? && !@awaiting.empty?
+        end
+
+        def would_exceed_message_limit?
+          @outstanding_messages + 1 > message_limit
+        end
+
+        def would_exceed_byte_limit? bytes_requested
+          @outstanding_bytes + bytes_requested > byte_limit
+        end
+      end
+    end
+
+    Pubsub = PubSub unless const_defined? :Pubsub
+  end
+end

data/lib/google/cloud/pubsub/project.rb

@@ -103,6 +103,16 @@ module Google
         #   * `:threads` (Hash) The number of threads to create to handle concurrent calls by the publisher:
         #     * `:publish` (Integer) The number of threads used to publish messages. Default is 2.
         #     * `:callback` (Integer) The number of threads to handle the published messages' callbacks. Default is 4.
+        #   * `:flow_control` (Hash) The client flow control settings for message publishing:
+        #     * `:message_limit` (Integer) The maximum number of messages allowed to wait to be published. Default is
+        #       `10 * max_messages`.
+        #     * `:byte_limit` (Integer) The maximum total size of messages allowed to wait to be published. Default is
+        #       `10 * max_bytes`.
+        #     * `:limit_exceeded_behavior` (Symbol) The action to take when publish flow control limits are exceeded.
+        #       Possible values include: `:ignore` - Flow control is disabled. `:error` - Calls to {Topic#publish_async}
+        #       will raise {FlowControlLimitError} when publish flow control limits are exceeded. `:block` - Calls to
+        #       {Topic#publish_async} will block until capacity is available when publish flow control limits are
+        #       exceeded. The default value is `:ignore`.
         #
         # @return [Google::Cloud::PubSub::Topic, nil] Returns `nil` if topic
         #   does not exist.
@@ -206,6 +216,16 @@ module Google
         #       Default is 2.
         #     * `:callback` (Integer) The number of threads to handle the published
         #       messages' callbacks. Default is 4.
+        #   * `:flow_control` (Hash) The client flow control settings for message publishing:
+        #     * `:message_limit` (Integer) The maximum number of messages allowed to wait to be published. Default is
+        #       `10 * max_messages`.
+        #     * `:byte_limit` (Integer) The maximum total size of messages allowed to wait to be published. Default is
+        #       `10 * max_bytes`.
+        #     * `:limit_exceeded_behavior` (Symbol) The action to take when publish flow control limits are exceeded.
+        #       Possible values include: `:ignore` - Flow control is disabled. `:error` - Calls to {Topic#publish_async}
+        #       will raise {FlowControlLimitError} when publish flow control limits are exceeded. `:block` - Calls to
+        #       {Topic#publish_async} will block until capacity is available when publish flow control limits are
+        #       exceeded. The default value is `:ignore`.
         # @param [String] schema_name The name of the schema that messages
         #   published should be validated against. Optional. The value can be a
         #   simple schema ID (relative name), in which case the current project
@@ -423,7 +443,7 @@ module Google
         #   * `BASIC` - Include the `name` and `type` of the schema, but not the `definition`.
         #   * `FULL` - Include all Schema object fields.
         #
-        #   The default value is `BASIC`.
+        #   The default value is `FULL`.
         # @param [String] project If the schema belongs to a project other
         #   than the one currently connected to, the alternate project ID can be
         #   specified here. Not used if a fully-qualified schema name is
@@ -444,7 +464,7 @@ module Google
         #   schema = pubsub.schema "my-schema"
         #   schema.name #=> "projects/my-project/schemas/my-schema"
         #   schema.type #=> :PROTOCOL_BUFFER
-        #   # schema.definition # nil - Use view: :full to load complete resource.
+        #   schema.definition # The schema definition
         #
         # @example Skip the lookup against the service with `skip_lookup`:
         #   require "google/cloud/pubsub"
@@ -458,21 +478,21 @@ module Google
         #   schema.type #=> nil
         #   schema.definition #=> nil
         #
-        # @example Get the schema definition with `view: :full`:
+        # @example Omit the schema definition with `view: :basic`:
         #   require "google/cloud/pubsub"
         #
         #   pubsub = Google::Cloud::PubSub.new
         #
-        #   schema = pubsub.schema "my-schema", view: :full
+        #   schema = pubsub.schema "my-schema", view: :basic
         #   schema.name #=> "projects/my-project/schemas/my-schema"
         #   schema.type #=> :PROTOCOL_BUFFER
-        #   schema.definition # The schema definition
+        #   schema.definition #=> nil
         #
         def schema schema_name, view: nil, project: nil, skip_lookup: nil
           ensure_service!
           options = { project: project }
           return Schema.from_name schema_name, view, service, options if skip_lookup
-          view ||= :BASIC
+          view ||= :FULL
           grpc = service.get_schema schema_name, view, options
           Schema.from_grpc grpc, service
         rescue Google::Cloud::NotFoundError
@@ -531,7 +551,7 @@ module Google
         #     * `BASIC` - Include the `name` and `type` of the schema, but not the `definition`.
         #     * `FULL` - Include all Schema object fields.
         #
-        #   The default value is `BASIC`.
+        #   The default value is `FULL`.
         # @param [String] token A previously-returned page token representing
         #   part of the larger set of results to view.
         # @param [Integer] max Maximum number of schemas to return.
@@ -561,7 +581,7 @@ module Google
         #
         def schemas view: nil, token: nil, max: nil
           ensure_service!
-          view ||= :BASIC
+          view ||= :FULL
           options = { token: token, max: max }
           grpc = service.list_schemas view, options
           Schema::List.from_grpc grpc, service, view, max

data/lib/google/cloud/pubsub/schema.rb

@@ -48,7 +48,7 @@ module Google
           @grpc = grpc
           @service = service
           @exists = nil
-          @view = view || :BASIC
+          @view = view || :FULL
         end
 
         ##
@@ -81,7 +81,7 @@ module Google
         #
         def definition
           return nil if reference?
-          @grpc.definition
+          @grpc.definition if @grpc.definition && !@grpc.definition.empty?
         end
 
         ##
@@ -141,7 +141,7 @@ module Google
         #   * `FULL` - Include all Schema object fields.
         #
         #   Optional. If not provided or `nil`, the last non-nil `view` argument to this method will be used if one has
-        #   been given, othewise `BASIC` will be used.
+        #   been given, othewise `FULL` will be used.
         #
         # @return [Google::Cloud::PubSub::Schema] Returns the reloaded schema.
         #
@@ -153,7 +153,7 @@ module Google
         #
         #   schema.reload!
         #
-        # @example Use the `view` option to load the full resource:
+        # @example Use the `view` option to load the basic or full resource:
         #   require "google/cloud/pubsub"
         #
         #   pubsub = Google::Cloud::PubSub.new
@@ -267,7 +267,7 @@ module Google
         #   require "google/cloud/pubsub"
         #
         #   pubsub = Google::Cloud::PubSub.new
-        #   schema = pubsub.schema "my-schema", view: :full
+        #   schema = pubsub.schema "my-schema"
         #
         #   schema.resource_full? #=> true
         #

data/lib/google/cloud/pubsub/topic.rb

@@ -684,6 +684,11 @@ module Google
         # @note At the time of this release, ordering keys are not yet publicly
         #   enabled and requires special project enablements.
         #
+        # Publisher flow control limits the number of outstanding messages that
+        # are allowed to wait to be published. See the `flow_control` key in the
+        # `async` parameter in {Project#topic} for more information about publisher
+        # flow control settings.
+        #
         # @param [String, File] data The message payload. This will be converted
         #   to bytes encoded as ASCII-8BIT.
         # @param [Hash] attributes Optional attributes for the message.
@@ -703,6 +708,13 @@ module Google
         #   message with an `ordering_key` that has already failed when
         #   publishing. Use {#resume_publish} to allow this `ordering_key` to be
         #   published again.
+        # @raise [Google::Cloud::PubSub::FlowControlLimitError] when publish flow
+        #   control limits are exceeded, and the `async` parameter key
+        #   `flow_control.limit_exceeded_behavior` is set to `:error` or `:block`.
+        #   If `flow_control.limit_exceeded_behavior` is set to `:block`, this error
+        #   will be raised only when a limit would be exceeded by a single message.
+        #   See the `async` parameter in {Project#topic} for more information about
+        #   `flow_control` settings.
         #
         # @example
         #   require "google/cloud/pubsub"

data/lib/google/cloud/pubsub/version.rb

@@ -16,7 +16,7 @@
 module Google
   module Cloud
     module PubSub
-      VERSION = "2.6.1".freeze
+      VERSION = "2.7.0".freeze
     end
 
     Pubsub = PubSub unless const_defined? :Pubsub

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: google-cloud-pubsub
 version: !ruby/object:Gem::Version
-  version: 2.6.1
+  version: 2.7.0
 platform: ruby
 authors:
 - Mike Moore
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-04-28 00:00:00.000000000 Z
+date: 2021-06-15 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: concurrent-ruby
@@ -233,6 +233,7 @@ files:
 - lib/google/cloud/pubsub/convert.rb
 - lib/google/cloud/pubsub/credentials.rb
 - lib/google/cloud/pubsub/errors.rb
+- lib/google/cloud/pubsub/flow_controller.rb
 - lib/google/cloud/pubsub/message.rb
 - lib/google/cloud/pubsub/policy.rb
 - lib/google/cloud/pubsub/project.rb
@@ -275,7 +276,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.2.16
+rubygems_version: 3.2.17
 signing_key: 
 specification_version: 4
 summary: API Client library for Google Cloud Pub/Sub