google-cloud-pubsub 2.4.0 → 2.7.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: a4148f77602c7b9031ed772a14735b25414034684472928fb5244a3c07c2c080
-  data.tar.gz: 247fd6d4697c8e4f04858487441a3552f30ebe3010720861a1e2c9e8198de219
+  metadata.gz: 55eb80f365cf82b22f7cb9383a88155e0c79e0d5c1e897f2c7ee6cb974e2989b
+  data.tar.gz: 4417a272576757638dbe5b122e832412303f55761fb6d4d9ae24706ecdc9842f
 SHA512:
-  metadata.gz: c926ed2d0bca88e81a0545e86ca4fdd916df9d24f1108e4e74cf9140c0a835b79fb6fc207074d62a920de0958e69071235d0c8874970aae9db3657e24c36b2e1
-  data.tar.gz: 3f719066bf2ffc015b88dd0ffc5f13c38d853c2e54603daa6190905420a87b4aeb9372e4d1bcb37d68863ac38151d147d465ec3891333608718ede84123dca92
+  metadata.gz: 8c1a8a9c9e62d737a468fbaca9c3909be1c608ec64d223f7ade2a478870c4e275cfad10e8c743dd213ce2f3d9084271f29cca6ba91205cda224177897d174016
+  data.tar.gz: 96cd5e4e593894f6c98401c9dff2ca99d7be2acdbb37504f5f53ab72cda2c9e29bde031facd7d779947deb892454716bc151f31d67eddf58b3f8944e46e66018

data/AUTHENTICATION.md

@@ -96,7 +96,8 @@ client = Google::Cloud::PubSub.new
 
 ### Configuration
 
-The **Project ID** and **Credentials JSON** can be configured instead of placing them in environment variables or providing them as arguments.
+The **Project ID** and the path to the **Credentials JSON** file can be configured
+instead of placing them in environment variables or providing them as arguments.
 
 ```ruby
 require "google/cloud/pubsub"

data/CHANGELOG.md

@@ -1,5 +1,60 @@
 # Release History
 
+### 2.7.1 / 2021-07-08
+
+#### Documentation
+
+* Update AUTHENTICATION.md in handwritten packages
+
+### 2.7.0 / 2021-06-15
+
+#### Features
+
+* Add Publisher Flow Control
+  * Add flow_control to async options in Project#create_topic and Project#topic
+  * Add FlowControlLimitError
+
+#### Bug Fixes
+
+* Fix Project#schema and #schemas to return full resource
+  * Include schema definition in default return values.
+  * Fix Schema#definition to return nil instead of empty string when not present.
+
+### 2.6.1 / 2021-04-28
+
+#### Bug Fixes
+
+* Add final flush of pending requests to Subscriber#wait!
+  * fix(pubsub): Add final flush of pending requests to Subscriber#wait!
+
+### 2.6.0 / 2021-04-19
+
+#### Features
+
+* Add ordering_key to Topic#publish
+  * Add ordering_key to BatchPublisher#publish
+
+#### Documentation
+
+* The immediate: false option is recommended to avoid adverse impacts on the performance of pull operations ([#11153](https://www.github.com/googleapis/google-cloud-ruby/issues/11153))
+* Update Subscription#pull docs and samples to recommend immediate: false
+
+### 2.5.0 / 2021-04-01
+
+#### Features
+
+* Add Schema support
+  * Add Schema
+  * Add Project#create_schema
+  * Add Project#schema
+  * Add Project#schemas (Schema::List)
+  * Add Project#valid_schema?
+  * Add schema options to Project#create_topic
+  * Add Topic#schema_name
+  * Add Topic#message_encoding
+  * Add Topic#message_encoding_binary?
+  * Add Topic#message_encoding_json?
+
 ### 2.4.0 / 2021-03-10
 
 #### Features

data/CONTRIBUTING.md

@@ -119,15 +119,14 @@ If you alter an example's title, you may encounter breaking tests.
 ### Pub/Sub Acceptance Tests
 
 The Pub/Sub acceptance tests interact with the live service API. Follow the
-instructions in the {file:AUTHENTICATION.md Authentication guide} for enabling
+instructions in the {file:AUTHENTICATION.md Authentication Guide} for enabling
 the Pub/Sub API. Occasionally, some API features may not yet be generally
 available, making it difficult for some contributors to successfully run the
 entire acceptance test suite. However, please ensure that you do successfully
 run acceptance tests for any code areas covered by your pull request.
 
 To run the acceptance tests, first create and configure a project in the Google
-Developers Console, as described in the {file:AUTHENTICATION.md Authentication
-guide}. Be sure to download the JSON KEY file. Make note of the PROJECT_ID and
+Developers Console, as described in the {file:AUTHENTICATION.md Authentication Guide}. Be sure to download the JSON KEY file. Make note of the PROJECT_ID and
 the KEYFILE location on your system.
 
 Before you can run the Pub/Sub acceptance tests, you must first create indexes

data/OVERVIEW.md

@@ -205,13 +205,16 @@ sleep
 Messages also can be pulled directly in a one-time operation. (See
 {Google::Cloud::PubSub::Subscription#pull Subscription#pull})
 
+The `immediate: false` option is recommended to avoid adverse impacts on the
+performance of pull operations.
+
 ```ruby
 require "google/cloud/pubsub"
 
 pubsub = Google::Cloud::PubSub.new
 
 sub = pubsub.subscription "my-topic-sub"
-received_messages = sub.pull
+received_messages = sub.pull immediate: false
 ```
 
 A maximum number of messages to pull can be specified:
@@ -222,7 +225,7 @@ require "google/cloud/pubsub"
 pubsub = Google::Cloud::PubSub.new
 
 sub = pubsub.subscription "my-topic-sub"
-received_messages = sub.pull max: 10
+received_messages = sub.pull immediate: false, max: 10
 ```
 
 ## Acknowledging a Message
@@ -263,7 +266,7 @@ require "google/cloud/pubsub"
 pubsub = Google::Cloud::PubSub.new
 
 sub = pubsub.subscription "my-topic-sub"
-received_messages = sub.pull
+received_messages = sub.pull immediate: false
 sub.acknowledge received_messages
 ```
 
@@ -328,7 +331,7 @@ require "google/cloud/pubsub"
 pubsub = Google::Cloud::PubSub.new
 
 sub = pubsub.subscription "my-topic-sub"
-received_messages = sub.pull
+received_messages = sub.pull immediate: false
 sub.modify_ack_deadline 120, received_messages
 ```
 
@@ -474,7 +477,7 @@ sub = pubsub.subscription "my-topic-sub"
 
 snapshot = sub.create_snapshot
 
-received_messages = sub.pull
+received_messages = sub.pull immediate: false
 sub.acknowledge received_messages
 
 sub.seek snapshot
@@ -521,5 +524,5 @@ sub.topic.name #=> "projects/other-project-id/topics/other-topic"
 ## Additional information
 
 Google Cloud Pub/Sub can be configured to use an emulator or to enable gRPC's
-logging. To learn more, see the {file:EMULATOR.md Emulator guide} and
+logging. To learn more, see the {file:EMULATOR.md Emulator guide}} and
 {file:LOGGING.md Logging guide}.

data/lib/google/cloud/pubsub.rb

@@ -33,8 +33,6 @@ module Google
     # See {file:OVERVIEW.md Google Cloud Pub/Sub Overview}.
     #
     module PubSub
-      # rubocop:disable Metrics/AbcSize
-
       ##
       # Creates a new object for connecting to the Pub/Sub service.
       # Each call creates a new connection.
@@ -110,8 +108,6 @@ module Google
         PubSub::Project.new service
       end
 
-      # rubocop:enable Metrics/AbcSize
-
       ##
       # Configure the Google Cloud PubSub library.
       #

data/lib/google/cloud/pubsub/async_publisher.rb

@@ -16,6 +16,7 @@
 require "monitor"
 require "concurrent"
 require "google/cloud/pubsub/errors"
+require "google/cloud/pubsub/flow_controller"
 require "google/cloud/pubsub/async_publisher/batch"
 require "google/cloud/pubsub/publish_result"
 require "google/cloud/pubsub/service"
@@ -65,14 +66,21 @@ module Google
         attr_reader :interval
         attr_reader :publish_threads
         attr_reader :callback_threads
+        attr_reader :flow_control
         ##
         # @private Implementation accessors
         attr_reader :service, :batch, :publish_thread_pool,
-                    :callback_thread_pool
+                    :callback_thread_pool, :flow_controller
 
         ##
         # @private Create a new instance of the object.
-        def initialize topic_name, service, max_bytes: 1_000_000, max_messages: 100, interval: 0.01, threads: {}
+        def initialize topic_name,
+                       service,
+                       max_bytes: 1_000_000,
+                       max_messages: 100,
+                       interval: 0.01,
+                       threads: {},
+                       flow_control: {}
           # init MonitorMixin
           super()
           @topic_name = service.topic_path topic_name
@@ -83,6 +91,10 @@ module Google
           @interval         = interval
           @publish_threads  = (threads[:publish] || 2).to_i
           @callback_threads = (threads[:callback] || 4).to_i
+          @flow_control = {
+            message_limit: 10 * @max_messages,
+            byte_limit: 10 * @max_bytes
+          }.merge(flow_control).freeze
 
           @published_at = nil
           @publish_thread_pool = Concurrent::ThreadPoolExecutor.new max_threads: @publish_threads
@@ -91,7 +103,7 @@ module Google
           @ordered = false
           @batches = {}
           @cond = new_cond
-
+          @flow_controller = FlowController.new(**@flow_control)
           @thread = Thread.new { run_background }
         end
 
@@ -121,13 +133,22 @@ module Google
         #
         def publish data = nil, attributes = nil, ordering_key: nil, **extra_attrs, &callback
           msg = Convert.pubsub_message data, attributes, ordering_key, extra_attrs
+          begin
+            @flow_controller.acquire msg.to_proto.bytesize
+          rescue FlowControlLimitError => e
+            stop_publish ordering_key, e if ordering_key
+            raise
+          end
 
           synchronize do
             raise AsyncPublisherStopped if @stopped
             raise OrderedMessagesDisabled if !@ordered && !msg.ordering_key.empty? # default is empty string
 
             batch = resolve_batch_for_message msg
-            raise OrderingKeyError, batch.ordering_key if batch.canceled?
+            if batch.canceled?
+              @flow_controller.release msg.to_proto.bytesize
+              raise OrderingKeyError, batch.ordering_key
+            end
             batch_action = batch.add msg, callback
             if batch_action == :full
               publish_batches!
@@ -305,6 +326,21 @@ module Google
           @batches[ordering_key]
         end
 
+        def stop_publish ordering_key, err
+          synchronize do
+            batch = resolve_batch_for_ordering_key ordering_key
+            return if batch.nil?
+            items = batch.cancel!
+            items.each do |item|
+              @flow_controller.release item.bytesize
+              next unless item.callback
+
+              publish_result = PublishResult.from_error item.msg, err
+              execute_callback_async item.callback, publish_result
+            end
+          end
+        end
+
         def publish_batches! stop: nil
           @batches.reject! { |_ordering_key, batch| batch.empty? }
           @batches.each_value do |batch|
@@ -325,7 +361,6 @@ module Google
         end
 
         # rubocop:disable Metrics/AbcSize
-        # rubocop:disable Metrics/MethodLength
 
         def publish_batch_sync topic_name, batch
           # The only batch methods that are safe to call from the loop are
@@ -337,6 +372,7 @@ module Google
             unless items.empty?
               grpc = @service.publish topic_name, items.map(&:msg)
               items.zip Array(grpc.message_ids) do |item, id|
+                @flow_controller.release item.bytesize
                 next unless item.callback
 
                 item.msg.message_id = id
@@ -363,6 +399,7 @@ module Google
           end
 
           items.each do |item|
+            @flow_controller.release item.bytesize
             next unless item.callback
 
             publish_result = PublishResult.from_error item.msg, e
@@ -374,7 +411,6 @@ module Google
         end
 
         # rubocop:enable Metrics/AbcSize
-        # rubocop:enable Metrics/MethodLength
 
         PUBLISH_RETRY_ERRORS = [
           GRPC::Cancelled, GRPC::DeadlineExceeded, GRPC::Internal,

data/lib/google/cloud/pubsub/async_publisher/batch.rb

@@ -129,8 +129,8 @@ module Google
           end
 
           ##
-          # Rebalances the batch by moving any queued items that will fit into
-          # the active item list.
+          # Fills the batch by sequentially moving the queued items that will
+          # fit into the active item list.
           #
           # This method is only intended to be used by the active publishing
           # job.
@@ -152,8 +152,6 @@ module Google
             end
           end
 
-          # rubocop:disable Metrics/MethodLength
-
           ##
           # Resets the batch after a successful publish. This clears the active
           # item list and moves the queued items that will fit into the active
@@ -202,8 +200,6 @@ module Google
             true
           end
 
-          # rubocop:enable Metrics/MethodLength
-
           ##
           # Cancel the batch and hault futher batches until resumed. See
           # {#resume!} and {#canceled?}.

data/lib/google/cloud/pubsub/batch_publisher.rb

@@ -13,6 +13,8 @@
 # limitations under the License.
 
 
+require "google/cloud/pubsub/convert"
+
 module Google
   module Cloud
     module PubSub
@@ -26,11 +28,12 @@ module Google
       #   pubsub = Google::Cloud::PubSub.new
       #
       #   topic = pubsub.topic "my-topic"
-      #   msgs = topic.publish do |t|
-      #     t.publish "task 1 completed", foo: :bar
-      #     t.publish "task 2 completed", foo: :baz
-      #     t.publish "task 3 completed", foo: :bif
+      #   msgs = topic.publish do |batch_publisher|
+      #     batch_publisher.publish "task 1 completed", foo: :bar
+      #     batch_publisher.publish "task 2 completed", foo: :baz
+      #     batch_publisher.publish "task 3 completed", foo: :bif
       #   end
+      #
       class BatchPublisher
         ##
         # @private The messages to publish
@@ -38,20 +41,40 @@ module Google
 
         ##
         # @private Create a new instance of the object.
-        def initialize data = nil, attributes = {}
+        def initialize data, attributes, ordering_key, extra_attrs
           @messages = []
           @mode = :batch
           return if data.nil?
           @mode = :single
-          publish data, attributes
+          publish data, attributes, ordering_key: ordering_key, **extra_attrs
         end
 
         ##
         # Add a message to the batch to be published to the topic.
         # All messages added to the batch will be published at once.
         # See {Google::Cloud::PubSub::Topic#publish}
-        def publish data, attributes = {}
-          @messages << create_pubsub_message(data, attributes)
+        #
+        # @param [String, File] data The message payload. This will be converted
+        #   to bytes encoded as ASCII-8BIT.
+        # @param [Hash] attributes Optional attributes for the message.
+        # @param [String] ordering_key Identifies related messages for which
+        #   publish order should be respected.
+        #
+        # @example Multiple messages can be sent at the same time using a block:
+        #   require "google/cloud/pubsub"
+        #
+        #   pubsub = Google::Cloud::PubSub.new
+        #
+        #   topic = pubsub.topic "my-topic"
+        #   msgs = topic.publish do |batch_publisher|
+        #     batch_publisher.publish "task 1 completed", foo: :bar
+        #     batch_publisher.publish "task 2 completed", foo: :baz
+        #     batch_publisher.publish "task 3 completed", foo: :bif
+        #   end
+        #
+        def publish data, attributes = nil, ordering_key: nil, **extra_attrs
+          msg = Convert.pubsub_message data, attributes, ordering_key, extra_attrs
+          @messages << msg
         end
 
         ##
@@ -69,28 +92,6 @@ module Google
             msgs
           end
         end
-
-        protected
-
-        def create_pubsub_message data, attributes
-          attributes ||= {}
-          if data.is_a?(::Hash) && attributes.empty?
-            attributes = data
-            data = nil
-          end
-          # Convert IO-ish objects to strings
-          if data.respond_to?(:read) && data.respond_to?(:rewind)
-            data.rewind
-            data = data.read
-          end
-          # Convert data to encoded byte array to match the protobuf defn
-          data_bytes = String(data).dup.force_encoding(Encoding::ASCII_8BIT).freeze
-
-          # Convert attributes to strings to match the protobuf definition
-          attributes = Hash[attributes.map { |k, v| [String(k), String(v)] }]
-
-          Google::Cloud::PubSub::V1::PubsubMessage.new data: data_bytes, attributes: attributes
-        end
       end
     end
 

data/lib/google/cloud/pubsub/errors.rb

@@ -78,6 +78,14 @@ module Google
           super "Can't publish message using #{ordering_key}."
         end
       end
+
+      ##
+      # Raised when the desired action is `error` and the message would exceed
+      # flow control limits, or when the desired action is `block` and the
+      # message would block forever against the flow control limits.
+      #
+      class FlowControlLimitError < Google::Cloud::Error
+      end
     end
 
     Pubsub = PubSub unless const_defined? :Pubsub