google-cloud-pubsub 1.1.3 → 1.2.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 1feb85392aa3adbb0358718fbd2b3f9cfad766c2dd30fa03ffdd93a0ab9199f3
-  data.tar.gz: 726f51f0d1851d82de1ee42230da2480a39d1dc18e3ebe418afdb03ce28fa6ea
+  metadata.gz: 75209fd682462904d589c3279cfad8bf7fdd1c546646a9f70c3ab419e14072a2
+  data.tar.gz: 11145f0b0969362c0ee516e69eb8d800b54f7e51a4f99fdb5febe3c6c3ec69dd
 SHA512:
-  metadata.gz: 9c9b2ac59f9fac674bb1bfbddf8d5a23cf768986fbd13e0e41128bf86574884b3d46cd88c564f9b28a531dfebaf6ba05bced34d5f6ee50664442876eea601d6a
-  data.tar.gz: e3e93f611b142aad3e8323d3aeae2339c810924f31369370e668063da54af64806b0270db81c75b2e6361537ec596d238a5bad4c8cdf79a4d818af08790efc6b
+  metadata.gz: 60ed4425c9b18550f4e8fdc849b374aa2dc5c0a6f9425df3e3afbdda85210f0d90d2e3dfde1ba97093c5beb6d22c1db713cd34dcc3a88160719c614ef45f9527
+  data.tar.gz: b1c6e793946e32530e5ad017a30951e08b287889e207d32a09e17db5b9d8d6f81cb697eaa42bd9067b707217cf260ab15bff359c6f70a99b5f9f41767600d359

data/CHANGELOG.md

@@ -1,5 +1,23 @@
 # Release History
 
+### 1.2.0 / 2020-01-09
+
+#### Features
+
+* Add Subscriber inventory settings
+  * Add the following settings to Subscriber:
+    * Subscriber#inventory_limit
+    * Subscriber#inventory_bytesize
+    * Subscriber#extension
+  * Allow Subscription#listen inventory argument to be a hash.
+* Update AsyncPublisher configuration defaults
+  * Update AsyncPublisher defaults to the following:
+    * max_bytes to 1MB, was 10MB.
+    * max_messages to 100, was 1,000.
+    * interval to 10 milliseconds, was 250 milliseconds.
+    * publish thread count to 2, was 4
+    * callback thread count to 4, was 8.
+
 ### 1.1.3 / 2019-12-18
 
 #### Bug Fixes

data/lib/google/cloud/pubsub/async_publisher.rb

@@ -44,20 +44,17 @@ module Google
       #
       #   topic.async_publisher.stop.wait!
       #
-      # @attr_reader [String] topic_name The name of the topic the messages
-      #   are published to. In the form of
+      # @attr_reader [String] topic_name The name of the topic the messages are published to. In the form of
       #   "/projects/project-identifier/topics/topic-name".
-      # @attr_reader [Integer] max_bytes The maximum size of messages to be
-      #   collected before the batch is published. Default is 10,000,000
-      #   (10MB).
-      # @attr_reader [Integer] max_messages The maximum number of messages to
-      #   be collected before the batch is published. Default is 1,000.
-      # @attr_reader [Numeric] interval The number of seconds to collect
-      #   messages before the batch is published. Default is 0.25.
-      # @attr_reader [Numeric] publish_threads The number of threads used to
-      #   publish messages. Default is 4.
-      # @attr_reader [Numeric] callback_threads The number of threads to
-      #   handle the published messages' callbacks. Default is 8.
+      # @attr_reader [Integer] max_bytes The maximum size of messages to be collected before the batch is published.
+      #   Default is 1,000,000 (1MB).
+      # @attr_reader [Integer] max_messages The maximum number of messages to be collected before the batch is
+      #   published. Default is 100.
+      # @attr_reader [Numeric] interval The number of seconds to collect messages before the batch is published. Default
+      #   is 0.01.
+      # @attr_reader [Numeric] publish_threads The number of threads used to publish messages. Default is 2.
+      # @attr_reader [Numeric] callback_threads The number of threads to handle the published messages' callbacks.
+      #   Default is 4.
       #
       class AsyncPublisher
         include MonitorMixin
@@ -71,18 +68,17 @@ module Google
 
         ##
         # @private Create a new instance of the object.
-        def initialize topic_name, service, max_bytes: 10_000_000, max_messages: 1000, interval: 0.25, threads: {}
+        def initialize topic_name, service, max_bytes: 1_000_000, max_messages: 100, interval: 0.01, threads: {}
           # init MonitorMixin
           super()
-
           @topic_name = service.topic_path topic_name
           @service    = service
 
           @max_bytes        = max_bytes
           @max_messages     = max_messages
           @interval         = interval
-          @publish_threads  = (threads[:publish] || 4).to_i
-          @callback_threads = (threads[:callback] || 8).to_i
+          @publish_threads  = (threads[:publish] || 2).to_i
+          @callback_threads = (threads[:callback] || 4).to_i
 
           @published_at = nil
           @publish_thread_pool = Concurrent::ThreadPoolExecutor.new max_threads: @publish_threads

data/lib/google/cloud/pubsub/project.rb

@@ -89,19 +89,15 @@ module Google
         #
         #   Hash keys and values may include the following:
         #
-        #   * `:max_bytes` (Integer) The maximum size of messages to be
-        #     collected before the batch is published. Default is 10,000,000
-        #     (10MB).
-        #   * `:max_messages` (Integer) The maximum number of messages to be
-        #     collected before the batch is published. Default is 1,000.
-        #   * `:interval` (Numeric) The number of seconds to collect messages
-        #     before the batch is published. Default is 0.25.
-        #   * `:threads` (Hash) The number of threads to create to handle
-        #     concurrent calls by the publisher:
-        #     * `:publish` (Integer) The number of threads used to publish
-        #       messages. Default is 4.
-        #     * `:callback` (Integer) The number of threads to handle the
-        #       published messages' callbacks. Default is 8.
+        #   * `:max_bytes` (Integer) The maximum size of messages to be collected before the batch is published. Default
+        #     is 1,000,000 (1MB).
+        #   * `:max_messages` (Integer) The maximum number of messages to be collected before the batch is published.
+        #     Default is 100.
+        #   * `:interval` (Numeric) The number of seconds to collect messages before the batch is published. Default is
+        #     0.01.
+        #   * `:threads` (Hash) The number of threads to create to handle concurrent calls by the publisher:
+        #     * `:publish` (Integer) The number of threads used to publish messages. Default is 2.
+        #     * `:callback` (Integer) The number of threads to handle the published messages' callbacks. Default is 4.
         #
         # @return [Google::Cloud::PubSub::Topic, nil] Returns `nil` if topic
         #   does not exist.
@@ -183,19 +179,15 @@ module Google
         #
         #   Hash keys and values may include the following:
         #
-        #   * `:max_bytes` (Integer) The maximum size of messages to be
-        #     collected before the batch is published. Default is 10,000,000
-        #     (10MB).
-        #   * `:max_messages` (Integer) The maximum number of messages to be
-        #     collected before the batch is published. Default is 1,000.
-        #   * `:interval` (Numeric) The number of seconds to collect messages
-        #     before the batch is published. Default is 0.25.
-        #   * `:threads` (Hash) The number of threads to create to handle
-        #     concurrent calls by the publisher:
-        #     * `:publish` (Integer) The number of threads used to publish
-        #       messages. Default is 4.
-        #     * `:callback` (Integer) The number of threads to handle the
-        #       published messages' callbacks. Default is 8.
+        #   * `:max_bytes` (Integer) The maximum size of messages to be collected before the batch is published. Default
+        #     is 1,000,000 (1MB).
+        #   * `:max_messages` (Integer) The maximum number of messages to be collected before the batch is published.
+        #     Default is 100.
+        #   * `:interval` (Numeric) The number of seconds to collect messages before the batch is published. Default is
+        #     0.01.
+        #   * `:threads` (Hash) The number of threads to create to handle concurrent calls by the publisher:
+        #     * `:publish` (Integer) The number of threads used to publish messages. Default is 2.
+        #     * `:callback` (Integer) The number of threads to handle the published messages' callbacks. Default is 4.
         #
         # @return [Google::Cloud::PubSub::Topic]
         #

data/lib/google/cloud/pubsub/subscriber.rb

@@ -54,8 +54,6 @@ module Google
       #   been enabled.
       # @attr_reader [Integer] streams The number of concurrent streams to open
       #   to pull messages from the subscription. Default is 4.
-      # @attr_reader [Integer] inventory The number of received messages to be
-      #   collected by subscriber. Default is 1,000.
       # @attr_reader [Integer] callback_threads The number of threads used to
       #   handle the received messages. Default is 8.
       # @attr_reader [Integer] push_threads The number of threads to handle
@@ -66,19 +64,16 @@ module Google
       class Subscriber
         include MonitorMixin
 
-        attr_reader :subscription_name, :callback, :deadline, :streams,
-                    :inventory, :message_ordering, :callback_threads,
+        attr_reader :subscription_name, :callback, :deadline, :streams, :message_ordering, :callback_threads,
                     :push_threads
 
         ##
         # @private Implementation attributes.
-        attr_reader :stream_inventory, :stream_pool, :thread_pool, :buffer,
-                    :service
+        attr_reader :stream_pool, :thread_pool, :buffer, :service
 
         ##
         # @private Create an empty {Subscriber} object.
-        def initialize subscription_name, callback, deadline: nil,
-                       message_ordering: nil, streams: nil, inventory: nil,
+        def initialize subscription_name, callback, deadline: nil, message_ordering: nil, streams: nil, inventory: nil,
                        threads: {}, service: nil
           super() # to init MonitorMixin
 
@@ -86,17 +81,19 @@ module Google
           @error_callbacks = []
           @subscription_name = subscription_name
           @deadline = deadline || 60
-          @streams = streams || 4
-          @inventory = inventory || 1000
+          @streams = streams || 2
+          @inventory = inventory
+          @inventory = { limit: @inventory } unless inventory.is_a? Hash
+          @inventory[:limit] = Integer(@inventory[:limit] || 1000)
+          @inventory[:bytesize] = Integer(@inventory[:bytesize] || 100_000_000)
+          @inventory[:extension] = Integer(@inventory[:extension] || 3600)
           @message_ordering = message_ordering
-          @callback_threads = (threads[:callback] || 8).to_i
-          @push_threads = (threads[:push] || 4).to_i
+          @callback_threads = Integer(threads[:callback] || 8)
+          @push_threads = Integer(threads[:push] || 4)
 
-          @stream_inventory = @inventory.fdiv(@streams).ceil
           @service = service
 
-          @started = nil
-          @stopped = nil
+          @started = @stopped = nil
 
           stream_pool = Array.new @streams do
             Thread.new { Stream.new self }
@@ -284,6 +281,36 @@ module Google
           synchronize { @last_error }
         end
 
+        ##
+        # The number of received messages to be collected by subscriber. Default is 1,000.
+        def inventory_limit
+          @inventory[:limit]
+        end
+        # @deprecated Use {#inventory_limit}.
+        alias inventory inventory_limit
+
+        ##
+        # The total bytesize of received messages to be collected by subscriber. Default is 100,000,000 (100MB).
+        def inventory_bytesize
+          @inventory[:bytesize]
+        end
+
+        ##
+        # The number of seconds that received messages can be held awaiting processing. Default is 3,600 (1 hour).
+        def inventory_extension
+          @inventory[:extension]
+        end
+
+        ##
+        # @private
+        def stream_inventory
+          {
+            limit:     @inventory[:limit].fdiv(@streams).ceil,
+            bytesize:  @inventory[:bytesize].fdiv(@streams).ceil,
+            extension: @inventory[:extension]
+          }
+        end
+
         # @private returns error object from the stream thread.
         def error! error
           error_callbacks = synchronize do

data/lib/google/cloud/pubsub/subscriber/inventory.rb

@@ -22,30 +22,39 @@ module Google
         ##
         # @private
         class Inventory
+          InventoryItem = Struct.new :bytesize, :pulled_at do
+            def self.from rec_msg
+              new rec_msg.to_proto.bytesize, Time.now
+            end
+          end
+
           include MonitorMixin
 
-          attr_reader :stream, :limit
+          attr_reader :stream, :limit, :bytesize, :extension
 
-          def initialize stream, limit
+          def initialize stream, limit:, bytesize:, extension:
             super()
-
             @stream = stream
             @limit = limit
-            @_ack_ids = []
+            @bytesize = bytesize
+            @extension = extension
+            @inventory = {}
             @wait_cond = new_cond
           end
 
           def ack_ids
-            @_ack_ids
+            @inventory.keys
           end
 
-          def add *ack_ids
-            ack_ids.flatten!
-            ack_ids.compact!
-            return if ack_ids.empty?
+          def add *rec_msgs
+            rec_msgs.flatten!
+            rec_msgs.compact!
+            return if rec_msgs.empty?
 
             synchronize do
-              @_ack_ids += ack_ids
+              rec_msgs.each do |rec_msg|
+                @inventory[rec_msg.ack_id] = InventoryItem.from rec_msg
+              end
               @wait_cond.broadcast
             end
           end
@@ -56,20 +65,34 @@ module Google
             return if ack_ids.empty?
 
             synchronize do
-              @_ack_ids -= ack_ids
+              @inventory.delete_if { |ack_id, _| ack_ids.include? ack_id }
+              @wait_cond.broadcast
+            end
+          end
+
+          def remove_expired!
+            synchronize do
+              extension_time = Time.new - extension
+              @inventory.delete_if { |_ack_id, item| item.pulled_at < extension_time }
               @wait_cond.broadcast
             end
           end
 
           def count
             synchronize do
-              @_ack_ids.count
+              @inventory.count
+            end
+          end
+
+          def total_bytesize
+            synchronize do
+              @inventory.values.sum(&:bytesize)
             end
           end
 
           def empty?
             synchronize do
-              @_ack_ids.empty?
+              @inventory.empty?
             end
           end
 
@@ -93,7 +116,9 @@ module Google
           end
 
           def full?
-            count >= limit
+            synchronize do
+              @inventory.count >= limit || @inventory.values.sum(&:bytesize) >= bytesize
+            end
           end
 
           protected

data/lib/google/cloud/pubsub/subscriber/stream.rb

@@ -181,8 +181,8 @@ module Google
             synchronize do
               return true if @inventory.empty?
 
-              @subscriber.buffer.renew_lease @subscriber.deadline,
-                                             @inventory.ack_ids
+              @inventory.remove_expired!
+              @subscriber.buffer.renew_lease @subscriber.deadline, @inventory.ack_ids
               unpause_streaming!
             end
 
@@ -243,17 +243,13 @@ module Google
                 # Cannot syncronize the enumerator, causes deadlock
                 response = enum.next
 
-                # Create a list of all the received ack_id values
-                received_ack_ids = response.received_messages.map(&:ack_id)
-
                 # Use synchronize so both changes happen atomically
                 synchronize do
                   # Create receipt of received messages reception
-                  @subscriber.buffer.modify_ack_deadline @subscriber.deadline,
-                                                         received_ack_ids
+                  @subscriber.buffer.modify_ack_deadline @subscriber.deadline, response.received_messages.map(&:ack_id)
 
                   # Add received messages to inventory
-                  @inventory.add received_ack_ids
+                  @inventory.add response.received_messages
                 end
 
                 response.received_messages.each do |rec_msg_grpc|

data/lib/google/cloud/pubsub/subscription.rb

@@ -556,8 +556,16 @@ module Google
         #   argument is not provided. See {#reference?}.
         # @param [Integer] streams The number of concurrent streams to open to
         #   pull messages from the subscription. Default is 4. Optional.
-        # @param [Integer] inventory The number of received messages to be
-        #   collected by subscriber. Default is 1,000. Optional.
+        # @param [Hash, Integer] inventory The settings to control how received messages are to be handled by the
+        #   subscriber. When provided as an Integer instead of a Hash only the `limit` will be set. Optional.
+        #
+        #   Hash keys and values may include the following:
+        #
+        #     * `:limit` (Integer) The number of received messages to be collected by subscriber. Default is 1,000.
+        #     * `:bytesize` (Integer) The total bytesize of received messages to be collected by subscriber. Default is
+        #       100,000,000 (100MB).
+        #     * `:extension` (Integer) The number of seconds that received messages can be held awaiting processing.
+        #       Default is 3,600 (1 hour).
         # @param [Hash] threads The number of threads to create to handle
         #   concurrent calls by each stream opened by the subscriber. Optional.
         #

data/lib/google/cloud/pubsub/version.rb

@@ -16,7 +16,7 @@
 module Google
   module Cloud
     module PubSub
-      VERSION = "1.1.3".freeze
+      VERSION = "1.2.0".freeze
     end
 
     Pubsub = PubSub unless const_defined? :Pubsub

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: google-cloud-pubsub
 version: !ruby/object:Gem::Version
-  version: 1.1.3
+  version: 1.2.0
 platform: ruby
 authors:
 - Mike Moore
@@ -9,7 +9,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2019-12-20 00:00:00.000000000 Z
+date: 2020-01-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: concurrent-ruby