google-cloud-pubsub 0.26.0 → 0.27.0

data/lib/google/cloud/pubsub/subscription.rb

@@ -13,10 +13,12 @@
 # limitations under the License.
 
 
+require "google/cloud/pubsub/convert"
 require "google/cloud/errors"
 require "google/cloud/pubsub/subscription/list"
 require "google/cloud/pubsub/received_message"
 require "google/cloud/pubsub/snapshot"
+require "google/cloud/pubsub/subscriber"
 
 module Google
   module Cloud
@@ -49,26 +51,15 @@ module Google
         # @private Create an empty {Subscription} object.
         def initialize
           @service = nil
-          @grpc = Google::Pubsub::V1::Subscription.new
-          @name = nil
+          @grpc = nil
+          @lazy = nil
           @exists = nil
         end
 
-        ##
-        # @private New lazy {Topic} object without making an HTTP request.
-        def self.new_lazy name, service, options = {}
-          new.tap do |s|
-            s.grpc = nil
-            s.service = service
-            s.instance_variable_set "@name",
-                                    service.subscription_path(name, options)
-          end
-        end
-
         ##
         # The name of the subscription.
         def name
-          @grpc ? @grpc.name : @name
+          @grpc.name
         end
 
         ##
@@ -98,6 +89,15 @@ module Google
           @grpc.ack_deadline_seconds
         end
 
+        def deadline= new_deadline
+          update_grpc = @grpc.dup
+          update_grpc.ack_deadline_seconds = new_deadline
+          @grpc = service.update_subscription update_grpc,
+                                              :ack_deadline_seconds
+          @lazy = nil
+          self
+        end
+
         ##
         # Indicates whether to retain acknowledged messages. If `true`, then
         # messages are not expunged from the subscription's backlog, even if
@@ -112,6 +112,15 @@ module Google
           @grpc.retain_acked_messages
         end
 
+        def retain_acked= new_retain_acked
+          update_grpc = @grpc.dup
+          update_grpc.retain_acked_messages = !(!new_retain_acked)
+          @grpc = service.update_subscription update_grpc,
+                                              :retain_acked_messages
+          @lazy = nil
+          self
+        end
+
         ##
         # How long to retain unacknowledged messages in the subscription's
         # backlog, from the moment a message is published. If
@@ -125,7 +134,17 @@ module Google
         #
         def retention
           ensure_grpc!
-          duration_to_number @grpc.message_retention_duration
+          Convert.duration_to_number @grpc.message_retention_duration
+        end
+
+        def retention= new_retention
+          update_grpc = @grpc.dup
+          update_grpc.message_retention_duration = \
+            Convert.number_to_duration new_retention
+          @grpc = service.update_subscription update_grpc,
+                                              :message_retention_duration
+          @lazy = nil
+          self
         end
 
         ##
@@ -159,12 +178,12 @@ module Google
         #   sub.exists? #=> true
         #
         def exists?
-          # Always true if we have a grpc object
-          return true unless @grpc.nil?
+          # Always true if the object is not set as lazy
+          return true unless lazy?
           # If we have a value, return it
           return @exists unless @exists.nil?
           ensure_grpc!
-          @exists = !@grpc.nil?
+          @exists = true
         rescue Google::Cloud::NotFoundError
           @exists = false
         end
@@ -180,10 +199,10 @@ module Google
         #   pubsub = Google::Cloud::Pubsub.new
         #
         #   sub = pubsub.get_subscription "my-topic-sub"
-        #   sub.lazy? #=> false
+        #   sub.lazy? #=> nil
         #
         def lazy?
-          @grpc.nil?
+          @lazy
         end
 
         ##
@@ -220,8 +239,6 @@ module Google
         # @param [Integer] max The maximum number of messages to return for this
         #   request. The Pub/Sub system may return fewer than the number
         #   specified. The default value is `100`, the maximum value is `1000`.
-        # @param [Boolean] autoack Automatically acknowledge the message as it
-        #   is pulled. The default value is `false`.
         #
         # @return [Array<Google::Cloud::Pubsub::ReceivedMessage>]
         #
@@ -250,15 +267,13 @@ module Google
         #   msgs = sub.pull immediate: false
         #   msgs.each { |msg| msg.acknowledge! }
         #
-        def pull immediate: true, max: 100, autoack: false
+        def pull immediate: true, max: 100
           ensure_service!
           options = { immediate: immediate, max: max }
           list_grpc = service.pull name, options
-          messages = Array(list_grpc.received_messages).map do |msg_grpc|
+          Array(list_grpc.received_messages).map do |msg_grpc|
             ReceivedMessage.from_grpc msg_grpc, self
           end
-          acknowledge messages if autoack
-          messages
         rescue Google::Cloud::DeadlineExceededError
           []
         end
@@ -272,8 +287,6 @@ module Google
         # @param [Integer] max The maximum number of messages to return for this
         #   request. The Pub/Sub system may return fewer than the number
         #   specified. The default value is `100`, the maximum value is `1000`.
-        # @param [Boolean] autoack Automatically acknowledge the message as it
-        #   is pulled. The default value is `false`.
         #
         # @return [Array<Google::Cloud::Pubsub::ReceivedMessage>]
         #
@@ -286,65 +299,79 @@ module Google
         #   msgs = sub.wait_for_messages
         #   msgs.each { |msg| msg.acknowledge! }
         #
-        def wait_for_messages max: 100, autoack: false
-          pull immediate: false, max: max, autoack: autoack
+        def wait_for_messages max: 100
+          pull immediate: false, max: max
         end
 
         ##
-        # Poll the backend for new messages. This runs a loop to ping the API,
-        # blocking indefinitely, yielding retrieved messages as they are
-        # received.
+        # Create a {Subscriber} object that receives  and processes messages
+        # using the code provided in the callback.
+        #
+        # @param [Numeric] deadline The default number of seconds the stream
+        #   will hold received messages before modifying the message's ack
+        #   deadline. The minimum is 10, the maximum is 600. Default is
+        #   {#deadline}. Optional.
+        # @param [Integer] streams The number of concurrent streams to open to
+        #   pull messages from the subscription. Default is 4. Optional.
+        # @param [Integer] inventory The number of received messages to be
+        #   collected by subscriber. Default is 1,000. Optional.
+        # @param [Hash] threads The number of threads to create to handle
+        #   concurrent calls by each stream opened by the subscriber. Optional.
+        #
+        #   Hash keys and values may include the following:
+        #
+        #     * `:callback` (Integer) The number of threads used to handle the
+        #       received messages. Default is 8.
+        #     * `:push` (Integer) The number of threads to handle
+        #       acknowledgement ({ReceivedMessage#ack!}) and delay messages
+        #       ({ReceivedMessage#nack!}, {ReceivedMessage#delay!}). Default is
+        #       4.
         #
-        # @param [Integer] max The maximum number of messages to return for this
-        #   request. The Pub/Sub system may return fewer than the number
-        #   specified. The default value is `100`, the maximum value is `1000`.
-        # @param [Boolean] autoack Automatically acknowledge the message as it
-        #   is pulled. The default value is `false`.
-        # @param [Number] delay The number of seconds to pause between requests
-        #   when the Google Cloud service has no messages to return. The default
-        #   value is `1`.
         # @yield [msg] a block for processing new messages
         # @yieldparam [ReceivedMessage] msg the newly received message
         #
+        # @return [Subscriber]
+        #
         # @example
         #   require "google/cloud/pubsub"
         #
         #   pubsub = Google::Cloud::Pubsub.new
         #
         #   sub = pubsub.subscription "my-topic-sub"
-        #   sub.listen do |msg|
+        #
+        #   subscriber = sub.listen do |msg|
         #     # process msg
+        #     msg.ack!
         #   end
         #
-        # @example Limit number of messages pulled per API request with `max`:
-        #   require "google/cloud/pubsub"
+        #   subscriber.start
         #
-        #   pubsub = Google::Cloud::Pubsub.new
+        #   # Shut down the subscriber when ready to stop receiving messages.
+        #   subscriber.stop.wait!
         #
-        #   sub = pubsub.subscription "my-topic-sub"
-        #   sub.listen max: 20 do |msg|
-        #     # process msg
-        #   end
-        #
-        # @example Automatically acknowledge messages with `autoack`:
+        # @example Configuring to increase concurrent callbacks:
         #   require "google/cloud/pubsub"
         #
         #   pubsub = Google::Cloud::Pubsub.new
         #
         #   sub = pubsub.subscription "my-topic-sub"
-        #   sub.listen autoack: true do |msg|
-        #     # process msg
+        #
+        #   subscriber = sub.listen threads: { callback: 16 } do |msg|
+        #     # store the message somewhere before acknowledging
+        #     store_in_backend msg.data # takes a few seconds
+        #     msg.ack!
         #   end
         #
-        def listen max: 100, autoack: false, delay: 1
-          loop do
-            msgs = wait_for_messages max: max, autoack: autoack
-            if msgs.any?
-              msgs.each { |msg| yield msg }
-            else
-              sleep delay
-            end
-          end
+        #   subscriber.start
+        #
+        def listen deadline: nil, streams: nil, inventory: nil, threads: {},
+                   &block
+          ensure_service!
+          deadline ||= self.deadline
+
+          Subscriber.new name, block, deadline: deadline, streams: streams,
+                                      inventory: inventory, threads: threads,
+                                      service: service
         end
 
         ##
@@ -406,6 +433,7 @@ module Google
           service.modify_ack_deadline name, ack_ids, new_deadline
           true
         end
+        alias_method :modify_ack_deadline, :delay
 
         ##
         # Creates a new {Snapshot} from the subscription. The created snapshot
@@ -633,16 +661,18 @@ module Google
           end
         end
 
-        protected
-
-        def duration_to_number duration
-          return nil if duration.nil?
-
-          return duration.seconds if duration.nanos == 0
-
-          duration.seconds + (duration.nanos / 1000000000.0)
+        ##
+        # @private New lazy {Topic} object without making an HTTP request.
+        def self.new_lazy name, service, options = {}
+          lazy_grpc = Google::Pubsub::V1::Subscription.new \
+            name: service.subscription_path(name, options)
+          from_grpc(lazy_grpc, service).tap do |s|
+            s.instance_variable_set :@lazy, true
+          end
         end
 
+        protected
+
         ##
         # @private Raise an error unless an active connection to the service is
         # available.
@@ -654,8 +684,8 @@ module Google
         # Ensures a Google::Pubsub::V1::Subscription object exists.
         def ensure_grpc!
           ensure_service!
-          return @grpc if @grpc
-          @grpc = service.get_subscription @name
+          @grpc = service.get_subscription name if lazy?
+          @lazy = nil
         end
 
         ##

data/lib/google/cloud/pubsub/topic.rb

@@ -14,8 +14,9 @@
 
 
 require "google/cloud/errors"
-require "google/cloud/pubsub/topic/publisher"
 require "google/cloud/pubsub/topic/list"
+require "google/cloud/pubsub/async_publisher"
+require "google/cloud/pubsub/batch_publisher"
 require "google/cloud/pubsub/subscription"
 require "google/cloud/pubsub/policy"
 
@@ -48,26 +49,43 @@ module Google
         # @private Create an empty {Topic} object.
         def initialize
           @service = nil
-          @grpc = Google::Pubsub::V1::Topic.new
-          @name = nil
+          @grpc = nil
+          @lazy = nil
           @exists = nil
+          @async_opts = {}
         end
 
         ##
-        # @private New lazy {Topic} object without making an HTTP request.
-        def self.new_lazy name, service, options = {}
-          new.tap do |t|
-            t.grpc = nil
-            t.service = service
-            t.instance_variable_set "@name", service.topic_path(name, options)
-          end
+        # AsyncPublisher object used to publish multiple messages in batches.
+        #
+        # @return [AsyncPublisher] Returns publisher object if calls to
+        #   {#publish_async} have been made, returns `nil` otherwise.
+        #
+        # @example
+        #   require "google/cloud/pubsub"
+        #
+        #   pubsub = Google::Cloud::Pubsub.new
+        #
+        #   topic = pubsub.topic "my-topic"
+        #   topic.publish_async "task completed" do |result|
+        #     if result.succeeded?
+        #       log_publish_success result.data
+        #     else
+        #       log_publish_failure result.data, result.error
+        #     end
+        #   end
+        #
+        #   topic.async_publisher.stop.wait!
+        #
+        def async_publisher
+          @async_publisher
         end
 
         ##
         # The name of the topic in the form of
         # "/projects/project-identifier/topics/topic-name".
         def name
-          @grpc ? @grpc.name : @name
+          @grpc.name
         end
 
         ##
@@ -234,11 +252,16 @@ module Google
         ##
         # Publishes one or more messages to the topic.
         #
-        # @param [String, File] data The message data.
+        # The message payload must not be empty; it must contain either a
+        # non-empty data field, or at least one attribute.
+        #
+        # @param [String, File] data The message payload. This will be converted
+        #   to bytes encoded as ASCII-8BIT.
         # @param [Hash] attributes Optional attributes for the message.
-        # @yield [publisher] a block for publishing multiple messages in one
+        # @yield [batch] a block for publishing multiple messages in one
         #   request
-        # @yieldparam [Topic::Publisher] publisher the topic publisher object
+        # @yieldparam [BatchPublisher] batch the topic batch publisher
+        #   object
         #
         # @return [Message, Array<Message>] Returns the published message when
         #   called without a block, or an array of messages when called with a
@@ -258,7 +281,8 @@ module Google
         #   pubsub = Google::Cloud::Pubsub.new
         #
         #   topic = pubsub.topic "my-topic"
-        #   msg = topic.publish File.open("message.txt")
+        #   file = File.open "message.txt", mode: "rb"
+        #   msg = topic.publish file
         #
         # @example Additionally, a message can be published with attributes:
         #   require "google/cloud/pubsub"
@@ -284,10 +308,68 @@ module Google
         #
         def publish data = nil, attributes = {}
           ensure_service!
-          publisher = Publisher.new data, attributes
-          yield publisher if block_given?
-          return nil if publisher.messages.count.zero?
-          publish_batch_messages publisher
+          batch = BatchPublisher.new data, attributes
+          yield batch if block_given?
+          return nil if batch.messages.count.zero?
+          publish_batch_messages batch
+        end
+
+        ##
+        # Publishes a message asynchonously to the topic.
+        #
+        # The message payload must not be empty; it must contain either a
+        # non-empty data field, or at least one attribute.
+        #
+        # @param [String, File] data The message payload. This will be converted
+        #   to bytes encoded as ASCII-8BIT.
+        # @param [Hash] attributes Optional attributes for the message.
+        # @yield [result] the callback for when the message has been published
+        # @yieldparam [PublishResult] result the result of the asynchonous
+        #   publish
+        #
+        # @example
+        #   require "google/cloud/pubsub"
+        #
+        #   pubsub = Google::Cloud::Pubsub.new
+        #
+        #   topic = pubsub.topic "my-topic"
+        #   topic.publish_async "task completed" do |result|
+        #     if result.succeeded?
+        #       log_publish_success result.data
+        #     else
+        #       log_publish_failure result.data, result.error
+        #     end
+        #   end
+        #
+        #   topic.async_publisher.stop.wait!
+        #
+        # @example A message can be published using a File object:
+        #   require "google/cloud/pubsub"
+        #
+        #   pubsub = Google::Cloud::Pubsub.new
+        #
+        #   topic = pubsub.topic "my-topic"
+        #   file = File.open "message.txt", mode: "rb"
+        #   topic.publish_async file
+        #
+        #   topic.async_publisher.stop.wait!
+        #
+        # @example Additionally, a message can be published with attributes:
+        #   require "google/cloud/pubsub"
+        #
+        #   pubsub = Google::Cloud::Pubsub.new
+        #
+        #   topic = pubsub.topic "my-topic"
+        #   topic.publish_async "task completed",
+        #                       foo: :bar, this: :that
+        #
+        #   topic.async_publisher.stop.wait!
+        #
+        def publish_async data = nil, attributes = {}, &block
+          ensure_service!
+
+          @async_publisher ||= AsyncPublisher.new(name, service, @async_opts)
+          @async_publisher.publish data, attributes, &block
         end
 
         ##
@@ -421,12 +503,14 @@ module Google
         #   topic.exists? #=> true
         #
         def exists?
-          # Always true if we have a grpc object
-          return true unless @grpc.nil?
+          # Always true if the object is not set as lazy
+          return true unless lazy?
           # If we have a value, return it
           return @exists unless @exists.nil?
           ensure_grpc!
-          @exists = !@grpc.nil?
+          @exists = true
+        rescue Google::Cloud::NotFoundError
+          @exists = false
         end
 
         ##
@@ -439,18 +523,29 @@ module Google
         #   pubsub = Google::Cloud::Pubsub.new
         #
         #   topic = pubsub.topic "my-topic"
-        #   topic.lazy? #=> false
+        #   topic.lazy? #=> nil
         #
         def lazy?
-          @grpc.nil?
+          @lazy
         end
 
         ##
         # @private New Topic from a Google::Pubsub::V1::Topic object.
-        def self.from_grpc grpc, service
-          new.tap do |f|
-            f.grpc = grpc
-            f.service = service
+        def self.from_grpc grpc, service, async: nil
+          new.tap do |t|
+            t.grpc = grpc
+            t.service = service
+            t.instance_variable_set :@async_opts, async if async
+          end
+        end
+
+        ##
+        # @private New lazy {Topic} object without making an HTTP request.
+        def self.new_lazy name, service, options = {}
+          lazy_grpc = Google::Pubsub::V1::Topic.new \
+            name: service.topic_path(name, options)
+          from_grpc(lazy_grpc, service).tap do |t|
+            t.instance_variable_set :@lazy, true
           end
         end
 
@@ -467,10 +562,8 @@ module Google
         # Ensures a Google::Pubsub::V1::Topic object exists.
         def ensure_grpc!
           ensure_service!
-          return @grpc if @grpc
-          @grpc = service.get_topic @name
-        rescue Google::Cloud::NotFoundError
-          nil
+          @grpc = service.get_topic name if lazy?
+          @lazy = nil
         end
 
         ##