google-cloud-pubsub 0.26.0 → 0.27.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 1c103954443d377024fc1a0c670bf809e48f2515
-  data.tar.gz: a2b506b3a345fb5b73756eb15ef6a96c76de39a5
+  metadata.gz: abd5adf64822c8eefa9b20faf514b0af57414e64
+  data.tar.gz: a93679512d85b7c038980ce0144dad4c28dc87c6
 SHA512:
-  metadata.gz: b5dda56bd07a86a42541f512fb1ea119333d257d9bc66dc65d73fb59180bfe584cbe2e55ebe51b337dfb6cafa0dd4c2ba013239657e269565b5126f6e0b09cf8
-  data.tar.gz: 98e0759f6a404373b5e793a1eab6cb1c631573e49d4cce29e769ee40e18995bc7bf7f7afddb1cadf0c096255eaf6d938cbdfb8b69f19654358e18d97d967d122
+  metadata.gz: 0af752e4b5d18bc90dfab1fb702af73a092c128c800e53754bba04c16d6f2a6c12492b9f4e528120c97ba087e4b24fa38446534acd31334b503cc177232367d7
+  data.tar.gz: dae4bcaac46a4b3c2a85ef0f9e0292fec333ed8694eabf25d68cdbe9e4eda63744bf14b2b3dfc3c286d37bd25826815afa4c28d0e4c1b362299b306192b9c217

data/lib/google/cloud/pubsub.rb

@@ -147,7 +147,30 @@ module Google
     #                     this: :that
     # ```
     #
-    # Multiple messages can be published at the same time by passing a block:
+    # Messages can also be published in batches asynchronously using
+    # `publish_async`. (See {Google::Cloud::Pubsub::Topic#publish_async} and
+    # {Google::Cloud::Pubsub::AsyncPublisher})
+    #
+    # ```ruby
+    # require "google/cloud/pubsub"
+    #
+    # pubsub = Google::Cloud::Pubsub.new
+    #
+    # topic = pubsub.topic "my-topic"
+    # topic.publish_async "task completed" do |result|
+    #   if result.succeeded?
+    #     log_publish_success result.data
+    #   else
+    #     log_publish_failure result.data, result.error
+    #   end
+    # end
+    #
+    # topic.async_publisher.stop.wait!
+    # ```
+    #
+    # Or multiple messages can be published in batches at the same time by
+    # passing a block to `publish`. (See
+    # {Google::Cloud::Pubsub::BatchPublisher})
     #
     # ```ruby
     # require "google/cloud/pubsub"
@@ -199,6 +222,29 @@ module Google
     # msgs = sub.wait_for_messages
     # ```
     #
+    # Messages can also be streamed from a subscription with a subscriber object
+    # that can be created using `listen`. (See
+    # {Google::Cloud::Pubsub::Subscription#listen} and
+    # {Google::Cloud::Pubsub::Subscriber})
+    #
+    # ```ruby
+    # require "google/cloud/pubsub"
+    #
+    # pubsub = Google::Cloud::Pubsub.new
+    #
+    # sub = pubsub.subscription "my-topic-sub"
+    #
+    # subscriber = sub.listen do |msg|
+    #   # process msg
+    #   msg.ack!
+    # end
+    #
+    # subscriber.start
+    #
+    # # Shut down the subscriber when ready to stop receiving messages.
+    # subscriber.stop.wait!
+    # ```
+    #
     # ## Acknowledging a Message
     #
     # Messages that are received can be acknowledged in Pub/Sub, marking the
@@ -311,9 +357,10 @@ module Google
     #
     # ## Listening for Messages
     #
-    # Long running workers are easy to create with `listen`, which runs an
-    # infinitely blocking loop to process messages as they are received. (See
-    # {Google::Cloud::Pubsub::Subscription#listen})
+    # A subscriber object can be created using `listen`, which streams messages
+    # from the backend and processes them as they are received. (See
+    # {Google::Cloud::Pubsub::Subscription#listen} and
+    # {Google::Cloud::Pubsub::Subscriber})
     #
     # ```ruby
     # require "google/cloud/pubsub"
@@ -321,28 +368,22 @@ module Google
     # pubsub = Google::Cloud::Pubsub.new
     #
     # sub = pubsub.subscription "my-topic-sub"
-    # sub.listen do |msg|
+    #
+    # subscriber = sub.listen do |msg|
     #   # process msg
+    #   msg.ack!
     # end
-    # ```
     #
-    # Messages are retrieved in batches for efficiency. The number of messages
-    # pulled per batch can be limited with the `max` option:
+    # subscriber.start
     #
-    # ```ruby
-    # require "google/cloud/pubsub"
-    #
-    # pubsub = Google::Cloud::Pubsub.new
-    #
-    # sub = pubsub.subscription "my-topic-sub"
-    # sub.listen max: 20 do |msg|
-    #   # process msg
-    # end
+    # # Shut down the subscriber when ready to stop receiving messages.
+    # subscriber.stop.wait!
     # ```
     #
-    # When processing time and the acknowledgement deadline are a concern,
-    # messages can be automatically acknowledged as they are pulled with the
-    # `autoack` option:
+    # The subscriber object can be configured to control the number of
+    # concurrent streams to open, the number of received messages to be
+    # collected, and the number of threads each stream opens for concurrent
+    # calls made to handle the received messages.
     #
     # ```ruby
     # require "google/cloud/pubsub"
@@ -350,19 +391,14 @@ module Google
     # pubsub = Google::Cloud::Pubsub.new
     #
     # sub = pubsub.subscription "my-topic-sub"
-    # sub.listen autoack: true do |msg|
-    #   # process msg
-    # end
-    # ```
     #
-    # ## Configuring timeout
-    #
-    # You can configure the request `timeout` value in seconds.
-    #
-    # ```ruby
-    # require "google/cloud/pubsub"
+    # subscriber = sub.listen threads: { callback: 16 } do |msg|
+    #   # store the message somewhere before acknowledging
+    #   store_in_backend msg.data # takes a few seconds
+    #   msg.ack!
+    # end
     #
-    # pubsub = Google::Cloud::Pubsub.new timeout: 120
+    # subscriber.start
     # ```
     #
     # ## Working Across Projects

data/lib/google/cloud/pubsub/async_publisher.rb

@@ -0,0 +1,328 @@
+# Copyright 2017 Google Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+require "monitor"
+require "concurrent"
+require "google/cloud/pubsub/publish_result"
+require "google/cloud/pubsub/service"
+
+module Google
+  module Cloud
+    module Pubsub
+      ##
+      # Used to publish multiple messages in batches to a topic. See
+      # {Google::Cloud::Pubsub::Topic#async_publisher}
+      #
+      # @example
+      #   require "google/cloud/pubsub"
+      #
+      #   pubsub = Google::Cloud::Pubsub.new
+      #
+      #   topic = pubsub.topic "my-topic"
+      #   topic.publish_async "task completed" do |result|
+      #     if result.succeeded?
+      #       log_publish_success result.data
+      #     else
+      #       log_publish_failure result.data, result.error
+      #     end
+      #   end
+      #
+      #   topic.async_publisher.stop.wait!
+      #
+      # @attr_reader [String] topic_name The name of the topic the messages
+      #   are published to. In the form of
+      #   "/projects/project-identifier/topics/topic-name".
+      # @attr_reader [Integer] max_bytes The maximum size of messages to be
+      #   collected before the batch is published. Default is 10,000,000
+      #   (10MB).
+      # @attr_reader [Integer] max_messages The maximum number of messages to
+      #   be collected before the batch is published. Default is 1,000.
+      # @attr_reader [Numeric] interval The number of seconds to collect
+      #   messages before the batch is published. Default is 0.25.
+      # @attr_reader [Numeric] publish_threads The number of threads used to
+      #   publish messages. Default is 4.
+      # @attr_reader [Numeric] callback_threads The number of threads to
+      #   handle the published messages' callbacks. Default is 8.
+      #
+      class AsyncPublisher
+        include MonitorMixin
+
+        attr_reader :topic_name, :max_bytes, :max_messages, :interval,
+                    :publish_threads, :callback_threads
+        ##
+        # @private Implementation accessors
+        attr_reader :service, :batch, :publish_thread_pool,
+                    :callback_thread_pool
+
+        ##
+        # @private Create a new instance of the object.
+        def initialize topic_name, service, max_bytes: 10000000,
+                       max_messages: 1000, interval: 0.25, threads: {}
+          @topic_name = service.topic_path topic_name
+          @service    = service
+
+          @max_bytes        = max_bytes
+          @max_messages     = max_messages
+          @interval         = interval
+          @publish_threads  = (threads[:publish] || 4).to_i
+          @callback_threads = (threads[:callback] || 8).to_i
+
+          @cond = new_cond
+
+          # init MonitorMixin
+          super()
+        end
+
+        ##
+        # Add a message to the async publisher to be published to the topic.
+        # Messages will be collected in batches and published together.
+        # See {Google::Cloud::Pubsub::Topic#publish_async}
+        def publish data = nil, attributes = {}, &block
+          msg = create_pubsub_message data, attributes
+
+          synchronize do
+            fail "Can't publish when stopped." if @stopped
+
+            if @batch.nil?
+              @batch ||= Batch.new self
+              @batch.add msg, block
+            else
+              unless @batch.try_add msg, block
+                publish_batch!
+                @batch = Batch.new self
+                @batch.add msg, block
+              end
+            end
+
+            init_resources!
+
+            publish_batch! if @batch.ready?
+
+            @cond.signal
+          end
+          nil
+        end
+
+        ##
+        # Begins the process of stopping the publisher. Messages already in
+        # the queue will be published, but no new messages can be added. Use
+        # {#wait!} to block until the publisher is fully stopped and all
+        # pending messages have been published.
+        #
+        # @return [AsyncPublisher] returns self so calls can be chained.
+        def stop
+          synchronize do
+            break if @stopped
+
+            @stopped = true
+            publish_batch!
+            @cond.signal
+            @publish_thread_pool.shutdown if @publish_thread_pool
+          end
+
+          self
+        end
+
+        ##
+        # Blocks until the publisher is fully stopped, all pending messages
+        # have been published, and all callbacks have completed. Does not stop
+        # the publisher. To stop the publisher, first call {#stop} and then
+        # call {#wait!} to block until the publisher is stopped.
+        #
+        # @return [AsyncPublisher] returns self so calls can be chained.
+        def wait! timeout = nil
+          synchronize do
+            if @publish_thread_pool
+              @publish_thread_pool.wait_for_termination timeout
+            end
+
+            if @callback_thread_pool
+              @callback_thread_pool.shutdown
+              @callback_thread_pool.wait_for_termination timeout
+            end
+          end
+
+          self
+        end
+
+        ##
+        # Forces all messages in the current batch to be published
+        # immediately.
+        #
+        # @return [AsyncPublisher] returns self so calls can be chained.
+        def flush
+          synchronize do
+            publish_batch!
+            @cond.signal
+          end
+
+          self
+        end
+
+        ##
+        # Whether the publisher has been started.
+        #
+        # @return [boolean] `true` when started, `false` otherwise.
+        def started?
+          !stopped?
+        end
+
+        ##
+        # Whether the publisher has been stopped.
+        #
+        # @return [boolean] `true` when stopped, `false` otherwise.
+        def stopped?
+          synchronize { @stopped }
+        end
+
+        protected
+
+        def init_resources!
+          @first_published_at   ||= Time.now
+          @publish_thread_pool  ||= Concurrent::FixedThreadPool.new \
+            @publish_threads
+          @callback_thread_pool ||= Concurrent::FixedThreadPool.new \
+            @callback_threads
+          @thread ||= Thread.new { run_background }
+        end
+
+        def run_background
+          synchronize do
+            until @stopped
+              if @batch.nil?
+                @cond.wait
+                next
+              end
+
+              time_since_first_publish = Time.now - @first_published_at
+              if time_since_first_publish > @interval
+                # interval met, publish the batch...
+                publish_batch!
+                @cond.wait
+              else
+                # still waiting for the interval to publish the batch...
+                @cond.wait(@interval - time_since_first_publish)
+              end
+            end
+          end
+        end
+
+        def publish_batch!
+          return unless @batch
+
+          publish_batch_async @topic_name, @batch
+          @batch = nil
+          @first_published_at = nil
+        end
+
+        def publish_batch_async topic_name, batch
+          Concurrent::Future.new(executor: @publish_thread_pool) do
+            begin
+              grpc = @service.publish topic_name, batch.messages
+              batch.items.zip(Array(grpc.message_ids)) do |item, id|
+                next unless item.callback
+
+                item.msg.message_id = id
+                publish_result = PublishResult.from_grpc(item.msg)
+                execute_callback_async item.callback, publish_result
+              end
+            rescue => e
+              batch.items.each do |item|
+                next unless item.callback
+
+                publish_result = PublishResult.from_error(item.msg, e)
+                execute_callback_async item.callback, publish_result
+              end
+            end
+          end.execute
+        end
+
+        def execute_callback_async callback, publish_result
+          Concurrent::Future.new(executor: @callback_thread_pool) do
+            callback.call publish_result
+          end.execute
+        end
+
+        def create_pubsub_message data, attributes
+          attributes ||= {}
+          if data.is_a?(::Hash) && attributes.empty?
+            attributes = data
+            data = nil
+          end
+          # Convert IO-ish objects to strings
+          if data.respond_to?(:read) && data.respond_to?(:rewind)
+            data.rewind
+            data = data.read
+          end
+          # Convert data to encoded byte array to match the protobuf defn
+          data_bytes = String(data).dup.force_encoding("ASCII-8BIT").freeze
+
+          # Convert attributes to strings to match the protobuf definition
+          attributes = Hash[attributes.map { |k, v| [String(k), String(v)] }]
+
+          Google::Pubsub::V1::PubsubMessage.new data: data_bytes,
+                                                attributes: attributes
+        end
+
+        ##
+        # @private
+        class Batch
+          attr_reader :messages, :callbacks
+
+          def initialize publisher
+            @publisher = publisher
+            @messages = []
+            @callbacks = []
+          end
+
+          def add msg, callback
+            @messages << msg
+            @callbacks << callback
+          end
+
+          def try_add msg, callback
+            if total_message_count + 1 > @publisher.max_messages ||
+               total_message_bytes + msg.to_proto.size >= @publisher.max_bytes
+              return false
+            end
+            add msg, callback
+            true
+          end
+
+          def ready?
+            total_message_count >= @publisher.max_messages ||
+              total_message_bytes >= @publisher.max_bytes
+          end
+
+          def total_message_count
+            @messages.count
+          end
+
+          def total_message_bytes
+            @messages.map(&:to_proto).map(&:size).inject(0, :+)
+          end
+
+          def items
+            @messages.zip(@callbacks).map do |msg, callback|
+              Item.new msg, callback
+            end
+          end
+
+          Item = Struct.new :msg, :callback
+        end
+      end
+    end
+  end
+end