google-cloud-pubsub 0.26.0 → 0.27.0

data/lib/google/cloud/pubsub/batch_publisher.rb

@@ -0,0 +1,98 @@
+# Copyright 2015 Google Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+module Google
+  module Cloud
+    module Pubsub
+      ##
+      # Topic Batch Publisher object used to publish multiple messages at
+      # once.
+      #
+      # @example
+      #   require "google/cloud/pubsub"
+      #
+      #   pubsub = Google::Cloud::Pubsub.new
+      #
+      #   topic = pubsub.topic "my-topic"
+      #   msgs = topic.publish do |t|
+      #     t.publish "task 1 completed", foo: :bar
+      #     t.publish "task 2 completed", foo: :baz
+      #     t.publish "task 3 completed", foo: :bif
+      #   end
+      class BatchPublisher
+        ##
+        # @private The messages to publish
+        attr_reader :messages
+
+        ##
+        # @private Create a new instance of the object.
+        def initialize data = nil, attributes = {}
+          @messages = []
+          @mode = :batch
+          return if data.nil?
+          @mode = :single
+          publish data, attributes
+        end
+
+        ##
+        # Add a message to the batch to be published to the topic.
+        # All messages added to the batch will be published at once.
+        # See {Google::Cloud::Pubsub::Topic#publish}
+        def publish data, attributes = {}
+          @messages << create_pubsub_message(data, attributes)
+        end
+
+        ##
+        # @private Create Message objects with message ids.
+        def to_gcloud_messages message_ids
+          msgs = @messages.zip(Array(message_ids)).map do |msg, id|
+            msg.message_id = id
+            Message.from_grpc msg
+          end
+          # Return just one Message if a single publish,
+          # otherwise return the array of Messages.
+          if @mode == :single && msgs.count <= 1
+            msgs.first
+          else
+            msgs
+          end
+        end
+
+        protected
+
+        def create_pubsub_message data, attributes
+          attributes ||= {}
+          if data.is_a?(::Hash) && attributes.empty?
+            attributes = data
+            data = nil
+          end
+          # Convert IO-ish objects to strings
+          if data.respond_to?(:read) && data.respond_to?(:rewind)
+            data.rewind
+            data = data.read
+          end
+          # Convert data to encoded byte array to match the protobuf defn
+          data_bytes = String(data).dup.force_encoding("ASCII-8BIT").freeze
+
+          # Convert attributes to strings to match the protobuf definition
+          attributes = Hash[attributes.map { |k, v| [String(k), String(v)] }]
+
+          Google::Pubsub::V1::PubsubMessage.new data: data_bytes,
+                                                attributes: attributes
+        end
+      end
+    end
+  end
+end

data/lib/google/cloud/pubsub/convert.rb

@@ -0,0 +1,63 @@
+# Copyright 2017 Google Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+require "time"
+
+module Google
+  module Cloud
+    module Pubsub
+      ##
+      # @private Helper module for converting Pub/Sub values.
+      module Convert
+        module ClassMethods
+          def time_to_timestamp time
+            return nil if time.nil?
+
+            # Force the object to be a Time object.
+            time = time.to_time
+
+            Google::Protobuf::Timestamp.new \
+              seconds: time.to_i,
+              nanos: time.nsec
+          end
+
+          def timestamp_to_time timestamp
+            return nil if timestamp.nil?
+
+            Time.at timestamp.seconds, Rational(timestamp.nanos, 1000)
+          end
+
+          def number_to_duration number
+            return nil if number.nil?
+
+            Google::Protobuf::Duration.new \
+              seconds: number.to_i,
+              nanos: (number.remainder(1) * 1000000000).round
+          end
+
+          def duration_to_number duration
+            return nil if duration.nil?
+
+            return duration.seconds if duration.nanos == 0
+
+            duration.seconds + (duration.nanos / 1000000000.0)
+          end
+        end
+
+        extend ClassMethods
+      end
+    end
+  end
+end

data/lib/google/cloud/pubsub/message.rb

@@ -13,6 +13,7 @@
 # limitations under the License.
 
 
+require "google/cloud/pubsub/convert"
 require "google/cloud/errors"
 
 module Google
@@ -36,12 +37,12 @@ module Google
       #   # Publish a message
       #   topic = pubsub.topic "my-topic"
       #   message = topic.publish "task completed"
-      #   message.data #=>  "task completed"
+      #   message.data #=> "task completed"
       #
       #   # Pull a message
       #   sub = pubsub.subscription "my-topic-sub"
       #   received_message = sub.pull.first
-      #   received_message.message.data #=>  "task completed"
+      #   received_message.message.data #=> "task completed"
       #
       class Message
         ##
@@ -61,13 +62,14 @@ module Google
         end
 
         ##
-        # The received data.
+        # The message payload. This data is a list of bytes encoded as
+        # ASCII-8BIT.
         def data
           @grpc.data
         end
 
         ##
-        # The received attributes.
+        # Optional attributes for the message.
         def attributes
           return @grpc.attributes.to_h if @grpc.attributes.respond_to? :to_h
           # Enumerable doesn't have to_h on Ruby 2.0, so fallback to this
@@ -82,6 +84,13 @@ module Google
         end
         alias_method :msg_id, :message_id
 
+        ##
+        # The time at which the message was published.
+        def published_at
+          Convert.timestamp_to_time @grpc.publish_time
+        end
+        alias_method :publish_time, :published_at
+
         ##
         # @private New Message from a Google::Pubsub::V1::PubsubMessage object.
         def self.from_grpc grpc

data/lib/google/cloud/pubsub/project.rb

@@ -18,6 +18,7 @@ require "google/cloud/env"
 require "google/cloud/pubsub/service"
 require "google/cloud/pubsub/credentials"
 require "google/cloud/pubsub/topic"
+require "google/cloud/pubsub/batch_publisher"
 require "google/cloud/pubsub/snapshot"
 
 module Google
@@ -83,25 +84,36 @@ module Google
         ##
         # Retrieves topic by name.
         #
-        # The topic will be created if the topic does not exist and the
-        # `autocreate` option is set to true.
-        #
         # @param [String] topic_name Name of a topic.
-        # @param [Boolean] autocreate Flag to control whether the requested
-        #   topic will be created if it does not exist. Ignored if `skip_lookup`
-        #   is `true`. The default value is `false`.
         # @param [String] project If the topic belongs to a project other than
         #   the one currently connected to, the alternate project ID can be
-        #   specified here.
+        #   specified here. Optional.
         # @param [Boolean] skip_lookup Optionally create a {Topic} object
         #   without verifying the topic resource exists on the Pub/Sub service.
         #   Calls made on this object will raise errors if the topic resource
-        #   does not exist. Default is `false`.
+        #   does not exist. Default is `false`. Optional.
+        # @param [Hash] async A hash of values to configure the topic's
+        #   {AsyncPublisher} that is created when {Topic#publish_async}
+        #   is called. Optional.
+        #
+        #   Hash keys and values may include the following:
+        #
+        #   * `:max_bytes` (Integer) The maximum size of messages to be
+        #     collected before the batch is published. Default is 10,000,000
+        #     (10MB).
+        #   * `:max_messages` (Integer) The maximum number of messages to be
+        #     collected before the batch is published. Default is 1,000.
+        #   * `:interval` (Numeric) The number of seconds to collect messages
+        #     before the batch is published. Default is 0.25.
+        #   * `:threads` (Hash) The number of threads to create to handle
+        #     concurrent calls by the publisher:
+        #     * `:publish` (Integer) The number of threads used to publish
+        #       messages. Default is 4.
+        #     * `:callback` (Integer) The number of threads to handle the
+        #       published messages' callbacks. Default is 8.
         #
         # @return [Google::Cloud::Pubsub::Topic, nil] Returns `nil` if topic
-        #   does not exist. Will return a newly created{
-        #   Google::Cloud::Pubsub::Topic} if the topic does not exist and
-        #   `autocreate` is set to `true`.
+        #   does not exist.
         #
         # @example
         #   require "google/cloud/pubsub"
@@ -115,12 +127,6 @@ module Google
         #   pubsub = Google::Cloud::Pubsub.new
         #   topic = pubsub.topic "non-existing-topic" # nil
         #
-        # @example With the `autocreate` option set to `true`.
-        #   require "google/cloud/pubsub"
-        #
-        #   pubsub = Google::Cloud::Pubsub.new
-        #   topic = pubsub.topic "non-existing-topic", autocreate: true
-        #
         # @example Create topic in a different project with the `project` flag.
         #   require "google/cloud/pubsub"
         #
@@ -133,14 +139,30 @@ module Google
         #   pubsub = Google::Cloud::Pubsub.new
         #   topic = pubsub.topic "another-topic", skip_lookup: true
         #
-        def topic topic_name, autocreate: nil, project: nil, skip_lookup: nil
+        # @example Configuring AsyncPublisher to increase concurrent callbacks:
+        #   require "google/cloud/pubsub"
+        #
+        #   pubsub = Google::Cloud::Pubsub.new
+        #   topic = pubsub.topic "my-topic",
+        #                        async: { threads: { callback: 16 } }
+        #
+        #   topic.publish_async "task completed" do |result|
+        #     if result.succeeded?
+        #       log_publish_success result.data
+        #     else
+        #       log_publish_failure result.data, result.error
+        #     end
+        #   end
+        #
+        #   topic.async_publisher.stop.wait!
+        #
+        def topic topic_name, project: nil, skip_lookup: nil, async: nil
           ensure_service!
           options = { project: project }
           return Topic.new_lazy(topic_name, service, options) if skip_lookup
           grpc = service.get_topic topic_name
-          Topic.from_grpc grpc, service
+          Topic.from_grpc grpc, service, async: async
         rescue Google::Cloud::NotFoundError
-          return create_topic(topic_name) if autocreate
           nil
         end
         alias_method :get_topic, :topic
@@ -150,6 +172,25 @@ module Google
         # Creates a new topic.
         #
         # @param [String] topic_name Name of a topic.
+        # @param [Hash] async A hash of values to configure the topic's
+        #   {AsyncPublisher} that is created when {Topic#publish_async}
+        #   is called. Optional.
+        #
+        #   Hash keys and values may include the following:
+        #
+        #   * `:max_bytes` (Integer) The maximum size of messages to be
+        #     collected before the batch is published. Default is 10,000,000
+        #     (10MB).
+        #   * `:max_messages` (Integer) The maximum number of messages to be
+        #     collected before the batch is published. Default is 1,000.
+        #   * `:interval` (Numeric) The number of seconds to collect messages
+        #     before the batch is published. Default is 0.25.
+        #   * `:threads` (Hash) The number of threads to create to handle
+        #     concurrent calls by the publisher:
+        #     * `:publish` (Integer) The number of threads used to publish
+        #       messages. Default is 4.
+        #     * `:callback` (Integer) The number of threads to handle the
+        #       published messages' callbacks. Default is 8.
         #
         # @return [Google::Cloud::Pubsub::Topic]
         #
@@ -159,10 +200,10 @@ module Google
         #   pubsub = Google::Cloud::Pubsub.new
         #   topic = pubsub.create_topic "my-topic"
         #
-        def create_topic topic_name
+        def create_topic topic_name, async: nil
           ensure_service!
           grpc = service.create_topic topic_name
-          Topic.from_grpc grpc, service
+          Topic.from_grpc grpc, service, async: async
         end
         alias_method :new_topic, :create_topic
 
@@ -206,161 +247,6 @@ module Google
         alias_method :find_topics, :topics
         alias_method :list_topics, :topics
 
-        ##
-        # Publishes one or more messages to the given topic. The topic will be
-        # created if the topic does previously not exist and the `autocreate`
-        # option is provided.
-        #
-        # A note about auto-creating the topic: Any message published to a topic
-        # without a subscription will be lost.
-        #
-        # @param [String] topic_name Name of a topic.
-        # @param [String, File] data The message data.
-        # @param [Hash] attributes Optional attributes for the message.
-        # @option attributes [Boolean] :autocreate Flag to control whether the
-        #   provided topic will be created if it does not exist.
-        # @yield [publisher] a block for publishing multiple messages in one
-        #   request
-        # @yieldparam [Topic::Publisher] publisher the topic publisher object
-        #
-        # @return [Message, Array<Message>] Returns the published message when
-        #   called without a block, or an array of messages when called with a
-        #   block.
-        #
-        # @example
-        #   require "google/cloud/pubsub"
-        #
-        #   pubsub = Google::Cloud::Pubsub.new
-        #
-        #   msg = pubsub.publish "my-topic", "task completed"
-        #
-        # @example A message can be published using a File object:
-        #   require "google/cloud/pubsub"
-        #
-        #   pubsub = Google::Cloud::Pubsub.new
-        #
-        #   msg = pubsub.publish "my-topic", File.open("message.txt")
-        #
-        # @example Additionally, a message can be published with attributes:
-        #   require "google/cloud/pubsub"
-        #
-        #   pubsub = Google::Cloud::Pubsub.new
-        #
-        #   msg = pubsub.publish "my-topic", "task completed", foo: :bar,
-        #                                                   this: :that
-        #
-        # @example Multiple messages can be sent at the same time using a block:
-        #   require "google/cloud/pubsub"
-        #
-        #   pubsub = Google::Cloud::Pubsub.new
-        #
-        #   msgs = pubsub.publish "my-topic" do |p|
-        #     p.publish "task 1 completed", foo: :bar
-        #     p.publish "task 2 completed", foo: :baz
-        #     p.publish "task 3 completed", foo: :bif
-        #   end
-        #
-        # @example With `autocreate`:
-        #   require "google/cloud/pubsub"
-        #
-        #   pubsub = Google::Cloud::Pubsub.new
-        #
-        #   msg = pubsub.publish "new-topic", "task completed", autocreate: true
-        #
-        def publish topic_name, data = nil, attributes = {}
-          # Fix parameters
-          if data.is_a?(::Hash) && attributes.empty?
-            attributes = data
-            data = nil
-          end
-          # extract autocreate option
-          autocreate = attributes.delete :autocreate
-          ensure_service!
-          publisher = Topic::Publisher.new data, attributes
-          yield publisher if block_given?
-          return nil if publisher.messages.count.zero?
-          publish_batch_messages topic_name, publisher, autocreate
-        end
-
-        ##
-        # Creates a new {Subscription} object for the provided topic. The topic
-        # will be created if the topic does previously not exist and the
-        # `autocreate` option is provided.
-        #
-        # @param [String] topic_name Name of a topic.
-        # @param [String] subscription_name Name of the new subscription. Must
-        #   start with a letter, and contain only letters ([A-Za-z]), numbers
-        #   ([0-9], dashes (-), underscores (_), periods (.), tildes (~), plus
-        #   (+) or percent signs (%). It must be between 3 and 255 characters in
-        #   length, and it must not start with "goog".
-        # @param [Integer] deadline The maximum number of seconds after a
-        #   subscriber receives a message before the subscriber should
-        #   acknowledge the message.
-        # @param [Boolean] retain_acked Indicates whether to retain acknowledged
-        #   messages. If `true`, then messages are not expunged from the
-        #   subscription's backlog, even if they are acknowledged, until they
-        #   fall out of the `retention_duration` window. Default is `false`.
-        # @param [Numeric] retention How long to retain unacknowledged messages
-        #   in the subscription's backlog, from the moment a message is
-        #   published. If `retain_acked` is `true`, then this also configures
-        #   the retention of acknowledged messages, and thus configures how far
-        #   back in time a {#seek} can be done. Cannot be more than 604,800
-        #   seconds (7 days) or less than 600 seconds (10 minutes). Default is
-        #   604,800 seconds (7 days).
-        # @param [String] endpoint A URL locating the endpoint to which messages
-        #   should be pushed.
-        # @param [String] autocreate Flag to control whether the topic will be
-        #   created if it does not exist.
-        #
-        # @return [Google::Cloud::Pubsub::Subscription]
-        #
-        # @example
-        #   require "google/cloud/pubsub"
-        #
-        #   pubsub = Google::Cloud::Pubsub.new
-        #
-        #   sub = pubsub.subscribe "my-topic", "my-topic-sub"
-        #   sub.name #=> "my-topic-sub"
-        #
-        # @example Wait 2 minutes for acknowledgement and push all to endpoint:
-        #   require "google/cloud/pubsub"
-        #
-        #   pubsub = Google::Cloud::Pubsub.new
-        #
-        #   sub = pubsub.subscribe "my-topic", "my-topic-sub",
-        #                          deadline: 120,
-        #                          endpoint: "https://example.com/push"
-        #
-        # @example With `autocreate`:
-        #   require "google/cloud/pubsub"
-        #
-        #   pubsub = Google::Cloud::Pubsub.new
-        #
-        #   sub = pubsub.subscribe "new-topic", "new-topic-sub",
-        #                          autocreate: true
-        #
-        def subscribe topic_name, subscription_name, deadline: nil,
-                      retain_acked: false, retention: nil, endpoint: nil,
-                      autocreate: nil
-          ensure_service!
-          options = { deadline: deadline, retain_acked: retain_acked,
-                      retention: retention, endpoint: endpoint }
-          grpc = service.create_subscription topic_name,
-                                             subscription_name, options
-          Subscription.from_grpc grpc, service
-        rescue Google::Cloud::NotFoundError => e
-          if autocreate
-            create_topic topic_name
-            return subscribe(topic_name, subscription_name,
-                             deadline: deadline, retain_acked: retain_acked,
-                             retention: retention, endpoint: endpoint,
-                             autocreate: false)
-          end
-          raise e
-        end
-        alias_method :create_subscription, :subscribe
-        alias_method :new_subscription, :subscribe
-
         ##
         # Retrieves subscription by name.
         #
@@ -497,15 +383,9 @@ module Google
 
         ##
         # Call the publish API with arrays of data data and attrs.
-        def publish_batch_messages topic_name, batch, autocreate = false
+        def publish_batch_messages topic_name, batch
           grpc = service.publish topic_name, batch.messages
           batch.to_gcloud_messages Array(grpc.message_ids)
-        rescue Google::Cloud::NotFoundError => e
-          if autocreate
-            create_topic topic_name
-            return publish_batch_messages topic_name, batch, false
-          end
-          raise e
         end
       end
     end