google-cloud-pubsub 1.0.2 → 1.1.0

data/lib/google/cloud/pubsub/message.rb

@@ -100,6 +100,48 @@ module Google
         end
         alias publish_time published_at
 
+        ##
+        # Identifies related messages for which publish order should be
+        # respected.
+        #
+        # Google Cloud Pub/Sub ordering keys provide the ability to ensure
+        # related messages are sent to subscribers in the order in which they
+        # were published. Messages can be tagged with an ordering key, a string
+        # that identifies related messages for which publish order should be
+        # respected. The service guarantees that, for a given ordering key and
+        # publisher, messages are sent to subscribers in the order in which they
+        # were published. Ordering does not require sacrificing high throughput
+        # or scalability, as the service automatically distributes messages for
+        # different ordering keys across subscribers.
+        #
+        # See {Topic#publish_async} and {Subscription#listen}.
+        #
+        # @return [String]
+        #
+        def ordering_key
+          @grpc.ordering_key
+        end
+
+        # @private
+        def hash
+          @grpc.hash
+        end
+
+        # @private
+        def eql? other
+          return false unless other.is_a? self.class
+          @grpc.hash == other.hash
+        end
+        # @private
+        alias == eql?
+
+        # @private
+        def <=> other
+          return nil unless other.is_a? self.class
+          other_grpc = other.instance_variable_get :@grpc
+          @grpc <=> other_grpc
+        end
+
         ##
         # @private New Message from a Google::Cloud::PubSub::V1::PubsubMessage
         # object.

data/lib/google/cloud/pubsub/project.rb

@@ -205,8 +205,7 @@ module Google
         #   pubsub = Google::Cloud::PubSub.new
         #   topic = pubsub.create_topic "my-topic"
         #
-        def create_topic topic_name, labels: nil, kms_key: nil,
-                         persistence_regions: nil, async: nil
+        def create_topic topic_name, labels: nil, kms_key: nil, persistence_regions: nil, async: nil
           ensure_service!
           grpc = service.create_topic topic_name,
                                       labels:              labels,
@@ -291,9 +290,7 @@ module Google
         def subscription subscription_name, project: nil, skip_lookup: nil
           ensure_service!
           options = { project: project }
-          if skip_lookup
-            return Subscription.from_name subscription_name, service, options
-          end
+          return Subscription.from_name subscription_name, service, options if skip_lookup
           grpc = service.get_subscription subscription_name
           Subscription.from_grpc grpc, service
         rescue Google::Cloud::NotFoundError

data/lib/google/cloud/pubsub/publish_result.rb

@@ -17,7 +17,12 @@ module Google
   module Cloud
     module PubSub
       ##
-      # # PublishResult
+      # The result of a publish operation. The message object is available on
+      # {#message} and will have {#message_id} assigned by the API.
+      #
+      # When the publish operation was successful the result will be marked
+      # {#succeeded?}. Otherwise, the result will be marked {#failed?} and the
+      # error raised will be availabe on {#error}.
       #
       class PublishResult
         ##

data/lib/google/cloud/pubsub/received_message.rb

@@ -91,6 +91,28 @@ module Google
         end
         alias msg_id message_id
 
+        ##
+        # Identifies related messages for which publish order should be
+        # respected.
+        #
+        # Google Cloud Pub/Sub ordering keys provide the ability to ensure
+        # related messages are sent to subscribers in the order in which they
+        # were published. Messages can be tagged with an ordering key, a string
+        # that identifies related messages for which publish order should be
+        # respected. The service guarantees that, for a given ordering key and
+        # publisher, messages are sent to subscribers in the order in which they
+        # were published. Ordering does not require sacrificing high throughput
+        # or scalability, as the service automatically distributes messages for
+        # different ordering keys across subscribers.
+        #
+        # See {Topic#publish_async} and {Subscription#listen}.
+        #
+        # @return [String]
+        #
+        def ordering_key
+          message.ordering_key
+        end
+
         ##
         # The time at which the message was published.
         def published_at
@@ -192,6 +214,26 @@ module Google
         alias nack! reject!
         alias ignore! reject!
 
+        # @private
+        def hash
+          @grpc.hash
+        end
+
+        # @private
+        def eql? other
+          return false unless other.is_a? self.class
+          @grpc.hash == other.hash
+        end
+        # @private
+        alias == eql?
+
+        # @private
+        def <=> other
+          return nil unless other.is_a? self.class
+          other_grpc = other.instance_variable_get :@grpc
+          @grpc <=> other_grpc
+        end
+
         ##
         # @private New ReceivedMessage from a
         # Google::Cloud::PubSub::V1::ReceivedMessage object.

data/lib/google/cloud/pubsub/service.rb

@@ -48,15 +48,14 @@ module Google
         def chan_args
           { "grpc.max_send_message_length"           => -1,
             "grpc.max_receive_message_length"        => -1,
-            "grpc.keepalive_time_ms"                 => 300000,
+            "grpc.keepalive_time_ms"                 => 300_000,
             "grpc.service_config_disable_resolution" => 1 }
         end
 
         def chan_creds
           return credentials if insecure?
           require "grpc"
-          GRPC::Core::ChannelCredentials.new.compose \
-            GRPC::Core::CallCredentials.new credentials.client.updater_proc
+          GRPC::Core::ChannelCredentials.new.compose GRPC::Core::CallCredentials.new credentials.client.updater_proc
         end
 
         def subscriber
@@ -126,8 +125,7 @@ module Google
 
         ##
         # Creates the given topic with the given name.
-        def create_topic topic_name, labels: nil, kms_key_name: nil,
-                         persistence_regions: nil, options: {}
+        def create_topic topic_name, labels: nil, kms_key_name: nil, persistence_regions: nil, options: {}
           if persistence_regions
             message_storage_policy = {
               allowed_persistence_regions: Array(persistence_regions)
@@ -147,8 +145,7 @@ module Google
         def update_topic topic_obj, *fields
           mask = Google::Protobuf::FieldMask.new paths: fields.map(&:to_s)
           execute do
-            publisher.update_topic \
-              topic_obj, mask, options: default_options
+            publisher.update_topic topic_obj, mask, options: default_options
           end
         end
 
@@ -240,6 +237,7 @@ module Google
           retain_acked = options[:retain_acked]
           mrd = Convert.number_to_duration options[:retention]
           labels = options[:labels]
+          message_ordering = options[:message_ordering]
 
           execute do
             subscriber.create_subscription \
@@ -249,6 +247,7 @@ module Google
               retain_acked_messages:      retain_acked,
               message_retention_duration: mrd,
               labels:                     labels,
+              enable_message_ordering:    message_ordering,
               options:                    default_options
           end
         end
@@ -256,8 +255,7 @@ module Google
         def update_subscription subscription_obj, *fields
           mask = Google::Protobuf::FieldMask.new paths: fields.map(&:to_s)
           execute do
-            subscriber.update_subscription \
-              subscription_obj, mask, options: default_options
+            subscriber.update_subscription subscription_obj, mask, options: default_options
           end
         end
 
@@ -363,8 +361,7 @@ module Google
         def update_snapshot snapshot_obj, *fields
           mask = Google::Protobuf::FieldMask.new paths: fields.map(&:to_s)
           execute do
-            subscriber.update_snapshot \
-              snapshot_obj, mask, options: default_options
+            subscriber.update_snapshot snapshot_obj, mask, options: default_options
           end
         end
 
@@ -387,9 +384,7 @@ module Google
               time = Convert.time_to_timestamp time_or_snapshot
               subscriber.seek subscription, time: time, options: default_options
             else
-              if time_or_snapshot.is_a? Snapshot
-                time_or_snapshot = time_or_snapshot.name
-              end
+              time_or_snapshot = time_or_snapshot.name if time_or_snapshot.is_a? Snapshot
               subscriber.seek subscription,
                               snapshot: snapshot_path(time_or_snapshot),
                               options:  default_options
@@ -465,14 +460,12 @@ module Google
         end
 
         def snapshot_path snapshot_name, options = {}
-          if snapshot_name.nil? || snapshot_name.to_s.include?("/")
-            return snapshot_name
-          end
+          return snapshot_name if snapshot_name.nil? || snapshot_name.to_s.include?("/")
           "#{project_path options}/snapshots/#{snapshot_name}"
         end
 
         def inspect
-          "#<#{self.class.name} #{@project}>"
+          "#<#{self.class.name} (#{@project})>"
         end
 
         protected

data/lib/google/cloud/pubsub/snapshot/list.rb

@@ -127,15 +127,13 @@ module Google
           #
           def all request_limit: nil
             request_limit = request_limit.to_i if request_limit
-            unless block_given?
-              return enum_for :all, request_limit: request_limit
-            end
+            return enum_for :all, request_limit: request_limit unless block_given?
             results = self
             loop do
               results.each { |r| yield r }
               if request_limit
                 request_limit -= 1
-                break if request_limit < 0
+                break if request_limit.negative?
               end
               break unless results.next?
               results = results.next

data/lib/google/cloud/pubsub/subscriber.rb

@@ -50,6 +50,8 @@ module Google
       # @attr_reader [Numeric] deadline The default number of seconds the stream
       #   will hold received messages before modifying the message's ack
       #   deadline. The minimum is 10, the maximum is 600. Default is 60.
+      # @attr_reader [Boolean] message_ordering Whether message ordering has
+      #   been enabled.
       # @attr_reader [Integer] streams The number of concurrent streams to open
       #   to pull messages from the subscription. Default is 4.
       # @attr_reader [Integer] inventory The number of received messages to be
@@ -65,7 +67,8 @@ module Google
         include MonitorMixin
 
         attr_reader :subscription_name, :callback, :deadline, :streams,
-                    :inventory, :callback_threads, :push_threads
+                    :inventory, :message_ordering, :callback_threads,
+                    :push_threads
 
         ##
         # @private Implementation attributes.
@@ -74,14 +77,16 @@ module Google
 
         ##
         # @private Create an empty {Subscriber} object.
-        def initialize subscription_name, callback, deadline: nil, streams: nil,
-                       inventory: nil, threads: {}, service: nil
+        def initialize subscription_name, callback, deadline: nil,
+                       message_ordering: nil, streams: nil, inventory: nil,
+                       threads: {}, service: nil
           @callback = callback
           @error_callbacks = []
           @subscription_name = subscription_name
           @deadline = deadline || 60
           @streams = streams || 4
           @inventory = inventory || 1000
+          @message_ordering = message_ordering
           @callback_threads = (threads[:callback] || 8).to_i
           @push_threads = (threads[:push] || 4).to_i
 
@@ -292,8 +297,7 @@ module Google
         ##
         # @private
         def to_s
-          "(subscription: #{subscription_name}, " \
-            "streams: [#{stream_pool.map(&:to_s).join(', ')}])"
+          "(subscription: #{subscription_name}, streams: [#{stream_pool.map(&:to_s).join(', ')}])"
         end
 
         ##

data/lib/google/cloud/pubsub/subscriber/inventory.rb

@@ -101,26 +101,24 @@ module Google
           def background_run
             delay_target = nil
 
-            synchronize do
-              until @stopped
-                if @_ack_ids.empty?
-                  delay_target = nil
-
-                  @wait_cond.wait # wait until broadcast
-                  next
-                end
+            until stopped?
+              if empty?
+                delay_target = nil
 
-                delay_target ||= calc_target
-                delay_gap = delay_target - Time.now
+                synchronize { @wait_cond.wait } # wait until broadcast
+                next
+              end
 
-                unless delay_gap.positive?
-                  delay_target = calc_target
-                  stream.renew_lease!
-                  next
-                end
+              delay_target ||= calc_target
+              delay_gap = delay_target - Time.now
 
-                @wait_cond.wait delay_gap
+              unless delay_gap.positive?
+                delay_target = calc_target
+                stream.renew_lease!
+                next
               end
+
+              synchronize { @wait_cond.wait delay_gap }
             end
           end
 

data/lib/google/cloud/pubsub/subscriber/sequencer.rb

@@ -0,0 +1,115 @@
+# Copyright 2019 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+require "monitor"
+
+module Google
+  module Cloud
+    module PubSub
+      class Subscriber
+        ##
+        # @private The sequencer's job is simple, keep track of all the
+        # streams's recieved message and deliver the messages with an
+        # ordering_key in the order they were recieved. The sequencer ensures
+        # only one callback can be performed at a time per ordering_key.
+        class Sequencer
+          include MonitorMixin
+
+          ##
+          # @private Create an empty Subscriber::Sequencer object.
+          def initialize &block
+            raise ArgumentError if block.nil?
+
+            @seq_hash = Hash.new { |hash, key| hash[key] = [] }
+            @process_callback = block
+
+            super() # to init MonitorMixin
+          end
+
+          ##
+          # @private Add a ReceivedMessage to the sequencer.
+          def add message
+            # Messages without ordering_key are not managed by the sequencer
+            if message.ordering_key.empty?
+              @process_callback.call message
+              return
+            end
+
+            perform_callback = synchronize do
+              # The purpose of this block is to add the message to the
+              # sequencer, and to return whether the message should be processed
+              # immediately, or whether it will be processed later by #next. We
+              # want to ensure that these operations happen atomically.
+
+              @seq_hash[message.ordering_key].push message
+              @seq_hash[message.ordering_key].count == 1
+            end
+
+            @process_callback.call message if perform_callback
+          end
+
+          ##
+          # @private Indicate a ReceivedMessage was processed, and the next in
+          # the queue can now be processed.
+          def next message
+            # Messages without ordering_key are not managed by the sequencer
+            return if message.ordering_key.empty?
+
+            next_message = synchronize do
+              # The purpose of this block is to remove the message that was
+              # processed from the sequencer, and to return the next message to
+              # be processed. We want to ensure that these operations happen
+              # atomically.
+
+              # The message should be at index 0, so this should be a very quick
+              # operation.
+              if @seq_hash[message.ordering_key].first != message
+                # Raising this error will stop the other messages with this
+                # ordering key from being processed by the callback (delivered).
+                raise OrderedMessageDeliveryError, message
+              end
+
+              # Remove the message
+              @seq_hash[message.ordering_key].shift
+
+              # Retrieve the next message to be processed, or nil if empty
+              next_msg = @seq_hash[message.ordering_key].first
+
+              # Remove the ordering_key from hash when empty
+              @seq_hash.delete message.ordering_key if next_msg.nil?
+
+              # Return the next message to be processed, or nil if empty
+              next_msg
+            end
+
+            @process_callback.call next_message unless next_message.nil?
+          end
+
+          # @private
+          def to_s
+            "#{@seq_hash.count}/#{@seq_hash.values.sum(&:count)}"
+          end
+
+          # @private
+          def inspect
+            "#<#{self.class.name} (#{self})>"
+          end
+        end
+      end
+    end
+
+    Pubsub = PubSub unless const_defined? :Pubsub
+  end
+end