google-cloud-pubsub 1.0.2 → 2.19.0

data/lib/google/cloud/pubsub/convert.rb

@@ -28,9 +28,7 @@ module Google
             # Force the object to be a Time object.
             time = time.to_time
 
-            Google::Protobuf::Timestamp.new \
-              seconds: time.to_i,
-              nanos:   time.nsec
+            Google::Protobuf::Timestamp.new seconds: time.to_i, nanos: time.nsec
           end
 
           def timestamp_to_time timestamp
@@ -42,9 +40,7 @@ module Google
           def number_to_duration number
             return nil if number.nil?
 
-            Google::Protobuf::Duration.new \
-              seconds: number.to_i,
-              nanos:   (number.remainder(1) * 1000000000).round
+            Google::Protobuf::Duration.new seconds: number.to_i, nanos: (number.remainder(1) * 1_000_000_000).round
           end
 
           def duration_to_number duration
@@ -52,7 +48,37 @@ module Google
 
             return duration.seconds if duration.nanos.zero?
 
-            duration.seconds + (duration.nanos / 1000000000.0)
+            duration.seconds + (duration.nanos / 1_000_000_000.0)
+          end
+
+          def pubsub_message data, attributes, ordering_key, extra_attrs
+            # TODO: allow data to be a Message object,
+            # then ensure attributes and ordering_key are nil
+            if data.is_a?(::Hash) && (attributes.nil? || attributes.empty?)
+              attributes = data.merge extra_attrs
+              data = nil
+            else
+              attributes = Hash(attributes).merge extra_attrs
+            end
+            # Convert IO-ish objects to strings
+            if data.respond_to?(:read) && data.respond_to?(:rewind)
+              data.rewind
+              data = data.read
+            end
+            # Convert data to encoded byte array to match the protobuf defn
+            data_bytes = String(data).dup.force_encoding(Encoding::ASCII_8BIT).freeze
+
+            # Convert attributes to strings to match the protobuf definition
+            attributes = attributes.to_h { |k, v| [String(k), String(v)] }
+
+            # Ordering Key must always be a string
+            ordering_key = String(ordering_key).freeze
+
+            Google::Cloud::PubSub::V1::PubsubMessage.new(
+              data:         data_bytes,
+              attributes:   attributes,
+              ordering_key: ordering_key
+            )
           end
         end
 

data/lib/google/cloud/pubsub/credentials.rb

@@ -14,7 +14,7 @@
 
 
 require "googleauth"
-require "google/cloud/pubsub/v1/credentials.rb"
+require "google/cloud/pubsub/v1/publisher/credentials"
 
 module Google
   module Cloud
@@ -38,7 +38,7 @@ module Google
       #
       #   pubsub.project_id #=> "my-project"
       #
-      class Credentials < Google::Cloud::PubSub::V1::Credentials
+      class Credentials < Google::Cloud::PubSub::V1::Publisher::Credentials
       end
     end
 

data/lib/google/cloud/pubsub/errors.rb

@@ -0,0 +1,93 @@
+# Copyright 2019 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+require "google/cloud/errors"
+
+module Google
+  module Cloud
+    module PubSub
+      ##
+      # Indicates that the {AsyncPublisher} has been stopped and cannot accept
+      # messages to publish.
+      #
+      class AsyncPublisherStopped < Google::Cloud::Error
+        def initialize message = "Can't publish when stopped."
+          super message
+        end
+      end
+
+      ##
+      # Indicates that the {AsyncPublisher} has not been enabled to publish
+      # messages with an ordering key. Use
+      # {AsyncPublisher#enable_message_ordering!} to enable publishing ordered
+      # messages.
+      #
+      class OrderedMessagesDisabled < Google::Cloud::Error
+        def initialize message = "Ordered messages are disabled."
+          super message
+        end
+      end
+
+      ##
+      # Indicates that the {Subscriber} for a {Subscription} with message
+      # ordering enabled has observed that a message has been delivered out of
+      # order.
+      #
+      class OrderedMessageDeliveryError < Google::Cloud::Error
+        attr_reader :ordered_message
+
+        def initialize ordered_message
+          @ordered_message = ordered_message
+
+          super "Ordered message delivered out of order."
+        end
+      end
+
+      ##
+      # Indicates that messages using the {#ordering_key} are not being
+      # published due to error. Future calls to {Topic#publish_async} with the
+      # {#ordering_key} will fail with this error.
+      #
+      # To allow future messages with the {#ordering_key} to be published, the
+      # {#ordering_key} must be passed to {Topic#resume_publish}.
+      #
+      # If this error is retrieved from {PublishResult#error}, inspect `cause`
+      # for the error raised while publishing.
+      #
+      # @!attribute [r] ordering_key
+      #   @return [String] The ordering key that is in a failed state.
+      #
+      class OrderingKeyError < Google::Cloud::Error
+        attr_reader :ordering_key
+
+        def initialize ordering_key
+          @ordering_key = ordering_key
+
+          super "Can't publish message using #{ordering_key}."
+        end
+      end
+
+      ##
+      # Raised when the desired action is `error` and the message would exceed
+      # flow control limits, or when the desired action is `block` and the
+      # message would block forever against the flow control limits.
+      #
+      class FlowControlLimitError < Google::Cloud::Error
+      end
+    end
+
+    Pubsub = PubSub unless const_defined? :Pubsub
+  end
+end

data/lib/google/cloud/pubsub/flow_controller.rb

@@ -0,0 +1,137 @@
+# Copyright 2021 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+require "google/cloud/pubsub/errors"
+require "concurrent/atomics"
+
+module Google
+  module Cloud
+    module PubSub
+      ##
+      # @private
+      #
+      # Used to control the flow of messages passing through it.
+      #
+      class FlowController
+        attr_reader :message_limit
+        attr_reader :byte_limit
+        attr_reader :limit_exceeded_behavior
+        ##
+        # @private Implementation accessors
+        attr_reader :outstanding_messages, :outstanding_bytes, :awaiting
+
+        def initialize message_limit: 1000, byte_limit: 10_000_000, limit_exceeded_behavior: :ignore
+          unless [:ignore, :error, :block].include? limit_exceeded_behavior
+            raise ArgumentError, "limit_exceeded_behavior must be one of :ignore, :error, :block"
+          end
+          if [:error, :block].include?(limit_exceeded_behavior) && message_limit < 1
+            raise ArgumentError,
+                  "Flow control message limit (#{message_limit}) exceeded by a single message, would block forever"
+          end
+          @mutex = Mutex.new
+          @message_limit = message_limit
+          @byte_limit = byte_limit
+          @limit_exceeded_behavior = limit_exceeded_behavior
+          @outstanding_messages = 0
+          @outstanding_bytes = 0
+
+          @awaiting = []
+        end
+
+        def acquire message_size
+          return if limit_exceeded_behavior == :ignore
+          @mutex.lock
+          if limit_exceeded_behavior == :error && would_exceed_message_limit?
+            raise FlowControlLimitError, "Flow control message limit (#{message_limit}) would be exceeded"
+          end
+          if limit_exceeded_behavior == :error && would_exceed_byte_limit?(message_size)
+            raise FlowControlLimitError,
+                  "Flow control byte limit (#{byte_limit}) would be exceeded, message_size: #{message_size}"
+          end
+          if limit_exceeded_behavior == :block && message_size > byte_limit
+            raise FlowControlLimitError,
+                  "Flow control byte limit (#{byte_limit}) exceeded by a single message, would block forever"
+          end
+
+          acquire_or_wait message_size
+        ensure
+          @mutex.unlock if @mutex.owned?
+        end
+
+        def release message_size
+          return if limit_exceeded_behavior == :ignore
+          @mutex.synchronize do
+            raise "Flow control messages count would be negative" if (@outstanding_messages - 1).negative?
+            raise "Flow control bytes count would be negative" if (@outstanding_bytes - message_size).negative?
+
+            @outstanding_messages -= 1
+            @outstanding_bytes -= message_size
+            @awaiting.first.set unless @awaiting.empty?
+          end
+        end
+
+        protected
+
+        # rubocop:disable Style/GuardClause
+
+        def acquire_or_wait message_size
+          waiter = nil
+          while is_new_and_others_wait?(waiter) ||
+                would_exceed_byte_limit?(message_size) ||
+                would_exceed_message_limit?
+
+            if waiter.nil?
+              waiter = Concurrent::Event.new
+              # This waiter gets added to the back of the line.
+              @awaiting << waiter
+            else
+              waiter = Concurrent::Event.new
+              # This waiter already in line stays at the head of the line.
+              @awaiting[0] = waiter
+            end
+            @mutex.unlock
+            waiter.wait
+            @mutex.lock
+          end
+          @outstanding_messages += 1
+          @outstanding_bytes += message_size
+
+          @awaiting.shift if waiter # Remove the newly released waiter from the head of the queue.
+
+          # There may be some surplus left; let the next message waiting try to acquire a permit.
+          if !@awaiting.empty? && @outstanding_bytes < byte_limit && @outstanding_messages < message_limit
+            @awaiting.first.set
+          end
+        end
+
+        # rubocop:enable Style/GuardClause
+
+        def is_new_and_others_wait? waiter
+          waiter.nil? && !@awaiting.empty?
+        end
+
+        def would_exceed_message_limit?
+          @outstanding_messages + 1 > message_limit
+        end
+
+        def would_exceed_byte_limit? bytes_requested
+          @outstanding_bytes + bytes_requested > byte_limit
+        end
+      end
+    end
+
+    Pubsub = PubSub unless const_defined? :Pubsub
+  end
+end

data/lib/google/cloud/pubsub/message.rb

@@ -50,7 +50,7 @@ module Google
       #   subscriber.start
       #
       #   # Shut down the subscriber when ready to stop receiving messages.
-      #   subscriber.stop.wait!
+      #   subscriber.stop!
       #
       class Message
         ##
@@ -62,7 +62,7 @@ module Google
         # This can be used to publish several messages in bulk.
         def initialize data = nil, attributes = {}
           # Convert attributes to strings to match the protobuf definition
-          attributes = Hash[attributes.map { |k, v| [String(k), String(v)] }]
+          attributes = attributes.to_h { |k, v| [String(k), String(v)] }
 
           @grpc = Google::Cloud::PubSub::V1::PubsubMessage.new(
             data:       String(data).dup.force_encoding(Encoding::ASCII_8BIT),
@@ -81,8 +81,7 @@ module Google
         # Optional attributes for the message.
         def attributes
           return @grpc.attributes.to_h if @grpc.attributes.respond_to? :to_h
-          # Enumerable doesn't have to_h on Ruby 2.0, so fallback to this
-          Hash[@grpc.attributes.to_a]
+          @grpc.attributes.to_a.to_h
         end
 
         ##
@@ -100,6 +99,48 @@ module Google
         end
         alias publish_time published_at
 
+        ##
+        # Identifies related messages for which publish order should be
+        # respected.
+        #
+        # Google Cloud Pub/Sub ordering keys provide the ability to ensure
+        # related messages are sent to subscribers in the order in which they
+        # were published. Messages can be tagged with an ordering key, a string
+        # that identifies related messages for which publish order should be
+        # respected. The service guarantees that, for a given ordering key and
+        # publisher, messages are sent to subscribers in the order in which they
+        # were published. Ordering does not require sacrificing high throughput
+        # or scalability, as the service automatically distributes messages for
+        # different ordering keys across subscribers.
+        #
+        # See {Topic#publish_async} and {Subscription#listen}.
+        #
+        # @return [String]
+        #
+        def ordering_key
+          @grpc.ordering_key
+        end
+
+        # @private
+        def hash
+          @grpc.hash
+        end
+
+        # @private
+        def eql? other
+          return false unless other.is_a? self.class
+          @grpc.hash == other.hash
+        end
+        # @private
+        alias == eql?
+
+        # @private
+        def <=> other
+          return nil unless other.is_a? self.class
+          other_grpc = other.instance_variable_get :@grpc
+          @grpc <=> other_grpc
+        end
+
         ##
         # @private New Message from a Google::Cloud::PubSub::V1::PubsubMessage
         # object.

data/lib/google/cloud/pubsub/policy.rb

@@ -68,7 +68,8 @@ module Google
       #   end
       #
       class Policy
-        attr_reader :etag, :roles
+        attr_reader :etag
+        attr_reader :roles
 
         ##
         # @private Creates a Policy object.
@@ -167,7 +168,7 @@ module Google
                 role:    role_name,
                 members: roles[role_name]
               )
-            end
+            end.compact
           )
         end