google-cloud-pubsub 1.9.0

data/lib/google/cloud/pubsub/snapshot.rb

@@ -0,0 +1,202 @@
+# Copyright 2017 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+require "google/cloud/errors"
+require "google/cloud/pubsub/snapshot/list"
+
+module Google
+  module Cloud
+    module PubSub
+      ##
+      # # Snapshot
+      #
+      # A named resource created from a subscription to retain a stream of
+      # messages from a topic. A snapshot is guaranteed to retain:
+      #
+      # * The existing backlog on the subscription. More precisely, this is
+      #   defined as the messages in the subscription's backlog that are
+      #   unacknowledged upon the successful completion of the
+      #   `create_snapshot` operation; as well as:
+      # * Any messages published to the subscription's topic following the
+      #   successful completion of the `create_snapshot` operation.
+      #
+      # @example
+      #   require "google/cloud/pubsub"
+      #
+      #   pubsub = Google::Cloud::PubSub.new
+      #   sub = pubsub.subscription "my-sub"
+      #
+      #   snapshot = sub.create_snapshot "my-snapshot"
+      #   snapshot.name #=> "projects/my-project/snapshots/my-snapshot"
+      #
+      class Snapshot
+        ##
+        # @private The Service object.
+        attr_accessor :service
+
+        ##
+        # @private The gRPC Google::Cloud::PubSub::V1::Snapshot object.
+        attr_accessor :grpc
+
+        ##
+        # @private Create an empty {Snapshot} object.
+        def initialize
+          @service = nil
+          @grpc = Google::Cloud::PubSub::V1::Snapshot.new
+        end
+
+        ##
+        # The name of the snapshot. Format is
+        # `projects/{project}/snapshots/{snap}`.
+        def name
+          @grpc.name
+        end
+
+        ##
+        # The {Topic} from which this snapshot is retaining messages.
+        #
+        # @return [Topic]
+        #
+        # @example
+        #   require "google/cloud/pubsub"
+        #
+        #   pubsub = Google::Cloud::PubSub.new
+        #   sub = pubsub.subscription "my-sub"
+        #
+        #   snapshot = sub.create_snapshot "my-snapshot"
+        #   snapshot.topic.name #=> "projects/my-project/topics/my-topic"
+        #
+        def topic
+          Topic.from_name @grpc.topic, service
+        end
+
+        ##
+        # The snapshot is guaranteed to exist up until this time.
+        # A newly-created snapshot expires no later than 7 days from the time of
+        # its creation. Its exact lifetime is determined at creation by the
+        # existing backlog in the source subscription. Specifically, the
+        # lifetime of the snapshot is 7 days - (age of oldest unacked message in
+        # the subscription). For example, consider a subscription whose oldest
+        # unacked message is 3 days old. If a snapshot is created from this
+        # subscription, the snapshot -- which will always capture this 3-day-old
+        # backlog as long as the snapshot exists -- will expire in 4 days.
+        #
+        # @return [Time] The time until which the snapshot is guaranteed to
+        #   exist.
+        #
+        # @example
+        #   require "google/cloud/pubsub"
+        #
+        #   pubsub = Google::Cloud::PubSub.new
+        #   sub = pubsub.subscription "my-sub"
+        #
+        #   snapshot = sub.create_snapshot "my-snapshot"
+        #   snapshot.topic.name #=> "projects/my-project/topics/my-topic"
+        #   snapshot.expiration_time
+        #
+        def expiration_time
+          self.class.timestamp_from_grpc @grpc.expire_time
+        end
+
+        ##
+        # A hash of user-provided labels associated with this snapshot.
+        # Labels can be used to organize and group snapshots.See [Creating and
+        # Managing Labels](https://cloud.google.com/pubsub/docs/labels).
+        #
+        # The returned hash is frozen and changes are not allowed. Use
+        # {#labels=} to update the labels for this snapshot.
+        #
+        # @return [Hash] The frozen labels hash.
+        #
+        def labels
+          @grpc.labels.to_h.freeze
+        end
+
+        ##
+        # Sets the hash of user-provided labels associated with this
+        # snapshot. Labels can be used to organize and group snapshots.
+        # Label keys and values can be no longer than 63 characters, can only
+        # contain lowercase letters, numeric characters, underscores and dashes.
+        # International characters are allowed. Label values are optional. Label
+        # keys must start with a letter and each label in the list must have a
+        # different key. See [Creating and Managing
+        # Labels](https://cloud.google.com/pubsub/docs/labels).
+        #
+        # @param [Hash] new_labels The new labels hash.
+        #
+        def labels= new_labels
+          raise ArgumentError, "Value must be a Hash" if new_labels.nil?
+          labels_map = Google::Protobuf::Map.new :string, :string
+          Hash(new_labels).each { |k, v| labels_map[String(k)] = String(v) }
+          update_grpc = @grpc.dup
+          update_grpc.labels = labels_map
+          @grpc = service.update_snapshot update_grpc, :labels
+        end
+
+        ##
+        # Removes an existing snapshot. All messages retained in the snapshot
+        # are immediately dropped. After a snapshot is deleted, a new one may be
+        # created with the same name, but the new one has no association with
+        # the old snapshot or its subscription, unless the same subscription is
+        # specified.
+        #
+        # @return [Boolean] Returns `true` if the snapshot was deleted.
+        #
+        # @example
+        #   require "google/cloud/pubsub"
+        #
+        #   pubsub = Google::Cloud::PubSub.new
+        #
+        #   pubsub.snapshots.each do |snapshot|
+        #     snapshot.delete
+        #   end
+        #
+        def delete
+          ensure_service!
+          service.delete_snapshot name
+          true
+        end
+
+        ##
+        # @private New Snapshot from a Google::Cloud::PubSub::V1::Snapshot
+        # object.
+        def self.from_grpc grpc, service
+          new.tap do |f|
+            f.grpc = grpc
+            f.service = service
+          end
+        end
+
+        ##
+        # @private Get a Time object from a Google::Protobuf::Timestamp object.
+        def self.timestamp_from_grpc grpc_timestamp
+          return nil if grpc_timestamp.nil?
+          Time.at grpc_timestamp.seconds, Rational(grpc_timestamp.nanos, 1000)
+        end
+
+        protected
+
+        ##
+        # @private Raise an error unless an active connection to the service is
+        # available.
+        def ensure_service!
+          raise "Must have active connection to service" unless service
+        end
+      end
+    end
+
+    Pubsub = PubSub unless const_defined? :Pubsub
+  end
+end

data/lib/google/cloud/pubsub/snapshot/list.rb

@@ -0,0 +1,178 @@
+# Copyright 2017 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+require "delegate"
+
+module Google
+  module Cloud
+    module PubSub
+      class Snapshot
+        ##
+        # Snapshot::List is a special case Array with additional values.
+        class List < DelegateClass(::Array)
+          ##
+          # If not empty, indicates that there are more snapshots
+          # that match the request and this value should be passed to
+          # the next {Google::Cloud::PubSub::Project#snapshots} to continue.
+          attr_accessor :token
+
+          ##
+          # @private Create a new Snapshot::List with an array of values.
+          def initialize arr = []
+            @prefix = nil
+            @token = nil
+            @max = nil
+            super arr
+          end
+
+          ##
+          # Whether there a next page of snapshots.
+          #
+          # @return [Boolean]
+          #
+          # @example
+          #   require "google/cloud/pubsub"
+          #
+          #   pubsub = Google::Cloud::PubSub.new
+          #
+          #   snapshots = pubsub.snapshots
+          #   if snapshots.next?
+          #     next_snapshots = snapshots.next
+          #   end
+          #
+          def next?
+            !token.nil?
+          end
+
+          ##
+          # Retrieve the next page of snapshots.
+          #
+          # @return [Snapshot::List]
+          #
+          # @example
+          #   require "google/cloud/pubsub"
+          #
+          #   pubsub = Google::Cloud::PubSub.new
+          #
+          #   snapshots = pubsub.snapshots
+          #   if snapshots.next?
+          #     next_snapshots = snapshots.next
+          #   end
+          #
+          def next
+            return nil unless next?
+            ensure_service!
+            next_snapshots
+          end
+
+          ##
+          # Retrieves remaining results by repeatedly invoking {#next} until
+          # {#next?} returns `false`. Calls the given block once for each
+          # result, which is passed as the argument to the block.
+          #
+          # An Enumerator is returned if no block is given.
+          #
+          # This method will make repeated API calls until all remaining results
+          # are retrieved. (Unlike `#each`, for example, which merely iterates
+          # over the results returned by a single API call.) Use with caution.
+          #
+          # @param [Integer] request_limit The upper limit of API requests to
+          #   make to load all snapshots. Default is no limit.
+          # @yield [snapshot] The block for accessing each snapshot.
+          # @yieldparam [Snapshot] snapshot The snapshot object.
+          #
+          # @return [Enumerator]
+          #
+          # @example Iterating each snapshot by passing a block:
+          #   require "google/cloud/pubsub"
+          #
+          #   pubsub = Google::Cloud::PubSub.new
+          #
+          #   snapshots = pubsub.snapshots
+          #   snapshots.all do |snapshot|
+          #     puts snapshot.name
+          #   end
+          #
+          # @example Using the enumerator by not passing a block:
+          #   require "google/cloud/pubsub"
+          #
+          #   pubsub = Google::Cloud::PubSub.new
+          #
+          #   snapshots = pubsub.snapshots
+          #   all_names = snapshots.all.map do |snapshot|
+          #     snapshot.name
+          #   end
+          #
+          # @example Limit the number of API calls made:
+          #   require "google/cloud/pubsub"
+          #
+          #   pubsub = Google::Cloud::PubSub.new
+          #
+          #   snapshots = pubsub.snapshots
+          #   snapshots.all(request_limit: 10) do |snapshot|
+          #     puts snapshot.name
+          #   end
+          #
+          def all request_limit: nil
+            request_limit = request_limit.to_i if request_limit
+            return enum_for :all, request_limit: request_limit unless block_given?
+            results = self
+            loop do
+              results.each { |r| yield r }
+              if request_limit
+                request_limit -= 1
+                break if request_limit.negative?
+              end
+              break unless results.next?
+              results = results.next
+            end
+          end
+
+          ##
+          # @private New Snapshots::List from a
+          # Google::Cloud::PubSub::V1::ListSnapshotsRequest object.
+          def self.from_grpc grpc_list, service, max = nil
+            subs = new(Array(grpc_list.snapshots).map do |grpc|
+              Snapshot.from_grpc grpc, service
+            end)
+            token = grpc_list.next_page_token
+            token = nil if token == "".freeze
+            subs.instance_variable_set :@token,   token
+            subs.instance_variable_set :@service, service
+            subs.instance_variable_set :@max,     max
+            subs
+          end
+
+          protected
+
+          ##
+          # @private Raise an error unless an active connection to the service
+          # is available.
+          def ensure_service!
+            raise "Must have active connection to service" unless @service
+          end
+
+          def next_snapshots
+            options = { prefix: @prefix, token: @token, max: @max }
+            grpc = @service.list_snapshots options
+            self.class.from_grpc grpc, @service, @max
+          end
+        end
+      end
+    end
+
+    Pubsub = PubSub unless const_defined? :Pubsub
+  end
+end

data/lib/google/cloud/pubsub/subscriber.rb

@@ -0,0 +1,399 @@
+# Copyright 2017 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+require "google/cloud/pubsub/service"
+require "google/cloud/pubsub/subscriber/stream"
+require "google/cloud/pubsub/subscriber/timed_unary_buffer"
+require "monitor"
+
+module Google
+  module Cloud
+    module PubSub
+      ##
+      # Subscriber object used to stream and process messages from a
+      # Subscription. See {Google::Cloud::PubSub::Subscription#listen}
+      #
+      # @example
+      #   require "google/cloud/pubsub"
+      #
+      #   pubsub = Google::Cloud::PubSub.new
+      #
+      #   sub = pubsub.subscription "my-topic-sub"
+      #
+      #   subscriber = sub.listen do |received_message|
+      #     # process message
+      #     received_message.acknowledge!
+      #   end
+      #
+      #   # Start background threads that will call the block passed to listen.
+      #   subscriber.start
+      #
+      #   # Shut down the subscriber when ready to stop receiving messages.
+      #   subscriber.stop.wait!
+      #
+      # @attr_reader [String] subscription_name The name of the subscription the
+      #   messages are pulled from.
+      # @attr_reader [Proc] callback The procedure that will handle the messages
+      #   received from the subscription.
+      # @attr_reader [Numeric] deadline The default number of seconds the stream
+      #   will hold received messages before modifying the message's ack
+      #   deadline. The minimum is 10, the maximum is 600. Default is 60.
+      # @attr_reader [Boolean] message_ordering Whether message ordering has
+      #   been enabled.
+      # @attr_reader [Integer] streams The number of concurrent streams to open
+      #   to pull messages from the subscription. Default is 4.
+      # @attr_reader [Integer] callback_threads The number of threads used to
+      #   handle the received messages. Default is 8.
+      # @attr_reader [Integer] push_threads The number of threads to handle
+      #   acknowledgement ({ReceivedMessage#ack!}) and delay messages
+      #   ({ReceivedMessage#nack!}, {ReceivedMessage#modify_ack_deadline!}).
+      #   Default is 4.
+      #
+      class Subscriber
+        include MonitorMixin
+
+        attr_reader :subscription_name, :callback, :deadline, :streams, :message_ordering, :callback_threads,
+                    :push_threads
+
+        ##
+        # @private Implementation attributes.
+        attr_reader :stream_pool, :thread_pool, :buffer, :service
+
+        ##
+        # @private Create an empty {Subscriber} object.
+        def initialize subscription_name, callback, deadline: nil, message_ordering: nil, streams: nil, inventory: nil,
+                       threads: {}, service: nil
+          super() # to init MonitorMixin
+
+          @callback = callback
+          @error_callbacks = []
+          @subscription_name = subscription_name
+          @deadline = deadline || 60
+          @streams = streams || 2
+          coerce_inventory inventory
+          @message_ordering = message_ordering
+          @callback_threads = Integer(threads[:callback] || 8)
+          @push_threads = Integer(threads[:push] || 4)
+
+          @service = service
+
+          @started = @stopped = nil
+
+          stream_pool = Array.new @streams do
+            Thread.new { Stream.new self }
+          end
+          @stream_pool = stream_pool.map(&:value)
+
+          @buffer = TimedUnaryBuffer.new self
+        end
+
+        ##
+        # Starts the subscriber pulling from the subscription and processing the
+        # received messages.
+        #
+        # @return [Subscriber] returns self so calls can be chained.
+        #
+        def start
+          start_pool = synchronize do
+            @started = true
+            @stopped = false
+
+            # Start the buffer before the streams are all started
+            @buffer.start
+            @stream_pool.map do |stream|
+              Thread.new { stream.start }
+            end
+          end
+          start_pool.map(&:join)
+
+          self
+        end
+
+        ##
+        # Immediately stops the subscriber. No new messages will be pulled from
+        # the subscription. All actions taken on received messages that have not
+        # yet been sent to the API will be sent to the API. All received but
+        # unprocessed messages will be released back to the API and redelivered.
+        # Use {#wait!} to block until the subscriber is fully stopped and all
+        # received messages have been processed or released.
+        #
+        # @return [Subscriber] returns self so calls can be chained.
+        #
+        def stop
+          stop_pool = synchronize do
+            @started = false
+            @stopped = true
+
+            @stream_pool.map do |stream|
+              Thread.new { stream.stop }
+            end
+          end
+          stop_pool.map(&:join)
+          # Stop the buffer after the streams are all stopped
+          synchronize { @buffer.stop }
+
+          self
+        end
+
+        ##
+        # Blocks until the subscriber is fully stopped and all received messages
+        # have been processed or released, or until `timeout` seconds have
+        # passed.
+        #
+        # Does not stop the subscriber. To stop the subscriber, first call
+        # {#stop} and then call {#wait!} to block until the subscriber is
+        # stopped.
+        #
+        # @param [Number, nil] timeout The number of seconds to block until the
+        #   subscriber is fully stopped. Default will block indefinitely.
+        #
+        # @return [Subscriber] returns self so calls can be chained.
+        #
+        def wait! timeout = nil
+          wait_pool = synchronize do
+            @stream_pool.map do |stream|
+              Thread.new { stream.wait! timeout }
+            end
+          end
+          wait_pool.map(&:join)
+
+          self
+        end
+
+        ##
+        # Stop this subscriber and block until the subscriber is fully stopped
+        # and all received messages have been processed or released, or until
+        # `timeout` seconds have passed.
+        #
+        # The same as calling {#stop} and {#wait!}.
+        #
+        # @param [Number, nil] timeout The number of seconds to block until the
+        #   subscriber is fully stopped. Default will block indefinitely.
+        #
+        # @return [Subscriber] returns self so calls can be chained.
+        #
+        def stop! timeout = nil
+          stop
+          wait! timeout
+        end
+
+        ##
+        # Whether the subscriber has been started.
+        #
+        # @return [boolean] `true` when started, `false` otherwise.
+        #
+        def started?
+          synchronize { @started }
+        end
+
+        ##
+        # Whether the subscriber has been stopped.
+        #
+        # @return [boolean] `true` when stopped, `false` otherwise.
+        #
+        def stopped?
+          synchronize { @stopped }
+        end
+
+        ##
+        # Register to be notified of errors when raised.
+        #
+        # If an unhandled error has occurred the subscriber will attempt to
+        # recover from the error and resume listening.
+        #
+        # Multiple error handlers can be added.
+        #
+        # @yield [callback] The block to be called when an error is raised.
+        # @yieldparam [Exception] error The error raised.
+        #
+        # @example
+        #   require "google/cloud/pubsub"
+        #
+        #   pubsub = Google::Cloud::PubSub.new
+        #
+        #   sub = pubsub.subscription "my-topic-sub"
+        #
+        #   subscriber = sub.listen do |received_message|
+        #     # process message
+        #     received_message.acknowledge!
+        #   end
+        #
+        #   # Register to be notified when unhandled errors occur.
+        #   subscriber.on_error do |error|
+        #     # log error
+        #     puts error
+        #   end
+        #
+        #   # Start listening for messages and errors.
+        #   subscriber.start
+        #
+        #   # Shut down the subscriber when ready to stop receiving messages.
+        #   subscriber.stop.wait!
+        #
+        def on_error &block
+          synchronize do
+            @error_callbacks << block
+          end
+        end
+
+        ##
+        # The most recent unhandled error to occur while listening to messages
+        # on the subscriber.
+        #
+        # If an unhandled error has occurred the subscriber will attempt to
+        # recover from the error and resume listening.
+        #
+        # @return [Exception, nil] error The most recent error raised.
+        #
+        # @example
+        #   require "google/cloud/pubsub"
+        #
+        #   pubsub = Google::Cloud::PubSub.new
+        #
+        #   sub = pubsub.subscription "my-topic-sub"
+        #
+        #   subscriber = sub.listen do |received_message|
+        #     # process message
+        #     received_message.acknowledge!
+        #   end
+        #
+        #   # Start listening for messages and errors.
+        #   subscriber.start
+        #
+        #   # If an error was raised, it can be retrieved here:
+        #   subscriber.last_error #=> nil
+        #
+        #   # Shut down the subscriber when ready to stop receiving messages.
+        #   subscriber.stop.wait!
+        #
+        def last_error
+          synchronize { @last_error }
+        end
+
+        ##
+        # The number of received messages to be collected by subscriber. Default is 1,000.
+        #
+        # @return [Integer] The maximum number of messages.
+        #
+        def max_outstanding_messages
+          @inventory[:max_outstanding_messages]
+        end
+        # @deprecated Use {#max_outstanding_messages}.
+        alias inventory_limit max_outstanding_messages
+        # @deprecated Use {#max_outstanding_messages}.
+        alias inventory max_outstanding_messages
+
+        ##
+        # The total byte size of received messages to be collected by subscriber. Default is 100,000,000 (100MB).
+        #
+        # @return [Integer] The maximum number of bytes.
+        #
+        def max_outstanding_bytes
+          @inventory[:max_outstanding_bytes]
+        end
+        # @deprecated Use {#max_outstanding_bytes}.
+        alias inventory_bytesize max_outstanding_bytes
+
+        ##
+        # The number of seconds that received messages can be held awaiting processing. Default is 3,600 (1 hour).
+        #
+        # @return [Integer] The maximum number of seconds.
+        #
+        def max_total_lease_duration
+          @inventory[:max_total_lease_duration]
+        end
+        # @deprecated Use {#max_total_lease_duration}.
+        alias inventory_extension max_total_lease_duration
+
+        ##
+        # The maximum amount of time in seconds for a single lease extension attempt. Bounds the delay before a message
+        # redelivery if the subscriber fails to extend the deadline. Default is 0 (disabled).
+        #
+        # @return [Integer] The maximum number of seconds.
+        #
+        def max_duration_per_lease_extension
+          @inventory[:max_duration_per_lease_extension]
+        end
+
+        ##
+        # @private
+        def stream_inventory
+          {
+            limit:                            @inventory[:max_outstanding_messages].fdiv(@streams).ceil,
+            bytesize:                         @inventory[:max_outstanding_bytes].fdiv(@streams).ceil,
+            extension:                        @inventory[:max_total_lease_duration],
+            max_duration_per_lease_extension: @inventory[:max_duration_per_lease_extension]
+          }
+        end
+
+        # @private returns error object from the stream thread.
+        def error! error
+          error_callbacks = synchronize do
+            @last_error = error
+            @error_callbacks
+          end
+          error_callbacks = default_error_callbacks if error_callbacks.empty?
+          error_callbacks.each { |error_callback| error_callback.call error }
+        end
+
+        ##
+        # @private
+        def to_s
+          "(subscription: #{subscription_name}, streams: [#{stream_pool.map(&:to_s).join(', ')}])"
+        end
+
+        ##
+        # @private
+        def inspect
+          "#<#{self.class.name} #{self}>"
+        end
+
+        protected
+
+        def coerce_inventory inventory
+          @inventory = inventory
+          if @inventory.is_a? Hash
+            @inventory = @inventory.dup
+            # Support deprecated field names
+            @inventory[:max_outstanding_messages] ||= @inventory.delete :limit
+            @inventory[:max_outstanding_bytes] ||= @inventory.delete :bytesize
+            @inventory[:max_total_lease_duration] ||= @inventory.delete :extension
+          else
+            @inventory = { max_outstanding_messages: @inventory }
+          end
+          @inventory[:max_outstanding_messages] = Integer(@inventory[:max_outstanding_messages] || 1000)
+          @inventory[:max_outstanding_bytes] = Integer(@inventory[:max_outstanding_bytes] || 100_000_000)
+          @inventory[:max_total_lease_duration] = Integer(@inventory[:max_total_lease_duration] || 3600)
+          @inventory[:max_duration_per_lease_extension] = Integer(@inventory[:max_duration_per_lease_extension] || 0)
+        end
+
+        def default_error_callbacks
+          # This is memoized to reduce calls to the configuration.
+          @default_error_callbacks ||= begin
+            error_callback = Google::Cloud::PubSub.configure.on_error
+            error_callback ||= Google::Cloud.configure.on_error
+            if error_callback
+              [error_callback]
+            else
+              []
+            end
+          end
+        end
+      end
+    end
+
+    Pubsub = PubSub unless const_defined? :Pubsub
+  end
+end