gcloud 0.10.0 → 0.11.0

data/lib/gcloud/pubsub/topic.rb

@@ -215,30 +215,23 @@ module Gcloud
       #     puts subscription.name
       #   end
       #
-      # @example With pagination: (See {Subscription::List#token})
+      # @example Retrieve all subscriptions: (See {Subscription::List#all})
       #   require "gcloud"
       #
       #   gcloud = Gcloud.new
       #   pubsub = gcloud.pubsub
       #
       #   topic = pubsub.topic "my-topic"
-      #   all_subs = []
-      #   tmp_subs = topic.subscriptions
-      #   while tmp_subs.any? do
-      #     tmp_subs.each do |subscription|
-      #       all_subs << subscription
-      #     end
-      #     # break loop if no more subscriptions available
-      #     break if tmp_subs.token.nil?
-      #     # get the next group of subscriptions
-      #     tmp_subs = topic.subscriptions token: tmp_subs.token
+      #   subscription = topic.subscriptions
+      #   subscriptions.all do |subscription|
+      #     puts subscription.name
       #   end
       #
       def subscriptions token: nil, max: nil
         ensure_service!
         options = { token: token, max: max }
         grpc = service.list_topics_subscriptions name, options
-        Subscription::List.from_grpc grpc, service
+        Subscription::List.from_topic_grpc grpc, service, name, max
       rescue GRPC::BadStatus => e
         raise Error.from_error(e)
       end
@@ -248,7 +241,7 @@ module Gcloud
       ##
       # Publishes one or more messages to the topic.
       #
-      # @param [String] data The message data.
+      # @param [String, File] data The message data.
       # @param [Hash] attributes Optional attributes for the message.
       # @yield [batch] a block for publishing multiple messages in one request
       # @yieldparam [Topic::Batch] batch the batch object
@@ -266,6 +259,15 @@ module Gcloud
       #   topic = pubsub.topic "my-topic"
       #   msg = topic.publish "new-message"
       #
+      # @example A message can be published using a File object:
+      #   require "gcloud"
+      #
+      #   gcloud = Gcloud.new
+      #   pubsub = gcloud.pubsub
+      #
+      #   topic = pubsub.topic "my-topic"
+      #   msg = topic.publish File.open("message.txt")
+      #
       # @example Additionally, a message can be published with attributes:
       #   require "gcloud"
       #

data/lib/gcloud/pubsub/topic/batch.rb

@@ -38,6 +38,13 @@ module Gcloud
         # All messages added will be published at once.
         # See {Gcloud::Pubsub::Topic#publish}
         def publish data, attributes = {}
+          # Convert IO-ish objects to strings
+          if data.respond_to?(:read) && data.respond_to?(:rewind)
+            data.rewind
+            data = data.read
+          end
+          # Convert data to encoded byte array to match the protobuf definition
+          data = String(data).force_encoding("ASCII-8BIT")
           # Convert attributes to strings to match the protobuf definition
           attributes = Hash[attributes.map { |k, v| [String(k), String(v)] }]
           @messages << [data, attributes]
@@ -47,10 +54,9 @@ module Gcloud
         # @private Create Message objects with message ids.
         def to_gcloud_messages message_ids
           msgs = @messages.zip(Array(message_ids)).map do |arr, id|
-            Message.from_grpc(Google::Pubsub::V1::PubsubMessage.new(
-                                data: String(arr[0]).encode("ASCII-8BIT"),
-                                attributes: arr[1],
-                                message_id: id))
+            Message.from_grpc(
+              Google::Pubsub::V1::PubsubMessage.new(
+                data: arr[0], attributes: arr[1], message_id: id))
           end
           # Return just one Message if a single publish,
           # otherwise return the array of Messages.

data/lib/gcloud/pubsub/topic/list.rb

@@ -28,21 +28,147 @@ module Gcloud
         attr_accessor :token
 
         ##
-        # Create a new Topic::List with an array of values.
-        def initialize arr = [], token = nil
+        # @private Create a new Topic::List with an array of values.
+        def initialize arr = []
           super arr
-          @token = token
-          @token = nil if @token == ""
+        end
+
+        ##
+        # Whether there a next page of topics.
+        #
+        # @return [Boolean]
+        #
+        # @example
+        #   require "gcloud"
+        #
+        #   gcloud = Gcloud.new
+        #   pubsub = gcloud.pubsub
+        #
+        #   topics = pubsub.topics
+        #   if topics.next?
+        #     next_topics = topics.next
+        #   end
+        #
+        def next?
+          !token.nil?
+        end
+
+        ##
+        # Retrieve the next page of topics.
+        #
+        # @return [Topic::List]
+        #
+        # @example
+        #   require "gcloud"
+        #
+        #   gcloud = Gcloud.new
+        #   pubsub = gcloud.pubsub
+        #
+        #   topics = pubsub.topics
+        #   if topics.next?
+        #     next_topics = topics.next
+        #   end
+        #
+        def next
+          return nil unless next?
+          ensure_service!
+          options = { token: token, max: @max }
+          grpc = @service.list_topics options
+          self.class.from_grpc grpc, @service, @max
+        rescue GRPC::BadStatus => e
+          raise Error.from_error(e)
+        end
+
+        ##
+        # Retrieves all topics by repeatedly loading {#next} until {#next?}
+        # returns `false`. Calls the given block once for each topic, which is
+        # passed as the parameter.
+        #
+        # An Enumerator is returned if no block is given.
+        #
+        # This method may make several API calls until all topics are retrieved.
+        # Be sure to use as narrow a search criteria as possible. Please use
+        # with caution.
+        #
+        # @param [Integer] request_limit The upper limit of API requests to make
+        #   to load all topics. Default is no limit.
+        # @yield [topic] The block for accessing each topic.
+        # @yieldparam [Topic] topic The topic object.
+        #
+        # @return [Enumerator]
+        #
+        # @example Iterating each topic by passing a block:
+        #   require "gcloud"
+        #
+        #   gcloud = Gcloud.new
+        #   pubsub = gcloud.pubsub
+        #
+        #   topics = pubsub.topics
+        #   topics.all do |topic|
+        #     puts topic.name
+        #   end
+        #
+        # @example Using the enumerator by not passing a block:
+        #   require "gcloud"
+        #
+        #   gcloud = Gcloud.new
+        #   pubsub = gcloud.pubsub
+        #
+        #   topics = pubsub.topics
+        #   all_names = topics.all.map do |topic|
+        #     topic.name
+        #   end
+        #
+        # @example Limit the number of API calls made:
+        #   require "gcloud"
+        #
+        #   gcloud = Gcloud.new
+        #   pubsub = gcloud.pubsub
+        #
+        #   topics = pubsub.topics
+        #   topics.all(request_limit: 10) do |topic|
+        #     puts topic.name
+        #   end
+        #
+        def all request_limit: nil
+          request_limit = request_limit.to_i if request_limit
+          unless block_given?
+            return enum_for(:all, request_limit: request_limit)
+          end
+          results = self
+          loop do
+            results.each { |r| yield r }
+            if request_limit
+              request_limit -= 1
+              break if request_limit < 0
+            end
+            break unless results.next?
+            results = results.next
+          end
         end
 
         ##
         # @private New Topic::List from a Google::Pubsub::V1::ListTopicsResponse
         # object.
-        def self.from_grpc grpc_list, service
-          topics = Array(grpc_list.topics).map do |grpc|
+        def self.from_grpc grpc_list, service, max = nil
+          topics = new(Array(grpc_list.topics).map do |grpc|
             Topic.from_grpc grpc, service
-          end
-          new topics, grpc_list.next_page_token
+          end)
+          token = grpc_list.next_page_token
+          token = nil if token == ""
+          topics.instance_variable_set "@token",   token
+          topics.instance_variable_set "@service", service
+          topics.instance_variable_set "@max",     max
+          topics
+        end
+
+        protected
+
+        ##
+        # @private Raise an error unless an active connection to the service is
+        # available.
+        def ensure_service!
+          fail "Must have active connection to service" unless @service
         end
       end
     end

data/lib/gcloud/resource_manager.rb

@@ -201,6 +201,30 @@ module Gcloud
   # resource_manager.undelete "tokyo-rain-123"
   # ```
   #
+  # ## Configuring Backoff
+  #
+  # The {Gcloud::Backoff} class allows users to globally configure how Cloud API
+  # requests are automatically retried in the case of some errors, such as a
+  # `500` or `503` status code, or a specific internal error code such as
+  # `rateLimitExceeded`.
+  #
+  # If an API call fails, the response will be inspected to see if the call
+  # should be retried. If the response matches the criteria, then the request
+  # will be retried after a delay. If another error occurs, the delay will be
+  # increased incrementally before a subsequent attempt. The first retry will be
+  # delayed one second, the second retry two seconds, and so on.
+  #
+  # ```ruby
+  # require "gcloud"
+  # require "gcloud/backoff"
+  #
+  # Gcloud::Backoff.retries = 5 # Raise the maximum number of retries from 3
+  # ```
+  #
+  # See the [Resource Manager error
+  # messages](https://cloud.google.com/resource-manager/docs/core_errors)
+  # for a list of error conditions.
+  #
   # ## Managing IAM Policies
   #
   # Google Cloud Identity and Access Management ([Cloud

data/lib/gcloud/resource_manager/connection.rb

@@ -14,6 +14,7 @@
 
 
 require "gcloud/version"
+require "gcloud/backoff"
 require "google/api_client"
 
 module Gcloud
@@ -43,14 +44,14 @@ module Gcloud
                    maxResults: max
                  }.delete_if { |_, v| v.nil? }
 
-        @client.execute(
+        execute(
           api_method: @res_man.projects.list,
           parameters: params
         )
       end
 
       def get_project project_id
-        @client.execute(
+        execute(
           api_method: @res_man.projects.get,
           parameters: { projectId: project_id }
         )
@@ -60,7 +61,7 @@ module Gcloud
         project_gapi = { projectId: project_id, name: name,
                          labels: labels }.delete_if { |_, v| v.nil? }
 
-        @client.execute(
+        execute(
           api_method: @res_man.projects.create,
           body_object: project_gapi
         )
@@ -73,7 +74,7 @@ module Gcloud
       def update_project project_gapi
         project_id = project_gapi["projectId"]
 
-        @client.execute(
+        execute(
           api_method: @res_man.projects.update,
           parameters: { projectId: project_id },
           body_object: project_gapi
@@ -81,28 +82,28 @@ module Gcloud
       end
 
       def delete_project project_id
-        @client.execute(
+        execute(
           api_method: @res_man.projects.delete,
           parameters: { projectId: project_id }
         )
       end
 
       def undelete_project project_id
-        @client.execute(
+        execute(
           api_method: @res_man.projects.undelete,
           parameters: { projectId: project_id }
         )
       end
 
       def get_policy project_id
-        @client.execute(
+        execute(
           api_method: @res_man.projects.get_iam_policy,
           parameters: { resource: project_id }
         )
       end
 
       def set_policy project_id, new_policy
-        @client.execute(
+        execute(
           api_method: @res_man.projects.set_iam_policy,
           parameters: { resource: project_id },
           body_object: { policy: new_policy }
@@ -110,7 +111,7 @@ module Gcloud
       end
 
       def test_permissions project_id, permissions
-        @client.execute(
+        execute(
           api_method: @res_man.projects.test_iam_permissions,
           parameters: { resource: project_id },
           body_object: { permissions: permissions }
@@ -120,6 +121,14 @@ module Gcloud
       def inspect
         "#{self.class}(#{@project})"
       end
+
+      protected
+
+      def execute options
+        Gcloud::Backoff.new.execute_gapi do
+          @client.execute options
+        end
+      end
     end
   end
 end

data/lib/gcloud/resource_manager/manager.rb

@@ -85,6 +85,7 @@ module Gcloud
       #   gcloud = Gcloud.new
       #   resource_manager = gcloud.resource_manager
       #   projects = resource_manager.projects
+      #
       #   projects.each do |project|
       #     puts project.project_id
       #   end
@@ -95,24 +96,26 @@ module Gcloud
       #   gcloud = Gcloud.new
       #   resource_manager = gcloud.resource_manager
       #   projects = resource_manager.projects filter: "labels.env:production"
+      #
       #   projects.each do |project|
       #     puts project.project_id
       #   end
       #
-      # @example With pagination: (See {Gcloud::ResourceManager::Project::List})
+      # @example Retrieve all projects: (See {Project::List#all})
       #   require "gcloud"
       #
       #   gcloud = Gcloud.new
       #   resource_manager = gcloud.resource_manager
-      #   projects = resource_manager.projects.all
-      #   projects.each do |project|
+      #   projects = resource_manager.projects
+      #
+      #   projects.all do |project|
       #     puts project.project_id
       #   end
       #
       def projects filter: nil, token: nil, max: nil
         resp = connection.list_project filter: filter, token: token, max: max
         if resp.success?
-          Project::List.from_response resp, self
+          Project::List.from_response resp, self, filter, max
         else
           fail ApiError.from_response(resp)
         end

data/lib/gcloud/resource_manager/project/list.rb

@@ -25,53 +25,132 @@ module Gcloud
         attr_accessor :token
 
         ##
-        # Create a new Project::List with an array of Project instances.
+        # @private Create a new Project::List with an array of Project
+        # instances.
         def initialize arr = []
           super arr
         end
 
         ##
-        # Whether there a next page of projects.
+        # Whether there is a next page of projects.
+        #
+        # @return [Boolean]
+        #
+        # @example
+        #   require "gcloud"
+        #
+        #   gcloud = Gcloud.new
+        #   resource_manager = gcloud.resource_manager
+        #
+        #   projects = resource_manager.projects
+        #   if projects.next?
+        #     next_projects = projects.next
+        #   end
+        #
         def next?
           !token.nil?
         end
 
         ##
         # Retrieve the next page of projects.
+        #
+        # @return [Project::List]
+        #
+        # @example
+        #   require "gcloud"
+        #
+        #   gcloud = Gcloud.new
+        #   resource_manager = gcloud.resource_manager
+        #
+        #   projects = resource_manager.projects
+        #   if projects.next?
+        #     next_projects = projects.next
+        #   end
+        #
         def next
           return nil unless next?
           ensure_manager!
-          @manager.projects token: token
+          @manager.projects token: token, filter: @filter, max: @max
         end
 
         ##
-        # Retrieves all projects by repeatedly loading pages until #next?
-        # returns false. Returns the list instance for method chaining.
+        # Retrieves all projects by repeatedly loading {#next} until {#next?}
+        # returns `false`. Calls the given block once for each project, which is
+        # passed as the parameter.
         #
-        # @example
+        # An Enumerator is returned if no block is given.
+        #
+        # This method may make several API calls until all projects are
+        # retrieved. Be sure to use as narrow a search criteria as possible.
+        # Please use with caution.
+        #
+        # @param [Integer] request_limit The upper limit of API requests to make
+        #   to load all projects. Default is no limit.
+        # @yield [project] The block for accessing each project.
+        # @yieldparam [Project] project The project object.
+        #
+        # @return [Enumerator]
+        #
+        # @example Iterating each project by passing a block:
+        #   require "gcloud"
+        #
+        #   gcloud = Gcloud.new
+        #   resource_manager = gcloud.resource_manager
+        #   projects = resource_manager.projects
+        #
+        #   projects.all do |project|
+        #     puts project.project_id
+        #   end
+        #
+        # @example Using the enumerator by not passing a block:
         #   require "gcloud"
         #
         #   gcloud = Gcloud.new
         #   resource_manager = gcloud.resource_manager
-        #   projects = resource_manager.projects.all # Load all projects
+        #   projects = resource_manager.projects
+        #
+        #   all_project_ids = projects.all.map do |project|
+        #     project.project_id
+        #   end
         #
-        def all
-          while next?
-            next_projects = self.next
-            push(*next_projects)
-            self.token = next_projects.token
+        # @example Limit the number of API calls made:
+        #   require "gcloud"
+        #
+        #   gcloud = Gcloud.new
+        #   resource_manager = gcloud.resource_manager
+        #   projects = resource_manager.projects
+        #
+        #   projects.all(request_limit: 10) do |project|
+        #     puts project.project_id
+        #   end
+        #
+        def all request_limit: nil
+          request_limit = request_limit.to_i if request_limit
+          unless block_given?
+            return enum_for(:all, request_limit: request_limit)
+          end
+          results = self
+          loop do
+            results.each { |r| yield r }
+            if request_limit
+              request_limit -= 1
+              break if request_limit < 0
+            end
+            break unless results.next?
+            results = results.next
           end
-          self
         end
 
         ##
         # @private New Projects::List from a response object.
-        def self.from_response resp, manager
+        def self.from_response resp, manager, filter = nil, max = nil
           projects = new(Array(resp.data["projects"]).map do |gapi_object|
             Project.from_gapi gapi_object, manager.connection
           end)
           projects.instance_variable_set "@token",   resp.data["nextPageToken"]
           projects.instance_variable_set "@manager", manager
+          projects.instance_variable_set "@filter",  filter
+          projects.instance_variable_set "@max",     max
           projects
         end