gcloud 0.3.0 → 0.3.1

data/lib/gcloud/bigquery/dataset/list.rb

@@ -35,7 +35,7 @@ module Gcloud
 
         ##
         # New Dataset::List from a response object.
-        def self.from_resp resp, conn #:nodoc:
+        def self.from_response resp, conn #:nodoc:
           datasets = List.new(Array(resp.data["datasets"]).map do |gapi_object|
             Dataset.from_gapi gapi_object, conn
           end)

data/lib/gcloud/bigquery/errors.rb

@@ -13,6 +13,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+require "gcloud/errors"
+
 module Gcloud
   module Bigquery
     ##

data/lib/gcloud/bigquery/job.rb

@@ -44,11 +44,7 @@ module Gcloud
     #   q = "SELECT COUNT(word) as count FROM publicdata:samples.shakespeare"
     #   job = bigquery.query_job q
     #
-    #   loop do
-    #     break if job.done?
-    #     sleep 1
-    #     job.refresh!
-    #   end
+    #   job.wait_until_done!
     #
     #   if job.failed?
     #     puts job.error
@@ -219,7 +215,7 @@ module Gcloud
 
       ##
       # Reloads the job with current data from the BigQuery service.
-      def refresh!
+      def reload!
         ensure_connection!
         resp = connection.get_job job_id
         if resp.success?
@@ -228,6 +224,34 @@ module Gcloud
           fail ApiError.from_response(resp)
         end
       end
+      alias_method :refresh!, :reload!
+
+      ##
+      # Refreshes the job until the job is +DONE+.
+      # The delay between refreshes will incrementally increase.
+      #
+      # === Example
+      #
+      #   require "gcloud"
+      #
+      #   gcloud = Gcloud.new
+      #   bigquery = gcloud.bigquery
+      #   dataset = bigquery.dataset "my_dataset"
+      #   table = dataset.table "my_table"
+      #
+      #   extract_job = table.extract "gs://my-bucket/file-name.json",
+      #                               format: "json"
+      #   extract_job.wait_until_done!
+      #   extract_job.done? #=> true
+      def wait_until_done!
+        backoff = ->(retries) { sleep 2 * retries + 5 }
+        retries = 0
+        until done?
+          backoff.call retries
+          retries += 1
+          reload!
+        end
+      end
 
       ##
       # New Job from a Google API Client object.

data/lib/gcloud/bigquery/job/list.rb

@@ -38,7 +38,7 @@ module Gcloud
 
         ##
         # New Job::List from a response object.
-        def self.from_resp resp, conn #:nodoc:
+        def self.from_response resp, conn #:nodoc:
           jobs = List.new(Array(resp.data["jobs"]).map do |gapi_object|
             Job.from_gapi gapi_object, conn
           end)

data/lib/gcloud/bigquery/project.rb

@@ -13,6 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+require "gcloud/gce"
 require "gcloud/bigquery/connection"
 require "gcloud/bigquery/credentials"
 require "gcloud/bigquery/errors"
@@ -77,7 +78,8 @@ module Gcloud
       def self.default_project #:nodoc:
         ENV["BIGQUERY_PROJECT"] ||
           ENV["GCLOUD_PROJECT"] ||
-          ENV["GOOGLE_CLOUD_PROJECT"]
+          ENV["GOOGLE_CLOUD_PROJECT"] ||
+          Gcloud::GCE.project_id
       end
 
       ##
@@ -143,11 +145,7 @@ module Gcloud
       #
       #   job = bigquery.query_job "SELECT name FROM [my_proj:my_data.my_table]"
       #
-      #   loop do
-      #     break if job.done?
-      #     sleep 1
-      #     job.refresh!
-      #   end
+      #   job.wait_until_done!
       #   if !job.failed?
       #     job.query_results.each do |row|
       #       puts row["name"]
@@ -268,6 +266,10 @@ module Gcloud
         end
       end
 
+      # rubocop:disable Metrics/AbcSize
+      # rubocop:disable Metrics/MethodLength
+      # Disabled rubocop because the level of abstraction is not violated here
+
       ##
       # Creates a new dataset.
       #
@@ -286,6 +288,11 @@ module Gcloud
       # <code>options[:expiration]</code>::
       #   The default lifetime of all tables in the dataset, in milliseconds.
       #   The minimum value is 3600000 milliseconds (one hour). (+Integer+)
+      # <code>options[:access]</code>::
+      #   The access rules for a Dataset using the Google Cloud Datastore API
+      #   data structure of an array of hashes. See {BigQuery Access
+      #   Control}[https://cloud.google.com/bigquery/access-control] for more
+      #   information. (+Array of Hashes+)
       #
       # === Returns
       #
@@ -311,7 +318,36 @@ module Gcloud
       #                                     name: "My Dataset",
       #                                     description: "This is my Dataset"
       #
+      # Access rules can be provided with the +access+ option:
+      #
+      #   require "gcloud"
+      #
+      #   gcloud = Gcloud.new
+      #   bigquery = gcloud.bigquery
+      #
+      #   dataset = bigquery.create_dataset "my_dataset",
+      #     access: [{"role"=>"WRITER", "userByEmail"=>"writers@example.com"}]
+      #
+      # Or access rules can be configured by using the block syntax:
+      # (See Dataset::Access)
+      #
+      #   require "gcloud"
+      #
+      #   gcloud = Gcloud.new
+      #   bigquery = gcloud.bigquery
+      #
+      #   dataset = bigquery.create_dataset "my_dataset" do |access|
+      #     access.add_writer_user "writers@example.com"
+      #   end
+      #
       def create_dataset dataset_id, options = {}
+        if block_given?
+          access_builder = Dataset::Access.new connection.default_access_rules,
+                                               "projectId" => project
+          yield access_builder
+          options[:access] = access_builder.access if access_builder.changed?
+        end
+
         ensure_connection!
         resp = connection.insert_dataset dataset_id, options
         if resp.success?
@@ -321,6 +357,9 @@ module Gcloud
         end
       end
 
+      # rubocop:enable Metrics/AbcSize
+      # rubocop:enable Metrics/MethodLength
+
       ##
       # Retrieves the list of datasets belonging to the project.
       #
@@ -387,7 +426,7 @@ module Gcloud
         ensure_connection!
         resp = connection.list_datasets options
         if resp.success?
-          Dataset::List.from_resp resp, connection
+          Dataset::List.from_response resp, connection
         else
           fail ApiError.from_response(resp)
         end
@@ -494,7 +533,7 @@ module Gcloud
         ensure_connection!
         resp = connection.list_jobs options
         if resp.success?
-          Job::List.from_resp resp, connection
+          Job::List.from_response resp, connection
         else
           fail ApiError.from_response(resp)
         end

data/lib/gcloud/bigquery/query_job.rb

@@ -126,11 +126,7 @@ module Gcloud
       #   q = "SELECT word FROM publicdata:samples.shakespeare"
       #   job = bigquery.query_job q
       #
-      #   loop do
-      #     break if job.done?
-      #     sleep 1
-      #     job.refresh!
-      #   end
+      #   job.wait_until_done!
       #   data = job.query_results
       #   data.each do |row|
       #     puts row["word"]

data/lib/gcloud/bigquery/table.rb

@@ -16,6 +16,7 @@
 require "gcloud/bigquery/view"
 require "gcloud/bigquery/data"
 require "gcloud/bigquery/table/list"
+require "gcloud/bigquery/table/schema"
 require "gcloud/bigquery/errors"
 require "gcloud/bigquery/insert_response"
 require "gcloud/upload"
@@ -36,46 +37,25 @@ module Gcloud
     #   gcloud = Gcloud.new
     #   bigquery = gcloud.bigquery
     #   dataset = bigquery.dataset "my_dataset"
-    #   table = dataset.create_table "my_table"
     #
-    #   schema = {
-    #     "fields" => [
-    #       {
-    #         "name" => "first_name",
-    #         "type" => "STRING",
-    #         "mode" => "REQUIRED"
-    #       },
-    #       {
-    #         "name" => "cities_lived",
-    #         "type" => "RECORD",
-    #         "mode" => "REPEATED",
-    #         "fields" => [
-    #           {
-    #             "name" => "place",
-    #             "type" => "STRING",
-    #             "mode" => "REQUIRED"
-    #           },
-    #           {
-    #             "name" => "number_of_years",
-    #             "type" => "INTEGER",
-    #             "mode" => "REQUIRED"
-    #           }
-    #         ]
-    #       }
-    #     ]
-    #   }
-    #   table.schema = schema
+    #   table = dataset.create_table "my_table" do |schema|
+    #     schema.string "first_name", mode: :required
+    #     schema.record "cities_lived", mode: :repeated do |nested_schema|
+    #       nested_schema.string "place", mode: :required
+    #       nested_schema.integer "number_of_years", mode: :required
+    #     end
+    #   end
     #
     #   row = {
     #     "first_name" => "Alice",
     #     "cities_lived" => [
     #       {
-    #         "place": "Seattle",
-    #         "number_of_years": 5
+    #         "place" => "Seattle",
+    #         "number_of_years" => 5
     #       },
     #       {
-    #         "place": "Stockholm",
-    #         "number_of_years": 6
+    #         "place" => "Stockholm",
+    #         "number_of_years" => 6
     #       }
     #     ]
     #   }
@@ -126,6 +106,51 @@ module Gcloud
         @gapi["tableReference"]["projectId"]
       end
 
+      ##
+      # The gapi fragment containing the Project ID, Dataset ID, and Table ID as
+      # a camel-cased hash.
+      def table_ref #:nodoc:
+        table_ref = @gapi["tableReference"]
+        table_ref = table_ref.to_hash if table_ref.respond_to? :to_hash
+        table_ref
+      end
+
+      ##
+      # The combined Project ID, Dataset ID, and Table ID for this table, in the
+      # format specified by the {Query
+      # Reference}[https://cloud.google.com/bigquery/query-reference#from]:
+      # +project_name:datasetId.tableId+. To use this value in queries see
+      # #query_id.
+      #
+      # :category: Attributes
+      #
+      def id
+        @gapi["id"]
+      end
+
+      ##
+      # The value returned by #id, wrapped in square brackets if the Project ID
+      # contains dashes, as specified by the {Query
+      # Reference}[https://cloud.google.com/bigquery/query-reference#from].
+      # Useful in queries.
+      #
+      # === Example
+      #
+      #   require "gcloud"
+      #
+      #   gcloud = Gcloud.new
+      #   bigquery = gcloud.bigquery
+      #   dataset = bigquery.dataset "my_dataset"
+      #   table = dataset.table "my_table"
+      #
+      #   data = bigquery.query "SELECT name FROM #{table.query_id}"
+      #
+      # :category: Attributes
+      #
+      def query_id
+        project_id["-"] ? "[#{id}]" : id
+      end
+
       ##
       # The name of the table.
       #
@@ -266,20 +291,66 @@ module Gcloud
       end
 
       ##
-      # The schema of the table.
+      # Returns the table's schema as hash containing the keys and values
+      # returned by the Google Cloud BigQuery {Rest API
+      # }[https://cloud.google.com/bigquery/docs/reference/v2/tables#resource].
+      # This method can also be used to set, replace, or add to the schema by
+      # passing a block. See Table::Schema for available methods. To set the
+      # schema by passing a hash instead, use #schema=.
+      #
+      # === Parameters
+      #
+      # +options+::
+      #   An optional Hash for controlling additional behavior. (+Hash+)
+      # <code>options[:replace]</code>::
+      #   Whether to replace the existing schema with the new schema. If
+      #   +true+, the fields will replace the existing schema. If
+      #   +false+, the fields will be added to the existing schema. When a table
+      #   already contains data, schema changes must be additive. Thus, the
+      #   default value is +false+. (+Boolean+)
+      #
+      # === Examples
+      #
+      #   require "gcloud"
+      #
+      #   gcloud = Gcloud.new
+      #   bigquery = gcloud.bigquery
+      #   dataset = bigquery.dataset "my_dataset"
+      #   table = dataset.create_table "my_table"
+      #
+      #   table.schema do |schema|
+      #     schema.string "first_name", mode: :required
+      #     schema.record "cities_lived", mode: :repeated do |nested_schema|
+      #       nested_schema.string "place", mode: :required
+      #       nested_schema.integer "number_of_years", mode: :required
+      #     end
+      #   end
       #
       # :category: Attributes
       #
-      def schema
+      def schema options = {}
         ensure_full_data!
-        s = @gapi["schema"]
-        s = s.to_hash if s.respond_to? :to_hash
-        s = {} if s.nil?
-        s
+        g = @gapi
+        g = g.to_hash if g.respond_to? :to_hash
+        s = g["schema"] ||= {}
+        return s unless block_given?
+        s = nil if options[:replace]
+        schema_builder = Schema.new s
+        yield schema_builder
+        self.schema = schema_builder.schema if schema_builder.changed?
       end
 
       ##
       # Updates the schema of the table.
+      # To update the schema using a block instead, use #schema.
+      #
+      # === Parameters
+      #
+      # +schema+::
+      #   A hash containing keys and values as specified by the Google Cloud
+      #   BigQuery {Rest API
+      #   }[https://cloud.google.com/bigquery/docs/reference/v2/tables#resource]
+      #   . (+Hash+)
       #
       # === Example
       #
@@ -385,7 +456,7 @@ module Gcloud
       # === Parameters
       #
       # +destination_table+::
-      #   The destination for the copied data. (+Table+)
+      #   The destination for the copied data. (+Table+ or +String+)
       # +options+::
       #   An optional Hash for controlling additional behavior. (+Hash+)
       # <code>options[:create]</code>::
@@ -409,7 +480,7 @@ module Gcloud
       #
       # Gcloud::Bigquery::CopyJob
       #
-      # === Example
+      # === Examples
       #
       #   require "gcloud"
       #
@@ -421,11 +492,28 @@ module Gcloud
       #
       #   copy_job = table.copy destination_table
       #
+      # The destination table argument can also be a string identifier as
+      # specified by the {Query
+      # Reference}[https://cloud.google.com/bigquery/query-reference#from]:
+      # +project_name:datasetId.tableId+. This is useful for referencing tables
+      # in other projects and datasets.
+      #
+      #   require "gcloud"
+      #
+      #   gcloud = Gcloud.new
+      #   bigquery = gcloud.bigquery
+      #   dataset = bigquery.dataset "my_dataset"
+      #   table = dataset.table "my_table"
+      #
+      #   copy_job = table.copy "other-project:other_dataset.other_table"
+      #
       # :category: Data
       #
       def copy destination_table, options = {}
         ensure_connection!
-        resp = connection.copy_table gapi, destination_table.gapi, options
+        resp = connection.copy_table table_ref,
+                                     get_table_ref(destination_table),
+                                     options
         if resp.success?
           Job.from_gapi resp.data, connection
         else
@@ -467,7 +555,7 @@ module Gcloud
       #
       def link source_url, options = {} #:nodoc:
         ensure_connection!
-        resp = connection.link_table gapi, source_url, options
+        resp = connection.link_table table_ref, source_url, options
         if resp.success?
           Job.from_gapi resp.data, connection
         else
@@ -516,7 +604,7 @@ module Gcloud
       #
       def extract extract_url, options = {}
         ensure_connection!
-        resp = connection.extract_table gapi, extract_url, options
+        resp = connection.extract_table table_ref, extract_url, options
         if resp.success?
           Job.from_gapi resp.data, connection
         else
@@ -542,6 +630,7 @@ module Gcloud
       #   * +csv+ - CSV
       #   * +json+ - {Newline-delimited JSON}[http://jsonlines.org/]
       #   * +avro+ - {Avro}[http://avro.apache.org/]
+      #   * +datastore_backup+ - Cloud Datastore backup
       # <code>options[:create]</code>::
       #   Specifies whether the job is allowed to create new tables. (+String+)
       #
@@ -558,6 +647,12 @@ module Gcloud
       #   * +append+ - BigQuery appends the data to the table.
       #   * +empty+ - An error will be returned if the table already contains
       #     data.
+      # <code>options[:projection_fields]</code>::
+      #   If the +format+ option is set to +datastore_backup+, indicates which
+      #   entity properties to load from a Cloud Datastore backup. Property
+      #   names are case sensitive and must be top-level properties. If not set,
+      #   BigQuery loads all properties. If any named property isn't found in
+      #   the Cloud Datastore backup, an invalid error is returned. (+Array+)
       #
       # === Returns
       #
@@ -589,7 +684,7 @@ module Gcloud
       #   file = bucket.file "file-name.csv"
       #   load_job = table.load file
       #
-      # Or, you can upload a smaller file directly.
+      # Or, you can upload a file directly.
       # See {Loading Data with a POST Request}[
       # https://cloud.google.com/bigquery/loading-data-post-request#multipart].
       #
@@ -603,6 +698,23 @@ module Gcloud
       #   file = File.open "my_data.csv"
       #   load_job = table.load file
       #
+      # === A note about large direct uploads
+      #
+      # You may encounter a broken pipe error while attempting to upload large
+      # files. To avoid this problem, add
+      # {httpclient}[https://rubygems.org/gems/httpclient] as a dependency to
+      # your project, and configure {Faraday}[https://rubygems.org/gems/faraday]
+      # to use it, after requiring Gcloud, but before initiating your Gcloud
+      # connection.
+      #
+      #   require "gcloud"
+      #
+      #   Faraday.default_adapter = :httpclient
+      #
+      #   gcloud = Gcloud.new
+      #   bigquery = gcloud.bigquery
+      #   dataset = bigquery.dataset "my_dataset"
+      #
       # :category: Data
       #
       def load file, options = {}
@@ -700,6 +812,22 @@ module Gcloud
         end
       end
 
+      ##
+      # Reloads the table with current data from the BigQuery service.
+      #
+      # :category: Lifecycle
+      #
+      def reload!
+        ensure_connection!
+        resp = connection.get_table dataset_id, table_id
+        if resp.success?
+          @gapi = resp.data
+        else
+          fail ApiError.from_response(resp)
+        end
+      end
+      alias_method :refresh!, :reload!
+
       ##
       # New Table from a Google API Client object.
       def self.from_gapi gapi, conn #:nodoc:
@@ -737,7 +865,7 @@ module Gcloud
         # Convert to storage URL
         file = file.to_gs_url if file.respond_to? :to_gs_url
 
-        resp = connection.load_table gapi, file, options
+        resp = connection.load_table table_ref, file, options
         if resp.success?
           Job.from_gapi resp.data, connection
         else
@@ -755,7 +883,7 @@ module Gcloud
 
       def load_resumable file, options = {}
         chunk_size = verify_chunk_size! options[:chunk_size]
-        resp = connection.load_resumable gapi, file, chunk_size, options
+        resp = connection.load_resumable table_ref, file, chunk_size, options
         if resp.success?
           Job.from_gapi resp.data, connection
         else
@@ -764,7 +892,7 @@ module Gcloud
       end
 
       def load_multipart file, options = {}
-        resp = connection.load_multipart gapi, file, options
+        resp = connection.load_multipart table_ref, file, options
         if resp.success?
           Job.from_gapi resp.data, connection
         else
@@ -822,6 +950,16 @@ module Gcloud
       def data_complete?
         !@gapi["creationTime"].nil?
       end
+
+      private
+
+      def get_table_ref table
+        if table.respond_to? :table_ref
+          table.table_ref
+        else
+          Connection.table_ref_from_s table, table_ref
+        end
+      end
     end
   end
 end