gcloud 0.3.0 → 0.3.1

checksums.yaml

@@ -1,15 +1,15 @@
 ---
 !binary "U0hBMQ==":
   metadata.gz: !binary |-
-    ZDhjZWRiZTI5NTJjY2FhMmNlNGUzNmRjNTYwYWY0MzhiYmFjOWU2ZA==
+    MzczYWVlN2JlNmY4YzI0MTBiNGM4ZjY5YjZlMjkzMDVmZjQ3MGQwNg==
   data.tar.gz: !binary |-
-    MGE5MDc4NzkzOTQ2ZWE3NWE0ZDZhMWI5ZDc2M2M0NGNiOTFkMGFjYg==
+    YmNkZGUyNzljMjY1YjlhMWQyZjliMGNhZGY2OTdhYjJiOTBiZTNhNw==
 SHA512:
   metadata.gz: !binary |-
-    ODkwZjlhNTcxNTU5ZmQ0OWUzODUxNmY4MDQ1NzIyZTEyOWRiZGI0ZDNkMzc2
-    MDZjNTk0YTM2YzE5M2MzYjI3YTUzYjQ0YzI4YjM3YzUwMDQxYzg4NzdhNWI5
-    NDFkNzE1MDk1YmNkMmEyZTc4ZmNjNzM4NDQzNmVmYTY2ZjA2YzY=
+    MzE5YjkzYmJmNmNmODY4NThhMjdlOTM2YmUxOTZmYWE4MDZjYjBhZDVmN2U1
+    NDMxMjY4NjlkZmZiNGJkNmYyNzczM2RhZWY3YmI4ODY5MmU0N2FiNjI5ZDNi
+    NzBhZGJmNDc4NDU2MWNkMmNkMTA5MzE3MjU0N2Y2N2ZiY2YzYmE=
   data.tar.gz: !binary |-
-    Y2FmNTFjMjhjMWJlNWI4NGQ4YmI4ZjQzYmJiYTU0ZGRhNzNmMTZiYjI3YTFk
-    NDI3YjdiNDZlOTE4M2FkOTlhMDViYWM4NmRkOGMzYmJhM2RjMTE0MjA2NjMz
-    NDA3NWM5N2ViMjk3NmE2ZGY1YzdkZDA0YzcwMjIwOGQxMTQ2ZDk=
+    OGU0MTQzYzRiOGU0NzJlZDllOGFmZWMzYTcwZDc3NTgwMTMxZTIxMjAxOTc4
+    MDQ0M2I2YWQyZGIyZmYyMzQ0MThmMTQ2ZjQ2NzJlNTAzMDI1Y2I2MWY3Y2U2
+    NjdjMjJjZGNlN2EyZmFmNDg3MTcwY2JiZmVkYzExYzZiNTZmZTY=

data/CHANGELOG.md

@@ -1,5 +1,26 @@
 # Release History
 
+### 0.3.1 / 2015-09-08
+
+#### Changes
+
+* Auto-discovery of project-id on Google Compute Engine
+* Support getting project id from GCE auth compute
+* New dataset access DSL for BigQuery
+* New table schema DSL for BigQuery
+* Add Code of Conduct
+
+#### Minor changes
+
+* Load data to BigQuery from Datastore backup
+* Add `Job#wait_until_complete` convenience method to BigQuery
+* Add String representation of tables in BigQuery
+* Add `refresh!` methods to Storage and BigQuery
+* Support `DATASTORE_DATASET` environment variable
+* Update Storage and BigQuery documentation for possible errors during large file uploads
+* Fix missing Pathname require
+* Truncate object representation in interactive output
+
 ### 0.3.0 / 2015-08-21
 
 #### Major changes

data/lib/gcloud.rb

@@ -253,9 +253,4 @@ module Gcloud
     require "gcloud/bigquery"
     Gcloud.bigquery @project, @keyfile, options
   end
-
-  ##
-  # Base Gcloud exception class.
-  class Error < StandardError
-  end
 end

data/lib/gcloud/bigquery.rb

@@ -171,11 +171,7 @@ module Gcloud
   #         "FROM publicdata:samples.shakespeare"
   #   job = bigquery.query_job sql
   #
-  #   loop do
-  #     break if job.done?
-  #     sleep 1
-  #     job.refresh!
-  #   end
+  #   job.wait_until_done!
   #   if !job.failed?
   #     job.query_results.each do |row|
   #       puts row["word"]
@@ -213,33 +209,13 @@ module Gcloud
   #   bigquery = gcloud.bigquery
   #   dataset = bigquery.dataset "my_dataset"
   #
-  #   schema = {
-  #     "fields" => [
-  #       {
-  #         "name" => "first_name",
-  #         "type" => "STRING",
-  #         "mode" => "REQUIRED"
-  #       },
-  #       {
-  #         "name" => "cities_lived",
-  #         "type" => "RECORD",
-  #         "mode" => "REPEATED",
-  #         "fields" => [
-  #           {
-  #             "name" => "place",
-  #             "type" => "STRING",
-  #             "mode" => "REQUIRED"
-  #           },
-  #           {
-  #             "name" => "number_of_years",
-  #             "type" => "INTEGER",
-  #             "mode" => "REQUIRED"
-  #           }
-  #         ]
-  #       }
-  #     ]
-  #   }
-  #   table = dataset.create_table "people", schema: schema
+  #   table = dataset.create_table "people" do |schema|
+  #     schema.string "first_name", mode: :required
+  #     schema.record "cities_lived", mode: :repeated do |nested_schema|
+  #       nested_schema.string "place", mode: :required
+  #       nested_schema.integer "number_of_years", mode: :required
+  #     end
+  #   end
   #
   # Because of the repeated field in this schema, we cannot use the CSV format
   # to load data into the table.
@@ -310,26 +286,11 @@ module Gcloud
   #   gcloud = Gcloud.new
   #   bigquery = gcloud.bigquery
   #   dataset = bigquery.dataset "my_dataset"
-  #   schema = {
-  #     "fields" => [
-  #       {
-  #         "name" => "name",
-  #         "type" => "STRING",
-  #         "mode" => "REQUIRED"
-  #       },
-  #       {
-  #         "name" => "sex",
-  #         "type" => "STRING",
-  #         "mode" => "REQUIRED"
-  #       },
-  #       {
-  #         "name" => "number",
-  #         "type" => "INTEGER",
-  #         "mode" => "REQUIRED"
-  #       }
-  #     ]
-  #   }
-  #   table = dataset.create_table "baby_names", schema: schema
+  #   table = dataset.create_table "baby_names" do |schema|
+  #     schema.string "name", mode: :required
+  #     schema.string "sex", mode: :required
+  #     schema.integer "number", mode: :required
+  #   end
   #
   #   file = File.open "names/yob2014.txt"
   #   load_job = table.load file, format: "csv"
@@ -340,6 +301,22 @@ module Gcloud
   # default format for load operations, the option is not actually necessary.
   # For JSON saved with a +.txt+ extension, however, it would be.
   #
+  # === A note about large uploads
+  #
+  # You may encounter a broken pipe error while attempting to upload large
+  # files. To avoid this problem, add
+  # {httpclient}[https://rubygems.org/gems/httpclient] as a dependency to your
+  # project, and configure {Faraday}[https://rubygems.org/gems/faraday] to use
+  # it, after requiring Gcloud, but before initiating your Gcloud connection.
+  #
+  #   require "gcloud"
+  #
+  #   Faraday.default_adapter = :httpclient
+  #
+  #   gcloud = Gcloud.new
+  #   bigquery = gcloud.bigquery
+  #   dataset = bigquery.dataset "my_dataset"
+  #
   # == Exporting query results to Google Cloud Storage
   #
   # The example below shows how to pass the +table+ option with a query in order
@@ -362,11 +339,7 @@ module Gcloud
   #         "ORDER BY count DESC"
   #   query_job = dataset.query_job sql, table: result_table
   #
-  #   loop do
-  #     break if query_job.done?
-  #     sleep 1
-  #     query_job.refresh!
-  #   end
+  #   query_job.wait_until_done!
   #
   #   if !query_job.failed?
   #
@@ -377,11 +350,7 @@ module Gcloud
   #
   #     extract_job = result_table.extract extract_url
   #
-  #     loop do
-  #       break if extract_job.done?
-  #       sleep 1
-  #       extract_job.refresh!
-  #     end
+  #     extract_job.wait_until_done!
   #
   #     # Download to local filesystem
   #     bucket.files.first.download "baby-names-sam.csv"

data/lib/gcloud/bigquery/connection.rb

@@ -13,6 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
+require "pathname"
 require "gcloud/version"
 require "google/api_client"
 require "digest/md5"
@@ -79,15 +80,11 @@ module Gcloud
       # fields that are provided in the submitted dataset resource.
       def patch_dataset dataset_id, options = {}
         project_id = options[:project_id] || @project
-        body = { friendlyName: options[:name],
-                 description: options[:description],
-                 defaultTableExpirationMs: options[:default_expiration]
-               }.delete_if { |_, v| v.nil? }
 
         @client.execute(
           api_method: @bigquery.datasets.patch,
           parameters: { projectId: project_id, datasetId: dataset_id },
-          body_object: body
+          body_object: patch_dataset_request(options)
         )
       end
 
@@ -315,6 +312,35 @@ module Gcloud
         result
       end
 
+      def default_access_rules
+        [
+          { "role" => "OWNER",  "specialGroup" => "projectOwners" },
+          { "role" => "WRITER", "specialGroup" => "projectWriters" },
+          { "role" => "READER", "specialGroup" => "projectReaders" },
+          { "role" => "OWNER",  "userByEmail"  => credentials.issuer }
+        ]
+      end
+
+      ##
+      # Extracts at least +tbl+ group, and possibly +dts+ and +prj+ groups,
+      # from strings in the formats: "my_table", "my_dataset.my_table", or
+      # "my-project:my_dataset.my_table". Then merges project_id and
+      # dataset_id from the default table if they are missing.
+      def self.table_ref_from_s str, default_table_ref
+        str = str.to_s
+        m = /\A(((?<prj>\S*):)?(?<dts>\S*)\.)?(?<tbl>\S*)\z/.match str
+        unless m
+          fail ArgumentError, "unable to identify table from #{str.inspect}"
+        end
+        default_table_ref.merge("projectId" => m["prj"],
+                                "datasetId" => m["dts"],
+                                "tableId" => m["tbl"])
+      end
+
+      def inspect #:nodoc:
+        "#{self.class}(#{@project})"
+      end
+
       protected
 
       ##
@@ -335,11 +361,20 @@ module Gcloud
           "kind" => "bigquery#dataset",
           "datasetReference" => {
             "projectId" => @project,
-            "datasetId" => dataset_id
-          },
+            "datasetId" => dataset_id },
           "friendlyName" => options[:name],
           "description" => options[:description],
-          "defaultTableExpirationMs" => options[:expiration]
+          "defaultTableExpirationMs" => options[:expiration],
+          "access" => options[:access]
+        }.delete_if { |_, v| v.nil? }
+      end
+
+      def patch_dataset_request options = {}
+        {
+          friendlyName: options[:name],
+          description: options[:description],
+          defaultTableExpirationMs: options[:default_expiration],
+          access: options[:access]
         }.delete_if { |_, v| v.nil? }
       end
 
@@ -457,16 +492,8 @@ module Gcloud
         {
           "configuration" => {
             "copy" => {
-              "sourceTable" => {
-                "projectId" => source["tableReference"]["projectId"],
-                "datasetId" => source["tableReference"]["datasetId"],
-                "tableId" => source["tableReference"]["tableId"]
-              }.delete_if { |_, v| v.nil? },
-              "destinationTable" => {
-                "projectId" => target["tableReference"]["projectId"],
-                "datasetId" => target["tableReference"]["datasetId"],
-                "tableId" => target["tableReference"]["tableId"]
-              }.delete_if { |_, v| v.nil? },
+              "sourceTable" => source,
+              "destinationTable" => target,
               "createDisposition" => create_disposition(options[:create]),
               "writeDisposition" => write_disposition(options[:write])
             }.delete_if { |_, v| v.nil? },
@@ -481,11 +508,7 @@ module Gcloud
           "configuration" => {
             "link" => {
               "sourceUri" => Array(urls),
-              "destinationTable" => {
-                "projectId" => table["tableReference"]["projectId"],
-                "datasetId" => table["tableReference"]["datasetId"],
-                "tableId" => table["tableReference"]["tableId"]
-              }.delete_if { |_, v| v.nil? },
+              "destinationTable" => table,
               "createDisposition" => create_disposition(options[:create]),
               "writeDisposition" => write_disposition(options[:write]),
               "sourceFormat" => source_format(path, options[:format])
@@ -504,11 +527,7 @@ module Gcloud
           "configuration" => {
             "extract" => {
               "destinationUris" => Array(storage_urls),
-              "sourceTable" => {
-                "projectId" => table["tableReference"]["projectId"],
-                "datasetId" => table["tableReference"]["datasetId"],
-                "tableId" => table["tableReference"]["tableId"]
-              }.delete_if { |_, v| v.nil? },
+              "sourceTable" => table,
               "destinationFormat" => dest_format
             }.delete_if { |_, v| v.nil? },
             "dryRun" => options[:dryrun]
@@ -523,14 +542,11 @@ module Gcloud
           "configuration" => {
             "load" => {
               "sourceUris" => Array(urls),
-              "destinationTable" => {
-                "projectId" => table["tableReference"]["projectId"],
-                "datasetId" => table["tableReference"]["datasetId"],
-                "tableId" => table["tableReference"]["tableId"]
-              }.delete_if { |_, v| v.nil? },
+              "destinationTable" => table,
               "createDisposition" => create_disposition(options[:create]),
               "writeDisposition" => write_disposition(options[:write]),
-              "sourceFormat" => source_format(path, options[:format])
+              "sourceFormat" => source_format(path, options[:format]),
+              "projectionFields" => projection_fields(options[:projection_fields])
             }.delete_if { |_, v| v.nil? },
             "dryRun" => options[:dryrun]
           }.delete_if { |_, v| v.nil? }
@@ -568,15 +584,22 @@ module Gcloud
         val = { "csv" => "CSV",
                 "json" => "NEWLINE_DELIMITED_JSON",
                 "newline_delimited_json" => "NEWLINE_DELIMITED_JSON",
-                "avro" => "AVRO" }[format.to_s.downcase]
+                "avro" => "AVRO",
+                "datastore" => "DATASTORE_BACKUP",
+                "datastore_backup" => "DATASTORE_BACKUP"}[format.to_s.downcase]
         return val unless val.nil?
         return nil if path.nil?
         return "CSV" if path.end_with? ".csv"
         return "NEWLINE_DELIMITED_JSON" if path.end_with? ".json"
         return "AVRO" if path.end_with? ".avro"
+        return "DATASTORE_BACKUP" if path.end_with? ".backup_info"
         nil
       end
 
+      def projection_fields array_or_str
+        Array(array_or_str) unless array_or_str.nil?
+      end
+
       # rubocop:enable all
 
       def load_media file, chunk_size = nil

data/lib/gcloud/bigquery/dataset.rb

@@ -16,7 +16,9 @@
 require "json"
 require "gcloud/bigquery/errors"
 require "gcloud/bigquery/table"
+require "gcloud/bigquery/table/schema"
 require "gcloud/bigquery/dataset/list"
+require "gcloud/bigquery/dataset/access"
 
 module Gcloud
   module Bigquery
@@ -34,7 +36,7 @@ module Gcloud
     #   bigquery = gcloud.bigquery
     #
     #   dataset = bigquery.create_dataset "my_dataset",
-    #                                     name: "My Dataset"
+    #                                     name: "My Dataset",
     #                                     description: "This is my Dataset"
     #
     class Dataset
@@ -73,6 +75,15 @@ module Gcloud
         @gapi["datasetReference"]["projectId"]
       end
 
+      ##
+      # The gapi fragment containing the Project ID and Dataset ID as a
+      # camel-cased hash.
+      def dataset_ref #:nodoc:
+        dataset_ref = @gapi["datasetReference"]
+        dataset_ref = dataset_ref.to_hash if dataset_ref.respond_to? :to_hash
+        dataset_ref
+      end
+
       ##
       # A descriptive name for the dataset.
       #
@@ -181,6 +192,89 @@ module Gcloud
         @gapi["location"]
       end
 
+      ##
+      # Retrieves the access rules for a Dataset using the Google Cloud
+      # Datastore API data structure of an array of hashes. The rules can be
+      # updated when passing a block, see Dataset::Access for all the methods
+      # available. See {BigQuery Access
+      # Control}[https://cloud.google.com/bigquery/access-control] for more
+      # information.
+      #
+      # === Examples
+      #
+      #   require "gcloud"
+      #
+      #   gcloud = Gcloud.new
+      #   bigquery = gcloud.bigquery
+      #   dataset = bigquery.dataset "my_dataset"
+      #
+      #   dataset.access #=> [{"role"=>"OWNER",
+      #                  #     "specialGroup"=>"projectOwners"},
+      #                  #    {"role"=>"WRITER",
+      #                  #     "specialGroup"=>"projectWriters"},
+      #                  #    {"role"=>"READER",
+      #                  #     "specialGroup"=>"projectReaders"},
+      #                  #    {"role"=>"OWNER",
+      #                  #     "userByEmail"=>"123456789-...com"}]
+      #
+      # Manage the access rules by passing a block.
+      #
+      #   require "gcloud"
+      #
+      #   gcloud = Gcloud.new
+      #   bigquery = gcloud.bigquery
+      #   dataset = bigquery.dataset "my_dataset"
+      #
+      #   dataset.access do |access|
+      #     access.add_owner_group "owners@example.com"
+      #     access.add_writer_user "writer@example.com"
+      #     access.remove_writer_user "readers@example.com"
+      #     access.add_reader_special :all
+      #     access.add_reader_view other_dataset_view_object
+      #   end
+      #
+      def access
+        ensure_full_data!
+        g = @gapi
+        g = g.to_hash if g.respond_to? :to_hash
+        a = g["access"] ||= []
+        return a unless block_given?
+        a2 = Access.new a, dataset_ref
+        yield a2
+        self.access = a2.access if a2.changed?
+      end
+
+      ##
+      # Sets the access rules for a Dataset using the Google Cloud Datastore API
+      # data structure of an array of hashes. See {BigQuery Access
+      # Control}[https://cloud.google.com/bigquery/access-control] for more
+      # information.
+      #
+      # This method is provided for advanced usage of managing the access rules.
+      # Calling #access with a block is the preferred way to manage access
+      # rules.
+      #
+      # === Example
+      #
+      #   require "gcloud"
+      #
+      #   gcloud = Gcloud.new
+      #   bigquery = gcloud.bigquery
+      #   dataset = bigquery.dataset "my_dataset"
+      #
+      #   dataset.access = [{"role"=>"OWNER",
+      #                      "specialGroup"=>"projectOwners"},
+      #                     {"role"=>"WRITER",
+      #                      "specialGroup"=>"projectWriters"},
+      #                     {"role"=>"READER",
+      #                      "specialGroup"=>"projectReaders"},
+      #                     {"role"=>"OWNER",
+      #                      "userByEmail"=>"123456789-...com"}]
+      #
+      def access= new_access
+        patch_gapi! access: new_access
+      end
+
       ##
       # Permanently deletes the dataset. The dataset must be empty before it can
       # be deleted unless the +force+ option is set to +true+.
@@ -236,10 +330,11 @@ module Gcloud
       # <code>options[:description]</code>::
       #   A user-friendly description of the table. (+String+)
       # <code>options[:schema]</code>::
-      #   A schema specifying fields and data types for the table. See the
+      #   A hash specifying fields and data types for the table. A block may be
+      #   passed instead (see examples.) For the format of this hash, see the
       #   {Tables resource
       #   }[https://cloud.google.com/bigquery/docs/reference/v2/tables#resource]
-      #   for more information. (+Hash+)
+      #   . (+Hash+)
       #
       # === Returns
       #
@@ -254,7 +349,35 @@ module Gcloud
       #   dataset = bigquery.dataset "my_dataset"
       #   table = dataset.create_table "my_table"
       #
-      # A name and description can be provided:
+      # You can also pass name and description options.
+      #
+      #   require "gcloud"
+      #
+      #   gcloud = Gcloud.new
+      #   bigquery = gcloud.bigquery
+      #   dataset = bigquery.dataset "my_dataset"
+      #   table = dataset.create_table "my_table"
+      #                                name: "My Table",
+      #                                description: "A description of my table."
+      #
+      # You can define the table's schema using a block.
+      #
+      #   require "gcloud"
+      #
+      #   gcloud = Gcloud.new
+      #   bigquery = gcloud.bigquery
+      #   dataset = bigquery.dataset "my_dataset"
+      #   table = dataset.create_table "my_table" do |schema|
+      #     schema.string "first_name", mode: :required
+      #     schema.record "cities_lived", mode: :repeated do |nested_schema|
+      #       nested_schema.string "place", mode: :required
+      #       nested_schema.integer "number_of_years", mode: :required
+      #     end
+      #   end
+      #
+      # Or, if you are adapting existing code that was written for the {Rest API
+      # }[https://cloud.google.com/bigquery/docs/reference/v2/tables#resource],
+      # you can pass the table's schema as a hash.
       #
       #   require "gcloud"
       #
@@ -288,20 +411,21 @@ module Gcloud
       #       }
       #     ]
       #   }
-      #   table = dataset.create_table "my_table",
-      #                                name: "My Table",
-      #                                schema: schema
+      #   table = dataset.create_table "my_table", schema: schema
       #
       # :category: Table
       #
       def create_table table_id, options = {}
         ensure_connection!
-        resp = connection.insert_table dataset_id, table_id, options
-        if resp.success?
-          Table.from_gapi resp.data, connection
-        else
-          fail ApiError.from_response(resp)
+        if block_given?
+          if options[:schema]
+            fail ArgumentError, "only schema block or schema option is allowed"
+          end
+          schema_builder = Table::Schema.new nil
+          yield schema_builder
+          options[:schema] = schema_builder.schema if schema_builder.changed?
         end
+        insert_table table_id, options
       end
 
       ##
@@ -447,7 +571,7 @@ module Gcloud
         ensure_connection!
         resp = connection.list_tables dataset_id, options
         if resp.success?
-          Table::List.from_resp resp, connection
+          Table::List.from_response resp, connection
         else
           fail ApiError.from_response(resp)
         end
@@ -516,11 +640,7 @@ module Gcloud
       #
       #   job = bigquery.query_job "SELECT name FROM my_table"
       #
-      #   loop do
-      #     break if job.done?
-      #     sleep 1
-      #     job.refresh!
-      #   end
+      #   job.wait_until_done!
       #   if !job.failed?
       #     job.query_results.each do |row|
       #       puts row["name"]
@@ -621,6 +741,15 @@ module Gcloud
 
       protected
 
+      def insert_table table_id, options
+        resp = connection.insert_table dataset_id, table_id, options
+        if resp.success?
+          Table.from_gapi resp.data, connection
+        else
+          fail ApiError.from_response(resp)
+        end
+      end
+
       ##
       # Raise an error unless an active connection is available.
       def ensure_connection!