gcloud 0.11.0 → 0.12.0

data/lib/gcloud/bigquery/table.rb

@@ -13,13 +13,14 @@
 # limitations under the License.
 
 
+require "gcloud/errors"
+require "gcloud/bigquery/service"
 require "gcloud/bigquery/view"
 require "gcloud/bigquery/data"
 require "gcloud/bigquery/table/list"
-require "gcloud/bigquery/table/schema"
-require "gcloud/bigquery/errors"
+require "gcloud/bigquery/schema"
 require "gcloud/bigquery/insert_response"
-require "gcloud/upload"
+require "google/apis/bigquery_v2"
 
 module Gcloud
   module Bigquery
@@ -65,8 +66,8 @@ module Gcloud
     #
     class Table
       ##
-      # @private The Connection object.
-      attr_accessor :connection
+      # @private The Service object.
+      attr_accessor :service
 
       ##
       # @private The Google API Client object.
@@ -75,8 +76,8 @@ module Gcloud
       ##
       # @private Create an empty Table object.
       def initialize
-        @connection = nil
-        @gapi = {}
+        @service = nil
+        @gapi = Google::Apis::BigqueryV2::Table.new
       end
 
       ##
@@ -87,7 +88,7 @@ module Gcloud
       # @!group Attributes
       #
       def table_id
-        @gapi["tableReference"]["tableId"]
+        @gapi.table_reference.table_id
       end
 
       ##
@@ -96,7 +97,7 @@ module Gcloud
       # @!group Attributes
       #
       def dataset_id
-        @gapi["tableReference"]["datasetId"]
+        @gapi.table_reference.dataset_id
       end
 
       ##
@@ -105,7 +106,7 @@ module Gcloud
       # @!group Attributes
       #
       def project_id
-        @gapi["tableReference"]["projectId"]
+        @gapi.table_reference.project_id
       end
 
       ##
@@ -113,7 +114,7 @@ module Gcloud
       # The gapi fragment containing the Project ID, Dataset ID, and Table ID as
       # a camel-cased hash.
       def table_ref
-        table_ref = @gapi["tableReference"]
+        table_ref = @gapi.table_reference
         table_ref = table_ref.to_hash if table_ref.respond_to? :to_hash
         table_ref
       end
@@ -128,7 +129,7 @@ module Gcloud
       # @!group Attributes
       #
       def id
-        @gapi["id"]
+        @gapi.id
       end
 
       ##
@@ -159,7 +160,7 @@ module Gcloud
       # @!group Attributes
       #
       def name
-        @gapi["friendlyName"]
+        @gapi.friendly_name
       end
 
       ##
@@ -168,7 +169,8 @@ module Gcloud
       # @!group Attributes
       #
       def name= new_name
-        patch_gapi! name: new_name
+        @gapi.update! friendly_name: new_name
+        patch_gapi! :friendly_name
       end
 
       ##
@@ -178,7 +180,7 @@ module Gcloud
       #
       def etag
         ensure_full_data!
-        @gapi["etag"]
+        @gapi.etag
       end
 
       ##
@@ -188,7 +190,7 @@ module Gcloud
       #
       def api_url
         ensure_full_data!
-        @gapi["selfLink"]
+        @gapi.self_link
       end
 
       ##
@@ -198,7 +200,7 @@ module Gcloud
       #
       def description
         ensure_full_data!
-        @gapi["description"]
+        @gapi.description
       end
 
       ##
@@ -207,7 +209,8 @@ module Gcloud
       # @!group Attributes
       #
       def description= new_description
-        patch_gapi! description: new_description
+        @gapi.update! description: new_description
+        patch_gapi! :description
       end
 
       ##
@@ -217,7 +220,11 @@ module Gcloud
       #
       def bytes_count
         ensure_full_data!
-        @gapi["numBytes"]
+        begin
+          Integer @gapi.num_bytes
+        rescue
+          nil
+        end
       end
 
       ##
@@ -227,7 +234,11 @@ module Gcloud
       #
       def rows_count
         ensure_full_data!
-        @gapi["numRows"]
+        begin
+          Integer @gapi.num_rows
+        rescue
+          nil
+        end
       end
 
       ##
@@ -237,7 +248,11 @@ module Gcloud
       #
       def created_at
         ensure_full_data!
-        Time.at(@gapi["creationTime"] / 1000.0)
+        begin
+          Time.at(Integer(@gapi.creation_time) / 1000.0)
+        rescue
+          nil
+        end
       end
 
       ##
@@ -249,8 +264,11 @@ module Gcloud
       #
       def expires_at
         ensure_full_data!
-        return nil if @gapi["expirationTime"].nil?
-        Time.at(@gapi["expirationTime"] / 1000.0)
+        begin
+          Time.at(Integer(@gapi.expiration_time) / 1000.0)
+        rescue
+          nil
+        end
       end
 
       ##
@@ -260,7 +278,11 @@ module Gcloud
       #
       def modified_at
         ensure_full_data!
-        Time.at(@gapi["lastModifiedTime"] / 1000.0)
+        begin
+          Time.at(Integer(@gapi.last_modified_time) / 1000.0)
+        rescue
+          nil
+        end
       end
 
       ##
@@ -269,7 +291,7 @@ module Gcloud
       # @!group Attributes
       #
       def table?
-        @gapi["type"] == "TABLE"
+        @gapi.type == "TABLE"
       end
 
       ##
@@ -278,7 +300,7 @@ module Gcloud
       # @!group Attributes
       #
       def view?
-        @gapi["type"] == "VIEW"
+        @gapi.type == "VIEW"
       end
 
       ##
@@ -289,16 +311,13 @@ module Gcloud
       #
       def location
         ensure_full_data!
-        @gapi["location"]
+        @gapi.location
       end
 
       ##
-      # Returns the table's schema as hash containing the keys and values
-      # returned by the Google Cloud BigQuery [Rest API
-      # ](https://cloud.google.com/bigquery/docs/reference/v2/tables#resource).
-      # This method can also be used to set, replace, or add to the schema by
-      # passing a block. See {Table::Schema} for available methods. To set the
-      # schema by passing a hash instead, use {#schema=}.
+      # Returns the table's schema. This method can also be used to set,
+      # replace, or add to the schema by passing a block. See {Schema} for
+      # available methods.
       #
       # @param [Boolean] replace Whether to replace the existing schema with the
       #   new schema. If `true`, the fields will replace the existing schema. If
@@ -308,6 +327,8 @@ module Gcloud
       # @yield [schema] a block for setting the schema
       # @yieldparam [Table::Schema] schema the object accepting the schema
       #
+      # @return [Gcloud::Bigquery::Schema]
+      #
       # @example
       #   require "gcloud"
       #
@@ -328,53 +349,20 @@ module Gcloud
       #
       def schema replace: false
         ensure_full_data!
-        g = @gapi
-        g = g.to_hash if g.respond_to? :to_hash
-        s = g["schema"] ||= {}
-        return s unless block_given?
-        s = nil if replace
-        schema_builder = Schema.new s
-        yield schema_builder
-        self.schema = schema_builder.schema if schema_builder.changed?
-      end
-
-      ##
-      # Updates the schema of the table.
-      # To update the schema using a block instead, use #schema.
-      #
-      # @param [Hash] new_schema A hash containing keys and values as specified
-      #   by the Google Cloud BigQuery [Rest API
-      #   ](https://cloud.google.com/bigquery/docs/reference/v2/tables#resource)
-      #   .
-      #
-      # @example
-      #   require "gcloud"
-      #
-      #   gcloud = Gcloud.new
-      #   bigquery = gcloud.bigquery
-      #   dataset = bigquery.dataset "my_dataset"
-      #   table = dataset.create_table "my_table"
-      #
-      #   schema = {
-      #     "fields" => [
-      #       {
-      #         "name" => "first_name",
-      #         "type" => "STRING",
-      #         "mode" => "REQUIRED"
-      #       },
-      #       {
-      #         "name" => "age",
-      #         "type" => "INTEGER",
-      #         "mode" => "REQUIRED"
-      #       }
-      #     ]
-      #   }
-      #   table.schema = schema
-      #
-      # @!group Attributes
-      #
-      def schema= new_schema
-        patch_gapi! schema: new_schema
+        schema_builder = Schema.from_gapi @gapi.schema
+        if block_given?
+          if replace
+            empty_schema = Google::Apis::BigqueryV2::TableSchema.new fields: []
+            schema_builder = Schema.from_gapi empty_schema
+          end
+          yield schema_builder
+          schema_builder.check_for_mutated_schema!
+          if schema_builder.changed?
+            @gapi.schema = schema_builder.to_gapi
+            patch_gapi! :schema
+          end
+        end
+        schema_builder.freeze
       end
 
       ##
@@ -383,10 +371,7 @@ module Gcloud
       # @!group Attributes
       #
       def fields
-        f = schema["fields"]
-        f = f.to_hash if f.respond_to? :to_hash
-        f = [] if f.nil?
-        f
+        schema.fields
       end
 
       ##
@@ -395,7 +380,7 @@ module Gcloud
       # @!group Attributes
       #
       def headers
-        fields.map { |f| f["name"] }
+        fields.map(&:name)
       end
 
       ##
@@ -441,14 +426,10 @@ module Gcloud
       # @!group Data
       #
       def data token: nil, max: nil, start: nil
-        ensure_connection!
+        ensure_service!
         options = { token: token, max: max, start: start }
-        resp = connection.list_tabledata dataset_id, table_id, options
-        if resp.success?
-          Data.from_response resp, self
-        else
-          fail ApiError.from_response(resp)
-        end
+        gapi = service.list_tabledata dataset_id, table_id, options
+        Data.from_gapi gapi, self
       end
 
       ##
@@ -505,54 +486,12 @@ module Gcloud
       # @!group Data
       #
       def copy destination_table, create: nil, write: nil, dryrun: nil
-        ensure_connection!
-        options = { create: create, write: write, dryrun: dryrun }
-        resp = connection.copy_table table_ref,
-                                     get_table_ref(destination_table),
-                                     options
-        if resp.success?
-          Job.from_gapi resp.data, connection
-        else
-          fail ApiError.from_response(resp)
-        end
-      end
-
-      ##
-      # @private
-      # Links the table to a source table identified by a URI.
-      #
-      # @param [String] source_url The URI of source table to link.
-      # @param [String] create Specifies whether the job is allowed to create
-      #   new tables.
-      #
-      #   The following values are supported:
-      #
-      #   * `needed` - Create the table if it does not exist.
-      #   * `never` - The table must already exist. A 'notFound' error is
-      #     raised if the table does not exist.
-      # @param [String] write Specifies how to handle data already present in
-      #   the destination table. The default value is `empty`.
-      #
-      #   The following values are supported:
-      #
-      #   * `truncate` - BigQuery overwrites the table data.
-      #   * `append` - BigQuery appends the data to the table.
-      #   * `empty` - An error will be returned if the destination table already
-      #     contains data.
-      #
-      # @return [Gcloud::Bigquery::Job]
-      #
-      # @!group Data
-      #
-      def link source_url, create: nil, write: nil, dryrun: nil
-        ensure_connection!
+        ensure_service!
         options = { create: create, write: write, dryrun: dryrun }
-        resp = connection.link_table table_ref, source_url, options
-        if resp.success?
-          Job.from_gapi resp.data, connection
-        else
-          fail ApiError.from_response(resp)
-        end
+        gapi = service.copy_table table_ref,
+                                  get_table_ref(destination_table),
+                                  options
+        Job.from_gapi gapi, service
       end
 
       ##
@@ -598,21 +537,17 @@ module Gcloud
       #
       def extract extract_url, format: nil, compression: nil, delimiter: nil,
                   header: nil, dryrun: nil
-        ensure_connection!
+        ensure_service!
         options = { format: format, compression: compression,
                     delimiter: delimiter, header: header, dryrun: dryrun }
-        resp = connection.extract_table table_ref, extract_url, options
-        if resp.success?
-          Job.from_gapi resp.data, connection
-        else
-          fail ApiError.from_response(resp)
-        end
+        gapi = service.extract_table table_ref, extract_url, options
+        Job.from_gapi gapi, service
       end
 
       ##
-      # Loads data into the table. You can pass a gcloud storage file path or
-      # a gcloud storage file instance. Or, you can upload a file directly.
-      # See [Loading Data with a POST Request](
+      # Loads data into the table. You can pass a gcloud-ruby storage file path
+      # or a gcloud-ruby storage file instance. Or, you can upload a file
+      # directly. See [Loading Data with a POST Request](
       # https://cloud.google.com/bigquery/loading-data-post-request#multipart).
       #
       # A `chunk_size` value can be provided in the options to be used in
@@ -698,10 +633,6 @@ module Gcloud
       #   file that BigQuery will skip when loading the data. The default value
       #   is `0`. This property is useful if you have header rows in the file
       #   that should be skipped.
-      # @param [Integer] chunk_size The number of bytes per chunk in a resumable
-      #   upload. Must be divisible by 256KB. If it is not divisible by 265KB
-      #   then it will be lowered to the nearest acceptable value. If no value
-      #   is provided it will use {Gcloud::Upload.default_chunk_size}. Optional.
       #
       # @return [Gcloud::Bigquery::LoadJob]
       #
@@ -715,7 +646,7 @@ module Gcloud
       #
       #   load_job = table.load "gs://my-bucket/file-name.csv"
       #
-      # @example Pass a gcloud storage file instance:
+      # @example Pass a gcloud-ruby storage file instance:
       #   require "gcloud"
       #   require "gcloud/storage"
       #
@@ -740,46 +671,18 @@ module Gcloud
       #   file = File.open "my_data.csv"
       #   load_job = table.load file
       #
-      # ### A note about large direct uploads
-      #
-      # You may encounter a Broken pipe (Errno::EPIPE) error when attempting to
-      # upload large files. To avoid this problem, add the
-      # [httpclient](https://rubygems.org/gems/httpclient) gem to your project,
-      # and the line (or lines) of configuration shown below. These lines must
-      # execute after you require gcloud but before you make your first gcloud
-      # connection. The first statement configures
-      # [Faraday](https://rubygems.org/gems/faraday) to use httpclient. The
-      # second statement, which should only be added if you are using a version
-      # of Faraday at or above 0.9.2, is a workaround for [this gzip
-      # issue](https://github.com/GoogleCloudPlatform/gcloud-ruby/issues/367).
-      #
-      # @example
-      #   require "gcloud"
-      #
-      #   # Use httpclient to avoid broken pipe errors with large uploads
-      #   Faraday.default_adapter = :httpclient
-      #
-      #   # Only add the following statement if using Faraday >= 0.9.2
-      #   # Override gzip middleware with no-op for httpclient
-      #   Faraday::Response.register_middleware :gzip =>
-      #                                           Faraday::Response::Middleware
-      #
-      #   gcloud = Gcloud.new
-      #   bigquery = gcloud.bigquery
-      #
       # @!group Data
       #
       def load file, format: nil, create: nil, write: nil,
                projection_fields: nil, jagged_rows: nil, quoted_newlines: nil,
                encoding: nil, delimiter: nil, ignore_unknown: nil,
-               max_bad_records: nil, quote: nil, skip_leading: nil, dryrun: nil,
-               chunk_size: nil
-        ensure_connection!
+               max_bad_records: nil, quote: nil, skip_leading: nil, dryrun: nil
+        ensure_service!
         options = { format: format, create: create, write: write,
                     projection_fields: projection_fields,
                     jagged_rows: jagged_rows, quoted_newlines: quoted_newlines,
                     encoding: encoding, delimiter: delimiter,
-                    ignore_unknown: ignore_unknown, chunk_size: chunk_size,
+                    ignore_unknown: ignore_unknown,
                     max_bad_records: max_bad_records, quote: quote,
                     skip_leading: skip_leading, dryrun: dryrun }
         return load_storage(file, options) if storage_url? file
@@ -824,14 +727,10 @@ module Gcloud
       #
       def insert rows, skip_invalid: nil, ignore_unknown: nil
         rows = [rows] if rows.is_a? Hash
-        ensure_connection!
+        ensure_service!
         options = { skip_invalid: skip_invalid, ignore_unknown: ignore_unknown }
-        resp = connection.insert_tabledata dataset_id, table_id, rows, options
-        if resp.success?
-          InsertResponse.from_gapi rows, resp.data
-        else
-          fail ApiError.from_response(resp)
-        end
+        gapi = service.insert_tabledata dataset_id, table_id, rows, options
+        InsertResponse.from_gapi rows, gapi
       end
 
       ##
@@ -852,13 +751,9 @@ module Gcloud
       # @!group Lifecycle
       #
       def delete
-        ensure_connection!
-        resp = connection.delete_table dataset_id, table_id
-        if resp.success?
-          true
-        else
-          fail ApiError.from_response(resp)
-        end
+        ensure_service!
+        service.delete_table dataset_id, table_id
+        true
       end
 
       ##
@@ -867,13 +762,9 @@ module Gcloud
       # @!group Lifecycle
       #
       def reload!
-        ensure_connection!
-        resp = connection.get_table dataset_id, table_id
-        if resp.success?
-          @gapi = resp.data
-        else
-          fail ApiError.from_response(resp)
-        end
+        ensure_service!
+        gapi = service.get_table dataset_id, table_id
+        @gapi = gapi
       end
       alias_method :refresh!, :reload!
 
@@ -883,77 +774,47 @@ module Gcloud
         klass = class_for gapi
         klass.new.tap do |f|
           f.gapi = gapi
-          f.connection = conn
+          f.service = conn
         end
       end
 
       protected
 
       ##
-      # Raise an error unless an active connection is available.
-      def ensure_connection!
-        fail "Must have active connection" unless connection
+      # Raise an error unless an active service is available.
+      def ensure_service!
+        fail "Must have active connection" unless service
       end
 
-      def patch_gapi! options = {}
-        ensure_connection!
-        resp = connection.patch_table dataset_id, table_id, options
-        if resp.success?
-          @gapi = resp.data
-        else
-          fail ApiError.from_response(resp)
-        end
+      def patch_gapi! *attributes
+        return if attributes.empty?
+        ensure_service!
+        patch_args = Hash[attributes.map do |attr|
+          [attr, @gapi.send(attr)]
+        end]
+        patch_gapi = Google::Apis::BigqueryV2::Table.new patch_args
+        @gapi = service.patch_table dataset_id, table_id, patch_gapi
       end
 
       def self.class_for gapi
-        return View if gapi["type"] == "VIEW"
+        return View if gapi.type == "VIEW"
         self
       end
 
-      def load_storage file, options = {}
+      def load_storage url, options = {}
         # Convert to storage URL
-        file = file.to_gs_url if file.respond_to? :to_gs_url
+        url = url.to_gs_url if url.respond_to? :to_gs_url
 
-        resp = connection.load_table table_ref, file, options
-        if resp.success?
-          Job.from_gapi resp.data, connection
-        else
-          fail ApiError.from_response(resp)
-        end
+        gapi = service.load_table_gs_url dataset_id, table_id, url, options
+        Job.from_gapi gapi, service
       end
 
       def load_local file, options = {}
-        if resumable_upload? file
-          load_resumable file, options
-        else
-          load_multipart file, options
-        end
-      end
-
-      def load_resumable file, options = {}
-        chunk_size = Gcloud::Upload.verify_chunk_size options[:chunk_size],
-                                                      file.length
-        resp = connection.load_resumable table_ref, file, chunk_size, options
-        if resp.success?
-          Job.from_gapi resp.data, connection
-        else
-          fail ApiError.from_response(resp)
-        end
-      end
-
-      def load_multipart file, options = {}
-        resp = connection.load_multipart table_ref, file, options
-        if resp.success?
-          Job.from_gapi resp.data, connection
-        else
-          fail ApiError.from_response(resp)
-        end
-      end
+        # Convert to storage URL
+        file = file.to_gs_url if file.respond_to? :to_gs_url
 
-      ##
-      # @private Determines if a resumable upload should be used.
-      def resumable_upload? file
-        ::File.size?(file).to_i > Upload.resumable_threshold
+        gapi = service.load_table_file dataset_id, table_id, file, options
+        Job.from_gapi gapi, service
       end
 
       def storage_url? file
@@ -976,17 +837,13 @@ module Gcloud
       end
 
       def reload_gapi!
-        ensure_connection!
-        resp = connection.get_table dataset_id, table_id
-        if resp.success?
-          @gapi = resp.data
-        else
-          fail ApiError.from_response(resp)
-        end
+        ensure_service!
+        gapi = service.get_table dataset_id, table_id
+        @gapi = gapi
       end
 
       def data_complete?
-        !@gapi["creationTime"].nil?
+        @gapi.is_a? Google::Apis::BigqueryV2::Table
       end
 
       private
@@ -995,7 +852,69 @@ module Gcloud
         if table.respond_to? :table_ref
           table.table_ref
         else
-          Connection.table_ref_from_s table, table_ref
+          Service.table_ref_from_s table, table_ref
+        end
+      end
+
+      ##
+      # Yielded to a block to accumulate changes for a patch request.
+      class Updater < Table
+        ##
+        # A list of attributes that were updated.
+        attr_reader :updates
+
+        ##
+        # Create an Updater object.
+        def initialize gapi
+          @updates = []
+          @gapi = gapi
+          @schema = nil
+        end
+
+        def schema replace: false
+          # Same as Table#schema, but not frozen
+          # TODO: make sure to call ensure_full_data! on Dataset#update
+          @schema ||= Schema.from_gapi @gapi.schema
+          if block_given?
+            if replace
+              @schema = Schema.from_gapi \
+                Google::Apis::BigqueryV2::TableSchema.new(fields: [])
+            end
+            yield @schema
+            check_for_mutated_schema!
+          end
+          # Do not freeze on updater, allow modifications
+          @schema
+        end
+
+        ##
+        # Make sure any access changes are saved
+        def check_for_mutated_schema!
+          return if @schema.nil?
+          @schema.check_for_mutated_schema!
+          return unless @schema.changed?
+          @gapi.schema = @schema.to_gapi
+          patch_gapi! :schema
+        end
+
+        def to_gapi
+          check_for_mutated_schema!
+          @gapi
+        end
+
+        protected
+
+        ##
+        # Change to a NOOP
+        def ensure_full_data!
+          # Do nothing because we trust the gapi is full before we get here.
+        end
+
+        ##
+        # Queue up all the updates instead of making them.
+        def patch_gapi! attribute
+          @updates << attribute
+          @updates.uniq!
         end
       end
     end