gcloud 0.11.0 → 0.12.0

data/lib/gcloud/bigquery/schema.rb

@@ -0,0 +1,359 @@
+# Copyright 2015 Google Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+module Gcloud
+  module Bigquery
+    ##
+    # # Table Schema
+    #
+    # A builder for BigQuery table schemas, passed to block arguments to
+    # {Dataset#create_table} and {Table#schema}. Supports nested and
+    # repeated fields via a nested block.
+    #
+    # @see https://cloud.google.com/bigquery/preparing-data-for-bigquery
+    #   Preparing Data for BigQuery
+    #
+    # @example
+    #   require "gcloud"
+    #
+    #   gcloud = Gcloud.new
+    #   bigquery = gcloud.bigquery
+    #   dataset = bigquery.dataset "my_dataset"
+    #   table = dataset.create_table "my_table"
+    #
+    #   table.schema do |schema|
+    #     schema.string "first_name", mode: :required
+    #     schema.record "cities_lived", mode: :repeated do |cities_lived|
+    #       cities_lived.string "place", mode: :required
+    #       cities_lived.integer "number_of_years", mode: :required
+    #     end
+    #   end
+    #
+    class Schema
+      def initialize
+        @nested = nil
+      end
+
+      def fields
+        @fields ||= @gapi.fields.map { |f| Field.from_gapi f }
+      end
+
+      def fields= new_fields
+        @gapi.fields = Array(new_fields).map(&:to_gapi)
+        @fields = @gapi.fields.map { |f| Field.from_gapi f }
+      end
+
+      def empty?
+        fields.empty?
+      end
+
+      # @private
+      def changed?
+        return false if frozen?
+        check_for_mutated_schema!
+        @original_json != @gapi.to_json
+      end
+
+      # @private
+      def freeze
+        @gapi = @gapi.dup.freeze
+        @gapi.fields.freeze
+        @fields = @gapi.fields.map { |f| Field.from_gapi(f).freeze }
+        @fields.freeze
+        super
+      end
+
+      ##
+      # @private Make sure any changes are saved.
+      def check_for_mutated_schema!
+        return if frozen?
+        return if @gapi.frozen?
+        return if @fields.nil?
+        gapi_fields = Array(@fields).map(&:to_gapi)
+        @gapi.update! fields: gapi_fields
+      end
+
+      # @private
+      def self.from_gapi gapi
+        gapi ||= Google::Apis::BigqueryV2::TableSchema.new fields: []
+        gapi.fields ||= []
+        new.tap do |s|
+          s.instance_variable_set :@gapi, gapi
+          s.instance_variable_set :@original_json, gapi.to_json
+        end
+      end
+
+      # @private
+      def to_gapi
+        check_for_mutated_schema!
+        @gapi
+      end
+
+      # @private
+      def == other
+        return false unless other.is_a? Schema
+        to_gapi.to_h == other.to_gapi.to_h
+      end
+
+      ##
+      # Adds a string field to the schema.
+      #
+      # @param [String] name The field name. The name must contain only
+      #   letters (a-z, A-Z), numbers (0-9), or underscores (_), and must
+      #   start with a letter or underscore. The maximum length is 128
+      #   characters.
+      # @param [String] description A description of the field.
+      # @param [Symbol] mode The field's mode. The possible values are
+      #   `:nullable`, `:required`, and `:repeated`. The default value is
+      #   `:nullable`.
+      def string name, description: nil, mode: :nullable
+        add_field name, :string, nil, description: description, mode: mode
+      end
+
+      ##
+      # Adds an integer field to the schema.
+      #
+      # @param [String] name The field name. The name must contain only
+      #   letters (a-z, A-Z), numbers (0-9), or underscores (_), and must
+      #   start with a letter or underscore. The maximum length is 128
+      #   characters.
+      # @param [String] description A description of the field.
+      # @param [Symbol] mode The field's mode. The possible values are
+      #   `:nullable`, `:required`, and `:repeated`. The default value is
+      #   `:nullable`.
+      def integer name, description: nil, mode: :nullable
+        add_field name, :integer, nil, description: description, mode: mode
+      end
+
+      ##
+      # Adds a floating-point number field to the schema.
+      #
+      # @param [String] name The field name. The name must contain only
+      #   letters (a-z, A-Z), numbers (0-9), or underscores (_), and must
+      #   start with a letter or underscore. The maximum length is 128
+      #   characters.
+      # @param [String] description A description of the field.
+      # @param [Symbol] mode The field's mode. The possible values are
+      #   `:nullable`, `:required`, and `:repeated`. The default value is
+      #   `:nullable`.
+      def float name, description: nil, mode: :nullable
+        add_field name, :float, nil, description: description, mode: mode
+      end
+
+      ##
+      # Adds a boolean field to the schema.
+      #
+      # @param [String] name The field name. The name must contain only
+      #   letters (a-z, A-Z), numbers (0-9), or underscores (_), and must
+      #   start with a letter or underscore. The maximum length is 128
+      #   characters.
+      # @param [String] description A description of the field.
+      # @param [Symbol] mode The field's mode. The possible values are
+      #   `:nullable`, `:required`, and `:repeated`. The default value is
+      #   `:nullable`.
+      def boolean name, description: nil, mode: :nullable
+        add_field name, :boolean, nil, description: description, mode: mode
+      end
+
+      ##
+      # Adds a timestamp field to the schema.
+      #
+      # @param [String] name The field name. The name must contain only
+      #   letters (a-z, A-Z), numbers (0-9), or underscores (_), and must
+      #   start with a letter or underscore. The maximum length is 128
+      #   characters.
+      # @param [String] description A description of the field.
+      # @param [Symbol] mode The field's mode. The possible values are
+      #   `:nullable`, `:required`, and `:repeated`. The default value is
+      #   `:nullable`.
+      def timestamp name, description: nil, mode: :nullable
+        add_field name, :timestamp, nil, description: description, mode: mode
+      end
+
+      ##
+      # Adds a record field to the schema. A block must be passed describing
+      # the nested fields of the record. For more information about nested
+      # and repeated records, see [Preparing Data for BigQuery
+      # ](https://cloud.google.com/bigquery/preparing-data-for-bigquery).
+      #
+      # @param [String] name The field name. The name must contain only
+      #   letters (a-z, A-Z), numbers (0-9), or underscores (_), and must
+      #   start with a letter or underscore. The maximum length is 128
+      #   characters.
+      # @param [String] description A description of the field.
+      # @param [Symbol] mode The field's mode. The possible values are
+      #   `:nullable`, `:required`, and `:repeated`. The default value is
+      #   `:nullable`.
+      # @yield [nested_schema] a block for setting the nested schema
+      # @yieldparam [Table::Schema] nested_schema the object accepting the
+      #   nested schema
+      #
+      # @example
+      #   require "gcloud"
+      #
+      #   gcloud = Gcloud.new
+      #   bigquery = gcloud.bigquery
+      #   dataset = bigquery.dataset "my_dataset"
+      #   table = dataset.create_table "my_table"
+      #
+      #   table.schema do |schema|
+      #     schema.string "first_name", mode: :required
+      #     schema.record "cities_lived", mode: :repeated do |cities_lived|
+      #       cities_lived.string "place", mode: :required
+      #       cities_lived.integer "number_of_years", mode: :required
+      #     end
+      #   end
+      #
+      def record name, description: nil, mode: nil
+        fail ArgumentError, "nested RECORD type is not permitted" if @nested
+        fail ArgumentError, "a block is required" unless block_given?
+        empty_schema = Google::Apis::BigqueryV2::TableSchema.new fields: []
+        nested_schema = self.class.from_gapi(empty_schema).tap do |s|
+          s.instance_variable_set :@nested, true
+        end
+        yield nested_schema
+        add_field name, :record, nested_schema.fields,
+                  description: description, mode: mode
+      end
+
+      protected
+
+      def add_field name, type, nested_fields, description: nil,
+                    mode: :nullable
+        # Remove any existing field of this name
+        fields.reject! { |f| f.name == name }
+        fields << Field.new(name, type, description: description,
+                                        mode: mode, fields: nested_fields)
+      end
+
+      class Field
+        # @private
+        MODES = %w( NULLABLE REQUIRED REPEATED )
+
+        # @private
+        TYPES = %w( STRING INTEGER FLOAT BOOLEAN TIMESTAMP RECORD )
+
+        def initialize name, type, description: nil,
+                       mode: :nullable, fields: nil
+          @gapi = Google::Apis::BigqueryV2::TableFieldSchema.new
+          @gapi.update! name: name
+          @gapi.update! type: verify_type(type)
+          @gapi.update! description: description if description
+          @gapi.update! mode: verify_mode(mode) if mode
+          if fields
+            @fields = fields
+            check_for_changed_fields!
+          end
+          @original_json = @gapi.to_json
+        end
+
+        def name
+          @gapi.name
+        end
+
+        def name= new_name
+          @gapi.update! name: new_name
+        end
+
+        def type
+          @gapi.type
+        end
+
+        def type= new_type
+          @gapi.update! type: verify_type(new_type)
+        end
+
+        def description
+          @gapi.description
+        end
+
+        def description= new_description
+          @gapi.update! description: new_description
+        end
+
+        def mode
+          @gapi.mode
+        end
+
+        def mode= new_mode
+          @gapi.update! mode: verify_mode(new_mode)
+        end
+
+        def fields
+          @fields ||= Array(@gapi.fields).map { |f| Field.from_gapi f }
+        end
+
+        def fields= new_fields
+          @fields = new_fields
+        end
+
+        ##
+        # @private Make sure any fields are saved.
+        def check_for_changed_fields!
+          return if frozen?
+          fields.each(&:check_for_changed_fields!)
+          gapi_fields = Array(fields).map(&:to_gapi)
+          gapi_fields = nil if gapi_fields.empty?
+          @gapi.update! fields: gapi_fields
+        end
+
+        # @private
+        def changed?
+          @original_json == to_gapi.to_json
+        end
+
+        # @private
+        def self.from_gapi gapi
+          new("to-be-replaced", "STRING").tap do |f|
+            f.instance_variable_set :@gapi, gapi
+            f.instance_variable_set :@original_json, gapi.to_json
+          end
+        end
+
+        # @private
+        def to_gapi
+          # make sure any changes are saved.
+          check_for_changed_fields!
+          @gapi
+        end
+
+        # @private
+        def == other
+          return false unless other.is_a? Field
+          to_gapi.to_h == other.to_gapi.to_h
+        end
+
+        protected
+
+        def verify_type type
+          upcase_type = type.to_s.upcase
+          unless TYPES.include? upcase_type
+            fail ArgumentError,
+                 "Type '#{upcase_type}' not found in #{TYPES.inspect}"
+          end
+          upcase_type
+        end
+
+        def verify_mode mode
+          upcase_mode = mode.to_s.upcase
+          unless MODES.include? upcase_mode
+            fail ArgumentError "Unable to determine mode for '#{mode}'"
+          end
+          upcase_mode
+        end
+      end
+    end
+  end
+end

data/lib/gcloud/bigquery/service.rb

@@ -0,0 +1,506 @@
+# Copyright 2015 Google Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+
+require "gcloud/version"
+require "gcloud/errors"
+require "google/apis/bigquery_v2"
+require "pathname"
+require "digest/md5"
+require "mime/types"
+
+module Gcloud
+  module Bigquery
+    ##
+    # @private Represents the Bigquery service and API calls.
+    class Service
+      ##
+      # Alias to the Google Client API module
+      API = Google::Apis::BigqueryV2
+
+      # @private
+      attr_accessor :project
+
+      # @private
+      attr_accessor :credentials
+
+      ##
+      # Creates a new Service instance.
+      def initialize project, credentials, retries: nil, timeout: nil
+        @project = project
+        @credentials = credentials
+        @credentials = credentials
+        @service = API::BigqueryService.new
+        @service.client_options.application_name    = "gcloud-ruby"
+        @service.client_options.application_version = Gcloud::VERSION
+        @service.request_options.retries = retries || 3
+        @service.request_options.timeout_sec = timeout if timeout
+        @service.authorization = @credentials.client
+      end
+
+      def service
+        return mocked_service if mocked_service
+        @service
+      end
+      attr_accessor :mocked_service
+
+      ##
+      # Lists all datasets in the specified project to which you have
+      # been granted the READER dataset role.
+      def list_datasets options = {}
+        service.list_datasets \
+          @project, all: options[:all], max_results: options[:max],
+                    page_token: options[:token]
+      rescue Google::Apis::Error => e
+        raise Gcloud::Error.from_error(e)
+      end
+
+      ##
+      # Returns the dataset specified by datasetID.
+      def get_dataset dataset_id
+        service.get_dataset @project, dataset_id
+      rescue Google::Apis::Error => e
+        raise Gcloud::Error.from_error(e)
+      end
+
+      ##
+      # Creates a new empty dataset.
+      def insert_dataset new_dataset_gapi
+        service.insert_dataset @project, new_dataset_gapi
+      rescue Google::Apis::Error => e
+        raise Gcloud::Error.from_error(e)
+      end
+
+      ##
+      # Updates information in an existing dataset, only replacing
+      # fields that are provided in the submitted dataset resource.
+      def patch_dataset dataset_id, patched_dataset_gapi
+        service.patch_dataset @project, dataset_id, patched_dataset_gapi
+      rescue Google::Apis::Error => e
+        raise Gcloud::Error.from_error(e)
+      end
+
+      ##
+      # Deletes the dataset specified by the datasetId value.
+      # Before you can delete a dataset, you must delete all its tables,
+      # either manually or by specifying force: true in options.
+      # Immediately after deletion, you can create another dataset with
+      # the same name.
+      def delete_dataset dataset_id, force = nil
+        service.delete_dataset @project, dataset_id, delete_contents: force
+      rescue Google::Apis::Error => e
+        raise Gcloud::Error.from_error(e)
+      end
+
+      ##
+      # Lists all tables in the specified dataset.
+      # Requires the READER dataset role.
+      def list_tables dataset_id, options = {}
+        service.list_tables @project, dataset_id, max_results: options[:max],
+                                                  page_token: options[:token]
+      rescue Google::Apis::Error => e
+        raise Gcloud::Error.from_error(e)
+      end
+
+      def get_project_table project_id, dataset_id, table_id
+        service.get_table project_id, dataset_id, table_id
+      rescue Google::Apis::Error => e
+        raise Gcloud::Error.from_error(e)
+      end
+
+      ##
+      # Gets the specified table resource by table ID.
+      # This method does not return the data in the table,
+      # it only returns the table resource,
+      # which describes the structure of this table.
+      def get_table dataset_id, table_id
+        get_project_table @project, dataset_id, table_id
+      rescue Google::Apis::Error => e
+        raise Gcloud::Error.from_error(e)
+      end
+
+      ##
+      # Creates a new, empty table in the dataset.
+      def insert_table dataset_id, new_table_gapi
+        service.insert_table @project, dataset_id, new_table_gapi
+      rescue Google::Apis::Error => e
+        raise Gcloud::Error.from_error(e)
+      end
+
+      ##
+      # Updates information in an existing table, replacing fields that
+      # are provided in the submitted table resource.
+      def patch_table dataset_id, table_id, patched_table_gapi
+        service.patch_table @project, dataset_id, table_id, patched_table_gapi
+      rescue Google::Apis::Error => e
+        raise Gcloud::Error.from_error(e)
+      end
+
+      ##
+      # Deletes the table specified by tableId from the dataset.
+      # If the table contains data, all the data will be deleted.
+      def delete_table dataset_id, table_id
+        service.delete_table @project, dataset_id, table_id
+      rescue Google::Apis::Error => e
+        raise Gcloud::Error.from_error(e)
+      end
+
+      ##
+      # Retrieves data from the table.
+      def list_tabledata dataset_id, table_id, options = {}
+        service.list_table_data @project, dataset_id, table_id,
+                                max_results: options.delete(:max),
+                                page_token: options.delete(:token),
+                                start_index: options.delete(:start)
+      rescue Google::Apis::Error => e
+        raise Gcloud::Error.from_error(e)
+      end
+
+      def insert_tabledata dataset_id, table_id, rows, options = {}
+        insert_rows = Array(rows).map do |row|
+          Google::Apis::BigqueryV2::InsertAllTableDataRequest::Row.new(
+            insert_id: Digest::MD5.base64digest(row.inspect),
+            # Hash[row.map{|(k,v)| [k.to_s,v]}] for Hash<String,Object>
+            json: row
+          )
+        end
+        insert_req = Google::Apis::BigqueryV2::InsertAllTableDataRequest.new(
+          rows: insert_rows,
+          ignore_unknown_values: options[:ignore_unknown],
+          skip_invalid_rows: options[:skip_invalid]
+        )
+
+        service.insert_all_table_data @project, dataset_id, table_id, insert_req
+      rescue Google::Apis::Error => e
+        raise Gcloud::Error.from_error(e)
+      end
+
+      ##
+      # Lists all jobs in the specified project to which you have
+      # been granted the READER job role.
+      def list_jobs options = {}
+        service.list_jobs \
+          @project, all_users: options[:all], max_results: options[:max],
+                    page_token: options[:token], projection: "full",
+                    state_filter: options[:filter]
+      rescue Google::Apis::Error => e
+        raise Gcloud::Error.from_error(e)
+      end
+
+      ##
+      # Returns the job specified by jobID.
+      def get_job job_id
+        service.get_job @project, job_id
+      rescue Google::Apis::Error => e
+        raise Gcloud::Error.from_error(e)
+      end
+
+      def insert_job config
+        job_object = API::Job.new(
+          configuration: config
+        )
+        service.insert_job @project, job_object
+      rescue Google::Apis::Error => e
+        raise Gcloud::Error.from_error(e)
+      end
+
+      def query_job query, options = {}
+        config = query_table_config(query, options)
+        service.insert_job @project, config
+      rescue Google::Apis::Error => e
+        raise Gcloud::Error.from_error(e)
+      end
+
+      def query query, options = {}
+        service.query_job @project, query_config(query, options)
+      rescue Google::Apis::Error => e
+        raise Gcloud::Error.from_error(e)
+      end
+
+      ##
+      # Returns the query data for the job
+      def job_query_results job_id, options = {}
+        service.get_job_query_results @project,
+                                      job_id,
+                                      max_results: options.delete(:max),
+                                      page_token: options.delete(:token),
+                                      start_index: options.delete(:start),
+                                      timeout_ms: options.delete(:timeout)
+      rescue Google::Apis::Error => e
+        raise Gcloud::Error.from_error(e)
+      end
+
+      def copy_table source, target, options = {}
+        service.insert_job @project, copy_table_config(source, target, options)
+      rescue Google::Apis::Error => e
+        raise Gcloud::Error.from_error(e)
+      end
+
+      def extract_table table, storage_files, options = {}
+        service.insert_job \
+          @project, extract_table_config(table, storage_files, options)
+      rescue Google::Apis::Error => e
+        raise Gcloud::Error.from_error(e)
+      end
+
+      def load_table_gs_url dataset_id, table_id, url, options = {}
+        service.insert_job \
+          @project, load_table_url_config(dataset_id, table_id, url, options)
+      rescue Google::Apis::Error => e
+        raise Gcloud::Error.from_error(e)
+      end
+
+      def load_table_file dataset_id, table_id, file, options = {}
+        service.insert_job \
+          @project, load_table_file_config(dataset_id, table_id, file, options),
+          upload_source: file, content_type: mime_type_for(file)
+      rescue Google::Apis::Error => e
+        raise Gcloud::Error.from_error(e)
+      end
+
+      ##
+      # Extracts at least `tbl` group, and possibly `dts` and `prj` groups,
+      # from strings in the formats: "my_table", "my_dataset.my_table", or
+      # "my-project:my_dataset.my_table". Then merges project_id and
+      # dataset_id from the default table if they are missing.
+      def self.table_ref_from_s str, default_table_ref
+        str = str.to_s
+        m = /\A(((?<prj>\S*):)?(?<dts>\S*)\.)?(?<tbl>\S*)\z/.match str
+        unless m
+          fail ArgumentError, "unable to identify table from #{str.inspect}"
+        end
+        str_table_ref_hash = {
+          project_id: m["prj"],
+          dataset_id: m["dts"],
+          table_id:   m["tbl"]
+        }.delete_if { |_, v| v.nil? }
+        new_table_ref_hash = default_table_ref.to_h.merge str_table_ref_hash
+        Google::Apis::BigqueryV2::TableReference.new new_table_ref_hash
+      end
+
+      def inspect
+        "#{self.class}(#{@project})"
+      end
+
+      protected
+
+      def table_ref_from tbl
+        return nil if tbl.nil?
+        API::TableReference.new(
+          project_id: tbl.project_id,
+          dataset_id: tbl.dataset_id,
+          table_id: tbl.table_id
+        )
+      end
+
+      def dataset_ref_from dts, pjt = nil
+        return nil if dts.nil?
+        if dts.respond_to? :dataset_id
+          API::DatasetReference.new(
+            project_id: (pjt || dts.project_id || @project),
+            dataset_id: dts.dataset_id
+          )
+        else
+          API::DatasetReference.new(
+            project_id: (pjt || @project),
+            dataset_id: dts
+          )
+        end
+      end
+
+      def load_table_file_opts dataset_id, table_id, file, options = {}
+        path = Pathname(file).to_path
+        {
+          destination_table: Google::Apis::BigqueryV2::TableReference.new(
+            project_id: @project, dataset_id: dataset_id, table_id: table_id),
+          create_disposition: create_disposition(options[:create]),
+          write_disposition: write_disposition(options[:write]),
+          source_format: source_format(path, options[:format]),
+          projection_fields: projection_fields(options[:projection_fields]),
+          allow_jagged_rows: options[:jagged_rows],
+          allow_quoted_newlines: options[:quoted_newlines],
+          encoding: options[:encoding], field_delimiter: options[:delimiter],
+          ignore_unknown_values: options[:ignore_unknown],
+          max_bad_records: options[:max_bad_records], quote: options[:quote],
+          schema: options[:schema], skip_leading_rows: options[:skip_leading]
+        }.delete_if { |_, v| v.nil? }
+      end
+
+      def load_table_file_config dataset_id, table_id, file, options = {}
+        load_opts = load_table_file_opts dataset_id, table_id, file, options
+        API::Job.new(
+          configuration: API::JobConfiguration.new(
+            load: API::JobConfigurationLoad.new(load_opts),
+            dry_run: options[:dryrun]
+          )
+        )
+      end
+
+      def load_table_url_opts dataset_id, table_id, url, options = {}
+        {
+          destination_table: Google::Apis::BigqueryV2::TableReference.new(
+            project_id: @project, dataset_id: dataset_id, table_id: table_id),
+          source_uris: Array(url),
+          create_disposition: create_disposition(options[:create]),
+          write_disposition: write_disposition(options[:write]),
+          source_format: source_format(url, options[:format]),
+          projection_fields: projection_fields(options[:projection_fields]),
+          allow_jagged_rows: options[:jagged_rows],
+          allow_quoted_newlines: options[:quoted_newlines],
+          encoding: options[:encoding], field_delimiter: options[:delimiter],
+          ignore_unknown_values: options[:ignore_unknown],
+          max_bad_records: options[:max_bad_records], quote: options[:quote],
+          schema: options[:schema], skip_leading_rows: options[:skip_leading]
+        }.delete_if { |_, v| v.nil? }
+      end
+
+      def load_table_url_config dataset_id, table_id, url, options = {}
+        load_opts = load_table_url_opts dataset_id, table_id, url, options
+        API::Job.new(
+          configuration: API::JobConfiguration.new(
+            load: API::JobConfigurationLoad.new(load_opts),
+            dry_run: options[:dryrun]
+          )
+        )
+      end
+
+      ##
+      # Job description for query job
+      def query_table_config query, options
+        dest_table = table_ref_from options[:table]
+        default_dataset = dataset_ref_from options[:dataset]
+        API::Job.new(
+          configuration: API::JobConfiguration.new(
+            query: API::JobConfigurationQuery.new(
+              query: query,
+              # tableDefinitions: { ... },
+              priority: priority_value(options[:priority]),
+              use_query_cache: options[:cache],
+              destination_table: dest_table,
+              create_disposition: create_disposition(options[:create]),
+              write_disposition: write_disposition(options[:write]),
+              allow_large_results: options[:large_results],
+              flatten_results: options[:flatten],
+              default_dataset: default_dataset
+            )
+          )
+        )
+      end
+
+      def query_config query, options = {}
+        dataset_config = dataset_ref_from options[:dataset], options[:project]
+
+        API::QueryRequest.new(
+          query: query,
+          max_results: options[:max],
+          default_dataset: dataset_config,
+          timeout_ms: options[:timeout],
+          dry_run: options[:dryrun],
+          use_query_cache: options[:cache]
+        )
+      end
+
+      ##
+      # Job description for copy job
+      def copy_table_config source, target, options = {}
+        API::Job.new(
+          configuration: API::JobConfiguration.new(
+            copy: API::JobConfigurationTableCopy.new(
+              source_table: source,
+              destination_table: target,
+              create_disposition: create_disposition(options[:create]),
+              write_disposition: write_disposition(options[:write])
+            ),
+            dry_run: options[:dryrun]
+          )
+        )
+      end
+
+      def extract_table_config table, storage_files, options = {}
+        storage_urls = Array(storage_files).map do |url|
+          url.respond_to?(:to_gs_url) ? url.to_gs_url : url
+        end
+        dest_format = source_format storage_urls.first, options[:format]
+        API::Job.new(
+          configuration: API::JobConfiguration.new(
+            extract: API::JobConfigurationExtract.new(
+              destination_uris: Array(storage_urls),
+              source_table: table,
+              destination_format: dest_format,
+              compression: options[:compression],
+              field_delimiter: options[:delimiter],
+              print_header: options[:header]
+            ),
+            dry_run: options[:dryrun]
+          )
+        )
+      end
+
+      def create_disposition str
+        { "create_if_needed" => "CREATE_IF_NEEDED",
+          "createifneeded" => "CREATE_IF_NEEDED",
+          "if_needed" => "CREATE_IF_NEEDED",
+          "needed" => "CREATE_IF_NEEDED",
+          "create_never" => "CREATE_NEVER",
+          "createnever" => "CREATE_NEVER",
+          "never" => "CREATE_NEVER" }[str.to_s.downcase]
+      end
+
+      def write_disposition str
+        { "write_truncate" => "WRITE_TRUNCATE",
+          "writetruncate" => "WRITE_TRUNCATE",
+          "truncate" => "WRITE_TRUNCATE",
+          "write_append" => "WRITE_APPEND",
+          "writeappend" => "WRITE_APPEND",
+          "append" => "WRITE_APPEND",
+          "write_empty" => "WRITE_EMPTY",
+          "writeempty" => "WRITE_EMPTY",
+          "empty" => "WRITE_EMPTY" }[str.to_s.downcase]
+      end
+
+      def priority_value str
+        { "batch" => "BATCH",
+          "interactive" => "INTERACTIVE" }[str.to_s.downcase]
+      end
+
+      def source_format path, format
+        val = { "csv" => "CSV",
+                "json" => "NEWLINE_DELIMITED_JSON",
+                "newline_delimited_json" => "NEWLINE_DELIMITED_JSON",
+                "avro" => "AVRO",
+                "datastore" => "DATASTORE_BACKUP",
+                "datastore_backup" => "DATASTORE_BACKUP" }[format.to_s.downcase]
+        return val unless val.nil?
+        return nil if path.nil?
+        return "CSV" if path.end_with? ".csv"
+        return "NEWLINE_DELIMITED_JSON" if path.end_with? ".json"
+        return "AVRO" if path.end_with? ".avro"
+        return "DATASTORE_BACKUP" if path.end_with? ".backup_info"
+        nil
+      end
+
+      def projection_fields array_or_str
+        Array(array_or_str) unless array_or_str.nil?
+      end
+
+      def mime_type_for file
+        mime_type = MIME::Types.of(Pathname(file).to_path).first.to_s
+        return nil if mime_type.empty?
+        mime_type
+      rescue
+        nil
+      end
+    end
+  end
+end