gcloud 0.2.0 → 0.3.0

data/lib/gcloud/bigquery/job/list.rb

@@ -0,0 +1,55 @@
+#--
+# Copyright 2015 Google Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+module Gcloud
+  module Bigquery
+    class Job
+      ##
+      # Job::List is a special case Array with additional values.
+      class List < DelegateClass(::Array)
+        ##
+        # If not empty, indicates that there are more records that match
+        # the request and this value should be passed to continue.
+        attr_accessor :token
+
+        # A hash of this page of results.
+        attr_accessor :etag
+
+        # Total number of jobs in this collection.
+        attr_accessor :total
+
+        ##
+        # Create a new Job::List with an array of jobs.
+        def initialize arr = []
+          super arr
+        end
+
+        ##
+        # New Job::List from a response object.
+        def self.from_resp resp, conn #:nodoc:
+          jobs = List.new(Array(resp.data["jobs"]).map do |gapi_object|
+            Job.from_gapi gapi_object, conn
+          end)
+          jobs.instance_eval do
+            @token = resp.data["nextPageToken"]
+            @etag = resp.data["etag"]
+            @total = resp.data["totalItems"]
+          end
+          jobs
+        end
+      end
+    end
+  end
+end

data/lib/gcloud/bigquery/load_job.rb

@@ -0,0 +1,199 @@
+#--
+# Copyright 2015 Google Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+module Gcloud
+  module Bigquery
+    ##
+    # = LoadJob
+    #
+    # A Job subclass representing a load operation that may be performed
+    # on a Table. A LoadJob instance is created when you call Table#load.
+    #
+    # See {Loading Data Into
+    # BigQuery}[https://cloud.google.com/bigquery/loading-data-into-bigquery]
+    # and the {Jobs API
+    # reference}[https://cloud.google.com/bigquery/docs/reference/v2/jobs]
+    # for details.
+    #
+    class LoadJob < Job
+      ##
+      # The URI or URIs representing the Google Cloud Storage files from which
+      # the operation loads data.
+      def sources
+        Array config["load"]["sourceUris"]
+      end
+
+      ##
+      # The table into which the operation loads data. This is the table on
+      # which Table#load was invoked. Returns a Table instance.
+      def destination
+        table = config["load"]["destinationTable"]
+        return nil unless table
+        retrieve_table table["projectId"],
+                       table["datasetId"],
+                       table["tableId"]
+      end
+
+      ##
+      # The delimiter used between fields in the source data. The default is a
+      # comma (,).
+      def delimiter
+        val = config["load"]["fieldDelimiter"]
+        val = "," if val.nil?
+        val
+      end
+
+      ##
+      # The number of header rows at the top of a CSV file to skip. The default
+      # value is +0+.
+      def skip_leading_rows
+        val = config["load"]["skipLeadingRows"]
+        val = 0 if val.nil?
+        val
+      end
+
+      ##
+      # Checks if the character encoding of the data is UTF-8. This is the
+      # default.
+      def utf8?
+        val = config["load"]["encoding"]
+        return true if val.nil?
+        val == "UTF-8"
+      end
+
+      ##
+      # Checks if the character encoding of the data is ISO-8859-1.
+      def iso8859_1?
+        val = config["load"]["encoding"]
+        val == "ISO-8859-1"
+      end
+
+      ##
+      # The value that is used to quote data sections in a CSV file.
+      # The default value is a double-quote (+"+). If your data does not contain
+      # quoted sections, the value should be an empty string. If your data
+      # contains quoted newline characters, #quoted_newlines? should return
+      # +true+.
+      def quote
+        val = config["load"]["quote"]
+        val = "\"" if val.nil?
+        val
+      end
+
+      ##
+      # The maximum number of bad records that the load operation can ignore. If
+      # the number of bad records exceeds this value, an error is
+      # returned. The default value is +0+, which requires that all records be
+      # valid.
+      def max_bad_records
+        val = config["load"]["maxBadRecords"]
+        val = 0 if val.nil?
+        val
+      end
+
+      ##
+      # Checks if quoted data sections may contain newline characters in a CSV
+      # file. The default is +false+.
+      def quoted_newlines?
+        val = config["load"]["allowQuotedNewlines"]
+        val = true if val.nil?
+        val
+      end
+
+      ##
+      # Checks if the format of the source data is
+      # {newline-delimited JSON}[http://jsonlines.org/]. The default is +false+.
+      def json?
+        val = config["load"]["sourceFormat"]
+        val == "NEWLINE_DELIMITED_JSON"
+      end
+
+      ##
+      # Checks if the format of the source data is CSV. The default is +true+.
+      def csv?
+        val = config["load"]["sourceFormat"]
+        return true if val.nil?
+        val == "CSV"
+      end
+
+      ##
+      # Checks if the source data is a Google Cloud Datastore backup.
+      def backup?
+        val = config["load"]["sourceFormat"]
+        val == "DATASTORE_BACKUP"
+      end
+
+      ##
+      # Checks if the load operation accepts rows that are missing trailing
+      # optional columns. The missing values are treated as nulls. If +false+,
+      # records with missing trailing columns are treated as bad records, and
+      # if there are too many bad records, an error is returned. The default
+      # value is +false+. Only applicable to CSV, ignored for other formats.
+      def allow_jagged_rows?
+        val = config["load"]["allowJaggedRows"]
+        val = false if val.nil?
+        val
+      end
+
+      ##
+      # Checks if the load operation allows extra values that are not
+      # represented in the table schema. If +true+, the extra values are
+      # ignored. If +false+, records with extra columns are treated as bad
+      # records, and if there are too many bad records, an invalid error is
+      # returned. The default is +false+.
+      def ignore_unknown_values?
+        val = config["load"]["ignoreUnknownValues"]
+        val = false if val.nil?
+        val
+      end
+
+      ##
+      # The schema for the data. Returns a hash. Can be empty if the table
+      # has already has the correct schema (see Table#schema= and Table#schema),
+      # or if the schema can be inferred from the loaded data.
+      def schema
+        val = config["load"]["schema"]
+        val = {} if val.nil?
+        val
+      end
+
+      ##
+      # The number of source files.
+      def input_files
+        stats["load"]["inputFiles"]
+      end
+
+      ##
+      # The number of bytes of source data.
+      def input_file_bytes
+        stats["load"]["inputFileBytes"]
+      end
+
+      ##
+      # The number of rows that have been loaded into the table. While an
+      # import job is in the running state, this value may change.
+      def output_rows
+        stats["load"]["outputRows"]
+      end
+
+      ##
+      # The number of bytes that have been loaded into the table. While an
+      # import job is in the running state, this value may change.
+      def output_bytes
+        stats["load"]["outputBytes"]
+      end
+    end
+  end
+end

data/lib/gcloud/bigquery/project.rb

@@ -0,0 +1,512 @@
+#--
+# Copyright 2015 Google Inc. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+require "gcloud/bigquery/connection"
+require "gcloud/bigquery/credentials"
+require "gcloud/bigquery/errors"
+require "gcloud/bigquery/dataset"
+require "gcloud/bigquery/job"
+require "gcloud/bigquery/query_data"
+
+module Gcloud
+  module Bigquery
+    ##
+    # = Project
+    #
+    # Projects are top-level containers in Google Cloud Platform. They store
+    # information about billing and authorized users, and they contain BigQuery
+    # data. Each project has a friendly name and a unique ID.
+    #
+    # Gcloud::Bigquery::Project is the main object for interacting with
+    # Google BigQuery. Gcloud::Bigquery::Dataset objects are created,
+    # accessed, and deleted by Gcloud::Bigquery::Project.
+    #
+    #   require "gcloud"
+    #
+    #   gcloud = Gcloud.new
+    #   bigquery = gcloud.bigquery
+    #   dataset = bigquery.dataset "my_dataset"
+    #   table = dataset.table "my_table"
+    #
+    # See Gcloud#bigquery
+    class Project
+      ##
+      # The Connection object.
+      attr_accessor :connection #:nodoc:
+
+      ##
+      # Creates a new Connection instance.
+      #
+      # See Gcloud.bigquery
+      def initialize project, credentials
+        project = project.to_s # Always cast to a string
+        fail ArgumentError, "project is missing" if project.empty?
+        @connection = Connection.new project, credentials
+      end
+
+      ##
+      # The BigQuery project connected to.
+      #
+      # === Example
+      #
+      #   require "gcloud"
+      #
+      #   gcloud = Gcloud.new "my-todo-project", "/path/to/keyfile.json"
+      #   bigquery = gcloud.bigquery
+      #
+      #   bigquery.project #=> "my-todo-project"
+      #
+      def project
+        connection.project
+      end
+
+      ##
+      # Default project.
+      def self.default_project #:nodoc:
+        ENV["BIGQUERY_PROJECT"] ||
+          ENV["GCLOUD_PROJECT"] ||
+          ENV["GOOGLE_CLOUD_PROJECT"]
+      end
+
+      ##
+      # Queries data using the {asynchronous
+      # method}[https://cloud.google.com/bigquery/querying-data].
+      #
+      # === Parameters
+      #
+      # +query+::
+      #   A query string, following the BigQuery {query
+      #   syntax}[https://cloud.google.com/bigquery/query-reference], of the
+      #   query to execute. Example: "SELECT count(f1) FROM
+      #   [myProjectId:myDatasetId.myTableId]". (+String+)
+      # <code>options[:priority]</code>::
+      #   Specifies a priority for the query. Possible values include
+      #   +INTERACTIVE+ and +BATCH+. The default value is +INTERACTIVE+.
+      #   (+String+)
+      # <code>options[:cache]</code>::
+      #   Whether to look for the result in the query cache. The query cache is
+      #   a best-effort cache that will be flushed whenever tables in the query
+      #   are modified. The default value is +true+. (+Boolean+)
+      # <code>options[:table]</code>::
+      #   The destination table where the query results should be stored. If not
+      #   present, a new table will be created to store the results. (+Table+)
+      # <code>options[:create]</code>::
+      #   Specifies whether the job is allowed to create new tables. (+String+)
+      #
+      #   The following values are supported:
+      #   * +needed+ - Create the table if it does not exist.
+      #   * +never+ - The table must already exist. A 'notFound' error is
+      #     raised if the table does not exist.
+      # <code>options[:write]</code>::
+      #   Specifies the action that occurs if the destination table already
+      #   exists. (+String+)
+      #
+      #   The following values are supported:
+      #   * +truncate+ - BigQuery overwrites the table data.
+      #   * +append+ - BigQuery appends the data to the table.
+      #   * +empty+ - A 'duplicate' error is returned in the job result if the
+      #     table exists and contains data.
+      # <code>options[:large_results]</code>::
+      #   If +true+, allows the query to produce arbitrarily large result tables
+      #   at a slight cost in performance. Requires <code>options[:table]</code>
+      #   to be set. (+Boolean+)
+      # <code>options[:flatten]</code>::
+      #   Flattens all nested and repeated fields in the query results. The
+      #   default value is +true+. <code>options[:large_results]</code> must be
+      #   +true+ if this is set to +false+. (+Boolean+)
+      # <code>options[:dataset]</code>::
+      #   Specifies the default dataset to use for unqualified table names in
+      #   the query. (+Dataset+ or +String+)
+      #
+      # === Returns
+      #
+      # Gcloud::Bigquery::QueryJob
+      #
+      # === Example
+      #
+      #   require "gcloud"
+      #
+      #   gcloud = Gcloud.new
+      #   bigquery = gcloud.bigquery
+      #
+      #   job = bigquery.query_job "SELECT name FROM [my_proj:my_data.my_table]"
+      #
+      #   loop do
+      #     break if job.done?
+      #     sleep 1
+      #     job.refresh!
+      #   end
+      #   if !job.failed?
+      #     job.query_results.each do |row|
+      #       puts row["name"]
+      #     end
+      #   end
+      #
+      def query_job query, options = {}
+        ensure_connection!
+        resp = connection.query_job query, options
+        if resp.success?
+          Job.from_gapi resp.data, connection
+        else
+          fail ApiError.from_response(resp)
+        end
+      end
+
+      ##
+      # Queries data using the {synchronous
+      # method}[https://cloud.google.com/bigquery/querying-data].
+      #
+      # === Parameters
+      #
+      # +query+::
+      #   A query string, following the BigQuery {query
+      #   syntax}[https://cloud.google.com/bigquery/query-reference], of the
+      #   query to execute. Example: "SELECT count(f1) FROM
+      #   [myProjectId:myDatasetId.myTableId]". (+String+)
+      # <code>options[:max]</code>::
+      #   The maximum number of rows of data to return per page of results.
+      #   Setting this flag to a small value such as 1000 and then paging
+      #   through results might improve reliability when the query result set is
+      #   large. In addition to this limit, responses are also limited to 10 MB.
+      #   By default, there is no maximum row count, and only the byte limit
+      #   applies. (+Integer+)
+      # <code>options[:timeout]</code>::
+      #   How long to wait for the query to complete, in milliseconds, before
+      #   the request times out and returns. Note that this is only a timeout
+      #   for the request, not the query. If the query takes longer to run than
+      #   the timeout value, the call returns without any results and with
+      #   QueryData#complete? set to false. The default value is 10000
+      #   milliseconds (10 seconds). (+Integer+)
+      # <code>options[:dryrun]</code>::
+      #   If set to +true+, BigQuery doesn't run the job. Instead, if the query
+      #   is valid, BigQuery returns statistics about the job such as how many
+      #   bytes would be processed. If the query is invalid, an error returns.
+      #   The default value is +false+. (+Boolean+)
+      # <code>options[:cache]</code>::
+      #   Whether to look for the result in the query cache. The query cache is
+      #   a best-effort cache that will be flushed whenever tables in the query
+      #   are modified. The default value is true. For more information, see
+      #   {query caching}[https://developers.google.com/bigquery/querying-data].
+      #   (+Boolean+)
+      # <code>options[:dataset]</code>::
+      #   Specifies the default datasetId and projectId to assume for any
+      #   unqualified table names in the query. If not set, all table names in
+      #   the query string must be qualified in the format 'datasetId.tableId'.
+      #   (+String+)
+      # <code>options[:project]</code>::
+      #   Specifies the default projectId to assume for any unqualified table
+      #   names in the query. Only used if +dataset+ option is set. (+String+)
+      #
+      # === Returns
+      #
+      # Gcloud::Bigquery::QueryData
+      #
+      # === Example
+      #
+      #   require "gcloud"
+      #
+      #   gcloud = Gcloud.new
+      #   bigquery = gcloud.bigquery
+      #
+      #   data = bigquery.query "SELECT name FROM [my_proj:my_data.my_table]"
+      #   data.each do |row|
+      #     puts row["name"]
+      #   end
+      #
+      def query query, options = {}
+        ensure_connection!
+        resp = connection.query query, options
+        if resp.success?
+          QueryData.from_gapi resp.data, connection
+        else
+          fail ApiError.from_response(resp)
+        end
+      end
+
+      ##
+      # Retrieves an existing dataset by ID.
+      #
+      # === Parameters
+      #
+      # +dataset_id+::
+      #   The ID of a dataset. (+String+)
+      #
+      # === Returns
+      #
+      # Gcloud::Bigquery::Dataset or nil if dataset does not exist
+      #
+      # === Example
+      #
+      #   require "gcloud"
+      #
+      #   gcloud = Gcloud.new
+      #   bigquery = gcloud.bigquery
+      #
+      #   dataset = bigquery.dataset "my_dataset"
+      #   puts dataset.name
+      #
+      def dataset dataset_id
+        ensure_connection!
+        resp = connection.get_dataset dataset_id
+        if resp.success?
+          Dataset.from_gapi resp.data, connection
+        else
+          return nil if resp.status == 404
+          fail ApiError.from_response(resp)
+        end
+      end
+
+      ##
+      # Creates a new dataset.
+      #
+      # === Parameters
+      #
+      # +dataset_id+::
+      #   A unique ID for this dataset, without the project name.
+      #   The ID must contain only letters (a-z, A-Z), numbers (0-9), or
+      #   underscores (_). The maximum length is 1,024 characters. (+String+)
+      # +options+::
+      #   An optional Hash for controlling additional behavior. (+Hash+)
+      # <code>options[:name]</code>::
+      #   A descriptive name for the dataset. (+String+)
+      # <code>options[:description]</code>::
+      #   A user-friendly description of the dataset. (+String+)
+      # <code>options[:expiration]</code>::
+      #   The default lifetime of all tables in the dataset, in milliseconds.
+      #   The minimum value is 3600000 milliseconds (one hour). (+Integer+)
+      #
+      # === Returns
+      #
+      # Gcloud::Bigquery::Dataset
+      #
+      # === Examples
+      #
+      #   require "gcloud"
+      #
+      #   gcloud = Gcloud.new
+      #   bigquery = gcloud.bigquery
+      #
+      #   dataset = bigquery.create_dataset "my_dataset"
+      #
+      # A name and description can be provided:
+      #
+      #   require "gcloud"
+      #
+      #   gcloud = Gcloud.new
+      #   bigquery = gcloud.bigquery
+      #
+      #   dataset = bigquery.create_dataset "my_dataset",
+      #                                     name: "My Dataset",
+      #                                     description: "This is my Dataset"
+      #
+      def create_dataset dataset_id, options = {}
+        ensure_connection!
+        resp = connection.insert_dataset dataset_id, options
+        if resp.success?
+          Dataset.from_gapi resp.data, connection
+        else
+          fail ApiError.from_response(resp)
+        end
+      end
+
+      ##
+      # Retrieves the list of datasets belonging to the project.
+      #
+      # === Parameters
+      #
+      # +options+::
+      #   An optional Hash for controlling additional behavior. (+Hash+)
+      # <code>options[:all]</code>::
+      #   Whether to list all datasets, including hidden ones. The default is
+      #   +false+. (+Boolean+)
+      # <code>options[:token]</code>::
+      #   A previously-returned page token representing part of the larger set
+      #   of results to view. (+String+)
+      # <code>options[:max]</code>::
+      #   Maximum number of datasets to return. (+Integer+)
+      #
+      # === Returns
+      #
+      # Array of Gcloud::Bigquery::Dataset (Gcloud::Bigquery::Dataset::List)
+      #
+      # === Examples
+      #
+      #   require "gcloud"
+      #
+      #   gcloud = Gcloud.new
+      #   bigquery = gcloud.bigquery
+      #
+      #   datasets = bigquery.datasets
+      #   datasets.each do |dataset|
+      #     puts dataset.name
+      #   end
+      #
+      # You can also retrieve all datasets, including hidden ones, by providing
+      # the +:all+ option:
+      #
+      #   require "gcloud"
+      #
+      #   gcloud = Gcloud.new
+      #   bigquery = gcloud.bigquery
+      #
+      #   all_datasets = bigquery.datasets, all: true
+      #
+      # If you have a significant number of datasets, you may need to paginate
+      # through them: (See Dataset::List#token)
+      #
+      #   require "gcloud"
+      #
+      #   gcloud = Gcloud.new
+      #   bigquery = gcloud.bigquery
+      #
+      #   all_datasets = []
+      #   tmp_datasets = bigquery.datasets
+      #   while tmp_datasets.any? do
+      #     tmp_datasets.each do |dataset|
+      #       all_datasets << dataset
+      #     end
+      #     # break loop if no more datasets available
+      #     break if tmp_datasets.token.nil?
+      #     # get the next group of datasets
+      #     tmp_datasets = bigquery.datasets token: tmp_datasets.token
+      #   end
+      #
+      def datasets options = {}
+        ensure_connection!
+        resp = connection.list_datasets options
+        if resp.success?
+          Dataset::List.from_resp resp, connection
+        else
+          fail ApiError.from_response(resp)
+        end
+      end
+
+      ##
+      # Retrieves an existing job by ID.
+      #
+      # === Parameters
+      #
+      # +job_id+::
+      #   The ID of a job. (+String+)
+      #
+      # === Returns
+      #
+      # Gcloud::Bigquery::Job or nil if job does not exist
+      #
+      # === Example
+      #
+      #   require "gcloud"
+      #
+      #   gcloud = Gcloud.new
+      #   bigquery = gcloud.bigquery
+      #
+      #   job = bigquery.job "my_job"
+      #
+      def job job_id
+        ensure_connection!
+        resp = connection.get_job job_id
+        if resp.success?
+          Job.from_gapi resp.data, connection
+        else
+          return nil if resp.status == 404
+          fail ApiError.from_response(resp)
+        end
+      end
+
+      ##
+      # Retrieves the list of jobs belonging to the project.
+      #
+      # === Parameters
+      #
+      # +options+::
+      #   An optional Hash for controlling additional behavior. (+Hash+)
+      # <code>options[:all]</code>::
+      #   Whether to display jobs owned by all users in the project.
+      #   The default is +false+. (+Boolean+)
+      # <code>options[:token]</code>::
+      #   A previously-returned page token representing part of the larger set
+      #   of results to view. (+String+)
+      # <code>options[:max]</code>::
+      #   Maximum number of jobs to return. (+Integer+)
+      # <code>options[:filter]</code>::
+      #   A filter for job state. (+String+)
+      #
+      #   Acceptable values are:
+      #   * +done+ - Finished jobs
+      #   * +pending+ - Pending jobs
+      #   * +running+ - Running jobs
+      #
+      # === Returns
+      #
+      # Array of Gcloud::Bigquery::Job (Gcloud::Bigquery::Job::List)
+      #
+      # === Examples
+      #
+      #   require "gcloud"
+      #
+      #   gcloud = Gcloud.new
+      #   bigquery = gcloud.bigquery
+      #
+      #   jobs = bigquery.jobs
+      #
+      # You can also retrieve only running jobs using the +:filter+ option:
+      #
+      #   require "gcloud"
+      #
+      #   gcloud = Gcloud.new
+      #   bigquery = gcloud.bigquery
+      #
+      #   running_jobs = bigquery.jobs filter: "running"
+      #
+      # If you have a significant number of jobs, you may need to paginate
+      # through them: (See Job::List#token)
+      #
+      #   require "gcloud"
+      #
+      #   gcloud = Gcloud.new
+      #   bigquery = gcloud.bigquery
+      #
+      #   all_jobs = []
+      #   tmp_jobs = bigquery.jobs
+      #   while tmp_jobs.any? do
+      #     tmp_jobs.each do |job|
+      #       all_jobs << job
+      #     end
+      #     # break loop if no more jobs available
+      #     break if tmp_jobs.token.nil?
+      #     # get the next group of jobs
+      #     tmp_jobs = bigquery.jobs token: tmp_jobs.token
+      #   end
+      #
+      def jobs options = {}
+        ensure_connection!
+        resp = connection.list_jobs options
+        if resp.success?
+          Job::List.from_resp resp, connection
+        else
+          fail ApiError.from_response(resp)
+        end
+      end
+
+      protected
+
+      ##
+      # Raise an error unless an active connection is available.
+      def ensure_connection!
+        fail "Must have active connection" unless connection
+      end
+    end
+  end
+end