gcloud 0.6.3 → 0.7.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +8 -8
- data/AUTHENTICATION.md +13 -9
- data/CHANGELOG.md +8 -3
- data/OVERVIEW.md +46 -8
- data/lib/gcloud.rb +123 -117
- data/lib/gcloud/backoff.rb +43 -15
- data/lib/gcloud/bigquery.rb +211 -195
- data/lib/gcloud/bigquery/connection.rb +9 -9
- data/lib/gcloud/bigquery/copy_job.rb +15 -16
- data/lib/gcloud/bigquery/credentials.rb +3 -3
- data/lib/gcloud/bigquery/data.rb +12 -11
- data/lib/gcloud/bigquery/dataset.rb +162 -216
- data/lib/gcloud/bigquery/dataset/access.rb +59 -43
- data/lib/gcloud/bigquery/dataset/list.rb +3 -3
- data/lib/gcloud/bigquery/errors.rb +9 -5
- data/lib/gcloud/bigquery/extract_job.rb +18 -18
- data/lib/gcloud/bigquery/insert_response.rb +7 -4
- data/lib/gcloud/bigquery/job.rb +48 -44
- data/lib/gcloud/bigquery/job/list.rb +3 -3
- data/lib/gcloud/bigquery/load_job.rb +24 -25
- data/lib/gcloud/bigquery/project.rb +145 -204
- data/lib/gcloud/bigquery/query_data.rb +10 -9
- data/lib/gcloud/bigquery/query_job.rb +23 -32
- data/lib/gcloud/bigquery/table.rb +238 -280
- data/lib/gcloud/bigquery/table/list.rb +3 -3
- data/lib/gcloud/bigquery/table/schema.rb +79 -87
- data/lib/gcloud/bigquery/view.rb +69 -82
- data/lib/gcloud/credentials.rb +3 -9
- data/lib/gcloud/datastore.rb +194 -170
- data/lib/gcloud/datastore/connection.rb +12 -8
- data/lib/gcloud/datastore/credentials.rb +6 -4
- data/lib/gcloud/datastore/dataset.rb +74 -141
- data/lib/gcloud/datastore/dataset/lookup_results.rb +6 -4
- data/lib/gcloud/datastore/dataset/query_results.rb +6 -4
- data/lib/gcloud/datastore/entity.rb +81 -76
- data/lib/gcloud/datastore/errors.rb +10 -8
- data/lib/gcloud/datastore/key.rb +41 -77
- data/lib/gcloud/datastore/properties.rb +3 -3
- data/lib/gcloud/datastore/proto.rb +7 -4
- data/lib/gcloud/datastore/query.rb +26 -3
- data/lib/gcloud/datastore/transaction.rb +12 -8
- data/lib/gcloud/dns.rb +180 -152
- data/lib/gcloud/dns/change.rb +16 -16
- data/lib/gcloud/dns/change/list.rb +3 -3
- data/lib/gcloud/dns/connection.rb +9 -10
- data/lib/gcloud/dns/credentials.rb +3 -3
- data/lib/gcloud/dns/errors.rb +9 -5
- data/lib/gcloud/dns/importer.rb +17 -23
- data/lib/gcloud/dns/project.rb +42 -64
- data/lib/gcloud/dns/record.rb +58 -46
- data/lib/gcloud/dns/record/list.rb +6 -7
- data/lib/gcloud/dns/zone.rb +198 -289
- data/lib/gcloud/dns/zone/list.rb +3 -3
- data/lib/gcloud/dns/zone/transaction.rb +56 -72
- data/lib/gcloud/errors.rb +174 -3
- data/lib/gcloud/gce.rb +3 -4
- data/lib/gcloud/grpc_utils.rb +76 -0
- data/lib/gcloud/logging.rb +308 -0
- data/lib/gcloud/logging/credentials.rb +29 -0
- data/lib/gcloud/logging/entry.rb +303 -0
- data/lib/gcloud/logging/entry/http_request.rb +141 -0
- data/lib/gcloud/logging/entry/list.rb +111 -0
- data/lib/gcloud/logging/entry/operation.rb +90 -0
- data/lib/gcloud/logging/logger.rb +307 -0
- data/lib/gcloud/logging/metric.rb +175 -0
- data/lib/gcloud/logging/metric/list.rb +98 -0
- data/lib/gcloud/logging/project.rb +650 -0
- data/lib/gcloud/logging/resource.rb +95 -0
- data/lib/gcloud/logging/resource_descriptor.rb +140 -0
- data/lib/gcloud/logging/resource_descriptor/list.rb +78 -0
- data/lib/gcloud/logging/service.rb +258 -0
- data/lib/gcloud/logging/sink.rb +233 -0
- data/lib/gcloud/logging/sink/list.rb +97 -0
- data/lib/gcloud/pubsub.rb +241 -199
- data/lib/gcloud/pubsub/credentials.rb +3 -3
- data/lib/gcloud/pubsub/message.rb +26 -20
- data/lib/gcloud/pubsub/project.rb +166 -233
- data/lib/gcloud/pubsub/received_message.rb +28 -38
- data/lib/gcloud/pubsub/service.rb +323 -0
- data/lib/gcloud/pubsub/subscription.rb +172 -242
- data/lib/gcloud/pubsub/subscription/list.rb +11 -9
- data/lib/gcloud/pubsub/topic.rb +152 -271
- data/lib/gcloud/pubsub/topic/batch.rb +66 -0
- data/lib/gcloud/pubsub/topic/list.rb +9 -7
- data/lib/gcloud/resource_manager.rb +158 -138
- data/lib/gcloud/resource_manager/connection.rb +6 -5
- data/lib/gcloud/resource_manager/credentials.rb +3 -3
- data/lib/gcloud/resource_manager/errors.rb +9 -5
- data/lib/gcloud/resource_manager/manager.rb +54 -86
- data/lib/gcloud/resource_manager/project.rb +69 -88
- data/lib/gcloud/resource_manager/project/list.rb +4 -5
- data/lib/gcloud/resource_manager/project/updater.rb +12 -14
- data/lib/gcloud/search.rb +158 -135
- data/lib/gcloud/search/api_client.rb +7 -7
- data/lib/gcloud/search/connection.rb +8 -8
- data/lib/gcloud/search/credentials.rb +3 -3
- data/lib/gcloud/search/document.rb +64 -87
- data/lib/gcloud/search/document/list.rb +5 -5
- data/lib/gcloud/search/errors.rb +9 -5
- data/lib/gcloud/search/field_value.rb +32 -38
- data/lib/gcloud/search/field_values.rb +50 -80
- data/lib/gcloud/search/fields.rb +44 -65
- data/lib/gcloud/search/index.rb +163 -204
- data/lib/gcloud/search/index/list.rb +5 -5
- data/lib/gcloud/search/project.rb +31 -47
- data/lib/gcloud/search/result.rb +27 -31
- data/lib/gcloud/search/result/list.rb +6 -6
- data/lib/gcloud/storage.rb +224 -190
- data/lib/gcloud/storage/bucket.rb +202 -227
- data/lib/gcloud/storage/bucket/acl.rb +83 -170
- data/lib/gcloud/storage/bucket/cors.rb +31 -34
- data/lib/gcloud/storage/bucket/list.rb +3 -3
- data/lib/gcloud/storage/connection.rb +11 -7
- data/lib/gcloud/storage/credentials.rb +3 -3
- data/lib/gcloud/storage/errors.rb +11 -8
- data/lib/gcloud/storage/file.rb +129 -171
- data/lib/gcloud/storage/file/acl.rb +51 -99
- data/lib/gcloud/storage/file/list.rb +3 -3
- data/lib/gcloud/storage/file/verifier.rb +3 -2
- data/lib/gcloud/storage/project.rb +111 -132
- data/lib/gcloud/upload.rb +4 -7
- data/lib/gcloud/version.rb +2 -4
- data/lib/google/api/annotations.rb +14 -0
- data/lib/google/api/http.rb +30 -0
- data/lib/google/api/label.rb +24 -0
- data/lib/google/api/monitored_resource.rb +25 -0
- data/lib/google/datastore/v1beta3/datastore.rb +115 -0
- data/lib/google/datastore/v1beta3/datastore_services.rb +33 -0
- data/lib/google/datastore/v1beta3/entity.rb +63 -0
- data/lib/google/datastore/v1beta3/query.rb +128 -0
- data/lib/google/devtools/cloudtrace/v1/trace.rb +78 -0
- data/lib/google/devtools/cloudtrace/v1/trace_services.rb +32 -0
- data/lib/google/example/library/v1/library.rb +91 -0
- data/lib/google/example/library/v1/library_services.rb +40 -0
- data/lib/google/iam/v1/iam_policy.rb +33 -0
- data/lib/google/iam/v1/iam_policy_services.rb +30 -0
- data/lib/google/iam/v1/policy.rb +25 -0
- data/lib/google/logging/type/http_request.rb +28 -0
- data/lib/google/logging/type/log_severity.rb +27 -0
- data/lib/google/logging/v2/log_entry.rb +44 -0
- data/lib/google/logging/v2/logging.rb +56 -0
- data/lib/google/logging/v2/logging_config.rb +59 -0
- data/lib/google/logging/v2/logging_config_services.rb +32 -0
- data/lib/google/logging/v2/logging_metrics.rb +51 -0
- data/lib/google/logging/v2/logging_metrics_services.rb +32 -0
- data/lib/google/logging/v2/logging_services.rb +31 -0
- data/lib/google/longrunning/operations.rb +50 -0
- data/lib/google/longrunning/operations_services.rb +29 -0
- data/lib/google/protobuf/any.rb +17 -0
- data/lib/google/protobuf/api.rb +31 -0
- data/lib/google/protobuf/descriptor.rb +0 -0
- data/lib/google/protobuf/duration.rb +17 -0
- data/lib/google/protobuf/empty.rb +15 -0
- data/lib/google/protobuf/field_mask.rb +16 -0
- data/lib/google/protobuf/source_context.rb +16 -0
- data/lib/google/protobuf/struct.rb +35 -0
- data/lib/google/protobuf/timestamp.rb +17 -0
- data/lib/google/protobuf/type.rb +79 -0
- data/lib/google/protobuf/wrappers.rb +48 -0
- data/lib/google/pubsub/v1/pubsub.rb +129 -0
- data/lib/google/pubsub/v1/pubsub_services.rb +56 -0
- data/lib/google/pubsub/v1beta2/pubsub.rb +126 -0
- data/lib/google/pubsub/v1beta2/pubsub_services.rb +56 -0
- data/lib/google/rpc/code.rb +32 -0
- data/lib/google/rpc/error_details.rb +61 -0
- data/lib/google/rpc/status.rb +19 -0
- data/lib/google/type/color.rb +20 -0
- data/lib/google/type/date.rb +18 -0
- data/lib/google/type/dayofweek.rb +23 -0
- data/lib/google/type/latlng.rb +17 -0
- data/lib/google/type/money.rb +18 -0
- data/lib/google/type/timeofday.rb +19 -0
- metadata +101 -4
- data/lib/gcloud/pubsub/connection.rb +0 -295
- data/lib/gcloud/pubsub/errors.rb +0 -93
data/lib/gcloud/backoff.rb
CHANGED
@@ -1,4 +1,3 @@
|
|
1
|
-
#--
|
2
1
|
# Copyright 2014 Google Inc. All rights reserved.
|
3
2
|
#
|
4
3
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
@@ -13,8 +12,7 @@
|
|
13
12
|
# See the License for the specific language governing permissions and
|
14
13
|
# limitations under the License.
|
15
14
|
|
16
|
-
|
17
|
-
# Google Cloud Backoff
|
15
|
+
|
18
16
|
module Gcloud
|
19
17
|
##
|
20
18
|
# Backoff allows users to control how Google API calls are retried.
|
@@ -25,9 +23,11 @@ module Gcloud
|
|
25
23
|
# retry will be delayed one second, the second retry will be delayed
|
26
24
|
# two seconds, and so on.
|
27
25
|
#
|
26
|
+
# @example
|
28
27
|
# require "gcloud/backoff"
|
29
28
|
#
|
30
29
|
# Gcloud::Backoff.retries = 5 # Set a maximum of five retries per call
|
30
|
+
#
|
31
31
|
class Backoff
|
32
32
|
class << self
|
33
33
|
##
|
@@ -36,6 +36,12 @@ module Gcloud
|
|
36
36
|
# The default value is 3.
|
37
37
|
attr_accessor :retries
|
38
38
|
|
39
|
+
##
|
40
|
+
# The GRPC Status Codes that should be retried.
|
41
|
+
#
|
42
|
+
# The default values are 14.
|
43
|
+
attr_accessor :grpc_codes
|
44
|
+
|
39
45
|
##
|
40
46
|
# The HTTP Status Codes that should be retried.
|
41
47
|
#
|
@@ -59,52 +65,74 @@ module Gcloud
|
|
59
65
|
end
|
60
66
|
# Set the default values
|
61
67
|
self.retries = 3
|
68
|
+
self.grpc_codes = [14]
|
62
69
|
self.http_codes = [500, 503]
|
63
70
|
self.reasons = %w(rateLimitExceeded userRateLimitExceeded)
|
64
71
|
self.backoff = ->(retries) { sleep retries.to_i }
|
65
72
|
|
66
73
|
##
|
74
|
+
# @private
|
67
75
|
# Creates a new Backoff object to catch common errors when calling
|
68
76
|
# the Google API and handle the error by retrying the call.
|
69
77
|
#
|
70
|
-
# Gcloud::Backoff.new(options).
|
78
|
+
# Gcloud::Backoff.new(options).execute_gapi do
|
71
79
|
# client.execute api_method: service.things.insert,
|
72
80
|
# parameters: { thing: @thing },
|
73
81
|
# body_object: { name: thing_name }
|
74
82
|
# end
|
75
|
-
def initialize options = {}
|
76
|
-
@
|
77
|
-
@
|
78
|
-
@
|
79
|
-
@
|
83
|
+
def initialize options = {}
|
84
|
+
@retries = (options[:retries] || Backoff.retries).to_i
|
85
|
+
@grpc_codes = (options[:grpc_codes] || Backoff.grpc_codes).to_a
|
86
|
+
@http_codes = (options[:http_codes] || Backoff.http_codes).to_a
|
87
|
+
@reasons = (options[:reasons] || Backoff.reasons).to_a
|
88
|
+
@backoff = options[:backoff] || Backoff.backoff
|
80
89
|
end
|
81
90
|
|
82
|
-
|
91
|
+
# @private
|
92
|
+
def execute_gapi
|
83
93
|
current_retries = 0
|
84
94
|
loop do
|
85
|
-
result = yield
|
86
|
-
return result
|
87
|
-
break result
|
95
|
+
result = yield
|
96
|
+
return result unless result.is_a? Google::APIClient::Result
|
97
|
+
break result if result.success? || !retry?(result, current_retries)
|
88
98
|
current_retries += 1
|
89
99
|
@backoff.call current_retries
|
90
100
|
end
|
91
101
|
end
|
92
102
|
|
103
|
+
# @private
|
104
|
+
def execute_grpc
|
105
|
+
current_retries = 0
|
106
|
+
loop do
|
107
|
+
begin
|
108
|
+
return yield
|
109
|
+
rescue GRPC::BadStatus => e
|
110
|
+
raise e unless @grpc_codes.include?(e.code) &&
|
111
|
+
(current_retries < @retries)
|
112
|
+
current_retries += 1
|
113
|
+
@backoff.call current_retries
|
114
|
+
end
|
115
|
+
end
|
116
|
+
end
|
117
|
+
|
93
118
|
protected
|
94
119
|
|
120
|
+
# @private
|
95
121
|
def retry? result, current_retries #:nodoc:
|
96
|
-
if current_retries < @
|
122
|
+
if current_retries < @retries
|
97
123
|
return true if retry_http_code? result
|
98
124
|
return true if retry_error_reason? result
|
99
125
|
end
|
100
126
|
false
|
101
127
|
end
|
102
128
|
|
129
|
+
# @private
|
103
130
|
def retry_http_code? result #:nodoc:
|
104
131
|
@http_codes.include? result.response.status
|
105
132
|
end
|
106
133
|
|
107
|
-
|
134
|
+
# @private
|
135
|
+
def retry_error_reason? result
|
108
136
|
if result.data &&
|
109
137
|
result.data["error"] &&
|
110
138
|
result.data["error"]["errors"]
|
data/lib/gcloud/bigquery.rb
CHANGED
@@ -12,40 +12,34 @@
|
|
12
12
|
# See the License for the specific language governing permissions and
|
13
13
|
# limitations under the License.
|
14
14
|
|
15
|
+
|
15
16
|
require "gcloud"
|
16
17
|
require "gcloud/bigquery/project"
|
17
18
|
|
18
|
-
#--
|
19
|
-
# Google Cloud BigQuery
|
20
19
|
module Gcloud
|
21
20
|
##
|
22
|
-
# Creates a new
|
21
|
+
# Creates a new `Project` instance connected to the BigQuery service.
|
23
22
|
# Each call creates a new connection.
|
24
23
|
#
|
25
|
-
#
|
24
|
+
# For more information on connecting to Google Cloud see the [Authentication
|
25
|
+
# Guide](https://googlecloudplatform.github.io/gcloud-ruby/#/docs/guides/authentication).
|
26
26
|
#
|
27
|
-
#
|
28
|
-
#
|
29
|
-
#
|
30
|
-
#
|
31
|
-
#
|
32
|
-
#
|
33
|
-
#
|
34
|
-
#
|
35
|
-
# the connection can access. See {Using OAuth 2.0 to Access Google
|
36
|
-
# APIs}[https://developers.google.com/identity/protocols/OAuth2]. (+String+
|
37
|
-
# or +Array+)
|
27
|
+
# @param [String] project Identifier for a BigQuery project. If not present,
|
28
|
+
# the default project for the credentials is used.
|
29
|
+
# @param [String, Hash] keyfile Keyfile downloaded from Google Cloud. If file
|
30
|
+
# path the file must be readable.
|
31
|
+
# @param [String, Array<String>] scope The OAuth 2.0 scopes controlling the
|
32
|
+
# set of resources and operations that the connection can access. See [Using
|
33
|
+
# OAuth 2.0 to Access Google
|
34
|
+
# APIs](https://developers.google.com/identity/protocols/OAuth2).
|
38
35
|
#
|
39
36
|
# The default scope is:
|
40
37
|
#
|
41
|
-
# *
|
42
|
-
#
|
43
|
-
# === Returns
|
38
|
+
# * `https://www.googleapis.com/auth/bigquery`
|
44
39
|
#
|
45
|
-
# Gcloud::Bigquery::Project
|
46
|
-
#
|
47
|
-
# === Example
|
40
|
+
# @return [Gcloud::Bigquery::Project]
|
48
41
|
#
|
42
|
+
# @example
|
49
43
|
# require "gcloud/bigquery"
|
50
44
|
#
|
51
45
|
# bigquery = Gcloud.bigquery
|
@@ -63,311 +57,333 @@ module Gcloud
|
|
63
57
|
end
|
64
58
|
|
65
59
|
##
|
66
|
-
#
|
60
|
+
# # Google Cloud BigQuery
|
67
61
|
#
|
68
62
|
# Google Cloud BigQuery enables super-fast, SQL-like queries against massive
|
69
63
|
# datasets, using the processing power of Google's infrastructure. To learn
|
70
|
-
# more, read
|
71
|
-
# BigQuery?
|
64
|
+
# more, read [What is
|
65
|
+
# BigQuery?](https://cloud.google.com/bigquery/what-is-bigquery).
|
72
66
|
#
|
73
67
|
# Gcloud's goal is to provide an API that is familiar and comfortable to
|
74
|
-
# Rubyists. Authentication is handled by Gcloud#bigquery. You can provide
|
68
|
+
# Rubyists. Authentication is handled by {Gcloud#bigquery}. You can provide
|
75
69
|
# the project and credential information to connect to the BigQuery service,
|
76
70
|
# or if you are running on Google Compute Engine this configuration is taken
|
77
71
|
# care of for you. You can read more about the options for connecting in the
|
78
|
-
#
|
72
|
+
# [Authentication
|
73
|
+
# Guide](https://googlecloudplatform.github.io/gcloud-ruby/#/docs/guides/authentication).
|
79
74
|
#
|
80
75
|
# To help you get started quickly, the first few examples below use a public
|
81
|
-
# dataset provided by Google. As soon as you have
|
82
|
-
# up
|
76
|
+
# dataset provided by Google. As soon as you have [signed
|
77
|
+
# up](https://cloud.google.com/bigquery/sign-up) to use BigQuery, and provided
|
83
78
|
# that you stay in the free tier for queries, you should be able to run these
|
84
79
|
# first examples without the need to set up billing or to load data (although
|
85
80
|
# we'll show you how to do that too.)
|
86
81
|
#
|
87
|
-
#
|
82
|
+
# ## Listing Datasets and Tables
|
88
83
|
#
|
89
84
|
# A BigQuery project holds datasets, which in turn hold tables. Assuming that
|
90
85
|
# you have not yet created datasets or tables in your own project, let's
|
91
|
-
# connect to Google's
|
86
|
+
# connect to Google's `publicdata` project, and see what you find.
|
92
87
|
#
|
93
|
-
#
|
88
|
+
# ```ruby
|
89
|
+
# require "gcloud"
|
94
90
|
#
|
95
|
-
#
|
96
|
-
#
|
91
|
+
# gcloud = Gcloud.new "publicdata"
|
92
|
+
# bigquery = gcloud.bigquery
|
97
93
|
#
|
98
|
-
#
|
99
|
-
#
|
94
|
+
# bigquery.datasets.count #=> 1
|
95
|
+
# bigquery.datasets.first.dataset_id #=> "samples"
|
100
96
|
#
|
101
|
-
#
|
102
|
-
#
|
97
|
+
# dataset = bigquery.datasets.first
|
98
|
+
# tables = dataset.tables
|
103
99
|
#
|
104
|
-
#
|
105
|
-
#
|
100
|
+
# tables.count #=> 7
|
101
|
+
# tables.map &:table_id #=> [..., "shakespeare", "trigrams", "wikipedia"]
|
102
|
+
# ```
|
106
103
|
#
|
107
104
|
# In addition listing all datasets and tables in the project, you can also
|
108
105
|
# retrieve individual datasets and tables by ID. Let's look at the structure
|
109
|
-
# of the
|
106
|
+
# of the `shakespeare` table, which contains an entry for every word in every
|
110
107
|
# play written by Shakespeare.
|
111
108
|
#
|
112
|
-
#
|
109
|
+
# ```ruby
|
110
|
+
# require "gcloud"
|
113
111
|
#
|
114
|
-
#
|
115
|
-
#
|
112
|
+
# gcloud = Gcloud.new "publicdata"
|
113
|
+
# bigquery = gcloud.bigquery
|
116
114
|
#
|
117
|
-
#
|
118
|
-
#
|
115
|
+
# dataset = bigquery.dataset "samples"
|
116
|
+
# table = dataset.table "shakespeare"
|
119
117
|
#
|
120
|
-
#
|
121
|
-
#
|
118
|
+
# table.headers #=> ["word", "word_count", "corpus", "corpus_date"]
|
119
|
+
# table.rows_count #=> 164656
|
120
|
+
# ```
|
122
121
|
#
|
123
122
|
# Now that you know the column names for the Shakespeare table, you can write
|
124
123
|
# and run a query.
|
125
124
|
#
|
126
|
-
#
|
125
|
+
# ## Running queries
|
127
126
|
#
|
128
127
|
# BigQuery offers both synchronous and asynchronous methods, as explained in
|
129
|
-
#
|
128
|
+
# [Querying Data](https://cloud.google.com/bigquery/querying-data).
|
130
129
|
#
|
131
|
-
#
|
130
|
+
# ### Synchronous queries
|
132
131
|
#
|
133
132
|
# Let's start with the simpler synchronous approach. Notice that this time you
|
134
133
|
# are connecting using your own default project. This is necessary for running
|
135
134
|
# a query, since queries need to be able to create tables to hold results.
|
136
135
|
#
|
137
|
-
#
|
136
|
+
# ```ruby
|
137
|
+
# require "gcloud"
|
138
138
|
#
|
139
|
-
#
|
140
|
-
#
|
139
|
+
# gcloud = Gcloud.new
|
140
|
+
# bigquery = gcloud.bigquery
|
141
141
|
#
|
142
|
-
#
|
143
|
-
#
|
144
|
-
#
|
142
|
+
# sql = "SELECT TOP(word, 50) as word, COUNT(*) as count " +
|
143
|
+
# "FROM publicdata:samples.shakespeare"
|
144
|
+
# data = bigquery.query sql
|
145
145
|
#
|
146
|
-
#
|
147
|
-
#
|
148
|
-
#
|
146
|
+
# data.count #=> 50
|
147
|
+
# data.next? #=> false
|
148
|
+
# data.first #=> {"word"=>"you", "count"=>42}
|
149
|
+
# ```
|
149
150
|
#
|
150
|
-
# The
|
151
|
-
# offered by BigQuery. See the
|
152
|
-
# Reference
|
151
|
+
# The `TOP` function shown above is just one of a variety of functions
|
152
|
+
# offered by BigQuery. See the [Query
|
153
|
+
# Reference](https://cloud.google.com/bigquery/query-reference) for a full
|
153
154
|
# listing.
|
154
155
|
#
|
155
|
-
#
|
156
|
+
# ### Asynchronous queries
|
156
157
|
#
|
157
158
|
# Because you probably should not block for most BigQuery operations,
|
158
159
|
# including querying as well as importing, exporting, and copying data, the
|
159
160
|
# BigQuery API enables you to manage longer-running jobs. In the asynchronous
|
160
|
-
# approach to running a query, an instance of Gcloud::Bigquery::QueryJob is
|
161
|
-
# returned, rather than an instance of Gcloud::Bigquery::QueryData.
|
161
|
+
# approach to running a query, an instance of {Gcloud::Bigquery::QueryJob} is
|
162
|
+
# returned, rather than an instance of {Gcloud::Bigquery::QueryData}.
|
162
163
|
#
|
163
|
-
#
|
164
|
+
# ```ruby
|
165
|
+
# require "gcloud"
|
164
166
|
#
|
165
|
-
#
|
166
|
-
#
|
167
|
+
# gcloud = Gcloud.new
|
168
|
+
# bigquery = gcloud.bigquery
|
167
169
|
#
|
168
|
-
#
|
169
|
-
#
|
170
|
-
#
|
170
|
+
# sql = "SELECT TOP(word, 50) as word, COUNT(*) as count " +
|
171
|
+
# "FROM publicdata:samples.shakespeare"
|
172
|
+
# job = bigquery.query_job sql
|
171
173
|
#
|
172
|
-
#
|
173
|
-
#
|
174
|
-
#
|
175
|
-
#
|
176
|
-
# end
|
174
|
+
# job.wait_until_done!
|
175
|
+
# if !job.failed?
|
176
|
+
# job.query_results.each do |row|
|
177
|
+
# puts row["word"]
|
177
178
|
# end
|
179
|
+
# end
|
180
|
+
# ```
|
178
181
|
#
|
179
182
|
# Once you have determined that the job is done and has not failed, you can
|
180
|
-
# obtain an instance of Gcloud::Bigquery::QueryData by calling
|
181
|
-
# Gcloud::Bigquery::QueryJob#query_results. The query results for both of
|
183
|
+
# obtain an instance of {Gcloud::Bigquery::QueryData} by calling
|
184
|
+
# {Gcloud::Bigquery::QueryJob#query_results}. The query results for both of
|
182
185
|
# the above examples are stored in temporary tables with a lifetime of about
|
183
186
|
# 24 hours. See the final example below for a demonstration of how to store
|
184
187
|
# query results in a permanent table.
|
185
188
|
#
|
186
|
-
#
|
189
|
+
# ## Creating Datasets and Tables
|
187
190
|
#
|
188
191
|
# The first thing you need to do in a new BigQuery project is to create a
|
189
|
-
# Gcloud::Bigquery::Dataset. Datasets hold tables and control access to
|
192
|
+
# {Gcloud::Bigquery::Dataset}. Datasets hold tables and control access to
|
193
|
+
# them.
|
190
194
|
#
|
191
|
-
#
|
195
|
+
# ```ruby
|
196
|
+
# require "gcloud/bigquery"
|
192
197
|
#
|
193
|
-
#
|
194
|
-
#
|
195
|
-
#
|
198
|
+
# gcloud = Gcloud.new
|
199
|
+
# bigquery = gcloud.bigquery
|
200
|
+
# dataset = bigquery.create_dataset "my_dataset"
|
201
|
+
# ```
|
196
202
|
#
|
197
203
|
# Now that you have a dataset, you can use it to create a table. Every table
|
198
204
|
# is defined by a schema that may contain nested and repeated fields. The
|
199
205
|
# example below shows a schema with a repeated record field named
|
200
|
-
#
|
201
|
-
#
|
202
|
-
# BigQuery
|
203
|
-
#
|
204
|
-
#
|
205
|
-
#
|
206
|
-
#
|
207
|
-
#
|
208
|
-
#
|
209
|
-
#
|
210
|
-
#
|
211
|
-
#
|
212
|
-
#
|
213
|
-
#
|
214
|
-
#
|
215
|
-
#
|
206
|
+
# `cities_lived`. (For more information about nested and repeated fields, see
|
207
|
+
# [Preparing Data for
|
208
|
+
# BigQuery](https://cloud.google.com/bigquery/preparing-data-for-bigquery).)
|
209
|
+
#
|
210
|
+
# ```ruby
|
211
|
+
# require "gcloud"
|
212
|
+
#
|
213
|
+
# gcloud = Gcloud.new
|
214
|
+
# bigquery = gcloud.bigquery
|
215
|
+
# dataset = bigquery.dataset "my_dataset"
|
216
|
+
#
|
217
|
+
# table = dataset.create_table "people" do |schema|
|
218
|
+
# schema.string "first_name", mode: :required
|
219
|
+
# schema.record "cities_lived", mode: :repeated do |nested_schema|
|
220
|
+
# nested_schema.string "place", mode: :required
|
221
|
+
# nested_schema.integer "number_of_years", mode: :required
|
216
222
|
# end
|
223
|
+
# end
|
224
|
+
# ```
|
217
225
|
#
|
218
226
|
# Because of the repeated field in this schema, we cannot use the CSV format
|
219
227
|
# to load data into the table.
|
220
228
|
#
|
221
|
-
#
|
229
|
+
# ## Loading records
|
222
230
|
#
|
223
231
|
# In addition to CSV, data can be imported from files that are formatted as
|
224
|
-
#
|
225
|
-
#
|
232
|
+
# [Newline-delimited JSON](http://jsonlines.org/) or
|
233
|
+
# [Avro](http://avro.apache.org/), or from a Google Cloud Datastore backup. It
|
226
234
|
# can also be "streamed" into BigQuery.
|
227
235
|
#
|
228
236
|
# To follow along with these examples, you will need to set up billing on the
|
229
|
-
#
|
237
|
+
# [Google Developers Console](https://console.developers.google.com).
|
230
238
|
#
|
231
|
-
#
|
239
|
+
# ### Streaming records
|
232
240
|
#
|
233
241
|
# For situations in which you want new data to be available for querying as
|
234
242
|
# soon as possible, inserting individual records directly from your Ruby
|
235
243
|
# application is a great approach.
|
236
244
|
#
|
237
|
-
#
|
238
|
-
#
|
239
|
-
#
|
240
|
-
#
|
241
|
-
#
|
242
|
-
#
|
243
|
-
#
|
244
|
-
#
|
245
|
-
#
|
246
|
-
#
|
247
|
-
#
|
248
|
-
#
|
249
|
-
#
|
250
|
-
#
|
251
|
-
#
|
252
|
-
#
|
253
|
-
#
|
254
|
-
#
|
255
|
-
#
|
256
|
-
#
|
257
|
-
#
|
258
|
-
#
|
259
|
-
#
|
260
|
-
#
|
261
|
-
#
|
262
|
-
#
|
263
|
-
#
|
264
|
-
#
|
265
|
-
#
|
266
|
-
#
|
267
|
-
#
|
268
|
-
#
|
245
|
+
# ```ruby
|
246
|
+
# require "gcloud"
|
247
|
+
#
|
248
|
+
# gcloud = Gcloud.new
|
249
|
+
# bigquery = gcloud.bigquery
|
250
|
+
# dataset = bigquery.dataset "my_dataset"
|
251
|
+
# table = dataset.table "people"
|
252
|
+
#
|
253
|
+
# rows = [
|
254
|
+
# {
|
255
|
+
# "first_name" => "Anna",
|
256
|
+
# "cities_lived" => [
|
257
|
+
# {
|
258
|
+
# "place" => "Stockholm",
|
259
|
+
# "number_of_years" => 2
|
260
|
+
# }
|
261
|
+
# ]
|
262
|
+
# },
|
263
|
+
# {
|
264
|
+
# "first_name" => "Bob",
|
265
|
+
# "cities_lived" => [
|
266
|
+
# {
|
267
|
+
# "place" => "Seattle",
|
268
|
+
# "number_of_years" => 5
|
269
|
+
# },
|
270
|
+
# {
|
271
|
+
# "place" => "Austin",
|
272
|
+
# "number_of_years" => 6
|
273
|
+
# }
|
274
|
+
# ]
|
275
|
+
# }
|
276
|
+
# ]
|
277
|
+
# table.insert rows
|
278
|
+
# ```
|
269
279
|
#
|
270
280
|
# There are some trade-offs involved with streaming, so be sure to read the
|
271
|
-
# discussion of data consistency in
|
272
|
-
# BigQuery
|
281
|
+
# discussion of data consistency in [Streaming Data Into
|
282
|
+
# BigQuery](https://cloud.google.com/bigquery/streaming-data-into-bigquery).
|
273
283
|
#
|
274
|
-
#
|
284
|
+
# ### Uploading a file
|
275
285
|
#
|
276
286
|
# To follow along with this example, please download the
|
277
|
-
#
|
287
|
+
# [names.zip](http://www.ssa.gov/OACT/babynames/names.zip) archive from the
|
278
288
|
# U.S. Social Security Administration. Inside the archive you will find over
|
279
289
|
# 100 files containing baby name records since the year 1880. A PDF file also
|
280
290
|
# contained in the archive specifies the schema used below.
|
281
291
|
#
|
282
|
-
#
|
292
|
+
# ```ruby
|
293
|
+
# require "gcloud"
|
283
294
|
#
|
284
|
-
#
|
285
|
-
#
|
286
|
-
#
|
287
|
-
#
|
288
|
-
#
|
289
|
-
#
|
290
|
-
#
|
291
|
-
#
|
295
|
+
# gcloud = Gcloud.new
|
296
|
+
# bigquery = gcloud.bigquery
|
297
|
+
# dataset = bigquery.dataset "my_dataset"
|
298
|
+
# table = dataset.create_table "baby_names" do |schema|
|
299
|
+
# schema.string "name", mode: :required
|
300
|
+
# schema.string "sex", mode: :required
|
301
|
+
# schema.integer "number", mode: :required
|
302
|
+
# end
|
292
303
|
#
|
293
|
-
#
|
294
|
-
#
|
304
|
+
# file = File.open "names/yob2014.txt"
|
305
|
+
# load_job = table.load file, format: "csv"
|
306
|
+
# ```
|
295
307
|
#
|
296
308
|
# Because the names data, although formatted as CSV, is distributed in files
|
297
|
-
# with a
|
309
|
+
# with a `.txt` extension, this example explicitly passes the `format` option
|
298
310
|
# in order to demonstrate how to handle such situations. Because CSV is the
|
299
311
|
# default format for load operations, the option is not actually necessary.
|
300
|
-
# For JSON saved with a
|
312
|
+
# For JSON saved with a `.txt` extension, however, it would be.
|
301
313
|
#
|
302
|
-
#
|
314
|
+
# ### A note about large uploads
|
303
315
|
#
|
304
|
-
# You may encounter a Broken pipe (Errno::EPIPE) error when attempting to
|
316
|
+
# You may encounter a Broken pipe (`Errno::EPIPE`) error when attempting to
|
305
317
|
# upload large files. To avoid this problem, add the
|
306
|
-
#
|
318
|
+
# [httpclient](https://rubygems.org/gems/httpclient) gem to your project, and
|
307
319
|
# the line (or lines) of configuration shown below. These lines must execute
|
308
320
|
# after you require gcloud but before you make your first gcloud connection.
|
309
|
-
# The first statement configures
|
321
|
+
# The first statement configures [Faraday](https://rubygems.org/gems/faraday)
|
310
322
|
# to use httpclient. The second statement, which should only be added if you
|
311
|
-
# are using a version of Faraday at or above 0.9.2, is a workaround for
|
312
|
-
# gzip issue
|
323
|
+
# are using a version of Faraday at or above 0.9.2, is a workaround for [this
|
324
|
+
# gzip issue](https://github.com/GoogleCloudPlatform/gcloud-ruby/issues/367).
|
313
325
|
#
|
314
|
-
#
|
326
|
+
# ```ruby
|
327
|
+
# require "gcloud"
|
315
328
|
#
|
316
|
-
#
|
317
|
-
#
|
329
|
+
# # Use httpclient to avoid broken pipe errors with large uploads
|
330
|
+
# Faraday.default_adapter = :httpclient
|
318
331
|
#
|
319
|
-
#
|
320
|
-
#
|
321
|
-
#
|
322
|
-
#
|
332
|
+
# # Only add the following statement if using Faraday >= 0.9.2
|
333
|
+
# # Override gzip middleware with no-op for httpclient
|
334
|
+
# Faraday::Response.register_middleware :gzip =>
|
335
|
+
# Faraday::Response::Middleware
|
323
336
|
#
|
324
|
-
#
|
325
|
-
#
|
337
|
+
# gcloud = Gcloud.new
|
338
|
+
# bigquery = gcloud.bigquery
|
339
|
+
# ```
|
326
340
|
#
|
327
|
-
#
|
341
|
+
# ## Exporting query results to Google Cloud Storage
|
328
342
|
#
|
329
|
-
# The example below shows how to pass the
|
343
|
+
# The example below shows how to pass the `table` option with a query in order
|
330
344
|
# to store results in a permanent table. It also shows how to export the
|
331
345
|
# result data to a Google Cloud Storage file. In order to follow along, you
|
332
346
|
# will need to enable the Google Cloud Storage API in addition to setting up
|
333
347
|
# billing.
|
334
348
|
#
|
335
|
-
#
|
349
|
+
# ```ruby
|
350
|
+
# require "gcloud"
|
336
351
|
#
|
337
|
-
#
|
338
|
-
#
|
339
|
-
#
|
340
|
-
#
|
341
|
-
#
|
352
|
+
# gcloud = Gcloud.new
|
353
|
+
# bigquery = gcloud.bigquery
|
354
|
+
# dataset = bigquery.dataset "my_dataset"
|
355
|
+
# source_table = dataset.table "baby_names"
|
356
|
+
# result_table = dataset.create_table "baby_names_results"
|
342
357
|
#
|
343
|
-
#
|
344
|
-
#
|
345
|
-
#
|
346
|
-
#
|
347
|
-
#
|
358
|
+
# sql = "SELECT name, number as count " +
|
359
|
+
# "FROM baby_names " +
|
360
|
+
# "WHERE name CONTAINS 'Sam' " +
|
361
|
+
# "ORDER BY count DESC"
|
362
|
+
# query_job = dataset.query_job sql, table: result_table
|
348
363
|
#
|
349
|
-
#
|
364
|
+
# query_job.wait_until_done!
|
350
365
|
#
|
351
|
-
#
|
366
|
+
# if !query_job.failed?
|
352
367
|
#
|
353
|
-
#
|
354
|
-
#
|
355
|
-
#
|
356
|
-
#
|
368
|
+
# storage = gcloud.storage
|
369
|
+
# bucket_id = "bigquery-exports-#{SecureRandom.uuid}"
|
370
|
+
# bucket = storage.create_bucket bucket_id
|
371
|
+
# extract_url = "gs://#{bucket.id}/baby-names-sam.csv"
|
357
372
|
#
|
358
|
-
#
|
373
|
+
# extract_job = result_table.extract extract_url
|
359
374
|
#
|
360
|
-
#
|
375
|
+
# extract_job.wait_until_done!
|
361
376
|
#
|
362
|
-
#
|
363
|
-
#
|
377
|
+
# # Download to local filesystem
|
378
|
+
# bucket.files.first.download "baby-names-sam.csv"
|
364
379
|
#
|
365
|
-
#
|
380
|
+
# end
|
381
|
+
# ```
|
366
382
|
#
|
367
383
|
# If a table you wish to export contains a large amount of data, you can pass
|
368
384
|
# a wildcard URI to export to multiple files (for sharding), or an array of
|
369
|
-
# URIs (for partitioning), or both. See
|
370
|
-
# BigQuery
|
385
|
+
# URIs (for partitioning), or both. See [Exporting Data From
|
386
|
+
# BigQuery](https://cloud.google.com/bigquery/exporting-data-from-bigquery)
|
371
387
|
# for details.
|
372
388
|
#
|
373
389
|
module Bigquery
|