gcloud 0.6.3 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +8 -8
- data/AUTHENTICATION.md +13 -9
- data/CHANGELOG.md +8 -3
- data/OVERVIEW.md +46 -8
- data/lib/gcloud.rb +123 -117
- data/lib/gcloud/backoff.rb +43 -15
- data/lib/gcloud/bigquery.rb +211 -195
- data/lib/gcloud/bigquery/connection.rb +9 -9
- data/lib/gcloud/bigquery/copy_job.rb +15 -16
- data/lib/gcloud/bigquery/credentials.rb +3 -3
- data/lib/gcloud/bigquery/data.rb +12 -11
- data/lib/gcloud/bigquery/dataset.rb +162 -216
- data/lib/gcloud/bigquery/dataset/access.rb +59 -43
- data/lib/gcloud/bigquery/dataset/list.rb +3 -3
- data/lib/gcloud/bigquery/errors.rb +9 -5
- data/lib/gcloud/bigquery/extract_job.rb +18 -18
- data/lib/gcloud/bigquery/insert_response.rb +7 -4
- data/lib/gcloud/bigquery/job.rb +48 -44
- data/lib/gcloud/bigquery/job/list.rb +3 -3
- data/lib/gcloud/bigquery/load_job.rb +24 -25
- data/lib/gcloud/bigquery/project.rb +145 -204
- data/lib/gcloud/bigquery/query_data.rb +10 -9
- data/lib/gcloud/bigquery/query_job.rb +23 -32
- data/lib/gcloud/bigquery/table.rb +238 -280
- data/lib/gcloud/bigquery/table/list.rb +3 -3
- data/lib/gcloud/bigquery/table/schema.rb +79 -87
- data/lib/gcloud/bigquery/view.rb +69 -82
- data/lib/gcloud/credentials.rb +3 -9
- data/lib/gcloud/datastore.rb +194 -170
- data/lib/gcloud/datastore/connection.rb +12 -8
- data/lib/gcloud/datastore/credentials.rb +6 -4
- data/lib/gcloud/datastore/dataset.rb +74 -141
- data/lib/gcloud/datastore/dataset/lookup_results.rb +6 -4
- data/lib/gcloud/datastore/dataset/query_results.rb +6 -4
- data/lib/gcloud/datastore/entity.rb +81 -76
- data/lib/gcloud/datastore/errors.rb +10 -8
- data/lib/gcloud/datastore/key.rb +41 -77
- data/lib/gcloud/datastore/properties.rb +3 -3
- data/lib/gcloud/datastore/proto.rb +7 -4
- data/lib/gcloud/datastore/query.rb +26 -3
- data/lib/gcloud/datastore/transaction.rb +12 -8
- data/lib/gcloud/dns.rb +180 -152
- data/lib/gcloud/dns/change.rb +16 -16
- data/lib/gcloud/dns/change/list.rb +3 -3
- data/lib/gcloud/dns/connection.rb +9 -10
- data/lib/gcloud/dns/credentials.rb +3 -3
- data/lib/gcloud/dns/errors.rb +9 -5
- data/lib/gcloud/dns/importer.rb +17 -23
- data/lib/gcloud/dns/project.rb +42 -64
- data/lib/gcloud/dns/record.rb +58 -46
- data/lib/gcloud/dns/record/list.rb +6 -7
- data/lib/gcloud/dns/zone.rb +198 -289
- data/lib/gcloud/dns/zone/list.rb +3 -3
- data/lib/gcloud/dns/zone/transaction.rb +56 -72
- data/lib/gcloud/errors.rb +174 -3
- data/lib/gcloud/gce.rb +3 -4
- data/lib/gcloud/grpc_utils.rb +76 -0
- data/lib/gcloud/logging.rb +308 -0
- data/lib/gcloud/logging/credentials.rb +29 -0
- data/lib/gcloud/logging/entry.rb +303 -0
- data/lib/gcloud/logging/entry/http_request.rb +141 -0
- data/lib/gcloud/logging/entry/list.rb +111 -0
- data/lib/gcloud/logging/entry/operation.rb +90 -0
- data/lib/gcloud/logging/logger.rb +307 -0
- data/lib/gcloud/logging/metric.rb +175 -0
- data/lib/gcloud/logging/metric/list.rb +98 -0
- data/lib/gcloud/logging/project.rb +650 -0
- data/lib/gcloud/logging/resource.rb +95 -0
- data/lib/gcloud/logging/resource_descriptor.rb +140 -0
- data/lib/gcloud/logging/resource_descriptor/list.rb +78 -0
- data/lib/gcloud/logging/service.rb +258 -0
- data/lib/gcloud/logging/sink.rb +233 -0
- data/lib/gcloud/logging/sink/list.rb +97 -0
- data/lib/gcloud/pubsub.rb +241 -199
- data/lib/gcloud/pubsub/credentials.rb +3 -3
- data/lib/gcloud/pubsub/message.rb +26 -20
- data/lib/gcloud/pubsub/project.rb +166 -233
- data/lib/gcloud/pubsub/received_message.rb +28 -38
- data/lib/gcloud/pubsub/service.rb +323 -0
- data/lib/gcloud/pubsub/subscription.rb +172 -242
- data/lib/gcloud/pubsub/subscription/list.rb +11 -9
- data/lib/gcloud/pubsub/topic.rb +152 -271
- data/lib/gcloud/pubsub/topic/batch.rb +66 -0
- data/lib/gcloud/pubsub/topic/list.rb +9 -7
- data/lib/gcloud/resource_manager.rb +158 -138
- data/lib/gcloud/resource_manager/connection.rb +6 -5
- data/lib/gcloud/resource_manager/credentials.rb +3 -3
- data/lib/gcloud/resource_manager/errors.rb +9 -5
- data/lib/gcloud/resource_manager/manager.rb +54 -86
- data/lib/gcloud/resource_manager/project.rb +69 -88
- data/lib/gcloud/resource_manager/project/list.rb +4 -5
- data/lib/gcloud/resource_manager/project/updater.rb +12 -14
- data/lib/gcloud/search.rb +158 -135
- data/lib/gcloud/search/api_client.rb +7 -7
- data/lib/gcloud/search/connection.rb +8 -8
- data/lib/gcloud/search/credentials.rb +3 -3
- data/lib/gcloud/search/document.rb +64 -87
- data/lib/gcloud/search/document/list.rb +5 -5
- data/lib/gcloud/search/errors.rb +9 -5
- data/lib/gcloud/search/field_value.rb +32 -38
- data/lib/gcloud/search/field_values.rb +50 -80
- data/lib/gcloud/search/fields.rb +44 -65
- data/lib/gcloud/search/index.rb +163 -204
- data/lib/gcloud/search/index/list.rb +5 -5
- data/lib/gcloud/search/project.rb +31 -47
- data/lib/gcloud/search/result.rb +27 -31
- data/lib/gcloud/search/result/list.rb +6 -6
- data/lib/gcloud/storage.rb +224 -190
- data/lib/gcloud/storage/bucket.rb +202 -227
- data/lib/gcloud/storage/bucket/acl.rb +83 -170
- data/lib/gcloud/storage/bucket/cors.rb +31 -34
- data/lib/gcloud/storage/bucket/list.rb +3 -3
- data/lib/gcloud/storage/connection.rb +11 -7
- data/lib/gcloud/storage/credentials.rb +3 -3
- data/lib/gcloud/storage/errors.rb +11 -8
- data/lib/gcloud/storage/file.rb +129 -171
- data/lib/gcloud/storage/file/acl.rb +51 -99
- data/lib/gcloud/storage/file/list.rb +3 -3
- data/lib/gcloud/storage/file/verifier.rb +3 -2
- data/lib/gcloud/storage/project.rb +111 -132
- data/lib/gcloud/upload.rb +4 -7
- data/lib/gcloud/version.rb +2 -4
- data/lib/google/api/annotations.rb +14 -0
- data/lib/google/api/http.rb +30 -0
- data/lib/google/api/label.rb +24 -0
- data/lib/google/api/monitored_resource.rb +25 -0
- data/lib/google/datastore/v1beta3/datastore.rb +115 -0
- data/lib/google/datastore/v1beta3/datastore_services.rb +33 -0
- data/lib/google/datastore/v1beta3/entity.rb +63 -0
- data/lib/google/datastore/v1beta3/query.rb +128 -0
- data/lib/google/devtools/cloudtrace/v1/trace.rb +78 -0
- data/lib/google/devtools/cloudtrace/v1/trace_services.rb +32 -0
- data/lib/google/example/library/v1/library.rb +91 -0
- data/lib/google/example/library/v1/library_services.rb +40 -0
- data/lib/google/iam/v1/iam_policy.rb +33 -0
- data/lib/google/iam/v1/iam_policy_services.rb +30 -0
- data/lib/google/iam/v1/policy.rb +25 -0
- data/lib/google/logging/type/http_request.rb +28 -0
- data/lib/google/logging/type/log_severity.rb +27 -0
- data/lib/google/logging/v2/log_entry.rb +44 -0
- data/lib/google/logging/v2/logging.rb +56 -0
- data/lib/google/logging/v2/logging_config.rb +59 -0
- data/lib/google/logging/v2/logging_config_services.rb +32 -0
- data/lib/google/logging/v2/logging_metrics.rb +51 -0
- data/lib/google/logging/v2/logging_metrics_services.rb +32 -0
- data/lib/google/logging/v2/logging_services.rb +31 -0
- data/lib/google/longrunning/operations.rb +50 -0
- data/lib/google/longrunning/operations_services.rb +29 -0
- data/lib/google/protobuf/any.rb +17 -0
- data/lib/google/protobuf/api.rb +31 -0
- data/lib/google/protobuf/descriptor.rb +0 -0
- data/lib/google/protobuf/duration.rb +17 -0
- data/lib/google/protobuf/empty.rb +15 -0
- data/lib/google/protobuf/field_mask.rb +16 -0
- data/lib/google/protobuf/source_context.rb +16 -0
- data/lib/google/protobuf/struct.rb +35 -0
- data/lib/google/protobuf/timestamp.rb +17 -0
- data/lib/google/protobuf/type.rb +79 -0
- data/lib/google/protobuf/wrappers.rb +48 -0
- data/lib/google/pubsub/v1/pubsub.rb +129 -0
- data/lib/google/pubsub/v1/pubsub_services.rb +56 -0
- data/lib/google/pubsub/v1beta2/pubsub.rb +126 -0
- data/lib/google/pubsub/v1beta2/pubsub_services.rb +56 -0
- data/lib/google/rpc/code.rb +32 -0
- data/lib/google/rpc/error_details.rb +61 -0
- data/lib/google/rpc/status.rb +19 -0
- data/lib/google/type/color.rb +20 -0
- data/lib/google/type/date.rb +18 -0
- data/lib/google/type/dayofweek.rb +23 -0
- data/lib/google/type/latlng.rb +17 -0
- data/lib/google/type/money.rb +18 -0
- data/lib/google/type/timeofday.rb +19 -0
- metadata +101 -4
- data/lib/gcloud/pubsub/connection.rb +0 -295
- data/lib/gcloud/pubsub/errors.rb +0 -93
data/lib/gcloud/backoff.rb
CHANGED
|
@@ -1,4 +1,3 @@
|
|
|
1
|
-
#--
|
|
2
1
|
# Copyright 2014 Google Inc. All rights reserved.
|
|
3
2
|
#
|
|
4
3
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
@@ -13,8 +12,7 @@
|
|
|
13
12
|
# See the License for the specific language governing permissions and
|
|
14
13
|
# limitations under the License.
|
|
15
14
|
|
|
16
|
-
|
|
17
|
-
# Google Cloud Backoff
|
|
15
|
+
|
|
18
16
|
module Gcloud
|
|
19
17
|
##
|
|
20
18
|
# Backoff allows users to control how Google API calls are retried.
|
|
@@ -25,9 +23,11 @@ module Gcloud
|
|
|
25
23
|
# retry will be delayed one second, the second retry will be delayed
|
|
26
24
|
# two seconds, and so on.
|
|
27
25
|
#
|
|
26
|
+
# @example
|
|
28
27
|
# require "gcloud/backoff"
|
|
29
28
|
#
|
|
30
29
|
# Gcloud::Backoff.retries = 5 # Set a maximum of five retries per call
|
|
30
|
+
#
|
|
31
31
|
class Backoff
|
|
32
32
|
class << self
|
|
33
33
|
##
|
|
@@ -36,6 +36,12 @@ module Gcloud
|
|
|
36
36
|
# The default value is 3.
|
|
37
37
|
attr_accessor :retries
|
|
38
38
|
|
|
39
|
+
##
|
|
40
|
+
# The GRPC Status Codes that should be retried.
|
|
41
|
+
#
|
|
42
|
+
# The default values are 14.
|
|
43
|
+
attr_accessor :grpc_codes
|
|
44
|
+
|
|
39
45
|
##
|
|
40
46
|
# The HTTP Status Codes that should be retried.
|
|
41
47
|
#
|
|
@@ -59,52 +65,74 @@ module Gcloud
|
|
|
59
65
|
end
|
|
60
66
|
# Set the default values
|
|
61
67
|
self.retries = 3
|
|
68
|
+
self.grpc_codes = [14]
|
|
62
69
|
self.http_codes = [500, 503]
|
|
63
70
|
self.reasons = %w(rateLimitExceeded userRateLimitExceeded)
|
|
64
71
|
self.backoff = ->(retries) { sleep retries.to_i }
|
|
65
72
|
|
|
66
73
|
##
|
|
74
|
+
# @private
|
|
67
75
|
# Creates a new Backoff object to catch common errors when calling
|
|
68
76
|
# the Google API and handle the error by retrying the call.
|
|
69
77
|
#
|
|
70
|
-
# Gcloud::Backoff.new(options).
|
|
78
|
+
# Gcloud::Backoff.new(options).execute_gapi do
|
|
71
79
|
# client.execute api_method: service.things.insert,
|
|
72
80
|
# parameters: { thing: @thing },
|
|
73
81
|
# body_object: { name: thing_name }
|
|
74
82
|
# end
|
|
75
|
-
def initialize options = {}
|
|
76
|
-
@
|
|
77
|
-
@
|
|
78
|
-
@
|
|
79
|
-
@
|
|
83
|
+
def initialize options = {}
|
|
84
|
+
@retries = (options[:retries] || Backoff.retries).to_i
|
|
85
|
+
@grpc_codes = (options[:grpc_codes] || Backoff.grpc_codes).to_a
|
|
86
|
+
@http_codes = (options[:http_codes] || Backoff.http_codes).to_a
|
|
87
|
+
@reasons = (options[:reasons] || Backoff.reasons).to_a
|
|
88
|
+
@backoff = options[:backoff] || Backoff.backoff
|
|
80
89
|
end
|
|
81
90
|
|
|
82
|
-
|
|
91
|
+
# @private
|
|
92
|
+
def execute_gapi
|
|
83
93
|
current_retries = 0
|
|
84
94
|
loop do
|
|
85
|
-
result = yield
|
|
86
|
-
return result
|
|
87
|
-
break result
|
|
95
|
+
result = yield
|
|
96
|
+
return result unless result.is_a? Google::APIClient::Result
|
|
97
|
+
break result if result.success? || !retry?(result, current_retries)
|
|
88
98
|
current_retries += 1
|
|
89
99
|
@backoff.call current_retries
|
|
90
100
|
end
|
|
91
101
|
end
|
|
92
102
|
|
|
103
|
+
# @private
|
|
104
|
+
def execute_grpc
|
|
105
|
+
current_retries = 0
|
|
106
|
+
loop do
|
|
107
|
+
begin
|
|
108
|
+
return yield
|
|
109
|
+
rescue GRPC::BadStatus => e
|
|
110
|
+
raise e unless @grpc_codes.include?(e.code) &&
|
|
111
|
+
(current_retries < @retries)
|
|
112
|
+
current_retries += 1
|
|
113
|
+
@backoff.call current_retries
|
|
114
|
+
end
|
|
115
|
+
end
|
|
116
|
+
end
|
|
117
|
+
|
|
93
118
|
protected
|
|
94
119
|
|
|
120
|
+
# @private
|
|
95
121
|
def retry? result, current_retries #:nodoc:
|
|
96
|
-
if current_retries < @
|
|
122
|
+
if current_retries < @retries
|
|
97
123
|
return true if retry_http_code? result
|
|
98
124
|
return true if retry_error_reason? result
|
|
99
125
|
end
|
|
100
126
|
false
|
|
101
127
|
end
|
|
102
128
|
|
|
129
|
+
# @private
|
|
103
130
|
def retry_http_code? result #:nodoc:
|
|
104
131
|
@http_codes.include? result.response.status
|
|
105
132
|
end
|
|
106
133
|
|
|
107
|
-
|
|
134
|
+
# @private
|
|
135
|
+
def retry_error_reason? result
|
|
108
136
|
if result.data &&
|
|
109
137
|
result.data["error"] &&
|
|
110
138
|
result.data["error"]["errors"]
|
data/lib/gcloud/bigquery.rb
CHANGED
|
@@ -12,40 +12,34 @@
|
|
|
12
12
|
# See the License for the specific language governing permissions and
|
|
13
13
|
# limitations under the License.
|
|
14
14
|
|
|
15
|
+
|
|
15
16
|
require "gcloud"
|
|
16
17
|
require "gcloud/bigquery/project"
|
|
17
18
|
|
|
18
|
-
#--
|
|
19
|
-
# Google Cloud BigQuery
|
|
20
19
|
module Gcloud
|
|
21
20
|
##
|
|
22
|
-
# Creates a new
|
|
21
|
+
# Creates a new `Project` instance connected to the BigQuery service.
|
|
23
22
|
# Each call creates a new connection.
|
|
24
23
|
#
|
|
25
|
-
#
|
|
24
|
+
# For more information on connecting to Google Cloud see the [Authentication
|
|
25
|
+
# Guide](https://googlecloudplatform.github.io/gcloud-ruby/#/docs/guides/authentication).
|
|
26
26
|
#
|
|
27
|
-
#
|
|
28
|
-
#
|
|
29
|
-
#
|
|
30
|
-
#
|
|
31
|
-
#
|
|
32
|
-
#
|
|
33
|
-
#
|
|
34
|
-
#
|
|
35
|
-
# the connection can access. See {Using OAuth 2.0 to Access Google
|
|
36
|
-
# APIs}[https://developers.google.com/identity/protocols/OAuth2]. (+String+
|
|
37
|
-
# or +Array+)
|
|
27
|
+
# @param [String] project Identifier for a BigQuery project. If not present,
|
|
28
|
+
# the default project for the credentials is used.
|
|
29
|
+
# @param [String, Hash] keyfile Keyfile downloaded from Google Cloud. If file
|
|
30
|
+
# path the file must be readable.
|
|
31
|
+
# @param [String, Array<String>] scope The OAuth 2.0 scopes controlling the
|
|
32
|
+
# set of resources and operations that the connection can access. See [Using
|
|
33
|
+
# OAuth 2.0 to Access Google
|
|
34
|
+
# APIs](https://developers.google.com/identity/protocols/OAuth2).
|
|
38
35
|
#
|
|
39
36
|
# The default scope is:
|
|
40
37
|
#
|
|
41
|
-
# *
|
|
42
|
-
#
|
|
43
|
-
# === Returns
|
|
38
|
+
# * `https://www.googleapis.com/auth/bigquery`
|
|
44
39
|
#
|
|
45
|
-
# Gcloud::Bigquery::Project
|
|
46
|
-
#
|
|
47
|
-
# === Example
|
|
40
|
+
# @return [Gcloud::Bigquery::Project]
|
|
48
41
|
#
|
|
42
|
+
# @example
|
|
49
43
|
# require "gcloud/bigquery"
|
|
50
44
|
#
|
|
51
45
|
# bigquery = Gcloud.bigquery
|
|
@@ -63,311 +57,333 @@ module Gcloud
|
|
|
63
57
|
end
|
|
64
58
|
|
|
65
59
|
##
|
|
66
|
-
#
|
|
60
|
+
# # Google Cloud BigQuery
|
|
67
61
|
#
|
|
68
62
|
# Google Cloud BigQuery enables super-fast, SQL-like queries against massive
|
|
69
63
|
# datasets, using the processing power of Google's infrastructure. To learn
|
|
70
|
-
# more, read
|
|
71
|
-
# BigQuery?
|
|
64
|
+
# more, read [What is
|
|
65
|
+
# BigQuery?](https://cloud.google.com/bigquery/what-is-bigquery).
|
|
72
66
|
#
|
|
73
67
|
# Gcloud's goal is to provide an API that is familiar and comfortable to
|
|
74
|
-
# Rubyists. Authentication is handled by Gcloud#bigquery. You can provide
|
|
68
|
+
# Rubyists. Authentication is handled by {Gcloud#bigquery}. You can provide
|
|
75
69
|
# the project and credential information to connect to the BigQuery service,
|
|
76
70
|
# or if you are running on Google Compute Engine this configuration is taken
|
|
77
71
|
# care of for you. You can read more about the options for connecting in the
|
|
78
|
-
#
|
|
72
|
+
# [Authentication
|
|
73
|
+
# Guide](https://googlecloudplatform.github.io/gcloud-ruby/#/docs/guides/authentication).
|
|
79
74
|
#
|
|
80
75
|
# To help you get started quickly, the first few examples below use a public
|
|
81
|
-
# dataset provided by Google. As soon as you have
|
|
82
|
-
# up
|
|
76
|
+
# dataset provided by Google. As soon as you have [signed
|
|
77
|
+
# up](https://cloud.google.com/bigquery/sign-up) to use BigQuery, and provided
|
|
83
78
|
# that you stay in the free tier for queries, you should be able to run these
|
|
84
79
|
# first examples without the need to set up billing or to load data (although
|
|
85
80
|
# we'll show you how to do that too.)
|
|
86
81
|
#
|
|
87
|
-
#
|
|
82
|
+
# ## Listing Datasets and Tables
|
|
88
83
|
#
|
|
89
84
|
# A BigQuery project holds datasets, which in turn hold tables. Assuming that
|
|
90
85
|
# you have not yet created datasets or tables in your own project, let's
|
|
91
|
-
# connect to Google's
|
|
86
|
+
# connect to Google's `publicdata` project, and see what you find.
|
|
92
87
|
#
|
|
93
|
-
#
|
|
88
|
+
# ```ruby
|
|
89
|
+
# require "gcloud"
|
|
94
90
|
#
|
|
95
|
-
#
|
|
96
|
-
#
|
|
91
|
+
# gcloud = Gcloud.new "publicdata"
|
|
92
|
+
# bigquery = gcloud.bigquery
|
|
97
93
|
#
|
|
98
|
-
#
|
|
99
|
-
#
|
|
94
|
+
# bigquery.datasets.count #=> 1
|
|
95
|
+
# bigquery.datasets.first.dataset_id #=> "samples"
|
|
100
96
|
#
|
|
101
|
-
#
|
|
102
|
-
#
|
|
97
|
+
# dataset = bigquery.datasets.first
|
|
98
|
+
# tables = dataset.tables
|
|
103
99
|
#
|
|
104
|
-
#
|
|
105
|
-
#
|
|
100
|
+
# tables.count #=> 7
|
|
101
|
+
# tables.map &:table_id #=> [..., "shakespeare", "trigrams", "wikipedia"]
|
|
102
|
+
# ```
|
|
106
103
|
#
|
|
107
104
|
# In addition listing all datasets and tables in the project, you can also
|
|
108
105
|
# retrieve individual datasets and tables by ID. Let's look at the structure
|
|
109
|
-
# of the
|
|
106
|
+
# of the `shakespeare` table, which contains an entry for every word in every
|
|
110
107
|
# play written by Shakespeare.
|
|
111
108
|
#
|
|
112
|
-
#
|
|
109
|
+
# ```ruby
|
|
110
|
+
# require "gcloud"
|
|
113
111
|
#
|
|
114
|
-
#
|
|
115
|
-
#
|
|
112
|
+
# gcloud = Gcloud.new "publicdata"
|
|
113
|
+
# bigquery = gcloud.bigquery
|
|
116
114
|
#
|
|
117
|
-
#
|
|
118
|
-
#
|
|
115
|
+
# dataset = bigquery.dataset "samples"
|
|
116
|
+
# table = dataset.table "shakespeare"
|
|
119
117
|
#
|
|
120
|
-
#
|
|
121
|
-
#
|
|
118
|
+
# table.headers #=> ["word", "word_count", "corpus", "corpus_date"]
|
|
119
|
+
# table.rows_count #=> 164656
|
|
120
|
+
# ```
|
|
122
121
|
#
|
|
123
122
|
# Now that you know the column names for the Shakespeare table, you can write
|
|
124
123
|
# and run a query.
|
|
125
124
|
#
|
|
126
|
-
#
|
|
125
|
+
# ## Running queries
|
|
127
126
|
#
|
|
128
127
|
# BigQuery offers both synchronous and asynchronous methods, as explained in
|
|
129
|
-
#
|
|
128
|
+
# [Querying Data](https://cloud.google.com/bigquery/querying-data).
|
|
130
129
|
#
|
|
131
|
-
#
|
|
130
|
+
# ### Synchronous queries
|
|
132
131
|
#
|
|
133
132
|
# Let's start with the simpler synchronous approach. Notice that this time you
|
|
134
133
|
# are connecting using your own default project. This is necessary for running
|
|
135
134
|
# a query, since queries need to be able to create tables to hold results.
|
|
136
135
|
#
|
|
137
|
-
#
|
|
136
|
+
# ```ruby
|
|
137
|
+
# require "gcloud"
|
|
138
138
|
#
|
|
139
|
-
#
|
|
140
|
-
#
|
|
139
|
+
# gcloud = Gcloud.new
|
|
140
|
+
# bigquery = gcloud.bigquery
|
|
141
141
|
#
|
|
142
|
-
#
|
|
143
|
-
#
|
|
144
|
-
#
|
|
142
|
+
# sql = "SELECT TOP(word, 50) as word, COUNT(*) as count " +
|
|
143
|
+
# "FROM publicdata:samples.shakespeare"
|
|
144
|
+
# data = bigquery.query sql
|
|
145
145
|
#
|
|
146
|
-
#
|
|
147
|
-
#
|
|
148
|
-
#
|
|
146
|
+
# data.count #=> 50
|
|
147
|
+
# data.next? #=> false
|
|
148
|
+
# data.first #=> {"word"=>"you", "count"=>42}
|
|
149
|
+
# ```
|
|
149
150
|
#
|
|
150
|
-
# The
|
|
151
|
-
# offered by BigQuery. See the
|
|
152
|
-
# Reference
|
|
151
|
+
# The `TOP` function shown above is just one of a variety of functions
|
|
152
|
+
# offered by BigQuery. See the [Query
|
|
153
|
+
# Reference](https://cloud.google.com/bigquery/query-reference) for a full
|
|
153
154
|
# listing.
|
|
154
155
|
#
|
|
155
|
-
#
|
|
156
|
+
# ### Asynchronous queries
|
|
156
157
|
#
|
|
157
158
|
# Because you probably should not block for most BigQuery operations,
|
|
158
159
|
# including querying as well as importing, exporting, and copying data, the
|
|
159
160
|
# BigQuery API enables you to manage longer-running jobs. In the asynchronous
|
|
160
|
-
# approach to running a query, an instance of Gcloud::Bigquery::QueryJob is
|
|
161
|
-
# returned, rather than an instance of Gcloud::Bigquery::QueryData.
|
|
161
|
+
# approach to running a query, an instance of {Gcloud::Bigquery::QueryJob} is
|
|
162
|
+
# returned, rather than an instance of {Gcloud::Bigquery::QueryData}.
|
|
162
163
|
#
|
|
163
|
-
#
|
|
164
|
+
# ```ruby
|
|
165
|
+
# require "gcloud"
|
|
164
166
|
#
|
|
165
|
-
#
|
|
166
|
-
#
|
|
167
|
+
# gcloud = Gcloud.new
|
|
168
|
+
# bigquery = gcloud.bigquery
|
|
167
169
|
#
|
|
168
|
-
#
|
|
169
|
-
#
|
|
170
|
-
#
|
|
170
|
+
# sql = "SELECT TOP(word, 50) as word, COUNT(*) as count " +
|
|
171
|
+
# "FROM publicdata:samples.shakespeare"
|
|
172
|
+
# job = bigquery.query_job sql
|
|
171
173
|
#
|
|
172
|
-
#
|
|
173
|
-
#
|
|
174
|
-
#
|
|
175
|
-
#
|
|
176
|
-
# end
|
|
174
|
+
# job.wait_until_done!
|
|
175
|
+
# if !job.failed?
|
|
176
|
+
# job.query_results.each do |row|
|
|
177
|
+
# puts row["word"]
|
|
177
178
|
# end
|
|
179
|
+
# end
|
|
180
|
+
# ```
|
|
178
181
|
#
|
|
179
182
|
# Once you have determined that the job is done and has not failed, you can
|
|
180
|
-
# obtain an instance of Gcloud::Bigquery::QueryData by calling
|
|
181
|
-
# Gcloud::Bigquery::QueryJob#query_results. The query results for both of
|
|
183
|
+
# obtain an instance of {Gcloud::Bigquery::QueryData} by calling
|
|
184
|
+
# {Gcloud::Bigquery::QueryJob#query_results}. The query results for both of
|
|
182
185
|
# the above examples are stored in temporary tables with a lifetime of about
|
|
183
186
|
# 24 hours. See the final example below for a demonstration of how to store
|
|
184
187
|
# query results in a permanent table.
|
|
185
188
|
#
|
|
186
|
-
#
|
|
189
|
+
# ## Creating Datasets and Tables
|
|
187
190
|
#
|
|
188
191
|
# The first thing you need to do in a new BigQuery project is to create a
|
|
189
|
-
# Gcloud::Bigquery::Dataset. Datasets hold tables and control access to
|
|
192
|
+
# {Gcloud::Bigquery::Dataset}. Datasets hold tables and control access to
|
|
193
|
+
# them.
|
|
190
194
|
#
|
|
191
|
-
#
|
|
195
|
+
# ```ruby
|
|
196
|
+
# require "gcloud/bigquery"
|
|
192
197
|
#
|
|
193
|
-
#
|
|
194
|
-
#
|
|
195
|
-
#
|
|
198
|
+
# gcloud = Gcloud.new
|
|
199
|
+
# bigquery = gcloud.bigquery
|
|
200
|
+
# dataset = bigquery.create_dataset "my_dataset"
|
|
201
|
+
# ```
|
|
196
202
|
#
|
|
197
203
|
# Now that you have a dataset, you can use it to create a table. Every table
|
|
198
204
|
# is defined by a schema that may contain nested and repeated fields. The
|
|
199
205
|
# example below shows a schema with a repeated record field named
|
|
200
|
-
#
|
|
201
|
-
#
|
|
202
|
-
# BigQuery
|
|
203
|
-
#
|
|
204
|
-
#
|
|
205
|
-
#
|
|
206
|
-
#
|
|
207
|
-
#
|
|
208
|
-
#
|
|
209
|
-
#
|
|
210
|
-
#
|
|
211
|
-
#
|
|
212
|
-
#
|
|
213
|
-
#
|
|
214
|
-
#
|
|
215
|
-
#
|
|
206
|
+
# `cities_lived`. (For more information about nested and repeated fields, see
|
|
207
|
+
# [Preparing Data for
|
|
208
|
+
# BigQuery](https://cloud.google.com/bigquery/preparing-data-for-bigquery).)
|
|
209
|
+
#
|
|
210
|
+
# ```ruby
|
|
211
|
+
# require "gcloud"
|
|
212
|
+
#
|
|
213
|
+
# gcloud = Gcloud.new
|
|
214
|
+
# bigquery = gcloud.bigquery
|
|
215
|
+
# dataset = bigquery.dataset "my_dataset"
|
|
216
|
+
#
|
|
217
|
+
# table = dataset.create_table "people" do |schema|
|
|
218
|
+
# schema.string "first_name", mode: :required
|
|
219
|
+
# schema.record "cities_lived", mode: :repeated do |nested_schema|
|
|
220
|
+
# nested_schema.string "place", mode: :required
|
|
221
|
+
# nested_schema.integer "number_of_years", mode: :required
|
|
216
222
|
# end
|
|
223
|
+
# end
|
|
224
|
+
# ```
|
|
217
225
|
#
|
|
218
226
|
# Because of the repeated field in this schema, we cannot use the CSV format
|
|
219
227
|
# to load data into the table.
|
|
220
228
|
#
|
|
221
|
-
#
|
|
229
|
+
# ## Loading records
|
|
222
230
|
#
|
|
223
231
|
# In addition to CSV, data can be imported from files that are formatted as
|
|
224
|
-
#
|
|
225
|
-
#
|
|
232
|
+
# [Newline-delimited JSON](http://jsonlines.org/) or
|
|
233
|
+
# [Avro](http://avro.apache.org/), or from a Google Cloud Datastore backup. It
|
|
226
234
|
# can also be "streamed" into BigQuery.
|
|
227
235
|
#
|
|
228
236
|
# To follow along with these examples, you will need to set up billing on the
|
|
229
|
-
#
|
|
237
|
+
# [Google Developers Console](https://console.developers.google.com).
|
|
230
238
|
#
|
|
231
|
-
#
|
|
239
|
+
# ### Streaming records
|
|
232
240
|
#
|
|
233
241
|
# For situations in which you want new data to be available for querying as
|
|
234
242
|
# soon as possible, inserting individual records directly from your Ruby
|
|
235
243
|
# application is a great approach.
|
|
236
244
|
#
|
|
237
|
-
#
|
|
238
|
-
#
|
|
239
|
-
#
|
|
240
|
-
#
|
|
241
|
-
#
|
|
242
|
-
#
|
|
243
|
-
#
|
|
244
|
-
#
|
|
245
|
-
#
|
|
246
|
-
#
|
|
247
|
-
#
|
|
248
|
-
#
|
|
249
|
-
#
|
|
250
|
-
#
|
|
251
|
-
#
|
|
252
|
-
#
|
|
253
|
-
#
|
|
254
|
-
#
|
|
255
|
-
#
|
|
256
|
-
#
|
|
257
|
-
#
|
|
258
|
-
#
|
|
259
|
-
#
|
|
260
|
-
#
|
|
261
|
-
#
|
|
262
|
-
#
|
|
263
|
-
#
|
|
264
|
-
#
|
|
265
|
-
#
|
|
266
|
-
#
|
|
267
|
-
#
|
|
268
|
-
#
|
|
245
|
+
# ```ruby
|
|
246
|
+
# require "gcloud"
|
|
247
|
+
#
|
|
248
|
+
# gcloud = Gcloud.new
|
|
249
|
+
# bigquery = gcloud.bigquery
|
|
250
|
+
# dataset = bigquery.dataset "my_dataset"
|
|
251
|
+
# table = dataset.table "people"
|
|
252
|
+
#
|
|
253
|
+
# rows = [
|
|
254
|
+
# {
|
|
255
|
+
# "first_name" => "Anna",
|
|
256
|
+
# "cities_lived" => [
|
|
257
|
+
# {
|
|
258
|
+
# "place" => "Stockholm",
|
|
259
|
+
# "number_of_years" => 2
|
|
260
|
+
# }
|
|
261
|
+
# ]
|
|
262
|
+
# },
|
|
263
|
+
# {
|
|
264
|
+
# "first_name" => "Bob",
|
|
265
|
+
# "cities_lived" => [
|
|
266
|
+
# {
|
|
267
|
+
# "place" => "Seattle",
|
|
268
|
+
# "number_of_years" => 5
|
|
269
|
+
# },
|
|
270
|
+
# {
|
|
271
|
+
# "place" => "Austin",
|
|
272
|
+
# "number_of_years" => 6
|
|
273
|
+
# }
|
|
274
|
+
# ]
|
|
275
|
+
# }
|
|
276
|
+
# ]
|
|
277
|
+
# table.insert rows
|
|
278
|
+
# ```
|
|
269
279
|
#
|
|
270
280
|
# There are some trade-offs involved with streaming, so be sure to read the
|
|
271
|
-
# discussion of data consistency in
|
|
272
|
-
# BigQuery
|
|
281
|
+
# discussion of data consistency in [Streaming Data Into
|
|
282
|
+
# BigQuery](https://cloud.google.com/bigquery/streaming-data-into-bigquery).
|
|
273
283
|
#
|
|
274
|
-
#
|
|
284
|
+
# ### Uploading a file
|
|
275
285
|
#
|
|
276
286
|
# To follow along with this example, please download the
|
|
277
|
-
#
|
|
287
|
+
# [names.zip](http://www.ssa.gov/OACT/babynames/names.zip) archive from the
|
|
278
288
|
# U.S. Social Security Administration. Inside the archive you will find over
|
|
279
289
|
# 100 files containing baby name records since the year 1880. A PDF file also
|
|
280
290
|
# contained in the archive specifies the schema used below.
|
|
281
291
|
#
|
|
282
|
-
#
|
|
292
|
+
# ```ruby
|
|
293
|
+
# require "gcloud"
|
|
283
294
|
#
|
|
284
|
-
#
|
|
285
|
-
#
|
|
286
|
-
#
|
|
287
|
-
#
|
|
288
|
-
#
|
|
289
|
-
#
|
|
290
|
-
#
|
|
291
|
-
#
|
|
295
|
+
# gcloud = Gcloud.new
|
|
296
|
+
# bigquery = gcloud.bigquery
|
|
297
|
+
# dataset = bigquery.dataset "my_dataset"
|
|
298
|
+
# table = dataset.create_table "baby_names" do |schema|
|
|
299
|
+
# schema.string "name", mode: :required
|
|
300
|
+
# schema.string "sex", mode: :required
|
|
301
|
+
# schema.integer "number", mode: :required
|
|
302
|
+
# end
|
|
292
303
|
#
|
|
293
|
-
#
|
|
294
|
-
#
|
|
304
|
+
# file = File.open "names/yob2014.txt"
|
|
305
|
+
# load_job = table.load file, format: "csv"
|
|
306
|
+
# ```
|
|
295
307
|
#
|
|
296
308
|
# Because the names data, although formatted as CSV, is distributed in files
|
|
297
|
-
# with a
|
|
309
|
+
# with a `.txt` extension, this example explicitly passes the `format` option
|
|
298
310
|
# in order to demonstrate how to handle such situations. Because CSV is the
|
|
299
311
|
# default format for load operations, the option is not actually necessary.
|
|
300
|
-
# For JSON saved with a
|
|
312
|
+
# For JSON saved with a `.txt` extension, however, it would be.
|
|
301
313
|
#
|
|
302
|
-
#
|
|
314
|
+
# ### A note about large uploads
|
|
303
315
|
#
|
|
304
|
-
# You may encounter a Broken pipe (Errno::EPIPE) error when attempting to
|
|
316
|
+
# You may encounter a Broken pipe (`Errno::EPIPE`) error when attempting to
|
|
305
317
|
# upload large files. To avoid this problem, add the
|
|
306
|
-
#
|
|
318
|
+
# [httpclient](https://rubygems.org/gems/httpclient) gem to your project, and
|
|
307
319
|
# the line (or lines) of configuration shown below. These lines must execute
|
|
308
320
|
# after you require gcloud but before you make your first gcloud connection.
|
|
309
|
-
# The first statement configures
|
|
321
|
+
# The first statement configures [Faraday](https://rubygems.org/gems/faraday)
|
|
310
322
|
# to use httpclient. The second statement, which should only be added if you
|
|
311
|
-
# are using a version of Faraday at or above 0.9.2, is a workaround for
|
|
312
|
-
# gzip issue
|
|
323
|
+
# are using a version of Faraday at or above 0.9.2, is a workaround for [this
|
|
324
|
+
# gzip issue](https://github.com/GoogleCloudPlatform/gcloud-ruby/issues/367).
|
|
313
325
|
#
|
|
314
|
-
#
|
|
326
|
+
# ```ruby
|
|
327
|
+
# require "gcloud"
|
|
315
328
|
#
|
|
316
|
-
#
|
|
317
|
-
#
|
|
329
|
+
# # Use httpclient to avoid broken pipe errors with large uploads
|
|
330
|
+
# Faraday.default_adapter = :httpclient
|
|
318
331
|
#
|
|
319
|
-
#
|
|
320
|
-
#
|
|
321
|
-
#
|
|
322
|
-
#
|
|
332
|
+
# # Only add the following statement if using Faraday >= 0.9.2
|
|
333
|
+
# # Override gzip middleware with no-op for httpclient
|
|
334
|
+
# Faraday::Response.register_middleware :gzip =>
|
|
335
|
+
# Faraday::Response::Middleware
|
|
323
336
|
#
|
|
324
|
-
#
|
|
325
|
-
#
|
|
337
|
+
# gcloud = Gcloud.new
|
|
338
|
+
# bigquery = gcloud.bigquery
|
|
339
|
+
# ```
|
|
326
340
|
#
|
|
327
|
-
#
|
|
341
|
+
# ## Exporting query results to Google Cloud Storage
|
|
328
342
|
#
|
|
329
|
-
# The example below shows how to pass the
|
|
343
|
+
# The example below shows how to pass the `table` option with a query in order
|
|
330
344
|
# to store results in a permanent table. It also shows how to export the
|
|
331
345
|
# result data to a Google Cloud Storage file. In order to follow along, you
|
|
332
346
|
# will need to enable the Google Cloud Storage API in addition to setting up
|
|
333
347
|
# billing.
|
|
334
348
|
#
|
|
335
|
-
#
|
|
349
|
+
# ```ruby
|
|
350
|
+
# require "gcloud"
|
|
336
351
|
#
|
|
337
|
-
#
|
|
338
|
-
#
|
|
339
|
-
#
|
|
340
|
-
#
|
|
341
|
-
#
|
|
352
|
+
# gcloud = Gcloud.new
|
|
353
|
+
# bigquery = gcloud.bigquery
|
|
354
|
+
# dataset = bigquery.dataset "my_dataset"
|
|
355
|
+
# source_table = dataset.table "baby_names"
|
|
356
|
+
# result_table = dataset.create_table "baby_names_results"
|
|
342
357
|
#
|
|
343
|
-
#
|
|
344
|
-
#
|
|
345
|
-
#
|
|
346
|
-
#
|
|
347
|
-
#
|
|
358
|
+
# sql = "SELECT name, number as count " +
|
|
359
|
+
# "FROM baby_names " +
|
|
360
|
+
# "WHERE name CONTAINS 'Sam' " +
|
|
361
|
+
# "ORDER BY count DESC"
|
|
362
|
+
# query_job = dataset.query_job sql, table: result_table
|
|
348
363
|
#
|
|
349
|
-
#
|
|
364
|
+
# query_job.wait_until_done!
|
|
350
365
|
#
|
|
351
|
-
#
|
|
366
|
+
# if !query_job.failed?
|
|
352
367
|
#
|
|
353
|
-
#
|
|
354
|
-
#
|
|
355
|
-
#
|
|
356
|
-
#
|
|
368
|
+
# storage = gcloud.storage
|
|
369
|
+
# bucket_id = "bigquery-exports-#{SecureRandom.uuid}"
|
|
370
|
+
# bucket = storage.create_bucket bucket_id
|
|
371
|
+
# extract_url = "gs://#{bucket.id}/baby-names-sam.csv"
|
|
357
372
|
#
|
|
358
|
-
#
|
|
373
|
+
# extract_job = result_table.extract extract_url
|
|
359
374
|
#
|
|
360
|
-
#
|
|
375
|
+
# extract_job.wait_until_done!
|
|
361
376
|
#
|
|
362
|
-
#
|
|
363
|
-
#
|
|
377
|
+
# # Download to local filesystem
|
|
378
|
+
# bucket.files.first.download "baby-names-sam.csv"
|
|
364
379
|
#
|
|
365
|
-
#
|
|
380
|
+
# end
|
|
381
|
+
# ```
|
|
366
382
|
#
|
|
367
383
|
# If a table you wish to export contains a large amount of data, you can pass
|
|
368
384
|
# a wildcard URI to export to multiple files (for sharding), or an array of
|
|
369
|
-
# URIs (for partitioning), or both. See
|
|
370
|
-
# BigQuery
|
|
385
|
+
# URIs (for partitioning), or both. See [Exporting Data From
|
|
386
|
+
# BigQuery](https://cloud.google.com/bigquery/exporting-data-from-bigquery)
|
|
371
387
|
# for details.
|
|
372
388
|
#
|
|
373
389
|
module Bigquery
|