google-cloud-bigquery 0.28.0 → 0.29.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +1 -1
- data/lib/google-cloud-bigquery.rb +2 -2
- data/lib/google/cloud/bigquery.rb +10 -12
- data/lib/google/cloud/bigquery/copy_job.rb +42 -6
- data/lib/google/cloud/bigquery/data.rb +129 -23
- data/lib/google/cloud/bigquery/dataset.rb +708 -66
- data/lib/google/cloud/bigquery/dataset/access.rb +533 -27
- data/lib/google/cloud/bigquery/dataset/list.rb +5 -3
- data/lib/google/cloud/bigquery/external.rb +2353 -0
- data/lib/google/cloud/bigquery/extract_job.rb +52 -11
- data/lib/google/cloud/bigquery/insert_response.rb +90 -2
- data/lib/google/cloud/bigquery/job.rb +160 -21
- data/lib/google/cloud/bigquery/load_job.rb +128 -11
- data/lib/google/cloud/bigquery/project.rb +187 -44
- data/lib/google/cloud/bigquery/query_job.rb +323 -13
- data/lib/google/cloud/bigquery/schema.rb +57 -1
- data/lib/google/cloud/bigquery/schema/field.rb +118 -17
- data/lib/google/cloud/bigquery/service.rb +196 -43
- data/lib/google/cloud/bigquery/table.rb +739 -49
- data/lib/google/cloud/bigquery/table/async_inserter.rb +280 -0
- data/lib/google/cloud/bigquery/version.rb +1 -1
- data/lib/google/cloud/bigquery/view.rb +306 -69
- metadata +18 -3
- data/lib/google/cloud/bigquery/query_data.rb +0 -234
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: google-cloud-bigquery
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.
|
4
|
+
version: 0.29.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Mike Moore
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2017-09
|
12
|
+
date: 2017-10-09 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: google-cloud-core
|
@@ -39,6 +39,20 @@ dependencies:
|
|
39
39
|
- - "~>"
|
40
40
|
- !ruby/object:Gem::Version
|
41
41
|
version: 0.14.0
|
42
|
+
- !ruby/object:Gem::Dependency
|
43
|
+
name: concurrent-ruby
|
44
|
+
requirement: !ruby/object:Gem::Requirement
|
45
|
+
requirements:
|
46
|
+
- - "~>"
|
47
|
+
- !ruby/object:Gem::Version
|
48
|
+
version: '1.0'
|
49
|
+
type: :runtime
|
50
|
+
prerelease: false
|
51
|
+
version_requirements: !ruby/object:Gem::Requirement
|
52
|
+
requirements:
|
53
|
+
- - "~>"
|
54
|
+
- !ruby/object:Gem::Version
|
55
|
+
version: '1.0'
|
42
56
|
- !ruby/object:Gem::Dependency
|
43
57
|
name: minitest
|
44
58
|
requirement: !ruby/object:Gem::Requirement
|
@@ -185,6 +199,7 @@ files:
|
|
185
199
|
- lib/google/cloud/bigquery/dataset.rb
|
186
200
|
- lib/google/cloud/bigquery/dataset/access.rb
|
187
201
|
- lib/google/cloud/bigquery/dataset/list.rb
|
202
|
+
- lib/google/cloud/bigquery/external.rb
|
188
203
|
- lib/google/cloud/bigquery/extract_job.rb
|
189
204
|
- lib/google/cloud/bigquery/insert_response.rb
|
190
205
|
- lib/google/cloud/bigquery/job.rb
|
@@ -192,12 +207,12 @@ files:
|
|
192
207
|
- lib/google/cloud/bigquery/load_job.rb
|
193
208
|
- lib/google/cloud/bigquery/project.rb
|
194
209
|
- lib/google/cloud/bigquery/project/list.rb
|
195
|
-
- lib/google/cloud/bigquery/query_data.rb
|
196
210
|
- lib/google/cloud/bigquery/query_job.rb
|
197
211
|
- lib/google/cloud/bigquery/schema.rb
|
198
212
|
- lib/google/cloud/bigquery/schema/field.rb
|
199
213
|
- lib/google/cloud/bigquery/service.rb
|
200
214
|
- lib/google/cloud/bigquery/table.rb
|
215
|
+
- lib/google/cloud/bigquery/table/async_inserter.rb
|
201
216
|
- lib/google/cloud/bigquery/table/list.rb
|
202
217
|
- lib/google/cloud/bigquery/time.rb
|
203
218
|
- lib/google/cloud/bigquery/version.rb
|
@@ -1,234 +0,0 @@
|
|
1
|
-
# Copyright 2015 Google Inc. All rights reserved.
|
2
|
-
#
|
3
|
-
# Licensed under the Apache License, Version 2.0 (the "License");
|
4
|
-
# you may not use this file except in compliance with the License.
|
5
|
-
# You may obtain a copy of the License at
|
6
|
-
#
|
7
|
-
# http://www.apache.org/licenses/LICENSE-2.0
|
8
|
-
#
|
9
|
-
# Unless required by applicable law or agreed to in writing, software
|
10
|
-
# distributed under the License is distributed on an "AS IS" BASIS,
|
11
|
-
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
12
|
-
# See the License for the specific language governing permissions and
|
13
|
-
# limitations under the License.
|
14
|
-
|
15
|
-
|
16
|
-
require "google/cloud/bigquery/service"
|
17
|
-
require "google/cloud/bigquery/data"
|
18
|
-
|
19
|
-
module Google
|
20
|
-
module Cloud
|
21
|
-
module Bigquery
|
22
|
-
##
|
23
|
-
# # QueryData
|
24
|
-
#
|
25
|
-
# Represents Data returned from a query a a list of name/value pairs.
|
26
|
-
class QueryData < Data
|
27
|
-
##
|
28
|
-
# @private The Service object.
|
29
|
-
attr_accessor :service
|
30
|
-
|
31
|
-
# @private
|
32
|
-
def initialize arr = []
|
33
|
-
@job = nil
|
34
|
-
super
|
35
|
-
end
|
36
|
-
|
37
|
-
# The total number of bytes processed for this query.
|
38
|
-
def total_bytes
|
39
|
-
Integer @gapi.total_bytes_processed
|
40
|
-
rescue
|
41
|
-
nil
|
42
|
-
end
|
43
|
-
|
44
|
-
# Whether the query has completed or not. When data is present this will
|
45
|
-
# always be `true`. When `false`, `total` will not be available.
|
46
|
-
def complete?
|
47
|
-
@gapi.job_complete
|
48
|
-
end
|
49
|
-
|
50
|
-
# Whether the query result was fetched from the query cache.
|
51
|
-
def cache_hit?
|
52
|
-
@gapi.cache_hit
|
53
|
-
end
|
54
|
-
|
55
|
-
##
|
56
|
-
# The schema of the data.
|
57
|
-
def schema
|
58
|
-
@schema ||= begin
|
59
|
-
s = Schema.from_gapi(@gapi.schema)
|
60
|
-
# call fields so they will be available
|
61
|
-
s.fields
|
62
|
-
s.freeze
|
63
|
-
end
|
64
|
-
end
|
65
|
-
|
66
|
-
##
|
67
|
-
# The fields of the data.
|
68
|
-
def fields
|
69
|
-
schema.fields
|
70
|
-
end
|
71
|
-
|
72
|
-
##
|
73
|
-
# The name of the columns in the data.
|
74
|
-
def headers
|
75
|
-
schema.headers
|
76
|
-
end
|
77
|
-
|
78
|
-
##
|
79
|
-
# Whether there is a next page of query data.
|
80
|
-
#
|
81
|
-
# @return [Boolean]
|
82
|
-
#
|
83
|
-
# @example
|
84
|
-
# require "google/cloud/bigquery"
|
85
|
-
#
|
86
|
-
# bigquery = Google::Cloud::Bigquery.new
|
87
|
-
# job = bigquery.job "my_job"
|
88
|
-
#
|
89
|
-
# data = job.query_results
|
90
|
-
# if data.next?
|
91
|
-
# next_data = data.next
|
92
|
-
# end
|
93
|
-
#
|
94
|
-
def next?
|
95
|
-
!token.nil?
|
96
|
-
end
|
97
|
-
|
98
|
-
##
|
99
|
-
# Retrieve the next page of query data.
|
100
|
-
#
|
101
|
-
# @return [QueryData]
|
102
|
-
#
|
103
|
-
# @example
|
104
|
-
# require "google/cloud/bigquery"
|
105
|
-
#
|
106
|
-
# bigquery = Google::Cloud::Bigquery.new
|
107
|
-
# job = bigquery.job "my_job"
|
108
|
-
#
|
109
|
-
# data = job.query_results
|
110
|
-
# if data.next?
|
111
|
-
# next_data = data.next
|
112
|
-
# end
|
113
|
-
#
|
114
|
-
def next
|
115
|
-
return nil unless next?
|
116
|
-
ensure_service!
|
117
|
-
gapi = service.job_query_results job_id, token: token
|
118
|
-
QueryData.from_gapi gapi, service
|
119
|
-
end
|
120
|
-
|
121
|
-
##
|
122
|
-
# Retrieves all rows by repeatedly loading {#next} until {#next?}
|
123
|
-
# returns `false`. Calls the given block once for each row, which is
|
124
|
-
# passed as the parameter.
|
125
|
-
#
|
126
|
-
# An Enumerator is returned if no block is given.
|
127
|
-
#
|
128
|
-
# This method may make several API calls until all rows are retrieved.
|
129
|
-
# Be sure to use as narrow a search criteria as possible. Please use
|
130
|
-
# with caution.
|
131
|
-
#
|
132
|
-
# @param [Integer] request_limit The upper limit of API requests to make
|
133
|
-
# to load all data. Default is no limit.
|
134
|
-
# @yield [row] The block for accessing each row of data.
|
135
|
-
# @yieldparam [Hash] row The row object.
|
136
|
-
#
|
137
|
-
# @return [Enumerator]
|
138
|
-
#
|
139
|
-
# @example Iterating each row by passing a block:
|
140
|
-
# require "google/cloud/bigquery"
|
141
|
-
#
|
142
|
-
# bigquery = Google::Cloud::Bigquery.new
|
143
|
-
# job = bigquery.job "my_job"
|
144
|
-
#
|
145
|
-
# data = job.query_results
|
146
|
-
# data.all do |row|
|
147
|
-
# puts row[:word]
|
148
|
-
# end
|
149
|
-
#
|
150
|
-
# @example Using the enumerator by not passing a block:
|
151
|
-
# require "google/cloud/bigquery"
|
152
|
-
#
|
153
|
-
# bigquery = Google::Cloud::Bigquery.new
|
154
|
-
# job = bigquery.job "my_job"
|
155
|
-
#
|
156
|
-
# data = job.query_results
|
157
|
-
# words = data.all.map do |row|
|
158
|
-
# row[:word]
|
159
|
-
# end
|
160
|
-
#
|
161
|
-
# @example Limit the number of API calls made:
|
162
|
-
# require "google/cloud/bigquery"
|
163
|
-
#
|
164
|
-
# bigquery = Google::Cloud::Bigquery.new
|
165
|
-
# job = bigquery.job "my_job"
|
166
|
-
#
|
167
|
-
# data = job.query_results
|
168
|
-
# data.all(request_limit: 10) do |row|
|
169
|
-
# puts row[:word]
|
170
|
-
# end
|
171
|
-
#
|
172
|
-
def all request_limit: nil
|
173
|
-
request_limit = request_limit.to_i if request_limit
|
174
|
-
unless block_given?
|
175
|
-
return enum_for(:all, request_limit: request_limit)
|
176
|
-
end
|
177
|
-
results = self
|
178
|
-
loop do
|
179
|
-
results.each { |r| yield r }
|
180
|
-
if request_limit
|
181
|
-
request_limit -= 1
|
182
|
-
break if request_limit < 0
|
183
|
-
end
|
184
|
-
break unless results.next?
|
185
|
-
results = results.next
|
186
|
-
end
|
187
|
-
end
|
188
|
-
|
189
|
-
##
|
190
|
-
# The BigQuery {Job} that was created to run the query.
|
191
|
-
def job
|
192
|
-
return @job if @job
|
193
|
-
return nil unless job?
|
194
|
-
ensure_service!
|
195
|
-
gapi = service.get_job job_id
|
196
|
-
@job = Job.from_gapi gapi, service
|
197
|
-
rescue Google::Cloud::NotFoundError
|
198
|
-
nil
|
199
|
-
end
|
200
|
-
|
201
|
-
##
|
202
|
-
# @private New Data from a response object.
|
203
|
-
def self.from_gapi gapi, service
|
204
|
-
if gapi.schema.nil?
|
205
|
-
formatted_rows = []
|
206
|
-
else
|
207
|
-
formatted_rows = Convert.format_rows gapi.rows, gapi.schema.fields
|
208
|
-
end
|
209
|
-
|
210
|
-
data = new formatted_rows
|
211
|
-
data.gapi = gapi
|
212
|
-
data.service = service
|
213
|
-
data
|
214
|
-
end
|
215
|
-
|
216
|
-
protected
|
217
|
-
|
218
|
-
##
|
219
|
-
# Raise an error unless an active connection is available.
|
220
|
-
def ensure_service!
|
221
|
-
fail "Must have active connection" unless service
|
222
|
-
end
|
223
|
-
|
224
|
-
def job?
|
225
|
-
@gapi.job_reference && @gapi.job_reference.job_id
|
226
|
-
end
|
227
|
-
|
228
|
-
def job_id
|
229
|
-
@gapi.job_reference.job_id
|
230
|
-
end
|
231
|
-
end
|
232
|
-
end
|
233
|
-
end
|
234
|
-
end
|