google-cloud-bigquery 1.18.0 → 1.21.1
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +48 -0
- data/TROUBLESHOOTING.md +2 -8
- data/lib/google/cloud/bigquery/argument.rb +197 -0
- data/lib/google/cloud/bigquery/copy_job.rb +18 -1
- data/lib/google/cloud/bigquery/data.rb +15 -0
- data/lib/google/cloud/bigquery/dataset.rb +379 -49
- data/lib/google/cloud/bigquery/dataset/list.rb +1 -2
- data/lib/google/cloud/bigquery/extract_job.rb +19 -2
- data/lib/google/cloud/bigquery/job.rb +198 -0
- data/lib/google/cloud/bigquery/job/list.rb +5 -5
- data/lib/google/cloud/bigquery/load_job.rb +273 -26
- data/lib/google/cloud/bigquery/model.rb +6 -4
- data/lib/google/cloud/bigquery/project.rb +82 -22
- data/lib/google/cloud/bigquery/project/list.rb +1 -2
- data/lib/google/cloud/bigquery/query_job.rb +292 -0
- data/lib/google/cloud/bigquery/routine.rb +1108 -0
- data/lib/google/cloud/bigquery/routine/list.rb +165 -0
- data/lib/google/cloud/bigquery/schema.rb +2 -2
- data/lib/google/cloud/bigquery/service.rb +96 -39
- data/lib/google/cloud/bigquery/standard_sql.rb +257 -53
- data/lib/google/cloud/bigquery/table.rb +410 -62
- data/lib/google/cloud/bigquery/table/async_inserter.rb +21 -11
- data/lib/google/cloud/bigquery/table/list.rb +1 -2
- data/lib/google/cloud/bigquery/version.rb +1 -1
- metadata +9 -6
@@ -73,6 +73,9 @@ module Google
|
|
73
73
|
# @private
|
74
74
|
def initialize table, skip_invalid: nil, ignore_unknown: nil, max_bytes: 10_000_000, max_rows: 500,
|
75
75
|
interval: 10, threads: 4, &block
|
76
|
+
# init MonitorMixin
|
77
|
+
super()
|
78
|
+
|
76
79
|
@table = table
|
77
80
|
@skip_invalid = skip_invalid
|
78
81
|
@ignore_unknown = ignore_unknown
|
@@ -88,9 +91,6 @@ module Google
|
|
88
91
|
@thread_pool = Concurrent::ThreadPoolExecutor.new max_threads: @threads
|
89
92
|
|
90
93
|
@cond = new_cond
|
91
|
-
|
92
|
-
# init MonitorMixin
|
93
|
-
super()
|
94
94
|
end
|
95
95
|
|
96
96
|
##
|
@@ -113,12 +113,13 @@ module Google
|
|
113
113
|
#
|
114
114
|
# @param [Hash, Array<Hash>] rows A hash object or array of hash
|
115
115
|
# objects containing the data.
|
116
|
-
# @param [Array<String
|
117
|
-
#
|
118
|
-
#
|
119
|
-
#
|
120
|
-
#
|
121
|
-
#
|
116
|
+
# @param [Array<String|Symbol>, Symbol] insert_ids A unique ID for each row. BigQuery uses this property to
|
117
|
+
# detect duplicate insertion requests on a best-effort basis. For more information, see [data
|
118
|
+
# consistency](https://cloud.google.com/bigquery/streaming-data-into-bigquery#dataconsistency). Optional. If
|
119
|
+
# not provided, the client library will assign a UUID to each row before the request is sent.
|
120
|
+
#
|
121
|
+
# The value `:skip` can be provided to skip the generation of IDs for all rows, or to skip the generation of
|
122
|
+
# an ID for a specific row in the array.
|
122
123
|
#
|
123
124
|
def insert rows, insert_ids: nil
|
124
125
|
return nil if rows.nil?
|
@@ -224,10 +225,14 @@ module Google
|
|
224
225
|
|
225
226
|
def validate_insert_args rows, insert_ids
|
226
227
|
rows = [rows] if rows.is_a? Hash
|
228
|
+
raise ArgumentError, "No rows provided" if rows.empty?
|
229
|
+
|
230
|
+
insert_ids = Array.new(rows.count) { :skip } if insert_ids == :skip
|
227
231
|
insert_ids = Array insert_ids
|
228
232
|
if insert_ids.count.positive? && insert_ids.count != rows.count
|
229
233
|
raise ArgumentError, "insert_ids must be the same size as rows"
|
230
234
|
end
|
235
|
+
|
231
236
|
[rows, insert_ids]
|
232
237
|
end
|
233
238
|
|
@@ -332,8 +337,13 @@ module Google
|
|
332
337
|
end
|
333
338
|
|
334
339
|
def addl_bytes_for json_row, insert_id
|
335
|
-
|
336
|
-
|
340
|
+
if insert_id == :skip
|
341
|
+
# "{\"json\":},".bytesize #=> 10
|
342
|
+
10 + json_row.to_json.bytesize
|
343
|
+
else
|
344
|
+
# "{\"insertId\":\"\",\"json\":},".bytesize #=> 24
|
345
|
+
24 + json_row.to_json.bytesize + insert_id.bytesize
|
346
|
+
end
|
337
347
|
end
|
338
348
|
end
|
339
349
|
|
@@ -78,8 +78,7 @@ module Google
|
|
78
78
|
def next
|
79
79
|
return nil unless next?
|
80
80
|
ensure_service!
|
81
|
-
|
82
|
-
gapi = @service.list_tables @dataset_id, options
|
81
|
+
gapi = @service.list_tables @dataset_id, token: token, max: @max
|
83
82
|
self.class.from_gapi gapi, @service, @dataset_id, @max
|
84
83
|
end
|
85
84
|
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: google-cloud-bigquery
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.
|
4
|
+
version: 1.21.1
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Mike Moore
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date:
|
12
|
+
date: 2020-05-28 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: concurrent-ruby
|
@@ -115,14 +115,14 @@ dependencies:
|
|
115
115
|
requirements:
|
116
116
|
- - "~>"
|
117
117
|
- !ruby/object:Gem::Version
|
118
|
-
version: '5.
|
118
|
+
version: '5.14'
|
119
119
|
type: :development
|
120
120
|
prerelease: false
|
121
121
|
version_requirements: !ruby/object:Gem::Requirement
|
122
122
|
requirements:
|
123
123
|
- - "~>"
|
124
124
|
- !ruby/object:Gem::Version
|
125
|
-
version: '5.
|
125
|
+
version: '5.14'
|
126
126
|
- !ruby/object:Gem::Dependency
|
127
127
|
name: minitest-autotest
|
128
128
|
requirement: !ruby/object:Gem::Requirement
|
@@ -185,14 +185,14 @@ dependencies:
|
|
185
185
|
requirements:
|
186
186
|
- - "~>"
|
187
187
|
- !ruby/object:Gem::Version
|
188
|
-
version: '0.
|
188
|
+
version: '0.18'
|
189
189
|
type: :development
|
190
190
|
prerelease: false
|
191
191
|
version_requirements: !ruby/object:Gem::Requirement
|
192
192
|
requirements:
|
193
193
|
- - "~>"
|
194
194
|
- !ruby/object:Gem::Version
|
195
|
-
version: '0.
|
195
|
+
version: '0.18'
|
196
196
|
- !ruby/object:Gem::Dependency
|
197
197
|
name: yard
|
198
198
|
requirement: !ruby/object:Gem::Requirement
|
@@ -240,6 +240,7 @@ files:
|
|
240
240
|
- TROUBLESHOOTING.md
|
241
241
|
- lib/google-cloud-bigquery.rb
|
242
242
|
- lib/google/cloud/bigquery.rb
|
243
|
+
- lib/google/cloud/bigquery/argument.rb
|
243
244
|
- lib/google/cloud/bigquery/convert.rb
|
244
245
|
- lib/google/cloud/bigquery/copy_job.rb
|
245
246
|
- lib/google/cloud/bigquery/credentials.rb
|
@@ -259,6 +260,8 @@ files:
|
|
259
260
|
- lib/google/cloud/bigquery/project.rb
|
260
261
|
- lib/google/cloud/bigquery/project/list.rb
|
261
262
|
- lib/google/cloud/bigquery/query_job.rb
|
263
|
+
- lib/google/cloud/bigquery/routine.rb
|
264
|
+
- lib/google/cloud/bigquery/routine/list.rb
|
262
265
|
- lib/google/cloud/bigquery/schema.rb
|
263
266
|
- lib/google/cloud/bigquery/schema/field.rb
|
264
267
|
- lib/google/cloud/bigquery/service.rb
|