google-cloud-bigquery 1.0.0 → 1.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/lib/google-cloud-bigquery.rb +25 -0
- data/lib/google/cloud/bigquery.rb +61 -11
- data/lib/google/cloud/bigquery/convert.rb +1 -1
- data/lib/google/cloud/bigquery/credentials.rb +6 -6
- data/lib/google/cloud/bigquery/data.rb +6 -6
- data/lib/google/cloud/bigquery/dataset.rb +16 -15
- data/lib/google/cloud/bigquery/dataset/access.rb +38 -30
- data/lib/google/cloud/bigquery/dataset/list.rb +1 -1
- data/lib/google/cloud/bigquery/external.rb +22 -20
- data/lib/google/cloud/bigquery/insert_response.rb +0 -2
- data/lib/google/cloud/bigquery/job.rb +39 -31
- data/lib/google/cloud/bigquery/job/list.rb +1 -1
- data/lib/google/cloud/bigquery/load_job.rb +4 -4
- data/lib/google/cloud/bigquery/project.rb +7 -15
- data/lib/google/cloud/bigquery/project/list.rb +1 -1
- data/lib/google/cloud/bigquery/query_job.rb +12 -12
- data/lib/google/cloud/bigquery/schema.rb +7 -7
- data/lib/google/cloud/bigquery/schema/field.rb +12 -12
- data/lib/google/cloud/bigquery/service.rb +44 -29
- data/lib/google/cloud/bigquery/table.rb +78 -21
- data/lib/google/cloud/bigquery/table/async_inserter.rb +42 -17
- data/lib/google/cloud/bigquery/table/list.rb +1 -1
- data/lib/google/cloud/bigquery/version.rb +1 -1
- metadata +11 -11
@@ -228,14 +228,20 @@ module Google
|
|
228
228
|
def push_batch_request!
|
229
229
|
return unless @batch
|
230
230
|
|
231
|
-
|
231
|
+
orig_rows = @batch.rows
|
232
|
+
json_rows = @batch.json_rows
|
232
233
|
Concurrent::Future.new(executor: @thread_pool) do
|
233
234
|
begin
|
234
|
-
|
235
|
-
|
236
|
-
|
237
|
-
|
238
|
-
|
235
|
+
raise ArgumentError, "No rows provided" if json_rows.empty?
|
236
|
+
options = { skip_invalid: @skip_invalid,
|
237
|
+
ignore_unknown: @ignore_unknown }
|
238
|
+
insert_resp = @table.service.insert_tabledata_json_rows(
|
239
|
+
@table.dataset_id, @table.table_id, json_rows, options
|
240
|
+
)
|
241
|
+
result = Result.new(
|
242
|
+
InsertResponse.from_gapi(orig_rows, insert_resp)
|
243
|
+
)
|
244
|
+
rescue StandardError => e
|
239
245
|
result = Result.new nil, e
|
240
246
|
ensure
|
241
247
|
@callback.call result if @callback
|
@@ -249,34 +255,55 @@ module Google
|
|
249
255
|
##
|
250
256
|
# @private
|
251
257
|
class Batch
|
252
|
-
attr_reader :max_bytes, :max_rows, :rows
|
258
|
+
attr_reader :max_bytes, :max_rows, :rows, :json_rows
|
253
259
|
|
254
260
|
def initialize max_bytes: 10000000, max_rows: 500
|
255
261
|
@max_bytes = max_bytes
|
256
262
|
@max_rows = max_rows
|
257
263
|
@rows = []
|
264
|
+
@json_rows = []
|
265
|
+
@current_bytes = 0
|
258
266
|
end
|
259
267
|
|
260
268
|
def insert row
|
261
|
-
|
269
|
+
json_row = to_json_row row
|
270
|
+
|
271
|
+
insert_rows_bytes row, json_row, addl_bytes_for_json_row(json_row)
|
262
272
|
end
|
263
273
|
|
264
274
|
def try_insert row
|
265
|
-
|
266
|
-
|
275
|
+
json_row = to_json_row row
|
276
|
+
addl_bytes = addl_bytes_for_json_row json_row
|
277
|
+
|
278
|
+
return false if @current_bytes + addl_bytes >= @max_bytes
|
267
279
|
return false if @rows.count + 1 >= @max_rows
|
268
280
|
|
269
|
-
|
281
|
+
insert_rows_bytes row, json_row, addl_bytes
|
270
282
|
true
|
271
283
|
end
|
272
284
|
|
273
285
|
def ready?
|
274
|
-
current_bytes >= @max_bytes || rows.count >= @max_rows
|
286
|
+
@current_bytes >= @max_bytes || rows.count >= @max_rows
|
287
|
+
end
|
288
|
+
|
289
|
+
private
|
290
|
+
|
291
|
+
def insert_rows_bytes row, json_row, addl_bytes
|
292
|
+
@rows << row
|
293
|
+
@json_rows << json_row
|
294
|
+
@current_bytes += addl_bytes
|
275
295
|
end
|
276
296
|
|
277
|
-
def
|
278
|
-
|
279
|
-
|
297
|
+
def to_json_row row
|
298
|
+
Convert.to_json_row row
|
299
|
+
end
|
300
|
+
|
301
|
+
def addl_bytes_for row
|
302
|
+
addl_bytes_for_json_row Convert.to_json_row(row)
|
303
|
+
end
|
304
|
+
|
305
|
+
def addl_bytes_for_json_row json_row
|
306
|
+
json_row.to_json.bytesize + 1
|
280
307
|
end
|
281
308
|
end
|
282
309
|
|
@@ -357,7 +384,6 @@ module Google
|
|
357
384
|
insert_response.success?
|
358
385
|
end
|
359
386
|
|
360
|
-
|
361
387
|
##
|
362
388
|
# The count of rows in the response, minus the count of errors for
|
363
389
|
# rows that were not inserted.
|
@@ -370,7 +396,6 @@ module Google
|
|
370
396
|
insert_response.insert_count
|
371
397
|
end
|
372
398
|
|
373
|
-
|
374
399
|
##
|
375
400
|
# The count of errors for rows that were not inserted.
|
376
401
|
#
|
metadata
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: google-cloud-bigquery
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 1.
|
4
|
+
version: 1.1.0
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- Mike Moore
|
@@ -9,7 +9,7 @@ authors:
|
|
9
9
|
autorequire:
|
10
10
|
bindir: bin
|
11
11
|
cert_chain: []
|
12
|
-
date: 2018-
|
12
|
+
date: 2018-02-27 00:00:00.000000000 Z
|
13
13
|
dependencies:
|
14
14
|
- !ruby/object:Gem::Dependency
|
15
15
|
name: google-cloud-core
|
@@ -17,28 +17,28 @@ dependencies:
|
|
17
17
|
requirements:
|
18
18
|
- - "~>"
|
19
19
|
- !ruby/object:Gem::Version
|
20
|
-
version: '1.
|
20
|
+
version: '1.2'
|
21
21
|
type: :runtime
|
22
22
|
prerelease: false
|
23
23
|
version_requirements: !ruby/object:Gem::Requirement
|
24
24
|
requirements:
|
25
25
|
- - "~>"
|
26
26
|
- !ruby/object:Gem::Version
|
27
|
-
version: '1.
|
27
|
+
version: '1.2'
|
28
28
|
- !ruby/object:Gem::Dependency
|
29
29
|
name: google-api-client
|
30
30
|
requirement: !ruby/object:Gem::Requirement
|
31
31
|
requirements:
|
32
32
|
- - "~>"
|
33
33
|
- !ruby/object:Gem::Version
|
34
|
-
version: 0.
|
34
|
+
version: 0.19.8
|
35
35
|
type: :runtime
|
36
36
|
prerelease: false
|
37
37
|
version_requirements: !ruby/object:Gem::Requirement
|
38
38
|
requirements:
|
39
39
|
- - "~>"
|
40
40
|
- !ruby/object:Gem::Version
|
41
|
-
version: 0.
|
41
|
+
version: 0.19.8
|
42
42
|
- !ruby/object:Gem::Dependency
|
43
43
|
name: googleauth
|
44
44
|
requirement: !ruby/object:Gem::Requirement
|
@@ -141,16 +141,16 @@ dependencies:
|
|
141
141
|
name: rubocop
|
142
142
|
requirement: !ruby/object:Gem::Requirement
|
143
143
|
requirements:
|
144
|
-
- - "
|
144
|
+
- - "~>"
|
145
145
|
- !ruby/object:Gem::Version
|
146
|
-
version: 0.
|
146
|
+
version: 0.50.0
|
147
147
|
type: :development
|
148
148
|
prerelease: false
|
149
149
|
version_requirements: !ruby/object:Gem::Requirement
|
150
150
|
requirements:
|
151
|
-
- - "
|
151
|
+
- - "~>"
|
152
152
|
- !ruby/object:Gem::Version
|
153
|
-
version: 0.
|
153
|
+
version: 0.50.0
|
154
154
|
- !ruby/object:Gem::Dependency
|
155
155
|
name: simplecov
|
156
156
|
requirement: !ruby/object:Gem::Requirement
|
@@ -250,7 +250,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
|
|
250
250
|
version: '0'
|
251
251
|
requirements: []
|
252
252
|
rubyforge_project:
|
253
|
-
rubygems_version: 2.7.
|
253
|
+
rubygems_version: 2.7.6
|
254
254
|
signing_key:
|
255
255
|
specification_version: 4
|
256
256
|
summary: API Client library for Google BigQuery
|