google-cloud-bigquery 1.35.0 → 1.38.0
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/CHANGELOG.md +36 -0
- data/lib/google/cloud/bigquery/dataset.rb +130 -42
- data/lib/google/cloud/bigquery/external/avro_source.rb +107 -0
- data/lib/google/cloud/bigquery/external/data_source.rb +2 -2
- data/lib/google/cloud/bigquery/external.rb +16 -14
- data/lib/google/cloud/bigquery/job.rb +34 -2
- data/lib/google/cloud/bigquery/load_job.rb +13 -13
- data/lib/google/cloud/bigquery/model.rb +8 -8
- data/lib/google/cloud/bigquery/project.rb +114 -68
- data/lib/google/cloud/bigquery/query_job.rb +33 -0
- data/lib/google/cloud/bigquery/routine.rb +2 -2
- data/lib/google/cloud/bigquery/schema/field.rb +25 -16
- data/lib/google/cloud/bigquery/schema.rb +13 -13
- data/lib/google/cloud/bigquery/service.rb +8 -0
- data/lib/google/cloud/bigquery/table.rb +29 -29
- data/lib/google/cloud/bigquery/version.rb +1 -1
- metadata +3 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 134aa6aa80aff038d9ef64cb47875c8147051f47d7259dcc61199fb28f7d4e4e
|
4
|
+
data.tar.gz: '023664494e99ff2e580a10ebda7ef369b7fbe6290b9e448ee3eef697ecf94b8a'
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 7563dc9d4e0474c9b2c0bd3f06bef6c059568e24051221542c2536f4b8a0576310d8721eaa5338815ee6fbc5c71b1a782bde9b892c5eabbe532ec30309731e5f
|
7
|
+
data.tar.gz: fd6f178cd53ad95f8eb5310854909c726d796beb74232f7537212f949e1358423531f80f8bbfc3e7b8f1afa5b90b1ab5f83549dd1fa9d7a8fc1c9f5653a6712e
|
data/CHANGELOG.md
CHANGED
@@ -1,5 +1,41 @@
|
|
1
1
|
# Release History
|
2
2
|
|
3
|
+
### 1.38.0 / 2021-11-16
|
4
|
+
|
5
|
+
#### Features
|
6
|
+
|
7
|
+
* Add session support
|
8
|
+
* Add create_session and session_id params to Project#query_job
|
9
|
+
* Add create_session and session_id params to Dataset#query_job
|
10
|
+
* Add session_id param to Project#query
|
11
|
+
* Add session_id param to Dataset#query
|
12
|
+
* Add Job#session_id
|
13
|
+
* Add QueryJob::Updater#create_session=
|
14
|
+
* Add QueryJob::Updater#session_id=
|
15
|
+
|
16
|
+
### 1.37.0 / 2021-10-21
|
17
|
+
|
18
|
+
#### Features
|
19
|
+
|
20
|
+
* Add support for Avro options to external data sources
|
21
|
+
* Add External::AvroSource
|
22
|
+
|
23
|
+
### 1.36.0 / 2021-09-22
|
24
|
+
|
25
|
+
#### Features
|
26
|
+
|
27
|
+
* Add Job#delete
|
28
|
+
|
29
|
+
#### Bug Fixes
|
30
|
+
|
31
|
+
* Add precision and scale to Field#add_field
|
32
|
+
|
33
|
+
### 1.35.1 / 2021-09-14
|
34
|
+
|
35
|
+
#### Documentation
|
36
|
+
|
37
|
+
* Wrap character class regex in backticks
|
38
|
+
|
3
39
|
### 1.35.0 / 2021-08-12
|
4
40
|
|
5
41
|
#### Features
|
@@ -69,8 +69,8 @@ module Google
|
|
69
69
|
##
|
70
70
|
# A unique ID for this dataset, without the project name.
|
71
71
|
#
|
72
|
-
# @return [String] The ID must contain only letters (
|
73
|
-
# (0-9), or underscores (_). The maximum length is 1,024 characters.
|
72
|
+
# @return [String] The ID must contain only letters (`[A-Za-z]`), numbers
|
73
|
+
# (`[0-9]`), or underscores (`_`). The maximum length is 1,024 characters.
|
74
74
|
#
|
75
75
|
# @!group Attributes
|
76
76
|
#
|
@@ -501,7 +501,7 @@ module Google
|
|
501
501
|
# you can pass the table's schema as a hash (see example.)
|
502
502
|
#
|
503
503
|
# @param [String] table_id The ID of the table. The ID must contain only
|
504
|
-
# letters (
|
504
|
+
# letters (`[A-Za-z]`), numbers (`[0-9]`), or underscores (`_`). The maximum
|
505
505
|
# length is 1,024 characters.
|
506
506
|
# @param [String] name A descriptive name for the table.
|
507
507
|
# @param [String] description A user-friendly description of the table.
|
@@ -630,7 +630,7 @@ module Google
|
|
630
630
|
# @see https://cloud.google.com/bigquery/docs/views Creating views
|
631
631
|
#
|
632
632
|
# @param [String] table_id The ID of the view table. The ID must contain
|
633
|
-
# only letters (
|
633
|
+
# only letters (`[A-Za-z]`), numbers (`[0-9]`), or underscores (`_`). The
|
634
634
|
# maximum length is 1,024 characters.
|
635
635
|
# @param [String] query The query that BigQuery executes when the view
|
636
636
|
# is referenced.
|
@@ -726,8 +726,8 @@ module Google
|
|
726
726
|
#
|
727
727
|
# @see https://cloud.google.com/bigquery/docs/materialized-views-intro Introduction to materialized views
|
728
728
|
#
|
729
|
-
# @param [String] table_id The ID of the materialized view table. The ID must contain only letters (
|
730
|
-
# numbers (0-9), or underscores (_). The maximum length is 1,024 characters.
|
729
|
+
# @param [String] table_id The ID of the materialized view table. The ID must contain only letters (`[A-Za-z]`),
|
730
|
+
# numbers (`[0-9]`), or underscores (`_`). The maximum length is 1,024 characters.
|
731
731
|
# @param [String] query The query that BigQuery executes when the materialized view is referenced.
|
732
732
|
# @param [String] name A descriptive name for the table.
|
733
733
|
# @param [String] description A user-friendly description of the table.
|
@@ -954,7 +954,7 @@ module Google
|
|
954
954
|
# {Routine::Updater#description=}.
|
955
955
|
#
|
956
956
|
# @param [String] routine_id The ID of the routine. The ID must contain only
|
957
|
-
# letters (
|
957
|
+
# letters (`[A-Za-z]`), numbers (`[0-9]`), or underscores (`_`). The maximum length
|
958
958
|
# is 256 characters.
|
959
959
|
# @yield [routine] A block for setting properties on the routine.
|
960
960
|
# @yieldparam [Google::Cloud::Bigquery::Routine::Updater] routine An updater to set additional properties on the
|
@@ -1244,13 +1244,19 @@ module Google
|
|
1244
1244
|
# Flattens all nested and repeated fields in the query results. The
|
1245
1245
|
# default value is `true`. `large_results` parameter must be `true` if
|
1246
1246
|
# this is set to `false`.
|
1247
|
+
# @param [Integer] maximum_billing_tier Limits the billing tier for this
|
1248
|
+
# job. Queries that have resource usage beyond this tier will fail
|
1249
|
+
# (without incurring a charge). WARNING: The billed byte amount can be
|
1250
|
+
# multiplied by an amount up to this number! Most users should not need
|
1251
|
+
# to alter this setting, and we recommend that you avoid introducing new
|
1252
|
+
# uses of it. Deprecated.
|
1247
1253
|
# @param [Integer] maximum_bytes_billed Limits the bytes billed for this
|
1248
1254
|
# job. Queries that will have bytes billed beyond this limit will fail
|
1249
1255
|
# (without incurring a charge). Optional. If unspecified, this will be
|
1250
1256
|
# set to your project default.
|
1251
1257
|
# @param [String] job_id A user-defined ID for the query job. The ID
|
1252
|
-
# must contain only letters (
|
1253
|
-
# (_), or dashes (
|
1258
|
+
# must contain only letters (`[A-Za-z]`), numbers (`[0-9]`), underscores
|
1259
|
+
# (`_`), or dashes (`-`). The maximum length is 1,024 characters. If
|
1254
1260
|
# `job_id` is provided, then `prefix` will not be used.
|
1255
1261
|
#
|
1256
1262
|
# See [Generating a job
|
@@ -1259,8 +1265,8 @@ module Google
|
|
1259
1265
|
# prepended to a generated value to produce a unique job ID. For
|
1260
1266
|
# example, the prefix `daily_import_job_` can be given to generate a
|
1261
1267
|
# job ID such as `daily_import_job_12vEDtMQ0mbp1Mo5Z7mzAFQJZazh`. The
|
1262
|
-
# prefix must contain only letters (
|
1263
|
-
# underscores (_), or dashes (
|
1268
|
+
# prefix must contain only letters (`[A-Za-z]`), numbers (`[0-9]`),
|
1269
|
+
# underscores (`_`), or dashes (`-`). The maximum length of the entire ID
|
1264
1270
|
# is 1,024 characters. If `job_id` is provided, then `prefix` will not
|
1265
1271
|
# be used.
|
1266
1272
|
# @param [Hash] labels A hash of user-provided labels associated with
|
@@ -1294,8 +1300,13 @@ module Google
|
|
1294
1300
|
# For additional information on migrating, see: [Migrating to
|
1295
1301
|
# standard SQL - Differences in user-defined JavaScript
|
1296
1302
|
# functions](https://cloud.google.com/bigquery/docs/reference/standard-sql/migrating-from-legacy-sql#differences_in_user-defined_javascript_functions)
|
1297
|
-
# @param [
|
1298
|
-
#
|
1303
|
+
# @param [Boolean] create_session If true, creates a new session, where the
|
1304
|
+
# session ID will be a server generated random id. If false, runs query
|
1305
|
+
# with an existing session ID when one is provided in the `session_id`
|
1306
|
+
# param, otherwise runs query in non-session mode. See {Job#session_id}.
|
1307
|
+
# The default value is false.
|
1308
|
+
# @param [String] session_id The ID of an existing session. See also the
|
1309
|
+
# `create_session` param and {Job#session_id}.
|
1299
1310
|
# @yield [job] a job configuration object
|
1300
1311
|
# @yieldparam [Google::Cloud::Bigquery::QueryJob::Updater] job a job
|
1301
1312
|
# configuration object for setting additional options for the query.
|
@@ -1371,8 +1382,7 @@ module Google
|
|
1371
1382
|
# bigquery = Google::Cloud::Bigquery.new
|
1372
1383
|
# dataset = bigquery.dataset "my_dataset"
|
1373
1384
|
#
|
1374
|
-
# job = dataset.query_job "SELECT name FROM my_table "
|
1375
|
-
# "WHERE id IN UNNEST(@ids)",
|
1385
|
+
# job = dataset.query_job "SELECT name FROM my_table WHERE id IN UNNEST(@ids)",
|
1376
1386
|
# params: { ids: [] },
|
1377
1387
|
# types: { ids: [:INT64] }
|
1378
1388
|
#
|
@@ -1387,8 +1397,9 @@ module Google
|
|
1387
1397
|
# require "google/cloud/bigquery"
|
1388
1398
|
#
|
1389
1399
|
# bigquery = Google::Cloud::Bigquery.new
|
1400
|
+
# dataset = bigquery.dataset "my_dataset"
|
1390
1401
|
#
|
1391
|
-
# job =
|
1402
|
+
# job = dataset.query_job "CREATE TABLE my_table (x INT64)"
|
1392
1403
|
#
|
1393
1404
|
# job.wait_until_done!
|
1394
1405
|
# if !job.failed?
|
@@ -1399,16 +1410,28 @@ module Google
|
|
1399
1410
|
# require "google/cloud/bigquery"
|
1400
1411
|
#
|
1401
1412
|
# bigquery = Google::Cloud::Bigquery.new
|
1413
|
+
# dataset = bigquery.dataset "my_dataset"
|
1402
1414
|
#
|
1403
|
-
# job =
|
1404
|
-
# "SET x = x + 1 " \
|
1405
|
-
# "WHERE x IS NOT NULL"
|
1415
|
+
# job = dataset.query_job "UPDATE my_table SET x = x + 1 WHERE x IS NOT NULL"
|
1406
1416
|
#
|
1407
1417
|
# job.wait_until_done!
|
1408
1418
|
# if !job.failed?
|
1409
1419
|
# puts job.num_dml_affected_rows
|
1410
1420
|
# end
|
1411
1421
|
#
|
1422
|
+
# @example Run query in a session:
|
1423
|
+
# require "google/cloud/bigquery"
|
1424
|
+
#
|
1425
|
+
# bigquery = Google::Cloud::Bigquery.new
|
1426
|
+
# dataset = bigquery.dataset "my_dataset"
|
1427
|
+
#
|
1428
|
+
# job = dataset.query_job "CREATE TEMPORARY TABLE temptable AS SELECT 17 as foo", create_session: true
|
1429
|
+
#
|
1430
|
+
# job.wait_until_done!
|
1431
|
+
#
|
1432
|
+
# session_id = job.session_id
|
1433
|
+
# data = dataset.query "SELECT * FROM temptable", session_id: session_id
|
1434
|
+
#
|
1412
1435
|
# @example Query using external data source, set destination:
|
1413
1436
|
# require "google/cloud/bigquery"
|
1414
1437
|
#
|
@@ -1435,16 +1458,52 @@ module Google
|
|
1435
1458
|
#
|
1436
1459
|
# @!group Data
|
1437
1460
|
#
|
1438
|
-
def query_job query,
|
1439
|
-
|
1440
|
-
|
1441
|
-
|
1461
|
+
def query_job query,
|
1462
|
+
params: nil,
|
1463
|
+
types: nil,
|
1464
|
+
external: nil,
|
1465
|
+
priority: "INTERACTIVE",
|
1466
|
+
cache: true,
|
1467
|
+
table: nil,
|
1468
|
+
create: nil,
|
1469
|
+
write: nil,
|
1470
|
+
dryrun: nil,
|
1471
|
+
standard_sql: nil,
|
1472
|
+
legacy_sql: nil,
|
1473
|
+
large_results: nil,
|
1474
|
+
flatten: nil,
|
1475
|
+
maximum_billing_tier: nil,
|
1476
|
+
maximum_bytes_billed: nil,
|
1477
|
+
job_id: nil,
|
1478
|
+
prefix: nil,
|
1479
|
+
labels: nil,
|
1480
|
+
udfs: nil,
|
1481
|
+
create_session: nil,
|
1482
|
+
session_id: nil
|
1442
1483
|
ensure_service!
|
1443
|
-
options = {
|
1444
|
-
|
1445
|
-
|
1446
|
-
|
1447
|
-
|
1484
|
+
options = {
|
1485
|
+
params: params,
|
1486
|
+
types: types,
|
1487
|
+
external: external,
|
1488
|
+
priority: priority,
|
1489
|
+
cache: cache,
|
1490
|
+
table: table,
|
1491
|
+
create: create,
|
1492
|
+
write: write,
|
1493
|
+
dryrun: dryrun,
|
1494
|
+
standard_sql: standard_sql,
|
1495
|
+
legacy_sql: legacy_sql,
|
1496
|
+
large_results: large_results,
|
1497
|
+
flatten: flatten,
|
1498
|
+
maximum_billing_tier: maximum_billing_tier,
|
1499
|
+
maximum_bytes_billed: maximum_bytes_billed,
|
1500
|
+
job_id: job_id,
|
1501
|
+
prefix: prefix,
|
1502
|
+
labels: labels,
|
1503
|
+
udfs: udfs,
|
1504
|
+
create_session: create_session,
|
1505
|
+
session_id: session_id
|
1506
|
+
}
|
1448
1507
|
|
1449
1508
|
updater = QueryJob::Updater.from_options service, query, options
|
1450
1509
|
updater.dataset = self
|
@@ -1566,6 +1625,8 @@ module Google
|
|
1566
1625
|
# When set to false, the values of `large_results` and `flatten` are
|
1567
1626
|
# ignored; the query will be run as if `large_results` is true and
|
1568
1627
|
# `flatten` is false. Optional. The default value is false.
|
1628
|
+
# @param [String] session_id The ID of an existing session. See the
|
1629
|
+
# `create_session` param in {#query_job} and {Job#session_id}.
|
1569
1630
|
# @yield [job] a job configuration object
|
1570
1631
|
# @yieldparam [Google::Cloud::Bigquery::QueryJob::Updater] job a job
|
1571
1632
|
# configuration object for setting additional options for the query.
|
@@ -1641,8 +1702,7 @@ module Google
|
|
1641
1702
|
# bigquery = Google::Cloud::Bigquery.new
|
1642
1703
|
# dataset = bigquery.dataset "my_dataset"
|
1643
1704
|
#
|
1644
|
-
# data = dataset.query "SELECT name FROM my_table "
|
1645
|
-
# "WHERE id IN UNNEST(@ids)",
|
1705
|
+
# data = dataset.query "SELECT name FROM my_table WHERE id IN UNNEST(@ids)",
|
1646
1706
|
# params: { ids: [] },
|
1647
1707
|
# types: { ids: [:INT64] }
|
1648
1708
|
#
|
@@ -1657,8 +1717,9 @@ module Google
|
|
1657
1717
|
# require "google/cloud/bigquery"
|
1658
1718
|
#
|
1659
1719
|
# bigquery = Google::Cloud::Bigquery.new
|
1720
|
+
# dataset = bigquery.dataset "my_dataset"
|
1660
1721
|
#
|
1661
|
-
# data =
|
1722
|
+
# data = dataset.query "CREATE TABLE my_table (x INT64)"
|
1662
1723
|
#
|
1663
1724
|
# table_ref = data.ddl_target_table # Or ddl_target_routine for CREATE/DROP FUNCTION/PROCEDURE
|
1664
1725
|
#
|
@@ -1666,13 +1727,25 @@ module Google
|
|
1666
1727
|
# require "google/cloud/bigquery"
|
1667
1728
|
#
|
1668
1729
|
# bigquery = Google::Cloud::Bigquery.new
|
1730
|
+
# dataset = bigquery.dataset "my_dataset"
|
1669
1731
|
#
|
1670
|
-
# data =
|
1671
|
-
# "SET x = x + 1 " \
|
1672
|
-
# "WHERE x IS NOT NULL"
|
1732
|
+
# data = dataset.query "UPDATE my_table SET x = x + 1 WHERE x IS NOT NULL"
|
1673
1733
|
#
|
1674
1734
|
# puts data.num_dml_affected_rows
|
1675
1735
|
#
|
1736
|
+
# @example Run query in a session:
|
1737
|
+
# require "google/cloud/bigquery"
|
1738
|
+
#
|
1739
|
+
# bigquery = Google::Cloud::Bigquery.new
|
1740
|
+
# dataset = bigquery.dataset "my_dataset"
|
1741
|
+
#
|
1742
|
+
# job = dataset.query_job "CREATE TEMPORARY TABLE temptable AS SELECT 17 as foo", create_session: true
|
1743
|
+
#
|
1744
|
+
# job.wait_until_done!
|
1745
|
+
#
|
1746
|
+
# session_id = job.session_id
|
1747
|
+
# data = dataset.query "SELECT * FROM temptable", session_id: session_id
|
1748
|
+
#
|
1676
1749
|
# @example Query using external data source, set destination:
|
1677
1750
|
# require "google/cloud/bigquery"
|
1678
1751
|
#
|
@@ -1699,10 +1772,25 @@ module Google
|
|
1699
1772
|
#
|
1700
1773
|
# @!group Data
|
1701
1774
|
#
|
1702
|
-
def query query,
|
1703
|
-
|
1704
|
-
|
1705
|
-
|
1775
|
+
def query query,
|
1776
|
+
params: nil,
|
1777
|
+
types: nil,
|
1778
|
+
external: nil,
|
1779
|
+
max: nil,
|
1780
|
+
cache: true,
|
1781
|
+
standard_sql: nil,
|
1782
|
+
legacy_sql: nil,
|
1783
|
+
session_id: nil,
|
1784
|
+
&block
|
1785
|
+
job = query_job query,
|
1786
|
+
params: params,
|
1787
|
+
types: types,
|
1788
|
+
external: external,
|
1789
|
+
cache: cache,
|
1790
|
+
standard_sql: standard_sql,
|
1791
|
+
legacy_sql: legacy_sql,
|
1792
|
+
session_id: session_id,
|
1793
|
+
&block
|
1706
1794
|
job.wait_until_done!
|
1707
1795
|
ensure_job_succeeded! job
|
1708
1796
|
|
@@ -1883,8 +1971,8 @@ module Google
|
|
1883
1971
|
# this option. Also note that for most use cases, the block yielded by
|
1884
1972
|
# this method is a more convenient way to configure the schema.
|
1885
1973
|
# @param [String] job_id A user-defined ID for the load job. The ID
|
1886
|
-
# must contain only letters (
|
1887
|
-
# (_), or dashes (
|
1974
|
+
# must contain only letters (`[A-Za-z]`), numbers (`[0-9]`), underscores
|
1975
|
+
# (`_`), or dashes (`-`). The maximum length is 1,024 characters. If
|
1888
1976
|
# `job_id` is provided, then `prefix` will not be used.
|
1889
1977
|
#
|
1890
1978
|
# See [Generating a job
|
@@ -1893,8 +1981,8 @@ module Google
|
|
1893
1981
|
# prepended to a generated value to produce a unique job ID. For
|
1894
1982
|
# example, the prefix `daily_import_job_` can be given to generate a
|
1895
1983
|
# job ID such as `daily_import_job_12vEDtMQ0mbp1Mo5Z7mzAFQJZazh`. The
|
1896
|
-
# prefix must contain only letters (
|
1897
|
-
# underscores (_), or dashes (
|
1984
|
+
# prefix must contain only letters (`[A-Za-z]`), numbers (`[0-9]`),
|
1985
|
+
# underscores (`_`), or dashes (`-`). The maximum length of the entire ID
|
1898
1986
|
# is 1,024 characters. If `job_id` is provided, then `prefix` will not
|
1899
1987
|
# be used.
|
1900
1988
|
# @param [Hash] labels A hash of user-provided labels associated with
|
@@ -0,0 +1,107 @@
|
|
1
|
+
# Copyright 2021 Google LLC
|
2
|
+
#
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
4
|
+
# you may not use this file except in compliance with the License.
|
5
|
+
# You may obtain a copy of the License at
|
6
|
+
#
|
7
|
+
# https://www.apache.org/licenses/LICENSE-2.0
|
8
|
+
#
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
12
|
+
# See the License for the specific language governing permissions and
|
13
|
+
# limitations under the License.
|
14
|
+
|
15
|
+
|
16
|
+
require "google/apis/bigquery_v2"
|
17
|
+
|
18
|
+
module Google
|
19
|
+
module Cloud
|
20
|
+
module Bigquery
|
21
|
+
module External
|
22
|
+
##
|
23
|
+
# # AvroSource
|
24
|
+
#
|
25
|
+
# {External::AvroSource} is a subclass of {External::DataSource} and
|
26
|
+
# represents a Avro external data source that can be queried
|
27
|
+
# from directly, even though the data is not stored in BigQuery. Instead
|
28
|
+
# of loading or streaming the data, this object references the external
|
29
|
+
# data source.
|
30
|
+
#
|
31
|
+
# @example
|
32
|
+
# require "google/cloud/bigquery"
|
33
|
+
#
|
34
|
+
# bigquery = Google::Cloud::Bigquery.new
|
35
|
+
#
|
36
|
+
# avro_url = "gs://bucket/path/to/*.avro"
|
37
|
+
# avro_table = bigquery.external avro_url do |avro|
|
38
|
+
# avro.use_avro_logical_types = 1
|
39
|
+
# end
|
40
|
+
#
|
41
|
+
# data = bigquery.query "SELECT * FROM my_ext_table",
|
42
|
+
# external: { my_ext_table: avro_table }
|
43
|
+
#
|
44
|
+
# # Iterate over the first page of results
|
45
|
+
# data.each do |row|
|
46
|
+
# puts row[:name]
|
47
|
+
# end
|
48
|
+
# # Retrieve the next page of results
|
49
|
+
# data = data.next if data.next?
|
50
|
+
#
|
51
|
+
class AvroSource < External::DataSource
|
52
|
+
##
|
53
|
+
# @private Create an empty AvroSource object.
|
54
|
+
def initialize
|
55
|
+
super
|
56
|
+
@gapi.avro_options = Google::Apis::BigqueryV2::AvroOptions.new
|
57
|
+
end
|
58
|
+
|
59
|
+
##
|
60
|
+
# Indicates whether to interpret logical types as the corresponding BigQuery data type (for example,
|
61
|
+
# `TIMESTAMP`), instead of using the raw type (for example, `INTEGER`).
|
62
|
+
#
|
63
|
+
# @return [Boolean]
|
64
|
+
#
|
65
|
+
# @example
|
66
|
+
# require "google/cloud/bigquery"
|
67
|
+
#
|
68
|
+
# bigquery = Google::Cloud::Bigquery.new
|
69
|
+
#
|
70
|
+
# avro_url = "gs://bucket/path/to/*.avro"
|
71
|
+
# avro_table = bigquery.external avro_url do |avro|
|
72
|
+
# avro.use_avro_logical_types = true
|
73
|
+
# end
|
74
|
+
#
|
75
|
+
# avro_table.use_avro_logical_types #=> true
|
76
|
+
#
|
77
|
+
def use_avro_logical_types
|
78
|
+
@gapi.avro_options.use_avro_logical_types
|
79
|
+
end
|
80
|
+
|
81
|
+
##
|
82
|
+
# Sets whether to interpret logical types as the corresponding BigQuery data type (for example, `TIMESTAMP`),
|
83
|
+
# instead of using the raw type (for example, `INTEGER`).
|
84
|
+
#
|
85
|
+
# @param [Boolean] new_use_avro_logical_types The new `use_avro_logical_types` value.
|
86
|
+
#
|
87
|
+
# @example
|
88
|
+
# require "google/cloud/bigquery"
|
89
|
+
#
|
90
|
+
# bigquery = Google::Cloud::Bigquery.new
|
91
|
+
#
|
92
|
+
# avro_url = "gs://bucket/path/to/*.avro"
|
93
|
+
# avro_table = bigquery.external avro_url do |avro|
|
94
|
+
# avro.use_avro_logical_types = true
|
95
|
+
# end
|
96
|
+
#
|
97
|
+
# avro_table.use_avro_logical_types #=> true
|
98
|
+
#
|
99
|
+
def use_avro_logical_types= new_use_avro_logical_types
|
100
|
+
frozen_check!
|
101
|
+
@gapi.avro_options.use_avro_logical_types = new_use_avro_logical_types
|
102
|
+
end
|
103
|
+
end
|
104
|
+
end
|
105
|
+
end
|
106
|
+
end
|
107
|
+
end
|
@@ -37,7 +37,7 @@ module Google
|
|
37
37
|
#
|
38
38
|
# bigquery = Google::Cloud::Bigquery.new
|
39
39
|
#
|
40
|
-
# avro_url = "gs://bucket/path/to
|
40
|
+
# avro_url = "gs://bucket/path/to/*.avro"
|
41
41
|
# avro_table = bigquery.external avro_url do |avro|
|
42
42
|
# avro.autodetect = true
|
43
43
|
# end
|
@@ -174,7 +174,7 @@ module Google
|
|
174
174
|
#
|
175
175
|
# bigquery = Google::Cloud::Bigquery.new
|
176
176
|
#
|
177
|
-
# avro_url = "gs://bucket/path/to
|
177
|
+
# avro_url = "gs://bucket/path/to/*.avro"
|
178
178
|
# avro_table = bigquery.external avro_url
|
179
179
|
#
|
180
180
|
# avro_table.format #=> "AVRO"
|
@@ -14,6 +14,7 @@
|
|
14
14
|
|
15
15
|
|
16
16
|
require "google/cloud/bigquery/external/data_source"
|
17
|
+
require "google/cloud/bigquery/external/avro_source"
|
17
18
|
require "google/cloud/bigquery/external/bigtable_source"
|
18
19
|
require "google/cloud/bigquery/external/csv_source"
|
19
20
|
require "google/cloud/bigquery/external/json_source"
|
@@ -101,28 +102,28 @@ module Google
|
|
101
102
|
# @private Determine source_format from inputs
|
102
103
|
def self.source_format_for urls, format
|
103
104
|
val = {
|
104
|
-
"csv" => "CSV",
|
105
105
|
"avro" => "AVRO",
|
106
|
-
"
|
107
|
-
"
|
108
|
-
"sheets" => "GOOGLE_SHEETS",
|
109
|
-
"google_sheets" => "GOOGLE_SHEETS",
|
110
|
-
"datastore" => "DATASTORE_BACKUP",
|
106
|
+
"bigtable" => "BIGTABLE",
|
107
|
+
"csv" => "CSV",
|
111
108
|
"backup" => "DATASTORE_BACKUP",
|
109
|
+
"datastore" => "DATASTORE_BACKUP",
|
112
110
|
"datastore_backup" => "DATASTORE_BACKUP",
|
113
|
-
"
|
111
|
+
"sheets" => "GOOGLE_SHEETS",
|
112
|
+
"google_sheets" => "GOOGLE_SHEETS",
|
113
|
+
"json" => "NEWLINE_DELIMITED_JSON",
|
114
|
+
"newline_delimited_json" => "NEWLINE_DELIMITED_JSON",
|
114
115
|
"orc" => "ORC",
|
115
116
|
"parquet" => "PARQUET"
|
116
117
|
}[format.to_s.downcase]
|
117
118
|
return val unless val.nil?
|
118
119
|
Array(urls).each do |url|
|
119
|
-
return "CSV" if url.end_with? ".csv"
|
120
|
-
return "NEWLINE_DELIMITED_JSON" if url.end_with? ".json"
|
121
|
-
return "PARQUET" if url.end_with? ".parquet"
|
122
120
|
return "AVRO" if url.end_with? ".avro"
|
121
|
+
return "BIGTABLE" if url.start_with? "https://googleapis.com/bigtable/projects/"
|
122
|
+
return "CSV" if url.end_with? ".csv"
|
123
123
|
return "DATASTORE_BACKUP" if url.end_with? ".backup_info"
|
124
124
|
return "GOOGLE_SHEETS" if url.start_with? "https://docs.google.com/spreadsheets/"
|
125
|
-
return "
|
125
|
+
return "NEWLINE_DELIMITED_JSON" if url.end_with? ".json"
|
126
|
+
return "PARQUET" if url.end_with? ".parquet"
|
126
127
|
end
|
127
128
|
nil
|
128
129
|
end
|
@@ -131,13 +132,14 @@ module Google
|
|
131
132
|
# @private Determine table class from source_format
|
132
133
|
def self.table_class_for format
|
133
134
|
case format
|
135
|
+
when "AVRO" then External::AvroSource
|
136
|
+
when "BIGTABLE" then External::BigtableSource
|
134
137
|
when "CSV" then External::CsvSource
|
138
|
+
when "GOOGLE_SHEETS" then External::SheetsSource
|
135
139
|
when "NEWLINE_DELIMITED_JSON" then External::JsonSource
|
136
140
|
when "PARQUET" then External::ParquetSource
|
137
|
-
when "GOOGLE_SHEETS" then External::SheetsSource
|
138
|
-
when "BIGTABLE" then External::BigtableSource
|
139
141
|
else
|
140
|
-
#
|
142
|
+
# DATASTORE_BACKUP, ORC
|
141
143
|
External::DataSource
|
142
144
|
end
|
143
145
|
end
|
@@ -74,8 +74,8 @@ module Google
|
|
74
74
|
##
|
75
75
|
# The ID of the job.
|
76
76
|
#
|
77
|
-
# @return [String] The ID must contain only letters (
|
78
|
-
# (0-9), underscores (_), or dashes (
|
77
|
+
# @return [String] The ID must contain only letters (`[A-Za-z]`), numbers
|
78
|
+
# (`[0-9]`), underscores (`_`), or dashes (`-`). The maximum length is 1,024
|
79
79
|
# characters.
|
80
80
|
#
|
81
81
|
def job_id
|
@@ -226,6 +226,16 @@ module Google
|
|
226
226
|
Array(@gapi.statistics.reservation_usage).map { |g| ReservationUsage.from_gapi g }
|
227
227
|
end
|
228
228
|
|
229
|
+
##
|
230
|
+
# The ID of the session if this job is part of one. See the `create_session` param in {Project#query_job} and
|
231
|
+
# {Dataset#query_job}.
|
232
|
+
#
|
233
|
+
# @return [String, nil] The session ID, or `nil` if not associated with a session.
|
234
|
+
#
|
235
|
+
def session_id
|
236
|
+
@gapi.statistics.session_info&.session_id
|
237
|
+
end
|
238
|
+
|
229
239
|
##
|
230
240
|
# The ID of a multi-statement transaction.
|
231
241
|
#
|
@@ -392,6 +402,28 @@ module Google
|
|
392
402
|
true
|
393
403
|
end
|
394
404
|
|
405
|
+
##
|
406
|
+
# Requests that a job is deleted. This call will return when the job is deleted.
|
407
|
+
#
|
408
|
+
# @return [Boolean] Returns `true` if the job was deleted.
|
409
|
+
#
|
410
|
+
# @example
|
411
|
+
# require "google/cloud/bigquery"
|
412
|
+
#
|
413
|
+
# bigquery = Google::Cloud::Bigquery.new
|
414
|
+
#
|
415
|
+
# job = bigquery.job "my_job"
|
416
|
+
#
|
417
|
+
# job.delete
|
418
|
+
#
|
419
|
+
# @!group Lifecycle
|
420
|
+
#
|
421
|
+
def delete
|
422
|
+
ensure_service!
|
423
|
+
service.delete_job job_id, location: location
|
424
|
+
true
|
425
|
+
end
|
426
|
+
|
395
427
|
##
|
396
428
|
# Created a new job with the current configuration.
|
397
429
|
#
|