google-cloud-bigquery 1.35.0 → 1.38.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +36 -0
- data/lib/google/cloud/bigquery/dataset.rb +130 -42
- data/lib/google/cloud/bigquery/external/avro_source.rb +107 -0
- data/lib/google/cloud/bigquery/external/data_source.rb +2 -2
- data/lib/google/cloud/bigquery/external.rb +16 -14
- data/lib/google/cloud/bigquery/job.rb +34 -2
- data/lib/google/cloud/bigquery/load_job.rb +13 -13
- data/lib/google/cloud/bigquery/model.rb +8 -8
- data/lib/google/cloud/bigquery/project.rb +114 -68
- data/lib/google/cloud/bigquery/query_job.rb +33 -0
- data/lib/google/cloud/bigquery/routine.rb +2 -2
- data/lib/google/cloud/bigquery/schema/field.rb +25 -16
- data/lib/google/cloud/bigquery/schema.rb +13 -13
- data/lib/google/cloud/bigquery/service.rb +8 -0
- data/lib/google/cloud/bigquery/table.rb +29 -29
- data/lib/google/cloud/bigquery/version.rb +1 -1
- metadata +3 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA256:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 134aa6aa80aff038d9ef64cb47875c8147051f47d7259dcc61199fb28f7d4e4e
|
4
|
+
data.tar.gz: '023664494e99ff2e580a10ebda7ef369b7fbe6290b9e448ee3eef697ecf94b8a'
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: 7563dc9d4e0474c9b2c0bd3f06bef6c059568e24051221542c2536f4b8a0576310d8721eaa5338815ee6fbc5c71b1a782bde9b892c5eabbe532ec30309731e5f
|
7
|
+
data.tar.gz: fd6f178cd53ad95f8eb5310854909c726d796beb74232f7537212f949e1358423531f80f8bbfc3e7b8f1afa5b90b1ab5f83549dd1fa9d7a8fc1c9f5653a6712e
|
data/CHANGELOG.md
CHANGED
@@ -1,5 +1,41 @@
|
|
1
1
|
# Release History
|
2
2
|
|
3
|
+
### 1.38.0 / 2021-11-16
|
4
|
+
|
5
|
+
#### Features
|
6
|
+
|
7
|
+
* Add session support
|
8
|
+
* Add create_session and session_id params to Project#query_job
|
9
|
+
* Add create_session and session_id params to Dataset#query_job
|
10
|
+
* Add session_id param to Project#query
|
11
|
+
* Add session_id param to Dataset#query
|
12
|
+
* Add Job#session_id
|
13
|
+
* Add QueryJob::Updater#create_session=
|
14
|
+
* Add QueryJob::Updater#session_id=
|
15
|
+
|
16
|
+
### 1.37.0 / 2021-10-21
|
17
|
+
|
18
|
+
#### Features
|
19
|
+
|
20
|
+
* Add support for Avro options to external data sources
|
21
|
+
* Add External::AvroSource
|
22
|
+
|
23
|
+
### 1.36.0 / 2021-09-22
|
24
|
+
|
25
|
+
#### Features
|
26
|
+
|
27
|
+
* Add Job#delete
|
28
|
+
|
29
|
+
#### Bug Fixes
|
30
|
+
|
31
|
+
* Add precision and scale to Field#add_field
|
32
|
+
|
33
|
+
### 1.35.1 / 2021-09-14
|
34
|
+
|
35
|
+
#### Documentation
|
36
|
+
|
37
|
+
* Wrap character class regex in backticks
|
38
|
+
|
3
39
|
### 1.35.0 / 2021-08-12
|
4
40
|
|
5
41
|
#### Features
|
@@ -69,8 +69,8 @@ module Google
|
|
69
69
|
##
|
70
70
|
# A unique ID for this dataset, without the project name.
|
71
71
|
#
|
72
|
-
# @return [String] The ID must contain only letters (
|
73
|
-
# (0-9), or underscores (_). The maximum length is 1,024 characters.
|
72
|
+
# @return [String] The ID must contain only letters (`[A-Za-z]`), numbers
|
73
|
+
# (`[0-9]`), or underscores (`_`). The maximum length is 1,024 characters.
|
74
74
|
#
|
75
75
|
# @!group Attributes
|
76
76
|
#
|
@@ -501,7 +501,7 @@ module Google
|
|
501
501
|
# you can pass the table's schema as a hash (see example.)
|
502
502
|
#
|
503
503
|
# @param [String] table_id The ID of the table. The ID must contain only
|
504
|
-
# letters (
|
504
|
+
# letters (`[A-Za-z]`), numbers (`[0-9]`), or underscores (`_`). The maximum
|
505
505
|
# length is 1,024 characters.
|
506
506
|
# @param [String] name A descriptive name for the table.
|
507
507
|
# @param [String] description A user-friendly description of the table.
|
@@ -630,7 +630,7 @@ module Google
|
|
630
630
|
# @see https://cloud.google.com/bigquery/docs/views Creating views
|
631
631
|
#
|
632
632
|
# @param [String] table_id The ID of the view table. The ID must contain
|
633
|
-
# only letters (
|
633
|
+
# only letters (`[A-Za-z]`), numbers (`[0-9]`), or underscores (`_`). The
|
634
634
|
# maximum length is 1,024 characters.
|
635
635
|
# @param [String] query The query that BigQuery executes when the view
|
636
636
|
# is referenced.
|
@@ -726,8 +726,8 @@ module Google
|
|
726
726
|
#
|
727
727
|
# @see https://cloud.google.com/bigquery/docs/materialized-views-intro Introduction to materialized views
|
728
728
|
#
|
729
|
-
# @param [String] table_id The ID of the materialized view table. The ID must contain only letters (
|
730
|
-
# numbers (0-9), or underscores (_). The maximum length is 1,024 characters.
|
729
|
+
# @param [String] table_id The ID of the materialized view table. The ID must contain only letters (`[A-Za-z]`),
|
730
|
+
# numbers (`[0-9]`), or underscores (`_`). The maximum length is 1,024 characters.
|
731
731
|
# @param [String] query The query that BigQuery executes when the materialized view is referenced.
|
732
732
|
# @param [String] name A descriptive name for the table.
|
733
733
|
# @param [String] description A user-friendly description of the table.
|
@@ -954,7 +954,7 @@ module Google
|
|
954
954
|
# {Routine::Updater#description=}.
|
955
955
|
#
|
956
956
|
# @param [String] routine_id The ID of the routine. The ID must contain only
|
957
|
-
# letters (
|
957
|
+
# letters (`[A-Za-z]`), numbers (`[0-9]`), or underscores (`_`). The maximum length
|
958
958
|
# is 256 characters.
|
959
959
|
# @yield [routine] A block for setting properties on the routine.
|
960
960
|
# @yieldparam [Google::Cloud::Bigquery::Routine::Updater] routine An updater to set additional properties on the
|
@@ -1244,13 +1244,19 @@ module Google
|
|
1244
1244
|
# Flattens all nested and repeated fields in the query results. The
|
1245
1245
|
# default value is `true`. `large_results` parameter must be `true` if
|
1246
1246
|
# this is set to `false`.
|
1247
|
+
# @param [Integer] maximum_billing_tier Limits the billing tier for this
|
1248
|
+
# job. Queries that have resource usage beyond this tier will fail
|
1249
|
+
# (without incurring a charge). WARNING: The billed byte amount can be
|
1250
|
+
# multiplied by an amount up to this number! Most users should not need
|
1251
|
+
# to alter this setting, and we recommend that you avoid introducing new
|
1252
|
+
# uses of it. Deprecated.
|
1247
1253
|
# @param [Integer] maximum_bytes_billed Limits the bytes billed for this
|
1248
1254
|
# job. Queries that will have bytes billed beyond this limit will fail
|
1249
1255
|
# (without incurring a charge). Optional. If unspecified, this will be
|
1250
1256
|
# set to your project default.
|
1251
1257
|
# @param [String] job_id A user-defined ID for the query job. The ID
|
1252
|
-
# must contain only letters (
|
1253
|
-
# (_), or dashes (
|
1258
|
+
# must contain only letters (`[A-Za-z]`), numbers (`[0-9]`), underscores
|
1259
|
+
# (`_`), or dashes (`-`). The maximum length is 1,024 characters. If
|
1254
1260
|
# `job_id` is provided, then `prefix` will not be used.
|
1255
1261
|
#
|
1256
1262
|
# See [Generating a job
|
@@ -1259,8 +1265,8 @@ module Google
|
|
1259
1265
|
# prepended to a generated value to produce a unique job ID. For
|
1260
1266
|
# example, the prefix `daily_import_job_` can be given to generate a
|
1261
1267
|
# job ID such as `daily_import_job_12vEDtMQ0mbp1Mo5Z7mzAFQJZazh`. The
|
1262
|
-
# prefix must contain only letters (
|
1263
|
-
# underscores (_), or dashes (
|
1268
|
+
# prefix must contain only letters (`[A-Za-z]`), numbers (`[0-9]`),
|
1269
|
+
# underscores (`_`), or dashes (`-`). The maximum length of the entire ID
|
1264
1270
|
# is 1,024 characters. If `job_id` is provided, then `prefix` will not
|
1265
1271
|
# be used.
|
1266
1272
|
# @param [Hash] labels A hash of user-provided labels associated with
|
@@ -1294,8 +1300,13 @@ module Google
|
|
1294
1300
|
# For additional information on migrating, see: [Migrating to
|
1295
1301
|
# standard SQL - Differences in user-defined JavaScript
|
1296
1302
|
# functions](https://cloud.google.com/bigquery/docs/reference/standard-sql/migrating-from-legacy-sql#differences_in_user-defined_javascript_functions)
|
1297
|
-
# @param [
|
1298
|
-
#
|
1303
|
+
# @param [Boolean] create_session If true, creates a new session, where the
|
1304
|
+
# session ID will be a server generated random id. If false, runs query
|
1305
|
+
# with an existing session ID when one is provided in the `session_id`
|
1306
|
+
# param, otherwise runs query in non-session mode. See {Job#session_id}.
|
1307
|
+
# The default value is false.
|
1308
|
+
# @param [String] session_id The ID of an existing session. See also the
|
1309
|
+
# `create_session` param and {Job#session_id}.
|
1299
1310
|
# @yield [job] a job configuration object
|
1300
1311
|
# @yieldparam [Google::Cloud::Bigquery::QueryJob::Updater] job a job
|
1301
1312
|
# configuration object for setting additional options for the query.
|
@@ -1371,8 +1382,7 @@ module Google
|
|
1371
1382
|
# bigquery = Google::Cloud::Bigquery.new
|
1372
1383
|
# dataset = bigquery.dataset "my_dataset"
|
1373
1384
|
#
|
1374
|
-
# job = dataset.query_job "SELECT name FROM my_table "
|
1375
|
-
# "WHERE id IN UNNEST(@ids)",
|
1385
|
+
# job = dataset.query_job "SELECT name FROM my_table WHERE id IN UNNEST(@ids)",
|
1376
1386
|
# params: { ids: [] },
|
1377
1387
|
# types: { ids: [:INT64] }
|
1378
1388
|
#
|
@@ -1387,8 +1397,9 @@ module Google
|
|
1387
1397
|
# require "google/cloud/bigquery"
|
1388
1398
|
#
|
1389
1399
|
# bigquery = Google::Cloud::Bigquery.new
|
1400
|
+
# dataset = bigquery.dataset "my_dataset"
|
1390
1401
|
#
|
1391
|
-
# job =
|
1402
|
+
# job = dataset.query_job "CREATE TABLE my_table (x INT64)"
|
1392
1403
|
#
|
1393
1404
|
# job.wait_until_done!
|
1394
1405
|
# if !job.failed?
|
@@ -1399,16 +1410,28 @@ module Google
|
|
1399
1410
|
# require "google/cloud/bigquery"
|
1400
1411
|
#
|
1401
1412
|
# bigquery = Google::Cloud::Bigquery.new
|
1413
|
+
# dataset = bigquery.dataset "my_dataset"
|
1402
1414
|
#
|
1403
|
-
# job =
|
1404
|
-
# "SET x = x + 1 " \
|
1405
|
-
# "WHERE x IS NOT NULL"
|
1415
|
+
# job = dataset.query_job "UPDATE my_table SET x = x + 1 WHERE x IS NOT NULL"
|
1406
1416
|
#
|
1407
1417
|
# job.wait_until_done!
|
1408
1418
|
# if !job.failed?
|
1409
1419
|
# puts job.num_dml_affected_rows
|
1410
1420
|
# end
|
1411
1421
|
#
|
1422
|
+
# @example Run query in a session:
|
1423
|
+
# require "google/cloud/bigquery"
|
1424
|
+
#
|
1425
|
+
# bigquery = Google::Cloud::Bigquery.new
|
1426
|
+
# dataset = bigquery.dataset "my_dataset"
|
1427
|
+
#
|
1428
|
+
# job = dataset.query_job "CREATE TEMPORARY TABLE temptable AS SELECT 17 as foo", create_session: true
|
1429
|
+
#
|
1430
|
+
# job.wait_until_done!
|
1431
|
+
#
|
1432
|
+
# session_id = job.session_id
|
1433
|
+
# data = dataset.query "SELECT * FROM temptable", session_id: session_id
|
1434
|
+
#
|
1412
1435
|
# @example Query using external data source, set destination:
|
1413
1436
|
# require "google/cloud/bigquery"
|
1414
1437
|
#
|
@@ -1435,16 +1458,52 @@ module Google
|
|
1435
1458
|
#
|
1436
1459
|
# @!group Data
|
1437
1460
|
#
|
1438
|
-
def query_job query,
|
1439
|
-
|
1440
|
-
|
1441
|
-
|
1461
|
+
def query_job query,
|
1462
|
+
params: nil,
|
1463
|
+
types: nil,
|
1464
|
+
external: nil,
|
1465
|
+
priority: "INTERACTIVE",
|
1466
|
+
cache: true,
|
1467
|
+
table: nil,
|
1468
|
+
create: nil,
|
1469
|
+
write: nil,
|
1470
|
+
dryrun: nil,
|
1471
|
+
standard_sql: nil,
|
1472
|
+
legacy_sql: nil,
|
1473
|
+
large_results: nil,
|
1474
|
+
flatten: nil,
|
1475
|
+
maximum_billing_tier: nil,
|
1476
|
+
maximum_bytes_billed: nil,
|
1477
|
+
job_id: nil,
|
1478
|
+
prefix: nil,
|
1479
|
+
labels: nil,
|
1480
|
+
udfs: nil,
|
1481
|
+
create_session: nil,
|
1482
|
+
session_id: nil
|
1442
1483
|
ensure_service!
|
1443
|
-
options = {
|
1444
|
-
|
1445
|
-
|
1446
|
-
|
1447
|
-
|
1484
|
+
options = {
|
1485
|
+
params: params,
|
1486
|
+
types: types,
|
1487
|
+
external: external,
|
1488
|
+
priority: priority,
|
1489
|
+
cache: cache,
|
1490
|
+
table: table,
|
1491
|
+
create: create,
|
1492
|
+
write: write,
|
1493
|
+
dryrun: dryrun,
|
1494
|
+
standard_sql: standard_sql,
|
1495
|
+
legacy_sql: legacy_sql,
|
1496
|
+
large_results: large_results,
|
1497
|
+
flatten: flatten,
|
1498
|
+
maximum_billing_tier: maximum_billing_tier,
|
1499
|
+
maximum_bytes_billed: maximum_bytes_billed,
|
1500
|
+
job_id: job_id,
|
1501
|
+
prefix: prefix,
|
1502
|
+
labels: labels,
|
1503
|
+
udfs: udfs,
|
1504
|
+
create_session: create_session,
|
1505
|
+
session_id: session_id
|
1506
|
+
}
|
1448
1507
|
|
1449
1508
|
updater = QueryJob::Updater.from_options service, query, options
|
1450
1509
|
updater.dataset = self
|
@@ -1566,6 +1625,8 @@ module Google
|
|
1566
1625
|
# When set to false, the values of `large_results` and `flatten` are
|
1567
1626
|
# ignored; the query will be run as if `large_results` is true and
|
1568
1627
|
# `flatten` is false. Optional. The default value is false.
|
1628
|
+
# @param [String] session_id The ID of an existing session. See the
|
1629
|
+
# `create_session` param in {#query_job} and {Job#session_id}.
|
1569
1630
|
# @yield [job] a job configuration object
|
1570
1631
|
# @yieldparam [Google::Cloud::Bigquery::QueryJob::Updater] job a job
|
1571
1632
|
# configuration object for setting additional options for the query.
|
@@ -1641,8 +1702,7 @@ module Google
|
|
1641
1702
|
# bigquery = Google::Cloud::Bigquery.new
|
1642
1703
|
# dataset = bigquery.dataset "my_dataset"
|
1643
1704
|
#
|
1644
|
-
# data = dataset.query "SELECT name FROM my_table "
|
1645
|
-
# "WHERE id IN UNNEST(@ids)",
|
1705
|
+
# data = dataset.query "SELECT name FROM my_table WHERE id IN UNNEST(@ids)",
|
1646
1706
|
# params: { ids: [] },
|
1647
1707
|
# types: { ids: [:INT64] }
|
1648
1708
|
#
|
@@ -1657,8 +1717,9 @@ module Google
|
|
1657
1717
|
# require "google/cloud/bigquery"
|
1658
1718
|
#
|
1659
1719
|
# bigquery = Google::Cloud::Bigquery.new
|
1720
|
+
# dataset = bigquery.dataset "my_dataset"
|
1660
1721
|
#
|
1661
|
-
# data =
|
1722
|
+
# data = dataset.query "CREATE TABLE my_table (x INT64)"
|
1662
1723
|
#
|
1663
1724
|
# table_ref = data.ddl_target_table # Or ddl_target_routine for CREATE/DROP FUNCTION/PROCEDURE
|
1664
1725
|
#
|
@@ -1666,13 +1727,25 @@ module Google
|
|
1666
1727
|
# require "google/cloud/bigquery"
|
1667
1728
|
#
|
1668
1729
|
# bigquery = Google::Cloud::Bigquery.new
|
1730
|
+
# dataset = bigquery.dataset "my_dataset"
|
1669
1731
|
#
|
1670
|
-
# data =
|
1671
|
-
# "SET x = x + 1 " \
|
1672
|
-
# "WHERE x IS NOT NULL"
|
1732
|
+
# data = dataset.query "UPDATE my_table SET x = x + 1 WHERE x IS NOT NULL"
|
1673
1733
|
#
|
1674
1734
|
# puts data.num_dml_affected_rows
|
1675
1735
|
#
|
1736
|
+
# @example Run query in a session:
|
1737
|
+
# require "google/cloud/bigquery"
|
1738
|
+
#
|
1739
|
+
# bigquery = Google::Cloud::Bigquery.new
|
1740
|
+
# dataset = bigquery.dataset "my_dataset"
|
1741
|
+
#
|
1742
|
+
# job = dataset.query_job "CREATE TEMPORARY TABLE temptable AS SELECT 17 as foo", create_session: true
|
1743
|
+
#
|
1744
|
+
# job.wait_until_done!
|
1745
|
+
#
|
1746
|
+
# session_id = job.session_id
|
1747
|
+
# data = dataset.query "SELECT * FROM temptable", session_id: session_id
|
1748
|
+
#
|
1676
1749
|
# @example Query using external data source, set destination:
|
1677
1750
|
# require "google/cloud/bigquery"
|
1678
1751
|
#
|
@@ -1699,10 +1772,25 @@ module Google
|
|
1699
1772
|
#
|
1700
1773
|
# @!group Data
|
1701
1774
|
#
|
1702
|
-
def query query,
|
1703
|
-
|
1704
|
-
|
1705
|
-
|
1775
|
+
def query query,
|
1776
|
+
params: nil,
|
1777
|
+
types: nil,
|
1778
|
+
external: nil,
|
1779
|
+
max: nil,
|
1780
|
+
cache: true,
|
1781
|
+
standard_sql: nil,
|
1782
|
+
legacy_sql: nil,
|
1783
|
+
session_id: nil,
|
1784
|
+
&block
|
1785
|
+
job = query_job query,
|
1786
|
+
params: params,
|
1787
|
+
types: types,
|
1788
|
+
external: external,
|
1789
|
+
cache: cache,
|
1790
|
+
standard_sql: standard_sql,
|
1791
|
+
legacy_sql: legacy_sql,
|
1792
|
+
session_id: session_id,
|
1793
|
+
&block
|
1706
1794
|
job.wait_until_done!
|
1707
1795
|
ensure_job_succeeded! job
|
1708
1796
|
|
@@ -1883,8 +1971,8 @@ module Google
|
|
1883
1971
|
# this option. Also note that for most use cases, the block yielded by
|
1884
1972
|
# this method is a more convenient way to configure the schema.
|
1885
1973
|
# @param [String] job_id A user-defined ID for the load job. The ID
|
1886
|
-
# must contain only letters (
|
1887
|
-
# (_), or dashes (
|
1974
|
+
# must contain only letters (`[A-Za-z]`), numbers (`[0-9]`), underscores
|
1975
|
+
# (`_`), or dashes (`-`). The maximum length is 1,024 characters. If
|
1888
1976
|
# `job_id` is provided, then `prefix` will not be used.
|
1889
1977
|
#
|
1890
1978
|
# See [Generating a job
|
@@ -1893,8 +1981,8 @@ module Google
|
|
1893
1981
|
# prepended to a generated value to produce a unique job ID. For
|
1894
1982
|
# example, the prefix `daily_import_job_` can be given to generate a
|
1895
1983
|
# job ID such as `daily_import_job_12vEDtMQ0mbp1Mo5Z7mzAFQJZazh`. The
|
1896
|
-
# prefix must contain only letters (
|
1897
|
-
# underscores (_), or dashes (
|
1984
|
+
# prefix must contain only letters (`[A-Za-z]`), numbers (`[0-9]`),
|
1985
|
+
# underscores (`_`), or dashes (`-`). The maximum length of the entire ID
|
1898
1986
|
# is 1,024 characters. If `job_id` is provided, then `prefix` will not
|
1899
1987
|
# be used.
|
1900
1988
|
# @param [Hash] labels A hash of user-provided labels associated with
|
@@ -0,0 +1,107 @@
|
|
1
|
+
# Copyright 2021 Google LLC
|
2
|
+
#
|
3
|
+
# Licensed under the Apache License, Version 2.0 (the "License");
|
4
|
+
# you may not use this file except in compliance with the License.
|
5
|
+
# You may obtain a copy of the License at
|
6
|
+
#
|
7
|
+
# https://www.apache.org/licenses/LICENSE-2.0
|
8
|
+
#
|
9
|
+
# Unless required by applicable law or agreed to in writing, software
|
10
|
+
# distributed under the License is distributed on an "AS IS" BASIS,
|
11
|
+
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
12
|
+
# See the License for the specific language governing permissions and
|
13
|
+
# limitations under the License.
|
14
|
+
|
15
|
+
|
16
|
+
require "google/apis/bigquery_v2"
|
17
|
+
|
18
|
+
module Google
|
19
|
+
module Cloud
|
20
|
+
module Bigquery
|
21
|
+
module External
|
22
|
+
##
|
23
|
+
# # AvroSource
|
24
|
+
#
|
25
|
+
# {External::AvroSource} is a subclass of {External::DataSource} and
|
26
|
+
# represents a Avro external data source that can be queried
|
27
|
+
# from directly, even though the data is not stored in BigQuery. Instead
|
28
|
+
# of loading or streaming the data, this object references the external
|
29
|
+
# data source.
|
30
|
+
#
|
31
|
+
# @example
|
32
|
+
# require "google/cloud/bigquery"
|
33
|
+
#
|
34
|
+
# bigquery = Google::Cloud::Bigquery.new
|
35
|
+
#
|
36
|
+
# avro_url = "gs://bucket/path/to/*.avro"
|
37
|
+
# avro_table = bigquery.external avro_url do |avro|
|
38
|
+
# avro.use_avro_logical_types = 1
|
39
|
+
# end
|
40
|
+
#
|
41
|
+
# data = bigquery.query "SELECT * FROM my_ext_table",
|
42
|
+
# external: { my_ext_table: avro_table }
|
43
|
+
#
|
44
|
+
# # Iterate over the first page of results
|
45
|
+
# data.each do |row|
|
46
|
+
# puts row[:name]
|
47
|
+
# end
|
48
|
+
# # Retrieve the next page of results
|
49
|
+
# data = data.next if data.next?
|
50
|
+
#
|
51
|
+
class AvroSource < External::DataSource
|
52
|
+
##
|
53
|
+
# @private Create an empty AvroSource object.
|
54
|
+
def initialize
|
55
|
+
super
|
56
|
+
@gapi.avro_options = Google::Apis::BigqueryV2::AvroOptions.new
|
57
|
+
end
|
58
|
+
|
59
|
+
##
|
60
|
+
# Indicates whether to interpret logical types as the corresponding BigQuery data type (for example,
|
61
|
+
# `TIMESTAMP`), instead of using the raw type (for example, `INTEGER`).
|
62
|
+
#
|
63
|
+
# @return [Boolean]
|
64
|
+
#
|
65
|
+
# @example
|
66
|
+
# require "google/cloud/bigquery"
|
67
|
+
#
|
68
|
+
# bigquery = Google::Cloud::Bigquery.new
|
69
|
+
#
|
70
|
+
# avro_url = "gs://bucket/path/to/*.avro"
|
71
|
+
# avro_table = bigquery.external avro_url do |avro|
|
72
|
+
# avro.use_avro_logical_types = true
|
73
|
+
# end
|
74
|
+
#
|
75
|
+
# avro_table.use_avro_logical_types #=> true
|
76
|
+
#
|
77
|
+
def use_avro_logical_types
|
78
|
+
@gapi.avro_options.use_avro_logical_types
|
79
|
+
end
|
80
|
+
|
81
|
+
##
|
82
|
+
# Sets whether to interpret logical types as the corresponding BigQuery data type (for example, `TIMESTAMP`),
|
83
|
+
# instead of using the raw type (for example, `INTEGER`).
|
84
|
+
#
|
85
|
+
# @param [Boolean] new_use_avro_logical_types The new `use_avro_logical_types` value.
|
86
|
+
#
|
87
|
+
# @example
|
88
|
+
# require "google/cloud/bigquery"
|
89
|
+
#
|
90
|
+
# bigquery = Google::Cloud::Bigquery.new
|
91
|
+
#
|
92
|
+
# avro_url = "gs://bucket/path/to/*.avro"
|
93
|
+
# avro_table = bigquery.external avro_url do |avro|
|
94
|
+
# avro.use_avro_logical_types = true
|
95
|
+
# end
|
96
|
+
#
|
97
|
+
# avro_table.use_avro_logical_types #=> true
|
98
|
+
#
|
99
|
+
def use_avro_logical_types= new_use_avro_logical_types
|
100
|
+
frozen_check!
|
101
|
+
@gapi.avro_options.use_avro_logical_types = new_use_avro_logical_types
|
102
|
+
end
|
103
|
+
end
|
104
|
+
end
|
105
|
+
end
|
106
|
+
end
|
107
|
+
end
|
@@ -37,7 +37,7 @@ module Google
|
|
37
37
|
#
|
38
38
|
# bigquery = Google::Cloud::Bigquery.new
|
39
39
|
#
|
40
|
-
# avro_url = "gs://bucket/path/to
|
40
|
+
# avro_url = "gs://bucket/path/to/*.avro"
|
41
41
|
# avro_table = bigquery.external avro_url do |avro|
|
42
42
|
# avro.autodetect = true
|
43
43
|
# end
|
@@ -174,7 +174,7 @@ module Google
|
|
174
174
|
#
|
175
175
|
# bigquery = Google::Cloud::Bigquery.new
|
176
176
|
#
|
177
|
-
# avro_url = "gs://bucket/path/to
|
177
|
+
# avro_url = "gs://bucket/path/to/*.avro"
|
178
178
|
# avro_table = bigquery.external avro_url
|
179
179
|
#
|
180
180
|
# avro_table.format #=> "AVRO"
|
@@ -14,6 +14,7 @@
|
|
14
14
|
|
15
15
|
|
16
16
|
require "google/cloud/bigquery/external/data_source"
|
17
|
+
require "google/cloud/bigquery/external/avro_source"
|
17
18
|
require "google/cloud/bigquery/external/bigtable_source"
|
18
19
|
require "google/cloud/bigquery/external/csv_source"
|
19
20
|
require "google/cloud/bigquery/external/json_source"
|
@@ -101,28 +102,28 @@ module Google
|
|
101
102
|
# @private Determine source_format from inputs
|
102
103
|
def self.source_format_for urls, format
|
103
104
|
val = {
|
104
|
-
"csv" => "CSV",
|
105
105
|
"avro" => "AVRO",
|
106
|
-
"
|
107
|
-
"
|
108
|
-
"sheets" => "GOOGLE_SHEETS",
|
109
|
-
"google_sheets" => "GOOGLE_SHEETS",
|
110
|
-
"datastore" => "DATASTORE_BACKUP",
|
106
|
+
"bigtable" => "BIGTABLE",
|
107
|
+
"csv" => "CSV",
|
111
108
|
"backup" => "DATASTORE_BACKUP",
|
109
|
+
"datastore" => "DATASTORE_BACKUP",
|
112
110
|
"datastore_backup" => "DATASTORE_BACKUP",
|
113
|
-
"
|
111
|
+
"sheets" => "GOOGLE_SHEETS",
|
112
|
+
"google_sheets" => "GOOGLE_SHEETS",
|
113
|
+
"json" => "NEWLINE_DELIMITED_JSON",
|
114
|
+
"newline_delimited_json" => "NEWLINE_DELIMITED_JSON",
|
114
115
|
"orc" => "ORC",
|
115
116
|
"parquet" => "PARQUET"
|
116
117
|
}[format.to_s.downcase]
|
117
118
|
return val unless val.nil?
|
118
119
|
Array(urls).each do |url|
|
119
|
-
return "CSV" if url.end_with? ".csv"
|
120
|
-
return "NEWLINE_DELIMITED_JSON" if url.end_with? ".json"
|
121
|
-
return "PARQUET" if url.end_with? ".parquet"
|
122
120
|
return "AVRO" if url.end_with? ".avro"
|
121
|
+
return "BIGTABLE" if url.start_with? "https://googleapis.com/bigtable/projects/"
|
122
|
+
return "CSV" if url.end_with? ".csv"
|
123
123
|
return "DATASTORE_BACKUP" if url.end_with? ".backup_info"
|
124
124
|
return "GOOGLE_SHEETS" if url.start_with? "https://docs.google.com/spreadsheets/"
|
125
|
-
return "
|
125
|
+
return "NEWLINE_DELIMITED_JSON" if url.end_with? ".json"
|
126
|
+
return "PARQUET" if url.end_with? ".parquet"
|
126
127
|
end
|
127
128
|
nil
|
128
129
|
end
|
@@ -131,13 +132,14 @@ module Google
|
|
131
132
|
# @private Determine table class from source_format
|
132
133
|
def self.table_class_for format
|
133
134
|
case format
|
135
|
+
when "AVRO" then External::AvroSource
|
136
|
+
when "BIGTABLE" then External::BigtableSource
|
134
137
|
when "CSV" then External::CsvSource
|
138
|
+
when "GOOGLE_SHEETS" then External::SheetsSource
|
135
139
|
when "NEWLINE_DELIMITED_JSON" then External::JsonSource
|
136
140
|
when "PARQUET" then External::ParquetSource
|
137
|
-
when "GOOGLE_SHEETS" then External::SheetsSource
|
138
|
-
when "BIGTABLE" then External::BigtableSource
|
139
141
|
else
|
140
|
-
#
|
142
|
+
# DATASTORE_BACKUP, ORC
|
141
143
|
External::DataSource
|
142
144
|
end
|
143
145
|
end
|
@@ -74,8 +74,8 @@ module Google
|
|
74
74
|
##
|
75
75
|
# The ID of the job.
|
76
76
|
#
|
77
|
-
# @return [String] The ID must contain only letters (
|
78
|
-
# (0-9), underscores (_), or dashes (
|
77
|
+
# @return [String] The ID must contain only letters (`[A-Za-z]`), numbers
|
78
|
+
# (`[0-9]`), underscores (`_`), or dashes (`-`). The maximum length is 1,024
|
79
79
|
# characters.
|
80
80
|
#
|
81
81
|
def job_id
|
@@ -226,6 +226,16 @@ module Google
|
|
226
226
|
Array(@gapi.statistics.reservation_usage).map { |g| ReservationUsage.from_gapi g }
|
227
227
|
end
|
228
228
|
|
229
|
+
##
|
230
|
+
# The ID of the session if this job is part of one. See the `create_session` param in {Project#query_job} and
|
231
|
+
# {Dataset#query_job}.
|
232
|
+
#
|
233
|
+
# @return [String, nil] The session ID, or `nil` if not associated with a session.
|
234
|
+
#
|
235
|
+
def session_id
|
236
|
+
@gapi.statistics.session_info&.session_id
|
237
|
+
end
|
238
|
+
|
229
239
|
##
|
230
240
|
# The ID of a multi-statement transaction.
|
231
241
|
#
|
@@ -392,6 +402,28 @@ module Google
|
|
392
402
|
true
|
393
403
|
end
|
394
404
|
|
405
|
+
##
|
406
|
+
# Requests that a job is deleted. This call will return when the job is deleted.
|
407
|
+
#
|
408
|
+
# @return [Boolean] Returns `true` if the job was deleted.
|
409
|
+
#
|
410
|
+
# @example
|
411
|
+
# require "google/cloud/bigquery"
|
412
|
+
#
|
413
|
+
# bigquery = Google::Cloud::Bigquery.new
|
414
|
+
#
|
415
|
+
# job = bigquery.job "my_job"
|
416
|
+
#
|
417
|
+
# job.delete
|
418
|
+
#
|
419
|
+
# @!group Lifecycle
|
420
|
+
#
|
421
|
+
def delete
|
422
|
+
ensure_service!
|
423
|
+
service.delete_job job_id, location: location
|
424
|
+
true
|
425
|
+
end
|
426
|
+
|
395
427
|
##
|
396
428
|
# Created a new job with the current configuration.
|
397
429
|
#
|