google-cloud-bigquery 1.1.0 → 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/README.md +27 -19
- data/lib/google/cloud/bigquery/convert.rb +106 -13
- data/lib/google/cloud/bigquery/copy_job.rb +168 -0
- data/lib/google/cloud/bigquery/dataset.rb +313 -112
- data/lib/google/cloud/bigquery/encryption_configuration.rb +124 -0
- data/lib/google/cloud/bigquery/extract_job.rb +149 -0
- data/lib/google/cloud/bigquery/job.rb +14 -3
- data/lib/google/cloud/bigquery/load_job.rb +770 -0
- data/lib/google/cloud/bigquery/project.rb +118 -20
- data/lib/google/cloud/bigquery/query_job.rb +403 -2
- data/lib/google/cloud/bigquery/schema/field.rb +22 -22
- data/lib/google/cloud/bigquery/service.rb +45 -328
- data/lib/google/cloud/bigquery/table.rb +375 -95
- data/lib/google/cloud/bigquery/version.rb +1 -1
- metadata +3 -2
@@ -18,6 +18,7 @@ require "google/cloud/bigquery/service"
|
|
18
18
|
require "google/cloud/bigquery/data"
|
19
19
|
require "google/cloud/bigquery/table/list"
|
20
20
|
require "google/cloud/bigquery/schema"
|
21
|
+
require "google/cloud/bigquery/encryption_configuration"
|
21
22
|
require "google/cloud/bigquery/external"
|
22
23
|
require "google/cloud/bigquery/insert_response"
|
23
24
|
require "google/cloud/bigquery/table/async_inserter"
|
@@ -782,6 +783,53 @@ module Google
|
|
782
783
|
schema.headers
|
783
784
|
end
|
784
785
|
|
786
|
+
##
|
787
|
+
# The {EncryptionConfiguration} object that represents the custom
|
788
|
+
# encryption method used to protect the table. If not set, default
|
789
|
+
# encryption is used.
|
790
|
+
#
|
791
|
+
# Present only if the table is using custom encryption.
|
792
|
+
#
|
793
|
+
# @see https://cloud.google.com/bigquery/docs/customer-managed-encryption
|
794
|
+
# Protecting Data with Cloud KMS Keys
|
795
|
+
#
|
796
|
+
# @return [EncryptionConfiguration, nil] The encryption configuration.
|
797
|
+
#
|
798
|
+
# @!group Attributes
|
799
|
+
#
|
800
|
+
def encryption
|
801
|
+
return nil if reference?
|
802
|
+
ensure_full_data!
|
803
|
+
return nil if @gapi.encryption_configuration.nil?
|
804
|
+
EncryptionConfiguration.from_gapi(@gapi.encryption_configuration)
|
805
|
+
.freeze
|
806
|
+
end
|
807
|
+
|
808
|
+
##
|
809
|
+
# Set the {EncryptionConfiguration} object that represents the custom
|
810
|
+
# encryption method used to protect the table. If not set, default
|
811
|
+
# encryption is used.
|
812
|
+
#
|
813
|
+
# Present only if the table is using custom encryption.
|
814
|
+
#
|
815
|
+
# If the table is not a full resource representation (see
|
816
|
+
# {#resource_full?}), the full representation will be retrieved before
|
817
|
+
# the update to comply with ETag-based optimistic concurrency control.
|
818
|
+
#
|
819
|
+
#
|
820
|
+
# @see https://cloud.google.com/bigquery/docs/customer-managed-encryption
|
821
|
+
# Protecting Data with Cloud KMS Keys
|
822
|
+
#
|
823
|
+
# @param [EncryptionConfiguration] value The new encryption config.
|
824
|
+
#
|
825
|
+
# @!group Attributes
|
826
|
+
#
|
827
|
+
def encryption= value
|
828
|
+
reload! unless resource_full?
|
829
|
+
@gapi.encryption_configuration = value.to_gapi
|
830
|
+
patch_gapi! :encryption_configuration
|
831
|
+
end
|
832
|
+
|
785
833
|
##
|
786
834
|
# The {External::DataSource} (or subclass) object that represents the
|
787
835
|
# external data source that the table represents. Data can be queried
|
@@ -1080,6 +1128,12 @@ module Google
|
|
1080
1128
|
# {Job#done?} to detect when the job is done, or simply block until the
|
1081
1129
|
# job is done by calling #{Job#wait_until_done!}. See also {#copy}.
|
1082
1130
|
#
|
1131
|
+
# The geographic location for the job ("US", "EU", etc.) can be set via
|
1132
|
+
# {CopyJob::Updater#location=} in a block passed to this method. If the
|
1133
|
+
# table is a full resource representation (see {#resource_full?}), the
|
1134
|
+
# location of the job will be automatically set to the location of the
|
1135
|
+
# table.
|
1136
|
+
#
|
1083
1137
|
# @param [Table, String] destination_table The destination for the
|
1084
1138
|
# copied data. This can also be a string identifier as specified by
|
1085
1139
|
# the [Query
|
@@ -1125,6 +1179,9 @@ module Google
|
|
1125
1179
|
# dashes. International characters are allowed. Label values are
|
1126
1180
|
# optional. Label keys must start with a letter and each label in the
|
1127
1181
|
# list must have a different key.
|
1182
|
+
# @yield [job] a job configuration object
|
1183
|
+
# @yieldparam [Google::Cloud::Bigquery::CopyJob::Updater] job a job
|
1184
|
+
# configuration object for setting additional options.
|
1128
1185
|
#
|
1129
1186
|
# @return [Google::Cloud::Bigquery::CopyJob]
|
1130
1187
|
#
|
@@ -1153,10 +1210,19 @@ module Google
|
|
1153
1210
|
job_id: nil, prefix: nil, labels: nil
|
1154
1211
|
ensure_service!
|
1155
1212
|
options = { create: create, write: write, dryrun: dryrun,
|
1156
|
-
job_id: job_id, prefix: prefix
|
1157
|
-
|
1158
|
-
|
1159
|
-
|
1213
|
+
labels: labels, job_id: job_id, prefix: prefix }
|
1214
|
+
updater = CopyJob::Updater.from_options(
|
1215
|
+
service,
|
1216
|
+
table_ref,
|
1217
|
+
get_table_ref(destination_table),
|
1218
|
+
options
|
1219
|
+
)
|
1220
|
+
updater.location = location if location # may be table reference
|
1221
|
+
|
1222
|
+
yield updater if block_given?
|
1223
|
+
|
1224
|
+
job_gapi = updater.to_gapi
|
1225
|
+
gapi = service.copy_table job_gapi
|
1160
1226
|
Job.from_gapi gapi, service
|
1161
1227
|
end
|
1162
1228
|
|
@@ -1166,6 +1232,12 @@ module Google
|
|
1166
1232
|
# generally handled as needed to complete the job. See also
|
1167
1233
|
# {#copy_job}.
|
1168
1234
|
#
|
1235
|
+
# The geographic location for the job ("US", "EU", etc.) can be set via
|
1236
|
+
# {CopyJob::Updater#location=} in a block passed to this method. If the
|
1237
|
+
# table is a full resource representation (see {#resource_full?}), the
|
1238
|
+
# location of the job will be automatically set to the location of the
|
1239
|
+
# table.
|
1240
|
+
#
|
1169
1241
|
# @param [Table, String] destination_table The destination for the
|
1170
1242
|
# copied data. This can also be a string identifier as specified by
|
1171
1243
|
# the [Query
|
@@ -1189,6 +1261,9 @@ module Google
|
|
1189
1261
|
# * `append` - BigQuery appends the data to the table.
|
1190
1262
|
# * `empty` - An error will be returned if the destination table
|
1191
1263
|
# already contains data.
|
1264
|
+
# @yield [job] a job configuration object
|
1265
|
+
# @yieldparam [Google::Cloud::Bigquery::CopyJob::Updater] job a job
|
1266
|
+
# configuration object for setting additional options.
|
1192
1267
|
#
|
1193
1268
|
# @return [Boolean] Returns `true` if the copy operation succeeded.
|
1194
1269
|
#
|
@@ -1213,20 +1288,10 @@ module Google
|
|
1213
1288
|
#
|
1214
1289
|
# @!group Data
|
1215
1290
|
#
|
1216
|
-
def copy destination_table, create: nil, write: nil
|
1217
|
-
job = copy_job destination_table, create: create, write: write
|
1291
|
+
def copy destination_table, create: nil, write: nil, &block
|
1292
|
+
job = copy_job destination_table, create: create, write: write, &block
|
1218
1293
|
job.wait_until_done!
|
1219
|
-
|
1220
|
-
if job.failed?
|
1221
|
-
begin
|
1222
|
-
# raise to activate ruby exception cause handling
|
1223
|
-
raise job.gapi_error
|
1224
|
-
rescue StandardError => e
|
1225
|
-
# wrap Google::Apis::Error with Google::Cloud::Error
|
1226
|
-
raise Google::Cloud::Error.from_error(e)
|
1227
|
-
end
|
1228
|
-
end
|
1229
|
-
|
1294
|
+
ensure_job_succeeded! job
|
1230
1295
|
true
|
1231
1296
|
end
|
1232
1297
|
|
@@ -1238,6 +1303,12 @@ module Google
|
|
1238
1303
|
# simply block until the job is done by calling #{Job#wait_until_done!}.
|
1239
1304
|
# See also {#extract}.
|
1240
1305
|
#
|
1306
|
+
# The geographic location for the job ("US", "EU", etc.) can be set via
|
1307
|
+
# {ExtractJob::Updater#location=} in a block passed to this method. If
|
1308
|
+
# the table is a full resource representation (see {#resource_full?}),
|
1309
|
+
# the location of the job will be automatically set to the location of
|
1310
|
+
# the table.
|
1311
|
+
#
|
1241
1312
|
# @see https://cloud.google.com/bigquery/exporting-data-from-bigquery
|
1242
1313
|
# Exporting Data From BigQuery
|
1243
1314
|
#
|
@@ -1281,7 +1352,9 @@ module Google
|
|
1281
1352
|
# dashes. International characters are allowed. Label values are
|
1282
1353
|
# optional. Label keys must start with a letter and each label in the
|
1283
1354
|
# list must have a different key.
|
1284
|
-
#
|
1355
|
+
# @yield [job] a job configuration object
|
1356
|
+
# @yieldparam [Google::Cloud::Bigquery::ExtractJob::Updater] job a job
|
1357
|
+
# configuration object for setting additional options.
|
1285
1358
|
#
|
1286
1359
|
# @return [Google::Cloud::Bigquery::ExtractJob]
|
1287
1360
|
#
|
@@ -1304,7 +1377,14 @@ module Google
|
|
1304
1377
|
options = { format: format, compression: compression,
|
1305
1378
|
delimiter: delimiter, header: header, dryrun: dryrun,
|
1306
1379
|
job_id: job_id, prefix: prefix, labels: labels }
|
1307
|
-
|
1380
|
+
updater = ExtractJob::Updater.from_options service, table_ref,
|
1381
|
+
extract_url, options
|
1382
|
+
updater.location = location if location # may be table reference
|
1383
|
+
|
1384
|
+
yield updater if block_given?
|
1385
|
+
|
1386
|
+
job_gapi = updater.to_gapi
|
1387
|
+
gapi = service.extract_table job_gapi
|
1308
1388
|
Job.from_gapi gapi, service
|
1309
1389
|
end
|
1310
1390
|
|
@@ -1314,6 +1394,12 @@ module Google
|
|
1314
1394
|
# transient errors are generally handled as needed to complete the job.
|
1315
1395
|
# See also {#extract_job}.
|
1316
1396
|
#
|
1397
|
+
# The geographic location for the job ("US", "EU", etc.) can be set via
|
1398
|
+
# {ExtractJob::Updater#location=} in a block passed to this method. If
|
1399
|
+
# the table is a full resource representation (see {#resource_full?}),
|
1400
|
+
# the location of the job will be automatically set to the location of
|
1401
|
+
# the table.
|
1402
|
+
#
|
1317
1403
|
# @see https://cloud.google.com/bigquery/exporting-data-from-bigquery
|
1318
1404
|
# Exporting Data From BigQuery
|
1319
1405
|
#
|
@@ -1335,11 +1421,13 @@ module Google
|
|
1335
1421
|
# exported data. Default is <code>,</code>.
|
1336
1422
|
# @param [Boolean] header Whether to print out a header row in the
|
1337
1423
|
# results. Default is `true`.
|
1338
|
-
#
|
1424
|
+
# @yield [job] a job configuration object
|
1425
|
+
# @yieldparam [Google::Cloud::Bigquery::ExtractJob::Updater] job a job
|
1426
|
+
# configuration object for setting additional options.
|
1339
1427
|
#
|
1340
1428
|
# @return [Boolean] Returns `true` if the extract operation succeeded.
|
1341
1429
|
#
|
1342
|
-
# @example
|
1430
|
+
# @example Extract to a JSON file:
|
1343
1431
|
# require "google/cloud/bigquery"
|
1344
1432
|
#
|
1345
1433
|
# bigquery = Google::Cloud::Bigquery.new
|
@@ -1348,25 +1436,29 @@ module Google
|
|
1348
1436
|
#
|
1349
1437
|
# table.extract "gs://my-bucket/file-name.json", format: "json"
|
1350
1438
|
#
|
1439
|
+
# @example Extract to a CSV file, attaching labels to the job:
|
1440
|
+
# require "google/cloud/bigquery"
|
1441
|
+
#
|
1442
|
+
# bigquery = Google::Cloud::Bigquery.new
|
1443
|
+
# dataset = bigquery.dataset "my_dataset"
|
1444
|
+
# table = dataset.table "my_table"
|
1445
|
+
#
|
1446
|
+
# table.extract "gs://my-bucket/file-name.csv" do |extract|
|
1447
|
+
# extract.labels = { "custom-label" => "custom-value" }
|
1448
|
+
# end
|
1449
|
+
#
|
1351
1450
|
# @!group Data
|
1352
1451
|
#
|
1353
1452
|
def extract extract_url, format: nil, compression: nil, delimiter: nil,
|
1354
|
-
header: nil
|
1355
|
-
job = extract_job extract_url,
|
1356
|
-
|
1357
|
-
|
1453
|
+
header: nil, &block
|
1454
|
+
job = extract_job extract_url,
|
1455
|
+
format: format,
|
1456
|
+
compression: compression,
|
1457
|
+
delimiter: delimiter,
|
1458
|
+
header: header,
|
1459
|
+
&block
|
1358
1460
|
job.wait_until_done!
|
1359
|
-
|
1360
|
-
if job.failed?
|
1361
|
-
begin
|
1362
|
-
# raise to activate ruby exception cause handling
|
1363
|
-
raise job.gapi_error
|
1364
|
-
rescue StandardError => e
|
1365
|
-
# wrap Google::Apis::Error with Google::Cloud::Error
|
1366
|
-
raise Google::Cloud::Error.from_error(e)
|
1367
|
-
end
|
1368
|
-
end
|
1369
|
-
|
1461
|
+
ensure_job_succeeded! job
|
1370
1462
|
true
|
1371
1463
|
end
|
1372
1464
|
|
@@ -1376,9 +1468,16 @@ module Google
|
|
1376
1468
|
# file directly. See [Loading Data with a POST Request](
|
1377
1469
|
# https://cloud.google.com/bigquery/loading-data-post-request#multipart).
|
1378
1470
|
#
|
1379
|
-
#
|
1380
|
-
#
|
1381
|
-
#
|
1471
|
+
# The geographic location for the job ("US", "EU", etc.) can be set via
|
1472
|
+
# {LoadJob::Updater#location=} in a block passed to this method. If the
|
1473
|
+
# table is a full resource representation (see {#resource_full?}), the
|
1474
|
+
# location of the job will be automatically set to the location of the
|
1475
|
+
# table.
|
1476
|
+
#
|
1477
|
+
# @param [File, Google::Cloud::Storage::File, String, URI,
|
1478
|
+
# Array<Google::Cloud::Storage::File, String, URI>] files
|
1479
|
+
# A file or the URI of a Google Cloud Storage file, or an Array of
|
1480
|
+
# those, containing data to load into the table.
|
1382
1481
|
# @param [String] format The exported file format. The default value is
|
1383
1482
|
# `csv`.
|
1384
1483
|
#
|
@@ -1488,6 +1587,8 @@ module Google
|
|
1488
1587
|
# dashes. International characters are allowed. Label values are
|
1489
1588
|
# optional. Label keys must start with a letter and each label in the
|
1490
1589
|
# list must have a different key.
|
1590
|
+
# @yield [load_job] a block for setting the load job
|
1591
|
+
# @yieldparam [LoadJob] load_job the load job object to be updated
|
1491
1592
|
#
|
1492
1593
|
# @return [Google::Cloud::Bigquery::LoadJob]
|
1493
1594
|
#
|
@@ -1513,6 +1614,19 @@ module Google
|
|
1513
1614
|
# file = bucket.file "file-name.csv"
|
1514
1615
|
# load_job = table.load_job file
|
1515
1616
|
#
|
1617
|
+
# @example Pass a list of google-cloud-storage files:
|
1618
|
+
# require "google/cloud/bigquery"
|
1619
|
+
# require "google/cloud/storage"
|
1620
|
+
#
|
1621
|
+
# bigquery = Google::Cloud::Bigquery.new
|
1622
|
+
# dataset = bigquery.dataset "my_dataset"
|
1623
|
+
# table = dataset.table "my_table"
|
1624
|
+
#
|
1625
|
+
# storage = Google::Cloud::Storage.new
|
1626
|
+
# bucket = storage.bucket "my-bucket"
|
1627
|
+
# file = bucket.file "file-name.csv"
|
1628
|
+
# load_job = table.load_job [file, "gs://my-bucket/file-name2.csv"]
|
1629
|
+
#
|
1516
1630
|
# @example Upload a file directly:
|
1517
1631
|
# require "google/cloud/bigquery"
|
1518
1632
|
#
|
@@ -1525,25 +1639,35 @@ module Google
|
|
1525
1639
|
#
|
1526
1640
|
# @!group Data
|
1527
1641
|
#
|
1528
|
-
def load_job
|
1642
|
+
def load_job files, format: nil, create: nil, write: nil,
|
1529
1643
|
projection_fields: nil, jagged_rows: nil,
|
1530
1644
|
quoted_newlines: nil, encoding: nil, delimiter: nil,
|
1531
1645
|
ignore_unknown: nil, max_bad_records: nil, quote: nil,
|
1532
1646
|
skip_leading: nil, dryrun: nil, job_id: nil, prefix: nil,
|
1533
1647
|
labels: nil, autodetect: nil, null_marker: nil
|
1534
1648
|
ensure_service!
|
1535
|
-
|
1536
|
-
|
1537
|
-
|
1538
|
-
|
1539
|
-
|
1540
|
-
|
1541
|
-
|
1542
|
-
|
1543
|
-
|
1544
|
-
|
1545
|
-
|
1546
|
-
|
1649
|
+
|
1650
|
+
updater = load_job_updater format: format, create: create,
|
1651
|
+
write: write,
|
1652
|
+
projection_fields: projection_fields,
|
1653
|
+
jagged_rows: jagged_rows,
|
1654
|
+
quoted_newlines: quoted_newlines,
|
1655
|
+
encoding: encoding,
|
1656
|
+
delimiter: delimiter,
|
1657
|
+
ignore_unknown: ignore_unknown,
|
1658
|
+
max_bad_records: max_bad_records,
|
1659
|
+
quote: quote, skip_leading: skip_leading,
|
1660
|
+
dryrun: dryrun, job_id: job_id,
|
1661
|
+
prefix: prefix, schema: schema,
|
1662
|
+
labels: labels, autodetect: autodetect,
|
1663
|
+
null_marker: null_marker
|
1664
|
+
|
1665
|
+
yield updater if block_given?
|
1666
|
+
|
1667
|
+
job_gapi = updater.to_gapi
|
1668
|
+
|
1669
|
+
return load_local(files, job_gapi) if local_file? files
|
1670
|
+
load_storage files, job_gapi
|
1547
1671
|
end
|
1548
1672
|
|
1549
1673
|
##
|
@@ -1552,9 +1676,16 @@ module Google
|
|
1552
1676
|
# file directly. See [Loading Data with a POST Request](
|
1553
1677
|
# https://cloud.google.com/bigquery/loading-data-post-request#multipart).
|
1554
1678
|
#
|
1555
|
-
#
|
1556
|
-
#
|
1557
|
-
#
|
1679
|
+
# The geographic location for the job ("US", "EU", etc.) can be set via
|
1680
|
+
# {LoadJob::Updater#location=} in a block passed to this method. If the
|
1681
|
+
# table is a full resource representation (see {#resource_full?}), the
|
1682
|
+
# location of the job will be automatically set to the location of the
|
1683
|
+
# table.
|
1684
|
+
#
|
1685
|
+
# @param [File, Google::Cloud::Storage::File, String, URI,
|
1686
|
+
# Array<Google::Cloud::Storage::File, String, URI>] files
|
1687
|
+
# A file or the URI of a Google Cloud Storage file, or an Array of
|
1688
|
+
# those, containing data to load into the table.
|
1558
1689
|
# @param [String] format The exported file format. The default value is
|
1559
1690
|
# `csv`.
|
1560
1691
|
#
|
@@ -1643,7 +1774,14 @@ module Google
|
|
1643
1774
|
# value is `0`. This property is useful if you have header rows in the
|
1644
1775
|
# file that should be skipped.
|
1645
1776
|
#
|
1646
|
-
# @
|
1777
|
+
# @yield [updater] A block for setting the schema of the destination
|
1778
|
+
# table and other options for the load job. The schema can be omitted
|
1779
|
+
# if the destination table already exists, or if you're loading data
|
1780
|
+
# from a Google Cloud Datastore backup.
|
1781
|
+
# @yieldparam [Google::Cloud::Bigquery::LoadJob::Updater] updater An
|
1782
|
+
# updater to modify the load job and its schema.
|
1783
|
+
#
|
1784
|
+
# @return [Boolean] Returns `true` if the load job was successful.
|
1647
1785
|
#
|
1648
1786
|
# @example
|
1649
1787
|
# require "google/cloud/bigquery"
|
@@ -1652,7 +1790,7 @@ module Google
|
|
1652
1790
|
# dataset = bigquery.dataset "my_dataset"
|
1653
1791
|
# table = dataset.table "my_table"
|
1654
1792
|
#
|
1655
|
-
#
|
1793
|
+
# success = table.load "gs://my-bucket/file-name.csv"
|
1656
1794
|
#
|
1657
1795
|
# @example Pass a google-cloud-storage `File` instance:
|
1658
1796
|
# require "google/cloud/bigquery"
|
@@ -1665,7 +1803,20 @@ module Google
|
|
1665
1803
|
# storage = Google::Cloud::Storage.new
|
1666
1804
|
# bucket = storage.bucket "my-bucket"
|
1667
1805
|
# file = bucket.file "file-name.csv"
|
1668
|
-
#
|
1806
|
+
# success = table.load file
|
1807
|
+
#
|
1808
|
+
# @example Pass a list of google-cloud-storage files:
|
1809
|
+
# require "google/cloud/bigquery"
|
1810
|
+
# require "google/cloud/storage"
|
1811
|
+
#
|
1812
|
+
# bigquery = Google::Cloud::Bigquery.new
|
1813
|
+
# dataset = bigquery.dataset "my_dataset"
|
1814
|
+
# table = dataset.table "my_table"
|
1815
|
+
#
|
1816
|
+
# storage = Google::Cloud::Storage.new
|
1817
|
+
# bucket = storage.bucket "my-bucket"
|
1818
|
+
# file = bucket.file "file-name.csv"
|
1819
|
+
# table.load [file, "gs://my-bucket/file-name2.csv"]
|
1669
1820
|
#
|
1670
1821
|
# @example Upload a file directly:
|
1671
1822
|
# require "google/cloud/bigquery"
|
@@ -1674,38 +1825,39 @@ module Google
|
|
1674
1825
|
# dataset = bigquery.dataset "my_dataset"
|
1675
1826
|
# table = dataset.table "my_table"
|
1676
1827
|
#
|
1677
|
-
# file = File.open "my_data.
|
1678
|
-
#
|
1828
|
+
# file = File.open "my_data.json"
|
1829
|
+
# success = table.load file do |j|
|
1830
|
+
# j.format = "newline_delimited_json"
|
1831
|
+
# end
|
1679
1832
|
#
|
1680
1833
|
# @!group Data
|
1681
1834
|
#
|
1682
|
-
def load
|
1835
|
+
def load files, format: nil, create: nil, write: nil,
|
1683
1836
|
projection_fields: nil, jagged_rows: nil, quoted_newlines: nil,
|
1684
1837
|
encoding: nil, delimiter: nil, ignore_unknown: nil,
|
1685
1838
|
max_bad_records: nil, quote: nil, skip_leading: nil,
|
1686
1839
|
autodetect: nil, null_marker: nil
|
1687
|
-
|
1688
|
-
projection_fields: projection_fields,
|
1689
|
-
jagged_rows: jagged_rows,
|
1690
|
-
quoted_newlines: quoted_newlines,
|
1691
|
-
encoding: encoding, delimiter: delimiter,
|
1692
|
-
ignore_unknown: ignore_unknown,
|
1693
|
-
max_bad_records: max_bad_records, quote: quote,
|
1694
|
-
skip_leading: skip_leading,
|
1695
|
-
autodetect: autodetect, null_marker: null_marker
|
1840
|
+
ensure_service!
|
1696
1841
|
|
1842
|
+
updater = load_job_updater format: format, create: create,
|
1843
|
+
write: write,
|
1844
|
+
projection_fields: projection_fields,
|
1845
|
+
jagged_rows: jagged_rows,
|
1846
|
+
quoted_newlines: quoted_newlines,
|
1847
|
+
encoding: encoding,
|
1848
|
+
delimiter: delimiter,
|
1849
|
+
ignore_unknown: ignore_unknown,
|
1850
|
+
max_bad_records: max_bad_records,
|
1851
|
+
quote: quote, skip_leading: skip_leading,
|
1852
|
+
schema: schema,
|
1853
|
+
autodetect: autodetect,
|
1854
|
+
null_marker: null_marker
|
1855
|
+
|
1856
|
+
yield updater if block_given?
|
1857
|
+
|
1858
|
+
job = load_local_or_uri files, updater
|
1697
1859
|
job.wait_until_done!
|
1698
|
-
|
1699
|
-
if job.failed?
|
1700
|
-
begin
|
1701
|
-
# raise to activate ruby exception cause handling
|
1702
|
-
raise job.gapi_error
|
1703
|
-
rescue StandardError => e
|
1704
|
-
# wrap Google::Apis::Error with Google::Cloud::Error
|
1705
|
-
raise Google::Cloud::Error.from_error(e)
|
1706
|
-
end
|
1707
|
-
end
|
1708
|
-
|
1860
|
+
ensure_job_succeeded! job
|
1709
1861
|
true
|
1710
1862
|
end
|
1711
1863
|
|
@@ -2049,29 +2201,157 @@ module Google
|
|
2049
2201
|
reload!
|
2050
2202
|
end
|
2051
2203
|
|
2052
|
-
def
|
2204
|
+
def ensure_job_succeeded! job
|
2205
|
+
return unless job.failed?
|
2206
|
+
begin
|
2207
|
+
# raise to activate ruby exception cause handling
|
2208
|
+
raise job.gapi_error
|
2209
|
+
rescue StandardError => e
|
2210
|
+
# wrap Google::Apis::Error with Google::Cloud::Error
|
2211
|
+
raise Google::Cloud::Error.from_error(e)
|
2212
|
+
end
|
2213
|
+
end
|
2214
|
+
|
2215
|
+
def load_job_gapi table_id, dryrun, job_id: nil, prefix: nil
|
2216
|
+
job_ref = service.job_ref_from job_id, prefix
|
2217
|
+
Google::Apis::BigqueryV2::Job.new(
|
2218
|
+
job_reference: job_ref,
|
2219
|
+
configuration: Google::Apis::BigqueryV2::JobConfiguration.new(
|
2220
|
+
load: Google::Apis::BigqueryV2::JobConfigurationLoad.new(
|
2221
|
+
destination_table: Google::Apis::BigqueryV2::TableReference.new(
|
2222
|
+
project_id: @service.project,
|
2223
|
+
dataset_id: dataset_id,
|
2224
|
+
table_id: table_id
|
2225
|
+
)
|
2226
|
+
),
|
2227
|
+
dry_run: dryrun
|
2228
|
+
)
|
2229
|
+
)
|
2230
|
+
end
|
2231
|
+
|
2232
|
+
def load_job_csv_options! job, jagged_rows: nil,
|
2233
|
+
quoted_newlines: nil,
|
2234
|
+
delimiter: nil,
|
2235
|
+
quote: nil, skip_leading: nil,
|
2236
|
+
null_marker: nil
|
2237
|
+
job.jagged_rows = jagged_rows unless jagged_rows.nil?
|
2238
|
+
job.quoted_newlines = quoted_newlines unless quoted_newlines.nil?
|
2239
|
+
job.delimiter = delimiter unless delimiter.nil?
|
2240
|
+
job.null_marker = null_marker unless null_marker.nil?
|
2241
|
+
job.quote = quote unless quote.nil?
|
2242
|
+
job.skip_leading = skip_leading unless skip_leading.nil?
|
2243
|
+
end
|
2244
|
+
|
2245
|
+
def load_job_file_options! job, format: nil,
|
2246
|
+
projection_fields: nil,
|
2247
|
+
jagged_rows: nil, quoted_newlines: nil,
|
2248
|
+
encoding: nil, delimiter: nil,
|
2249
|
+
ignore_unknown: nil, max_bad_records: nil,
|
2250
|
+
quote: nil, skip_leading: nil,
|
2251
|
+
null_marker: nil
|
2252
|
+
job.format = format unless format.nil?
|
2253
|
+
unless projection_fields.nil?
|
2254
|
+
job.projection_fields = projection_fields
|
2255
|
+
end
|
2256
|
+
job.encoding = encoding unless encoding.nil?
|
2257
|
+
job.ignore_unknown = ignore_unknown unless ignore_unknown.nil?
|
2258
|
+
job.max_bad_records = max_bad_records unless max_bad_records.nil?
|
2259
|
+
load_job_csv_options! job, jagged_rows: jagged_rows,
|
2260
|
+
quoted_newlines: quoted_newlines,
|
2261
|
+
delimiter: delimiter,
|
2262
|
+
quote: quote,
|
2263
|
+
skip_leading: skip_leading,
|
2264
|
+
null_marker: null_marker
|
2265
|
+
end
|
2266
|
+
|
2267
|
+
def load_job_updater format: nil, create: nil,
|
2268
|
+
write: nil, projection_fields: nil,
|
2269
|
+
jagged_rows: nil, quoted_newlines: nil,
|
2270
|
+
encoding: nil, delimiter: nil,
|
2271
|
+
ignore_unknown: nil, max_bad_records: nil,
|
2272
|
+
quote: nil, skip_leading: nil, dryrun: nil,
|
2273
|
+
schema: nil, job_id: nil, prefix: nil, labels: nil,
|
2274
|
+
autodetect: nil, null_marker: nil
|
2275
|
+
new_job = load_job_gapi table_id, dryrun, job_id: job_id,
|
2276
|
+
prefix: prefix
|
2277
|
+
LoadJob::Updater.new(new_job).tap do |job|
|
2278
|
+
job.location = location if location # may be table reference
|
2279
|
+
job.create = create unless create.nil?
|
2280
|
+
job.write = write unless write.nil?
|
2281
|
+
job.schema = schema unless schema.nil?
|
2282
|
+
job.autodetect = autodetect unless autodetect.nil?
|
2283
|
+
job.labels = labels unless labels.nil?
|
2284
|
+
load_job_file_options! job, format: format,
|
2285
|
+
projection_fields: projection_fields,
|
2286
|
+
jagged_rows: jagged_rows,
|
2287
|
+
quoted_newlines: quoted_newlines,
|
2288
|
+
encoding: encoding,
|
2289
|
+
delimiter: delimiter,
|
2290
|
+
ignore_unknown: ignore_unknown,
|
2291
|
+
max_bad_records: max_bad_records,
|
2292
|
+
quote: quote,
|
2293
|
+
skip_leading: skip_leading,
|
2294
|
+
null_marker: null_marker
|
2295
|
+
end
|
2296
|
+
end
|
2297
|
+
|
2298
|
+
def load_storage urls, job_gapi
|
2053
2299
|
# Convert to storage URL
|
2054
|
-
|
2055
|
-
|
2300
|
+
urls = [urls].flatten.map do |url|
|
2301
|
+
if url.respond_to? :to_gs_url
|
2302
|
+
url.to_gs_url
|
2303
|
+
elsif url.is_a? URI
|
2304
|
+
url.to_s
|
2305
|
+
else
|
2306
|
+
url
|
2307
|
+
end
|
2308
|
+
end
|
2309
|
+
|
2310
|
+
unless urls.nil?
|
2311
|
+
job_gapi.configuration.load.update! source_uris: urls
|
2312
|
+
if job_gapi.configuration.load.source_format.nil?
|
2313
|
+
source_format = Convert.derive_source_format_from_list urls
|
2314
|
+
unless source_format.nil?
|
2315
|
+
job_gapi.configuration.load.source_format = source_format
|
2316
|
+
end
|
2317
|
+
end
|
2318
|
+
end
|
2056
2319
|
|
2057
|
-
gapi = service.load_table_gs_url
|
2320
|
+
gapi = service.load_table_gs_url job_gapi
|
2058
2321
|
Job.from_gapi gapi, service
|
2059
2322
|
end
|
2060
2323
|
|
2061
|
-
def load_local file,
|
2062
|
-
|
2063
|
-
|
2324
|
+
def load_local file, job_gapi
|
2325
|
+
path = Pathname(file).to_path
|
2326
|
+
if job_gapi.configuration.load.source_format.nil?
|
2327
|
+
source_format = Convert.derive_source_format path
|
2328
|
+
unless source_format.nil?
|
2329
|
+
job_gapi.configuration.load.source_format = source_format
|
2330
|
+
end
|
2331
|
+
end
|
2064
2332
|
|
2065
|
-
gapi = service.load_table_file
|
2333
|
+
gapi = service.load_table_file file, job_gapi
|
2066
2334
|
Job.from_gapi gapi, service
|
2067
2335
|
end
|
2068
2336
|
|
2069
|
-
def
|
2070
|
-
|
2071
|
-
|
2072
|
-
|
2073
|
-
|
2074
|
-
|
2337
|
+
def load_local_or_uri file, updater
|
2338
|
+
job_gapi = updater.to_gapi
|
2339
|
+
job = if local_file? file
|
2340
|
+
load_local file, job_gapi
|
2341
|
+
else
|
2342
|
+
load_storage file, job_gapi
|
2343
|
+
end
|
2344
|
+
job
|
2345
|
+
end
|
2346
|
+
|
2347
|
+
def storage_url? files
|
2348
|
+
[files].flatten.all? do |file|
|
2349
|
+
file.respond_to?(:to_gs_url) ||
|
2350
|
+
(file.respond_to?(:to_str) &&
|
2351
|
+
file.to_str.downcase.start_with?("gs://")) ||
|
2352
|
+
(file.is_a?(URI) &&
|
2353
|
+
file.to_s.downcase.start_with?("gs://"))
|
2354
|
+
end
|
2075
2355
|
end
|
2076
2356
|
|
2077
2357
|
def local_file? file
|