google-cloud-bigquery 1.1.0 → 1.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -18,6 +18,7 @@ require "google/cloud/bigquery/service"
18
18
  require "google/cloud/bigquery/data"
19
19
  require "google/cloud/bigquery/table/list"
20
20
  require "google/cloud/bigquery/schema"
21
+ require "google/cloud/bigquery/encryption_configuration"
21
22
  require "google/cloud/bigquery/external"
22
23
  require "google/cloud/bigquery/insert_response"
23
24
  require "google/cloud/bigquery/table/async_inserter"
@@ -782,6 +783,53 @@ module Google
782
783
  schema.headers
783
784
  end
784
785
 
786
+ ##
787
+ # The {EncryptionConfiguration} object that represents the custom
788
+ # encryption method used to protect the table. If not set, default
789
+ # encryption is used.
790
+ #
791
+ # Present only if the table is using custom encryption.
792
+ #
793
+ # @see https://cloud.google.com/bigquery/docs/customer-managed-encryption
794
+ # Protecting Data with Cloud KMS Keys
795
+ #
796
+ # @return [EncryptionConfiguration, nil] The encryption configuration.
797
+ #
798
+ # @!group Attributes
799
+ #
800
+ def encryption
801
+ return nil if reference?
802
+ ensure_full_data!
803
+ return nil if @gapi.encryption_configuration.nil?
804
+ EncryptionConfiguration.from_gapi(@gapi.encryption_configuration)
805
+ .freeze
806
+ end
807
+
808
+ ##
809
+ # Set the {EncryptionConfiguration} object that represents the custom
810
+ # encryption method used to protect the table. If not set, default
811
+ # encryption is used.
812
+ #
813
+ # Present only if the table is using custom encryption.
814
+ #
815
+ # If the table is not a full resource representation (see
816
+ # {#resource_full?}), the full representation will be retrieved before
817
+ # the update to comply with ETag-based optimistic concurrency control.
818
+ #
819
+ #
820
+ # @see https://cloud.google.com/bigquery/docs/customer-managed-encryption
821
+ # Protecting Data with Cloud KMS Keys
822
+ #
823
+ # @param [EncryptionConfiguration] value The new encryption config.
824
+ #
825
+ # @!group Attributes
826
+ #
827
+ def encryption= value
828
+ reload! unless resource_full?
829
+ @gapi.encryption_configuration = value.to_gapi
830
+ patch_gapi! :encryption_configuration
831
+ end
832
+
785
833
  ##
786
834
  # The {External::DataSource} (or subclass) object that represents the
787
835
  # external data source that the table represents. Data can be queried
@@ -1080,6 +1128,12 @@ module Google
1080
1128
  # {Job#done?} to detect when the job is done, or simply block until the
1081
1129
  # job is done by calling #{Job#wait_until_done!}. See also {#copy}.
1082
1130
  #
1131
+ # The geographic location for the job ("US", "EU", etc.) can be set via
1132
+ # {CopyJob::Updater#location=} in a block passed to this method. If the
1133
+ # table is a full resource representation (see {#resource_full?}), the
1134
+ # location of the job will be automatically set to the location of the
1135
+ # table.
1136
+ #
1083
1137
  # @param [Table, String] destination_table The destination for the
1084
1138
  # copied data. This can also be a string identifier as specified by
1085
1139
  # the [Query
@@ -1125,6 +1179,9 @@ module Google
1125
1179
  # dashes. International characters are allowed. Label values are
1126
1180
  # optional. Label keys must start with a letter and each label in the
1127
1181
  # list must have a different key.
1182
+ # @yield [job] a job configuration object
1183
+ # @yieldparam [Google::Cloud::Bigquery::CopyJob::Updater] job a job
1184
+ # configuration object for setting additional options.
1128
1185
  #
1129
1186
  # @return [Google::Cloud::Bigquery::CopyJob]
1130
1187
  #
@@ -1153,10 +1210,19 @@ module Google
1153
1210
  job_id: nil, prefix: nil, labels: nil
1154
1211
  ensure_service!
1155
1212
  options = { create: create, write: write, dryrun: dryrun,
1156
- job_id: job_id, prefix: prefix, labels: labels }
1157
- gapi = service.copy_table table_ref,
1158
- get_table_ref(destination_table),
1159
- options
1213
+ labels: labels, job_id: job_id, prefix: prefix }
1214
+ updater = CopyJob::Updater.from_options(
1215
+ service,
1216
+ table_ref,
1217
+ get_table_ref(destination_table),
1218
+ options
1219
+ )
1220
+ updater.location = location if location # may be table reference
1221
+
1222
+ yield updater if block_given?
1223
+
1224
+ job_gapi = updater.to_gapi
1225
+ gapi = service.copy_table job_gapi
1160
1226
  Job.from_gapi gapi, service
1161
1227
  end
1162
1228
 
@@ -1166,6 +1232,12 @@ module Google
1166
1232
  # generally handled as needed to complete the job. See also
1167
1233
  # {#copy_job}.
1168
1234
  #
1235
+ # The geographic location for the job ("US", "EU", etc.) can be set via
1236
+ # {CopyJob::Updater#location=} in a block passed to this method. If the
1237
+ # table is a full resource representation (see {#resource_full?}), the
1238
+ # location of the job will be automatically set to the location of the
1239
+ # table.
1240
+ #
1169
1241
  # @param [Table, String] destination_table The destination for the
1170
1242
  # copied data. This can also be a string identifier as specified by
1171
1243
  # the [Query
@@ -1189,6 +1261,9 @@ module Google
1189
1261
  # * `append` - BigQuery appends the data to the table.
1190
1262
  # * `empty` - An error will be returned if the destination table
1191
1263
  # already contains data.
1264
+ # @yield [job] a job configuration object
1265
+ # @yieldparam [Google::Cloud::Bigquery::CopyJob::Updater] job a job
1266
+ # configuration object for setting additional options.
1192
1267
  #
1193
1268
  # @return [Boolean] Returns `true` if the copy operation succeeded.
1194
1269
  #
@@ -1213,20 +1288,10 @@ module Google
1213
1288
  #
1214
1289
  # @!group Data
1215
1290
  #
1216
- def copy destination_table, create: nil, write: nil
1217
- job = copy_job destination_table, create: create, write: write
1291
+ def copy destination_table, create: nil, write: nil, &block
1292
+ job = copy_job destination_table, create: create, write: write, &block
1218
1293
  job.wait_until_done!
1219
-
1220
- if job.failed?
1221
- begin
1222
- # raise to activate ruby exception cause handling
1223
- raise job.gapi_error
1224
- rescue StandardError => e
1225
- # wrap Google::Apis::Error with Google::Cloud::Error
1226
- raise Google::Cloud::Error.from_error(e)
1227
- end
1228
- end
1229
-
1294
+ ensure_job_succeeded! job
1230
1295
  true
1231
1296
  end
1232
1297
 
@@ -1238,6 +1303,12 @@ module Google
1238
1303
  # simply block until the job is done by calling #{Job#wait_until_done!}.
1239
1304
  # See also {#extract}.
1240
1305
  #
1306
+ # The geographic location for the job ("US", "EU", etc.) can be set via
1307
+ # {ExtractJob::Updater#location=} in a block passed to this method. If
1308
+ # the table is a full resource representation (see {#resource_full?}),
1309
+ # the location of the job will be automatically set to the location of
1310
+ # the table.
1311
+ #
1241
1312
  # @see https://cloud.google.com/bigquery/exporting-data-from-bigquery
1242
1313
  # Exporting Data From BigQuery
1243
1314
  #
@@ -1281,7 +1352,9 @@ module Google
1281
1352
  # dashes. International characters are allowed. Label values are
1282
1353
  # optional. Label keys must start with a letter and each label in the
1283
1354
  # list must have a different key.
1284
- #
1355
+ # @yield [job] a job configuration object
1356
+ # @yieldparam [Google::Cloud::Bigquery::ExtractJob::Updater] job a job
1357
+ # configuration object for setting additional options.
1285
1358
  #
1286
1359
  # @return [Google::Cloud::Bigquery::ExtractJob]
1287
1360
  #
@@ -1304,7 +1377,14 @@ module Google
1304
1377
  options = { format: format, compression: compression,
1305
1378
  delimiter: delimiter, header: header, dryrun: dryrun,
1306
1379
  job_id: job_id, prefix: prefix, labels: labels }
1307
- gapi = service.extract_table table_ref, extract_url, options
1380
+ updater = ExtractJob::Updater.from_options service, table_ref,
1381
+ extract_url, options
1382
+ updater.location = location if location # may be table reference
1383
+
1384
+ yield updater if block_given?
1385
+
1386
+ job_gapi = updater.to_gapi
1387
+ gapi = service.extract_table job_gapi
1308
1388
  Job.from_gapi gapi, service
1309
1389
  end
1310
1390
 
@@ -1314,6 +1394,12 @@ module Google
1314
1394
  # transient errors are generally handled as needed to complete the job.
1315
1395
  # See also {#extract_job}.
1316
1396
  #
1397
+ # The geographic location for the job ("US", "EU", etc.) can be set via
1398
+ # {ExtractJob::Updater#location=} in a block passed to this method. If
1399
+ # the table is a full resource representation (see {#resource_full?}),
1400
+ # the location of the job will be automatically set to the location of
1401
+ # the table.
1402
+ #
1317
1403
  # @see https://cloud.google.com/bigquery/exporting-data-from-bigquery
1318
1404
  # Exporting Data From BigQuery
1319
1405
  #
@@ -1335,11 +1421,13 @@ module Google
1335
1421
  # exported data. Default is <code>,</code>.
1336
1422
  # @param [Boolean] header Whether to print out a header row in the
1337
1423
  # results. Default is `true`.
1338
- #
1424
+ # @yield [job] a job configuration object
1425
+ # @yieldparam [Google::Cloud::Bigquery::ExtractJob::Updater] job a job
1426
+ # configuration object for setting additional options.
1339
1427
  #
1340
1428
  # @return [Boolean] Returns `true` if the extract operation succeeded.
1341
1429
  #
1342
- # @example
1430
+ # @example Extract to a JSON file:
1343
1431
  # require "google/cloud/bigquery"
1344
1432
  #
1345
1433
  # bigquery = Google::Cloud::Bigquery.new
@@ -1348,25 +1436,29 @@ module Google
1348
1436
  #
1349
1437
  # table.extract "gs://my-bucket/file-name.json", format: "json"
1350
1438
  #
1439
+ # @example Extract to a CSV file, attaching labels to the job:
1440
+ # require "google/cloud/bigquery"
1441
+ #
1442
+ # bigquery = Google::Cloud::Bigquery.new
1443
+ # dataset = bigquery.dataset "my_dataset"
1444
+ # table = dataset.table "my_table"
1445
+ #
1446
+ # table.extract "gs://my-bucket/file-name.csv" do |extract|
1447
+ # extract.labels = { "custom-label" => "custom-value" }
1448
+ # end
1449
+ #
1351
1450
  # @!group Data
1352
1451
  #
1353
1452
  def extract extract_url, format: nil, compression: nil, delimiter: nil,
1354
- header: nil
1355
- job = extract_job extract_url, format: format,
1356
- compression: compression,
1357
- delimiter: delimiter, header: header
1453
+ header: nil, &block
1454
+ job = extract_job extract_url,
1455
+ format: format,
1456
+ compression: compression,
1457
+ delimiter: delimiter,
1458
+ header: header,
1459
+ &block
1358
1460
  job.wait_until_done!
1359
-
1360
- if job.failed?
1361
- begin
1362
- # raise to activate ruby exception cause handling
1363
- raise job.gapi_error
1364
- rescue StandardError => e
1365
- # wrap Google::Apis::Error with Google::Cloud::Error
1366
- raise Google::Cloud::Error.from_error(e)
1367
- end
1368
- end
1369
-
1461
+ ensure_job_succeeded! job
1370
1462
  true
1371
1463
  end
1372
1464
 
@@ -1376,9 +1468,16 @@ module Google
1376
1468
  # file directly. See [Loading Data with a POST Request](
1377
1469
  # https://cloud.google.com/bigquery/loading-data-post-request#multipart).
1378
1470
  #
1379
- # @param [File, Google::Cloud::Storage::File, String, URI] file A file
1380
- # or the URI of a Google Cloud Storage file containing data to load
1381
- # into the table.
1471
+ # The geographic location for the job ("US", "EU", etc.) can be set via
1472
+ # {LoadJob::Updater#location=} in a block passed to this method. If the
1473
+ # table is a full resource representation (see {#resource_full?}), the
1474
+ # location of the job will be automatically set to the location of the
1475
+ # table.
1476
+ #
1477
+ # @param [File, Google::Cloud::Storage::File, String, URI,
1478
+ # Array<Google::Cloud::Storage::File, String, URI>] files
1479
+ # A file or the URI of a Google Cloud Storage file, or an Array of
1480
+ # those, containing data to load into the table.
1382
1481
  # @param [String] format The exported file format. The default value is
1383
1482
  # `csv`.
1384
1483
  #
@@ -1488,6 +1587,8 @@ module Google
1488
1587
  # dashes. International characters are allowed. Label values are
1489
1588
  # optional. Label keys must start with a letter and each label in the
1490
1589
  # list must have a different key.
1590
+ # @yield [load_job] a block for setting the load job
1591
+ # @yieldparam [LoadJob] load_job the load job object to be updated
1491
1592
  #
1492
1593
  # @return [Google::Cloud::Bigquery::LoadJob]
1493
1594
  #
@@ -1513,6 +1614,19 @@ module Google
1513
1614
  # file = bucket.file "file-name.csv"
1514
1615
  # load_job = table.load_job file
1515
1616
  #
1617
+ # @example Pass a list of google-cloud-storage files:
1618
+ # require "google/cloud/bigquery"
1619
+ # require "google/cloud/storage"
1620
+ #
1621
+ # bigquery = Google::Cloud::Bigquery.new
1622
+ # dataset = bigquery.dataset "my_dataset"
1623
+ # table = dataset.table "my_table"
1624
+ #
1625
+ # storage = Google::Cloud::Storage.new
1626
+ # bucket = storage.bucket "my-bucket"
1627
+ # file = bucket.file "file-name.csv"
1628
+ # load_job = table.load_job [file, "gs://my-bucket/file-name2.csv"]
1629
+ #
1516
1630
  # @example Upload a file directly:
1517
1631
  # require "google/cloud/bigquery"
1518
1632
  #
@@ -1525,25 +1639,35 @@ module Google
1525
1639
  #
1526
1640
  # @!group Data
1527
1641
  #
1528
- def load_job file, format: nil, create: nil, write: nil,
1642
+ def load_job files, format: nil, create: nil, write: nil,
1529
1643
  projection_fields: nil, jagged_rows: nil,
1530
1644
  quoted_newlines: nil, encoding: nil, delimiter: nil,
1531
1645
  ignore_unknown: nil, max_bad_records: nil, quote: nil,
1532
1646
  skip_leading: nil, dryrun: nil, job_id: nil, prefix: nil,
1533
1647
  labels: nil, autodetect: nil, null_marker: nil
1534
1648
  ensure_service!
1535
- options = { format: format, create: create, write: write,
1536
- projection_fields: projection_fields,
1537
- jagged_rows: jagged_rows,
1538
- quoted_newlines: quoted_newlines, encoding: encoding,
1539
- delimiter: delimiter, ignore_unknown: ignore_unknown,
1540
- max_bad_records: max_bad_records, quote: quote,
1541
- skip_leading: skip_leading, dryrun: dryrun,
1542
- job_id: job_id, prefix: prefix, labels: labels,
1543
- autodetect: autodetect, null_marker: null_marker }
1544
- return load_storage(file, options) if storage_url? file
1545
- return load_local(file, options) if local_file? file
1546
- raise Google::Cloud::Error, "Don't know how to load #{file}"
1649
+
1650
+ updater = load_job_updater format: format, create: create,
1651
+ write: write,
1652
+ projection_fields: projection_fields,
1653
+ jagged_rows: jagged_rows,
1654
+ quoted_newlines: quoted_newlines,
1655
+ encoding: encoding,
1656
+ delimiter: delimiter,
1657
+ ignore_unknown: ignore_unknown,
1658
+ max_bad_records: max_bad_records,
1659
+ quote: quote, skip_leading: skip_leading,
1660
+ dryrun: dryrun, job_id: job_id,
1661
+ prefix: prefix, schema: schema,
1662
+ labels: labels, autodetect: autodetect,
1663
+ null_marker: null_marker
1664
+
1665
+ yield updater if block_given?
1666
+
1667
+ job_gapi = updater.to_gapi
1668
+
1669
+ return load_local(files, job_gapi) if local_file? files
1670
+ load_storage files, job_gapi
1547
1671
  end
1548
1672
 
1549
1673
  ##
@@ -1552,9 +1676,16 @@ module Google
1552
1676
  # file directly. See [Loading Data with a POST Request](
1553
1677
  # https://cloud.google.com/bigquery/loading-data-post-request#multipart).
1554
1678
  #
1555
- # @param [File, Google::Cloud::Storage::File, String, URI] file A file
1556
- # or the URI of a Google Cloud Storage file containing data to load
1557
- # into the table.
1679
+ # The geographic location for the job ("US", "EU", etc.) can be set via
1680
+ # {LoadJob::Updater#location=} in a block passed to this method. If the
1681
+ # table is a full resource representation (see {#resource_full?}), the
1682
+ # location of the job will be automatically set to the location of the
1683
+ # table.
1684
+ #
1685
+ # @param [File, Google::Cloud::Storage::File, String, URI,
1686
+ # Array<Google::Cloud::Storage::File, String, URI>] files
1687
+ # A file or the URI of a Google Cloud Storage file, or an Array of
1688
+ # those, containing data to load into the table.
1558
1689
  # @param [String] format The exported file format. The default value is
1559
1690
  # `csv`.
1560
1691
  #
@@ -1643,7 +1774,14 @@ module Google
1643
1774
  # value is `0`. This property is useful if you have header rows in the
1644
1775
  # file that should be skipped.
1645
1776
  #
1646
- # @return [Google::Cloud::Bigquery::LoadJob]
1777
+ # @yield [updater] A block for setting the schema of the destination
1778
+ # table and other options for the load job. The schema can be omitted
1779
+ # if the destination table already exists, or if you're loading data
1780
+ # from a Google Cloud Datastore backup.
1781
+ # @yieldparam [Google::Cloud::Bigquery::LoadJob::Updater] updater An
1782
+ # updater to modify the load job and its schema.
1783
+ #
1784
+ # @return [Boolean] Returns `true` if the load job was successful.
1647
1785
  #
1648
1786
  # @example
1649
1787
  # require "google/cloud/bigquery"
@@ -1652,7 +1790,7 @@ module Google
1652
1790
  # dataset = bigquery.dataset "my_dataset"
1653
1791
  # table = dataset.table "my_table"
1654
1792
  #
1655
- # load_job = table.load_job "gs://my-bucket/file-name.csv"
1793
+ # success = table.load "gs://my-bucket/file-name.csv"
1656
1794
  #
1657
1795
  # @example Pass a google-cloud-storage `File` instance:
1658
1796
  # require "google/cloud/bigquery"
@@ -1665,7 +1803,20 @@ module Google
1665
1803
  # storage = Google::Cloud::Storage.new
1666
1804
  # bucket = storage.bucket "my-bucket"
1667
1805
  # file = bucket.file "file-name.csv"
1668
- # load_job = table.load_job file
1806
+ # success = table.load file
1807
+ #
1808
+ # @example Pass a list of google-cloud-storage files:
1809
+ # require "google/cloud/bigquery"
1810
+ # require "google/cloud/storage"
1811
+ #
1812
+ # bigquery = Google::Cloud::Bigquery.new
1813
+ # dataset = bigquery.dataset "my_dataset"
1814
+ # table = dataset.table "my_table"
1815
+ #
1816
+ # storage = Google::Cloud::Storage.new
1817
+ # bucket = storage.bucket "my-bucket"
1818
+ # file = bucket.file "file-name.csv"
1819
+ # table.load [file, "gs://my-bucket/file-name2.csv"]
1669
1820
  #
1670
1821
  # @example Upload a file directly:
1671
1822
  # require "google/cloud/bigquery"
@@ -1674,38 +1825,39 @@ module Google
1674
1825
  # dataset = bigquery.dataset "my_dataset"
1675
1826
  # table = dataset.table "my_table"
1676
1827
  #
1677
- # file = File.open "my_data.csv"
1678
- # load_job = table.load_job file
1828
+ # file = File.open "my_data.json"
1829
+ # success = table.load file do |j|
1830
+ # j.format = "newline_delimited_json"
1831
+ # end
1679
1832
  #
1680
1833
  # @!group Data
1681
1834
  #
1682
- def load file, format: nil, create: nil, write: nil,
1835
+ def load files, format: nil, create: nil, write: nil,
1683
1836
  projection_fields: nil, jagged_rows: nil, quoted_newlines: nil,
1684
1837
  encoding: nil, delimiter: nil, ignore_unknown: nil,
1685
1838
  max_bad_records: nil, quote: nil, skip_leading: nil,
1686
1839
  autodetect: nil, null_marker: nil
1687
- job = load_job file, format: format, create: create, write: write,
1688
- projection_fields: projection_fields,
1689
- jagged_rows: jagged_rows,
1690
- quoted_newlines: quoted_newlines,
1691
- encoding: encoding, delimiter: delimiter,
1692
- ignore_unknown: ignore_unknown,
1693
- max_bad_records: max_bad_records, quote: quote,
1694
- skip_leading: skip_leading,
1695
- autodetect: autodetect, null_marker: null_marker
1840
+ ensure_service!
1696
1841
 
1842
+ updater = load_job_updater format: format, create: create,
1843
+ write: write,
1844
+ projection_fields: projection_fields,
1845
+ jagged_rows: jagged_rows,
1846
+ quoted_newlines: quoted_newlines,
1847
+ encoding: encoding,
1848
+ delimiter: delimiter,
1849
+ ignore_unknown: ignore_unknown,
1850
+ max_bad_records: max_bad_records,
1851
+ quote: quote, skip_leading: skip_leading,
1852
+ schema: schema,
1853
+ autodetect: autodetect,
1854
+ null_marker: null_marker
1855
+
1856
+ yield updater if block_given?
1857
+
1858
+ job = load_local_or_uri files, updater
1697
1859
  job.wait_until_done!
1698
-
1699
- if job.failed?
1700
- begin
1701
- # raise to activate ruby exception cause handling
1702
- raise job.gapi_error
1703
- rescue StandardError => e
1704
- # wrap Google::Apis::Error with Google::Cloud::Error
1705
- raise Google::Cloud::Error.from_error(e)
1706
- end
1707
- end
1708
-
1860
+ ensure_job_succeeded! job
1709
1861
  true
1710
1862
  end
1711
1863
 
@@ -2049,29 +2201,157 @@ module Google
2049
2201
  reload!
2050
2202
  end
2051
2203
 
2052
- def load_storage url, options = {}
2204
+ def ensure_job_succeeded! job
2205
+ return unless job.failed?
2206
+ begin
2207
+ # raise to activate ruby exception cause handling
2208
+ raise job.gapi_error
2209
+ rescue StandardError => e
2210
+ # wrap Google::Apis::Error with Google::Cloud::Error
2211
+ raise Google::Cloud::Error.from_error(e)
2212
+ end
2213
+ end
2214
+
2215
+ def load_job_gapi table_id, dryrun, job_id: nil, prefix: nil
2216
+ job_ref = service.job_ref_from job_id, prefix
2217
+ Google::Apis::BigqueryV2::Job.new(
2218
+ job_reference: job_ref,
2219
+ configuration: Google::Apis::BigqueryV2::JobConfiguration.new(
2220
+ load: Google::Apis::BigqueryV2::JobConfigurationLoad.new(
2221
+ destination_table: Google::Apis::BigqueryV2::TableReference.new(
2222
+ project_id: @service.project,
2223
+ dataset_id: dataset_id,
2224
+ table_id: table_id
2225
+ )
2226
+ ),
2227
+ dry_run: dryrun
2228
+ )
2229
+ )
2230
+ end
2231
+
2232
+ def load_job_csv_options! job, jagged_rows: nil,
2233
+ quoted_newlines: nil,
2234
+ delimiter: nil,
2235
+ quote: nil, skip_leading: nil,
2236
+ null_marker: nil
2237
+ job.jagged_rows = jagged_rows unless jagged_rows.nil?
2238
+ job.quoted_newlines = quoted_newlines unless quoted_newlines.nil?
2239
+ job.delimiter = delimiter unless delimiter.nil?
2240
+ job.null_marker = null_marker unless null_marker.nil?
2241
+ job.quote = quote unless quote.nil?
2242
+ job.skip_leading = skip_leading unless skip_leading.nil?
2243
+ end
2244
+
2245
+ def load_job_file_options! job, format: nil,
2246
+ projection_fields: nil,
2247
+ jagged_rows: nil, quoted_newlines: nil,
2248
+ encoding: nil, delimiter: nil,
2249
+ ignore_unknown: nil, max_bad_records: nil,
2250
+ quote: nil, skip_leading: nil,
2251
+ null_marker: nil
2252
+ job.format = format unless format.nil?
2253
+ unless projection_fields.nil?
2254
+ job.projection_fields = projection_fields
2255
+ end
2256
+ job.encoding = encoding unless encoding.nil?
2257
+ job.ignore_unknown = ignore_unknown unless ignore_unknown.nil?
2258
+ job.max_bad_records = max_bad_records unless max_bad_records.nil?
2259
+ load_job_csv_options! job, jagged_rows: jagged_rows,
2260
+ quoted_newlines: quoted_newlines,
2261
+ delimiter: delimiter,
2262
+ quote: quote,
2263
+ skip_leading: skip_leading,
2264
+ null_marker: null_marker
2265
+ end
2266
+
2267
+ def load_job_updater format: nil, create: nil,
2268
+ write: nil, projection_fields: nil,
2269
+ jagged_rows: nil, quoted_newlines: nil,
2270
+ encoding: nil, delimiter: nil,
2271
+ ignore_unknown: nil, max_bad_records: nil,
2272
+ quote: nil, skip_leading: nil, dryrun: nil,
2273
+ schema: nil, job_id: nil, prefix: nil, labels: nil,
2274
+ autodetect: nil, null_marker: nil
2275
+ new_job = load_job_gapi table_id, dryrun, job_id: job_id,
2276
+ prefix: prefix
2277
+ LoadJob::Updater.new(new_job).tap do |job|
2278
+ job.location = location if location # may be table reference
2279
+ job.create = create unless create.nil?
2280
+ job.write = write unless write.nil?
2281
+ job.schema = schema unless schema.nil?
2282
+ job.autodetect = autodetect unless autodetect.nil?
2283
+ job.labels = labels unless labels.nil?
2284
+ load_job_file_options! job, format: format,
2285
+ projection_fields: projection_fields,
2286
+ jagged_rows: jagged_rows,
2287
+ quoted_newlines: quoted_newlines,
2288
+ encoding: encoding,
2289
+ delimiter: delimiter,
2290
+ ignore_unknown: ignore_unknown,
2291
+ max_bad_records: max_bad_records,
2292
+ quote: quote,
2293
+ skip_leading: skip_leading,
2294
+ null_marker: null_marker
2295
+ end
2296
+ end
2297
+
2298
+ def load_storage urls, job_gapi
2053
2299
  # Convert to storage URL
2054
- url = url.to_gs_url if url.respond_to? :to_gs_url
2055
- url = url.to_s if url.is_a? URI
2300
+ urls = [urls].flatten.map do |url|
2301
+ if url.respond_to? :to_gs_url
2302
+ url.to_gs_url
2303
+ elsif url.is_a? URI
2304
+ url.to_s
2305
+ else
2306
+ url
2307
+ end
2308
+ end
2309
+
2310
+ unless urls.nil?
2311
+ job_gapi.configuration.load.update! source_uris: urls
2312
+ if job_gapi.configuration.load.source_format.nil?
2313
+ source_format = Convert.derive_source_format_from_list urls
2314
+ unless source_format.nil?
2315
+ job_gapi.configuration.load.source_format = source_format
2316
+ end
2317
+ end
2318
+ end
2056
2319
 
2057
- gapi = service.load_table_gs_url dataset_id, table_id, url, options
2320
+ gapi = service.load_table_gs_url job_gapi
2058
2321
  Job.from_gapi gapi, service
2059
2322
  end
2060
2323
 
2061
- def load_local file, options = {}
2062
- # Convert to storage URL
2063
- file = file.to_gs_url if file.respond_to? :to_gs_url
2324
+ def load_local file, job_gapi
2325
+ path = Pathname(file).to_path
2326
+ if job_gapi.configuration.load.source_format.nil?
2327
+ source_format = Convert.derive_source_format path
2328
+ unless source_format.nil?
2329
+ job_gapi.configuration.load.source_format = source_format
2330
+ end
2331
+ end
2064
2332
 
2065
- gapi = service.load_table_file dataset_id, table_id, file, options
2333
+ gapi = service.load_table_file file, job_gapi
2066
2334
  Job.from_gapi gapi, service
2067
2335
  end
2068
2336
 
2069
- def storage_url? file
2070
- file.respond_to?(:to_gs_url) ||
2071
- (file.respond_to?(:to_str) &&
2072
- file.to_str.downcase.start_with?("gs://")) ||
2073
- (file.is_a?(URI) &&
2074
- file.to_s.downcase.start_with?("gs://"))
2337
+ def load_local_or_uri file, updater
2338
+ job_gapi = updater.to_gapi
2339
+ job = if local_file? file
2340
+ load_local file, job_gapi
2341
+ else
2342
+ load_storage file, job_gapi
2343
+ end
2344
+ job
2345
+ end
2346
+
2347
+ def storage_url? files
2348
+ [files].flatten.all? do |file|
2349
+ file.respond_to?(:to_gs_url) ||
2350
+ (file.respond_to?(:to_str) &&
2351
+ file.to_str.downcase.start_with?("gs://")) ||
2352
+ (file.is_a?(URI) &&
2353
+ file.to_s.downcase.start_with?("gs://"))
2354
+ end
2075
2355
  end
2076
2356
 
2077
2357
  def local_file? file