google-cloud-bigquery 1.14.0 → 1.42.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (51) hide show
  1. checksums.yaml +4 -4
  2. data/AUTHENTICATION.md +17 -54
  3. data/CHANGELOG.md +377 -0
  4. data/CONTRIBUTING.md +328 -116
  5. data/LOGGING.md +1 -1
  6. data/OVERVIEW.md +21 -20
  7. data/TROUBLESHOOTING.md +2 -8
  8. data/lib/google/cloud/bigquery/argument.rb +197 -0
  9. data/lib/google/cloud/bigquery/convert.rb +155 -173
  10. data/lib/google/cloud/bigquery/copy_job.rb +74 -26
  11. data/lib/google/cloud/bigquery/credentials.rb +5 -12
  12. data/lib/google/cloud/bigquery/data.rb +109 -18
  13. data/lib/google/cloud/bigquery/dataset/access.rb +474 -52
  14. data/lib/google/cloud/bigquery/dataset/list.rb +7 -13
  15. data/lib/google/cloud/bigquery/dataset/tag.rb +67 -0
  16. data/lib/google/cloud/bigquery/dataset.rb +1044 -287
  17. data/lib/google/cloud/bigquery/external/avro_source.rb +107 -0
  18. data/lib/google/cloud/bigquery/external/bigtable_source/column.rb +404 -0
  19. data/lib/google/cloud/bigquery/external/bigtable_source/column_family.rb +945 -0
  20. data/lib/google/cloud/bigquery/external/bigtable_source.rb +230 -0
  21. data/lib/google/cloud/bigquery/external/csv_source.rb +481 -0
  22. data/lib/google/cloud/bigquery/external/data_source.rb +771 -0
  23. data/lib/google/cloud/bigquery/external/json_source.rb +170 -0
  24. data/lib/google/cloud/bigquery/external/parquet_source.rb +148 -0
  25. data/lib/google/cloud/bigquery/external/sheets_source.rb +166 -0
  26. data/lib/google/cloud/bigquery/external.rb +50 -2256
  27. data/lib/google/cloud/bigquery/extract_job.rb +226 -61
  28. data/lib/google/cloud/bigquery/insert_response.rb +1 -3
  29. data/lib/google/cloud/bigquery/job/list.rb +10 -14
  30. data/lib/google/cloud/bigquery/job.rb +289 -14
  31. data/lib/google/cloud/bigquery/load_job.rb +810 -136
  32. data/lib/google/cloud/bigquery/model/list.rb +5 -9
  33. data/lib/google/cloud/bigquery/model.rb +247 -16
  34. data/lib/google/cloud/bigquery/policy.rb +432 -0
  35. data/lib/google/cloud/bigquery/project/list.rb +6 -11
  36. data/lib/google/cloud/bigquery/project.rb +509 -250
  37. data/lib/google/cloud/bigquery/query_job.rb +594 -128
  38. data/lib/google/cloud/bigquery/routine/list.rb +165 -0
  39. data/lib/google/cloud/bigquery/routine.rb +1227 -0
  40. data/lib/google/cloud/bigquery/schema/field.rb +413 -63
  41. data/lib/google/cloud/bigquery/schema.rb +221 -48
  42. data/lib/google/cloud/bigquery/service.rb +204 -112
  43. data/lib/google/cloud/bigquery/standard_sql.rb +269 -53
  44. data/lib/google/cloud/bigquery/table/async_inserter.rb +86 -43
  45. data/lib/google/cloud/bigquery/table/list.rb +6 -11
  46. data/lib/google/cloud/bigquery/table.rb +1470 -377
  47. data/lib/google/cloud/bigquery/time.rb +6 -0
  48. data/lib/google/cloud/bigquery/version.rb +1 -1
  49. data/lib/google/cloud/bigquery.rb +4 -6
  50. data/lib/google-cloud-bigquery.rb +14 -13
  51. metadata +66 -38
@@ -37,8 +37,8 @@ module Google
37
37
  # bigquery = Google::Cloud::Bigquery.new
38
38
  # dataset = bigquery.dataset "my_dataset"
39
39
  #
40
- # gs_url = "gs://my-bucket/file-name.csv"
41
- # load_job = dataset.load_job "my_new_table", gs_url do |schema|
40
+ # gcs_uri = "gs://my-bucket/file-name.csv"
41
+ # load_job = dataset.load_job "my_new_table", gcs_uri do |schema|
42
42
  # schema.string "first_name", mode: :required
43
43
  # schema.record "cities_lived", mode: :repeated do |nested_schema|
44
44
  # nested_schema.string "place", mode: :required
@@ -62,14 +62,18 @@ module Google
62
62
  # The table into which the operation loads data. This is the table on
63
63
  # which {Table#load_job} was invoked.
64
64
  #
65
+ # @param [String] view Specifies the view that determines which table information is returned.
66
+ # By default, basic table information and storage statistics (STORAGE_STATS) are returned.
67
+ # Accepted values include `:unspecified`, `:basic`, `:storage`, and
68
+ # `:full`. For more information, see [BigQuery Classes](@todo: Update the link).
69
+ # The default value is the `:unspecified` view type.
70
+ #
65
71
  # @return [Table] A table instance.
66
72
  #
67
- def destination
73
+ def destination view: nil
68
74
  table = @gapi.configuration.load.destination_table
69
75
  return nil unless table
70
- retrieve_table table.project_id,
71
- table.dataset_id,
72
- table.table_id
76
+ retrieve_table table.project_id, table.dataset_id, table.table_id, metadata_view: view
73
77
  end
74
78
 
75
79
  ##
@@ -114,8 +118,7 @@ module Google
114
118
  # `false` otherwise.
115
119
  #
116
120
  def iso8859_1?
117
- val = @gapi.configuration.load.encoding
118
- val == "ISO-8859-1"
121
+ @gapi.configuration.load.encoding == "ISO-8859-1"
119
122
  end
120
123
 
121
124
  ##
@@ -191,14 +194,13 @@ module Google
191
194
 
192
195
  ##
193
196
  # Checks if the format of the source data is [newline-delimited
194
- # JSON](http://jsonlines.org/). The default is `false`.
197
+ # JSON](https://jsonlines.org/). The default is `false`.
195
198
  #
196
199
  # @return [Boolean] `true` when the source format is
197
200
  # `NEWLINE_DELIMITED_JSON`, `false` otherwise.
198
201
  #
199
202
  def json?
200
- val = @gapi.configuration.load.source_format
201
- val == "NEWLINE_DELIMITED_JSON"
203
+ @gapi.configuration.load.source_format == "NEWLINE_DELIMITED_JSON"
202
204
  end
203
205
 
204
206
  ##
@@ -220,8 +222,27 @@ module Google
220
222
  # `false` otherwise.
221
223
  #
222
224
  def backup?
223
- val = @gapi.configuration.load.source_format
224
- val == "DATASTORE_BACKUP"
225
+ @gapi.configuration.load.source_format == "DATASTORE_BACKUP"
226
+ end
227
+
228
+ ##
229
+ # Checks if the source format is ORC.
230
+ #
231
+ # @return [Boolean] `true` when the source format is `ORC`,
232
+ # `false` otherwise.
233
+ #
234
+ def orc?
235
+ @gapi.configuration.load.source_format == "ORC"
236
+ end
237
+
238
+ ##
239
+ # Checks if the source format is Parquet.
240
+ #
241
+ # @return [Boolean] `true` when the source format is `PARQUET`,
242
+ # `false` otherwise.
243
+ #
244
+ def parquet?
245
+ @gapi.configuration.load.source_format == "PARQUET"
225
246
  end
226
247
 
227
248
  ##
@@ -350,10 +371,169 @@ module Google
350
371
  end
351
372
 
352
373
  ###
353
- # Checks if the destination table will be time-partitioned. See
374
+ # Checks if hive partitioning options are set.
375
+ #
376
+ # @see https://cloud.google.com/bigquery/docs/hive-partitioned-loads-gcs Loading externally partitioned data
377
+ #
378
+ # @return [Boolean] `true` when hive partitioning options are set, or `false` otherwise.
379
+ #
380
+ # @!group Attributes
381
+ #
382
+ def hive_partitioning?
383
+ !@gapi.configuration.load.hive_partitioning_options.nil?
384
+ end
385
+
386
+ ###
387
+ # The mode of hive partitioning to use when reading data. The following modes are supported:
388
+ #
389
+ # 1. `AUTO`: automatically infer partition key name(s) and type(s).
390
+ # 2. `STRINGS`: automatically infer partition key name(s). All types are interpreted as strings.
391
+ # 3. `CUSTOM`: partition key schema is encoded in the source URI prefix.
392
+ #
393
+ # @see https://cloud.google.com/bigquery/docs/hive-partitioned-loads-gcs Loading externally partitioned data
394
+ #
395
+ # @return [String, nil] The mode of hive partitioning, or `nil` if not set.
396
+ #
397
+ # @!group Attributes
398
+ #
399
+ def hive_partitioning_mode
400
+ @gapi.configuration.load.hive_partitioning_options.mode if hive_partitioning?
401
+ end
402
+
403
+ ###
404
+ # The common prefix for all source uris when hive partition detection is requested. The prefix must end
405
+ # immediately before the partition key encoding begins. For example, consider files following this data layout:
406
+ #
407
+ # ```
408
+ # gs://bucket/path_to_table/dt=2019-01-01/country=BR/id=7/file.avro
409
+ # gs://bucket/path_to_table/dt=2018-12-31/country=CA/id=3/file.avro
410
+ # ```
411
+ #
412
+ # When hive partitioning is requested with either `AUTO` or `STRINGS` mode, the common prefix can be either of
413
+ # `gs://bucket/path_to_table` or `gs://bucket/path_to_table/` (trailing slash does not matter).
414
+ #
415
+ # @see https://cloud.google.com/bigquery/docs/hive-partitioned-loads-gcs Loading externally partitioned data
416
+ #
417
+ # @return [String, nil] The common prefix for all source uris, or `nil` if not set.
418
+ #
419
+ # @!group Attributes
420
+ #
421
+ def hive_partitioning_source_uri_prefix
422
+ @gapi.configuration.load.hive_partitioning_options.source_uri_prefix if hive_partitioning?
423
+ end
424
+
425
+ ###
426
+ # Checks if Parquet options are set.
427
+ #
428
+ # @see https://cloud.google.com/bigquery/docs/loading-data-cloud-storage-parquet Loading Parquet data from Cloud
429
+ # Storage
430
+ #
431
+ # @return [Boolean] `true` when Parquet options are set, or `false` otherwise.
432
+ #
433
+ # @!group Attributes
434
+ #
435
+ def parquet_options?
436
+ !@gapi.configuration.load.parquet_options.nil?
437
+ end
438
+
439
+ ###
440
+ # Indicates whether to use schema inference specifically for Parquet `LIST` logical type.
441
+ #
442
+ # @see https://cloud.google.com/bigquery/docs/loading-data-cloud-storage-parquet Loading Parquet data from Cloud
443
+ # Storage
444
+ #
445
+ # @return [Boolean, nil] The `enable_list_inference` value in Parquet options, or `nil` if Parquet options are
446
+ # not set.
447
+ #
448
+ # @!group Attributes
449
+ #
450
+ def parquet_enable_list_inference?
451
+ @gapi.configuration.load.parquet_options.enable_list_inference if parquet_options?
452
+ end
453
+
454
+ ###
455
+ # Indicates whether to infer Parquet `ENUM` logical type as `STRING` instead of `BYTES` by default.
456
+ #
457
+ # @see https://cloud.google.com/bigquery/docs/loading-data-cloud-storage-parquet Loading Parquet data from Cloud
458
+ # Storage
459
+ #
460
+ # @return [Boolean, nil] The `enum_as_string` value in Parquet options, or `nil` if Parquet options are not set.
461
+ #
462
+ # @!group Attributes
463
+ #
464
+ def parquet_enum_as_string?
465
+ @gapi.configuration.load.parquet_options.enum_as_string if parquet_options?
466
+ end
467
+
468
+ ###
469
+ # Checks if the destination table will be range partitioned. See [Creating and using integer range partitioned
470
+ # tables](https://cloud.google.com/bigquery/docs/creating-integer-range-partitions).
471
+ #
472
+ # @return [Boolean] `true` when the table is range partitioned, or `false` otherwise.
473
+ #
474
+ # @!group Attributes
475
+ #
476
+ def range_partitioning?
477
+ !@gapi.configuration.load.range_partitioning.nil?
478
+ end
479
+
480
+ ###
481
+ # The field on which the destination table will be range partitioned, if any. The field must be a
482
+ # top-level `NULLABLE/REQUIRED` field. The only supported type is `INTEGER/INT64`. See
483
+ # [Creating and using integer range partitioned
484
+ # tables](https://cloud.google.com/bigquery/docs/creating-integer-range-partitions).
485
+ #
486
+ # @return [String, nil] The partition field, if a field was configured, or `nil` if not range partitioned.
487
+ #
488
+ # @!group Attributes
489
+ #
490
+ def range_partitioning_field
491
+ @gapi.configuration.load.range_partitioning.field if range_partitioning?
492
+ end
493
+
494
+ ###
495
+ # The start of range partitioning, inclusive. See [Creating and using integer range partitioned
496
+ # tables](https://cloud.google.com/bigquery/docs/creating-integer-range-partitions).
497
+ #
498
+ # @return [Integer, nil] The start of range partitioning, inclusive, or `nil` if not range partitioned.
499
+ #
500
+ # @!group Attributes
501
+ #
502
+ def range_partitioning_start
503
+ @gapi.configuration.load.range_partitioning.range.start if range_partitioning?
504
+ end
505
+
506
+ ###
507
+ # The width of each interval. See [Creating and using integer range partitioned
508
+ # tables](https://cloud.google.com/bigquery/docs/creating-integer-range-partitions).
509
+ #
510
+ # @return [Integer, nil] The width of each interval, for data in range partitions, or `nil` if not range
511
+ # partitioned.
512
+ #
513
+ # @!group Attributes
514
+ #
515
+ def range_partitioning_interval
516
+ return nil unless range_partitioning?
517
+ @gapi.configuration.load.range_partitioning.range.interval
518
+ end
519
+
520
+ ###
521
+ # The end of range partitioning, exclusive. See [Creating and using integer range partitioned
522
+ # tables](https://cloud.google.com/bigquery/docs/creating-integer-range-partitions).
523
+ #
524
+ # @return [Integer, nil] The end of range partitioning, exclusive, or `nil` if not range partitioned.
525
+ #
526
+ # @!group Attributes
527
+ #
528
+ def range_partitioning_end
529
+ @gapi.configuration.load.range_partitioning.range.end if range_partitioning?
530
+ end
531
+
532
+ ###
533
+ # Checks if the destination table will be time partitioned. See
354
534
  # [Partitioned Tables](https://cloud.google.com/bigquery/docs/partitioned-tables).
355
535
  #
356
- # @return [Boolean, nil] `true` when the table will be time-partitioned,
536
+ # @return [Boolean] `true` when the table will be time-partitioned,
357
537
  # or `false` otherwise.
358
538
  #
359
539
  # @!group Attributes
@@ -363,11 +543,12 @@ module Google
363
543
  end
364
544
 
365
545
  ###
366
- # The period for which the destination table will be partitioned, if
546
+ # The period for which the destination table will be time partitioned, if
367
547
  # any. See [Partitioned Tables](https://cloud.google.com/bigquery/docs/partitioned-tables).
368
548
  #
369
- # @return [String, nil] The partition type. Currently the only supported
370
- # value is "DAY", or `nil` if not present.
549
+ # @return [String, nil] The time partition type. The supported types are `DAY`,
550
+ # `HOUR`, `MONTH`, and `YEAR`, which will generate one partition per day,
551
+ # hour, month, and year, respectively; or `nil` if not present.
371
552
  #
372
553
  # @!group Attributes
373
554
  #
@@ -376,13 +557,13 @@ module Google
376
557
  end
377
558
 
378
559
  ###
379
- # The field on which the destination table will be partitioned, if any.
380
- # If not set, the destination table will be partitioned by pseudo column
381
- # `_PARTITIONTIME`; if set, the table will be partitioned by this field.
560
+ # The field on which the destination table will be time partitioned, if any.
561
+ # If not set, the destination table will be time partitioned by pseudo column
562
+ # `_PARTITIONTIME`; if set, the table will be time partitioned by this field.
382
563
  # See [Partitioned Tables](https://cloud.google.com/bigquery/docs/partitioned-tables).
383
564
  #
384
- # @return [String, nil] The partition field, if a field was configured.
385
- # `nil` if not partitioned or not set (partitioned by pseudo column
565
+ # @return [String, nil] The time partition field, if a field was configured.
566
+ # `nil` if not time partitioned or not set (partitioned by pseudo column
386
567
  # '_PARTITIONTIME').
387
568
  #
388
569
  # @!group Attributes
@@ -392,28 +573,29 @@ module Google
392
573
  end
393
574
 
394
575
  ###
395
- # The expiration for the destination table partitions, if any, in
576
+ # The expiration for the destination table time partitions, if any, in
396
577
  # seconds. See [Partitioned
397
578
  # Tables](https://cloud.google.com/bigquery/docs/partitioned-tables).
398
579
  #
399
580
  # @return [Integer, nil] The expiration time, in seconds, for data in
400
- # partitions, or `nil` if not present.
581
+ # time partitions, or `nil` if not present.
401
582
  #
402
583
  # @!group Attributes
403
584
  #
404
585
  def time_partitioning_expiration
405
- @gapi.configuration.load.time_partitioning.expiration_ms / 1_000 if
406
- time_partitioning? &&
407
- !@gapi.configuration.load.time_partitioning.expiration_ms.nil?
586
+ return nil unless time_partitioning?
587
+ return nil if @gapi.configuration.load.time_partitioning.expiration_ms.nil?
588
+
589
+ @gapi.configuration.load.time_partitioning.expiration_ms / 1_000
408
590
  end
409
591
 
410
592
  ###
411
593
  # If set to true, queries over the destination table will require a
412
- # partition filter that can be used for partition elimination to be
594
+ # time partition filter that can be used for partition elimination to be
413
595
  # specified. See [Partitioned
414
596
  # Tables](https://cloud.google.com/bigquery/docs/partitioned-tables).
415
597
  #
416
- # @return [Boolean] `true` when a partition filter will be required,
598
+ # @return [Boolean] `true` when a time partition filter will be required,
417
599
  # or `false` otherwise.
418
600
  #
419
601
  # @!group Attributes
@@ -427,10 +609,15 @@ module Google
427
609
  ###
428
610
  # Checks if the destination table will be clustered.
429
611
  #
612
+ # See {LoadJob::Updater#clustering_fields=}, {Table#clustering_fields} and
613
+ # {Table#clustering_fields=}.
614
+ #
430
615
  # @see https://cloud.google.com/bigquery/docs/clustered-tables
431
- # Introduction to Clustered Tables
616
+ # Introduction to clustered tables
617
+ # @see https://cloud.google.com/bigquery/docs/creating-clustered-tables
618
+ # Creating and using clustered tables
432
619
  #
433
- # @return [Boolean, nil] `true` when the table will be clustered,
620
+ # @return [Boolean] `true` when the table will be clustered,
434
621
  # or `false` otherwise.
435
622
  #
436
623
  # @!group Attributes
@@ -445,14 +632,16 @@ module Google
445
632
  # be first partitioned and subsequently clustered. The order of the
446
633
  # returned fields determines the sort order of the data.
447
634
  #
448
- # See {LoadJob::Updater#clustering_fields=}.
635
+ # BigQuery supports clustering for both partitioned and non-partitioned
636
+ # tables.
637
+ #
638
+ # See {LoadJob::Updater#clustering_fields=}, {Table#clustering_fields} and
639
+ # {Table#clustering_fields=}.
449
640
  #
450
- # @see https://cloud.google.com/bigquery/docs/partitioned-tables
451
- # Partitioned Tables
452
641
  # @see https://cloud.google.com/bigquery/docs/clustered-tables
453
- # Introduction to Clustered Tables
642
+ # Introduction to clustered tables
454
643
  # @see https://cloud.google.com/bigquery/docs/creating-clustered-tables
455
- # Creating and Using Clustered Tables
644
+ # Creating and using clustered tables
456
645
  #
457
646
  # @return [Array<String>, nil] The clustering fields, or `nil` if the
458
647
  # destination table will not be clustered.
@@ -473,6 +662,7 @@ module Google
473
662
  ##
474
663
  # @private Create an Updater object.
475
664
  def initialize gapi
665
+ super()
476
666
  @updates = []
477
667
  @gapi = gapi
478
668
  @schema = nil
@@ -560,13 +750,19 @@ module Google
560
750
  # See {Schema#string}.
561
751
  #
562
752
  # @param [String] name The field name. The name must contain only
563
- # letters (a-z, A-Z), numbers (0-9), or underscores (_), and must
753
+ # letters (`[A-Za-z]`), numbers (`[0-9]`), or underscores (`_`), and must
564
754
  # start with a letter or underscore. The maximum length is 128
565
755
  # characters.
566
756
  # @param [String] description A description of the field.
567
757
  # @param [Symbol] mode The field's mode. The possible values are
568
758
  # `:nullable`, `:required`, and `:repeated`. The default value is
569
759
  # `:nullable`.
760
+ # @param [Array<String>, String] policy_tags The policy tag list or
761
+ # single policy tag for the field. Policy tag identifiers are of
762
+ # the form `projects/*/locations/*/taxonomies/*/policyTags/*`.
763
+ # At most 1 policy tag is currently allowed.
764
+ # @param [Integer] max_length The maximum UTF-8 length of strings
765
+ # allowed in the field.
570
766
  #
571
767
  # @example
572
768
  # require "google/cloud/bigquery"
@@ -578,8 +774,8 @@ module Google
578
774
  # end
579
775
  #
580
776
  # @!group Schema
581
- def string name, description: nil, mode: :nullable
582
- schema.string name, description: description, mode: mode
777
+ def string name, description: nil, mode: :nullable, policy_tags: nil, max_length: nil
778
+ schema.string name, description: description, mode: mode, policy_tags: policy_tags, max_length: max_length
583
779
  end
584
780
 
585
781
  ##
@@ -588,13 +784,17 @@ module Google
588
784
  # See {Schema#integer}.
589
785
  #
590
786
  # @param [String] name The field name. The name must contain only
591
- # letters (a-z, A-Z), numbers (0-9), or underscores (_), and must
787
+ # letters (`[A-Za-z]`), numbers (`[0-9]`), or underscores (`_`), and must
592
788
  # start with a letter or underscore. The maximum length is 128
593
789
  # characters.
594
790
  # @param [String] description A description of the field.
595
791
  # @param [Symbol] mode The field's mode. The possible values are
596
792
  # `:nullable`, `:required`, and `:repeated`. The default value is
597
793
  # `:nullable`.
794
+ # @param [Array<String>, String] policy_tags The policy tag list or
795
+ # single policy tag for the field. Policy tag identifiers are of
796
+ # the form `projects/*/locations/*/taxonomies/*/policyTags/*`.
797
+ # At most 1 policy tag is currently allowed.
598
798
  #
599
799
  # @example
600
800
  # require "google/cloud/bigquery"
@@ -606,8 +806,8 @@ module Google
606
806
  # end
607
807
  #
608
808
  # @!group Schema
609
- def integer name, description: nil, mode: :nullable
610
- schema.integer name, description: description, mode: mode
809
+ def integer name, description: nil, mode: :nullable, policy_tags: nil
810
+ schema.integer name, description: description, mode: mode, policy_tags: policy_tags
611
811
  end
612
812
 
613
813
  ##
@@ -616,13 +816,17 @@ module Google
616
816
  # See {Schema#float}.
617
817
  #
618
818
  # @param [String] name The field name. The name must contain only
619
- # letters (a-z, A-Z), numbers (0-9), or underscores (_), and must
819
+ # letters (`[A-Za-z]`), numbers (`[0-9]`), or underscores (`_`), and must
620
820
  # start with a letter or underscore. The maximum length is 128
621
821
  # characters.
622
822
  # @param [String] description A description of the field.
623
823
  # @param [Symbol] mode The field's mode. The possible values are
624
824
  # `:nullable`, `:required`, and `:repeated`. The default value is
625
825
  # `:nullable`.
826
+ # @param [Array<String>, String] policy_tags The policy tag list or
827
+ # single policy tag for the field. Policy tag identifiers are of
828
+ # the form `projects/*/locations/*/taxonomies/*/policyTags/*`.
829
+ # At most 1 policy tag is currently allowed.
626
830
  #
627
831
  # @example
628
832
  # require "google/cloud/bigquery"
@@ -634,25 +838,48 @@ module Google
634
838
  # end
635
839
  #
636
840
  # @!group Schema
637
- def float name, description: nil, mode: :nullable
638
- schema.float name, description: description, mode: mode
841
+ def float name, description: nil, mode: :nullable, policy_tags: nil
842
+ schema.float name, description: description, mode: mode, policy_tags: policy_tags
639
843
  end
640
844
 
641
845
  ##
642
- # Adds a numeric number field to the schema. Numeric is a
643
- # fixed-precision numeric type with 38 decimal digits, 9 that follow
644
- # the decimal point.
846
+ # Adds a numeric number field to the schema. `NUMERIC` is a decimal
847
+ # type with fixed precision and scale. Precision is the number of
848
+ # digits that the number contains. Scale is how many of these
849
+ # digits appear after the decimal point. It supports:
850
+ #
851
+ # Precision: 38
852
+ # Scale: 9
853
+ # Min: -9.9999999999999999999999999999999999999E+28
854
+ # Max: 9.9999999999999999999999999999999999999E+28
855
+ #
856
+ # This type can represent decimal fractions exactly, and is suitable
857
+ # for financial calculations.
645
858
  #
646
859
  # See {Schema#numeric}
647
860
  #
648
861
  # @param [String] name The field name. The name must contain only
649
- # letters (a-z, A-Z), numbers (0-9), or underscores (_), and must
862
+ # letters (`[A-Za-z]`), numbers (`[0-9]`), or underscores (`_`), and must
650
863
  # start with a letter or underscore. The maximum length is 128
651
864
  # characters.
652
865
  # @param [String] description A description of the field.
653
866
  # @param [Symbol] mode The field's mode. The possible values are
654
867
  # `:nullable`, `:required`, and `:repeated`. The default value is
655
868
  # `:nullable`.
869
+ # @param [Array<String>, String] policy_tags The policy tag list or
870
+ # single policy tag for the field. Policy tag identifiers are of
871
+ # the form `projects/*/locations/*/taxonomies/*/policyTags/*`.
872
+ # At most 1 policy tag is currently allowed.
873
+ # @param [Integer] precision The precision (maximum number of total
874
+ # digits) for the field. Acceptable values for precision must be:
875
+ # `1 ≤ (precision - scale) ≤ 29`. Values for scale must be:
876
+ # `0 ≤ scale ≤ 9`. If the scale value is set, the precision value
877
+ # must be set as well.
878
+ # @param [Integer] scale The scale (maximum number of digits in the
879
+ # fractional part) for the field. Acceptable values for precision
880
+ # must be: `1 ≤ (precision - scale) ≤ 29`. Values for scale must
881
+ # be: `0 ≤ scale ≤ 9`. If the scale value is set, the precision
882
+ # value must be set as well.
656
883
  #
657
884
  # @example
658
885
  # require "google/cloud/bigquery"
@@ -664,8 +891,71 @@ module Google
664
891
  # end
665
892
  #
666
893
  # @!group Schema
667
- def numeric name, description: nil, mode: :nullable
668
- schema.numeric name, description: description, mode: mode
894
+ def numeric name, description: nil, mode: :nullable, policy_tags: nil, precision: nil, scale: nil
895
+ schema.numeric name,
896
+ description: description,
897
+ mode: mode,
898
+ policy_tags: policy_tags,
899
+ precision: precision,
900
+ scale: scale
901
+ end
902
+
903
+ ##
904
+ # Adds a bignumeric number field to the schema. `BIGNUMERIC` is a
905
+ # decimal type with fixed precision and scale. Precision is the
906
+ # number of digits that the number contains. Scale is how many of
907
+ # these digits appear after the decimal point. It supports:
908
+ #
909
+ # Precision: 76.76 (the 77th digit is partial)
910
+ # Scale: 38
911
+ # Min: -5.7896044618658097711785492504343953926634992332820282019728792003956564819968E+38
912
+ # Max: 5.7896044618658097711785492504343953926634992332820282019728792003956564819967E+38
913
+ #
914
+ # This type can represent decimal fractions exactly, and is suitable
915
+ # for financial calculations.
916
+ #
917
+ # See {Schema#bignumeric}
918
+ #
919
+ # @param [String] name The field name. The name must contain only
920
+ # letters (`[A-Za-z]`), numbers (`[0-9]`), or underscores (`_`), and must
921
+ # start with a letter or underscore. The maximum length is 128
922
+ # characters.
923
+ # @param [String] description A description of the field.
924
+ # @param [Symbol] mode The field's mode. The possible values are
925
+ # `:nullable`, `:required`, and `:repeated`. The default value is
926
+ # `:nullable`.
927
+ # @param [Array<String>, String] policy_tags The policy tag list or
928
+ # single policy tag for the field. Policy tag identifiers are of
929
+ # the form `projects/*/locations/*/taxonomies/*/policyTags/*`.
930
+ # At most 1 policy tag is currently allowed.
931
+ # @param [Integer] precision The precision (maximum number of total
932
+ # digits) for the field. Acceptable values for precision must be:
933
+ # `1 ≤ (precision - scale) ≤ 38`. Values for scale must be:
934
+ # `0 ≤ scale ≤ 38`. If the scale value is set, the precision value
935
+ # must be set as well.
936
+ # @param [Integer] scale The scale (maximum number of digits in the
937
+ # fractional part) for the field. Acceptable values for precision
938
+ # must be: `1 ≤ (precision - scale) ≤ 38`. Values for scale must
939
+ # be: `0 ≤ scale ≤ 38`. If the scale value is set, the precision
940
+ # value must be set as well.
941
+ #
942
+ # @example
943
+ # require "google/cloud/bigquery"
944
+ #
945
+ # bigquery = Google::Cloud::Bigquery.new
946
+ # dataset = bigquery.dataset "my_dataset"
947
+ # job = dataset.load_job "my_table", "gs://abc/file" do |schema|
948
+ # schema.bignumeric "total_cost", mode: :required
949
+ # end
950
+ #
951
+ # @!group Schema
952
+ def bignumeric name, description: nil, mode: :nullable, policy_tags: nil, precision: nil, scale: nil
953
+ schema.bignumeric name,
954
+ description: description,
955
+ mode: mode,
956
+ policy_tags: policy_tags,
957
+ precision: precision,
958
+ scale: scale
669
959
  end
670
960
 
671
961
  ##
@@ -674,13 +964,17 @@ module Google
674
964
  # See {Schema#boolean}.
675
965
  #
676
966
  # @param [String] name The field name. The name must contain only
677
- # letters (a-z, A-Z), numbers (0-9), or underscores (_), and must
967
+ # letters (`[A-Za-z]`), numbers (`[0-9]`), or underscores (`_`), and must
678
968
  # start with a letter or underscore. The maximum length is 128
679
969
  # characters.
680
970
  # @param [String] description A description of the field.
681
971
  # @param [Symbol] mode The field's mode. The possible values are
682
972
  # `:nullable`, `:required`, and `:repeated`. The default value is
683
973
  # `:nullable`.
974
+ # @param [Array<String>, String] policy_tags The policy tag list or
975
+ # single policy tag for the field. Policy tag identifiers are of
976
+ # the form `projects/*/locations/*/taxonomies/*/policyTags/*`.
977
+ # At most 1 policy tag is currently allowed.
684
978
  #
685
979
  # @example
686
980
  # require "google/cloud/bigquery"
@@ -692,8 +986,8 @@ module Google
692
986
  # end
693
987
  #
694
988
  # @!group Schema
695
- def boolean name, description: nil, mode: :nullable
696
- schema.boolean name, description: description, mode: mode
989
+ def boolean name, description: nil, mode: :nullable, policy_tags: nil
990
+ schema.boolean name, description: description, mode: mode, policy_tags: policy_tags
697
991
  end
698
992
 
699
993
  ##
@@ -702,13 +996,19 @@ module Google
702
996
  # See {Schema#bytes}.
703
997
  #
704
998
  # @param [String] name The field name. The name must contain only
705
- # letters (a-z, A-Z), numbers (0-9), or underscores (_), and must
999
+ # letters (`[A-Za-z]`), numbers (`[0-9]`), or underscores (`_`), and must
706
1000
  # start with a letter or underscore. The maximum length is 128
707
1001
  # characters.
708
1002
  # @param [String] description A description of the field.
709
1003
  # @param [Symbol] mode The field's mode. The possible values are
710
1004
  # `:nullable`, `:required`, and `:repeated`. The default value is
711
1005
  # `:nullable`.
1006
+ # @param [Array<String>, String] policy_tags The policy tag list or
1007
+ # single policy tag for the field. Policy tag identifiers are of
1008
+ # the form `projects/*/locations/*/taxonomies/*/policyTags/*`.
1009
+ # At most 1 policy tag is currently allowed.
1010
+ # @param [Integer] max_length The maximum the maximum number of
1011
+ # bytes in the field.
712
1012
  #
713
1013
  # @example
714
1014
  # require "google/cloud/bigquery"
@@ -720,8 +1020,8 @@ module Google
720
1020
  # end
721
1021
  #
722
1022
  # @!group Schema
723
- def bytes name, description: nil, mode: :nullable
724
- schema.bytes name, description: description, mode: mode
1023
+ def bytes name, description: nil, mode: :nullable, policy_tags: nil, max_length: nil
1024
+ schema.bytes name, description: description, mode: mode, policy_tags: policy_tags, max_length: max_length
725
1025
  end
726
1026
 
727
1027
  ##
@@ -730,13 +1030,17 @@ module Google
730
1030
  # See {Schema#timestamp}.
731
1031
  #
732
1032
  # @param [String] name The field name. The name must contain only
733
- # letters (a-z, A-Z), numbers (0-9), or underscores (_), and must
1033
+ # letters (`[A-Za-z]`), numbers (`[0-9]`), or underscores (`_`), and must
734
1034
  # start with a letter or underscore. The maximum length is 128
735
1035
  # characters.
736
1036
  # @param [String] description A description of the field.
737
1037
  # @param [Symbol] mode The field's mode. The possible values are
738
1038
  # `:nullable`, `:required`, and `:repeated`. The default value is
739
1039
  # `:nullable`.
1040
+ # @param [Array<String>, String] policy_tags The policy tag list or
1041
+ # single policy tag for the field. Policy tag identifiers are of
1042
+ # the form `projects/*/locations/*/taxonomies/*/policyTags/*`.
1043
+ # At most 1 policy tag is currently allowed.
740
1044
  #
741
1045
  # @example
742
1046
  # require "google/cloud/bigquery"
@@ -748,8 +1052,8 @@ module Google
748
1052
  # end
749
1053
  #
750
1054
  # @!group Schema
751
- def timestamp name, description: nil, mode: :nullable
752
- schema.timestamp name, description: description, mode: mode
1055
+ def timestamp name, description: nil, mode: :nullable, policy_tags: nil
1056
+ schema.timestamp name, description: description, mode: mode, policy_tags: policy_tags
753
1057
  end
754
1058
 
755
1059
  ##
@@ -758,13 +1062,17 @@ module Google
758
1062
  # See {Schema#time}.
759
1063
  #
760
1064
  # @param [String] name The field name. The name must contain only
761
- # letters (a-z, A-Z), numbers (0-9), or underscores (_), and must
1065
+ # letters (`[A-Za-z]`), numbers (`[0-9]`), or underscores (`_`), and must
762
1066
  # start with a letter or underscore. The maximum length is 128
763
1067
  # characters.
764
1068
  # @param [String] description A description of the field.
765
1069
  # @param [Symbol] mode The field's mode. The possible values are
766
1070
  # `:nullable`, `:required`, and `:repeated`. The default value is
767
1071
  # `:nullable`.
1072
+ # @param [Array<String>, String] policy_tags The policy tag list or
1073
+ # single policy tag for the field. Policy tag identifiers are of
1074
+ # the form `projects/*/locations/*/taxonomies/*/policyTags/*`.
1075
+ # At most 1 policy tag is currently allowed.
768
1076
  #
769
1077
  # @example
770
1078
  # require "google/cloud/bigquery"
@@ -776,8 +1084,8 @@ module Google
776
1084
  # end
777
1085
  #
778
1086
  # @!group Schema
779
- def time name, description: nil, mode: :nullable
780
- schema.time name, description: description, mode: mode
1087
+ def time name, description: nil, mode: :nullable, policy_tags: nil
1088
+ schema.time name, description: description, mode: mode, policy_tags: policy_tags
781
1089
  end
782
1090
 
783
1091
  ##
@@ -786,13 +1094,17 @@ module Google
786
1094
  # See {Schema#datetime}.
787
1095
  #
788
1096
  # @param [String] name The field name. The name must contain only
789
- # letters (a-z, A-Z), numbers (0-9), or underscores (_), and must
1097
+ # letters (`[A-Za-z]`), numbers (`[0-9]`), or underscores (`_`), and must
790
1098
  # start with a letter or underscore. The maximum length is 128
791
1099
  # characters.
792
1100
  # @param [String] description A description of the field.
793
1101
  # @param [Symbol] mode The field's mode. The possible values are
794
1102
  # `:nullable`, `:required`, and `:repeated`. The default value is
795
1103
  # `:nullable`.
1104
+ # @param [Array<String>, String] policy_tags The policy tag list or
1105
+ # single policy tag for the field. Policy tag identifiers are of
1106
+ # the form `projects/*/locations/*/taxonomies/*/policyTags/*`.
1107
+ # At most 1 policy tag is currently allowed.
796
1108
  #
797
1109
  # @example
798
1110
  # require "google/cloud/bigquery"
@@ -804,8 +1116,8 @@ module Google
804
1116
  # end
805
1117
  #
806
1118
  # @!group Schema
807
- def datetime name, description: nil, mode: :nullable
808
- schema.datetime name, description: description, mode: mode
1119
+ def datetime name, description: nil, mode: :nullable, policy_tags: nil
1120
+ schema.datetime name, description: description, mode: mode, policy_tags: policy_tags
809
1121
  end
810
1122
 
811
1123
  ##
@@ -814,13 +1126,17 @@ module Google
814
1126
  # See {Schema#date}.
815
1127
  #
816
1128
  # @param [String] name The field name. The name must contain only
817
- # letters (a-z, A-Z), numbers (0-9), or underscores (_), and must
1129
+ # letters (`[A-Za-z]`), numbers (`[0-9]`), or underscores (`_`), and must
818
1130
  # start with a letter or underscore. The maximum length is 128
819
1131
  # characters.
820
1132
  # @param [String] description A description of the field.
821
1133
  # @param [Symbol] mode The field's mode. The possible values are
822
1134
  # `:nullable`, `:required`, and `:repeated`. The default value is
823
1135
  # `:nullable`.
1136
+ # @param [Array<String>, String] policy_tags The policy tag list or
1137
+ # single policy tag for the field. Policy tag identifiers are of
1138
+ # the form `projects/*/locations/*/taxonomies/*/policyTags/*`.
1139
+ # At most 1 policy tag is currently allowed.
824
1140
  #
825
1141
  # @example
826
1142
  # require "google/cloud/bigquery"
@@ -832,8 +1148,44 @@ module Google
832
1148
  # end
833
1149
  #
834
1150
  # @!group Schema
835
- def date name, description: nil, mode: :nullable
836
- schema.date name, description: description, mode: mode
1151
+ def date name, description: nil, mode: :nullable, policy_tags: nil
1152
+ schema.date name, description: description, mode: mode, policy_tags: policy_tags
1153
+ end
1154
+
1155
+ ##
1156
+ # Adds a geography field to the schema.
1157
+ #
1158
+ # See {Schema#geography}.
1159
+ #
1160
+ # @see https://cloud.google.com/bigquery/docs/gis-data Working with BigQuery GIS data
1161
+ #
1162
+ # @param [String] name The field name. The name must contain only
1163
+ # letters (`[A-Za-z]`), numbers (`[0-9]`), or underscores (`_`), and must
1164
+ # start with a letter or underscore. The maximum length is 128
1165
+ # characters.
1166
+ # @param [String] description A description of the field.
1167
+ # @param [Symbol] mode The field's mode. The possible values are
1168
+ # `:nullable`, `:required`, and `:repeated`. The default value is
1169
+ # `:nullable`.
1170
+ # @param [Array<String>, String] policy_tags The policy tag list or
1171
+ # single policy tag for the field. Policy tag identifiers are of
1172
+ # the form `projects/*/locations/*/taxonomies/*/policyTags/*`.
1173
+ # At most 1 policy tag is currently allowed.
1174
+ #
1175
+ # @example
1176
+ # require "google/cloud/bigquery"
1177
+ #
1178
+ # bigquery = Google::Cloud::Bigquery.new
1179
+ # dataset = bigquery.dataset "my_dataset"
1180
+ # job = dataset.load_job "my_table", "gs://abc/file" do |schema|
1181
+ # schema.record "cities_lived", mode: :repeated do |cities_lived|
1182
+ # cities_lived.geography "location", mode: :required
1183
+ # cities_lived.integer "number_of_years", mode: :required
1184
+ # end
1185
+ # end
1186
+ #
1187
+ def geography name, description: nil, mode: :nullable, policy_tags: nil
1188
+ schema.geography name, description: description, mode: mode, policy_tags: policy_tags
837
1189
  end
838
1190
 
839
1191
  ##
@@ -846,7 +1198,7 @@ module Google
846
1198
  # See {Schema#record}.
847
1199
  #
848
1200
  # @param [String] name The field name. The name must contain only
849
- # letters (a-z, A-Z), numbers (0-9), or underscores (_), and must
1201
+ # letters (`[A-Za-z]`), numbers (`[0-9]`), or underscores (`_`), and must
850
1202
  # start with a letter or underscore. The maximum length is 128
851
1203
  # characters.
852
1204
  # @param [String] description A description of the field.
@@ -923,7 +1275,7 @@ module Google
923
1275
  # The following values are supported:
924
1276
  #
925
1277
  # * `csv` - CSV
926
- # * `json` - [Newline-delimited JSON](http://jsonlines.org/)
1278
+ # * `json` - [Newline-delimited JSON](https://jsonlines.org/)
927
1279
  # * `avro` - [Avro](http://avro.apache.org/)
928
1280
  # * `orc` - [ORC](https://cloud.google.com/bigquery/docs/loading-data-cloud-storage-orc)
929
1281
  # * `parquet` - [Parquet](https://parquet.apache.org/)
@@ -934,8 +1286,7 @@ module Google
934
1286
  # @!group Attributes
935
1287
  #
936
1288
  def format= new_format
937
- @gapi.configuration.load.update! \
938
- source_format: Convert.source_format(new_format)
1289
+ @gapi.configuration.load.update! source_format: Convert.source_format(new_format)
939
1290
  end
940
1291
 
941
1292
  ##
@@ -955,8 +1306,7 @@ module Google
955
1306
  # @!group Attributes
956
1307
  #
957
1308
  def create= new_create
958
- @gapi.configuration.load.update! \
959
- create_disposition: Convert.create_disposition(new_create)
1309
+ @gapi.configuration.load.update! create_disposition: Convert.create_disposition(new_create)
960
1310
  end
961
1311
 
962
1312
  ##
@@ -977,8 +1327,7 @@ module Google
977
1327
  # @!group Attributes
978
1328
  #
979
1329
  def write= new_write
980
- @gapi.configuration.load.update! \
981
- write_disposition: Convert.write_disposition(new_write)
1330
+ @gapi.configuration.load.update! write_disposition: Convert.write_disposition(new_write)
982
1331
  end
983
1332
 
984
1333
  ##
@@ -999,8 +1348,7 @@ module Google
999
1348
  if new_fields.nil?
1000
1349
  @gapi.configuration.load.update! projection_fields: nil
1001
1350
  else
1002
- @gapi.configuration.load.update! \
1003
- projection_fields: Array(new_fields)
1351
+ @gapi.configuration.load.update! projection_fields: Array(new_fields)
1004
1352
  end
1005
1353
  end
1006
1354
 
@@ -1199,8 +1547,7 @@ module Google
1199
1547
  if new_options.nil?
1200
1548
  @gapi.configuration.load.update! schema_update_options: nil
1201
1549
  else
1202
- @gapi.configuration.load.update! \
1203
- schema_update_options: Array(new_options)
1550
+ @gapi.configuration.load.update! schema_update_options: Array(new_options)
1204
1551
  end
1205
1552
  end
1206
1553
 
@@ -1238,21 +1585,28 @@ module Google
1238
1585
  #
1239
1586
  # @!group Attributes
1240
1587
  def encryption= val
1241
- @gapi.configuration.load.update!(
1242
- destination_encryption_configuration: val.to_gapi
1243
- )
1588
+ @gapi.configuration.load.update! destination_encryption_configuration: val.to_gapi
1244
1589
  end
1245
1590
 
1246
1591
  ##
1247
1592
  # Sets the labels to use for the load job.
1248
1593
  #
1249
1594
  # @param [Hash] val A hash of user-provided labels associated with
1250
- # the job. You can use these to organize and group your jobs. Label
1251
- # keys and values can be no longer than 63 characters, can only
1252
- # contain lowercase letters, numeric characters, underscores and
1253
- # dashes. International characters are allowed. Label values are
1254
- # optional. Label keys must start with a letter and each label in
1255
- # the list must have a different key.
1595
+ # the job. You can use these to organize and group your jobs.
1596
+ #
1597
+ # The labels applied to a resource must meet the following requirements:
1598
+ #
1599
+ # * Each resource can have multiple labels, up to a maximum of 64.
1600
+ # * Each label must be a key-value pair.
1601
+ # * Keys have a minimum length of 1 character and a maximum length of
1602
+ # 63 characters, and cannot be empty. Values can be empty, and have
1603
+ # a maximum length of 63 characters.
1604
+ # * Keys and values can contain only lowercase letters, numeric characters,
1605
+ # underscores, and dashes. All characters must use UTF-8 encoding, and
1606
+ # international characters are allowed.
1607
+ # * The key portion of a label must be unique. However, you can use the
1608
+ # same key with multiple resources.
1609
+ # * Keys must start with a lowercase letter or international character.
1256
1610
  #
1257
1611
  # @!group Attributes
1258
1612
  #
@@ -1261,15 +1615,325 @@ module Google
1261
1615
  end
1262
1616
 
1263
1617
  ##
1264
- # Sets the partitioning for the destination table. See [Partitioned
1618
+ # Sets the mode of hive partitioning to use when reading data. The following modes are supported:
1619
+ #
1620
+ # 1. `auto`: automatically infer partition key name(s) and type(s).
1621
+ # 2. `strings`: automatically infer partition key name(s). All types are interpreted as strings.
1622
+ # 3. `custom`: partition key schema is encoded in the source URI prefix.
1623
+ #
1624
+ # Not all storage formats support hive partitioning. Requesting hive partitioning on an unsupported format
1625
+ # will lead to an error. Currently supported types include: `avro`, `csv`, `json`, `orc` and `parquet`.
1626
+ #
1627
+ # See {#format=} and {#hive_partitioning_source_uri_prefix=}.
1628
+ #
1629
+ # @see https://cloud.google.com/bigquery/docs/hive-partitioned-loads-gcs Loading externally partitioned data
1630
+ #
1631
+ # @param [String, Symbol] mode The mode of hive partitioning to use when reading data.
1632
+ #
1633
+ # @example
1634
+ # require "google/cloud/bigquery"
1635
+ #
1636
+ # bigquery = Google::Cloud::Bigquery.new
1637
+ # dataset = bigquery.dataset "my_dataset"
1638
+ #
1639
+ # gcs_uri = "gs://cloud-samples-data/bigquery/hive-partitioning-samples/autolayout/*"
1640
+ # source_uri_prefix = "gs://cloud-samples-data/bigquery/hive-partitioning-samples/autolayout/"
1641
+ # load_job = dataset.load_job "my_new_table", gcs_uri do |job|
1642
+ # job.format = :parquet
1643
+ # job.hive_partitioning_mode = :auto
1644
+ # job.hive_partitioning_source_uri_prefix = source_uri_prefix
1645
+ # end
1646
+ #
1647
+ # load_job.wait_until_done!
1648
+ # load_job.done? #=> true
1649
+ #
1650
+ # @!group Attributes
1651
+ #
1652
+ def hive_partitioning_mode= mode
1653
+ @gapi.configuration.load.hive_partitioning_options ||= Google::Apis::BigqueryV2::HivePartitioningOptions.new
1654
+ @gapi.configuration.load.hive_partitioning_options.mode = mode.to_s.upcase
1655
+ end
1656
+
1657
+ ##
1658
+ # Sets the common prefix for all source uris when hive partition detection is requested. The prefix must end
1659
+ # immediately before the partition key encoding begins. For example, consider files following this data
1660
+ # layout:
1661
+ #
1662
+ # ```
1663
+ # gs://bucket/path_to_table/dt=2019-01-01/country=BR/id=7/file.avro
1664
+ # gs://bucket/path_to_table/dt=2018-12-31/country=CA/id=3/file.avro
1665
+ # ```
1666
+ #
1667
+ # When hive partitioning is requested with either `AUTO` or `STRINGS` mode, the common prefix can be either of
1668
+ # `gs://bucket/path_to_table` or `gs://bucket/path_to_table/` (trailing slash does not matter).
1669
+ #
1670
+ # See {#hive_partitioning_mode=}.
1671
+ #
1672
+ # @see https://cloud.google.com/bigquery/docs/hive-partitioned-loads-gcs Loading externally partitioned data
1673
+ #
1674
+ # @param [String] source_uri_prefix The common prefix for all source uris.
1675
+ #
1676
+ # @example
1677
+ # require "google/cloud/bigquery"
1678
+ #
1679
+ # bigquery = Google::Cloud::Bigquery.new
1680
+ # dataset = bigquery.dataset "my_dataset"
1681
+ #
1682
+ # gcs_uri = "gs://cloud-samples-data/bigquery/hive-partitioning-samples/autolayout/*"
1683
+ # source_uri_prefix = "gs://cloud-samples-data/bigquery/hive-partitioning-samples/autolayout/"
1684
+ # load_job = dataset.load_job "my_new_table", gcs_uri do |job|
1685
+ # job.format = :parquet
1686
+ # job.hive_partitioning_mode = :auto
1687
+ # job.hive_partitioning_source_uri_prefix = source_uri_prefix
1688
+ # end
1689
+ #
1690
+ # load_job.wait_until_done!
1691
+ # load_job.done? #=> true
1692
+ #
1693
+ # @!group Attributes
1694
+ #
1695
+ def hive_partitioning_source_uri_prefix= source_uri_prefix
1696
+ @gapi.configuration.load.hive_partitioning_options ||= Google::Apis::BigqueryV2::HivePartitioningOptions.new
1697
+ @gapi.configuration.load.hive_partitioning_options.source_uri_prefix = source_uri_prefix
1698
+ end
1699
+
1700
+ ##
1701
+ # Sets whether to use schema inference specifically for Parquet `LIST` logical type.
1702
+ #
1703
+ # @see https://cloud.google.com/bigquery/docs/loading-data-cloud-storage-parquet Loading Parquet data from
1704
+ # Cloud Storage
1705
+ #
1706
+ # @param [Boolean] enable_list_inference The `enable_list_inference` value to use in Parquet options.
1707
+ #
1708
+ # @example
1709
+ # require "google/cloud/bigquery"
1710
+ #
1711
+ # bigquery = Google::Cloud::Bigquery.new
1712
+ # dataset = bigquery.dataset "my_dataset"
1713
+ #
1714
+ # gcs_uris = ["gs://mybucket/00/*.parquet", "gs://mybucket/01/*.parquet"]
1715
+ # load_job = dataset.load_job "my_new_table", gcs_uris do |job|
1716
+ # job.format = :parquet
1717
+ # job.parquet_enable_list_inference = true
1718
+ # end
1719
+ #
1720
+ # load_job.wait_until_done!
1721
+ # load_job.done? #=> true
1722
+ #
1723
+ # @!group Attributes
1724
+ #
1725
+ def parquet_enable_list_inference= enable_list_inference
1726
+ @gapi.configuration.load.parquet_options ||= Google::Apis::BigqueryV2::ParquetOptions.new
1727
+ @gapi.configuration.load.parquet_options.enable_list_inference = enable_list_inference
1728
+ end
1729
+
1730
+ ##
1731
+ # Sets whether to infer Parquet `ENUM` logical type as `STRING` instead of `BYTES` by default.
1732
+ #
1733
+ # @see https://cloud.google.com/bigquery/docs/loading-data-cloud-storage-parquet Loading Parquet data from
1734
+ # Cloud Storage
1735
+ #
1736
+ # @param [Boolean] enum_as_string The `enum_as_string` value to use in Parquet options.
1737
+ #
1738
+ # @example
1739
+ # require "google/cloud/bigquery"
1740
+ #
1741
+ # bigquery = Google::Cloud::Bigquery.new
1742
+ # dataset = bigquery.dataset "my_dataset"
1743
+ #
1744
+ # gcs_uris = ["gs://mybucket/00/*.parquet", "gs://mybucket/01/*.parquet"]
1745
+ # load_job = dataset.load_job "my_new_table", gcs_uris do |job|
1746
+ # job.format = :parquet
1747
+ # job.parquet_enum_as_string = true
1748
+ # end
1749
+ #
1750
+ # load_job.wait_until_done!
1751
+ # load_job.done? #=> true
1752
+ #
1753
+ # @!group Attributes
1754
+ #
1755
+ def parquet_enum_as_string= enum_as_string
1756
+ @gapi.configuration.load.parquet_options ||= Google::Apis::BigqueryV2::ParquetOptions.new
1757
+ @gapi.configuration.load.parquet_options.enum_as_string = enum_as_string
1758
+ end
1759
+
1760
+ ##
1761
+ # Sets the field on which to range partition the table. See [Creating and using integer range partitioned
1762
+ # tables](https://cloud.google.com/bigquery/docs/creating-integer-range-partitions).
1763
+ #
1764
+ # See {#range_partitioning_start=}, {#range_partitioning_interval=} and {#range_partitioning_end=}.
1765
+ #
1766
+ # You can only set range partitioning when creating a table. BigQuery does not allow you to change
1767
+ # partitioning on an existing table.
1768
+ #
1769
+ # @param [String] field The range partition field. the destination table is partitioned by this
1770
+ # field. The field must be a top-level `NULLABLE/REQUIRED` field. The only supported
1771
+ # type is `INTEGER/INT64`.
1772
+ #
1773
+ # @example
1774
+ # require "google/cloud/bigquery"
1775
+ #
1776
+ # bigquery = Google::Cloud::Bigquery.new
1777
+ # dataset = bigquery.dataset "my_dataset"
1778
+ #
1779
+ # gcs_uri = "gs://my-bucket/file-name.csv"
1780
+ # load_job = dataset.load_job "my_new_table", gcs_uri do |job|
1781
+ # job.schema do |schema|
1782
+ # schema.integer "my_table_id", mode: :required
1783
+ # schema.string "my_table_data", mode: :required
1784
+ # end
1785
+ # job.range_partitioning_field = "my_table_id"
1786
+ # job.range_partitioning_start = 0
1787
+ # job.range_partitioning_interval = 10
1788
+ # job.range_partitioning_end = 100
1789
+ # end
1790
+ #
1791
+ # load_job.wait_until_done!
1792
+ # load_job.done? #=> true
1793
+ #
1794
+ # @!group Attributes
1795
+ #
1796
+ def range_partitioning_field= field
1797
+ @gapi.configuration.load.range_partitioning ||= Google::Apis::BigqueryV2::RangePartitioning.new(
1798
+ range: Google::Apis::BigqueryV2::RangePartitioning::Range.new
1799
+ )
1800
+ @gapi.configuration.load.range_partitioning.field = field
1801
+ end
1802
+
1803
+ ##
1804
+ # Sets the start of range partitioning, inclusive, for the destination table. See [Creating and using integer
1805
+ # range partitioned tables](https://cloud.google.com/bigquery/docs/creating-integer-range-partitions).
1806
+ #
1807
+ # You can only set range partitioning when creating a table. BigQuery does not allow you to change
1808
+ # partitioning on an existing table.
1809
+ #
1810
+ # See {#range_partitioning_field=}, {#range_partitioning_interval=} and {#range_partitioning_end=}.
1811
+ #
1812
+ # @param [Integer] range_start The start of range partitioning, inclusive.
1813
+ #
1814
+ # @example
1815
+ # require "google/cloud/bigquery"
1816
+ #
1817
+ # bigquery = Google::Cloud::Bigquery.new
1818
+ # dataset = bigquery.dataset "my_dataset"
1819
+ #
1820
+ # gcs_uri = "gs://my-bucket/file-name.csv"
1821
+ # load_job = dataset.load_job "my_new_table", gcs_uri do |job|
1822
+ # job.schema do |schema|
1823
+ # schema.integer "my_table_id", mode: :required
1824
+ # schema.string "my_table_data", mode: :required
1825
+ # end
1826
+ # job.range_partitioning_field = "my_table_id"
1827
+ # job.range_partitioning_start = 0
1828
+ # job.range_partitioning_interval = 10
1829
+ # job.range_partitioning_end = 100
1830
+ # end
1831
+ #
1832
+ # load_job.wait_until_done!
1833
+ # load_job.done? #=> true
1834
+ #
1835
+ # @!group Attributes
1836
+ #
1837
+ def range_partitioning_start= range_start
1838
+ @gapi.configuration.load.range_partitioning ||= Google::Apis::BigqueryV2::RangePartitioning.new(
1839
+ range: Google::Apis::BigqueryV2::RangePartitioning::Range.new
1840
+ )
1841
+ @gapi.configuration.load.range_partitioning.range.start = range_start
1842
+ end
1843
+
1844
+ ##
1845
+ # Sets width of each interval for data in range partitions. See [Creating and using integer range partitioned
1846
+ # tables](https://cloud.google.com/bigquery/docs/creating-integer-range-partitions).
1847
+ #
1848
+ # You can only set range partitioning when creating a table. BigQuery does not allow you to change
1849
+ # partitioning on an existing table.
1850
+ #
1851
+ # See {#range_partitioning_field=}, {#range_partitioning_start=} and {#range_partitioning_end=}.
1852
+ #
1853
+ # @param [Integer] range_interval The width of each interval, for data in partitions.
1854
+ #
1855
+ # @example
1856
+ # require "google/cloud/bigquery"
1857
+ #
1858
+ # bigquery = Google::Cloud::Bigquery.new
1859
+ # dataset = bigquery.dataset "my_dataset"
1860
+ #
1861
+ # gcs_uri = "gs://my-bucket/file-name.csv"
1862
+ # load_job = dataset.load_job "my_new_table", gcs_uri do |job|
1863
+ # job.schema do |schema|
1864
+ # schema.integer "my_table_id", mode: :required
1865
+ # schema.string "my_table_data", mode: :required
1866
+ # end
1867
+ # job.range_partitioning_field = "my_table_id"
1868
+ # job.range_partitioning_start = 0
1869
+ # job.range_partitioning_interval = 10
1870
+ # job.range_partitioning_end = 100
1871
+ # end
1872
+ #
1873
+ # load_job.wait_until_done!
1874
+ # load_job.done? #=> true
1875
+ #
1876
+ # @!group Attributes
1877
+ #
1878
+ def range_partitioning_interval= range_interval
1879
+ @gapi.configuration.load.range_partitioning ||= Google::Apis::BigqueryV2::RangePartitioning.new(
1880
+ range: Google::Apis::BigqueryV2::RangePartitioning::Range.new
1881
+ )
1882
+ @gapi.configuration.load.range_partitioning.range.interval = range_interval
1883
+ end
1884
+
1885
+ ##
1886
+ # Sets the end of range partitioning, exclusive, for the destination table. See [Creating and using integer
1887
+ # range partitioned tables](https://cloud.google.com/bigquery/docs/creating-integer-range-partitions).
1888
+ #
1889
+ # You can only set range partitioning when creating a table. BigQuery does not allow you to change
1890
+ # partitioning on an existing table.
1891
+ #
1892
+ # See {#range_partitioning_start=}, {#range_partitioning_interval=} and {#range_partitioning_field=}.
1893
+ #
1894
+ # @param [Integer] range_end The end of range partitioning, exclusive.
1895
+ #
1896
+ # @example
1897
+ # require "google/cloud/bigquery"
1898
+ #
1899
+ # bigquery = Google::Cloud::Bigquery.new
1900
+ # dataset = bigquery.dataset "my_dataset"
1901
+ #
1902
+ # gcs_uri = "gs://my-bucket/file-name.csv"
1903
+ # load_job = dataset.load_job "my_new_table", gcs_uri do |job|
1904
+ # job.schema do |schema|
1905
+ # schema.integer "my_table_id", mode: :required
1906
+ # schema.string "my_table_data", mode: :required
1907
+ # end
1908
+ # job.range_partitioning_field = "my_table_id"
1909
+ # job.range_partitioning_start = 0
1910
+ # job.range_partitioning_interval = 10
1911
+ # job.range_partitioning_end = 100
1912
+ # end
1913
+ #
1914
+ # load_job.wait_until_done!
1915
+ # load_job.done? #=> true
1916
+ #
1917
+ # @!group Attributes
1918
+ #
1919
+ def range_partitioning_end= range_end
1920
+ @gapi.configuration.load.range_partitioning ||= Google::Apis::BigqueryV2::RangePartitioning.new(
1921
+ range: Google::Apis::BigqueryV2::RangePartitioning::Range.new
1922
+ )
1923
+ @gapi.configuration.load.range_partitioning.range.end = range_end
1924
+ end
1925
+
1926
+ ##
1927
+ # Sets the time partitioning for the destination table. See [Partitioned
1265
1928
  # Tables](https://cloud.google.com/bigquery/docs/partitioned-tables).
1266
1929
  #
1267
- # You can only set the partitioning field while creating a table.
1930
+ # You can only set the time partitioning field while creating a table.
1268
1931
  # BigQuery does not allow you to change partitioning on an existing
1269
1932
  # table.
1270
1933
  #
1271
- # @param [String] type The partition type. Currently the only
1272
- # supported value is "DAY".
1934
+ # @param [String] type The time partition type. The supported types are `DAY`,
1935
+ # `HOUR`, `MONTH`, and `YEAR`, which will generate one partition per day,
1936
+ # hour, month, and year, respectively.
1273
1937
  #
1274
1938
  # @example
1275
1939
  # require "google/cloud/bigquery"
@@ -1277,8 +1941,8 @@ module Google
1277
1941
  # bigquery = Google::Cloud::Bigquery.new
1278
1942
  # dataset = bigquery.dataset "my_dataset"
1279
1943
  #
1280
- # gs_url = "gs://my-bucket/file-name.csv"
1281
- # load_job = dataset.load_job "my_new_table", gs_url do |job|
1944
+ # gcs_uri = "gs://my-bucket/file-name.csv"
1945
+ # load_job = dataset.load_job "my_new_table", gcs_uri do |job|
1282
1946
  # job.time_partitioning_type = "DAY"
1283
1947
  # end
1284
1948
  #
@@ -1288,26 +1952,25 @@ module Google
1288
1952
  # @!group Attributes
1289
1953
  #
1290
1954
  def time_partitioning_type= type
1291
- @gapi.configuration.load.time_partitioning ||= \
1292
- Google::Apis::BigqueryV2::TimePartitioning.new
1955
+ @gapi.configuration.load.time_partitioning ||= Google::Apis::BigqueryV2::TimePartitioning.new
1293
1956
  @gapi.configuration.load.time_partitioning.update! type: type
1294
1957
  end
1295
1958
 
1296
1959
  ##
1297
- # Sets the field on which to partition the destination table. If not
1298
- # set, the destination table is partitioned by pseudo column
1299
- # `_PARTITIONTIME`; if set, the table is partitioned by this field.
1960
+ # Sets the field on which to time partition the destination table. If not
1961
+ # set, the destination table is time partitioned by pseudo column
1962
+ # `_PARTITIONTIME`; if set, the table is time partitioned by this field.
1300
1963
  # See [Partitioned
1301
1964
  # Tables](https://cloud.google.com/bigquery/docs/partitioned-tables).
1302
1965
  #
1303
- # The destination table must also be partitioned. See
1966
+ # The destination table must also be time partitioned. See
1304
1967
  # {#time_partitioning_type=}.
1305
1968
  #
1306
- # You can only set the partitioning field while creating a table.
1969
+ # You can only set the time partitioning field while creating a table.
1307
1970
  # BigQuery does not allow you to change partitioning on an existing
1308
1971
  # table.
1309
1972
  #
1310
- # @param [String] field The partition field. The field must be a
1973
+ # @param [String] field The time partition field. The field must be a
1311
1974
  # top-level TIMESTAMP or DATE field. Its mode must be NULLABLE or
1312
1975
  # REQUIRED.
1313
1976
  #
@@ -1317,8 +1980,8 @@ module Google
1317
1980
  # bigquery = Google::Cloud::Bigquery.new
1318
1981
  # dataset = bigquery.dataset "my_dataset"
1319
1982
  #
1320
- # gs_url = "gs://my-bucket/file-name.csv"
1321
- # load_job = dataset.load_job "my_new_table", gs_url do |job|
1983
+ # gcs_uri = "gs://my-bucket/file-name.csv"
1984
+ # load_job = dataset.load_job "my_new_table", gcs_uri do |job|
1322
1985
  # job.time_partitioning_type = "DAY"
1323
1986
  # job.time_partitioning_field = "dob"
1324
1987
  # job.schema do |schema|
@@ -1332,21 +1995,20 @@ module Google
1332
1995
  # @!group Attributes
1333
1996
  #
1334
1997
  def time_partitioning_field= field
1335
- @gapi.configuration.load.time_partitioning ||= \
1336
- Google::Apis::BigqueryV2::TimePartitioning.new
1998
+ @gapi.configuration.load.time_partitioning ||= Google::Apis::BigqueryV2::TimePartitioning.new
1337
1999
  @gapi.configuration.load.time_partitioning.update! field: field
1338
2000
  end
1339
2001
 
1340
2002
  ##
1341
- # Sets the partition expiration for the destination table. See
2003
+ # Sets the time partition expiration for the destination table. See
1342
2004
  # [Partitioned
1343
2005
  # Tables](https://cloud.google.com/bigquery/docs/partitioned-tables).
1344
2006
  #
1345
- # The destination table must also be partitioned. See
2007
+ # The destination table must also be time partitioned. See
1346
2008
  # {#time_partitioning_type=}.
1347
2009
  #
1348
2010
  # @param [Integer] expiration An expiration time, in seconds,
1349
- # for data in partitions.
2011
+ # for data in time partitions.
1350
2012
  #
1351
2013
  # @example
1352
2014
  # require "google/cloud/bigquery"
@@ -1354,8 +2016,8 @@ module Google
1354
2016
  # bigquery = Google::Cloud::Bigquery.new
1355
2017
  # dataset = bigquery.dataset "my_dataset"
1356
2018
  #
1357
- # gs_url = "gs://my-bucket/file-name.csv"
1358
- # load_job = dataset.load_job "my_new_table", gs_url do |job|
2019
+ # gcs_uri = "gs://my-bucket/file-name.csv"
2020
+ # load_job = dataset.load_job "my_new_table", gcs_uri do |job|
1359
2021
  # job.time_partitioning_type = "DAY"
1360
2022
  # job.time_partitioning_expiration = 86_400
1361
2023
  # end
@@ -1366,48 +2028,44 @@ module Google
1366
2028
  # @!group Attributes
1367
2029
  #
1368
2030
  def time_partitioning_expiration= expiration
1369
- @gapi.configuration.load.time_partitioning ||= \
1370
- Google::Apis::BigqueryV2::TimePartitioning.new
1371
- @gapi.configuration.load.time_partitioning.update! \
1372
- expiration_ms: expiration * 1000
2031
+ @gapi.configuration.load.time_partitioning ||= Google::Apis::BigqueryV2::TimePartitioning.new
2032
+ @gapi.configuration.load.time_partitioning.update! expiration_ms: expiration * 1000
1373
2033
  end
1374
2034
 
1375
2035
  ##
1376
2036
  # If set to true, queries over the destination table will require a
1377
- # partition filter that can be used for partition elimination to be
2037
+ # time partition filter that can be used for time partition elimination to be
1378
2038
  # specified. See [Partitioned
1379
2039
  # Tables](https://cloud.google.com/bigquery/docs/partitioned-tables).
1380
2040
  #
1381
2041
  # @param [Boolean] val Indicates if queries over the destination table
1382
- # will require a partition filter. The default value is `false`.
2042
+ # will require a time partition filter. The default value is `false`.
1383
2043
  #
1384
2044
  # @!group Attributes
1385
2045
  #
1386
2046
  def time_partitioning_require_filter= val
1387
- @gapi.configuration.load.time_partitioning ||= \
1388
- Google::Apis::BigqueryV2::TimePartitioning.new
1389
- @gapi.configuration.load.time_partitioning.update! \
1390
- require_partition_filter: val
2047
+ @gapi.configuration.load.time_partitioning ||= Google::Apis::BigqueryV2::TimePartitioning.new
2048
+ @gapi.configuration.load.time_partitioning.update! require_partition_filter: val
1391
2049
  end
1392
2050
 
1393
2051
  ##
1394
- # Sets one or more fields on which the destination table should be
1395
- # clustered. Must be specified with time-based partitioning, data in
1396
- # the table will be first partitioned and subsequently clustered.
2052
+ # Sets the list of fields on which data should be clustered.
1397
2053
  #
1398
2054
  # Only top-level, non-repeated, simple-type fields are supported. When
1399
2055
  # you cluster a table using multiple columns, the order of columns you
1400
2056
  # specify is important. The order of the specified columns determines
1401
2057
  # the sort order of the data.
1402
2058
  #
1403
- # See {LoadJob#clustering_fields}.
2059
+ # BigQuery supports clustering for both partitioned and non-partitioned
2060
+ # tables.
2061
+ #
2062
+ # See {LoadJob#clustering_fields}, {Table#clustering_fields} and
2063
+ # {Table#clustering_fields=}.
1404
2064
  #
1405
- # @see https://cloud.google.com/bigquery/docs/partitioned-tables
1406
- # Partitioned Tables
1407
2065
  # @see https://cloud.google.com/bigquery/docs/clustered-tables
1408
- # Introduction to Clustered Tables
2066
+ # Introduction to clustered tables
1409
2067
  # @see https://cloud.google.com/bigquery/docs/creating-clustered-tables
1410
- # Creating and Using Clustered Tables
2068
+ # Creating and using clustered tables
1411
2069
  #
1412
2070
  # @param [Array<String>] fields The clustering fields. Only top-level,
1413
2071
  # non-repeated, simple-type fields are supported.
@@ -1418,8 +2076,8 @@ module Google
1418
2076
  # bigquery = Google::Cloud::Bigquery.new
1419
2077
  # dataset = bigquery.dataset "my_dataset"
1420
2078
  #
1421
- # gs_url = "gs://my-bucket/file-name.csv"
1422
- # load_job = dataset.load_job "my_new_table", gs_url do |job|
2079
+ # gcs_uri = "gs://my-bucket/file-name.csv"
2080
+ # load_job = dataset.load_job "my_new_table", gcs_uri do |job|
1423
2081
  # job.time_partitioning_type = "DAY"
1424
2082
  # job.time_partitioning_field = "dob"
1425
2083
  # job.schema do |schema|
@@ -1436,11 +2094,27 @@ module Google
1436
2094
  # @!group Attributes
1437
2095
  #
1438
2096
  def clustering_fields= fields
1439
- @gapi.configuration.load.clustering ||= \
1440
- Google::Apis::BigqueryV2::Clustering.new
2097
+ @gapi.configuration.load.clustering ||= Google::Apis::BigqueryV2::Clustering.new
1441
2098
  @gapi.configuration.load.clustering.fields = fields
1442
2099
  end
1443
2100
 
2101
+ def cancel
2102
+ raise "not implemented in #{self.class}"
2103
+ end
2104
+
2105
+ def rerun!
2106
+ raise "not implemented in #{self.class}"
2107
+ end
2108
+
2109
+ def reload!
2110
+ raise "not implemented in #{self.class}"
2111
+ end
2112
+ alias refresh! reload!
2113
+
2114
+ def wait_until_done!
2115
+ raise "not implemented in #{self.class}"
2116
+ end
2117
+
1444
2118
  ##
1445
2119
  # @private Returns the Google API client library version of this job.
1446
2120
  #